\documentclass[english]{jnlp_3.3}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage[dvipdfmx,nosetpagesize]{graphicx}
%%%%\usepackage[dvipdfmx,nosetpagesize]{color}
%%%%\usepackage[dvipdfmx,nosetpagesize]{colortbl}
%%%%\usepackage{amsmath}
\usepackage{amsthm}
%%%%\usepackage{amsmath, amsthm, amssymb}
\usepackage{mathtools}
%%%%\usepackage[stable]{footmisc}
%%%%\usepackage{placeins}
%%%%\usepackage{tabularx}
%%%%\usepackage{scalerel}
%%%%\usepackage{soul}
%%%%\usepackage{mathtools}
\usepackage{booktabs}
\newtheorem{prop}{Proposition}
%%%%\usepackage{xcolor}
%%%%\usepackage{paralist}
%%%%\usepackage{multirow} 
%%%%\usepackage[linesnumbered, ruled]{algorithm2e}
%%%%\SetKw{KwBy}{by}
%%%%\SetKwInOut{Parameter}{Parameter}
%%%%%%% Coloring the comment as blue
%%%%\newcommand\mycommfont[1]{\footnotesize\ttfamily\textcolor{blue}{#1}}
%%%%\SetCommentSty{mycommfont}
%%%%\SetKwInput{KwInput}{Input}                % Set the Input
%%%%\SetKwInput{KwOutput}{Output}              % set the Output
%%%%\makeatletter 
%%%%\g@addto@macro{\@algocf@init}{\SetKwInOut{Parameter}{Parameters}} 
%%%%\makeatother
%%%%\usepackage{color}
%%%%\usepackage{tabularx}
%%%%\usepackage{graphicx}
%%%%\usepackage{caption}
%%%%\usepackage{subcaption}
%%%%\definecolor{chromeyellow}{rgb}{0.91, 0.84, 0.42}
%%%%\definecolor{burntorange}{rgb}{0.8, 0.33, 0.0}
%%%%\definecolor{mypink2}{RGB}{219, 48, 122}
%%%%\newcommand{\hu}[1]{{\color{violet}[H: #1]}}
%%%%\definecolor{OliveGreen}{rgb}{0,0.6,0}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{28}
\Number{4}
\Month{December}
\Year{2021}
\Issuetitle{Special Issue: XXXXXXXX}

\received{2021}{3}{1}
\revised{2021}{6}{1}
%%%%\rerevised{2006}{5}{6}
\accepted{2021}{7}{10}

\setcounter{page}{1053}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{One-class Text Classification with Multi-modal Deep Support Vector Data Description}

\eauthor{Chenlong Hu\affiref{Author_1} \and Yukun Feng\affiref{Author_1} \and Hidetaka Kamigaito\affiref{Author_1}  \and
Hiroya Takamura\affiref{Author_2} 
\and \\ Manabu Okumura\affiref{Author_1} } 

\eabstract{%
This work presents multi-modal deep SVDD (mSVDD) for one-class text classification. By extending the uni-modal SVDD to a multiple modal one, we build mSVDD with multiple hyperspheres, that enable us to build a much better description for target one-class data. Additionally, the end-to-end architecture of mSVDD can jointly handle neural feature learning and one-class text learning. We also introduce a mechanism for incorporating negative supervision in the absence of real negative data, which can be beneficial to one-class text models including mSVDD model. We conduct experiments on \textit{Reuters}, \textit{20 Newsgroup}, and \textit{TREC} datasets, and the experimental results demonstrate that mSVDD outperforms uni-modal SVDD and mSVDD can get further improvements when negative supervision is incorporated.
}
\ekeywords{One-class Classification,  Text Classification, Support Vector Data Description, Negative Supervision}

\headauthor{Hu et al.}
\headtitle{One-class Text Classification with Multi-modal Deep Support Vector Data Description}

\affilabel{Author_1}{}{Tokyo Institute of Technology}
\affilabel{Author_2}{}{National Institute of Advanced Industrial Science and Technology (AIST)}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
\label{sec:introduction}

One-class classification (OCC), a special classification problem, aims to learn a model on the basis of training samples only from \textit{one} class. 
The learned model is expected to make an accurate description of the class (so called \textit{target} or \textit{normal}) and then to distinguish the \textit{target} from samples for negative classes during testing \cite{moya1993one,taxphd}. 
The one-class classification problem has arisen in many real-world applications, including anomaly or novelty detection \cite{roberts1999novelty,chandola2010anomaly,gupta2013outlier},  bioinformatics \cite{alashwal2006one}, and especially computer vision \cite{rodner2011one,ruff2018deep}.

One-class text classification can be applied to the scenario where a large number of labelled negative samples (e.g., web pages, spam emails ) are hard or time-consuming to obtain \cite{yu2004pebl,heron2009technologies}, while positive training data samples (e.g., ham emails) are available.
One of the early work on one-class text classification is \citeA{manevitz2001one}, who implemented versions of one-class support vector machines (\textit{OC-SVM}) \cite{scholkopf2001estimating}
and showed good performances over the \textit{Reuters} dataset  \cite{dumais98inductive}.  
OC-SVM and support vector data description (\textit{SVDD})  \cite{tax2004support} are \textit{boundary}-based one-class methods \cite{taxphd}. 
Both try to describe the target data using a boundary. 
SVDD learns an optimal hypersphere with the minimum radius to include the most target data, 
while OC-SVM builds a hyperplane to maximally separate the data points from the origin where outlier examples lie around. 

\textit{Reconstruction}-based one-class approaches, including AutoEncoder and its variants \cite{oja1982simplified,rumelhart1985learning,manevitz2007one} and  principal component analysis (PCA) \cite{pearson1901liii,hotelling1933analysis,hoffmann2007kernel},  
which aim to learn a more compact representation for the description of  target data. 
The compact representation is a set of prototypes or subspaces obtained by optimizing a reconstruction error on the target training data.

Regarding the features for representing text in OCC,
document-to-word co-occurrence matrices or hand-crafted features 
have been commonly used in most of the previous work \cite{manevitz2001one,manevitz2007one,kumaraswamy2015anomaly}.
Pre-trained vectors have been popular for many NLP tasks \cite{mikolov2013distributed,bengio2003neural}. 
The recent \textit{context vector data description (CVDD)}, proposed by \citeA{ruff2019self}, fully uses word embedding knowledge and a neural network structure to process one-class text classification problems.

\citeA{ruff2018deep} introduced \textit{deep support vector data description (deep SVDD)}, a fully unsupervised method for deep one-class classification for image data.
Deep SVDD learns to extract the common factors of target training samples with a neural network to  minimize the radius of a hypersphere that encloses the network representations of the data. 
The learned hypersphere, with a center $\boldsymbol{c}$ and a neural feature transformer $\phi(\boldsymbol{x})$, can be an end-to-end feature learning and one-class classification model.
Another piece of work would be \textit{one-class neural networks (OC-NN)} \cite{Chalapathy2018}, which extends OC-SVM to an end-to-end neural architecture.
Instead of a hypersphere in deep SVDD, OC-NN builds a hyperplane on features learned by a feed-forward neural network.

Target data samples may have distinctive distributions that are located in different regions. 
Therefore, uni-modal deep SVDD with one hypersphere may not be enough to describe the target samples.
In this work, we extend deep SVDD to multiple modes, where each mode describes the target samples from a distinctive aspect.
Given our \textit{multi-modal deep SVDD}, \textit{mSVDD} in short, we can create an ensemble set of hyperspheres with different centers to build a better one-class model. 
\citeA{ghafoori2020deep} proposed \textit{deep multi-sphere SVDD} (\textit{DMSVDD}), a similar but different work from ours. 
We will also discuss the relationship between the two and compare them in the experiments. 

In one-class classification, only samples from the target class are available for training, while the model needs to discriminate between the target class and other classes in testing. 
Due to the unavailability of training samples from negative classes,
it is hard for the one-class models to learn effective discrimination information, especially for mSVDD with a multi-layer neural structure.  
In this study, we also propose an architecture for improving the discrimination ability of mSVDD by incorporating \emph{negative supervision}.  
Specifically, we define two kinds of losses, \textit{contrastive} and \textit{triplet}, for joint training with the objective function of mSVDD, which is expected to enhance the discriminative power of mSVDD.

In summary, the main contributions of this work are as follows.
1) We propose a general one-class neural learning framework, called mSVDD, to extend the uni-modal deep SVDD to end-to-end multi-modal. 
2) We also prove that four one-class models, deep SVDD, CVDD, DMSVDD, and OC-NN, are all special cases of the mSVDD model.
3) We propose two approaches for effectively incorporating negative supervision information to improve the performance of the proposed mSVDD and other related one-class text classification methods. 


The remainder of this paper is structured as follows: 
Section \ref{sec:Preliminary} introduces notations and baseline models. 
Section \ref{sec:msvdd} describes the proposed mSVDD.
Section \ref{sec:msvdd_neg} describes the combinatorial use of mSVDD and negative supervision.
We then analyze evaluation results by comparing mSVDD and the baseline models in Section \ref{sec:experiment}.
Section \ref{sec:related_work} summarizes related work for this paper.
Finally, we conclude this paper in Section~7. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Preliminary}
\label{sec:Preliminary}

Before describing our mSVDD, we first introduce SVDD \cite{tax2004support} and OC-SVM \cite{scholkopf2001estimating}  and their extensions, deep SVDD \cite{ruff2018deep} and OC-NN \cite{Chalapathy2018}. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{SVDD and One-class SVM}
\label{ssec:SVDD_OCSVM}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{SVDD}
\label{ssec:SVDD}

Support vector data descriptions (SVDD) is a support vector learning method for one-class classification.
It aims at constructing an optimal boundary in a feature space that includes almost all normal target data, 
given only the target training samples, $T=\{\boldsymbol{x}_1,...,\boldsymbol{x}_n\}$, $\boldsymbol{x}_i \in \mathcal{X} $, where $n\in \mathbb{N}$ is the size of the training data, and  $\mathcal{X}  $ is  a compact subset of $ \mathbb{R}^d $. 
The main idea of SVDD is to optimize a hypersphere with a center $\boldsymbol{c} $ and radius $R$, that encloses the majority of the data. 
SVDD solves the following quadratic problem:
\begin{gather} \label{eq:svddq} 
 \min_{\mathnormal{R},\boldsymbol{c},\boldsymbol{\xi}  }  \quad   \mathnormal{R}^2 + \mathnormal{C} \sum_{\mathnormal{i}} \xi_{\mathnormal{i}}  \\
 s.t. \;    \|  \boldsymbol{x}_i  - \boldsymbol{c}   \|_{2}^2 \leq \mathnormal{R}^2 + \xi_i , \; \xi_i \geq 0 ,  \quad \forall  \mathnormal{i} = 1,...,n, \nonumber
\end{gather}
where $\xi_i $ is a slack variable for allowing a flexible boundary.
$C$ is a regularization parameter, that is usually represented by $\frac{1}{\nu n} $, where $\nu \in (0,1]$ is a parameter that controls the tradeoff between the radius of the hypersphere and the penalties $\xi_i$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{OC-SVM}
\label{ssec:oc-svm}

One-class Support Vector Machines (OC-SVM) \cite{scholkopf2001estimating} is another commonly used approach for solving one-class classification problems. 
Unlike SVDD, OC-SVM intends to seek the best hyperplane by maximizing the margin between the data points and the origin.
Formally, given the training data $T=\{\boldsymbol{x}_1,...,\boldsymbol{x}_n\}$, $\boldsymbol{x}_i \in \mathcal{X} $ like SVDD,  
OC-SVM solves the following  optimization problem:
\begin{gather} \label{eq:oc-svm} %\mathnormal{R}
\min_{\mathnormal{\rho},\boldsymbol{w},\boldsymbol{\xi} }  \quad    \frac{1}{2}\| \boldsymbol{w} \|_{2}^2 - \mathnormal{\rho} + \mathnormal{\frac{1}{\nu n}} \sum_{\mathnormal{i}} \xi_{\mathnormal{i}}  \\
 s.t. \;     \boldsymbol{w}^T   \boldsymbol{x}_i    \geq  \mathnormal{\rho}  -  \xi_i , \; \xi_i \geq 0 ,  \quad \forall  \mathnormal{i} = 1,...,n, \nonumber
\end{gather}
where $\mathnormal{\rho}$ is the margin between the origin and the hyperplane $\boldsymbol{w}$, and $\xi_i$ is a slack variable for allowing data to cross the hyperplane boundary.
As with SVDD, $\nu \in (0,1]$ is a hyperparameter that controls the trade-off between the distance from  the origin to the hyperplane and the number of data points that are allowed to cross the hyperplane. 

The kernel trick \cite{973} can be incorporated into OC-SVM and SVDD to obtain more flexible descriptions. 
Assuming $\mathcal{H}$ is \textit{reproducing kernel Hilbert space} (RKHS) \cite{Saitoh2003} and $K(\boldsymbol{x},\boldsymbol{x}')=  \Phi(\boldsymbol{x}) \cdot \Phi(\boldsymbol{x}')$ is a kernel function, 
$\Phi(x): \mathcal{X} \rightarrow \mathcal{H}$ becomes an implicit feature mapping function from the input space $\mathcal{X}$ to the feature space $\mathcal{H}$.
In this case, the objectives of the kernel-based OC-SVM and SVDD (Eq. (\ref{eq:oc-svm}) and  Eq. (\ref{eq:svddq})) are to find respectively a hyperplane and a hypersphere in feature space $\mathcal{H}$.
Then, both can be solved in the corresponding dual optimization problems, which are quadratic functions involving the dot products of the transformed vectors. 
And every dot product can be replaced with a nonlinear kernel function, without knowing the explicit form of $\Phi(x)$. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Relationship between SVDD and OC-SVM}
\label{ssec:relation_svdd_ocsvm}

The identity of SVDD and OC-SVM has been mentioned in several papers \cite{tsybakov1997nonparametric,taxphd,vert2006consistency}.
\citeA{taxphd}\footnote{Section 2.5.} proved that although the hyperplane obtained by OC-SVM is not a closed boundary around the data that SVDD optimized, both can be represented by an equivalent formulation when data samples are normalized to unit norm vectors. 
Also for the standard Gaussian kernel, where the data is implicitly normalized, these two one-class approaches are also equivalent to each other.  


Several efforts have been proposed to extend kernel-induced OC-SVM and SVDD  with multi-modal form.
\citeA{10.1007/978-3-540-73325-6_75} was early work to use multi-sphere SVDD, which was used for multi-class tasks. 
For one-class tasks, \cite{xiao2009multi} used multi-sphere SVDD to encode multi-distribution target data. 
Multiple kernel learning was proposed for OC-SVM \cite{das2010multiple}.
Two more efforts have been proposed by \cite{le2010theoretical,le2013fuzzy}, which found the optimal solution by an iterative algorithm consisting of the following two steps: 1) calculate radii and centers, and 2) calculate the assignments of data to centers.
While one limitation of SVDD and OC-SVM, along with their extensions, would be that it has to perform hand-crafted feature engineering \cite{pal2010feature},  
the limitation can be solved by incorporating neural models into SVDD. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Deep SVDD and One-class Neural Networks}
\label{ssec:dsvdd_ocnn}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Deep SVDD}
\label{ssec:dsvdd}

Deep support vector data description (deep SVDD) \cite{ruff2018deep} is an end-to-end deep neural model that not only optimizes the SVDD objective loss but also learns a neural feature transformation. 
Given target training samples $T=\{\boldsymbol{x}_1,...,\boldsymbol{x}_n\}$,
deep SVDD first transforms instance $\boldsymbol{x}$ into a data point of the output feature space with $\phi$, which is a multi-layer neural network of $L\in \mathbb{N}$ layers with parameters $ \mathcal{W} = \{\boldsymbol{W}^1, ..., \boldsymbol{W}^L \}$. 
Deep SVDD defines two kinds of loss functions:


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Soft-boundary deep SVDD:}

\begin{equation} \label{eq:svdd_soft_hinge}
\frac{1}{\nu n} \sum_{i} max(0, \|   \phi(      \boldsymbol{x}_i; \mathcal{W}) - \boldsymbol{c}   \|_{2}^2 -     \mathnormal{R}^2  )
+ \mathnormal{R}^2   + \frac{\lambda}{2}\sum_{\mathnormal{l}} \|   \boldsymbol{W}^l \|_{F}^{2}
\end{equation}
The first penalty term is for samples lying outside the sphere, i.e., when the distance of $\boldsymbol{x}_i$ to the center, $\|\phi( \boldsymbol{x}_i; \mathcal{W}) - \boldsymbol{c}  \|_{2}^2$, is greater than radius $R$ after the transformation by network $\phi$. 
The above loss also regularizes the radius and neural weight parameters in the second term.  
As with SVDD, parameter $\nu \in (0,1]$ adjusts the tradeoff between the radius of the hypersphere and the points outside the hypersphere.



\citeA{scholkopf2001estimating} proved that, in single-class classification, 
$\nu$ is the upper bound of the fraction of anomalies, and the lower bound of the fraction of training samples being anomalies or on the optimal boundary.
\citeA{ruff2018deep} proved that this $\nu$-property still holds for uni-modal soft-boundary deep SVDD.


Another simplified objective that minimizes the mean distance of all positive training samples to the center, the \textit{one-class} form, can be defined as follows:


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{One-class deep SVDD (simplified form):} 

\begin{equation}
\frac{1}{n} \sum_{i}  \|  \phi(  \boldsymbol{x}_i; \mathcal{W}) - \boldsymbol{c}  \|_{2}^2  +   \frac{\lambda}{2}\sum_{\mathnormal{l}}\| \boldsymbol{W}^{\mathnormal{l}} \|_{F}^{2}
\end{equation}
Here, we can rewrite both the above in a unified form:
\begin{equation}% \label{eq:svdd_soft_hinge}
\mathcal{L}_{DSVDD} = C \sum_{i} [ \|   \phi(   \boldsymbol{x}_i; \mathcal{W}) - \boldsymbol{c}   \|_{2}^2 -    \beta  ]_{+} 
+   \beta  + \frac{\lambda}{2}\sum_{\mathnormal{l}} \|   \boldsymbol{W}^\mathnormal{l} \|_{F}^{2},  
\end{equation}
where $[ \cdot]_{+} = max\{0,\cdot\}$,  $\beta \in \{0, R^2 \}$ and regularization parameter  $C\in \{\frac{1}{n},\frac{1}{\nu n} \}$ correspond to the two types of forms. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{One-class Neural Networks}
\label{ssec:ocnn}

\citeA{Chalapathy2018} proposed one-class neural networks (OC-NN), which extends OC-SVM to a neural architecture.
As with deep SVDD, OC-NN also uses an end-to-end neural network to perform feature learning and to optimize a OC-SVM equivalent loss objective function.
Given a training set $T=\{\boldsymbol{x}_1,...,\boldsymbol{x}_n\}$
and an $L$-layer neural network $\phi$ with parameter $\mathcal{W}$,
the objective of OC-NN is:
\begin{equation}\label{eq:ocnn}
\frac{1}{2}\| \boldsymbol{w} \|_{2}^2+  \frac{1}{\nu n} \sum_{i} max(0, \mathnormal{\rho} -    \boldsymbol{w}^T \phi(      \boldsymbol{x}_i; \mathcal{W})  )
-   \mathnormal{\rho}  + \frac{\lambda}{2}\sum_{\mathnormal{l}} \|   \boldsymbol{W}^l \|_{F}^{2},
\end{equation}
where the key idea is to use $\phi$ to learn the feature transformation for each instance $\boldsymbol{x}$. 
Similar to soft-boundary deep SVDD (Eq. (\ref{eq:svdd_soft_hinge})), OC-NN also uses $\nu$ to regularize the loss.

We will discuss the relationship between deep SVDD and OC-NN in Section \ref{ssec:msvdd_and_ocnn_relation}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Multi-modal Deep SVDD}
\label{sec:msvdd}

In this section, we present our mSVDD, a method for deep one-class classification. 
Unlike a uni-modal model with a hypersphere, mSVDD uses a set of hyperspheres to describe target class data and to reject samples from negative classes.
Fig.~\ref{fig:msvdd_general} shows the general idea of mSVDD with two modes.
Consider that we have $m$ modes, each of which is described by a hypersphere $\boldsymbol{M}_j$ with center $\boldsymbol{c}_j$ and radius $R_j$;
mSVDD uses each $\boldsymbol{M}_j$ to describe a distinctive aspect of the target class and then ensemble them.
This ensembled deep mSVDD model can provide better descriptions for the target data.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{28-4ia5f1.pdf} 
\end{center}
\hangcaption{mSVDD with two modes. $\phi$ is a neural network. Fivestars denote  centers, and black points denote positive target samples, while triangles denote negative outliers that need to be rejected by hyperspheres. }
\label{fig:msvdd_general}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

As with deep SVDD, given target training samples $T=\{\boldsymbol{x}_1,...,\boldsymbol{x}_n\}$,
mSVDD first transforms instance $\boldsymbol{x}$ into a data point of the output feature space with $ \phi $, where $ \phi$ is a deep neural network of $L\in \mathbb{N}$ layers with parameters $ \mathcal{W} = \{\boldsymbol{W}^1, ..., \boldsymbol{W}^L \}$. 
In contrast to deep SVDD, mSVDD uses $m$ hyperspheres to include almost all of the target data with the minimum radii, i.e., $\frac{1}{m}\sum_{j=1}^{m}{R_j^2} $. 
The objective of kernel SVDD and deep SVDD is to minimize the volume of a data-enclosing hypersphere in feature space that is represented by center $\boldsymbol{c}$ and radius $R$.
Therefore, to minimize the volume, deep SVDD punishes points lying outside the sphere, i.e., if the distance of $\boldsymbol{x}$ to the center $\boldsymbol{c}$, $\|   \phi( \boldsymbol{x};  \mathcal{W}) - \boldsymbol{c} \|_{2}^2$, is greater than radius $R$.
Since we have a set of hyperspheres $M=\{\boldsymbol{M}_1,...,\boldsymbol{M}_m\}$, one choice would be to punish $\boldsymbol{x}$ with respect to each hypersphere by adding $\sum_j \max \left( 0, \|   \phi  \left( \boldsymbol{x};  \mathcal{W} \right) - \boldsymbol{c}_j   \|_{2}^2 -   R_j ^2\right)$ to the loss function.
However, the above penalty term does not take into account the different influences from multiple hyperspheres together.
Therefore, we would like to take another ensembled constraint by incorporating attention mechanism.
Attention mechanism has been adapted to various tasks, allowing models to learn the weights of focusing on different contexts \cite{luong2015effective}.
Thus, it is naturally to incorporate this mechanism to assign a weight for each sample to each center.
Given non-negative \emph{attention weight} $\alpha_{ij}$ for $\boldsymbol{x}_i$ to each $\boldsymbol{M}_j$, the penalty term can be computed as the weighted average over $m$ constraints.
Now, only one ensembled constraint is required, i.e., \emph{the sum of radii is greater than the sum distance to the center}. 
Formally, we can define our mSVDD objective as follows:
\begin{equation}\label{eq:msvdd_weight}
\mathcal{L}_{\scalebox{0.7}{soft-mSVDD}}=  \frac{1}{\nu n} \sum_{i} \max (0, \sum_{j} \alpha_{ij}  ( \|   \phi \left( \boldsymbol{x}_i; \mathcal{W} \right) - \boldsymbol{c}_j   \|_{2}^2 -      R_{j}^2  )  )    +  \frac{1}{m}\sum_{j}{R_{j}^2}   +
\frac{\lambda}{2} \sum_{\mathnormal{l}} \|   \boldsymbol{W}^l \|_{F}^{2},  
\end{equation}
where $\frac{1}{m}\sum_{j=1}^{m}{R_j^2} $ is the regularization term for radii from all $m$ hyperspheres to get a closer boundary around  the target data.
This form can be seen as mSVDD with weighted soft-boundary constraints, which we call \textit{soft-boundary mSVDD}.

Although the $\nu$-property, mentioned in Section~\ref{ssec:dsvdd}, does not hold true for our multi-modal case as it is in general, it is still true when the attention weight $\alpha_{ij}$ is constant for different hyperspheres.
This will give us an intuition on the role of $\nu$.

\begin{prop}\label{prop:nu_property}{\hskip -0.5em}\footnote{The proofs of propositions can be found in the appendix.}{\hskip 0.5em}
The $\nu$-property holds if we set equal attention weight to each hypersphere:  
i.)  $\nu $ is an upper bound for the fraction of outlier samples and ii.) $\nu $ is a lower bound for the fraction of training samples being rejected or on the optimal boundary.
\end{prop}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{One-class mSVDD (Simplified Form)}
\label{ssec:oneclass_msvdd}

As in deep SVDD, we also have the simplified form and called: \textit{one-class\footnote{The term \textit{``one-class''} is used following \cite{ruff2018deep}. Note that most of the models discussed in this paper are one-class models.} mSVDD}.
If we assume that the majority of the training data is not anomalous,
then the radius can be ignored and we can define the simplified mSVDD as follows: 
\begin{equation} \label{eq:msvdd_oneclass}
\mathcal{L}_{\scalebox{0.7}{oc-mSVDD}}= \frac{1}{n}  \sum_{i}   \sum_{j} \alpha_{ij} \|   \phi \left( \boldsymbol{x}_i; \mathcal{W}\right)  - \boldsymbol{c}_j  \|_{2}^2   + \frac{\lambda}{2}  \sum_{\mathnormal{l}} \|   \boldsymbol{W}^l \|_{F}^{2}, 
\end{equation}
where the attention weight $\alpha_{ij}$ will be kept, while the penalty of radius $R$ is deleted. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Unified Form of mSVDD}
\label{ssec:msvdd_unify}

We can write the two variants of mSVDD (i.e., \emph{soft-boundary}  mSVDD and  simplified \emph{one-class} mSVDD) in a unified form:
\begin{equation}\label{eq:msvdd_unify}
\mathcal{L}_{\scalebox{0.99}{mSVDD}}= C \sum_{i}  [ \sum_{j} \alpha_{ij}\left( \| \phi \left( \boldsymbol{x}_i; \mathcal{W} \right) - 
\boldsymbol{c}_j  \|_{2}^2 - \beta_{j}  \right) ]_{+}  + \frac{1}{m}  \sum_{j} \beta_{j}  + \frac{\lambda}{2}  \sum_{\mathnormal{l}} \|   \boldsymbol{W}^l \|_{F}^{2},
\end{equation}
where $\beta_{j} \in \{0, R_{j}^2 \}$. $\beta_{j}=0$ corresponds to
simplified one-class mSVDD, and $\beta_{j}=R_{j}^2$ corresponds to soft-boundary mSVDD. 
\pagebreak
For $\boldsymbol{x}_i$ to the $j$-th hypersphere, attention weight $\alpha_{ij}$ should be inversely proportional to its distance to center $\boldsymbol{c}_j$.
Thus, we define:
\begin{equation}
\label{eq:attention_weight}
\alpha_{ij} =
\frac{ \exp(d(\boldsymbol{x}_i,\boldsymbol{c}_j)/\delta))}{  \sum\nolimits_{k=1}^{m} \exp(  d(\boldsymbol{x}_i,\boldsymbol{c}_k)/\delta)} = 
\frac{ \exp(  \| \phi \left( \boldsymbol{x}_i; \mathcal{W} \right) - \boldsymbol{c}_j  \|_{2}^2   /\delta)}{  \sum\nolimits_{k=1}^{m} \exp(  \| \phi \left( \boldsymbol{x}_i; \mathcal{W} \right) - \boldsymbol{c}_k\|_{2}^2/\delta)}
\end{equation}
where $\delta < 0$ is a temperature hyperparameter. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Discussions on Relationships between mSVDD and Other Models}
\label{ssec:msvdd_unify_2}

In this subsection, we would like to discuss the relationships between mSVDD and other related models.
First, it will help us to understand the differences, and more importantly the common characteristics of deep SVDD-related approaches. 
Second, these discussions would give us an unified view on the development of deep SVDD-related approaches in a limited range.
Further, it also provides possible future directions. 
For example, if one model obtained improvements from certain method, e.g., negative supervision introduced later, this may motivate other deep SVDD-related models. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Relationship between mSVDD and Uni-modal Deep SVDD}

Their relationship is obvious and can be summarized by the following proposition.

\begin{prop}\label{prop:svdd} Deep SVDD is a special case of the unified form of mSVDD with one hypersphere used.
\end{prop}
\begin{proof}
Obviously, mSVDD becomes uni-modal \cite{ruff2018deep} if we use only one hypersphere, i.e., $m=1$.
\end{proof}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Relationship between mSVDD and CVDD}

CVDD \cite{ruff2019self}  is a one-class model for text data. 
In CVDD, each training sample $x_i$ (i.e., a text) is represented by $r$ self-attention feature vectors $  S_i=\left( \boldsymbol{s}_{i1},...,\boldsymbol{s}_{ir}\right)$ \cite{lin2017structured}.  
CVDD uses a group of $r$ \textit{context vectors} $C =\left(\boldsymbol{c}_1,...,\boldsymbol{c}_r \right)$ to describe the target one-class data, where  $\boldsymbol{c}_k \in \mathbb{R}^{p}$.
CVDD tries to reduce the one-to-one \textit{reconstruction} distance between feature vectors $S_i$ and context vectors $C$.
The loss can be defined as:
\begin{equation}\label{eq:losscvdd}
\mathcal{L}_{CVDD}   =   \frac{1}{n}  \sum_{i} \sum_{k}  \sigma_{ik} d(\boldsymbol{c}_k, \boldsymbol{s}_{ik}), 
\end{equation}
where $d(\boldsymbol{c}_k, \boldsymbol{s}_{ik})$ computes the distance, and $\sigma_{ik}$ denotes the attention weight.
The following proposition implies the close connection between the two models for one-class text classification.

\begin{prop} CVDD is a very special case of  one-class mSVDD when mSVDD is applied to text-based tasks under certain conditions.
\end{prop}

\begin{proof}
W.L.O.G., rewrite the loss function of one-class mSVDD in a simplified form for each sample as follows:
\[
\begin{split}  
\mathcal{L}_{mSVDD}  & =  \frac{1}{n}  \sum_{i}\sum_{j} \alpha_{ij}  \|   \phi_{j} \left( \boldsymbol{x}_{i}; \mathcal{W} \right) - \boldsymbol{c}_j   \|^2   \\
 & =\frac{1}{n}  \sum_{i} \sum_{j} \sigma_{ij}  d(\boldsymbol{x}_{i}, \boldsymbol{c}_j ) \approx  \mathcal{L}_{ CVDD } , 
\end{split}
\]
where we drop the regularization terms for weights of $\phi$ and radii, and set $m = r$, $\sigma_{ij} = \alpha_{ij}$, $d(\boldsymbol{x}_{i}, \boldsymbol{c}_{j}) = \|   \phi_j \left( \boldsymbol{x}_i; \mathcal{W} \right) -\boldsymbol{c}_{j}   \|^2 $. $\phi \left( \boldsymbol{x}_i; \mathcal{W} \right)$ has to be a self-attention neural model, $\phi_{j} \left( \boldsymbol{x}_i; \mathcal{W} \right)$ is the  $j-$th feature vector of sample $\boldsymbol{x}_i$, and $\boldsymbol{c}_j$ is the $j-$th context vector of target samples. 
Now the loss functions of CVDD and one-class mSVDD are almost the same.
\end{proof}

[A-(d)]
As stated before, CVDD uses a different multi-head structure for  text feature learning, while the common point would be both adopt the simplified form of mSVDD loss.
In the experiments, we will show that the proposed negative supervision for mSVDD (Section \ref{sec:msvdd_neg}) can also be used for CVDD due to the similarity of both.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Relationship between mSVDD and DMSVDD}
\label{ssec:msvdd_dmsvdd}

DMSVDD \cite{ghafoori2020deep} also uses multi-hyperspheres to extend SVDD.
The loss function of DMSVDD is as follows:
\begin{equation}\label{eq:dmsvdd}
\mathcal{L}_{DMSVDD} = \frac{1}{\nu n} \sum_{i} [ \|   \phi(   \boldsymbol{x}_i; \mathcal{W}) - \boldsymbol{c}_{i^*}   \|_{2}^2 -   R_{i^*}^2  ]_{+}
+\frac{1}{K} \sum_{k}  R_k^2   + \frac{\lambda}{2}\sum_{\mathnormal{l}} \|   \boldsymbol{W}^l \|_{F}^{2},
\end{equation}
where $K$ is the number of hyperspheres\footnote{In DMSVDD, $K$ changes dynamically. However, we ignore this difference and focus on the comparison of the models.},  $\boldsymbol{c}_{i^*}$ is the \textit{nearest center} of sample $\boldsymbol{x}_i$ and $R_{i^*}$ is its radius.

\begin{prop} DMSVDD can be seen as a \textit{hard}-version of soft-boundary mSVDD if we set the attention weight in some way.
\end{prop}

\begin{proof}
In the calculation of the attention weight for mSVDD with $\alpha_{ij} = \frac{\exp(   d(\boldsymbol{x}_i,\boldsymbol{c}_j/\delta)) }{   \sum_{k=1}^{m} \exp( d(\boldsymbol{x}_i,\boldsymbol{c}_k)/\delta) }$, the temperature parameter $\delta$ can influence the assignment of center $c_k$.
If we set $\delta \rightarrow  0^{-}$, the above formula acts as the \emph{argmin} operation.\footnote{$\delta$ approaches 0 from the negative side.} 
In this case,  $\alpha_{ii*}= 1$ if  $ i*=   \mathop{\arg\min}_{k=1,...,K} d(\boldsymbol{x}_i,\boldsymbol{c}_k)$, and  0 otherwise.
Now, we can get the form of DMSVDD from soft-boundary mSVDD (Eq. (\ref{eq:msvdd_weight})) through the adjustment of the attention weight. 
Therefore, we can prove that DMSVDD is also a special case of mSVDD.
\end{proof}

The above relation illustrates key difference between them: DMSVDD puts value on one hypersphere with the largest weight. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Relationship between mSVDD and OC-NN}
\label{ssec:msvdd_and_ocnn_relation}

As mentioned, OC-NN and deep SVDD \cite{Chalapathy2018,ruff2018deep} extend one-class SVM and SVDD \cite{scholkopf2001estimating,tax2004support} to their neural network structures, respectively. 
\citeA{taxphd} has discussed and proved the equivalence of OC-SVM and SVDD. 
Therefore, in this subsection, we discuss the relationships among the three neural models: OC-NN, deep SVDD  and mSVDD.
Since deep SVDD is the uni-modal version of mSVDD (See  Proposition 2), we focus on the discussion of the relationship between mSVDD and OC-NN.
We provide their relationship in the following proposition. 

\begin{prop}
i.) OC-NN can be extended to the multi-modal version.    ii.) Soft-boundary mSVDD can be seen as multi-modal OC-NN under certain conditions.
\end{prop}

\begin{proof}
\textbf{Ad (i).} First, we show that OC-NN can be extended to its multi-modal version. 
OC-NN uses one hyperplane to separate the target data from the origin. Actually we can also use a set of hyperplanes to separate target class data in the feature space, which is a similar idea to mSVDD.
We give the objective function of multi-modal OC-NN as follows:
\begin{equation}\label{eq:msvm}
\mathcal{L}_{mOCNN}=  \frac{1}{\nu n} \sum_{i}  \max (0, \sum_{j} \alpha_{ij}  (    \mathnormal{\rho}_j -    \boldsymbol{w}_j^T \phi(      \boldsymbol{x}_i; \mathcal{W}) )  )    
+  \frac{1}{m}\sum_{j}{\| \boldsymbol{w}_j \|_{2}^2}  
- \frac{1}{m} \sum_{j} \mathnormal{\rho}_j 
+ \frac{\lambda}{2} \sum_{\mathnormal{l}} \| 
\boldsymbol{W}^l \|_{F}^{2},  
\end{equation}
where $\boldsymbol{w}_j$ and $\mathnormal{\rho}_j$ denote the norm perpendicular and the margin of the $j$-th hyperplane in the neural feature space, respectively.   
For one data sample $\boldsymbol{x}_i$, the ensembled constraint term, $\max (0, \sum_{j} \alpha_{ij}  (    \mathnormal{\rho}_j -  \boldsymbol{w}_j^T \phi(      \boldsymbol{x}_i; \mathcal{W}) )$, denotes that 
\emph{the sum of margins should be less than the sum of $x_i$'s distances to the origin}. 
The definition of weight $\alpha$ is the same as in mSVDD (Eq. (\ref{eq:attention_weight})).

\textbf{Ad (ii).}  
Then, we prove the equivalence of soft-boundary mSVDD and multi-modal OC-NN.
For clarity, we use $\boldsymbol{f}_i=\phi( \boldsymbol{x}_i; \mathcal{W})$ to denote the feature vector of $\boldsymbol{x}_i$.
And we also assume the feature vector $\boldsymbol{f}$ is  normalized ($\hat {\boldsymbol{f}} =  \boldsymbol{f}/   \sqrt{\sum\left|f_{i}\right|^{2} }$), so does $\boldsymbol{c}$.
As shown in Fig.~\ref{fig:msvdd_msvm_unit}, in this case, all the data representations and centers are mapped onto an unit hypersphere.
In this case,  a simplified formulation for the loss of mSVDD ( Eq. (\ref{eq:msvdd_weight})) with normalized $\hat {\boldsymbol{f}}$ and $\hat{\boldsymbol{c}}$ would be:
\begin{equation}\label{eq:msvdd_weight_simplified}
\mathcal{L}_{mSVDD}    = \frac{1}{\nu n} \sum_{i}  \max (0, \sum_{j} \alpha_{ij}  ( \|  \hat{\boldsymbol{f}_i} - \hat{\boldsymbol{c}_j}   \|_{2}^2 -      R_{j}^2  )  )    +  \frac{1}{m}\sum_{j}{R_{j}^2},
\end{equation}  
We further transform Eq. (\ref{eq:msvdd_weight_simplified}) by using the relation $ \|\hat{\boldsymbol{f}_i} - \hat{\boldsymbol{c}_j} \|_{2}^2 
= \|\hat{\boldsymbol{f}_i}\|_{2}^2 -2 \hat{\boldsymbol{f}_i}^{T}\hat{\boldsymbol{c}_j}+ \|\hat{\boldsymbol{c}_j}\|_{2}^2 
=  2-2 \hat{\boldsymbol{f}_i}^{T}\hat{\boldsymbol{c}_j}$,
\pagebreak
\begin{align}\label{eq:msvdd_weight_simplified_2}
\mathcal{L}_{mSVDD} &   = \frac{1}{\nu n} \sum_{i}  \max (0, \sum_{j} \alpha_{ij}  (  2-2 \hat{\boldsymbol{f}_i}^{T}\hat{\boldsymbol{c}_j} -    R_{j}^2  )  )   +  \frac{1}{m}\sum_{j}{R_{j}^2}\nonumber \\
&  = \frac{1}{\nu n} \sum_{i}   \max (0, \sum_{j} \alpha_{ij} 2 ( \underbrace{ 1- \frac{1}{2} R_{j}^2}_{
\mathclap{\text{redefine:}\mathnormal{\rho}_j'=1- \frac{1}{2} R_{j}^2 }}- \hat{\boldsymbol{c}_j}^{T}{\hat{\boldsymbol{f}_i}}    )  )   +  \frac{1}{m}\sum_{j}{\underbrace{R_{j}^2}_{
\mathclap{R_{j}^2=2 -2\mathnormal{\rho}_j'  }}} \nonumber \\
&  = \frac{2}{\nu n} \sum_{i}   \max (0, \sum_{j} \alpha_{ij} (  \mathnormal{\rho}_j'-  \underbrace{\hat{\boldsymbol{c}_j}^{T}}_{
\mathclap{\text{redefine:}\hat{\boldsymbol{w}_j}=\hat{\boldsymbol{c}_j}  }}{\hat{\boldsymbol{f}_i}}  ) )   +  \frac{2}{m}\sum_{j}{ (1- \mathnormal{\rho}_j') }\nonumber \\    
& =2 \left(\frac{1}{\nu n} \sum_{i}  \max (0, \sum_{j} \alpha_{ij}( \mathnormal{\rho}_j'-\hat{\boldsymbol{w}_j}^{T}{\hat{\boldsymbol{f}_i}} )  )  +  \frac{1}{m}\sum_{j}{\| \hat{\boldsymbol{w}_j} \|_{2}^2}  
- \frac{1}{m} \sum_{j} \mathnormal{\rho}_j' \right) \nonumber \\  
& =2 \mathcal{L}_{mOCNN},
\end{align}  
where we redefine $ \mathnormal{\rho}_j'=1- \frac{1}{2} R_{j}^2,   \hat{\boldsymbol{w}_j}=\hat{\boldsymbol{c}_j}, \| \hat{\boldsymbol{w}_j} \|_{2}^2=1, \quad \forall j \in \{1,...,m\}$. 
The above formulation indicates that the loss mSVDD is equal to the loss of multi-modal OC-NN, $\mathcal{L}_{mOCNN}$ (Eq. (\ref{eq:msvm})), up to a factor 2.
Therefore, we can prove the equivalence of   soft-boundary mSVDD and multi-modal OC-NN under the condition of normalization.
Certainly, the identity can hold when we use uni-modal deep SVDD and OC-NN.
\end{proof}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% fig. 2
\begin{figure}[t]
\begin{center}
\includegraphics{28-4ia5f2.pdf} 
\end{center}
%%%\label{fig:sub:msvdd_unit_hyperplane}
%%%%\label{fig:sub:ocnn_unit_hyperplane}
\hangcaption{Multi-modal Deep SVDD and OC-NN with two modes, where the data representations are mapped to unit norm hypersphere.
Black points denote normalized data samples.}
\label{fig:msvdd_msvm_unit}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

As for the geometric interpretation, Fig.~\ref{fig:msvdd_msvm_unit} illustrates how the two models show when data representations are normalized and mapped onto unit hypersphere. 
The two subfigures in Fig.~\ref{fig:msvdd_msvm_unit} show multi-modal OC-NN and SVDD with two modes, respectively. 
As shown in  Fig.~\ref{fig:msvdd_msvm_unit}, each data sample can be regarded as a point on the unit hypersphere, and maximizing the margins of hyperplanes from the origin in OC-NN is equal to minimizing the radii of hyperspheres in SVDD.
We can also get the relationship from $ \mathnormal{\rho}_j'=1- \frac{1}{2} R_{j}^2$ which shows that the margin $ \mathnormal{\rho}_j' $ is inversely proportional to the radius $R_{j}$.
During the (maximizing/minimizing) optimization process in OC-NN and mSVDD, these two neural models can also perform the feature learning such that data representations are rotated to their ``center'' representations (i.e., $\boldsymbol{w}$ and $\boldsymbol{c}$) on the unit hypersphere \cite{liu2017sphereface,deng2019arcface,yokoi2020word}. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Summarizing mSVDD}
\label{ssec:msvdd_summarization}

We summarize the proposed mSVDD in accordance with the discussions presented above.
The proposed  multi-modal deep SVDD (mSVDD) learns a compact description of one-class data with multiple hyperspheres.
mSVDD is also a generic framework that includes deep SVDD, CVDD, DMSVDD, and OC-NN if the corresponding conditions are met.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Multi-modal Deep SVDD with Negative Supervision}
\label{sec:msvdd_neg}

In this section, we incorporate negative supervision into the training of mSVDD.
The SVDD-related models are usually trained with only positive samples from the target one-class,
while, if negative samples are available, the models can be extended to train with them to improve the description \cite{taxphd}.\footnote{Section 2.2.}
Note that these samples are \emph{not} necessarily required to be from ``\emph{real}'' negative class. 
In our experiment, external data can be seen as one of the choices for the incorporated \textit{pseudo}-negative samples.
Note that, the proposed negative supervision can be compatible with other SVDD-related one-class text classification methods, e.g., CVDD (see Section \ref{sssec:cvdd_neg}), not just mSVDD.

Given a set of extended training samples $T'=\{(\boldsymbol{x}_1,y_1),...,(\boldsymbol{x}_{n'},y_{n'})\}$, where the first $n$ samples are labeled $y_i = 1$, denoting positive, whereas the others are labeled $y_i = 0$, which denotes negative samples that should be rejected by mSVDD.
Our mSVDD is represented with $m$ hyperspheres and is formulated as $M= \{  \boldsymbol{M}_1(\boldsymbol{c}_1, R_1), ......, \boldsymbol{M}_m(\boldsymbol{c}_m, R_m) \}$. 
It is required that the positive samples should be inside the $m$ hyperspheres, while the negative samples should lie outside. 
Given training samples composed of positive and a negative samples,
we can first get their corresponding distances to each center $\boldsymbol{c}_j$. 
The goal of optimization should be to pull the positive samples closer the center and to push the negative ones away. 
Formally, we define the distance between one sample $\boldsymbol{x}_i$ and one center $\boldsymbol{c}_j $  as $d_{ij}=d(\boldsymbol{x}_i, \boldsymbol{c}_j)= \| \phi(   \boldsymbol{x}_i; \mathcal{W}) - \boldsymbol{c}_j \|_{2}^2  $. 
There are usually two types of losses to obtain the discriminative loss.

\emph{Contrastive type}: The contrastive-type loss directly optimizes the distance by encouraging the distance between a positive sample and a center to be smaller,
while it forces the larger distance to a negative sample:
\begin{equation}\label{eq:loss_contrastive_singlesample}
\mathcal{L}_{Con\_d}^{(ij)}= y_i [ d_{ij} - R_j^2]_+  + \left( 1- y_i \right )  [  R_j^2 -  d_{ij}]_+,
\end{equation} %\scalebox{0.5}{Con}
where $R_j^2 $ can be seen as a \emph{margin} (or \emph{threshold}) with a function that prevents too much effort from being wasted in enlarging/reducing distances \cite{hadsell2006dimensionality}. 

\emph{Triplet type}: 
The triplet-type loss is defined for a pair of positive sample $\boldsymbol{x}_{i}$ and
negative sample $\boldsymbol{x}_{i'}$.
If we consider center $\boldsymbol{c}_j$ as an \emph{anchor} representative of target data, the triplet loss punishes only when $\mathnormal{d}_{ij}$, the distance from $\boldsymbol{x}_{i}$ to $\boldsymbol{c}_j$, is greater than $d_{i'j}$, the distance from $\boldsymbol{x}_{i'}$ to $\boldsymbol{c}_j$, with a margin $\tau>0$: 
\begin{equation}\label{eq:loss_triple_singlesample}
\mathcal{L}_{Tri\_d}^{(ii'j)} =  \left [   d_{ij} -  d_{i'j}+ \tau \right ]_{+}.  
\end{equation}
For clarity, Eqs. (\ref{eq:loss_contrastive_singlesample}) and (\ref{eq:loss_triple_singlesample}) show only the two types of losses for one hypersphere.
Multi-modal version can be obtained by sum operation over $j \in \{1,...,m\}$. 

The triplet loss forces only positive samples to be closer to the center than negative samples,
and the contrastive loss requires only keeping the distances for negative samples above the radius.  
These two types of objectives are easy to achieve, especially when we assume that negative samples are ``\emph{not real}.''
This may result in failing to make a full use of negative supervision.
Therefore, we will reformulate both $\mathcal{L}_{Tri\_d}$ and $\mathcal{L}_{Con\_d}$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Reformulating Contrastive and Triplet Losses}
\label{ssec:reformulation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Normalization layer}
In neural models with the contrastive or triplet loss, 
it is a common strategy to normalize the feature representations of samples for training stability \cite{schroff2015facenet,wang2017normface}.
Therefore, we apply the normalization to the input vectors: 
$\hat {\boldsymbol{x}} =  \boldsymbol{x}/   \sqrt{\sum\left|x_{i}\right|^{2}  +\epsilon}$,
where $\epsilon>0$ is a value avoiding division by zero. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Reformulation}
\label{pa:reformulation}
Given a center $\boldsymbol{c}_j$ and positive and negative samples, 
we can use the probability form in the optimization objective, rather than the two \textit{non-probabilistic} ones: $\mathcal{L}_{Con\_d}$ and $\mathcal{L}_{Tri\_d}$. 
We introduce $p(y_i=1|\boldsymbol{x}_i,\boldsymbol{c}_j)$, which is the probability that a hypersphere with center $\boldsymbol{c}_j$ \emph{accepts} the sample $\boldsymbol{x}_i$, and define it as follows: 
\begin{equation}\label{eq:prob_each_sample}
p(y_i=1|\boldsymbol{x}_i,\boldsymbol{c}_j) = \sigma (  s\hat{\boldsymbol{f}_i}^T  \hat{\boldsymbol{c}_j} ),  
\end{equation}
where $\sigma(x) = \frac{1}{1+\exp(-x)}$,  $\boldsymbol{f}_i=\phi( \boldsymbol{x}_i; \mathcal{W})$ denotes the feature output vector of $\boldsymbol{x}_i$, 
\pagebreak
and $s$ is a scale hyper-parameter for preventing failed convergence \cite{wu2018unsupervised} after the normalization.
For each sample $\boldsymbol{x}_i$, $\boldsymbol{c}_j$ acts as a \emph{pseudo}-weight vector for the classification of the $j$-th hypersphere of mSVDD. 
Thus, given $p(y_i=1|\boldsymbol{x}_i,\boldsymbol{c}_j)$, the probability of a sample being accepted by hypersphere $\boldsymbol{M}_j$, 
we can reformulate the two discriminative losses with the probability. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{\emph{Contrastive type loss}: } 
\begin{align}\label{eq:loss_contrastive_singlesample_reformulate}
\mathcal{L}_{Con}^{(ij)}& =  -  y_i \log  p(y_{i}=1|\boldsymbol{x}_{i },\boldsymbol{c}_j) 
\nonumber - \left( 1- y_i \right )  \log  p(y_{i }=0|\boldsymbol{x}_{i },\boldsymbol{c}_j) \\  
&= -  y_i \log \sigma  (  s\hat{\boldsymbol{f}_i}^T   \hat{\boldsymbol{c}_j} )  -  ( 1- y_i   )  \log \sigma  ( - s\hat{\boldsymbol{f}_i}^T  \hat{\boldsymbol{c}_j} ),
\end{align} 
This loss maximizes the likelihood of training positive samples being accepted or negative rejected.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{\emph{Triplet type loss}:}
\begin{equation}\label{eq:loss_triple_singlesample_reformulate}
\begin{split}
\mathcal{L}_{Tri}^{(ii'j)} &=   
\left [   \log  p(y_{i'}=1|\boldsymbol{x}_{i'},\boldsymbol{c}_j)-      \log  p(y_i=1|\boldsymbol{x}_i,\boldsymbol{c}_j)  +     \tau \right ]_{+} \\
&=  \left [  \log \sigma (s  \hat{\boldsymbol{f}_{i'}}^T  \hat{\boldsymbol{c}_j } ) - \log \sigma  (  s\hat{\boldsymbol{f}_i}^T  \hat{\boldsymbol{c}_j } )   +     \tau \right ]_{+} 
\end{split}
\end{equation}

The loss will punish when the log probability of a negative sample is greater than a positive sample with a margin $\tau$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Reformulating Contrastive and Triplet Losses for Multiple Modes}

While Eqs. (\ref{eq:prob_each_sample}), (\ref{eq:loss_contrastive_singlesample_reformulate}), and (\ref{eq:loss_triple_singlesample_reformulate}) show the uni-modal case, 
for the multi-modal one, we have to consider $m$ different centers $\{\boldsymbol{c}_1, \ldots,\boldsymbol{c}_m\}$ in the calculation of the two reformulated discriminative losses.
Therefore, we propose two strategies as follows:
\begin{equation}\label{eq:prob_max_mean}
p(y_i=1|\boldsymbol{x}_i)=
\begin{cases}
\max \limits_{1 \leqslant j \leqslant   m}p(y_i=1|\boldsymbol{x}_i,\boldsymbol{c}_j) \quad \textit{Max} \\ 
\frac{1}{m} \sum_{j}p(y_i=1|\boldsymbol{x}_i,\boldsymbol{c}_j)  \quad \textit{Mean}, 
\end{cases}       
\end{equation}
where \textit{Max} references only $\boldsymbol{M}_j$ with the max logit output, while \textit{Mean} takes account of all hyperspheres equally.
Then, we can obtain the corresponding \emph{Contrastive}  and \emph{Triplet} losses by substituting Eqs. (\ref{eq:loss_contrastive_singlesample_reformulate}) and (\ref{eq:loss_triple_singlesample_reformulate}) with the probability term (Eq. (\ref{eq:prob_max_mean})).  


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Training Loss}

The final training loss for the mSVDD with negative supervision  can be formulated as:
\begin{equation}\label{eq:final_loss}
\mathcal{L}=  \mathcal{L}_{mSVDD}+ \gamma   \mathcal{L}_{Con|Tri },
\end{equation}
where $\gamma $ adjusts between the mSVDD loss and the discrimination with negative supervision. 
In the training process, $\mathcal{L}_{Con|Tri }$ will sum the loss from one batch samples with  Eqs. (\ref{eq:loss_contrastive_singlesample_reformulate}) and (\ref{eq:loss_triple_singlesample_reformulate}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Relationship between mSVDD and the Use of Negative Supervision}
mSVDD and negative supervision are not two independent sub-architectures. 
Negative supervision, including \emph{contrastive} and \emph{triplet} losses, are specially equipped to mSVDD.
Specifically, these two components are closely connected by the center of the hypersphere, $\boldsymbol{c}_j$.
Both mSVDD (Eq. (\ref{eq:msvdd_unify})) and negative supervision (Eq. (\ref{eq:loss_contrastive_singlesample_reformulate}) or  
(\ref{eq:loss_triple_singlesample_reformulate})) contain $\boldsymbol{c}_j$. 
Since there is no \emph{real} negative data,  external data are used as \emph{pseudo} negative samples to complete negative supervision.
The use of negative supervision can improve the discrimination ability of mSVDD. 
In training, negative supervision loss forces mSVDD to reject \textit{unseen}  samples since real negative data in testing are also unseen in training. 
This improves inter-class discrepancy, compared with intra-class loss mSVDD optimized. 
However, in testing, the decision function will be the same as mSVDD trained only with positive samples.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}
\label{sec:experiment}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Datasets and Implementation Details}
\label{ssec:implementation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Datasets}
Experiments were conducted on two datasets: \textit{20 Newsgroups}\footnote{\url{http://qwone.com/json/20Newsgroups}} and  \textit{Reuters},\footnote{\url{http://daviddlewis.com/resources/testcollections/reuters21578/}} 
which have been commonly used in other one-class text classification work \cite{manevitz2001one,ruff2019self}.
We also conducted experiments on another dataset \textit{TREC}\footnote{\url{https://cogcomp.seas.upenn.edu/Data/QA/QC/}} \cite{li2002learning}.
We follow the same one-class classification setting in \cite{ruff2019self},  
i.e., one of the classes in each dataset is considered as the target class, we call it as \textit{target} or \textit{positive},
while the remaining classes are considered \textit{negative}.
Note that we labelled positive samples as $y=1$, while negative ones $y=0$.
We used the same pre-processing steps as the ones used in earlier work  \cite{ruff2019self}, including lowercasing, removing stopwords, and tokenization.
We used the external data for negative supervision in the absence of ``\emph{real}'' labeled negative instances.
We followed the similar logic for choosing our external data as the one in the field of pretrained word vectors, in which one general corpus, such as Wikipedia articles, is often adopted as the training dataset  \cite{mikolov2013distributed}. 
So we also chose one publicly available corpus WikiText-2 \cite{merity2016pointer}, extracted from Wikipedia articles, as our external data.
In the training, data loader loads one batch of negative samples, i.e., sentences from WikiText-2, which are labeled with $0$. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Encoder}
For encoding the text input, i.e., $\phi(\boldsymbol{x},W)$, 
\pagebreak
we used a Bidirectional LSTM with attention  \cite{hochreiter1997long,xu2015show}, with the number of hidden units being $150$. 
For the pre-trained word embeddings, we experimented with GloVe Vectors \cite{pennington-etal-2014-glove} and set the dimension to 300.
In our experiments, we did not adopt the widely used BERT model \cite{devlin2019bert}, as \citeA{ruff2019self} showed that BERT model did not improve the performance.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Settings}
As for the optimization of parameters, Adam  \cite{kingma2014adam} with a base learning rate of $0.001$ was used for $50$ epochs.
The batch sizes were set to $32$, $32$, and $64$ for \textit{Reuters}, \textit{TREC}, and  \textit{Newsgroups}, respectively.  
For the initialization of mSVDD model, we employed two operation steps.
In the absence of negative samples, mSVDD was first pre-trained on target samples by using an AutoEncoder with two objectives: 1) warm-up and 2) reducing the \emph{reconstruction} error for the target samples, such that the model can be more robust to noise or anomalous inputs \cite{jacobs1995methods,hinton2006reducing}. 
An AutoEncoder feed-forward network with a $0.5$ compression rate, which consists of an encoder and a decoder, was put on the back of the BiLSTM feature network.
Then, the weights of the $m$ hyperspheres in mSVDD were initialized by running $k$-means clustering on the features learned before \cite{lloyd1982least}.
As for the regularization term of mSVDD, 
$\boldsymbol{c}_j$ was regularized \cite{ng2004feature}, and a weight decay with $0.95$ was applied for the parameters.  
As for the number of hyperspheres, different settings, $1, 3 , 5, 10$, were tested. 
For the hyperparameters, we set parameter $s=1.2$ for scale, $\nu=0.1$, $\delta=-0.9 $ for the attention weight,  $\tau=0.1$ for the triplet loss, $\epsilon=1\mathrm{e} - 6$ for norm, and $\gamma=1$ for the training loss.
The results were averaged over 10 runs with different random seeds. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Evaluation metrics}
The performance was measured by the \textit{area under the receiver operating characteristics} (ROC) curve (AUCs),  
a commonly used metric for one-class text classification \cite{manevitz2001one,ruff2019self}. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Results}
\label{ssec:results}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Comparison between multi-modal and unit-modal}
\label{ex:msvdd}

Table~\ref{table:result_msvdd_one_centernum_05} shows the performance of mSVDD with different choices of $m$, i.e., the number of hyperspheres. 
Here, mSVDD(1) represents uni-modal deep SVDD (DSVDD) \cite{ruff2018deep}.
The results show that: 
1) mSVDD outperformed the uni-modal DSVDD under most target classes over three datasets. 
Generally, multi-modal mSVDD provided better performances by obtaining best scores in more times than the uni-modal one.
As for the \textit{one-class} version, compared to mSVDD(3), mSVDD with more hyperspheres (5) setting performed better in \textit{Reuters} and \textit{TREC}, while there was no improvement found on \textit{Newsgroup}. 
mSVDD(3) also shows improvements on four target classes on \textit{TREC} datset. % also be observed from greater than X was observed.
Similar results can also be observed for \textit{soft-boundary} version. 
On the three target classes, \textit{rec}, \textit{sci} and \textit{misc}, \textit{soft-boundary} mSVDD(3) shows consistent performances.
It shows also that the \textit{soft-boundary} mSVDD(5) achieved the best scores on five out of six cases in \textit{TREC} dataset.
The results show the effects of incorporating more hyperspheres to better describe the target data.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[t]
\input{05table01.tex}
\hangcaption{Results of mSVDD with different settings of $m$. Numbers in brackets denote the number of hyperspheres in mSVDD. \textit{One-v} and \textit{Soft-v} denote the two versions of mSVDD, \textit{One-class} and \textit{Soft-boundary},  respectively.  AUCs in \% on the \textit{Reuters} (upper part), \textit{20 Newsgroup} (middle part) and \textit{TREC} (lower part) datasets. Best scores in each \emph{row}  are presented in \textbf{bold} within each \textit{One-v} and \textit{Soft-v} group. Significant improvements over uni-modal have been marked with a dagger $^\dag$($p<0.05$).}
\label{table:result_msvdd_one_centernum_05}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

2) The performance of mSVDD did not improve linearly along with $m$. 
For examples, except for \textit{comp} in \textit{Newsgroup} dataset, mSVDD(10) with \textit{one-class} form didn't achieve best scores.
While mSVDD(3 or 5) perform the best in most target classes over three datasets in the \textit{one-class} loss setting. 
This indicates that the incorporation of more descriptions is not necessary sometimes. 
Fig.~\ref{fig:vis_feature} shows an example with t-SNE \cite{van2008visualizing} visualization of feature representations learned by DSVDD, mSVDD, and CVDD. 
For the data inputs for target \textit{rec}, mSVDD maps data to multiple clusters in the feature space which matches the assumption that there are multiple sub-groups on \textit{rec}. 
While DSVDD maps them into one cluster, and CVDD outputs feature representations separated by five heads.
As for the performance, both forms of mSVDD obtained better scores than CVDD or DSVDD on \textit{rec}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[b]
\vspace{-0.75\Cvs}
\begin{center}
\includegraphics{28-4ia5f3.pdf} 
\end{center}
%%%%\label{fig:sub:vis_dsvdd}
%%%%\label{fig:sub:vis_msvdd}
%%%%\label{fig:sub:vis_cvdd}
\hangcaption{t-SNE visualization of feature representations learned by DSVDD, mSVDD and CVDD over \textit{rec} on \textit{Newsgroup} dataset}
\label{fig:vis_feature}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

As for uni-modal has good results in some cases such as \textit{ABBR}, \textit{pol}, and \textit{rel}. 
We can explain this from the following aspects.
As for the model, mSVDD with more centers means that it has more parameters and a complex model structure, which is hard to be optimized, especially on the data with a small training size (e.g., \textit{pol} or \textit{rel}.)
As for the data, some data might have simple data distributions without the need for more modes.
Another aspect would be the attention weights of multiple hyperspheres. 
\citeA{ghafoori2020deep} showed that focusing on some ``\textit{good}'' hyperspheres would be beneficial rather than over all hyperspheres. 
In the calculation of attentions,  we did not adjust $\delta$ so as to have a large weight for one specific hypersphere.
This may cause limited improvements. 
We will compare mSVDD with DMSVDD later. 

As the choice of $m$,  the number of hyperspheres of mSVDD for different situations,
it would be beneficial to use multiple hyperspheres if we have prior knowledge about the target class such as the existence of  subtopics as shown in Fig.~\ref{fig:vis_feature}.
Besides the prior information, 
we can also use the information in training process, e.g., the number of close points of each hypersphere, to determine whether some ones should be combined or be discarded \cite{ghafoori2020deep}. 
Another method is cross validation, especially leave-one-out method,  which is commonly used to automatically optimize the values for parameters \cite{stone1974cross,tax2001uniform}.
Considerably more work will need to be done to determine the right number of $m$. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Comparison between mSVDD and other models}

Table \ref{table:msvdd_vs_others} shows the results of the comparison between mSVDD and other models.
From the discussion in the last subsection, we used $m=3$ for mSVDD in this subsection. 
As for the three baseline methods, kernel OC-SVM, SVDD, and CVDD, we adopted their \textit{best} scores on \textit{Reuters} and \textit{20 Newsgroup} from ~\protect\cite{ruff2019self}.
As for \textit{TREC}, we conducted experiments with the same setting with \protect\cite{ruff2019self} and reported the best results for the above three approaches.
For the method of DMSVDD, we reported the results in the setting of the initial number of spheres $K_{init}=10$.
We also ran experiments with one-class neural networks (OC-NN) \cite{Chalapathy2018}.
Since OC-NN\footnote{\url{https://github.com/raghavchalapathy/oc-nn}} was originally designed and applied for computer vision tasks,  we re-implemented it to match our experiments with PyTorch \cite{NEURIPS2019_9015}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[b]
\input{05table02.tex}
\hangcaption{Results of mSVDD and other models. Average AUCs in \% on the \textit{Reuters} (left part), \textit{20 Newsgroup} (middle part),  and \textit{TREC} (right part).\_\textit{One} and \_\textit{Soft} mean \textit{One-class} and \textit{Soft-boundary} forms, respectively. Best scores in each \textit{column} are presented in \textbf{bold}, while the second best are \underline{underlined}. }
\label{table:msvdd_vs_others}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

From the results in Table \ref{table:msvdd_vs_others}, we have the following observations. 
In general, CVDD performed better than other neural models on \textit{Reuters} dataset, while mSVDD achieved better performances than other models on other two datasets, \textit{Newsgroup} and \textit{TREC}.
As for the two traditional kernel methods, they performed well on  \textit{Reuters}, especially for \textit{earn and acq}.
As regard the comparison between DMSVDD and mSVDD, DMSVDD puts value on one hypersphere and performed slightly better over mSVDD(3) in some cases (e.g., \textit{earn, acq} and \textit{comp}).
This indicates one inspiration that discarding ``\textit{bad}'' hyperspheres is sometimes necessary. 
Regarding OC-NN, another uni-modal method,  mSVDD performed better than it in most target classes, 
while OC-NN obtained high scores in  the cases of \textit{ABBR} and \textit{pol}.
Therefore, OC-NN and its multi-modal version deserve to be explored and discussed in-depth in the future.
Even mSVDD did not obtain the best AUC scores on \textit{Reuters}, it still attained the second best in three cases. 
And the performances on the most cases for other two datasets showed the advantage of the proposed mSVDD over other related baseline models. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Results of mSVDD with negative supervision}
\label{ex:msvdd_neg}

Table \ref{table:result_msvdd_with_neg} shows the performance of mSVDD trained with negative supervision.
As mentioned, we still used $m=3$ in this subsection. 
To perform negative supervision for mSVDD, we evaluated four approaches where different losses and their reformulated probability forms were selected. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[b]
\input{05table03.tex}
\hangcaption{Results of mSVDD with negative supervision. Average AUCs in \% on the \textit{Reuters} (left part), \textit{20 Newsgroup} (middle part),  and \textit{TREC} (right part). +Triple+Max, which denotes mSVDD with \textit{Triplet} loss with \textit{Max} probability strategy, followed by three other negative supervision methods. In the rows of mSVDD\_\textit{Soft} and mSVDD\_\textit{One}, `+' following numbers means that there were improvements with negative supervision (three  of four methods.) Best scores in each \textit{column} are presented in \textbf{bold} within each \textit{\_One} and \textit{\_Soft} group.}
\label{table:result_msvdd_with_neg}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

For \textit{Reuters}, the results indicate that mSVDD benefited from the joint training of the discrimination losses, except for \textit{acq} and \textit{ship} where negative supervision did not show consistent improvements (three of four methods).
We can see more obvious improvements for \textit{20 Newsgroup}.
All four negative supervision methods improved mSVDD markedly for all target classes of \textit{20 Newsgroup}.
For example, mSVDD with negative supervision gained 2--3 points for \textit{comp}. 
For different losses for negative supervision, the \textit{contrastive} type loss, which has larger punishment over negative data, performs better than the \textit{triplet} type loss, which uses a relatively small margin. 
In addition, the performance of \textit{Con+Max} was greater than the \textit{Con+Mean} strategy to reformulate the probability.
We hypothesize that focusing on one of the hyperspheres is effective when we used mSVDD with the \textit{contrastive} loss.


As for the third \textit{TREC} dataset, mSVDD with negative supervision achieved the best results on all target classes. 
The greatest gain came from \textit{DESC} when the \textit{one-class} form mSVDD trained with \textit{contrastive} type loss with \textit{mean} probability strategy. 
As for the probability strategy on \textit{TREC}, \textit{max} strategy would be beneficial to the \textit{one-class} form, while  \textit{max} for \textit{soft-boundary} mSVDD.
In summary,  Table \ref{table:result_msvdd_with_neg} validated  that  mSVDD  can be further improved by incorporating the  proposed negative supervision methods. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[b]
\input{05table04.tex}
\hangcaption{Results of CVDD with the proposed negative supervision. Average AUCs in \% on the \textit{Reuters} (upper part), \textit{20 Newsgroup} (middle part), and \textit{TREC}(lower part) datasets. Number $r$ in brackets denotes the number of heads in CVDD. \textbf{Bold} means the better AUCs score.}
\label{table:result_cvdd_neg}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Results of CVDD with negative supervision}
\label{sssec:cvdd_neg}

Table \ref{table:result_cvdd_neg} shows the results of CVDD with the proposed negative supervision for mSVDD.
As mentioned in Section \ref{ssec:msvdd_unify_2}, CVDD can be seen as a special case of mSVDD. 
Therefore, the proposed negative supervision approaches to mSVDD can be also applied to CVDD theoretically.
To highlight the usefulness of the negative supervision, we conducted the experiments to use the \textit{triplet} loss with \textit{Max} probability for CVDD.  
As for the implementation, since CVDD uses a different multi-head structure, we also used a different form to incorporate \textit{Triplet+Max} to CVDD (See Section \ref{app:sec:cvdd_neg}
in the appendix for the details of the implementation.). 

Overall, we can see that the proposed negative supervision enhanced CVDD in most cases on the three datasets. 
The overall performance mainly shows the following: 
1) The improvement by the negative supervision to CVDD is consistent with mSVDD due to the similarity between the two.
2) The generality of the negative supervision can be shown, as \textit{Triple+Max} was successfully applied to the different multi-head structure. 

Regarding different target-classes, \textit{ship} with the smaller training data size may cause worse performance, so does \textit{real} with CVDD(3), which are similar phenomena with mSVDD. 
While for \textit{TREC}, almost all the target classes can be improved by using negative supervision.
In addition, the negative supervision also prevented over-fitting for CVDD. 
For example, CVDD(3) with the minimal parameters achieved the best score for \textit{comp} when varying ``\textit{r}'' among $3$, $5$ and  $10$. 
In contrast,    
when the negative supervision was used, CVDD(10) with the maximal parameters attained the best and also performed better for all six target classes of the \textit{20 Newsgroup} dataset.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Negative supervision using word replacement with TF-IDF}
\label{sssec:neg_tfreplace}

In the former parts, we have introduced that mSVDD can be further improved by incorporating negative supervision. 
And negative supervision was trained on \textit{pseudo}-negative samples that were randomly sampled sentences from a certain publicly available corpus, WikiText-2 \cite{merity2016pointer}. 
While this sentence-level sampling strategy can provide a wealth of out-of-domain samples with diverse semantics, it is not easy to control the generation of samples and the adaptability of diverse target classes settings over different datasets. 

Thus, we provide another method for \textit{pseudo}-negative construction without the use of out-of-domain corpus. 
This idea was motivated by the recent remarkable data augmentation techniques, especially by unsupervised data augmentation method \cite{xie2019unsupervised}. 
In general, we propose a word-level data augmentation method to construct \textit{pseudo}-negative samples from the only given the positive target training dataset.  

Our method is very simple. 
First, we assume that : 
1) The \textit{pseudo}-negative samples should have more common words to target samples than samples from external corpus in the one-class classification setting. 
2) While \textit{pseudo}-negative samples constructed should \textit{not} contain words that are more informative to the instances from the target class.  
Therefore, we proposed a method that replaces \emph{informative} words with high TF-IDF scores while keeping those with low values. 
Now, for a given target training dataset, this approach could construct \textit{pseudo}-negative samples on the basis of itself, rather than any other external corpus.    	 
	
Formally, for a word $w$ in one target training text sequence, we compute its TF-IDF value as TFIDF($w$)= TF($w$)*IDF($w$), 
where TF($w$) is the \textit{term frequency} score for word  $w$ in this sequence,
and IDF($w$) denotes the \textit{inverse document frequency} calculated on the only target training dataset. 
Specifically, we sample words to be replaced with weights according to the TF-IDF value, 
i.e., sampling with weights $p(w) \propto$ TFIDF($w$)/$Z$, where $Z$ is the sum of TF-IDF scores in one sequence.  
When a word is replaced, we randomly sample one word from the whole vocabulary for the replacement.
Once completed the replacement for one input sequence, we can get one corresponding \textit{pseudo}-negative sample. 
In the training,  for one batch of positive training samples, we randomly sampled its \textit{pseudo}-negative 
sample from the generated augmented set. 
Then, same as before, the constructed word-level \textit{pseudo}-negative samples were used for negative supervision. 

Table \ref{table:result_msvdd_with_neg_tfreplace} shows the results of mSVDD with negative supervision using word replacement with TF-IDF. 
In general, it shows that the proposed negative supervision  still works with the \textit{pseudo}-negative samples generated by the method of TFIDF replacement.  
We can see that mSVDD obtained improvements in most cases over the three datasets, 
as the same with the use of external dataset shown in Table \ref{table:result_msvdd_with_neg}.
Besides, for \textit{acq} and \textit{ship} on \textit{Reuters},  mSVDD still did not benefit from negative supervision. 
And \textit{ABBR} also did not show consistent improvements for four approaches. 
While differently, there were more obvious variations for four negative supervision approaches, especially on the the former two datasets.  
\textit{Con+Mean} strategy performed better than other three methods on ten target classes, except for \textit{ship} where this strategy degraded the performance of mSVDD. 
As for \textit{TREC} dataset, \textit{max} strategy would be the best choice for the \textit{triplet} loss type which beats other combinations on \textit{HUM} and  \textit{LOC}. 
While \textit{mean} strategy would be still beneficial to \textit{constrastive} type loss, as with other two datasets. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Affects of parameter $\boldsymbol{\gamma}$}
\label{sssec:gamma_test}

Parameter $\gamma$ in Eq. (\ref{eq:final_loss}) controls the trade-off between the intra-class loss mSVDD optimized and the discrimination loss negative supervision learned.  
In the former experiments on negative supervision using TFIDF word-level replacement method, we fixed $\gamma=1$.
Here, we would like to conduct experiments with varying $\gamma$ settings to verify how the trade-off affects the overall performance.
There are four negative supervision approaches that can be jointly trained with mSVDD.
As shown in Table \ref{table:result_msvdd_with_neg_tfreplace}, other three choices outperformed \textit{Triple+Max} in many cases. 
Thus we chose \textit{Triple+Max} for our sensitivity analysis experiments here.
We expected \textit{Triple+Max} can obtain improvements through the adjustment of $\gamma$. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[t]
\input{05table05.tex}
\hangcaption{Results of mSVDD with negative supervision using TFIDF replacement method. Settings are the same with Table \ref{table:result_msvdd_with_neg}.}
\label{table:result_msvdd_with_neg_tfreplace}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[t]
\begin{center}
\includegraphics{28-4ia5f4.pdf} 
\end{center}
%%%%\label{fig:msvdd_one_reuters_gamma}
%%%%\label{fig:msvdd_one_news_gamma}
%%%%\label{fig:msvdd_one_trec_gamma}
%%%%\label{fig:msvdd_soft_reuters_gamma}
%%%%\label{fig:msvdd_soft_news_gamma}
%%%%\label{fig:msvdd_soft_trec_gamma}
\hangcaption{Results of mSVDD using the method of \textit{Triple+Max} with  parameter $\gamma$ varying from $0.1$, $1$ to $10$.}
\label{fig:gamma_test}
\vspace{-0.5\Cvs}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Fig.~\ref{fig:gamma_test} illustrates the relationship of $\gamma$ and the evaluation of mSVDD+\textit{Triple+Max} on different target classes over three datasets.
As for the two versions of mSVDD on \textit{Reuters}, 
%%%%Figs. \ref{fig:msvdd_one_reuters_gamma} and \ref{fig:msvdd_soft_reuters_gamma}
    Figs.~\ref{fig:gamma_test}a and \ref{fig:gamma_test}d
show that a larger $\gamma$ (10 or 1) would be better setting on the former three target classes, especially on \textit{acq}.  
While on \textit{trade} or \textit{money-fx}, mSVDD achieved better performances with smaller value of $\gamma =0.1$. 
%%%%Figs. \ref{fig:msvdd_one_news_gamma}
    Figs.~\ref{fig:gamma_test}b
%%%%and \ref{fig:msvdd_soft_news_gamma} indicate that when $\gamma$ is set to 1 or 10, 
    and \ref{fig:gamma_test}e indicate that when $\gamma$ is set to 1 or 10, 
mSVDD can become slighter better for most target classes of  \textit{Newsgroup} dataset.  
While for \textit{rel} with smaller data size, mSVDD became worse by focusing more on negative supervision. 
We can observe a clear relationship between the increases of $\gamma$ and the improvements of mSVDD 
%%%%from Figs. \ref{fig:msvdd_one_trec_gamma} and \ref{fig:msvdd_soft_trec_gamma}. 
    from Figs.~\ref{fig:gamma_test}c and \ref{fig:gamma_test}f. 
These two subfigures show that mSVDD gained steady improvements over most target classes on \textit{TREC} for both mSVDD forms. 
Besides, the verification performance of the negative supervision remains stable across different settings of $\gamma$ over a wide range of target class settings. 

\newpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}
\label{sec:related_work}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{One-class learning}
\label{related:one-class-learning}
The first related area would be one-class learning or classification.
Generally, one-class classification approaches can be divided into the following groups: the density estimation-based methods, the boundary-based methods, and the reconstruction-based  methods \cite{taxphd,khan2009survey,khan2014one}.
\textit{Density estimation}-based methods, including early work uses different probability models \cite{parzen1962estimation,hardle1990applied,kim2012robust},  estimate the density of the training data and sets the threshold for the \textit{target} of high probabilities in the distribution. 
\textit{Reconstruction-based} methods, including AutoEncoder \cite{rumelhart1985learning} and  principal component analysis (PCA) \cite{pearson1901liii}, attempt to learn a more compact representation for the description of target data.
Most of these methods learn a model that is optimized to reduce the reconstruction error of normal data samples. 
CVDD \cite{ruff2019self}, that optimizes the one-to-one reconstruction distance between context vectors and self-attention feature vectors, can be viewed as a kind of  Reconstruction-based approach.
\textit{Boundary-based} methods learn only a boundary around the target set.
As introduced in Section \ref{sec:Preliminary}, OC-SVM seeks a hyperplane to separate the data from the origin, while SVDD finds a hypersphere to enclose the data.  
Their neural extensions, DSVDD \cite{ruff2018deep} and OC-NN \cite{Chalapathy2018}, can also optimize the related boundaries in the neural feature space. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Metric learning}
\label{related:metric learning}
The incorporation of negative supervision in Section \ref{sec:msvdd_neg} is related to metric learning \cite{guillaumin2009you}.
Specifically, in our proposed mSVDD with the negative supervision framework, 
\textit{pseudo}-negative samples are used to calculate the \textit{contrastive} and \textit{triplet} type losses. 
Through adding these losses to mSVDD, mSVDD can learn discriminative features that keep \emph{target} data points close, while pushing \textit{pseudo}-negative data apart in the feature space.
This shares a similar concept with metric learning, such as $\epsilon$-SVM, which minimizes the within-class distances by adding constraints to SVM \cite{do2012metric}.
In terms of using neural networks to learn features, our methods are more relevant to deep metric learning \cite{hoffer2015deep,kaya2019deep}.
Different from Mahalanobis metric learning \cite{weinberger2009distance}, that finds only a linear transform, recent work on deep metric learning can learn non-linear neural feature mapping, such that the distance of a positive pair is smaller than a negative one \cite{hu2014large,wu2017sampling,Sun_2020_CVPR}. 
Formally, deep metric learning methods usually define a distance between data points as $d(x_i, x_j)= \|f(x_i)-f(x_j)\|$, where $f(x)$ is an embedding of a sample $x$ learned by a neural network.  
Given three data points $a, p, n$, where $a$ is an anchor, $p$ is a positive of the same class as the anchor, and $n$ is a negative of a different class, the triplet loss that the deep metric learning optimized is: $\mathcal{L}= max\{0, d(a,p)- d(a,n) + margin \}$.
If we assume that the \emph{centers} in mSVDD are anchors, data from the target class are positive, and external samples are negative, the loss of mSVDD with negative supervision can be written in similar formulation. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Other related work}
The \textit{pseudo}-negative samples used in our negative supervision are from publicly available corpus, WikiText-2. 
We then use these external data to complete auxiliary tasks with \textit{contrastive} and \textit{triplet} losses. 
This can be seen as one kind of \textit{weakly supervision} method, which learns classification models from data that are incompletely or scarcely labeled \cite{lee2013pseudo,zhou2018brief}. 
Since the discriminative information is learned from external data that is not directly related to the original training data in our negative supervision,  
this also follows the idea from \textit{transfer learning} \cite{pan2010survey}.
\citeA{perera2019learning} proposed a much more related work, where they also incorporated the external image dataset to learn discriminative features for a one-class problem in the computer vision area.
As for the difference between our work \cite{perera2019learning}, the primary purpose of \cite{perera2019learning} is to learn features for one-class classification with two stages setting: \textit{feature learning} and \textit{k-nearest-neighbor classifier} usage. 
In contrast, our purpose is to design a specific structure for mSVDD such that it can be trained in an end-to-end neural structure.
Another related area is \textit{self-supervised learning}, that constructs kinds of \textit{contrastive learning} tasks for unsupervised representation learning \cite{khosla2020supervised,ohashi2020text,chen2020simple}.  
For example, in the contrastive learning framework used in Momentum Contrast (MoCo) \cite{he2020momentum}, they learned similar/dissimilar representations from data that are grouped into pairs by optimizing a so-called InfoNCE loss function \cite{oord2018representation}. 
The contrastive loss (Eq.  (\ref{eq:loss_contrastive_singlesample_reformulate})) used in our work can be viewed as a simplified InfoNCE loss function.   


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
\label{sec:conclusion}

In this work, we proposed mSVDD, a new generic one-class text classification framework that uses multi-modal deep SVDD.
Rather than the uni-modal deep SVDD, mSVDD can enhance the description ability to the target one-class data with multiple hyperspheres.
We also proved that this generic framework can include four variants, deep SVDD, DMSVDD, OC-NN, and CVDD under certain conditions.
In addition, in the absence of  ``\emph{real}'' negative training data, we also proposed approaches for effectively adding negative supervision to further improve the performance of one-class text classification methods. 
The experiments validated that the proposed mSVDD provides better performance compared to uni-modal SVDD. 
The experiments also showed the further improvements in most cases when negative supervision was used for mSVDD and CVDD.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

The authors would like to gratefully acknowledge the reviewers for their time and their valuable comments. Hu thanks the support from China Scholarship Council. 
This paper is an extended version of the paper \cite{hu2021one} published at the 16th Conference of the European Chapter of the Association for Computational Linguistics (EACL 2021).
The extended version introduces more baseline models in Section \ref{sec:Preliminary}, discusses their relationships with other models in  Section \ref{sec:msvdd}, and shows the evaluation results in Section \ref{sec:experiment}.
More experiments and discussions on the incorporation of negative supervision are also included.
Section \ref{sec:related_work} is newly added for the summary of related work. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\bibliography{05refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix 
\section{Proofs of Proposition 1}

\textbf{Proposition 1.} \textit{($\nu$-property )%\footnote{Rewrite this proposition in the main body} 
The hyperparamter $\nu \in (0,1]$ in soft-boundary deep mSVDD holds if we set an equal attention weight to each hypersphere: 
\begin{enumerate}
\item [i.] $\nu $ is an upper bound on the fraction of outlier samples.
\item [ii.] $\nu $ is a lower bound on the fraction of training samples being rejected or on the optimal boundary.
\end{enumerate}
}

\begin{proof}
\textbf{Ad (i).} 
For each training instance $\boldsymbol{x}_i$,  
its loss function is defined as \textit{hinge-loss}:  
$l(f(\boldsymbol{x}_i))=  \max \{ 0, \sum_{j} \alpha_{ij}\left( \|   \phi  ( \boldsymbol{x}_i; \mathcal{W} \right) - \boldsymbol{c}_j   \|^2 - R_{j}^{2} ) \}  $, where  $f$ is the model with parameters.
Let us define $d_{i}=\sum_j\alpha_{ij}d(\boldsymbol{x}_i,\boldsymbol{c}_j)$. 
Assume $\alpha_{ij} = 1/m $, we have $d_{i}=\frac{1}{m}\sum_j d(\boldsymbol{x}_i,\boldsymbol{c}_j)$.
We also define 
$R_s= \frac{1}{m}\sum_j R_{j}^2$.
And W.L.O.G, we also assume $ d_{1}\leqslant ... \leqslant d_{n}$ which means  $d_{n}$ is $n$-th farthest sum distance. 
The number of outliers  is  given by  $n_{out} = | \{ i | d_i > R_s \}  |$. 
Rewrite the objective of soft-boundary deep mSVDD ( Eq. (\ref{eq:msvdd_weight})) as: 
\begin{equation}
\mathcal{L}_{\scalebox{0.7}{soft-mSVDD}}=R_s  - \frac{n_{out}}{\nu n}R_s = (1 - \frac{n_{out}}{\nu n})R_s
\end{equation}
Since the objective of  mSVDD is to get a minimum $R_s$, therefore $1 - \frac{n_{out}}{\nu n}$ should be positive,
Thus, $n_{out} \leqslant \nu n  $ must hold in the training.
It implies that at most $\nu n $  outliers should be rejected. 

\textbf{Ad (ii).}   
The optimal $R_s^*$ has to hold the inequality $n_{out} \leqslant \nu n$. 
If $R_s^* >= d_n $, then $n_{out}$ takes the minimum value of $0$ which means the boundary includes all the samples. 
Since $n_{out}$ is increased as long as $R_s$ decreased. 
If $n_{out}$ take the maximum value of $\nu n$ under condition  \textbf{(i)}, we can have the minimal  $R_s^* = d_{i*}$, where $ i*=n - n_{out} $ means $d_{i*}$ is $(n - n_{out}) $-th farthest distance. 
We define $ \{ \boldsymbol{x}_i | d_i \geqslant R_s^* \}$ is the set of training samples being rejected ($ d_i > R_s^* $) or on the optimal boundary ($ d_i = R_s^* $). 
Then we have inequality: $ | \{ \boldsymbol{x}_i | d_i \geqslant R_s^* \}  | = | \{ \boldsymbol{x}_i | d_i > R_s^* \} \cup  \{ \boldsymbol{x}_i | d_i = R_s^* \}  |  \geqslant n_{out} +1 \geqslant  \nu n$. 
This implies that at least $\nu n$ samples being rejected or just on the optimal boundary. 
\end{proof}

\textbf{Proposition} \ref{prop:nu_property} and its proof refer to works \cite{ruff2018deep,chen2005tutorial,scholkopf2000new}. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Implementation of CVDD with Negative Supervision}
\label{app:sec:cvdd_neg}
CVDD uses a group of $r$ \textit{context vectors} $C =\left(\boldsymbol{c}_1,...,\boldsymbol{c}_r \right)$ to describe the target one-class data, where $\boldsymbol{c}_k \in \mathbb{R}^{p}$.
Given one context vector $\boldsymbol{c}_k$, and a pair of training positive and negative samples, $( x_i, x_{i'})$,
we can get the reformulated probability form.
First, CVDD maps a training sample $x_i$ to $r$ heads of feature vectors 
$S_i=\left( \boldsymbol{s}_{i1},...,\boldsymbol{s}_{ir}\right)$.
Then, we denote $p(y_i=1|\boldsymbol{s}_{{ik}},\boldsymbol{c}_k)$ as the probability that $k$-th $\boldsymbol{s}_{ik}$ reconstructs  $k$-th context vector $\boldsymbol{c}_k$ well.  
\begin{equation}\label{eq:prob_each_sample_cvdd}
p(y_i=1|\boldsymbol{s}_{ik}, \boldsymbol{c}_k) = \sigma (  \hat{\boldsymbol{s}_{ik}}^T  \hat{\boldsymbol{c}_k} )  
\end{equation}
And  with \textit{triplet} and  \textit{Max} probability strategy, we can define the negative supervision loss as:
\begin{equation}\label{eq:cvdd_neg_triple_max}
\begin{split}
\mathcal{L}_{Tri}^{(ii')} &=   
[  \log  p(y_{i'}=1|\boldsymbol{x}_{i'})-  \log  p(y_i=1|\boldsymbol{x}_i ) + \tau  ]_{+}  \\
&     =   [  \log  \mathop{\max}_{{k=1,...,r}}  p(y_{i'}=1|\boldsymbol{s}_{i'k},\boldsymbol{c}_k) -  \log  \mathop{\max}_{{k=1,...,r}} p(y_{i}=1|\boldsymbol{s}_{ik},\boldsymbol{c}_k)  + \tau ]_{+}   \\ 
\end{split}
\end{equation}
where  $\tau$ is a margin. Then, $ \mathcal{L}_{Tri}^{(ii')}$  can then be added to Eq. (\ref{eq:losscvdd}) to obtain the training loss.  


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{\textbf{Chenlong Hu}}{
He received his Ph.D. from Tokyo Institute of Technology.
Before that, he received M.E. from Xidian University, Xi'an. 
His research interests include natural language processing and machine learning.
}

\bioauthor[:]{\textbf{Yukun Feng}}{%
He is currently a Ph.D. candidate at Department of Information and Communications Engineering, Tokyo Institute of Technology. 
Before that, he received M.E. from Tokyo Institute of Technology in 2020 and B.E. from Beijing Language and Culture University in 2016. 
His research interests include natural language processing and machine learning, particularly word representation learning and language modeling.
}
\bioauthor[:]{\textbf{Hidetaka Kamigaito}}{He received his Ph.D. from Tokyo Institute of Technology, and is currently an assistant professor at Institute of Innovative Research, Tokyo Institute of Technology. 
His current research interests are in natural language processing with a specific focus on document-level text processing.
}
\bioauthor[:]{\textbf{Hiroya Takamura}}{
He received his Ph.D. from Nara Institute of Science and Technology.
He worked as a professor at Tokyo Institute of Technology, and currently is a research team leader at AI Research Center of Advanced Industrial Science and Technology.
His current research interests include natural language processing.
}
\bioauthor[:]{\textbf{Manabu Okumura}}{
Manabu Okumura was born in 1962. 
He received B.E., M.E. and Dr. Eng. from Tokyo Institute of Technology in 1984, 1986 and 1989 respectively. 
He was an assistant at the Department of Computer Science, Tokyo Institute of Technology from 1989 to 1992, and an associate professor at the School of Information Science, Japan Advanced Institute of Science and Technology from 1992 to 2000. 
He is currently a professor at Institute of Innovative Research, Tokyo Institute of Technology. 
His current research interests include natural language processing, especially text summarization, computer assisted language learning, sentiment analysis, and text data mining.
}
\end{biography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate




\end{document}

