\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{array}
\allowdisplaybreaks

\usepackage{hangcaption_jnlp}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition

% \usepackage{graphicx}
\usepackage{booktabs}
\usepackage{adjustbox}
\usepackage{colortbl}
\usepackage{multirow}
\usepackage{pifont}

%%%%%\newcommand{\cmark}{\textcolor{green!80!black}{\ding{51}}}
%%%%%\newcommand{\xmark}{\textcolor{red}{\ding{55}}}

%%%%%\usepackage{bm}

\definecolor{yellow-green}{rgb}{0.6, 0.8, 0.2}
\definecolor{bluegray}{rgb}{0.4, 0.6, 0.8}
\definecolor{orange-red}{rgb}{1.0, 0.27, 0.0}

\usepackage[linesnumbered,ruled,vlined]{algorithm2e}
% \newcommand\mycommfont[1]{\scriptsize\ttfamily\textcolor{blue}{#1}}
%%%%%\newcommand\mycommfont[1]{\small\ttfamily\textcolor{blue}{#1}}
%%%%%\SetCommentSty{mycommfont}

%%%%%\usepackage{afterpage}
%%%%%\usepackage{here}

%%%%%\usepackage{lipsum}
% \newcommand\blfootnote[1]{%
%   \begingroup
%   \renewcommand\thefootnote{}\footnote{#1}%
%   \addtocounter{footnote}{-1}%
%   \endgroup
% }

\makeatletter
\renewcommand{\algocf@capseparator}{~~}%
\makeatother

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{32}
\Number{1}
\Month{March}
\Year{2025}
%%%%\Issuetitle{「　　　　」特集号}

\received{2024}{7}{30}
\revised{2024}{11}{8}
\accepted{2024}{12}{13}



\setcounter{page}{252}



%%%%   Title, Author, etc.            %%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{DiLM: Distilling Dataset into Language Model \\for Text-level Dataset Distillation}
\eauthor{Aru Maekawa\affiref{TokyoTech} \and Satoshi Kosugi\affiref{TokyoTech} \and Kotaro Funakoshi\affiref{TokyoTech} \and Manabu Okumura\affiref{TokyoTech}}
\eabstract{Dataset distillation aims to compress a training dataset by creating a few informative synthetic samples such that the neural networks trained on them perform as best as those trained on the original training dataset. Current text dataset distillation methods create each synthetic sample as a sequence of word embeddings instead of text data to apply gradient-based optimization; however, such embedding-level distilled datasets cannot be used for training other models whose word embedding weights are different from the model used for distillation. To address this issue, we propose a novel text dataset distillation approach, called \emph{distilling dataset into language model (DiLM)}, which trains a language model to generate informative synthetic training samples as text data, rather than directly optimizing synthetic samples. We evaluated DiLM on various text classification datasets and showed that the distilled synthetic datasets from DiLM outperformed those from the current coreset selection methods. DiLM achieved remarkable generalization performance in training different types of models and in the in-context learning of large language models. Our code is available at \url{https://github.com/arumaekawa/DiLM}.\footnotetext{This paper was published in Findings of the Association for Computational Linguistics: NAACL 2024 \protect\cite{maekawa-etal-2024-dilm} and arXiv \protect\url{https://doi.org/10.48550/arXiv.2404.00264}.}
}

\ekeywords{Dataset Distillation, Language Model, Text Generation}

\headauthor{Maekawa et al.}
\headtitle{DiLM: Distilling Dataset into Language Model for Text-level Dataset Distillation}



%%%%     Affiliation     %%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\affilabel{TokyoTech}{}{Institute of Science Tokyo}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S1
\section{Introduction}

There have been successful advancements in machine learning (ML) in a wide range of fields owing to the scaling up of deep neural networks and large training datasets. 
In the field of natural language processing (NLP), large language models (LLMs) that are pre-trained with a large amount of text, such as BERT- and GPT-family models \cite{devlin-etal-2019-bert,liu-2019-roberta,radford2019language,brown2020gpt3}, have exhibited remarkable capabilities for various NLP tasks. 
However, training such large-scale models requires considerable computational resources and time, making it difficult to develop new LLMs and even fine-tune them.

To address this issue, dataset distillation \cite{wang2018dataset}, which aims to reduce training costs by compressing training datasets, has garnered significant attention from the ML community. 
In contrast to traditional coreset selection approaches \cite{wolf2011facility,sener2018active,welling2009herding} that heuristically select a small subset of representative training samples from the original dataset, dataset distillation creates more informative synthetic samples by distilling knowledge from the original dataset. 
Using this approach, synthetic samples are optimized using gradient descent according to the objective functions for dataset distillation, including meta-learning \cite{wang2018dataset}, gradient matching \cite{zhao2021dataset}, training trajectory matching \cite{Cazenavette_2022_CVPR}, and feature distribution matching \cite{wang2022cafe,zhao2023distribution}.
The recent remarkable performance of dataset distillation, particularly in the field of computer vision (CV), has led to studies on its various applications, including neural architecture search \cite{pmlr-v119-such20a,medvedev2022learning}, federated learning \cite{zhang2022dense,xiong2023feddm}, continual learning \cite{wiewel-2021-condensed-conposite-memory,sangermano-2022-sample-condensation}, and privacy preservation \cite{pmlr-v162-dong22c,chen-2022-private-set}. 

Although most previous studies have applied dataset distillation only to image classification datasets, some have focused on text dataset distillation \cite{sucholutsky-2021-soft-label,li-2021-data-distillation,maekawa-etal-2023-dataset,sahni2023multilingual}. 
In contrast to images, which can be applied to gradient-based optimization by considering them as continuous pixel-wise data, the discrete nature of text makes dataset distillation challenging \cite{Geng2023survey,yu2023review}. 
To address this issue, all existing text dataset distillation methods employ a widely used neural NLP technique known as embedding, which optimizes a synthetic dataset as continuous input word embeddings instead of discrete text. 
However, such embedding-level distilled synthetic datasets cannot be used to train other models with different word embedding weights, which is a crucial issue in terms of practical applications. 
Furthermore, distilled word embedding sequences are completely unreadable by humans, making it difficult to interpret and analyze the original training dataset by observing distilled synthetic samples. 

Motivated by these shortcomings, this study is the first to explore text dataset distillation to obtain distilled synthetic datasets at the text level. 
We propose the first text-level dataset distillation approach called ``\textbf{D}istilling dataset \textbf{i}nto \textbf{L}anguage \textbf{M}odel (\textbf{DiLM})''. 
To overcome the difficulty of optimizing discrete text, DiLM uses a language model as a surrogate continuous optimization target, rather than directly optimizing the text of a synthetic sample.
Specifically, the DiLM trains a language model to minimize the gradient matching loss \cite{zhao2021dataset} of the generated synthetic samples as a dataset distillation objective.
To enable backpropagation of the gradient matching loss to the language model, we design a differentiable backward pass via loss weighting with generation probabilities to bypass the non-differentiable generated text (Figure \ref{fig:overview}).



%%%%%%%%%%%%%%%%%%%%%%%%%%%%F1
\begin{figure*}[b]
{\vskip -9pt}
\begin{center}
\includegraphics{32-1ia9f1.eps}
\end{center}
    \hangcaption{Overview of training with DiLM. Gradient matching loss is computed on the learner model between real samples from the original dataset and generated samples from the generator model. Thereafter, it is backpropagated to the generator model via generation probabilities, which weight the learner loss for each generated sample. (a) Representative teacher for computing real sample's gradients, improving the performance and accelerate convergence using K-center samples; it represents the original dataset, instead of randomly sampled ones. (b) Diverse mini-batch sampling, enabling the generator model to explore diverse synthetic samples in each training step.}
    \label{fig:overview}
\end{figure*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



In our experiments, we applied DiLM to distill three text classification datasets from the GLUE benchmark \cite{wang-etal-2018-glue}: SST-2, QQP, and MNLI-m.
The results indicated that the synthetic datasets distilled with DiLM outperformed the representative real samples selected from the original datasets with the current coreset selection methods. 
Our distilled datasets also achieved remarkable generalization performance not only for training different types of pre-trained models but also for in-context learning of LLMs as few-shot prompts.

The main contributions of this study are as follows:
%%%%%
\begin{itemize}
    \item To the best of our knowledge, this is the first study to distill a text dataset into a text-level synthetic dataset that are applicable for training models independent of word embedding weights.
    \item We propose DiLM, which addresses the discreteness of text by using a language model as a surrogate optimization target and backpropagating the distillation loss to the model, bypassing non-differentiable generated text.
    \item Our experimental results indicate that DiLM outperformed the current coreset selection methods not only for training the same model used for distillation but also for training different models independent of the word embedding weights, architectures, and training processes.
\end{itemize}
%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2
\section{Related Work}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.1
\subsection{Coreset Selection}

Coreset selection or data selection is a traditional approach for training dataset reduction in the NLP and ML fields.
Coreset selection aims to extract a subset of effective samples from the original training datasets according to heuristic criteria.
For instance, K-centers \cite{wolf2011facility,sener2018active} construct a representative subset by clustering the original training data and selecting centroid samples. Herding \cite{welling2009herding,chen2010herding} selects samples in a greedy manner to minimize the distance in the feature space between the center of the selected subset and that of the entire original dataset. 
CRAIG \cite{pmlr-v119-mirzasoleiman20a} finds a coreset that minimizes the difference in gradients computed by the coreset and the original dataset with the same network.
In the NLP field, \citeA{moore-lewis-2010-intelligent} proposed selecting samples by comparing the cross-entropy between domain- and non-domain-specific language models.

Coreset selection methods effectively reduce model training costs and have proven useful in several applications such as active learning \cite{sener2018active} and continual learning \cite{aljundi2019gradient}.
However, the coreset selection depends on the presence of representative samples useful for model training in the original dataset.
Therefore, their performance is inherently restricted when reducing datasets to an extremely small coreset, whereas they work well for approximately 10\% of the original dataset size.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.2
\subsection{Dataset Distillation}
Dataset distillation was proposed by \cite{wang2018dataset} as an alternative approach to training dataset reduction.
Dataset distillation aims to create extremely small number of synthetic samples that are more informative than the real samples in the original datasets and that are effective for model training.
Dataset distillation has garnered considerable attention because of its theoretical interest and practical applications for reducing network training costs.
Inspired by meta-learning-based hyperparameter optimization \cite{maclaurin2015gradient}, \citeA{wang2018dataset} optimized a small synthetic dataset by gradient descent, such that models trained on it have a lower training loss for the original dataset.
Several surrogate objectives have recently been proposed to improve the performance and efficiency of dataset distillation.
DC \cite{zhao2021dataset} and DSA \cite{pmlr-v139-zhao21a} focused on the gradient matching between real and synthetic samples.
DM \cite{zhao2023distribution} and CAFE \cite{wang2022cafe} proposed feature distribution matching, which requires less GPU memory to optimize synthetic datasets.
MTT \cite{Cazenavette_2022_CVPR} and TESLA \cite{pmlr-v202-cui23e} optimized synthetic samples to approximate the trajectories of the model parameters trained with real data. 
SLDD \cite{sucholutsky-2021-soft-label} and LDD \cite{bohdal-2020-flexible-dataset} introduced learnable soft labels that were optimized together with the input images to make each synthetic sample more informative.

Although most current studies on dataset distillation involve only image classification datasets, some studies have focused on text classification datasets. 
\citeA{sucholutsky-2021-soft-label} and \citeA{li-2021-data-distillation} applied the original meta-learning based method by \citeA{wang2018dataset} to text datasets. 
To overcome the discrete nature of text, which makes it difficult to apply gradient-based methods, they optimized synthetic samples in a pre-trained GloVe word embedding space {\linebreak}\cite{pennington-etal-2014-glove} rather than using actual words of text as the optimization target. 
\citeA{maekawa-etal-2023-dataset} extended the text dataset distillation to the pre-trained BERT model and improved its performance by introducing learnable attention labels, which directly guide the self-attention probabilities of the models.
\citeA{sahni2023multilingual} explored dataset distillation in multilingual text classification datasets in the context of fairness, interpretability, and cross-architecture generalization. 
Although these methods perform well for text classification datasets, distilled synthetic datasets obtained with them cannot be used for training other models that have different word embedding weights.
Although \citeA{sucholutsky-2021-soft-label} and \citeA{sahni2023multilingual} transformed their distilled synthetic samples into text by finding the word with the nearest neighbor embedding, the converted text comprised unrelated words and did not make sense, which made it difficult to interpret and analyze them. 
Moreover, the performance of distilled datasets after their conversion to text has not been investigated.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.3
\subsection{Generative Models}
\label{sec:related_work/generative_models}
Recent studies on dataset distillation in the CV field have used generative adversarial networks (GANs) \cite{ian2014gan}, that is, training model parameters and/or their latent input noises, instead of synthetic images.
These methods generalize the distilled synthetic images to different model architectures by restricting them to the generative distribution learned from the original dataset.
DiM \cite{wang2023dim} fine-tuned a GAN to generate informative synthetic images from randomly sampled latent noise, where distilled datasets of different sizes were produced without retraining the model.
GTNs \cite{pmlr-v119-such20a} trained a GAN to generate informative images instead of realistic images to accelerate the neural architecture search. 
GTNs also learned the latent noise for each synthetic image as a curriculum for training the learner networks.
IT-GAN \cite{zhao2022itgan} and GLaD \cite{cazenavette2023generative} used a pre-trained GAN as a generative prior for synthetic samples and only optimized the latent noise.

Inspired by these studies, we also introduce a generative model with a different motivation for text dataset distillation. We optimize the continuous parameters of a generative model to generate distilled synthetic samples to avoid the difficulties of directly optimizing discrete text.
However, because all the previous studies that used generative models for image dataset distillation trained them and/or their input latent noises by backpropagating the distillation loss to them via generated images, none of them can be applied to text data, which are non-differentiable owing to their discrete nature.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3
\section{Methodology}

In this section, we introduce DiLM, which distills text datasets into text data, not word embeddings, for model-agnostic applicability and interpretability of the distilled synthetic datasets. 
The main idea of DiLM is to avoid the optimization difficulties of discrete text by training the continuous parameters of a language model as a surrogate optimization target for dataset distillation. 



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.1
\subsection{Overview}
Given a training dataset $\mathcal{D}_\text{real} = \{x_i\}_{i=1}^{|\mathcal{D}_\text{real}|}$, the goal of DiLM is to obtain a generator model, parameterized by $\phi$, that generates a distilled synthetic dataset $\mathcal{D}_\text{syn} = \{\tilde{x}_i\}_{i=1}^{|\mathcal{D}_\text{syn}|}$ ($|\mathcal{D}_\text{syn}| \ll |\mathcal{D}_\text{real}|$), such that a learner model, parameterized by $\theta$, trained on $\mathcal{D}_\text{syn}$ performs well.
To achieve this goal, the overall procedure of DiLM comprises the following three steps:
%%%%%
\begin{enumerate}
\item We first simply train the generator model to generate synthetic training samples that belong to the same distribution as in the original dataset $\mathcal{D}_\text{real}$ (Section \ref{sec:text_generation_model}). 
\item Thereafter, we fine-tune the generator model to generate ``informative'' training samples by minimizing the gradient matching loss between generated and real samples (Section~\ref{sec:text_gtn}). 
\item We obtain distilled dataset $\mathcal{D}_\text{syn}$ by generating synthetic samples with the generator model and selecting representative samples from them by using a clustering-based coreset selection method (Section \ref{sec:sample_selection}). 
\end{enumerate}
%%%%%
Details of each step are described in the following sections.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2
\subsection{Synthetic Training Data Generation with Language Model}
\label{sec:text_generation_model}

Inspired by the remarkable text generation capability of pre-trained transformer language models \cite{radford2019language}, we use them as the generator model to generate synthetic training samples of sufficient quality for use in the training models. 
Before training the generator model to generate synthetic samples that are more informative than the real samples in the original dataset, we first train a language model to generate training samples that belong to the same distribution as the original training dataset for the initial parameters of the generator model. 

When text classification tasks are targeted, the generator model must be controlled to generate samples for each class. 
Therefore, we introduce class-specific beginning-of-sentence tokens \verb|<bos_i>|, which are added to the head of each training sample to train the generator model to generate samples of the corresponding class following it.
For each training sample, an end-of-sentence token \verb|<eos>| is also added, and the sample is fed to the generator model as follows:
%%%%%
\begin{quote}
    \verb|<bos_i>| \textit{sentence of class $i$} \verb|<eos>|.
\end{quote}
%%%%%
To involve text classification tasks that specify the relation between two sentences, such as semantic similarity and natural language inference (NLI), we use a separate token \verb|<sep>| to split two sentences as
%%%%%
\begin{quote}
    \verb|<bos_i>| \textit{sentence~1} \verb|<sep>| \textit{sentence~2} \verb|<eos>|.
\end{quote}
%%%%%

The generator model is trained using the language modeling loss $l_\phi(x_i)$ as
\begin{align}
    l_\phi(x_i) = - \frac{1}{|x_i|}\sum_{t=1}^{\|x_i|} \log p_\phi(w_t|w_{<t}),
\end{align}
where $w_t$ denotes the $t$-th token in $x_i$ and $|x_i|$ denotes the length of $x_i$. In this manner, we pre-train the generator model parameters $\phi$ to generate synthetic training data, such as real data, and use them as the initial parameters for training for gradient matching, as described in the following section.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.3
\subsection{Training for Gradient Matching}
\label{sec:text_gtn}

In this section, we explain how to fine-tune the pre-trained generator model, described in Section \ref{sec:text_generation_model}, to generate synthetic training samples that are more informative than the real samples in the original dataset.
Specifically, we describe gradient matching, an optimization objective for dataset distillation, and a model updating procedure to address text discreteness.
We also introduce two techniques to improve DiLM: representative teacher and diverse mini-batch sampling.

%%%%%
\noindent\textbf{Gradient Matching.}
To distill the knowledge of the original dataset $\mathcal{D}_\text{real}$ into synthetic samples generated by the generator model, we optimized the gradient matching loss \cite{zhao2021dataset} as the objective for dataset distillation.
Given a mini-batch of real samples $\{x_i\}_{i=1}^M$ and a mini-batch of synthetic samples $\{\tilde{x}_i\}_{i=1}^N$, which are generated from the generator model, the gradient matching loss $\mathcal{L}_\text{GM}$ on the learner model parameters $\theta$ is calculated as follows:
%%%%%
\begin{align}
    \begin{split}
    \mathcal{L}_\text{GM} &= D\left(
        \nabla_\theta \mathcal{L}_\text{real}, \nabla_\theta \mathcal{L}_\text{syn}
    \right)\label{eq:gradient_matching_loss} \quad \text{where} \\
    \mathcal{L}_\text{real} &= \frac{1}{M}\sum_{i=1}^M l_\theta(x_i), \ 
    \mathcal{L}_\text{syn} = \frac{1}{N}\sum_{i=1}^N l_\theta(\tilde{x}_i),
    \end{split}
\end{align}
%%%%%
where $l_\theta(\cdot)$ denotes the loss function for learning tasks, such as cross-entropy loss, and $D(\cdot,\cdot)$ denotes the cosine similarity-based distance function, expressed as follows:
%%%%%
\begin{align}
    D(A, B) = 1-\frac{A\cdot B}{\|A\|\|B\|}.
    \label{eq:distance_function}
\end{align}
%%%%%
Following a previous study \cite{zhao2021dataset}, we separately calculated the gradient matching loss for each class and combined them to update the generator model parameters $\phi$. 
To consider the gradient of the learner model parameters $\theta$ throughout the training process, the generator model is trained using the nested loop algorithm, including the outer loop, which initializes $\theta$ at the beginning, and the inner loop, which updates $\theta$ for $K$ steps with real samples (see Algorithm~\ref{alg:text_gtn}).



%%%%%%%%%%%%%%%%%%%%%%%A1
\begin{algorithm}[t]
\input{09algo1.tex}
    \caption{Optimization for DiLM}
    \label{alg:text_gtn}
\end{algorithm}
%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%
\noindent\textbf{Generator Updating.}
As described in Section \ref{sec:related_work/generative_models}, the gradient matching loss $\mathcal{L}_\text{GM}$ cannot be directly backpropagated to the generator model parameters $\phi$ through the generated samples $\{\tilde{x}_i\}_{i=1}^N$, as in the case of image datasets, because they comprise discrete text.
Although some solutions to the discrete backpropagation issue in text generation have been explored in the NLP research field, most standard approaches, including soft-argmax \cite{zhang-etal-2017-textgan} and policy gradient \cite{yu-etal-2017-seqgan}, cannot be applied to this case (see details in Appendix \ref{sec:appendix/background}).
To address this issue, we design an alternative backward pass inspired by a previous study \cite{hiraoka-etal-2020-optimizing} that optimizes a tokenization model for the loss of the downstream task through a non-differentiable procedure.
When computing the generated sample loss $\mathcal{L}_\text{syn}$, rather than simply averaging the losses for each generated sample, as in Eq. \eqref{eq:gradient_matching_loss}, we weigh them using their generation probabilities $p_\phi(\tilde{x}_i)$ as follows:
%%%%%
\begin{align}
    \mathcal{L}_\text{syn} = \sum_{i=1}^N a_i \, l_\theta(\tilde{x}_i),\\
    a_i = \frac{p_\phi(\tilde{x}_i)}{\sum_{j=1}^N p_\phi(\tilde{x}_j)}.
\end{align}
%%%%%
Therefore, $\mathcal{L}_\text{GM}$ can be backpropagated to $\phi$ through the differentiable pass via the loss weights $a_i$, as illustrated in Figure \ref{fig:overview}. 
Intuitively, the generator model is updated to increase the generation probabilities of synthetic samples that improve gradient similarity. 

%%%%%
\noindent\textbf{Representative Teacher.}
To improve DiLM, we consider enhancing the gradient teacher of real samples using representative samples for each mini-batch of real samples instead of randomly selected ones.
Inspired by \citeA{liu2023dream}, we select representative samples with K-centers \cite{wolf2011facility,sener2018active}, which is a clustering-based coreset selection method (Figure \ref{fig:overview}a).
Specifically, we divide all real training samples for each class into $M$ sub-clusters using the K-means algorithm in the feature space of the learner model and select the center sample of each sub-cluster.
As shown in \citeA{liu2023dream}, the representative samples selected by the K-centers provide the proper teacher gradient by including diverse samples that cover the overall distribution for each class and eliminating samples near the decision boundaries, which have dominant gradients with large norms.
Considering coverage and robustness, we generate 10 representative sample sets by running the K-means algorithm with different random seeds at the beginning of the training.
In each training step, one of the representative sample sets is randomly selected and used as a mini-batch of real samples.\footnote{\citeA{liu2023dream} repeatedly re-generated the K-center representative samples by performing clustering on the feature space of the different states of the learner model throughout the inner loop. However, this is time consuming with BERT as the learner model, as in our study.}

%%%%
\noindent\textbf{Diverse Mini-batch Sampling.}
The diversity in a mini-batch of generated samples for each step affects the sample space explored by the generator model during training. 
If the generator model generates only several samples that are similar to each other, this leads to a biased optimization of the generator model.
To address this issue, we introduce diverse mini-batch sampling of the generated samples in the training process of DiLM (Figure \ref{fig:overview}b).
Rather than generating $N$ synthetic samples for each step, the generator model simultaneously generates $N\times I_\text{int}$ synthetic samples, where $I_\text{int}$ denotes the generation interval. The generated synthetic samples are then divided into $N$ sub-clusters using the K-means algorithm, and a mini-batch of synthetic samples for each step is constructed by randomly selecting one sample from each sub-cluster.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.4
\subsection{Generate Synthetic Dataset}
\label{sec:sample_selection}
We obtained a distilled dataset $\mathcal{D}_\text{syn}$ by generating synthetic samples using a trained generator model.
To include representative samples of the generative distribution $p_\phi(\tilde{x})$ of the model, we use the coreset selection method again to select the generated synthetic samples.
Specifically, we generate 100 times as many synthetic samples as the distilled dataset size $|\mathcal{D}_\text{syn}|$ by top-$p$ sampling with $p=0.95$, considering the diversity, and subsequently construct $\mathcal{D}_\text{syn}$ using the K-center representative samples.
This causes $\mathcal{D}_\text{syn}$ to include diverse synthetic samples by removing redundant samples caused by the biased generative distribution of the model.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4
\section{Experimental Setup}

%%%%%
\noindent\textbf{Datasets.}
We evaluated DiLM by distilling three major text classification datasets, SST-2, QQP, and MNLI-m, from the GLUE benchmark \cite{wang-etal-2018-glue}. 
Following \citeA{wang-etal-2018-glue}, we report the accuracy for SST-2 and MNLI-m and the average accuracy and F1 score for QQP.
More details on each dataset are provided in Appendix \ref{sec:appendix/datasets}.

%%%%%
\noindent\textbf{Baselines.} Following previous studies on dataset distillation in the CV field, we compared the performance of DiLM with three coreset selection methods: Random, K-centers \cite{wolf2011facility,sener2018active}, and herding \cite{welling2009herding}, as well as TDD \cite{sucholutsky-2021-soft-label}, which is a recent embedding-level distillation method.
The TDD also trains the learnable soft labels and learning rates for each training step, along with the input word embeddings.
We also evaluated the vanilla LM, which skips training for gradient matching (in Section \ref{sec:text_gtn}), to validate its effectiveness. Note that we also applied the K-center representative sample selection (in Section \ref{sec:sample_selection}) to the vanilla LM.
Although reducing the model size may be considered a baseline, it addresses a different challenge, and dataset distillation tackles issues that model compression cannot resolve. 
Moreover, reducing the model sufficiently for training with the entire original dataset at a computational cost comparable to DiLM and coreset methods is impractical.
The details of each baseline are provided in Appendix \ref{sec:appendix/baselines}.

%%%%%
\noindent\textbf{Evaluation.} We used BERT$_\text{BASE}$, RoBERTa$_\text{BASE}$, BERT$_\text{LARGE}$, and XLNet$_\text{BASE}$ as learner models for the evaluation (see more details in Appendix \ref{sec:appendix/learner_models}).
We trained a learner model on distilled datasets for 200 steps using AdamW \cite{loshchilov2019adamw} with a learning rate of $1.0\times10^{-4}$ and a batch size of 64.\footnote{We did not follow this training protocol for TDD, because TDD optimizes learning rates as well for each step with a specific synthetic sample order.} 
For Herding and TDD, we trained the learner model on their datasets 100 times each. For the other methods, we generated 20 datasets with different random seeds and trained the learner models five times.
We report the averages and standard deviations of 100 models.
In the result tables, `$*$' indicates significant difference of DiLM from K-centers ($p < 0.05$, Welch's t-test). Note that the standard deviations in our results inevitably increased because we trained the models with few selected/generated samples from different initial model parameters. However, our evaluation procedure, which includes 100 runs, supports the reliability of our experimental results for discussing the effectiveness of the proposed method.

%%%%%
\noindent\textbf{Implementation.}
We used the 128M parameter version of GPT-2\footnote{\url{https://huggingface.co/gpt2}} \cite{radford2019language} as the generator model of DiLM,
and used BERT$_\text{BASE}$ \cite{devlin-etal-2019-bert} as the learner model to calculate the gradient matching loss.
To reduce computational costs, we calculated the gradient matching loss only for the randomly initialized last layer parameters, which tend to have a dominantly larger gradient than the pre-trained parameters.
We set the number of loops for training DiLM to $S=2000$, $T=10$, and $K=20$, and the generation interval to $I_\text{int}=200$ according to our preliminary experiments.
The mini-batch sizes of the real and synthetic samples were respectively set to $M=200$ and $N=64$.
Training DiLM took 1 d on an RTX 3090 (24~GB) for SST-2, 1.5 d on an A100 PCIe (80~GB) for QQP, and 2.5 d on an A100 PCIe (80~GB) for MNLI-m.
Further details of our implementation are provided in Appendix \ref{sec:appendix/implementation_details}.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5
\section{Results and Discussion}
\label{sec:results}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1
\subsection{Performance for BERT$_\text{BASE}$}

As summarized in Table \ref{tab:main_results}, we first compared DiLM with the other baselines for training BERT$_\text{BASE}$, on which DiLM trained the gradient matching. 
They were evaluated for different sizes of distilled synthetic datasets using 5/10/20 DPC settings.



%%%%%%%%%%%%%%%%%%%T1
\begin{table*}[b]
\input{09table01.tex}
    \hangcaption{Performance comparison of DiLM with the coreset selection methods and TDD for training BERT$_\text{BASE}$ model. \colorbox{yellow-green!25}{Green} highlighted results indicate that DiLM outperformed the coreset selection methods. \colorbox{orange-red!25}{Red} highlighted results indicate the performance degradation of distilled datasets from TDD after being converted to text. Note: experiments on TDD with larger data-per-class (DPC) settings are not conducted owing to GPU memory restrictions.}
    \label{tab:main_results}
\end{table*}
%%%%%%%%%%%%%%%%%%%



The vanilla LM, which was trained only for synthetic training sample generation without gradient matching, clearly underperformed the coreset selection methods. 
This indicates that, as expected, the quality of the generated synthetic samples was lower than that of the real samples in the original datasets.
However, DiLM, which fine-tuned the vanilla LM with gradient matching, improved its performance and outperformed the coreset selection methods overall. 
Note that the performance gains from the K-centers indicate that DiLM generated synthetic training samples that were more effective for model training than the real samples in the original datasets.

Focusing on the differences among the three datasets, the performance gains of DiLM on QQP and MNLI-m were larger than those on SST-2.
We believe that this is because QQP and MNLI-m, which are tasks used to specify the relationship between two sentences, are intuitively less likely to have real samples that represent the task than SST-2, which is a relatively simple negative/positive classification task.
In addition, it may also be related to the size of the original training dataset of QQP and MNLI-m, which is five times larger than that of SST-2.
Because the generator model was trained by gradient matching with self-generated synthetic samples, it can explore broader sample space by pre-training with the original dataset that contains sufficiently diverse samples, which results in the effective performance of DiLM.

For the TDD, we evaluated its distilled datasets as text data by converting them into discrete tokens with nearest neighbor embeddings.
When directly using distilled datasets as word embeddings, the TDD achieved remarkable performance, even when compared with the full datasets. 
However, after conversion to text, its performance degrades catastrophically, even to lower-bound performances with random prediction.
This suggests that the distilled datasets from the TDD are strictly overfitted at the word embedding level and cannot be converted into text without acceptable performance degradation, which is necessary for applying them to other models.
This is a clear advantage of DiLM, which distills synthetic datasets at the text level.




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T2
\begin{table}[b]
\input{09table02.tex}
    \hangcaption{Cross-model generalization performance for settings of DPC = 20. (\textbf{S}) indicates the source model for gradient matching of DiLM and feature extractor for K-centers.}
    \label{tab:cross_model_generalization}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%





%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.2
\subsection{Cross-model Generalization}

In contrast to the current embedding-level distillation methods, text-level synthetic datasets from DiLM can be leveraged 
for training different models independent of their word embedding weights. 
To emphasize this advantage, we evaluated the distilled synthetic datasets for training three models that were different from BERT$_\text{BASE}$, with which the distilled synthetic datasets were obtained, 
that is, RoBERTa$_\text{BASE}$, BERT$_\text{LARGE}$, and XLNet$_\text{BASE}$.
Table \ref{tab:cross_model_generalization} summarizes the performances of Random, K-centers, and DiLM with DPC = 20, where DiLM achieved stable and good performance.\footnote{We also show the results with other DPC settings in Appendix \ref{sec:appendix/additional_results}.} 
The results indicate that the distilled datasets from DiLM consistently performed well for training the different models, even though DiLM trained the gradient matching only for the BERT$_\text{BASE}$ model parameters. 
Notably, our distilled datasets exhibit successful generalization performance not only for training RoBERTa$_\text{BASE}$ and BERT$_\text{LARGE}$, which have the same model architecture as BERT$_\text{BASE}$, but also for training XLNet$_\text{BASE}$, which is an autoregressive model using the hidden state of the \verb|<eos>| token for classification, whereas BERT$_\text{BASE}$ is an autoencoding model using the hidden state of the \verb|[CLS]| token.

We also evaluated the distilled datasets from DiLM as few-shot prompts for the in-context learning of LLMs.
Table \ref{tab:few-shot} summarizes the performances of Random, K-centers, and DiLM for in-context learning for SST-2 with three different sizes of LLMs: GPT-2-XL \cite{radford2019language}, OPT \cite{zhang2022opt}, and Llama~2 \cite{touvron2023llama2}.
Surprisingly, the distilled datasets from DiLM consistently performed well for in-context learning compared with Random and K-centers.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T3
\begin{table}[t]
\input{09table03.tex}
    \hangcaption{Performance of distilled datasets as five-shot prompts for in-context learning of SST-2. Each score is the average and standard deviation for 100 prompts with 20 distilled datasets and five random orders.}
    \label{tab:few-shot}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



These remarkable generalization performances across the models and training processes {\linebreak}strongly support the advantage of DiLM to distill datasets at the text level.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.3
\subsection{Analysis and Discussion}

%%%%%
\noindent\textbf{Ablation Study.} Table \ref{tab:ablation} summarizes the results of the ablation study for the performance improvement techniques of the representative teacher for gradient matching, diverse mini-batch sampling of synthetic samples during the training of DiLM (in Section \ref{sec:text_gtn}), and representative sample selection with K-centers during synthetic dataset generation (in Section \ref{sec:sample_selection}).
The results demonstrate that all three techniques were consistently effective for DiLM.



%%%%%%%%%%%%%%%%%%%%%%%%%%%T4
%%%%%\newcolumntype{Y}{>{\centering\arraybackslash}p{4.5em}}
%%%%%\newcolumntype{C}{>{\centering\arraybackslash}p{2.5em}}
\begin{table}[t]
\input{09table04.tex}
\hangcaption{Ablation study on the performance improvement techniques of DiLM with the DPC = 5 setting. RT, DMS, and Selection indicate representative teacher, diverse mini-batch sampling, and sample selection with K-centers, respectively.}
\label{tab:ablation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



\noindent\textbf{Scaling of DPC.}
We investigated the performance of DiLM by increasing the size of the synthetic datasets.
Note that DiLM does not require retraining of the generator model to generate distilled synthetic datasets for different DPCs, which is also an advantage of using generative models for dataset distillation.
As shown in Figure \ref{fig:graph_dpc}, the performance of the distilled datasets is generally scaled with increasing DPC. 


%%%%%
\noindent\textbf{Distilled Data Examples.}
We provided examples of distilled synthetic samples for each dataset in Appendix \ref{sec:appendix/examples}.
We discovered that DiLM successfully generated interpretable synthetic samples that were appropriate for the tasks of the original datasets.
Although DiLM consistently generated high quality synthetic samples for SST-2 and QQP, a repetition problem was observed in some lengthy samples for MNLI-m.
This suggests that room still exists for performance improvement of DiLM using a larger and more sophisticated pre-trained language model for the generator model than the small GPT-2 used in our current experiments.

We also investigated the proportion of synthetic data generated by DiLM that exactly matches the samples in the original datasets.
As summarized in Table \ref{tab:original_example}, most of the synthetic data generated by DiLM are novel samples that are not present in the original dataset. 
We further compared these results with the performances in Table \ref{tab:main_results}, particularly in relation to the performance gains from the K-centers.
A larger number of novel samples correlated with a larger performance improvement achieved by DiLM over K-centers.
This suggests that the ability of DiLM to generate synthetic data that are more useful than real data contributes to the performance improvement of dataset distillation. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6
\section{Conclusions}

This study proposed the first text-level dataset distillation approach, called DiLM, which trains a language model to generate informative synthetic samples as text data for model-agnostic applicability and interpretability of distilled datasets.
Experimental results across various text classification datasets indicated that the distilled datasets from DiLM achieved successful performance in training various types of models beyond the source model used for distillation, even for the in-context learning of LLMs.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F2
\begin{figure}[t]
\begin{center}
\includegraphics{32-1ia9f2.eps}
\end{center}
    \hangcaption{Performance for increasing the number of synthetic samples with DPC $\in\{1,5,10,20,50,100, 200\}$. The mean and 95\% confidence interval for 100 models trained on distilled datasets from DiLM are plotted.}
    \label{fig:graph_dpc}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T5
\begin{table}[t]
\input{09table05.tex}
    \hangcaption{Portion of the synthetic data generated with DiLM that matches the samples in the original training dataset.}
    \label{tab:original_example}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%





%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S7
\section{Limitations}
The limitations of this study are as follows.
(i) Although DiLM achieved remarkable performance as a text-level distillation method, there was still a performance gap compared to that of full datasets.
However, the performance improvements from K-centers are sufficiently large to demonstrate the effectiveness of DiLM, considering the fundamental difficulty of the dataset distillation problem in cases where synthetic data are restricted to the text level. Moreover, DiLM has the potential for further performance improvement by employing larger and more sophisticated pre-trained language models as the generator model or by using other dataset distillation objectives as an alternative to gradient matching.
(ii) In our experiments, DiLM was applied to distill only text classification task datasets. 
However, DiLM can be applied to text generation tasks as well by considering the entire original training dataset as the data for a single label. 
In the future, we will explore the application of DiLM in more difficult settings, such as text generation tasks and full-scratch training of language models. 
(iii) Both the proposed and existing dataset distillation methods generally incur higher costs for distillation compared to training with the full dataset. 
However, dataset distillation is useful for reducing the training costs by distilling a dataset into a small synthetic dataset, and subsequently, reusing it to train new models. From this perspective, the proposed approach for obtaining synthetic datasets as text shows potential for practical applications.
(iv) Although privacy preservation of the original training datasets is one of the applications of dataset distillation, applying DiLM to privacy preservation is difficult because the distilled synthetic datasets from DiLM may include real samples from the original dataset owing to the memorization of the training data of the language model.
However, we believe that the advantage of DiLM in generating distilled synthetic datasets at the text level, enabling the training of models independent of word embedding weights, is more valuable than its application to privacy preservation in terms of practical applications.



% Start bibliography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{09refs}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%SA
\section{Background Details of DiLM}
\label{sec:appendix/background}

In this section, we provide a detailed background of the techniques introduced in Section \ref{sec:text_gtn} to enable backpropagation computing by bypassing the non-differentiable discrete text via loss weighting according to the generation probabilities. 

As described in Section \ref{sec:text_gtn}, although two standard approaches exist to the non-differentiable problem in discrete text generation, that is, soft-argmax \cite{zhang-etal-2016-softargmax} and policy gradient \cite{yu-etal-2017-seqgan}, neither of them can be applied in training DiLM.
For soft-argmax and the same type of approach, the vocabulary of the generator model should be the same as that of the learner model, which receives the text generated by the generator model as input.
However, this is not true for DiLM, where we used GPT-2 for the generator model and BERT$_\text{BASE}$ for the learner model.

The policy gradient can be applied with sample-level gradient similarity as the reward function.
However, the gradients for synthetic data for gradient matching loss should be calculated as an average for samples in a mini-batch and not for a single sample. 
Moreover, the calculation of the per-sample gradient is computationally inefficient.
Therefore, we did not use a policy gradient with per-sample gradient similarity.

However, the basic idea of the proposed approach, which aims to update the generator model to increase its generation probabilities for synthetic samples that improve the gradient similarity, is essentially the same as the policy gradient.
Notably, the proposed approach can also be formulated as the policy gradient manner.
Letting $g_\theta(\tilde{x})=\frac{\partial l_\theta(\tilde{x})}{\partial\theta}$ and $r(\cdot)$ be the reward function, the gradient of the generator parameter $\phi$ can be represented as follows:
%%%%%
\begin{align*}
    & \nabla_\phi\mathcal{L}_\phi = \sum_{n=1}^N \nabla_\phi l_\phi (\tilde{x}_n)\cdot r(\tilde{x}_n)  \quad \text{where}\\
    & r(\tilde{x}_n) = a_n\left\{
    g_\theta(\tilde{x}_n) - g_\theta(\tilde{X})\right\}^T
    \left( -\frac{\partial \mathcal{L}_\mathrm{GM}}{\partial g_\theta(\tilde{X})} \right).
\end{align*}
%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%SB
\section{Datasets}
\label{sec:appendix/datasets}

We used three text classification datasets in the GLUE benchmark \cite{wang-etal-2018-glue} from huggingface datasets.\footnote{\url{https://huggingface.co/datasets/glue}}
SST-2 is a banally sentiment classification (negative/positive) task for movie reviews.
QQP identifies whether a question pair is semantically equivalent.
MNLI-m is an NLI task that predicts whether a premise sentence entails or contradicts a hypothesis sentence (neutral). We reported the evaluation results for the validation set in Section \ref{sec:results}, because the test set is not publicly available.
For MNLI-m, we used a matched-domain validation set for evaluation.
The statistics for each dataset are summarized in Table \ref{tab:datasets}.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T6
\begin{table}[b]
{\vskip -15pt}
\input{09table06.tex}
    \caption{Summary of statistics of evaluation datasets}
    \label{tab:datasets}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%SC
\section{Baselines Details}
\label{sec:appendix/baselines}

In this section, we describe the baseline methods used in our experiments.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%SC.1
\subsection{Coreset Selection}

%%%%%
\noindent\textbf{Random} is the simplest baseline, which randomly selects real samples from the original training dataset.

%%%%%
\noindent\textbf{K-centers} \cite{wolf2011facility,sener2018active} is a standard coreset selection method that selects the center samples of sub-clusters as a coreset, thereby eliminating redundant samples and covering the distribution of the original dataset.

%%%%%
\noindent\textbf{Herding} \cite{welling2009herding} is also a standard coreset selection method that greedily selects real samples to match their mean embedding with that of the original dataset.

For K-centers and Herding, we used the last hidden state of the \verb|[CLS]| token in the BERT$_\text{BASE}$ model as a feature of each training sample. 



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%SC.2
\subsection{Embedding-level Dataset Distillation}

%%%%%
\noindent\textbf{TDD}\footnote{We used the implementation by \citeA{maekawa-etal-2023-dataset}, because it also employs BERT as the learner model.} \cite{sucholutsky-2021-soft-label} is the current embedding level text dataset distillation method. 
TDD also optimizes learnable soft labels and learning rates, together with input word embeddings, using the original meta-learning approach \cite{wang2018dataset}.
Following the best performing settings in \citeA{maekawa-etal-2023-dataset}, which applied this approach to the BERT model, we used one synthetic sample per class as a mini-batch of a single gradient step and fixed the order of the synthetic samples, which implies that the learner model is trained with five gradient steps in the experiments in Section \ref{sec:results} with DPC = 5. 
Similar to DiLM, the TDD used BERT$_\text{BASE}$ as the learner model for distillation.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%SD
\section{Learner Models}
\label{sec:appendix/learner_models}

%%%%%
\noindent\textbf{BERT$_\text{BASE}$}\footnote{\url{https://huggingface.co/bert-base-uncased}} \cite{devlin-etal-2019-bert} was used as the source model for training for dataset distillation and the feature extractor of the coreset selection methods. Following the fine-tuning settings in \citeA{devlin-etal-2019-bert}, we used a randomly initialized linear layer on top of the last hidden state of the \verb|[CLS]| token.

%%%%%
\noindent\textbf{RoBERTa$_\text{BASE}$}\footnote{\url{https://huggingface.co/roberta-base}} is a BERT derivative model proposed by \citeA{liu-2019-roberta}. This model has the same size and architecture as BERT$_\text{BASE}$, but has different parameters pre-trained with the masked language modeling task, without the next sentence prediction (NSP) task, on a larger corpus than the BERT models.

%%%%%
\noindent\textbf{BERT$_\text{LARGE}$}\footnote{\url{https://huggingface.co/bert-large-uncased}} is the 24 layer, 340M parameter version of BERT, whereas BERT$_\text{BASE}$ has 12 layers and 110M parameters.

%%%%%
\noindent\textbf{XLNet$_\text{BASE}$}\footnote{\url{https://huggingface.co/xlnet-base-cased}} is an autoregressive model in contrast to BERT and RoBERTa. 
Following \citeA{yang-etal-2019-xlnet}, we used a randomly initialized linear layer on the top of the last hidden state of the \verb|<eos>| token, which involves entire tokens in the sequence.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%SE
\section{Implementation Details}
\label{sec:appendix/implementation_details}

Table \ref{tab:hyperparameters} lists details of the hyperparameter settings used in the experiments.
Our implementation was based on PyTorch 2.1.0, and we used pre-trained models from hugging face transformers 4.30.0.
All model training and evaluations in our experiments were performed with half-precision (BFloat16) on a single RTX 3090 (24GB), RTX A6000 (48GB), or A100 PCIe (80GB) according to the required GPU memory size for each experiment.



%%%%%%%%%%%%%%%%%%%%%%%%T7
\begin{table}[t]
\input{09table07.tex}
    \caption{Hyperparameter settings in our experiments}
    \label{tab:hyperparameters}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%SF
\section{Results for Cross-model Generalization}
\label{sec:appendix/additional_results}

Tables \ref{tab:cross_model_generalization_dpc_5} and \ref{tab:cross_model_generalization_dpc_10} show the cross-model generalization performances with DPC = 5,10 settings.
As in the setting of DPC = 20 in Table \ref{tab:cross_model_generalization}, DiLM performed better in training different models than the source model.



%%%%%%%%%%%%%%%%%%%%%%%%T8
\begin{table}[t]
\input{09table08.tex}
    \hangcaption{Cross-model generalization performance for the setting of DPC = 5. (\textbf{S}) indicates the source model for gradient matching of DiLM and feature extractor for K-centers.}
    \label{tab:cross_model_generalization_dpc_5}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%T9
\begin{table}[t]
\input{09table09.tex}
    \hangcaption{Cross-model generalization performance for the setting of DPC = 10. (\textbf{S}) indicates the source model for gradient matching of DiLM and feature extractor for K-centers.}
    \label{tab:cross_model_generalization_dpc_10}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%SG
\section{Distilled Synthetic Data Examples}
\label{sec:appendix/examples}

Examples of the distilled synthetic samples from DiLM are provided in Tables \ref{tab:example_sst2}, \ref{tab:example_qqp}, and \ref{tab:example_mnli}. 
The synthetic examples generated using DiLM were interpretable and appear to represent the tasks of the original training dataset. 



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T10
\begin{table*}
\input{09table10.tex}
    \caption{Distilled synthetic samples for SST-2 with DPC = 5}
    \label{tab:example_sst2}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T11
\begin{table*}
\input{09table11.tex}
    \caption{Distilled synthetic samples for QQP with DPC = 5}
    \label{tab:example_qqp}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%T12
\begin{table*}
\input{09table12.tex}
    \caption{Distilled synthetic samples for MNLI-m with DPC = 5}
    \label{tab:example_mnli}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%


\clearpage


% Start biography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Aru Maekawa}{%
Aru Maekawa received his B.E. and M.E. degrees from Tokyo Institute of Technology in 2022 and 2024, respectively. He is currently a Ph.D. student at Institute of Science Tokyo. His Research interests include natural language processing and machine learning, particularly dataset distillation.
}

\bioauthor[:]{Satoshi Kosugi}{%
Satoshi Kosugi received his B.S., M.S., and Ph.D. degrees in information and communication engineering from The University of Tokyo in 2018, 2020, and 2023, respectively. He is currently an Assistant Professor at Institute of Science Tokyo. His research interests include natural language processing and computer vision.
}

\bioauthor[:]{Kotaro Funakoshi}{%
Kotaro Funakoshi received his Dr.\hspace{0.33em}Eng.\hspace{0.33em}degree from the Tokyo Institute of Technology, Japan, and is currently an associate professor at the Institute of Integrated Research at Institute of Science Tokyo. Formerly with Honda Research Institute Japan Co., Ltd., he served as a program-specific associate professor with the Cooperative Intelligence Joint Chair at Kyoto University since 2017. His research interests include natural language processing, particularly computational linguistics, multi-modal dialogue systems, and human--machine interaction.
}

\bioauthor[:]{Manabu Okumura}{%
Manabu Okumura was born in 1962. He earned his B.E., M.E., and Dr.\ Eng.\ from Tokyo Institute of Technology in 1984, 1986, and 1989, respectively. He served as an assistant at the Department of Computer Science, Tokyo Institute of Technology, from 1989 to 1992, and as an associate professor at the School of Information Science, Japan Advanced Institute of Science and Technology, from 1992 to 2000. Currently he is a professor at the Institute of Integrated Research at Institute of Science Tokyo, Japan. His research interests include natural language processing, particularly text summarization, computer-assisted language learning, sentiment analysis, and text data mining.
}

\end{biography}

 \biodate   %%%% Following settings will be edited by editors.

\end{document}






