%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\documentclass[english]{jnlp_3.6}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{array}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition


%%%%%\usepackage{times}
%%%%%\usepackage{latexsym}
%%%%%\usepackage[T1]{fontenc}
%%%%%\usepackage[utf8]{inputenc}
%%%%%\usepackage{url}
%%%%%\usepackage{latexsym}
%%%%%\usepackage{amsmath}
%%%%%\usepackage{amssymb}
\usepackage{multirow}
%%%%%\usepackage{siunitx}
\usepackage{booktabs}
\usepackage{paralist}
%%%%%\usepackage{adforn}
%%%%\usepackage{graphics}
%%%%%\usepackage{float}
%%%%%\usepackage[export]{adjustbox} 
\usepackage{enumitem}
\usepackage{color}
%%%%%\newcommand{\draft}[1]{\textcolor{gray}{#1}}
%%%%%\newcommand{\verify}[1]{\textcolor{red}{#1}}
%%%%%\makeatletter
%%%%%\def\blfootnote{\xdef\@thefnmark{}\@footnotetext}
%%%%%\makeatother

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{30}
\Number{3}
\Month{September}
\Year{2023}
%%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2023}{1}{20}
\revised{2023}{4}{24}
%%%%%\rerevised{2006}{5}{6}
\accepted{2023}{6}{14}

\setcounter{page}{1011}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Plug-and-Play Attribute-Aware Text Infilling via A New Attention Mechanism and Two-Level Positional Encoding}
\eauthor{Dongyuan Li\affiref{lrlab}  \and Kotaro Funakoshi\affiref{lrlab} \and Manabu Okumura\affiref{lrlab}} 
\eabstract{%
Text infilling aims to restore incomplete texts by filling in blanks and has attracted increasing attention recently because of its wide application in ancient text restoration, conversation generation, and text rewriting. However, attribute-aware text infilling is yet to be explored, and existing methods seldom focus on the infilling length of each blank or the number and location of the blanks. In this study, we propose a plug-and-play \underline{\textbf{A}}ttribute-aware \underline{\textbf{T}}ext \underline{\textbf{I}}nfilling method using a \underline{\textbf{P}}re-trained language model (A-TIP) that contains a text-infilling component and a plug-and-play discriminator. Specifically, we first designed a unified text-infilling component with modified attention mechanisms and intra- and inter-blank positional encoding to better perceive the number of blanks and the infilling length for each blank. We then propose a plug-and-play discriminator to guide generation and improve attribute relevance without decreasing text fluency. Finally, automatic and human evaluations on three open-source datasets indicate that, compared to all the baselines, A-TIP achieves state-of-the-art performance. An additional ablation study demonstrated the robustness of \textsc{A-TIP}.
}
\ekeywords{Text Infilling, Language Model, Language Generation}

\headauthor{Li et al.}
\headtitle{A-TIP: Attribute-aware Text Infilling via Pre-trained Language Model}
\affilabel{lrlab}{}{Tokyo Institute of Technology}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
Originating from Cloze tests \cite{taylor1953cloze}, text infilling\footnote{This paper is an extended version of our preliminary paper \cite{li-etal-2022-tip} presented in the Proceedings of the 29th International Conference on Computational Linguistics (COLING-2022).} aims to fill in missing blanks in a sentence or paragraph using preceding and subsequent texts.
For example, given the two infilling tasks E1 and E2 shown in Figure \ref{example1}, text infilling models are supposed to provide fine-grained control over the location of any number of blanks and infill a variable number of missing tokens for each blank.
Text infilling has gained increasing attention in numerous research fields, including ancient text restoration \cite{assael-etal-2019-restoring,lazar-etal-2021-filling}, text editing and rewriting \cite{su-etal-2021-keep,agrawal-etal-2021-non}, and conversation generation \cite{roberta,Conversation}.
For example, as can be observed in Figure \ref{example_ancient}, text infilling model can effectively help restore the ancient broken text, provide hints to historians \cite{AssaelSSBPCAPF22}, and sheds light on recovering millions of books without much human labor.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% fig.1
\begin{figure}[t]
\begin{center}
\includegraphics{30-3ia6f1.pdf}
\end{center}
\hangcaption{A-TIP can generate more fluent, diverse and attribute relevant infilling content in two examples.}
\label{example1}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% fig.2
\begin{figure}[t]
\begin{center}
\includegraphics{30-3ia6f2.pdf}
\end{center}
\hangcaption{Example of ancient text restoration \protect\cite{AssaelSSBPCAPF22}. 
(A) Original inscription with 378 missing characters; (B) restoration of the missing characters proposed in the authoritative edition of Rhodes--Osborne, which we use as ground truth for evaluation; (C) text infilling model's restoration showing 74 mismatches with the Rhodes--Osborne edition.}
\label{example_ancient}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


However, current text-infilling methods are based only on bidirectional semantic constraints \cite{Conversation}, and other abundant attribute-based constraints, such as sentiment, domain knowledge, and topics, remain to be studied.
In reality, infilling attribute-aware content can better satisfy human needs and introduce more diversity into the generated content.
For example, as shown in Figure \ref{example1}, our proposed method, called \underline{\textbf{A}}ttribute-aware \underline{\textbf{T}}ext \underline{\textbf{I}}nfilling method using a \underline{\textbf{P}}re-trained language model (A-TIP),  can fill in blanks under the guidance of a specific pre-defined attribute, such as sentiment or domain knowledge. 
By contrast, current text-infilling models mainly focus on fluency, which leads to meaningless and monotonous infilling  of content \cite{sun-etal-2021-generating,Leakgan,Maskgan,TIGS,DECO}.


However, designing a simple and efficient attribute-aware text-infilling model remains challenging.
(1) To achieve attribute awareness, simply modifying a text-infilling model architecture or fine-tuning it with attribute-specific data will destroy the ability of the language model to infill blanks or incur a significant cost for re-training \cite{PPLM}.
(2) Second, if the model infills blanks to improve text attributes, avoiding ill-formedness between the infilled content and its bidirectional context becomes a challenge. 
For instance, ``The movie \underline{interesting} and \underline{perfect} us'' with \_ as blanks.
(3) Finally, Markov assumption-based methods \cite{TIGS,DECO} and masked language model (MLM)-based methods \cite{BERT,roberta} can recognize the number and location of blanks using special [Mask] tokens; however, they lack the ability to generate content with variable infilling lengths for each blank.
Generative language model (LM)-based methods \cite{ilm} can generate content with variable infilling lengths for each blank. However, they lack fine-grained control over the automatic determination of the number and location of blanks.

To circumvent this dilemma, we propose A-TIP, in which a plug-and-play discriminator provides fine-grained control over bidirectional well-formed fluency and attribute relevance.\footnote{Sentences with higher accuracy of attribute classification are said to have higher attribute relevance \cite{PPLM}. For example, as shown in Figure \ref{example1},  E1 of A-TIP has higher positive relevance than that of the compared algorithms.}
Specifically,
\begin{inparaenum}
    \item we first propose a general text filling framework that fine-tunes a standard LM with many artificially-masked examples in an auto-regressive manner.
    Moreover, to ensure that the number of infilling contents equals the number of blanks, we designed a new attention mechanism, where unmasked tokens can attend to each other but masked tokens can attend only to the preceding context (Figure \ref{overall} (A)).
    We also adopted two-level positional encoding to combine inter- and intra-blank positional information to automatically learn the length of the blanks.
    \item To achieve attribute-aware generation without modifying the model's architecture or re-training from scratch, we propose a plug-and-play discriminator that shifts the output distribution of the text infilling model towards the semantic space of the given guide attributes.
\end{inparaenum}
We also introduce two additional strategies to ensure that the infilling content is well formed in its bidirectional context without decreasing attribute relevance.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% fig.3
\begin{figure}[t]
\begin{center}
\includegraphics{30-3ia6f3.pdf}
\end{center}
\hangcaption{Model overview. We first fine-tune an off-the-shelf GPT-2 by adopting a new attention mechanism and two-level positional encoding to infill blanks. Then, we design a plug-and-play discriminator to guide generation in the direction of improving attribute relevance. We also adopt KL divergence and a threshold-based strategy to provide fine-grained control over fluency and attribute relevance.
Please note that [E] is a special token representing the end of each blank.}
\label{overall}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The main contributions of this study are summarized as follows:

\begin{itemize}
    \item We propose a unified text infilling model that adopts a new attention mechanism and two-level positional encoding to enable our model to learn the number/location of blanks and infilling length for each blank automatically.
    \item To the best of our knowledge, A-TIP is the first attribute-aware text infilling model that does not require any modification of the language model's architecture or re-training on specific attributed datasets. 
    Further, our plug-and-play discriminator can provide fine-grained control over fluency and attribute relevance, and can be applied to any transformer decoder-based text infilling model.
    \item The experimental results on three open datasets show that A-TIP achieves state-of-the-art performance compared with all baselines.
\end{itemize}

The remainder of this paper is organized as follows: 
In Section~\ref{related}, we provide an overview of related work. 
In Section~\ref{preliminary}, we provide the basic information. 
Section~\ref{method} introduces details of the proposed approach. 
We present detailed experimental results in Section~\ref{experiment}. 
Finally, we conclude the paper in Section~\ref{conclusion}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}
\label{related}
In this section, we briefly review the most relevant studies on pre-trained LMs, text infilling, and constrained text generation.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Pre-trained Language Models}
Pretrained LMs have made significant improvements to many natural language processing tasks by adopting self-supervised learning with abundant web text  \cite{chay-intr-etal-2021-character,you-etal-2022-joint}.
Pre-trained LMs can be classified into three types. 
The first uses the \textit{\textbf{encoder model}}.
For example, BERT \cite{BERT} and its variations are pre-trained as masked LMs to obtain bidirectional contextualized word representations.
The second type adopts an \textit{\textbf{encoder--decoder architecture}}, which is pretrained for seq2seq tasks such as MASS \cite{DBLP:conf/icml/SongTQLL19} and T5 \cite{2020t5}.
The third one adopts an \textit{\textbf{ autoregressive model}} that follows a left-to-right approach to text generation, such as GPT-2 \cite{radford2019language} and XLNet \cite{XLNet}.
Although we adopted GPT-2 as the LM in this study, our method can easily migrate to any type of pre-trained LM.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Text Infilling Approaches}
\label{Text-infilling}
The current text-infilling algorithms can be classified into four categories. 
1) \textit{\textbf{Generative adversarial network (GAN)-based methods}} train GANs to ensure that the generator can generate highly reliable infilling content to fool the discriminator \cite{Leakgan,Maskgan}.  
2) \textit{\textbf{Intricate inference-based methods}} adopt dynamic programming or gradient search to find infilling content with a high likelihood within its surrounding context \cite{TIGS,DECO}. 
3) \textit{\textbf{Masked LM-based methods}} generate infilling content by using bidirectional contextual word embedding \cite{BERT,BLM}.
4) \textit{\textbf{LM-based methods}} fine-tune off-the-shelf LMs in an autoregressive manner. Several methods change the input format by placing an infilling answer after the masked input \cite{ilm}, whereas others do not change the input format \cite{ti}.
Unlike the aforementioned methods, we propose solving a more complex task: attribute-aware text infilling.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Constrained Text Generation}
Traditional controlled generation models involve either fine-tuning existing models \cite{he2021cbart} or training conditional generative models \cite{CTRL}. 
Dathathri et al. \shortcite{PPLM} proposed a plug-and-play controlled generation model (PPLM) that does not modify or retrain the parameters of the original LM but can achieve comparable performance to fine-tuned methods. 
For example, PPCM \cite{PPLM-2} updates the hidden state in 
direction of the attribute enhancement to generate attribute-aware conversations. 
Pascual \shortcite{PPLM-3} designed a complex plug-and-play architecture to ensure that the generated content contains specific keywords. 
GeDi \cite{PPLM-4} and its extension \cite{PPLM-5} can accelerate the decoding process of the PPLM, but they assume that the model is trained using large-scale labeled datasets, which are unrealizable for text infilling.

Although the PPLM can generate attribute-aware content well, it tends to generate repetitive and discontinuous contextual content.
In contrast to the traditional PPLM, 
(1) A-TIP has input and output formats different from those of the PPLM and similar to T5 \cite{2020t5} to adopt the model for the text-infilling task.
(2) We propose a novel attention mechanism inspired by the prefix language model \cite{2020t5} and UL2 \cite{Tay2022UnifyingLL}  to help the text-infilling task generate attribute-aware, fluent, and context-sensitive text-infilling content.
(3) To further ensure the fluency of the generated text, we use KL divergence to minimize the PPLM-based modified and unmodified word distributions, and we use attribute gain to determine whether we should change a generated word with an attribute-aware one.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Preliminaries}
\label{preliminary}
To clarify our method, we first introduce essential background knowledge and then define the task of attribute-aware text infilling.

\textbf{LMs} reveal the degree of how much a sentence (a sequence of words) is likely to be a realistic sequence of a human language.
Formally, let $\mathcal{W}$ be the vocabulary set and $\boldsymbol{w}_{1:n} = \{w_{1}, \dots, w_{n}\}$ be a sentence with $n$ words, where $w_{i} \in \mathcal{W}$.
An LM measures the joint probability by individually decomposing the sequence.
\begin{equation}
    p(\boldsymbol{w}_{1:n}) = \prod_{i=1}^{n}p(w_{i}|\boldsymbol{w}_{<i}), 
\end{equation}
where $\boldsymbol{w}_{<i} = \{w_{1},\dots, w_{i-1}\}$.

\textbf{Constrained Text Generation:} Given $k$ explicit constraints $\textbf{c} = \{c_{1},\dots,c_{k}\}$, our goal is to generate a sentence $\boldsymbol{w}$ that maximizes the conditional probability $p(\boldsymbol{w}|\textbf{c})$: 
\begin{equation}
        p(\boldsymbol{w}|\textbf{c}) =\prod_{i=1}^{n} p(w_{i}| \boldsymbol{w}_{<i},\textbf{c}).
\end{equation}

\textbf{Task Definition:} Attribute-aware text infilling takes incomplete text $\widetilde{\boldsymbol{w}}$, containing one or more missing blanks, and returns completed text $\boldsymbol{w}$ under the constraints of \textbf{c}. 
As shown in Figure \ref{example1}, several attributes are listed in \textbf{c}. 
Specifically, let [Mask] be a placeholder of a contiguous sequence of one or more missing tokens.
Then, $\widetilde{\boldsymbol{w}}$ is a sequence of tokens in which a number of tokens are a [mask].
To map $\widetilde{\boldsymbol{w}}$ to $\boldsymbol{w}$ constrained by attribute \textbf{c}, an infilling strategy must specify the number of tokens generated for each [Mask].
Note that many logical $\boldsymbol{w}$ values may exist for a given $\widetilde{\boldsymbol{w}}$.
Hence, we are interested in learning distribution $p(\boldsymbol{w}|\widetilde{\boldsymbol{w}},\textbf{c})$.
Specifically, in accordance with Bayes' theorem, we formulate the probability of predicting token $w_{i}$ for its corresponding [Mask] as
\begin{equation}
    p(w_{i}|\boldsymbol{w}_{<i},\textbf{c})
    \propto p(w_{i}|\boldsymbol{w}_{<i}) \cdot p(\textbf{c}|\boldsymbol{w}_{1:i}),\label{bayes3}
\end{equation}
where $p(w_{i}|\boldsymbol{w}_{<i},\textbf{c})$ can be decomposed into two parts that deal with the LM for $p(w_{i}|\boldsymbol{w}_{<i})$ and the discriminator for $p(\textbf{c}|\boldsymbol{w}_{1:i})$.
In Section~\ref{method}, we describe these two aspects in detail.
We assume that any two constraints are independent, $p(\textbf{c}|\boldsymbol{w}_{1:i})=\prod_{j=1}^{k}p(c_{j}|\boldsymbol{w}_{1:i})$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Methodology}
\label{method}
The overall framework of A-TIP is shown in Figure \ref{overall}.
A-TIP contains two components: a \textit{ text-infilling model} and a \textit{plug-and-play attribute-aware controller}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Text Infilling Model}
Given a corpus comprising complete text examples, we first create \textit{infilling examples} and then train GPT-2 using these examples.
Specifically, given an input example $\boldsymbol{w}_{1:n}$ with $n$ tokens, we first randomly replace $m$ non-overlapping word spans $\mathbf{S} = \{\boldsymbol{s}_{1}, \dots, \boldsymbol{s}_{m}\}$ in $\boldsymbol{w}$ with [Mask] tokens to form a corrupted text $\widetilde{\boldsymbol{w}}$.
We also assume that each span $\boldsymbol{s}_{i}$ contains $n_{i}$ consecutive tokens [$s_{(i,1)}, \dots, s_{(i,n_{i})}$].
Subsequently, we concatenate the spans $S$ separated by the [Answer] tokens to form the training target
$\widetilde{\mathbf{S}} = $ \{[Answer], $s_{(1,1)}$, \dots, $s_{(1,n_1)}$, [Answer], \dots, [Answer], $s_{(m,1)}$, \dots, $s_{(m,n_m)}$ \}.
Finally, we construct a complete infilling example by concatenating $\widetilde{\boldsymbol{w}}$ and $\widetilde{\mathbf{S}}$ (see the Token Embedding in Figure \ref{overall}).

There are two advantages in designing such an input format.
(1) We add only two $m$ additional tokens (one [Mask] and one [Answer] per blank area, as shown in Figure \ref{overall} ``Token Embedding'' add four tokens for two spans).
Although the memory usage for GPT-2 grows quadratically with the sequence length, as $m$ is small, the additional training time complexity will be minimal.
(2) Based on the attention mechanism of the prefix language model described in T5 \cite{2020t5} and UL2 \cite{Tay2022UnifyingLL}, 
we can apply two different attention strategies to the corrupted text $\widetilde{\boldsymbol{w}}$ and training target text $\widetilde{\boldsymbol{S}}$.
As shown in Figure \ref{overall} (A), while tokens in the corrupted text have attentions on all other tokens in $\widetilde{\boldsymbol{w}}$, tokens in the training target can have attentions only on its previous tokens.
Further, our new attention mechanism is applied to each Transformer Block. Figure \ref{overall} (A) shows an example of attention mechanism when generating the token $x_{5}$. 
The attention mechanism of the prefix language model described in T5 can help the model generate the next token in an autoregressive manner.
In contrast, the attention mechanism in A-TIP can not only generate tokens in an autoregressive manner, but also help capture the positional relationship between [Mask] tokens in the corrupted text and subsequently generate [Answer] tokens. That is, 
the attention mechanism can help generate an equal number of [Answer] tokens for the corresponding [Mask] tokens, and it will focus on the bidirectional context of the $i$-th blank when we fill in the $i$-th blank $\boldsymbol{s}_{i}$, which can ensure the well-formedness and rationality of the whole sentence.


As mentioned in the Introduction,  the current methods, Markov assumption-based and generative LM-based text-infilling models, do not automatically perceive the number/location and infilling length for each blank.
Although we can provide a certain number of masks and their positions for these methods, we cannot provide this information in real scenarios, which limits their real-world applications.
We designed a two-level positional encoding system that can provide fine-grained control.  Specifically, each token is encoded using two positional IDs.  
Intra-position encoding can be formally defined as 
\begin{equation}
    \text{Intra-position} = 1 + i  \quad \text{where i represents the $i$-th order of words in the blank},
\end{equation}
and it aims to identify the positional relationship of the infilling content within one blank. 
For example, as shown in Figure \ref{overall},
the intra-positions for [[A], $x_{5}$, $x_{6}$] are [1, 2, 3], where 1 represents the special token [Answer], and 2 \& 3 represent the intra-position of $x_{5}$ \& $x_{6}$. 
The interposition can be formally defined as follows: 
\begin{equation}
    \text{Inter-position}= \text{the position of its corresponding}\,[\text{Mask}]\,\text{token in the sentence}, 
\end{equation}
which aims to identify the positional relationships among the blanks.
For example, as shown in Figure \ref{overall}, the inter-positions for [[A], $x_{2}$, [A], $x_{5}$, $x_{6}$] are [2, 2, 5, 5, 5]. 
Intra-position helps the model perceive the infilling length of each blank.
Interposition can help the model perceive the number/location of blanks that need to be infilled. 
By fine-tuning LMs, such as GPT-2, A-TIP can automatically recognize that each [Mask] token represents broken parts that need to be infilled.
A-TIP can also effectively utilize the bidirectional context of each blank to determine how many words should be infilled to ensure the fluency and attribute relevance of each blank. 
When the infilling content can connect with its bidirectional context, a special token [E] is generated to indicate that the content generation of [Mask] should be stopped (as shown in the output of Figure \ref{overall}).

Please note that as discussed in Section 5.1.3\footnote{To make our description clearer, we give a detailed example in Section 5.1.3 about how A-TIP determines the number/location and the infilling length of blanks.}, we do not set the intra-position and inter-position IDs in the inference process because A-TIP has the ability to automatically determine the number of blanks, their location in the sentence,  and the infilling length for each blank. 

Finally, A-TIP trains GPT-2 using \textit{infilling examples} in an autoregressive manner.
When predicting missing tokens in each blank, A-TIP has access to the corrupted text $\widetilde{\boldsymbol{w}}$ and the previously predicted blanks.
Formally, the probability of generating the $i$th blank $\boldsymbol{s}_{i}$ is
\begin{equation}\label{equation1}
    p_{\theta}(\boldsymbol{s}_{i}|\widetilde{\boldsymbol{w}},\boldsymbol{s}_{<i}) = \prod_{j=1}^{n_{i}} p(s_{i,j}|\widetilde{\boldsymbol{w}},\boldsymbol{s}_{<i}, \boldsymbol{s}_{i,<j}),
\end{equation}
where $\theta$ are parameters for the GPT-2, $n_{i}$ represents the number of tokens in $\boldsymbol{s}_{i}$, $s_{i,j}$ represents the $j$-th token in the span $\boldsymbol{s}_{i}$, $\boldsymbol{s}_{<i}$ represents previously predicted blanks, and $\boldsymbol{s}_{i,<j} = \{s_{(i,1)},\cdots, s_{(i,j-1)}\}$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Plug-and-play Attribute-aware Controller}

To clarify our approach, we follow the notation in  \cite{PPLM} and define the GPT-2 decoding process (Eq.(\ref{equation1})) recursively.
Specifically, we first define $\mathbf{H}_{t}$, which contains all historical key-value pairs, i.e., $\mathbf{H}_{t} =[(\mathbf{K}_{t}^{(1)},\mathbf{V}_{t}^{(1)}), \dots, (\mathbf{K}_{t}^{(l)}, \mathbf{V}_{t}^{(l)})]$, where $(\mathbf{K}_{t}^{(l)},\mathbf{V}_{t}^{(l)})$ stores all key-value pairs of $t$ tokens in the $l$-th layer.
Subsequently, we formally define the recurrent decoding process for generating the $i$th token as
\begin{equation}
    \boldsymbol{o}_{i}, \mathbf{H}_{i} = \mbox{GPT-2}(\boldsymbol{w}_{<i},\mathbf{H}_{i-1}),
\end{equation}
where $\boldsymbol{o}_{i}$ denotes the hidden state of the input at the $i$-th time step.
Then, we sample the $i$th generated token from the following distribution using a beam search \cite{beam_search}:
\begin{equation}
    w_{i} \sim p_{i} = \mbox{Softmax}(\mathbf{W} \boldsymbol{o}_{i}),
\end{equation}
where $\mathbf{W}$ is a parameter matrix that maps the hidden state $\boldsymbol{o}_{i}$ to a vector of vocabulary size.


In accordance with the Bayes' theorem, Eq.(\ref{bayes3}), we obtain $p(w_{i}| \boldsymbol{w}_{<i},\textbf{c}) \propto p(w_{i}|\boldsymbol{w}_{<i}) \cdot p(\textbf{c}|\boldsymbol{w}_{1:i})$.
To achieve attribute-aware text infilling, when we infill the $i$-th word, we shift the history matrix $\mathbf{H}_{i-1}$ in the direction of the sum of two gradients, as follows: 
1) Maximize the log-likelihood of attribute $\textbf{c}$ under the conditional attribute model $p(\textbf{c}|\boldsymbol{w}_{1:i})$. 
2) To ensure high fluency during text infilling, let $p(w_{t}|\boldsymbol{w}_{<i})$.
We update only $\mathbf{H}_{i-1}$ and keep the other model parameters unchanged because the next-token prediction depends only on the past key-value pairs via $\mathbf{H}_{i-1}$.
Thus, we propose gradually updating $\mathbf{H}_{i-1}$ to guide future generations in the desired direction.

Let $\Delta \mathbf{H}_{i-1}$ be an update to $\mathbf{H}_{i-1}$ to shift the generation-infilling content towards the desired attribute direction $\textbf{c}$. 
At the beginning of the generation, $\Delta \mathbf{H}_{i-1}$ is initialized to zero and we obtain an unmodified distribution as $p_{i}$.
We then update $\Delta \mathbf{H}_{i-1}$ with the gradients from the attribute model, which measures the extent to which the generated text possesses the desired attribute.
Following \cite{PPLM}, we rewrite  $p(\textbf{c}|\boldsymbol{w}_{1:i})$ as Pb = $p(\textbf{c}|\mathbf{H}_{i-1} + \Delta \mathbf{H}_{i-1})$ and define the gradient update for $\Delta \mathbf{H}_{i-1}$ as
\begin{equation}\label{equation3}
    \Delta \mathbf{H}_{i-1} \leftarrow \Delta \mathbf{H}_{i-1} + \alpha \frac{\nabla_{\Delta \mathbf{H}_{i-1}} Pb} {\|\nabla_{\Delta \mathbf{H}_{i-1}} Pb\|^{\gamma}},
\end{equation}
where $\alpha$ is the learning rate, and $\gamma$ is the scaling coefficient for the normalization term to control the relevance of the attribute.
We repeat Eq.(\ref{equation3}) less than 10 times to generate attribute-aware tokens.
Subsequently, the new $\widetilde{\mathbf{H}}_{i-1} = \mathbf{H}_{i-1} + \Delta \mathbf{H}_{i-1}$ is computed, and a new token is generated using $\widetilde{\boldsymbol{o}}_{i}, \mathbf{H}_{i} = \mbox{GPT-2}(\boldsymbol{w}_{<i}, \widetilde{\mathbf{H}}_{i-1})$.
The described optimization process is repeated for every token in the generated sequence.
Compared with the unconditional LM-based text generation task, this process does not require much time (see details in the experiments).

Although we can generate attribute-aware infilling content, we can also easily generate low-quality, repetitive, and low-fluency texts.
Thus, we add two components to ensure the fluency and quality of the generated infilling content within its bidirectional context.
First, we minimize the KL divergence between the unmodified distribution $p_{i}$ and the modified distribution $\widetilde{p}_{i}$ for the $i$-th token: 
\begin{equation}
        \min{ \,\,\, \mbox{D}_{KL}(\widetilde{p}_{i}\,||\,p_{i})}.
\end{equation}
Our objective function can be reformulated as follows:
\begin{equation}
    Loss = p(\textbf{c} | \widetilde{\mathbf{H}}_{i-1}) + \lambda \mbox{D}_{KL}(\widetilde{p}_{i}\,||\,p_{i}),
\end{equation}
where $\lambda$ is a parameter that balances the fluency and attribute relevance.
Then, we update $\Delta \mathbf{H}_{i-1}$ as follows:
\begin{equation}\label{equation4}
    \Delta \mathbf{H}_{i-1} \leftarrow \Delta \mathbf{H}_{i-1} + \alpha \frac{\partial_{\Delta \mathbf{H}_{i-1}} Loss}{ \|\partial_{\Delta \mathbf{H}_{i-1}} Loss \|^{\gamma}}.
\end{equation}


Intuitively, we can generally find many words that have different levels of correlations with the specific attribute \cite{VAD}.
For example, \{perfect, good, bad, like\} can mainly determine the sentiment of a sentence.
Thus, we define the \textit{ gain } from the attributes to determine whether to change a generated word.
As shown in Figure \ref{overall},
Two candidate words were sampled from the unmodified distribution (before backpropagation) and the modified distribution (after backpropagation).
\textit{Gain} between two candidate words in the conditional model can be formulated as
\begin{equation}
     Gain = p(\textbf{c}|\boldsymbol{w}_{<i}, \widetilde{w}_{i})- p(\textbf{c}|\boldsymbol{w}_{<i}, w_{i}), 
\end{equation}
where $\widetilde{w}_{i}$ and $w_{i}$ are samples from the modified and unmodified distributions, respectively.

To better control the relevance of the attribute, we define a threshold $\delta$ to determine whether a word should be generated from the modified distribution.
Specifically, \textit{ gain } \textgreater $\delta$ indicates that the word generated from the modified distribution has a significant effect on the attributes.
Otherwise, if the discriminator does not guide well at certain steps (\textit{Gain} \textless $\delta$), we select the word generated from the unmodified distribution to maintain the same fluency as the original unconditional text-infilling model to the greatest extent.

\textbf{Discriminator Construction:}
As shown in Figure \ref{overall} (B), for simplicity, we train a linear classifier $f$ as a discriminator with annotated datasets, indicating a sentence and label pair as ($\boldsymbol{w},\boldsymbol{y}$).
Specifically, for each sentence $\boldsymbol{w}$ of length $ t $, we compute the set of hidden states $\boldsymbol{o}$ = $\{o_{1},\dots,o_{t}\}$ from GPT-2.
We then compute the mean of $\boldsymbol{o}$ as $\bar{\boldsymbol{o}}$ and train $f$ using the cross-entropy between the true label distribution $\boldsymbol{y}$ and the predicted label distribution $f(\bar{\boldsymbol{o}})$.
The number of parameters in this layer is (embedding dimension $\times$ number of attributes + number of attributes), which is negligible compared to the number of parameters in the text-infilling model itself.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiment}
\label{experiment}
%In this section, 
As presented in Table~\ref{T1}, we evaluated the proposed methods on three tasks to demonstrate that our framework is not custom-tailored to a single domain: sentiment awareness, domain knowledge awareness, and topic-aware text infilling.
In addition, we realized a case study of these tasks.
We determined whether A-TIP can generate infilling text that satisfies the desired attribute and whether it can infill high-quality text in blanks using both automated methods and human annotators. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% tab.1
\begin{table}[t]
\input{06table01.tex}%
\caption{Descriptive statistics of datasets and their attributes.}
\label{T1}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Experimental Settings}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Datasets}
In addition to using the datasets listed in Table~\ref{T1} to train the text infilling model,
we also adopted the sentiment labels in SST-5 \cite{sst} for sentiment-aware text infilling, research area labels in Abstracts \cite{ilm} for domain-knowledge-aware text infilling, and topic labels in ROCStories \cite{story} for topic-aware text infilling.
For datasets with attribute labels such as SST-5 and Abstracts, we can directly use their labels to train our plug-and-play discriminator.
However, considering that most datasets like ROCStories have no labels, we extended our method to address this situation. Intuitively, we can construct a general attribute-based plug-and-play discriminator to guide different datasets to generate different infilling content. However, in practical operations, it is unrealistic to build such an available attribute-based discriminator to guide the infilling generation because the downstream datasets have a variety of different attribute requirements. 
Therefore, we must generate specific category labels for different downstream datasets to satisfy their specific attribute-related needs, and then use these attribute-related labels to guide infilling generation.
Specifically, we extended our model to other applications by combining it with a topic exploration algorithm to mine topic labels on unlabeled datasets.
For instance, we adopted combinedTM \cite{attribute} to detect topic attributes for the ROCStories dataset using two methods: \textbf{Contextual-TM} and \textbf{Combined-TM}.

Specifically, \textbf{Contextual-TM} is a neural topic model based on a variational autoencoder architecture \cite{KingmaW13}.
\textbf{Contextual-TM} first uses a neural encoding layer, called SBERT \cite{reimers-gurevych}, to capture the contextualized embeddings of the input sentences.
Subsequently, a Gaussian distribution component is used to sample topic-related embeddings from the input embeddings to characterize the topic distribution. 
Finally, a decoder network is designed to generate topic-related words based on the hidden embedding of the Gaussian distribution component. 
Compared with \textbf{Contextual-TM}, \textbf{Combined-TM} alters the input by concatenating the contextual embedding and BoW embedding of the input sentence.
ProdLDA \cite{srivastava2017autoencoding}, which is a neural topic modeling approach based on the Variational Auto-Encoder, is used to extract the BoW embedding of the input sentence. 
SBERT \cite{reimers-gurevych} is used to capture the contextualized document embeddings.  
The contextual and BoW embeddings are concatenated to form the input encoding, and the same Gaussian distribution component and decoder network used in \textbf{Contextual-TM} are used to generate topic-related words.



We adopted three metrics to evaluate the quality of the attributes of the ROCStories dataset: \textbf{Topic Coherence}, \textbf{Inverted RBO}, and \textbf{NPMI}.
\begin{itemize}
    \item \textbf{Topic Coherence} measures the similarity of words with a topic. 
Following \cite{ding-etal-2018-coherence}, we first calculated the average pairwise cosine similarity of the top-10 words in a topic using \cite{Mikolov} embeddings.
Then, we took the average of these values across all topics, which provided an external measure of topic coherence. 
\item \textbf{Inverted RBO} means Inversed Rank-Biased Overlap \cite{TerragniFM21}, and it evaluates how diverse the topics generated by a single model are. 
Inverted RBO compares the 10-top words of two topics, which allows disjointedness between the lists of topics and uses weighted ranking, i.e., two lists that share some of the same words, albeit at different rankings, are penalized less than two lists that share the same words at the highest ranks.
\item \textbf{NPMI} means normalized pointwise mutual information \cite{lau-etal-2014-machine}, and it measures how related
the 10-top words of a topic are to each other, considering the word's empirical frequency in the original corpus. 
\textit{NPMI} is a symbolic metric and relies on co-occurrence. 
As \cite{ding-etal-2018-coherence} pointed out, topic coherence computed on the original data is inherently limited. 
Coherence computed on an external corpus, meanwhile, correlates much more to human judgment, but it may be expensive to estimate.
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% tab.2
\begin{table}[tb]
\input{06table02.tex}%
\hangcaption{Contextual-based and Combined-based topic detection algorithms evaluate three widely used metrics: Topic Coherence, Inverted RBO, and NPMI.}
\label{topic} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

As presented in Table~\ref{topic},  we chose 13 topics as our final labels because they exhibited the best performance on average for all metrics.   
 

We split the datasets into $80\%/10\%/10\%$ as training, validation, and test data, respectively.
Following TIGS \cite{TIGS} and BLM \cite{BLM}, we randomly masked $r\%$ of the tokens in each document.
To ensure that all experiments were performed using the same data, we removed the infilling examples that exceeded the training sequence length of 256 tokens.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Evaluation Metrics}
In an automated evaluation, perplexity is a measure of fluency in open-domain text generation.\footnote{Overlap-based metrics such as BLEU scores \cite{BLUE} are not appropriate for evaluating infilling as there are many realistic infills that have no word-level overlap with the original.} We measured this using GPT-2.
The text diversity was measured using the number of distinct n-grams (normalized by text length), as in  \cite{diversity}.
We reported Dist1, Dist2, and Dist3 scores for the distinct 1, 2, 3-grams. 
Following \cite{PPLM}, we used an external classifier to evaluate the \textit{ accuracy } (macro-average F-score) of sentence attribute labels. 
We evaluated attribute control for sentiment (SST-5) using two external sentiment classifiers:  XLNet \cite{XLNet} and self-explaining  \cite{DBLP:journals/corr/abs-2012-01786}.
We chose a BERT-based classifier \cite{CF2} and another classifier, Big Bird \cite{DBLP:conf/nips/ZaheerGDAAOPRWY20} for the abstract dataset. 
A t-test was used to evaluate significant performance differences between the two approaches \cite{test} for both automated and human evaluations. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Baselines}
We compared A-TIP with six baselines, which can be classified into four classes (Section~\ref{Text-infilling}):
\begin{itemize}
    \item \textbf{Inference-based:} We trained TIGS \cite{TIGS}, an RNN-based seq2seq model. At inference time, we iteratively searched tokens in continuous space and projected their vectors to real words.
\item \textbf{GAN-based:} We trained the generator of MaskGan \cite{Maskgan} on PLM with a seq2seq architecture. The discriminator can make word distributions of the generator closer to those of the real word distribution.
\item \textbf{Masked LM-based:}
We used representations of blanks as seeds to fine-tune BERT \cite{BERT} and RoBERTa \cite{roberta}. At inference time, blanks are infilled one after another and are conditioned on the previous generation.
We trained BLM \cite{BLM} with a seq2seq architecture, where the encoder module is a transformer (base) and the decoder process adopts beam search.
\item \textbf{LM-based:} We trained ILM \cite{ilm} by fine-tuning GPT-2 to output a full document from a masked input. Note that it may have invalid outputs that do not match the input format.
\end{itemize}

For the Inference-based, GAN-based, and Masked LM-based methods, the length of the infilling content should be determined in advance.  For example, if the infilling length is four words, four special [mask] tokens must be used to replace the four words to ensure that they can generate the required length for text infilling. 
For LM-based methods, we used only one [Mask] token to replace the varied lengths of the infilling content. LM-based methods, such as GPT-2, can automatically determine the generated length for each blank.
Our A-TIP can also use one [mask] token to replace various infilling content lengths and automatically determine the infilling length for each blank.


Please note that none of the baseline models are attribute-aware because they are not designed for attribute-aware text infilling. However, they are the best-performing large-scale text-generation and text-infilling models. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Implementation Details}
In our experiments, we set the learning rate $\alpha=1e-4$ and the scaling coefficient $\gamma=0.5$ for Eq.~(\ref{equation4}).  
Sequence representations were obtained using the GPT-2 module ($12$ layers, $12$ heads, $n_{embd}=768$, $n_{ctx}=1024$, batch size $=24$).
We applied the Adam \cite{Adam} optimizer with an initial learning rate of $1\mathrm{e}-4$, and the weight decay and dropout were turned on based on the loss of validation data.
Our discriminator had a linear layer on the head of GPT-2.
For a fair comparison, we followed the default parameter settings of the baselines and repeated all the experiments $10$ times to report the average accuracy.
An unpaired t-test was used to evaluate the significant differences between any two approaches as multiple comparisons for both automated and human evaluations. We trained the models using early stopping.
For the models trained on Stories and Abstracts, 
We trained the models to converge using early stopping. 
based on the validation set perplexity (PPL) of each model and computed only on masked tokens.



\textbf{Benjamini--Hochberg procedure:}
 The Benjamini--Hochberg (\textit{B-H}) procedure is a powerful tool that decreases the false discovery rate.
Adjusting the rate helps control for the fact that sometimes small p-values (less than 5\%) occur by chance, which could lead to incorrect rejection of the true null hypotheses \cite{benjam}.

Considering the reproducibility of multiple significance tests, we carefully introduced the \textit{B--H} procedure and selected the hyper-parameter values.
Specifically, we first adopted a t-test \cite{test} with default parameters\footnote{\url{https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.ttest_ind.html}} to calculate the p-value between each compared algorithm and A-TIP.
We then placed the individual p-values in ascending order and assigned ranks to each p-value. For example, the smallest had a rank of 1, and the largest had a value of 6 (we had six compared algorithms).
We defined the p-value corrected by B--H as the q-value, which can be calculated as
\begin{equation}
    \mbox{q-value}(i) = \frac{i}{m}  Q,
\end{equation}
where $i$ is the rank index of each individual p-value, q-value(i) represents the q-value of the $i$-th p-value, $m=6$ is the total number of tests, and $Q$ is the false discovery rate. We chose $Q = 0.05$ because this is the widely used default value.\footnote{\url{https://www.statsmodels.org/dev/generated/statsmodels.stats.multitest.multipletests.html}} 
Finally, we found the largest p-value that was smaller than its q value.
\begin{equation}
      \mbox{cut-off} = \max \,\,\mbox{p-value}(i) \,\, \text{such that} \,\, i \in \{1,\dots,6\} \,\,\text{and}\,\, \mbox{p-value}(i) < \mbox{q-value}(i).
\end{equation}
If the p-value was smaller than the cutoff, we concluded that the two groups of data were significantly different.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Parameter Sensitivity}
A-TIP uses two hyperparameters.
$\lambda$ dominates the attribute relevance of the generated text, and $\delta$ controls the fluency of the infilling content.
We analyzed the parameter sensitivity of all three validation datasets and selected the validation data of SST-5 as an example to determine the parameter sensitivity of A-TIP.
As shown in Figure \ref{parameter} (A--C), we observed how $\lambda$ and $\delta$ affected the performance of A-TIP by varying $\lambda$ from 0.2 to 0.6 in 0.1 intervals and $\delta$ from 0.008 to 0.012 in 0.001 intervals.
The results indicated that A-TIP achieves the best performance when $\lambda \in [0.4,0.5]$ and $\delta \in [0.010,0.011]$.
These parameters can affect the results, because when $\lambda < 0.4$, the attribute relevance becomes stronger and fluency is destroyed.
$\lambda > 0.5$ weakens both attribute relevance and text diversity.
When $\delta< 0.01$, A-TIP tended to preserve the modified words, leading to low fluency.
When $\delta >0.012$, A-TIP preserved the original unmodified words, resulting in low attribute relevance and text diversity.
To achieve balanced performance, we set $\lambda=0.4$ and $\delta=0.01$ for all datasets in our experiments.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% fig.4
\begin{figure}[t]
\begin{center}
\includegraphics{30-3ia6f4.pdf}
\end{center}
\hangcaption{Based on the validation data of SST-5, we evaluated the parameter effect for Perplexity (A), Dist1 (B), and Accuracy (C). We draw the effect of mask rate on performance of text infilling for Perplexity (D) and Accuracy (E).}
\label{parameter}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Considering that the mask rate $r$ is also a hyperparameter, we analyzed its effect on the results by varying it from 10\% to 70\%.
We observed the same trend for all datasets and used SST-5 as an example.
As shown in Figure \ref{parameter} (D), the fluency decreased when $r$ varies from 10\% to 40\% because infilling content may be well-formed with its bidirectional context.
As $r$ increases from 40\% to 70\%, the fluency of the text mainly depends on the baseline's original generation ability, which is stable.
Figure \ref{parameter} (E) shows that when $r$ increases, the baselines cannot recover the attributes of the infilling content well.
However, A-TIP can generate attribute-aware text to improve classification accuracy.
All baselines can obtain stable fluency and classification accuracy when $r$ = 50\%; we fixed $r$= 50\% to show the numerical experimental results in later experiments.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Automated Evaluation}

We evaluated the performance of A-TIP on attribute-aware text infilling by measuring PPL, Dist1, Dist2, Dist3, and ACC on the test data.
As can be observed in Table~\ref{automated}, A-TIP outperformed the other baselines, indicating that our proposed framework can take advantage of bidirectional context and attribute information.
Additionally, ILM can achieve good results in PPL because it also uses GPT-2 for text infilling. 
However, compared with the one-layer positional encoding and auto-regression attention mechanisms in ILM, A-TIP/Dis (A-TIP without discriminator) achieves better fluency (PPL) because it adopts a modified attention mechanism (Figure \ref{overall} (A)) to effectively learn the length for each blank, and focus on the number/location of blanks by two-level positional encoding (intra- and inter-blank).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% tab.3
\begin{table}[t]
\input{06table03.tex}%
\hangcaption{Overall performance comparison. PPL is perplexity, Dist scores measure divergence, and ACC is classification accuracy. $\dag$ shows that our results significantly surpass all baselines using t-test with $p<0.005$. Underlines mean that our ablation algorithm can achieve better results than A-TIP for a metric.} 
\label{automated}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Dist1, Dist2, and Dist3 are used to count the number of occurrences of one, two, and three words, respectively, which are widely used to measure the diversity of the generated content.
Under the premise of generating words with the highest probability for each selection,
we found that LMs such as BERT and RoBERTa tend to generate repetitive and fluent infilling content for blanks because many [Mask] tokens in one sentence cause the contextual information of each blank to be the same, which severely reduces the diversity of text generation. 
However, attribute-aware text generation guides the model to generate more diverse content based on different attribute information, which helps generate diverse and meaningful infilling content under the same contextual information of blanks.

A-TIP obtained more accurate sentence attributes than the other baselines, demonstrating that A-TIP can generate text that satisfies the desired attributes.
Although the accuracy improved by 8\% compared to the baselines, we observed that ILM and BERT also yielded high classification accuracy.
This was because we randomly masked 50\% of the tokens in the original input without considering whether the token had a specific attribute.
We did not generally mask attribute-relevant tokens, which helped the sentence maintain its original attributes.
Better results can be obtained if all the attribute-relevant tokens are masked.
For a fair comparison, we randomly masked the tokens instead of masking specific tokens.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% tab.4
\begin{table}[b]
\input{06table04.tex}%
\hangcaption{Overall performance comparison of attribute-aware classification accuracy. Original ACC represents the classification accuracy of the original sentences. Broken ACC represents the classification accuracy of broken sentences with [Mask] tokens in their sentences. Generated ACC represents the classification accuracy of generated sentences by baselines.}
\label{automated_accuracy}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Please note that we adopt ``Accuracy'' to measure if the generated sentences have the desired attributes.
To make this clearer, as can be observed from Table~\ref{automated_accuracy}, we report the attribute accuracy of the original sentences, broken sentences, and sentences generated by the baselines.
Instead of simply using the discriminator trained by us (XLNet for the sentiment classifier and BERT-based classifier for the Abstracts dataset), we introduced off-the-shelf external classifiers to help judge the attribute relevance of the baselines. 
Specifically, we used the best-performing sentiment analysis baseline self-explanation  \cite{DBLP:journals/corr/abs-2012-01786} and the best-performing abstract classification model 
Big Bird \cite{DBLP:conf/nips/ZaheerGDAAOPRWY20} as an external classifier. 
For a fair comparison, we evaluated the performance of all test examples to avoid the testing bias caused by manual sample selection. 
%Analysis
Table~\ref{automated_accuracy} indicates that A-TIP can obtain SOTA performance on both the SST-5 and Abstracts datasets on classification accuracy. 
A-TIP cannot achieve the same accuracy as the original sentence because it has a trade-off between the attribute relevance and fluency of the generated content.


Please note that external classifiers may not provide sufficient reasonability and risk using accuracy as a performance metric for attribute-aware text infilling.
As presented in Table~\ref{automated_accuracy}, external classifiers cannot obtain 100\% classification accuracy on the original sentences of SST-5 and Abstract, which suggests that the poor performance of the baselines is due to the classifier; rather, the baselines cannot generate attribute-relevant content.
However, A-TIP and all the baselines use the same classifier, such as XLNet and BERT-based, which provides a fair comparison environment to a certain extent.
In addition, we used two different external classifiers for each dataset, and the results showed that A-TIP was better than the comparison algorithms, which also proves that the superior results of A-TIP are not related to the classifier.


Some baselines, such as TIGS and BLM on SST-5 and TIGS, MaskGan, BERT and RoBERTa on Abstracts, perform better than broken sentences because they tend to generate random content that can be considered noise for classification (some even generate opposite content of the target attribute). 
For instance, the original sentence is ``Amy likes to 
play with her friends and family. And the broken sentence is ``[Mask] likes to [Mask] [Mask] [Mask] [Mask] [Mask] [Mask],''  which also has a positive sentiment. 
However, after text infilling by BERT, the sentence becomes ``She likes to see movies but hates the actors,'' which provides a negative sentiment.  

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Ablation Study}
To verify the effect of each component in A-TIP, we conducted an ablation study.
Specifically, A-TIP/Dis does not include a plug-and-play discriminator and the text-infilling part remains unchanged.
A-TIP/KL does not include KL loss or a threshold-based strategy. %and the text infilling part and discriminator remain unchanged.
Table~\ref{automated} indicates that A-TIP/Dis can improve text fluency while reducing the attribute relevance.
A-TIP/KL increases attribute relevance and decreases text fluency.
The discriminator can guide generation in an attribute-aware direction while losing fluency to a certain extent.
By incorporating KL and the threshold,
A-TIP achieves a better-balanced performance. %for both the fluency and attribute of generated text.

We also conducted a detailed ablation study to explore the effectiveness of the new attention mechanism and two-level positional encoding.   
As presented in Table~\ref{automated}, A-TIP/attention represents A-TIP without the new attention mechanism. We used the original attention mechanism of the Transformers in A-TIP/Attention. 
We obtained poor perplexity and low diversity of generated content because the original attention mechanism could not capture the bidirectional contextual information of each blank, demonstrating the effectiveness of our new attention mechanism. 
A-TIP/position represents A-TIP without two-level positional encoding. 
We used the original word embedding and positional encoding of the Transformers in A-TIP/Position. 
As can be observed from Table~\ref{automated}, A-TIP/Position outperforms all the baselines in terms of perplexity and attribute-aware accuracy. 
However, A-TIP/Position performs worse than A-TIP because A-TIP/Position cannot accurately identify the length of the infilling content and the numbers/locations of the text infilling.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Human Evaluation}

As shown in Figure \ref{case2}, we considered two types of human annotation: fluency and attribute relevance (Attri-Rele).
Annotators were asked to evaluate the fluency/attribute relevance of each individual sample on a scale of 1$\sim$5, with 1 being not fluent/not relevant at all, and 5 being very fluent/very relevant, as in \cite{human}.
We randomly selected 100 samples for each baseline from each test dataset and asked ten people on Amazon Mechanical Turk to identify the fluency and attribute relevance of each sample.
The ten people had a US Bachelor's degree, and we asked them to complete all human evaluations within three days. 
The maximum time that a worker has to work on a single task was 8~h.
We then used the average scores of the annotations as the final scores.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% fig.5
\begin{figure}[b]
\begin{center}
\includegraphics{30-3ia6f5.pdf}
\end{center}
\caption{Human evaluations on Amazonmturk.}
\label{case2}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% tab.5
\begin{table}[t]
\input{06table05.tex}%
\caption{Human evaluation on three datasets. $\dag$ indicates that the results significantly surpass others.}
\label{human_evl}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

As presented in Table~\ref{human_evl}, A-TIP achieved the highest score compared with the baselines, indicating that sentences filled by A-TIP can be not only more fluent, but also more attribute-relevant.
Somewhat surprisingly, we observed that BERT, TIGS, and MaskGan yielded the worst performance.
BERT performed poorly because of the intrinsic difficulty in finding convincing infilling content of suitable length.
TIGS and MaskGan may have performed poorly because, unlike ILM and A-TIP, they were not initialized from a large-scale pre-trained LM.

However, almost all baseline Attri-Rele scores were higher than 3.0, which demonstrates that humans might not evaluate the pure ability of the attribute-aware discriminator technique for the following two reasons: 
(1) as shown in Figure \ref{case2}, we did not provide more fine-grained scoring of Attri-Rele. 
We only used 3.0 to indicate that the attributes of sentences were not obvious,  4.0 to represent that the sentences had the desired attributes; and 5.0 to represent the sentences that had strong relevance to the desired attributes.
Thus, human evaluation could not provide sufficient evidence that A-TIP performs better than the baseline in terms of attribute relevance.
(2) For a fair comparison and simulation of real-world scenarios, we randomly masked the tokens instead of selectively masking the attribute-relevant tokens. 
The corrupted text retained many attribute-aware tokens. 
Even if the language models do not generate attribute-relevant contents, sentences after infilling can still be correctly classified into corresponding classes, e.g., ``[Mask] is [Mask] happy [Mask] like it.''
After generating the infilling content from all baselines, this sentence can be successfully classified as positive.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Running Time Comparison}


To generate attribute-aware tokens, we updated Eq.(\ref{equation4}) less than ten times for each token.
As shown in Figure \ref{running_time}, we compared the running time between A-TIP/Dis and A-TIP to ensure that we require less additional time.  Specifically, we randomly selected 30 samples from the SST-5 and ROCStories datasets, where SST-5 contained short sentences, and ROCStories contained long sentences.
We then changed the mask rate from 30\% to 70\% for each selected sample to improve the reliability of our results.  
As shown in Figure \ref{running_time}, compared with the unconditional LM-based text generation task, updating the hidden state towards attribute-relevant direction takes less additional time.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% fig.6
\begin{figure}[b]
\begin{center}
\includegraphics{30-3ia6f6.pdf}
\end{center}
\hangcaption{Running time comparison between A-TIP/Dis and A-TIP on SST-5 and ROCStories. We change the mask rate from 30\% to 70\%.}
\label{running_time}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%% fig.7
\begin{figure}[b]
\begin{center}
\includegraphics{30-3ia6f7.pdf}
\end{center}
\caption{Case study for sentiment content infilling and expert knowledge infilling.}
\label{case}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Case Study}

We conducted a case study to demonstrate the infilling ability of A-TIP.
In particular, as shown in Figure \ref{case}, we first propose to infill the blanks with sentimental words. 
We chose RoBERTa and BLM for comparison. This is because these two methods yield the best results.
We can see that RoBERTa infill blanks with two contradictory words (fun and heartbreaking), whereas humans do not have such contradictory and complex emotional expressions. 
BLM can unify the expression of emotions but cannot ensure the fluency of the generated sentence.
By contrast, A-TIP can be used to generate positive or negative infilling content with high fluency.

In the second case, we wanted to explore whether A-TIP could generate domain knowledge for a specific area. 
We chose BERT and TIGS as examples for comparison. These two methods obtained the best results for domain-knowledge infilling.
We found that they were unable to generate expert knowledge-infilling content. Additionally, they tended to generate correct and high-frequency infilling content, but it was generally meaningless and monotonous \cite{sun-etal-2021-generating,lazar-etal-2021-filling,su-etal-2021-keep}.
However, we can control A-TIP to generate both CS-related and Math-related infilling content by constraining the attributes to CS and Math.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
\label{conclusion}
In this paper, we present a simple strategy for text-infilling, A-TIP, that leverages an LM by introducing new attention mechanisms and two-level positional encoding. By fine-tuning A-TIP on small-scale datasets, such as SST-5, Abstracts, and ROCStories, it effectively improves the quality of generation.
Furthermore, our plug-and-play discriminator can guide the generation to improve text attribute relevance.
In future work, we plan to incorporate the plug-and-play discriminator into additional systems that assist humans in the writing process. We hope that our work will encourage further investigation on text infilling. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment
First, we would like to thank all the reviewers for their valuable suggestions, which helped us improve the quality of our manuscript.
This study is an extended version of our preliminary paper \cite{li-etal-2022-tip} presented in the Proceedings of the 29th International Conference on Computational Linguistics (COLING-2022).
We provided more detailed explanations in the Introduction to clarify our contributions. 
We summarized the related works in Section 2. 
We also extended Section 4 by conducting additional experiments, such as topic detection and parameter tuning.
Finally, Dongyuan Li acknowledges the support from the China Scholarship Council (CSC).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{06refs}

% Start biography
\begin{biography}



\bioauthor[:]{Dongyuan Li}{%
D. Li received his bachelor's degree in computer science from Dalian University of Technology in 2018. He received the Master's degree from the School of Computer Science and Technology, Xidian University.  
He is currently pursuing the Ph.D. degree at the School of Engineering, Tokyo Institute of Technology.
His research interests include natural language processing, machine learning, data mining, social network analyses, and bio-informatics.
}

\bioauthor[:]{Kotaro Funakoshi}{
K. Funakoshi received his Dr. Eng. degree from the Tokyo Institute of Technology, Japan and is currently an associate professor at the Institute of Innovation Research, Tokyo Institute of Technology. 
Formerly with Honda Research Institute Japan Co., Ltd., he has served as a program-specific associate professor with the Cooperative Intelligence Joint Chair at Kyoto University since 2017. 
His research interests include natural language processing, particularly computational linguistics, multi-modal dialogue systems, and human-machine interaction.
}

\bioauthor[:]{Manabu Okumura}{%
M. Okumura received his Dr. Eng. degree from the Tokyo Institute of Technology and is currently a professor at the Institute of Innovative Research. His current research interests include natural language processing, particularly, text summarization, computer-assisted language learning, sentiment analysis, and text mining.
}
\end{biography}

\biodate

\end{document}
