\documentclass[english]{jnlp_3.3}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
%%%%\usepackage{array}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage{textcomp}
\usepackage[dvipdfmx]{xcolor}
%%%%\usepackage{graphics}
%%%%\usepackage{subcaption}
\usepackage{enumitem}
\newlist{myQuoteEnumerate}{enumerate}{2}% Set max nesting depth
\setlist[myQuoteEnumerate,1]{label=(\alph*)}
\setlist[myQuoteEnumerate,2]{label=(\alph*)}%
\newenvironment{MyQuote}{%
    \begin{myQuoteEnumerate}[resume=*,series=MyQuoteSeries]%
    \item \begin{quote}%
}{%
    \end{quote}%
    \end{myQuoteEnumerate}%
}%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{28}
\Number{2}
\Month{June}
\Year{2021}
%%%%\Issuetitle{「　　　　」特集号}


\received{2020}{11}{1}
\revised{2021}{2}{9}
%%%%\rerevised{2006}{5}{6}
\accepted{2021}{3}{25}

\setcounter{page}{651}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Heterogeneous Graph Based Extractive Summarization Considering Discourse and Coreference Relations}
\eauthor{Yin Jou Huang\affiref{Author_1} \and Sadao Kurohashi\affiref{Author_1}} 
\eabstract{%
Modeling the relations between text spans in a document is a crucial yet challenging problem for extractive summarization. Various kinds of relations exist among text spans of different granularity, such as discourse relations between elementary discourse units and coreference relations between phrase mentions. In this paper, we utilize heterogeneous graphs that contain multiple edge/node types to model the input document as well as the various relations among text spans in it. Also, we propose a heterogeneous graph based model for extractive summarization that considers the heterogeneity of the document graph. Experimental results on a benchmark summarization dataset verify the effectiveness of our proposed method.
}
\ekeywords{Extractive Summarization, Graph Neural Network, Heterogeneous Graph, Discourse Relation, Coreference Relation}

\headauthor{Huang and Kurohashi}
\headtitle{Heterogeneous Graph Based Extractive Summarization}

\affilabel{Author_1}{}{Graduate School of Informatics, Kyoto University}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Automatic summarization aims to condense the information of the input document into a shorter summary.
Previous works on the task of automatic summarization can be categorized into two main paradigms: abstractive summarization and extractive summarization.
Abstractive summarization methods encode the input document and generate output summaries from scratch.
Abstractive summarizers can generate concise and flexible summaries, but they also suffer from the problem of not being able to reproduce factual details correctly \cite{get_to_the_point}.
On the other hand, extractive summarization methods aim to select salient text spans (mostly sentences) from the input document as summaries. 
Conceptually, extractive summarization resembles the process of highlighting the important parts of the input document.
Due to the above nature, extractive summarizers have the advantage of being computationally efficient and factually reliable.
In this paper, we will focus on extractive summarization.

Most existing extractive summarization models operate on sentence level.
However, the extracted sentences may contain redundant details that harm the conciseness of the summary.
This kind of sentence-level redundancies exist pervasively even in the salient sentences. 
Consider the following examples:
\begin{MyQuote}
    \textit{
    \textbf{{[Boston native Mark Wahlberg will star in a film about the Boston Marathon bombing and the manhunt,]}\textsubscript{$EDU_1^a$}}
    {\color{gray}{[Deadline reported Wednesday.]}\textsubscript{$EDU_2^a$}}
    }
\end{MyQuote}
\begin{MyQuote}
    \textit{
    \textbf{{[Yahya Rashid,]}\textsubscript{$EDU_1^b$}} 
    {\color{gray}{[a UK national from northwest London,]}\textsubscript{$EDU_2^b$}} 
    \textbf{{[was detained at Luton Airport on Tuesday]}\textsubscript{$EDU_3^b$}} 
    \textbf{{[after he arrived on a flight from Istanbul, ]}\textsubscript{$EDU_4^b$}} 
    {\color{gray}{[police said.]}\textsubscript{$EDU_5^b$}}
    }
\end{MyQuote}
Both sentence (a) and sentence (b) contain important information about the salient events (highlighted in bold font in the above example).
However, there are also plenty of peripheral details that could safely be excluded from the summary without harming the overall understanding of the events (colored in gray in the above example).

To avoid the sentence-level redundancy as shown in the examples above, we perform extraction on a smaller granularity---Elementary Discourse Unit (EDU)---in this work.
EDU is a sub-sentence unit that originated from discourse analysis \cite{rst}.
In the examples above, sentence (a) can be further segmented into two EDUs, and sentence (b) can be segmented into five EDUs.
Among them, only $EDU_1^a$, $EDU_1^b$, $EDU_3^b$ and $EDU_4^b$ present central concept of the sentences. 
By performing EDU-level extractive summarization, we can eliminate unwanted trivial details like $EDU_2^a$, $EDU_2^b$, and $EDU_5^b$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{28-2ia14f1.pdf}
\end{center}
\caption{Relations among text spans of different granularity.}
\label{fig:rels}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The goal of extractive summarization is to identify salient text spans that represent the central ideas of the input document.
Thus, it is crucial to model the overall document structure and the various relations between the text spans across the document.
Natural language documents have a hierarchical nature, with each level corresponding to a different level of granularity: document, sentences, EDUs, words and phrases (Figure \ref{fig:rels}(a)).
Between text spans of different granularity, there exist many different kinds of relations.
For example, discourse relations exist between EDUs within a document and coreference relations exist between mention phrases that correspond to the same real-world entity (Figure \ref{fig:rels}(b)).
The discourse relations between EDUs provide important clues for extractive summarization.
The discourse structure describes the high-level linguistic structure of the document, and the concept of nucleus and satellite discourse units in the Rhetorical Structure Theory \cite{rst} defines the relative saliency of discourse units.
Thus, the discourse relations are helpful in identifying the salient EDUs of an input document.
Also, since salient entities are often mentioned multiple times in a given document, the coreference relations between mention phrases can implicitly capture the narrative structure of the document.
Figure \ref{fig:coref_narr} shows a document where the protagonist `Yahya Rashid' is mentioned multiple times.
By observing each mention of the entity `Yahya Rashid' as well as its context, we can observe the narrative revolving around the given protagonist.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{28-2ia14f2.pdf}
\end{center}
\hangcaption{Coreference relations around a given entity reflects the narrative structure in which the entity being the protagonist.}
\label{fig:coref_narr}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Due to its complex nature, modeling the various relations among text spans of a document remains an open challenge.
Some recent works capture inter-sentential relations by utilizing recurrent neural networks (RNNs) or Transformer \cite{attn} based encoders on top of the acquired sentence representations \cite{cheng-lapata-2016-neural,summarunner,presumm}.
However, empirical observations show that these sentence-level encoders do not bring much performance gain \cite{presumm}.
Graph structure is an intuitive way to model long-range relations among text spans throughout a document.
Early works build connectivity graphs based on content similarity between sentences \cite{lexrank,textrank}.
Some recent works incorporate discourse or coreference relations into the graph structure and utilize graph neural networks (GNNs) to obtain a high-level representation of text spans \cite{multidoc_adg,xu2019neural,discobert}.
Most of these works operate on homogeneous graphs such as Approximate Discourse Graph (ADG) \cite{adg} or Rhetorical Structure Theory (RST) \cite{rst} dependency graph.
By definition, a homogeneous graph contains only one type of node as well as only one type of edge.
On the other hand, a heterogeneous graph consists of more than one type of nodes or more than one type of edge.
As illustrated in Figure \ref{fig:rels}, the various types of relations exist between text spans of different granularity.
To model the various text spans of different granularity (nodes) as well as the various types of relations (edges) among them, a heterogeneous graph is a more natural choice than homogeneous graphs.

In this paper, we propose a novel heterogeneous graph based model for extractive summarization.
First, we model each input document as a heterogeneous document graph.
The heterogeneous graph contains three types of nodes of different granularity: sentence nodes, EDU nodes, and entity nodes.
We encode both discourse and coreference relations into the heterogeneous document graph.
The discourse relations are encoded with the edges between EDU nodes.
As for the coreference relations, edges between EDU nodes and entity nodes are introduced.
In addition, we model the hierarchical structure of the document by adding edges between sentence nodes and their constituent EDU nodes.
Further, we adopt a Graph Attention network (GAT) \cite{gat} based graph encoder to capture the various relations in the heterogeneous document graph.
Different from the original GAT network, which is proposed for handling homogeneous graphs, our proposed graph encoder considers the heterogeneous nature of the document graph and applies separate processing for each type of node.
To the best of our knowledge, we are the first to utilize heterogeneous graphs to incorporate multiple types of relations (discourse relations and coreference relations) simultaneously for extractive summarization.

Our main contribution is threefold:
(1) We propose a heterogeneous document graph that incorporates multiple types of relations simultaneously for extractive summarization.
(2) We propose a GAT-based graph encoder that considers the heterogeneity of the document graph.
(3) We conducted experiments on summarization benchmark corpora CNN/DailyMail (CNNDM) \cite{cnndm} and New York Times \cite{nyt} and verified the effectiveness of our proposed method.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Document Graph Construction}
\label{graph_construction}

In this section, we introduce the construction of document graphs.
Section \ref{prepro} gives details of the pre-processing steps to obtain the discourse and coreference information required for the construction of document graphs.
Section \ref{homo-doc-g} gives a brief introduction of the homogeneous document graphs used in previous works as well as their limitations.
At last, Section \ref{g1} and \ref{g2} introduce two heterogeneous document graphs which tackle the limitations of the homogeneous document graphs.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Pre-processing}
\label{prepro}

Both discourse information and coreference information are helpful in deciding the content saliency, and have been incorporate as external knowledge into extractive summarization systems.

Given an input document $D$, we first perform the following pre-processing steps and construct the document graphs based on the discourse information and coreference information acquired.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Discourse Parsing}

Given document $D$ with $m$ sentences $\{s_1,...,s_m\}$, we first segment the sentences into $n$ contiguous, adjacent and non-overlapping EDUs $\{d_1,...,d_n\}$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[b]
\begin{center}
\includegraphics{28-2ia14f3.pdf}
\end{center}
\hangcaption{An example of RST discourse parse tree. Based on the rhetorical relations among the discourse units, we can derive the RST dependency graph.}
\label{fig:rst_conv}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Further, we perform RST discourse parsing to identify the rhetorical relations between the EDUs.
In the RST framework, the discourse structure of document $D$ is represented by a continuous constituency tree.
For example, Figure \ref{fig:rst_conv} illustrates the RST parse tree consisting of five EDUs.
The RST parse tree is constructed by continuously merging adjacent discourse units into a larger discourse unit.
In addition, the merged discourse units are tagged as either nucleus(N) or satellite(S), which indicates their relative nuclearity/saliency.
Nucleus units are considered more salient, while satellite units are less important in content.
RST defines two types of rhetorical relations between discourse units: 
\begin{itemize}
    \item \textbf{Mononuclear relation} that links a satellite unit and a nucleus unit. Such as relation between $EDU_1$ and $EDU_2$.
    \item \textbf{Multinuclear relation} that links two nucleus units. Such as the relations between $EDU_{1-2}$ and $EDU_{3-5}$.
\end{itemize}

Similar to \cite{discobert}, we convert the RST tree to the dependency form based on the rhetorical relations in the RST tree.
The RST dependency graph consists of EDU nodes.
In the case of mononuclear relation, the dependency graph contains directed edges from satellite nodes to the corresponding nucleus nodes.
In the case of multinuclear relation, we link the nucleus nodes in both directions.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Coreference Resolution}

In addition, we perform coreference resolution to identify coreferent relations among mention phrases in the input document $D$.
The goal is to identify mention phrases that refer to the same real-world entity, such as the mentions 'Boston native Mark Wahlberg` and 'Wahlberg` (highlighted in red) in Figure \ref{fig:rels}.
The mentions in document $D$ are clustered into $k$ entities $\{e_1,...,e_k\}$, with each entity $e_i$ representing a cluster of mentions among which coreference relations holds.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[p]
\begin{center}
\includegraphics{28-2ia14f4.pdf}
\end{center}
\caption{Homogeneous and heterogeneous document graphs.}
\label{fig:hetG}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Homogeneous Document Graphs}
\label{homo-doc-g}

In many works on graph-based extractive summarization, discourse information and coreference information are used to build homogeneous document graphs with sentence/EDU nodes.
By definition, homogeneous graphs contain only one type of node and one type of edge.

For example, we can embed discourse relations with the RST dependency graph (Figure \ref{fig:rst_conv}).
Also, we can build a homogeneous coreference graph (Figure \ref{fig:hetG}(a)) with EDU nodes in which EDU nodes containing a common entity are linked.
However, it is not straightforward to incorporate discourse relations and coreference relations together in a single homogeneous graph.

Some previous works like DiscoBERT \cite{discobert} utilize two different graphs to embed discourse and coreference relations separately.
However, this method neglects the interaction between different relation types.
Some other works use weighted graphs (Figure \ref{fig:hetG}(b)) like ADG to embed multiple types of textual relations, with the edge weights represents the overall `strength' of relations between text nodes \cite{adg}.
Although it is possible to combine multiple different relation types within the same graph in this manner, it is difficult to design the proper edge weights.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Heterogeneous Document Graph with Multiple Edge Types}
\label{g1}

To tackle the limitations of homogeneous document graphs, we consider a heterogeneous document graph with multiple edge types.
As shown in Figure \ref{fig:hetG}(c), we represent each input document $D$ with a heterogeneous graph $G_1=\{V, E\}$, with $V$ and $E$ being the set of nodes and edges, respectively.

In the pre-processing step, we perform discourse parsing and coreference resolution on $D$ (Section \ref{prepro}).
Based on the pre-processing results, we construct the heterogeneous document graph $G_1$.

$V=\{d_1,...,d_n\}$ consists of the EDU nodes, with each node $d_i$ representing the $i^{th}$ EDU in document $D$.

$E$ consists of three types of edges:
\begin{itemize}
    \item \textbf{Discourse edge}: Based on the discourse parsing results, we add the RST dependency edges between EDU nodes.
    \item \textbf{Coreference edge}: Based on the coreference resolution results, the two EDUs which contain mentions of the same entity are connected. 
    \item \textbf{Same-sentence edge}: EDU nodes that belong to the same sentence are connected.
\end{itemize}

In this work, we use the above-mentioned document graph $G_1$ as a baseline to compare with our proposed heterogeneous document graph with multiple node types (introduced in Section \ref{g2}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Heterogeneous Document Graph with Multiple Node Types}
\label{g2}

In Figure \ref{fig:hetG}(a)--(c), the coreference relations are represented by the edges between EDU nodes.
However, the important information about the coreference entities is overlooked.
For example, in Figure \ref{fig:hetG}(c), a coreference edge only indicates that the two EDUs share one or more common entities, but does not give information regarding what entities actually are shared by the EDUs.

To tackle the above limitations, we propose a heterogeneous document graph with multiple node types.
As shown in Figure \ref{fig:hetG}(d), input document $D$ is represented with a heterogeneous document graph $G_2=\{V, E\}$, where $V$ and $E$ are the set of nodes and edges.

$V$ contains three types of nodes:
\begin{itemize}
    \item \textbf{Sentence nodes} $V_s = \{s_1,...,s_m\}$, with $s_i$ representing the $i^{th}$ sentence in $D$.
    \item \textbf{EDU nodes} $V_d = \{d_1,...,d_n\}$, with $d_i$ representing the $i^{th}$ EDU in $D$.
    \item \textbf{Entity nodes} $V_e = \{e_1,...,e_k\}$, with each $e_i$ representing a phrase entity in $D$.
\end{itemize}

We utilize the pre-processing results acquired in Section \ref{prepro} to construct the heterogeneous document graph $G_2$ as follows:
\begin{itemize}
    \item \textbf{Discourse edge}: We add these RST dependency edges between EDU units to our document graph $G_2$ to model the discourse structure of the document.
    \item \textbf{Coreference edge}: We use edges between EDU nodes and entity nodes to embed the coreference relations. If EDU $d_i$ contains a mention of entity $e_j$, then we add an undirected edge ($d_i$, $e_j$) to $E$. That is, each entity node indirectly connects all EDUs that contain mentions of the entity. In this way, the subgraph around a specific entity node implicitly models the narrative structure related to the entity.
    \item \textbf{Same-sentence edge}: Based on the discourse segmentation result, we connect each sentence node to its constituent EDU nodes.
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Proposed Method}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Overview}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Problem Formulation}

Given an input document $D$ with $n$ EDUs $\{d_1, d_2, ..., d_n\}$, we formulate EDU-based extractive summarization task as a sequence labeling problem.
The model predicts a sequence of binary labels $Y=\{y_1, y_2, ..., y_n\}$, where $y_i=1$ indicates that the $i^{th}$ EDU, $d_i$, should be included in the summary.

From the human-written summary, we heuristically obtain the oracle labels $\{y_1^*, y_2^*, ..., y_n^*\}$, which can be used to train our extractive summarization model.
Further details will be given in Section \ref{oracle}.
In the inference stage, the model predicts the binary labels for each EDU in the input document.
The EDUs with label $y_i=1$ will be concatenated to form the summary.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Model Overview}

Figure \ref{fig:sys} provides an overview of our proposed model.
First, the input document $D$ is feed into a Longformer \cite{longformer} document encoder.
With the self-attention based EDU and entity encoders, we acquire the initial node representation of the heterogeneous document graph (Section \ref{graph_init}).
We then apply a heterogeneous graph encoder to obtain high-level node representations considering the document graph structure (Section \ref{gat_layer}).
Finally, we make predictions based on the learned EDU node representations (Section \ref{predict_layer}).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[t]
\begin{center}
\includegraphics{28-2ia14f5.pdf}
\end{center}
\caption{System overview.}
\label{fig:sys}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Graph Node Initialization}
\label{graph_init}

Following the settings in \cite{presumm}, we utilize pretrained Longformer \cite{longformer} to encode the input document $D$.
We insert the $\langle CLS \rangle$ and $\langle SEP \rangle$ special tokens to the beginning and the end of each sentence $s_i$, respectively.
$\langle CLS \rangle$ special token was originally used to aggregate features from one or a pair of sentences.

With the output vectors of Longformer, we acquire the initial node representations.
In the following, we describe how we obtain the initial representations of sentence nodes, EDU nodes, and entity nodes, respectively.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Sentence Node Representations}

For each sentence node $s_i$ in $V_s$, we take the Longformer output vector of the $\langle CLS \rangle$ token before $s_i$ as the sentence node representation $h_i^s$.
We expect the $\langle CLS \rangle$ symbol to aggregate the information of the tokens of the sentences that follows it.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{EDU Node Representations}

We use a self-attention based EDU encoder to encode each EDU node in $V_d$.
Given an EDU $d_i$ consisting of tokens $\{w_i^1, w_i^2, ..., w_i^N\}$, we obtain the node representation $h_i^d$ by taking self-attention on the Longformer output vectors $\{v_i^1, v_i^2,..., v_i^N\}$ of its consisting tokens:
\begin{gather}
    \alpha_{ij}=v_2 ReLU(W_1v_i^{j} + b_1) \\
    a_{ij} = \frac{exp(\alpha_{ij})}{\sum_{k=1}^{N} exp(\alpha_{ik})} \\[1ex]
    h_i^d = \sum_{j}a_{ij}v_i^j
\end{gather}
where $W_1$, $b_1$, $v_2$ are trainable weights.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Entity Node Representations}
\label{t}

The structure of the entity encoder is identical to the EDU encoder. 
For each entity $e_i$ in $V_e$, we consider all mentions of it.
By taking self-attention among the Longformer output vectors which correspond to tokens of these mentions, we can acquire the entity representation $h_i^e$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Heterogeneous Graph Encoder}
\label{gat_layer}

We initialize the representation of each node in $G$ with the sentence representations ($\{h_i^s\}$), EDU representations ($\{h_i^d\}$), and entity representations ($\{h_i^e\}$) acquired in Section \ref{t}.
We then use the heterogeneous graph encoder to learn high-level node representations considering the structure of the document graph.

Figure \ref{fig:gat_layer} illustrates the structure of our heterogeneous graph encoder.
Each graph encoding layer $L_i$ consists of two Transformer sub-layers and a graph attention sub-layer.
The Transformer sub-layers aim to model the interactions among nodes with the same granularity.
In the graph attention sub-layer, we apply graph attention network (GAT) \cite{gat} to model the interactions among all types of nodes in $G$ and update the node representations.
The complete heterogeneous graph encoder consists of $N$ stacked graph encoding layer $\{L_1,...,L_N\}$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[t]
\begin{center}
\includegraphics{28-2ia14f6.pdf}
\end{center}
\caption{Structure of the heterogeneous graph encoder.}
\label{fig:gat_layer}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Transformer Sub-layers}

We first feed the sentence vectors and EDU vectors to the sentence-level and the EDU-level Transformer \cite{attn} sub-layers.

At the center of the Transformer sub-layer is a multi-head self-attention layer follow by a feed-forward layer.
The self-attention mechanism can be seen as a fully-connected version of a graph attention network. %%
To model the interactions among nodes of the same granularity, we utilized two types of Transformer layers, which operate on the sentence-level and the EDU-level, respectively.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Graph Attention Sub-layer}

The graph attention sub-layer consists of a multi-head GAT network followed 
\pagebreak
by a feed-forward layer.
Taking the document graph and the node representations as input, the purpose of the graph attention sub-layer is to learn a higher-level representation of each node by aggregating information from its neighboring nodes.
Here, we introduce two types of GAT networks, the \textbf{vanilla GAT network} and the \textbf{heterogeneous GAT network}.
Similar to the one proposed in \cite{gat}, the vanilla GAT network handles the document graph $G$ as a homogeneous graph and treats all types of nodes in the same way.
On the other hand, our proposed heterogeneous GAT network considers the heterogeneity of $G$ and applies different processing to different node types.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Vanilla GAT network}

We apply graph attention networks (GAT) \cite{gat} to update the node representations in $G$.
For the $i^{th}$ node, we update the representation $h_i$ of node $i$ with the representations of its neighbors $\{h_j\}$:
\begin{gather}
    \alpha_{ij} = LeakyReLU(W_a[W_qh_i;W_kh_j]) \\
    a_{ij} = \frac{exp(\alpha_{ij})}{\sum_{k} exp(\alpha_{ik})}\\[1ex]
    h_i \leftarrow W_t(\sigma(\sum_{j}a_{ij}W_vh_j) + h_i)
\end{gather}
where $W_a$, $W_q$, $W_k$, $W_v$, $W_k$ are trainable weights.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 7
\begin{figure}[t]
\begin{center}
\includegraphics{28-2ia14f7.pdf}
\end{center}
\caption{Graph attention sub-layer.}
\label{fig:3}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Figure \ref{fig:3} illustrates an example of graph attention mechanism.
The subgraph centering around node $EDU_1$ is highlighted in Figure \ref{fig:3}(a).
$EDU_1$ is connected to a sentence node $sent_1$, two EDU nodes $EDU_2$ and $EDU_3$, and two entity nodes $entity_1$ and $entity_2$.
With vanilla GAT network, we calculate the attention weight across the five neighbors of $EDU_1$ and updated the node representation of $EDU_1$ ($h^d_1$) accordingly (Figure \ref{fig:3}(b)).

Although a single GAT network only considers the first-degree neighbors, by stacking several layers of GAT network, we can obtain a higher-level representation for each node in $G$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Heterogeneous GAT Network}

The vanilla GAT network disregards the heterogeneity of the document graph and treats different types of nodes identically.
Take Figure \ref{fig:3} for example, all three types of neighboring nodes of $EDU_1$ (sentence, EDU, and entity nodes) use the same equation for calculating attention scores $\alpha$ (Figure \ref{fig:3}(b)).

Considering the heterogeneous nature of the document graph $G_2$, we introduce a heterogeneous version of the GAT network (Figure \ref{fig:3}(c)).
To incorporate the multiple node types of heterogeneous document graph $G_2$, we introduce different query and key matrices for each type of node.
For example, we use query matrix $W_q^d$ and key matrix $W_k^d$ for EDU nodes, query matrix $W_q^e$ and key matrix $W_k^e$ for entity nodes, and query matrix $W_q^s$ and key matrix $W_k^s$ for sentence nodes.

Different query ($\{W_q^d, W_q^e, W_q^e\}$) and key matrices ($\{W_k^d, W_k^e, W_k^e\}$) are used during the attention calculation process for different types of nodes:
\begin{gather}
    \alpha_{ij} = LeakyReLU(W_a[W_q^{t_i}h_i;W_k^{t_j}h_j]) \\
    a_{ij} = \frac{exp(\alpha_{ij})}{\sum_{k} exp(\alpha_{ik})} \\[1ex]
    h_i \leftarrow W_t(\sigma(\sum_{j}a_{ij}W_vh_j) + h_i)
\end{gather}
where $t_i$ indicates the type of node $i$, and $W_a$, $\{W_q^d, W_q^e, W_q^e\}$, $\{W_k^d, W_k^e, W_k^e\}$, $W_v$, $W_k$ are trainable weights.

For example, the attention score $\alpha$ between $sent_1$ and $EDU_1$ is calculated with the key matrix for sentence nodes $W_k^s$, while the attention score $\alpha$ between $entity_1$ and $EDU_1$ is calculated with the key matrix for entity nodes $W_k^e$.

As for the baseline document graph $G_1$, we consider a heterogeneous GAT network that considers the multiple edge types.
We introduce binary variables $d_{ij}$, $c_{ij}$ and $s_{ij}$ to indicate the existence of discourse edge, coreference edge and same-sentence edge between node $i$ and node $j$.
For example, if there is a discourse edge between node $i$ and node $j$, then $d_{ij}=1$, otherwise, $d_{ij}=0$.
The attention weight between node $i$ and node $j$ is calculated as follows:
\begin{equation}
    \alpha_{ij} = LeakyReLU(W_a[W_qh_i;d_{ij}W^d_kh_j;c_{ij}W^c_kh_j;s_{ij}W^s_kh_j])
\end{equation}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Prediction Layer}
\label{predict_layer}

We feed the final representation of the EDU nodes ($h_i^d$) to the prediction layer with sigmoid activation to predict binary labels:
\begin{equation}
    \hat{y_i}=\sigma(W_ph_i^d + b_p)
\end{equation}
The training loss of the model is the binary cross-entropy loss $L$ against the oracle extraction labels:
\begin{equation}
    L = -\sum_{i} y_i^* log(\hat{y_i}) + (1-y_i^*) log(1 - \hat{y_i})
\end{equation}
in which $\{y_i^*\}$ are the oracle labels and $\{\hat{y_i}\}$ are the binary labels predicted by the model.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiment}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Experimental Settings}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Dataset} \label{oracle}

We evaluated our proposed model on the benchmark CNN/DailyMail dataset \cite{cnndm} and New York Times dataset \cite{nyt}.

For the CNN/DailyMail dataset, we use the non-anonymized version of the standard split, which contains 287,227/13,368/11,490 documents for training, validation, and test split, respectively.
As for the New York Times (NYT) dataset, we follow previous works and split the dataset into 137,7787/17,222/17,223 documents for training, validation, and test split, respectively.

Since both datasets only contain abstractive gold summaries, we have to construct oracle labels heuristically.
Similar to \cite{presumm}, we created the oracle labels on EDU-level by greedily selecting EDUs which maximize the ROUGE scores \cite{rouge}.
We used the average of ROUGE\textsubscript{1}-F\textsubscript{1} and ROUGE\textsubscript{2}-F\textsubscript{1} as selection criteria.
For each document, we selected up to 5 EDUs.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Pre-processing}

We used the Stanford CoreNLP \cite{corenlp} to split sentences.
Further, we used the RST discourse parser proposed by \cite{rst_parser} for both discourse segmentation and discourse parsing.
For coreference resolution, we used the spanBERT-based \cite{spanbert} version of the end-to-end coreference resolver proposed by \cite{end2end_coref}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Hyper-parameter Settings}

We used the `longformer-base-4096' version of Longformer to encode the input document.
The length of each document is truncated to 1024 BPEs.
The hidden size of the EDU encoder and the entity encoder is 128.
Based on the evaluation losses on the validation set, we set the number of stacked graph encoding layers to $N=2$.
For both Transformer sub-layers and the graph attention sub-layers, the number of attention heads is set to 8, with each head having a hidden size of 96.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Training and Evaluation}

During training, we used a batch size of 20.
We used Adam optimizer with $\beta_1=0.9$ and $\beta_2=0.999$ and followed the learning rate ($lr$) scheduling in \cite{attn} with warm-up of 4000 steps ($n_{warmup}$):
\pagebreak
\[
lr = 2e^{-3} min({n_{step}}^{-0.5}, n_{step}{n_{warmup}}^{-1.5})
\]

All models are trained for 60000 steps.
We selected the top-3 checkpoints based on the evaluation losses on the validation set and report the average scores of them on the test set.
During the inference phase, the trained model is used to obtain likelihood score for each EDU.
The top-5 EDUs with the highest likelihood scores are concatenated to generate the final summary.
We also perform trigram blocking in the inference phase, which is a simple yet effective way to reduce redundancy in extractive summarization.

We adopt ROUGE \cite{rouge} as the evaluation metrics.
We report the F\textsubscript{1} scores of the ROUGE\textsubscript{1}, ROUGE\textsubscript{2}, and ROUGE\textsubscript{L} metrics of our proposed models.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{14table01.tex}
\caption{Results on CNN/DailyMail dataset and NYT dataset.}
\label{table:res}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Results and Analysis}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Results on CNN/DailyMail Dataset}

Table \ref{table:res} shows the results on CNN/DailyMail dataset.
The first part contains the \textbf{LEAD-3} baseline and \textbf{Oracle} upper bounds.
The second part of the table includes the sentence-based extractive models, and the third part includes the EDU-based extractive models.
In the third part of the table, we present the evaluation scores of our proposed models with \textbf{vanillaGAT} and the \textbf{hetGAT} version.
heterogeneous document graph $G_2$ (introduced in Section \ref{g2}) is used here.
Also, we included results of maximum input size 1024 and 768.
For the input size 768, we use BERT \cite{bert} as encoder instead of Longformer, which is the same setting as DiscoBERT \cite{discobert}.

As Table \ref{table:res} shows, all our proposed models outperform the \textbf{LEAD-3} and all sentence based extractive baseline models.
Compared to the \textbf{BertSum(sent)} baseline, our proposed model (\textbf{hetGAT, 1024}) achieved a higher ROUGE on all three metrics (R-1/R-2/R-L).
We conclude that EDU-based extraction is a promising direction in extractive summarization.

Our proposed model (\textbf{hetGAT}) also outperform the \textbf{BertSum(EDU)} baseline by a significant margin in all three metrics (R-1/R-2/R-L).
This result shows the effectiveness of our graph encoder module to capture the complex relations among the text spans of the input documents.

Compared to the state-of-the-art EDU-extraction model \textbf{DiscoBERT}, our proposed model (\textbf{hetGAT, 1024}) achieved comparable performance on R-1/R-2 metrics and outperformed it on the R-L metrics by 0.45 of F\textsubscript{1} score.
However, the performance of the proposed model with 768 maximum input size (\textbf{hetGAT, 768}) has worse performance compare to DiscoBERT on R-1/R-2 metrics.
\textbf{DiscoBERT} incorporates a strict RST-based rule during oracle construction and post-processing to ensure discourse consistency.
Since the purpose of this paper is to propose a heterogeneous graph based method for modeling text span relations, we will leave the question of discourse consistency to future work.\footnote[1]{%
The SpanBERT based end-to-end coreference resolver ($\text{F1}=0.77$, OntoNote corpus) has a better performance than the Stanford CoreNLP coreference resolver ($\text{F1}=0.69$, OntoNote corpus) used in DiscoBERT.

Compared to DiscoBERT, our proposed model can better adapt to long input documents.
First, the homogeneous structure of their work is not efficient in embedding coreference relations.
Take the example of an entity with k mentions, the homogeneous coreference graph (like Figure 4(a)) needs to use $\frac{k(k-1)}{2}$ edges, while our proposed heterogeneous graph only needs $k$ edges to represent the coreference relation.
Since both GCN (used in DiscoBERT) and GAT have time complexity $O(|E|)$, the larger number of edges will make it difficult to adapt to longer documents.
Also, unlike their GCN-based method, the GAT-based method we adopted is exempted from eigen-compositions and costly matrix operations.
}

Finally, for both types of input length (768 and 1024), we can observe that the \textbf{hetGAT} model outperforms the \textbf{vanillaGAT} model.
This shows the effectiveness of our proposed heterogeneous GAT networks in capturing and aggregating the various text relations in the heterogeneous document graphs.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Results on New York Times Dataset}

The results on New York Times dataset is also included in Table \ref{table:res}.
Our proposed method (\textbf{hetGAT}) outperforms the \textbf{BertSum(EDU)} baseline in all R-1/R-2/R-L metrics with a significant margin.
Compared to the state-of-the-art model \textbf{DiscoBERT}, our proposed model is inferior in R-1/R-2 metrics but outperforms \textbf{DiscoBERT} in R-L metrics.

Similar to the results on CNN/Daily Mail dataset, the performance of the \textbf{hetGAT} models are better than the \textbf{vanillaGAT} models, for both maximum input size settings (768 and 1024).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Ablation Study}

We conduct ablation studies on the CNN/Daily Mail dataset by removing components from our proposed document graph (heterogeneous document graph $G_2$ introduced in Section \ref{g2}).
The results of the ablation studies are shown in Table \ref{table:ablation}.

The first part shows the ablation study of the proposed (\textbf{hetGAT}) model.
First, we remove the RST dependency edges between EDU nodes (\textbf{-discourse}).
Next, we remove the coreferential edges between EDU nodes and entity nodes (\textbf{-coref}).
We can see that both discourse and coreference information contributes significantly to the model performance, with discourse information being slightly more important than the coreference information.
We also try to remove the edges between sentence nodes and their constituent EDU nodes (\textbf{-sent}).
However, linking the sentence and EDU nodes does not seem to have a significant impact on model performance.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[b]
\input{14table02.tex}
\caption{Ablation studies on CNN/DailyMail dataset.}
\label{table:ablation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The second part of Table \ref{table:ablation} shows the ablation study of the proposed (\textbf{vanillaGAT}) model.
The results of the ablation study show a similar tendency compared to the \textbf{hetGAT} model.
We can observe from the results that both discourse and coreference information contributes to the model performance separately, but these two different types of information do not aggregate well compared to the case in the \textbf{hetGAT} model.
This result illustrates the effectiveness of our proposed heterogeneous GAT network in handling various types of text relations simultaneously, compared to the vanilla GAT networks proposed for homogeneous graphs.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Results of Different Graph Structure}

We perform experiments on different graph structures with the CNN/Daily Mail dataset.
We compare the system performance on the baseline document graph $G_1$ (Section \ref{g1}) and our proposed heterogeneous document graph $G_2$ (Section \ref{g2}).

As shown in Table \ref{table:res-g}, for both \textbf{vanillaGAT} and \textbf{hetGAT}, using document graph $G_2$ gives a better performance then using $G_1$.
Although both $G_1$ and $G_2$ embeds the discourse, coreference and same-sentence information within the document graph, $G_2$ includes richer information of the coreferent entities and sentences.
The experimental results show the effectiveness of introducing extra entity and sentence nodes in the document graph.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[b]
\input{14table03.tex}
\caption{Results of different graph structure on CNN/Daily Mail dataset.}
\label{table:res-g}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Results of Different Hyper-parameter Settings}

We perform experiments on different hyper-parameter settings to observe how the model performance changes under the changes of hyper-parameters.
Same as ablation study, we use heterogeneous graph $G_2$ in the following experiments.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Number of stacked graph encoding layers}
We modify the number of stacked graph encoding layers ($N$) and observe how it affects the performance.
We report the performance on the validation set from $N=1$ to $N=3$ in terms of validation loss, ROUGE-1, ROUGE-2 and ROUGE-L in Figure \ref{fig:var_n}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 8
\begin{figure}[t]
\begin{center}
\includegraphics{28-2ia14f8.pdf}
\end{center}
\hangcaption{Result of different number of stacked graph encoding layers (N) on validation set of CNN/Daily Mail dataset.}
\label{fig:var_n}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

As can be observed in Figure \ref{fig:var_n}, we can see that there is a significant performance gap between $N=1$ and $N=2$.
On the other hand, there is no significant difference in model performance from $N=2$ to $N=3$.

The phenomenon can be explained by the theory of meta-path of the document graph.
Meta path is a widely used concept proposed to model the various types of relations between nodes in a heterogeneous graph.
A meta path captures a specific type of relations within the given graph.
Figure \ref{fig:mp} illustrates the various meta paths in our proposed heterogeneous document graph:
\begin{enumerate}[label=(\alph*)]
    \item \textbf{EDU-EDU meta path} represents the discourse relation between discourse units.
    \item \textbf{EDU-entity-EDU meta path} connects two EDU nodes sharing a coreferent entity. This 2-hop meta path describes the coreference relation of the entity in the center.
    \item \textbf{entity-EDU-entity meta path} describes the collocation relation between two entities that appear in the same EDU unit.
    \item \textbf{EDU-sent-EDU meta path} represents the hierarchical structure of a sentence and its consisting EDUs.
\end{enumerate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 9
\begin{figure}[t]
\begin{center}
\includegraphics{28-2ia14f9.pdf}
\end{center}
\caption{Meta paths in the heterogeneous document graph.}
\label{fig:mp}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The $N=1$ model cannot capture the 2-hop meta-paths like EDU-entity-EDU, entity-EDU-entity, and EDU-sent-EDU.
We speculate that this could account for the performance loss of the $N=1$ model.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Maximum Input Length}
We modify the maximum input length (in BPEs) and observe how it affects the performance.
We report the ROUGE-1, ROUGE-2 and ROUGE-L scores on the validation set with the maximum input size set to 512, 768, 1024, 2048, 4096 BPEs in Figure \ref{fig:var_mlen}.

Generally, we can observe a performance gain by increasing the maximum input length.
The performance gain is significant for the maximum input size under 1024.
However, the gain is less significant if we increase the input size further.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 10
\begin{figure}[t]
\includegraphics{28-2ia14f10.pdf}
\caption{Results of different maximum input size on CNN/Daily Mail dataset.}
\label{fig:var_mlen}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The average document length of the CNN/DailyMail dataset is around 864 BPEs, with a standard deviation of 443 BPEs.
Also, consider the fact that summary-worthy, salient sentences tend to appear at the beginning of the document.
The above two facts altogether support that setting the maximum input length to 1024 BPEs should give satisfactory results.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Qualitative Analysis}

We also conduct a qualitative analysis of the proposed model.
The effectiveness of discourse relations is more straightforward and widely studied in previous research.
Thus, we focus on the analysis of the role of coreference information in our proposed summarization model.

In the heterogeneous document graph, EDUs containing the same entity phrase are indirectly connected through the node of the given entity.
By analyzing the output of the full proposed model and the model without coreference information (\textbf{-coref}), we found that the models rank the importance of coreferent EDUs differently.
Table \ref{table:qa} indicates a common pattern of the improved cases by incorporating coreference information.
The table shows examples of coreferent EDUs and the ranking of their likelihood scores to be included in the summary.
Comparing the EDU ranking of the full model (\textbf{Rank\textsubscript{coref}}) and the model without coreference information (\textbf{Rank\textsubscript{w/o coref}}), we argue that the model with coreference information is better in discriminating the important EDUs among all EDUs sharing the same entity.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{14table04.tex}
\caption{Qualitative studies on CNN/DailyMail dataset.}
\label{table:qa}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection*{Graph-based Summarization}

Graph-based summarization models have been broadly explored.
Early works build connectivity document graphs based on inter-sentential similarity \cite{lexrank,textrank}.
With the promising results of graph neural networks (GNNs) \cite{gcn,gat}, some recent works utilize GNN to incorporate external knowledge into the model.
For instance, \cite{multidoc_adg} utilizes a sentence-level ADG graph to model discourse and coreference relations.
Some works convert the RST tree of the input document into dependency form in either sentence or EDU level \cite{xu2019neural,discobert}.
Most of these models operate on homogeneous graphs with only one type of node.
Fewer summarization models operate on heterogeneous graphs with different types of nodes.
\cite{hetersum_wei} introduces a heterogeneous graph of sentence, word, and topic nodes, and \cite{hetersum} also utilizes a heterogeneous graph of sentence and word nodes.
However, neither of the above works incorporates external knowledge into the graph.
\cite{topic_sum} performs sentence-based extractive summarization based on the heterogeneous sentence node and nodes representing latent topics.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection*{EDU-based Extractive Summarization}

\cite{role_edu_sum} illustrates the potential of using EDU as the extraction unit for summarization.
\cite{discobert} also introduces an end-to-end EDU-based extractive summarization model.
By using a heuristic based on RST dependency structure, they enhanced the grammaticality and discourse consistency of the extracted summary.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

In this paper, we proposed a novel heterogeneous graph based model for extractive summarization.
By introducing nodes of different granularity, the heterogeneous document graph has the capacity to embed various types of relations between text spans.
In addition, we proposed a heterogeneous GAT network that considers the heterogeneous nature of the document graph.
Experiments on the benchmark datasets illustrated the effectiveness of our proposed method.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\bibliography{14refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Yin Jou Huang}{%
Yin Jou Huang received his B.S. in Taiwan University, Taipei, Taiwan in 2015.
From 2015 she became a student of Graduate School of Informatics, Kyoto University and received her M.S. here in 2017.
She is currently a program-specific researcher in Graduate School of Informatics, Kyoto University.
Her research interest center on natural language processing, particularly graph-based neural networks.
}

\bioauthor[:]{Sadao Kurohashi}{%
Sadao Kurohashi received the B.S., M.S., and Ph.D. in Electrical Engineering from Kyoto University in 1989, 1991, and 1994, respectively.
He has been a visiting researcher of IRCS, the University of Pennsylvania in 1994.
He is currently a professor of the Graduate School of Informatics in Kyoto University.
His research interest include natural language processing and knowledge acquisition/representation.
}
\end{biography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate





\end{document}

