\documentclass[english]{jnlp_3.3}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
%%%%\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage[dvipdfmx,nosetpagesize]{graphicx}
%%%%\usepackage[dvipdfmx,nosetpagesize]{color}
%%%%\usepackage[dvipdfmx,nosetpagesize]{colortbl}
%%%%\usepackage{amsmath}
\usepackage{amsthm}
%%%%\usepackage{amsmath, amsthm, amssymb}
%%%%\usepackage[stable]{footmisc}
%%%%\usepackage{placeins}
%%%%\usepackage{tabularx}
%%%%\usepackage{scalerel}
%%%%\usepackage{soul}
%%%%%add by me
\usepackage{multirow}
%%%%\usepackage{subcaption}
%%%%\usepackage{amsmath}
%%%%\usepackage{amssymb}
\newcommand{\R}{\mathbb{R}}
%%%%\usepackage{graphicx}
\usepackage{csquotes}
%%%%\usepackage[utf8x]{inputenc}
%%%%\usepackage{rotating}
%%%%\usepackage{makecell, multirow}
%%%%\renewcommand\theadfont{\normalsize}
%%%%\renewcommand\theadgape{}
%%%%\usepackage{booktabs}
%%%%\newtheorem{prop}{Proposition}
%%%%\usepackage{xcolor}
    \usepackage[table]{xcolor}
%%%%\usepackage{paralist}
%%%%%\newtheorem{hyp}{Hypothesis}
%%%%\usepackage{multirow} 
%%%%\usepackage[linesnumbered, ruled,vlined]{algorithm2e}
%%%%\SetKwInOut{Parameter}{Parameter}
%%%%%%% Coloring the comment as blue
%%%%\newcommand\mycommfont[1]{\footnotesize\ttfamily\textcolor{blue}{#1}}
%%%%\SetCommentSty{mycommfont}
%%%%\SetKwInput{KwInput}{Input}                % Set the Input
%%%%\SetKwInput{KwOutput}{Output}              % set the Output
%%%%\makeatletter 
%%%%\g@addto@macro{\@algocf@init}{\SetKwInOut{Parameter}{Parameters}} 
%%%%\makeatother
%%%%\usepackage{color}
%%%%\usepackage{tabularx}
%%%%\usepackage{graphicx}
%%%%%\usepackage{subcaption}
%%%%%\usepackage{caption}
%%%%%\graphicspath{ {./images/} }
%%%%\definecolor{chromeyellow}{rgb}{0.91, 0.84, 0.42}
%%%%\definecolor{burntorange}{rgb}{0.8, 0.33, 0.0}
\definecolor{lightgray}{gray}{0.9}
%%%%%\definecolor{chromeyellow}{rgb}{1.0, 0.65, 0.0}
%%%%\definecolor{mypink2}{RGB}{219, 48, 122}
%%%%%\newcommand{\authorA}[1]{{\color{cincinnati-red}[A: #1]}}
%%%%\newcommand{\hu}[1]{{\color{violet}[H: #1]}}
%%%%%\newcommand{\hu}[1]{{\color{blue}[H: #1]}}
%%%%\definecolor{OliveGreen}{rgb}{0,0.6,0}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{28}
\Number{4}
\Month{December}
\Year{2021}
\Issuetitle{Special Issue: XXXXXXXX}

\received{2021}{7}{31}
%%%%\revised{2006}{3}{4}
%%%%\rerevised{2006}{5}{6}
\accepted{2021}{9}{6}

\setcounter{page}{1247}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Metric-Type Identification for Multilevel Header Numerical Tables in Scientific Papers}
\eauthor{Lya Hulliyyatus Suadaa\affiref{Author_1} \and   Hidetaka Kamigaito\affiref{Author_1} \and Manabu Okumura\affiref{Author_1}  \and Hiroya Takamura\affiref{Author_1}\affiref{Author_2}} 

\eabstract{
Numerical tables are widely used to present experimental results in scientific papers. For table understanding, a metric-type is essential to discriminate numbers in the tables. Herein, we introduce a new information extraction task, i.e., metric-type identification from multilevel header numerical tables, and provide a dataset extracted from scientific papers comprising header tables, captions, and metric-types. We propose joint-learning neural classification and generation schemes featuring pointer-generator-based and pretrained-based models. Our results show that the joint models can manage both in-header and out-of-header metric-type identification problems. Furthermore, transfer learning using fine-tuned pretrained-based models successfully improves the performance. The domain-specific of BERT-based model, SciBERT, achieves the best performance. Results achieved by a fine-tuned T5-based model are comparable to those obtained using our BERT-based model under a multitask setting.
}
\ekeywords{Metric-Type Identification, Table Understanding, Information Extraction}

\headauthor{Suadaa et al.}
\headtitle{Metric-Type Identification for Multilevel Header Numerical Tables}

\affilabel{Author_1}{}{Tokyo Institute of Technology}
\affilabel{Author_2}{}{National Institute of Advanced Industrial Science and Technology (AIST)}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Tables are an effective tool for presenting data efficiently in rows and columns. In scientific papers, numerical tables are typically used to present experimental results to facilitate data analysis. Examples of numerical tables presented in scientific papers are shown in Figure~\ref{example-table}.

Multiple categories can be represented in table headers by incorporating several header sets in a hierarchical view; this is known as multilevel header tables. The presentation of tables in scientific papers must adhere to strict guidelines. For example, similar types of texts are to be written at the same level of header. 
%%%%Figure \ref{example-table-a} shows a multilevel header example in the column section, 
    Figure \ref{example-table}a shows a multilevel header example in the column section, 
with the task (\textit{Task 1} and \textit{Task 2}) presented in the first header-level and the metric (\textit{Prec} and \textit{Rec}) in the second. The table contains a row header specifying the model (\textit{Model A}, \textit{Model B}, \textit{Model C}, and \textit{Model D}). The header information is typically limited owing to the unknown table schema. However, we assume that tables presented in scientific papers adhere to the rule whereby similar types of header names are categorized at the same header level.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[t]
\begin{center}
\includegraphics{28-4ia12f1.pdf}
\end{center}
%%%%\label{example-table-a}
%%%%\label{example-table-b}
\caption{Examples of tables presented in scientific papers. Bold indicates metric-type.}
\label{example-table}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

To understand numbers in the tables, metric-types are important for discriminating the numbers. A comparison between numbers is applied for numbers in the same metric-type with different categories. 
%%%%For the table shown in Figure \ref{example-table-a}, 
    For the table shown in Figure \ref{example-table}a, 
we cannot compare the number $60$ for \textit{Model A} in the first column with $60$ in the second column because they represent different metric-types: \textit{Prec} and \textit{Rec}. Computing numbers of different metric-types will result in inaccurate analyses.  

Header names may be written differently in different tables, e.g., using abbreviations such as \textit{p}, \textit{pre}, or \textit{prec} to refer to \textit{precision}. 
Due to the lexical diversity of header names, metric-type identification becomes more challenging. Using rule-based metric-type tagging or a limited set of metric-types in a dictionary is insufficient to encompass the diversity of metric-types. As tables presented in scientific papers are typically provided with logical captions and a logical categorization of the header level, we introduce a metric-type identification task that locates the metric-type in the headers using the caption and header names as inputs. 
%%%%For the example shown in Figure \ref{example-table-a}, 
    For the example shown in Figure \ref{example-table}a, 
the metric-type is indicated at the second level of the column header.

Furthermore, we consider tables that do not include metric-types in their header (out-of-headers), 
%%%%as shown in Figure \ref{example-table-b}. 
    as shown in Figure \ref{example-table}b. 
In these cases, the metric-types are provided in the caption. To consider tables with metric-types located and not located in the headers, we propose a joint framework of metric-type location prediction and metric-type token generation for the metric-type identification task in multilevel header tables. We adopted a pointer-generator \cite{pointer-generator} to generate metric-type tokens, combined with a softmax layer to predict the metric-type location. 

We collected tables from scientific papers and hired workers to identify the metric-types. Because our annotated dataset is limited, we proposed transfer learning by fine-tuning available pretrained models trained on a large corpus in our task. We used BERT, a pretrained model with bi-directional transformer encoders, to take advantage of its ability in producing more contexts from two directions. To perform prediction and generation tasks for solving our metric-type identification problem, we fine-tuned a pretrained encoder-decoder T5, which successfully solved multitask NLP problems with its unified framework.


Our contributions are as follows:
\begin{itemize}
\item We introduce a metric-type identification task for multilevel header tables and propose joint location prediction and generation models to solve the task.
\item We provide a dataset comprising multilevel header numerical tables, captions, and metric-types extracted from scientific papers. Our dataset is publicly available.\footnote{Dataset is available on \url{https://github.com/titech-nlp/metrictable}}
\item We introduce a multilevel header table encoder mechanism to obtain table header representations and propose a pointer-generator-based model to cover out-of-headers in the metric-type identification task.
\item We fine-tune a general pretrained encoder (BERT) and a domain-specific encoder \linebreak
(SciBERT) in our task and present the experimental results. We show that models incorporating the pretrained encoders yield significant performance gains, particularly domain-specific encoders.
\item We fine-tune a pretrained encoder-decoder, T5, for our task in a multitask setting and present the experimental results.
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Studies}

Table information extraction is beneficial for covering unknown table schemes and understanding the table contents. 
\shortciteA{milosevic2019} proposed a framework for table information extraction in biomedical domains by defining rules for all possible variables. Specifically, for numerical variables, they retrieved metric-types by searching a set of possible tokens in their dictionary. Focusing on numerical tables, \shortciteA{Spread} extracted metric-types in earning reports by using similarity scores between stored metric-types and the corresponding non-numeric text for the leftmost cells. They investigated only header texts to identify metric-type from a limited set of tokens in a vocabulary. Dealing with out-of-vocabulary issues, we elaborate the caption information as additional inputs for our proposed framework.

The study that is the most similar to ours is that of  \shortciteA{hou-etal-2019-identification}, who used tables from the experimental result section, combined with the title and abstract as document representations to extract triples comprising tasks, datasets, and metrics for leaderboard construction. In our study, we represent tables more generically to preserve the original table structure in a multilevel header form that encompasses different structures in a real table. Therefore, our framework can be easily implemented in all types of tables. \shortciteA{milosevic2006} investigated multilevel tables to automatically detect table structures from XML tables.

Our pointer-generator-based model in the metric-type generation scheme is inspired by the promising results of the pointer-generator network \cite{pointer-generator} for the summarization task. The network deals with the out-of-vocabulary issue by jointly copying from source texts and generating from vocabularies.

Recent studies demonstrate that pretrained models can be successfully fine-tuned for downstream NLP tasks, thereby obviating the necessity to train a new model from scratch. Due to the effectiveness of transfer learning for a limited dataset, we propose fine-tuning pretrained models and utilizing universal language representations trained on a large corpus. A pretrained model that appropriately facilitates our framework for understanding the context of our table representation is BERT. We used the original BERT model \cite{devlin-etal-2019-bert} trained on the BooksCorpus (800M words) and Wikipedia (2,500M words). 


To improve contextualized representations in the scientific domain, \shortciteA{beltagy-etal-2019-scibert} introduced a domain-specific BERT model, i.e., SciBERT, which was trained on 1.14M papers from Semantic Scholar. \shortciteA{friedrich-etal-2020-sofc} used SciBERT on their models to solve the information extraction task in the same domain and achieved significant performances. Similarly, we fine-tuned SciBERT in our proposed BERT-based model as we used scientific papers as our dataset sources.


To encompass the prediction and generation tasks of our proposed framework, we fine-tuned a pretrained encoder-decoder, T5, which can be easily adapted to multitask settings. \shortciteA{Raffelt5} introduced T5 as a unified framework that converts NLP problems into text-to-text tasks using the same loss function and decoding procedure. They demonstrated that their approach can be successfully applied to various tasks such as summarization, question-answering, and natural language inference.  


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Metric-Type Identification for Numerical Tables}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dataset}
\label{mt_datasets}

We automatically extracted tables from PDF files of scientific papers in the computational linguistics domain using PDFMiner\footnote{\url{https://github.com/euske/pdfminer}} and Tabula\footnote{\url{https://github.com/tabulapdf/tabula}} as extraction tools and used only numerical tables associated with experimental results using the keywords \textit{evaluation}, \textit{result}, \textit{comparison}, and \textit{performance}. We used papers from the ACL and EMNLP conferences (2016 to 2019) on the ACL Anthology\footnote{\url{https://www.aclweb.org/anthology/}} website as data sources.

In tables presented in scientific papers, information regarding table semantics is rarely provided. Based on the manner by which information is \enquote{read} from a table, \shortciteA{hurst_thesis} separated functional table areas into access cells and data cells. Access cells comprise column headers and/or row headers. We define the data structure based on their functional areas: table caption ($capt$), row headers ($rh$), column headers ($ch$), and cells. The headers in the row and column have several levels, and we assume that header names at the same level are of the same type. Figure \ref{table-structure} shows the structure of the table. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{28-4ia12f2.pdf}
\end{center}
\caption{Table structure.}
\label{table-structure}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We asked several qualified workers in the computer science field to manually verify the extracted table structure to ensure the separation of row headers, column headers, and cells, as shown in Figure \ref{sample-preprocessing}. Based on the table structure, a header-level was defined as the location order of a group of header names in the same column of row headers, or in the same row of column headers. Subsequently, the workers annotated  the metric-types, which are a unit of measurement for numbers in the table, by prioritizing the location of the metric-type in a specific header-level. 
Detailed annotation instructions are provided in Figure \ref{annotation-instruc} in the Appendix.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[b]
\begin{center}
\includegraphics{28-4ia12f3.pdf}
\end{center}
\caption{Illustration of table preprocessing.}
\label{sample-preprocessing}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The annotators successfully identified the metric-types of approximately 70\% of the tables in their headers, and they determined the metric-types of the remainder based on information provided in the table captions. When no metric-type was provided in the headers, we assumed the metric-type was the same for all values in the table. The structure from the example shown in Figure \ref{sample-preprocessing} is $capt$: \enquote{{\it  model comparison in task 1 and 2}}; $rh$ level 1: ({\it models, models, models, models}); $rh$ level 2: ({\it model a, model b, model c, model d}); $ch$ level 1: ({\it task 1, task 1, task 2, task 2}); $ch$ level 2: ({\it prec, rec, prec, rec}); and metric-type: ({\it prec, rec, prec, rec}) (identified in $ch$ level 2).

We double annotated 10\% of our corpus and obtained near-perfect inter-rater agreement (0.813) using Krippendorf's alpha \cite{krippendorff} in identifying whether metric-type was located in the row header, column header, or neither. A substantial agreement (0.762) was achieved in determining metric-type tokens using the caption information for not-in-header metric-type cases.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{12table01.tex}
\caption{Dataset statistics. }
\label{stat-table}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The statistics of our dataset are provided in Table ~\ref{stat-table}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Problem Definition}

Let $\text{Table} = (capt, rh^{i}_{k}, ch^{j}_{l}, cell_{ij}) \text{, where } 1 \leq i \leq n_{r}, 1 \leq j \leq n_{c}, 1 \leq k \leq u, 1 \leq l \leq v$, denote an $n_{r} \times n_{c}$ table with the $u$ levels of $rh$ and $v$ levels of $ch$. 
The task is to identify a tuple of metric-type tokens, denoted by $\hat{m}$. When the tuple is in the $k$-th level of the row header, we extract the $rh_{k}$ as $\hat{m}$. When the tuple is in the $l$-th level of the column header, we extract $ch_{l}$ as $\hat{m}$. If both the row and column headers do not contain a tuple of metric-type tokens, we generate $\hat{m}$ using information from the table caption ($capt$). The formulation for the metric-type identification is as follows:
\begin{equation}
\label{modelformulation}
  \hat{m} =
    \begin{cases}
      (rh^{1}_{k}, rh^{2}_{k}, ..., rh^{n_{r}}_{k}), k \in \{1,...,u\} & \text{if the tuple is in the row header} \\
      (ch^{1}_{l}, ch^{2}_{l}, ..., ch^{n_{c}}_{l}), l \in \{1,...,v\} & \text{if the tuple is in the column header} \\
      \begin{aligned}[b](\text{w}_{m}, \text{w}_{m}, ..., \text{w}_{m}), \text{w}_{m} \in W_{m} \text{ or } \text{w}_{m} \in capt 
      \end{aligned}& \text{otherwise,}
    \end{cases}       
\end{equation}

where $W_{m}$ is the set of metric-types in the vocabulary. 
If the table headers do not contain any metric-types, then the table is likely to exhibit the values of a single metric-type. In other words, when $\hat{m}$ is not located in header $rh$ or $ch$, we will find a metric-type token $\text{w}_m$ from $W_{m}$ or $capt$, and copy $\text{w}_m$ to all the columns\footnote{We chose columns instead of rows simply because majority metrics are mentioned in their column headers.} from 1 to $n_c$,
resulting in 
$w^{1}_{m}= w^{2}_{m} = ...= w^{n_{c}}_{m}=\text{w}_{m}$, where $w^{m}_{i}$ is a metric-type token in position $i$. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Models}

We propose neural models to identify the metric-type for multilevel header tables using a joint model that enables metric-type location prediction and metric-type token generation.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Pointer-Generator Supervised Attention Model}

We obtained the representations of captions and header-levels using a bidirectional long short-term memory (BiLSTM) encoder and then captured the header-level weights using supervised attention between the header-level encoder and the metric-type header-location outputs. In the generation scheme, we adopted the pointer-generator network to consider captions as source texts and metric-type vocabulary in the metric-type generation gate. The architecture of the proposed model is shown in Figure \ref{arch-pointergen}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[t]
\begin{center}
\includegraphics{28-4ia12f4.pdf}
\end{center}
\caption{Architecture of proposed pointer-generator-based model for identifying metric-types in tables.}
\label{arch-pointergen}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Header encoder}
We obtained the vector representation of each header-level by averaging the vectors of all header name tokens at the same level. 
\pagebreak
Let $E_{{rh}_{k}}$ denote the average vector of the initial vector representations of the row header tokens at level $k$. Similarly, let $E_{{ch}_{l}}$ denote the average vector of the initial vector representations of the column header tokens at level $l$. We used BiLSTM as our encoder, which learns bidirectional long-term dependencies between time steps of sequence data. We input 
a sequence of row header vectors $E_{{rh}_{1:u}}$ to the BiLSTM encoder to obtain the  representation of the $k$-th level $h_{{rh}_{k}}$, while considering both the entire history $E_{{rh}_{1:k}}$ and the entire future $E_{{rh}_{k:u}}$ as follows:
\begin{equation}
h_{{rh}_{k}} = BiLSTM(E_{{rh}_{1:u}},k).
\end{equation}
Similar to the row headers, for a sequence of column header vectors $E_{{ch}_{1:v}}$, we used the BiLSTM encoder to obtain the representation of the $l$-th level $h_{{ch}_{l}}$, while considering both the entire history $E_{{ch}_{1:l}}$ and entire future $E_{{ch}_{l:v}}$ as follows:
\begin{equation}
h_{{ch}_{l}} = BiLSTM(E_{{ch}_{1:v}},l).
\end{equation}

To obtain the contexts of a sequence of row header-level $C_{rh}$ and column header-level $C_{ch}$, we incorporated the dot attention mechanism proposed by \shortciteA{luong-etal-2015-effective}. We selected the hidden state of the last level and combined it with the weighted hidden states as follows:
\pagebreak
\begin{align}
C_{rh} &= [h_{{rh}_{u}};\sum _{k=1}^{u}a_{{rh}_{k}}h_{{rh}_{k}}],\\
C_{ch} &= [h_{{ch}_{v}};\sum _{l=1}^{v}a_{{ch}_{l}}h_{{ch}_{l}}],
\end{align}
where attention $a_{{rh}_{k}}$ and $a_{{ch}_{l}}$ are derived by comparing the final encoder outputs $h_{rh_{u}}$ and $h_{ch_{v}}$ with each source hidden state $E_{{rh}_{k}}$ and $E_{{ch}_l}$, respectively, as follows:
\begin{align}
a_{{rh}_{k}} & = \frac{exp(h^{\intercal}_{rh_{u}}E_{{rh}_k})}{\sum _{k'}exp(h^{\intercal}_{rh_{u}}E_{{rh}_{{k}^{'}}})},\\
a_{{ch}_{l}} &= \frac{exp(h^{\intercal}_{ch_{v}}E_{{ch}_l})}{\sum _{l'}exp(h^{\intercal}_{ch_{v}}E_{{ch}_{{l}^{'}}})}.
\end{align}
Note that $[x;y]$ denotes the concatenation of vectors x and y.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Caption encoder}
Let $E_{{capt}_{x}}$ denote the initial vector representation of the caption token at position $x$. Similar to the headers, we used the BiLSTM encoder with attention  $a_{capt_x}$ to compute the context vector of a caption sequence $C_{capt}$ with length $t$, while considering
both the entire history $E_{{capt}_{1:x}}$ and entire future $E_{{capt}_{x:t}}$ as follows:
\begin{align}
h_{{capt}_{x}} &= BiLSTM(E_{{capt}_{1:t}},x),\\
a_{{capt}_{x}} &= \frac{exp(h^{\intercal}_{capt_{t}}E_{{capt}_x})}{\sum _{{x'}}exp(h^{\intercal}_{capt_{t}}E_{{capt}_{{x}^{'}}})},\\
C_{capt} &= [h_{{capt}_{t}};\sum _{x=1}^{t}a_{{capt}_{x}}h_{{capt}_{x}}].
\end{align}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Metric-type header-location gates}
We input the concatenation of the row and column header contexts to the softmax layer with linear transformation to obtain the metric-type header-location probability, as follows:
\begin{equation}
p_{hloc} = \text{softmax}(W([C_{rh};C_{ch}] + b),
\end{equation}
which includes the probabilities of the metric-types located in the row headers ($p_{rh}$) and column headers ($p_{ch}$), or not located in the headers ($p_{capt}$), where $p_{rh}+p_{ch}+p_{capt}=1$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Metric-type header-level gates} Since the attention scores $a_{{rh}_{k}}$ and $a_{{ch}_{l}}$ capture the relevant header-level information in rows and columns, these attention scores are used as header-level weights as follows:
\begin{equation}
\text{w}_{{hlvl}_{i}}=
\begin{cases}
a_{{rh}_{i}}p_{rh} & \text{if }i \le u\\
a_{{ch}_{i-u}}p_{ch} & \text{if }i>u\\
\end{cases},
\end{equation}
where $i\in\{1,...,u,(u+1),...,(u+v)\}$ is a header-level index.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Metric-type generation gates}
In our pointer-generator network, we used the sigmoid layer with linear transformation to obtain a switch copy probability as follows:
\begin{equation}
p_{copy} = \text{sigmoid}(W(C_{capt}) + b),
\end{equation}
which allows us to select between copying a word $w_{capt}$ from a table caption and generating a word $w_{m}$ from the metric-type vocabulary, where $p_{copy} \in [0,1]$.
We used a softmax function with linear transformation to compute the probability distribution over the metric-type vocabulary:
\begin{equation}
P_{vocab}(w_{m}) = \text{softmax}(W(C_{capt}) + b).
\end{equation}

Subsequently, we obtained the probability distribution over the extended vocabulary, as follows:
\begin{equation}
P(w_{m}) = p_{copy}\sum _{i:w_{i}=(w_{m})}^{n}a_{{capt}_{i}}  + (1-p_{copy})P_{vocab}(w_{m}),
\end{equation}
where $i$ is the index of metric-type tokens in the vocabulary.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Learning objective}
For training, we used the negative log-likelihood objective as the loss function. In addition, we adopted supervised attention \cite{liu-etal-2016-neural} to jointly supervise the row and column header-level attention to obtain the metric-type header-level. We combined all loss functions in the location classification and token generation model, and defined $\alpha$ as the weight, as follows:
\begin{equation}
\mathcal{L} = -((1-\alpha)(\sum _{c} z_{hloc_{c}} \log{p}_{hloc_{c}} + \sum_{i=1}^{u+v}\log{\text{w}}_{{hlvl}_{i}}) + \alpha(\log{p}_{copy} + \log P_{vocab}(w_{m}))),
\end{equation}
where $c \in  \{capt, rh, ch\}$ is the metric-type header-location class and $z_{hloc_{c}}$ is the binary indicator (0 or 1) of each corresponding class.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Fine-tuning BERT-based Model}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Input representation}
The input text in a fine-tuned BERT-based model is preprocessed by inserting two special tokens, i.e., [CLS] and [SEP]. In the original BERT architecture, [CLS] is appended to the beginning of the input as the representation of the entire input sequence, whereas [SEP] is inserted after each input type as a sign for a segment boundary. For example, in question-answering tasks with two types of input text, i.e., pairs of questions and answers, a
[CLS] token is appended before the question tokens,
and [SEP] tokens are placed after question and answer tokens to separate the question and answer segments. Based on the study by  \shortciteA{liu-lapata-2019-text}, 
we customized these preprocessing schemes by inserting [CLS] before each segment and inserting [SEP] after each segment. We partitioned our inputs into several segments: caption, row header level $1$ to $u$, and column header level $1$ to $v$. 

After preprocessing, the input text is denoted as a sequence of tokens $X = (x_{1}, x_{2}, \text{· · ·} , x_{n})$. 
Three types of embedding are assigned to each $x_{i}$: token embeddings representing the meaning of each token, segmentation embeddings indicating the segment boundaries of a sequence of tokens, and position embeddings indicating the token position within the sequences. Because BERT only includes two segments in its input, we named the odd segment as segment A and the even segment as segment B. The sum of these three embeddings was input to the bidirectional transformer layer of the BERT.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[b]
\begin{center}
\includegraphics{28-4ia12f5.pdf}
\end{center}
\caption{Architecture of proposed BERT-based model for identifying metric-types in tables.}
\label{arch-bert}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We used the token representations from the top hidden layers of the pretrained transformer as context embeddings. We assumed that the context vectors of each CLS token can  represent the segment sequences more effectively. As shown in Figure \ref{arch-bert}, we labeled the input embedding as $E$, the final hidden vector of the [CLS] token for the 
 $i^{th}$ input segment as $C_{i} \in \R_{H}$, and the final hidden vector for the $j^{th}$ input token as $T_{j} \in \R_{H}$.



We used a metric-type header-location gate and a metric-type header-level gate for metric-type location classification, and a metric-type generation gate to generate metric-type tokens from vocabulary encompassing out-of-header metric-types. The proposed BERT-based model architecture is shown in Figure \ref{arch-bert}. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Metric-type header-location gates}
We input the first segment context $C_{1}$ to the softmax layer via linear transformation to obtain the metric-type header-location probability, as follows:
\begin{equation}
p_{hloc} = \text{softmax}(W(C_{1}) + b).
\end{equation}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Metric-type header-level gates}
In our task, segments were used to represent the table section that is related the most closely with the metric-type. We incorporated the segment context $C_{i}$ into the sigmoid layer via linear transformation to obtain the probability that the metric-type is located at a specific header-level, as follows:
\begin{equation}
p_{hlvl_{i}} = \text{sigmoid}(W(C_{i}) + b).
\end{equation}

Subsequently, the probabilities were normalized to all segments as the weight score of the header level, as follows:
\begin{equation}
\text{w}_{hlvl_{i}} = \frac{p_{hlvl_{i}}}{\sum^{n}_{i=1} p_{hlvl_{i}}}.
\end{equation}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Metric-type generation gates} 
We used a softmax function with linear transformation based on the first segment context $C_{1}$ to compute the probability distribution over the metric vocabulary, as follows:
\begin{equation}
P_{vocab}(w_{m}) = \text{softmax}(W(C_{1}) + b).
\end{equation}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Learning objective}
We combined all loss functions in the metric-type header-location, metric-type header-level, and metric-type generation gates as follows:
\begin{equation}
\mathcal{L} = -((1-\alpha)(\sum _{c} z_{hloc_{c}} \log{p}_{hloc_{c}} + \sum_{i=1}^{n}\log{\text{w}}_{{hlvl}_{i}}) + \alpha\log P_{vocab}(w_{m})),
\end{equation}
where $\alpha$ is the weight of the metric-type generation functions.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Fine-tuning T5-based Model}

The input text in the fine-tuned T5-based model was preprocessed by inserting several specific tokens to discriminate tokens in different locations. Tokens [CAPT], [ROW], and [COL] were placed before the caption, row-name, and column-name tokens, respectively. [ROW$i$] and [COL$j$] were inserted before the row-name tokens at the $i$ level and column-name tokens at the $j$ level, respectively; each name was separated by the [SEP] token. We fine-tuned the T5 model to perform multiple tasks, metric-type header-location classification, and metric-type token generation. For each task, we appended a prefix string to the input of the model, i.e.,  \enquote{identify location:} for the first task and \enquote{identify metric:} for the second task. The architecture of the proposed T5-based model is shown in Figure \ref{arch-t5}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[t]
\begin{center}
\includegraphics{28-4ia12f6.pdf}
\end{center}
\caption{Architecture of proposed T5-based model for identifying metric-types in tables.}
\label{arch-t5}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experimental Settings}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Baseline Model}

Because our task pertain primarily to location classification and token prediction from metric-type vocabularies, we selected SVM as a baseline owing to its effectiveness in high-dimensional spaces, particularly in text classification. We used two SVM classification models  as baselines: a metric-type location prediction model and a metric-type token prediction model from the vocabulary of metric-types. We used \enquote{tf.idf} from the concatenation header name tokens for all levels as an input representation in the first model and \enquote{tf.idf} of the caption tokens in the second one. 
%%%%We performed a grid search to tune the hyperparameters of the SVM model on the development set for search space $c ∈ \{ 0.1, 1, 10, 100, 1000\}$ and $gamma ∈ \{0.001, 0.0001\}$; 
    We performed a grid search to tune the hyperparameters of the SVM model on the development set for search space $c \in  \{ 0.1, 1, 10, 100, 1000\}$ and $gamma \in \{0.001, 0.0001\}$; 
subsequently, we selected the  $c$ and $gamma$ parameters of the SVM that indicated the best accuracy.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Evaluation Metrics}

The \enquote{accuracy} metric was used to evaluate the metric-type location and generated metric-type tokens.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Metric-type location accuracy}
The target of the metric-type location prediction model is the metric-type located in the row headers, in the column headers, or neither. The accuracy of the header-location ($acc_{hloc}$) is the rate of correct header-location predictions.

Details regarding the metric-type location at the header-level are required to identify metric-type token lists. We computed the accuracy of the metric-type header-level ($acc_{hlvl}$) using the ratio of correct header-level predictions to the total number of predictions.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Metric-type token accuracy} 
Let $\hat{m} = (\hat{w}_{m_{1}}, ..., \hat{w}_{m_{n}})$ denote the sequence of predicted metric-type tokens for $n_r$ rows or $n_c$ columns (depending on the header-location prediction), and $m = (w_{m_{1}}, ..., w_{m_{n}})$ denote the target ones, e.g., $\hat{m} = (\text{f1, f1, f1})$ and $m = (\text{f-1, f-1, f-1})$. We calculate the metric-type token accuracy using the string matching of all token lists in $\hat{m}$ and $m$, i.e.,
\begin{equation}
acc_{m}^{sm} = \frac{\text{\# correct }\hat{m}}{\text{\# }\hat{m}},
\end{equation}
and string matching of each token pair $\hat{w}_{m_{i}}$ and $w_{m_{i}}$ in the token lists, i.e.,
\begin{equation}
acc_{m\_token}^{sm} = \frac{\text{\# correct }\hat{w}_{m}}{\text{\# }\hat{w}_{m}}.
\end{equation}

To account for token predictions involving an abbreviation, we compute the metric-type token accuracy based on ordered character matching, as follows:
\begin{equation}
acc_{m\_token}^{ocm} = \frac{d}{\text{\# }\hat{w}_{m}},
\end{equation}
where $d$ is the number of $\hat{w}_{m}$, 
whose characters appear in $w_{m}$ in the same order. For example, the predicted token {\it RG1} is regarded as correct when the reference token is {\it ROUGE-1}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Implementation Details}

We implemented our models using the AllenNLP library \cite{gardner-etal-2018-allennlp}. In our pointer-generator-based model, we used pretrained word embeddings for initialization and two-layer BiLSTMs with 256 hidden sizes in both the caption and header-level encoders. We added a dropout \cite{srivastava-hovy-2014-vector} with a probability of $p = 0.1$ to our header and caption encoder.
We evaluated our models using $k$-fold cross-validation. Because we had imbalanced classes of metric-type location with row-header metric-type instances below 2\%, we used $k=5$ to increase the probability of minority class instances being evaluated.

To perform optimization in the training phase, we used Adam as the optimizer with a batch size of $10$ and learning rates of $3 \times 10^{-3}$ and $3 \times 10^{-5}$ in the pointer-generator-based and BERT-based models, respectively, with a slanted triangular schedule \cite{howard-ruder-2018-universal}. 
We trained the model for a maximum of $20$ epochs and implemented early stopping on the validation set (patience of $10$), and we set $\alpha$ to $0.5$. We used the original BERT and the domain-specific SciBERT uncased model to fine-tune our BERT-based model. For our T5-based model, we fine-tuned the model using Adafactor optimizer with a constant learning rate of 0.001 \cite{Raffelt5}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Results}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Experimental Results}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Model comparison} 
The performances of the proposed and baseline models are shown in Table ~\ref{exp-results}. As shown, the pointer-generator supervised attention model initialized by Glove embeddings outperformed the baseline (SVM) in predicting the metric-type location. The accuracy of this model for metric-type generation was better than that of the baseline.   
A slight difference in the pointer-generator supervised-attention performance of the proposed model as compared with the SVM implies that the deep neural network architecture afforded minimal improvement in our limited dataset. Adding more datasets is suggested when training a deep learning model from scratch. Furthermore, we demonstrated that using only pretrained embeddings of a large pretrained model decreased the accuracy. The performance of our pointer-generator-based model deteriorated significantly when the input was initialized using BERT and SciBERT.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[b]
\input{12table02.tex}
\hangcaption{Accuracy scores (\%) of different metric-type identification models using five-fold cross-validation. Scores differed significantly those of fine-tuned SciBERT with paired-bootstrap-resampling test ($p < 0.05$). \textbf{Bold} indicates the best score. }
\label{exp-results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We fine-tuned the large pretrained BERT, SciBERT, and T5 to exploit deep neural models trained on a larger corpus. For our fine-tuned BERT-based models, we used the context representation of their transformer encoders and added gates to solve our tasks. 
The accuracy of our fine-tuned BERT-based models was significantly better than that of pointer-generator-based models trained on our limited corpus, where header-location and header-level prediction accuracies exceeding 88\% and generation accuracy improvement exceeding 3 points (\%) were achieved. The fine-tuned BERT-based model using the domain-specific SciBERT led to significant improvements in all metrics because its corpus is similar to ours.


The performance of our fine-tuned T5 with an encoder-decoder architecture is comparable to that of the fine-tuned encoder-only BERT model. The unified framework of T5 was easily adapted to our tasks by inserting different prefixes in the input. Unlike fine-tuned BERT-based models, no added layer was used in the fine-tuning of T5. \\ 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Effect of copy mechanism}
We evaluated our pointer-generator-based model using an ablation test, as presented in Table~\ref{ablation}. As shown, the performance of our generation model without a copy mechanism decreased. This shows that incorporating the copy mechanism is beneficial to metric-type token generation. Our model demonstrated the worst accuracy when it was executed without a pointer-generator network because the location prediction model alone failed to manage out-of-header metric-types.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{12table03.tex}
\hangcaption{Accuracy scores (\%) of ablation test of our pointer-generator-based model obtained using five-fold cross-validation. Scores of \enquote{without copy} and \enquote{without copy and generation} models differed significantly from proposed pointer-generator model with paired-bootstrap-resampling test ($p < 0.05$).}
\label{ablation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Effect of segment embeddings}
Table \ref{ablation2} shows the effect of segment embeddings in our BERT-based model. The accuracies of the fine-tuned BERT and the SciBERT models without segment embeddings both decreased. This implies that segment embeddings successfully discriminated the header-level boundaries in the input representation of the BERT-based models.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{12table04.tex}
\hangcaption{Accuracy scores (\%) of ablated BERT-based models without segment embeddings obtained using five-fold cross-validation. Scores of models with segment embeddings differed significantly from models without segment embeddings  with paired-bootstrap-resampling test ($p < 0.05$).}
\label{ablation2}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Qualitative Analysis}

As introduced in the model formulation (Eq.~\ref{modelformulation}), our models include row-header, column-header, and not-in-header metrics based on metric-type location. We evaluated the performances of our models for each location category based on the precision ($P$), recall ($R$), and F1-score ($F$) of the metric-type location prediction task as shown in Table \ref{crossval_results}.

As shown in Table \ref{crossval_results}, F-SciBERT demonstrated the best performance for all metric-type location cases. Due to the small number of instances in the dataset, the overall performance of the row-header metric-type category was worse than that of the others. Applying a specific procedure to handle the imbalanced class problems is left for future work.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[t]
\input{12table05.tex}
\hangcaption{Performance scores (\%) of metric-type location prediction for each header-location class using five-fold cross-validation. \textbf{Bold} indicates the best score.}
\label{crossval_results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The pointer-generator-based and fine-tuned BERT-based model outperformed the baseline in predicting row-header and column-header metric-type cases. Specifically, a significant margin was observed in predicting the row-header metric-type as the minority class. This indicates that combining the metric-type header-location and the header-level gate improved the model's ability to determine the metric-type location in all cases. The text-to-text mechanism of the fine-tuned T5 failed to perform the metric-type location prediction of the minority cases.


For the not-in-header metric-type cases, the models successfully generated metric-type tokens from the vocabulary. Because we added the copying ability to our pointer-generated-based models, we present both the generating and copying performances in  Table \ref{crossval_results}. As shown, our proposed model performed better than the generating model in terms of copying.



Additionally, we investigated the errors in the predicted metric-type tokens. We discovered that the models generated more generic  metric-types; for example, they extracted \textit{score} as a prediction of the target \textit{accuracy}. By contrast, our models generated terms similar to the ground truth metric, such as the metric-type \textit{pearson's} for the target \textit{r}. Examples of table captions, headers, and metric-type for each metric-type location are shown in Table \ref{example-outputs}.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

In this study, 
\pagebreak
we extracted multilevel header numerical table datasets comprising header tables, captions, and metric-types from scientific papers. We introduced a metric-type identification task for multilevel header numerical tables, and proposed joint location prediction and generation models to solve the task. We demonstrated that our proposed model can identify metric-types from the multilevel header tables, both when the metric-types were/were not included in the headers. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[t]
\input{12table06.tex}
\caption{Example of table caption, headers, and predicted metric-type.}
\label{example-outputs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Our datasets were based on scientific papers in the computational linguistic domain. The generalization of our results beyond the domain is challenging because of difficulties in acquiring comparable datasets in other domains without requiring additional annotations by human experts.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

This paper is a revised version of a previous paper by  \shortciteA{suadaa-etal-2021-metric}, 
which was published in the proceedings of the 16th European Chapter of the Association for Computational Linguistics (EACL-2021). This study was partially supported by JST PRESTO (Grant Number JPMJPR1655). We thank the anonymous reviewers for their discussions pertaining to this study and their comments on previous drafts of the paper.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\bibliography{12refs}


\newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix

\section{Annotation Guideline} 
Figure \ref{annotation-instruc} shows annotation instructions for metric-type identification task.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 7
\begin{figure}[h]
\begin{center}
\includegraphics{28-4ia12f7.pdf}
\end{center}
\caption{Metric-type annotation instructions.}
\label{annotation-instruc}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Lya Hulliyyatus Suadaa}{is a Ph.D. student at the Department of Information and Communications Engineering, School of Engineering, the Tokyo Institute of Technology. She  is supported by the Indonesian Endowment Fund for Education (LPDP). Her current research interests include natural language generation and information extraction.
}

\bioauthor[:]{Hidetaka Kamigaito}{received his Ph.D. from the Tokyo Institute of Technology, and is currently an assistant professor at the Institute of Innovative Research, the Tokyo Institute of Technology. His current research interests include natural language processing with emphasis on document-level text processing.
}

\bioauthor[:]{Manabu Okumura}{received his Ph.D. from the Tokyo Institute of Technology in 1989. He was an assistant at the Department of Computer Science, the Tokyo Institute of Technology, from 1989 to 1992; and an associate professor at the School of Information Science, the Japan Advanced Institute of Science and Technology, from 1992 to 2000. He is currently a professor at the Institute of Innovative Research, the Tokyo Institute of Technology. His current research interests include natural language processing, particularly text summarization, computer-assisted language learning, sentiment analysis, and text data mining.
}

\bioauthor[:]{Hiroya Takamura}{received his Ph.D. from the Nara Institute of Science and
Technology. He worked as a professor at the Tokyo Institute of Technology and is currently a research team leader at the AI Research Center of Advanced Industrial Science and Technology. His current research interests include natural language processing. 
}
\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate






\clearpage





















\clearpage


















\end{document}

