\documentclass[english]{jnlp_3.7}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{array}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage{times}
%%%%\usepackage{latexsym}
%%%%\usepackage[T1]{fontenc}
%%%%\usepackage[utf8]{inputenc}
%%%%\usepackage{url}
%%%%\usepackage{latexsym}
%%%%\usepackage{amsmath}
%%%%\usepackage{amssymb}
\usepackage{multirow}
%%%%\usepackage{siunitx}
\usepackage{booktabs}
%%%%\usepackage{paralist}
%%%%\usepackage{adforn}
%%%%\usepackage{graphics}
%%%%\usepackage{float}
%%%%\usepackage[export]{adjustbox} 
%%%%\usepackage{enumitem}
%%%%\usepackage{color}
%%%%% my commands
%%%%\newcommand{\draft}[1]{\textcolor{gray}{#1}}
%%%%\newcommand{\verify}[1]{\textcolor{red}{#1}}
%%%%\makeatletter
%%%%\def\blfootnote{\xdef\@thefnmark{}\@footnotetext}
%%%%\makeatother
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{31}
\Number{1}
\Month{March}
\Year{2024}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2023}{7}{27}
\revised{2023}{11}{1}
\accepted{2023}{12}{4}

\setcounter{page}{189}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{A Table Question Alignment based Cell-Selection Method for Table-Text QA}
\eauthor{Jian Wu\affiref{lrlab} \and Yicheng Xu\affiref{NTU} \and B\"{o}rje F. Karlsson\affiref{BAAI} \and Manabu Okumura\affiref{lrlab}} 
\eabstract{%
Hybrid Question-Answering (HQA), which targets reasoning over tables and passages linked from table cells, has witnessed significant research in recent years. A common challenge in HQA and other passage-table QA datasets is that it is generally unrealistic to iterate over all table rows, columns, and linked passages to retrieve evidence. Such a challenge made it difficult for previous studies to show their reasoning ability in retrieving answers. To bridge this gap, we propose a novel \textbf{T}able-\textbf{a}lignment-based \textbf{C}ell-selection and \textbf{R}easoning model (TACR) for hybrid text and table QA, evaluated on the HybridQA and WikiTableQuestions datasets. In evidence retrieval, we design a table-question-alignment enhanced cell-selection method to retrieve fine-grained evidence. In answer reasoning, we incorporate a QA module that treats the row containing selected cells as context. Experimental results over the HybridQA and WikiTableQuestions (WTQ) datasets show that TACR achieves state-of-the-art results on cell selection and outperforms fine-grained evidence retrieval baselines on HybridQA, while achieving competitive performance on WTQ. We also conducted a detailed analysis to demonstrate that being able to align questions to tables in the cell-selection stage can result in important gains from experiments of over 90\% table row and column selection accuracy, meanwhile also improving output explainability.
}
\ekeywords{Table Cell Selection, Table QA, Question Answering}

\headauthor{Wu et al.}
\headtitle{A Table Question Alignment based Cell-Selection Method for Table-Text QA}

\affilabel{lrlab}{}{Tokyo Institute of Technology}
\affilabel{NTU}{}{Nanyang Technological University}
\affilabel{BAAI}{}{Beijing Academy of Artificial Intelligence}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Text-based question-answering datasets derive answers based on reasoning over given passages \cite{rajpurkar2016squad,chen-etal-2017-reading,Joshi2017TriviaQAAL,yang-etal-2018-hotpotqa}, while table-based QA datasets collect tables from sources such as WikiTables \cite{pasupat-liang-2015-compositional,zhong2017seq2sql,chen2019tabfact}.\footnote{This paper is an extended version of our preliminary paper \cite{wu2023tacr}, presented in Findings of the 61st Annual Meeting of the Association for Computational Linguistics (ACL'23).} 
However, datasets combining textual passages and tables, like HybridQA \cite{chen2020hybridqa}, OTT-QA \cite{Chen2020OpenQA}, and TAT-QA \cite{zhu2021tat} are more realistic benchmarks. As the answer to a given question may come from either table cells or linked passages, current hybrid QA models usually consist of two components, a retriever to learn evidence and a reasoner to leverage the evidence to derive answers. Such models retrieve evidence from different granularities, coarse-grained (e.g., row or column) or fine-grained (e.g., cell), and directly use a span-based reading comprehension model to reason the answer.  

\citeA{Kumar2021MultiInstanceTF}, for example, chooses coarse-grained regions as evidence, e.g., a table row. \citeA{chen2020hybridqa} and \citeA{Eisenschlos2021MATEMA}, however, focus on fine-grained units, table cells, and linked passages. To preserve the advantages and eliminate the disadvantages of different-granularity evidence, \citeA{Sun2021EndtoEndMR} propose MuGER$^2$, which performs multi-granularity evidence retrieval and answer reasoning. 

\citeA{Wang2022MuGER2ME} conducts extensive experiments to prove that a coarse-grained retriever contributes less than a fine-grained retriever. Moreover, fine-grained methods, although giving an exact position of candidate cells, fail to illustrate why the selected cells are chosen, while our method is based on row and column selection probabilities. We thus further extend the fine-grained method by aligning questions with tables, letting our approach know which parts of questions are accounted for by which modalities. Intuitively, multi-hop questions in the text-table QA task usually contain two pieces of information from different modalities, tables and passages. Moreover, tables and passages are connected with evidence contained in tabular data. Our method implicitly decomposes the questions for different modalities to locate evidence and improve cell-selection accuracy. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\vspace{-0.5\Cvs}
\begin{center}
\includegraphics{31-1ia7f1.pdf}
\end{center}
\hangcaption{Example from the HybridQA dataset. The top sentence is the original question, and words in different colors show different parts of questions required for reasoning in different modalities. The two headers in blue-dashed boxes are column names aligned with the given question. TACR first uses a method based on table-question-alignment to align the original question with table columns to help obtain golden table cells and then retrieve the final answer based on linked passages.}
\label{Figure Example}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

As illustrated in Figure \ref{Figure Example}, 
\pagebreak
an example from the HybridQA dataset shows how humans work on multi-hop and multi-modal QA tasks. The original question \emph{``What is the middle name of the player with the second most National Football League career rushing yards ?''} can be divided into two parts, \emph{``What is the middle name of''} and \emph{``the player with the second most National Football League career rushing yards?''} for passages and tables, respectively. Such sub-questions are connected with the evidence of a cell (\emph{``Walter Payton''}). For humans, we first locate who was the player in the second rank, which requires information from two columns: \emph{``Rank''} and \emph{``Player''}. After locating the cell, we can finally determine Walter Payton's middle name from the passage. 
Such reasoning process inspired us to develop TACR, a \textbf{T}able-\textbf{a}lignment-based \textbf{C}ell-selection and \textbf{R}easoning model, which incorporates a fine-grained evidence-retrieval module that utilizes table-question-alignment to learn which parts of the question are used for retrieving evidence from different modalities and reasoning towards answers. 

To explicitly and correctly show the reasoning process in the text-table QA task, in the evidence retrieval stage, TACR first selects the golden cells and avoids redundant information in multi-granularity evidence that would lower the performance of the answer-reasoning module. The table-cell-selection module of TACR is designed to navigate the fine-grained evidence for the reader by fusing well-learned information from the table-question-alignment module. Compared with current fine-grained retrievers, the table-question-alignment module of TACR can help our model learn which parts of questions are used for reasoning in which modality, and which parts of tables contain candidate cells. Together with the alignment module, TACR preserves both high golden cell-selection accuracy and shows competitive QA performance on the HybridQA and WikiTableQuestions (WTQ) datasets, while providing improved explainability. 

Our contributions are as follows: (1) TACR is the first model able to apply the table header alignment method in the passage-table QA task; (2) We jointly train the cell-selection and table-question alignment modules to improve golden cell selection performance and preserve the QA reader's performance; and (3) We conduct extensive experiments on the HybridQA and WTQ datasets to demonstrate the effectiveness of TACR. 

The remainder of this paper is organized as follows. 
In Section~\ref{related}, we overview the related work. 
In Section~\ref{framework}, we introduce details of the proposed framework. 
We show detailed experimental results in Section~\ref{experiment}. 
Finally, we conclude the paper in Section~\ref{conclusion}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}
\label{related}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Table Question Answering}

Table QA has gained much attention, as shown by benchmark datasets such as WikiTableQuestions \cite{Pasupat2015CompositionalSP}, WikiSQL \cite{zhong2017seq2sql}, SPIDER \cite{Yu2018SpiderAL}, and TABFACT \cite{chen2019tabfact}. However, these datasets mainly focus on reasoning on tables and ignore important knowledge stored in the textual corpus. Consequently, QA covering both tabular and textual knowledge has gained increasing interest. \citeA{chen2020hybridqa} pioneered a passage-table QA benchmark, HybridQA, with Wikipedia tables linked to relevant free-form text passages (e.g., Wikipedia entity-definition pages). The OTT-QA \cite{Chen2020OpenQA} benchmark extended HybridQA to the open domain setting, where a system needs to retrieve a relevant set of tables and passages first before trying to answer questions. Moreover, the links from the table and passage are not provided explicitly. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Table-Question Alignment}

There are several table-question-alignment methods. Schema-linking-based methods, such as RAT-SQL \cite{Wang2019RATSQLRS}, introduce a relation-aware transformer encoder to improve the joint encoding of a question and schema. \citeA{Liu2022SemanticET} propose a similarity learning-based question-schema-alignment method to obtain a semantic schema-linking graph and observed how the pre-trained language model (PLM) embeddings for the schema items are affected. \citeA{Zhao2022TablebasedFV} use the same words that appear in both the natural language statement and the table as weak supervised key points and design an interaction network to explore the correlation between the representations of the natural language statements and tables. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Hybrid QA}

Studies on hybrid QA usually retrieve different granularities of evidence from heterogeneous data to retrieve the final answer. Hybrider, proposed by \citeA{chen2020hybridqa}, is a two-phase pipeline framework to retrieve gold table cells as evidence and input their values and linked passages into a QA model to extract the final answer. \citeA{Sun2021IterativeHA} proposes Dochopper, an end-to-end multi-hop retrieval model that directly concatenates rows with related textual evidence as its inputs. \citeA{Pan2020UnsupervisedMQ} explores an unsupervised multi-hop QA model, called MQA-QG, which can generate human-like multi-hop questions by building a reasoning graph from heterogeneous data resources. \citeA{Kumar2021MultiInstanceTF} propose MITQA, which applies multiple-instance training objectives to retrieve coarse-grained evidence. On the contrary, \citeA{Eisenschlos2021MATEMA} introduces a transformer-based model with row- and column-wise attention for fine-grained evidence retrieval, e.g., table cells. \citeA{Wang2022MuGER2ME} propose a unified retriever that tries to preserve the advantages and eliminates the disadvantages of different-granularity evidence retrieval methods. 

TACR differs from the above models mainly in two aspects: (1) TACR focuses on providing an explicit reasoning process by aligning multi-hop questions to tables, so it learns which parts of multi-hop questions are accounted for by retrieving evidence from which modality; and (2) The table-question alignment can enhance the reasoning ability of the table cell selection module with the help of our generated hybrid alignment dataset. TACR shows competitive performance to that of other table QA models on the HybridQA and WTQ datasets on the basis of high row, column, and cell selection accuracy. To the best of our knowledge, no text-table QA system handles the challenge of explicitly showing its reasoning process and multi-hop question table alignment.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Table Cell Retrieval}

\citeA{jauhar-etal-2016-tables} construct a multiple-choice table QA benchmark that includes over 9,000 question-table pairs via crowd-sourcing and proposed a table-cell search model based on calculating all relevance scores between each cell and question. Such a model is reasonable and intuitive but time-consuming. TACR selects gold cells based on row and column selection. Suppose that a table contains $n$ rows and $m$ columns; the table cell search method must calculate $n*m$ scores for each cell, while TACR needs to calculate only $n+m$ scores for each row and column, and selects the gold cell in the row and column with the highest score. \citeA{Sun2016TableCS} focus on extracting entities from questions and building a row graph and then mapping the question to the pair of cells in the same row of a table. However, some entities may not appear in both questions and table cells, e.g., an entity of the question in Figure \ref{Figure Example} that should be extracted is \emph{National Football League}, but it cannot be mapped into any cells.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Methodology}
\label{framework}

As described in the previous section, both coarse- and fine-grained approaches fail to provide a reasoning process showing which parts of multi-hop questions map to which modality and evidence. Here we describe the details of TACR and its three main components: (1) data augmentation for training the table-question alignment module; (2) a multi-task learning module for table-question alignment and table-cell-selection; and (3) a text-based multi-hop QA module for retrieving answers. Figure \ref{Figure pipeline} shows the overall architecture of TACR.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[t]
\begin{center}
\includegraphics{31-1ia7f2.pdf}
\end{center}
\hangcaption{TACR model architecture. From left to right, we first construct a hybrid alignment dataset to jointly train the table-question-alignment and table-cell-selection modules. We then concatenate filtered linked passages with selected top-k candidate cells as paragraphs and feed them into a text-based multi-hop QA module to retrieve answers.}
\label{Figure pipeline}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Task Definition}

Given a question $Q$ (a sequence of tokens) and $N$ rows of table $T$ together with linked passages $P$, where each table column has a header $h_{i=1}^{i=M}$ ($M$ is the number of table headers), %. Given the input, 
the task is to find a candidate cell $c_{i,j}$ that contains the answer $\alpha$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Data Construction}

We generate multi-hop questions from tables and linked passages. At the same time, we also generate table-question alignment labels from questions and table columns for training the table-question-alignment module. However, such supervision information is not offered in the HybridQA dataset and other text-table QA datasets, which makes the alignment task difficult. We use an unsupervised text-table QA-generation method to generate questions as well as alignment labels.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Alignment Generation.} We follow the settings of the MQA-QG method \cite{Pan2020UnsupervisedMQ}, using a pre-trained Google T5 \cite{Raffel2019ExploringTL}, fine-tuned on the SQuAD dataset \cite{Rajpurkar2018KnowWY}, to generate multi-hop questions from tables and passages based on a bridge entity, a table cell that contains the bridge entity, and a linked passage that describes the bridge entity. The bridge entity is critical in reasoning because it connects the tables and passages, which are difficult to locate in the original HybridQA dataset.

Such bridge entity provides us with additional information to align table headers with generated questions based on the column containing golden cells and the column containing the bridge entity. We align the columns that contain bridge entities and answers to questions following two schema-linking alignment rules:

\textbf{Name-based Linking.} This rule refers to exact or partial occurrences of the column/table names in the question, such as the occurrences of ``player'' in the question in Figure \ref{Figure Example}. Textual matches are the most explicit evidence of table-question alignment and, as such, one might expect them to be directly beneficial to the table-question alignment module. 

\textbf{Value-based Linking.} Table-question alignment also occurs when the question mentions any values that occur in the table and consequently participate in the table-cell selection, such as ``the second most'' in Figure \ref{Figure Example}. While it is common for examples to make the alignment explicit by mentioning the column name (e.g., ``Rank''), many real-world questions do not (like in the example). Consequently, linking a value mentioned in the question to the corresponding column also requires background knowledge.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Passage Filtering}

In this stage, we aim to filter out linked passages unrelated to a question, namely keeping almost noise-free passages for the following modules. Moreover, the total number of tokens in passages linked to table cells can be large, exceeding the maximum input sequence length of current LMs. Thus, we utilize Sentence-BERT \cite{Reimers2019SentenceBERTSE} to obtain question and passage embeddings and rank the top-k sentences based on their text similarities. We expand the cells with the filtered top k-related sentences to both fit in the max input length of language models and to preserve the useful information from passages. More details on this stage are provided in Section \ref{sec:passage filtration}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Table Alignment \& Cell Selection}

In this stage, we jointly train a multi-task model based on the ALBERT architecture with the objectives of selecting the expanded cell that contains the answer and table-question alignment to different modalities to enhance the previous objective. TACR accepts the full table as inputs and outputs the probabilities of selected cells based on the probabilities of row and column selection.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Table-Question Alignment}

Given a natural language question $ Q = \left \{ q_{1},....q_{\left | Q \right |} \right \} $, a table consisting of several column headers $C = \left \{c_{1}....c_{\left | C \right |}\right \} $, and the corresponding table-question alignment labels $L = \left \{ l_{1}, ... l_{\left | C \right |} \right \} $ where $l_{i} \in [0,1]$ (0 meaning the column header is unrelated to the question $Q$ and 1 meaning the column header is related to $Q$). The goal of our table-question alignment module is to learn the relevance between table-column headers and questions. Table-question relations aid TACR by aligning column references in the question to the corresponding table columns.

We first feed the questions and table columns into the pre-trained model and map them into hidden representations. The question and table-column headers can be denoted as $ \left \{ q_{1},....q_{\left | Q \right |} \right \}$ and $ \left \{c_{1}....c_{\left | C \right |}\right \}$, respectively. Our goal is to induce a function $f(q,c)$ to capture the relevance of a question $q$ has on the representation of column header $c$. The supervision information for training the table-question alignment module is the label we generated from the data construction. Figure \ref{Figure alignment} shows the structure of the alignment module.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia7f3.pdf}
\end{center}
\hangcaption{The table-question-alignment module of TACR. We treat the alignment objective as a similarity learning task.}
\label{Figure alignment}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Specifically, we use ALBERT \cite{Lan2019ALBERTAL} as the encoder to learn the representations of tables and column headers. Here we concatenate column headers as a pseudo sentence. The representations of the question ({$h_{q} \in \mathbf{R}^{Q \times d} $}) and the column headers sequence ($h_{c} \in \mathbf{R}^{C \times d}$) are first computed independently, $d$ is the dimension of the hidden representation. The relevance between header and question where each column header $c_{i}$ is the target of the question is then given by using softmax. The respective equations are as follows:
\begin{align}
h_{q} & = \texttt{BERT(Q)}, \\
h_{c} & = \texttt{BERT(C)}, \\
f(q,c) & = \texttt{softmax}(h_{c} \times (h_{q} W)^T + b ).
\end{align}
where $W \in \mathbf{R}^{d \times d}$ and $\times$ is cross product. The $f(q,c)$ is the probability of the relevance.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Table-Cell Selection}

Inspired by the previous idea of modeling the attention on rows and columns \cite{Eisenschlos2021MATEMA}, we design a cell-selection module based on row and column selection. The probabilities of each row and column are given and the cells with the top-k highest scores are returned as the candidate answers, or to aid in locating the relevant passage. However, unlike in MATE \cite{Eisenschlos2021MATEMA}, we can derive probabilities of candidate cells from the probabilities of row and column. 

We utilize the Row-Column-Intersection (RCI) model, designed for the single-hop table-QA task \cite{Glass2021CapturingRA} (based on ALBERT \cite{Lan2019ALBERTAL}), as our backbone and decompose the table QA task into two subtasks: projection - corresponding to identifying columns; and selection - identifying rows. Every row and column identification is a binary sequence pair classification. We concatenate the question as the first sequence and the row or column as the second sequence. We feed concatenated two sequences, with standard separator tokens $[CLS]$ and $[SEP]$, as the input to the model. The representation of the final hidden state is sent to the linear layer, followed by a softmax to classify whether the column or row contains the answer or not. Each row and column is assigned a probability of containing the answer. This module finally outputs the top-k cells with the sum of row and column probabilities. Therefore, given a table $T$ with $N$ rows and $M$ columns, we can obtain two sets of scores produced from the RCI model: $P_{r} = p_{1},....p_{N}$ for rows and $P_{c} = p_{1},....p_{M}$ for columns. We then calculate the overall probability score for each cell.

The final training loss is the summation of table-question-alignment loss, table-row-selection loss, and table-column-selection loss:
\begin{equation}
    \begin{split}
         L &= \texttt{L\_{row}} + \texttt{L\_{column}} \\
         &+ \sigma \times \texttt{BCE}(pred\_{headers}, target\_{headers}),
    \end{split}
\end{equation}
where $\sigma$ is a hyper-parameter to balance cell-selection training and table-question-alignment training. The details of choosing the best $\sigma$ are provided in Section \ref{sec:implementation details}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Passage Question-Answering}

Previous research often simply treats the answer-reasoning task as a span-extraction task, considers the first span matching the answer text as the gold span, and uses that for training. Such consideration is incorrect because the answer text may appear in multiple passages, but only one of them is right. Therefore, using all text matches for training span extraction may introduce a large amount of training noise. As not all instances are the gold answer text that has relations with questions, after obtaining the top-k cells from the cell-selection module, we train the text-based QA module to predict the final answer that also takes into account the cell-selection scores. 
In inference stage, for answer appears in multiple cells, the QA model extracts the k candidate answers from different k candidate cells and output the answer with the highest score as the final answer. For answer appear in the same cell multiple times, the QA model extracts all answers and also output one answer with the highest scores.

Specifically, we select clean training instances where the gold answer text appears only once and train an initial QA model. In this stage, we use RoBERTa \cite{Liu2019RoBERTaAR} as our backbone model. Other BERT variants, e.g., either SpanBERT \cite{Joshi2019SpanBERTIP} or DeBERTa \cite{He2020DeBERTaDB}, could be also used in this module. Our goal is to obtain a span $s$ in a given expanded table cell $c$ with its filtered passage $p$ and the input question $q$. We compute a span representation as follows:
       \begin{align}
        & h_{start} = \texttt{RoBERTa}_r(q, c)[\texttt{START}(s)], \\
        & h_{end}  = \texttt{RoBERTa}_r(q, c)[\texttt{END}(s)], \\
        & S_{span}(q, c)  = \texttt{MLP}([h\_{start}, h\_{end}]).
        \end{align} 
where the $h_{start}$ and $h_{end}$ denote the corresponding position of the answer in the expanded cell $c$. The $S_{span}(q, c)$ denotes the logits of the start and end positions of the answer span.
We also consider other cells in the same row as the retrieved candidate gold cells as the necessary context. We linearize and concatenate the row into a passage with the designed template: ``The {\textless}column header{\textgreater} is {\textless}cell content{\textgreater}''. We retrieve the top-k cells and thus have k samples. Since not all selected cells contain the gold answer text, we treat one sample as positive and the others as negative samples. For each data point, we generate $k$ samples %as described in the previous paragraph, 
and match these with the answer text. Let $K = \{ q_{i}, A_{i}, P^{+}_{i}, P^{-}_{i,1}, \dots, P^{-}_{i,k-1} \}^{k}_{i=1}$ be the training data that consist of $k$ instances, where $k$ is the number of selected candidate cells. Each instance contains one question $q_{i}$, the gold answer text $A_{i}$, and one correct (positive) passage text $P^{+}_{i}$, along with $k-1$ wrong passages $P^{-}_{i,j}$. For positive samples, the answer is the text span of the passage, while for negative samples, the answers are $-1$. 

For the instances with noise information like gold text appears many times, we here just apply the well-trained QA module and output the answer spans with the highest scores. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiment}
\label{experiment}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Datasets}

For training a table-question alignment module, we utilize the MQA-QG to unsupervisedly generate supervision information on the HybridQA dataset that adds the labels on table headers and questions.

For train and evaluate the performance of cell-selection and QA, we utilized HybridQA and WikiTableQuestions.
\textbf{HybridQA} \cite{chen2020hybridqa} is the first large-scale multi-hop QA dataset that requires reasoning over hybrid knowledge, including tables and linked Wikipedia passages. The dataset contains 62,682 instances in the training set, 3,466 instances in the development set, and 3,463 instances in the test set. 

\textbf{WikiTableQuestions} \cite{pasupat-liang-2015-compositional}, WTQ for short, consists of 22,033 complex questions and 2,108 semi-structured Wikipedia tables. The questions are designed by crowd-sourcing to contain a wide range of domains. The answers are derived from several operations such as table lookup, aggregation, superlatives, arithmetic operations, joins, and unions. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{07table01.tex}%
\caption{Statistics of HybridQA dataset}
\label{Statistics}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

To verify the performance of TACR, we first conduct experiments on HybridQA \cite{chen2020hybridqa}, a dataset of multi-hop question-answering over tabular and textual data. The basic statistics of HybridQA are listed in Table \ref{Statistics}. The dataset contains three partitions: `In-Table', where the answer derives from table cell values; `In-Passage', where the answer exists in a linked passage; and `Compute', where the answer should be computed by executing numerical operations. We mainly focus on the first two types. We also provide results over WTQ to illustrate TACR's capabilities in table-focused QA.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Baselines}

\textbf{MQA-QG}, proposed by \citeA{Pan2020UnsupervisedMQ}, is an unsupervised question-generation framework that generates multi-hop questions from tables and linked passages, and uses the generated questions to train an HQA model. \\
\textbf{Table-Only} \citeA{chen2020hybridqa} only retrieves the tabular information to find an answer by parsing the question into a symbolic form and executing it. \\
\textbf{Passage-Only} \citeA{chen2020hybridqa} only retrieves answers from the table-linked passages. \\
\textbf{Hybrider} \citeA{chen2020hybridqa} addresses HQA using a two-stage pipeline framework to retrieve the gold table cell and extract an answer in its value or linked passages.\\
\textbf{Dochopper} \citeA{Sun2021IterativeHA} first converts a table with its hyperlinked passages into a long document then concatenates column headers, cell text, and linked passages in each row of tables as a paragraph. \\
\textbf{MATE} \citeA{Eisenschlos2021MATEMA} applies sparse attention to rows and columns in a table. To apply it to the HybridQA dataset, the authors propose a PointR module, which expands a cell using the description of its entities, selects the golden cells, then retrieves answers from them. \\
\textbf{MITQA} \citeA{Kumar2021MultiInstanceTF} designs a multi-instance training method based on distant supervision to filter the noisy information from multiple answer spans. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Quantitative Analysis}

We use exact match (EM) and F1 scores as evaluation metrics on the HybridQA dataset to compare the performance of TACR with that of previous baselines. As shown in Table \ref{tab:results}, TACR outperforms most baselines and achieved competitive performance to state-of-the-art (SOTA) models (e.g., MITQA) in both EM and F1 scores over the HybridQA dataset. Table \ref{tab:wtq_results} reports the accuracy performance on WTQ. Though TACR is trained on a base model, it presents comparable accuracy to the large SOTA models and outperforms other base models. It is important to note that, besides both using much larger LMs than TACR (GPT-3 and BART-large respectively, versus RoBERTa-base), neither Binder nor Omnitab-large provide explainability. With the help of the table-question-alignment module, TACR boosts relative accuracy by $+18.5\%$ on the test set compared with RCI \cite{Glass2021CapturingRA}, which is also based on cell selection. This competitive performance is mainly based on the high cell selection along with table-question alignment. We further verified the effectiveness of the table-question-alignment module in an ablation study discussed in Section \ref{sec_ablation}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{07table02.tex}%
\hangcaption{EM and F1 results of models on the HybridQA dataset. In-Table and In-Passage subsets refer to the location of answers.}
\label{tab:results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{07table03.tex}%
\caption{Execution-accuracy results of models on WTQ}
\label{tab:wtq_results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Qualitative Analysis}

We compare the cell-selection accuracy of TACR and baseline models, as shown in Table \ref{cell retrieval}. The high cell selection accuracy is based on the high row- and column-selection accuracies shown in Table \ref{row column and cell select accuracy}. On the HybirdQA dataset, TACR shows SOTA performance and 0.4\% higher than that of MATE \cite{Eisenschlos2021MATEMA} in the top 3 cell-selection accuracies due to its 89.3\% row-selection accuracy and 98.3\% column-selection accuracy, as shown in Table \ref{row column and cell select accuracy}. Moreover, by achieving soft question decomposition (i.e., showing which parts of questions are connected to reasoning in the different modalities), TACR both improves the explainability of its results and provides valuable signals for future improvements. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{07table04.tex}%
\caption{ Comparison of cell-retrieval results on HybridQA dataset (dev set)}
\label{cell retrieval}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8 --> table 5
\begin{table}[t]
\input{07table05.tex}%
\hangcaption{Performance of TACR with different backbone models. Top-k rows and columns selection accuracies on HybridQA and WTQ datasets, where $\text{k}=1, 3, 5$. Results demonstrate the effectiveness of TACR.}
\label{row column and cell select accuracy}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Ablation Study} 
\label{sec_ablation}

To evaluate the impact of the table-question-alignment module, we conduct an ablation study, shown in Table \ref{ablation}. We test DeBERTa-base, ALBERT-base, and RoBERTa-base models as TACR backbones for generality. Different top-k results show that the alignment module consistently significantly improves results; with the best model based on ALBERT improving cell-selection accuracy by 2.5, 3.9, and 4.3\% in top 1, 3, and 5 cell selection respectively; and mean reciprocal rank (MRR) improving by 3.7\%. The results indicate that the table-question-alignment module has an important role in the table-question-reasoning stage to select the most related cells that support the answer to the question.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[t]
\input{07table06.tex}%
\hangcaption{Ablation study of table-question-alignment module impact. Experiment results of cell-retrieval on HybridDQA (dev set) show the effectiveness of this module in the table-cell-selection stage.}\label{ablation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

To evaluate the impact of the passage filtration module, we conduct an ablation study in Table \ref{ablation_passage_filteration}. We test the passage filtration module in different top-k sentences on the dev set of the HybridQA dataset. Top 1, 3 and 5 means we only select 1, 3, and 5 sentences from the passage and fill in the related cells. The experiment results show that we could get the best performance when $\text{k}=3$. Based on our analysis, when selecting only one sentence, it may lose many other useful information. When $\text{k}=5$, there are too much information that should be discarded from passage and damage to the cell selection.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[t]
\input{07table07.tex}%
\hangcaption{Ablation study of passage filtration module impact. Experiment results of cell selection performance on HybridDQA (dev set) show the impact of selecting top 1,3 and 5 sentences in the table-cell-selection stage.}
\label{ablation_passage_filteration}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{LLM Evaluation} 
\label{sec_LLM}

To evaluate the LLM performance of HybridQA, we introduce ChatGPT (3.5), shown in Table \ref{llm} for Table cell selection. 
Different top 1,2,3 results show that the shot number affects the table selection performance. We also linearize the table into a sequence and finetune llama to do the table cell selection. The ChatGPT with 3-shots improves the accuracy by 6.1, 5.2, and 5.0\% to 1 shot.
The results indicate that the shot numbers could help improve performance and due to the limitation of the input length of ChatGPT, we can not explore more with the numbers of shots.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5 --> table 8
\vspace{-1\Cvs}
\begin{table}[b]
\input{07table08.tex}%
\hangcaption{The table cell selection performance of LLMs on dev set of HybridQA. We here select 200 samples from the training set for ChatGPT with different shots. }
\label{llm}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia7f4.pdf}
\end{center}
\caption{Heatmap of question and table-header relevance - Case 1}
\label{Figure case}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Case Study}

To illustrate TACR can successfully learn which parts of tables contain golden cells and which parts of questions are required for reasoning in the different modalities, we choose two examples from the HybridQA development set. Section 4.10 includes Figures \ref{Figure case} and \ref{Figure case2} showing their word relevances heatmap and analysis.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[t]
\begin{center}
\includegraphics{31-1ia7f5.pdf}
\end{center}
\caption{Heatmap of question and table header relevance - Case 2}
\label{Figure case2}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The question in Case 1 is \textit{``Who is the athlete in a city located on the Mississippi River ?''}. The concatenated table headers string for the corresponding table is \textit{``Year Score Athlete Place''}. The table-question-alignment module helps TACR learn that header terms \textit{``Athlete''} and \textit{``Place''} have higher relevance to the question than the headers of other columns, thus guiding cell-selection. Figure \ref{Figure case} shows its relevance heatmap. TACR again learns which parts of the question account for retrieving evidence in tables. 

The question in Case 2 is \textit{``What is the middle name of the player with the second most National Football League career rushing yards ?''}. The concatenated table headers string for it is \textit{``Rank Player Team(s) by season Carries Yards Average''}. The table-question-alignment module helps TACR learn that the sub-question \textit{``the player with the second most National Football League career rushing yards''} has a higher relevance to the table headers than that of other parts of the original question, thus guiding modality relevance. Figure \ref{Figure case2} shows its relevance heatmap.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Error Analysis}

To further analyze TACR, we also calculate statistics for error cases in the model predictions. The error statistics are based on the development set of HybridQA. Through the cell-selection accuracy statistics in Table \ref{cell retrieval}, we find there are 347 tables whose cells are incorrectly selected. 

To better understand the advantages and disadvantages of table-question alignment-based cell selection, we manually sample and examined 20 such error cases (i.e., where TACR does not provide the correct answer in the correct row, column, and cell position). Out of the 20 samples, we find that five error cases (25\%) are due to requiring numerical reasoning operations that cross several cells (which is out of scope for TACR). The majority of errors, 13 of the remaining incorrect cases, are in the same column with a correct answer while in the wrong row. Only one case is from a different row but the same column with the correct answer and only one incorrect case is in a completely different row and column to the correct answer.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Passage Filtering}
\label{sec:passage filtration}

Passage filtering plays an important role in cell selection as well as answer extraction. Pre-trained language models such as BERT, RoBERTa, and LLMs have the limitation of max input sequence length. Passage filtering ensures that it is unlikely to lose information relevant to the questions, while fitting model input limits. We used the well-trained DistilBert-based model to obtain question and passage embeddings to rank and filter relevant passages.\footnote{\url{https://huggingface.co/sebastian-hofstaetter/distilbert-dot-tas_b-b256-msmarco}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Alignment Analysis}
\label{sec:alignment analysis}

Here we provide example heatmaps showing the relevance of questions and table headers. The relevance is in the [0,1] range, where the higher relevance between words from questions and column headers is shown in the warmer colors and vice versa. Figure \ref{Figure case} shows that the column headers ``athlete'' and ``place'' have more relevance to the question, which helps TACR identify which columns contain potential gold cells. In Figure \ref{Figure case2}, the words ``player with second most national football league'' from the question have more relevance to columns, which help TACR learn which parts of the question better use to retrieve gold cells.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Implementation Details}\label{sec:implementation details}

TACR is implemented using Pytorch version 1.13 and the Huggingface transformers \cite{wolf-etal-2020-transformers} library. We trained TACR using two NVIDIA A6000 GPUs. The cell selection and table--question-alignment modules are trained for four epochs and we selected the best model based on the dev fold performance. AdamW is used as an optimizer algorithm with a learning rate of $5\times 10^{-5}$ and a batch size of 32. We set the per-GPU train batch size to 16 while training the span-based QA model. Final answers are evaluated using EM and F1 scores. We also automatically iterated through increments of 0.1 in the range [0, 1] to select the best $\sigma$ to balance the multi-task training. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Hyper-parameter Details:}
We tune hyper-parameters based on the loss on the development set and use the following range of values for selecting the best hyper-parameters: \\
\textbullet{} Batch size: [8, 16, 32, 64] \\
\textbullet{} Learning rate: [1e-3, 1e-4, 1e-5, 1e-6, 3e-3, 3e-4, 3e-5, 3e-6, 5e-3, 5e-4, 5e-5, 5e-6] \\
\textbullet{} $\sigma$ : [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]\\
\textbullet{} $top  K$ sentence: here we select $\text{k}=1,3,5$ for passage filteration.\\
\textbullet{} $top  K$ cells: here we select $\text{k}=1,3,5$ for cell selection.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
\label{conclusion}

This paper presents TACR, a \textbf{T}able question \textbf{A}lignment-based \textbf{c}ell selection and \textbf{R}easoning model for hybrid text and table QA, evaluated on the HybridQA and WikiTableQuestions datasets. When answering questions given retrieved table cells and passages, TACR attempts to align multi-hop questions to different modalities for correct evidence retrieval. To enhance the QA module with better table cell-selection and table-question-alignment ability, we construct a hybrid alignment dataset generated from the HybridQA dataset. TACR shows state-of-the-art performance in retrieving intermediate gold table cells and competitive performance on the HybridQA and WikiTableQuestions datasets, while improving output explainability.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Acknowledgments}

This paper is an extended version of our preliminary paper \cite{wu2023tacr}, presented in Findings of the 61st Annual Meeting of the Association for Computational Linguistics (ACL'23).

We acknowledge the support of Yan Gao and Jian-Guang Lou from Microsoft Research Asia (MSRA) for the discussions and collaboration on the work.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\begingroup
%%%%\addtolength{\baselineskip}{-0.5pt}
\bibliography{07refs}
\endgroup


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Jian Wu}{%
He received a bachelor's degree in computer science from Southwest University in 2018. He received a master's degree from the School of Computer Science, University of Hong Kong.  
He is currently pursuing a Ph.D. degree in the School of Engineering, Tokyo Institute of Technology.
His research interests include Natural Language Processing, Question Answering, and Table QA.
}

\bioauthor[:]{Yicheng Xu}{
He received the bachelor degree in Electronic and Informaion Engineering from  East China Normal University, in 2020, and the master degree in Computer Control and Automation from Nanyang Technological University in 2021. He currently works as a Research Associate in the school of Electrical and Electronic Engineering at Nanyang Technological University. His research interests include machine learning, deep neural networks and natural language processing.
}


\bioauthor[:]{B\"{o}rje F. Karlsson}{
He is a researcher at the Beijing Academy of Artificial Intelligence (BAAI) Now.
He received his doctorate in Computer Science from PUC-Rio, Brazil in 2010 and his BSc from UFPE, also in Brazil.
His main interests include machine learning systems, multi-modality reasoning, intelligent agents, multilingual knowledge pipelines, and digital entertainment.
}

\bioauthor[:]{Manabu Okumura}{%
He received his Dr.~Eng.\ from the Tokyo Institute of Technology, and is currently a professor at the Institute of Innovative Research. His current research interests include Natural Language Processing, especially text summarizing, computer-assisted language learning, sentiment analysis, and text mining.
}
\end{biography}




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate



\end{document}

