\documentclass[english]{jnlp_3.3}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{array}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage{soul}
\usepackage{xcolor}
%%%%\usepackage{subcaption}
\usepackage{wtref}
\newref{eq,fig,tab,sec}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{28}
\Number{4}
\Month{December}
\Year{2021}
%%%%\Issuetitle{「　　　　」特集号}

\received{2021}{3}{1}
\revised{2021}{5}{31}
%%%%\rerevised{2006}{5}{6}
\accepted{2021}{7}{5}

\setcounter{page}{938}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Probing Simple Factoid Question Answering Based on Linguistic Knowledge}
\eauthor{Namgi Han\affiref{SOKENDAI}\affiref{NII}\affiref{AIST} \and Hiroshi Noji\affiref{AIST}\affiref{LEAP} \and Katsuhiko Hayashi\affiref{OSAKA} \and Hiroya Takamura\affiref{AIST}\affiref{TOKOU} \and Yusuke Miyao\affiref{TOKYO}} 
\eabstract{
Recent studies have indicated that existing systems for simple factoid question answering over a knowledge base are not robust for different datasets. We evaluated the ability of a pretrained language model, BERT, to perform this task on four datasets, Free917, FreebaseQA, SimpleQuestions, and WebQSP, and found that, like other existing systems, the existing BERT-based system also can not solve them robustly. To investigate the reason for this problem, we employ a statistical method, partial least squares path modeling (PLSPM), with 24 BERT models and two probing tasks, SentEval and GLUE. Our results reveal that the existing BERT-based system tends to depend on the surface and syntactic features of each dataset, and it disturbs the generality and robustness of the system performance. We also discuss the reason for this phenomenon by considering the features of each dataset and the method that was used to evaluate the simple factoid question answering task.
}
\ekeywords{Question Answering, Language Model, Evaluation, Interpretability}

\headauthor{Han et al.}
\headtitle{Probing Simple Factoid Question Answering Based on Linguistic Knowledge}

\affilabel{SOKENDAI}{}{The Graduate University for Advanced Studies}
\affilabel{NII}{}{National Institute of Informatics}
\affilabel{AIST}{}{Artificial Intelligence Research Center, AIST}
\affilabel{OSAKA}{}{Gunma University}
\affilabel{TOKOU}{}{Tokyo Institute of Technology}
\affilabel{TOKYO}{}{The University of Tokyo}
\affilabel{LEAP}{}{LeapMind Inc.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Question answering over knowledge base (QAKB) has been studied as an important task in natural language processing.
This task demands the prediction of entities and relations that can reach the correct answer with a knowledge base from a given question.
For example, given the question, \textit{``Which country is Albert Bolender from?''}, then the subject \textit{Albert\_Bolender} and the relation \textit{people.person.nationality} are required to reach the correct object \textit{United\_States} in Freebase.
Although various approaches have been suggested for this task, neural network-based classifier models \shortcite{yu17,petrochuk18,mohammed18,huang19} have reported state-of-the-art accuracies recently.
For example, simple factoid question answering, a subtask in the QAKB field, is considered to be a task that has already been solved \shortcite{petrochuk18}.
This task is a simplified version of QAKB because simple factoid questions require only one fact \textit{(subject, relation, object)} to be solved.
Many papers have reported successful accuracies with SimpleQuestions \shortcite{bordes15}, the largest benchmark dataset of simple factoid question answering.
However, the high accuracy of SimpleQuestions does not mean that simple factoid question answering has been conquered.

Simple factoid questions in other datasets applicable to the QAKB task, such as Free917 \shortcite{cai13}, WebQSP \shortcite{yih16}, and FreebaseQA \shortcite{jiang19}, are not solved, even by the systems that are successful for SimpleQuestions.
\shortciteA{coling2020} discussed this problem by involving four datasets and systems, respectively.
They revealed that existing systems such as BuboQA \shortcite{mohammed18}, HR-BiLSTM \shortcite{yu17}, KBQA-Adapter \shortcite{wu19}, and KEQA \shortcite{huang19} cannot reach the upper bound accuracies for simple factoid questions in WebQSP and FreebaseQA.
Moreover, they reported that existing systems showed a lack of transferability in the experiment across two datasets.
Although their results indicate that the systems proposed in previous studies are limited in terms of general simple question answering, the effectiveness of pretrained language models for this problem has not yet been examined.


In this paper, we examine the effectiveness, transferability, and robustness of BERT by assessing its performance with respect to simple factoid question answering.
Because many studies have reported successful results for various natural language processing tasks with BERT \shortcite{devlin19,wolf20transformers}, we expected that BERT would be able to solve simple factoid question answering regardless of the differences among datasets.
We employed an advanced version of BuboQA \shortcite{mohammed18}, which employs an LSTM and a CNN to encode a given question, by replacing all the encoders of BuboQA with BERT following \shortciteA{lukovnikov19} to test our hypothesis in the same experimental setting as \shortciteA{coling2020}.
In our experiments, even though our BERT-based system attained accuracies higher than those of BuboQA on the four datasets, we also found that our BERT-based system still limited in robustness and transferability, similar to the original BuboQA.

We conducted a statistical analysis to examine the inner working of BERT to determine why BERT fails in general simple factoid question answering.
Recently, several studies \shortcite{jawahar19bert,ravishankar19multilingual,kovaleva19revealing} attempted to explain the inner working of BERT using probing tasks such as SentEval \shortcite{conneau18senteval} and GLUE \shortcite{wang19glue}.
As a result of depending on one or a few observations on which to base their conclusions, their results may be conflicting, similar to those of previous studies that were conducted to prove word embeddings \shortcite{schnabel15,chiu16,wang19,lrec2020}.

Here, we propose a different approach in which this problem is defined as a statistical examination of causal relationships between the result of probing tasks and the result of simple factoid question answering.
\shortciteA{lrec2020} employed partial least squares path modeling (PLSPM) \shortcite{wold82} to investigate causal relationships between linguistic knowledge and NLP downstream tasks on word embeddings.
Compared with other statistical methods, the advantage of PLSPM is its robustness to a smaller sample size \shortcite{tenenhaus05pls}.
In addition, it is easy to apply this method to analyze experimental results because it requires fewer assumptions on the observed variable \shortcite{tenenhaus05}. 
Hence, we estimated PLSPM models using 24 BERT models \shortcite{turc19well} to explain the results of our BERT-based system based on the result of the probing tasks.
As a result, we found that the accuracy of our BERT-based system can be causally explained by the accuracy of probing tasks on the surface and the syntactic information in our PLSPM models.
This indicates that existing simple factoid question answering systems may to a large extent depend on the surface and syntactic information of the target dataset.

Our study makes the following contributions.
\begin{itemize}
    \item Our findings showed that the system that employs pretrained language models, such as BERT, still experiences the same problems as other existing systems with respect to transferability and robustness.
    \item We employ PLSPM, a statistical method to probe and examine the inner working of BERT and linguistic knowledge.
    \item The PLSPM analysis with SentEval and GLUE revealed that the accuracies of probing tasks for semantic understanding are not causally related to the accuracies of our BERT-based system, whereas the accuracies of surface and syntactic tasks can explain the accuracies of our BERT-based system with significant path coefficients.
    \item Our findings suggest that the method to evaluate simple factoid question answering and the source of each dataset play an important role in this phenomenon, according to additional error analyses. 
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Background}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Question answering over knowledge base}
\seclabel{section:2-1}

Question answering over a knowledge base is one task of semantic parsing.
This task aims to find the correct answer for a given question from a target knowledge base, such as Freebase. 
Free917 \shortcite{cai13} is one of the early datasets for this task over Freebase.
Free917 was constructed by two annotators who wrote 917 questions using Freebase Commons, a subset of Freebase.
However, this dataset contains only 917 questions and is too small to train a machine-learning model sufficiently. 
Furthermore, \shortciteA{berant13} mentioned that the questions in Free917 tend to contain the label of the gold relation directly.
Therefore, other datasets, such as WebQuestion \shortcite{berant13}, SimpleQuestions \shortcite{bordes15}, and FreebaseQA \shortcite{jiang19}, have been proposed on an ongoing basis.

First, WebQuestion was proposed.
This dataset contains 5,810 questions that were created using Google Suggest API to ensure the naturalness of the question.
Although WebQuestion succeeds in aggregating more naturally written questions than Free917, this dataset does not contain formal queries to be executed.
To overcome this problem, \shortciteA{yih16} suggested WebQSP, a subset of WebQuestion containing annotated SPARQL queries.
SimpleQuestions is the most popular dataset for this task because of its size of over 100,000 questions.
The questions in SimpleQuestions were generated by crowd workers referring to randomly sampled facts from Freebase.
The approach FreebaseQA uses to aggreate questions differs from that of the above-mentioned  datasets.
\shortciteA{jiang19} annotated 28,348 questions in TriviaQA \shortcite{joshi17} with Freebase knowledge because \shortciteA{jiang19} aimed to suggest a dataset that includes more difficult and naturally written questions such as those used in trivia quizzes.

Previously, researchers tried to solve this task by transforming questions into logical forms \shortcite{berant13,reddy16,trivedi17}.
Recently, many studies have reported state-of-the-art accuracies for the above datasets by using systems based on neural network \shortcite{yu17,petrochuk18,mohammed18,huang19}. 
For example, BuboQA \shortcite{mohammed18}, which is well-known benchmark system for this task, consists of the following four submodules:
\begin{itemize}
    \item entity detection, a sequence-tagging network to find the span of an entity in a given question.
    \item entity linking, a string-match module to link an entity in a knowledge base with the span of an entity.
    \item relation prediction, a sentence classifier network to predict a relation in a knowledge base from a given question.
    \item evidence integration, a scoring module to rerank predicted entities and relations.
\end{itemize}

\shortciteA{petrochuk18} argued that their neural network-based model nearly solved SimpleQuestions.
They examined the upper bound accuracy of SimpleQuestions and showed that their system, which employs a similar network to BuboQA, almost reached the upper bound accuracy of SimpleQuestions.
However, the success of SimpleQuestions does not mean that the QAKB task is generally successful.
\shortciteA{coling2020} examined whether four systems, for which state-of-the-art performance was reported for SimpleQuestions, could also solve simple factoid questions in other datasets: Free917, WebQSP, and FreebaseQA.
They revealed that the existing systems did not reach a level of accuracy similar to that of SimpleQuestions on other datasets.
Moreover, their analysis showed that existing systems were not robust in terms of their transferability.
Therefore, it is difficult to conclude that simple factoid question answering is genearlly solved.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{BERT and BERTology}
\seclabel{bertology}

Contextual embeddings, such as ELMo \shortcite{peters18} and BERT \shortcite{devlin19}, have become indispensable tools in natural language processing, in addition to non-contextual distributional word representations, such as word2vec \shortcite{mikolov13} and fastText \shortcite{bojanowski17}.
Despite many studies in which contextual embeddings were employed resulting in state-of-the-art performance for a variety of natural language processing tasks, their usefulness was not clearly explained.
Therefore, researchers have investigated the inner working of contextual embeddings to explain their effectiveness, especially with respect to BERT.

These studies, known as BERTology \shortcite{rogers20primer}, usually accepted the traditional hypothesis that encoded linguistic knowledge in a language model can explain the accuracies of downstream tasks in NLP \shortcite{chiu16}.
Thus, BERTology has included various analyses involving both the structural features of BERT and the linguistic knowledge encoded in BERT.
The linguistic knowledge encoded in BERT has been examined in various ways, such as attention analysis \shortcite{liu19linguistic}, edge probing \shortcite{tenney19you}, and in comparison with intrinsic evaluations such as SentEval and GLUE.
For example, \shortciteA{tenney19bert} showed that BERT encodes syntactic information, such as the part-of-speech, chucking span, and its syntactic and semantic role by edge probing analysis.

Although BERTology has succeeded in determining the type of and the way in which linguistic knowledge is encoded in BERT, they usually derived their conclusions from one or a few observations without any statistical analysis or verification.
Because many language models have been proposed, observation with one or a few samples is not sufficient to make a general conclusion for BERTology.
Previous studies conducted to probe word embeddings sometimes reported conflicting results \shortcite{schnabel15,chiu16,rogers18,wang19,lrec2020}.
Therefore, the general relationship between encoded linguistic knowledge and the performance of downstream applications is not yet clear.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Statistical analysis of word embeddings}
\seclabel{plspm}

Traditionally, encoded linguistic knowledge in the language model is believed to be helpful for solving the downstream tasks of natural language processing \shortcite{chiu16}.
Before BERTology, many researchers \shortcite{schnabel15,chiu16,rogers18,wang19} have attempted to prove this intuition with distributional word representations, such as word2vec \shortcite{mikolov13}.
\shortciteA{lrec2020} argued that previous studies have two limitations.
First, they only conducted a correlation analysis between the results of two tasks, for example, the accuracy of word similarity and the accuracy of POS tagging.
Because a downstream task in NLP usually requires multiple linguistic knowledge to be solved, correlation analysis between the probing task and the downstream task in NLP limited ability to understand their causal relationship.
Second, as we mentioned in Section~\secref{bertology}, researchers sometimes reported conflicting results for the same issue because their conclusions were usually based on few observations \shortcite{schnabel15,chiu16,rogers18,wang19,lrec2020}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{28-4ia1f1.pdf}
\end{center}
\caption{Example of a causal diagram}
\figlabel{figure:sample}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In the field of statistics, researchers employed structural equation modeling (SEM) \shortcite{joreskog70} to prove causal assumptions between variables.
In SEM, a causal diagram representing causal assumptions for the target variables is first suggested by a user.
Figure~\figref{figure:sample} shows an example of a causal diagram involving probing tasks for semantic knowledge and the QAKB task.
Figure~\figref{figure:sample} contains the causal assumptions listed below:
\begin{itemize}
    \item Encoded semantic information ($y1$) affects the accuracies of probing tasks ($x1_{1}, x1_{2}, ..., x1_{n}$).
    \item The performance on the QAKB task ($y2$) affects the accuracies of entity detection ($x2_{1}$) and relation prediction ($x2_{2}$).
    \item Encoded semantic information ($y1$) affects the performance on the QAKB task ($y2$).
\end{itemize}
Based on a given causal diagram and the observed variables, SEM estimates the regression formulas for each causal hypothesis.
We refer to an estimated regression as a \textit{structural equation}.
Compared to correlation analysis, the advantage of SEM is its ability to handle multiple variables at once.
Moreover, it can provide many reliable indexes for proving causal assumptions.
For example, the score of $y2$ can be predicted by the following structural equation:
\[
    y2=\beta_{1}y1+\zeta_1
\]
where $\beta_1$ is the estimated weight, and $\zeta_1$ is the estimated error term.
The use of analysis enables us to analyze the number of latent variable scores that can be predicted by the structural equation, which the researcher has referred to as the explainability of that latent variable.

\shortciteA{lrec2020} employed partial least squares path modeling (PLSPM) \shortcite{wold82}, one of the methods of SEM, to investigate the relationship between the encoded linguistic knowledge in a language model and the accuracies of NLP tasks.
They selected PLSPM because of its fewer requirements for observed variables and robustness with small samples.
They examined their causal diagrams, which assume the causal relationship between the accuracies of probing tasks and downstream tasks, by using training algorithms, a corpus, and hyperparameters as variables.
Although they proved some relationships between linguistic knowledge and the accuracy of downstream tasks in their causal diagrams, they did not consider contextual embeddings. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Simple Factoid Question Answering using BERT}
\seclabel{bertqa}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Experimental settings}

In this study, we employ the system suggested by \shortciteA{lukovnikov19}, an extended version of BuboQA using BERT.
BuboQA, the benchmark system for simple factoid question answering \shortcite{mohammed18}, consists of four submodules: entity detection, entity linking, relation prediction, and evidence integration, as mentioned in Section~\secref{section:2-1}.
Among them, entity detection and relation prediction use a machine-learning algorithm based on a neural network, whereas the other submodules employ string matching or rule-based weight calculation.
The original BuboQA employed LSTM and CNN-based encoders for entity detection and relation prediction; however, \shortciteA{lukovnikov19} reported that the performance was increased by replacing LSTM and CNN-based encoders with the BERT model.
Therefore, we employ the model proposed in this paper as a BERT-based simple factoid question answering system.

Note that we implemented it ourselves because the official GitHub repository of \shortciteA{lukovnikov19} is no longer available. 
Our implementation followed the instructions in the original paper as much as possible, except for the design of the network for entity detection and relation prediction. 
In their original paper, they combined two submodules into one classifier to improve the accuracy of the proposed system. %effectiveness.
However, our aim was to examine the performance of BERT compared with the original BuboQA.
Therefore, we did not change the original design of BuboQA and changed only its encoder.
Hereinafter, we refer to this system as BertQA to distinguish this system from the original BuboQA and that suggested by \shortciteA{lukovnikov19}.

The other experimental settings we used are the same as those of of \shortciteA{coling2020}.
They prepared four datasets, Free917, FreebaseQA, SimpleQuestions, and WebQSP to examine the robustness and transferability of existing systems.
Because Free917, FreebaseQA, and WebQSP were not designed for simple factoid question answering, they contain questions that require multiple facts to be answered.
To guarantee the same level of domain and difficulty as SimpleQuestions, \shortciteA{coling2020} removed all questions that require two or more facts as the answer or cannot be solved by the FB2M dataset \shortcite{bordes14}, which is the source dataset of SimpleQuestions and a subset of Freebase.
We employed their filtered datasets, hereinafter F917, FBQ, SQ, and WQ, for our experiments.
Table~\tabref{table:dataset} provides details of the data statistics for these datasets.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{01table01.tex}
\hangcaption{Data statistics for datasets. Note that we do not use F917 as a training dataset, because of its small size.}
\tablabel{table:dataset}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In this study, we evaluated BertQA across F917, FBQ, SQ, and WQ.
First, we conducted a standard evaluation on a single dataset to examine whether BertQA could reach the upper bound accuracies for each dataset, such as SimpleQuestions.
In other words, this evaluation aimed to examine whether BertQA could solve simple factoid questions from the four datasets.
Second, we evaluated BertQA with revised datasets that employ test splits from other datasets.
For example, if we train BertQA with the train split of FBQ, then we test BertQA with the test split of SQ rather than the test split of FBQ.
This evaluation aimed to examine whether BertQA can learn the ability to solve simple factoid questions that are not specific to a single dataset.
Therefore, the robustness and transferability of BertQA would be evaluated in our experiments.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Results}
\seclabel{bertqa-result}

Table~\tabref{table:single} lists the end-to-end top-1 accuracies of BertQA for F917, FBQ, SQ, and WQ.
Our findings indicated that BertQA achieves higher accuracies than BuboQA, except for only one case.
However, BertQA does not reach the upper bound accuracy\footnote{Note that we do not consider the upper bound accuracy of F917, which consists of only the test split in our experiment because of its small size.} of FBQ and WQ, similar to SQ.
\shortciteA{coling2020} reported the upper bound accuracies of FBQ (84\%) and WQ (87\%), which are similar to the upper bound accuracy of SimpleQuestions, 83\% \shortcite{petrochuk18}.
In Table~\tabref{table:single}, the accuracies of BertQA for FBQ and WQ were much lower than the upper bound accuracies of FBQ and WQ.
This indicates that BertQA has difficulty solving simple factoid questions in FBQ and WQ.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[b]
\input{01table02.tex}
\hangcaption{Results of BertQA and BuboQA for the datasets. The reported value for the upper bound accuracies of each dataset was the same as that employed in previous studies \protect\shortcite{petrochuk18,coling2020}.}
\tablabel{table:single}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table~\tabref{table:single} indicates that BertQA becomes weaker when the training data and the test data are different, even though the level of domain and difficulty of the datasets are the same.
This indicates that BertQA shows a lack of transferability.
To examine the reason for the lower transferability, we examined error cases when BertQA, which was trained with the train split of WQ, tried to solve the validation split of WQ.
We only considered the SQ and WQ cases because \shortciteA{coling2020} reported that FBQ contains unfaithful questions.
For example, we found that FBQ annotated a gold subject \textit{Africa} and a gold relation \textit{location.contains} for a question \textit{What is the highest volcano in Africa?}.
This question should be answered by multiple facts consisting of a fact for volcanos in Africa and the fact that it restricts the answer to the highest volcano.
However, FreebaseQA annotated this question as a simple factoid question.

Because it indicates that the problems BertQA experiences when solving FBQ may be caused by FBQ itself, we excluded FBQ from further analyses in this study.

Table~\tabref{table:labeling} indicates that approximately 70\% of questions failed with resprect to relation prediction.
This problem was also reported by \shortciteA{coling2020}, who argued that the errors in relation prediction were caused by the discrepancies in question styles across the datasets.
For example, the \textit{people.person.profession} relation is mainly related to the \textit{Who is ...?} question in WQ.
However, the same relation tends to be related to \textit{What is ... 's profession?} question in SQ.
According to \shortciteA{coling2020}, the four state-of-the-art systems for SimpleQuestions have the same problem with relation prediction, regardless of the difference in the structure and the approach followed by those systems.
Considering that our experiment with BertQA experiences the same problem as \shortciteA{coling2020}, a pretrained language model may not be the solution for general simple factoid question answering.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{01table03.tex}
\hangcaption{Labeling for questions on WQ validation split, which fails when the training data changes from WQ to SQ.}
\tablabel{table:labeling}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Statistical Analysis of Simple Factoid Question Answering with BERT}

In this section, we present a statistical analysis of the results of BertQA to understand why BertQA still has the same problem as other existing systems.
\shortciteA{coling2020} attempted to explain the results they obtained for simple factoid question answering systems, but their analyses were limited to ad-hoc error analysis.
This prompted us to employ PLSPM analysis, which is a priori way to investigate the inner working of existing systems.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Causal diagram and data preparation}

In this study, we aimed to investigate the inner working of BertQA when solving simple factoid question answering.
As explained in Section~\secref{plspm}, causal assumptions for the target variables should be expressed as a causal diagram as motivation to employ PLSPM.
We drew inspiration for our causal diagram from previous studies, which assumed that the accuracies of probing tasks can explain the accuracies of NLP tasks \shortcite{chiu16} on the analysis of word embedding models.
Following the traditional assumption, we suggest the use of the causal diagrams shown in Figure~\figref{figure:cd}, which consist of causal hypotheses among linguistic knowledge and simple factoid question answering.
Our causal diagrams assume that the same language model solves all probing tasks and simple factoid question answering. 
Because the probing task was designed to examine encoded linguistic knowledge in the language model \shortcite{conneau18senteval}, we only employed BertQA, which only consists of BERT and one dense decoder, in this study.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{28-4ia1f2.pdf}
\end{center}
%%%%\label{fig:no-1}
%%%%\label{fig:no-2}
\caption{Causal diagrams}
\figlabel{figure:cd}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Our causal diagrams can be divided into two parts: a part for probing tasks as the cause and the other part for submodules in the BertQA system as the effect.
Referring to previous studies \shortcite{jawahar19bert,kovaleva19revealing,hao19visualizing,schneider20language}, we employed SentEval \shortcite{conneau18senteval} and GLUE \shortcite{wang19glue} as probing tasks to examine the accuracy of the encoded linguistic knowledge in BERT.
Then, we divided BertQA into its submodules, including entity detection, entity linking, relation prediction, and evidence integration, following the original paper of BuboQA.
Because entity linking and evidence integration do not use the BERT-based machine-learning system, we set causal hypotheses only between the probing tasks and two submodules of BertQA, including entity detection and relation prediction.
Therefore, our PLSPM models based on the diagrams shown in Figure~\figref{figure:cd} aim to estimate structural equations for the following hypotheses:
\begin{itemize}
    \item The accuracies of probing tasks, SentEval and GLUE, can explain the accuracy of entity detection and the accuracy of relation prediction.
    \item The accuracy of entity detection can explain the accuracy of entity linking.
    \item The accuracy of entity linking and the accuracy of relation prediction can explain the accuracy of evidence integration.
\end{itemize}

We employed 24 BERT models from \shortciteA{turc19well} to prepare sample data for our causal diagrams.
These BERT models share their standard training recipe with BERT-Base \shortcite{devlin19}, only differing in the number of layers (2, 4, 6, 8, 10, 12) and hidden embedding sizes (128, 256, 512, 768).
Even though PLSPM allows a smaller sample size compared with other structural equation modeling methods \shortcite{tenenhaus05,sanchez13}, we did not combine SentEval and GLUE into one PLSPM model to decrease the number of parameters of PLSPM models.
Furthermore, we estimated PLSPM models for FBQ, SQ, and WQ, respectively; that is, we prepared six PLSPM models in total.
Hereinafter, we use the following naming convention for a PLSPM model estimated with the accuracies of the specific probing dataset and BertQA for a simple factoid question answering dataset: \texttt{(the name of the probing dataset)}-\texttt{(the name of the dataset for simple factoid question answering)}.
For example, \textit{SentEval-FBQ} indicates that this PLSPM model is estimated with the accuracies of SentEval and the accuracies of BertQA for the FBQ dataset.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[b]
\input{01table04.tex}
\hangcaption{List of tasks used for PLSPM models. Gray tasks were not used in our experiments because of the low correlation coefficients.}
\tablabel{table:tasks}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

For the observed variables in our PLSPM models, we prepared the results of SentEval, GLUE, and submodules of BertQA with 24 BERT models.
Table~\tabref{table:tasks} provides the details of the tasks we employed.
Note that we did not conduct any preprocessing for the observed variables except for normalization, whereas SentEval and GLUE used many kinds of performance indicators, such as Top-$n$ accuracy, F1-score, and Matthews correlation coefficient.
For our causal diagram, we need to define the original indicator for measuring the performance of each task.
Furthermore, we also structured our employed tasks according to their original paper to obtain the composite latent variables, except for QQP and WNLI in the GLUE dataset.
These tasks have low correlation coefficients with other tasks in the same category, which negatively affects the reliability of PLSPM models.
Our code will be shared in the GitHub repository for reproducibility.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Results}
\seclabel{plspmresult}

First, we examine the extent to which the accuracy of SentEval can predict the accuracy of BertQA by PLSPM analysis.
Table~\tabref{table:coef-senteval} lists the path coefficients of each path in SentEval-FBQ, SentEval-SQ, and SentEval-WQ.
When the PLSPM model is interpreted in the statistical field, the path coefficient is considered as the explainability of the target path.
We do not list paths for which the $p$-value is larger than 0.05.
Note that it is considered to be a meaningful index even though the path coefficient is a negative value because a meaningless path would be rejected on the basis of the $p$-value.
As a result, we can conclude that the accuracies of SentEval can meaningfully explain the accuracies of BertQA, except for the semantic tasks of SentEval.
This indicates that BertQA cannot overcome the discrepancy between different datasets because the inconsistency in the distribution of questions between different datasets is related to surface and syntactic knowledge.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[b]
\input{01table05.tex}
\hangcaption{Path coefficient for PLSPM models with SentEval. If the $p$-value of the path equation is higher than 0.05, we reject this path.}
\tablabel{table:coef-senteval}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We find it to be more difficult to interpret the result of PLSPM models using the GLUE dataset as is clear from Table~\tabref{table:coef-glue}.
Only the accuracies of the inference tasks can explain the accuracy of entity detection with the $p$-value $< 0.05$ among GLUE tasks, yet they also report negative coefficients for explaining the accuracies of relation prediction.
In this regard, the accuracies of single-sentence tasks, such as CoLA, mainly explain the relation prediction.
One interesting point is that the accuracies of the similarity and paraphrase tasks are rejected to explain the accuracy of BertQA with a $p$-value $> 0.05$.
These datasets of the similarity and paraphrase tasks, such as MRPC, require an understanding of the semantic knowledge of the given sentences to solve the questions.
This means that encoded semantic knowledge and the ability to understand the given sentences are not very helpful in explaining the accuracies of BertQA, even in our PLSPM model with the GLUE dataset.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[b]
\input{01table06.tex}
\hangcaption{Path coefficient for PLSPM models with GLUE. If the $p$-value of the path equation is higher than 0.05, we reject this path.}
\tablabel{table:coef-glue}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[b]
\input{01table07.tex}
\caption{Goodness-of-Fit index for each of the PLSPM models.}
\tablabel{table:gof}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In terms of the overall results of our PLSPM models, we found another problem regarding the discrepancy between each simple factoid question answering dataset.
Table~\tabref{table:gof} lists the goodness-of-fit indexes of each PLSPM model.
The goodness-of-fit value of the PLSPM model indicates the extent to which this model can explain the observed and latent variables.
Although all PLSPM models share the same causal diagram, the PLSPM models for the WQ dataset reported lower goodness-of-fit indices than the PLSPM models for other datasets.
In particular, SentEval-WQ shows a lower goodness-of-fit index than 0.6, which indicates that this model is not well explained by a given causal diagram.
This result implies that WQ is solved with different encoded linguistic knowledge compared with FBQ and SQ.

Another issue is the discrepancy in the GoF values between the PLSPM models using SentEval and the PLSPM models using GLUE.
As in Table~\tabref{table:gof}, we find that the PLSPM models using GLUE have higher GoF values than those using SentEval.
This indicates that the linguistic knowledge measured by SentEval has lower explainability than that measured by GLUE, following our causal diagrams in Figure~\figref{figure:cd}.
We suppose two reasons for this phenomenon.
First, \shortciteA{conneau18senteval} reported that sometimes the accuracy of their probing task does not show any correlation with the accuracy of downstream tasks.
For example, the tree depth and bigram shift tasks only correlate with one downstream task among the 17 downstream tasks.
This means that the probing tasks in SentEval are not sufficiently robust to explain the accuracy of downstream tasks.
Second, SentEval was proposed for sentence encoders such as SkipThought \shortcite{kiros2015skip} and InferSent \shortcite{conneau2017infersent} before the proposal of contextual embeddings.
We suppose that this is why the probing task in SentEval has lower explainability for the BERT-based model.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Discussion}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{BertQA and semantic understanding}

The results in Table~\tabref{table:coef-senteval} show that BertQA largely depends on the encoded surface and syntactic knowledge.
This also means that the surface and syntactic features of each dataset affect the accuracy of the BertQA model.
On the other hand, the accuracy of semantic information tasks cannot explain the accuracies of BertQA.
As mentioned in Section~\secref{bertqa-result}, semantic understanding is required to solve ambiguous questions among datasets, such as \textit{people.person.profession}.
Therefore, the lack of semantic understanding in BertQA is an important factor responsible for the failure of general simple factoid question answering.

The method used to evaluate the simple factoid question answering task, namely matching the predicted subject and relation with gold data \shortcite{bordes15}, is one reason for this problem.
Particularly in SimpleQuestions, the subject and the relation can be extracted from the question without semantic understanding, because the questions usually contain labels of the subject and the relation \shortcite{serban16}.
This means that the evaluation method compels existing QA models to concentrate on surface and syntactic features.
Furthermore, the possibility of multiple correct facts for the given questions could be another problem with this evaluation method.
For example, when BertQA predicted \textit{people.person.profession} for a given question then it was able to find the correct answer in Freebase, but the traditional evaluation method may reject this prediction if the gold relation in the dataset is \textit{common.topic.notable\_type}.
Hereinafter, we refer to the traditional evaluation method as the \textit{matching accuracy} because the criteria of this evaluation are based on matching the correct subject and relation from a given question.

To examine the effect of the evaluation method on the accuracy of BertQA, we conducted additional experiments.
We employed an older evaluation method \shortcite{berant13,berant14semantic}, which considers an object, the answer to a given question in the QAKB task, to overcome the limitation of the matching accuracy.
Evidence integration, a submodule of BertQA, combines the predicted subjects and predicted relations for a given input question, and evaluates its result with the gold fact given by the dataset.
Here, our employed evaluation method compares the predicted object, which is derived from a predicted subject and relation, with the gold object in the dataset.
Moreover, we extended this method to entity linking and relation prediction.
In particular, we aggregated all facts from FB2M to automatically examine whether the predicted result of each submodule can reach the gold object.
We refer to this employed evaluation method as the \textit{reachability accuracy} because the criteria of this evaluation are based on the reachability of the correct answer in a knowledge base. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[b]
\input{01table08.tex}
\hangcaption{Results of BertQA for QAKB datasets. The matching accuracy was calculated by checking whether the predicted subject and relation were the same as the gold data. The reachability accuracy was calculated by checking whether the predicted subject and relation can reach the gold object.}
\tablabel{table:evaluation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table~\tabref{table:evaluation} compares the matching and reachability accuracy for the experimental results.
BertQA still does not reach the upper bound accuracies of each dataset, even when evaluating the reachability accuracy.
However, BertQA achieves higher accuracies on average with the reachability accuracy.
This means that many entities and relations, which can reach the correct answer in a knowledge base, were scored as incorrect predictions with the previous evaluation method.
This indicates that the previous evaluation method does not consider paraphrasing or synonyms, strongly related to semantic understanding.

We also estimated the PLSPM models with the reachability accuracies of BertQA.
Note that we did not change our causal models at all, because we only employed the reachability accuracies of BertQA as observed variables.
The new PLSPM models still rejected the structural equations between the probing tasks for semantic information and BertQA.
However, the new PLSPM models reported a higher Goodness-of-Fit value for all datasets on average than the PLSPM models with matching accuracies, as indicated in Table~\tabref{table:newgof}.
In particular, in Table~\tabref{table:r2}, the difference in the $R^2$ value of latent variables is generally larger for WQ than for SQ.
Because it is related to the features of WebQuestions, we continue this discussion in the next section.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 9
\begin{table}[t]
\input{01table09.tex}
\caption{Comparison of Goodness-of-Fit indexes between evaluation methods.}
\tablabel{table:newgof}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[t]
\input{01table10.tex}
\hangcaption{$R^2$ (explanatory power) for each variable in our PLSPM models. A higher value indicates a higher explainablilty for the target variable. ED is entity detection, EL is entity linking, RP is relation prediction, and EI is the end-to-end accuracy from evidence integration.}
\tablabel{table:r2}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Special characteristics of WQ}

As we reported in Section~\secref{plspmresult}, the Goodness-of-Fit values of the PLSPM models for WQ are lower than those of the PLSPM models for other datasets.
This indicates that the same causal diagram, which assumes that encoded linguistic knowledge can explain the accuracies of BertQA, does not match for WQ, unlike other datasets.
Table~\tabref{table:r2} contains the $R^2$ value of each variable in our PLSPM models.
$R^2$ denotes the explanatory power of each variable. 
For example, the $R^2$ value of $0.787$ for ED means that approximately 79\% of the ED variables can be explained by the PLSPM model.
We found that the observed variables of entity detection (ED) and relation prediction (RP) for WQ are not explained more satisfactorily by linguistic knowledge than ED and RP for other datasets.
Furthermore, the result obtained for entity linking (EL) for WQ is not explained by the accuracy of entity detection for WQ, unlike the result obtained for other datasets.
These results indicate that WQ is an especially difficult dataset for predicting the correct entity from a given question using the existing QA model.
Therefore, we conducted a qualitative analysis with question sentences of WQ to determine the reason for this phenomenon.

We investigated the error cases of BertQA for the validation split of WQ to examine why these problems occur.
We manually labeled 20 error questions, which were evaluated as being correct in entity detection but as incorrect in entity linking, as in Table~\tabref{table:ellabel}.
As a result, we find that two main problems occur when linking the entity in given questions.
First, Freebase links too many entities with a single entity label.
For example, when we find an entity with the label \textit{mexico} in the preprocessed index by BuboQA, we obtain 2,830 results.
Because the entity linking of BuboQA and BertQA does not have any scoring process to sort ambiguous results, BertQA sometimes cannot find the correct entity for a given question within the top-$n$ results.
The second problem is that the string label for the entity in Freebase and the written string for the subject in WQ are not identical.
For example, the given question in WQ has the phrase, \textit{communist party of china}, but the label of the entity for this phrase in Freebase is \textit{Chinese communist party}.
Therefore, the characteristics of WQ are the reason why entity detection and linking of WQ are not explained well by our PLSPM models.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 11
\begin{table}[b]
\input{01table11.tex}
\caption{Twenty error cases in entity linking for WQ.}
\tablabel{table:ellabel}
\end{table}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[b]
\input{01table12.tex}
\hangcaption{Question patterns for the relation \textit{people.person.profession} in SQ and WQ. Number means the number of questions requiring \textit{people.person.profession} as the gold relation.}
\tablabel{table:profession}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

For relation prediction, we find that sometimes the same relation is written with different patterns by annotators on each dataset.
We examined the relation \textit{people.person.profession} as the sample relation to investigate the difference in writing patterns between datasets.
If the term \textit{profession} appears directly in the question, we label it as \textit{directly specify relation}.
If a term \textit{job, work, ...} that is similar to \textit{profession} appears in the question, we use the \textit{indirectly specify relation label}, and if there is no term similar to \textit{profession} in the question, \textit{paraphrasing} is used for annotation.
Table~\tabref{table:profession} contains the result of our annotation on the validation split of SQ and WQ.
In SQ, approximately 80\% of the questions for \textit{people.person.profession} contain a term that indicates the relation either directly or indirectly, whereas in WQ only 13\% of the questions contain such a term.
This result indicates that the relation prediction of WQ demands more complex linguistic knowledge than surface and syntactic understanding, unlike SQ.
Therefore, the relation prediction of WQ is not explained more satisfactorily than that of SQ in our PLSPM models.

As discussed above, WQ has different characteristics for entities and relations compared with other datasets, especially SQ. 
According to our analysis, entity linking for WQ requires the ability to disambiguate too many candidate entities for the same label in Freebase. 
Furthermore, relation prediction for WQ involves semantic understanding to predict the gold relation because the questions in WQ tend to contain a paraphrased term or synonym of the label for the gold relation. 
\shortciteA{coling2020} reported that BuboQA and KEQA, other state-of-the-art systems for SimpleQuestions, have the same difficulty with entity linking and relation prediction for WQ as BertQA. 
This indicates that WQ is a more challenging dataset than SQ, even for other state-of-the-art systems.

One main reason for this phenomenon is the difference in the method that is used to generate questions in the dataset.
For WebQuestions, the source of WQ, the questions were generated by Google Suggest API as naturally written user queries \shortcite{berant13}.
On the other hand, the question in SimpleQuestions, the source of SQ, were written artificially by crowd workers based on the suggested fact \shortcite{bordes15}.
Therefore, the difference in the method that was used to create each dataset is the reason for the difference in distribution among the datasets.
Moreover, this is also the reason why each dataset is solved using different linguistic knowledge in our PLSPM models.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

In this paper, we examined whether the pretrained language model, BERT, could robustly solve simple factoid question answering.
BERT-based models proved to be robust in other natural language processing areas \shortcite{talmor19}.
However, we found that BertQA failed to solve various tasks successfully, as in previous systems.
We conducted PLSPM analysis to investigate why BertQA could not overcome the discrepancy in the distribution among datasets.
As a result, our experiments revealed that even BERT depends on the surface and syntactic features of each dataset, rather than the semantic understanding required for general simple factoid question answering.
This indicates that a pretrained language model is not sufficient to generalize the distribution discrepancy among existing datasets.
In addition, we discuss the source of each dataset and the particular method that was used to evaluate simple factoid question answering, which are important to understand why even BERT depends on the surface and syntactic features.

\shortciteA{coling2020} mentioned other possible approaches to general simple factoid question answering, such as improving the quality of the dataset and distributionally robust optimization \shortcite{delage10,oren19} for generalizing biased datasets.
In addition, new datasets have been proposed that consider the distributions and difficulties of questions \shortcite{gu20iid}.
Although these studies yielded promising results, we additionally suggest that it would also be necessary to reconsider the evaluation method for simple factoid question answering.
For example, changing an objective function from the subject and relation to an object may improve the semantic understanding of the QA system for the given questions.
In future work, we hope to suggest a more robust system for simple factoid question answering, based on the findings of this study and those of other researchers.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment
This paper is based on results obtained from a project JPNP20006, commissioned by the New Energy and Industrial Technology Development Organization (NEDO). For experiments, computational resource of AI Bridging Cloud Infrastructure (ABCI) provided by National Institute of Advanced Industrial Science and Technology (AIST) was used.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\bibliography{01refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Namgi Han}{%
received his Ph.D. from The Graduate University for Advanced Studies, SOKENDAI, in 2021. He is currently a post-doctoral fellow at Ulsan National Institute of Science and Technology. His research interests include natural language processing and neural network interpretability.
}

\bioauthor[:]{Hiroshi Noji}{%
received his Ph.D. from the Graduate University for Advanced Studies and was an assistant professor at Nara Institute of Science and Technology. He is currently a machine learning engineer at LeapMind Inc.\ and a visiting researcher at AI Research Center of Advanced Industrial Science and Technology. His research interests include computational linguistics, in particular syntactic parsing.
}

\bioauthor[:]{Katsuhiko Hayashi}{%
received PhD in Engineering from the graduate school of Information Science at the Nara Institute of Science and Technology in 2013.
He is currently an associate professor in Gunma University and a visiting researcher in the Riken Center for Advanced Intelligence Project.
His research interests include natural language processing and statistical relational learning.
}

\bioauthor[:]{Hiroya Takamura}{%
received his Ph.D. from Nara Institute of Science and Technology.
He worked as a professor at Tokyo Institute of Technology, and currently is a research team leader at AI Research Center of Advanced Industrial Science and Technology.
His current research interests include natural language processing.

}

\bioauthor[:]{Yusuke Miyao}{%
received his Ph.D. from the University of Tokyo in 2006. He has been Assistant Professor at the University of Tokyo from 2001 to 2010, Associate Professor at National Institute of Informatics since 2010, and Professor at the University of Tokyo from 2018.
}
\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate



\end{document}

