\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{dcolumn}
\usepackage{hangcaption_jnlp}
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\newcommand\blfootnote[1]{
%%%%    \begingroup
%%%%    \renewcommand\thefootnote{}\footnote{#1}
%%%%    \addtocounter{footnote}{-1}
%%%%    \endgroup
%%%%}
%%%%\usepackage{soul}
%%%%\usepackage{appendix}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{31}
\Number{3}
\Month{September}
\Year{2024}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2024}{2}{1}
\revised{2024}{5}{1}
\accepted{2024}{6}{14}

\setcounter{page}{1140}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{SciCap+: A Knowledge Augmented Dataset to Study the Challenges of Scientific Figure Captioning}
\eauthor{Zhishen Yang\affiref{Author_1} \and Raj Dabre\affiref{Author_2}\and Hideki Tanaka\affiref{Author_2}\and Naoaki Okazaki\affiref{Author_1}} 
\eabstract{
Figures in scholarly documents provide a straightforward method of communicating scientific findings to readers. Automating figure caption generation enhances model understanding of scientific documents beyond text and helps authors write informative captions. Unlike previous studies, we refer to scientific figure captioning as a knowledge-augmented image-captioning task in which models must utilize knowledge embedded across modalities for caption generation. To this end, we extend the large-scale SciCap dataset \cite{hsu-etal-2021-scicap-generating} to SciCap+, which includes mention paragraphs (paragraphs mentioning figures) and OCR tokens. We then conducted experiments using the M4C-Captioner (a multimodal transformer-based model with a pointer network) as a baseline for our study. Our results indicate that mention paragraphs serve as additional context knowledge, significantly boosting automatic standard image caption evaluation scores compared to figure-only baselines. Human evaluations further reveal the challenges associated with generating figure captions that are informative to readers. The code and SciCap+ dataset are publicly available: \url{https://github.com/ZhishenYang/scientific_figure_captioning_dataset}
}
\ekeywords{Figure Captioning, Multimodal Machine Learning, Understanding Scientific Documents}

\headauthor{Yang et al.}
\headtitle{SciCap+: A Knowledge Augmented Scientific Figure Captioning Dataset}

\affilabel{Author_1}{}{Tokyo Institute of Technology}
\affilabel{Author_2}{}{National Institute of Information and Communications Technology}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

\renewcommand{\thefootnote}{}
\footnote[0]{This study is an extension of a previously accepted work presented at the AAAI-23 Workshop on Scientific Document Understanding. The paper can be accessed at \protect\url{https://ceur-ws.org/Vol-3656/paper13.pdf}}
\renewcommand{\thefootnote}{\arabic{footnote}}


\vspace{-1\Cvs}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Scholar documents are the primary source of scientific knowledge sharing. These documents are available in various formats such as journal articles, book chapters, and conference proceedings. Most of these documents is text, and concurrently with figures and tables, they help to communicate knowledge to readers. Figures provide visual representations of complex information that facilitates the efficient and straightforward sharing of scientific findings with readers. The standard practice in scientific writing is to write a caption for each figure, accompanied by paragraphs with detailed explanations. Figures and captions should be standalone, and readers should be able to understand them without referring to the main text. Helping authors write appropriate and informative captions for figures will improve the quality of scientific documents, thereby enhancing the speed and quality of communication. In this study, we focused on automating the generation of captions for figures in scientific papers.

Figure captioning is a variant of image captioning. However, with the same goal of generating a caption, it has two unique challenges: 1. Figures are not natural images: In contrast to natural images, visual objects are texts and data points in scientific figures. 2. Figure captions should not merely identify objects and texts but also provide an analysis that the authors intend to present and highlight.  

In a previous study \cite{hsu-etal-2021-scicap-generating}, SciCap defined the scientific figure-captioning task as a figure-to-caption task: a model generates captions by referring only to figures. Their work reported relatively low scores as measured by automatic evaluation metrics, indicating considerable room for improvement. Intuitively, writing appropriate figure captions without sufficient background knowledge is difficult because even humans struggle to interpret a figure and write a caption unless some background knowledge is available. Based on this observation, it is infeasible to generate appropriate captions without adding context knowledge to the caption generation model. This context has two forms: background knowledge from the running text and OCR texts in the figure, both of which should help reduce the burden on the captioning model. Thus far, we augment existing large-scale scientific figure captioning dataset SciCap with mention paragraphs and OCR texts and call the resultant dataset SciCap+. We then posed scientific figure captioning as a multimodal summarization task and used the M4C captioner model \cite{sidorov2020textcaps} (which utilizes multimodal knowledge to generate captions) as a baseline for studying the scientific figure captioning task. Experimental results of the automatic evaluation demonstrate that using knowledge embedded in different modalities, particularly  in the form of mention paragraphs and OCR tokens, significantly boosts performance.  

In addition to the experiments using automatic evaluation metrics, we performed human generation and evaluation tasks to establish the inherent difficulties of scientific figure captioning. The human evaluation revealed three findings: 1. Multimodal knowledge helps models outperform humans in caption generation tasks. 2. Model-generated captions are almost as informative as ground-truth captions and human evaluators do not prefer either type of caption. 3. Even when referring to paragraphs, it remains challenging for humans to write captions that are close to the ground truth. To the best of our knowledge, this is the first study to pose scientific figure captioning as a multimodal summarization task and show that mention paragraphs and OCR tokens substantially enhance the quality of the generated captions.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{31-3ia11f1.pdf}
\end{center}
\hangcaption{Example figure~\protect\cite{zhang2019cheetah} with its captions and mention-paragraph and the texts recognized via OCR. Without referring to the mention-paragraph and the OCR texts to tie the figure and the mention, we cannot have a proper interpretation of the data presented in the figure, which is communication cost comparison and speed up of CHEETAH over GAZELLE.}
\label{pre_study}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In traditional image-captioning tasks, an image is captured to describe the appearance or nature of recognized objects and illustrate the relationships between recognized objects. Unlike typical image captioning tasks, these figures do not contain visual scenes. Rather, the captions provide interpretations of the data presented in figures to highlight the scientific findings that the authors aim to present to the readers. With this unique characteristic, without referring to paragraphs that typically refer to the figure, it is extremely challenging for a human to properly interpret figures. This is because they may lack the background knowledge of the domain or context of the figure. As shown in Figure~\ref{pre_study}, by only looking at the figure, we do not know what ``comm.(KB)'' stands for; therefore lacking the knowledge to write informative captions is challenging. However, the mention-paragraph contains ``communication cost'' and this is also present in the caption, indicating that such background knowledge should help in writing accurate captions. 



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Problem Formulation}

The previous study \cite{hsu-etal-2021-scicap-generating} defined this task as an image-captioning task: Given a figure $I$, the model generates a caption $C=[c_0,c_1,...,c_N]$ subword tokens. However, we reframe the scientific figure captioning task as a knowledge-augmented image-captioning task that requires knowledge extracted from the text and vision modalities. For a figure, we define the paragraph that mentions it (mention-paragraph) and the text within the figure, extracted via OCR, as text modalities. The figure itself and the visual appearance of the OCR text are visual modalities. Given a scientific figure $I$ and knowledge extracted from the text and vision modalities $K_{text}$ and $K_{vision}$, we define the figure caption generation task as $P(C|I,K_{text}, K_{vision})$


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{SciCap+ Dataset}

SciCap is a large-scale figure-caption dataset comprising graphplots. These plots were extracted from ten years of collections from arXiv's computer science sections under the topics of computer science (cs) and machine learning (stat.ML).


We used approximately 414k figures from SciCap and augmented each figure with its mention-paragraph and OCR tokens (OCR texts and bounding boxes). This section details the dataset creation and data augmentation processes. Figure~\ref{dataset_creation} illustrates the workflow behind the creation of SciCap+.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Mention-paragraph Extraction}

We first obtained papers in PDF format from Kaggle arXiv dastaset.\footnote{\url{https://www.kaggle.com/datasets/Cornell-University/arxiv}} The reason for using PDFs is that not all papers have source files and some are overly complicated to parse. 
After obtaining the PDFs, we used PDFFigures 2.0\footnote{\url{https://github.com/allenai/pdffigures2}} \cite{clark2016pdffigures} to extract the body text from each paper. PDFFigure 2.0 is a tool that extracts figures, captions, tables, and text from scholarly PDFs in computer science. In scholarly documents, authors label figures with numbers (for example, Figure 1. Fig. 1). For each figure, we used its figure number in a regular expression to locate the paragraph that mentions it.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{OCR Extraction}

The SciCap dataset also provides text extracted from figures 
\pagebreak
as metadata but does not provide location information for each text. To include location information for each text in a figure, we used Google Vision OCR API to extract text tokens from each figure with the coordinates of the bounding boxes.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia11f2.pdf}
\end{center}
\hangcaption{Overall workflow of the data augmentation for creating SciCap+ dataset. For each figure in SciCap+, we extracted its mention-paragraphs and OCR tokens (OCR texts and bounding boxes).}  
\label{dataset_creation}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Data Statistics}

The SciCap dataset is split at the figure level. Therefore, figures from the same study may appear with different splits. This leads to unfair evaluations because the information from one figure in one split may coincidentally overlap with the information from another figure. Therefore, we have re-split the figures at the document level to eliminate the overlapping problem. 
\citeA{hsu-etal-2021-scicap-generating} show that text normalization and figure filtering do not improve model performance. Hence, we retained the original captions and all figures (with/without subfigures) in the SciCap+ dataset. For each figure, we retained only the first paragraph (mention-paragraph) that mentions it in the body text. In addition, we included bounding box information. Table~\ref{data_diff} compares the SciCap+ dataset with the previous figure-captioning datasets. The proposed SciCap+ dataset leverages data from real-world scientific papers and includes additional in-context knowledge. Table~\ref{data_stats} lists the statistics for the SciCap+ dataset. In all the three splits, approximately 90\% of the captions contained fewer than 66 words. All figures are graphplots.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[t]
\input{11table01.tex}%
\hangcaption{Comparison with the previous figure captioning datasets. The proposed SciCap+ dataset builds upon the SciCap dataset by incorporating additional in-context knowledge and utilizing data from real-world scientific papers.}
\label{data_diff}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{11table02.tex}%
\caption{Statistics of the SciCap+ dataset.}
\label{data_stats}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dataset Quality Evaluation} 

Before conducting the experiments, we conducted a human evaluation of SciCap+, in which we verified mention-paragraphs and OCR tokens extraction quality. The goal was to establish whether the paragraphs and OCR tokens were extracted correctly and relevant to the figures and their captions. To achieve this objective, we randomly chose 200 figures from the training set. Two human annotators assigned scores ranging from 1 to 5 for each figure. On this scale, 1 signified no relevance, 2 indicated low relevance, 3 denoted moderate relevance, 4 represented high relevance, and 5 signified perfect relevance. We instructed the annotators to examine the figures and assess the relevance of the mention-paragraphs and OCR tokens separately. Finally, they were asked to provide a joint relevance score for the mention-paragraphs and OCR tokens concerning figure captions and figures.

Unlike natural image captioning, human evaluation tasks in the figure captioning domain require expert knowledge. Two colleagues were recruited to perform the evaluation task. Both have Ph.D. degrees in computer science and work as researchers in natural language processing (NLP). Their backgrounds implied that they had adequate experience evaluating figure captions. 

Figure~\ref{dataset_quality_check} shows the distribution of the relevance scores. Two annotators provided the most figures (evaluator 1: 64\% and evaluator 2: 79.5\%), with relevance scores greater than three and a cohen kappa score of 0.28. These evaluation results indicate that the mention-paragraphs and OCR tokens have satisfactory extraction quality, and that the annotators considered most of them relevant to the figure and its caption. However, the two annotators seem to have a relatively low agreement (0.28) regarding which figures and captions are relevant to their mention-paragraphs and OCR tokens. We attribute this to the subjective nature of evaluating figure captions. Determining relevance involves manually assessing the extent to which the text aligns with the figure and relies on the annotators' understanding of the figures and text.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia11f3.pdf}
\end{center}
\hangcaption{Score distribution on correlations between mention-paragraph, OCR tokens and figure captions. Both evaluators judged most figures and captions with at least moderate correlations with their mention-pargraphs and OCR tokens. }  
\label{dataset_quality_check}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Figure \ref{fig:dataset_casestudy} depicts a case study highlighting score differences between two annotators. Annotator 1 observed excessive text in the mention-paragraph, making it difficult to determine which parts were relevant to the caption. Therefore, Annotator 1 concluded a moderate relevance between the caption and the mention-paragraph and noted the presence of Japanese words (incorrect recognition) in the OCR tokens. Consequently, Annotator 1 assigned a total score of 2. However, Annotator 2 perceived that the mention-paragraph contained a crucial keyword, ``two-way coupling.'' with in-detailed explanations. Despite incorrect OCR recognition, Annotator 2 believed that it did not significantly impact overall comprehension and thus assigned a score of 5. This case study further demonstrates that, despite providing score criteria to annotators, the definition of the relevance level is highly subjective.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia11f4.pdf}
\end{center}
\hangcaption{Case study on dataset quality evaluation. Two annotators subjectively weigh the contributions of mention-paragraphs and OCR tokens, resulting in significant differences in scores.}  
\label{fig:dataset_casestudy}
\vspace{1\Cvs}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}

We conducted experiments using SciCap+ to empirically prove that scientific figure captioning is inherently a knowledge-augmented task that benefits from the knowledge coming from both text and vision modalities. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Figure Captioning Model}


The figure depicted in Figure~\ref{workflow_model} outlines the overall workflow of a scientific figure-captioning task. Our approach leverages the M4C-Captioner \cite{sidorov2020textcaps} as the baseline model to study the challenges of the task. 

The M4C-Captioner is based on a multimodal multicopy mesh (M4C) \cite{hu2020iterative} which jointly learns representations across input modalities. It incorporates a pointer network capable of selecting text from OCR tokens or a predefined fixed dictionary to address out-of-vocabulary issues during caption generation. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[t]
\includegraphics{31-3ia11f5.pdf}
\hangcaption{Overall framework for scientific figure captioning is centred around the architecture derived from M4C-Captioner \protect\cite{sidorov2020textcaps}. This core component is designed to learn representations collaboratively from various input modalities. It incorporates a pointed network to dynamically choose text from OCR tokens or a predefined dictionary. }
\label{workflow_model}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Implementation and Training}

Our implementation of the M4C-Captioner was based on the MMF framework \cite{singh2020mmf} and Pytorch. The implementation allows users to specify diverse pre-trained encoders for each modality, which can be fine-tuned or frozen during training. The M4C-captioner itself has $D=768$  hidden dimension size, $K=4$ transformer layers and 12 attention heads. We used sentencepiece \cite{kudo-richardson-2018-sentencepiece} to obtain a dictionary of 32000 subwords built from both mention-paragraphs and OCR tokens. This is used as the M4C-captioner's vocabulary. We followed the BERT-BASE hyperparameter \cite{devlin-etal-2019-bert} settings and trained from scratch.

Regarding the encoders that feed features to the M4C-captioner, we used a pre-trained Resnet-152 as the vision encoder of the figure. For each figure, we applied a 2D adaptive average pooling over the outputs from layer 5 to obtain a global visual feature vector with dimensions of 2048. 
During training, layers 2, 3, and 4 were fine-tuned to acquire specialized features from the figures \cite{yosinski2014transferable}. For mention-paragraph features, SciBERT \cite{beltagy-etal-2019-scibert} was used to encode\footnote{We only used the first 3 layers of SciBERT for lightweightness.} it into 758-dimensional feature vectors. The number of vectors equals the number of sub-word tokens in the mention-paragraph, which we limit to 192 (around 90\% mention-pargarphs have less than 190 words, with additional start and end sentence symbols), including additional start and end sentence symbols. The mention-paragraph encoder was also fine-tuned during training. Finally, for OCR tokens, we used both text and visual features. We selected FastText \cite{almazan2014word} as the word encoder and Pyramidal Histogram of Characters (PHOC) \cite{bojanowski2017enriching} as the character encoder, respectively. Regarding the visual feature encoder of OCR tokens, we first extracted Faster R-CNN fc6 features and then applied fc7 weights to obtain 2048-dimensional appearance features for the bounding boxes of OCR tokens. The fc7 weights were fine-tuned during the training. We retained a maximum of 95 OCR tokens per figure, with approximately 90\% of the figures containing fewer than 95 OCR tokens.

We trained a model on a GPU server using eight Nvidia Tesla V100 GPUs. The training of a model with a complete set of features required 13 hours. During training, we used a batch size of 128. We selected CIDEr as the evaluation metric. Evaluations were conducted every 2000 iterations, and the training was stopped if the CIDEr score did not improve for four intervals. The optimizer used was Adam with a learning rate of $0.001$ and $\epsilon= 1.0\mathrm{E}{-08} $. We also used a multistep learning rate schedule with warmup iterations of 1000 and a warmup factor of 0.2. During decoding, we set the maximum number of steps to 67, including the start and end sentence symbols (with approximately 90\% of captions in the training set containing fewer than 65 words).

For evaluation, we used five standard metrics for evaluating image captions: BLEU-4 \cite{papineni-etal-2002-bleu}, METEOR \cite{banerjee2005meteor}, ROUGE-L \cite{lin2004rouge}, CIDEr \cite{vedantam2015cider} and SPICE \cite{anderson2016spice}. Since figure captions contain scientific terms which can be seen as uncommon words among all five metrics, we are particularly interested in CIDEr since it emphasizes them.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Results}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Main Result}

Experimental results in Table~\ref{main_result} demonstrate that using the mention-paragraph and OCR tokens (\#6) significantly improves scores on all five exact-matching metrics compared to the figure-only baseline (\#1). The experimental results align with our hypothesis and preliminary study that scientific figure captioning is a knowledge-augmented image captioning task, and that OCR tokens and knowledge embedded in mention-paragraphs help in composing informative captions. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[b]
\input{11table03.tex}%
\hangcaption{Automatic exact-matching evaluation scores of M4C-captioning on SciCap+ dataset. Aggregate knowledge from text and vision modalities significantly boosts the model performance compared to the figure-only baseline.}
\label{main_result}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We established a baseline M4C-Captioner (Figure only) with figures as the only input modality to the M4C-Captioner model in row \#1. This baseline was conducted in a non-knowledge setting. Therefore, low scores for all metrics indicate that the model requires knowledge of other modalities. Using only the mention in row \#2 shows that the mention contains a large amount of useful information, as evidenced by the increase in performance. When OCR features are added to the figure input in row \#3, the scores for all metrics have significant gains compared to the figure-only baseline (\#1), but are still weaker than when only mentions are used (\#2). Since results indicate that mention embeds more information than OCR, we combine mention with the figure as inputs to the model (\#4); compared with row\#2, the figure gives additional improvements across all metrics.

This motivates the combination of mentions and OCR tokens and in row \#5, compared to the figure-only baseline (\#1), mention-only baseline (\#2), figure-OCR-only baseline (\#3) and figure-mention-only baseline (\#4), the performance further improves. All these results indicate that explicitly extracted multimodal knowledge helps to compose informative captions.


Exact-matching metrics tend to favour machine-generated captions over high-quality human-generated captions. However, soft-matching scores demonstrate a better correlation with human judgments \cite{Kasai2021TransparentHE}. Given the variation in style among figure captions, relying solely on exact-matching scores may not effectively evaluate them. Therefore, we complement the exact-matching scores by including soft-matching scores, specifically RefCLIPScore \cite{hessel-etal-2021-clipscore}, in Table~\ref{main_result_soft}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{11table04.tex}%
\hangcaption{Automatic soft-matching evaluation scores of M4C-captioning on SciCap+ dataset. Aggregate knowledge from text and vision modalities significantly boosts the model performance compared to the figure-only baseline.}
\label{main_result_soft}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Overall, the findings from RefCLIPScore evaluations align with the exact-matching metric evaluations. Using only figures (\#1) resulted in the lowest score, whereas gradually incorporating knowledge from the mention-paragraphs and OCR tokens improved the scores. Combining the mention-paragraphs and OCR tokens (\#6) led to the highest score. The results of the soft-matching metric evaluation further confirm the effectiveness of utilizing multimodal knowledge to compose informative captions.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Ablation Studies}

We conducted an ablation study on figures by removing visual feature vectors (row \#6). The CIDEr score increases slightly, suggesting that the visual features may act as noise for the model when the text modality contains sufficient information. Conversely, when the OCR is missing from the input (row \#4), the model can utilize information from the figure to improve scores, albeit slightly compared to using mention-paragraphs only (row \#2). However, these scores still fell below the settings utilizing both the OCR and mention-paragraphs (rows \#5 and \#6). This discrepancy may have arisen because the ResNet-152 visual encoder was not specifically trained on figures. Consequently, the visual features extracted from figures are less informative when the text modality contains sufficient information.

We enriched the representations of the OCR tokens by adding text, visual, and spatial features. Ablation studies aimed to reveal the impacts of each OCR token feature. All comparisons are with row \#5, although row \#6 yielded slightly better scores. When the OCR features in row \#4 were completely removed, the CIDEr scores decreased by 5.3. Using only the OCR spatial features in row \#7, the CIDEr score dropped by 7.8. After removing the OCR spatial features in row \#8, the CIDEr scores dropped by 1.2. Removing the OCR visual features in row \#9, resulted in a CIDEr score similar to the removal of spatial features. 


The ablation study described above indicates that the enriched OCR token features contribute to the informativeness of the generated captions. Unlike OCR token features, where appearance features are helpful to the model, removing the visual features of figures increases CIDEr scores, further indicating that a specific vision encoder is required for figures to provide meaningful features. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Human Evaluation}
\label{sec:humaneval}

Having established that knowledge helps a model perform figure captioning, we conducted human evaluation activities to determine the subjective quality.
Human caption generation and evaluation tasks are conducted. Human generation tasks examine whether humans can write better captions than models. The evaluation task evaluates how appropriate the model-generated captions are versus the ground-truth captions. Both tasks were performed by the same human subject for the quality assessment of the dataset.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Figure Caption Generation Task}

The figure caption generation task was to generate captions under two conditions separately: 1. Figure-only: Human annotators wrote captions given only figures. They did this to compare with the captions generated by the M4C-Captioner, which only has access to figures and OCR features. 2. Figure-Mention: Human annotators wrote captions for given figures and their mention-paragraphs. We randomly selected 100 figures from the test set and compared the the human-generated captions with the captions generated by M4C-Captioner. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[t]
\input{11table05.tex}%
\hangcaption{Automatic exact-matching evaluation scores on human-generated captions. The model has similar performances when the figure is the only available source. Using knowledge from vision and text modality, the model gains more on CIDEr scores. } 
\label{human_result_generation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table~\ref{human_result_generation} shows the automatic evaluation results for the human caption generation tasks. Given only the figures (rows \#1, 2), both annotators obtained low scores across all the metrics. Among these, Annotator 2 led all metrics except SPICE. As humans perform OCR naturally with their eyes, we compared it with the M4C-Captioner (Figure and OCR features). It had the best SPICE score, and although it outperformed Annotator 1 in four of the five evaluation metrics, it achieved a similar performance to Annotator 2. This indicates that humans are not better than machines without additional knowledge.

However, given mention-paragraphs and figures (rows \#4, 5), compared to the figure-only condition, both annotators got improved scores in BLEU-4, METEOR, ROUGE-L, and SPICE but lower scores in CIDEr. Previous research suggested that CIDEr is a more informative evaluation metric for caption generation tasks, particularly when the captions contain named entities \cite{biten2019good}. Given that figure captions incorporate named entities from both OCR tokens and mention-paragraphs, lower CIDEr scores imply that humans may face challenges integrating additional knowledge from these sources. However, with access to full features, the M4C-Captioner gained a significantly better CIDEr score than the human annotators. The automatic evaluation results of the human generation tasks show a steep difficulty in writing figure captions close to the ground truth.

Even given mention-paragraphs, our annotators wrote captions with low scores across all standard image captioning evaluation metrics. We ascribe it as figure captions are highly subjective and require in-domain knowledge to write. Although our annotators are researchers, they cannot be professional in all knowledge existing in the computer science domain. Granted mention-paragraphs and OCR tokens as external knowledge sources, and with a large amount of data training, the model can significantly outperform humans.

Table~\ref{human_result_generation_soft_score} presents the soft-matching evaluation results. In the setting where the mention-paragraphs are not provided, we observed that the results are consistent with exact-matching evaluations. When considering the mention-paragraphs, Annotator 2 achieved a higher RefCLIPScore than the model, although the model still outperformed Annotator 1. This discrepancy highlights the complexity and distinctiveness of the scientific figure captioning task. Further investigation needs to be conducted into the correlations between soft-matching scores, such as RefCLIPScore, and figure captions.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[t]
\input{11table06.tex}%
\hangcaption{Automatic soft-matching evaluation scores on human-generated captions. Humans obtained higher RefCLIPscore than models.} 
\label{human_result_generation_soft_score}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Appropriateness Evaluation}

This task evaluated the appropriateness of the model-generated and ground-truth captions. We used the same set of 100 figures as in the figure caption generation task and placed the ground-truth captions and model-generated captions in random order. Human evaluators then ranked each caption to give appropriateness scores (1--4) to each caption. The evaluation scale: 1. Inappropriate: a caption does not match the figure, is not a sentence, is wrong, or is misleading. 2. Not sure: Judging appropriateness solely from the figures is impossible. 3. Possible: A possible candidate that is incomplete but not wrong. 4. Appropriate: An informative caption that interprets the figure well with context knowledge. An appropriate figure caption should be self-contained, allowing readers to grasp the intended message of the figure without consulting the body text. Therefore, we withhold the mention-paragraphs from the evaluators during the evaluation process.

Table~\ref{appropriatness_score} lists the evaluation results. The two evaluators gave low average scores for both model-generated captions and ground-truth captions. In addition, evaluators only reached fair agreements on scoring (0.23--0.36). Using the mention and OCR tokens (row \#2), gets the best human evaluation scores and this is in line with the corresponding score in Table~\ref{main_result}, which also achieved the best CIDEr performance, indicating that human evaluation is reliable despite the fair agreement. Evaluation results indicate that the model-generated and ground-truth captions are not always informative for both evaluators, which reveals the need to improve the caption writing quality and model performance. We observed that captions tended to be written without following specific rules, which may have contributed to the lack of agreement. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[tb]
\input{11table07.tex}%
\hangcaption{Average appropriateness score on model-generated and ground truth captions. Two evaluators gave low scores on both model-generated and ground truth captions, with the fair inter-annotator agreement.}
\label{appropriatness_score}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Figure~\ref{fig:bad_caption} illustrates two ground-truth captions with low appropriateness scores. These are examples of poorly written captions because they are overly brief and lack sufficient information. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia11f6.pdf}
\end{center}
\hangcaption{Examples of ground-truth captions with low appropriateness scores. These two captions receive low appropriateness scores due to their brevity and lack of meaningful interpretation.}
\label{fig:bad_caption}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

With low inter-rater agreements, we found how informative a figure caption is highly subjective and depends on in-domain background knowledge evaluators have and the writing style evaluators favour.

Figure \ref{fig:case_study_appropriate_human} illustrates a sample of evaluation results where two annotators have significant assessment discrepancies. The ground-truth caption in this case study also serves as an example of poor caption writing because it offers only an overall description without interpreting the visual content in the figure. Annotator 1 consistently assigned a score of 2 to most captions, whereas Annotator 2 was predominantly awarded a score of 4. Annotator 1 expressed uncertainty regarding the presence of keywords such as ``bests set'' in captions 1, 2, and 4 and ``fakenewsamt'' in caption 5. For caption 3, annotator 1 suggested that the figure may depict the accuracy of the proposed method and, therefore, gave a score of 3. Annotator 1 exhibited a more cautious approach, whereas Annotator 2 displayed flexibility, even with captions that provided only general figure descriptions.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 7
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia11f7.pdf}
\end{center}
\hangcaption{Case study on appropriateness evaluations of figure captions. Two annotators exhibit significant discrepancies in their assessments regarding which caption scores higher.}
\label{fig:case_study_appropriate_human}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Qualitative Study}

This section presents a case study comparing human-generated and machine-generated figure captions, as shown in Figure~\ref{fig:case_study}. Given that an appropriate caption should be self-sufficient, conveying background knowledge and visual interpretations, our analysis highlights mention-paragraphs' role in providing background information for composing captions. Furthermore, we note the subjective nature of caption writing styles, which can vary significantly among authors.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 8
\begin{figure}[p]
\begin{center}
\includegraphics{31-3ia11f8.pdf}
\end{center}
\hangcaption{Case study on human-generated and model-generated captions. The mention-paragraph provides major information for the model and human annotators to compose informative captions.}
\label{fig:case_study}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Without reference to the mention-paragraph, both annotators relied solely on the text visible in the figure. Without access to the background information provided in the mention-paragraph, the annotators lack a contextual understanding of the figure. Consequently, their captions tended to resemble the summaries of the visual elements depicted in the figure. 

Access to the mention-paragraph empowers human annotators with the necessary background knowledge to interpret figures accurately and compose informative captions. For instance, with access to the mention-paragraph, both annotators correctly interpret the abbreviation ``Comm. (KB)'' as the communication cost, as shown in the figure. However, despite this advantage, both annotators failed to write captions separately for each sub-figure.

 A similar phenomenon is observed in the captions generated using the model. In scenarios where access to the mention-paragraph is lacking, even when provided with OCR tokens, the model struggles to position OCR text within the caption appropriately but manages to create sub-captions for each sub-figure (the model wrote sub-captions (a) and (b) separately, but failed to place OCR texts CHEETAH and GAZELLE in the caption).

When only given access to a mention-paragraph, the model copied the first sentence of the paragraph directly. However, when provided with both the mention-paragraph and OCR tokens, the model effectively incorporated OCR texts and summarized the figure with accurate information.

However, when presented with all features, including both the mention-paragraph and the OCR, the model appears to struggle to integrate information from multiple modalities. It fails to incorporate OCR texts from the figure and generates captions for each sub-figure instead of utilizing the information effectively (it does not split the figure caption into (a) and (b), and it does not include OCR texts CHEETAH and GAZELLE in the figure). When confronted with multiple heterogeneous modalities, the model encounters challenges in integrating information cohesively.

The two annotators adopted different writing styles; Annotator 1 tended to produce more extended captions with more detailed information than Annotator 2. However, despite these differences, both annotators failed to employ the same writing style for the ground-truth caption, which used subcaptions for each subfigure. This suggests that caption writing is highly variable, subjective, and influenced by individual preferences and interpretations.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}

Unlike natural image captioning, figure captioning has rarely been studied. SciCap \cite{hsu-etal-2021-scicap-generating}, the most recent work on scientific figure captioning, released a large-scale scientific figure captioning dataset that included figures from academic papers in the arXiv dataset. Before SciCap, FigCAP \cite{chen2019figure,Chen_2020_WACV} and FigureQA \cite{kahou2017figureqa} are two figure captioning datasets; however, their figures are synthesized. We decided to extend and study the SciCap dataset because its figures were obtained from real-world scientific papers. In this study, we also have leveraged multimodal knowledge using pre-trained models. 

The task of generating text from tables is closely related to generating captions for figures. A recently proposed benchmark, Chart-to-Text \cite{kantharaj-etal-2022-chart}, addresses table-to-text generation. It offers two problem settings: one where the underlying data table of the chart is provided and another where data needs to be extracted from chart images. The experimental findings suggest that utilizing the underlying data table results in table captions with fewer factual errors. Another study \cite{suadaa-etal-2021-towards} introduced a table-to-text generation dataset comprising pairs of tables and paragraphs describing the tables. The authors employed pre-trained language models with copy mechanisms to enhance numerical reasoning. Compared with the table-to-text generation task, captioning figures present greater challenges. Figures lack a fixed structure, such as tables, and captions must interpret numeric values, visual content, and contextual knowledge, thus introducing additional complexities.


Multimodal machine learning is to model knowledge across various modalities. The closest multimodal task to figure captioning is image captioning. A popular architecture is the encoder-decoder, where the decoder learns to generate captions conditioned on visual features extracted from the encoder. Recent studies on integrating text in natural images for visual question-answering and image-captioning tasks are based on a transformer architecture augmented with a pointer network \cite{hu2020iterative,sidorov2020textcaps}. The transformer enriches representations by integrating knowledge from both text and visual modalities. The pointer network dynamically selects words from a fixed dictionary or OCR texts during the generation.  

Using knowledge embedded in pre-trained models is common in solving multimodal tasks. In this study, we used SciBert \cite{beltagy-etal-2019-scibert}, a BERT model \cite{devlin-etal-2019-bert} that was pre-trained in scientific papers, to obtain informative representations of the texts extracted from computer science papers. Because the terms in the figures may be uncommon, we also used FastText \cite{Bojanowski2017EnrichingWV} to obtain word embeddings with subword information. For the visual modality, we used Renst152 \cite{he2016deep} and Faster R-CNN \cite{NIPS2015_14bfa6bb} to extract features from the images and bounding boxes.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

This study examines the challenges of a scientific figure-captioning task. Extending from a previous study \cite{hsu-etal-2021-scicap-generating}, we reframe this task as a knowledge-augmented image-captioning task; that is, the model needs to use knowledge extracted across modalities to generate captions. To this end, we released a new version of the SciCap dataset, SciCap+, by augmenting figures with the mention-paragraphs and OCR tokens. We used the M4C-captioner model as a baseline model to utilize knowledge across three modalities: mention-paragraphs, figures, and OCR tokens. Automatic evaluation experiments further revealed that using knowledge significantly improved evaluation metric scores. Compared to human-generated captions, we found that the models generated better captions than humans in terms of automatic evaluation metrics. However, human evaluations have demonstrated that writing scientific figure captioning is challenging, even for humans. Despite reasonable automatic evaluation scores, model-generated captions are still far from meeting human standards. The release of the SciCap+ dataset promotes further development of scientific figure captioning.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Limitations and Future Work}

Although our evaluators possess PhD degrees and expertise in natural language processing, we cannot guarantee that they have comprehensive knowledge across all topics in computer science and machine learning. However, their backgrounds are sufficient to assess the readability and informativeness of figure captions and interpret basic information from figures.

In this study, human evaluation and generation of figure captions are inherently subjective, as they rely on annotators' background knowledge and individual writing preferences. Moreover, figure captions, serving as context-level interpretations of figures, lack rigid writing standards. Authors may create captions of diverse quality levels, spanning from concise introductions to those neglecting the interpretation of visual content. For future work, we aim to expand our pool of annotators and conduct preliminary studies to select high-quality figure captions that provide detailed in-context and visual interpretations for human evaluation and generation tasks.

We performed pre-extraction of mention-paragraphs from papers and OCR tokens from figures, this preprocessing step represents another limitation of the paper. Our future work aims to develop methods to automatically reference knowledge from entire papers and perform automatic OCR token extractions.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

These research results were partly obtained from the commissioned research (No.~225) by National Institute of Information and Communications Technology (NICT), Japan, and partly obtained from the first author's internship research under NICT.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{11refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Zhishen Yang}{%
Zhishen Yang is currently a Ph.D. student at the School of Computing, Tokyo Institute of Technology, Japan. He received his master of engineering in artificial intelligence from the same institute in 2020 and his bachelor of computer science degree with first class honours (cooperative program) from Dalhousie University, Canada, in 2016. His research focuses on multimodal natural language processing.
}

\bioauthor[:]{Raj Dabre}{%
Raj Dabre (raj.dabre@nict.go.jp) is a senior researcher at the Advanced Translation Technology Laboratory, National Institute of Information and Communications Technology (NICT), Japan and an Adjunct Faculty at IIT Madras, India. He received his Ph.D. from Kyoto University and Masters from IIT Bombay. His primary interests are in low-resource NLP, language modeling and efficiency. He has published in ACL, EMNLP, NAACL, TMLR, AAAI, AACL, IJCNLP and CSUR. Additional information is available at \url{https://prajdabre.github.io/}.
}

\bioauthor[:]{Hideki Tanaka}{%
Hideki Tanaka is a Research Executive Director of the Advanced Translation Technology Laboratory of the National Institute of Information and Communications Technology (NICT) in Japan. Before NICT, he was with the Science and Technology Research Laboratories of NHK and Advanced Telecommunications Research Institute International (ATR). His research interest lies in Natural Language Processing and Machine Learning. He received a BS., MS., and Ph.~D. from Kyushu University in 1982, 1984, and 1995 respectively.
}

\bioauthor[:]{Naoaki Okazaki}{%
Naoaki Okazaki is a professor in School of Computing, Tokyo Institute of Technology, Japan. Prior to this faculty position, he worked as a post-doctoral researcher in University of Tokyo (2007--2011), and as an associate professor in Tohoku University (2011--2017). He is also a visiting research scholar of the Artificial Intelligence Research Center (AIRC), National Institute of Advanced Industrial Science and Technology (AIST). His research areas include Natural Language Processing (NLP), Artificial Intelligence (AI), and Machine Learning.
}


\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate



\end{document}

