\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{array}
\usepackage{hangcaption_jnlp}
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
\usepackage{bm}
%%%%\usepackage{here}
%%%%\usepackage{amsfonts}
%%%%\usepackage{graphicx}
%%%%\usepackage{tablefootnote}
%%%%\usepackage{booktabs}
%%%%\usepackage{enumitem}
%%%%\usepackage{booktabs}
\usepackage{paralist}
%%%%\usepackage{bm}
\usepackage{multirow}
%%%%\usepackage{color}
    \usepackage[dvipdfmx]{xcolor}
\definecolor{my-orange}{cmyk}{0,0.75,0.95,0}
\definecolor{my-blue}{cmyk}{1,0.45,0,0}
%%%%\usepackage[subrefformat=parens]{subcaption}
%%%%\usepackage{mathtools}
%%%%\renewcommand{\thefootnote}{\fnsymbol{footnote}}
%%%%\newcommand{\myfig}[2][width=\linewidth]{
%%%%  \vspace{-3ex}
%%%%  \begin{figure}[H]
%%%%    \centering
%%%%    \includegraphics[keepaspectratio, #1]{#2}
%%%%  \end{figure}%
%%%%  \vspace{-5ex}
%%%%}
%%%%\captionsetup[subfigure]{labelformat=simple}
%%%%\renewcommand{\thesubfigure}{(\alph{subfigure})}
\renewcommand{\theequation}{\alph{equation}}
\makeatletter
\def\EqNumText{\refstepcounter{equation}\tagform@\theequation}%
\makeatother
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{31}
\Number{4}
\Month{December}
\Year{2024}
%%%%\Issuetitle{「　　　　」特集号}

\received{2024}{4}{23}
\revised{2024}{7}{29}
\accepted{2024}{9}{11}

\setcounter{page}{1691}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{How Domain Adaptation of BERT Improves \\ Syntactic Parsing of Math Text}
\eauthor{Runa Yoshida\affiref{Author_1} \and Takuya Matsuzaki\affiref{Author_1}} 
\eabstract{%
This study clarifies how the domain adaptation of bidirectional encoder representations from transformers (BERT) contributes to the syntactic analysis of mathematical texts and their limitations.  Experimental results show that the domain adaptation of BERT is highly effective, even with a relatively small amount of raw in-domain data. This improves the accuracy of the syntactic dependency analysis by up to four points without any annotated in-domain data. By analyzing the improvement, we found that numerous errors involving the mathematical expressions have been corrected. Errors related to structures that are not frequent in the out-domain fine-tuning data were difficult to improve by only the domain adaptation of BERT. This study also revealed that the effectiveness of BERT depends on the representation of the mathematical expressions in the input. Among several different representations of mathematical expressions, the highest dependency accuracy was achieved using a simple method where an entire mathematical expression is replaced with a dedicated special token.}
\ekeywords{BERT, Domain Adaptation, Dependency Parsing}

\headauthor{Yoshida and Matsuzaki}
\headtitle{Domain Adaptation of BERT for Syntactic Parsing of Math Text}

\affilabel{Author_1}{}{Tokyo University of Science}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

\renewcommand{\thefootnote}{}
\footnote[0]{This paper is a revised version of \cite{Yoshida2023MathBERT}, which has been submitted for Proceedings of the 29th Annual Conference of the Association for Natural Language Processing, 2023.}
\renewcommand{\thefootnote}{\arabic{footnote}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Large language models (LLMs), such as GPT-4 \cite{openai2023gpt4}, can be used in various natural language processing tasks.
However, there are some challenges, such as combating hallucination \cite{maynez-etal-2020-faithfulness} and handling logical inference tasks \cite{dziri2023faith}.
One promising approach is to combine a rigorous verification or inference method with the flexibility of LLMs by transforming natural language statements into logical expressions \cite{wu2023empirical}.
In this process (i.e., semantic parsing), a sentence with logically complex content naturally requires an accurate parser for processing. 
Hence, syntactic parsing remains crucial as an interface between LLMs and logical inference systems.

This implies that accurate syntactic parsing is particularly important in domains where logical inference is naturally required, such as mathematics and other sciences, laws, and technical documents.
However, creating a syntactically annotated corpus for each text genre is costly because it requires language analysis skills and a fair amount of domain knowledge.

Recent studies have shown that high-accuracy dependency parsing can be achieved using bidirectional encoder representations from transformers (BERT) \cite{devlin-etal-2019-bert,Shibata2019bert}.
Furthermore, the effectiveness of using domain-specific text for BERT pre-training has been demonstrated in syntactic analysis of biomedical texts \cite{BERT_DEP}, which provides a remedy for the cost-intensive nature of syntactic annotation.

However, it is not clear what specific improvements achieved by the domain adaptation of BERT.
By analyzing the improvements and remaining errors, this study clarifies the extent to which the syntax analysis of mathematical texts is improved without annotated in-domain data.
Specifically, we used pre-university mathematical questions in Japanese as test data.
We found that, by only the domain adaptation of BERT, many errors related to mathematical expressions were corrected.
However, it is difficult to improve the analysis of structures that are not frequently found in the dependency-annotated out-domain corpus. 

This study also explores how to treat mathematical expressions in the input to BERT.
It is unclear whether they warranted the same treatment as the main-language tokens, such as Japanese and English, since mathematical expressions have completely different vocabulary and grammar from the main language. 
Similar considerations should also be applied to other genre-specific sublanguages such as program code fragments, chemical formulas, and board game records such as chess notation.

Therefore, we compared different methods for representing mathematical expressions in the input to BERT.
As a result, a simple method that replaced the entire mathematical expression with a dedicated special token outperformed a more complex method that considers the components of mathematical expressions. 
We examined the embeddings of the mathematical expressions produced by these two methods and investigated why the simpler method performed better for parsing.

The rest of this paper is organized as follows. 
First, we describe the representation of mathematical expressions (\S \ref{sec:Represention}), basic characteristics of Japanese dependency structures, and the parsing model (\S \ref{sec:Dependency-Parsing}). 
We then present the experimental setting (\S\ref{sec:Experiments}) and results (\S\ref{sec:Results}). 
In \S \ref{sec:Analysis-Errors}, we analyze why some errors were not corrected by the domain adaptation of BERT and require training with dependency-annotated mathematical texts.
In \S \ref{sec:analysis_Differences_math_expression}, we compare the embeddings produced by different representations of the mathematical expressions. 
Finally, we conclude the paper in \S \ref{sec:Conclusion}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Representation of Mathematical Expressions}
\label{sec:Represention}

We treated each of the following as a mathematical expression: 
compound expressions, such as $\int_{0}^{1} f(x) dx = 3$, compound names such as $\triangle ABC$, and single variable $x$.

There are two straightforward approaches to representing mathematical expressions: 
\begin{inparaenum}[(I)]
\item representing them with a single special token and 
\item representing them as a sequence of symbols.
\end{inparaenum}
Approach (I) has the advantage of minimizing the number of tokens regardless of the complexity of the mathematical expressions in the input.
In approach (II), we can learn the embeddings of individual mathematical symbols by adding them to the vocabulary of BERT and may be able to capture the compositional nature of the mathematical expressions.
Additionally, we can combine the advantages of (I) and (II) using an encoder for mathematical expressions.

Therefore, we compared the following four methods:
\begin{description}
\item[\texttt{\lbrack UNK\rbrack}:]An entire mathematical expression is represented with a single unknown token \texttt{[UNK]}.
\item[\texttt{\lbrack MATH\rbrack}:]An entire mathematical expression is represented with a dedicated special token \texttt{[MATH]}.
\item[Mixed:]The vocabulary of BERT is extended with the mathematical symbols (henceforth referred to as \textit{math tokens}).
The input is a sequence of the main language and math tokens.
\item[ExprBERT:]BERT is stacked on top of another BERT model (ExprBERT) specialized for mathematical expressions.
That is, a mathematical expression is first fed into ExprBERT as a sequence of math tokens, and its \texttt{[CLS]} vector is input into BERT 
in the second layer, along with the tokens of the main language. The structure of ExprBERT is identical to that of BERT-base.
\end{description} 

%%%%Figures \ref{fig:Mixed} and \ref{fig:ExprBERT} illustrate the input into a model using the Mixed and ExprBERT method, respectively.
    Figures \ref{figure01}(a) and \ref{figure01}(b) illustrate the input into a model using the Mixed and ExprBERT method, respectively.
Table \ref{tab:math4} shows examples of the inputs using the four representation methods.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[t]
\begin{center}
\includegraphics{31-4ia10f1.pdf}
\end{center}
%%%% (a) \label{fig:Mixed}
%%%% (b) \label{fig:ExprBERT}
\caption{Input into BERT using the Mixed and ExprBERT method.}
\label{figure01}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[t]
\input{10table01.tex}%
\hangcaption{\label{tab:math4}Examples of the input with the four methods of mathematical expression representation for ``$f(x)=0$ has solution $x=1$.'' On the last row, $\mathbf{e}_1$ and $\mathbf{e}_2$ represent the \texttt{[CLS]} vectors of ExprBERT obtained for ``$f(x)=0$'' and ``$x=1$.''}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In the Mixed and ExprBERT methods, 
a mathematical expression is tokenized into a sequence of math tokens. 
The vocabulary of math tokens is comprised of symbols similar to \LaTeX, including alphabetic characters, 
operators, superscripts, and subscript symbols.
For example, the expression 
``$a_{n}=n^{2}+100n$'' 
is represented as 
\begin{center}
$a\text{ }$ \_$\lbrace\text{ }$ $n\text{ }$ $\rbrace\text{ }$ $=\text{ }$ $n\text{ }$ $^{\wedge}\lbrace\text{ }$ $2\text{ }$ $\rbrace\text{ }$ $+\text{ }$ $100\text{ }$ $n$
\end{center}
(i.e., a sequence of 12 math tokens). 

The vocabulary of the math tokens was selected based on their frequency in the mathematical texts used for the additional pretraining of BERT 
with a threshold of five or more. 
The vocabulary size was 241. 
When a math token has the same string as the main language token, a different token ID is assigned to the math token.

In the domain adaptation of BERT using masked language modeling (MLM), we randomly selected 15\% of the (main language) tokens and mathematical expressions for prediction.
We replace 80\% of the selected tokens with the \texttt{\lbrack MASK\rbrack} token, 10\% of them using a random token and maintained 10\% of them unchanged.
When performing MLM using the \texttt{\lbrack MATH\rbrack} or the Mixed methods, we did not distinguish between math and main language tokens during replacing tokens.
In doing MLM with BERT stacked on top of ExprBERT, when we mask a mathematical expression, the task was defined as selecting a vector corresponding to the masked mathematical expression from the word embeddings of the BERT vocabulary and \texttt{[CLS]} vectors of ExprBERT for all mathematical expressions in a mini-batch. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Dependency Parsing using BERT}
\label{sec:Dependency-Parsing}

In this section, 
we describe the basic characteristics of Japanese dependency structures, and the dependency-parsing model that we used. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Japanese Dependency Structure}

In Japanese, a bunsetsu chunk (henceforth, referred to as a \textit{chunk}) is a syntactic unit that consists of one or more content words followed by zero or more function words.
The dependency between chunks in Japanese is always from a preceding chunk to a following chunk.
Non-projective dependency structures are very rare in Japanese.
Dependency parsing was performed at the chunk level.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Parsing Model}

To perform dependency parsing at the chunk level we combine \citeA{dozat2017deep}'s biaffine model with BERT.
Let $\bm{f}_{i}$ and $\bm{h}_{j}$ be the output vectors of BERT for the initial subtokens of the $i$-th chunk's word form morpheme and $j$-th chunk's head morpheme, respectively.
We followed the definitions of word form and head morphemes by \citeA{kudo2002conll}.
The dependency score of $i$-th chunk depending on $j$-th chunk is calculated as follows: 
\begin{equation}
\textrm{score}(i,\ j) = \bm{f}_{i}^{T}\textbf{A}\bm{h}_{j} + \bm{w}^{T}\bm{h}_{j} \label{formula:score} \tag{1} 
\end{equation}
where $\textbf{A}$ and $\bm{w}$ are the parameters introduced in the fine-tuning phase for the dependency prediction.
We used the model by Dozat and Manning only as the scoring model and used a greedy algorithm for dependency parsing; that is, for each chunk $c$, we chose the highest-scored chunk that follows $c$ as the head of $c$.
We assume that the dependencies between chunks are always from the preceding chunk to the following chunk: the predicted dependency structure is a tree even when it includes crossed dependencies.
The SoftMax cross-entropy was used as the loss function. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}
\label{sec:Experiments}

In order to investigate the effectiveness and the limitations of the domain adaptation of BERT and determine the most suitable representation of mathematical expressions, 
we created a total of 10 analysis models and conducted experiments as shown in Table \ref{tab:result}.
They differ in 
\begin{inparaenum}[(i)]
\item whether or not additional pre-training was performed using mathematical text,
\item which of the four methods of representing mathematical expressions was used, and
\item whether or not a dependency-annotated in-domain data was used for fine-tuning dependency prediction.
\end{inparaenum}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Datasets}

We used entrance exam mathematical questions of Japanese universities from 1957 to 2020 \cite{math_problem_corpus} (henceforth referred to as MathRaw) for the domain adaptation of BERT. 
We used two dependency-annotated corpora.
One was the Kyoto University Text Corpus Version 4.0 \cite{text_corpus} (henceforth referred to as KTC).
Its text was taken from newspapers.
It was used as the out-domain training data for the BERT-based dependency parsers.
The other was a mathematical problem text corpus with dependency annotations.
In consists of the National Center Test mathematical questions from 1997 to 2011 and the entrance exam mathematical questions of seven Japanese national universities from 1994 to 2014 (henceforth referred to as MathDep).
MathDep follows the same syntactic annotation standards as KTC.
In MathRaw and MathDep, the mathematical expressions are encoded using Mathematical Markup Language (MathML).
We treat all the elements of the \texttt{<math>} tag in MathML, and only them, as comprising mathematical expressions. 
When a math token had the same string as a main language token, a different token ID was assigned to the math token.
For example, ``$A$'' appearing in a sentence such as ``The suspect $A$ was ...'' is a main language token, while 
``$A$'' appearing in a sentence such as ``The vertex \texttt{<math>} $A$ \texttt{<\textbackslash math>} was ...'' is a math token. 
Tokenization of the mathematical expressions is done based on the MathML annotations. 

We used MathDep as the in-domain training data for the parsers and also as the evaluation data (by cross-validation).
Table \ref{tab:dataSize} shows the size of these datasets. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{10table02.tex}%
\hangcaption{\label{tab:dataSize}Statistics of the datasets. 
``\%Exprs.'' is the proportion of mathematical expressions to all tokens when each expression is counted as a single token.}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Additional Pre-training}
\label{sec:preTrain}

We used a Japanese BERT-base model released by Tohoku University\footnote{\url{https://github.com/cl-tohoku/bert-japanese}} (Tohoku-BERT) in all dependency parsing methods.
It has been pre-trained on Japanese Wikipedia consisting of approximately 17M sentences. 
Mathematical expressions are treated as strings in Tohoku-BERT, just like other words. 
Using Tohoku-BERT as the initial values, 
we performed MLM using MathRaw for the domain adaptation. 
Specifically, we prepared three domain-adapted models, using \texttt{\lbrack MATH\rbrack}, Mixed, and ExprBERT for mathematical expression representation, respectively.
In the additional pre-training on MLM, 
we set the maximum number of epochs to 50 and used the parameters after the epoch where the loss on the validation set was minimized.
The mini-batch size was set to 16, 
and AdamW \cite{loshchilov2018decoupled} was used as the optimizer with a learning rate of 5 $\times$ 10$^{-5}$.
We input MathRaw by an entire problem to BERT and set the maximum length of the input token sequence to 256.
The maximum length of the math token sequence for a mathematical expression was set to 36 in Mixed representation and the input to ExprBERT.
For other hyperparameters, we followed the setting of the MLM task by \citeA{devlin-etal-2019-bert}.
ExprBERT was jointly trained with BERT in doing MLM. 
In the joint training, we masked the input to the BERT stacked on the ExprBERT; i.e., we masked the main language tokens and the outputs from the ExprBERT but not a part of a mathematical expression.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Fine-tuning for Dependency Parsing}
\label{sec:fine-tuning}

When only KTC was used as the dependency training data, 
90\% of it was used as the training data,
and the remaining 10\% was used as the validation data.
When both KTC and MathDep were used as the dependency training data,
the entire KTC and 90\% of MathDep were used as the training data, and the remaining 10\% of MathDep was used as the validation data.  

All evaluation was done on MathDep.
When MathDep was used also as the dependency training data, 
the evaluation was performed using five-fold cross-validation.

We didn't use parsing algorithms such as CKY \cite{sakai1961syntax}, i.e., 
we evaluated the accuracy of the head prediction according to the dependency score in Eq. (\ref{formula:score}).
This is because we found that the predictions very rarely resulted in a cross-dependency even without the projectivity restriction imposed by the parsing algorithm (See ``Crossed deps.'' in Table \ref{tab:result}).

In the fine-tuning for dependency prediction, 
we used the same hyperparameter settings as the additional pre-training, 
except that the maximum number of epochs was set to five.
The parameters of ExprBERT, which were trained through the domain adaptation of BERT using MLM, were frozen during the fine-tuning.
\footnote{When the parameters of ExprBERT were not frozen, the accuracy decreased in all models.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{10table03.tex}%
\hangcaption{\label{tab:result} Evaluation results for different combinations of pre-training data, dependency training data, and representation of mathematical expressions. There were 6,875 instances of MATH→MATH, 15,619 instances of MATH→WORD, 9,593 instances of WORD→MATH, and 26,008 instances of WORD→WORD.}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Results}
\label{sec:Results}

Table \ref{tab:result} presents the evaluation results for the 10 configurations. 
In configurations \eqref{M-E}\eqref{M-MIX}\eqref{M-MATH} and \eqref{MME}\eqref{MMMIX}\eqref{MMMATH}, 
we used the same mathematical expression representation in both BERT adaptation and in-domain fine-tuning.
In addition to the accuracy, 
the table provides the $\textrm{F}_{1}$-score for different types of gold-standard dependent-head pairs that are classified according to whether the dependent and the head include a mathematical expression (coded as ``MATH'') or not (``WORD'').
For instance, the row labeled ``MATH→WORD'' provides the $\textrm{F}_{1}$-score for the pairs in which the dependent includes a mathematical expression and the head does not.
In Table \ref{tab:result}, ``accuracy'' means unlabeled attachment score (UAS) and ``Crossed dep. (\%sent.)'' refers to the percentage of sentences including crossing dependency arcs. 
``Crossed dep. (\%arc)'' refers to the percentage of crossing dependency arcs. 
We measured the accuracy by combining all the predictions made in the five-fold cross-validation.

It can be seen that the accuracy was improved by the additional pre-training of BERT despite that the mathematical texts used for it was far smaller (less than 1\slash 100) than the pre-training data.
Although it improved the accuracy whether or not we do fine-tuning with in-domain dependency annotation data, 
it was especially effective when the in-domain dependency annotation data (i.e., MathDep) was not used for training.

In particular, when mathematical expressions were replaced by \texttt{[MATH]}, 
the accuracy was improved by about four points, 
and the $\textrm{F}_{1}$-score for MATH→MATH dependency type was improved by about 11 points compared to the result obtained with no additional pre-training on in-domain data (i.e., between \eqref{--U} and \eqref{M-MATH} in Table \ref{tab:result}).
However, 
there still remained a difference of about three points in accuracy between the models trained on in-domain dependency annotation data and those not trained on it (e.g., between \eqref{M-MATH} and \eqref{-MMATH} in Table \ref{tab:result}).
We will investigate the reason for this in \S\ref{sec:Analysis-Errors}.

Among the three methods of representing mathematical expressions, 
the \texttt{[MATH]} method produced the best results followed by Mixed and ExprBERT.
The most significant difference among them appeared when the in-domain dependency data was not used for training (\eqref{M-E}, \eqref{M-MIX} and \eqref{M-MATH}).
We will investigate the reason for this difference in \S\ref{sec:analysis_Differences_math_expression}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Analysis of Errors Corrected by Domain Adaptation and by Training with Annotated Mathematical Texts}
\label{sec:Analysis-Errors}

To examine the effect of the domain adaptation of BERT and its limitations, 
we analyzed the errors that were corrected by the domain adaptation of BERT only (\S \ref{sec:analysis_B}) and the errors that were corrected only after the training with dependency-annotated mathematical texts (\S \ref{sec:analysis_D}).

Table \ref{tab:prediction-by} shows the four groups of errors analyzed in this section.
We analyzed 100 errors of each category in Table \ref{tab:prediction-by}, 
which were sampled from one of the five-fold of cross-validations on MathDep.
In Table \ref{tab:prediction-by}, ``Out-domain only \eqref{--U}'', 
``+ BERT adapt. \eqref{M-MATH}'', and ``+ In-domain Dep. train. \eqref{MMMATH}'' respectively refer to the first, the fourth, and the sixth model in Table \ref{tab:result}.
We shall see the effect of the domain adaptation by examining the examples that were improved or deteriorated after it (the first and second rows of Table \ref{tab:prediction-by}).
Similarly, we will clarify the effect of the annotated in-domain data (= limitation of domain adaptation) by examining the errors in the third and fourth rows of Table \ref{tab:prediction-by}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{10table04.tex}%
\caption{\label{tab:prediction-by} The four groups of errors analyzed in \S \ref{sec:Analysis-Errors}.}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In this section's figures, 
black and \textcolor{my-blue}{blue} arrows stand for the correct dependencies, 
and \textcolor{my-orange}{orange} arrows stand for the incorrect predicted dependencies.
A slash ( \slash\ ) signifies the boundary of a chunk.
In this section's tables, 
we show only the error types that account for more than 5\% of the errors and differ in the number of the errors by more than 10 between the two configurations.
A more extensive classification list is provided in the Appendix \ref{sec:LIST}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Effect of the Domain Adaptation}
\label{sec:analysis_B}

We classify the errors corrected by the domain adaptation of BERT in Table \ref{tab:analysis_B_g}. 

The error type ``appositional relation with coordinated math expressions'' is an error where a noun denoting the type of coordinated mathematical expressions does not depend on the last conjunct, which is the correct head according to the annotation guideline.

%%%%%%%%%%%%%%%%%%%%
\vspace{1\Cvs}
\begin{center}
\includegraphics{31-4ia10f1_1.pdf}
\end{center}
\vspace{1\Cvs}
%%%%%%%%%%%%%%%%%%%%

The error type ``coordinated math expressions'' is an error where an element in a comma-separated list of mathematical expressions does not depend on the next element, which is the correct head.

%%%%%%%%%%%%%%%%%%%%
\vspace{1\Cvs}
\begin{center}
\includegraphics{31-4ia10f1_2.pdf}
\end{center}
\vspace{1\Cvs}
%%%%%%%%%%%%%%%%%%%%

The error type ``imperative form without nominative argument'' in Table \ref{tab:analysis_B_g} is an error where a noun with the nominative case marker (or one without a case marker) depends on an imperative verb.
In particular, 
there were several instances of errors where a part of coordinated mathematical expressions depends on an imperative verb.

%%%%%%%%%%%%%%%%%%%%
\vspace{1\Cvs}
\begin{center}
\includegraphics{31-4ia10f1_3.pdf}
\end{center}
\vspace{1\Cvs}
%%%%%%%%%%%%%%%%%%%%

While the domain adaptation of BERT corrected 717 errors, 
there were 266 errors made only after the domain adaptation. 
We classify them in Table \ref{tab:analysis_B_b}.
A quarter of those errors were due to annotation errors (i.e., the predictions were correct).
The number of errors deteriorated after the domain adaptation is less than half the number of improved.
Therefore, we may say that the errors related to a coordination of mathematical expressions have improved through the domain adaptation.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5 and 6
\begin{table}[t]
\setlength{\captionwidth}{184pt}
\begin{minipage}[t]{200pt}
\input{10table05.tex}%
\hangcaption{Classification of errors corrected by the domain adaptation of BERT.}
\label{tab:analysis_B_g}
\end{minipage}
\hfill
\raisebox{6.5pt}[0pt][0pt]{%
\begin{minipage}[t]{200pt}
\input{10table06.tex}%
\hangcaption{Classification of errors caused by the domain adaptation of BERT.}
\label{tab:analysis_B_b}
\end{minipage}
}%
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Discussion and Further Analysis}

The analysis revealed that many errors related to a coordination of mathematical expressions were corrected by the domain adaptation of BERT. 
Here is an explanation for this.

First, MLM training with the special token \texttt{[MATH]} allows BERT to learn that mathematical expressions are likely to appear in close proximity to each other. 
Then, the dependency prediction training with KTC allows the model to learn that similar tokens in close proximity are likely to be in a coordination, 
making it easier to correctly analyze a structure including a coordination of mathematical expressions.

On the other hand, 
when we pre-trained and fine-tuned BERT only with out-domain data and replaced the mathematical expressions in the test data with \texttt{[UNK]}, the model often failed to analyze a coordination of mathematical expressions represented as adjacent \texttt{[UNK]} tokens. 
This is because the \texttt{[UNK]} tokens are rather scattered in the pre-training data of BERT (Japanese Wikipedia), 
and hence there is no way for the model to learn that mathematical expressions often appear close to each other. 
In fact, 
in 98\% of the paragraphs of Japanese Wikipedia, 
the \texttt{[UNK]} token appears at most once and even in the remaining 2\% of the paragraphs, 
there are three or more tokens between two \texttt{[UNK]} tokens in 81\% of the paragraphs (see Appendix \ref{sec:How-[UNK]-Tokens-Appear-in-Japanese-Wikipedia} for the details).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Effect of Fine-tuning with In-domain Dependency-annotated Data}
\label{sec:analysis_D}

We analyzed the errors corrected only after the head prediction training with dependency-annotated mathematical text.
Table \ref{tab:analysis_D_g} shows the results of error classification.
The error type ``coordinated math expressions'' and ``appositional relation with coordinated math expressions'' in Table \ref{tab:analysis_D_g} are the same kind of errors as explained in \S\ref{sec:analysis_B}.

The error type ``definition by a small clause'' is an error in the analysis of the construction ``NP$_1$ wa NP$_2$ to suru'' (Let NP$_1$ be NP$_2$), 
which is very frequent in mathematical texts.
In these errors, 
the first NP with the topic marker ``wa'' wrongly depends on the main verb ``suru'' (let) instead of its nominal predicate NP$_2$ in the small clause ``NP$_1$ wa NP$_2$'' (NP$_1$ is NP$_2$).
The wrong output structure roughly corresponds to an incorrect interpretation ``NP$_1$ lets (something) be NP$_2$.''

%%%%%%%%%%%%%%%%%%%%
\vspace{1\Cvs}
\begin{center}
\includegraphics{31-4ia10f1_4.pdf}
\end{center}
\vspace{1\Cvs}
%%%%%%%%%%%%%%%%%%%%

The error type ``condition modifying an imperative verb'' in Table \ref{tab:analysis_D_g} is an error where a condition expressed by a pronoun depends on an imperative verb.
The wrong output structure corresponds to an incorrect interpretation 
``Only if the condition is met, then do something (e.g., prove a statement).''

%%%%%%%%%%%%%%%%%%%%
\vspace{1\Cvs}
\begin{center}
\includegraphics{31-4ia10f1_5.pdf}
\end{center}
\vspace{1\Cvs}
%%%%%%%%%%%%%%%%%%%%

The error type ``argument cluster coordination'' is the analysis errors on the structures such as ``I gave John an apple and Mary an orange'' and ``Let $A$ be a positive number and $B$ a negative number.''
In the example below, the first argument cluster ``$A$ o sei'' ($A$-ACC positive) wrongly depends on the main verb ``suru'' (let) while the annotation guideline defines that the correct head is the second cluster (or a small clause) ``$B$ o fu'' ($B$-ACC negative).

%%%%%%%%%%%%%%%%%%%%
\vspace{1\Cvs}
\begin{center}
\includegraphics{31-4ia10f1_6.pdf}
\end{center}
\vspace{1\Cvs}
%%%%%%%%%%%%%%%%%%%%

Table \ref{tab:analysis_D_b} classifies the errors made only by training on the dependency-annotated mathematical text. 
While training on the dependency-annotated mathematical text corrected 296 errors, 
there were 120 errors made only after it. 
Therefore, we may say the errors classified in Table \ref{tab:analysis_D_g} were improved through training on the annotated mathematical text.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7 and 8
\begin{table}[t]
\setlength{\captionwidth}{184pt}
\begin{minipage}[t]{200pt}
\input{10table07.tex}%
\hangcaption{Classification of errors corrected by training with annotated \mbox{mathematical} texts.}
\label{tab:analysis_D_g}
\end{minipage}
\hfill
\raisebox{19.5pt}[0pt][0pt]{%
\begin{minipage}[t]{200pt}
\input{10table08.tex}%
\hangcaption{Classification of errors made only after training with annotated \mbox{mathematical} texts.}
\label{tab:analysis_D_b}
\end{minipage}
}%
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Discussion and Further Analysis}

The errors corrected by the training with dependency-annotated 
mathematical texts include not only structures including a coordination of mathematical expressions, 
but also errors related to
\begin{inparaenum}[(1)]
\item a small clause,
\item a condition modifying an imperative verb, and 
\item an argument cluster coordination.
\end{inparaenum}
This can be explained as follows.

The erroneous structures predicted for the case (1) and (2) were grammatically acceptable but semantically wrong.
These errors were most provably made because the sentence patterns (1) and (2) are less common in newspapers, 
and even when similar sentence patterns occur, 
they often have structures that differ from the correct structures found in mathematical texts.
Regarding the pattern (1), in KTC, 
there are only four sentences that include the construction ``NP$_1$ wa NP$_2$ to suru'', 
while in MathDep, there are 284 such sentences.
As for the pattern (2),
in KTC, there are 31 sentences that include the condition marker ``toki'' and one or more verbs, 
but none of them includes an imperative verb.
Among them, in 24 sentences, ``toki'' depends on the main verb at the end of the sentence.
On the other hand, in MathDep, 
there are 158 sentences that include ``toki'' and an imperative verb, 
and in none of these ``toki'' depends on the imperative verb.

As for the argument cluster coordinations, 
we found that the sentences including that structure accounted for less than 1\% of the sentences in KTC while it is very frequent in mathematical texts.

Therefore, 
these errors were supposedly difficult to improve through dependency training using only KTC while the training with dependency-annotated mathematical texts improved many errors related to those structures that are far less frequent in KTC.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Comparison of Corrected Errors on Coordinations}
\label{sec:analysis_parallelism}

Many errors on coordinations were corrected both by the domain adaptation of BERT (\S \ref{sec:analysis_B}) and by the training on the in-domain dependency data (\S \ref{sec:analysis_D}). 
Why some of them were improved by the domain adaptation only, and others were not?

We compared the errors related to coordinated mathematical expressions that were corrected 
\begin{inparaenum}[(i)]
\item by the domain adaptation of BERT and 
\item only by the training with in-domain dependency data.
\end{inparaenum}
Table \ref{tab:para} shows the number of conjuncts in the corrected errors of type (i) and (ii).
For instance, ``$x$, $y$'' includes two conjuncts and 
``$a_{n}$, $a_{n+1}$, $a_{n+2}$'' includes three. 

As Table \ref{tab:para} shows,
the domain adaptation improved many errors in coordination analysis but most of them include only two conjuncts.
In contrast,
the in-domain dependency data improved the analysis of coordinations including three or more conjuncts, in addition to those with two conjuncts.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[t]
\input{10table09.tex}%
\caption{\label{tab:para}Number of conjuncts in a coordinated mathematical expressions.}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[t]
\input{10table10.tex}%
\caption{\label{tab:heirestu}Number of conjuncts in a coordination.}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table \ref{tab:heirestu} shows the distribution of the number of conjuncts in a comma-separated list of noun phrases in KTC and MathDep. 
It shows that coordination structures of this type are less frequent in newspaper text than in mathematical text and also the number of conjuncts tends to be small in newspaper text.

For a noun coordination including three (or more) conjuncts,
two structures are equally conceivable:

%%%%%%%%%%%%%%%%%%%%
\vspace{0.5\Cvs}
\begin{center}
\includegraphics{31-4ia10f2_1.pdf}
\end{center}
\vspace{0.5\Cvs}
%%%%%%%%%%%%%%%%%%%%

\noindent
The choice of one of them as the gold-standard is a rather arbitrary decision though it is necessary for fixing a guideline.
For this reason, 
correcting errors related to coordinations with many conjuncts is challenging without a sufficient amount of annotation data.
It explains why the errors on coordinations with many conjuncts were not significantly improved with only MLM training on MathRaw and dependency training on KTC.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Differences in Mathematical Expression Embeddings}
\label{sec:analysis_Differences_math_expression}

Why did the \texttt{[MATH]} method outperformed ExprBERT method, 
even though it doesn't take into account the individual components of the mathematical expressions?
To clarify it, we examined the embeddings of mathematical expressions produced by ExprBERT method and \texttt{[MATH]} method. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[t]
\begin{center}
\includegraphics{31-4ia10f2.pdf}
\end{center}
%%%%\label{fig:ve}
%%%%\label{fig:vm}
\caption{The output vector for the mathematical expression by ExprBERT and  \texttt{\lbrack MATH\rbrack} method.}
\label{figure02}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We input a sentence including a mathematical expression to the model pre-trained with Wikipedia and MathRaw and fine-tuned on KTC, 
using either ExprBERT or \texttt{[MATH]} method.
We call the input sentence a \textit{query}.
We denote the output vector for the mathematical expression by ExprBERT method as $v_{E}$ and that by \texttt{[MATH]} method as $v_{M}$.
Note that $v_{E}$ is not the \texttt{[CLS]} vector of the mathematical expression encoder but the final output of the BERT that receives the output of ExprBERT.
It is thus affected by the context of the expression. 
We then searched for the mathematical expressions in MathRaw that have the output vector by ExprBERT (resp. by \texttt{[MATH]}) closest to $v_{E}$ (resp. to $v_{M}$).
%%%%The \textcolor{my-blue}{blue} arrows in Figure \ref{fig:ve} and Figure \ref{fig:vm} illustrate 
    The \textcolor{my-blue}{blue} arrows in Figure \ref{figure02}(a) and Figure \ref{figure02}(b) illustrate 
the output vector for the mathematical expression by the ExprBERT and \texttt{[MATH]} method.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 11
\begin{table}[t]
\input{10table11.tex}%
\hangcaption{Mathematical expressions with similar embedding vector to ``$A^{3}=E$'' in ``$A^{3}=E$\ を みたす 上三角行列 $A$を すべて 求めよ.'' (``Find all upper triangular matrices $A$ that satisfy $A^{3}=E$.'') by ExprBERT method.}
\label{fig:A3EifExprOut}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[t]
\input{10table12.tex}%
\hangcaption{Mathematical expressions with similar embedding vector to ``$A^{3}=E$'' in ``$A^{3}=E$\ を みたす 上三角行列 $A$を すべて 求めよ.'' (``Find all upper triangular matrices $A$ that satisfy $A^{3}=E$.'') by \texttt{[MATH]} method.}
\label{fig:A3EifMATHOut}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Tables \ref{fig:A3EifExprOut} and \ref{fig:A3EifMATHOut} show the results for the query ``$A^{3}=E$\ をみたす上三角行列$A$をすべて求めよ.''
This query means ``Find all upper triangular matrices $A$ that satisfy $A^{3}=E$.''
We provide the results in Japanese along with their English translations.
Table \ref{fig:A3EifExprOut} shows that, by ExprBERT method, 
``$A^{3}=E$'' in the query is closest to the expressions in the form ``$A^{k}=\pm E$'' regardless of their context.
In contrast, Table \ref{fig:A3EifMATHOut} shows, 
by \texttt{[MATH]} method, 
``$A^{3}=E$'' in the query is closest to the expression \textit{expr}$_{2}$ in the pattern of 
\pagebreak
\[
\text{Find $\lbrace$NP$_{\text{(appositive)}}$$\rbrace$ $\lbrace$\textit{expr}$_{1}\rbrace$ that satisfies \underline{$\lbrace$\textit{expr}$_{2}\rbrace$}}
\]
regardless of the content of \textit{expr}$_{2}$.
This is because the embeddings produced by \texttt{[MATH]} are determined solely by the context, 
regardless of the components of the mathematical expressions.
Table \ref{fig:A3EifMATHOut} reveals that the similarity of the mathematical expression embeddings by \texttt{[MATH]} is affected by not only a few surrounding words, 
but also the syntactic structure of a sentence as a whole.

We confirmed that a similar observation holds for a number of queries including a mathematical expression such as:
coordinates (e.g., $(x, y)$), 
variables (e.g., $x$), 
sequences (e.g., $a_{n}$), 
equations of circles (e.g., $x^{2}+y^{2} = r^{2}$), 
linear equations (e.g., $y=ax+b$), and so on. 
See Appendix \ref{sec:Instance-of-Differences-in-Math-Expression-Embeddings} for another example.

This different nature of the embeddings explains a typical dependency error made by ExprBERT but not by \texttt{[MATH]}.
In the example below, 
the correct conjuncts in ``hen AD to hen BC'' (side AD and BC) are ``hen AD (to)'' (side AD) and ``hen BC'' (side BC) but the chunk ``AD to'' (AD and) wrongly depends on the appositive noun ``hen'' (side).

%%%%%%%%%%%%%%%%%%%%
\vspace{0.5\Cvs}
\begin{center}
\includegraphics{31-4ia10f2_2.pdf}
\end{center}
\vspace{0.5\Cvs}
%%%%%%%%%%%%%%%%%%%%

\noindent
To parse coordinated mathematical expressions with appositive nouns,
the context and the syntactic similarity are crucial, 
rather than the content of the mathematical expressions.
This is because the components of the coordinated expressions are not necessarily similar to each other.
It explains why the \texttt{[MATH]} representation was better in analyzing such structures than ExprBERT.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Limitations of Replacing Mathematical Expressions with a Single Token}

There are some sentences where even humans cannot determine the dependency structure without considering the components of the mathematical expressions.
For instance, consider the following sentence:

%%%%%%%%%%%%%%%%%%%%
\vspace{0.5\Cvs}
\begin{center}
\includegraphics{31-4ia10f2_3.pdf}
\end{center}
\vspace{0.5\Cvs}
%%%%%%%%%%%%%%%%%%%%

\noindent
We cannot choose the head of {\it chokusen} (straight line) between the second and the third \texttt{\lbrack MATH\rbrack} without knowing the mathematical expressions.
If the sentence is ``直線$l_{1}, l_{2}$および $C$で囲まれた図形の面積は...'', we can predict that $C$ is not a straight line and {\it chokusen} is dependent on $l_{2}$ (the second \texttt{\lbrack MATH\rbrack}).
If the sentence is ``直線$l_{1}, l_{2}$および $l_{3}$で囲まれた図形の面積は...'', we can predict that $l_{1}, l_{2}$ and $l_{3}$ are straight line and {\it chokusen} is dependent on $l_{3}$ (the third \texttt{\lbrack MATH\rbrack}).
We in fact observed that, on some of such sentences, a correct analysis was obtained with ExprBERT but not with \texttt{\lbrack MATH\rbrack}.

However, such sentences were extremely rare. 
It is presumably because a mathematical text is written with special care so that it can be read in a unique way.
A phrase like the one above is often expressed as ``3直線 $l_{1}, l_{2}, l_{3}$ で囲まれた図形'' (the figure surrounded by the three lines $l_{1}, l_{2}$, and $ l_{3}$) or ``直線 $l_{1}, l_{2}$ および曲線 C で囲まれた図形'' (the figure surrounded by the straight lines $l_{1}, l_{2}$ and the curve $C$).
We suppose that it is a reason for the superiority of \texttt{\lbrack MATH\rbrack} method over ExprBERT method.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}

Several domain-specific large language models have been developed 
\cite<e.g., >{DBLP:journals/corr/abs-1904-05342,lee2020biobert,taylor2022galactica,wu2023bloomberggpt}. 
These models use huge target domain data for training. 
In contrast, our work highlights the potential for leveraging a limited amount of domain-specific data to improve the performance of a general-domain LLM through additional pre-training.

\citeA{joshi-etal-2018-extending} conducted domain adaptation to improve the performance of statistical parsers in domains they were not trained on.
While they focused on the adaptation of the parsers by training on in-domain annotated data, 
we investigated the effectiveness of domain adaptation through additional pre-training of BERT using only raw data.

Previous studies on mathematical expression embeddings mainly aimed to improve the performance of mathematical information retrieval (MIR) tasks.
\citeA{krstovski2018equation} applied word2vec \cite{DBLP:journals/corr/abs-1301-3781} and showed that a method that considers the composition of formulas achieved better performance in MIR tasks compared to another method that ignores the composition of formulas.
\citeA{DBLP:conf/clef/DadurePB21,Dadure2022embedding} showed that BERT-based formula embedding approach can capture the syntax and semantics of mathematical formulas when trained on more than 20M formulas.

However, for syntactic analysis, 
we have shown that representing a mathematical expression as a single token resulted in higher parsing accuracy using sentences including only about 46k mathematical expressions as training data.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
\label{sec:Conclusion}

The domain adaptation of BERT improved the parsing accuracy,
especially when an entire mathematical expression was replaced with a dedicated special token.
However, the parsing accuracy was still lower than the model trained on in-domain dependency-annotation data.

The domain adaptation of BERT was especially effective for correcting the errors involving mathematical expressions. 
It was however difficult to improve the accuracy on the structures that are infrequent in out-domain data used for fine-tuning. 
This indicates that newspaper texts do not provide enough exposure to the linguistic patterns that are characteristic to the mathematical domain and the domain adaptation of BERT did not fill the gap.

The method of replacing an entire mathematical expression with a dedicated special token produced an embedding that is highly influenced by the context compared to other methods.
In syntactic parsing, 
such embeddings were more beneficial compared to those that capture similarity based on the sequence of mathematical symbols.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

We acknowledge Mr. Yoshinori Hiromatsu for providing the mathematical problem texts and the member of ``Todai Robot Project'' for providing the mathematical problem text corpus with dependency annotations.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{10refs}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix

\section{Classification of Error}
\label{sec:LIST}

We show a more extensive classification list of the error analysis presented in \S \ref{sec:Analysis-Errors}. 
In this section's table, ``Others'' consists of various error types each of which accounts for less than 3\%.

We classify the errors corrected by the domain adaptation of BERT in Table \ref{tab:analysis_B_g_de}.
Table \ref{tab:analysis_B_g_de} is an extension of Table \ref{tab:analysis_B_g} in \S \ref{sec:analysis_B}.
We classify the errors made only after the domain adaptation of BERT in Table \ref{tab:analysis_B_b_de}.
Table \ref{tab:analysis_B_b_de} is an extension of Table \ref{tab:analysis_B_b} in \S \ref{sec:analysis_B}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 13 and 14
\begin{table}[h]
\setlength{\captionwidth}{178pt}
\begin{minipage}[t]{200pt}
\input{10table13.tex}%
\hangcaption{Classification of errors corrected by the domain adaptation of BERT.}
\label{tab:analysis_B_g_de}
\end{minipage}
\hfill
\raisebox{6.5pt}[0pt][0pt]{%
\begin{minipage}[t]{200pt}
\input{10table14.tex}%
\hangcaption{Classification of errors caused by the domain adaptation of BERT.}
\label{tab:analysis_B_b_de}
\end{minipage}
}%
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We classify the errors corrected by the training with dependency-annotated mathematical texts in Table \ref{tab:analysis_D_g_de}.
Table \ref{tab:analysis_D_g_de} is an extension of Table \ref{tab:analysis_D_g} in \S \ref{sec:analysis_D} .
We classify the errors made only after the training with dependency-annotated mathematical texts in Table \ref{tab:analysis_D_b_de}. 
Table \ref{tab:analysis_D_b_de} is an extension of Table \ref{tab:analysis_D_b} in \S \ref{sec:analysis_D}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 15 and 16
\begin{table}[t]
\setlength{\captionwidth}{188pt}
\begin{minipage}[t]{200pt}
\input{10table15.tex}%
\hangcaption{Classification of errors corrected by training with annotated \mbox{mathematical} texts.}
\label{tab:analysis_D_g_de}
\end{minipage}
\hfill
\raisebox{6.5pt}[0pt][0pt]{%
\begin{minipage}[t]{200pt}
\input{10table16.tex}%
\hangcaption{Classification of errors made only after training with annotated \mbox{mathematical} texts.}
\label{tab:analysis_D_b_de}
\end{minipage}
}%
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The error type  ``Ambiguous'' is an error where it is not possible to decide the correct dependency based solely on the input sentence.
For instance, consider the following sentence: 

%%%%%%%%%%%%%%%%%%%%
\vspace{0.5\Cvs}
\begin{center}
\includegraphics{31-4ia10f2_4.pdf}
\end{center}
\vspace{0.5\Cvs}
%%%%%%%%%%%%%%%%%%%%

This sentence can be interpreted in two different ways: either ``Assuming $a<1$, find the intersection of the circle and the straight line $f(x)$, both determined in (1).'' or ``Assuming $a<1$, find the intersection of the circle determined in (1) and the straight line $f(x)$.'' 
They differ in whether the straight line $f(x)$ was found in (1) or not.
If the straight line $f(x)$ is given in the problem statement (e.g, ``$f(x) = ax - 1$''),  ``求めた (determined)'' is dependent on ``the circle.''
If the straight line $f(x)$ is determined in (1), ``求めた'' is dependent on ``(the circle and) the straight line $f(x)$''.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{How \texttt{[UNK]} Tokens Appear in Japanese Wikipedia}
\label{sec:How-[UNK]-Tokens-Appear-in-Japanese-Wikipedia}

We conducted an investigation into the occurrence of \texttt{[UNK]} tokens in Japanese Wikipedia. 
We randomly sampled 10,000 articles from Japanese Wikipedia and analyzed the distribution of the number of \texttt{[UNK]} tokens in a paragraph. 
The results are presented in Table \ref{tab:UNK}.

According to Table \ref{tab:UNK}, 
in most paragraphs of Japanese Wikipedia, either no \texttt{[UNK]} tokens appear or only one appears.

We also investigated the distance between adjacent \texttt{[UNK]} tokens in a paragraph including two or more \texttt{[UNK]} tokens. 
Table \ref{tab:UNKtoken} shows the result.
According to Table \ref{tab:UNKtoken}, 
there are about 10\% of cases where the pattern `\texttt{[UNK][UNK]}' occurs. 
But in most cases, there are three or more tokens between the two \texttt{[UNK]} tokens. 
Therefore, the unknown token \texttt{[UNK]} is scattered throughout Japanese Wikipedia rather than occurring in continuous sequences.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 17
\begin{table}[t]
\input{10table17.tex}%
\caption{\label{tab:UNK}Distribution of the number of \texttt{[UNK]} tokens appearing in a paragraph of Japanese Wikipedia.}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 18
\begin{table}[h]
\input{10table18.tex}%
\caption{\label{tab:UNKtoken} The distance between adjacent \texttt{[UNK]} tokens.}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{More Instance of Differences in Mathematical Expression Embeddings}
\label{sec:Instance-of-Differences-in-Math-Expression-Embeddings}

Tables \ref{fig:x2y21ExprOut} and \ref{fig:x2y21MATHOut} show the mathematical expressions with embedding vectors similar to $v_{E}$ and $v_{M}$ for ``$x^{2}+y^{2}=1$'' in ``円$x^{2}+y^{2}=1$\ と放物線 $C$：$y=x^{2}+1$の両方に接する直線は$3$本ある.'' 
This query means ``There are three lines that are tangent to both the circle $x^{2}+y^{2}=1$ and the parabola $C:y=x^{2}+1$.''
Table \ref{fig:x2y21ExprOut} shows that, by ExprBERT method, 
``$x^{2}+y^{2}=1$'' in the query is closest to the expressions in the form ``$x^{2}±y^{2}=1$'' regardless of their context. 
In contrast, Table \ref{fig:x2y21MATHOut} shows, 
by \texttt{[MATH]} method, 
``$x^{2}+y^{2}=1$'' in the query is closest to the expression \textit{expr}$_{1}$ in the pattern of 
\[
\text{$\lbrace$NP$_{\text{(appositive)}}$$\rbrace$ \underline{$\lbrace$\textit{expr}$_{1}\rbrace$} and $\lbrace$NP$_{\text{(appositive)}}$$\rbrace$ $\lbrace$\textit{expr}$_{2}\rbrace$}
\]
regardless of the content of  \textit{expr}$_{1}$.
Thus, the similarity of the mathematical expression embeddings by \texttt{[MATH]} is affected by surrounding words.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 19
\begin{table}[t]
\input{10table19.tex}%
\hangcaption{\label{fig:x2y21ExprOut}Mathematical expressions with similar embedding vectors to 
``$x^{2}+y^{2}=1$'' in ``円 $x^{2}+y^{2}=1$\ と 放物線 $C$：$y=x^{2}+1$の 両方に 接する 直線は $3$本 ある.'' by ExprBERT method.}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 20
\begin{table}[t]
\input{10table20.tex}%
\hangcaption{\label{fig:x2y21MATHOut}Mathematical expressions with similar embedding vectors to ``$x^{2}+y^{2}=1$'' in ``円 $x^{2}+y^{2}=1$\ と 放物線 $C$：$y=x^{2}+1$の 両方に 接する 直線は $3$本 ある.'' by \texttt{[MATH]} method.}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\clearpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 
\begin{biography}

\bioauthor[:]{Runa Yoshida}{
Runa  Yoshida received the bachelor's degree in science from the Tokyo University of Science, Japan, in 2023.
She is currently pursuing a master's degree.
She is a member of ANLP and JSAI.
}

\bioauthor[:]{Takuya Matsuzaki}{
Takuya Matsuzaki received a Ph.D. in Information Science and Technology from the University of Tokyo, Japan, in 2007.
He is currently a professor at the Faculty of Science Division I, Tokyo University of Science.
Prior to that, he worked as an assistant professor at the University of Tokyo, as a project associate professor at the National Institute of Information Science, and as an associate professor at Nagoya University.
He is a member of ANLP, JSAI, and IPSJ.
}
\end{biography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate




\end{document}


