\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{array}
\usepackage{hangcaption_jnlp}
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage{algorithm}
%%%%\usepackage{algpseudocode}
%%%%\usepackage{amsmath,amssymb}
\usepackage{bm}
\usepackage{booktabs}
%%%%\usepackage{bussproofs}
%%%%\usepackage{color}
%%%%\usepackage[dvipdfmx]{graphicx}
%%%%\usepackage{comment}
\usepackage{enumerate}
%%%%\PassOptionsToPackage{hyphens}{url}
%%%%\usepackage{hyperref}
%%%%\usepackage{mathtools}
\usepackage{multirow}
\usepackage{physics}
%%%%\usepackage{setspace}
%%%%\usepackage{soul}
%%%%\usepackage{stmaryrd}
%%%%\usepackage{tikz}
%%%%\usetikzlibrary{arrows}
%%%%\usetikzlibrary{arrows.meta}
%%%%\usepackage[normalem]{ulem}
%%%%\usepackage{url}
%%%%\usepackage[hang,small,bf]{caption}
%%%%\usepackage[subrefformat=parens]{subcaption}
%%%%\usepackage{jnlpbbl_2.1}
%%self macro
\newcommand{\citet}[1]{\citeauthor{#1} \citeyear{#1}}
%%%%\newcommand{\citeu}[1]{\citeauthor{#1} \citeyear{#1}}
%%%%\newcommand{\todo}[1]{\textcolor{red}{#1}}
%%%%\newcommand{\ctext}[3][RGB]{%
%%%%  \begingroup
%%%%  \definecolor{hlcolor}{#1}{#2}\sethlcolor{hlcolor}%
%%%%  \hl{#3}%
%%%%  \endgroup
%%%%}
%%%%\newcommand{\highlight}[1]{\ctext[RGB]{179,228,228}{#1}}
%%%%\newcommand{\changed}[1]{\ctext[RGB]{179,228,179}{#1}}
%%%%\newcommand{\deleted}[1]{\ctext[RGB]{228,179,179}{#1}}
\newcommand{\hz}{\hphantom{0}}
\newcommand{\hzz}{\hphantom{00}}
\newcommand{\hzzc}{\hphantom{00,}}
% Functions
\newcommand{\fbilstmseq}{\textsf{biLSTM}_\textsf{seq}}
\newcommand{\fbilstmctx}{\textsf{biLSTM}_\textsf{ctx}}
\newcommand{\flstmf}{\textsf{LSTM}_\textsf{f}}
\newcommand{\flstmb}{\textsf{LSTM}_\textsf{b}}
\newcommand{\fshift}[1]{\textsc{Shift}[#1]}
\newcommand{\freduce}[1]{\textsc{Reduce}[#1]}
\newcommand{\fleft}[2]{\textsc{Left}_{#1}[#2]}
\newcommand{\fright}[2]{\textsc{Right}_{#1}[#2]}
% Functions for DRS
\newcommand{\fattribute}{\textsf{Attribute}}
\newcommand{\fname}{\textsf{Name}}
\newcommand{\fpivot}{\textsf{Pivot}}
\newcommand{\ftheme}{\textsf{Theme}}
\newcommand{\ftime}{\textsf{Time}}
\newcommand{\fvalue}{\textsf{Value}}
% Model name
\newcommand{\counter}{\textsc{Counter}}
\newcommand{\bistparser}{\textsc{BistParser}}
\newcommand{\uuparser}{\textsc{Uuparser}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\Volume{31}
\Number{3}
\Month{September}
\Year{2024}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2024}{2}{1}
\revised{2024}{5}{8}
\accepted{2024}{6}{15}

\setcounter{page}{1193}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{A Cross-linguistic Analysis of the Effects of Character-level Information in Neural Models}
\eauthor{Tomoya Kurosawa\affiref{UTokyo} \and Hitomi Yanaka\affiref{UTokyo}}
\eabstract{%
Characters are the smallest units of natural language, and humans understand texts from characters. Past studies have attempted to train language models with the information obtained from character sequences (character-level information) in addition to tokens to improve the performance of these models in various natural language processing tasks in various languages. However, they treated the performance improvement by character-level information as a performance difference between with and without characters. The extent to which these models use character-level information to solve these tasks remains unclear. The effects of linguistic features such as morphological factors on differences in the performance across languages are also under investigation. In this study, we examine existing character-employed neural models and the variation in their performance with character-level information. We focus on four languages: English, German, Italian, and Dutch, and three tasks: part-of-speech (POS) tagging, dependency parsing, and Discourse Representation Structure (DRS) parsing. The experimental results show that character-level information has the greatest effects on model performance on POS tagging and dependency parsing tasks in German and on a DRS parsing task in Italian. Based on these results, we hypothesize that the significant effects on model performance in German is caused by the average lengths of the words and the forms of common nouns. A detailed analysis reveals a strong correlation between the average lengths of the words and effectiveness on POS tagging in German.
}
\ekeywords{Character-level Information, Linguistic Features, Cross-lingual, Neural Networks, Part-of-Speech Tagging, Dependency Parsing, Semantic Parsing, Discourse Representation Structures}

\affilabel{UTokyo}{}{The University of Tokyo}

\headauthor{Kurosawa and Yanaka}
\headtitle{Cross-linguistic Analysis of the Effects of Character-level Information}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

\renewcommand{\thefootnote}{}
\footnote[0]{A part of this study is reported at the 37th Annual Conference of the Japanese Society for Artificial Intelligence, the 12th Joint Conference on Lexical and Computational Semantics (Kurosawa and Yanaka 2023), the 18th Young Researcher Association for NLP Studies, and the 257th Special Interest Group on Natural Language Processing.}
\renewcommand{\thefootnote}{\arabic{footnote}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\vspace{-1.5\Cvs}
\section{Introduction}
\label{section:introduction}

Characters are symbols that compose natural language text.
There are different types of character sets{, and within each type, there are} different characters.
\pagebreak
Characters {can} sometimes {be} helpful {for} understanding the {meanings} of words {that are not very familiar to us}.
For example, we can guess{, from the characters alone,} that the English words \textit{wake} and \textit{awake} have a strong similarity.
{Such similarity can also indicate contrastive meanings such as \textit{include} and \textit{exclude}.}

Neural network models, on the other hand, treat a mass of characters as tokens and ignore the original characters, even though {these characters may} have clues {from which} to infer meaning.
{For example, BERT} \cite{devlin-etal-2019-bert} {and GPT} \cite{openai-2023-gpt4} {do not split each of the words \textit{wake} and \textit{awake} into multiple tokens.}
In fact, these models have struggled to understand texts involving antonymy and numerical reasoning, which require {capturing} every letter \cite{naik-etal-2018-stress}.
To overcome this difficulty, some studies have directly provided characters ({referred to hereafter as} \textbf{character-level information}) for their models (\textbf{character-employed models}) with sequences of tokens, achieving performance improvements compared to those {that have not been provided with} characters.
Improvements due to character-level information have been reported in several languages \cite{ballesteros-etal-2015-improved,plank-etal-2016-multilingual,van-noord-etal-2020-character}.
However, these studies calculate the improvement as the difference between the performance with and without character-level information.
{These} values do not reveal the extent to which these models use character-level information other than word-level information.

Simultaneously, {pre-trained} language models have been shown to be able to read multilingual text and have achieved high performance in multiple languages \cite{devlin-etal-2019-bert,openai-2023-gpt4}.
Such performance, however, is limited to languages with sufficient data.
It is known that language models are difficult to construct {for} low-resource {languages} \cite{ponti-etal-2019-modeling}.
To compensate for this weakness, some {earlier} studies have tried to use linguistic features already discovered in the field of linguistic typology, {such as the order of subject, object, and verb} \cite{zhang-barzilay-2015-hierarchical,ammar-etal-2016-many}.
Instead of using facts found in linguistic typology, some studies have explored linguistic features from neural networks \cite{stanczak-etal-2022-neurons,varda-marelli-2023-data,stanczak-etal-2023-latent}.
These studies focused on obtaining common features between languages, {thus leaving space} to explore language-specific features in neural network-based models.
\citet{gutierrez-vasques-etal-2023-languages} is one of the few studies that attempted to induce linguistic typology.
{On the other hand, whereas} \citet{gutierrez-vasques-etal-2023-languages} focused on the tokenization method and {analyzed its} results,
we aim to find linguistic features from the results obtained by neural network-based approaches {on} various {NLP} grounding tasks.

In this study, we investigate the extent of the effects of character-level information in {existing} character-employed neural models trained with character-level and word-level information on multiple tasks in multiple languages.
Because characters are the smallest units, we restrict attention to them rather than {use} subwords such as{, as in the case of} WordPiece \cite{schuster-nakajima-2012-japanese}.
We {mainly} focus on four languages: English, German, Italian, and Dutch, which all use Latin scripts as their writing system.
We use three {types} of tasks: a morphological task, a syntactical task, and a semantical task.
These are all classified as {language analysis tasks} in NLP rather than {as} applied tasks such as machine translation and text generation.
{In our study, language analysis is treated as a means to analyze the behavior of neural models.}
{Models} for text generation tasks output texts all at once, although its process consists of multiple operations at morphological, syntactical, and semantical levels.
We analyze how neural models handle character-level information through language analysis at {each of} these levels.
Specifically, {for the three {language analysis tasks},} we {perform our study on} part-of-speech (POS) tagging \cite{marcus-etal-1993-building}, dependency parsing \cite{nivre-etal-2016-universal}, and Discourse Representation Structure (DRS) parsing \cite{abzianidze-etal-2017-parallel}, respectively.

Our main contributions in this study are {as follows}:
\begin{enumerate}[1.]
    \item We propose a new method to calculate how much a character-employed model uses character-level information compared to word-level information {by providing unobserved characters to the model in the test phase}.
    \item We {determine} several tendencies in a particular language from our experimental results. 
    \item We build hypotheses from the linguistic features obtained in the experiments and verify {these} by conducting additional experiments.
\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Background}
\label{section:background}

In this section, we introduce three models of {POS} tagging, dependency parsing, and {DRS} parsing, which we use in our experiments.
Note that each {of the three models} has bidirectional long short-term {memory (bi-LSTM) components} in its architecture.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Notation}
{Unless indicated otherwise}, small letters without any emphasis (e.g., $x$) {are used} for scalar values, bold letters (e.g., $\bm{x}$) {are used} for vectors, and bold capitals (e.g., $\bm{X}$) {are used} for matrices.
{Furthermore,} $\bm{x}_{1:n}$ denotes a vector sequence consisting of $\bm{x}_1, \bm{x}_2, \ldots, \bm{x}_n${; $\bm{a}\circ \bm{b}\in \mathbb{R}^{a+b}$ denotes a concatenation between $\bm{a}\in \mathbb{R}^a$ and $\bm{b}\in \mathbb{R}^b$}.
$\sigma$ {represents} a sigmoid function $\sigma(x) = \dfrac{1}{1+e^{-x}}$, which can be applied to vectors.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{POS tagging}
\label{section:bg_pos}

In our study, we use an off-the-shelf POS tagger by \citet{plank-etal-2016-multilingual}, partially shown in Figure \ref{figure:model_pos}.
This tagger adopts bi-LSTMs as {its} basic neural architecture.
When an input sequence is {composed} only {of} words, the tagger first converts each of them into its embedding (word embedding, dimension $d_w$).
The trainable word embeddings $\bm{w}_1, \ldots, \bm{w}_n$ are input into a context bi-LSTM, outputting
\[
    \bm{v}_i = \fbilstmctx(\bm{w}_{1:n}, i) = \flstmf({\bm{w}_{1:i}})\circ\flstmb({\bm{w}_{n:i}})
\]
for each position $i${, where}
$\flstmf({\bm{w}_{1:i}})$ and $\flstmb({\bm{w}_{n:i}})$ are functions that return the final hidden states of forward and backward LSTMs, respectively.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia13f1.pdf}
\end{center}
\hangcaption{A part of the architecture of the POS tagger by \protect\citet{plank-etal-2016-multilingual}. $\textbf{bi-LSTM}_{\textbf{seq}}$ and $\textbf{bi-LSTM}_{\textbf{ctx}}$ stand for a sequence and context bi-LSTM, respectively.}
\label{figure:model_pos}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

When {the} inputs {include both} words and their characters, the tagger first converts each character {constituting} the word into its embedding (character embedding, dimension $d_c$).
The character embedding of word $w$ consisting of $m$ characters $c_{1:m}$ is calculated as
\[
    \bm{c} = \fbilstmseq(\bm{c}_{1:m}') = \flstmf(\bm{c}_{1:m}')\circ\flstmb(\bm{c}_{m:1}'),
\]
where $\bm{c}_i'$ is a trainable embedding for each character $c_i$.
This bi-LSTM is called a sequence bi-LSTM.
Then, the core bi-LSTM reads the concatenation of a trainable word embedding and the corresponding character embedding.
{This tagger is trained with} stochastic gradient descent with cross-entropy loss for the optimization algorithm.

{In their study,} \citet{plank-etal-2016-multilingual} examined the capability of character embeddings for 22 languages, including English, German, Italian, and Dutch.
The dataset they used {was} Universal Dependency v1.2\footnote{\url{https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1548}} \cite{nivre-etal-2016-universal}.
The Universal Dependencies (UD) project,\footnote{\url{https://universaldependencies.org/}} a framework for consistent annotation across different languages, has 17 POS tags in its definition.
In their experiments, the dimension of the word and character embeddings {were} set to 128 and 100, respectively. %, and $\sigma = 0.2$.
Experimental results showed that {for each language, the} values obtained using both word and character embeddings are higher than {those obtained} using either word or character embeddings.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dependency parsing}
\label{section:bg_dep}

In our study, we use a dependency parser \uuparser{} by \citet{de-lhoneux-etal-2017-raw}, partially shown in Figure \ref{figure:model_dep}.
This parser originated from \bistparser{} proposed by \citet{kiperwasser-goldberg-2016-simple}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{31-3ia13f2.pdf}
\end{center}
\hangcaption{A part of the architecture of \uuparser{}. $\textbf{bi-LSTM}_{\textbf{seq}}$ and $\textbf{bi-LSTM}_{\textbf{ctx}}$ stand for a sequence and context bi-LSTM, respectively.}
\label{figure:model_dep}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{\bistparser{}}

\bistparser{} is a neural network-based dependency parser {that uses} a bi-LSTM.
Given an input sentence $s$ {that} consists of $n$ words $w_1, w_2, \ldots, w_n$ and gold POS tags $t_1, t_2, \ldots, t_n$, this parser create input vectors of word $w_i (1\leq i \leq n)$ as $\bm{x}_i = \bm{w}_i\circ \bm{t}_i$, where $\bm{w}_i$ and $\bm{t}_i$ are embeddings for words and POS tags, respectively.
Then, {it calculates} context vectors $\bm{v}_i = \fbilstmctx({\bm{x}_{1:n}, i})$ for each position $i$.

The core architecture of \bistparser{} is transition-based \cite{nivre-2008-algorithms}.{Transition-based} \linebreak
parsers parse sentences according to the rules of transitions.
\bistparser{} applies the arc-hybrid system \cite{kuhlmann-etal-2011-dynamic}.

{Herein, we} introduce the {term} \textit{oracle}, {defined as} a strategy {for choosing} a transition.
\bistparser{} has a dynamic oracle.
A concept of the dynamic oracle was proposed by \citet{goldberg-nivre-2012-dynamic} to address {how to choose an} optimal transition based on the state of each point.
Based on the loss between the gold and predicted dependency tree, this system introduces the cost of transition $T$.
Finally, the system follows the oracle, choosing a transition with the smallest cost.
The classifier of this parser is a multi-layer perceptron and scores each possible transition.

{In their study,} \citet{kiperwasser-goldberg-2016-simple} evaluated the ability of \bistparser{} in English and Chinese.
Their implementation of \bistparser{} has a word dropout mechanism \cite{iyyer-etal-2015-deep} to deal with unknown words.
In \bistparser{}, a word $w$ will be replaced {by} the unknown symbol with {a} probability {of} $\alpha / (\#(w) + \alpha)$ where $\alpha$ is a hyperparameter, and $\#(w)$ is the number of {appearances of} $w$ in the training dataset.
This probability {will be} high {if} the word $w$ hardly appears in the dataset.
Their experimental results showed that in both English and Chinese, \bistparser{} achieved higher scores than {those obtained using earlier-developed} parsers \cite{dyer-etal-2015-transition,ballesteros-etal-2016-training}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{\uuparser{}}

\uuparser{} is a dependency parser modified from \bistparser{} in several {aspects}.
One of {these} is the composition of input vectors.
\uuparser{} removes POS tagging embeddings and adds character embeddings into input vectors {that will be fed into} a context bi-LSTM.
In \uuparser{}, an input vector of a word $w_i$ is calculated as
\[
    x_i = \bm{w}_i\circ \bm{p}_i\circ \bm{c}_i,
\]
where $\bm{p}_i$ is a pre-trained word embedding, and $\bm{c}_i$ is a character vector of the word $w_i$.
When a word $w$ is written in an $m$-character sequence $c_{1:m}$, its character embedding is calculated as $\fbilstmseq({\bm{c}_{1:m}'})$, where $\bm{c}_i'$ is an embedding for each character $c_i$.
{\textsc{Uuparser} still has a mechanism of word replacement in \textsc{BistParser}.}

\uuparser{} was created for the shared task in Conference on Computational Natural Language Learning (CoNLL) 2017 \cite{zeman-etal-2017-conll} {that} is rooted from UD v2.0.
The CoNLL 2017 shared task {was} dependency parsing {in} 49 languages, including English, German, Italian, and Dutch.
In the main evaluation phase, \uuparser{} obtained a {micro-averaged labeled attachment score (LAS)} F1-score of 65.11, which ranked 23rd out of 33 systems.
In the post-evaluation ranking, {it} achieved an F1-score of 70.49, which ranked in the top ten positions.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{{DRS} parsing}
\label{section:bg_drs}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Overview}

Discourse Representation Structure (DRS) parsing is one of the semantic parsing tasks in NLP.
In {this} task, given an input sequence of tokens $s_1, \ldots, s_n$, we determine semantic relations involving the tokens based on Discourse Representation Theory {(DRT, Kamp and Reyle 1993)}.
A DRS is a logical representation based on an analysis of theological linguistics.
{A DRS consists of a set of variables (referred to as discourse referents) and a set of conditions.
We usually describe DRSs in a box-style format (\textit{box form}), with discourse referents in the first row and conditions in the second.}
Figure \ref{figure:drs_example_100825_box} shows a DRS in {the} box form for the sentence \textit{I don't have a bicycle}.
{We can express linguistic phenomena by using two or more DRSs in one sentence.}
Because DRSs in {the} box form {are} hard to read for computers, \citet{van-noord-etal-2018-exploring} proposed {the} \textit{clausal form}{, in which DRSs are written as} sequences of clauses.
Figure \ref{figure:drs_example_100825_rel_gold} {shows how} the same DRS in Figure \ref{figure:drs_example_100825_box} {is represented} in {the} clausal form.
{See van Noord et al. (2018) %\citet{van-noord-etal-2018-exploring}
for details about how to convert from the box form of DRSs to its clausal form.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3 and 4
\begin{figure}[b]
\begin{minipage}{200pt}
\setlength{\captionwidth}{200pt}
\begin{center}
\includegraphics{31-3ia13f3.pdf}
\end{center}
\hangcaption{A DRS for the sentence \textit{I don't have a bicycle}.}
\label{figure:drs_example_100825_box}
\end{minipage}
\hfill
\begin{minipage}{200pt}
\setlength{\captionwidth}{200pt}
\begin{center}
\includegraphics{31-3ia13f4.pdf}
\end{center}
\hangcaption{A DRS in {the} clausal form in Figure \ref{figure:drs_example_100825_box}.}
\label{figure:drs_example_100825_rel_gold}
\end{minipage}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Model}

In our study, we use a DRS-based semantic parser by \citet{van-noord-etal-2020-character}{, who proposed the} current state-of-the-art parsing model for the Parallel Meaning Bank (PMB).

{Van} noord et al. (2020) provided multiple setups in the baseline architecture, bi-LSTM and Transformer \cite{vaswani-etal-2017-attention}.
First, {they} prepared nine representations for the input {into} the neural architecture, word-based models such as GloVe \cite{pennington-etal-2014-glove} and fastText \cite{bojanowski-etal-2017-enriching}, and pre-trained {language} models such as BERT \cite{devlin-etal-2019-bert} and RoBERTa \cite{liu-etal-2019-roberta}.
In these setups, an input sequence $s_{1:n}$ is converted {into} an embedding sequence $\bm{s}_{1:l}$, where $\bm{s}_i$ is an embedding vector of {the $i$-th token in an input sentence} $s_i$ {for} each architecture.
Second, {they} employed several {pieces of} linguistic information such as POS tags and semantic tags ({i.e., language-neutral and} semantically informative tags, Abzianidze and Bos 2017). \nocite{abzianidze-bos-2017-towards}
They concatenated these {pieces of} linguistic information after each token, resulting in a mix of word tokens and tokens for linguistic information.
Third, {for} the most noteworthy component, {they} added character-level information to the architecture to experiment whether the information is useful on DRS-based semantic parsing.

The {model by} \citet{van-noord-etal-2020-character} can be extended by adding another encoder (the same as the existing one) into the baseline architecture.
When the model has two encoders and can train with character-level information, the embeddings of tokens in an input sentence are given to the first encoder, and character embeddings are given to the second encoder (see Figure \ref{figure:model_drs}).
They also experimented with both char-CNN \cite{kim-etal-2016-character} in a single encoder and attention mechanism \cite{vaswani-etal-2017-attention} in multiple encoders.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[b]
\begin{center}
\includegraphics{31-3ia13f5.pdf}
\end{center}
\hangcaption{A part of the architecture of the DRS parser by \protect\citet{van-noord-etal-2020-character}. $\textbf{bi-LSTM}_{\textbf{ctx}}$ stands for a context bi-LSTM. Note that embeddings of characters (e.g., those for $\mathrm{i}_1$ and $\mathrm{i_2}$) are identical, but the context vectors are different.}
\label{figure:model_drs}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Each of the character-employed models {previously described} in {Sections} \ref{section:bg_pos} and \ref{section:bg_dep} use a sequence bi-LSTM and compute character embeddings per token.
On the other hand, the {model by} \citet{van-noord-etal-2020-character} uses a context bi-LSTM and {computes} character embeddings for a whole sentence.
This model obtains a context embedding for characters as the averaged vector of each context character {embedding}, i.e., $\bm{c}^\ast = \qty(\sum_{i=1}^l {\bm{c}_i}) / l$ where $l$ is the length of {words} of an input sentence.
This calculation is also applied for word embeddings, i.e., $\bm{w}^\ast = \qty(\sum_{i=1}^n {\bm{v}_i}) / n${,} where $n$ is the number of tokens of an input sentence, including whitespaces.
These averaged vectors are used for the initial state of the decoder as \[
    \bm{d}_0 = \tanh(\bm{W}_{\textrm{init}}(\bm{w}^\ast\circ\bm{c}^\ast)),
\]
where $\bm{W}_{\textrm{init}}$ is a trainable matrix.

{In} {experiments} conducted using PMB {versions} 2.2.0 and 3.0.0\footnote{\url{https://pmb.let.rug.nl/data.php}}, \citet{van-noord-etal-2020-character} {determined} that the most {utilized} input representation was the BERT-base rather than {modified} models such as RoBERTa{;} {the most utilized} linguistic information {were} the semantic tags{;} and character-level information contributed to performance improvements of the neural model.
{Compared with} a single bi-LSTM setup with BERT-base embeddings and semantic tags, adding another encoder for character-level information contributes to performance improvements {of} 0.5\% (from 87.9\% to 88.4\%) {for} the English dev set in PMB version 3.0.0.
{When an encoder for character-level information was added for the PMB version 3.0.0 test sets in German, Italian, and Dutch, this} model gained performance improvements of 2.6\%, 1.5\%, and 2.1\%, respectively.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Method}
\label{section:method}

To calculate the effects of character-level information in character-employed neural models, we construct a series of methodologies from our prior work \cite{kurosawa-yanaka-2023-character}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[b]
\begin{center}
\includegraphics{31-3ia13f6.pdf}
\end{center}
\caption{An overview of the existing method to find contribution of character-level information.}
\label{figure:method_contrib}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 7
\begin{figure}[b]
\begin{center}
\includegraphics{31-3ia13f7.pdf}
\end{center}
\hangcaption{An overview of our method to find effectiveness of character-level information. (ii) and (iii) correspond to the processes in Section \ref{section:mt_effect}, respectively.}
\label{figure:method_effect}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Performance effects of character-level information}
\label{section:mt_effect}

{Past studies} evaluated the contribution of character-level information as {the} difference {in the performance} with and without character-level information \cite{ballesteros-etal-2015-improved,plank-etal-2016-multilingual}, as illustrated in Figure \ref{figure:method_contrib}.
In both Setup \#1 (with character-level information) and Setup \#2 (without character-level information), a character-employed model is trained and evaluated in the same input style, {i.e., a model in Setup \#1 where both tokens and corresponding characters are input; and a model in Setup \#2, where only tokens are input}.
Because a model {in Setup \#2} is trained without characters, {this model is optimized under only tokens, and} the evaluated score (contribution by character-level information) {does not} reveal {the effect of character-level information in Figure} \ref{figure:method_contrib}{, i.e.,} how much the model uses character-level information rather than word-level information.
We measure the extent of {the} effects of character-level information in a character-employed model by the following procedure:
\begin{enumerate}[(i)]
    \item We train the model using the original training data and a series of correct character sequences.
    \item We evaluate the trained model using the original test data and a series of correct character sequences. Let $x$ denote the obtained value.
    \item We evaluate the trained model using the original test data and a series of character sequences that consist of \textbf{fully unobserved characters} in the training phase. Let $y$ denote the obtained value.
    \item We compare the results of (ii) and (iii) and define \textbf{effectiveness} as {the} ratio between the performance of the model with correct character-level information and {the} difference {in} the {performance} of the model with correct and {with} unobserved character-level information, i.e., $\dfrac{x-y}{x}$.
\end{enumerate}
A character sequence with fully unobserved characters {is obtained by} changing all characters {except} whitespaces into an unobserved character in the training phase.
For example, {if the} input is \textit{Yuri\textvisiblespace{}is\textvisiblespace{}so\textvisiblespace{}cute}, {then the corresponding} unobserved character-level information of the input {is} \textit{\textbackslash{}\textbackslash{}\textbackslash{}\textbackslash{}\textvisiblespace{}\textbackslash{}\textbackslash{}\textvisiblespace{}\textbackslash{}\textbackslash{}\textvisiblespace{}\textbackslash{}\textbackslash{}\textbackslash{}\textbackslash{}}, where {\textit{\textbackslash{}}} is one of the unobserved characters in the training phase.
Figure \ref{figure:method_effect} illustrates (ii) and (iii) of our procedure to find effectiveness of character-level information.
{When provided with} fully unobserved characters in the test phase, the character-employed model trained with correct character-level information cannot predict labels from the information.
Hence, the obtained result reflects the benefit of character-level information in such a model.
To evaluate models in a setting that eliminates the influence of knowledge about relationships between tokens and characters obtained during training, we do not use any character that has been observed at least once in the training phase.
{To see the performance effects based on the overall model performance with correct character-level information}, we adopt ratio rather than difference (e.g., $x-y$) in (iv).
We also evaluate the {performance of the model} without character inputs and encoders {to replicate earlier studies for comparison}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Reliance on a single character}
\label{section:mt_reliance}

We measure the extent of {the} reliance {of a character-employed model} on a single character.
{In an} experiment in our prior work \cite{kurosawa-yanaka-2023-character}{, we} input characters comprising only \texttt{a} in the test phase to see the performance of the model {to make} any information from the input characters {unusable}.
This methodology does not reveal the effects of character-level information because any character that appeared in the training phase may lead to {the misprediction of} labels.
Hence, we measure the reliance on each character by changing {steps} (iii) and (iv) of the procedure in Section \ref{section:mt_effect} into (iii)' and (iv)', respectively{, as follows}:
\begin{enumerate}[(i)']
    \setcounter{enumi}{2}
    \item We evaluate the trained model using the original test data and a series of character sequences that consist of \textbf{a single character} (\textbf{unified} character-level information) {observed} in the training phase. Let $y_i$ denote the obtained value, where $i$ is a character.
    \item We compare the results of (ii) and (iii)' and define \textbf{reliance} on the character $i$ as {the} ratio between the performance of the model with correct character-level information and {the} difference {in} the {performance} of the model with correct and {with} unified character-level information, i.e., $\dfrac{x-y_i}{x}$, where $i$ is a character.
\end{enumerate}
A sequence of unified characters {is obtained by} changing all characters {except} whitespaces into a single character.
{For} this method, we {perform experiments} for each ASCII\footnote{American Standard Code for Information Interchange} character from code 33 (\texttt{!}) to 126 (\texttt{\textasciitilde}), {as listed} in Table \ref{table:ascii}.
By unifying {them} into single {characters} and providing them {to the model} in the test phase, we can {determine} how much the model uses a single character to predict labels.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[t]
\input{13table01.tex}%
\hangcaption{{Character} table enumerated in ASCII code. Each value in the header row and column is hexadecimal. x in a header column will be filled with a value in the header row. For example, {the character} \texttt{A} is located in row 4x and column 1, {and thus, its} ASCII code is 41 in hexadecimal (65 in decimal). [DEL] is a control character for deletion, which is not used in real-world texts.}
\label{table:ascii}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Character order awareness}

A sentence in unsettled order is {difficult} to read for humans (e.g., \textit{a black race car starts up in front of a crowd of people} vs. \textit{of black up race starts in car a of people front a crowd}).
On the other hand, neural models {are} still {able to} predict labels correctly in multiple tasks even if the order of an input sentence is unnatural \cite{pham-etal-2021-order,sinha-etal-2021-unnatural}.
Is this phenomenon also true for character order?
{In our} work \cite{kurosawa-yanaka-2023-character}{, we} experimented ahead on DRS parsing and {determined} that Italian is the only language where {the} shuffled character sequences lead to {misprediction}.
In this {study}, we measure how much a character-employed model uses {the} order of character sequences to solve tasks, including DRS parsing, by changing {steps} (iii) and (iv) of the procedure in Section \ref{section:mt_effect} into (iii)'' and (iv)'', respectively{, as follows}:
\begin{enumerate}[(i)'']
    \setcounter{enumi}{2}
    \item We evaluate the trained model using the original test data and a series of character sequences \textbf{shuffled} in the training phase. Let $y'$ denote the obtained value.
    \item We compare the results of (ii) and (iii)'' and define \textbf{order awareness} as {the} ratio between the performance of the model with correct character-level information and {the} difference {in} the {performance} of the model with correct and {with} shuffled character-level information, i.e., $\dfrac{x-y'}{x}$.
\end{enumerate}
For example, possible shuffled character-level information {for} the sentence \textit{Yuri is so cute} is \textit{uirY si so ctue}.
Some of the characters may not be moved to another position.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Order awareness for a whole sentence}
The DRS parsing model described in Section \ref{section:bg_drs} obtains context vectors for all characters in a sentence.
{In our} prior work \cite{kurosawa-yanaka-2023-character}{, we} obtained {the performance} of the model with shuffled character-level information over word separators.
{In this study, we additionally} measure order awareness for a whole sentence using our results from \citet{kurosawa-yanaka-2023-character}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Capability of extended characters}

In all {languages that use Latin scripts}, including English, German, Italian, and Dutch, the {numbers} of {characters are} very small compared to {those used by} several East Asian languages{,} such as Chinese and Japanese.
{Furthermore}, there is a huge gap between the numbers of characters and tokens in each language.
In this {study}, we explore {the} capabilities {of characters} to possess more information than {those found in} unigram sequences{,} following our prior work \cite{kurosawa-yanaka-2023-character}.
{For each span of sentence, we} create {a} bigram sequence.
For example, a bigram sequence {for the} sentence \textit{Yuri is so cute} {is} (\textit{Yu}, \textit{ur}, \textit{ri}, \textit{i}, $\emptyset$, \textit{is}, \textit{s}, $\emptyset$, \textit{so}, \textit{o}, $\emptyset$, \textit{cu}, \textit{ut}, \textit{te}, \textit{e}),
where $\emptyset$ expresses a word separator that is not handled as a (bigram) character.
Other than the previous methodologies, we train the character-employed model and evaluate it {using the} bigram sequences.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experimental Setup}
\label{section:setup}

In this section, we explain {the different aspects of the} experimental {setup,} such as {the} datasets and model parameters.
{Fundamentally}, most of the settings {applied in this study} follow the previous works \cite{plank-etal-2016-multilingual,de-lhoneux-etal-2017-raw,van-noord-etal-2020-character,kurosawa-yanaka-2023-character}.
We {perform tests} in four languages: English, German, Italian, and Dutch.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{POS tagging}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Dataset}
In {our} experiments {on} POS tagging, we use the Universal Dependencies (UD) v1.2\footnote{\url{https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1548}} \cite{nivre-etal-2016-universal}.
Table \ref{table:dataset_ud_conll} shows {statistical information on} the dataset used in {the} POS tagging experiments.
For each language, the model is trained on the training set and evaluated on the test set.
We do not use the development set for each language.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Model}
We use the POS tagger by \citet{plank-etal-2016-multilingual} described {earlier} in Section \ref{section:bg_pos}{.}
{The} implementation {of which} is {available} in GitHub.\footnote{\url{https://github.com/bplank/bilstm-aux}}
We use hyperparameters that \citet{plank-etal-2016-multilingual} determined {for} the English development set.
{Therein, they}
{did not split the training set into subsets for multiple training steps}.
See Table \ref{table:setup_pos_dep_hyperpara} for other hyperparameters.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Evaluation metrics}
We evaluate the model {based on} the averaged accuracy of three runs.
The accuracy of each output is calculated as {the} ratio {between} the number of tokens whose tags are matched {and} the {total} number of tokens.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{13table02.tex}%
\caption{The dataset statistics of UD v1.2 and CoNLL 2017.}
\label{table:dataset_ud_conll}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{13table03.tex}%
\caption{Hyperparameters for POS tagging and dependency parsing (DEP).}
\label{table:setup_pos_dep_hyperpara}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dependency parsing}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Dataset}
In {our} experiments {on} dependency parsing, we use a dataset {that was utilized} for the shared task in the Conference on Computational Natural Language Learning (CoNLL) 2017.
{The training} and development data are {from UD} v2.0\footnote{\url{https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1983}} \cite{nivre-etal-2016-universal}.
The test data {were} constructed originally by \citet{zeman-etal-2017-conll}.
In this dataset, {there are multiple corpora in English and Italian}.
We use English Web Treebank\footnote{\url{https://github.com/UniversalDependencies/UD_English-EWT}} for English and Italian Stanford Dependency Treebank\footnote{\url{https://github.com/UniversalDependencies/UD_Italian-ISDT/tree/master}} for Italian.
Table \ref{table:dataset_ud_conll} shows {statistical information on} the dataset used in dependency parsing experiments.
For each language, the model is trained on the training sets {with validation on the development sets} and evaluated on the test set.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Model}
We use the dependency parser proposed by \citet{de-lhoneux-etal-2017-raw} described in Section \ref{section:bg_dep}{.}
{The} implementation {of which} is {available} in GitHub.\footnote{\url{https://github.com/UppsalaNLP/uuparser}}
{The} hyperparameters {we use are} described in \citet{de-lhoneux-etal-2017-raw} {and outlined in} Table \ref{table:setup_pos_dep_hyperpara}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Evaluation metrics}
We evaluate the model {based on} averaged labeled attachment score (LAS) of three runs.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{DRS parsing}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Dataset}
In {our} experiments {on} DRS parsing, we use the Parallel Meaning Bank version 3.0.0 {(PMB v3, Abzianidze et al. 2017)}.
Following \citet{van-noord-etal-2020-character} and our prior work \cite{kurosawa-yanaka-2023-character}, {for each language, we prepare training sets for \textit{coarse training}, i.e., training with a large amount of data that potentially include noise; and \textit{fine training}, i.e., training with a small amount of data of relatively high quality, as follows}:
\begin{enumerate}[(a)]
    \item If PMB v3 contains {a} gold train set and {a} silver train set {with a} sufficient {number of} sentences {(more than 10k)}, the {coarse training} data {will comprise} gold and silver train sets, {whereas} the {fine training} data {will become} a gold train set.
    \item If PMB v3 contains {a} gold train set and {a} silver train set {with an insufficient number of sentences (less than 10k)}, the {coarse training} data {will comprise} all of the train sets (gold, silver, and bronze), {whereas} the {fine training} data {will become} a concatenation of gold and silver train sets.
    \item If PMB v3 does not contain {a} gold train {set}, the {coarse training} data {will comprise} all of the train sets (silver and bronze), {whereas} the {fine training} data {will become} a silver train set.
\end{enumerate}
{Note that documents in the PMB are divided into the three splits (gold, silver, and bronze) based on annotation quality.
Gold documents are fully annotated by humans,
silver documents are partially manually corrected from the trained model, and
bronze documents are annotated by an annotation model trained on data that are annotated manually.}
{Case (a) applies to English, case (b) applies to German, and case (c) applies to Italian and Dutch}.
Table \ref{table:dataset_pmb} shows {statistical information on} of the dataset constructed {in the aforementioned manner}.
{We conducted a series of experiments before, in which the numbers of data were aligned across the languages.
See the Appendix A for more detail.}  % \ref{appendix:data_aligned}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{13table04.tex}%
\hangcaption{The dataset statistics of PMB v3. In each language, {the underlined and bolded} numbers are the ones of data used for the {coarse} and {fine training}, respectively.}
\label{table:dataset_pmb}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Model}
We use the DRS parser proposed by \citet{van-noord-etal-2020-character} described {earlier} in Section \ref{section:bg_drs}{.}
{The} implementation {of which} is {available} in GitHub.\footnote{\url{https://github.com/RikVN/Neural_DRS}}
{The} tokenization method is the same as multilingual BERT (\texttt{bert-base-multilingual-uncased}, Devlin et al. 2019) in all languages.
The parameter of BERT is set to be frozen.
The architecture {used} in {our} experiments has two bi-LSTM encoders{, one} for {subwords} and {the other for} characters.
{We use bi-LSTMs rather than transformers because \citet{van-noord-etal-2020-character} reported that scores of the model with bi-LSTMs were higher than those with transformers.}
{The} series of inputs is a tokenized sequence without additional information, such as semantic tags.
{The} representation format of DRSs is a clausal form with relative naming \cite{van-noord-etal-2018-exploring}.
We use gold tokenized sentences in PMB v3 rather than outputs of the off-the-shelf tokenizer.
{The} hyperparameters {we use are} described in \citet{van-noord-etal-2020-character}, \citet{kurosawa-yanaka-2023-character}, and \citet{van-noord-etal-2019-linguistic}; {and outlined in} Table \ref{table:setup_drs_hyperpara}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[b]
\input{13table05.tex}%
\caption{Hyperparameters for DRS parsing.}
\label{table:setup_drs_hyperpara}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Evaluation metrics}
We evaluate the model {based on the} averaged micro F-score of nine runs: three runs of both pre-training and fine-tuning.
{The} F-score of a prediction is calculated {using} \counter{} {(van Noord et al., 2018a)}. % \cite{van-noord-etal-2018-evaluating}.
\counter{} explores the best matching between variables in the two DRSs.
This evaluation tool returns an F1-score between 0 and 1 when the optimal one-to-one variable mapping {is obtained}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Environmental setup}

We run each of the experiments on Intel Xeon Gold 6148 {processors} with one NVIDIA V100 (16GiB).
{The computation times obtained} in {the experiments are} as follows: less than 2.5 hours {for} POS tagging, less than 12.5 hours {for} dependency parsing, and less than 6.5 hours {for} DRS parsing.
See Appendix \ref{appendix:computation_time} for more detail.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[b]
\input{13table06.tex}%
\hangcaption{{Average accuracies using} correct and unobserved character-level information, effectiveness of character-level information, and {average accuracies using} only word embeddings (\%) on POS tagging.}
\label{table:result_pos_effectiveness}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Result}
\label{section:result}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{POS tagging}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Effectiveness}
\label{section:res_pos_effectiveness}
Table \ref{table:result_pos_effectiveness} shows the scores (accuracy and effectiveness) {observed in} the experiment on POS tagging.
{For} each of the four languages, we {obtained} positive effectiveness, which {indicates} that character-level information contributes to performance improvements.
{The} largest effectiveness {across the languages is that for} German (44.61), {which is} more than twice {the} value {for each} of the other languages.
These values suggest that character-level information {has a greater effect on POS tagging in German than on those in the other languages}.
We also {observed} that {for each language,} the accuracy {of the model} with correct character-level information is larger than that {of the model} without character-level information.
The {largest} difference between the two scores {is} {that observed for} Dutch ($93.10-85.56=7.54$), but the effectiveness {of the model} in {executing tasks in} this language is {only} the third largest of the four (15.82).
{For reference,} the state-of-the-art value for the English language is 97.45 in UD v2.3, as reported by \citet{straka-etal-2019-evaluating}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Reliance on a single character}
\label{section:res_pos_reliance}
Table \ref{table:result_pos_disturbance} shows the average {accuracies} (reliance) grouped into four based on {the type} of {character} (capitals, smalls, numbers, and symbols), {as} evaluated {using} unified characters.
{The largest values for} the capitals and smalls {across the languages are those for German}.
{In particular}, the value {in} the capitals column {for} German is 50 percent points larger than that {for} English.
See also Appendix {C} for individual values. % \ref{appendix:full_results}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[b]
\input{13table07.tex}%
\hangcaption{Average reliance with standard error (\%) {for} each {type} of {character}, {as} obtained {using} unified character-level information{,} on POS tagging.}
\label{table:result_pos_disturbance}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

To {determine the} factors that {led to} some {of the} values in Table \ref{table:result_pos_disturbance},
the values {for} the {number characters for} all languages and {for} the capitals and smalls {for} German, we focus {on} two pairs of a {type} of {character} and a POS tag: numbers and \textsf{NUM} (numeral) and capitals and \textsf{PROPN} (proper noun) because we can observe strong correlation between each pair (see Appendix {C} for full results). % \ref{appendix:full_results}
Table \ref{table:result_pos_tag_num} and Table \ref{table:result_pos_tag_propn} show the numbers of gold and predicted tags {for} these two pairs.
For numbers and \textsf{NUM}, we can observe relatively consistent (incorrect) ratio of the number of predictions across all languages ({approximately} 70\%).
The model might {have} output \textsf{NUM} to tokens when it found numbers in each, achieving very high {accuracies} but low F-scores.
{On the other hand, for} capitals and \textsf{PROPN}, the predictions of the tag {for} German occupy more than 90\% of all, far more than those {for} other languages (16\% in English, 42\% in Italian, and 32\% in Dutch).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[t]
\input{13table08.tex}%
\hangcaption{{Numbers} of all tags and \textsf{NUM} tags, {numbers} of times the POS tagger predicts \textsf{NUM}, {accuracies}, and {F-scores for} unified character sequences of numbers.
{Herein, each accuracy is calculated by dividing by the number of data with a gold tag of \textsf{NUM}.}}
\label{table:result_pos_tag_num}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[t]
\input{13table09.tex}%
\hangcaption{{Numbers} of all tags and \textsf{PROPN} tags, {numbers} of times the POS tagger predicts \textsf{PROPN},{accuracies}, and {F-scores for} unified character sequences of capitals.}
\label{table:result_pos_tag_propn}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Order awareness}
Table \ref{table:result_pos_order} shows {the accuracies of the models} with correct and {with} shuffled character-level information and {their} order {awareness}.
{The} largest order awareness is Dutch (15.82).
This suggests that the model trained with character-level information is most sensitive to character order {when executing tasks} in Dutch.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[t]
\input{13table10.tex}%
\hangcaption{{Accuracies of the models using} correct and shuffled character-level information and order {awarenesses} (\%) on POS tagging.}
\label{table:result_pos_order}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Bigram extension}
Table \ref{table:result_pos_ngram} shows {the accuracies of the models} with unigram and {with} bigram character-level information and {their} differences.
We can observe that {for each language, the accuracies of the models} with {bigrams are} lower than {those of the models} with {unigrams}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 11
\begin{table}[t]
\input{13table11.tex}%
\hangcaption{{Accuracies of the model} with unigram and {with} bigram character-level information and the {differences} (\%) on POS tagging.}
\label{table:result_pos_ngram}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[t]
\input{13table12.tex}%
\hangcaption{LAS scores {of models} with correct and {with} unobserved character-level information, effectiveness of character-level information, and LAS {scores of models} with only word embeddings (\%) on dependency parsing.}
\label{table:result_dep_effectiveness}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dependency parsing}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Effectiveness}
\label{section:res_dep_effectiveness}
Table \ref{table:result_dep_effectiveness} shows the scores (LAS scores and effectiveness) {observed in} the experiment on dependency parsing.
{Similar to the results of our} experiment on POS tagging, we {obtained} positive effectiveness {for} each of the four languages, {indicating} that character-level information contributes to performance improvements.
{The} largest effectiveness {across the languages is also that for} German (38.02), {which is} nearly twice as much as the values {for} the other languages.
These values suggest that character-level information {has a greater effect on dependeincy parsing in German than on those in the other languages}.
{Similar to the results for} POS tagging, we also {observed} that {for each language,} the accuracy {of the model} with correct character-level information is larger than that {of the model} without character-level information.
The {largest} difference between the two scores {is} {that observed for} Dutch ($75.90 - 69.16 = 6.74$), but the effectiveness {of the model in executing tasks} in this language is {only} the second largest of the four (19.99).
{For reference,} the state-of-the-art value for the English language is 88.45 in UD v2.3, as reported by \citet{straka-etal-2019-evaluating}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Reliance on a single character}
\label{section:res_dep_reliance}
Table \ref{table:result_dep_disturbance} shows the average LAS scores (reliance) grouped into four based on {the type} of {character}, {as} evaluated {using} unified characters.
{The largest values for} the capitals and smalls {across the languages are those for German}.
{In particular}, compared to the {corresponding} value {in the} capitals {column for the} English {language}, {that for} German {exhibits an increase of approximately} 50 percent {points}, {similar to the results of} the POS tagging experiment.
{See also Appendix C for individual values.} % \ref{appendix:full_results}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 13
\begin{table}[b]
\input{13table13.tex}%
\hangcaption{Average reliance with standard {error} (\%) {for} each {type} of {character}, {as} obtained {using} unified character-level information{,} on dependency parsing. Note that the character sequence unifying to * caused an error in all languages.}
\label{table:result_dep_disturbance}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Order awareness}
Table \ref{table:result_dep_order} shows the LAS {scores of the models} with correct and {with} shuffled character-level information and {their} order {awarenesses}.
{The} largest order awareness is German (16.29).
This could suggest that the model trained with character-level information is most sensitive to character order {when executing tasks} in German.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 14
\begin{table}[b]
\input{13table14.tex}%
\hangcaption{LAS scores {of the model} with correct and {with} shuffled character-level information and order {awarenesses} (\%) on dependency parsing.}
\label{table:result_dep_order}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Bigram extension}
Table \ref{table:result_dep_ngram} shows LAS scores {of the models} with unigram and {with} bigram character-level information and their differences.
We can observe that {for each language,} the LAS {scores of the models} with {bigrams are} lower than {those of the models} with {unigrams}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 15
\begin{table}[b]
\input{13table15.tex}%
\hangcaption{LAS scores with bigram and {with} unigram character-level information and the {differences} (\%) on dependency parsing.}
\label{table:result_dep_ngram}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{{Common hyperparameters on POS tagging and dependency parsing}}
We conducted a series of experiments on POS tagging and dependency parsing to analyze whether the obtained results are affected by the differences in the hyperparameters.
We set the dimensions of the word embeddings and character embeddings to 100 and the dimensions of the pre-trained word embeddings to 0.
Table \ref{table:result_pos_dep_united} shows effectiveness of the experiments.
Both values for German are still among the top for the four languages, whereas the value for Italian on POS tagging is much higher than that in Table \ref{table:result_pos_effectiveness} (37.09\% vs. 20.72\%).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 16
\begin{table}[b]
\input{13table16.tex}%
\hangcaption{{Effectiveness (\%) on POS tagging and dependency parsing, with common dimensions of word embeddings, pre-trained word embeddings, and character embeddings.}}
\label{table:result_pos_dep_united}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{DRS parsing}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Effectiveness}
\label{section:res_drs_effectiveness}
Table \ref{table:result_drs_effectiveness} shows the scores (F-scores and effectiveness) {observed in} the experiment on DRS parsing.
Unlike {on} POS tagging and dependency parsing, {only a tiny value of} effectiveness (0.13) {is observed for} English, suggesting that the character-employed model does not make much use of character-level information for solving {DRS parsing tasks in the English language}.
{On the other hand, for} each of the {three other} languages, we {obtained} positive effectiveness, which {indicates} that {for these languages,} character-level information contributes to performance improvements.
Unlike the results {on} the other two tasks, the largest effectiveness {across the languages is that for} in Italian (52.53), suggesting {that} the model uses character-level information as much {as it uses} word-level information.
This result {is consistent with} our prior work \cite{kurosawa-yanaka-2023-character}, but note that {in this experiment,} we do not use information from semantic tags in {the} English {language}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 17
\begin{table}[b]
\input{13table17.tex}%
\hangcaption{F-scores {of models} with correct and {with} unobserved character-level information, effectiveness of character-level information, and F-scores {of models} with only word embeddings (\%) on DRS parsing.
F-scores {on models} with only word embeddings are {obtained} from our prior study \protect\cite{kurosawa-yanaka-2023-character}.}
\label{table:result_drs_effectiveness}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Reliance on a single character}
\label{section:res_drs_reliance}
Table \ref{table:result_drs_disturbance} shows the average F-scores (reliance) grouped into four based on {the type} of {character}, {as} evaluated {using} unified characters.
{For all character types}, {the largest value across the languages is that for} Italian.
Unlike the {tendencies observed on} POS tagging and dependency parsing, smaller differences {are exhibited} among the four values {for} each language.
This suggests that the character-employed model uses multiple characters, rather than a single character{, in executing DRS parsing tasks}.
{See also Appendix C for individual values.} % \ref{appendix:full_results}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 18
\begin{table}[b]
\input{13table18.tex}%
\hangcaption{Average reliance with standard {error} (\%) {for} each {type} of {character}, {as} obtained {using} unified character-level information{,} on DRS parsing. Note that the character sequence unifying to * caused an error in all languages.}
\label{table:result_drs_disturbance}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 19
\begin{table}[b]
\input{13table19.tex}%
\hangcaption{{F-scores} with correct and {with} shuffled character-level information and {order awarenesses} (\%) on DRS parsing.}
\label{table:result_drs_order}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Order awareness}
Table \ref{table:result_drs_order} shows the F-scores {of the models} with correct and {with} shuffled character-level information and {their} order {awarenesses}.
{For the English language, as with the tiny effectiveness value reported earlier, a tiny order awareness value is also observed}.
{On the other hand, for Dutch, even though character-level information has earlier demonstrated effectiveness for tasks in this language, a small order awareness of approximately 2\% is also observed}.
{The} largest order awareness {across the languages is that for} Italian (25.73), {for} which we {also documented} the largest effectiveness in Table \ref{table:result_drs_effectiveness}.
This could suggest that the model trained with character-level information is most sensitive to character order {when executing tasks} in Italian.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{For a whole sentence}
Table \ref{table:result_drs_order_sent} shows {the} order awareness for a whole sentence from our prior work \cite{kurosawa-yanaka-2023-character}.
{A} significantly large value {is observed for} Italian, nearly double the order awareness within a word.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 20
\begin{table}[t]
\input{13table20.tex}%
\caption{Order awareness (\%) for a whole sentence on DRS parsing.}
\label{table:result_drs_order_sent}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Bigram extension}
Table \ref{table:result_drs_ngram} shows {the} F-scores {of the models} with unigram and {with} bigram character-level information and their differences, which are {obtained} from our prior work \cite{kurosawa-yanaka-2023-character}.
We can observe that {for each language,} the F-score {of the model} with {bigrams} is higher than that {of the model} with {unigrams} in each language.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 21
\begin{table}[t]
\input{13table21.tex}%
\hangcaption{F-scores with bigram and {with} unigram character-level information and the {differences} (\%) on DRS parsing.}
\label{table:result_drs_ngram}
\vspace{1\Cvs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Summary}

From the experimental results, we observed the following:
\begin{enumerate}[1.]
    \item The models tend to use a single character for {prediction} on POS tagging and dependency parsing.
    \item The models tend to use the order of characters on each task {for} each language{,} except on DRS parsing in English, {for} which we cannot observe significant effectiveness of character-level information.
    \item {For all languages, the models observing bigrams instead of unigrams tend to perform better on DRS parsing but not on POS tagging and dependency parsing.
    This might be caused by the number of target tokens.
    For DRS parsing, the model has to output one of over 1k target tokens.
    By contrast, the models for POS tagging and dependency parsing output one of less than 100 target tokens.}
\end{enumerate}

We focus on {the following} three tendencies for analyzing fine-grained hereafter.
\begin{itemize}
    \item Unified character-level information in uppercase disrupts the character-employed models on POS tagging and dependency parsing in German as mentioned in Sections \ref{section:res_pos_reliance} and \ref{section:res_dep_reliance}.
    \item Character-level information {has a greater effect on} model performance on POS tagging and dependency parsing tasks in German {than on those in other languages,} as mentioned in Sections \ref{section:res_pos_effectiveness} and \ref{section:res_dep_effectiveness}.
    \item {On the other hand, character-level} information {has a greater effect on} model performance on DRS parsing task in Italian {than on those in other languages,} as mentioned in Section \ref{section:res_drs_effectiveness}.
\end{itemize}

Before beginning {the} analysis, we provide a possible reason {for} the first {of these observations}.
As we {have documented}, reliance {on the} capital letters are 20.85\% and 14.42\% for POS tagging and dependency parsing, respectively, and each value {has an approximately} 50\% {difference} from the {corresponding} score {for} English.
This might {have been} caused by the {rules for writing} common nouns in German{, which state that} common nouns {are always first-letter capitalized}.
This differs from the {writing rules for the} other three languages, {in which} common nouns are {normally} written in small letters.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Analysis}
\label{section:analysis}

We picked up two tendencies observed {among} the results{, except for the} linguistic nature {of} German nouns.
In this section, we build hypotheses to the linguistic features of the tendencies.
Then, we conduct more detailed experiments to judge and discuss the hypothesis.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{{For POS tagging and dependency parsing in German, why is \linebreak
character-level information used the most?}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Hypothesis}

{On POS tagging and dependency parsing, the} German character-employed models trained with character-level information {exhibited} larger {values of} effectiveness by character-level information.
We hypothesize that this tendency is caused by \textbf{word length}.\footnote{Another experiment shows that performance changes by word length are larger than those by capitalization of common nouns in German, which we mentioned at the end of Section \ref{section:result}. See Appendix \ref{appendix:uncased} for details.}
To show that German words are relatively longer than {words from} other languages, we calculate the average numbers of characters per token (word length) in XNLI, a cross-lingual Natural Language Inference corpus \cite{conneau-etal-2018-xnli}.
XNLI {comprises} 15 languages across multiple families (listed in Table \ref{table:analysis_xnli_length}).
In XNLI, each pair of sentences (premise and hypothesis) is translated into these languages by professional experts.
Thus, we can observe language characteristics by analyzing the contents in XNLI.
Note that {of} the four languages {focused upon in this study}, only English and German are included in XNLI.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 22
\begin{table}[b]
\input{13table22.tex}%
\hangcaption{Average tokens per sentence and per token {for} each language in XNLI. {Number} of sentences {per} each language is 4980.}
\label{table:analysis_xnli_length}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 23
\begin{table}[b]
\input{13table23.tex}%
\hangcaption{Average tokens per sentence and characters per token in the CoNLL 2017 dataset (sum of train, dev, and test sets).}
\label{table:analysis_conll_ext_stats}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table \ref{table:analysis_xnli_length} shows the average number of tokens per sentence and characters per token.
As {has been outlined}, the average number of characters per token in German is precisely one character {more} than in English.
{Furthermore, among all 15 languages}, German is the third {highest in terms of the average number of characters per word}, following Turkish and Russian.
To compare the average {numbers of} characters {of German} with {those of} Italian and Dutch, we also calculate {the} average {numbers of} characters in the CoNLL 2017 dataset (the four upper rows in Table \ref{table:analysis_conll_ext_stats}).
We can observe that the average number of characters per token in German is also {greater} than {those} of Italian and Dutch.
The longer {the} word length is, the more informative it is for models {trained} with character-level information.
Hence, we suppose that the reason {behind} the largest effectiveness by character-level information {observed for} in German is {its} word length.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Method}

To {assess} whether the {aforementioned} hypothesis is true, we conduct an experiment in Catalan, Spanish, Finnish, French, and Slovak, which are included in the CoNLL 2017 dataset.
The lower rows in Table \ref{table:analysis_conll_ext_stats} show {statistical information on} the CoNLL 2017 dataset for these languages.
We choose these five languages {primarily because of their use of} Latin scripts{.}
{Furthermore,} the number of sentences {in each language} is between 10k and 20k {for the} train, dev, and test sets {combined} (we {exclude} Czech {from our experiments because the original dataset includes} 87,913 sentences {in this language, which} is five times more than the {corresponding} numbers {for} other languages).
For each language, we experiment {on} POS tagging and dependency parsing, as {described} in Section \ref{section:mt_effect}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Result}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{POS tagging}
Table \ref{table:analysis_pos_add_lang} shows the experimental {results for} POS tagging.
{Among all the additional languages (ca, es, fi, fr, and sk) and the target languages (en, de, it, and nl),}
the largest effectiveness {obtained is that for} Finnish (75.22\%).
Because Finnish also has the longest average word length {among} these languages, this result suggests a strong correlation between the average word length and effectiveness on POS tagging.
We plot the data in Table \ref{table:result_pos_effectiveness} and Table \ref{table:analysis_pos_add_lang} into Figure \ref{figure:analysis_graph_pos}.
We obtain a Pearson's correlation coefficient of 0.9077 for all {of the} nine languages.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 24
\begin{table}[b]
\input{13table24.tex}%
\caption{{Accuracies} and effectiveness {for} the additional languages on POS tagging.}
\label{table:analysis_pos_add_lang}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Dependency parsing}
Table \ref{table:analysis_dep_add_lang} shows the experimental {results for} dependency tagging.
\linebreak
{Among all the additional languages (ca, es, fi, fr, and sk) and the target languages (en, de, it, and nl),}
the largest effectiveness {obtained is that for} Slovak (58.87\%).
Because Slovak does not have the longest average word length, and {because an} effectiveness {value of} 39.90\% {is observed for} Finnish, which has the longest average word length, these results suggest that there is no strong correlation between the average word length and effectiveness on dependency parsing.
We plot the data in Table \ref{table:result_dep_effectiveness} and Table \ref{table:analysis_dep_add_lang} into Figure \ref{figure:analysis_graph_dep}.
We obtain a Pearson correlation coefficient of 0.6003 for all {of the} nine languages.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 8
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia13f8.pdf}
\end{center}
\hangcaption{{Scatter} graph plotted {the} data (average length per token vs effectiveness) from {experiments} on POS tagging with an approximate line.}
\label{figure:analysis_graph_pos}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 25
\begin{table}[t]
\input{13table25.tex}%
\caption{LAS scores and effectiveness in the additional languages on dependency parsing.}
\label{table:analysis_dep_add_lang}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Testing}
We conduct statistical hypothesis tests for the obtained correlation coefficients.
Our hypothesis is correct when {the} Pearson's correlation coefficient is more than 0.5.
In this testing, the null hypothesis is that the population correlation coefficient is equal to 0.5{, whereas} the alternative hypothesis is that the population correlation coefficient is more than 0.5{.}
{Based on these,} we conduct a statistical significance one-sided test where the significance level is 5\%.
We can reject the null hypothesis for POS tagging because we obtain {a} $p$-value of 0.0091.
Thus, our hypothesis {for POS tagging is correct}.
However, we cannot reject the null hypothesis for dependency parsing because we obtain {a} $p$-value of 0.3619.
Thus, our hypothesis {for dependency parsing is not correct}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 9
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 9
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia13f9.pdf}
\end{center}
\hangcaption{{Scatter} graph plotted {the} data (average length per token vs effectiveness) from {experiments} on dependency parsing with an approximate line.}
\label{figure:analysis_graph_dep}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{{For DRS parsing in Italian, why is character-level information used the most?}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Hypothesis}
The character-employed models in Italian {exhibited} larger {values of} effectiveness on DRS parsing.
We hypothesize that this tendency is caused by {the} \textbf{conjugation of verbs}{, of which} Italian {has the most} varied in the four languages.
{To demonstrate, whereas}
Table \ref{table:amare_conjugation} shows all of {the} simple conjugations of the Italian verb \textit{amare} (\textit{love} in English){,}
Italian verbs {also} have present {participle}, past {participle}, and \textit{gerundio} (gerund) {forms}.
For example, \textit{amare} {can be conjugated} into present {participle} \textit{amato}, past {participle} \textit{amante}, and gerund \textit{amando}.
Because conjugated forms are tokenized differently in the multilingual BERT tokenizer, some conjugated forms in the test set might be handled as out-of-vocabulary words.
Table \ref{table:analysis_amare_tok} shows each conjugated form and its tokens for the indicative present tense of the Italian verb \textit{amare} (\!\textit{love} in English).
We can find that the conjugated verb remains as a token for singular subjectives but is separated into two tokens for plural subjectives.
Such variation of conjugations in Italian is considered to {have caused the} significant effectiveness of character-level information.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 26
\begin{table}[t]
\input{13table26.tex}%
\caption{{Conjugations} of Italian verb \textit{amare}. sg. and pl. {denote} singular and plural, respectively.}
\label{table:amare_conjugation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 27
\begin{table}[t]
\input{13table27.tex}%
\hangcaption{Conjugated forms and their tokens by the multilingual BERT tokenizer for the indicative present tense of \textit{amare}.}
\label{table:analysis_amare_tok}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Method}

To {assess} whether the {aforementioned} hypothesis is true, we conduct an experiment {on} the PMB v3 data where each verb is converted to its lemma (intransitive form).
For example, the English sentence \textit{I played the piano} is converted to \textit{I play the piano}, and the Italian sentence \textit{ti amo} (\textit{I love you} in English) is converted to \textit{ti amare}.
Note that the Italian sentence \textit{ti amo} has only an object \textit{ti} (\textit{you}) and a verb \textit{amo} (\textit{love}) and does not have a subject because we can infer it from the form of \textit{amo}, the {first-person} singular form of \textit{amare}.
We aim to fix the form with one token in all conjugated occurrences rather than handling tense correctly.
We use Stanza \cite{qi-etal-2020-stanza} for {the} lemmatization.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Result}

Table \ref{table:analysis_drs_vlemma_effectiveness} shows the experimental {results}.
Focusing on {the} effectiveness, where each verb is replaced {with} its lemma prepared for this analysis (Vlemma),
we can observe a negative difference between two values of effectiveness only {for} Italian ($-8.80$),
where we observe significant effectiveness on DRS parsing (Normal).
Nevertheless, this difference is not relatively low {owing} to the differences in the other languages.
If {the} conjugations of verbs cause large effectiveness, we should confirm {the same for} the languages where the {numbers of conjugations are} large, {i.e.}, Italian (57) and German (14).
However, we can observe a positive difference between the two values of effectiveness (14.02) using character-level information in the data prepared for this analysis (Vlemma).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 28
\begin{table}[t]
\input{13table28.tex}%
\hangcaption{F-scores {of models} with correct and {with} unobserved character-level information, effectiveness of character-level information {on} data where all verbs are replaced {with} their lemmas, effectiveness from Table \ref{table:result_drs_effectiveness}, and differences between {the} two values of effectiveness on DRS parsing.}
\label{table:analysis_drs_vlemma_effectiveness}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

From the results of the {aforementioned} experiment, we conclude that the number of conjugations of verbs does not {contribute much to the} large effectiveness of character-level information on DRS parsing.
One possible reason is {the} unbalanced quality between languages in the PMB dataset.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}
\label{section:related_work}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Character usage in language models}
\label{section:rw_characters}

Character-based neural models calculate character embeddings throughout a corpus to obtain {character-level information} {that major language models do not read}.
\citet{sutskever-etal-2011-generating}, one of the earliest attempts to obtain character embeddings, calculates representations of characters by using recurrent neural networks.
\citet{zhang-etal-2015-character} and \citet{kim-etal-2016-character} also obtained character embeddings by convolutional neural networks.
\citet{ling-etal-2015-finding} proposed a methodology to obtain character embeddings.
{On the other hand}, \citet{cao-2023-best} developed a character-level language model by using a system in attention mechanism \cite{jaegle-etal-2022-perceiver}.

Some studies have explored characters in languages {other than} English more deeply.
{For example, Si et al. (2023)} {sought} to tokenize a Chinese character {using} its \textit{pinyin}, the writing system in Latin {script} for {the} Chinese {language}.

From tokenization to machine translation, information within characters has contributed to performance improvements in neural models.
{Moreover}, thanks to works across multiple languages, such as Universal Dependencies \cite{nivre-etal-2016-universal}, characters are {now} known to be useful in various languages {and can be employed with} a single architecture of models.
One of the reasons {behind the usiefulness of} character-level information {to} models is the capability {for} out-of-vocabulary words \cite{vania-etal-2018-character}.
{For} tokenization, {which is the} task to segment text into words, \citet{evang-etal-2013-elephant} proposed {focusing} on each character in an input text, labeled with symbols that express chunks.
This method achieved performance improvements in English, Italian, and Dutch.
{For} POS tagging, \citet{santos-etal-2014-learning} obtained word representations from character-level embeddings and achieved high performance in English and Portuguese.
\citet{plank-etal-2016-multilingual} obtained character embeddings calculated {using} bi-LSTMs and observed higher scores{, compared to those of models} without character-level representations in 22 languages, including five non-Indo-European languages.
{For} dependency parsing, \citet{ballesteros-etal-2015-improved} obtained character embeddings {using} the {method of Ling et al. (2015)} and achieved high performance with character-level information in various languages with different scripts: Latin, Arabic, Hebrew, and Korean.
{For} translation, \citet{luong-manning-2016-achieving} adopted character-level representations only for some words and achieved high performance {on} an English-Czech translation dataset.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Exploring commonalities and differences between languages}
\label{section:rw_language}

A major part of NLP studies that use or mention the backgrounds of languages relies on linguistic typology.
Linguistic typology is the field of classifying the {languages of the world} based on their syntactical features.
{The} World Atlas of Language Structures {(WALS, Dryer and Haspelmath 2013)},\footnote{\url{https://wals.info/}} a data bank piling language classification from many perspectives, is often used by NLP studies.
\nocite{wals-2013}
For example, \citet{zhang-barzilay-2015-hierarchical} trained a multilingual dependency parser by adding typological features from WALS, such as {the} order of subject, object, and verb.
The WALS data {are} also used for learning languages with relatively low resources.
\citet{ammar-etal-2016-many} defined language embeddings, which include all typological properties stored in WALS, and {used them to train a model}.
\citet{ponti-etal-2019-modeling} {argued} that typological information benefits not only morphological and syntactic tasks such as dependency parsing but also tasks requiring language transfer.

{Thus far, previous} studies that explore linguistic features, such as those in WALS from resulting values, {have not been} common.
\citet{plank-etal-2016-multilingual} clustered languages into their language family and analyzed results with respect to the families but did not attempt to derive any linguistic features.
Our prior work \cite{kurosawa-yanaka-2023-character}, which explored a cause {for the} value differences between languages, also did not induce any features.

We can see several studies that explore linguistic features from trained neural networks.
\citet{stanczak-etal-2022-neurons} tried to {determine} common representations of morphosyntactic categories across 43 languages in multilingual BERT \cite{devlin-etal-2019-bert} and XLM-R \cite{conneau-etal-2020-unsupervised}, {which are} multilingual pre-trained models.
\citet{gonen-etal-2022-analyzing} explored whether {multilingual BERT} has common representations for gender across English, French, and Spanish{, whereas}
\citet{varda-marelli-2023-data} explored whether {multilingual BERT} and XLM-R have common representations for number agreement across English, German, French, Hebrew, and Russian.
As a result, each of them found some common representations across multiple languages in the multilingual language models.
Our study is similar to {that by} \citet{gutierrez-vasques-etal-2023-languages} in attempting to find some different language-specific features from the results of byte-pair encoding (BPE) compression \cite{gage-1994-new}.
{For} each of {the} 47 languages {that they included in their study}, they ran the algorithm of BPE compression until 200 merges between bytes {occurred} and plotted them into 3D space {consisting} of {the productivityof each word}, degree of variation of the subword contained in {the} word, {idiosyncrasy}, a degree of deviation of the subword contained in {the word}, and frequency.
They revealed a novel tendency that is different from clustering based on WALS.
However, our study differs {from Gutierrez-Vasques et al. (2023)} in {that we explore} such features {based on the performance in a number of} natural language understanding tasks.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion and Future Work}
\label{section:conclusion}

In this study, we focused on character-employed neural models and {determined} how much these models use character-level information.
{Whereas earlier} studies {defined} the contribution of character-level information to the models as {the} difference between the {performance} with and without the information, we proposed \textit{effectiveness}, {which is} defined as {the} ratio of {the} difference between the performances with the information and {with} unobserved character-level information, to consider the extent {to} which the models use character-level information. 
By evaluating {existing} character-employed models in multiple languages on multiple NLP tasks, we found some peculiarities of effectiveness in German and Italian.
We hypothesized these peculiarities and known linguistic features, which {had} rarely been done in {past} studies.
We {determined} via hypothesis tests that the longer the average {character length} per word of a language, the larger the effectiveness on POS tagging, but not on dependency parsing{, in that language}.
On the other hand, we could not find any correlation between the variety of verb conjugations and effectiveness on DRS parsing.
{We believe that our methods help to understand the extent to which newly developed language models correctly handle character-level information across languages.}

It is noteworthy to mention that our study is restricted by several difficulties involving linguistic nature.
First, {corpora that have parallelized multilingual data are thus far insufficient}.
Each of the three datasets we used in this study is not completely parallelized in terms of the number, quality, and complexity of cases for each {included} language.
Although researchers have emphasized the importance of constructing such parallelized corpora, it has been difficult because {this undertaking} requires much time and many linguistic experts for each language.

Second, our study focused only on the languages with Latin alphabets and {uncovered only} a limited number of possible linguistic features that cause peculiarities.
In {reality}, we can find many scripts other than Latin, such as Chinese, Cyrillic, Greek, Hebrew, Korean, {and} Thai, {among others}.
{Moreover}, the {languages of the world} can be classified according to their writing systems.
{The writing systems of many} European languages{, such as} Latin, Cyrillic such as Russian, and Greek are \textit{alphabets}, a {type} of segmental scripts.
In an alphabet, each letter represents a vowel or consonant.
However, we can find {other types} of segmental scripts: \textit{abjad} (\textit{consonantary}) and \textit{abugida} (\textit{alphasyllabary}).
{Furthermore}, there are languages whose writing {systems do} not involve segmental scripts{; these writing systems include} feature scripts and logograms.
Feature scripts are {composed of} phonograms in units smaller than a letter.
Korean is the only language {that uses} feature scripts.
In logograms, each letter represents a word or morpheme.
Chinese and Japanese (only for \textit{kanji}) are logogram languages.
Different writing systems may reflect different tendencies in character-level information.
It would be interesting to analyze effectiveness of character-level information in character-employed models for such languages, which we leave for future work.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

This work was supported by JST,
PRESTO grant number JPMJPR21C8, Japan.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{13refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix
\label{section:appendix}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 29
\begin{table}[b]
\input{13table29.tex}%
\caption{{Two splits we set for the previous experiments.}}
\label{table:prev_exp_splits}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 30
\begin{table}[b]
\input{13table30.tex}%
\caption{{Reliance values of \texttt{a} for the two splits.}}
\label{table:prev_exp_res}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{{Previous Experiments on DRS Parsing}}
\label{appendix:data_aligned}

We described the results of the experiments on DRS parsing, in which the numbers of data were aligned across the languages.
We set two splits as shown in Table \ref{table:prev_exp_splits} and conducted an experiment using the method described in Section \ref{section:mt_reliance}.
We only reported reliance values of the character \texttt{a} for English, German, Italian, and Dutch, as shown in Table \ref{table:prev_exp_res}.
Note that we could not perform an experiment for Dutch in Split \#2 due to insufficient data.
In Split \#1, the largest reliance was observed for Italian.
In Split \#2, the largest reliance was observed for Dutch, and the second largest one for Italian.
Based on these observations, the tendencies we observed in Section \ref{section:result} might not be caused by the quality of the PMB dataset.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Computation Time}
\label{appendix:computation_time}

Table \ref{table:app_setup_compute_time} shows {the computation times in the experiments for each task} in each language.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Full Results}
\label{appendix:full_results}

Table \ref{table:app_result_pos_full}, \ref{table:app_result_dep_full}, and \ref{table:app_result_drs_full} show all reliance {values for} each character in the experiments described in Section \ref{section:mt_reliance} {on} POS tagging, dependency parsing, and DRS parsing, respectively.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{{Effectiveness in Uncased Sentences}}
\label{appendix:uncased}

We experimented on POS tagging and dependency parsing by changing sentences in the data into uncased (all characters are lowercase).
Table \ref{table:app_result_uncased} shows the obtained values of effectiveness in English, German, Italian, and Dutch.
The effectiveness in German is still the highest on both tasks.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 31
\begin{table}[h]
\input{13table31.tex}%
\hangcaption{{Computation times} for an experiment {for each task in each language}. POS stands for POS tagging, DEP stands for dependency parsing, DRS (c) stands for coarse training on DRS parsing, and DRS (f) stands for fine training on DRS parsing. }
\label{table:app_setup_compute_time}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 32
\begin{table}[p]
\input{13table32.tex}%
\hangcaption{All reliance (\%) in the experiment described in Section \ref{section:mt_reliance} on POS tagging (Full table of Table \ref{table:result_pos_disturbance}). {Characters in columns with bold values are unobserved characters for each language.}}
\label{table:app_result_pos_full}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 33
\begin{table}[p]
\input{13table33.tex}%
\hangcaption{All reliance (\%) in the experiment described in Section \ref{section:mt_reliance} on dependency parsing (Full table of Table \ref{table:result_dep_disturbance}). {Characters in columns with bold values are unobserved characters for each language.}}
\label{table:app_result_dep_full}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 34
\begin{table}[p]
\input{13table34.tex}%
\hangcaption{All reliance (\%) in the experiment described in Section \ref{section:mt_reliance} on DRS parsing (Full table of Table \ref{table:result_drs_disturbance}). {Characters in columns with bold values are unobserved characters for each language.}}
\label{table:app_result_drs_full}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\clearpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 35
\begin{table}[t]
\input{13table35.tex}%
\caption{Effectiveness on POS tagging and dependency parsing in uncased sentences.}
\label{table:app_result_uncased}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Tomoya Kurosawa}{%
Tomoya Kurosawa received B.E. and M.E. degrees in information science from the University of Tokyo, Japan, in 2022 and 2024, respectively.
After working at LY Corp., he is currently working as a natural language processing researcher at SB Intuitions Corp. since July 2024.
His research interests include computational linguistics and natural language processing.
}

\bioauthor[:]{Hitomi Yanaka}{%
Hitomi Yanaka received the Ph.D. degree in engineering from the University of Tokyo, Japan, in 2018.
From 2018 to 2021, she was a postdoctoral researcher at the Riken Center for Advanced Intelligence Project (AIP).
She is currently an associate professor (Excellent Young Researcher) at the Graduate School of Information Science and Technology, the University of Tokyo.
She is also a visiting researcher at the Riken AIP.
Her research interests include computational linguistics and natural language processing, with focus on Natural Language Inference.
}
\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate





\clearpage
























\clearpage









































































\end{document}

