\documentclass[english]{jnlp_3.6}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage{nccmath}
%%%%\usepackage{amssymb}
%%%%\usepackage{float}
%%%%\usepackage{booktabs}
\usepackage{multirow}
%%%%\usepackage{subfiles}
%%%%\usepackage{xcolor}
    \usepackage[dvipdfmx]{xcolor}
%%%%\usepackage{subcaption}
\usepackage{arydshln}  % dashed line
%%%%% \usepackage{subfigure}
%%%%% my commands
%%%%\newcommand{\draft}[1]{\textcolor{gray}{#1}}
%%%%\newcommand{\verify}[1]{\textcolor{red}{#1}}
%%%%\makeatletter
%%%%\def\blfootnote{\xdef\@thefnmark{}\@footnotetext}
%%%%\makeatother
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{30}
\Number{2}
\Month{June}
\Year{2023}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2022}{8}{1}
\revised{2022}{11}{28}
\accepted{2023}{1}{16}

\setcounter{page}{372}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Character-Based Thai Word Segmentation with Multiple Attentions}
\eauthor{Thodsaporn Chay-intr\affiref{lrlab} \and Hidetaka Kamigaito\affiref{naist} \and Manabu Okumura\affiref{lrlab-fac}} 

\eabstract{%
Character-based word segmentation models have been extensively applied to Asian languages, including Thai, owing to their promising performance. These models estimate the word boundaries from a character sequence; however, a Thai character unit in a sequence has no inherent meaning, in contrast with word, subword, and character cluster units that represent more meaningful linguistic information. In this paper, we propose a Thai word segmentation model that uses various types of information, including words, subwords, and character clusters, from a character sequence. Our model applies multiple attentions to refine segmentation inferences by estimating the significant relationships among characters and various unit types. We evaluated our model on three Thai datasets, and the experimental results show that our model outperforms other Thai word segmentation models, demonstrating the validity of using character clusters over subword units. A case study on sample Thai text supported these results. Thus, according to our analysis, particularly the case study, our model can segment Thai text accurately, while other existing models yield incorrect results that violate the Thai writing system.}
\ekeywords{Thai Word Segmentation, Representation Learning}

\headauthor{Chay-intr et al.}
\headtitle{Character-Based Thai Word Segmentation with Multiple Attentions}

\affilabel{lrlab}{}{School of Engineering, Tokyo Institute of Technology}
\affilabel{naist}{}{Division of Information Science, Nara Institute of Science and Technology}
\affilabel{lrlab-fac}{}{Institute of Innovative Research, Tokyo Institute of Technology}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

\renewcommand{\thefootnote}{}
\footnote[0]{Part of this work was presented at the 13th Recent Advances in Natural Language Processing (RANLP 2021) conference \protect\cite{chay-intr-etal-2021-character}.}
\renewcommand{\thefootnote}{\arabic{footnote}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Word segmentation is an early step in natural language processing (NLP) for most Asian languages, such as Thai, Japanese, and Chinese. 
In such languages, characters are fundamental units, which, in various combinations, can form new words with different roles, meanings, or grammatical properties.
Failing to precisely segment word boundaries tends to affect the performance of downstream tasks.
For example, Thai running text contains neither essential word boundaries nor sentence periods.
Spaces are arbitrarily used to separate words, phrases, clauses, and sentences.
These characteristics make word segmentation in Thai more difficult than in other languages such as English, German, and Finnish, which have structured spaces and periods to identify word and sentence boundaries, respectively.

Thai word segmentation can be categorised as a sequence-labelling task that assigns a word boundary label to each character in a character sequence.
Each character can be assigned by a specific tagging scheme; for example, a fine-grained tagging scheme, that is, beginning, inside, end, and singleton (BIES) \cite{Xue2003}, as shown in Figure~\ref{fig:thbies}.
In addition, several tagging schemes, such as character and word level, have been used to avoid overestimation in domain-adaptation scenarios  \cite{limkonchotiwat-etal-2020-domain,limkonchotiwat-etal-2021-handling}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{30-2ia4f1.pdf}
\end{center}
\hangcaption{Thai word segmentation as a sequence-labeling task with BIES (beginning, inside, end, and singleton) tagging scheme.}
\label{fig:thbies}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Neural-network models have been applied to character-based Thai word segmentation with promising results.
For example, \citeA{Jousimo2017} has applied bidirectional recurrent neural networks with gated recurrent units (GRUs), while \citeA{Chormai2019} proposed AttaCut \cite{Chormai2019},\footnote{\url{https://github.com/PyThaiNLP/attacut}} a convolutional-neural-network (CNN)-based model, the main advantage of which is faster and more accurate word deduction powered by DeepCut \cite{Kittinaradorn2019}.
Although neural network models with linguistic knowledge can perform character-based word segmentation almost perfectly, adapting models with pre-trained neural networks, such as word vectors and language models, remains useful \cite{Shao2018,yan-etal-2020-bert}.

Despite the successful application of pre-trained models (PTMs), such as the bidirectional encoder representation from transformers (BERT) \cite{devlin-etal-2019-bert}, for word segmentation mostly on Chinese datasets \cite{yang-etal-2019-bert,qiu-etal-2020-concise,ke-etal-2020-unified,huang-etal-2020-towards,ke-etal-2021-pre}, only \citeA{seeha-etal-2020-thailmcut} have proposed a transfer-learning approach for Thai word segmentation using PTMs for character-based word segmentation. 
Although their model exhibits state-of-the-art performance, it merely uses characters and does not use other information such as words and subwords \cite{sennrich-etal-2016-neural,kudo-2018-subword}.
Moreover, reliance on the transfer-learning method enables interaction  only with the input and output layers of the models; everything in between thus behaves as a ``black box,'' which is considered a limitation of such models \cite{limkonchotiwat-etal-2020-domain}. 
\citeA{limkonchotiwat-etal-2020-domain,limkonchotiwat-etal-2021-handling} have proposed alternative approaches to focus particularly on domain-dependent word segmentation and lessen the black box limitation of transfer learning.
Their work obtained results that were comparable to those of the transfer learning method.

Linguistic information other than character sequence, for example, word, subword, and Thai character clusters (CCs) \cite{Theeramunkong2000}, has also been successfully used for word segmentation and downstream tasks \cite{Sutantayawalee2015,Lapjaturapit2018,Nararatwong2018,yang-etal-2019-subword,li-etal-2019-word-segmentation}.
Furthermore, an attention mechanism \cite{Bahdanau2015} has been effectively applied to various downstream tasks, particularly character-based sequence-labelling tasks that exploit more than one linguistic characteristic \linebreak
\cite{higashiyama-etal-2019-incorporating,tian-etal-2020-joint}.

To exploit multiple types of linguistic information, we propose a character-based Thai word-segmentation model with multiple attentions that jointly uses corresponding words, CCs, and subwords.
Our model is based on the bidirectional long short-term memory with a conditional random field (BiLSTM-CRF) architecture, which is the baseline model for this study because it has been successfully applied in sequence-labelling tasks for Thai and other languages \cite{Jousimo2017,Nararatwong2018,higashiyama-etal-2019-incorporating,seeha-etal-2020-thailmcut,tian-etal-2020-joint}.

Transformer-based PTMs have not yet been incorporated into Thai word segmentation.
Thus, we expand the potential of PTMs, particularly Transformer-based PTMs, in Thai word segmentation, by applying BERT in our model.
To the best of our knowledge, this work is the first to adopt PTMs for Thai word segmentation. 

The contributions of this study are as follows:
\begin{itemize}
%%%%    \setlength\itemsep{0.01em}
    \item We use word, CC, and subword characteristics with multiple attentions to estimate the relationships between characters in character-based Thai word segmentation.
    \item We conduct experiments to evaluate the validity of using CC over subword units in terms of segmentation performance and a segmentation result that does not violate the rules of the Thai writing system.
    \item Our model demonstrates an improvement over the baseline model and outperforms previous works in Thai word segmentation.
    \item Our proposed model outperforms recent works that focus on domain-dependent word segmentation on Thai domain-dependent datasets.
    \item We incorporate multilingual BERT (M-BERT) into character-based Thai word segmentation and conduct a comparison between our BiLSTM-based model and BERT-based model. Moreover, we find that most BERT-based models are unable to outperform our proposed BiLSTM-based model.
    \item Our code is publicly available.\footnote{\url{https://github.com/tchayintr/thwcc-attn}}
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Thai Word Segmentation Revisited}

In the early stages of Thai word segmentation, dictionary-based learning techniques were used along with machine-learning techniques, such as Markov models \cite{Kawtrakul1997}, decision trees \cite{Sornlertlamvanich2000,Theeramunkong2001}, and CRFs \cite{Haruechaiyasak2008}.
CRFs have been shown to be particularly suitable for Thai sequence-labelling tasks \cite{Kruengkrai2006,Haruechaiyasak2009,Kruengkrai2009,Nararatwong2018}.

In parallel with CRFs, neural network models such as CNNs \cite{Kittinaradorn2019,Chormai2019}, LSTM \cite{Treeratpituk2017},\footnote{\url{https://github.com/pucktada/cutkum}} and BiLSTM \cite{Jousimo2017} have been applied character-based Thai word segmentation, with excellent results.
However, using additional knowledge, such as CC \cite{Lapjaturapit2018,Nararatwong2018}, transfer learning \cite{seeha-etal-2020-thailmcut}, and stacking ensemble \cite{limkonchotiwat-etal-2020-domain,limkonchotiwat-etal-2021-handling}, along with neural network models, may improve model performance.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Character Clusters in Thai Word Segmentation}

Thai has various types of characters, such as consonants, vowels, tones, and special characters.
A word can be formed using a combination of these characters.
Thai also exhibits unique linguistic phenomena; for example, some sequential characters tend to be indivisible units.
Thus, \citeA{Theeramunkong2000} introduced the concept of a CC, which is a set of predefined rules for indivisible units based on the Thai writing system.

A CC is smaller than a word but larger than a character.
This concept is roughly comparable to a subword unit that is also in the middle of a character, as well as a word in terms of its length. 
Using subword units as well as CCs \cite{Theeramunkong2004,Sutantayawalee2015,Lapjaturapit2018,Nararatwong2018}, word-segmentation tasks can yield good segmentation performance \cite{yang-etal-2019-subword,li-etal-2019-word-segmentation}.
However, decomposing subword units from words requires specific settings, such as Byte Pair Encoding, whereas CCs do not require any additional settings.

A CC helps avoid segmenting that might violate a writing system \cite{Limcharoen2009}, while subword units will likely not thoroughly exploit language morphology \cite{provilkov-etal-2020-bpe}, which could generate noise and weaken the segmentation performance. 
This generally makes CCs smaller than subword units, as shown in Figure~\ref{fig:tccsub}, which enables a sample comparison of the segmentation results from coarse to fine (top-down).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[t]
\begin{center}
\includegraphics{30-2ia4f2.pdf}
\end{center}
\hangcaption{Comparison of segmentation results. S, W, Sub, CC, and C indicate segmentation levels of sentence, word, subword, character cluster, and character, respectively.}
\label{fig:tccsub}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Pre-trained Models in Thai Word Segmentation}

Recent studies have shown that PTMs are useful for many downstream tasks, particularly Chinese Word Segmentation (CWS) \cite{yang-etal-2019-bert,qiu-etal-2020-concise,ke-etal-2020-unified,huang-etal-2020-towards,ke-etal-2021-pre}.
\citeA{yang-etal-2019-bert} adopted a multi-layer BERT to subword-based CWS.
Compared with several neural models, that is, CNNs and BiLSTM, their work yielded better segmentation performance.

As mentioned prior, to the best of our knowledge, only \citeA{seeha-etal-2020-thailmcut} has applied PTMs to Thai word segmentation. 
They pre-trained a character-based Language Model with BiLSTM architecture and then transferred its parameters for fine-tuning character-based word segmentation. 
This approach achieves state-of-the-art performance on the BEST2010 dataset\footnote{\url{https://thailang.nectec.or.th}} which is the most well-known Thai dataset for evaluating Thai word segmentation.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Attention Mechanism}

An attention mechanism was initially proposed 
\pagebreak
by \citeA{Bahdanau2015} for neural machine translation, focusing on proper parts of sentences and, particularly, of long sentences. 
The attention mechanism estimates the attention vectors between source and target states.
It has been successfully applied to downstream tasks, including machine translation \cite{luong-etal-2015-effective,Vaswani2017}, constituency parsing \cite{kitaev-klein-2018-constituency}, and sequence labelling \cite{higashiyama-etal-2019-incorporating,tian-etal-2020-joint-chinese}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Approach}

Incorporating candidate words into the character-based BiLSTM-CRF architecture with an attention mechanism, as shown in Figure~\ref{fig:base-model}, may yield good segmentation performance \linebreak
\cite{higashiyama-etal-2019-incorporating}.
An attention mechanism has the advantage of being flexible for use with additional linguistic information such as CCs and subword units.
Thus, we used the concept of CCs with an attention mechanism in character-based word segmentation, as shown in Figure \ref{fig:main-model}, by extending the BiLSTM-CRF architecture with word attention from \citeA{higashiyama-etal-2019-incorporating}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[b]
\begin{center}
\includegraphics{30-2ia4f3.pdf}
\end{center}
\hangcaption{Character-based BiLSTM-CRF architecture used as a baseline, where $x$ represents individual characters from an input sequence and $y$ represents the predicted label for each character; $\textbf{e}^{c}$ and $\textbf{h}$ indicate an embedding representation and BiLSTM-encoded representation for each character, respectively.} 
\label{fig:base-model}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[t]
\begin{center}
\includegraphics{30-2ia4f4.pdf}
\end{center}
\hangcaption{Proposed model, integrating word and CC attentions into a character-based BiLSTM-CRF architecture.}
\label{fig:main-model}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Our model estimates that \textit{CC-integrated character vectors} ${\textbf{z}}$ are incorporated on top of \textit{word-integrated character vectors} ${\textbf{g}}$, which are almost identical in their architecture.
In the sections that follow, we discuss the major components of our model, that is, the character-embedding layer, word- and CC-embedding layers, BiLSTM layers for character representation, attention integrations with the BiLSTM layers for integrated representations, CRF layer, and optional BERT layers.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Character-Embedding Layer}

Given a sentence $s$ with $n$ characters that can be represented as $x_{1:n} \equiv (x_1, x_2, \dots, x_n)$, each character $x_{i} \in x_{1:n}$ is transformed into a character embedding $\textbf{e}_{i}^{c}$ of a $d_{c}$-dimensional vector using a lookup table operation \cite{Bengio2003,Collobert2011}.
The lookup table is defined as $\text{E}^{c} \in \mathbb{R}^{d_{c} \times |V_c|}$, where $d_{c}$ denotes the dimensions of the embeddings and $V_c$ denotes a vocabulary of characters.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Word- and CC-Embedding Layers}\label{section:word-cc-embed}

Using the word-embedding layer as an example, let $V_{w}$ be a word vocabulary.
Given the character sequence $x_{1:n}$, $V_{w}$ is searched for words within a maximum word length $K$, corresponding to that of the character subsequence. 
A candidate word list $\mathcal{W}_{x}$ $\equiv (w_{1}, \dots, w_{m})$ (each of size within $K$) with $m$ candidate words is then obtained, as shown in Figure~\ref{fig:main-model}.
Each word $w_{j} \in \mathcal{W}_{x} \subseteq V_{w}$ was transformed into a word embedding $\textbf{e}^{w}$ of a $d_{w}$-dimensional vector. 
The word-embedding matrix is defined as $\text{E}^{w} \in \mathbb{R}^{d_{w} \times |V_{w}|}$, where $d_{w}$ denotes the dimensions of the embeddings.
A similar procedure is applied to obtain a candidate CC list $\mathcal{CC}_{x}$, which is transformed into a CC-embedding layer $\textbf{e}^{cc}$ of a $d_{cc}$-dimensional vector. 
The CC-embedding matrix is defined as $\text{E}^{cc} \in \mathbb{R}^{d_{cc} \times |V_{cc}|}$, where $d_{cc}$ denotes the dimensions of the embeddings, and $V_{cc}$ denotes a CC vocabulary.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{BiLSTM Layers for Character Representation}\label{section:bilstm-for-char}

The character embedding sequence $\textbf{e}_{1:n}^{c}$ is provided to the BiLSTM \cite{Hochreiter1997,Gers2000} layers to acquire the \textit{character context vectors} $\textbf{h}_{1:n}$.

The current character context vector $\textbf{h}_{i}^{l} \in \textbf{h}_{1:n}^{l}$ of the $l$-th layer BiLSTM can be computed bidirectionally, as follows:
\begin{equation}
    \begin{split}
        \textbf{h}^{l}_{i} &= \text{BiLSTM}(\textbf{h}^{l-1}_{1:n}, i) \\
        & \equiv \text{LSTM}_f(\textbf{h}^{l-1}_{1:n}, i) \\
        & \oplus \text{LSTM}_b(\textbf{h}^{l-1}_{n:1}, n-i+1),
    \end{split}
    \label{eq:bilstm}
\end{equation}
where $\textbf{h}^{0}_{1:n} = \textbf{e}_{1:n}^{c}$, $\text{LSTM}_{f}$ denotes forward LSTM, $\text{LSTM}_{b}$ denotes backward LSTM, $\oplus$ denotes concatenation, and $\textbf{h} \in \mathbb{R}^{2d_{r}}$ and $d_{r}$ are hyperparameters.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Attention Integrations with BiLSTM Layers for Integrated Representations}\label{section:attn-integration}

We use two attention integrations, word attention and CC attention, to estimate a \textit{word-integrated summary vector} $\textbf{a}_{i}^{w}$ and \textit{CC-integrated summary vector} $\textbf{a}_{i}^{cc}$, respectively, for each character in the character sequence.
These integrations, which are equal in architecture, summarise the relationships among characters, words, and CCs.

We apply the composition function \textit{weight concatenation} (WCON) \cite{higashiyama-etal-2019-incorporating,higashiyama-2022-wordseg} to estimate both summary vectors. 
This function produces a word-integrated summary vector based on the relationship between a character and the corresponding candidate word.
It can also be used implicitly to produce a CC-integrated summary vector based on the relationship between a character and the corresponding candidate words and candidate CCs.

Starting with word-attention integration, we estimate the word-importance score $u_{ij}^{w}$ and word-attention weight $\alpha_{ij}^{w}$ based on the character context vector $\textbf{h}_{i}$ and the candidate word embedding $\textbf{e}_{j}^{w}$, as follows:
\begin{gather}
    u_{ij}^{w} = \textbf{h}_i^{T}W_{a}^{w}\textbf{e}_{j}^{w},
    \label{eq:word-important} \\
    \alpha_{ij}^{w} = \frac{\delta_{ij}\exp(u_{ij}^{w})}{\sum_{k=1}^{m}\delta_{ik}\exp(u_{ik}^{w})},
    \label{eq:word-attn-weight}
\end{gather}
where $W_{a}^{w} \in \mathbb{R}^{2d_{r} \times d_{w}}$ denotes a trainable weight matrix and $\delta_{ij} \in \{0,1\}$ indicates whether character $x_{i}$ is included in the candidate word $w_{j}$.
The word-integrated summary vector $\textbf{a}_{i}^{w}$ for the character $x_{i}$ can be calculated as
\begin{equation}
    \textbf{a}_{i}^{w} = \text{WCON}^{w}(x_{i}, \{w_{j}\}_{j=1}^{m}) {=} \bigoplus_{l=1}^{L^{w}}\alpha_{i,i_{l}}^{w}\textbf{e}_{i_{l}}^{w},
    \label{eq:wcon}
\end{equation}
where \{$w_{j}$\} = $\mathcal{W}_{x}$. 
If $K^{w}$ is the maximum word length, then $L^{w} = \sum_{k=1}^{K^{w }}k$. The symbol $\bigoplus$ denotes concatenation and $i_{l}$ is the corresponding index of the candidate word list $\mathcal{W}_{x}$ for character $x_{i}$, that is, $\{w^{\prime}_{1},\dots,w^{\prime}_{L^{w}}\} \equiv \bigcup_{k=1}^{K^{w}}\bigcup_{s=-k+1}^{0}\{x_{i+s:i+s+k-1}\}$.
A zero vector is applied to Equation \ref{eq:wcon} when $w^{\prime}_{l} \notin V_{w}$.

For example, to compose the list of words $w^{\prime}_{4}$ containing a character $x_{4}$ for estimating the word-importance score $u^{w}_{4,j}$, word-attention weight $\alpha^{w}_{4,j}$, and summary vector $\textbf{a}^{w}_{4}$, let the candidate word list $\mathcal{W}_{x} = \{w_{1},\dots,w_{8}\}$, maximum word length $K^{w} = 4$, and $L^{w} = \sum_{k=1}^{K^{w }}k = 10$. The procedure can be performed as follows:
\begin{equation}
    \begin{split}
        w^{\prime}_{4} &= \bigcup_{k=1}^{4}\bigcup_{s=-k+1}^{0} \{x_{4+s:4+s+k-1}\} \\
        & = \{x_{4:4},x_{3:4},x_{4:5},x_{2:4},x_{3:5},x_{4:6},x_{1:4},x_{2:5},x_{3:6},x_{4:7}\},
    \end{split}
    \label{eq:wcon-x4}
\end{equation}
where the words $x \in w^{\prime}_{4}$ contained in the training vocabulary are used.

We then use the BiLSTM layers to transform the word-integrated summary vectors $\textbf{a}^{w}$ into word-integrated character vectors $\textbf{g}$ using the corresponding character context vectors $\textbf{h}$, as follows:
\begin{equation}
    \textbf{g}_{i} = \text{BiLSTM}(\textbf{h}_{i}\oplus\textbf{a}_{i}^{w}).
    \label{eq:wcon-concat}
\end{equation}

However, candidate CCs that correspond to the character are used on top of $\textbf{g}_{i}$ as

\begin{gather}
    u_{ip}^{cc} = \textbf{g}_i^{T}W_{a}^{cc}\textbf{e}_{p}^{cc}, \label{eq:ccwcon-score}\\
    \alpha_{ip}^{cc} = \frac{\delta_{ip}\exp(u_{ip}^{cc})}{\sum_{k=1}^{q}\delta_{ik}\exp(u_{ik}^{cc})},
\end{gather}
where $W_{a}^{cc} \in \mathbb{R}^{4d_{r} \times d_{cc}}$ denotes a trainable weight matrix and $\delta_{ip} \in \{0,1\}$ indicates whether character $x_{i}$ is included in candidate CC $cc_{p}$.
The CC-integrated summary vector $\textbf{a}_{i}^{cc}$ for the character $x_{i}$ can be calculated as
\begin{equation}
    \textbf{a}_{i}^{cc} = \text{WCON}^{cc}(x_{i}, \{cc_{p}\}_{p=1}^{q}) {=} \bigoplus_{l=1}^{L^{cc}}\alpha_{i,i_{l}}^{cc}\textbf{e}_{i_{l}}^{cc},
    \label{eq:ccwcon}
\end{equation}
where \{$cc_{p}$\} = $\mathcal{CC}_{x}$. 
If $K^{cc}$ is the maximum CC length, then $L^{cc} = \sum_{k=1}^{K^{cc}}k$. Here, $i_{l}$ is the corresponding index of the potential CC list $\mathcal{CC}_{x}$ for character $x_{i}$, which is represented by $\{cc^{\prime}_{1},\dots,cc^{\prime}_{L^{cc}}\} \equiv \bigcup_{k=1}^{K^{cc}}\bigcup_{s=-k+1}^{0}\{x_{i+s:i+s+k-1}\}$.
As before, a zero vector is applied to Equation \ref{eq:ccwcon} when $cc^{\prime}_{l} \notin \mathcal{V}_{cc}$.

Next, we use additional BiLSTM layers to transform the CC-integrated summary vectors $\textbf{a}^{cc}$ into CC-integrated character vectors $\textbf{z}$ based on a cluster-integrated summary vector $\textbf{a}_{i}^{cc}$ and its corresponding word-integrated character vector $\textbf{g}_{i}$, as
\begin{equation}
    \textbf{z}_{i} = \text{BiLSTM}(\textbf{g}_{i}\oplus\textbf{a}_{i}^{cc}).
    \label{eq:ccwcon-concat}
\end{equation}
Finally, a CRF is used to estimate the probability of the optimal label sequence $y$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{CRF Layer}

A CRF \cite{Lafferty2001}, combined with explicit consideration of the correlations between adjacent labels, has been successfully applied to sequence-labelling-related tasks \cite{Collobert2011}.
We developed a CRF layer as follows. Let $A \in \mathbb{R}^{|T| \times |T|}$ be a transition matrix for correlations between adjacent labels, where $T$ denotes a set of all possible label sequences; for instance, $T = \{B, I, E, S\}$.
The CC-integrated character vector $\textbf{z}_{i}$ is transformed into an un-normalized label score $\textbf{s}_{i}$ of the $|T|$-dimensional vector for character $x_i$, as follows:
\begin{equation}
    \textbf{s}_i = W_{s}\textbf{z}_{i} + \textbf{b}_{s},
\end{equation}
where $W_{s} \in \mathbb{R}^{|T| \times 4d_{r}}$ denotes a trainable weight matrix and $\textbf{b}_{s} \in \mathbb{R}^{|T|}$ denotes a trainable bias. 
Given the input sequence $x_{1:n}$, the corresponding scores for the label sequence $y_{1:n}$ are computed from the transition matrix $A$ and segmentation label scores $\textbf{s}$, as follows:
\begin{equation}
    \text{score}(x, y) = \sum_{i=1}^{n}(A_{y_{i-1},{y_i}} + \textbf{s}_i[y_{i}]).
\end{equation}
The probability of the label sequence can then be obtained as
\begin{equation}
    P(y|x) = \frac{\text{score}(x,y)}{\sum_{y' \in T^{n}}\text{score}(x,y')}.
\end{equation}
The optimal label sequence $y^{\star}$ can be obtained by maximising the sentence score using the Viterbi algorithm:
\begin{equation}
    y^{\star} = \text{argmax}_{y \in T^{n}}\text{score}(x,y).
\end{equation}
The loss function $\mathcal{L}$ is minimised by backpropagation during the training process:
\begin{equation}
    \mathcal{L}(x,y) = - \log P(y|x).
\end{equation}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{BERT Layers}

In this study, BERT layers were used for comparison with BiLSTM-based models. 
A $\text{BERT}_{\text{base}}$ layer\footnote{\url{https://huggingface.co/bert-base-multilingual-cased}} was used to extract three types of representations: contextual-character-, word-, and CC-representation.
Specifically, the BiLSTM layers for character representation in Section \ref{section:bilstm-for-char} were replaced with a BERT-encoder layer to compute the \textit{character context vectors} $\textbf{h}_{1:n}$, as shown in Figure~\ref{fig:bert-model}. 
In this architecture, the current character context vector $\textbf{h}_{i}$ is computed as follows: 
\begin{gather}
    \textbf{h}_{i}^{0} = {\textbf{e}^{c}}_{i} + {\textbf{e}^{p}_{i}}, \\
    \textbf{h}_{i}^{l} = \text{Transformer}(\textbf{h}_{i}^{l-1}),
\end{gather}
where $l = \{1, 2, \dots, L\}$ is the number of transformer layers \cite{Vaswani2017}, $\textbf{e}^{c}$ is the embedding obtained from the BERT embedding layer, and $\textbf{e}^{p}$ is the positional embedding.
Because we adopted $\text{BERT}_{\text{base}}$, $L=12$ and $e^{c} \in \mathbb{R}^{d_{c} \times |V_{\text{BERT}}|}$, where $d_{c} = 768$ and $V_{\text{BERT}}$ denotes the vocabulary in $\text{BERT}_{\text{base}}$.
The word- and CC-representations (Section~\ref{section:word-cc-embed} are obtained simply from the BERT embedding layer. 
In other words, word- and CC-embedding layers (as well as subword-embedding layers) were replaced with a BERT-embedding layer.
When using the BERT embedding layers, the word- and CC-embedding matrices are defined as $\text{E}^{w} \in \mathbb{R}^{d_{w} \times |V_{\text{BERT}}|}$ and $\text{E}^{cc} \in \mathbb{R}^{d_{cc} \times |V_{\text{BERT}}|}$, where $d_{w}=768$ and $d_{cc}=768$.
Finally, the BERT-pooler layer and CRF layer conditionally project the BERT representations from the encoder layers into the optimal label sequences $y^{\star}$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[t]
\begin{center}
\includegraphics{30-2ia4f5.pdf}
\end{center}
\caption{Character-based word-segmentation model with incorporated BERT pre-trained model.}
\label{fig:bert-model}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiment}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Datasets} 

We trained and evaluated several versions of our model on three datasets: BEST2010, TNHC,\footnote{\url{https://attapol.github.io/tlc}} and VISTEC.\footnote{\url{https://github.com/mrpeerat/OSKut/tree/main/VISTEC-TP-TH-2021}}
BEST2010 corpus consists of four different domains of textual documents---articles, encyclopaedia, news, and novels---and has frequently been used to evaluate Thai word segmentation models.
TNHC and VISTEC are collections of Thai classical literature and social media datasets, respectively, for Thai text processing.
The latter two datasets were recently used to evaluate domain-dependent word segmentation.
For our purposes, we randomly split the BEST2010 corpus into three sets:\footnote{\url{https://resources.aiat.or.th/thwcc-attn/datasets}} 80\% for training, 10\% for validation, and 10\% for testing.
For TNHC and VISTEC, we applied that data splits as in \citeA{limkonchotiwat-etal-2020-domain,limkonchotiwat-etal-2021-handling}.
The sizes of all the datasets are listed in Table~\ref{tab:dataset}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[t]
\input{04table01.tex}%
\hangcaption{Data size, including the number of sentences (S), words (W), and characters (Ch), as well as the size of the vocabulary (V).}
\label{tab:dataset}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Subword-Integration}

Although subword units have been successfully applied to word-segmentation tasks, when the word dictionary already exists, they can generate noise that decreases segmentation performance.
Therefore, using the different versions of our proposed model, we conducted a comparison between using subword units and CCs because of their similarity.
For instance, let $V_{sw}$ be a subword vocabulary decomposed from a dataset.
We simply replaced the CC vocabulary $V_{cc}$ with the decomposed subword vocabulary $V_{sw}$.
Thus, a candidate list of subwords $\mathcal{SW}_{x}$ can be acquired and used for subword attention integration by applying Equations \ref{eq:ccwcon-score} and \ref{eq:ccwcon}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Attention Integration Order}

Since the attention integration is flexible, the integration order in our model can be switched.
For instance, in the base version, our model executes CC attention integration to estimate the relationship between characters and CCs before it conducts word attention integration.
This might affect the segmentation performance because each integration provides different information.
Thus, we implemented a swapped version of our model (Swap) in which the integration order was switched and compared its segmentation performance with that of the base model.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Pre-Trained Model Integration}

Fine-tuning a PTM for word segmentation has proven to be effective in previous research.
Thus, as mentioned prior, we conducted character-based Thai word segmentation with a PTM (shown in Figure \ref{fig:bert-model}), by simply replacing the character BiLSTM layers in Figure \ref{fig:base-model} with BERT layers.
We chose M-BERT\footnote{\url{https://huggingface.co/bert-base-multilingual-cased}} for our experiment because of its originality and accessibility. 
Furthermore, we applied this approach to the baseline model and our proposed model, where words, CCs, or subword units were used for model comparison.

In our experiment, each character $x_{i}$ was transformed into a BERT-token id.
Thereafter, each token id was encoded with a BERT encoder to obtain a character BERT representation.
Similarly, candidate word $w_{j}$ and either candidate CC $cc_{p}$ or candidate subword unit $sw_{p}$, obtained from Section \ref{section:attn-integration}, were also transformed into a BERT-token id.
Unlike the character BERT-representation, such candidate units are directly encoded into word, CC, and subword BERT-representations from the BERT-embedding layer.
The character BERT-representation, along with the corresponding word BERT-representation and either CC BERT-representation or subword BERT-representation, were used to estimate the importance scores $u$ and the attention weights $\alpha$ (Section \ref{section:attn-integration}).
Note that the BERT tokenizer was not used to tokenise the sentence $s$ into subword units.
Thus, the candidate word, CC, and subword unit, can be unknown tokens (UNK); in other words, tokenisation depends on which PTM was used as well as an approach to construct subword units in this experiment.

In addition, we included special tokens, including CLS and SEP tokens, in the training, validation, and test steps as the S (single-word) label of BIES tagging scheme.
However, these special tokens were not counted during the testing step to avoid overestimation.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Compared Models}

The following models were evaluated:

\begin{itemize}
%%%%    \setlength\itemsep{0.01em}
    \item\textbf{Baseline}:
        A character-based BiLSTM-CRF architecture, as shown in Figure~\ref{fig:base-model}.
    \item\textbf{Baseline w/ Word}:
        An extension of the Baseline that integrates word attention (BiLSTM-CRF with word attention) \cite{higashiyama-etal-2019-incorporating}.
    \item\textbf{Baseline w/o BiLSTM w/ BERT}:
        A character-based BERT-CRF architecture that replaces BiLSTM layers for character representation with BERT layers, as shown in Figure \ref{fig:bert-model}.
    \item\textbf{Baseline w/ BERT w/ Word}:
        The Baseline w/ Word that replaces BiLSTM layers for character representation with BERT (BERT-BiLSTM-CRF with word attention).
    \item\textbf{OURS}:
        Our proposed model that integrates word and CC attentions (BiLSTM-CRF with word and CC attentions), as shown in Figure~\ref{fig:main-model}.
    \item\textbf{OURS w/o CC w/ Sub}:
        Our proposed model that replaces the CC with various sizes of subword units (800--12,800) (BiLSTM-CRF with word and subword attentions).
    \item\textbf{OURS w/o Word}:
        Our proposed model that removes word attention (BiLSTM-CRF with CC attention).
    \item\textbf{OURS Swap}:
        Our proposed model that swaps the order of word and CC attentions.
    \item\textbf{OURS w/o CC w/ Sub Swap}: 
        OURS w/o CC w/ Sub model that swaps the order of word and subword unit attentions.
    \item\textbf{OURS w/ BERT}:
        Our proposed model that integrates word and CC attentions, and replaces BiLSTM layers for character representation with BERT (BERT-BiLSTM-CRF with word and CC attentions).
    \item\textbf{OURS w/ BERT w/o CC w/ Sub}:
        Our proposed model that integrates word and subword unit attentions, and replaces BiLSTM layers for character representation with BERT (BERT-BiLSTM-CRF with word and subword attentions).
    \item\textbf{Others}: Reproduced Thai non-neural/neural word-segmentation models, including well-known models, the state-of-the-art Thai word-segmentation model \cite{seeha-etal-2020-thailmcut}, and recent domain-adaptation models \cite{limkonchotiwat-etal-2020-domain,limkonchotiwat-etal-2021-handling}.
\end{itemize}

We used CCs from publicly available libraries, including \citeA{pythainlp} and TCCSEG,\footnote{\url{https://github.com/tchayintr/tccseg}} to build the CC vocabulary $V_{cc}$.
To generate the subword vocabulary $V_{sw}$ for each dataset, we decomposed raw sentences from the training sets into subword units of various sizes using byte-pair encoding \cite{sennrich-etal-2016-neural} implemented in SentencePiece \cite{kudo-richardson-2018-sentencepiece}. The generated subword vocabulary for each dataset was not shared with the other experimental datasets.

We used common hyperparameters for training the different versions of our proposed model (hereafter, referred to as ``our models''), as shown in Table~\ref{tab:hyp}.
Dropout \cite{Srivastava2014} was applied to the BiLSTM layers to avoid overfitting and non-recurrent layers \cite{Zaremba2015}.
The model parameters were optimised using the Adam optimiser \cite{Kingma2015}.
We trained our models up to 20 epochs and chose the best one based on a validation process involving word-level evaluation (CoNLL\footnote{\url{https://github.com/spyysalo/conlleval.py}}).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{04table02.tex}%
\hangcaption{Common hyperparameters for Baselines and our models (top/middle) with exclusive values for our models (bottom). CC hyperparameters can also be applied to subword-integration.}
\label{tab:hyp}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We evaluated our models on the test data using three evaluation metrics: word-level evaluation (CoNLL-format), as above; BIES tagging scheme (character-level evaluation); and bound (boundary-level evaluation) \cite{seeha-etal-2020-thailmcut}.
We also used binary-character-level evaluation similar to that of \citeA{limkonchotiwat-etal-2020-domain,limkonchotiwat-etal-2021-handling} to compare our model with recent Thai domain-adaptation word segmentation works.
Note that our F$_{1}$ scores were based on the micro-averaged F$_{1}$ scores for all evaluation matrices.
We conducted statistical significance tests using paired bootstrap resampling \cite{Koehn2004} on our results, particularly on OURS with state-of-the-art Thai word segmentation \cite{seeha-etal-2020-thailmcut}, OURS with Baseline w/ Word \cite{higashinaka-etal-2021-integrated}, and OURS w/o CC w/ Sub.
We set the resampling size to 100,000 iterations and the sample size for each resampling to 10\% of the test data.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Results and Discussion}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Main Results}\label{section:main-results}

Table~\ref{tab:main-results}\footnote{LexTo (\url{http://www.sansarn.com/lexto}), TLTK (\url{https://github.com/attapol/tltk}), Multi-cut, and New Multi-cut (\url{https://github.com/PyThaiNLP}) were used to produce non-neural segmentation results.} illustrates the evaluation results of the compared models on BERT2010, while \linebreak
Table~\ref{tab:main-results-desc} shows the architectures of the compared models.
According to these results, the best of our models was OURS, which performed better than all the other models.
From the statistical significance test results, we concluded that OURS surpasses the state-of-the-art model and the strong baseline models Baseline w/ Word and OURS w/o CC w/ Sub.
Although using subword units showed the potential to slightly improve the performance compared with that when using CCs, OURS outperformed OURS w/o CC w/ Sub in multiple runs and performed significantly better at p-level $< 0.05$. 
This indicates that CCs are better than subword units in terms of segmentation performance. Moreover, unlike incorporating subwords, CCs do not require hyperparameter adjustments.
OURS, which used word and CC information, outperformed Baseline w/ Word, indicating that CCs can be used to complement linguistic knowledge, particularly word information.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{04table03.tex}%
\hangcaption{Comparison between our models, baselines, and others on BEST2010. Best score for each metric is indicated in \textbf{bold}. OURS was significantly better than the state-of-the-art Thai word segmentation model, Baseline w/ Word, and OURS w/o CC w/ Sub (\underline{underlined} scores) at p-level $< 0.05$ in pairwise comparison. All models were evaluated based on the same dataset division. The scores were obtained from the mean of three runs. OURS w/o CC w/ Sub scores were reported from the best validation performance among various subword vocabulary sizes. The symbols $\circ$, $\odot$, and $\star$ indicate reproduced, non-neural, and the state-of-the-art Thai word segmentation models, respectively; $\sigma$ represents population standard deviation.}
\label{tab:main-results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[b]
\input{04table04.tex}%
\hangcaption{Comparison of architectures among our models, baselines, and others. A {\checkmark} indicates whether Words, CCs, Subwords, LSTM, and PTMs were used in the model.}
\label{tab:main-results-desc}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We implemented an additional model that replaces the BiLSTM layers with transformer layers \cite{Vaswani2017} in Baseline, using the hyperparameters for the transformer layers from \citeA{Vaswani2017}. However, the results were noticeably poorer than those of other models.

No non-neural models were able to outperform any neural model.
However, TLTK and New Multi-Cut, which incorporate additional linguistic information, that is, Syllables and CCs, exhibited benefits over LexTo and Multi-Cut in terms of segmentation performance.
The results indicate that incorporating Syllables boosts the Word-F1 score and Bound-F1, whereas incorporating CCs can improve BIES-F1.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[b]
\input{04table05.tex}%
\hangcaption{Results of our subword-integration model (OURS w/o CC w/ Sub) with various subword vocabulary sizes from 800 to 12,800 tokens. The best result of three runs is reported. An \underline{underline} indicates the model that obtained the best score in the validation process.}
\label{tab:subword-results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Analysis}

\noindent\textbf{Subword-Integration Performance:}
We implemented OURS w/o CC w/ Sub on various subword unit vocabulary sizes, as shown in Table~\ref{tab:subword-results}.
The results indicate that vocabulary size affects the performance of subword-integration.
Specifically, providing a larger subword vocabulary to the model consistently improves its overall performance.
It is suggested that OURS w/o CC w/ Sub may match the performance of the OURS when the subword vocabulary size is sufficiently large.
However, it may be a challenge to determine the appropriate vocabulary size to benefit the model.
For instance, OURS w/o CC w/Sub with 12,800 subword tokens (OURS w/o CC w/ Sub12800) failed to improve on the performance of those with 3,200 and 6,400 subword tokens.
Thus, the size of the subword vocabulary is a crucial parameter that affects the performance of this model.

We chose the best subword-integration model based on validation performance to compare with OURS, as shown in Table~\ref{tab:main-results}. 
Both subword- and CC-integration tended to act as an additional filter layer for word-integrated character representations and improved the segmentation performance.
However, OURS outperformed OURS w/o CC w/Sub in every evaluation.
We believe that the main reason for this is that subword units contain noise, while CCs do not.
For example, there may be a unit that is included in the subword vocabulary but does not exist in the Thai word vocabulary and violates the Thai writing system (for example, a combination of one Thai vowel followed by one Thai consonant), whereas CCs will not include this type of unit.


%%%%\medskip
\noindent\textbf{Order-of-Integration Performance:}
As mentioned prior, we also compared the performance of our model when the order of attention integration was swapped.
Table~\ref{tab:main-results} shows that the swapped models exhibited decreased segmentation performance compared with the original models, especially in the case of the swapped subword-integration model.
We suggest that subword-integration adds initial noise to character representations, as explained above, making it difficult for the model to complement such representations in the word-attention integration. 
OURS Swap slightly increased in performance compared with OURS w/o CC w/ Sub Swap because the CC vocabulary consists of smaller units that reflect the Thai writing system and includes no noise information.
Results showed that OURS outperformed both swapped models and that word information is preferential to complement a character representation, whereas fine-grained information, that is, CCs and subword units, is more suitable for use after word information as an additional filter layer.


\noindent\textbf{Pre-Trained Model Performance}
In this experiment, we replaced BiLSTM layers with BERT layers to extract \textit{character context vectors} while simply extracting word and CC representations from the BERT-embedding layer. 
From the results shown in Table~\ref{tab:main-results}, applying the pre-trained BERT model to the baseline improved its segmentation performance on BEST2010.
However, the BERT-integrated models that incorporated additional linguistic knowledge, that is, Baseline w/ BERT w/ Word, OURS w/ BERT, and OURS w/ BERT w/o CC w/ Sub, exhibited clearly reduced segmentation performance compared with the original models.
We believe that the main reason is that the pre-trained BERT model can produce unknown tokens (UNK) to represent unknown characters or candidate words, CCs, and subword units, which are not included in the BERT vocabulary, instead of using their own representation. 
This can be considered a limitation of applying the existing pre-trained BERT model for straightforward fine-tuning without expanding the BERT vocabulary to cover all possible candidate words, CCs, and subword units.
We expect that the segmentation performance could be improved if the pre-trained BERT model were pre-trained with the training data, and the BERT vocabulary is appended with candidate words, CCs, and subword units.

Although the segmentation performance in our model, that is, OURS w/ BERT and OURS w/ BERT w/o CC w/ Sub, decreased when it was combined with the pre-trained BERT model, their results still outperformed Baseline w/ BERT w/ Word. 
This indicates that incorporating a pre-trained BERT model with multiple types of linguistic information, including word layers, and either CC- or subword-unit layers, is useful.
By comparing the BiLSTM-based models with the BERT-integrated models, most of the BiLSTM-based models outperformed their own BERT-integrated versions; only Baseline did not outperform Baseline w/o BiLSTM w/ BERT. 

\noindent\textbf{Comparison with Thai Domain-Adaption Models}
To evaluate our models on additional datasets, we compared them with recent studies that focused on domain-adaptation scenarios using specific datasets, including TNHC and VISTEC. 
The baseline and proposed models were evaluated in an in-domain scenario, that is, the models were trained on the targeted domain training data and evaluated on the target domain test data. 
For reference, the results of previous models \cite{limkonchotiwat-etal-2020-domain,limkonchotiwat-etal-2021-handling} trained in a domain adaptation scenario (trained using both source and target domain training data) were reported.
Specifically, we performed our model based on training, validation, and test splits for TNHC and VISTEC.
Table~\ref{tab:domain-adapt-results}\footnote{Results for the DSE-DC method from \citeA{limkonchotiwat-etal-2021-handling} were obtained using the same dataset division as their official model from \url{https://github.com/mrpeerat/OSKut}.} shows the results of our models and other domain-adaptation models in character-level and word-level evaluations, as in the work of \cite{limkonchotiwat-etal-2020-domain,limkonchotiwat-etal-2021-handling}. 
OURS and OURS w/ BERT surpassed all domain-adaptation methods and outperformed every reproduced model.
Specifically, mirroring the tendency observed in Section \ref{section:main-results}, OURS was statistically significantly better than OURS w/o CC w/ Sub at p-level $p < 0.05$.
Moreover, OURS outperformed OURS without CC w/Sub in multiple runs.
This emphasises the benefits of using CCs over subword units.
Although we reproduced \citeA{limkonchotiwat-etal-2021-handling} from their original repository, the reproduced results were lower than their reported scores: by 3.4 for the word-F1 scores on TNHC and by 2.29--4.43 for the char/word-F1 scores on VISTEC.
The results of the non-neural models showed the same tendency as in Section \ref{section:main-results}, that is, no non-neural model was able to outperform any neural models.
In addition, the results of the non-neural models also supported the hypothesis that the use of additional linguistic information, that is, Syllables and CCs, could improve segmentation performance, as in TLTK and New Multi-Cut.
Specifically, by incorporating Syllables, TLTK achieved outstanding Word-F1 scores, while New Multi-Cut, which incorporates CCs, performed well with regards to Char-F1. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[t]
\input{04table06.tex}%
\hangcaption{Comparison between our models and domain-adaptation models on TNHC and VISTEC datasets. Best score for each metric is indicated in \textbf{bold}. Scores were obtained from the mean of three runs. OURS performed significantly better than Baseline w/ Word and OURS w/o CC w/ Sub (\underline{underlined} scores) at p-level $< 0.05$. A $\bullet$ indicates a reported score from literature; $^\circ$ and $\odot$ indicate scores reproduced from neural models and non-neural models, respectively (results were obtained from OSKUT by reproducing their official trained models, https://github.com/mrpeerat/OSKut); $\sigma$ represents population standard deviation.}
\label{tab:domain-adapt-results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%\medskip
\noindent\textbf{Case Study:}\label{section:case-study}
Figure~\ref{fig:case-study} shows examples of segmentation results from four models: Baseline, Baseline/w Word, OURS, and OURS w/o CC w/ Sub.
OURS perfectly segmented the example sentence; however, other models yielded incorrect results. 
Specifically, \textcolor{red}{\underline{underlined word}} violates the Thai writing system by combining the two consonants in the word.
We believe that CC-integration successfully filters out this type of violation from the word-integrated character representations, enabling OURS to outperform the other models.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[t]
\begin{center}
\includegraphics{30-2ia4f6.pdf}
\end{center}
\hangcaption{Examples of segmentation results from baseline models and our models. The ground-truth segmentation result is indicated as ``Reference'' and incorrect segmentation results are in \textcolor{red}{red}. \underline{Underlines} indicates segmentation results that violate the Thai writing system.}
\label{fig:case-study}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion and Future Work}

In this paper, we proposed a character-based Thai word segmentation model that uses multiple types of linguistic knowledge, that is, words, subwords, and CCs.
The best version of our model outperformed the state-of-the-art Thai word segmentation model by using word attention along with CC attention in the BiLSTM-CRF architecture.
In general, comparison between BiLSTM-based models and BERT-based models indicated that BiLSTM-based models surpassed BERT-based models, particularly when using our method.
However, our approach showed an improvement over the baseline model after integrating a pre-trained BERT model into our proposed model.
Further analysis also indicates that CCs are better than subword units for the word segmentation task.
In future work, multiple attentions can be merged into a single attention to reduce the model size as well as the time complexity.
Additionally, instead of using existing PTMs, the construction of new PTMs for fine-tuning the proposed model can also be considered.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment
This paper is an extended version of our preliminary paper \cite{chay-intr-etal-2021-character} presented in the 13th Recent Advances in Natural Language Processing (RANLP 2021) conference. 
In this paper, we add detailed explanations and the following new information to the preliminary version: BERT and recent domain-dependent Thai word segmentation in the Introduction; pre-trained models for Thai word segmentation in Related Work; an explanation of BERT layers in Approach; four non-neural models for comparison; two new domain-dependent datasets (with their statistics) for comparing our model with recent domain-dependent models; hyperparameters for pre-trained model integration; experimental results for pre-trained model integration; comparison of our model and those used in recent Thai domain-dependency studies on two domain-dependent datasets; analysis of pre-trained model integration results and a comparison with domain-dependent models; population standard deviation in neural models; and statistical significance tests for the strong baseline models in Main Results and Analysis. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{04refs}


%%%%\bibliography{anthology,e_yourrefs}
\begin{biography}

\bioauthor[:]{Thodsaporn Chay-intr}{%
T. Chay-intr received his M.Eng.\ from Sirindhorn Institute of Technology and is currently a doctoral student at the Tokyo Institute of Technology. His current research interests include natural language processing, particularly text classification; computer linguistics; and text mining.
}

\bioauthor[:]{Hidetaka Kamigaito}{%
H. Kamigaito received his Ph.D. from the Tokyo Institute of Technology and is currently an associate professor at the Nara Institute of Science and Technology. His current research interests include natural language processing, with a specific focus on document-level text processing and knowledge graph completion.
}

\bioauthor[:]{Manabu Okumura}{%
M. Okumura received his Dr. Eng. from the Tokyo Institute of Technology and is currently a professor at the Institute of Innovative Research, Tokyo Institute of Technology. His current research interests include natural language processing, particularly text summarising; computer-assisted language learning; sentiment analysis; and text mining.
}
\end{biography}



%%%%\bibliography{anthology,e_yourrefs}
\biodate





\end{document}

