    \documentclass[english]{jnlp_1.4}

\usepackage{jnlpbbl_1.3}
\usepackage[dvips]{graphicx}
\usepackage{hangcaption_jnlp}

\usepackage{url}
\usepackage{amsmath}
\DeclareMathOperator*{\argmax}{arg\,max}
\newcommand{\twocomb}[2]{}
\newcommand{\threecomb}[3]{}


\Volume{19}
\Number{1}
\Month{March}
\Year{2012}

\received{2011}{6}{21}
\revised{2011}{8}{29}
\rerevised{2011}{11}{17}
\accepted{2011}{12}{20}

\setcounter{page}{25}

\etitle{Study on Supervised Learning of Vietnamese Word Sense Disambiguation Classifiers}
\eauthor{Minh Hai Nguyen\affiref{Author_1} \and Kiyoaki Shirai\affiref{Author_2}}
\eabstract{
It is said that Vietnamese is a language with highly ambiguous words.
However, there has been no published Word Sense Disambiguation (WSD hereafter) research on this language.
This current research is the first attempt to study Vietnamese WSD. 
Especially, we would like to explore the effective features for training WSD classifiers and verify the
applicability of the `pseudoword' technique to both investigating effectiveness of features and training WSD classifiers.
Three tasks have been conducted, using two corpora which were built manually based on Vietnamese Treebank
and automatically by applying pseudowords technique.
Experiment results showed that Bag-Of-Word feature performs well for all three categories of words (verbs, nouns, and adjectives).
However,
its combination with POS, 
Collocation or Syntactic features can not significantly improve the performance of WSD classifiers.
Moreover, the experiment results confirmed that pseudoword is a suitable technique to explore the effectiveness of features in disambiguation of Vietnamese verbs and adjectives.
Furthermore, 
we empirically evaluated the applicability of the pseudoword technique as an unsupervised learning method for real Vietnamese WSD.}
\ekeywords{Word Sense Disambiguation, Vietnamese, Supervised Machine Learning, Feature for WSD, Pseudoword}

\headauthor{Nguyen and Shirai}
\headtitle{Study on Supervised Learning of Vietnamese WSD Classifiers}

\affilabel{Author_1}{}{School of Information Science, Japan Advanced Institute of Science and Technology. nhminh@jaist.ac.jp}
\affilabel{Author_2}{}{School of Information Science, Japan Advanced Institute of Science and Technology. kshirai@jaist.ac.jp}


\begin{document}

\maketitle

\section{Introduction} \label{section:introduction}
WSD plays an important role in natural language processing applications, such as machine translation, information retrieval, speech processing, etc.
So far, this problem has been studied for English, Japanese and many other languages for more than half a century,
and many effective knowledge sources as well as disambiguation methods have been discovered.
Vietnamese is said to be a language including many highly ambiguous words.
For example, the word `bien' in Vietnamese can have different meanings: the sea, a sign-board, a large group of people. 
Hence, WSD is also an important task in Vietnamese language processing. 
However, to the best of our knowledge, there is no research on Vietnamese WSD.
Vietnamese is an isolating language with some general characteristics as follows:
\begin{itemize}
\item Words do not have morphological forms.
Vietnamese has a number of tense markers to indicate the tense of a sentence.
 Therefore, the grammatical relationship is expressed by word order and auxiliary words.
\item Word boundary is not obviously determined by blank.
\item There are many `classifiers' which come before nouns like Chinese.
\item Vietnamese also has the same basic SVO word order as English.
\end{itemize}

In this study, 
one of our goals is to carry out the first attempt to establish a WSD method for Vietnamese. 
Since approaches based on supervised machine learning achieved great success in WSD, the present authors are also interested in it.
Especially, this paper will discuss the following two issues:
\begin{itemize}
\item What are effective features in Vietnamese WSD?

Various types of features for WSD were proposed in previous work.
Our question here is, ``What kinds of features are effective for disambiguation of word senses in Vietnamese?''
\item Is pseudoword technique applicable for Vietnamese WSD?

For supervised learning of WSD classifiers,
a sense-tagged corpus is required as training data.
However, there is no Vietnamese sense-tagged corpus available to the public.
Pseudoword technique is often used to evaluate supervised WSD methods when no training data is available.
Two words $w_1$ and $w_2$ are regarded as an imaginary word (pseudoword) $p$, 
then machine learning methods are applied to train classifiers which predict if the original word of $p$ in texts is $w_1$ or $w_2$.
The performance of trained classifiers can be evaluated without heavy human intervention.
Our interest is whether the pseudoword technique is useful for Vietnamese WSD or not.
\end{itemize}

Considering the above issues, this paper has three goals.
The first one is to empirically explore effective features for Vietnamese WSD.
Supervised WSD classifiers with several kinds of features are trained,
then their performance is compared.
Effectiveness of feature combination is also considered.
The second is to check the applicability of the pseudoword technique.
This paper will investigate the possibility of the pseudoword technique for finding the most effective features.
The last goal is, as an alternative to unsupervised methods,
we explore a method to apply the pseudoword technique for training WSD classifiers when no sense-tagged corpus is available.

In the next section, we will discuss some work related to our research. 
Then, we describe the development of our system for Vietnamese WSD in Section \ref{section:method}. 
Section \ref{section:task} introduces three tasks which were conducted in this research. 
Section \ref{section:eval} shows results and some discussion. 
Finally, we summarize the research and indicate future work in Section \ref{section:conclude}.


\section{Related work} \label{section:relatedwork}
The first experiment by Kaplan proved that just one or two words on both sides of an ambiguous word can be evidence to disambiguate that word \cite{Kaplan1950}.
Later, more useful information from context was discovered by numerous works in WSD. 
Yarowsky introduced simple set of features (context around the ambiguous words) in accent restoration task \cite{Yarowsky1994}. 
This led to many other improved sets of features, 
    such as syntactic dependencies \cite{martinez:02:a,Dang2002,Yarowsky2002},
or cross language evidence \cite{GaleChurch1992}. 
Beside the approaches utilizing the evidence provided by the surrounding context of the ambiguous word, 
there are many other researches which take advantage of knowledge bases without using any corpus evidence,
such as approaches using dictionaries, thesauri, and lexical knowledge bases \cite{Lesk1986,Agirre2001}.
These knowledge sources have been used in various ways to improve WSD systems in English. 
Numerous studies have also been devoted to WSD in languages other than English.
However, Vietnamese WSD has not been studied so far.
Vietnamese is a language with characteristics different from those of English. 
For example, words in Vietnamese are not separated by empty spaces, an adjective can be a subject of a sentence, etc.
It is necessary to investigate the effective features for Vietnamese WSD.

According to the knowledge sources used in sense disambiguation, 
methods in WSD are classified as knowledge-based, unsupervised corpus-based, supervised corpus-based, 
and combinations of these \cite{WSDbook}. 
Among these methods, the approach to supervised learning is the hot topic,
since it has been one of the most successful approaches in the last fifteen years in WSD. 
However, the biggest problem of supervised learning methods is the knowledge acquisition bottleneck, 
which poses challenges to the supervised learning approach for WSD.
For Vietnamese WSD, the problem is serious, 
since no sense-tagged corpus is available to the public.
Dinh attempted to construct a sense-tagged corpus in Vietnamese by using English semantically-tagged corpus and bilingual English-Vietnamese texts \cite{Dinh2002}. 
However, he mainly annotated English texts, in order to disambiguate English words to be applied in an English-Vietnamese machine translation system. 
And there was no evaluation of WSD based on his corpus, either. 

Gale et al. introduced a technique called `pseudowords' to overcome the obstacles of supervised methods \cite{Gale1992}.
However, two words to be combined as a pseudoword in Gale's experiments are randomly chosen.
Thus pseudowords may have different linguistic characteristics from real ambiguous words.
Lu et al. presented `equivalent' pseudowords \cite{Lu2006},
in which they built up pseudowords based on real ambiguous words. 
However, they only performed evaluation on pseudowords,
and have no comparison between pseudowords and real ambiguous words.
The task of classifying two different words may be easier than distinguishing two senses of the same word. 
Therefore, our research aims to empirically evaluate the validity of the `pseudoword' method for Vietnamese WSD.


\section{Our method} \label{section:method}
In this section, we describe our method to disambiguate word senses. 
SVM is used as a machine learning algorithm which is introduced in Subsection \ref{svm}. 
Features used in the SVM classifiers are also explained in Subsection \ref{featureset}. 

\subsection{Support Vector Machine as classifier for WSD}
\label{svm}
Support Vector Machine (SVM)~\cite{Cortes1995} learns a linear discriminant hyperplane that separates two classes of data represented as high-dimensional vectors.
In this research, the number of senses for an ambiguous word is limited to two, 
since it is rather difficult to prepare a large scale corpus covering all senses of an ambiguous word\footnote{This assumption is obviously not realistic.
However, 
we believe that results of the experiments reported in this paper would provide somewhat reliable information about Vietnamese WSD.}. 
The linear kernel is used for training WSD classifiers, because in high dimensional space (when the number of features is large),
we expect that mapping data to a higher dimensional space does not improve performance. 
We actually found that other kernels gave poorer results than linear kernel in our preliminary experiment. 

\subsection{Feature set}
\label{featureset}
For each target instance $w$, we encode its surrounding context as a feature vector. 
The feature set $F$ of $w$ is denoted as in \eqref{eq:featureset}, where $ f_{i} $ represents a feature.
\begin{equation}\label{eq:featureset}
 F = \{f_{1},f_{2},...,f_{n}\}
\end{equation} 
 In our experiment, the feature vector is weighted according to the context of target instances in the training corpus (Eq. \eqref{eq:featurevector}), where $ \omega_{i} $ is a weight of $f_{i}$. Methods for defining $f_{i}$ and $\omega_{i}$ will be described in detail for each type of feature.
 \begin{equation}\label{eq:featurevector}
 
 \vec{f} = (\omega_{1},\omega_{2},...,\omega_{n})
 \end{equation}
 
\subsubsection{Bag-Of-Words}
Bag-Of-Words (BOW hereafter) feature encodes single words around the target word in a sentence. 
For example, in the following sentence, 
\[\frac{\textit{Hoang hon }}{(sunset;Noun)} \frac{\textit{tren }}{(on;Preposition)} \frac{\textbf{\textit{bien}} }{(sea;Noun)} \frac{\textit{that} }{(so;Adverb)} \frac{\textit{dep }}{(beautiful;Adjective)} \]
the BOW of the target word `\textit{bien}' is \textit{\{hoang hon, dep\}}. 
Therefore, $f_{i}$ corresponds to a word appearing in the context of a target word.

Function words\footnote{Function word is defined by POS. 
In our method, 
classifier, 
unit noun, 
pronoun, 
quantifier, 
adverb, 
preposition, 
connector, 
interjection, 
introductory word (a kind of particle), 
abbreviation 
and untagged word
are regarded as function words.
},
proper nouns, numbers and punctuation marks are not used as features, 
since they would not be effective clues for WSD.
For BOW feature, $F$ is a set of all possible words appearing in the context of target instances in the training corpus. 
For each sentence $l$ containing a target instance $w$ in the training corpus, $f_{i}$ is weighted as in Eq. \eqref{eq:weight_unigram_train}.
\begin{equation}\label{eq:weight_unigram_train}
\omega_{i} = \left \{ \begin{array}{ll} t_i^1 & \mbox{if $f_{i}$ appears in $l$ and sense of $w$ is $s_{1}$}\\ t_i^2 & \mbox{if $f_{i}$ appears in $l$ and sense of $w$ is $s_{2}$} \\ 0 & \mbox{if $f_{i}$ does not appear in $l$} \end{array} \right.
\end{equation}
where $t_i^j$ is the frequency of $f_i$ that appears in the context of sense $s_{j}$ of $w$ in the training corpus.
While $f_i$ is weighted as in Eq. \eqref{eq:weight_unigram_test} in the test data,
since the sense of $w$ is unknown\footnote{
	We also tried weighting both test and training data as in Eq.~\eqref{eq:weight_unigram_test}. However, the accuracy was 81.2, which was worse than our weighting method (94.0; the accuracy of the classifier with only BOW feature for all words shown in Table \ref{table:rw_all}). In Eq.~\eqref{eq:weight_unigram_train}, association between a BOW feature and a sense is considered in the training phase. It seems useful to improve the accuracy.}.
\begin{equation}\label{eq:weight_unigram_test}
\omega_{i} = \left \{ \begin{array}{ll} (t_i^1 + t_i^2)/2 & \mbox{if $f_{i}$ appears in $l$} \\ 
0 & \mbox{if $f_{i}$ does not appear in $l$} \end{array} \right.
\end{equation}

\subsubsection{POS}
This feature encodes part-of-speech of each word in a context window \textit{c} around the target instance \textit{w} as in Eq. \eqref{eq:fi_pos}, 
where $p_{i}$ is the position of the word and $P_{i}$ is its POS. $p_{i}$ is an integer in the range $[-c,c]$ indicating the distance between a target word and a word in the context. 
If $p_{i}$ is positive, the context word appears in the context after the target word. 
Similarly, $p_{i}$ is negative for words in the context before the target word. 
If $p_{i}$ exceeds the sentence boundary,
$P_{i}$ is denoted by the null symbol $\epsilon$. 
For POS feature, $F$ is a set of all possible pairs of the position of the word in the context and its POS found in the training corpus. For each sentence in the corpus, 
$f_i$ is weighted by $\omega_i$ as in Eq. \eqref{eq:weight_pos}. 
Note that POS categories used in our classifiers are coarse, 
such as A (Adjective), V (Verb), N (Noun) and E (Preposition).
\begin{gather}
\label{eq:fi_pos}
f_{i}=(p_{i},P_{i})  \\
\label{eq:weight_pos}
 \omega_{i} = \left \{ \begin{array}{ll} 1 & \mbox{if POS of the word at the position $p_{i}$ is $P_{i}$;}\\ 0 & \mbox{otherwise} \end{array} \right.
\end{gather}


\subsubsection{Collocation}
Collocation feature  (COL feature hereafter) encodes a sequence of words (n-grams) that co-occurs with the target word. 
Let $w_{i}$ denote the \textit{i-th} word to the right (or left if \textit{i} is negative) of the target instance $w_{0}$.
If the \textit{i-th} word exceeds the sentence boundary, $w_{i}=\epsilon$. 
A collocation string is defined as in Eq. \eqref{eq:coll_string}.
\begin{equation}\label{eq:coll_string}
C_{l,r} = w_{l}w_{l+1}...w_{r} 
\end{equation}
For each target instance in the corpus, we extracted 9 collocation strings:
$ C_{-1,0}$; $C_{0,1}$;
$ C_{-2,0}$; $C_{-1,1}$; $C_{0,2}$;
$C_{-3,0}$;$ C_{-2,1}$; $C_{-1,2}$; $C_{0,3} $.
Each feature $f_{i}$ is extracted as in Eq. \eqref{eq:fi_coll}, where $l_{i}$ and $r_{i}$ are the start and end positions of a collocation string $(1<r_{i}-l_{i}<4, l_{i}=-3,...,0, r_{i}=0,...,3)$. 
Unlike the case of BOW, we do not remove punctuation symbols or numbers in the collocations. 
For the COL feature, $F$ is a set of all possible collocation strings with $w$ in the training data. For each sentence $l$ containing the target word $w$ in the corpus, $f_i$ is weighted by $\omega_i$ as in Eq. \eqref{eq:weight_coll}. 
\begin{gather}
 \label{eq:fi_coll}
f_{i} = (l_{i},r_{i},C_{l_{i},r_{i}}) \\
 \label{eq:weight_coll}
 \omega_{i} = \left \{ \begin{array}{ll} 1 & \mbox{if $C_{l_{i},r_{i}}$ is found in $l$;}\\ 0 & \mbox{otherwise} \end{array} \right.
\end{gather}

\subsubsection{Syntactic}
Syntactic relations can be extracted from an annotated syntactic tree, such as subject-verb, verb-object, etc. 
In this paper, target words are supposed to verbs, nouns or adjectives.
For each category of target word, we used different features according to Vietnamese grammar. 
Since characteristics of Vietnamese are different from English, 
the extracted features are not the same as in the previous approaches based on syntactic relations of English.
For example, an adjective can be subject of a sentence in Vietnamese, while it is impossible in English.
Table \ref{table:syn_feature} shows the list of syntactic feature (SYN feature hereafter) used in our WSD classifiers.
In Table \ref{table:syn_feature},
each type of syntactic feature is presented as `R-P' (e.g. Subj-N)
where R stands for syntactic relation between the target word and the word used as a feature, 
and P stands for POS of a feature word.
The SYN feature vector is constructed in the same manner as in POS and Collocation features. 
Let $sl_{i}$ denotes the syntactic relation (Subj-V,Mod-A,...), 
$t_{i}$ is a word which has a syntactic relation $sl_{i}$ with the target word. 
Each syntactic feature is represented as in \eqref{eq:fi_syn}. 
For Syntactic feature, $F$ is a set of all possible words that have some syntactic relations with the target word in the training corpus. 
For each sentence $l$ containing target instance $w$ in the corpus, $f_i$ is weighted as in Eq. \eqref{eq:weight_syn}.
\begin{gather}
 \label{eq:fi_syn}
f_{i}=(sl_{i},t_{i}) \\
 \label{eq:weight_syn}
 \omega_{i} = \left \{ \begin{array}{ll} 1 & \mbox{if $w$ and $t_{i}$ are in the syntactic relation $sl_{i}$ in $l$}\\ 0 & \mbox{otherwise} \end{array} \right.
\end{gather}

\begin{table}[t]
\caption{List of syntactic features.}
\label{table:syn_feature}
\input{03table01.txt}
\end{table}


In addition to 4 types of features, the feature combinations are considered as in Table \ref{table:feature_combi}.
In feature combination, feature vectors for target instances are built by just concatenating vectors for individual features.

\begin{table}[t]
\caption{Combined feature sets.}
\label{table:feature_combi}
\input{03table02.txt}
\vspace*{-1\baselineskip}
\end{table}




\section{Tasks} \label{section:task}
This section describes three tasks which were conducted to explore the effective features for learning
Vietnamese WSD classifiers, as well as to evaluate pseudoword technique.
Since there is no sense-tagged corpus for Vietnamese WSD,
two kinds of sense-tagged corpora were built based on Vietnamese Treebank \cite{Nguyen2009},
a corpus which contains around 10,000 sentences manually annotated with syntactic trees.
Details of these two corpora are explained in the succeeding sections.

\subsection{Real Word task}
\label{real_task}
We first conducted the ordinary WSD experiments in order to investigate which features are effective for Vietnamese WSD classifiers.
We called this task Real Word task (RW task hereafter).
Since there is no sense-tagged corpus for Vietnamese WSD,
in order to train SVM classifiers, a manually sense-tagged corpus named `RW corpus' is built using Vietnamese Treebank \cite{Nguyen2009}\footnote{Vietnamese Treebank contains about 10,000 sentences which come from news articles.
Vietnamese Treebank has already been available at http://vlsp.vietlp.org:8080/demo/?page=resources.
At the same site, some other Vietnamese language resources 
(such as machine readable dictionary and bilingual corpus) are available.}.
The tagging process was conducted as follows: 
we first choose 9 verbs, 11 nouns and adjectives for target words. 
These words are chosen considering the following conditions:
it is a high frequency word in Vietnamese Treebank,
it is ambiguous and both senses of it are expected to appear sufficiently in the Treebank.
For each target word, about 100 sentences were chosen for sense tagging, 
resulted in around 3,000 sentences for all verbs, nouns and adjectives. 
Two Vietnamese native speakers were invited to judge independently which sense a target word had in those sentences.
Chosen senses are those defined in VDict Vietnamese dictionary\footnote{VDict---An online Vietnamese-Vietnamese dictionary \texttt{http://vdict.com/}, online accessed 2009-11-01.}.
Average number of senses for target words in VDict is 3.1.
However, not all but only two coarse grained senses for each target word are annotated.
The inter-annotator aggreement is 90.63\%.
For the disagreed sentences, two annotators discussed together and determined the final sense.
We call the above sense tagged corpus `RW corpus'.
The average numbers of sentences for verbs, nouns and adjectives are 92.3, 116.7 and 92.1, respectively.
Full lists of chosen target words and their senses are shown in Figure \ref{listword}.

\begin{figure}[b]
\begin{center}
\includegraphics{19-1ia929f1.eps}
\end{center}
  \caption{List of ambiguous words and their senses}
  \label{listword}
\end{figure}


\subsection{Pseudoword task}
\label{pw_task}
Although using ordinary WSD classifiers can give us more reliable results,
the problem is a sense tagged corpus is not easily built.
Therefore, we applied the pseudoword technique to automatically develop a sense-tagged corpus,
and trained WSD classifiers from it.
We call this task Pseudoword task (PW task).
The main goal of this task is to evaluate the applicability of pseudoword technique for exploring effective features of WSD by comparing results between RW and PW tasks.

Let us suppose $V_{1}$ and $V_{2}$ are two different words. 
Pseudoword $V_{1}$-$V_{2}$ is an imaginary word implying it is $V_{1}$ or $V_{2}$. Then $V_{1}$ or $V_{2}$ in the corpus are replaced with the pseudoword $V_{1}$-$V_{2}$. 
Now we can regard the original word $V_{1}$ or $V_{2}$ as a sense (we call it `pseudo-sense' hereafter) of $V_{1}$-$V_{2}$. 
Note that the corpus after $V_{1}$ or $V_{2}$ are replaced by $V_{1}$-$V_{2}$ can be regarded as a sense tagged corpus.
Pseudoword task (PW task hereafter) is a task to determine the pseudo-sense ($V_{1}$ or $V_{2}$) of the pseudoword $V_{1}$-$V_{2}$ in a sentence.
We call the obtained corpus `PW corpus'. 
Although it is not a real WSD, a pseudo-sense tagged corpus can be easily created without any human intervention.

In many previous studies applying pseudoword technique to evaluate WSD methods, 
two words $V_{1}$ and $V_{2}$ are selected randomly. 
However, in this research, $V_{1}$ and $V_{2}$ are chosen considering the meanings of a certain word, 
similar to `equivalent pseudoword' proposed by Lu et al. \cite{Lu2006}.

\begin{figure}[t]
\begin{center}
\includegraphics{19-1ia929f2.eps}
\end{center}
  \caption{List of pseudowords and their pseudo-senses}
  \label{listpw}
\vspace*{-0.5\Cvs}
\end{figure}

Let us suppose $w$ is a target word.
 We use VDict to look up meanings of $w$. Let $s_{1}, s_{2}$ be two meanings (or senses) of $w$. 
 Then, we find two Vietnamese words $V_{1}, V_{2}$ that reflect the meanings of $s_{1}, s_{2}$ respectively. $V_{1}, V_{2}$ are supposed to be monosemous. 
Disambiguation of the pseudoword $V_{1}$-$V_{2}$ would simulate the disambiguation of the original target word $w$.
For example, the Vietnamese verb `\textit{mang}' has two meanings: ``to bring something'' and ``to contain some characteristic of something''. Then `\textit{dem}' (bring) and `\textit{chua}' (contain) are selected as pseudo-senses of `\textit{mang}'.
We chose 9 verbs, 9 nouns, and 5 adjectives as target words in PW task,
which are the subset of target words in RW task. 
Some target words in RW task are discarded in PW task because of the lack of data in our corpus.
Figure \ref{listpw} reveals the target words and their two pseudo-senses of verbs, nouns and adjectives, respectively.
Note that in order to increase the number of training and test instances, a pseudo-sense is sometimes represented as a set of synonymous words such as V2, N1 and A1.
The figure also includes pseudo-senses of the 2 remaining target nouns and 4 adjectives in RW task,
whose IDs are shown in italics.
Since we could not get enough training data for these pseudo-senses, 
we removed them from target words of PW task\footnote{It is possible to prepare a lot of Vietnamese texts to obtain more training examples for these words. However, manually annotated syntactic trees are used to derive SYN features. Thus we used Vietnamese Treebank, which is a relatively small corpus.}.
The PW corpus comprises 1,162 sentences for verbs, 1,483 sentences for nouns and 568 sentences for adjectives. The average samples of pseudo-verbs, pseudo-nouns and pseudo-adjectives are 129.1, 164.8 and 113.6, respectively.
The number of adjective instances is less than verbs and nouns because the frequency of ambiguous adjectives in the corpus is low.
Also, since the adjectives have fine-grained senses, 
it is more difficult to disambiguate them.


\subsection{Pseudoword and Real Word task}
We will present a method to train WSD classifiers without sense-tagged corpora in this subsection.
In Pseudoword and Real Word task (PW-RW task hereafter), 
we use PW corpus for training WSD classifiers, 
then classifiers are tested using RW corpus.
This task is conducted in order to evaluate the effectiveness of pseudoword technique applied to real WSD. 
Since the target words are shared in our PW and RW tasks, and a pseudo-sense ($V_{1}$ or $V_{2}$) in PW task corresponds to a sense ($s_{1}$ or $s_{2}$) in RW task, WSD classifiers trained from PW corpus could be applicable for RW task.
The attractive advantage of this approach is that no sense-tagged corpus is required for supervised learning of WSD systems.




\section{Evaluation} \label{section:eval}
For each experiment, we first evaluate the effectiveness of each feature separately, 
then the feature combinations.
LIBSVM \cite{LibSVM} is used for training SVM classifiers. 
Experiments in RW task and PW task are conducted by 10-fold cross validation.
For PW-RW task, PW corpus is used as training set and RW corpus is used as test set.

The Baseline used in the experiments is the most frequent sense method. 
That is, all test instances of a target word are determined to be the most frequent sense appearing in the training data. 

The evaluation criteria for WSD systems is the accuracy of sense classification defined as in Eq. \eqref{eq:acc}.
\begin{equation}\label{eq:acc}
 acc=\frac{\text{number of correct instances}}{\text{total number of instances}} 
\end{equation}

In each task, 
15
feature sets are used for training WSD classifiers.
The first four utilize one feature type, 
while the others utilize two, 
three, or 
four
feature types (feature combination).
In following subsections, accuracies of trained WSD classifiers for individual target words are reported. 
Average accuracies for verbs, nouns, adjectives and all target words are also shown.
For the results of individual target words, 
not all but only the first and second ranked feature combinations are shown.


\subsection{Results of Real Word task}
\label{exp_rw}

\begin{table}[b]
\caption{Accuracy in RW task for each target word.}
\label{table:rw_individual}
\input{03table03.txt}
\end{table}

Table \ref{table:rw_individual} shows results for each target word,
while Table \ref{table:rw_all} shows the average accuracies for verbs, nouns, adjectives and all target words in RW task.
Results of SVM classifiers are verified by McNemar's test ($p<0.05$).
$*$ means the case that it significantly outperforms Baseline. 
The bold number indicates the best accuracy achieved when one feature type is used, 
or when two or more feature types are used. 
If $\dagger$ is attached, 
the system significantly outperforms the second best system
among one feature or combined feature groups.
To clearly show the effectiveness of feature combination,
$\ddagger$ is attached if the difference between the best single and combined feature is statistically significant\footnote{Tables \ref{table:pw_individual}, \ref{table:pw_all}, \ref{table:pwrw_individual} and \ref{table:pwrw_all} are also denoted in the same format.}.

\begin{table}[t]
\caption{Average accuracy in RW task for verbs, nouns, adjectives and all target words.}
\label{table:rw_all}
\input{03table04.txt}
\vspace*{-1\Cvs}
\end{table}


First, we see that almost all WSD classifiers of single features 
except POS and SYN for adjectives,
are significantly better than the Baseline method.
When only a single feature is used, 
BOW was better than the other three features in almost all words.
This is reasonable because BOW can capture the most contextual information of a target word.
As a human usually does when facing an ambiguous word, 
BOW utilizes the context around the target word to find the key words that help disambiguate it. 
The POS feature only contains the grammatical information of several words around the target word,
but not the \textit{`meanings'} of these words.
So, their surrounding POS may not be clearly discriminative. 
The results of POS feature are usually the lowest in comparison with the others,
even with baseline.
SYN feature is also not so effective for adjectives
(only 1.9\% higher than Baseline),
since we only use 4 syntactic relations for an adjective. 
This may cause data sparseness for training SVM classifiers. 
However, SYN feature works well on verbs and nouns (with 10.6\% accuracies higher than Baseline for verb and 17.3\% for noun).
On average, when applying a single feature in Vietnamese WSD,
BOW is the most effective feature, followed by COL, SYN and POS feature. 

In Table \ref{table:rw_individual}, WSD classifiers with combined feature sets got equal or higher results compared to individual features
for some target words.
In Table \ref{table:rw_all}, 
the best feature combination outperforms the best single feature BOW for nouns and adjectives on average. However, 
BOW+SYN, which is the best feature combination for all words,
are not higher than BOW.
Note that the differences between the best single and combined feature sets are insignificant (not marked by $\ddagger$),
indicating that combining several features is not obviously better or worse than the use of only one type of feature.
Increasing the number of feature types in feature combination could not lead to the improvement of accuracies.
The 4 feature types combination is better than the combination of 2 or 3 features only for one verb (V7).
Furthermore, the best feature combinations are different for individual target words,
and differences 
between the best and second best of feature combination
are insignificant (not marked by $\dagger$)
because of the relatively small size of the training corpus.
Therefore, we cannot conclude what is the best feature combination for Vietnamese WSD from our result.


\begin{table}[b]
\vspace*{-0.5\Cvs}
\caption{Accuracy in PW task of each pseudoword.}
\label{table:pw_individual}
\input{03table05.txt}
\end{table}

\subsection{Results of Pseudoword task}
\label{exp_pw}

Table \ref{table:pw_individual} shows results of each pseudoword in PW task,
\pagebreak
and Table \ref{table:pw_all} shows the average accuracies for pseudo-verbs, pseudo-nouns, pseudo-adjectives and all target words.

We can see that results when only a single feature is used are similar to RW task,
in which BOW feature gave the best performance.
As we discussed in Subsection \ref{exp_rw},
BOW contains the most lexical information around the target word.
Results of POS feature are not always the lowest in comparison with the others,
however in some cases, they are lower than the Baseline
(3 of 9 verbs, 1 of 9 nouns, 2 of 5 adjectives).
COL feature also gave relatively high results for all parts-of-speech.
This is because usages of two target words in two classes are different, 
so their collocations are very different.
However, COL still could not perform better than BOW. 

\begin{table}[t]
\hangcaption{Average accuracy in PW task of pseudo-verbs, pseudo-nouns, pseudo-adjectives and pseudowords all.}
\label{table:pw_all}
\input{03table06.txt}
\end{table}

When two or more features are combined together, 
WSD classifiers gave better results compared to single features for 
8 of 9 verbs, 6 of 9 nouns, and all adjectives.
Table \ref{table:pw_all} showed that the most effective feature combination is 
BOW+COL+SYN for verbs and adjectives,
while BOW+COL is most effective for nouns.
However, the differences among feature combinations including BOW are not so great.
The combinations without BOW are worse,
since they do not take advantage of referring to the wide range of lexical information around the target word as BOW does.
Similar to RW task, the best feature combinations in PW task vary for individual target words as shown in Table \ref{table:pw_individual}.
This might be because our training corpus is not large enough.

\subsubsection{Comparison of Effective Features in RW and PW task}
If the best feature set found in PW task is same as one in RW task, it indicates that, 
even when we do not have a word sense tagged corpus,
we can apply pseudoword technique to find the effective features for Vietnamese WSD.
As shown in Table \ref{table:pw_all},
on average, BOW is the most effective feature,
followed by COL, SYN and POS features in PW task.
The order is the same as for the RW task (in Table \ref{table:rw_all}).
Thus investigation of effective features by pseudoword sense disambiguation is reasonable.

\begin{table}[t]
\caption{The best feature comparison for each target word.}
\label{table:bestfitcount}  
\input{03table07.txt}
\end{table}

Looking deeper to the similarity between results of PW task and RW task helps us to verify the applicability of pseudoword technique for investigating effective features in more details.
Table \ref{table:bestfitcount} reveals two numbers in the form of \textit{a/b}:
\textit{a} is the number of target words where the best (or one of the best) feature set is the same in PW and RW tasks,
while \textit{b} is total number of target words shared in PW and RW tasks. 
The `Single' column indicates the case in which the best single feature sets are the same, 
while `Combined' column indicates the case of combined feature sets. 
As shown in the table, 
pseudoword is only appropriate for choosing the best single feature when the target word is a verb or an adjective,
since the best single feature of all target verbs and 4 of 5 target adjectives in PW task agreed with those in RW task.
It seems ineffective for choosing the best single feature for nouns,
 as well as the best feature combination for all categories.

The reason why there are too few target nouns sharing the best feature sets in PW and RW tasks might be because nouns are used in a wide range of domains,
compared to verbs and adjectives in the corpus. 
For example, the first sense of the ambiguous verb \textit{`V4.chuyen'} is `\textit{to send}'. 
This sense can only be used in text related to email, postcard or documents. 
Similarly, the second sense of the adjective `\textit{A5.nang}' is `\textit{serious}'. 
This sense can only be used in a context related to health and disease. 
However, domains for using nouns are very large. 
For example, the second sense of the ambiguous noun \textit{`N6.gio'} is \textit{`now'}. 
This sense can be used in various topics, such as sports, news, literature, etc. 
However, since the corpus is small, its pseudoword cannot cover all possible contexts in which the real word might appear.

\begin{table}[t]
\caption{Accuracy in PW-RW task for each target word.}
\label{table:pwrw_individual}
\input{03table08.txt}
\vspace*{-0.5\Cvs}
\end{table}
\begin{table}[t]
\caption{Average accuracies in PW-RW task for verbs, nouns, adjectives and all words.}
\label{table:pwrw_all}
\input{03table09.txt}
\vspace*{-0.5\Cvs}
\end{table}

\subsection{Results of pseudoword and Real Word task}
\label{exp_pwrw}
In this task, we use two baselines. 
The first baseline, MFS-PW, is the system which always chooses the most frequent sense in PW corpus, the second one, MFS-RW, is the system choosing the most frequent sense in RW corpus. 
Comparison between these two baselines also enables us to verify how well pseudoword can simulate real word WSD.
Table \ref{table:pwrw_individual} shows results for each target word.
Table \ref{table:pwrw_all} shows average results for verbs, nouns, adjectives and all target words\footnote{In Tables \ref{table:pwrw_individual} and \ref{table:pwrw_all}, $*$ indicates that the system significantly outperforms MFS-RW.}.



Comparing results in RW task (Tables \ref{table:rw_individual} and \ref{table:rw_all}) and PW-RW task (Tables \ref{table:pwrw_individual} and \ref{table:pwrw_all}), 
we can see that accuracies of WSD systems in RW-PW task are worse than those in RW task in all feature sets.
It seems that WSD classifiers trained from PW corpus could not perform as well as ones trained from RW corpus,
although two words of pseudo-senses were not randomly chosen but related with real senses.
The first reason is that pseudowords are not actually real words, 
so there are certain differences among features extracted from PW corpus, and features from RW corpus. 
The second reason is that the most frequent sense of pseudowords in some cases totally different from the real most frequent sense.
This can be empirically observed by seeing that there are great gaps between MFS-PW and MFS-RW in Table \ref{table:pwrw_individual}. 
For example, MFS-PW of \textit{`V7.mat'} is 19.2\% while its MFS-RW is 80.8\%.
Therefore, the training data for the least frequent sense in PW corpus could not learn the behavior of that sense in the RW corpus (which is the most frequent sense indeed).
The worst case is adjectives where disagreement of the most frequent sense is found in 4 of 5 adjectives. 
This is also the reason why the accuracies for adjectives are much lower than for verbs and nouns.

As shown in Table \ref{table:pwrw_individual}, 
classifiers trained from PW corpus do not significantly outperform MFS-RW except for V1,
N6 and N7 (marked by $*$). 
This might be because the training data (Vietnamese Treebank) used in our experiment is not so large. 
One way to enlarge the size of training data is to use not manually annotated but automatically analyzed syntactic trees for SYN features.
However, no public syntactic parser for Vietnamese is currently available.

On average,
in Table \ref{table:pwrw_all},
systems without BOW feature achieved relatively better results.
Although BOW works well on RW and PW task, 
it performs poorest compared to other feature sets.
One of the reasons might be
the mismatch of words appearing in the context of target words in PW
and RW corpus. Many words in the test RW corpus might be `unknown'
in the training PW corpus, causing the decline of accuracy.
Comparing BOW and POS, BOW would suffer from the mismatch,
since the variety of words (feature space of BOW in other words) is
much broader than that of POS. This assumption would be supported by the
fact that POS is better than BOW in Table \ref{table:pwrw_all}.



\section{Discussion} \label{section:discuss}
In this section, we will discuss three issues:
comparison between SVM and Naive Bayes model in \ref{NBclassifier}, 
differences of effective WSD features for different languages in \ref{Otherlanguage},
and the previous work on the pseudoword technique in \ref{PWpreviouswork}

\subsection{Naive bayes classifier} \label{NBclassifier}
In order to compare our results with other learning method, we also performed experiments using a Naive Bayes classifier (NB hereafter) \cite{duda:01}.
Naive Bayes classifier is trained with the same feature set described in Section \ref{section:task}.
Sense of a target word $w_i$ is determined by Eq. \eqref{eq:nb}.
\begin{equation}\label{eq:nb}
S(w_i) = \argmax_{S_k} \left[ \ln P(S_k) + \sum_{x_j \in F_i} \ln P(x_j | S_k) \right]
\end{equation}
where $S_k$ is one of the two senses ($k={0,1}$), 
$P(S_k)$ is the probability of sense $S_k$ in the training corpus,
$x_j$ is a value of feature vector $F_i$,
$P(x_j | S_k)$ is the conditional probability of $x_j$.

\begin{table}[t]
\caption{Average accuracies of Naive Bayes for all words in three tasks.}
\label{table:acc_nb}
\input{03table10.txt}
\end{table}

Table \ref{table:acc_nb} shows results of Naive Bayes classifiers,
i.e. averages of accuracies for all words in three tasks.
The accuracies of NB are almost always worse than SVM. 
$\triangleright$ indicates that NB is significantly worse than SVM by McNemar's test ($p<0.05$). 
One exceptional case is POS in PW-RW task where NB is better than SVM. 
However, the difference is not statistically significant. 
From these results, 
SVM might be a more appropriate learning algorithm for Vietnamese WSD.
In RW task,
BOW is the best single feature for SVM,
while POS and SYN for NB.
However,
the accuracy of SVM using BOW feature is much higher than NB using POS or SYN.
We still conclude that BOW is the best feature for Vietnamese.

\subsection{Comparison of effective features for different languages} \label{Otherlanguage}
For English, several papers have reported empirical evaluation of
different types of features.
Lee and Ng evaluated a variety of features and supervised
learning algorithms (SVM, Naive Bayes, AdaBoost and Decision Tree)
on the SENSEVAL-2 and SENSEVAL-1 data~\cite{lee:02:a}.
Among 4 learning algorithms, SVM achieved the best performance.
They compared features similar to BOW, POS, COL and SYN in this paper,
and reported that COL was the best feature type,
followed by BOW, POS and SYN.
When we implemented the SVM classifiers with the exactly same BOW, POS and COL
feature proposed by \cite{lee:02:a} and evaluated the performance of them for Vietnamese WSD,
we found that COL was also the best 
(the average accuracy was 85.3 for all words),
followed by SYN (83.4), POS (79.5) and BOW (79.3)\footnote{We could not
implement the exactly same feature for SYN, because Lee and Ng used dependency trees to extract syntactic features,
while we used constituent trees.
Actually, they converted constituent trees to dependency trees, 
but did not show the conversion algorithm.
Furthermore, they applied feature selection for SYN features,
but detailed algorithm was not explained, either.}.
On the other hand,
when we used our own features described in Subsection \ref{featureset},
BOW was significantly better than COL for Vietnamese WSD
as shown in Table \ref{table:rw_all}.
Our features seem more appropriate for Vietnamese WSD than Lee's ones,
since the accuracy of our method was much better\footnote{Considering feature combination,
our method was also better than Lee's method.
When all 4 features were used, 
the accuracy of our method was 93.3,
while Lee's method achieved 88.7. 
Without SYN feature (BOW+POS+COL), the accuracy is 93.2 for our method,
while 86.9 for Lee's method.}.
We may say that
local collocations near the target word would be useful for English WSD,
while words in the context in a wide range would be effective
for Vietnamese.

Mart\'{i}nez et~al. explored the contribution of syntactic features 
by training Decision List and AdaBoost
on the SENSEVAL-2 English data~\cite{martinez:02:a}.
The paper revealed that COL was more effective than SYN,
although syntactic features contributed to the gain of WSD precision
when they combined with COL and BOW.
Mohammad and Pedersen have also reported
similar results~\cite{mohammad:04:a}.
They trained Decision Tree on the data of SENSEVAL-2, SENSEVAL-1 and others,
and showed that (1) COL was better feature than SYN, 
(2) simple ensemble of two classifiers using COL and SYN
achieved the increase of the accuracy.
As shown in Table \ref{table:rw_all},
SYN was also less effective than COL for Vietnamese WSD.
Seeing results of two feature combinations with SYN (BOW+SYN, POS+SYN and COL+SYN),
SYN contributed to the gain of accuracies when it combined with POS and COL, but not with BOW since the performance of BOW was much better than SYN.

Murata et al. worked on the comprehensive study of supervised machine
learning of Japanese WSD~\cite{murata:03:a}.
They evaluated several machine learning methods
(SVM, Naive Bayes, Decision List and ensembles of them)
with several feature sets (COL, POS, SYN, BOW as well as topics of documents)
on the data of SENSEVAL-2 Japanese dictionary task.
The results of Naive Bayes classifiers,
which was the best system except for ensembles of multiple learning
algorithms, showed that 
the most effective feature was COL, followed by BOW, SYN and POS.
Our results showed that BOW would be the most effective for Vietnamese
WSD, but it might be less useful than COL in Japanese, like English.

Note that the above discussions are just rough comparisons
between languages,
since the feature sets used in previous work and ours are not exactly same.
Furthermore, the effectiveness of features might be dependent
not only on languages but also other factors, such as target words,
sense definitions (fine or coarse grained),
genres of texts and machine learning algorithms\footnote{
  For example, the order of the effectiveness of features
  were different according to the machine learning algorithms
  in \cite{murata:03:a}.
}.
To more precisely explore differences of effective features
among different languages,
more sophisticated designs of experiments would be required.
That is,
we should prepare parallel corpora with annotations of senses,
use bilingual or multilingual lexicons
to define the same set of target words and their senses,
train WSD classifiers using the same machine learning algorithm,
and use the exactly same feature set.
Such an experiment is beyond the scope of this paper,
since currently we do not have the necessary language resources.



\subsection{Previous work on pseudoword} \label{PWpreviouswork}
Gale et al. introduced the `pseudoword' technique at first in English \cite{Gale1992}. 
They built a pseudo-ambiguous word by combining two or three randomly chosen unambiguous words and tried to disambiguate these two or three pseudo-senses.
The unambiguous words came from definition sentences in a dictionary,
and they were chosen so that the frequencies of pseudowords were equal.
Although this is not a real WSD system,
the idea of pseudoword helps to develop large amounts of training material.
In the study of \cite{Gaustad:01}, 
the author constructed experiments to compare the performance of Naive Bayes classifier for real ambiguous word and pseudoword.
Pseudowords were created by choosing words with the same frequency ratios to that of real senses.
The paper reported that accuracies of pseudoword disambiguation were different from that of real WSD,
indicating that pseudoword technique would not be valid for evaluation of WSD systems.

In most previous work,
semantic properties of senses were not considered for the choice of pseudowords.
While Lu et al. proposed the method for Chinese WSD to automatically choose unambiguous pseudowords similar to real senses using a thesaurus \cite{Lu2006}.
Furthermore,
like our PW-RW task,
pseudowords in an unannotated corpus were used to estimate the probabilities of Naive Bayes model for real WSD.
The trained NB achieved good results, 
even higher than supervised classifiers trained from a relatively small amount of sense tagged corpus.

Our pseudoword technique is similar to \cite{Lu2006}, which considers semantic properties of pseudowords.
One of the differences is that pseudowords were automatically chosen using the Chinese thesaurus in \cite{Lu2006}, while manually chosen in this paper.
Lu's method seems preferable to ours,
since manual choice of pseudowords might be arbitrary.
Another difference is the size of the training corpus.
As discussed in \ref{exp_pwrw},
pseudoword technique did not work well in our experiment of PW-RW task,
while it worked well with a large amount of training data in \cite{Lu2006}.
From another point of view,
the lack of language resources and tools in Vietnamese,
such as a thesaurus (for automatic selection of pseudowords)
and a syntactic parser (to obtain a large training corpus with parse tree),
might be an obstacle to application of pseudoword technique for Vietnamese WSD.



\section{Conclusion} \label{section:conclude}
In this research, we have developed a WSD system for Vietnamese language on two corpora: 
RW corpus (which was manually built) and PW corpus (collected automatically). 
In RW task, the best average accuracy for all words is 94.0\%.
We have experimented using three tasks to evaluate the effectiveness of each feature and feature combinations with and without a sense-tagged corpus.
For the first goal to explore effective features, we found that BOW is the most effective one.
Combinations of BOW and other features enhance the performance of WSD system
in some cases, but not significantly.
For the other goal to check the applicability of the pseudoword technique, 
we found that it is 
useful to rank feature types according to effectiveness for WSD 
and find best single feature for individual target verbs and adjectives.
In addition, pseudoword technique might be an alternative WSD approach when there is no training data.

However, there are some disadvantages in this research. 
For example, the data sparseness is problematic for training classification models, 
and the assumption of two senses per target word may not be realistic. 
Therefore, it will be interesting to investigate the effective features for WSD multi-class classifiers along with increasing the corpus size. 
Also, we could not clearly find the best feature combination. 
More large-scaled sense tagged corpus enables us to explore the best feature combination for Vietnamese WSD.
Effectiveness of other types of features should also be investigated. 
For example, Cai et al. used features about the topics of documents \cite{cai:07:a}, 
which are derived by Latent Dirichlet Allocation \cite{blei:03:a}. 
They reported that topic features were effective for English, but not sure for Vietnamese.
Although the results of our experiments in PW-RW task showed that pseudoword technique did not work well as unsupervised WSD method, 
it should be evaluated again with a larger corpus.
Another interesting proposal is comparing the effective features between Vietnamese WSD and other languages
in precise experiments as discussed in Subsection \ref{Otherlanguage}.


\bibliographystyle{jnlpbbl_1.5}
\begin{thebibliography}{}

\bibitem[\protect\BCAY{Agirre \BBA\ Edmonds}{Agirre \BBA\
  Edmonds}{2006}]{WSDbook}
Agirre, E.\BBACOMMA\ \BBA\ Edmonds, P. \BBOP 2006\BBCP.
\newblock {\Bem Word Sense Disambiguation, Algorithms and Application},
  \lowercase{\BVOL}~33.
\newblock Text, Speech and Language Technology.

\bibitem[\protect\BCAY{Agirre \BBA\ Martinez}{Agirre \BBA\
  Martinez}{2001}]{Agirre2001}
Agirre, E.\BBACOMMA\ \BBA\ Martinez, D. \BBOP 2001\BBCP.
\newblock \BBOQ Learning Class-to-class Selectional Preferences.\BBCQ\
\newblock In {\Bem ConLL '01: Proceedings of the 2001 Workshop on Computational
  Natural Language Learning}, \mbox{\BPGS\ 1--8}. Association for Computational
  Linguistics.

\bibitem[\protect\BCAY{Blei, Ng, \BBA\ Jordan}{Blei et~al.}{2003}]{blei:03:a}
Blei, D.~M., Ng, A.~Y., \BBA\ Jordan, M.~I. \BBOP 2003\BBCP.
\newblock \BBOQ Latent Dirichlet Allocation.\BBCQ\
\newblock {\Bem Journal of Machine Learning Research}, {\Bbf 3}, \mbox{\BPGS\
  993--1022}.

\bibitem[\protect\BCAY{Cai, Lee, \BBA\ Teh}{Cai et~al.}{2007}]{cai:07:a}
Cai, J.~F., Lee, W.~S., \BBA\ Teh, Y.~W. \BBOP 2007\BBCP.
\newblock \BBOQ Improving Word Sense Disambiguation Using Topic Features.\BBCQ\
\newblock In {\Bem Proceedings of the 2007 Joint Conference on Empirical
  Methods in Natural Language Processing and Computational Natural Language
  Learning}, \mbox{\BPGS\ 1015--1023}.

\bibitem[\protect\BCAY{Chang \BBA\ Lin}{Chang \BBA\ Lin}{2001}]{LibSVM}
Chang, C.-C.\BBACOMMA\ \BBA\ Lin, C.-J. \BBOP 2001\BBCP.
\newblock {\Bem LIBSVM: a library for support vector machines}.
\newblock Software available at
  \texttt{http://www.csie.ntu.edu.tw/~cjlin/libsvm}.

\bibitem[\protect\BCAY{Corinna \BBA\ Vladimir}{Corinna \BBA\
  Vladimir}{1995}]{Cortes1995}
Corinna, C.\BBACOMMA\ \BBA\ Vladimir, V. \BBOP 1995\BBCP.
\newblock \BBOQ Support-Vector Networks.\BBCQ\
\newblock {\Bem Machine Learning}, {\Bbf 20}, \mbox{\BPGS\ 273--297}.

\bibitem[\protect\BCAY{Dang, Chia, Palmer, \BBA\ Chiou}{Dang
  et~al.}{2002}]{Dang2002}
Dang, H., Chia, C.-Y., Palmer, M., \BBA\ Chiou, F.-D. \BBOP 2002\BBCP.
\newblock \BBOQ Simple Features for {C}hinese Word Sense Disambiguation.\BBCQ\
\newblock {\Bem Proceedings of the 19th International Conference On
  Computational Linguistics}, {\Bbf 1}, \mbox{\BPGS\ 1--7}.

\bibitem[\protect\BCAY{Dinh}{Dinh}{2002}]{Dinh2002}
Dinh, D. \BBOP 2002\BBCP.
\newblock \BBOQ Building a Training Corpus for Word Sense Disambiguation in
  {E}nglish-to-{V}ietnamese Machine Translation.\BBCQ\
\newblock In {\Bem {COLING}-02 on Machine Translation in Asia}, \mbox{\BPGS\
  1--7}, Morristown, NJ, USA. Association for Computational Linguistics.

\bibitem[\protect\BCAY{Duda, Hart, \BBA\ Stork}{Duda et~al.}{2001}]{duda:01}
Duda, R.~O., Hart, P.~E., \BBA\ Stork, D.~G. \BBOP 2001\BBCP.
\newblock {\Bem Pattern Classification, 2nd Edition}.
\newblock Wiley-Interscience.

\bibitem[\protect\BCAY{Gale, Church, \BBA\ Yarowsky}{Gale
  et~al.}{1992a}]{GaleChurch1992}
Gale, W.~A., Church, K.~W., \BBA\ Yarowsky, D. \BBOP 1992a\BBCP.
\newblock \BBOQ One Sense per Discourse.\BBCQ\
\newblock In {\Bem HLT '91: Proceedings of the Workshop on Speech and Natural
  Language}, \mbox{\BPGS\ 233--237}, Morristown, NJ, USA. Association for
  Computational Linguistics.

\bibitem[\protect\BCAY{Gale, Church, \BBA\ Yarowsky}{Gale
  et~al.}{1992b}]{Gale1992}
Gale, W.~A., Church, K.~W., \BBA\ Yarowsky, D. \BBOP 1992b\BBCP.
\newblock \BBOQ Work on Statistical Methods for Word Sense
  Disambiguation.\BBCQ\
\newblock In {\Bem AAAI Fall Symposium Series, Probabilistic Approaches to
  Natural Language}, \mbox{\BPGS\ 54--60}. AAAI Press.

\bibitem[\protect\BCAY{Gaustad}{Gaustad}{2001}]{Gaustad:01}
Gaustad, T. \BBOP 2001\BBCP.
\newblock \BBOQ Statistical Corpus-Based Word Sense Disambiguation: Pseudowords
  vs Real Ambiguous Words.\BBCQ\
\newblock In {\Bem Proceedings of the 39th Annual Meeting of the Association
  for Computational Linguistics (ACL/EACL 2001). Proceedings of the Student
  Research Workshop}.

\bibitem[\protect\BCAY{Kaplan}{Kaplan}{1955}]{Kaplan1950}
Kaplan, A. \BBOP 1955\BBCP.
\newblock \BBOQ An Experiment Study of Ambiguity and Context.\BBCQ\
\newblock {\Bem Mechanical Translation}, {\Bbf 2}, \mbox{\BPGS\ 39--46}.

\bibitem[\protect\BCAY{Lee \BBA\ Ng}{Lee \BBA\ Ng}{2002}]{lee:02:a}
Lee, Y.~K.\BBACOMMA\ \BBA\ Ng, H.~T. \BBOP 2002\BBCP.
\newblock \BBOQ An Empirical Evaluation of Knowledge Sources and Learning
  Algorithms for Word Sense Disambiguation.\BBCQ\
\newblock In {\Bem Proceedings of the Conference on Empirical Methods in
  Natural Language Processing ({EMNLP})}, \mbox{\BPGS\ 41--48}.

\bibitem[\protect\BCAY{Lesk \BBA\ Michael}{Lesk \BBA\ Michael}{1986}]{Lesk1986}
Lesk\BBACOMMA\ \BBA\ Michael \BBOP 1986\BBCP.
\newblock \BBOQ Automatic Sense Disambiguation Using Machine Readable
  Dictionaries: How to Tell a Pine Cone from an Ice Cream Cone.\BBCQ\
\newblock In {\Bem SIGDOC '86: Proceedings of the 5th Annual International
  Conference on Systems Documentation}, \mbox{\BPGS\ 24--26}, New York, NY,
  USA. ACM.

\bibitem[\protect\BCAY{Lu, Wang, Yao, Liu, \BBA\ Li}{Lu et~al.}{2006}]{Lu2006}
Lu, Z., Wang, H., Yao, J., Liu, T., \BBA\ Li, S. \BBOP 2006\BBCP.
\newblock \BBOQ An Equivalent Pseudoword Solution to {C}hinese Word Sense
  Disambiguation.\BBCQ\
\newblock In {\Bem Proceedings of the 21st International Conference on
  Computational Linguistics and 44th Annual Meeting of the Association for
  Computational Linguistics}, \mbox{\BPGS\ 457--464}, Sydney, Australia.
  Association for Computational Linguistics.

\bibitem[\protect\BCAY{Mart\'{i}nez, Agirre, \BBA\ M\`{a}rquez}{Mart\'{i}nez
  et~al.}{2002}]{martinez:02:a}
Mart\'{i}nez, D., Agirre, E., \BBA\ M\`{a}rquez, L. \BBOP 2002\BBCP.
\newblock \BBOQ Syntactic Features for High Precision Word Sense
  Disambiguation.\BBCQ\
\newblock In {\Bem Proceedings of the 19th International Conference on
  Computational linguistics ({COLING})}, \mbox{\BPGS\ 626--632}.

\bibitem[\protect\BCAY{Mohammad \BBA\ Pedersen}{Mohammad \BBA\
  Pedersen}{2004}]{mohammad:04:a}
Mohammad, S.\BBACOMMA\ \BBA\ Pedersen, T. \BBOP 2004\BBCP.
\newblock \BBOQ Combining Lexical and Syntactic Features for Supervised Word
  Sense Disambiguation.\BBCQ\
\newblock In {\Bem Proceedings of the Conference on Computational Natural
  Language Learning}, \mbox{\BPGS\ 25--32}.

\bibitem[\protect\BCAY{Murata, Utiyama, Uchimoto, Ma, \BBA\ Isahara}{Murata
  et~al.}{2003}]{murata:03:a}
Murata, M., Utiyama, M., Uchimoto, K., Ma, Q., \BBA\ Isahara, H. \BBOP
  2003\BBCP.
\newblock \BBOQ {CRL} at {J}apanese Dictionary-based Task of
  {SENSEVAL}-2---Comparison of Various Types of Machine Learning Methods and
  Features in Japanese Word Sense Disambiguation---.\BBCQ\
\newblock {\Bem Journal of Natural Language Processing}, {\Bbf 10}  (3),
  \mbox{\BPGS\ 115--133}.
\newblock (in {J}apanese).

\bibitem[\protect\BCAY{Nguyen, Vu, Nguyen, Nguyen, \BBA\ Le}{Nguyen
  et~al.}{2009}]{Nguyen2009}
Nguyen, P.~T., Vu, X.~L., Nguyen, T. M.~H., Nguyen, V.~H., \BBA\ Le, H.~P.
  \BBOP 2009\BBCP.
\newblock \BBOQ Building a Large Syntactically-Annotated Corpus of
  {V}ietnamese.\BBCQ\
\newblock In {\Bem Proceedings of the Third Linguistic Annotation Workshop},
  \mbox{\BPGS\ 182--185}, Suntec, Singapore. Association for Computational
  Linguistics.

\bibitem[\protect\BCAY{Yarowsky}{Yarowsky}{1994}]{Yarowsky1994}
Yarowsky, D. \BBOP 1994\BBCP.
\newblock \BBOQ Decision Lists for Lexical Ambiguity Resolution: Application to
  Accent Restoration in Spanish and French.\BBCQ\
\newblock In {\Bem Proceedings of the 32nd Annual Meeting of the Association
  for Computational Linguistics}, \mbox{\BPGS\ 88--95}. ACL.

\bibitem[\protect\BCAY{Yarowsky \BBA\ Florian}{Yarowsky \BBA\
  Florian}{2002}]{Yarowsky2002}
Yarowsky, D.\BBACOMMA\ \BBA\ Florian, R. \BBOP 2002\BBCP.
\newblock \BBOQ Evaluating Sense Disambiguation across Diverse Parameter
  spaces.\BBCQ\
\newblock {\Bem Journal of Natural Language Engineering}, {\Bbf 8},
  \mbox{\BPGS\ 293--310}.

\end{thebibliography}

\begin{biography}

\bioauthor[:]{Minh Hai Nguyen}{
received the B.S from University of Science, Vietnam in 2007
and M.S from Japan Advanced Institute of Science and Technology in 2010.
She was a teaching assistant at University of Science,
Vietnam from 2007 to 2009.
She is currently a Ph.D. student at School of Information Science,
Japan Advanced Institute of Science and Technology.
Her current research field is natural language processing, specifically
sentence retrieval methods and applications.
}

\bioauthor[:]{Kiyoaki Shirai}{
received the B.E., M.E. and Dr. Eng. from Tokyo
Institute of Technology in 1993, 1995 and 1998, respectively.
He was an assistant at the Graduate School of Information Science
and Engineering, Tokyo Institute of Technology from 1998 to 2001.
He is currently an associate professor at School of Information
Science, Japan Advanced Institute of Science and Technology.
His current research interests include natural language processing,
especially corpus-based methods and their applications.
He is a member of the Association for Natural Language Processing,
the Information Processing Society of Japan and
the Japanese Society of Artificial Intelligence.
}
\end{biography}

\biodate




\end{document}
