    \documentclass[english]{jnlp_2.1}

\usepackage{jnlpbbl_1.3}
\usepackage[dvipdfmx]{graphicx}
\usepackage{array}
\usepackage{hangcaption_jnlp}


\Volume{25}
\Number{4}
\Month{September}
\Year{2018}

\received{2017}{12}{27}
\revised{2018}{4}{25}
\accepted{2018}{6}{18}

\setcounter{page}{463}

\esubject{Paper}
\etitle{Domain Adaptation using Word Embeddings for Word Sense Disambiguation}
\eauthor{Kanako Komiya\affiref{Ibaraki} \and Minoru Sasaki\affiref{Ibaraki} \and Hiroyuki Shinnou\affiref{Ibaraki} \and Manabu Okumura\affiref{TITECH}}
\eabstract{
In this paper, we propose domain adaptation using word embeddings for word sense disambiguation (WSD). The validity for WSD of word embeddings derived from a huge corpus such as Wikipedia had already been shown, but their validity in a domain adaptation framework has not been previously discussed. If word embeddings are valid in this new context, the impact of the document type of the corpora on WSD is still unknown. Therefore, we investigate the performances of domain adaptation in WSD using word embeddings from the source, target and general corpora and examine (1) whether the word embeddings are valid for domain adaptation of WSD and (2) if they are, the effects of the document type of the corpora from which the word embeddings are derived. We used three corpora of distinct document types and performed domain adaptation experiments using the document types as the domains. The experiments, conducted using Japanese corpora, revealed that the accuracy of WSD was highest when we used the word embeddings obtained from the target corpora together with a general corpora.
}
\ekeywords{Word Sense Disambiguation, Domain Adaptation, Word Embeddings, Word2vec}

\headauthor{Komiya  et al.}
\headtitle{Domain Adaptation using Word Embeddings for Word Sense Disambiguation}

\affilabel{Ibaraki}{}{Ibaraki University}
\affilabel{TITECH}{}{Tokyo Institution of Technology}


\begin{document}

\maketitle

\section{Introduction}

Word embeddings \cite{Mikolov1,Mikolov2,Mikolov3} are the vector representations of word meanings that have compositionality. 
They are effective for word sense disambiguation (WSD) tasks \cite{Sugawara} since they are low-dimensional vectors and the sparseness of the meaning representations can be greatly alleviated through their use.

Domain adaptation involves adapting a classifier that has been trained from data in one domain (source domain) to data in another domain (target domain). 
Domain adaptation, including domain adaptation in WSD (see Section \ref{Related Work}), has been studied extensively. 
First, we propose using word embeddings for domain adaptation of WSD (see Section \ref{Domain Adaptation for Word Sense Disambiguation using Word Embeddings}). 
Since domain adaptation suffers from data sparseness caused by shifting between source and target domains, we suppose that the use of word embeddings can improve the performance of WSD in a domain adaptation framework.

However, one problem with domain adaptation is the shift of the priors of the word senses in the texts \cite{Shinnou1}, which means that the word meanings in one domain could be different from those in another domain. 

Therefore, second, we investigate the performances of domain adaptation in WSD using the word embeddings obtained from the source, target and general corpora, and we examine the effects according to the document types. 
Our experiments were with Japanese WSD, and the experiments (described in Section \ref{Experiment}) revealed that the accuracy of WSD was highest when we used word embeddings obtained from the corpus of the target domain and those obtained from the general domain together. 
The observed dependence on the corpus indicated to us that not only did the size of the corpus used to generate the word embeddings affect the results but so did the document type of the corpus. Finally, we discuss the results (Section \ref{Discussion}) and conclude this paper (Section \ref{Conclusion}).

We have three sense-tagged corpora of different document types: Q \& A sites, newspapers and white papers, so we performed the domain adaptation experiments regarding these document types as the domains. 
Considering that, the same topic can be described in both Q \& A sites and newspapers, the main difference among these corpora is not the topic of the documents but rather the manner of writing. 


\section{Related Work}
\label{Related Work}

The use of word embeddings is a considerably active area of current and recent research. Some important examples of this research include the works of \citeA{Sugawara} and \citeA{Taghipour}. The former study investigated the impact of the use of word embeddings for WSD, whereas the latter study proposed semi-supervised WSD using word embeddings and revealed that the word embeddings of a general or specific domain improved the performance of WSD.

Research on domain adaptation is not limited to the area of natural language processing. 
The domain adaptation problem can be categorized into three types depending on the information to be learned, namely, supervised, semi-supervised and unsupervised approaches. 
According to Daum\'e \cite{Daume2010}, in a supervised approach, a classifier is developed from a large quantity of labelled source data and a small amount of labelled target data. In a semi-supervised approach, a classifier is developed from a large amount of labelled source data, a small amount of labelled target data and a large amount of unlabelled target data. 
Finally, in an unsupervised approach, a classifier is developed from a large amount of labelled source data and unlabelled target data. 
Our focus in the research reported in this paper is on unsupervised domain adaptation in Japanese WSD.

\citeA{Chan2006,Chan2007} carried out the domain adaptations of WSD by estimating class priors using an EM algorithm, which utilizes unsupervised domain adaptation, supervised domain adaptation and active learning. 
For supervised domain adaptation, \citeA{Daume2007} augmented an input space and made triple-length features that were general, source specific and target specific. 
\citeA{Daume2010} then extended the earlier work to semi-supervised domain adaptation. 
\citeA{Agirre2008,Agirre2009} applied singular value decomposition (SVD) to a matrix of unlabelled target data and a large amount of unlabelled source data and then used them to train a classifier. 
\citeA{Kunii} proposed the combined use of topic models and unsupervised domain adaptation in WSD. 
When domain adaptation was applied, \citeA{Jiang} demonstrated that performance increased as examples were weighted. 
\citeA{Shinnou1} reported that active learning can be used to remove source instances for domain adaptation in WSD. 
Shinnou et al., also proposed learning under a covariate shift for domain adaptation in WSD \cite{Shinnou2}. 
\citeA{Kouno} performed unsupervised domain adaptation in WSD using a stacked denoising autoencoder.
The research by \citeA{Blitzer} involved work on semi-supervised domain adaptation, where they calculated the weight of words around pivot features (words that appeared frequently in both source and target data and behaved similarly in each) to model words in one domain that behaved similarly in another. 
To achieve this, they applied SVD to the matrix of the weights, generated a new feature space and used the new features with the original features.

Given a triple of the target word type of WSD, source data and target data, \citeA{KomiyaIJCNLP,KomiyaJ12} determined an optimal method of domain adaptation using decision tree learning. 
They discussed which features influenced how the best method was determined and proposed determining the optimal training data set for each instance using the degree of confidence for supervised domain adaptation in WSD \cite{KomiyaPaclic}.

That work which is closest to ours is that of \citeA{Sugawara}, which investigated the context representation using word embeddings for WSD. 
\citeA{Iacobacci} also employed word embeddings for word sense disambiguation, but they did not attempt domain adaptation. 
\citeA{Yamaki} also proposed a method that employed context word embeddings for supervised word sense disambiguation. We propose using the context representation derived from word embeddings for domain adaptation of WSD and furthermore show that word embeddings can improve the performance of WSD in a domain adaptation framework. 
Additionally, we investigate the performances of domain adaptation in WSD using the word embeddings obtained from the source, target and general corpora and examine the effects according to their document types.


\section{Domain Adaptation in WSD using Word Embeddings}
\label{Domain Adaptation for Word Sense Disambiguation using Word Embeddings}

In addition to the baseline features in both the source and target data, we carry out domain adaptation for WSD by concatenating the features of the word embeddings of words surrounding the target word of WSD. 
We followed \citeA{Sugawara} and used Context-Word-Embeddings; a Context-Word-Embedding is a vector formed by the concatenation of the real-valued vectors consisting of the words in the context window of the target word of WSD.\footnote{\citeA{Sugawara} reported that Context-Word-Embeddings improved the results of WSD more than Average-Word-Embeddings, which are the averages of vector representations of words in the context window of the target word of WSD.}
As Sugawara et al., described, Context-Word-Embeddings of a target word of WSD is a vector concatenating $v_{w_{-N}}, \ldots , v_{w_{-1}}, v_{w_{+1}}, \ldots ,v_{w_{+N}}$ when the window size is $N$ and the words appearing in the context window are $w_{-N}, \ldots , w_{-1}, w_{+1}, \ldots ,w_{+N}$, where $v_w$ represents an embedding of word $w$. 
If the dimension of each word embedding is $d$, the size of this feature vector is $2N \times d$. 
Table \ref{tab:Context-Word-Embeddings} and Figure \ref{tab:Context-Word-Embeddings} show a simple example of Context-Word-Embeddings. 
Figure \ref{tab:Context-Word-Embeddings} shows the Context-Word-Embeddings of an instance vector for the word ``大きい'' (big) in the context of the phrase ``効果が大きいため、'' (Because the effect is big,) as shown in Table \ref{tab:Context-Word-Embeddings}. 
In this example, $N$ is two and $d$ is three.
 
\begin{table}[b]
\caption{Simple example of Context-Word-Embeddings}
\label{tab:Context-Word-Embeddings}
\input{06table01.tex}
\end{table}

\begin{figure}[t]
\begin{center}
\includegraphics{25-4ia6f1.eps}
\end{center}
\caption{Example of Context-Word-Embeddings (after concatenation)}
\label{Fig:concept text}
\end{figure}

We investigated the following nine methods of domain adaptation which vary by the document types of the corpora used to generate the word embeddings. 

\begin{description}
\item {\bf Add Target} Word embeddings obtained from the target data are used.
\item {\bf Add Source} Word embeddings obtained from the source data are used. 
\item {\bf Add Wiki} Word embeddings obtained from Wikipedia, a huge, general corpus, are used.
\item {\bf Add Target \& Source} Word embeddings obtained from the source and those obtained from the target data are used together by concatenating them.
\item {\bf Add Target \& Wiki} Word embeddings obtained from the target data and those obtained from the huge, general corpus are used together by concatenating them.
\item {\bf Add Target Large} Word embeddings obtained from the large target data are used.
\item {\bf Add Source Large} Word embeddings obtained from the large source data are used.
\item {\bf Add Target Large \& Source Large} Word embeddings obtained from the large target data and those obtained from the large source data are used together by concatenating them.
\item {\bf Add Target Large \& Wiki} Word embeddings obtained from the large target data and those obtained from the huge, general corpus are used together by concatenating them.
\end{description}


\section{Experiment}
\label{Experiment}

We used word2vec\footnote{https://code.google.com/archive/p/word2vec/} \cite{Mikolov1,Mikolov2,Mikolov3} to generate the word embeddings. 
To generate the word embeddings, the vector size (which is $d$ in Section \ref{Domain Adaptation for Word Sense Disambiguation using Word Embeddings}) and the window size were set to 200 and 5, respectively. We used a skip-gram algorithm. 
The window size of each target word (which is $N$ in Section \ref{Domain Adaptation for Word Sense Disambiguation using Word Embeddings}) was set to 2. 
We used default settings for other parameters.
Libsvm \cite{LIBSVM}, which supports multi-class classification, was used as the classifier for WSD.\footnote{We used the -b option of libsvm.}
A linear kernel was used in accordance with the results obtained from preliminary experiments. 
Twenty features, which were the same as \cite{KomiyaIJCNLP} and \cite{KomiyaPaclic}, were introduced as baseline features to train the classifier.

\begin{itemize}
\item Morphological features used include:
	\begin{itemize}
	\item Bag-of-words,
	\item Part-of-speech (POS), and
        \item Finer subcategory of POS.
        \end{itemize}
\item Baseline syntactic features are as follows.
	\begin{itemize}
	\item If the POS of a target word is a noun, the verb that the target word modifies is used. 
        \item If the POS of a target word is a verb, the case element of `ヲ' (wo, objective) for the verb is used. 
        \end{itemize}
\item The semantic feature of the baseline was 
	\begin{itemize}
	\item Semantic classification code.
        \end{itemize}
\end{itemize}

Morphological features and a semantic feature were extracted from the surrounding words (two words to the right and left) of the target word and from the target word itself. 
POS and the finer subcategory of POS could be obtained by using a morphological analyser. 
We used Mecab\footnote{https://github.com/jordwest/mecab-docs-en} as a morphological analyser, CaboCha\footnote{http://sourceforge.net/projects/cabocha/} as a syntactic parser and the Word List by Semantic Principles \cite{Bunruigoihyo} for semantic classification codes. (For example, the code of the `program' was 1.3162.)

\begin{table}[p]
\caption{Minimum, maximum and average number of instances of each word type for each corpus}
\label{tab:minimum, maximum, and average}
\input{06table02.tex}
\end{table}
\begin{table}[p]
\caption{Percentage of word types in the target corpus unknown in the source corpus} 
\label{tab:unknown word ratio}
\input{06table03.tex}
\end{table}
\begin{table}[p]
\caption{List of target word types} \label{tab:The list of target words}
\input{06table04.tex}
\end{table}


\subsection{Data}
\label{sec:DATA}

Three labelled data sets were used for the experiments: 
(1) the sub-corpus of white papers in the Balanced Corpus of Contemporary Japanese (BCCWJ) \cite{BCCWJ}, 
(2) the sub-corpus of documents from a Q \& A site on the WWW in the BCCWJ and 
(3) Real World Computing (RWC) text databases (newspaper articles) \cite{RWC}. 
Domain adaptation was conducted in six directions in accordance with the possible combinations of source and target data. 
Word senses were annotated in these corpora in accordance with a Japanese dictionary, that is, the Iwanami Kokugo Jiten \cite{Iwanami}. 
It has three levels for sense IDs, and we used the middle-level sense in the experiments. 
Multi-sense words that appeared at least 50 times in each data set were selected as the target words in the experiment; 36-word types were used in the experiments. 
Table \ref{tab:minimum, maximum, and average} lists the minimum, maximum and average number of instances of each word type for each corpus. 
Table \ref{tab:unknown word ratio} shows the percentage of word types in the target corpus unknown in the source corpus. 
For example, 40.82\% of the words appearing in white papers did not appear in the Q \& A site corpus. 
Table \ref{tab:The list of target words} summarizes the list of target word types. 
The first column labelled `No. of senses' is the number of the senses in the dictionary of each word type. 
For example, the word type ``考える''  (think) has two senses in the dictionary. 
Please note that there is no guarantee that all the senses in the dictionary will appear in the corpora.

In addition, we used seven types of unlabelled data to generate the word embeddings: 
(1) a large collection of white papers in BCCWJ, 
(2) a large collection of documents from a Q \& A site on the WWW in the BCCWJ, 
(3) a large collection of the newspaper articles of the Mainichi Shimbun 1994 including RWC text databases, 
(4) collections of the Diet minutes and Budgets and Audits at the National Diet from 1947 to 2012,\footnote{http://www2.ninjal.ac.jp/lrc/index.php} 
(5) a subset of Q \& A sites on the WWW, specifically, ``Yahoo! Chiebukuro data (2nd edition)'',\footnote{ http://www.nii.ac.jp/dsc/idr/en/yahoo/chiebkr2/Y\_chiebukuro.html}
(6) a large collection of the newspaper articles of the Mainichi Shimbun from 1991 to 2005 including RWC text databases, and 
(7) dumped Japanese Wikipedia data (2015/10/02).\footnote{ https://dumps.wikimedia.org/jawiki/}
Of these, corpora (1), (2) and (3), are the unlabelled corpora that is larger than the labelled data for WSD. 
It includes the corresponding labelled data. Corpora (4), (5) and (6) are used as larger corpora that correspond to the first three. Technically, the texts used to form corpus (4), the Diet minutes and Budgets and Audits, are not white papers, and corpus (4) does not include corpus (1). 
However, we used corpus (4) as a larger corpus corresponding to corpus (1) because we could not find a larger collection of white papers and we believe that the two classes of documents are similar to one another. 
As for Q \& A site corpora, we used two distinct larger corpora; corpus (5a) a subset of Q \& A sites on the WWW that includes corpus (2) as a subset, and corpus (5b) is another subset of the Q \& A sites on the WWW that excludes corpus (2). 
This allowed us to investigate the impact of the inclusion of the unlabelled data of labelled data.\footnote{Half of them are overlapped each other.}
Table \ref{tab:bunsan ratio} shows the number of word tokens and sentences according to the corpora from which the word embeddings generated, the number of word types of the word embeddings and their ratios to those of Wikipedia in accordance with each document type. 
The numbers of types in the table are different from the number of word types of each corpus because word2vec generated the meaning representation vector if and only if a word appeared in the corpus at least five times.

\begin{table}[t] 
\hangcaption{Number of word tokens, sentences and word types of each document type and their ratios to that of Wikipedia}
\label{tab:bunsan ratio}
\input{06table05.tex}
\end{table}
\begin{table}[t]
\caption{Corpora for word embeddings according to  method and source and target corpora}
\label{tab:word embedding corpora}
\input{06table06.tex}
\end{table}

Table \ref{tab:word embedding corpora} shows the corpora for word embeddings according to the method, the source corpus and target corpus. 
For example, (5) a large collections of documents from a Q \& A site on the WWW in the BCCWJ was used for the domain adaptation from (1) the sub-corpus of white papers in the BCCWJ to (2) the sub-corpus of documents from a Q \& A site on the WWW in the BCCWJ by the method {\it Add Target Large}. 
Table \ref{tab:unknown W2V} shows the ratio of unknown word types of the tagged corpus in the word embedding vectors. For example, 12.31\% of the words in white papers did not have word embeddings. See line (1), white papers, in Table \ref{tab:bunsan ratio}.\footnote{Although distribution differences of word embeddings among corpora are helpful to analyse the effects of our method for each corpus, we could not map the word embeddings onto the same feature space because we created them separately.}
The average accuracies over six directions \footnote{Six directions are (1) $\to$ (2), (1) $\to$ (3), (2) $\to$ (1), (2) $\to$ (3), (3) $\to$ (1), and (3) $\to$ (2). } (Table \ref{tab:accuracy}) and the average accuracies over two directions according to the target corpora \footnote{For example, the accuracies of column `Newspaper' are the averages over accuracies of domain adaptation from white papers to newspaper ((1) $\to$ (3)) and from Q \& A site to newspaper ((2) $\to$ (3)) except for `Self', which is the result of (3) $\to$ (3).} (Table \ref{tab:accuracy document type}) were evaluated in Section \ref{Result}.

\begin{table}[p]
\caption{Unknown word type ratio of the tagged corpus in the word embedding corpus} 
\label{tab:unknown W2V}
\input{06table07.tex}
\end{table}
\begin{table}[p] 
\caption{Summary of accuracies}
\label{tab:accuracy}
\input{06table08.tex}
\end{table}

\begin{table}[t] 
\caption{Average accuracies of WSD according to corpora and methods of domain adaptation}
\label{tab:accuracy document type}
\input{06table09.tex}
\end{table}


\subsection{Results}
\label{Result}

In what follows, the micro-average is the average over word tokens, and the macro-average is that over word types. 
Table \ref{tab:accuracy} lists the micro- and macro-averaged accuracies of WSD for the whole data set by method of domain adaptation, and 
Table \ref{tab:accuracy document type} summarizes the micro- and macro-averaged accuracies of WSD by the corpus and method of domain adaptation. 
We tested {\it Self}, which is standard supervised learning with the whole target data, by five-fold cross-validation, assuming that fully annotated data were obtained and could be used for learning with word embedding features,\footnote{{\it ``Self''} was learned using word embeddings from the target corpus itself. For example, ``Self'' for (1) white papers was learned using word embeddings from (1) white papers.}
{\it MFS}, which is the most frequent sense of the target corpus\footnote{Note that we cannot know the most frequent sense in the target corpus without the labeled target data.}, and 
{\it Baseline features}, which is standard supervised learning without word embedding features, as references. 
Please note that all the methods except for {\it Self}, including {\it Baseline features} and {\it MFS}, are performed in the domain adaptation scenario, where the domains of the training and test data are different. 

{\it Self} was an upper bound, and {\it Baseline features} features was a baseline. Excluding {\it Self} and {\it MFS}, the highest accuracies for each corpus are shown in bold typeface in Tables \ref{tab:accuracy} and \ref{tab:accuracy document type}. 
A plus signlus or a minus sign after the percentage indicate that the accuracy of the proposed method is higher or lower, respectively, than that of {\it Baseline features}. 
The asterisk is used to indicate that the difference between accuracies of the proposed method and {\it Baseline features} is statistically significant according to a chi-square test.
The level of significance in the test was 0.05. 


\section{Discussion}
\label{Discussion}

First, Table \ref{tab:accuracy}  demonstrates that the macro-averaged accuracies of all the methods outperformed {\it Baseline features} features, which indicates that the word embedding features are effective in the domain adaptation framework. 
In addition, the same table shows that the micro-averaged accuracies of all methods except {\it Add Wiki} and {\it Add Target Large and Source Large} (namely, Add Target, {\it Add Target}, {\it Add Source}, {\it Add Target and Source}, {\it Add Target and Wiki}, two kinds of {\it Add Target Large}, {\it Add Source Large} and {\it Add Target Large and Wiki}) outperformed {\it Baseline features}. 
Moreover, the differences in micro-averaged accuracies between {\it Baseline features} and all methods except {\it Add Source} and {\it Add Wiki} are statistically significant.

When we used {\it Add Wiki}, macro-averaged accuracy increased but the micro-averaged accuracy decreased. 
Table \ref{tab:accuracy document type} suggests that the reason for the decline is the micro-averaged accuracy of white papers. 
We think this is because the senses in the corpus of white papers are very biased as shown in Table \ref{tab:accuracy document type} 
and for these cases the word embedding features generated from the general corpus cannot improve the accuracy. 
Table \ref{tab:accuracy document type} also shows that 
not only {\it Add Wiki} but also 
{\it Add Target}, {\it Add Source}, {\it Add Source Large} and {\it Add Target Large and Source Large} could not beat {\it Baseline features} for white papers. 
We believe this to be for the same reason. 

The decline of micro-averaged accuracy in {\it Add Target Large and Source Large} also came from white papers. The WSD accuracy of a word ``技術'' (technique), which had more than 7,000 instances, dropped considerably compared with {\it Baseline features} when domain adaptation from newspaper to white paper was conducted.   

Next, Table \ref{tab:accuracy} shows that both micro- and macro-averaged accuracies of {\it Add Target Large}  and the micro-averaged accuracy of {\it Add Target} outperformed {\it Add Wiki}. 
In addition, the differences between {\it Add Target} and {\it Add Wiki} or the differences between {\it Add Target Large} and {\it Add Wiki} in micro-averaged accuracies are statistically significant.
These results indicate that we should use the word embeddings obtained from the target corpus rather than the general corpora.
In particular, the ratios of the vocabulary size of the target corpora to that of Wikipedia range from 1\% to 7\% and those of the larger target corpora are from 6\% to 36\%, according to Table \ref{tab:bunsan ratio}. 
This indicates that {\it Add Target} or {\it Add Target Large} could improve the accuracies of WSD even though the number of word types appearing in the corpora is much smaller than the number of word types in Wikipedia. 

On the other hand, Table \ref{tab:accuracy} shows that in macro-averaged accuracies the best method is Add {\it Add Target Large and Wiki}. 
It also shows that the best method in micro-averaged accuracies is {\it Add Target and Wiki}. 
The same table indicates that the second-best method in micro-averaged accuracy is {\it Add Target Large and Wiki} and {\it Add Target and Wiki} in macro-averaged accuracy. 
Therefore, we should use the word embeddings obtained from the target corpus and the general corpora together.



Finally, we will discuss why the word embedding features obtained from the target corpora are effective for domain adaptation.  
We think that the word embedding features are effective for WSD tasks because their use greatly alleviates the sparseness of the meaning representations. 
Our experiments show that the results of {\it Add Target Large} are always better than those of {\it Add Target}, which indicates that the word embeddings obtained from the larger corpora did improve the performance of WSD when the document types were the same. 
However, we cannot explain all the improvements
of the results in a domain adapta
tion framework only through the alleviation of the sparseness alone. 
Table \ref{tab:ratio of unknown words} shows the ratios of the unknown word types of each method; the ratio of the unknown words of {\it Add Target} (8.53\%) is greater than that of {\it Add Wiki} (2.31\%) in the target data. 

\begin{table}[b]
\caption{Ratio of unknown word types}
\label{tab:ratio of unknown words}
\input{06table10.tex}
\end{table}

This demonstrates that the improvement in the results comes from not only the alleviation of the sparseness, that is, the size of the word embeddings, but also the corresponding document type. 
In particular, when we compared {\it Add Target} with {\it Add Wiki}, the experimental results show that the document type of the corpus used to generate the word embeddings affected the results more than the size of the corpus.

Next, let us investigate the effect of inclusion of the unlabelled data with labelled data. 
 According to Table \ref{tab:accuracy}, {\it Add Target Large excl. (2)} is higher than that of {\it Add Target Large incl. (2)}, which indicates that the accuracy was higher when the unlabelled data were not included. 
Table \ref{tab:bunsan ratio} demonstrates that the number of word tokens and sentences of the corpous from which the word embeddings were generated and the number of word types of the word embeddings of {\it Add Target Large excl. (2)} are greater than those of {\it Add Target Large incl. (2)}. In addition, Table \ref{tab:ratio of unknown words} shows that ratio of unknown word types of {\it Add Target Large excl. (2)} 
(2.12\%) is smaller than that of {\it Add Target Large incl. (2)} (2.16\%). Again, these results indicate that the word embeddings obtained from larger corpora improved the performance of WSD when their document type was the same. 

Moreover, Table \ref{tab:accuracy document type} shows that accuracy of {\it Add Target Large} was better than that of {\it Baseline features} and {\it Add Wiki} for white papers. 
This indicates that collections of the Diet minutes and Budgets and Audits at the National Diet were useful as target corpus for white papers. 
This suggests that minor differences of document types do not affect the accuracies of domain adaptation for word sense disambiguation.

Finally, we should note that it has yet to be determined if our method is applicable to domain adaptation for corpora of other document types. 
It is also unclear more generally under what conditions our method is valid. 
It is difficult to analyse that because one should take into consideration not only the differences among the document types but also the differences among the distributions of the word senses of each target word. 
Therefore, we treat that problem as future work. 
However, we believe that our method is generally applicable to corpora of other document types because most domain adaptation methods are not effective for the three corpora we used when the averaged accuracies 
are evaluated, whereas ours was effective. 
We also believe that this method is applicable to domain adaptation for English word sense disambiguation because the original method, Context-Word-Embeddings, is effective for English 
word sense disambiguation. 


\section{Conclusions}
\label{Conclusion}

We proposed the use of word embedding features generated through word2vec and showed their validity for domain adaptation in Japanese WSD tasks. 
We investigated the performances of domain adaptation in WSD using various methods, 
namely, {\it Add Target}, {\it Add Source}, {\it Add Wiki}, {\it Add Target \& Source}, {\it Add Target \& Wiki} , {\it Add Target Large}, {\it Add Source Large}, {\it Add Target Large \& Source Large}, and {\it Add Target Large \& Wiki}. 
Furthermore, we examined the effects of the types of document in the corpus used to generate the word embeddings. 
Our experiments demonstrated that word embeddings that were obtained from the target corpus together with a huge, general corpus improved the accuracies of WSD more than any other method. 
They also revealed that improvement in the results of domain adaptation came not only from the alleviation of the sparseness, that is, the size of the word embeddings, but also from the document types of the corpus used to generate the word embeddings.  


\acknowledgment

This paper is the revised version of Komiya, Suzuki, Sasaki, \citeA{cicling2017}, which is published in the proceedings of the CICLING 2017. 
We would like to thank the Yahoo Japan Corporation and the National Institute of Informatics that provide us the ``Yahoo! Chiebukuro data (2nd edition).''
This work was partially supported by JSPS KAKENHI Grant Number 15K16046. 


\bibliographystyle{jnlpbbl_1.5}
\begin{thebibliography}{}

\bibitem[\protect\BCAY{Agirre \BBA\ de~Lacalle}{Agirre \BBA\
  de~Lacalle}{2008}]{Agirre2008}
Agirre, E.\BBACOMMA\ \BBA\ de~Lacalle, O.~L. \BBOP 2008\BBCP.
\newblock \BBOQ On Robustness and Domain Adaptation using SVD for Word Sense
  Disambiguation.\BBCQ\
\newblock In {\Bem Proceedings of COLING 2008}, \mbox{\BPGS\ 17--24}.

\bibitem[\protect\BCAY{Agirre \BBA\ de~Lacalle}{Agirre \BBA\
  de~Lacalle}{2009}]{Agirre2009}
Agirre, E.\BBACOMMA\ \BBA\ de~Lacalle, O.~L. \BBOP 2009\BBCP.
\newblock \BBOQ Supervised Domain Adaption for WSD.\BBCQ\
\newblock In {\Bem Proceedings of EACL 2009}, \mbox{\BPGS\ 42--50}.

\bibitem[\protect\BCAY{Blitzer, McDonald, \BBA\ Pereira}{Blitzer
  et~al.}{2006}]{Blitzer}
Blitzer, J., McDonald, R., \BBA\ Pereira, F. \BBOP 2006\BBCP.
\newblock \BBOQ Domain Adaptation with Structural Correspondence
  Learning.\BBCQ\
\newblock In {\Bem Proceedings of EMNLP 2006}, \mbox{\BPGS\ 120--128}.

\bibitem[\protect\BCAY{Chan \BBA\ Ng}{Chan \BBA\ Ng}{2006}]{Chan2006}
Chan, Y.~S.\BBACOMMA\ \BBA\ Ng, H.~T. \BBOP 2006\BBCP.
\newblock \BBOQ Estimating Class Priors in Domain Adaptation for Word Sense
  Disambiguation.\BBCQ\
\newblock In {\Bem Proceedings of COLING-ACL 2006}, \mbox{\BPGS\ 89--96}.

\bibitem[\protect\BCAY{Chan \BBA\ Ng}{Chan \BBA\ Ng}{2007}]{Chan2007}
Chan, Y.~S.\BBACOMMA\ \BBA\ Ng, H.~T. \BBOP 2007\BBCP.
\newblock \BBOQ Domain Adaptation with Active Learning for Word Sense
  Disambiguation.\BBCQ\
\newblock In {\Bem Proceedings of ACL 2007}, \mbox{\BPGS\ 49--56}.

\bibitem[\protect\BCAY{Chang \BBA\ Lin}{Chang \BBA\ Lin}{2001}]{LIBSVM}
Chang, C.-C.\BBACOMMA\ \BBA\ Lin, C.-J. \BBOP 2001\BBCP.
\newblock {\Bem LIBSVM: a library for support vector machines}.
\newblock Software available at
  http://www.csie.ntu.edu.tw/{\textasciitilde}cjlin/libsvm.

\bibitem[\protect\BCAY{Daum{\'{e}}~III}{Daum{\'{e}}~III}{2007}]{Daume2007}
Daum{\'{e}}~III, H. \BBOP 2007\BBCP.
\newblock \BBOQ Frustratingly Easy Domain Adaptation.\BBCQ\
\newblock In {\Bem Proceedings of ACL 2007}, \mbox{\BPGS\ 256--263}.

\bibitem[\protect\BCAY{Daum{\'{e}}~III, Kumar, \BBA\ Saha}{Daum{\'{e}}~III
  et~al.}{2010}]{Daume2010}
Daum{\'{e}}~III, H., Kumar, A., \BBA\ Saha, A. \BBOP 2010\BBCP.
\newblock \BBOQ Frustratingly Easy Semi-Supervised Domain Adaptation.\BBCQ\
\newblock In {\Bem Proceedings of the 2010 Workshop on Domain Adaptation for
  Natural Language Processing, ACL 2010}, \mbox{\BPGS\ 23--59}.

\bibitem[\protect\BCAY{Hashida, Isahara, Tokunaga, Hashimoto, Ogino, \BBA\
  Kashino}{Hashida et~al.}{1998}]{RWC}
Hashida, K., Isahara, H., Tokunaga, T., Hashimoto, M., Ogino, S., \BBA\
  Kashino, W. \BBOP 1998\BBCP.
\newblock \BBOQ The RWC Text Databases.\BBCQ\
\newblock In {\Bem Proceedings of the 1st International Conference on Language
  Resource and Evaluation}, \mbox{\BPGS\ 457--461}.

\bibitem[\protect\BCAY{Iacobacci, Pilehvar, \BBA\ Navigli}{Iacobacci
  et~al.}{2016}]{Iacobacci}
Iacobacci, I., Pilehvar, M.~T., \BBA\ Navigli, R. \BBOP 2016\BBCP.
\newblock \BBOQ Embeddings for Word Sense Disambiguation: An Evaluation
  Study.\BBCQ\
\newblock In {\Bem Processing of ACL 2016}, \mbox{\BPGS\ 897--907}.

\bibitem[\protect\BCAY{Jiang \BBA\ Zhai}{Jiang \BBA\ Zhai}{2007}]{Jiang}
Jiang, J.\BBACOMMA\ \BBA\ Zhai, C. \BBOP 2007\BBCP.
\newblock \BBOQ Instance Weighting for Domain Adaptation in NLP.\BBCQ\
\newblock In {\Bem Proceedings of ACL 2007}, \mbox{\BPGS\ 264--271}.

\bibitem[\protect\BCAY{Komiya \BBA\ Okumura}{Komiya \BBA\
  Okumura}{2011}]{KomiyaIJCNLP}
Komiya, K.\BBACOMMA\ \BBA\ Okumura, M. \BBOP 2011\BBCP.
\newblock \BBOQ Automatic Determination of a Domain Adaptation Method for Word
  Sense Disambiguation Using Decision Tree Learning.\BBCQ\
\newblock In {\Bem Proceedings of IJCNLP 2011}, \mbox{\BPGS\ 1107--1115}.

\bibitem[\protect\BCAY{Komiya \BBA\ Okumura}{Komiya \BBA\
  Okumura}{2012a}]{KomiyaPaclic}
Komiya, K.\BBACOMMA\ \BBA\ Okumura, M. \BBOP 2012a\BBCP.
\newblock \BBOQ Automatic Domain Adaptation for Word Sense Disambiguation Based
  on Comparison of Multiple Classifiers.\BBCQ\
\newblock In {\Bem Proceedings of PACLIC 2012}, \mbox{\BPGS\ 77--85}.

\bibitem[\protect\BCAY{Komiya \BBA\ Okumura}{Komiya \BBA\
  Okumura}{2012b}]{KomiyaJ12}
Komiya, K.\BBACOMMA\ \BBA\ Okumura, M. \BBOP 2012b\BBCP.
\newblock \BBOQ Automatic Selection of Domain Adaptation Method for WSD using
  Decision Tree Learning.\BBCQ\
\newblock {\Bem Journal of NLP (In Japanese)}, {\Bbf 19}  (3), \mbox{\BPGS\
  143--166}.

\bibitem[\protect\BCAY{Komiya, Suzuki, Sasaki, Shinnou, \BBA\ Okumura}{Komiya
  et~al.}{2017}]{cicling2017}
Komiya, K., Suzuki, S., Sasaki, M., Shinnou, H., \BBA\ Okumura, M. \BBOP
  2017\BBCP.
\newblock \BBOQ Domain Adaptation for Word Sense Disambiguation using Word
  Embeddings.\BBCQ\
\newblock In {\Bem Proceedings of CICLING 2017, \textup{no. 57}}.

\bibitem[\protect\BCAY{Kouno, Shinnou, Sasaki, \BBA\ Komiya}{Kouno
  et~al.}{2015}]{Kouno}
Kouno, K., Shinnou, H., Sasaki, M., \BBA\ Komiya, K. \BBOP 2015\BBCP.
\newblock \BBOQ Unsupervised Domain Adaptation for Word Sense Disambiguation
  using Stacked Denoising Autoencoder.\BBCQ\
\newblock In {\Bem Proceedings of PACLIC-29}, \mbox{\BPGS\ 224--231}.

\bibitem[\protect\BCAY{Kunii \BBA\ Shinnou}{Kunii \BBA\ Shinnou}{2013}]{Kunii}
Kunii, S.\BBACOMMA\ \BBA\ Shinnou, H. \BBOP 2013\BBCP.
\newblock \BBOQ Combined Use of Topic Models on Unsupervised Domain Adaptation
  for Word Sense Disambiguation.\BBCQ\
\newblock In {\Bem Proceedings of PACLIC-27}, \mbox{\BPGS\ 224--231}.

\bibitem[\protect\BCAY{Maekawa, Yamazaki, Ogiso, Maruyama, Ogura, Kashino,
  Koiso, Yamaguchi, Tanaka, \BBA\ Den}{Maekawa et~al.}{2014}]{BCCWJ}
Maekawa, K., Yamazaki, M., Ogiso, T., Maruyama, T., Ogura, H., Kashino, W.,
  Koiso, H., Yamaguchi, M., Tanaka, M., \BBA\ Den, Y. \BBOP 2014\BBCP.
\newblock \BBOQ Balanced Corpus of Contemporary Written Japanese.\BBCQ\
\newblock In {\Bem Language Resources and Evaluation}, \lowercase{\BVOL}~48,
  \mbox{\BPGS\ 345--371}.

\bibitem[\protect\BCAY{Mikolov, Chen, Corrado, \BBA\ Dean}{Mikolov
  et~al.}{2013a}]{Mikolov1}
Mikolov, T., Chen, K., Corrado, G., \BBA\ Dean, J. \BBOP 2013a\BBCP.
\newblock \BBOQ Efficient Estimation of Word Representations in Vector
  Space.\BBCQ\
\newblock In {\Bem Proceedings of ICLR Workshop 2013}, \mbox{\BPGS\ 1--12}.

\bibitem[\protect\BCAY{Mikolov, Sutskever, Chen, Corrado, \BBA\ Dean}{Mikolov
  et~al.}{2013b}]{Mikolov2}
Mikolov, T., Sutskever, I., Chen, K., Corrado, G., \BBA\ Dean, J. \BBOP
  2013b\BBCP.
\newblock \BBOQ Distributed Representations of Words and Phrases and their
  Compositionality.\BBCQ\
\newblock In {\Bem Proceedings of NIPS 2013}, \mbox{\BPGS\ 1--9}.

\bibitem[\protect\BCAY{Mikolov, tau Yih, \BBA\ Zweig}{Mikolov
  et~al.}{2013c}]{Mikolov3}
Mikolov, T., tau Yih, W., \BBA\ Zweig, G. \BBOP 2013c\BBCP.
\newblock \BBOQ Linguistic Regularities in Continuous Space Word
  Representations.\BBCQ\
\newblock In {\Bem Proceedings of NAACL 2013}, \mbox{\BPGS\ 746--751}.

\bibitem[\protect\BCAY{{National Institute for Japanese Language and
  Linguistics}}{{National Institute for Japanese Language and
  Linguistics}}{1964}]{Bunruigoihyo}
{National Institute for Japanese Language and Linguistics} \BBOP 1964\BBCP.
\newblock {\Bem Word List by Semantic Principles}.
\newblock Shuuei Shuppan, In Japanese.

\bibitem[\protect\BCAY{Nishio, Iwabuchi, \BBA\ Mizutani}{Nishio
  et~al.}{1994}]{Iwanami}
Nishio, M., Iwabuchi, E., \BBA\ Mizutani, S. \BBOP 1994\BBCP.
\newblock {\Bem Iwanami Kokugo Jiten Dai Go Han}.
\newblock Iwanami Publisher, In Japanese.

\bibitem[\protect\BCAY{Shinnou, Onodera, Sasaki, \BBA\ Komiya}{Shinnou
  et~al.}{2015a}]{Shinnou1}
Shinnou, H., Onodera, Y., Sasaki, M., \BBA\ Komiya, K. \BBOP 2015a\BBCP.
\newblock \BBOQ Active Learning to Remove Source Instances for Domain
  Adaptation for Word Sense Disambiguation.\BBCQ\
\newblock In {\Bem Proceedings of PACLING-2015}, \mbox{\BPGS\ 224--231}.

\bibitem[\protect\BCAY{Shinnou, Sasaki, \BBA\ Komiya}{Shinnou
  et~al.}{2015b}]{Shinnou2}
Shinnou, H., Sasaki, M., \BBA\ Komiya, K. \BBOP 2015b\BBCP.
\newblock \BBOQ Learning under Covariate Shift for Domain Adaptation for Word
  Sense Disambiguation.\BBCQ\
\newblock In {\Bem Proceedings of PACLIC-29}, \mbox{\BPGS\ 215--223}.

\bibitem[\protect\BCAY{Sugawara, Takamura, Sasano, \BBA\ Okumura}{Sugawara
  et~al.}{2015}]{Sugawara}
Sugawara, H., Takamura, H., Sasano, R., \BBA\ Okumura, M. \BBOP 2015\BBCP.
\newblock \BBOQ Context Representation with Word Embeddings for WSD.\BBCQ\
\newblock In {\Bem Proceedings of PACLING 2015}.

\bibitem[\protect\BCAY{Taghipour \BBA\ Ng}{Taghipour \BBA\
  Ng}{2015}]{Taghipour}
Taghipour, K.\BBACOMMA\ \BBA\ Ng, H.~T. \BBOP 2015\BBCP.
\newblock \BBOQ Semi-Supervised Word Sense Disambiguation Using Word Embeddings
  in General and Specific Domains.\BBCQ\
\newblock In {\Bem Proceedings of NAACL-HLT2015}, \mbox{\BPGS\ 314--323}.

\bibitem[\protect\BCAY{Yamaki, Shinnou, Komiya, \BBA\ Sasaki}{Yamaki
  et~al.}{2016}]{Yamaki}
Yamaki, S., Shinnou, H., Komiya, K., \BBA\ Sasaki, M. \BBOP 2016\BBCP.
\newblock \BBOQ Supervised Word Sense Disambiguation with Sentences
  Similarities from Context Word Embeddings.\BBCQ\
\newblock In {\Bem Proceedings of PACLIC 2016}.
\newblock Y16-1010.

\end{thebibliography}


\begin{biography}
\bioauthor[:]{Kanako Komiya}{
received her Ph.D. degree in Tokyo University of Agriculture and Technology in 2009. After being a postdoctoral fellow in Tokyo Institute of Technology and an assistant professor in Tokyo University of Agriculture and Technology, she is currently a lecturer in Ibaraki University. She is interested in natural language processing. She is a member of IPSJ, JSAI, and ANLP.
}

\bioauthor[:]{Minoru Sasaki}{
received his Ph.D. degree in Tokushima University. After being a research assistant in Ibaraki University, he is currenly a lecturer in Ibaraki University. His research interests include information retrieval and natural language processing. He is a member of IPSJ and ANLP.
}

\bioauthor[:]{Hiroyuki Shinnou}{
received his Ph.D. degree in Tokyo Institute of Technology. After being a researcher in Fuji Xerox Co., Ltd. and Panasonic Corporation and a research assistant, a lecturer, and an associate professor in Ibaraki University, he is currently a professor in Ibaraki University. His research interests include Bayes statistics, machine learning and natural language processing. He is a member of IPSJ, JSAI, and ANLP.
}

\bioauthor[:]{Manabu Okumura}{
was born in 1962. He received B.E., M.E. and Dr. Eng. from Tokyo Institute of Technology in 1984, 1986, and 1989 respectively. 
He was an assistant at the Department of Computer Science, Tokyo Institute of Technology from 1989 to 1992, and an associate professor at the School of Information Science,
Japan Advanced Institute of Science and Technology from 1992 to 2000. 
He is currently a professor at Institute of Innovative Research, Tokyo Institute of Technology.
His current research interests include natural language processing, especially text summarization, computer assisted language learning, sentiment analysis and text data mining. 
}

\end{biography}


\biodate




\end{document}
