\documentclass[english]{jnlp_3.6}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
\usepackage{multirow}
%%%%\usepackage{amssymb}
%%%%\usepackage{subcaption}
\usepackage{xcolor}
%%%%\usepackage{multirow}
\usepackage{arydshln}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{30}
\Number{1}
\Month{March}
\Year{2023}
%%%%\Issuetitle{Special Issue: XXXXXXXX}


\received{2022}{7}{30}
\revised{2022}{11}{18}
\accepted{2022}{12}{20}

\setcounter{page}{156}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{A Simple and Effective Method for Injecting Word-level Information into Character-aware Neural Language Models}
\eauthor{Yukun Feng\affiref{Author_1} \and Hidetaka Kamigaito\affiref{Author_1} \and Hiroya Takamura\affiref{Author_2} \and Manabu Okumura\affiref{Author_1} }
\eabstract{
In this study, we propose a simple and effective method to inject word-level information into character-aware neural language models. Unlike previous approaches, which typically inject word-level information as input to a long short-term memory (LSTM) network, we inject such information into the softmax function. The resultant model can be considered a combination of a character-aware language model and a simple word-level language model. Our injection method can be used in conjunction with previous methods. The results of experiments on 14 typologically diverse languages are provided to empirically show that our injection method performed better than previous methods that inject word-level information at the input, including a gating mechanism, averaging, and concatenation of word vectors. Our method can also be used together with previous injection methods. Finally, we provide a comprehensive comparison with previous injection methods and analyze the effectiveness of word-level information in character-aware language models and the properties of our injection method in detail.
}
\ekeywords{Character-aware, Neural Language Model, Word Embeddings}

\headauthor{Feng et al.}
\headtitle{A Simple and Effective Usage of Word Clusters for CBOW Model}

\affilabel{Author_1}{}{Tokyo Institute of Technology}
\affilabel{Author_2}{}{National Institute of Advanced Industrial Science and Technology (AIST)}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
\label{sec:intro}

Language modeling (LM) is an important task in natural language processing, involving various
applications such as speech recognition \cite{Mikolov2010RecurrentNN}, machine translation
\cite{koehn2009statistical}, and summarization \cite{filippova2015sentence}.  Recently,
neural language models (NLMs) have shown considerable success and now reliably perform better than traditional
count-based methods
\cite{bengio2003neural,Mikolov2010RecurrentNN}. Standard NLMs typically maintain fixed vocabulary and map
each word as a continuous representation. 
The word representations obtained through NLMs are usually close to each other in the induced vector space if they are semantically similar.
However, standard NLMs involve two main problems. First, they cannot handle out-of-vocabulary words \textcolor{black}{\cite{D16-1209,gerz2018language,vania-lopez:2017:Long}}.
These words are typically replaced with special unknown symbols. 
Another problem is that these models are relatively ineffective in learning relationships between infrequent words.
\textcolor{black}{\cite{D16-1209,gerz2018language,vania-lopez:2017:Long}}.
For example, although words the  ``husbandman'' and ``salesman'' share the suffix ``man'', standard NLMs cannot capture such information in obtaining the relationship between the two words.
One solution is to use smaller units, such as bytes, characters,
or word pieces learned from tokens \cite{wu2016google,sennrich-etal-2016-neural}. However, this approach must process longer
sequences than word-based alternatives and may lead to increased modeling and computational challenges \cite{cherry-etal-2018-revisiting,Al-Rfou_Choe_Constant_Guo_Jones_2019}. \textcolor{black}{In addition, \citeA{wang19iclr} indicated that in some cases subwords perform notably worse than word-level models with subword-awareness in neural machine translation tasks.}
Another way to deal with these issues is to use the character-level information of each word to calculate the word representation, which is often referred to as character-aware NLMs
\cite{ling2015,kim2016character,vania-lopez:2017:Long,gerz2018language,feng-etal-2021-improving}. Character-aware NLMs are still word-based models because each word is processed sequentially.
In this study, we consider injecting word-level information into character-aware NLMs that utilize only character-level information.

\textcolor{black}{The present work is not the first to propose injecting word-level information into character-aware NLMs. However, previous methods injected word-level information at the input of LSTM layers through a gating mechanism, or by averaging or concatenating word vectors \cite{kang2011mandarin,D16-1209,kim2016character,E17-1040,keskomon2020thai}. 
Given that these approaches generally target the input vectors, word-level information is not explicitly considered at the output layer for predicting the next word; thus, these methods may not make full use of word-level information. To address this problem,} \textcolor{black}{we propose to inject the information of current and previous words at the output layer}. \textcolor{black}{Our method can be considered a combination of a modern character-aware NLM and a simple $n$-gram word-level language model. This approach is strongly inspired by the success of $n$-gram language models.} 

\textcolor{black}{In this study, we focus on injecting word-level information to character-aware language models, even though our injection method is not restricted to only character-aware language models. For example, we can also inject current and previous words into the output layer in a standard word-level language model. There are two main reasons for this focus. First, we expect that our injection method is more helpful in character-aware language models, as a standard character-aware language model should benefit more from word-level information than other models. For example, our injection method improves character-aware language models more than word-level language models on most datasets. \textcolor{black}{This experiment can be found in Sec.~\ref{sec:proposal_on_standard} in Appendix.} Second, we also focus on analyzing the effectiveness of word-level information in character-aware language models and apply our injection method to these models in various languages. To evaluate the effectiveness of word-level information, we analyzed the effects of rare words and the type of words that worked best when injected. We compared our injection method with previous methods that inject word-level information at the input in character-aware language models and analyzed the effects of combining our method with these methods. We also tested several variants of our injection method such as injecting character-level information or a combination of word- and character-level information into the output layer. These comparisons reveal additional properties of our injection method in character-aware language models.}

\textcolor{black}{In our experiments, we selected 14 datasets with typologically diverse languages.} \textcolor{black}{The results show that our injection method performed better than previous methods that inject word-level information at the input, and our method can be also used with these previous injection methods.} \textcolor{black}{Finally, we also applied our injection method over AWD-LSTM-LM \cite{merity2018regularizing}, and the results show that our approach was effective for models with different architectures.}
         


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}
\label{sec:related_work}

In recent years, many studies have attempted to improve character-aware NLMs. For example, \citeA{assylbekov-takhanov-2018-reusing}
proposed several methods for reusing the weights in character-aware NLMs. 
\citeA{gerz2018language} achieved an improved
result on 50 typologically diverse languages by injecting subword-level information into word vectors using a softmax function. 
For a thorough review of the relevant literature,
readers are referred to the work by \citeA{vania-lopez:2017:Long}, who performed a
systematic comparison across different models based on different subword units (characters, character trigrams, BPE, etc.).

One approach for the proposed method is injecting word-level information into character-aware neural models. 
In addition to language modeling, \citeA{santos2014learning} and \citeA{santos2015ner} first used a convolutional neural network (CNN) to encode characters and subsequently concatenated these encoded character- and word-level representations for part-of-speech tagging and named entity recognition.  
\citeA{luong_hybird_nmt2016} introduced a character-word neural machine translation model that only consults character-level representations for rare words encoded with a deep LSTM. 

Among studies on language models, \citeA{kang2011mandarin} used a simple character-word NLM designed for Chinese.
\citeA{D16-1209} introduced a gate mechanism between word embeddings and character embeddings
obtained from a bidirectional LSTM (BiLSTM) model for English. 
\citeA{E17-1040} and \citeA{keskomon2020thai} directly concatenated word and character embeddings without other subnetworks to encode characters for English, Dutch, and Thai. 
\textcolor{black}{These efforts were mainly targeted at injecting word-level information at the input side. Instead, \citeA{Takase_Suzuki_Nagata_2019} adopted a method similar to our method to inject a mixture of character- and word-level information into the output layer. However, there are two main differences. First, we only inject word-level information, whereas their method also injected character-level information. Thus, our proposal and their methods may be effective in different cases, which are analyzed in Sec.~\ref{sec:injecting_different_using_takase}. Second, compared to our proposal, one disadvantage of the injection method adopted by \citeA{Takase_Suzuki_Nagata_2019} is that it involves the computations of encoding each word of the whole output vocabulary at each training step, which may slow down the training process. We provide further details on experiments in which we implemented their method with our character encoder in Sec.~\ref{sec:injecting_different_using_takase}.}


\textcolor{black}{
 Most existing methods \cite{kang2011mandarin,D16-1209,kim2016character,E17-1040} inject word-level information at the input of character-aware language models, while our proposed method injects this into the output. As we show in subsequent sections, our method can be used in conjunction with most existing injection methods. However, when our proposed approach and existing methods are used together, word-level information is used at both the input and output in character-aware language models. In this situation, our injection method appears similar to residual connections or the method of \citeA{takase-etal-2017-input}, who proposed the use of information from an input word to modify the output of word-level language models. In this situation, our injection method can be interpreted as adding a residual connection from the input word to the output in character-aware language models. However, three differences should be noted when comparing residual connection-like methods in this situation.  First, our method may include previous words (when $n>1$). We show that this can facilitate improvisation when including previous words in our analysis. Second, our proposed method is not necessarily used together with such existing methods. When our injection method is used alone in a standard character-aware language model, it performs better than existing methods that inject word-level information at the input, as shown in Sec.~\ref{sec:compare_baseline_models}. However, we did not use residual connection-like methods at all. This is possible when the current input words are rare, and discarding them can improve performance and reduce the parameters, as shown in Sec.~\ref{sec:effet_infreq_word}. In these cases, we can still inject word embeddings of previous words that exclude rare words or pretrained $n$-gram/span embedding (e.g., ngram2vec\footnote{\url{https://github.com/zhezhaoa/ngram2vec}}) into the output. This may occur particularly in languages with a high type-token ratio that contain many rare words.
Third, when our proposal and existing injection methods are used together, and we only inject current words without including previous words, comparisons with residual connection-like methods are involved. However, the application of residual connection-like methods in character-aware language models differs notably from our proposed approach. \textcolor{black}{This is the case because the input} is a combination of character- and word-level information. It is natural to apply residual connection-like methods to the combined information because we might want to notify the output layer of both word- and character-level information. However, a straightforward method to inject character-level information into the output layer degrades the performance, as analyzed in Sec.~\ref{sec:injecting_different_info_using_proposal}. Thus, the natural application of residual connection-like methods in character-aware language models performs worse than our proposed method. In this case, when existing methods are used together and only the current word is injected, our proposed approach can be considered as an adaption of residual connection-like methods to a character-aware language model.
}

Although several studies have considered the use of both character-level and word-level
information, these methods feed the two types of information only to an LSTM model, whereas our model injects word-level information into the softmax function. 
Previous work on this topic has been generally conducted in a limited number of languages, but a comprehensive comparison of different injection methods has rarely been considered. 
We compared our method with the existing methods mentioned in this section on 14 typologically diverse languages. \textcolor{black}{Although the models in this work are smaller and less powerful than current pretrained language models based on large-scale corpora, e.g., GPT \cite{radford2019language}, they are still used in cases where computational resources are limited.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Model Description}

For language modeling, we used an LSTM network.
\cite{hochreiter1997long}. We incorporate
word-level information using a neural network %architecture 
as shown in Figure~\ref{fig:model}. The details of this model are described in the following subsections.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[t]
\begin{center}
\includegraphics{30-1ia6f1.pdf}
\end{center}
\hangcaption{Our character-aware LSTM language model with injection of word-level information with an example word ``cats''. Symbols \^{} and \$ respectively represent the beginning and the end of a word.}
\label{fig:model}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{\textcolor{black}{Existing Methods}}
\label{sec:input_word}

\textcolor{black}{We describe the existing methods for combining word- and character-level information here.} We use BiLSTM to encode character $n$-grams to obtain a character-level representation.
We set $n$ to 3 for all languages except Japanese and Chinese, for which we set $n$ to 1. 
We used these settings because BiLSTM over character 3-grams obtained the best results on most LM datasets in the study
of \citeA{vania-lopez:2017:Long}, but Japanese and Chinese are more ideographic than the others, and
we expected that a smaller $n$ would be more effective.

\textcolor{black}{We denote the hidden state of LSTM for the $t$-th word $w_{t}$ as $\mathbf{h}_{t} \in \mathbb{R}^{d}$, where $d$ is the embedding size.}
Given a word $w_{t}$,
we denote its embedding from a lookup table $ \mathbf{W}_{in} \in \mathbb{R}^{d \times |V|}$ as $
\mathbf{w_{t}} \in \mathbb{R}^{d}$, where $|V|$ is the vocabulary size.
We compute the character-level representation of $w_{t}$ as follows.
\begin{equation}
\mathbf{c}_{t}=\mathbf{W}_{f} \mathbf{h}_{l}^{f w}+\mathbf{W}_{b} \mathbf{h}_{0}^{b w}+\mathbf{b},
\end{equation}
where $\mathbf{h}_{l}^{f w}$ and $\mathbf{h}_{0}^{b w}$ $\in \mathbb{R}^{d}$ respectively denote the last states of the forward and
backward LSTMs. $\mathbf{W}_{f}$, $\mathbf{W}_{b}$ $\in \mathbb{R}^{d \times d}$ and $\mathbf{b}$ $\in \mathbb{R}^{d}$ are trainable
parameters.
We define the following \textcolor{black}{existing} methods to obtain the combination $\mathbf{w}_{t}'$ from $\mathbf{w}_{t}$ and $\mathbf{c}_{t}$.
\begin{itemize}
\item[-] \textbf{gate}: we use the same gating mechanism as \citeA{D16-1209}, which is
    described later to combine $\mathbf{w}_{t}$ and $\mathbf{c}_{t}$.
\item[-] \textbf{avg, add, cat}:  we obtain $\mathbf{w}_{t}'$ through averaging, addition,
    and concatenation of $\mathbf{w}_{t}$ and $\mathbf{c}_{t}$, respectively.
\end{itemize}
In the gating mechanism, we compute $\mathbf{w}_{t}'$ as follows. 
\begin{align}
%%%%\label{eq:gate}
    g_{w_{t}}^{in} & = \sigma\left(\mathbf{v}_{g}^{\top} \mathbf{w}_{t}+b_{g}\right),\label{eq:gate}\\
    \mathbf{w}_{t}' & = (1 - g_{w_{t}}^{in})\mathbf{w}_{t} + g_{w_{t}}^{in}\mathbf{c}_{t},
    \label{eq:new_wt}
\end{align}
where $\mathbf{v}_{g} \in \mathbb{R}^{d}$ and $b_{g} \in \mathbb{R}$ are trainable
parameters and $\sigma(\cdot)$ is a sigmoid function.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{\textcolor{black}{Our Proposal}}
\label{sec:output}

Our proposed method combines $\mathbf{h}_{t}$ with $\mathbf{w_{t}}$  to better inform the softmax
function of word-level information. The combination $\mathbf{h}_{t}'$ is computed as follows.
\pagebreak
\begin{equation}
    \mathbf{h}_{t}' = \mathbf{h}_{t} + g_{w_{t}}^{out}\mathbf{w}_{t},
    \label{eq:new_ht}
\end{equation}
where $g_{w_{t}}^{out}$ is a gate value. In our experiments, we set up two types of gates. One
was a fixed value $g_{w_{t}}^{out}=0.5$. The second was similar to the definition given in Eq.~(\ref{eq:gate}),
which adaptively outputs a gate value depending on $w_{t}$.
\begin{equation}
     g_{w_{t}}^{out}= \sigma\left(\mathbf{v}_{k}^{\top}\mathbf{w}_{t}+b_{k}\right),
\end{equation}
where $\mathbf{v}_{k}^{\top} \in \mathbb{R}^{d}$ and $b_{k} \in \mathbb{R}$ are trainable parameters.
In Eq.~(\ref{eq:new_ht}), the gate is used only
on word-level information to determine the amount of information $\mathbf{w}_{t}$ that should be obtained.\footnote{We tested the above other methods, such as avg, add and cat, for combining $\mathbf{h}_{t}$ and $\mathbf{w}_{t}$, in place of the gate, 
and observed that they did not perform well.}

In Eq.~(\ref{eq:new_ht}), if we remove the term $\mathbf{h}_{t}$, then the resultant model is a simple word-level language model $P(w_{t+1}|w_{t})$. Based on this
observation, we can simply extend our method to contain word-level information for previous words
\textcolor{black}{
\begin{equation}
    \mathbf{h}_{t}^{word} = \sum^{n}_{i=1}  g_{t+1-i} \mathbf{Q}_{t+1-i}\mathbf{w}_{t+1-i},
\end{equation}
}
where $\mathbf{Q}_{t+1-i} \in \mathbb{R}^{d \times d}$ is a linear transformation applied to $\mathbf{w}_{t+1-i}$, and  $n$ is the number of current and previous words used to calculate $\mathbf{h}_{t}^{word}$ and \textcolor{black}{each previous word has its gate, which is defined in Eq.~\ref{eq:gate}. We set $g_{t}$ (the gate for the current word) to 1, so that the current word is not weighted. We expect these gates to learn to weigh the importance of each previous word and use the weighted sum of the previous words for the following computation}.
The hidden state $\mathbf{h}_{t}'$ now can be calculated as follows.
\begin{equation}
    \mathbf{h}_{t}' = \mathbf{h}_{t} + g_{w_{t}}^{out}\mathbf{h}_{t}^{word}.
    \label{eq:ext_ht}
\end{equation}
The final language modeling task computes the probability of a given sentence $w_{1},\dotso,w_{T}$.
\begin{equation}
   P(w_{1},\dotso,w_{T}) = \prod_{t=1}^{T} P(w_{t}|w_{1},\dotso,w_{t-1}).
\end{equation}
We use a softmax function based on $\mathbf{h}_{t}'$ to generate a probability distribution over the vocabulary as given below.
\begin{equation}
    P(w_{t+1}|w_{1},\ldots,w_{t}) = \mbox{softmax}( \textbf{W}_{out}^{T}{\textbf{h}_{t}}'),
\label{eq:softmax}
\end{equation}
where $\mathbf{W}_{out} \in \mathbb{R}^{d \times |V|}$ denotes the output word embeddings.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{06table01.tex}
\hangcaption{Hyperparameters of our model. We use $d$ for the sizes of the character/word embeddings and for the number of hidden units of LSTM and BiLSTM.}
\label{tab:hyper}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Model Variants}
\label{sec:models}

\textcolor{black}{We largely followed the model settings from \citeA{gerz2018language} for all our models listed below, and these settings are shown in Table~\ref{tab:hyper}}.
The learning rate decreased when no improvement was observed in
the validation dataset. Several baseline models used for comparison and our models are listed as follows.
\begin{itemize}
\item[-] \textbf{Char-BiLSTM-LSTM}: We used a BiLSTM architecture to encode character trigrams without injecting word-level information. \textcolor{black}{We chose this model as our character-aware NLM baseline according to the experimental results of \citeA{vania-lopez:2017:Long}, in which the authors performed a detailed comparison over different character encoders and found that this model performed best on most datasets, as noted above in Sec.~\ref{sec:input_word}.}

\item[-] \textbf{Word-LSTM}: Standard word-level LSTM model. \textcolor{black}{This standard baseline was chosen to verify whether the above Char-BiLSTM-LSTM performed better, especially on datasets with many infrequent words.}

\item[-] \textbf{Char-BiLSTM-gate/avg/add/cat-Word-LSTM}:  \textcolor{black}{We inject word-level information at the input of Char-BiLSTM-LSTM through previous injection methods gate/avg/add/cat, mentioned in Sec.~\ref{sec:input_word}. These models were used to evaluate the effectiveness of previous injection methods and also used for comparison with our proposed method.}


\item[-] \textbf{Char-BiLSTM-LSTM-Word}: \textcolor{black}{We applied our proposed method to Char-BiLSTM-LSTM by injecting word-level information into the softmax function. This model was used to verify the effectiveness of our proposal.}

\item[-] \textbf{Char-BiLSTM-gate/avg/add/cat-Word-LSTM-Word}: \textcolor{black}{Because our proposed \linebreak method and the abovementioned previous methods target different positions when injecting word-level information, we applied them and our proposal at the same time over Char-BiLSTM-LSTM. This model was used to evaluate the extent to which previous injection methods make full use of word-level information and whether our proposal improved previous injection methods.}
\end{itemize}
For both Char-BiLSTM-LSTM-Word and Char-BiLSTM-gate/avg/add/cat-Word-LSTM-Word, we used
$g=0.5/\textrm{adaptive}$ and \textcolor{black}{$n=\{1,2,3\}$} to represent specific injection methods. For example, 
Char-BiLSTM-LSTM-Word ($g=0.5,n=2$) indicates that
we used a fixed gate value on the word-level information in Eq.~(\ref{eq:new_ht}), and we injected the information of the current and preceding words into the softmax function.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments on 14 Languages}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Datasets}

Common language modeling datasets for evaluating character-aware NLMs were obtained from the
work by \citeA{botha2014compositional}. Although these datasets contain languages with rich morphology, 
They included only five languages. Perhaps the most large-scale language modeling
datasets were provided in the work of \citeA{gerz2018language}, who released 50 language modeling
datasets covering typologically diverse languages. The difference between the newly
released datasets, and the previous common datasets is that unseen words are retained in the testing set. Thus, on these datasets, 
our method was tested using a real LM setup.  
The languages from \citeA{gerz2018language} were selected to represent a wide spectrum
with different morphological
systems and many low-frequency or unseen words.  Therefore, these datasets were considered preferable to evaluate the performance of character-aware NLMs.\footnote{To test our models against previous methods, we also included experiments on common datasets, as described below.}

To simplify the experiments without loss of generality,
We only chose datasets of 14 languages from these datasets and tried to cover different language
typologies as well as different type/token ratios (TTRs).
The statistics for the chosen datasets are listed in Table~\ref{tab:data_sti}. 
We used all the words observed in the training data and one special unknown
token for out-of-vocabulary words as an output vocabulary to create the
setting the same as \citeA{gerz2018language}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{06table02.tex}
\caption{The statistics of our language modeling datasets. TTR represents type/token ratio.}
\label{tab:data_sti}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Main Experimental Results}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{\textcolor{black}{Comparison of Standard NLMs and Character-aware NLMs}}
\label{sec:compare_baseline_models}

\textcolor{black}{As aforementioned, character-aware NLMs are more powerful than standard NLMs, particularly on datasets with many infrequent words. In this subsection, we describe our experimental verification of whether character-aware NLMs perform better than standard NLMs.}
The results of Word-LSTM and Char-BiLSTM-LSTM\footnote{\textcolor{black}{Our implemented standard NLMs and character-aware NLMs were better than those from \citeA{gerz2018language} on all datasets. We do not include their results in Table \ref{tab_baseline_inject}.}} \textcolor{black}{The dashed lines in the table show the relevant models.}
\textcolor{black}{As shown in the table, Char-BiLSTM-LSTM performed better than Word-LSTM on all the datasets.
This matches our hypothesis because there were many infrequent and unseen words in our datasets, making it difficult for Word-LSTM to learn effectively. However, character-aware models can encode characters from unseen and infrequent words, making it possible to process these words well.}
\textcolor{black}{The results also show that Char-BiLSTM-LSTM tended to achieve better results than Word-LSTM on datasets with a higher ratio of infrequent words, which again verifies the effectiveness of character-aware NLMs for learning infrequent words. These experimental results show that character-aware NLMs are worth studying to improve their performance further. In the next subsection, we describe our investigation of these models by injecting word-level information.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[b]
\input{06table03.tex}
\hangcaption{Perplexity of several baseline models and our proposed models on 14 language modeling datasets. The best results among all models are shown in bold.}
\label{tab_baseline_inject}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{\textcolor{black}{Comparison with Previous Injection Methods That Target at Input Side}}
\label{sec:compare_with_prev_injection}

\textcolor{black}{Although character-aware NLMs were shown to perform better than standard NLMs above, injecting word-level information into character-aware NLMs is a common approach because it provides information from different aspects. This subsection has three purposes, including: 1) we evaluated the effectiveness of our approach for injecting word-level information at the input side using previous methods, 2) we also sought to evaluate the effectiveness of our proposal that targets the output layer, and 3) we compared the effectiveness of our proposed method with previous methods that target the input side. In particular, we implemented four previous methods, including Char-BiLSTM-gate/avg/add/cat-Word-LSTM introduced in Sec.~\ref{sec:models}.}
The results of these methods and our proposal on 14 language modeling datasets are presented in Table~\ref{tab_baseline_inject}.


\textcolor{black}{Comparing Char-BiLSTM-gate/avg/cat-Word-LSTM and Char-BiLSTM-LSTM, we found that injecting word-level information at the input side with gate/avg/cat did not help Char-BiLSTM-LSTM much on most datasets, indicating these methods were not effective.}
We also found that some previous studies had similar results.  
For example, \citeA{kim2016character} reported that some basic methods (e.g., concatenation, averaging, and adaptive weighting schemes) for injecting word-level information degrade the performance of their character-aware NLMs. 
\citeA{D16-1209} found that a concatenation method for injecting word-level information also degraded their model.
\textcolor{black}{However, the addition method (Char-BiLSTM-add-Word-LSTM) helped Char-BiLSTM-LSTM achieve improved results on 13 out of 14 datasets, while this simple method was less mentioned in the previous work. This indicates that injecting word-level information at the input side can be helpful when using an appropriate method. We also experimented with another baseline that backs off characters only if a word is infrequent. However, this baseline underperformed Char-BiLSTM-add-Word-LSTM, and we provide the experimental details in Sec.~\ref{sec:back_off} in the Appendix.}

Our proposed Char-BiLSTM-LSTM-Word ($g=0.5,n=1$) achieved better results than 
Char-BiLSTM-LSTM on all datasets, suggesting that our approach of targeting the output layer was effective. Char-BiLSTM-LSTM-Word ($g=0.5,n=1$) performed better than Char-BiLSTM-LSTM-Word ($g=\textrm{adaptive},n=1$) on most datasets, indicating that a simple fixed gate value in our proposal may be sufficiently effective. 
Compared with previous injection methods that target the input side, our proposed method achieved the best results on most datasets (bold scores in Table~\ref{tab_baseline_inject}). 
This suggests that our injection method, aimed at a different position from the input of LSTM, \textcolor{black}{makes better use of word-level information and is thus more effective than previous methods. Further details are provided in Sec.~\ref{sec:analysis}.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{\textcolor{black}{Combination of Our Proposal and Previous Methods Targeting at Input Side}}
\label{sec:combination_with_prev_injection}

\textcolor{black}{We showed both our proposal and previous injection method that targets the input side are effective in the previous subsection. However, our proposed method and most previous methods target different positions when injecting word-level information. One natural idea is to verify whether the model can make better use of word-level information when our proposal and previous injection methods are used together. In this subsection, we consider this possibility. We simultaneously injected word-level information at the input and output sides.}

\textcolor{black}{The combination of our proposal Char-BiLSTM-LSTM-Word ($g=0.5,n=1$) and the previous injection method Char-BiLSTM-add-Word-LSTM is denoted as Char-BiLSTM-add-Word-LSTM-Word ($g=0.5,n=1$). As shown in Table~\ref{tab:comp_inject}, this combination can help achieve further improvements on all the datasets.}
The results indicate that previous injection methods did not make full use of word-level information, whereas our method
which injects word-level information into a different position, specifically, softmax, can help the previous models make better use of
word-level information.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{06table04.tex}
\hangcaption{Perplexity of the combination of our injection method with the previous methods on 14 language modeling datasets.}
\label{tab:comp_inject}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{\textcolor{black}{Including Previous Words for Our Proposal}}
\label{sec:include_prev}

As mentioned in \textcolor{black}{Sec.~\ref{sec:output}}, \textcolor{black}{our proposed method can include previous words motivated by the success of the $n$-gram language model. In this subsection, we describe the results of an experiment conducted to verify the effectiveness of our proposed method when including the injection of previous words. We showed above that when our proposal and previous injection methods are used together, the model achieved better performance. Thus, we tested the effectiveness of including previous words based on the combination of our proposal and the previous injection method.} 
The number of words used in the injection method is denoted as $n$. 
In our experiments, we only set $n$ to 1, 2, and 3, as we observed no obvious improvements when using a larger $n$. 
\textcolor{black}{As shown in Table~\ref{tab:inc_prev_word}, the performance was further improved when including previous words for our proposal on most datasets. This indicates that word-level information can be better utilized by including previous words.}
Because our current method of including word-level information for previous words is simple, a more advanced method can be further exploited in future work.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[t]
\input{06table05.tex}
\hangcaption{\textcolor{black}{Perplexity of our Char-BiLSTM-add-Word-LSTM-Word including word-level information for previous words on 14 language modeling datasets.} }
\label{tab:inc_prev_word}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Comparison with a Previous Method Targeting the Output}
\label{sec:injecting_different_using_takase}

\textcolor{black}{
\citeA{Takase_Suzuki_Nagata_2019} adopted a method to notify the softmax layer 
\pagebreak
of both character-level information, computed from their novel character encoder and word-level information. Their injection method can be regarded as targeting the output layer, which is similar to our method. Thus, this subsection compares our proposed method with Takase et al.'s  method in detail. Because we have shown that our model can achieve better performance when previous words are included, we used this setting when compared with their injection method. In particular, we expected to find two main differences. First, our proposal only aims to inject word-level information, while their method aims to inject character-level information into the output layer. Thus, our proposal and their method may be effective in different cases (e.g., different languages with high and low ratios of infrequent words). Second, the training speed may reduce for their injection method because it involves more computations during injection than ours.}

\textcolor{black}{
Regarding the details of Takase et al.'s  method, they first applied their character encoder to each word of the output vocabulary of the softmax layer to obtain the character-level representations, and then aggregated the embeddings from the character encoder with standard word embeddings from the input vocabulary. During training, these embeddings from their character encoder must be re-computed at each step as the parameters of the character encoder are updated at each step. \textcolor{black}{Thus, the training speed may be slowed down}. In this section, we implemented this injection method in our baseline and compared it with our approach. Because \citeA{Takase_Suzuki_Nagata_2019} also aggregated the embeddings computed from their character encoder and word embeddings at the input, we chose to implement their injection method in Char-BiLSTM-add-Word-LSTM to follow a similar setting. We denote the model with its injection method as Char-BiLSTM-add-Word-LSTM-ComputeOut. The results are presented in Table \ref{tab:takase_inject_compare}.
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[b]
\input{06table06.tex}
\hangcaption{\textcolor{black}{Perplexity and training hours of models using our proposal and the method from Takase et al. (2019). All models are trained for 40 epochs.}}
\label{tab:takase_inject_compare}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\textcolor{black}{
\textcolor{black}{The result in the table matches our first expected difference. As we can see,} the trend shows that the injection method of \citeA{Takase_Suzuki_Nagata_2019} is more effective than our injection method on datasets with more infrequent words but worse than our injection method on datasets with less infrequent words. This may be because their method injected character-level information for each embedding in the output vocabulary of softmax layer, and thus, during prediction, it is effective for languages with many infrequent words. \textcolor{black}{The analysis for injecting character-level information with our proposal is mentioned in Sec.~\ref{sec:injecting_different_info_using_proposal}.}
}

\textcolor{black}{In the second expected difference, we mentioned the injection method from \citeA{Takase_Suzuki_Nagata_2019} may slow down the training progress as it involves more computations during injection}. To demonstrate this, we include the training time in Table \ref{tab:takase_inject_compare}. \textcolor{black}{Note that the training time depends on the type of the character encoder and its implementation. Here, we used our BiLSTM as the character encoder, whereas \citeA{Takase_Suzuki_Nagata_2019} used the character encoder based on self-attention, which is faster.} It may be observed that applying their method \textcolor{black}{using our character encoder} took 15 to 36 times longer, depending on the datasets, than applying our injection method. In addition, the training time was further increased with the size of the vocabulary. Unlike their injection method, which changes each word embedding in the output layer at each step, our proposed method only changes the hidden embeddings of the last LSTM layer. These hidden embeddings are used to compute the probability distribution with word embeddings in the output layer.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Analysis}
\label{sec:analysis}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{\textcolor{black}{Effects of Injecting Character-level Information Using Our Proposal}}
\label{sec:injecting_different_info_using_proposal}

\textcolor{black}{Because our proposal is not restricted to only injecting word-level information, we also investigated the effect of injecting character-level information with our proposal.} The following baseline configurations were used for our analysis.
\begin{itemize}
    \item[\textbullet] Char-BiLSTM-LSTM-Char: In Char-BiLSTM-LSTM, we injected the character-level embeddings of a word using our proposal into the softmax function where we set $n=1$.
    \item[\textbullet] Char-BiLSTM-add-Word-LSTM-WordChar: In Char-BiLSTM-add-Word-LSTM, we inject combined embeddings from characters and words into the softmax function.
\end{itemize}
As shown in Table \ref{tab:other_injections}, Char-BiLSTM-LSTM-Char degraded its baseline on the eight datasets. This result suggests that injecting the embeddings computed from the character encoder into softmax does not benefit substantially. When comparing Char-BiLSTM-add-Word-LSTM-WordChar and Char-BiLSTM-add-Word-LSTM-Word, we obtained similar results: injecting only word embeddings achieved better results than injecting both. The results indicate that the proposed method is more suitable for word-level information.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[b]
\input{06table07.tex}
\hangcaption{\textcolor{black}{Perplexity of different baseline configurations on 14 language modeling datasets. When we inject embeddings to softmax function, we set $n=1$ and $g=0.5$ for all models.}}
\label{tab:other_injections}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\textcolor{black}{
To further investigate why this occurs, we used the proposed model as a combination of a modern LSTM-based language model and a simple $n$-gram language model that directly uses input to predict the next word. We also note that our proposal was motivated by this view. Specifically, we disabled the character-aware NLM component in Char-BiLSTM-LSTM-Word and only used the left simple language model. Because we set $n$ to 1, we refer to it as Bigram-Word. Similarly, Bigram-Char corresponds to the remaining simple language model in Char-BiLSTM-LSTM-Char. The result is shown in Table \ref{tab:simple_lm_res}. As can be seen, Bigram-Word obtained better results for most datasets. Notably, even on several datasets with a high ratio of infrequent words, Bigram-Word performed better than Bigram-Char. This may be contrasted with the comparison between Word-LSTM and Char-BiLSTM-LSTM in Table \ref{tab_baseline_inject} where Char-BiLSTM-LSTM achieved better results for all datasets. One possible reason is that the character encoder may perform better when more parameters and nonlinearity are involved.
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[t]
\input{06table08.tex}
\caption{\textcolor{black}{Perplexity of two simple language models on 14 language modeling datasets.}}
\label{tab:simple_lm_res}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Effects of Rare Words}
\label{sec:effet_infreq_word}

To check whether rare words help our character-aware NLMs, we conducted several experiments by discarding some rare words based on their word frequency. 
Two independent vocabularies were maintained. The first is the input vocabulary, which is used to inject word-level information.
We obtained the word embeddings in our previous injection methods using the lookup table
$\mathbf{W}_{in}$ as described in Sec.~\ref{sec:input_word}. The other was the output vocabulary used
for word prediction as described in Sec.~\ref{sec:output}. 
When we discarded rare words, we narrowed down the input vocabulary and did not change the output vocabulary. 
Thus, the perplexity scores were still comparable to those in the above experiments. 
For example, when our model processes the sentence ``the salesman brought some samples'' in the training phase, where ``salesman'' is
a rare word in the training data, our model can still try to predict the word ``salesman'' given
the previous word ``the'' because ``salesman'' is in our output vocabulary. 
When inputting the word ``salesman'' to predict the word ``brought,'' we do not inject word-level information for the word ``salesman.'' 
We only used its character-level representation obtained through our BiLSTM over characters to perform the language modeling task.

We denoted the frequency threshold by $\theta$ and set its value to 5, 15, and 25. 
If the frequency of a word in the training data is less than or equal to $\theta$, we discarded it. 
We refer to the model that discards infrequent words as
Char-BiLSTM-add-Word-LSTM-Word ($g=0.5,n=1$, $\theta=5/15/25$). 
The results are presented in Table~\ref{tab:freq_ppl}. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[b]
\input{06table09.tex}
\hangcaption{Perplexity of Char-BiLSTM-add-Word-LSTM-Word ($g=0.5,n=1$) with different frequency thresholds on 14 language modeling datasets.}
\label{tab:freq_ppl}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

When discarding words with a frequency less than or equal to 15, the
model obtains better results on only two out of the 14 datasets than 
Char-BiLSTM-add-Word-LSTM-Word ($g=0.5,n=1$). 
This indicates that some infrequent words were helpful.
When we further increased the frequency threshold to 25, the performance of the model was
dropped compared with Char-BiLSTM-add-Word-LSTM-Word ($g=0.5,n=1,\theta=15$), as more frequent words were discarded. 
However, we found that a relatively small frequency threshold $\theta=5$ works effectively. Char-BiLSTM-add-Word-LSTM-Word ($g=0.5,n=1,\theta=5$) achieved better results than
Char-BiLSTM-add-Word-LSTM-Word ($g=0.5,n=1$) on seven of the 14 datasets. 
It appears that discarding rare words with $\theta=5$ is useful for datasets with \textcolor{black}{ a high ratio of infrequent words.}
Because many words in natural languages are rare, as described in Zipf's law, we can reduce the size of the input vocabulary significantly, even with a small $\theta$. The sizes of the full input vocabulary and reduced vocabulary with different frequency threshold values are shown in Table \ref{tab:input_vocab_size}. 
As can be observed, when $\theta$ was set to five, our model achieved better results with fewer parameters.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[t]
\input{06table10.tex}
\hangcaption{The size of input vocabulary seen in the training data on 14 datasets with different frequency threshold.}
\label{tab:input_vocab_size}
\vspace{-1\Cvs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{\textcolor{black}{Analyzing Which Word-level Information Is Most Useful}}
\label{sec:targeted_perp}

\textcolor{black}{In this subsection, we analyze which word-level information was most useful when injecting them into character-aware NLMs. We aimed to analyze this by measuring the improvement when injecting words with different frequency ranges.} Specifically,
when the current input word was within a specified frequency range, we calculated the perplexity of the next word. \textcolor{black}{We then measure the averaged improvement across 14 languages, achieved by Char-BiLSTM-add-Word-LSTM-Word based on Char-BiLSTM-LSTM given input words of different frequency ranges.} A similar analysis of language models is available in \citeA{vania-lopez:2017:Long}. 
The results in Figure \ref{fig:which_word} show that the performance was best when the injected words are medium-frequency words. \textcolor{black}{The improvement starts to decrease as the input words became more frequent.} This may be because the character encoder can learn to represent high-frequency words well, as they appear many times during training, and injecting these word embeddings does not provide much new information for the model. This improvement also starts to decrease when low-frequency words are injected. \textcolor{black}{This is probably because the embeddings of infrequent words are not reliable, as they are updated not many times during training.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[t]
\vspace{-0.5\Cvs}
\begin{center}
\includegraphics{30-1ia6f2.pdf}
\end{center}
\hangcaption{The averaged improvement of Char-BiLSTM-add-Word-LSTM-Word compared with Char-BiLSTM-LSTM over 14 languages with input words in different frequency range.}
\label{fig:which_word}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{\textcolor{black}{Effects of Our Proposal on Different Architecture}}

\textcolor{black}{In this subsection, we tested the effectiveness of our proposal on a different model architecture. In particular, we chose the widely used AWD-LSTM-LM\footnote{\url{https://github.com/salesforce/awd-lstm-lm}}} \cite{merity2018regularizing}. We replaced the word embedding layer of AWD-LSTM-LM with our BiLSTM character encoder and refer to it as Char-BiLSTM-AWD-LSTM. When our injection method was applied to Char-BiLSTM-AWD-LSTM, we refer to this model as Char-BiLSTM-AWD-LSTM-Word ($g=0.5$, $n=2$). The original AWD-LSTM-LM code includes a training phase and fine-tuning phase, and each phase included hundreds of training epochs (e.g., 750 epochs on WikiText-2 in one phase). Owing to time constraints, we set the number of training epochs on all datasets to 200, without a further fine-tuning phase. We refer to the original AWD-LSTM-LM which is a word-level Word-AWD-LSTM. For the other parameters, we followed the settings of the source code. The results are presented in Table \ref{tab:awd_res}.  As can be seen, Char-BiLSTM-AWD-LSTM achieves better results than Word-AWD-LSTM on most datasets with a high ratio of infrequent words. This was similar to the previous results with the standard LSTM. When our method was applied, Char-BiLSTM-AWD-LSTM-Word obtained better results on most datasets. Similar to before, Char-BiLSTM-AWD-LSTM-Word improves Char-BiLSTM-AWD-LSTM more on datasets with a low ratio of infrequent words than those with a high ratio of infrequent words. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 11
\begin{table}[b]
\input{06table11.tex}
\caption{\textcolor{black}{Perplexity based on AWD-LSTM-LM on 14 language modeling datasets.}}
\label{tab:awd_res}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[b]
\input{06table12.tex}
\caption{ Data statistics of our 6 language modeling datasets.}
\label{tab:data_sti_6}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments on 6 Common Datasets}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{\textcolor{black}{Datasets}}

In addition to the above datasets, we set up six common language modeling datasets: English Penn
Treebank (PTB) \cite{marcus1993building} and five non-English datasets with rich morphology from the
2013 ACL Workshop on Machine Translation,\footnote{\url{http://www.statmt.org/wmt13/translation-task.html}} which is commonly used for evaluating
character-aware NLMs \cite{botha2014compositional,kim2016character,bojanowski2017enriching,assylbekov-takhanov-2018-reusing}. Because some previous studies have tested their model on PTB,  we also included PTB in
our experiment. We used a preprocessed small version of the non-English dataset by
\citeA{botha2014compositional}  and followed the same split as the previous work.  The data statistics are
provided in Table \ref{tab:data_sti_6}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 13
\begin{table}[b]
\input{06table13.tex}
\caption{Perplexity of our models and previous work on 6 language modeling datasets.}
\label{tab:res_6_data}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{\textcolor{black}{Results}}

The results of our proposed models and those of previous studies are listed in Table \ref{tab:res_6_data}.  We
used Char-BiLSTM-LSTM and Char-BiLSTM-add-Word-LSTM as the baseline models. For our models, we
set the frequency threshold $\theta$ to 5 and $n$ to 2, as these settings help improve our
character-aware NLMs as discussed in Sec.~\ref{sec:effet_infreq_word} and Sec.
\ref{sec:include_prev}.
The language models used in previous work were improved in different aspects, and most of them were
based on the standard LSTM, as in our study.
\citeA{botha2014compositional} used the morphological logbilinear (MLBL) model, which takes into
account morpheme information.  \citeA{kim2016character} used CNN as their character encoder and
trained an LSTM language model, where the input representation of a word was the sum of the
Morpheme embeddings of words.
\citeA{bojanowski2017enriching} trained the word embeddings through skip-gram models with subword-level
information and used these word embeddings to initialize the lookup table of word embeddings of a
word-level language models.  \citeA{assylbekov-takhanov-2018-reusing} focused on reusing embeddings
and weights in a character-aware language model.  The input of the model is also
the sum of the morpheme embeddings of words. As shown in the table, Char-BiLSTM-LSTM underperforms
Previous work on PTB.  One reason may be that we did not tune the hyperparameters of the
models on PTB. The hyperparameters were maintained in all the experiments on the 20 datasets.  As we can
see, Char-BiLSTM-LSTM achieved better results than most previous studies on non-English datasets. Our
models achieved the best results for non-English datasets.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

\textcolor{black}{Character-aware neural language models are powerful tools for languages that contain many infrequent words. Previous studies have attempted to further enhance character-aware neural language models with word-level information, and they usually inject word-level information into the input. In this study, we found that previous approaches still do not make full use of word-level information, and we have proposed a method to inject the information of current and previous words into the softmax function. The proposed model can be viewed as a combination of a modern character-aware neural language model and a simple $n$-gram word-level language model. We evaluated our method, previous methods, and a combination of our method and previous methods on 14 typologically diverse languages. The results show that our injection method performed better than previous methods that inject word-level information at the input, and that our injection method can also be used together with them. Finally, we analyzed our models on the effects of rare words, what kinds of words perform best when injected, and the effectiveness of our proposal. In future work, we plan to extend our model to other tasks, such as text generation.
}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

\textcolor{black}{
This paper is an extended version of \cite{feng-etal-2019-simple} accepted for publication by the 23rd Conference on Computational Natural Language Learning (CoNLL 2019). This extended version contains the following changes.
\begin{itemize}
\item[\textbullet] Sec.~\ref{sec:targeted_perp} was added to analyze when the method works best.
\item[\textbullet] The format of several tables with long rows was adopted.
\end{itemize}
}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{06refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix
\section{Applying Our Proposal on Standard NLMs}
\label{sec:proposal_on_standard}

Another point worth noting is that, while our proposal can also be applied to a standard word-level language model, we expect that our method is more helpful in character-aware language models. This is because a standard character-aware language model would benefit more from word-level information than a standard word-level language model, as mentioned in Sec.~\ref{sec:intro}. Here, we show the improvement percentage of our proposal applied to a standard character-aware language model and word-level language model in Table~\ref{tab:improvement_percentage}. We can see that on 10 datasets out of 14，character-aware language models obtain more improvements from the injected word-level information than the word-level language model.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 14
\begin{table}[b]
\input{06table14.tex}
\hangcaption{\textcolor{black}{Improvement percentage of our proposal applied to Word-LSTM and Char-BiLSTM-LSTM. The percentage on Word-LSTM is computed from Word-LSTM and Word-LSTM-Word in Table \ref{tab:other_injections}. For Char-BiLSTM-LSTM, it is computed from  Char-BiLSTM-LSTM and Char-BiLSTM-LSTM-Word ($g = 0.5$, $n = 1$) in Table \ref{tab_baseline_inject}.}}
\label{tab:improvement_percentage}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 15
\begin{table}[b]
\input{06table15.tex}
\hangcaption{Perplexity of InfreqChar-BiLSTM-FreqWord-LSTM and Char-BiLSTM-add-Word-LSTM on 14 language modeling datasets. The best results among all models are in bold.}
\label{tab:back_off}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Another Baseline That Backs Off to Characters for Infrequent Words}
\label{sec:back_off}

We denote \textbf{InfreqChar-BiLSTM-FreqWord-LSTM} as another straightforward baseline for using character- and word-level information. Specifically, this model uses only word embeddings if the word frequency is greater than 15; otherwise, it uses embeddings from the character encoder for infrequent words. The comparison between this model and Char-BiLSTM-add-Word-LSTM, can verify whether we need to combine both word and characters simultaneously.

\textcolor{black}{The result is shown in Table \ref{tab:back_off}. Compared with InfreqChar-BiLSTM-FreqWord-LSTM, Char-BiLSTM-add-Word-LSTM achieved better results on 13 out of 14 datasets. This indicates that using both word- and character-level information simultaneously is more beneficial. This may be because the character encoder of Char-BiLSTM-add-Word-LSTM has more chances to be trained as frequent words will also be encoded.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{\textbf{Yukun Feng}}{
He is currently a Ph.D. candidate at Department of Information and Communications Engineering, Tokyo Institute of Technology.  Before that, he received M.E. from Tokyo Institute of Technology in 2020 and B.E. from Beijing Language and Culture University in 2016. His research interests include natural language processing and machine learning, particularly word representation learning and language modeling.
}

\bioauthor[:]{\textbf{Hidetaka Kamigaito}}{He received his Ph.D. from Tokyo Institute of Technology, and is currently an associate professor in Nara Institute of Science and Technology. He worked as an assistant professor at Institute of Innovative Research, Tokyo Institute of Technology. His current research interests are in natural language processing with a specific focus on document-level text processing.
}
\bioauthor[:]{\textbf{Hiroya Takamura}}{
He received his Ph.D. from Nara Institute of Science and Technology. He worked as a professor at Tokyo Institute of Technology, and currently is a research team leader at AI Research Center of Advanced Industrial Science and Technology. His current research interests include natural language processing.
}
\bioauthor[:]{\textbf{Manabu Okumura}}{
Manabu Okumura was born in 1962.  He received B.E., M.E. and Dr. Eng. from Tokyo Institute of Technology in 1984, 1986 and 1989 respectively.  He was an assistant at the Department of Computer Science, Tokyo Institute of Technology from 1989 to 1992, and an associate professor at the School of Information Science, Japan Advanced Institute of Science and Technology from 1992 to 2000.  He is currently a professor at Institute of Innovative Research, Tokyo Institute of Technology.  His current research interests include natural language processing, especially text summarization, computer assisted language learning, sentiment analysis, and text data mining.
}
\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate




\end{document}

