\documentclass[english]{jnlp_3.5}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{xcolor}
\usepackage{array}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{29}
\Number{2}
\Month{June}
\Year{2022}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2021}{11}{1}
\revised{2022}{2}{18}
%%%%\rerevised{2006}{5}{6}
\accepted{2022}{3}{28}

\setcounter{page}{638}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%   Title, Author, etc.            %%%
%%%%\usepackage{okumacro}
%%%%\usepackage{bxpapersize} % printing b5
%%%%\usepackage{color}
%%%%\usepackage{multirow, booktabs, comment, url}
    \usepackage{multirow, booktabs}
%%%%\usepackage[dvipdfmx]{graphicx}
\usepackage{algorithm}
\usepackage[noend]{algpseudocode}
%%%%\usepackage{bm}
%%%%\usepackage{subfiles}
%%%%\usepackage{otf}
\usepackage{tabularx}
\usepackage{seqsplit}
% universal color
%アクセントカラー
\definecolor{cudred}{rgb}{1,0.15686,0}
\definecolor{cudyellow}{rgb}{0.98039,0.96078,0}
\definecolor{cudgreen}{rgb}{0.20784,0.63137,0.41961}
\definecolor{cudblue}{rgb}{0,0.2549,1}
%ベースカラー
\definecolor{cudlightpink}{rgb}{1,0.81961,0.81961}
\definecolor{cudcream}{rgb}{1,1,0.6}
\definecolor{cudlightyellowishgreen}{rgb}{0.79608,0.94902,0.4}
\definecolor{cudlightskyblue}{rgb}{0.70588,0.92157,0.98039}
%無彩色
\definecolor{cudwhite}{rgb}{1,1,1}
\definecolor{cudlightgray}{rgb}{0.78431,0.78431,0.79608}
\definecolor{cudgray}{rgb}{0.49804,0.52941,0.56078}
\definecolor{cudblack}{rgb}{0,0,0}
\newcommand{\red}[1]{\textcolor{cudred}{\textbf{#1}}}
\newcommand{\blue}[1]{\textcolor{cudblue}{\textbf{#1}}}
\newcommand{\green}[1]{\textcolor{cudgreen}{\textbf{#1}}}
\newcommand{\mdd}{\mathrm{DATE}}
\newcommand{\mduration}{\mathrm{DURATION}}
\newcommand{\mambiguous}{\mathrm{AMBIGUOUS}}
\newcommand{\mnull}{\mathrm{NULL}}
\newcommand{\tdd}{$\mathrm{DATE}$}
\newcommand{\tduration}{$\mathrm{DURATION}$}
\newcommand{\tambiguous}{$\mathrm{AMBIGUOUS}$}
\newcommand{\tnull}{$\mathrm{NULL}$}
\newcommand{\gi}{$\rm(\hspace{.18em}i\hspace{.18em})$}
\newcommand{\gii}{$\rm(\hspace{.08em}ii\hspace{.08em})$}
\newcommand{\giii}{$\rm(i\hspace{-.08em}i\hspace{-.08em}i)$}
\newcommand{\giv}{$\rm(i\hspace{-.08em}v\hspace{-.06em})$}
\newcommand{\gv}{$\rm(\hspace{.06em}v\hspace{.06em})$}
\newcommand{\gvi}{$\rm(\hspace{-.06em}v\hspace{-.08em}i)$}
\newcommand{\mgi}{\rm(\hspace{.18em}i\hspace{.18em})}
\newcommand{\mgii}{\rm(\hspace{.08em}ii\hspace{.08em})}
\newcommand{\mgiii}{\rm(i\hspace{-.08em}i\hspace{-.08em}i)}
\newcommand{\mgiv}{\rm(i\hspace{-.08em}v\hspace{-.06em})}
\newcommand{\mgv}{\rm(\hspace{.06em}v\hspace{.06em})}
\newcommand{\mgvi}{\rm(\hspace{-.06em}v\hspace{-.08em}i)}
%%%%\newcommand*{\timesfont}{\fontfamily{ptm}\selectfont}
%%%%\DeclareTextFontCommand{\textmyfont}{\timesfont}
%%%%\newcommand{\hiragana}[1]{\textit{\timesfont{#1}}}
    \newcommand{\hiragana}[1]{\textit{#1}}
%%%%\usepackage[dvipdfmx]{graphicx}
%%%%\usepackage{amsmath}
%%%%\usepackage{url}
%%%%\renewcommand\UrlFont{\rmfamily}
%%% For revision
%%%%\usepackage{soul,xcolor}
\newcommand{\Add}[1]{\textcolor{black}{#1}}
\newcommand{\Erase}[1]{\setstcolor{red}{\st{#1}}}
\newcommand{\AddUnderline}[1]{\textcolor{red}{\underline{\textcolor{black}{#1}}}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Leveraging a Bilingual Corpus to Resolve Date--Duration Ambiguity in Japanese Numeric Day Expressions}
\eauthor{Kazutaka Kinugawa\affiref{Author_1} \and Hideya Mino\affiref{Author_1} \and Isao Goto\affiref{Author_1} \and Ichiro Yamada\affiref{Author_1}} 
\eabstract{
In Japanese, time expressions are often unaccompanied by explicit temporal markers, and thus their temporal types are not always obvious. One of the most representative cases is date--duration ambiguity arising from the commonly used time expression, ``$\ast\ast$\linebreak
日[$\ast\ast\ $\hiragana{nichi}].'' To build a supervised classifier for this ambiguity while minimizing the annotation burden, we introduce an automatic label generation method using a bilingual corpus. Inspired by an annotation projection technique, we associate Japanese time expressions with their corresponding English words. Ambiguity in Japanese time expressions is comparatively easily resolved using their associated English words. We prepared several simple rules to determine temporal type labels from sentence pairs, and automatically created a training set for this task. Through a human evaluation, we verified that 98.7\% of the sampled labels match the hand-crafted labels. We then developed a classification model on these training examples and compared our automatically created examples with existing manually annotated data. Experimental results show that the produced examples improve classification models by up to 14.0\% accuracy points. Hence, our label generation method not only minimized the annotation task but is also sufficiently reliable for building temporal type classifiers.
}
\ekeywords{Time Expression, Automatic Label Generation}

\headauthor{Kinugawa et al.}
\headtitle{Leveraging Bilingual Corpus to Resolve Date--Duration Ambiguity in Ja Timexes}

\affilabel{Author_1}{}{NHK Science and Technology Research Laboratories}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
\label{sec:introduction}

The study of time expressions (timexes) has a long history in natural language processing (NLP).
In many studies, temporal types are tied to timexes, which represent the notions of time \cite{Pustejovsky2005TheSL,styler-iv-etal-2014-temporal,tabassum-etal-2016-tweetime}.
For example, DATE, TIME, DURATION, and SET are defined as temporal types in the TimeML scheme \cite{Pustejovsky2005TheSL}.
Identifying temporal types of timexes in given texts is also an important preparatory step for enabling NLP systems to understand natural language.
Specifically, this process has been studied as a subtask of temporal processing in TempEval-2 \cite{verhagen-etal-2010-semeval}, TempEval-3 \cite{uzzaman-etal-2013-semeval}, and Clinical TempEval \cite{bethard-etal-2015-semeval,bethard-etal-2016-semeval,bethard-etal-2017-semeval}.
In this subtask, surrounding temporal markers, such as ``for'' or ``during,'' are important clues.
For example, in TempEval-2, \citeA{derczynski-gaizauskas-2010-usfd2} achieved 90\% accuracy points using only three simple rules including the above temporal markers.
In Japanese, however, timexes are often unaccompanied by such explicit temporal markers, and thus their temporal types are sometimes unclear.
One of the most representative cases is ambiguity arising from the commonly used timex, ``$\ast\ast$日[$\ast\ast\ $\hiragana{nichi}].''
Consider the following examples (A) and (B) from news articles:
\begin{enumerate}
\item[(A)] 関西の空港　\blue{５日}の欠航便\footnote{The actual date is modified here to ``５日[\hiragana{itsuka}]'' \Add{(``the 5th'')} for the purposes of the example.}

Canceled flights at Kansai airport for \blue{July 5}

\item[(B)] 不適切調査の報告　\blue{５日}の遅れ 

Reporting of faulty surveys delayed by \blue{five days}
\end{enumerate}
In the above two examples, 
while the former timex ``\blue{５日}[\hiragana{itsuka}]'' (``the 5th'') indicates a specific date, i.e., DATE, 
the latter timex ``\blue{５日}[\hiragana{itsuka}]'' (``five days'') means a duration of time, i.e., DURATION.
Specifically, the former is the same as ``\underline{７月}\blue{５日}[\hiragana{\underline{shichigatsu} itsuka}]'' \Add{(``July 5''),} and the latter is the same as ``\blue{５日}\underline{間}[\hiragana{itsuka \underline{kan}}]'' \Add{(``for five days'').}\ 
However, the adjacent temporal clues ``\underline{７月}[\hiragana{\underline{shichigatsu}}]'' \Add{(``July'')} and ``\underline{間}[\hiragana{\underline{kan}}]'' \Add{(``for'')} are omitted from the sentences.
The difference in the temporal types thus comes from the following common nouns ``欠航便[\hiragana{kekkobin}]'' (``canceled flights'') and ``遅れ[\hiragana{okure}]'' (``delay'').

Resolving this ambiguity should be helpful for several downstream tasks because this timex frequently appears in real-world documents.
For example, one possible application we assume is as preprocessing for Japanese-English machine translation in the news domain.
In Japanese news articles, ``$\ast\ast$日[$\ast\ast\ $\hiragana{nichi}]'' is often unaccompanied by a month when it means a specific date.
However, such expressions are often translated into non-equivalent expressions such as ``$\langle$a day of the week$\rangle$'' or ``$\langle$month$\rangle\langle$cardinal number$\rangle$'' in English.
To fill this gap in machine translation, one possible solution is to insert an appropriate month before the timex (or replace the timex with a day of the week) in the source sentence before translation.
Temporal type identification is required here because the above insertion (or replacement) must be performed only for date timexes.
If the phrase were to be inserted before a duration timex (or the duration timex were to be replaced with the phrase), an incorrect translation would be generated.
Taking (A) as an example, to translate ``５日[\hiragana{itsuka}]'' \Add{(``the 5th'')} into ``July 5,'' a system has to identify this timex as DATE, and then insert ``７月[\hiragana{shichigatsu}]'' \Add{(``July'')} before the timex.
As for (B), if the phrase were to be mistakenly inserted, the meaning of the source sentence would be changed, e.g., ``７月５日の遅れ[\hiragana{shichigatsu ituska no okure}]'' (``delay of July 5'').

While rule-based approaches are effective for temporal type identification in English \cite{strotgen-gertz-2010-heideltime,chang-manning-2012-sutime}, 
manually building rules that cover the vast number of possible word co-occurrences, such as ``欠航便[\hiragana{kekkobin}]'' (``canceled flights'') or ``遅れ[\hiragana{okure}]'' (``delay''), is challenging in Japanese.
Supervised learning is thus a reasonable alternative for such disambiguation.
However, even though temporal type identification is also addressed in named entity recognition (NER) in Japanese \cite{sekine-etal-2002-extended,asahara-matsumoto-2003-japanese,Taira2009BaseNPST,misawa-etal-2017-character,mai-etal-2018-empirical}, 
an annotated training set specialized for this ambiguity is still required to further improve prediction accuracy. 
The problem here is that manually annotating additional training examples is time-consuming.

In this paper, we present an automatic label generation technique for the above \textsl{date--duration ambiguity}.
The target timex is specifically described as ``$\langle$cardinal number$\rangle$日[\hiragana{nichi}]''; for instance, ``５日[\hiragana{itsuka}]'' \Add{(``the 5th'' or ``five days'').}
\Add{As noted, this type of timex frequently appears in the real-world documents.
For instance, the Japanese NER dataset IREX CRL\footnote{\url{https://nlp.cs.nyu.edu/irex/index-j.html}} 
contains 3,567 date examples, and 1,325 (37.1\%) of them have the above forms and 1,113 of 1,325 examples are unaccompanied by a month. 
Thus, correctly identifying this form of timex would be important for processing Japanese timexes.}
Specifically, we refer to this form of timex as a \textbf{D}ate--\textbf{D}uration-\textbf{tim}e \textbf{ex}pression (henceforth DD timex).
Our aim is to automatically generate binary labels indicating the DATE or DURATION for DD timexes, 
and then use the created training examples to develop a date--duration classifier.
We borrowed the idea from a traditional \Add{annotation projection technique using a bilingual corpus \cite{yarowsky-etal-2001-inducing,fu-etal-2011-generating,ehrmann-etal-2011-building}.}
The key lies in the assumption that, even if the source-side words are ambiguous, they can be easily resolved using the target-side words.
In the above examples, the temporal types of the Japanese timexes can be disambiguated using their distinguishable translations ``\blue{July 5}'' and ``\blue{five days}.'' 
We configured several heuristic rules and applied them to bilingual sentence pairs to produce binary labels.
We aggregated the DD timexes, crafted labels, and built a date--duration training set in a cost-effective manner.
We manually inspected the produced labels and verified that 98.7\% of the sampled labels overlap with the hand-crafted annotations.
\Add{Next,} we conducted a date--duration classification experiment to confirm the effectiveness of our training set.
Results showed that a supervised classification model developed on our training examples 
\Add{
outperforms a classifier trained on existing manually annotated examples by up to 14.0\% accuracy points.
}
This means that our label generation method is sufficiently reliable for developing temporal type classifiers.
\Add{
In addition, we conducted an ablation study to discuss the effectiveness of our example generation technique.
}
\Add{
Finally, we investigated the degree to which publicly available temporal type classifiers resolve date--duration ambiguity. 
We evaluated the existing systems on several test sets, conducted an error analysis, and discussed the difficulties associated with this problem.
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Date--Duration Example Generation}
\label{sec:date_duration_example_generation}

In this section, we explain our automatic label generation method for temporal type ambiguity.
We retrieved timexes, which are described as ``$\langle$cardinal number$\rangle$日[\hiragana{nichi}],'' from a Japanese--English bilingual corpus and attached temporal type labels using their translations.
For simplicity, other similar timexes such as ``毎日[\hiragana{mainichi}]'' (``every day'') or ``今日[\hiragana{kyo}]'' (``today'') are not considered in this paper.
Although various temporal types have been introduced in individual tasks \cite{sekine-etal-2002-extended,Pustejovsky2005TheSL,styler-iv-etal-2014-temporal,tabassum-etal-2016-tweetime}, 
we consider the most common temporal types, DATE and DURATION, for the target timex.
Note that another possible temporal type can also be considered for this timex.
For example, ``エスカレーター事故はほぼ\blue{２日}に１件'' (``Escalator accidents occur about once every \blue{2 days}'') can be interpreted as a SET in the TimeML scheme \cite{Pustejovsky2005TheSL}.
(This timex is regarded as DURATION in this paper.)


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Preprocess}
\label{subsec:preprocess}

Given a bilingual corpus, we first ran GIZA++
\footnote{
1) The Japanese sentences were tokenized using MeCab \cite{kudo-etal-2004-applying}, and English sentences were tokenized using the \texttt{tokenizer.perl} script in Moses \cite{koehn-etal-2007-moses} beforehand.
2) When running GIZA++, we used the {\tt train-model.perl} script with {\tt grow-diag-final-and} option in Moses \cite{koehn-etal-2007-moses}.
}
\cite{och03} on the entire corpus to obtain alignment pairs.
We then performed a simple regular-expression-based pattern matching 
against each Japanese sentence to \Add{extract sentence pairs containing a DD timex}.\footnote{The regular expression unintentionally extracts non-time expressions, e.g., ``野球U\blue{１８日}本代表[\hiragana{yakyu under juhachi nihon daihyo}]'' (``Japan's U-18 national baseball team''). We eliminated such sentence pairs using simple rules.}
\Add{
The regular expression is designed to recognize DD timexes as ``$\langle$cardinal number$\rangle$日[\hiragana{nichi}].'' Cardinal numbers in DD timexes were assumed to be Arabic numerals. 
}
After this filtering, we obtained a set of tuples $(s, w_{s}, t, w_{t})$, 
where $s$ and $t$ denote a source (Japanese) sentence and a target (English) sentence, respectively, 
and $w_{s}$ is a DD timex, and $w_{t}$ denotes the English tokens to which $w_{s}$ is aligned, e.g., ($w_{s}$, $w_{t}$) = (``５日[\hiragana{itsuka}],'' ``five days'').
When a Japanese sentence contains multiple DD timexes $(w^{1}_{s}, w^{2}_{s}, ..., w^{n}_{s})$, 
the corresponding multiple tuples $\{(s, w^{1}_{s}, t, w^{1}_{t})$, $(s, w^{2}_{s}, t, w^{2}_{t})$, ..., $(s, w^{n}_{s}, t, w^{n}_{t})\}$ are produced from this sentence pair.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Automatic Annotation Rules}
\label{subsec:automatic_annotation_rules}

Next, we provided each tuple with a temporal type label $y \in \{\mathrm{DATE}, \mathrm{DURATION}\}$.
To produce reliable and diverse examples, we set up a two-fold process to determine $y$.
We preliminarily investigated the lexical patterns of both $w_{s}$ and $w_{t}$, and configured annotation rules in both languages independently.
Specifically, given a tuple $(s, w_{s}, t, w_{t})$, we first divided this into the Japanese side $(s, w_{s})$ and the English side $(t, w_{t})$.
The Japanese-side rules and the English-side rules were separately applied to $(s, w_{s})$ and $(t, w_{t})$ to produce tentative temporal type labels $y_{s}$ and $y_{t}$.
We then compared $y_{s}$ and $y_{t}$ to determine the final label $y$. 
When $y_{s}$ and $y_{t}$ were inconsistent with each other, we discarded the tuple to ensure label validity.
We extracted date--duration examples $(s, w_{s}, y)$ from the remaining tuples to build a date--duration training set.
Note that our technique assumes Japanese--English sentence pairs, but another language pair Japanese--X can also be used by setting up similar rules for X.
The rules we used are described in detail in the remainder of this subsection.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{English-Side Rules}
As mentioned in Section \ref{sec:introduction}, $w_{t}$ is lexically distinguishable as DATE or DURATION when compared with $w_{s}$.
Therefore, on the English side, simple look-up tables should work well for identifying temporal types.
We investigated lexical patterns in sentence pairs from our corpus.
(The bilingual corpus we used is explained in Section \ref{sec:date_duration_dataset}.)
Our preliminary findings are summarized as follows:

\begin{enumerate}
\item When $w_{s}$ is a DATE, $w_{t}$ tends to be (a) ``$\langle$month$\rangle\langle$cardinal number$\rangle$,'' (b) ``$\langle$days of the week$\rangle$,'' or (c) ``$\langle$ordinal number$\rangle$.''
\begin{enumerate}
\item[(a)] \blue{２３日}の桑名市は、午後３時前に２６度８分を観測し、夏日となりました。% ID:1029390

The temperature in Kuwana City on \blue{May 23} was 26.8 degrees Celsius shortly before 3 p.m., and it was a hot summer day.

\item [(b)] 第１戦は\blue{１４日}午後７時からナゴヤドームで行われます。% ID:1160790

The first of this year's Japan All-Star Games will be held in Nagoya Dome on \blue{Friday} at 7 pm.


\item[(c)] \blue{２３日}も危険な暑さのおそれ　引き続き熱中症に厳重警戒を % ID:916173

Concern for dangerous levels of heat again on the \blue{23rd}; continued strict caution for heatstroke

\end{enumerate}
\item When $w_{s}$ is a DURATION, $w_{t}$ tends to include (d) ``day,'' (e) derivatives of ``day,'' or (f) compounds of ``day.''
\begin{enumerate}
\item [(d)] \blue{１日}の感染者数が５０人を下回るのは、\green{４日}連続です。% ID:1111988

For \green{the fourth consecutive day}, the number of those infected with the virus \blue{per day} was below 50.

\item [(e)] 同時に、\blue{１日}単位ではなく、時間単位で有給休暇を取得できるようにするなど、働き方改革も進めました。 % ID:316708

At the same time, he promoted work style reforms, such as making it possible to take paid holidays not on a \blue{daily} basis but on an hourly basis.

\item[(f)] 総合運動公園の事務所によりますと、大会は２２日から\blue{３日}間の日程で開かれていました。% ID:1017609

According to the sports park office, the \blue{three-day} competition began on July 22.

\end{enumerate}
\end{enumerate}
Based on these findings, we build four dictionaries, 
\texttt{Dict\_Month} (corresponding to (a)), 
\texttt{Dict\_Week} (corresponding to (b)), 
\texttt{Dict\_Ord} (corresponding to (c)),  
and \texttt{Dict\_Day} (corresponding to (d), (e), and (f)), as listed in Table \ref{table:en_dict}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b] 
\input{14table01.tex}
\caption{Tokens contained in the dictionaries. Tokens are counted in a case-insensitive manner.}
\label{table:en_dict}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Algorithm \ref{alg:get_en_label} presents the label generation rule on the English side.
Given $(t, w_{t})$, we checked the four dictionaries in order, and once $w_{t}$ was found in one dictionary, the corresponding temporal type was readily assigned to $y_{t}$.
When $w_{t}$ was contained in \texttt{Dict\_Month}, \texttt{Dict\_Week} or \texttt{Dict\_Ord}, DATE was assigned to $y_{t}$.
Similarly, when $w_{t}$ was contained in \texttt{Dict\_Day}, DURATION was assigned to $y_{t}$.
When $w_{t}$ contained several tokens, we checked the dictionary token by token.
In addition to this, we prepared one more heuristic rule: when $w_{t}$ is not found in any dictionary, we then try the adjacent tokens of $w_{t}$.
In our preliminary trial, we sometimes encountered the case that $w_{t}$ was not found in any dictionary because of misalignment.
Taking the previous sentence pair (b) as an example, ``１４日[\hiragana{juyokka}]'' \Add{(``the 14th'')} should be aligned to ``Friday'' or ``on Friday,'' but sometimes it was only aligned to ``on.''
If the adjacent tokens were also not found in any dictionary, we attached a dummy label NULL, which indicates it does not match any prepared rules, to $y_{t}$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% algo. 1
\begin{algorithm}[t]
\caption{~~Temporal Type Label Generation on the English Side}
\begin{algorithmic}[1]
\Procedure{LookupDict}{$w$}
\If{$w$ in \texttt{Dict\_Week} or $w$ in \texttt{Dict\_Month}}
\State \Return{\tdd}
\EndIf
\If{$w$ in \texttt{Dict\_Day}}
\State \Return{\tduration}
\EndIf
\If{$w$ in \texttt{Dict\_Ord}}
\State \Return{\tdd}
\EndIf
\State \Return{\tnull}
\EndProcedure
\\
\Procedure{GenEnLabel}{$t$, $w_{t}$}
\State{$y_{t} \leftarrow \Call{LookupDict}{w_{t}}$}
\If{$y_{t} == \mnull$}
\State{$w^{\mathrm{adj}}_{t} \leftarrow \Call{ExtractAdjacentTokens}{t, w_{t}}$}
\State{$y_{t} \leftarrow  \Call{LookupDict}{w^{\mathrm{adj}}_{t}}$}
\EndIf
\State \Return{$y_{t}$}
\EndProcedure
\end{algorithmic}
\label{alg:get_en_label}
\end{algorithm}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Japanese-Side Rules}
Even though some DD timexes are ambiguous, not all DD timexes are challenging.
Typical expressions accompanying explicit temporal markers are easily disambiguated by a simple pattern matching in Japanese.
Examples are as follows:
\begin{enumerate}
\item[] クルーズ船は\underline{８月}\blue{１６日}までの\green{８日}\underline{間}、運航されます。% ID:67767p

The cruise ship will be running for a total of \green{eight days} through \blue{August 16}.

\end{enumerate}
In this case, we can easily identify the temporal type of ``\underline{８月}\blue{１６日}[\hiragana{\underline{hachigatsu} jurokunichi}]'' (``August 16'') as a DATE by virtue of the previous words ``\underline{８月}[\hiragana{\underline{hachigatsu}}]'' \Add{(``August'').}\ 
Similarly, ``\green{８日}\underline{間}[\hiragana{yoka \underline{kan}}]'' (``eight days'') can be interpreted as a DURATION because of the following word ``\underline{間}[\hiragana{\underline{kan}}]'' \Add{(``for'').} \ 
These DD timexes are deterministically identified by adjacent words.
Such \textsl{easy examples} should also be useful for developing a temporal type classifier.
To retrieve these examples, we manually enumerated adjacent word patterns and configured five regular expressions, 
$\texttt{Month\_Reg\_Date}$, 
$\texttt{Prev\_Reg\_Date}$, 
$\texttt{Next\_Reg\_Date}$, 
$\texttt{Prev\_Reg\_Duration}$, 
and $\texttt{Next\_Reg\_Duration}$, as described in Table \ref{table:ja_regrex}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t] 
\input{14table02.tex}
\hangcaption{Descriptions of the regular expressions. $\texttt{DD}$, $\texttt{MM}$, and $\texttt{WW}$ are the component variables used in the five regular expressions.}
\label{table:ja_regrex}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% algo. 2
\begin{algorithm}[t]
\caption{~~Temporal Type Label Generation on the Japanese Side}
\begin{algorithmic}[1]
\Procedure{GenJaLabel}{$s, w_{s}$}
\State{$d \leftarrow \Call{ExtractInteger}{w_{t}}$}
\If{$d$ is more than 31}
\State{$y_{s} \leftarrow \mduration$}
\State \Return{$y_{s}$}
\EndIf
\If{$\texttt{Month\_Reg\_Date}.\mathrm{match}(s, w_{s})$ or $\texttt{Next\_Reg\_Date}.\mathrm{match}(s, w_{s})$}
\State{$y_{s} \leftarrow \mdd$}
\State \Return{$y_{s}$}
\EndIf
\If{$\texttt{Prev\_Reg\_Duration}.\mathrm{match}(s, w_{s})$ or $\texttt{Next\_Reg\_Duration}.\mathrm{match}(s, w_{s})$}
\State{$y_{s} \leftarrow \mduration$}
\State \Return{$y_{s}$}
\EndIf
\If{$\texttt{Prev\_Reg\_Date}.\mathrm{match}(s, w_{s})$}
\State{$y_{s} \leftarrow \mdd$}
\State \Return{$y_{s}$}
\EndIf
\State{$y_{s} \leftarrow \mnull$}
\State \Return{$y_{s}$}
\EndProcedure
\end{algorithmic}
\label{alg:get_ja_label}
\end{algorithm}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Algorithm \ref{alg:get_ja_label} shows the entire process on the Japanese side.
Given $(s, w_{s})$, we applied the five pattern matchings to $w_{s}$ in order, and the label was determined in a greedy manner.
If $w_{s}$ matches $\texttt{Month\_Reg\_Date}$, $\texttt{Prev\_Reg\_Date}$, or $\texttt{Next\_Reg\_Date}$, DATE is assigned to $y_{s}$.
And if $w_{s}$ matches $\texttt{Prev\_Reg\_Duration}$ or $\texttt{Next\_Reg\_Duration}$, DURATION is assigned to $y_{s}$.
Moreover, we used a simple heuristic rule at the beginning of the procedure: if the number in $w_{s}$ is more than 31, DURATION is assigned to $y_{s}$.
If $w_{s}$ does not match any rules, NULL is assigned to $y_{s}$. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Determination of the Final Label}
The final label $y$ is determined by the tentative labels $y_{s}$ and $y_{t}$ obtained in the previous steps.
Algorithm \ref{alg:get_ja_en_label} presents the entire procedure.
If a temporal type is consistently assigned to both $y_{s}$ and $y_{t}$, e.g., $(y_{s}, y_{t}) = (\mdd, \mdd)$,
the same label is assigned to $y$.
By contrast, if the tentative labels are inconsistent, e.g., $(y_{s}, y_{t}) = (\mdd, \mduration)$, 
$y$ is labelled as NULL.
Another possibility is that one of the labels is NULL and the other is a DATE or DURATION.
In this case, the non-null label is adopted as the final label.
Finally, if both $y_{s}$ and $y_{t}$ are NULL, NULL is assigned to $y$.
Date--duration examples ($s$, $w_{s}$, $y$) are extracted from the tuples.
After filtering out examples where $y$ is NULL, the remaining set is used as the date--duration training example.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% algo. 3
\begin{algorithm}[t]
\caption{~~Date--Duration Example Generation}
\label{alg:get_ja_en_label}
\begin{algorithmic}[1]
\Procedure{GenerateDate-DurationExample}{$s, w_{s}, t, w_{t}$}
\State{$y_{s} \leftarrow \textproc{GenJaLabel}(s, w_{s})$}
\State{$y_{t} \leftarrow \textproc{GenEnLabel}(t, w_{t})$}
\If{$y_{s} \in \{\mdd, \mduration\}$ and $y_{t}  \in \{\mdd, \mduration\}$}
\If{$y_{s} == y_{t}$}
\State{$y \leftarrow y_{s}$}
\Else
\State{$y \leftarrow \mnull$}
\EndIf
\ElsIf{$y_{s} \in \{\mdd, \mduration\}$ and $y_{t} == \mnull$}
\State{$y \leftarrow y_{s}$}
\ElsIf{$y_{s} == \mnull$ and $y_{t} \in \{\mdd, \mduration\}$}
\State{$y \leftarrow y_{t}$}
\ElsIf{$y_{s} == \mnull$ and $y_{t} == \mnull$}
\State{$y \leftarrow \mnull$}
\EndIf
\State \Return{$(s, w_{s}, y)$}
\EndProcedure
\end{algorithmic}
\end{algorithm}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Date--Duration Dataset}
\label{sec:date_duration_dataset}

This section describes statistics of a bilingual corpus we used and the generated date--duration examples.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Bilingual Corpus}
\label{subsec:bilingual_corpus}

We applied the proposed label generation method to an in-house NHK Japanese-English parallel corpus.
This corpus is a set of news articles where each Japanese sentence is manually translated into one or more English sentences, 
and news articles where each English sentence is manually translated into one or more Japanese sentences.\footnote{When performing the word alignment, multiple sentences were concatenated with the separator symbol ``$|||$.''}
This corpus is suitable for this task because it contains many DD timexes.
The corpus statistics are shown in Table \ref{table:stat}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Derivation of the Labels}
\label{subsec:label_derivation}

\newcommand{\fen}{$R_{t}$}
\newcommand{\fja}{$R_{s}$}
Next, we analyzed the coverage and reliability of the generated labels from the perspective of the derivation of labels.
As described in Section \ref{sec:date_duration_example_generation}, we produced labels by combining the English-side rules with the Japanese-side rules.
In this subsection, the respective label-generation procedures, which are presented in Algorithms \ref{alg:get_en_label} and \ref{alg:get_ja_label}, 
are denoted as $R_{t} := \textproc{GenEnLabel}$ and $R_{s} := \textproc{GenJaLabel}$, respectively.
Moreover, the set of non-null temporal types is denoted as $Y_{dd} := \{\mdd, \mduration\}$.
Generated labels are grouped into the following five categories as follows:

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t] 
\input{14table03.tex}
\hangcaption{Bilingual corpus statistics. ``DATE \#,'' ``DURATION \#,'' and ``NULL \#'' indicate the number of DD timexes labeled as DATE, DURATION, and NULL, respectively with our proposed method described in Section \ref{sec:date_duration_example_generation}.}
\label{table:stat}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{enumerate}

\item[\gi] labels consistently derived from \fen \ and \fja, i.e., $y_{t} \in Y_{dd}$, $y_{s} \in Y_{dd}$ and $y_{t} = y_{s}$. 

\item[\gii] labels derived from \fen, but not from \fja, i.e., $y_{t} \in Y_{dd}$ and $y_{s} = \mnull$

\item[\giii] labels derived from \fja, but not from \fen, i.e., $y_{t} = \mnull$ and $y_{s} \in Y_{dd}$

\item[\giv] labels inconsistently derived from \fen \ and \fja, i.e., $y_{t} \in Y_{dd}$, $y_{s} \in Y_{dd}$ and $y_{t} \neq y_{s}$

\item[\gv] labels derived from neither  \fen \ nor \fja, i.e., $y_{t} = y_{s} = \mnull$

\end{enumerate}
DD timexes in groups \gi--\giii \ are given with a final temporal type label $y \in Y_{dd}$ \ and are contained in the date-duration examples.
In contrast, DD timexes in groups \giv--\gv \ are labeled as $y = \mnull$ and excluded from the training set.
The percentages of groups \gi--\gv \ are listed in Table \ref{table:ja_vs_en}.
The subscripts of the underlined numbers in Table \ref{table:ja_vs_en} indicate the corresponding group enumerated above.
Our first observation was that the total percentage of DD timexes with $y \in Y_{dd}$ is 98.0\% ($\underline{61.9\%}_{\mgi} + \underline{33.1\%}_{\mgii} + \underline{3.0\%}_{\mgiii}$).
Although the total number of rules is not very large, most DD timexes are covered by these simple rules, largely by virtue of the English-side rules.
Whereas the Japanese-side rules cover only 65.3\% ($\underline{61.9\%}_{\mgi} + \underline{0.4\%}_{\mgiv} + \underline{3.0\%}_{\mgiii}$) of the DD timexes, the English-side rules cover 95.4\% ($\underline{61.9\%}_{\mgi} + \underline{0.4\%}_{\mgiv} + \underline{33.1\%}_{\mgii}$).
This fact is consistent with our claim that DD timexes are more tractable in English than in Japanese.
Moreover, the percentage of group \giv \ is only $\underline{0.4\%}_{\mgiv}$.
This suggests that the Japanese-side rules and the English-side rules are both consistently designed.
We observed that inconsistency between $y_{t}$ and $y_{s}$ in group \giv \ mainly comes from incorrect word alignments or under-translations in the target sentences.
Finally, it is shown that the DD timexes that are attributed using only the English-side rules are $\underline{33.1\%}_{\mgii}$ and those attributed using the Japanese-side rules are 65.3\% ($\underline{61.9\%}_{\mgi} + \underline{0.4\%}_{\mgiv} + \underline{3.0\%}_{\mgiii}$).
Our assumption is that the Japanese-side rules can recognize typical expressions, which are accompanied by explicit temporal markers, but cannot discriminate ambiguous timexes that lack such explicit clues.  
This means that approximately one-third ($\underline{33.1\%}_{\mgii}$) of the DD timexes are \textsl{difficult examples}, and the remaining two-thirds (65.3\%) are \textsl{easy examples}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}
\label{sec:experiment}

In this section, we discuss two experiments.
First, we manually checked the quality of the automatically produced labels \Add{in Section \ref{subsec:manual_inspection}}.
Second, to demonstrate the effectiveness of our generated examples, 
we conducted a date--duration classification experiment \Add{in Section \ref{subsec:date_duration_classification}.} 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[b] 
\input{14table04.tex}
\hangcaption{Derivation of the labels. ``Ja'' and ``En'' indicate the Japanese-side rules $R_{s}$ and the English-side rules $R_{t}$, respectively. ``DD'' represents DD timexes successfully labeled as either DATE or DURATION, and ``NULL'' represents those labeled as NULL. The subscripts of the underlined numbers indicate the corresponding groups.}
\label{table:ja_vs_en}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Manual Inspection}
\label{subsec:manual_inspection}

To evaluate the quality of the generated labels, we performed a manual inspection of the labels sampled from the DD examples.
To check our rule design over a wide range of data, examples were sampled from the different groups described in Section \ref{subsec:label_derivation}.
Specifically, 381, 310, and 356 examples were \Add{randomly} taken from groups \gi, \gii, \ and \giii, respectively.
\Add{
This proportion of the sampled numbers is different from that of the population, which is 61.9 : 33.1 : 3.0 as shown in Table \ref{table:ja_vs_en}.
The proportion of the population is biased towards group \gi \ because this group contains many typical expressions such as ``８月１６日[\hiragana{hachigatsu jurokunichi}]'' (``August 16'').
As mentioned in Section \ref{subsec:automatic_annotation_rules}, these typical expressions are easily recognized and labeled.
To prevent such easy examples from dominating the manual inspection data, 
we sampled examples from the three groups so that the proportion was not significantly biased. 
For the same reason, 
even though the ratio of date examples to duration examples in the population data is approximately 10 : 1, as shown in Table \ref{table:stat}, 
we adjusted the extracted examples so that the ratio in the manual inspection data was not significantly biased.
}
Two workers annotated a total of 1,047 examples with gold labels for temporal type.
Because some DD timexes were intrinsically ambiguous, these DD timexes were labeled as \tambiguous.
For instance, ``\blue{1日}の出来高は１１億３８１５万株でした。'' can be interpreted as either 
DATE (``The trading volume \blue{on the 1st} was 1.13815 billion shares.''), % ID: 445173p
or DURATION (``The \blue{daily} trading volume was 1.13815 billion shares.''). % ID: 461105p
Such examples were excluded from the evaluation.
The gold labels and generated labels are compared in Table \ref{table:silver_vs_gold}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[t] 
\input{14table05.tex}
\hangcaption{Comparison of automatically produced labels and human-annotated labels. The former are referred to as ``silver'' and the latter are referred to as ``gold'' in the table. Bold-faced numbers indicate the number of DD timexes whose silver labels are coincident with the gold labels. ``AMBIGUOUS'' indicates intrinsically ambiguous DD timexes, and they were not considered in this evaluation.}
\label{table:silver_vs_gold}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The percentage of generated labels that match human-annotated labels is 98.7\%. %silver labels have high accuracies of 98.7\%.
This demonstrates the quality of our label generation method. 
In contrast, 7 of the 13 unmatched DD timexes were incorrectly labeled because of incompleteness in the English-side rules.
An example is as follows:
\begin{enumerate}
\item[] 日経平均株価、午前の終値は、\blue{２４日}の終値より４４円７１銭安い２万１２４１円２８銭、東証株価指数=トピックスは、０.５８上がって１５４８.３２、午前の出来高は４億８７０８万株でした。

The Nikkei Stock Average morning closing price was down 44.71 yen from \blue{the previous day} to 21,241.28 yen, while the Tokyo Stock Price Index, or TOPIX, was up 0.58 points at 1,548.32 points on a volume of 487.08 million shares.
\end{enumerate}
In this sentence pair, ``２４日[\hiragana{nijuyokka}]'' \Add{(``the 24th'')} indicates a date, which is expressed as ``the previous day'' in English, but the alignment pair was $(w_{s}, w_{t}) = (``２４日[\hiragana{nijuyokka}],'' \mathrm{``day''})$.
Because ``day'' is contained in $\texttt{Dict\_Day}$, ``２４日[\hiragana{nijuyokka}]'' \Add{(``the 24th'')} was incorrectly labeled as DURATION.
Other mistakes were due to misalignment (two examples) and under-translation (four examples).
\Add{As for the derivation of labels, 12 examples of the 13 unmatched examples were from group \gii \ and the remaining one example was from group \gi. The 13 ambiguous examples were all from group \gii.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Date--Duration Classification}
\label{subsec:date_duration_classification}

In the following section, we present the results of the \Add{date--duration classification experiments.}
The task was to predict binary temporal type labels given a Japanese sentence containing DD timexes.
Following the settings in TempEval-2 \cite{verhagen-etal-2010-semeval}, the evaluation metric was the date--duration classification accuracy over the DD timexes that have gold annotations, i.e., the number of correct examples divided by the number of test set examples.
\Add{
First, we describe the main experiment in Section \ref{subsubsec:comparision_dataset}, where our automatically created examples are compared with existing manually annotated examples. 
We trained supervised classification models on respective training sets and evaluated prediction accuracies to demonstrate the effectiveness of our automatic example generation technique.
Second, we investigate the extent to which existing timex recognizers resolve date--duration ambiguity in Section \ref{subsubsec:comparision_recognizer}. 
We evaluated existing systems on several test sets, conducted an error analysis, and discuss the difficulty of date--duration ambiguity.  
}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{\Add{Comparison with existing timex training data}}
\label{subsubsec:comparision_dataset}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Dataset}
\Add{
To conduct a date--duration classification experiment, we prepared several types of timex datasets.
The first dataset was constructed on DD timexes from the NHK Ja-En News Corpus shown in Table \ref{table:stat}.
We built development data and test data based on the manual inspection data described in Section \ref{subsec:manual_inspection}.
To evaluate examples, we additionally asked annotators to label 228 DD timexes which were randomly taken from groups \giv \ (seven examples) and \gv \ (221 examples), 
and added them to the manual inspection data.
The resulting data consist of examples from all groups, 
and 
the numbers of easy and difficult examples in this set are not significantly biased.
(According to Section \ref{subsec:label_derivation}, timexes in groups \gi, \giv, and \giii \ tend to be easy examples, 
and timexes in groups \gii \ and \gv \ tend to be difficult.)
Moreover, from the perspective of the four cells in Table \ref{table:ja_vs_en}, 
i.e., groups \gi \ and \giv, group \gii, group \giii, \ and group \gv, 
the number of examples from each cell in the resulting data is not significantly biased. 
}
We eliminated a total of 25 AMBIGUOUS examples \Add{from this set}, and then split the annotated examples into 628 examples for development and 622 examples for testing 
\Add{
so that the proportions of examples in respective groups and the label distributions were approximately the same in the two sets. 
}
Next, we extracted training examples from the remaining set so that a set of news articles containing training examples was disjointed from that containing development/test examples.
The number of the resulting training examples is 219,954.
\Add{We refer to this dataset as NHK-all in the following experiments.}

\Add{
The second dataset is from BCCWJ-TimeBank \cite{asahara-etal-2013-bccwj}.
This corpus is composed of a set of Japanese documents where timexes are manually annotated. 
Specifically, each timex is marked with a TIMEX3 tag and provided with one of the four temporal types, i.e., DATE, TIME, DURATION and SET.
This corpus is also suitable for this experiment,  
and 559 of 3,715 date timexes and 124 of 1,210 duration timexes correspond to DD timexes. 
We extracted sentences containing at least one tagged DD timex and built a date--duration dataset. 
Because the number of DD timexes are limited, we did not use standard train-dev-test splits.
Instead, we assumed the following training/test combinations and split the date--duration dataset in different ways, as follows.
}
\begin{enumerate}
\item [\Add{(1)}] \Add{For a model trained on BCCWJ-TimeBank and evaluated on other test sets: \\
We split the extracted DD timexes into a training set (546 examples) and a development set (137 examples) 
so that the ratio of the training set to the development set was approximately 8 : 2, and the ratio of date examples to duration examples in the training set was approximately the same as that of the development set.}
\item [\Add{(2)}] \Add{For models trained on other examples and evaluated on BCCWJ-TimeBank: \\
All DD timexes were used as a test set.}
\item[\Add{(3)}] \Add{For a model trained on BCCWJ-TimeBank and evaluated on BCCWJ-TimeBank: \\
Specifically, this splitting was for comparing the model with models in (2). 
We divided the examples in a similar way to a five-fold cross validation.  
First, we split all DD timexes into five disjointed test sets so that the union of the five sets was equal to all DD timexes in the BCCWJ-TimeBank, i.e., the test set in (2).    
Examples other than each test set were further divided into training and development data so that the ratio of the training data to the development data was approximately 8 : 2, 
and the ratios of date examples to duration examples in all sets were approximately the same.
By virtue of this splitting, we could obtain five trained models and five results on their corresponding test sets.
Finally, we summed up correct/incorrect examples in the five test sets and computed a total accuracy point. }
\end{enumerate}

\Add{  
For comparison with BCCWJ-TimeBank, we prepared the NHK-small dataset.
We further split the development set of NHK-all into a training set (546 examples) and a development set (82 examples).
The size of the training set was the same as that of the training set in BCCWJ-TimeBank (1) explained above.
The ratio of date examples to duration examples in the training set was the approximately same as that of the development set.
}

\Add{
In addition to the above dataset, we used a Japanese NER corpus IREX CRL.\footnote{\url{https://nlp.cs.nyu.edu/irex/index-j.html}}
This corpus is composed of news articles and contains various NER tags including DATE. 
Although DURATION tags are not included, 1,325 out of 3,567 date examples correspond to DD timexes in this corpus.
We used these date examples as a test set.
Because DURATION tags did not exist, we computed the recall score for this test set, i.e., 
the number of examples correctly classified as DATE out of 1,325 examples, instead of accuracy.
}

\Add{
To conduct experiments with our settings, we performed several preprocesses. 
As described in Section \ref{subsec:preprocess}, DD timexes are defined as ``$\langle$cardinal number$\rangle$日[\hiragana{nichi}]'' in our settings.
Gold labels in NHK-all and NHK-small were also based on this definition.
However, segmentations of some DD timexes in BCCWJ-TimeBank and IREX CRL differ from our definition. 
For instance, in the sentence ``２人は今年１月１１日に結婚したばかりだった。'' (``They had just gotten married on January 11 of this year.'') from BCCWJ-TimeBank, the Timex3 tag is attached to ``今年１月１１日'' (``on January 11 of this year'').
In our definition, the DD timex is ``１１日[\hiragana{juichinichi}]'' (``the 11th'') in this case.
Taking this gap into consideration, we modified segmentations for such DD timexes to our definition. 
Second, while most of the DD timexes in NHK-all and NHK-small are expressed by Arabic numerals, e.g., ``７月５日[\hiragana{shichigatsu itsuka}]'' (``July 5''), 
DD timexes in BCCWJ-TimeBank and IREX CRL are frequently expressed by Chinese numerals, e.g., ``七月五日[\hiragana{shichigatsu itsuka}]'' (``July 5'').
Such DD timexes were all converted to Arabic numerals as mentioned in Section \ref{subsec:preprocess}.
}

The date--duration dataset statistics are shown in Table \ref{table:split}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[b] 
\input{14table06.tex}
\hangcaption{Date--duration dataset statistics. In the fifth row, i.e., ``BCCWJ-TimeBank (3),'' one of the five splittings is listed.}
\label{table:split}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Model}
\Add{As a temporal type classifier,} we built a simple bidirectional long short-term memory (Bi-LSTM) classification model, which is a simplified version of \citeA{laparra-etal-2018-characters}'s model.
Let a left-to-right LSTM be denoted by $\mathrm{LSTM}_{f}$ and a right-to-left one be denoted as $\mathrm{LSTM}_{b}$.
\begin{align}
\overrightarrow{h}_{k} & = \mathrm{LSTM}_{f}(e_{k}, \overrightarrow{h}_{k-1}), \\
\overleftarrow{h}_{k} & = \mathrm{LSTM}_{b}(e_{k}, \overleftarrow{h}_{k-1}),
\end{align}
where $e_{k}$ indicates the $k$-th embedding vector in an input sequence, 
and $\overrightarrow{h}_{k}$ and$\overleftarrow{h}_{k}$ are the corresponding hidden vectors from $\mathrm{LSTM}_{f}$ and $\mathrm{LSTM}_{b}$, respectively.
Let the start and end points of a target DD timex in the input sequence be $i$ and $j$, respectively.
Two hidden vectors corresponding to positions $i$ and $j$ are extracted from the hidden vector sequences, and then concatenated.
Finally, the concatenated vector $h_{i:j}$ is projected to compute an output probability $\alpha$ using the softmax function.
\begin{gather}
h_{i:j} = \mathrm{Concat}(\overleftarrow{h}_{i}, \overrightarrow{h}_{j}), \label{hij} \\
\alpha = \mathrm{Softmax}(Wh_{i:j}+b), \label{linear}
\end{gather}
where $W$ and $b$ are the output projection matrix and the bias, respectively.

Our Bi-LSTM model is composed of single-layer LSTMs with 128 units, and the number of dimensions of the word embedding vectors is 128.
\Add{
The word embedding vectors, all the weights and biases in the LSTMs, and the weight and bias in Eq. \ref{linear}
were randomly initialized from $\mathcal{N}(0,\,1)\,$, $\mathcal{U}(-1/\sqrt{128},\,1/\sqrt{128})$, and $\mathcal{U}(-1/\sqrt{256},\,1/\sqrt{256})$, respectively.
}
Our system was implemented using PyTorch \cite{NEURIPS2019_9015}.
During training, the size of a mini-batch was 32.
We used Adam \cite{kingma14} as the optimizer and the initial learning rate was set to 0.001.
We evaluated accuracy scores on the development set and save checkpoints every 2000 iterations.
The early stopping patience was set to 16 checkpoints, and the dropout rate of embeddings and LSTMs was set to 0.2.
When constructing the vocabulary, words of frequency less than 5 were replaced with ``$\langle$UNK$\rangle$.''


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Results}
Table \ref{table:comparison_dataset} shows the experimental results.
The model trained on NHK-all outperformed the other two models on all test sets.
This suggests that a small number of human-annotated examples is not enough and automatically created training examples effectively work in this task. 
Next, it was observed that the models trained on manually annotated training sets (NHK-small and BCCWJ-TimeBank) both achieved 84\%--92\% accuracy points.
The models trained on BCCWJ-TimeBank training data obtained better results on IREX CRL test data and BCCWJ-TimeBank test data. 
In contrast, this model tends to incorrectly predict duration examples on NHK-all test data. 
As shown in Table \ref{table:split}, there were fewer duration examples than date examples in the training data.
To improve prediction accuracy, more duration examples are required. 
Conversely, the model trained on NHK-small training data achieved a higher accuracy point on NHK-all test data 
but obtained a worse result on IREX CRL test data and BCCWJ-TimeBank test data. 
(Note that NHK-small comes from the same example set as NHK-all test data and thus has the same properties as NHK-all test data.) 
More date examples are needed to obtain more accurate results on IREX CRL test data and BCCWJ-TimeBank test data.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[t]
\input{14table07.tex}
\hangcaption{Experimental results. Bold-faced numbers indicate the number of examples whose predicted labels are coincident with the gold labels. ``Unk'' indicates the proportion of test DD timexes replaced with ``$\langle$UNK$\rangle$.'' Note that, in the third row, the models in the first and second columns and the model in the third column are different. While the former models were trained on ``BCCWJ-TimeBank (1)'' in Table \ref{table:split}, the latter model was trained on ``BCCWJ-TimeBank (3).'' The sizes of models in the second and third rows were halved because models in this setting obtained slightly better results on development data.}
\label{table:comparison_dataset}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Ablation Study}
\Add{
Next, to investigate contributions of respective label groups in NHK-all, 
we conducted an ablation study.
In the above experiments, NHK-all training data comprised of examples in groups \gi, \gii, and \giii.
We changed the combination and created two additional training sets.
The first training data were examples created using only the Japanese-side rules, i.e., a set of examples from groups \gi, \giii, and \giv.
The second training data were created using only the English-side rules, i.e., a set of examples from groups \gi, \gii, and \giv.
For examples in group \giv, labels from the corresponding language side were adopted in each training set.
We refer to the above two training sets as NHK-ja and NHK-en.
We trained two Bi-LSTM models on NHK-ja and NHK-en independently, and computed evaluation scores.
The results are shown in Table \ref{table:ablation_study}. 
The model trained on NHK-en consistently outperforms the model trained on NHK-ja. 
This shows that examples produced using the English-side rules are more helpful in this task. 
In addition, the model trained on NHK-en achieved competitive results to the model trained on NHK-all in Table \ref{table:comparison_dataset}.
As shown in Table \ref{table:ja_vs_en}, the English-side rules cover most of the DD timexes (95.4\%) in the original corpus.
Thus, NHK-all largely overlap with NHK-en. (Actually, 97.2\% of examples in NHK-all are also contained in NHK-en.)
On the other hand, even though all DD examples covered by the Japanese-side rules have typical expression patterns, 
the model trained on NHK-ja achieves 94\%--95\% accuracy points. 
This suggests that the trained model does not only rely on typical expressions but also uses other information such as contexts in input sentences to predict temporal type labels. 
Thus, such easy DD timexes are also useful training examples in this task.
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[t]
\input{14table08.tex}
\hangcaption{Ablation study. The first row is the same as the results in Table \ref{table:comparison_dataset}. Numbers in parentheses in the second and third rows indicate accuracy drops/improvements compared with the levels of the model trained on NHK-all.}
\label{table:ablation_study}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Comparison with existing timex recognizers}
\label{subsubsec:comparision_recognizer}

\Add{
In this section, we evaluate two existing systems and our model to demonstrate the difficulty of date--duration ambiguity.
The test sets in Section \ref{subsubsec:comparision_dataset} were also used in this experiment.
It should be noted that this is not a completely fair comparison. 
While our model concentrates on date--duration ambiguity and tackles this task as a binary-label classification, 
comparison systems partly address this problem in a multi-label classification. 
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Comparison System}
\Add{W}e first adopted a Japanese time expression recognizer, normalizeNumexp.\footnote{\url{https://github.com/nullnull/normalizeNumexp}}
Using a rule-based approach, this tool extracts timexes \Add{and numerical expressions} in a given sentence and classifies them into three temporal types \Add{and one additional type}: \textsl{abstime}, \textsl{reltime}, \textsl{duration}\Add{, and \textsl{numerical}}. 
To conduct an experiment in our binary classification setting, we treated \textsl{abstime} as DATE, and \textsl{duration} as DURATION.

The second comparison method is a publicly available NER model, GiNZA v5.0.2.\footnote{\url{https://megagonlabs.github.io/ginza/}}
The underlying model in GiNZA is a Hugging Face Transformer released as 
\texttt{\seqsplit{transformers-ud-japanese-electra-base-discriminator}}.\footnote{\url{https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator}}
The Transformer was pre-trained on Japanese monolingual data extracted from mC4 \cite{2019t5}, 
and fine-tuned on the GSK2014-A corpus.\footnote{\url{https://www.gsk.or.jp/catalog/gsk2014-a}}
The NER tags are based on Sekine's Extended Named Entity Hierarchy\footnote{\url{http://liat-aip.sakura.ne.jp/ene/ene8/definition_jp/html/enedetail.html}}
\cite{sekine-etal-2002-extended} and OntoNote5.\footnote{\url{https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf}}
In this definition, two tags \textsl{Date} and \textsl{Period\_Day} simply correspond to DATE and DURATION in our setting, respectively.

\Add{
When input sentences are fed into the respective models, recognized timexes can be differently segmented depending on the models. 
Consider the phrase ``ここ３０日ほどの間に[\hiragana{koko sanju nichi hodo no aida ni}]'' (``in the past 30 days'') as an example.\footnote{The whole sentence, the gold label and the predicted labels are listed in the fifth row in Table \ref{table:error_analysis}.} 
While GiNZA recognizes the timex as ``３０日[\hiragana{sanju nichi}]'' (``30 days''), 
normalizeNumExp recognizes the timex as ``３０日ほど[\hiragana{sanju nichi hodo}]'' (``about 30 days''). 
Taking this gap into consideration, when computing accuracy points, 
we checked whether the temporal type assigned to the segmentation including a focused DD timex, e.g., ``３０日[\hiragana{sanju nichi}]'' (``30 days''), coincided with the gold label.
In the above example, the gold label of ``３０日[\hiragana{sanju nichi}]'' (``30 days'') is DURATION and the predicted label of ``３０日ほど [\hiragana{sanju nichi hodo}]'' (``about 30 days'') in normalizeNumExp is also DURATION. 
Even though the timex segmentation of normalizeNumExp differs from our definition, 
the predicted label is regarded as correct in this case.
}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Results}

Table \ref{table:experimental_results} shows the experimental results.
As shown in the table, 
\Add{
our Bi-LSTM model consistently outperforms other systems. 
Target classes are differently defined depending on models, and thus, we cannot simply compare the results. 
However, even if examples not labeled as DATE or DURATION in the respective comparison systems (``OTHERWISE'' in the first and second rows in Table \ref{table:experimental_results}) are counted as correct,  
our model achieves higher accuracy points than both systems for all test data.
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[t]
\input{14table09.tex}
\hangcaption{Experimental results. ``OTHERWISE'' in the first row, i.e., ``normalizeNumExp,'' indicates examples labeled as \textsl{reltime} or \textsl{numerical}. ``OTHERWISE'' in the second row, i.e., ``GiNZA,'' refers to examples not labeled as \textsl{Date} or \textsl{Period\_Day} and examples not provided with any NER labels. ``Ours'' is the same as the model in the first row in Table \ref{table:comparison_dataset}. When evaluating normalizeNumExp and GiNZA, we tried both the original test set and the test set where all DD timexes are expressed by Arabic numerals. The results of the latter version are listed because they were slightly better.}
\label{table:experimental_results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\definecolor{caddback}{rgb}{0.90, 0.98, 0.96}
\definecolor{cadd}{rgb}{0, 0.47, 0.34}
\definecolor{cdelback}{rgb}{1, 0.94, 0.92}
\definecolor{cdel}{rgb}{0.83, 0.32, 0.16}
\newcommand{\true}[1]{\colorbox{caddback}{\textcolor{cadd}{#1}}}
\newcommand{\false}[1]{\colorbox{cdelback}{\textcolor{cdel}{#1}}}
\newcolumntype{C}{>{\raggedright\arraybackslash}X}
\setlength\extrarowheight{7pt}
\begin{table}[p] 
\input{14table10.tex}
\caption{Test set and label examples in NHK-all. Proper nouns are replaced with ``$\ast\ast$.''}
\label{table:error_analysis}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Add{Focusing on the results on NHK-all test data, the} difference mainly comes from the number of duration timexes correctly classified.
Though normalizeNumExp correctly classifies many date timexes, 
this tool tends to incorrectly label duration timexes not accompanied by explicit temporal markers.
To classify such timexes, models are required to discriminate surrounding words in some way, 
but it is difficult to build classification rules covering the extensive range of possible surrounding words.
GiNZA detects duration timexes better than normalizeNumExp.
However, 46 timexes were labeled as neither DATE nor DURATION.
In contrast, our supervised learning approach successfully classifies duration timexes by virtue of the created training set.
Table \ref{table:error_analysis} shows test set examples and the output labels of different models.
Although the timexes in the first and second rows are unaccompanied by explicit temporal markers, 
our trained model correctly identifies the temporal types.
The timex ``\underline{中}\blue{５日}[\hiragana{\underline{naka} itsuka}]'' (``5 day's rest'') in the third row has the adjacent temporal marker ``\underline{中}[\underline{\hiragana{naka}}],'' \Add{(``rest''),}
and this timex frequently appears in the sports domain.
To cover such a specific timex, automatically retrieving training examples is an effective approach.
On the other hand, our model fails to classify the timexes in the fourth and the fifth rows.
In contrast, normalizeNumExp is able to identify the temporal types of these timexes using the adjacent patterns such as ``\Add{５日}\underline{未満}[\Add{\hiragana{itsuka}} \underline{\hiragana{miman}}]'' \Add{(``less than five days''),} ``\underline{ここ}\Add{５日}[\underline{\hiragana{koko}} \Add{\hiragana{itsuka}}]'' \Add{(``last 5 days''),} and ``\Add{５日}\underline{ほど}[\Add{\hiragana{itsuka}} \underline{\hiragana{hodo}}]'' \Add{(``about 5 days'').} 
One possible reason for these mispredictions is numeric bias and label imbalance in the training set.
Let $d$ denote an integer contained in $w_{s}$, e.g., ($w_{s}$, $d$) = (``１４日[\hiragana{juyokka}],'' 14).
To investigate the cause of the above vulnerability,
we grouped all examples in the NHK-all training set into 31 bins according to $d$, and counted the number of labels in each bin.
The result is illustrated in Figure \ref{fig:histogram}.
As shown in the histogram, the number of all examples in each bin is not significantly different across $d \in [3, 20]$. 
However, the number of duration examples consistently decreases as $d$ increases except for $d = 10$.
Specifically, almost all timexes with $d \in [21,31]$ are date examples.
This means that models have fewer chance to learn duration examples with larger $d$.
As a result, models built on this training set tend to be biased to DATE against such timexes, 
and thus, the timexes such as $(d, y) = (14, \mathrm{DURATION})$ or $(d, y) = (30, \mathrm{DURATION})$ were incorrectly labeled as DATE.
To alleviate such a numeric bias, the number of examples in each bin should be sensitively adjusted.
A similar phenomenon was reported by \citeA{prabhakaran-etal-2019-perturbation}.
They found that output scores are affected by changing a person's name in an input sentence in sentiment and toxicity analysis. 
Specifically, controversial people are likely to yield negative scores.
One of the possible reasons is that names of such people tend to be accompanied by negative words in contexts, and this imbalance consequently biases models.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[t]
\begin{center}
\includegraphics{29-2ia14f1.pdf}
\end{center}
\hangcaption{Correlation between the numbers in the timexes and the label distributions in the NHK-all training set. Because examples where $d \in [11, 20]$ and $d \in [21,31]$ have similar distributions inside their respective ranges, they are both represented as the averaged values in the histogram. Examples where $d > 31$, which are all labeled as \tduration \ in our technique, are excluded from this figure.}
\label{fig:histogram}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}
\label{sec:related_work}

Extracting timexes in given texts is a fundamental process for understanding texts.
This process includes two tasks: recognition \cite{zhong-etal-2017-time,Ding_Gao_Shi_Qu_2019} and normalization \cite{ding-etal-2021-automatic-rule}.
The first task is to detect timexes from free text, and the second one is to annotate the detected timexes with temporal types and then translate the timexes into machine-readable formats.
\Add{Other timex-related tasks, such as temporal relation extraction \cite{cheng-miyao-2017-classifying,meng-etal-2017-temporal}, rely on such essential information.}
Although timex recognition and normalization have long been studied \cite{angeli-etal-2012-parsing,angeli-uszkoreit-2013-language,ning-etal-2018-cogcomptime,laparra-etal-2018-characters}, 
identification of timexes in natural language text is still challenging.
\citeA{Tissot2019NormalisationOI} pointed out that imprecise timexes, such as ``less than 2 years'' or ``several weeks,'' cannot be well normalized by existing timex schemes.
\citeA{may-etal-2021-extraction} proposed a novel timex scheme for such vague timexes.
This study concerns another aspect: temporal type ambiguity in Japanese timexes.

In the Japanese timex domain, \citeA{asahara-etal-2013-bccwj} annotated a Japanese corpus based on the TimeML scheme \cite{Pustejovsky2005TheSL}.
However, the actively studied area is temporal relation extraction \cite{kaneko-bekki-2014-building,kaneko-etal-2016-annotation,cheng-etal-2020-dynamically}, and research about timex recognition and normalization is limited.
Apart from the timex domain, temporal type identification is also addressed as a part of the NER task.
Temporal types in NER include not only basic timex tags, such as \textsl{Date} and \textsl{Time} \cite{asahara-matsumoto-2003-japanese,Taira2009BaseNPST,misawa-etal-2017-character}, but also fine-grained tags, such as \textsl{Era} and \textsl{Period\_Year} \cite{sekine-etal-2002-extended,mai-etal-2018-empirical}.
Our focus is discrimination between DATE and DURATION, and this study explicitly tackled this ambiguity.

Recently, some studies have aimed to supply blind spot examples that are not often included in existing datasets in individual tasks.
\citeA{elkahky-etal-2018-challenge} indicated that recent part-of-speech (POS) tag prediction models, which have achieved high accuracies in benchmark test sets, do not yet work well for \textsl{noun--verb ambiguous} words such as ``mark.''
They aggregated noun--verb ambiguous words and built a specialized dataset for this specific problem.
\citeA{zhang-etal-2019-paws} found that classifiers trained on existing paraphrase identification datasets are easily mislead by non-paraphrase sentence pairs that have high lexical overlap, e.g., ``flights from New York to Florida'' and ``flights from Florida to New York.''\
They proficiently scrambled words in existing resources to create adversarial examples.
This paper aggregated date and duration timexes to build a date--duration training set.

\Add{In} order to create date--duration examples, we borrowed the idea of a traditionally used \Add{annotation projection technique using bilingual corpora \cite{yarowsky-etal-2001-inducing,fu-etal-2011-generating,ehrmann-etal-2011-building}}.
\Add{
The aim of this approach is to complement annotated examples in a target language using annotations in a source language.
Generally, annotations are projected from languages where a focused task is comparatively easily addressed. 
The basic process is as follows:
First, sentences in the source language are annotated by existing systems, 
and then the annotations are transferred to the corresponding sentences in the target language.
To perform this, sentences and words are aligned by existing toolkits \cite{fu-etal-2011-generating}.
This technique is used in various tasks such as POS tagging \cite{yarowsky-ngai-2001-inducing,yarowsky-etal-2001-inducing}, noun phrase bracketing \cite{yarowsky-ngai-2001-inducing} and NER \cite{fu-etal-2011-generating,ehrmann-etal-2011-building}. 
For example, \citeA{fu-etal-2011-generating} used this approach to project annotations from English to Chinese. 
They claimed that Chinese NER is more difficult than English NER due to the lack of capitalization and the uncertainty in word segmentation. 
In our case, we used this concept to project temporal type annotations from English to Japanese. 
Temporal type classification in Japanese is more difficult than in English because Japanese timexes are often unaccompanied by explicit temporal markers.
Following the report that English timexes are comparatively easily classified using rule-based approaches \cite{derczynski-gaizauskas-2010-usfd2}, 
we manually built automatic annotation rules in English and more directly annotated English timexes with temporal type labels compared with \citeA{fu-etal-2011-generating}.
In addition, to ensure label quality, we also prepared simple pattern matching rules in Japanese and configured a two-fold process to determine temporal type labels.
We performed rule-based annotation for both languages independently 
and then compared tentative labels obtained from respective language sides.
Through this process, inconsistent results were excluded and examples which could be recognized only by the English-side rules or the Japanese-side rules were also covered.
}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
\label{sec:conclusion}

This paper presented an automatic label generation method for date-duration ambiguity.
We specifically focused on the commonly used Japanese DD timex, ``$\ast\ast$日[$\ast\ast\ $\hiragana{nichi}].''
To employ a machine learning approach, we first generated temporal type labels using a bilingual corpus.
The motivation for this method was based on our finding that English timexes corresponding to Japanese DD timexes can be comparatively easily labeled using a rule-based approach.
To ensure label validity and coverage, we prepared automatic annotation rules in both Japanese and English.
Our simple rules covered DD timexes extensively and consequently produced a number of training examples while minimizing the annotation burden.
We conducted a human evaluation of the automatically produced labels and observed that 98.7\% of sampled labels match the human annotated labels with a high accuracy.
\Add{
Then, we set up a date--duration classification experiment.
Experimental results showed that a supervised learning model developed on our training set outperforms models trained on existing manually annotated examples by up to 14.0\% accuracy points. 
}
Our label generation method is thus sufficiently reliable for training temporal type classifiers.
\Add{
Finally, we observed the extent to which existing timex recognizers resolve date--duration ambiguity. 
We evaluated prediction accuracies, conducted an error analysis, and discussed the difficulty of this problem. 
}

For future work, we will expand the proposed label generation method to other Japanese timexes such as ``年[\hiragana{nen}]'' (``year''), ``時間[\hiragana{jikan}]'' (``hour''), or ``分[\hiragana{hun}]'' (``minute'').
Automatic annotation rules for other kinds of Japanese timexes can also be developed in the same manner as that used for ``$\ast\ast$日[$\ast\ast\ $\hiragana{nichi}].''
Building label generation rules for temporal type labels other than DATE and DURATION, such as SET, would also be interesting. 
Another direction would be to apply our technique to comparable corpora.
The bilingual corpus in this paper consists of sets of sentence pairs, where each sentence is translated to one or more sentences, 
and word alignment pairs are thus comparatively easily obtained.
However, the availability of large parallel corpora is still limited.
By incorporating an unsupervised word sense disambiguation method for comparable bilingual corpora such as \citeA{kaji-morimoto-2002-unsupervised} and \citeA{kaji-2003-word}, our technique would be more flexible.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

The authors would like to acknowledge the reviewers for their valuable comments. 
We also thank Kimberly Moravec, PhD, from Edanz, for editing a draft of this manuscript.
Finally, we are grateful to all of the workers for annotating the temporal type labels.  


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\bibliography{14refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Kazutaka Kinugawa}{ 
received his B.S. and M.S. degrees from the University of Tokyo in 2016 and 2018, respectively.
He joined NHK (Japan Broadcasting Corporation) in 2018 and has worked for NHK Science and Technical Research Laboratories since 2020.
His research interests include natural language processing.
}

\bioauthor[:]{Hideya Mino}{
received his B.S., M.S., and Ph.D. degrees in computer science from the Tokyo Institute of Technology in 2002, 2004 and 2022, respectively. 
He joined NHK (Japan Broadcasting Corporation) in 2004 and has worked for NHK Science and Technical Research Laboratories since 2009. 
He was a Visiting Researcher at the National Institute of Information and Communications Technology from 2013 to 2017. 
His research interests include natural language processing.
}
\bioauthor[:]{Isao Goto}{
received his B.S. and M.S. degrees in electrical engineering from Waseda University in 1995 and 1997, respectively, and received his Ph.D. degree from Kyoto University in 2014.
He joined NHK (Japan Broadcasting Corporation) in 1997 and has worked for NHK Science and Technical Research Laboratories since 1999.
He was a Visiting Researcher at Advanced Telecommunications Research Institute International from 2004 to 2006 and a Visiting Researcher at the National Institute of Information and Communications Technology from 2008 to 2013.
His research interests include machine translation.
}
\bioauthor[:]{Ichiro Yamada}{
received his B.S., M.S., and Ph.D. degrees from Nagoya University in 1991, 1993 and 2009, respectively. 
He joined NHK (Japan Broadcasting Corporation) in 1993 and has worked for NHK Science and Technical Research Laboratories since 1996. 
He was a Visiting Scholar at Stanford University from 2003 to 2004 and a Visiting Researcher at the National Institute of Information and Communications Technology from 2008 to 2011. 
His research interests include machine translation and knowledge acquisition.
}
\end{biography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate



\end{document}



