\documentclass[english]{jnlp_1.4}

\usepackage{jnlpbbl_1.3}
\usepackage[dvips]{graphicx}
\usepackage{hangcaption_jnlp}
\usepackage{amssymb}
\usepackage{amsmath}
\newcommand{\RowHalfline}[1]{}

\usepackage{tabularx}



\Volume{20}
\Number{1}
\Month{March}
\Year{2013}

\received{2012}{10}{9}
\accepted{2012}{11}{25}

\setcounter{page}{27}

\etitle{Database of Human Evaluations of Machine Translation Systems for Patent Translation}
\eauthor{Isao Goto\affiref{NICT} \and 
Bin Lu\affiref{HKIEd}\affiref{CityU} \and
Ka Po Chow\affiref{HKIEd} \and
Eiichiro Sumita\affiref{NICT} \and \\
Benjamin K. Tsou\affiref{HKIEd}\affiref{CityU} \and
Masao Utiyama\affiref{NICT} \and
Keiji Yasuda\affiref{NICT} 
} 
\eabstract{
This paper discusses a database of human evaluations of patent machine translation, from Chinese to English, Japanese to English, and English to Japanese. 
The evaluations were conducted for the NTCIR-9 Patent Machine Translation Task (PatentMT). 
Different types of systems, such as research systems and commercial systems, and rule-based systems and statistical machine translation systems were evaluated. 
Since human evaluation results are important when investigating automatic evaluation of translation quality, the database of the evaluation results is valuable. 
From the NTCIR project, resources including the human evaluation database, translation results, and test/reference data are available for research purposes.
}
\ekeywords{Human evaluation, Patent translation, Machine translation}

\headauthor{Goto et al.}
\headtitle{Database of Human Evaluations of MT Systems for Patent Translation}

\affilabel{NICT}{}{National Institute of Information and Communications Technology}
\affilabel{HKIEd}{}{Hong Kong Institute of Education}
\affilabel{CityU}{}{City University of Hong Kong}


\begin{document}

\maketitle

\section{Introduction}

Automatic evaluation of translation quality is important for development of machine translation systems; 
thus, it is an active area of research \cite{papineni-EtAl:2002:ACL,lin-hovy:2003:NAACL,isozaki-EtAl:2010:EMNLP}. 
Human evaluation resources, which can be used as fundamental data for the research, have been published. NIST Metrics for Machine Translation (MetricsMATR)
\footnote{LDC Catalog No.: LDC2009T05 and LDC2011T05}
published human evaluation resources for the news domain and for Arabic/Chinese-to-English translations. 
The Workshop on Statistical Machine Translation (WMT)\footnote{http://www.statmt.org/wmt12/results.html} \cite{callisonburch-EtAl:2012:WMT} published human evaluation resources for news and European Parliament Proceedings and translations between European languages.

The database of human evaluations that we introduce in this paper is different from the above-mentioned human evaluation resources in that it targets the patent domain and focuses on translations that include the Asian languages of Japanese and Chinese.
This human evaluation database was produced through the Patent Machine Translation Task (PatentMT) at NTCIR-9\footnote{The NTCIR-9 workshop was held December 6--9, 2011 in Tokyo. http://research.nii.ac.jp/ntcir/ntcir-9/}. 


PatentMT at NTCIR-9 builds on the previous patent translation tasks performed at NTCIR-7 \cite{Fujii-EtAl:NTCIR7} and NTCIR-8 \cite{Fujii-EtAl:NTCIR8} and contains two additions that were not present in the previous two tasks: 
Chinese-to-English translation and acceptability evaluations. 
The need to translate patent information from the Chinese language has increased, and therefore, a Chinese-to-English subtask was added
\footnote{We did not add the English-to-Chinese subtask because of three reasons: 
1) Since human evaluation incurs significant costs, such as expense and labor, conducting human evaluations for four subtasks was difficult at that time. 
2) We thought that the Chinese-to-English translation was more popular than the English-to-Chinese translation. 
3) We could not provide a large-scale monolingual patent corpus in Chinese.
}. 
A comparison of NTCIR-7, 8, and 9 is summarized in Table \ref{table:comparison_between_ntcir}. 
A human evaluation for \textit{acceptability} was conducted, aiming to realize a more practical evaluation than is afforded by adequacy. 
The human evaluation reveals the number of test sentences that can be understood from the translations.

\begin{table}[t]
\caption{Comparison of NTCIR-7, 8, and 9.}
\label{table:comparison_between_ntcir}
\input{02table01.txt}
\end{table}

This paper\footnote{This paper is based on a presentation given at the NTCIR-9 workshop \cite{Goto-EtAl:NTCIR9}.}
 gives a brief overview of NTCIR-9 PatentMT and describes the human evaluation results obtained at NTCIR-9 PatentMT.
The effectiveness of the different state-of-the-art machine translation systems can be evaluated by comparing the human evaluation results of translations done by the different systems using the same test data. 
The database of the human evaluations is useful for research in the automatic evaluation of translation quality. 

This paper is organized as follows: 
Section \ref{sec:task_design} explains the task design, 
Section \ref{sec:participants_and_submissions} lists the participants and submissions, 
Section \ref{sec:human_evaluation_results} describes the human evaluation results, 
Section \ref{sec:validation_human_evaluation} shows the validation of human evaluations, 
Section \ref{sec:meta-evaluation} gives a meta-evaluation of the automatic evaluation measure of BLEU, 
Section \ref{sec:obtain_method} shows the method for obtaining the database, 
and Section \ref{sec:conclusion} concludes the paper.


\section{Task design}
\label{sec:task_design}

PatentMT had three patent machine translation subtasks: Chinese to English (CE), Japanese to English (JE), and English to Japanese (EJ). 
Participants chose the subtasks in which they wished to participate and were provided with training data, development data, and test data. 
Participants translated the test data using their machine translation systems and submitted the translations to the PatentMT organizers. 
The PatentMT organizers evaluated the submitted translations and returned the evaluation results to the participants. 
Finally, the participants presented their research results at the NTCIR-9 workshop. 

\subsection{Data provided to the participants}

The provided data consisted of training data, development data, test data, context documents, and reference data. 
The reference data was provided after the submission of translation results. 
The training data consisted of a parallel corpus and a monolingual corpus. 
The parallel sentence pairs for the training, development, and test/reference data were drawn from patent description sentences 
(patent documents consist of a title, abstract, claim, and description).
The parallel sentence pairs for the training data were automatically extracted from patent documents using bilingual dictionaries.
The Chinese--English parallel sentence pairs were extracted from Patent Cooperation Treaty (PCT) patents in Chinese and English
 \cite{Lu-EtAl:CLP2010}. 
The Japanese--English parallel sentence pairs were extracted from the patent family in Japanese and English
 \cite{Utiyama-Isahara:MTS2007}. 
The training data was built from patent documents published between 1993 and 2005.
The number of patent parallel sentence pairs for the training data was: 1 million for Chinese--English and approximately 3.2 million for Japanese--English.
The training data of the monolingual corpus was a monolingual patent corpus in the target language spanning 13 years (1993--2005).
The test data was built by randomly selecting parallel sentences from a portion of the automatically built patent parallel sentence pairs published in 2006 and 2007, 
manually judging whether the sentence pairs were correct translations, 
then selecting 2,000 correct sentence pairs as the test data and their reference data. 
The patent documents from which the test sentences were extracted were provided as context documents for the test data.


\subsection{Evaluation methodology}

We conducted human evaluations and regarded these as the primary evaluation.

Human evaluations were carried out by paid evaluation experts\footnote{
Because we evaluated machine translations and not human translations, we did not perform evaluations in a similar manner as human translations. 
The evaluators were not patent experts for the domains. 
This evaluation did not check whether the translations of technical terms were perfect. 
The writing style for the test data is a general style because all of the test sentences were from description sections of patents and were not from claim sections. 
Therefore, the evaluators, who are not patent experts, can distinguish whether a translated sentence represents the source sentence meaning. 
The evaluator profiles were as follows: 
For CE, adequacy: Chinese native speakers who can understand English; 
acceptability: Chinese native speakers whose English abilities are very high.
For JE, adequacy and acceptability: English native speakers who can understand Japanese.
For EJ, adequacy and acceptability: Japanese native speakers who can understand English.
} and employed the criteria of \textit{adequacy} and \textit{acceptability}, which will be explained later.
For each criterion, three evaluators evaluated 100 sentences per system. 
The three evaluators evaluated different sentences. 
Thus, 300 sentences were evaluated per system.
The 300 sentences were randomly selected from the test sentences.
In this evaluation, the evaluators looked at a source sentence and its translation results to be evaluated.


\subsubsection{Adequacy}

We conducted a 5-scale (1 to 5) adequacy evaluation. 
The main purpose of the adequacy evaluation was to compare the systems. 

Adequacy can be defined in multiple ways.
White \cite{White-EtAl:AMTA1994} defined it as how much of the information from a fragment of a reference sentence is contained in the translation results. 
They insisted that fragmentation is intended to avoid biasing the results in favor of linguistic compositional approaches (which may do relatively better on longer, clause-level strings) or statistical approaches (which may do better on shorter strings not associated with syntactic constituency).
However, this evaluation cannot evaluate whether the sentence meaning is correct or not because simply containing all of the fragments of the reference information does not guarantee a correct sentence meaning. 
The NTCIR-7 Patent Translation Task \cite{Fujii-EtAl:NTCIR7} conducted adequacy evaluations using a criterion based on the degree of preservation of sentence-level meaning instead of the degree of fragments of the reference information contained. 

We believed that the degree of sentence-level meaning preservation was better than that of fragments of reference information contained for the evaluation of translation quality. 
However, since the cost of checking sentence meanings is high, we evaluated quality considering the clause-level meanings for adequacy.

The instructions for the adequacy criterion are given in Appendix \ref{sec:apn:adequacy}. 
Examples of adequacy values and translations are shown in Appendix \ref{sec:apn:adq_example}.

The systems were ranked based on adequacy using the average system scores.


\subsubsection{Acceptability}
\label{sec:acceptability}

We conducted a 5-scale acceptability evaluation as shown in Fig. \ref{fig:acceptability_criterion}. 
The main purpose of an acceptability evaluation is to clarify the percentage of translated sentences for which the source sentence meanings can be understood from randomly selected test sentences.
\footnote{\textit{Informative}, shown in the ALPAC report \cite{Pierce-EtAl:ALPAC-report:1966} p.70, is also an evaluation criterion of translation quality. 
Informative is a measure of how informative the original version is perceived to be after the translation has been seen using a scale from 0 to 9. 
The upper grades are decided on the basis of whether a translation includes word-level or sentence structure-level errors. 
Because there are cases where the source sentence meaning can be understood as well as cases where it cannot be understood when a translation includes sentence structure-level errors, informative cannot clarify the difference in understandability. 
In contrast, acceptability can judge whether the source sentence meaning can be understood independent of the existence of word-level or sentence-structure-level errors.} 
Acceptability is an evaluation of sentence-level meaning. 
The acceptability criterion used in this evaluation is aimed more at practical evaluation as opposed to adequacy. 
For example, if the requirement of a translation system is that the source sentence meaning can be understood, translations of C or higher are useful; however, if the requirement is that the source sentence meaning can be understood and the sentence is grammatically correct, then only translations of A or higher are useful. 
We can then know the number of sentences from a system would be useful for each requirement. 
An adequacy criterion cannot answer these requirements.

\begin{figure}[b]
\begin{center}
\includegraphics{20-1ia964f1.eps}
\end{center}
\caption{Acceptability}
\label{fig:acceptability_criterion}
\end{figure}

Acceptability also contains an evaluation of \textit{fluency} that measures fluency in the target language, since it also affects the differences in grading from C to AA. 
If the adequacy of a translation is very low, then the translation is not correct even if the fluency is high. 
If the integrated evaluation score is calculated by averaging the adequacy and fluency scores, then those translations could be overvalued. 
Acceptability avoids this problem, allowing us to consider fluency.

The instructions for the acceptability criterion are shown in Appendix \ref{sec:apn:acceptability}.
Examples of acceptability values and translations are shown in Appendix \ref{sec:apn:acp_example}.

We ranked the systems based on acceptability using a pairwise comparison, which will now be explained. 
The \textit{pairwise score} for a system A reflects how frequently it was judged to be better than or equal to other systems. 
Suppose there are five systems to be compared. 
For each input sentence, system A is included in four pairwise comparisons (against the other systems). 
System A is rewarded as 1.0 for each of the comparisons in which system A is ranked the highest of the two, and 
0.5 for each of the comparisons in which system A is in a tie. 
System A's score is the total rewarded score in the pairwise comparisons divided by the total number of pairwise comparisons involving system A.

Note that the average score of acceptability was not used for system ranking. 
The reason is as follows. 
Here we assume that the differences between the grades are measured by general usability. 
It is important to be able to understand the contents from the source sentence. 
There is a large difference in usability between F and C. 
However, at the A-level, while the translations are at a non-native level, the contents from the source sentences can be understood and they are grammatically correct; thus, they have the potential to be useful in many cases. 
Thus, it is believed that the difference in usability between A and AA is smaller than that between F and C. 
In addition, we think that useful grades depend on specific usage. 
Therefore, it is difficult to give an appropriate score for each grade, and we avoided the conversion of grades to scores and calculation of averages.


\subsubsection{Human evaluation procedure}

We conducted human evaluation training before the main evaluation to normalize the evaluators' criteria. 
In the training, all evaluators evaluated 100 translations, and a meeting was held to determine common results for each subtask. 
The main evaluation was then performed.  
The common results produced at the training were used as the reference results for the main evaluation.

The instructions for the human evaluation procedure are shown in Appendix \ref{sec:apn:eval_procedure}.


\subsection{Schedule}

Translations were done over a two-week period in May 2011. 


\section{Participants and submissions}
\label{sec:participants_and_submissions}

\begin{table}[b]
\caption{Participants and subtasks participated in.}
\label{table:participant}
\input{02table02.txt}
\end{table}

We received submissions from 21 groups. 
The number of groups for each subtask was: 18 for CE, 12 for JE, and 9 for EJ. 
Table \ref{table:participant} shows the Group IDs, the participant organizations, system description papers, and the subtasks in which they participated. 
The types of translation systems are statistical machine translation (SMT),  rule-based machine translation (RBMT), example-based machine translation (EBMT), or hybrids of two or more types (HYBRID).

In addition to the submissions from the participants, the organizers submitted results for baseline systems that consisted of 2 SMT systems, 5 commercial RBMT systems, and 1 online SMT system. 
The baseline systems are shown in Table \ref{table:baseline_system}. 
The SMT baseline systems consisted of publicly available software, and the procedures for building the systems and translating using the systems were published on the PatentMT web page\footnote{http://ntcir.nii.ac.jp/PatentMT/} so that those with the training data can build the SMT baseline systems and compare their results. 
The commercial RBMT systems and the Google online translation system\footnote{http://www.google.com} were operated by the organizers. 
The translation results from the Google translation system were created by translating the test data via their web interface. 
We note that these RBMT companies and Google did not submit themselves. 
Since our objective does not include comparing the commercial RBMT systems of companies who did not themselves participate, the System IDs of the commercial RBMT systems are kept anonymous in this paper. 


\begin{table}[b]
\caption{Baseline systems.}
\label{table:baseline_system}
\input{02table03.txt}
\end{table}

Each participant is allowed to submit as many translated results (``runs'') as desired, but the submitted runs should be prioritized by the group.
In this paper, we distinguish their runs using a Run ID expressed by Group ID (or System ID for the baseline systems) and a priority number connected by ``-''.
The resource information used by each run is indicated by 
\begin{itemize} 
\item \textit{Resource B}: The system used the bilingual training data provided by the organizers. 
\item \textit{Resource M}: The system used the monolingual training data provided by the organizers. 
\item \textit{Resource E}: The system used external knowledge other than data provided by the organ\-izers or the system uses a rule-based system.
\end{itemize}






\section{Human evaluation results}
\label{sec:human_evaluation_results}

We evaluated the adequacy for at least all of the first priority submissions. 
However, because of budget limitations, acceptability was evaluated for only selected systems. 


\subsection{Chinese to English}

\subsubsection{Adequacy evaluation}

\begin{table}[b]
\caption{Results of CE adequacy.}
\label{table:results_CE_adequacy}
\input{02table04.txt}
\end{table}

Table \ref{table:results_CE_adequacy} shows the results of the adequacy evaluation. 
Table \ref{table:sign_test_CE_adequacy} shows the results of the statistical significance test of the adequacy evaluation using a sign test.
In the tables showing the results of a statistical significance test, the marks (``$\gg$'', ``$>$'', ``-'') indicate whether the Run ID to the left of a mark is significantly better than that above the mark. 

From these results, we can observe the following:
\begin{itemize}
\item All of the top systems are SMT systems. 
The top system, BBN-1, shows a significantly higher adequacy than the other systems. 
\item The adequacy score for Moses' hierarchical phrase-based SMT system (BASELINE1-1) is higher than that for Moses' phrase-based SMT system (BASELINE2-1). 
\item The adequacy scores for Moses' hierarchical phrase-based SMT system (BASELINE1-1) and Moses' phrase-based SMT system (BASELINE2-1) are higher than those for the two RBMT baseline systems (RBMT2-1 and RBMT1-1).
\end{itemize}

\begin{table}[t]
\hangcaption{Sign test of CE adequacy. ``$\gg$'': significantly different at $\alpha=0.01$, ``$>$'': significantly different at $\alpha=0.05$, ``-'': not significantly different at $\alpha=0.05$.}
\label{table:sign_test_CE_adequacy}
\input{02table05.txt}
\end{table}

To improve translation quality, the top BBN-1 system \cite{BBN:NTCIR9} used the following techniques: generalization of infrequent numerical expressions, optimization of Chinese word segmentation, adaptation of language models, addition of features, and utilization of English dependency structures.
Effectiveness of the system using these techniques was shown.


\subsubsection{Acceptability evaluation}

Table \ref{table:results_CE_acceptability} shows the results of the acceptability evaluation. 
Table \ref{table:sign_test_CE_acceptability} shows the results of the statistical significance test of the acceptability evaluation using a sign test.

\begin{table}[t]
\caption{Results of CE acceptability.}
\label{table:results_CE_acceptability}
\input{02table06.txt}
\end{table}
\begin{table}[t]
\hangcaption{Sign test of CE acceptability. ``$\gg$'': significantly different at $\alpha=0.01$, ``$>$'': significantly different at $\alpha=0.05$, ``-'': not significantly different at $\alpha=0.05$.}
\label{table:sign_test_CE_acceptability}
\input{02table07.txt}
\end{table}

From the results, we can see that the meaning of the source language could be understood (C-rank and above) for 79.7\% of the translated sentences in the best-ranked system (BBN-1). 
This result significantly surpasses the others.


\subsection{Japanese to English}

\subsubsection{Adequacy evaluation}

Table \ref{table:results_JE_adequacy} shows the results of the adequacy evaluation. 
Table \ref{table:sign_test_JE_adequacy} shows the results of the statistical significance test of the adequacy evaluation using a sign test.

\begin{table}[b]
\caption{Results of JE adequacy.}
\label{table:results_JE_adequacy}
\input{02table08.txt}
\end{table}

The top five systems, JAPIO-1, RBMT1-1, EIWA-1, RBMT3-1, and RBMT2-1, are either commercial RBMT systems or systems that use commercial RBMT systems.
From these results, the following are observed:
\begin{itemize}
\item The commercial RBMT systems had higher adequacies than the state-of-the-art SMT systems for patent machine translation from Japanese to English.
\item The adequacy score for Moses' hierarchical phrase-based SMT (BASELINE1-1) is slightly higher than that for Moses' phrase-based SMT (BASELINE2-1).
\end{itemize}

The reason that the SMT systems could not achieve adequacy scores as high as those from the top RBMT systems is thought to be because of word ordering.  
Since the word order in Japanese and English is significantly different (Japanese is a Subject-Object-Verb (SOV) language and English is a Subject-Verb-Object (SVO) language), word ordering is difficult for Japanese--English translation.
The current SMT performs well for word selection, but not for difficult word ordering of Japanese--English translation. 
On the other hand, the baseline commercial RBMT systems perform well for difficult word ordering of Japanese--English translations. 

\begin{table}[t]
\hangcaption{Sign test of JE adequacy. ``$\gg$'': significantly different at $\alpha=0.01$, ``$>$'': significantly different at $\alpha=0.05$, ``-'': not significantly different at $\alpha=0.05$.}
\label{table:sign_test_JE_adequacy}
\input{02table09.txt}
\end{table}

The results showing that RBMT systems were better than SMT systems were the same as the previous human evaluation results at NTCIR-7 \cite{Fujii-EtAl:NTCIR7}. 


\subsubsection{Acceptability evaluation}

Table \ref{table:results_JE_acceptability} shows the results of the acceptability evaluation. 
Table \ref{table:sign_test_JE_acceptability} shows the results of the statistical significance test of the acceptability evaluation using a sign test.

From the results, we can see that the source sentence meaning could be understood (C-rank and above) for 63\% of the sentences in the best-ranked system using RBMT (JAPIO-1). 
For the best-ranked SMT system (NTT-UT-1), the source sentence meaning could be understood for 25\% of the translated sentences (C-rank and above).

\begin{table}[t]
\caption{Results of JE acceptability.}
\label{table:results_JE_acceptability}
\input{02table10.txt}
\end{table}
\begin{table}[t]
\hangcaption{Sign test of JE acceptability. ``$\gg$'': significantly different at $\alpha=0.01$, ``$>$'': significantly different at $\alpha=0.05$, ``-'': not significantly different at $\alpha=0.05$.}
\label{table:sign_test_JE_acceptability}
\input{02table11.txt}
\end{table}

There was a large difference in the ability to retain the sentence-level meanings between the top-level commercial RBMT systems and the SMT systems for Japanese-to-English patent translation.


\subsection{English to Japanese}

\subsubsection{Adequacy evaluation}

Table \ref{table:results_EJ_adequacy} shows the results of the adequacy evaluation. 
Table \ref{table:sign_test_EJ_adequacy} shows the results of the statistical significance test of the adequacy evaluation using a sign test.

\begin{table}[p]
\caption{Results of EJ adequacy.}
\label{table:results_EJ_adequacy}
\input{02table12.txt}
\end{table}
\begin{table}[p]
\hangcaption{Sign test of EJ adequacy. ``$\gg$'': significantly different at $\alpha=0.01$, ``$>$'': significantly different at $\alpha=0.05$, ``-'': not significantly different at $\alpha=0.05$.}
\label{table:sign_test_EJ_adequacy}
\input{02table13.txt}
\end{table}


NTT-UT-1 and NTT-UT-3 are the top systems for the SMT systems and RBMT6-1, JAPIO-1, RBMT4-1, and RBMT5-1 are the top RBMT systems.
From these results, the following are observed: 
\begin{itemize}
\item The top SMT systems NTT-UT-1 and NTT-UT-3 achieved human 
evaluation scores (adequacy) that were equal to or better than the top-level commercial RBMT systems. 
This was not the case for any SMT system at NTCIR-7, and it is believed to be the first time that this is being achieved. 
\item The adequacy scores for the commercial RBMT systems were higher than those for SMT systems other than NTT-UT-1 and NTT-UT-3.
\end{itemize}

English-to-Japanese translation is difficult for SMT because the English and Japanese word order is significantly different.
However, the top SMT systems achieved results equal to or better than the RBMT systems. 
There was one feature in the top SMT systems that improved translation quality. 
This feature, used in NTT-UT-1 and NTT-UT-3 \cite{NTT-UT:NTCIR9}
\footnote{The NTT-UT system for Japanese-to-English translation also used a pre-ordering method. 
However, the system did not achieve an improvement as in English-to-Japanese translation because the pre-ordering method was different from the pre-ordering method for English-to-Japanese translation.
The head finalization rules cannot be applied to Japanese-to-English translation since English is not a head final language.}, is that the systems utilize a method that pre-orders English input sentences using parse results and head finalization rules \cite{isozaki-EtAl:2010:WMT} and then translates in almost monotone word orders. 
Since NTT-UT-1 uses a combination of three MT systems (two pre-ordering systems and one forest-to-string system), the effectiveness of the pre-ordering method was not clear from just the NTT-UT-1 evaluation result. 
However, NTT-UT-3 consisted of only an MT system with the pre-ordering method using the head finalization rules. 
This allowed the effectiveness of the pre-ordering method using the head finalization rules to be seen from the results. 


\subsubsection{Acceptability evaluation}

Table \ref{table:results_EJ_acceptability} shows the results of the acceptability evaluation. 
Table \ref{table:sign_test_EJ_acceptability} shows the results of the statistical significance test of the acceptability evaluation using a sign test.

\begin{table}[t]
\caption{Results of EJ acceptability.}
\label{table:results_EJ_acceptability}
\input{02table14.txt}
\end{table}
\begin{table}[t]
\hangcaption{Sign test of EJ acceptability. ``$\gg$'': significantly different at $\alpha=0.01$, ``$>$'': significantly different at $\alpha=0.05$, ``-'': not significantly different at $\alpha=0.05$.}
\label{table:sign_test_EJ_acceptability}
\input{02table15.txt}
\end{table}

For the best SMT system (NTT-UT-1), the source sentence meaning could be understood (C and above) for 60\% of the sentences.
Of the systems using RBMT, the source sentence meaning could be understood (C or above) for 60\% of the translated sentences in the best system (RBMT6-1).

The translation quality of the top SMT system (NTT-UT-1) was equal to or better than that of the top-level commercial RBMT systems for retaining the sentence-level meanings. 


\section{Validation of Human Evaluation Results}
\label{sec:validation_human_evaluation}

To discuss reliability of the human evaluation, we present the correlation between the evaluation results for divided data.
We validated the reliability of human evaluation as follows: 
\begin{enumerate}
\item The human evaluation data was divided into the first half data (Half-1) and the second half data (Half-2). 
Each contains half of all of the sentences evaluated by each evaluator.
\item Scores for the systems based on the halved data were calculated. 
\item Correlation of system comparisons between the halved data was calculated. 
\end{enumerate}
Since the test data were built by random selection, it is assumed that the evaluation is not affected by differences in the halved data. 
Under this assumption, the following is true: 
If the evaluation is reliable, the top systems based on the first half data will also be the top systems based on the second half data, and the lower-ranking systems based on the first half data will also be the lower-ranking systems based on the second half data, i.e., there is good correlation between system comparison results of the two halved data. 
On the other hand, if the evaluation is not reliable, the top systems based on the first half data would be the lower-ranking systems based on the second half data, or the lower-ranking systems based on the first half data would be the top systems based on the second half data, i.e., there is poor correlation between system comparison results of the two halved data.
Therefore, we validated the reliability based on the correlation between the evaluation results for the divided data.
In this section, pairwise scores for systems were used for normalization purposes.
A pairwise score for a system reflects the frequency with which it was judged to be better than or equal to other systems. 
A detailed explanation of the pairwise score is given in Section \ref{sec:acceptability}. 

Figures \ref{fig:comparison_data_CE_adequacy}--\ref{fig:comparison_data_EJ_acceptability} show the evaluation results for the first half of the data (Half-1), the second half of the data (Half-2), and all of the data (All).
In the figures, the vertical axis is the pairwise score, and the horizontal axis is the Run ID.
Although there are slight differences between the half data, there are no large differences that reverse the high-ranked and low-ranked systems.

Table \ref{table:correlation_between_data} shows the Pearson correlation coefficients of the system evaluation scores between the half data.
The Pearson correlation coefficients are close to 1.0 for all of the data pairs.

\begin{figure}[b]
\setlength{\captionwidth}{192pt}
\begin{minipage}[b]{200pt}
\centerline{\includegraphics{20-1ia964f2.eps}}
\hangcaption{Comparison between data for CE adequacy}
\label{fig:comparison_data_CE_adequacy}
\end{minipage}
\hfill
\begin{minipage}[b]{200pt}
\centerline{\includegraphics{20-1ia964f3.eps}}
\hangcaption{Comparison between data for CE acceptability}
\label{fig:comparison_data_CE_acceptability}
\end{minipage}
\end{figure}
\begin{figure}[t]
\setlength{\captionwidth}{192pt}
\begin{minipage}[b]{200pt}
\centerline{\includegraphics{20-1ia964f4.eps}}
\hangcaption{Comparison between data for JE adequacy}
\label{fig:comparison_data_JE_adequacy}
\end{minipage}
\hfill
\begin{minipage}[b]{200pt}
\centerline{\includegraphics{20-1ia964f5.eps}}
\hangcaption{Comparison between data for JE acceptability}
\label{fig:comparison_data_JE_acceptability}
\end{minipage}
\end{figure}

These indicate that the evaluations of 150 sentences are thought to be consistent for system comparison, and this consistency shows the reliability of the evaluation results.
The evaluation results of 300 sentences are thought to be more reliable than the evaluation results of 150 sentences because the number of sentences is larger.

\begin{figure}[t]
\setlength{\captionwidth}{192pt}
\begin{minipage}[b]{200pt}
\begin{center}
\includegraphics{20-1ia964f6.eps}
\end{center}
\hangcaption{Comparison between data for EJ adequacy}
\label{fig:comparison_data_EJ_adequacy}
\end{minipage}
\hfill
\begin{minipage}[b]{200pt}
\begin{center}
\includegraphics{20-1ia964f7.eps}
\end{center}
\hangcaption{Comparison between data for EJ acceptability}
\label{fig:comparison_data_EJ_acceptability}
\end{minipage}
\end{figure}
\begin{table}[t]
\caption{Pearson correlation coefficient between data.}
\label{table:correlation_between_data}
\input{02table16.txt}
\end{table}
\begin{table}[t]
\caption{Pearson correlation coefficient between evaluators by different data sets.}
\label{table:correlation_between_evaluators}
\input{02table17.txt}
\end{table}

In addition to the above main validation for reliability, we also checked the differences between evaluators. 
For each subtask and criterion, three evaluators evaluated the translations of 100 different source sentences. 
We checked the correlation between the evaluation results based on the 100 source sentences evaluated by the same evaluator. 
Table \ref{table:correlation_between_evaluators} shows the Pearson correlation coefficients for the system evaluation scores between evaluators.
These values indicate that there is a high correlation between evaluators. 
Thus, even when the evaluators and the data are different, the evaluations are thought to be consistent for system comparison. 


\section{Meta-Evaluation of the Automatic Evaluation Measure of BLEU}
\label{sec:meta-evaluation}

We calculated the BLEU scores based on the 2,000 test sentences to investigate the reliability of the automatic evaluation measure of BLEU \cite{papineni-EtAl:2002:ACL}, which is widely used to evaluate translation quality, in the patent domain for the language pairs of CE, JE, and EJ. 

The Spearman rank-order correlation coefficients and the Pearson correlation coefficients between human evaluations (average adequacy scores) and the BLEU scores are shown in Table \ref{table:correlation_between_adequacy_and_auto}.
From Table \ref{table:correlation_between_adequacy_and_auto}, it can be seen that the BLEU scores have a high correlation with the human evaluation for the CE evaluation, but do not have a high correlation with the human evaluation for the JE and EJ evaluations including RBMT systems.

\begin{table}[t]
\caption{Correlation coefficients between adequacy and the BLEU scores}
\label{table:correlation_between_adequacy_and_auto}
\input{02table18.txt}
\end{table}
\begin{table}[t]
\caption{Correlation coefficients between adequacy and the BLEU scores excluding RBMT systems}
\label{table:correlation_between_adequacy_and_auto_excludingRBMT}
\input{02table19.txt}
\end{table}

The Spearman rank-order correlation coefficients and the Pearson correlation coefficients between human evaluation and the BLEU scores excluding the RBMT systems for JE and EJ are shown in Table \ref{table:correlation_between_adequacy_and_auto_excludingRBMT}.
The correlations excluding RBMT systems for JE and EJ are higher than those including the RBMT systems.
Therefore, the reliability of the BLEU scores of the comparisons between systems without the RBMT systems is higher than that of the BLEU scores of the comparisons between systems including the RBMT systems for the automatic evaluation of the quality of the JE and EJ patent translations.


\section{Method for Obtaining the Database}
\label{sec:obtain_method}

This section explains the method used to obtain resources. 
The available resources for research purposes consist of a human evaluation database, test data, reference data, and submission data (translated data). 
Resources are provided by the NTCIR project\footnote{http://research.nii.ac.jp/ntcir/}. 
The method used to obtain the resources is given at the URLs shown in Table \ref{table:url}.
Applicants are asked to sign a user agreement (memorandum on permission to use) to obtain the resources. 
The use of these resources is free of charge. 

\begin{table}[b]
\caption{URLs for obtaining the database.}
\input{02table20.txt}
\end{table}



\section{Conclusion}
\label{sec:conclusion}

This paper presented information regarding the database of human evaluations from the NTCIR-9 Patent Machine Translation Task and the knowledge obtained from these evaluations. 
The evaluations showed the effectiveness of a number of machine translation systems in the patent translation field. 
Database of human evaluations is valuable for translation quality evaluation research. 
The resources will also be useful for system combination research. 
Resources including the human evaluation database, translation results, and test/reference data are available from the NTCIR project for research purposes.



\acknowledgment

We would like to thank all of the evaluators for constructing the database of human evaluations. 

\bibliographystyle{jnlpbbl_1.5}
\begin{thebibliography}{}

\bibitem[\protect\BCAY{Callison-Burch, Koehn, Monz, Post, Soricut, \BBA\
  Specia}{Callison-Burch et~al.}{2012}]{callisonburch-EtAl:2012:WMT}
Callison-Burch, C., Koehn, P., Monz, C., Post, M., Soricut, R., \BBA\ Specia,
  L. \BBOP 2012\BBCP.
\newblock \BBOQ Findings of the 2012 Workshop on Statistical Machine
  Translation.\BBCQ\
\newblock In {\Bem Proceedings of the Seventh Workshop on Statistical Machine
  Translation}, \mbox{\BPGS\ 10--51}, Montr{\'e}al, Canada. Association for
  Computational Linguistics.

\bibitem[\protect\BCAY{Chang, Huang, Yen, Jiang, Huang, Chang, \BBA\
  Yang}{Chang et~al.}{2011}]{NTHU:NTCIR9}
Chang, J., Huang, S.-T., Yen, H.-C., Jiang, M.-J., Huang, C.-C., Chang, J.~S.,
  \BBA\ Yang, P.-C. \BBOP 2011\BBCP.
\newblock \BBOQ {[PatentMT] Summary Report of Team III\_CYUT\_NTHU}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 684--688}.

\bibitem[\protect\BCAY{Chao \BBA\ Li}{Chao \BBA\ Li}{2011}]{BUAA:NTCIR9}
Chao, W.\BBACOMMA\ \BBA\ Li, Z. \BBOP 2011\BBCP.
\newblock \BBOQ {ZZX\_MT: the BeiHang MT System for NTCIR-9 PatentMT
  Task}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 629--633}.

\bibitem[\protect\BCAY{Ehara}{Ehara}{2011}]{EIWA:NTCIR9}
Ehara, T. \BBOP 2011\BBCP.
\newblock \BBOQ {Machine translation system for patent documents combining
  rule-based translation and statistical post-editing applied to the PatentMT
  Task}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 623--628}.

\bibitem[\protect\BCAY{Feng, Schmidt, Wuebker, Peitz, Freitag, \BBA\ Ney}{Feng
  et~al.}{2011}]{RWTH:NTCIR9}
Feng, M., Schmidt, C., Wuebker, J., Peitz, S., Freitag, M., \BBA\ Ney, H. \BBOP
  2011\BBCP.
\newblock \BBOQ {The RWTH Aachen System for NTCIR-9 PatentMT}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 600--605}.

\bibitem[\protect\BCAY{Fujii, Utiyama, Yamamoto, \BBA\ Utsuro}{Fujii
  et~al.}{2008}]{Fujii-EtAl:NTCIR7}
Fujii, A., Utiyama, M., Yamamoto, M., \BBA\ Utsuro, T. \BBOP 2008\BBCP.
\newblock \BBOQ {Overview of the Patent Translation Task at the NTCIR-7
  Workshop}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-7}, \mbox{\BPGS\ 389--400}.

\bibitem[\protect\BCAY{Fujii, Utiyama, Yamamoto, Utsuro, Ehara, Echizen-ya,
  \BBA\ Shimohata}{Fujii et~al.}{2010}]{Fujii-EtAl:NTCIR8}
Fujii, A., Utiyama, M., Yamamoto, M., Utsuro, T., Ehara, T., Echizen-ya, H.,
  \BBA\ Shimohata, S. \BBOP 2010\BBCP.
\newblock \BBOQ {Overview of the Patent Translation Task at the {NTCIR-8}
  Workshop}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-8}, \mbox{\BPGS\ 371--376}.

\bibitem[\protect\BCAY{Goto, Lu, Chow, Sumita, \BBA\ Tsou}{Goto
  et~al.}{2011}]{Goto-EtAl:NTCIR9}
Goto, I., Lu, B., Chow, K.~P., Sumita, E., \BBA\ Tsou, B.~K. \BBOP 2011\BBCP.
\newblock \BBOQ {Overview of the Patent Translation Task at the {NTCIR-9}
  Workshop}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 559--578}.

\bibitem[\protect\BCAY{He, Shi, \BBA\ Wang}{He et~al.}{2011}]{ISTIC:NTCIR9}
He, Y., Shi, C., \BBA\ Wang, H. \BBOP 2011\BBCP.
\newblock \BBOQ {ISTIC Statistical Machine Translation System for Patent
  Machine Translation in NTCIR-9}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 634--637}.

\bibitem[\protect\BCAY{Hoang, Koehn, \BBA\ Lopez}{Hoang
  et~al.}{2009}]{iwslt09:TP:hoang}
Hoang, H., Koehn, P., \BBA\ Lopez, A. \BBOP 2009\BBCP.
\newblock \BBOQ {A Unified Framework for Phrase-Based, Hierarchical, and
  Syntax-Based Statistical Machine Translation}.\BBCQ\
\newblock In {\Bem Proceedings of the International Workshop on Spoken Language
  Translation}, \mbox{\BPGS\ 152--159}.

\bibitem[\protect\BCAY{Isozaki, Hirao, Duh, Sudoh, \BBA\ Tsukada}{Isozaki
  et~al.}{2010a}]{isozaki-EtAl:2010:EMNLP}
Isozaki, H., Hirao, T., Duh, K., Sudoh, K., \BBA\ Tsukada, H. \BBOP 2010a\BBCP.
\newblock \BBOQ {Automatic Evaluation of Translation Quality for Distant
  Language Pairs}.\BBCQ\
\newblock In {\Bem Proceedings of the 2010 Conference on Empirical Methods in
  Natural Language Processing}, \mbox{\BPGS\ 944--952}.

\bibitem[\protect\BCAY{Isozaki, Sudoh, Tsukada, \BBA\ Duh}{Isozaki
  et~al.}{2010b}]{isozaki-EtAl:2010:WMT}
Isozaki, H., Sudoh, K., Tsukada, H., \BBA\ Duh, K. \BBOP 2010b\BBCP.
\newblock \BBOQ Head Finalization: A Simple Reordering Rule for SOV
  Languages.\BBCQ\
\newblock In {\Bem Proceedings of the Joint Fifth Workshop on Statistical
  Machine Translation and MetricsMATR}, \mbox{\BPGS\ 244--251}, Uppsala,
  Sweden. Association for Computational Linguistics.

\bibitem[\protect\BCAY{Jiang, Xu, Lin, \BBA\ Zhang}{Jiang
  et~al.}{2011}]{BJTUX:NTCIR9}
Jiang, J., Xu, J., Lin, Y., \BBA\ Zhang, Y. \BBOP 2011\BBCP.
\newblock \BBOQ {System Description of BJTU-NLP SMT for NTCIR-9
  PatentMT}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 638--640}.

\bibitem[\protect\BCAY{Koehn, Hoang, Birch, Callison-Burch, Federico, Bertoldi,
  Cowan, Shen, Moran, Zens, Dyer, Bojar, Constantin, \BBA\ Herbst}{Koehn
  et~al.}{2007}]{koehn-EtAl:2007:PosterDemo}
Koehn, P., Hoang, H., Birch, A., Callison-Burch, C., Federico, M., Bertoldi,
  N., Cowan, B., Shen, W., Moran, C., Zens, R., Dyer, C., Bojar, O.,
  Constantin, A., \BBA\ Herbst, E. \BBOP 2007\BBCP.
\newblock \BBOQ {Moses: Open Source Toolkit for Statistical Machine
  Translation}.\BBCQ\
\newblock In {\Bem Proceedings of the 45th Annual Meeting of the Association
  for Computational Linguistics Companion Volume Proceedings of the Demo and
  Poster Sessions}, \mbox{\BPGS\ 177--180}.

\bibitem[\protect\BCAY{Kondo, Komachi, Matsumoto, Sudoh, Duh, \BBA\
  Tsukada}{Kondo et~al.}{2011}]{NAIST:NTCIR9}
Kondo, S., Komachi, M., Matsumoto, Y., Sudoh, K., Duh, K., \BBA\ Tsukada, H.
  \BBOP 2011\BBCP.
\newblock \BBOQ {Learning of Linear Ordering Problems and its Application to
  J-E Patent Translation in NTCIR-9 PatentMT}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 641--645}.

\bibitem[\protect\BCAY{Lee, Xiang, Zhao, Franz, Roukos, \BBA\ Al-Onaizan}{Lee
  et~al.}{2011}]{IBM:NTCIR9}
Lee, Y.-S., Xiang, B., Zhao, B., Franz, M., Roukos, S., \BBA\ Al-Onaizan, Y.
  \BBOP 2011\BBCP.
\newblock \BBOQ {IBM Chinese-to-English PatentMT System for NTCIR-9}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 606--613}.

\bibitem[\protect\BCAY{Lin \BBA\ Hovy}{Lin \BBA\
  Hovy}{2003}]{lin-hovy:2003:NAACL}
Lin, C.-Y.\BBACOMMA\ \BBA\ Hovy, E. \BBOP 2003\BBCP.
\newblock \BBOQ {Automatic Evaluation of Summaries Using N-gram Co-occurrence
  Statistics}.\BBCQ\
\newblock In {\Bem Proceedings of the 2003 Human Language Technology Conference
  of the North American Chapter of the Association for Computational
  Linguistics}, \mbox{\BPGS\ 71--78}.

\bibitem[\protect\BCAY{Lu, Tsou, Jiang, Kwong, \BBA\ Zhu}{Lu
  et~al.}{2010}]{Lu-EtAl:CLP2010}
Lu, B., Tsou, B.~K., Jiang, T., Kwong, O.~Y., \BBA\ Zhu, J. \BBOP 2010\BBCP.
\newblock \BBOQ {Mining Large-scale Parallel Corpora from Multilingual Patents:
  An English-Chinese example and its application to SMT}.\BBCQ\
\newblock In {\Bem Proceedings of the 1st CIPS-SIGHAN Joint Conference on
  Chinese Language Processing (CLP-2010)}, Beijing, China.

\bibitem[\protect\BCAY{Ma \BBA\ Matsoukas}{Ma \BBA\
  Matsoukas}{2011}]{BBN:NTCIR9}
Ma, J.\BBACOMMA\ \BBA\ Matsoukas, S. \BBOP 2011\BBCP.
\newblock \BBOQ {BBN's Systems for the Chinese-English Sub-task of NTCIR-9
  Patent MT Evaluation}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 579--584}.

\bibitem[\protect\BCAY{Murakami \BBA\ Tokuhisa}{Murakami \BBA\
  Tokuhisa}{2011}]{TORI:NTCIR9}
Murakami, J.\BBACOMMA\ \BBA\ Tokuhisa, M. \BBOP 2011\BBCP.
\newblock \BBOQ {Statistical Machine Translation with Rule based Machine
  Translation}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 646--651}.

\bibitem[\protect\BCAY{Na, Li, Kim, \BBA\ Lee}{Na et~al.}{2011}]{KLE:NTCIR9}
Na, H., Li, J.-J., Kim, S.-J., \BBA\ Lee, J.-H. \BBOP 2011\BBCP.
\newblock \BBOQ {POSTECH's Statistical Machine Translation Systems for the
  NTCIR-9 PatentMT Task (English-to-Japanese)}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 652--656}.

\bibitem[\protect\BCAY{Nakazawa \BBA\ Kurohashi}{Nakazawa \BBA\
  Kurohashi}{2011}]{KYOTO:NTCIR9}
Nakazawa, T.\BBACOMMA\ \BBA\ Kurohashi, S. \BBOP 2011\BBCP.
\newblock \BBOQ {EBMT System of KYOTO Team in PatentMT Task at NTCIR-9}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 657--660}.

\bibitem[\protect\BCAY{Ohio, Mitsuhashi, \BBA\ Kakita}{Ohio
  et~al.}{2011}]{JAPIO:NTCIR9}
Ohio, T., Mitsuhashi, T., \BBA\ Kakita, T. \BBOP 2011\BBCP.
\newblock \BBOQ {Use of the Japio Technical Field Dictionaries for
  NTCIR-PatentMT}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 614--617}.

\bibitem[\protect\BCAY{Papineni, Roukos, Ward, \BBA\ Zhu}{Papineni
  et~al.}{2002}]{papineni-EtAl:2002:ACL}
Papineni, K., Roukos, S., Ward, T., \BBA\ Zhu, W.-J. \BBOP 2002\BBCP.
\newblock \BBOQ {Bleu: a Method for Automatic Evaluation of Machine
  Translation}.\BBCQ\
\newblock In {\Bem Proceedings of 40th Annual Meeting of the Association for
  Computational Linguistics}, \mbox{\BPGS\ 311--318}.

\bibitem[\protect\BCAY{Pierce, Carroll, Hamp, Hays, Hockett, Oettinger, \BBA\
  Perlis}{Pierce et~al.}{1966}]{Pierce-EtAl:ALPAC-report:1966}
Pierce, J.~R., Carroll, J.~B., Hamp, E.~P., Hays, D.~G., Hockett, C.~F.,
  Oettinger, A.~G., \BBA\ Perlis, A. \BBOP 1966\BBCP.
\newblock \BBOQ Language and Machines: Computers in Translation and
  Linguistics.\BBCQ\ ALPAC report, National Academy of Sciences, National
  Research Council, Washington, DC.

\bibitem[\protect\BCAY{Schwenk \BBA\ Abdul-Rauf}{Schwenk \BBA\
  Abdul-Rauf}{2011}]{LIUM:NTCIR9}
Schwenk, H.\BBACOMMA\ \BBA\ Abdul-Rauf, S. \BBOP 2011\BBCP.
\newblock \BBOQ {LIUM's Statistical Machine Translation System for the NTCIR
  Chinese/English Patent Translation Task}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 618--622}.

\bibitem[\protect\BCAY{Sudoh, Duh, Tsukada, Nagata, Wu, Matsuzaki, \BBA\
  Tsujii}{Sudoh et~al.}{2011}]{NTT-UT:NTCIR9}
Sudoh, K., Duh, K., Tsukada, H., Nagata, M., Wu, X., Matsuzaki, T., \BBA\
  Tsujii, J. \BBOP 2011\BBCP.
\newblock \BBOQ {NTT-UT Statistical Machine Translation in NTCIR-9
  PatentMT}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 585--592}.

\bibitem[\protect\BCAY{Tseng, Liu, Tsai, Wang, Chuang, \BBA\ Jeng}{Tseng
  et~al.}{2011}]{NCW:NTCIR9}
Tseng, Y.-H., Liu, C.-L., Tsai, C.-C., Wang, J.-P., Chuang, Y.-H., \BBA\ Jeng,
  J. \BBOP 2011\BBCP.
\newblock \BBOQ {Statistical Approaches to Patent Translation for PatentMT -
  Experiments with Various Settings of Training Data}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 661--665}.

\bibitem[\protect\BCAY{Utiyama \BBA\ Isahara}{Utiyama \BBA\
  Isahara}{2007}]{Utiyama-Isahara:MTS2007}
Utiyama, M.\BBACOMMA\ \BBA\ Isahara, H. \BBOP 2007\BBCP.
\newblock \BBOQ {A Japanese-English patent parallel corpus}.\BBCQ\
\newblock In {\Bem Proceedings of Machine Translation Summit XI}, Copenhagen,
  Denmark.

\bibitem[\protect\BCAY{White, O'Connell, \BBA\ O'Mara}{White
  et~al.}{1994}]{White-EtAl:AMTA1994}
White, J.~S., O'Connell, T.~A., \BBA\ O'Mara, F.~E. \BBOP 1994\BBCP.
\newblock \BBOQ {The ARPA MT Evaluation Methodologies: Evolution, Lessons, and
  Future Approaches}.\BBCQ\
\newblock In {\Bem Proceedings of AMTA}.

\bibitem[\protect\BCAY{Wu, Matsuzaki, \BBA\ Tsujii}{Wu
  et~al.}{2011}]{UOTTS:NTCIR9}
Wu, X., Matsuzaki, T., \BBA\ Tsujii, J. \BBOP 2011\BBCP.
\newblock \BBOQ {SMT Systems in the University of Tokyo for NTCIR-9
  PatentMT}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 666--672}.

\bibitem[\protect\BCAY{Xiao, Li, Lu, Zhang, Ding, Yao, Xu, Fei, Zhu, Ren, \BBA\
  Wang}{Xiao et~al.}{2011}]{NEU:NTCIR9}
Xiao, T., Li, Q., Lu, Q., Zhang, H., Ding, H., Yao, S., Xu, X., Fei, X., Zhu,
  J., Ren, F., \BBA\ Wang, H. \BBOP 2011\BBCP.
\newblock \BBOQ {The NiuTrans Machine Translation System for NTCIR-9
  PatentMT}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 593--599}.

\bibitem[\protect\BCAY{Xiong, Song, Meng, L\"{u}, \BBA\ Liu}{Xiong
  et~al.}{2011}]{ICT:NTCIR9}
Xiong, H., Song, L., Meng, F., L\"{u}, Y., \BBA\ Liu, Q. \BBOP 2011\BBCP.
\newblock \BBOQ {The ICT's Patent MT System Description for NTCIR-9}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 673--678}.

\bibitem[\protect\BCAY{Zheng, Ge, Meng, \BBA\ Yu}{Zheng
  et~al.}{2011}]{FRDC:NTCIR9}
Zheng, Z., Ge, N., Meng, Y., \BBA\ Yu, H. \BBOP 2011\BBCP.
\newblock \BBOQ {HPB SMT of FRDC Assisted by Paraphrasing for the NTCIR-9
  PatentMT}.\BBCQ\
\newblock In {\Bem Proceedings of NTCIR-9}, \mbox{\BPGS\ 679--683}.

\end{thebibliography}



\appendix

\section{Adequacy Criterion}

\subsection{Instructions for the Adequacy Criterion}
\label{sec:apn:adequacy}

\subsubsection{Evaluation Criterion}

\begin{table}[b]
\caption{Adequacy criterion.}
\input{02table21.txt}
\end{table}

Adequacy is scored according to how well the meaning of a translation matches the meaning of the reference (source) translation for each sentence. 
Adequacy evaluations are done according to the 5-level scale shown in Table \ref{table:adequacy_criterion}.


\subsubsection{Notes}

\begin{enumerate}
\item Adequacy estimates the sentence meaning by evaluating fragments of a sentence. 
\item The main reason for using fragments is to reduce evaluation costs. When sentences are long, fragment-level evaluation is easier than sentence-level ones.
\item Fragment size:
\begin{enumerate}
\item Clause-level (first priority) or 
\item ``subject and its predicate'' level (second priority) or 
\item phrase-level (third priority). 
\end{enumerate}
\item Supplementary definitions to reduce criterion ambiguity:
\begin{enumerate}
\item A score of 5 indicates that the sentence-level meaning (subject, predicate and object) is correct.
\item Relative comparison: 
\begin{itemize}
\item A sentence whose sentence-level meaning is not correct would be evaluated as 1--4 not only by the absolute criterion (most, much, little, and none) but also a relative comparison among the multiple translation outputs. 
\item The relative comparison must be consistent in all of the data.
\end{itemize}
\end{enumerate}
\end{enumerate}


\subsection{Example Values of Adequacy}
\label{sec:apn:adq_example}

Examples of adequacy values and translations are shown in Table \ref{table:adq_example}.

\begin{table}[b]
\caption{Examples of adequacy values and translations.}
\label{table:adq_example}
\input{02table22.txt}
\end{table}


\section{Acceptability Criterion}

\subsection{Instructions for the Acceptability Criterion}
\label{sec:apn:acceptability}

\subsubsection{Evaluation Criterion}

Acceptability evaluations are done using the 5-level scale in Figure \ref{fig:acceptability_criterion}.

\subsubsection{Notes}
\begin{enumerate}
\item Evaluations are performed from the perspective of whether the machine-translated English sentence conveys the important information and the content of the source sentence and not on the completeness of a literal translation.
\item What is ``important information''?
``Important information'' is the information that is necessary for a conversation between two people. This information is what needs to be conveyed by the machine translation results for the conversation partner to understand the content of the source sentence.
\item What does ``contents of the source sentence can be understood'' mean?
It refers to when two people can begin a conversation and the machine-translated results allow the conversation partner to understand the contents of the conversation.
\item The first step and the second step of the chart can be merged; therefore, ``F'' means that either not all of the important information is included or the contents from the source sentence cannot be understood.
\item The level of correctness for the ``Grammatically correct'' step indicates whether the translation is grammatical enough to convey the meaning of the source sentence. Strict adequateness (e.g., Editor's emendation level) for each expression is not required here. Therefore, if there are sentences that include expressions which cannot be considered to fully express the patent or technological terms, but the meaning itself is expressed, then it can be evaluated as A.
\item On the ``Native level'' step, natural English sentences that do not need any correction are to be evaluated as AA. Therefore, all minimum required grammatical check points (including punctuation) for a natural English sentence are needed. 
\item If there is a sentence in unnatural English that lacks a subject (nominative), and if the sentence could be easily understood and is grammatically correct if it were transformed from the active sentence to the passive voice, it can be evaluated as ``B,'' as the sentence is grammatically incorrect. 
\item The following type of differences is permissible:
The character is the same but the character code is not the same.
e.g., ``１２３'' and ``123'' are considered to be the same.
\item Special characters such as Greek letters in the source sentences are replaced as letters enclosed by periods or enclosed by ampersands and semicolons. These replacements are permissible. 
e.g., ``５μｍ'' → ``5 .mu.m'' or ``5 \&mu;m''
\item Some translations mistakenly include segments of characters from the source language. These segments are ignored if the translation works out appropriately without the segments. 
\end{enumerate}


\subsection{Example Values of Acceptability}
\label{sec:apn:acp_example}

\begin{table}[b]
\vspace{-0.5\Cvs}
\caption{Examples of acceptability values and translations.}
\input{02table23.txt}
\end{table}

Examples of acceptability values and translations are shown in Table \ref{table:acp_example}.


\section{Instructions for the Human Evaluation Procedure}
\label{sec:apn:eval_procedure}


\subsection{Evaluation Method for Training and Main Evaluations}
\begin{itemize}
\item The criteria for evaluation are based on the guidelines.
\item One input sentence (or one reference sentence) and all of the system outputs are shown simultaneously to compare systems.
\item An evaluator evaluates all of the translations for the same input sentence. 
\item The MT output sentences for each input sentence are given to the evaluators in a random order.
\item The evaluators can review the evaluations.
\end{itemize}


\subsection{Training}

Before the main evaluation, a trial evaluation is done. 
All of the evaluators evaluate translation results for the trial evaluations. 
The conditions for all evaluators are the same. 
After the trial evaluation, a consensus meeting is held to make corrections to the differences in the evaluations obtained from all of the evaluators and to decide on common evaluations for the translation results for the trial evaluation.


\vspace{3\Cvs}
\begin{biography}

\bioauthor[:]{Isao Goto}{
received an M.E. in Electrical Engineering from Waseda University in 1997.  
He is a Research Expert at the National Institute of Information and Communications Technology (NICT). 
His research interests include machine translation.
}
\bioauthor[:]{Bin Lu}{
received an M.S. in Computer Science from Peking University in 2007.
He is currently a PhD candidate at the Department of Chinese, Translation
and Linguistics, City University of Hong Kong. He has published more than
20 academic papers in the area of Natural Language Processing and
Computational Linguistics and holds two China patents. His research
focuses on Statistical Machine Translation (especially in the patent
domain) and Sentiment Analysis and Opinion Mining.
}
\bioauthor[:]{Ka Po Chow}{
received an MPhil in Electrical and Electronic Engineering in 2001. He is currently a project officer at the Research Centre on Linguistics and Language Information Sciences, Hong Kong Institute of Education, managing research projects and developing software in various areas including computational linguistics, multilingual sentence alignment, statistical machine translation and sentiment analysis. He has published a number of papers on parallel computing and natural language processing.
}
\bioauthor[:]{Eiichiro Sumita}{
received an M.S. in Computer Science
  from the University of Electro-Communications in 1982 and 
  a Ph.D. in Engineering from Kyoto University in 1999.
  Dr. Sumita is the Director of the NICT Multilingual Trans-\linebreak[2]lation Laboratory.
  His research interests include machine translation and \linebreak e-Learning.
}
\bioauthor[:]{Benjamin K. Tsou}{
received the MA degree from Harvard University, Cambridge, Massachusetts, and the Ph.D. degree from the University of California, Berkeley. He is a member of the Royal Academy of Overseas Sciences of Belgium and the chair professor of Linguistics and Asian Languages, as well as the director of the Language Information Sciences Research Center of the City University of Hong Kong. Since 1995, he has developed and cultivated the
largest (350 million characters, 1.5 million word types by 2008) synchronous corpus of Chinese LIVAC (http://www.livac.org), which makes unique provisions for monitoring linguistic and related trends for application in NLP. He has authored several books and monographs, and more than 100 articles, mostly on computational linguistics and Chinese linguistics. He has served on the editorial boards of Natural Language Processing (Japan), the International Journal of Computational Linguistics and Chinese Language Processing (IJCLCLP) (Taipei), the International Journal of Computer Processing of Oriental Languages (Singapore), and the monograph series on natural language processing from John Benjamins (Amsterdam), among others. His current research interests include sentiment analysis, computational lexicography and lexicology, and resource developing and natural language processing. He is the founding president of the Asian Federation of Natural Language Processing (AFNLP) and was the chairman of SIGHAN, ACL. He also serves on the Executive Board of the Chinese Information Processing Society of China. He is a member of the IEEE.
}
\bioauthor[:]{Masao Utiyama}{
received a B.S. in Computer Science
  from the University of Tsukuba, Japan in 1992, an M.S. in 
  Computer Science from the University of Tsukuba in 1994, a Ph.D. 
  in Engineering from the University of Tsukuba in 1997.  From 1997 to
  1999, he was a Research Associate at Shinshu University,
  Japan. He has been a member of the National Institute of Information and
  Communications Technology (NICT), Japan since 1999. He is a Senior
  Researcher at NICT.
}
\bioauthor[:]{Keiji Yasuda}{
received an M.E. and a Ph.D. from Doshisha University in 2001 
and 2004, respectively. He is currently a Senior Researcher at the National Institute of 
Communications Technology (NICT). His research interests include speech 
recognition, natural language processing, and e-Learning. He received a 
society paper award from the IEICE-ISS in 2006. He is a member of IEICE, 
IPSJ, and ASJ.
}
\end{biography}


\biodate



\end{document}
