    \documentclass[english]{jnlp_1.3e}

\usepackage[dvips]{graphicx}
\usepackage{jnlpbbl_1.1}
\usepackage{hangcaption_jnlp}
\setlength{\captionwidth}{\textwidth}

\Volume{14}
\Number{5}
\Month{Oct.}
\Year{2007}
\received{2006}{5}{22}
\revised{2006}{7}{18}
\accepted{2007}{5}{21}

\setcounter{page}{3}

\etitle{A Study of the Position of Discourse Markers: Focusing on the Texts Whose Target Audience Was Intermediate Non-native Speakers of English}
\eauthor{Xinyu Deng\affiref{KyotoUniv} \and  Jun-ichi Nakamura} 
\eabstract{
As an international language, English has become more and more important for
non-native speakers. Therefore, writers ought to consider the
needs of non-native speakers, i.e. write English in a way that can be
understood quite well by non-native audience. In this paper, we investigate
the position of six discourse markers within the texts whose target audience 
was intermediate non-native speakers of English. The six discourse markers 
are: \textit{because} and \textit{since}, which represent ``reason'' relation; \textit{if} and
\textit{when}, which represent ``condition'' relation; \textit{although} and
\textit{while}, which represent ``concession''/``contrast'' relation. 
First, we created a corpus (200,000 words) containing the texts 
(domain: \textit{natural and pure science}) whose target audience was 
intermediate non-native speakers. We selected 1072 examples of the six
discourse markers from the corpus, and annotated them. Second,
a machine learning program C4.5 was applied to induce the 
classification models of the position of the discourse markers.
And then we used Support Vector Machine (SVM) to verify 
the experiment results of C4.5. To our knowledge, 
this study is the first one on exploring the position of
discourse markers within the texts whose target audience was intermediate
non-native speakers. The experiment results can be applied to text generation
and homepage creation for intermediate non-native speakers of English.
}
\ekeywords{discourse marker, position, intermediate non-native speakers of English, machine learning program C4.5, Support Vector Machine (SVM)}

\headauthor{Deng, Nakamura}
\headtitle{A Study of the Position of Discourse Markers}

\affilabel{KyotoUniv}{
	Kyoto University, Graduate School of Informatics}{Kyoto University, Graduate School of Informatics}


\begin{document}

\maketitle


\section{Background} 

It is estimated that the world has about 375 million people who speak
English as a first language, and about 750 million more people
who speak English as a foreign language. In natural language processing, 
although some text generation systems \cite{McKeown85,Goldberg94,Bateman97} 
had been developed, their target users were native speakers of English. 
Since the population of non-native speakers is huge, text generation
for non-native speakers whose reading ability is lower is a promising 
application area. Now, we are developing a text generation system
(the SILK system) that can generate texts (domain: \textit{natural and pure science})
appropriate for non-native users on discourse level. In this study, non-native
speakers are divided into three levels (see Section 2.2 for details): primary
(middle school level), intermediate (high school level) and advanced (university level).
The users of the SILK system are assumed to be at intermediate level. 

While writing an article, authors often signal the relation between 
discourses by providing explicit discourse markers, such as 
\textit{because, for example}, and \textit{if}. Discourse markers
play an important role in keeping the coherence of texts. 
Therefore, the research concerned with discourse 
markers is one of the important issues in natural language generation. 
In \cite{Eugenio97}, the authors mention three aspects which need to be 
considered while generating discourse markers: 
\textit{occurrence}, whether to generate a discourse marker or not; 
\textit{position}, where to place the discourse marker; 
and \textit{selection}, which discourse marker to use. Moreover, 
the study of \cite{Will04} puts forward some factors, 
such as the position of discourse markers, which have influence on readability
at discourse level. As the first step of developing the SILK system, 
this study concentrates on investigating the position of discourse markers 
within the texts whose target audience was intermediate non-native 
speakers. For example, from the viewpoint of text generation, text (a) and
(b) below have the same meaning and abstract structure. However, 
the position of discourse marker \textit{if} is different. In text (a), 
\textit{if} is placed in the first span, while in text (b), \textit{if}
is placed in the second span. What we want to know is which text 
is more appropriate for intermediate non-native speakers.

(a)~~You failed the exam. \textit{But if} you study hard, you can master English.

(b)~~You failed the exam. \textit{But} you can master English, \textit{if} you study hard. 

We use machine learning program C4.5 \cite{Quinlan93} to induce the classification models 
of the position of six discourse markers, and then verify the results of 
C4.5 by Support Vector Machine (SVM) \cite{Vapnik99}. The six discourse markers are: 
\textit{because} and \textit{since}, which represent ``reason'' relation; 
\textit{if} and \textit{when}, which represent ``condition'' relation; 
\textit{although} and \textit{while}, which represent ``contrast'' 
relation\footnote{In this study, ``contrast'' relation refers to both ``contrast'' relation and ``concession'' relation.}. 
The rest of the paper is arranged as follows. In Section 2, we describe how to investigate the
position of discourse markers. Section 3 and Section 4 introduce the 
experiment results of C4.5 and SVM respectively. Section 5 discusses the 
results of this study. In Section 6, we introduce the related work. 
Section 7 explains the importance of this study. In Section 8, we draw conclusion.

\section{Investigating the position of discourse markers}  

This section describes how to annotate discourse markers, and the 
experiments of investigating the position of discourse markers.

\subsection{Definition of discourse markers}

Discourse markers function ``with great reliability as a marker of new
information and focus \cite{Underhill88}''. Their role is to indicate how one
piece of discourse is connected to another piece of discourse. In this study, 
we follow \cite{Knott96} (pp.66-67) to classify discourse markers into five syntactic 
categories:

\begin{itemize}

  \item \textbf{Coordinators}. They link the clauses which are equal constituents
of a sentence. They always appear in between the clauses they link. For example:

(c)~~John likes tea \textit{and} his sister likes coffee.

  \item \textbf{Subordinators}. ``They introduce subordinate clauses in complex
sentences. The subordinate clause can be on the left or the right of the main clause,
but the subordinator is always on the left of the subordinate clause \cite{Knott96}''.
For example:

(d)~~He went to school, \textit{although} he had a fever.

  \item \textbf{Conjunct adverbs}. ``They modify whole clauses, and can
appear at different points within them, although there is often a default
position for particular phrases \cite{Knott96}''. For example:

(e)~~I had a cold. \textit{Therefore}, I couldn't go swimming.

  \item \textbf{Prepositional phrases}. ``They often contain propositional
anaphora referring back to the previous clause \cite{Knott96}''. For example:

(f)~~She answered the questions without difficulty. \textit{From then on} she
   knew she would pass the exam.

  \item \textbf{Phrases which take sentential complements}. ``They often
introduce a particular intentional stance with respect to the content of
the clause they introduce \cite{Knott96}.'' For example:

(g)~~On the moon, the sky looks black and the sun white. \textit{This is because}
there is no atmosphere. 

\end{itemize}

Compared with other four syntactic categories, most subordinators have no default
position, so we investigate the position of subordinators. In this study, 
discourse markers refer to subordinators.  

\subsection{Creating corpus TANN}

\textit{ACTFL Proficiency Guidelines}\footnote{http://www.sil.org/lingualinks/languagelearning/OtherResources/ACTFLProficiencyGuidelines/contents.htm} were developed by the American Council for the Teaching of
Foreign Languages (ACTFL). The aim is to provide a means of assessing the proficiency of
a foreign language speaker. Now the guidelines are widely applied to teaching and learning of
foreign languages (include English). In the guidelines, non-native speakers are divided into
five levels: novice, intermediate, advanced, superior and distinguished. In this study, primary, intermediate
and advanced refer to novice, intermediate and the combination of advanced, superior and distinguished 
prescribed in the guidelines respectively.

The intermediate proficiency prescribed in \textit{ACTFL Proficiency Guidelines} corresponds
with the English proficiency of high school students which prescribed in China\footnote{In China, English proficiency of high school students is prescribed in ``English Course Guidelines for high school students'', which is issued by the Ministry of Education. See http://www.e4in1.com/index/ca11567.htm} and in Japan\footnote{In Japan, English proficiency of high school students is prescribed in ``Guidelines for the Course of Study'', which is issued by the Ministry of Education, Culture, Sports, Science and Technology. See http://www.mext.go.jp/b\_menu/shuppan/sonota/990301/03122603/009.htm}.
Therefore, we selected the texts from high school students' English textbooks published in
the two countries and created a corpus. Using textbooks published in different non-English
speaking countries can make the results of this study have great generality, i.e. the results
might be appropriate for intermediate non-native audiences with different mother tongues.

The corpus we created is called TANN (Target Audience was intermediate Non-Native speakers) 
(size: 200,000 words, domain: \textit{natural and pure science}),
which was annotated on discourse level by Rhetorical
Structure Theory (RST) \cite{Mann88}\footnote{At present, 
the largest publicly available discourse-annotated corpus using the framework of RST 
is RST-DTC (171,383 words), which is smaller than TANN.}. 
The method of creating TANN is as follows. The first author got all selected texts into electronic
form by a scanner. Since scanning is not a completely accurate way of making a text
machine-readable, the first author checked for misinterpreted characters, and corrected them
if necessary. And then, another person was asked to check the electronic texts again to make
sure that no misinterpreted character existed in the corpus.

TANN consists of three parts (Table 1). The texts of Part I were selected 
from 15 textbooks published in Japan. The texts of Part II and III were 
selected from 11 textbooks and 11 supplementary textbooks published in China. 
Furthermore, the Flesch Reading Ease score\footnote{The Flesch reading Ease Scale is the most widely used formula to measure readability. The formula can be represented as follows: 206.835 - (1.015 $\times$ ASL) -(84.6 $\times$ ASW). ASL represents average sentence length (number of words divided by the number of sentences). ASW represents average number of syllables per word (number of syllables divided by number of words). The Flesch Reading Ease score is based on a 100 point scale. The higher the score, the easier a text is to read. For example, 100 means ``very easy to read'', ``65'' means plain English, ``0'' means extremely difficult to read.} (the sixth row) shows that the three parts have the same readability.  

\begin{table}[t]
 \caption{An introduction to TANN}
\input{01t01.txt}
\end{table}

\begin{table}[b]
 \caption{Five portions of discourse markers functioning as subordinators}
\input{01t02.txt}
\end{table}



\subsection{Selecting discourse markers from TANN}

92 English discourse markers functioning as subordinators are listed in \cite{Knott96} (pp. 161-169).
Within TANN, these discourse markers are divided into five portions (Table 2). 
In this study, we only explore the position of six discourse markers shown in Portion 1. 
We have the following four reasons of not considering other discourse markers. 

\begin{enumerate}
   \item Discourse marker \textit{as} (Portion 2) causes ambiguity
easily because it can signal three relations, i.e. ``manner'', ``reason'' and ``time''. 
   \item Discourse markers \textit{so} and \textit{so that} (Portion 3) have default position,
i.e. they always occur in the final position.
   \item Time clauses are not very common in academic prose \cite{Biber99}. 
The domain of TANN is \textit{natural and pure science}, which is similar to
academic prose. Therefore, we do not explore the discourse markers 
signaling ``time'' relation, such as \textit{after}, \textit{before} and 
\textit{until} (Portion 4).
   \item The frequency of the discourse markers shown in Portion 5 is 
lower than 25, which is too low to explore. 
\end{enumerate}

Table 3 shows the number of discourse markers selected from TANN. For the discourse 
markers that can signal two relations, we use ``other'' 
to represent the discourse relation we do not consider. 
For example, for discourse marker \textit{while} which can signal ``contrast'' 
and ``time'' relation, ``other'' refers to ``time'' relation. 
Moreover, we do not consider the structures 
such as ``not because...but because'' and ``if..., or if...'' as well. 
Lastly, 1072 examples were selected. 

\begin{table}[t]
\caption{1072 discourse markers selected from TANN}
\input{01t03.txt}
\end{table}



\subsection{Rhetorical Structure Theory}

Rhetorical Structure Theory (RST) was originally developed for text generation by a
team at Information Sciences Institute of University of Southern California.
RST offers an explanation of the coherence of texts, and is intended to describe
text structure. The main opinions of RST are as follows: 1. The discourse relations
that hold between text spans make text coherent. 2. Each text span is categorized
as a nucleus or a satellite. 3. Compared with satellites, nuclei play a crucial role
in keeping the coherence of a text.

\begin{figure}[b]
\begin{center}
\includegraphics{14-5ia1f1.eps}
\end{center}
 \caption{An example text}
\end{figure}
\begin{figure}[b]
\begin{center}
\includegraphics{14-5ia1f2.eps}
\end{center}
 \caption{The RST analysis of the text shown in Fig. 1}
\end{figure}


In Fig. 1, discourse marker \textit{because} represents ``reason'' 
relation between the main clause B.1 (nucleus) and the subordinate clause B.2 
(satellite), i.e. B.2 provides a reason for the situation presented
in B.1. On the other hand, B.1 and B.2 constitute discourse B. 
Discourse marker \textit{when} represents ``condition'' relation 
between the non-finite clause A (satellite) and discourse B (nucleus). 
That is, the truth mentioned in B is a consequence of fulfillment 
of the condition in A. This text can be represented schematically 
by a tree (Fig. 2). In the tree, the non-terminal nodes
(i.e. Node 0 and Node 1) represent ``condition'' and ``reason'' relation,
while the terminal nodes (i.e. Node 2, Node 3 and Node 4) represent A, B.1
and B.2 respectively. In this study, we define two kinds of structures. 
One is \textit{embedded structure}, such as B.1 and B.2, 
which is embedded in the nucleus of a ``condition'' relation. 
Another is \textit{whole structure}, such as the ``condition''
relation that is signaled by \textit{when}.

We have the following two reasons to apply RST to annotation: 1. 
RST is suitable for representing the discourse structure of any 
genre of texts. Therefore, it is no problem to use RST to annotate 
the texts whose domain is \textit{natural and pure science}.
2. In RST, the discourse relations initially defined is an open set.
The researchers can add or modify relations according to their 
needs. For example, in \cite{Carlson01}, 78 discourse relations 
were defined. Within RST-DTC, discourse marker \textit{because} 
and \textit{while} signal at least five discourse relations. 
Discourse marker \textit{because} can signal ``consequence-n'', 
``Cause-Result'', ``reason'', ``explanation-argumentative'', 
``result'' and ``cause'' relation. Discourse marker \textit{while} 
can signal ``temporal-same-time'', ``circumstance'', ``antithesis'', 
``comparison'', and ``Contrast'' relation. However, we do not think 
it is necessary to define so many discourse relations in this study, 
because the structure of texts within TANN is much simpler. 
We defined 12 discourse relations: background, condition, contrast,
elaboration, evaluation, example, list, purpose, reason, restatement, 
summary, time. All relations except ``list'' are mononuclear relations, 
that is, they have one nucleus and one satellite, e.g. ``condition'' 
and ``reason'' relation shown in Fig. 2. ``List'' relation is a 
multinuclear relation, that is, it has two nuclei. In this study, 
we do not consider the relation having three nuclei or more.

\subsection{Annotating discourse markers by RST}

Fig. 3 demonstrates the definition of ``example'' relation. 
We can see that ``Computers are already being used in agriculture.'' is
the nucleus (N), while ``For example, in greenhouses, computers control 
the growing conditions of vegetables.'' is the satellite (S) because 
it gives an example to the proposition presented in the nucleus.

\begin{figure}[b]
\begin{center}
\includegraphics{14-5ia1f3.eps}
\end{center}
 \caption{Definition of ``example'' relation}
\end{figure}

Annotation can be divided into four steps. 

\begin{enumerate}
  \item Annotating the nucleus (N) and the satellite (S) of the \textit{embedded structure}
in round \linebreak
brackets. E.g. ``When exposed to air, (the substance expands) -N--reason--S- (because it reacts with oxygen).''
  \item Annotating the boundary of the nucleus and the satellite of the
\textit{whole structure} in angle brackets. E.g. ``$\langle$When exposed to air,$\rangle$ $\langle$(the substance expands) -N--reason--S- (because it reacts with oxygen)$\rangle$.''
  \item Labeling the discourse relation of the \textit{whole structure}.
E.g. ``$\langle$When exposed to air,$\rangle$ --condition-- $\langle$(the substance expands) -N-reason-S- (because it reacts with oxygen)$\rangle$.''
  \item Annotating the nucleus and the satellite of the \textit{whole structure}. E.g. ``$\langle$When exposed to air,$\rangle$ -S--condition--N- $\langle$(the substance expands) -N--reason--S- (because it reacts with oxygen)$\rangle$.''
\end{enumerate}

Two coders (one was main coder, another was reliability coder) 
took part in annotation. In order to make the annotation results 
precise and reliable, we wrote a reference manual, in which we 
introduced RST and definitions of the 12 discourse relations 
(A concise manual is shown in Appendix A. See \cite{Deng06} for more details). 
Before annotation, the two coders were asked to read the manual 
and marked some discourse relations in a small test corpus 
according to their understanding of the manual. We compared 
the annotation results of the two coders and analyzed the problems 
that caused disagreement, based on which we revised the manual and 
trained the coders until the agreement became satisfactory 
(more details about training the coders could be found in Appendix B). 
The main coder annotated all the 1072 examples, while the reliability 
coder annotated the first 30 examples of each discourse marker. 
In the experiments, we will use the annotation results of the main coder. 

\begin{table}[b]
\caption{Rate of agreement of the two coders}
\input{01t04.txt}
\vspace{-1\baselineskip}
\end{table}


We follow \cite{Moser95a}'s approach to assess the 
reliability of annotation. Using the first 30 examples of each 
discourse marker, we compared the annotation results 
of the main coder with those of the reliability coder. 
Since the subordinate clauses beginning with the discourse markers 
to be investigated are included in the \textit{embedded structure},
the discourse relations between the main clauses and the subordinate
clauses are determined. For example, for ``reason'' relation signaled
by \textit{because}, it is no doubt that the main clause is the nucleus
and the subordinate clause is the satellite. Therefore, we only assessed 
the annotation results of the \textit{whole structure} of the 180 
examples from three aspects: boundary, discourse relation, 
nucleus and satellite. Table 4 shows that the rate of agreement 
of the two coders was 82.3\%. This result was higher than that 
mentioned in \cite{Moser95a}. We think that the reference 
manual of annotation was very helpful for the coders, because the 
precise definition of each relation avoided misunderstanding.
Furthermore, the two trained coders had linguistic background,
so they could quite grasp the meaning of the manual.  



\subsection{Features}

Each data point in our dataset is characterized by 18 features. 
Feature value can either be a numeric value or a user-defined symbolic 
value. In this study, four features have numeric values with continuous 
ranges, such as Mg and Sg, while others have discrete values, such as R and P. 
We divided the 18 features into two groups: embedded structure features 
and whole structure features. Recall that the \textit{embedded structure} 
consists of the main clause and the subordinate clause, and the subordinate 
clause begins with the discourse marker to be investigated. On the other hand, 
the \textit{embedded structure} is a part of the \textit{whole structure}. 

The two groups of features and their possible values are as follows:  

\begin{itemize}
 \item Embedded structure features
   \begin{enumerate}
    \item Mt. Tense of the main clause: past, present, future.  
    \item Mv. Voice of the main clause: active, passive.
    \item Mg. Length of the main clause (in words): integer.
    \item Ms. Structure of the main clause: simple, other.  
    \item Mi. Information contained in the main clause: new, old.
    \item St. Tense of the subordinate clause: past, present, future.
    \item Sv. Voice of the subordinate clause: active, passive.
    \item Sg. Length of the subordinate clause (in words): integer.
    \item Ss. Structure of the subordinate clause: simple, other.
    \item Si: Information contained in the subordinate clause: new, old.
\end{enumerate}
\end{itemize}

\begin{itemize}
 \item Whole structure features
   \begin{enumerate}
   \item R. Discourse relation: background, condition, contrast, elaboration, evaluation, example, list, purpose, reason, restatement, summary, time.
   \item X. Whether the discourse relation is signaled by discourse marker or not: yes, no.
   \item E. Role of the \textit{embedded structure}: nucleus, satellite.
   \item P. Position of the \textit{embedded structure}: first span, second span.
   \item Eg. Length of the span containing the \textit{embedded structure}: integer.
   \item Es. Structure of the span containing the \textit{embedded structure}: complex, other. 
   \item Og. Length of the span not containing the \textit{embedded structure} (in words): integer.
   \item Os. Structure of the span not containing the \textit{embedded structure}: simple, other.
  \end{enumerate}
\end{itemize}

As shown above, there are 10 embedded structure features. Mt and St represent 
tense of the main clause and the subordinate clause respectively. Their 
values could be ``past'', ``present'' or ``future''. Mv and Sv represent 
voice of the main clause and the subordinate clause. Their values could 
be ``active'' or ``passive''. In \cite{Will03}, the author presents that sentence length
could affect readability on discourse level. We therefore 
considered length (in words) of the main clause (Mg) and the 
subordinate clause (Sg). Moreover, structure of the main clause (Ms) 
and the subordinate clause (Ss) were considered as well. Their 
values could be ``simple sentence'' or ``other'' (i.e. structure
which is more complicated than simple sentence, such as compound 
sentence and complex sentence). In \cite{Biber99}, the authors
note that sometimes the position of discourse markers is concerned with information
structuring. For example, some subordinate clauses in the first span contain
old information that mentioned in the preceding discourse, while the main 
clauses in the second span contain new information. On the contrary, when 
the main clauses contain old information, the subordinate clauses which 
contain new information tend to be in the second span. We therefore 
considered two features related to information structuring: 
Mi (i.e. information contained in the main clause) 
and Si (i.e. information contained in the subordinate clause). 
The values of Mi and Si could be ``new'' or ``old''.  

In addition, there are 8 whole structure features. R represents the
discourse relation labeled on the \textit{whole structure}. Its
value could be one of the 12 discourse relations we defined. X represents whether 
the \textit{whole structure} is signaled by discourse marker or not. 
E represents the role of the \textit{embedded structure} which is included
in the \textit{whole structure}. Its value could be ``nucleus'' or 
``satellite''. P represents position of the \textit{embedded structure} in the 
\textit{whole structure}. Its value could be ``first span'' or ``second span''. 
Moreover, length and structure of the two spans of the \textit{whole structure} 
were considered. Eg and Es represent the length and structure of the 
span containing the \textit{embedded structure}. The value of Es could be 
``complex sentence'' or ``other'' (i.e. structure which is more
complicated than complex sentence). Og and Os represent length and
structure of the span not containing the \textit{embedded structure}. 
The value of Os could be ``simple sentence'' or ``other'' (i.e. all structures
except simple sentence).  

We use Fig. 1 to introduce how to represent the 18 features (Table 5). 
Here, we only explain 9 features in detail. For the main clause in 
the \textit{embedded structure} (i.e. ``the substance expands''), 
its tense (Mt) and voice (Mv) are ``present'' and ``active'' respectively. 
The length of the main clause (Mg) is 3, and the main clause is a 
simple sentence (Ms). The subject of the main clause (i.e. ``the substance'') 
refers to the same thing in the preceding discourse (i.e. ``when exposed to air'') 
although it is deleted there. So the information contained in the
main clause is old (Mi). On the other hand, the discourse relation of 
the \textit{whole structure} (R) is ``condition''. Since this relation 
is signaled by \textit{when}, the value of feature X is ``yes''.
The \textit{embedded structure} is the nucleus (E) of the ``condition'' 
relation, and occurs in the second span (P). Of the 18 features, 7 related 
to the \textit{embedded structure} can be extracted automatically. 
For example, features Mt, Mv, St and Sv can be extracted by a syntactic 
parser called SYNTPARSE\footnote{http://www.langsoft.ch/parsing.htm}. 
The features Mg, Sg and Eg can be extracted by a program. 
However, 11 features related to the \textit{whole structure}
can not be extracted automatically. At present, although 
discourse parser SPADE (Sentence-level PArsing for DiscoursE)\footnote{http://www.isi.edu/licensed-sw/spade/} can be publicly available, it can only build sentence-level discourse 
parse trees. If a discourse parser that can parse a text containing 
several sentences on discourse level can be obtained in the future, 
then all features can be extracted automatically.

\begin{table}[t] 
\caption{18 features and their values of the text shown in Fig. 1}
\input{01t05.txt}
\vspace{-1\normalbaselineskip}
\end{table}


\subsection{Six sets of experiments}

We do six sets of experiments to induce the classification models of the 
position of the six discourse markers. Each set of experiments
contains 38 experiments using different models represented by different 
subsets of the 18 features. The first 18 experiments use 
the models represented by individual features, corresponding to each feature
mentioned in Section 2.6. Another 20 experiments use the models 
represented by the combinations of the available features (Table 6). 
These models can be divided into three parts. In the first part, the 
8 models consider the combinations of the embedded structure features. 
All.e contains all 10 embedded structure features. Main.e and Sub.e 
contain the features related to the main clause and the 
subordinate clause respectively. Len.e and Str.e contain the features 
related to length and structure. Inf.e contains the features related to 
information. T+V.e contains the features related to tense and voice. 
I+L.e contains the features related to information and length.
In the second part, the 6 models consider the combinations of the 
whole structure features. All.w contains all 8 whole structure features. 
Emb.w contains the features related to the \textit{embedded structure}. Len.w
and Str.w contain the features related to length and structure. -R.w contains
all whole structure features except R. -RX.w contains all whole structure
features except R and X. In the third part, the 6 models consider the combinations
of both embedded structure features and whole structure 
features. All.ew contains all 18 features. Len.ew and Str.ew 
contain the features related to length and structure respectively. 
-R.ew contains all features except R. -RI.ew contains all features except R, 
Mi and Si. -I.ew contains all features except Mi and Si.

\begin{table}[t]
\caption{Models containing different feature sets}
\input{01t06.txt}
\end{table}


In order to make the experiment results accurate and reliable,
we use C4.5 to induce the classification models, and then verify 
the experiment results of C4.5 by SVM. For each discourse marker, we use 
datasets with different sizes to see if the results change. 
If the number of examples is more than 200, 
we first do experiments using all examples, and then 
repeat the experiments using a smaller dataset 
containing 120 examples (the reason is shown in Step 1 of Appendix B).
If the number of examples is less than 100, we
first do the experiments using all examples, and then repeat the
experiments by reducing the size of the dataset till
the experiment results change. 

\section{Experiment results of C4.5} 

C4.5 is a set of computer programs whose main function is identifying and
analyzing patterns in amount of data. This section introduces the evaluation 
method of C4.5 and the results of six sets of experiments. 

\subsection{Evaluation method} 

Within each set of experiments, the baseline can be obtained by choosing 
the majority class. For example, in Experiment Set 1, 272 examples of
discourse marker \textit{because} were used. In these examples, 
69.1\% (188/272) of subordinate clauses beginning with \textit{because} 
occur in the second span. That is, if a subordinate clause beginning 
with discourse marker \textit{because} was placed directly in the second 
span, one would be wrong 30.9\% of the times. So 30.9\% is the error 
rate of the baseline model. 

In this study, the error rates of the learned classification models are estimated by
the method of 10-fold cross-validation \cite{Weiss91}. That is, data for
learning is randomly divided into 10 test sets. The program is run for 10 times,
each run uses 9 test sets as the training set and the remaining one as the test set.
The error rate of a classification model obtained by using the whole dataset for training 
is assumed to be the average error rate on the test set over the 10 runs 
\cite{Eugenio97}. The advantage of this method is that all examples 
are eventually used for testing, and almost all examples are used in any 
given training run \cite{Litman96}. 

For each set of experiments, the method of evaluating and identifying the best
classification model(s) can be divided into two steps.

First, we compute the 95\% confidence interval for error rate of each model by (1). 
\begin{equation}
   \bar{x}~~\pm~~t_{N-1}\left( \frac{\alpha}{2} \right)~~\sqrt{\frac{S^2}{N}}
\end{equation}
where $\bar{x}$ is average error rate (i.e. $\varepsilon$) of a learned classification
model, ``N-1'' is degree of freedom. Since N is 10 in this study, the degree of
freedom is 9 (i.e. 10-1). A 95\% confidence interval for a sample represents that
the true population mean has a 95\% chance of falling within the confidence interval.
Here, $\alpha$/2 is 0.025. In {\textit{t}}-Table, {\textit{t}} distribution is 2.262,
$S^2$ is sample variance (2), i.e. 
\begin{equation}
   S^2 = \frac{\left( x_1 - \bar{x} \right)^2 + \left( x_2 - \bar{x} \right)^2 + \cdots + \left( x_N - \bar{x} \right)^2}{N-1}
\end{equation}

Second, we identify and select the best learned model(s) by 
comparing it (their) error rate(s) with the error rates of the 
other learned models and with the error rate of the baseline model. 
We follow \cite{Eugenio97}'s approach to 
determine whether two error rates $\varepsilon$1 and $\varepsilon$2 
are significantly different. That is, if the upper bound of the 
95\% confidence interval for error rate $\varepsilon$1 is lower 
than the lower bound of the 95\% confidence interval for error rate 
$\varepsilon$2, then the difference between $\varepsilon$1 and $\varepsilon$2 is
considered to be significant. 

\subsection{Experiment results}

For each set of experiments, we will report the error rates of all learned
classification models. To identify the best performing learned model, we will
compare the error rates of learned models with each other and with the error rate of
the baseline model. In addition, we will represent the learned classification model
that performs better than the baseline model by decision tree. 
 
\subsubsection{Position of discourse marker \textit{because}}

In Experiment Set 1, firstly, 272 examples of discourse marker 
\textit{because} were used. In these examples, 69.1\% (188/272) of 
subordinate clauses beginning with \textit{because} occur in the second span. 
Therefore, the error rate of the baseline model is 30.9\%. 

In Table 7, four classification models learned from R, All.w, All.ew 
and -I.ew (in bold type) perform better than the baseline model. 
Of these models, the one learned from feature R is the best classification
model because it performs significantly better than the three others. 
All the four learned models say that if the discourse relation 
of the \textit{whole structure} (R) is ``contrast'', ``example'' or 
``reason'', then in the \textit{embedded structure}, the subordinate 
clause beginning with \textit{because} occurs in the first span; 
otherwise occurs in the second span (Fig. 4).

In Fig. 5, the ``reason'' relation signaled by \textit{because} 
between the subordinate clause B.1 and the main clause B.2 
is the nucleus of the ``contrast'' relation signaled by discourse 
marker \textit{but}. Therefore, in the \textit{embedded structure}
which contains ``reason'' relation, discourse marker \textit{because} is 
placed in the first span. The subordinate 
clause B.1 puts strong emphasis on the response from the 
public on global warming, and provides contrast with the consequences 
caused by global warming mentioned in sentence A. 

\begin{table}[t]
\hangcaption{95\%-confidence intervals for the error rates for all learned classification models of discourse marker \textit{because} (272 examples)}
\input{01t07.txt}
\end{table}
\begin{figure}[t]
\begin{center}
\includegraphics{14-5ia1f4.eps}
\end{center}
 \caption{Decision tree learned from R, All.w, All.ew, and -I.ew}
\end{figure}
\begin{figure}[t]
\begin{center}
\includegraphics{14-5ia1f5.eps}
\end{center}
 \caption{An example of placing \textit{because} in the first span}
\end{figure}

The experiment results also show that feature R can improve performance
of the learned models, because the classification models learned
from the feature sets including feature R (i.e. R, All.w, All.ew and -I.ew) 
perform better than the baseline model. On the contrary, the baseline model
performs better than the classification models learned from the 
feature sets not including feature R (e.g. Mt, Main.e and Emb.w).  

Discourse marker \textit{because} often occurs in the second span \cite{Quirk72}.  
This conclusion was proved through corpus analysis by \cite{Moser95a}. In their corpus, 
100\% (13/13) of subordinate clauses beginning with \textit{because} occur 
in the second span. In this study, this conclusion is proved as well,
since 69.1\% (188/272) of subordinate clauses beginning with \textit{because} 
occur in the second span within TANN. 

We also did another set of experiments using 120 examples (Table 8). 
The result did not change. That is, the classification model learned from
feature R is the best performing learned model. 

\begin{table}[t]
\hangcaption{95\%-confidence intervals for the error rates for all learned classification models of discourse marker \textit{because} (120 examples)}
\input{01t08.txt}
\end{table}



\subsubsection{Position of discourse marker \textit{since}}

In Experiment Set 2, firstly, 46 examples of discourse marker \textit{since} 
were used. In these examples, 56.5\% (26/46) of subordinate clauses
beginning with \textit{since} occur in the first span.
Therefore, the error rate of the baseline model is 43.5\%. 

In Table 9, four classification models learned from Mi, Main.e, All.ew
and -R.ew (in bold type) perform better than the baseline model.
Of these models, the one learned from feature Mi is the best 
classification model because it performs significantly 
better than the three others. All the four learned models say that if 
the information contained in the main clause (Mi) is new, 
then discourse marker \textit{since} is placed 
in the first span; otherwise is placed in the second span (Fig. 6). 

\begin{table}[t]
\hangcaption{95\%-confidence intervals for the error rates for all learned  classification models of discourse marker \textit{since} (46 examples)}
\input{01t09.txt}
\end{table}
\begin{figure}[t]
\begin{center}
\includegraphics{14-5ia1f6.eps}
\end{center}
 \caption{Decision tree learned from Mi, Main.e, All.ew, and -R.ew}
\end{figure}

Discourse marker \textit{since} typically occurs in the first span \cite{Quirk72}. 
The authors of \cite{Moser95a} indicate that discourse marker \textit{since}
has a strong preference of occurring in the initial position because 
95.7\% (22/23) of subordinate clauses beginning with \textit{since} occur in the first
span in their corpus. However, in this study, discourse marker \textit{since}
has a slight preference of occurring in the first span, because only 56.5\% (26/46) 
of subordinate clauses beginning with \textit{since} occur in the first span. 
This conclusion is a little different from those mentioned in \cite{Quirk72} and 
in \cite{Moser95a}. Fig. 7 demonstrates why sometimes discourse
marker \textit{since} occurs in the second span. We can see that both 
the main clause B and the subordinate clause C contain old information.
In the main clause B, the word ``this'' refers to the proposition
of sentence A (i.e. ``Many regions of the world will face water crisis.''),
while in the subordinate clause C, the word ``water'' refers to the same word
``water'' in sentence A. It is obvious that the word ``this'' in B is stronger and
more important than the word ``water'' in C. It is because that the word ``this''
refers to the whole sentence A, but the word ``water'' only refers to a part
of sentence A. Therefore, placing the main clause B in the first span
can not only emphasize the proposition of sentence A but also attract
the attention of the intermediate non-native audience. 
The experiment results illustrate that information contained in the main clause (Mi)
plays an important role in determining the position of discourse 
marker \textit{since}.

\begin{figure}[t]
\begin{center}
\includegraphics{14-5ia1f7.eps}
\end{center}
 \caption{An example of placing \textit{since} in the second span}
\end{figure}
\begin{table}[t]
\hangcaption{95\%-confidence intervals for the error rates for all learned  classification models of discourse marker \textit{since} (30 examples)}
\input{01t10.txt}
\end{table}


We also did several other sets of experiments, and found that
the best performing learned model did not change until the dataset was smaller 
than 30 (Table 10). That is, when the size of dataset is 29, 
no learned model performs better than the baseline model, and discourse
marker \textit{since} always occurs in the first span.




\subsubsection{Position of discourse marker \textit{if}}

In Experiment Set 3, firstly, 381 examples of discourse marker \textit{if} 
were used. In these examples, 83.2\% (317/381) of subordinate
clauses beginning with \textit{if} occur in the first span. 
Therefore, the error rate of the baseline model is 16.8\%.  

\begin{table}[b]
\hangcaption{95\%-confidence intervals for the error rates for all learned classification models of discourse marker \textit{if} (381 examples)}
\input{01t11.txt}
\end{table}
\begin{table}[b]
\caption{Frequency of discourse marker \textit{if} per million words in different registers}
\input{01t12.txt}
\end{table}


Table 11 presents the error rates of the 38 learned classification models. 
We can see that no classification model performs significantly better 
than the baseline model. 

Conditional clauses have a slight preference 
of occurring in the initial position in the written registers \cite{Biber99}. 
However, in this study, discourse marker \textit{if} has a strong 
preference of occurring in the first span, because 83.2\% (317/381) 
of conditional clauses beginning with \textit{if} occur in 
the first span within TANN. Moreover, the frequency of discourse marker 
\textit{if} per million words across different registers 
(the first four columns of Table 12) is mentioned in \cite{Biber99}. 
We infer that if TANN contained one million words,
the frequency of discourse marker \textit{if} would be 2265 (453 $\times$ 5) 
(the fifth column of Table 12). It is obvious that the frequency of 
\textit{if} within TANN (domain: \textit{natural and pure science})
follows the same pattern as that in Academy register 
(the forth column of Table 12). 

\begin{table}[t]
\hangcaption{95\%-confidence intervals for the error rates for all learned classification models of discourse marker \textit{if} (120 examples)}
\input{01t13.txt}
\end{table}

We also did another set of experiments using 120 examples 
(Table 13). The result did not change. That is,
discourse marker \textit{if} always occurs in the first
span. 


\subsubsection{Position of discourse marker \textit{when}}

In Experiment Set 4, firstly, 228 examples of discourse marker \textit{when} 
were used. In these examples, 61.8\% (141/228) of subordinate clauses
beginning with \textit{when} occur in the first span. Therefore, 
the error rate of the baseline model is 38.2\%.  

In Table 14, eleven learned classification models (in bold type) perform 
better than the baseline model. Of these models, 
the one learned from Inf.e (i.e. Mi and Si) is the best 
classification model because it performs significantly better 
than the ten others.

The classification model learned from Mg is identical to 
those learned from Main.e, Len.e and Len.ew. All the four 
learned models say that if length of the main clause (Mg) is less than or equal 
to 6, then discourse marker \textit{when} is placed in the second 
span; otherwise is placed in the first span (Fig. 8). Moreover, 
feature Mg can improve the performance of 
classification models. For example, the upper bound of the 95\% 
confidence interval for the error rate of Sg (40.63\%) is higher than the
error rate of the baseline model. However, the model 
learned from Len.e (i.e. Mg and Sg) performs significantly 
better than the baseline model, because the upper bound of 
the 95\% confidence interval for the error rate of Len.e (34.46\%)
is lower than the error rate of the baseline model.

\begin{table}[t]
\hangcaption{95\%-confidence intervals for the error rates for all learned classification models of discourse marker \textit{when} (228 examples)}
\input{01t14.txt}
\end{table}
\begin{figure}[t]
\begin{center}
\includegraphics{14-5ia1f8.eps}
\end{center}
 \caption{Decision tree learned from Mg, Main.e, Len.e, and Len.ew}
\end{figure}
\begin{figure}[t]
\begin{center}
\includegraphics{14-5ia1f9.eps}
\end{center}
 \caption{Decision tree learned from Si, and Sub.e}
\end{figure}


The models learned from Si and Sub.e perform significantly better
than the baseline model as well. Both of the models say that if the 
information contained in the subordinate clause (Si) is new, 
then discourse marker \textit{when} is placed in the second 
span; otherwise is placed in the first span (Fig. 9). 

In addition, five classification models learned from All.e, Inf.e, I+L.e, All.ew 
and -R.ew perform better than the baseline model.
These classification models, which are identical to each other,
say that if the information contained in the subordinate clause (Si)
is old, then discourse marker \textit{when} is placed 
in the first span; if the information contained in the subordinate 
clause (Si) is new and the information contained in the main clause (Mi) is also new, 
then discourse marker \textit{when} is placed in the first span; if the information
contained in the subordinate clause (Si) is new and the information
contained in the main clause (Mi) is old, then discourse marker 
\textit{when} is placed in the second span (Fig. 10). 
We can see that the information contained in the main clause (Mi)
and the information contained in the subordinate clause (Si)
determine the position of \textit{when}.

\begin{figure}[t]
\begin{center}
\includegraphics{14-5ia1f10.eps}
\end{center}
 \caption{Decision tree learned from All.e, Inf.e, I+L.e, All.ew, and -R.ew}
\end{figure}
\begin{table}[t]
\hangcaption{95\%-confidence intervals for the error rates for all learned classification models of discourse marker \textit{when} (120 examples)}
\input{01t15.txt}
\end{table}


We also did another set of experiments using 120 examples (Table 15). 
The result did not change. That is, the classification model learned from
Inf.e is the best performing learned model.  


\subsubsection{Position of discourse marker \textit{although}}

In Experiment Set 5, firstly, 83 examples of discourse marker \textit{although} 
were used. In these examples, 69.9\% (58/83) of subordinate 
clauses beginning with \textit{although} occur in the first span. 
Therefore, the error rate of the baseline model is 30.1\%.  

\begin{table}[b]
\hangcaption{95\%-confidence intervals for the error rates for all learned classification models of discourse marker \textit{although} (83 examples)}
\input{01t16.txt}
\end{table}
\begin{figure}[b]
\begin{center}
\includegraphics{14-5ia1f11.eps}
\end{center}
 \caption{Decision tree learned from All.e, Inf.e, I+L.e, All.ew, and -R.ew}
\end{figure}
\begin{figure}[b]
\begin{center}
\includegraphics{14-5ia1f12.eps}
\end{center}
 \caption{An example of placing \textit{although} in the first span}
\end{figure}




In Table 16, five classification models learned from All.e, Inf.e, I+L.e, 
All.ew and -R.ew perform significantly better than the baseline model.
Of these models, the one learned from Inf.e (i.e. Mi and Si) 
is the best classification model because it performs significantly better 
than the four others. All the five learned models 
say three rules (Fig. 11). Rule 1 says that if the information
contained in the subordinate clause (Si) is old, then discourse marker \textit{although} 
is placed in the first span. In Fig. 12, the word ``they'' in the subordinate
clause B is old information, because it 
refers to ``whales'' in the preceding sentence A. So discourse marker
\textit{although} is placed in the first span. Rule 2 says that 
if the information contained in the subordinate clause (Si) is new and the information
contained in the main clause (Mi) is also new, then discourse marker \textit{although}
is placed in the first span. Rule 3 says that if the information contained in the 
subordinate clause (Si) is new and the information contained in the main 
clause (Mi) is old, then discourse marker \textit{although} is placed in the second span. 
In Fig. 13, ``tides'' in the main clause 
B is old information, because it refers to the same word ``tides'' 
in the preceding sentence A. Moreover, the subordinate clause C 
contains new information. Therefore, discourse marker \textit{although}
is placed in the second span. 

\begin{figure}[t]
\begin{center}
\includegraphics{14-5ia1f13.eps}
\end{center}
 \caption{An example of placing \textit{although} in the second span}
\end{figure}
\begin{table}[t]
\hangcaption{95\%-confidence intervals for the error rates for all learned classification models of discourse marker \textit{although} (49 examples)}
\input{01t17.txt}
\end{table}



We also did several other sets of experiments, and found that
the best performing learned model did not change until the dataset was smaller 
than 49 (Table 17). That is, when the size of dataset is 48, 
no best performing learned model is obtained, and discourse marker
\textit{although} always occurs in the first span.



\subsubsection{Position of discourse marker \textit{while}}  

In Experiment Set 6, firstly, 62 examples of discourse marker \textit{while} 
were used. In these examples, 85.5\% (53/62) of subordinate clauses
beginning with \textit{while} occur in the second span. 
Therefore, the error rate of the baseline model is 14.5\%. 

\begin{table}[b]
\hangcaption{95\%-confidence intervals for the error rates for all learned classification models of discourse marker \textit{while} (62 examples)}
\input{01t18.txt}
\end{table}

Table 18 presents the error rates of the 38 learned classification models. 
We can see that no classification model performs significantly better than 
the baseline model. 



Contrast clauses show a slight preference of occurring in final 
position, and this preference is shared across all registers \cite{Biber99}. 
In this study, discourse marker \textit{while} shows a strong 
preference of occurring in the second span, because 85.5\% (53/62) 
of contrast clauses beginning with \textit{while} occur in the second span
within TANN. In addition, discourse marker \textit{while} 
can signal both ``contrast'' relation and ``time'' relation. 
In \cite{Biber99}, the authors indicate that in 
academic prose, about 90\% of subordinate clauses beginning with \textit{while} 
are contrast clauses, about 10\% are time clauses. However, within TANN, 
about 60.2\% (62/103) of subordinate clauses beginning with \textit{while} are contrast
clauses, about 39.8\% (41/103) are time clauses (Table 3).

Fig. 14 and Fig. 15 demonstrate the position of discourse marker 
\textit{while} when it is used to signal ``contrast'' relation. 
In Fig. 14, discourse marker \textit{while} contrasts the effect of 
``moderate rainfall'' with that of ``continuous heavy rain''. In Fig. 15, 
discourse marker \textit{while} compares the population of left-handed with
that of right-handed. Although the position of \textit{while} is different,
it contrasts the proposition of the main clause with that of the subordinate clause
in both examples.

We also did several other sets of experiments using different
sizes of datasets. The result did not change. That is,
discourse marker \textit{while} always occurs in the second span.

\begin{figure}[t]
\begin{center}
\includegraphics{14-5ia1f14.eps}
\end{center}
 \caption{An example of placing \textit{while} in the first span}
\end{figure}
\begin{figure}[t]
\begin{center}
\includegraphics{14-5ia1f15.eps}
\end{center}
 \caption{An example of placing \textit{while} in the second span}
\end{figure}




\section{Verifying the experiment results of C4.5 by SVM}

SVM is supervised machine learning algorithm that is designed for binary classification. It
maps input feature vectors to a high-dimensional space, and divides the space into a positive
class side and a negative class side. Then SVM finds a hyperplane with the maximal margin
in this high-dimensional space. That is, the hyperplane separates the training data into two
classes, and the distance between the hyperplane and the nearest points of the two classes
is the maximal.

\subsection{Evaluation method}

Within each set of experiments, the baseline accuracy is the ratio of the examples that belong
to the majority class. For example, in Experiment Set 1, 272 examples of discourse marker
\textit{because} were used. In these examples, 69.1\% (188/272) of subordinate clauses beginning with
\textit{because} occur in the second span. That is, the baseline accuracy is 69.1\%.

For each set of experiments, the best learned model should satisfy the following two
conditions: (a) the accuracy of the best learned model is higher than the baseline accuracy,
(b) the best learned model produces the highest accuracy among all learned models.

\subsection{Experiment results}

We choose to use the LIBSVM\footnote{http://www.csie.ntu.edu.tw/$\sim$cjlin/libsvm}
package in our experiments, because it is a easy-to-use and
efficient software for support vector classification. According to the proposed procedure
introduced in \textit{A Practical Guide to Support Vector Classification} (i.e. data scaling, parameter
selection (a 10-fold cross-validation is used to determine the optimal value of the parameters),
training and predicating), we induce the classification models of the six discourse markers. 

\begin{table}[t]
\caption{The best learned models obtained from SVM and C4.5}
\input{01t19.txt}
\end{table}


Table 19 shows the experiment results of LIBSVM. The second and the forth column show 
the classification accuracy of the best learned model, while the third and the fifth column show
the baseline accuracy. For discourse markers \textit{if} and \textit{while}, no best performing learned model
is obtained, because no learned model whose classification accuracy is higher than the baseline
accuracy. The best learned classification models of discourse markers \textit{because}, 
\textit{since}, \textit{when} and \textit{although} are learned from R, Mi, Inf.e and Inf.e 
respectively (the sixth column). We can see
that the experiment results of SVM are consistent with those of C4.5 (the seventh column).



\section{Discussion}  

The experiment results of this study show that information 
structuring plays an important role in determining the position
of discourse markers. In Experiment Set 2, the best classification 
model of \textit{since} is learned from feature Mi 
(i.e. information contained in the main clause). 
In Experiment Set 4 and 5, for both \textit{when} and \textit{although}, 
the best classification models are learned from Inf.e (i.e. Mi and Si). 
The results of this study also show that \textit{since}, 
\textit{if}, \textit{when} and \textit{although} tend to 
be placed in the first span. One possible explanation is that the 
intermediate non-native speakers sometimes can not understand the
meaning of the texts quite well. Placing discourse markers in the 
initial position can not only emphasize the proposition of the subordinate clauses
but also attract the attention of the audience whose reading ability
is lower. The results of this study and those of the previous
ones were summarized in Table 20.

\begin{table}[b]
\caption{Results of this study and those of the previous ones}
\input{01t20.txt}
\end{table}
\begin{table}[b]
\caption{An example in the questionnaire}
\input{01t21.txt}
\end{table}




Although we investigated the position of six discourse markers
by experiments, we still do not know if the experiment 
results are appropriate for intermediate non-native speakers. 
So we designed a questionnaire and asked the human subjects 
to assess readability. In the questionnaire, each question
has two texts: one is the original text of BNC (British National
Corpus), another is its paraphrased text using the results of 
this study. The subjects were asked to choose one text that  
he/she preferred. Since almost all examples of discourse marker \textit{while} 
within BNC occur in the second span, which is nearly the same 
as the result of this study, we do not consider discourse marker 
\textit{while}.

The questionnaire contains 20 questions. 4 questions were tested
for each discourse marker. Table 21 shows an example taken from
the questionnaire. In order to emphasize the discourse markers
investigated and make human subjects get a better understanding
of them, we underlined the discourse markers in the questionnaire.  


50 intermediate non-native subjects (25 Chinese and 25 Japanese) answered the
questionnaire. Broadly speaking, a person's English proficiency is dependent on his/her
school background. In this study, a person whose school background is lower than university
and higher than middle school is regarded to be an intermediate non-native speaker. Of the
subjects, 21 were high school students, 11 were junior college students, 18 graduated from
junior college. Moreover, 26 were male, 24 were female. The age of the 
subjects ranged from 17 to 53; the mean was 28. We contacted with 
them by Internet and asked them to finish the questionnaire in their 
spare time.

\begin{table}[t]
\caption{The questionnaire results}
\input{01t22.txt}
\end{table}

Table 22 shows the results of the questionnaire. It is obvious that
compared with the original texts of BNC, most human subjects preferred
their paraphrased texts. It means that the results of this study can improve
readability and make the texts easy to read for intermediate non-native speakers.



\section{Related work} 

This section introduces the related work from two aspects: 
one is the research on the position of discourse markers, another is 
the study about making English texts easier to understand in natural
language generation.

Until now, some researches had been done on the position of discourse
markers. The study of \cite{Moser95b} explores the occurrence and position 
of discourse markers by corpus analysis. In \cite{Moser95a},
the authors compare the position distribution of discourse marker 
\textit{because} with that of discourse marker \textit{since}. 
Using C4.5, the authors of \cite{Eugenio97} identify the features that predict 
the occurrence and position of discourse markers in tutorial explanations. 
Moreover, a full-scale study on discourse markers is conducted in \cite{Biber99}.
For example, the authors explore the position of discourse markers across four registers, i.e. 
conversation, fiction, news and academic prose. They 
discuss the factors that have influence on determining the position
of discourse markers, such as information structuring, cohesion 
and structural considerations. They also investigate the 
usage of discourse markers with multiple semantic roles,
for example, \textit{since} (which can represent ``reason'' 
and ``time'' relation) and \textit{as} (which can represent ``manner'', 
``reason'' and ``time'' relation).

This study can be regarded as the first step of making 
English texts easier to understand for intermediate non-native speakers
on discourse level. Future work will concentrate on generating English 
texts easier to read by placing discourse markers in adequate position. 
In natural language generation, many other methods about 
making English documents easier to understand were put forward. 
For example, generating discourse markers whenever possible could make 
a text easier to comprehend \cite{Scott90}, substituting common words for uncommon 
words \cite{Devlin98}, reducing multiple-clause sentences to single-clause sentences 
\cite{Chandrasekar97}. Furthermore, the study of \cite{Devlin00} 
simplifies newspaper articles for aphasic readers (partial or total 
loss of ability to speak or understand spoken language, caused 
by damage to the brain) by simplification of syntactic structures and 
lexical simplification. In \cite{Will04}, the author mentions at least three aspects 
(e.g. between-text-span punctuation, position and selection of discourse markers) 
that affect readability on discourse level for poor readers of native speakers (missed school, 
learning difficulties, short-term memory, etc.).
Recently, more and more researchers pay attention to the research on generating
English texts easier to read for the audience whose reading 
ability is lower.  




\section{Importance of the study}  

At present, the population of non-native speakers is twice that of native
speakers of English. As a tool of global communication, English plays an important 
role in people's daily lives. Therefore, it is necessary to write English 
in a way that can be understood quite well by international audience. 
The results of this study can make people become more aware of
the usage of discourse markers (especially the position of discourse 
markers) within the texts whose target audience was intermediate 
non-native speakers of English. 

In addition, the opinion of this study (i.e. making English 
texts easier to understand for non-native speakers whose reading 
ability is lower) is important and meaningful as well. Actually, this study is 
only a small step on investigating the word usage for
non-native speakers. We hope that a full-scale investigation can 
be conducted on the texts whose target audience is intermediate 
non-native speakers in the future (like the project introduced in 
\textit{Longman grammar of spoken and written English} \cite{Biber99}). 
If it comes true, people could create homepages 
(e.g. Official site of Olympic Games and FIFA 
World Cup) in the way that is easy to read for non-native 
speakers through referring to the investigation results. These 
homepages will attract countless non-native users. On the other hand, 
for the machine translation systems whose target language is English, 
such as Japanese-English translation system, their target 
users are assumed to be good readers of native speakers of English at
present. Researchers could develop generation rules referring to
the investigation results to make the translation results appropriate 
for intermediate non-native speakers. The machine translation systems 
can be used by not only native speakers but also a great number of 
non-native speakers all over the world, for example, Mongolian, Russian, 
and Vietnamese whose English level is not high. 




\section{Conclusion}  

The position of discourse markers is one of the factors that affect
readability on discourse level. This paper introduces the study on investigating 
the position of discourse markers within the texts whose target audience was 
intermediate non-native speakers of English. We did experiments to induce 
the classification models of the position of six discourse markers 
(i.e. \textit{because, since, if, when, although} and \textit{while}) by C4.5 and SVM. 

The first two sets of experiments induced the classification models
of the position of \textit{because} and \textit{since} which signal 
``reason'' relation. For discourse marker \textit{because}, 
the best performing classification model is learned from feature R. 
Discourse marker \textit{because} tends to occur in the second 
span. On the other hand, for discourse marker \textit{since}, the best 
performing classification model is learned from feature Mi. Discourse 
marker \textit{since} has a slight preference of occurring in the first span.

The third and forth sets of experiments induced the classification models
of the position of \textit{if} and \textit{when} which signal 
``condition'' relation. For discourse marker \textit{if}, 
no best learned model was obtained. Discourse marker \textit{if} typically 
occurs in the first span. On the other hand, the best performing 
classification model of \textit{when} is learned from Inf.e (i.e. Mi and Si). 
The information contained in the main clause (Mi) and the information 
contained in the subordinate clause (Si) play an important role 
in determining the position of \textit{when}. Moreover, discourse marker 
\textit{when} has a slight preference of occurring in the first span.

The fifth and sixth sets of experiments induced the classification
models of the position of \textit{although} and \textit{while} which 
signal ``contrast'' relation. For discourse marker \textit{although}, 
the best performing classification model is learned from Inf.e (i.e. Mi and Si). 
Discourse marker \textit{although} has strong preference 
of occurring in the first span. On the other hand, for discourse marker 
\textit{while}, no best learned model was obtained. Discourse marker 
\textit{while} typically occurs in the second span.

Future work will concentrate on generating English texts that are
easier to read for intermediate non-native speakers on discourse
level. The SILK system, which we are developing, can generate text
using Genetic Algorithm. In the system, four factors
(i.e. nucleus position, between-text-span punctuation, embedded discourse
markers and punctuation pattern) are regarded to affect readability 
on discourse level. We think that it is the preferences among these 
factors rather than the factors themselves that decide readability. 
For example, if the position of a discourse marker is consistent with 
the experiment results of this study, then a high score will be assigned. 
In addition, as mentioned above, the results of this study can be 
applied to other fields in natural language processing, such as 
homepage creation and machine translation.

\section*{Appendix A}

\begin{center} \large{Concise Reference for Discourse Annotation}   \end{center}

1. Introduction

This reference manual presents a guideline that is used to 
develop a discourse-annotated corpus using the framework of 
Rhetorical Structure Theory. Four steps of annotation are introduced 
in Section 2.5. In this reference, the definitions of 11 discourse 
relations are given.

2. Definitions of discourse relations


2.1 Background: mononuclear

Definition: in a background relation, the situation presented in the satellite provides the context in which the situation presented in the nucleus should be interpreted. However, the satellite is not the reason of the situation presented in the nucleus. The situation presented in the satellite is objective, and the reader/writer intentions are irrelevant in determining whether such a relation holds. 

Example: (The World Wide Web was invented by English computer scientist Sir Tim Berners-Lee in 1989.) -N--background--S- (This momentous invention has transformed many aspects of our world.)


2.2 Condition: mononuclear

Definition: in a condition relation, the truth of the proposition of the nucleus is a consequence of the fulfillment of the condition in the satellite. 

Example: (If you do not study hard,) -S--condition--N- (you can not master English.)


2.3 Contrast: mononuclear

Definition: in a contrast relation, the situation presented in the nucleus is contrary to expectation in the light of the information presented in the satellite, or the situation presented in the nucleus comes in contrast with the situation presented in the satellite. 

Examples:

\begin{enumerate}
  \item (Although he had a headache,) -S--contrast--N- (he attended the meeting on time.)
  \item (I like black coffee) -N--contrast--S- (while my sister prefers red tea.)
\end{enumerate}


2.4 Elaboration: mononuclear

Definition: in an elaboration relation, the satellite gives additional information (or detail) about the situation or a part of the situation presented in the nucleus. 

Example: (Greenland is the largest island in the world.) -N--elaboration--S- (It covers 840,000 sq mi. The population of Greenland is about 55,000.)


2.5 Evaluation: mononuclear

Definition: in an evaluation relation, the satellite assesses the situation presented in the nucleus. An evaluation can be an appraisal, estimation, rating. The evaluation can be the viewpoint of the writer or another agent in the text. 

Example: (Some people think skipping breakfast may not hurt health.) -N--evaluation--S- (This is wrong.)


2.6 Example: mononuclear (See Fig. 3 in Section 2.5.)


2.7 List: multinuclear

Definition: a list relation is a multinuclear relation whose elements can be listed, but which are not in a contrast relation. 

Example: This book has two parts. (The first part is about the history of this country.) -N--list--N- (The second part focuses on the influence of globalization.)


2.8 Purpose: mononuclear

Definition: in a purpose relation, the situation presented in the satellite is only putative, i.e. it is not yet to be achieved. Most of it can be paraphrased as ``nucleus in order to satellite''. 

Example: (To protect wild animals,) -S--purpose--N- (hunting is forbidden since this year.)


2.9 Reason: mononuclear

Definition: in a reason relation, the situation presented in the satellite is the reason of the situation presented in the nucleus. 

Example: (Because stars are far away,) -S--reason--N- (they look like shining dots.)


2.10 Restatement: mononuclear

Definition: in a restatement relation, the satellite reiterates the information presented in the nucleus, typically with slightly different wording. It does not add to or interpret the information. 

Example: (Save the Earth.) -N--restatement--S- (Save our planet.)


2.11 Summary: mononuclear

Definition: in a summary relation, the satellite summarizes the information presented in the nucleus. 

Example: (The war was over. John returned home. His wife fully recovered the memory.) -N--summary--S- (In a word, the book had a happy end.)


2.12 Time: mononuclear

Definition: in a time relation, the situation presented in the nucleus occurs after (or before, or at the same time) the situation presented in the satellite. 

Example: (John had many things to do) -N--time--S- (before he left for New York).


\section*{Appendix B}

\begin{center} \large{Training the two coders}  \end{center}

In this study, training coders can be divided into eight steps:

\begin{enumerate}
  \item Before training, we created a small test corpus. On one hand, the size of the 
test corpus ought to be as small as possible because the coders will be asked to annotate 
it in a short period of time. On the other hand, if the size of the
test corpus is too small, it is difficult to find the differences in annotation 
results between before training and after training by C4.5. 
In \cite{Eugenio97}, four experiments were done to explore
the usage of discourse markers for native speakers using C4.5. The sizes of 
the datasets used in the four experiments were 127, 155, 124 and 100. 
Since the average size of the four datasets was nearly 120, we used the 
first 120 examples of discourse marker \textit{because} 
to create a small test corpus in this study. We think that doing experiment by
C4.5 using a dataset containing 120 examples would be effective and reliable. 
  \item Two independent coders annotated the 120 examples in the test
corpus according to their understanding of the manual. It lasted two weeks. 
  \item We analyzed the annotation results of the two coders 
from three aspects of the \textit{whole structure} (see Section 2.5), 
i.e. boundary, discourse relation, nucleus and satellite. 
The rate of agreement of the two coders was 45.9\% (the second column of Table 23). 
Then we analyzed the problems that caused the disagreement of the two 
coders and revised the manual. It lasted two weeks. 
  \item The two coders annotated the test corpus 
using the revised manual. It lasted two weeks. 
  \item The rate of agreement of the two coders became 64.9\% (the third column of Table 23). 
We analyzed the problems that caused the disagreement of the two coders and revised 
the manual again. It lasted two weeks.
  \item The two coders annotated the test corpus using 
the newly revised manual. It lasted two weeks.
  \item Since the rates of agreement of the two coders were equal to or higher 
than 95\% on the three aspects (the forth column of Table 23), we stopped 
training the coders. 
  \item The main coder annotated the left 952 (i.e. 1072-120) examples, 
which lasted 3 months. In order to assess the reliability of the annotation, 
the reliability coder annotated the first 30 examples of the six discourse 
markers except \textit{because} at the same time.
\end{enumerate}

\begin{table}[t]
\caption{Rate of agreement of the two coders}
\input{01t23.txt}
\end{table}


As shown in Table 23, the rate of agreement of the two coders after training 
was 42.1\% (i.e. $88.0\% - 45.9\%$) higher than that before training. 
In addition, we did two sets of experiments using the annotation results of 
the main coder to verify the effectiveness of training. First, annotation results 
before training were used to do 38 experiments (see Section 2.7). No best learned
model was obtained. Then we used the annotation results after training to do another 
38 experiments. The model learned from feature R is the best performing model (Table 8).
It is obvious that training coders can improve the performance of the learned models.




\acknowledgment

I would like to thank the anonymous JNLP reviewer for the helpful comments on this paper.

\bibliographystyle{jnlpbbl_1.3}
\begin{thebibliography}{}

\bibitem[\protect\BCAY{Bateman}{Bateman}{1997}]{Bateman97}
Bateman, J. \BBOP 1997\BBCP.
\newblock \BBOQ Enabling technology for multilingual natural language
  generation: The KPML development environment\BBCQ\
\newblock {\Bem Natural Language Engineering}, {\Bbf 3}  (1), \mbox{\BPGS\
  15--55}.

\bibitem[\protect\BCAY{Biber, Johansson, Leech, Conrad, \BBA\ Finegan}{Biber
  et~al.}{1999}]{Biber99}
Biber, D., Johansson, S., Leech, G., Conrad, S., \BBA\ Finegan, E. \BBOP
  1999\BBCP.
\newblock {\Bem Longman grammar of spoken and written English}.
\newblock Pearson Education Limited, England.

\bibitem[\protect\BCAY{Carlson \BBA\ Marcu}{Carlson \BBA\
  Marcu}{2001}]{Carlson01}
Carlson, L.\BBACOMMA\ \BBA\ Marcu, D. \BBOP 2001\BBCP.
\newblock \BBOQ Discourse tagging reference manual\BBCQ\
\newblock \BTR, USC Information Science Institute (ISI).

\bibitem[\protect\BCAY{Chandrasekar \BBA\ Srinivas}{Chandrasekar \BBA\
  Srinivas}{1997}]{Chandrasekar97}
Chandrasekar, R.\BBACOMMA\ \BBA\ Srinivas, B. \BBOP 1997\BBCP.
\newblock \BBOQ Automatic induction of rules for text simplification\BBCQ\
\newblock {\Bem Knowledge-Based System}, {\Bbf 10}  (3), \mbox{\BPGS\
  183--190}.

\bibitem[\protect\BCAY{Deng \BBA\ Nakamura}{Deng \BBA\ Nakamura}{2006}]{Deng06}
Deng, X.\BBACOMMA\ \BBA\ Nakamura, J. \BBOP 2006\BBCP.
\newblock \BBOQ Annotation of the discourse markers within the texts whose
  target audience was intermediate non-native speakers of English\BBCQ\
\newblock In {\Bem Proc. of the 85th Meeting of Informatics (IPSJ SIGFI-85)},
  \mbox{\BPGS\ 45--52}.

\bibitem[\protect\BCAY{Devlin, Canning, Tait, Carrol, Minnen, \BBA\
  Pearce}{Devlin et~al.}{2000}]{Devlin00}
Devlin, S., Canning, Y., Tait, J., Carrol, J., Minnen, G., \BBA\ Pearce, D.
  \BBOP 2000\BBCP.
\newblock \BBOQ Making accessible international communication for people with
  language comprehension difficulties\BBCQ\
\newblock In {\Bem Proc. of The 7th International Conference on Computers
  Helping People with Special Needs}, \mbox{\BPGS\ 135--142}.

\bibitem[\protect\BCAY{Devlin \BBA\ Tait}{Devlin \BBA\ Tait}{1998}]{Devlin98}
Devlin, S.\BBACOMMA\ \BBA\ Tait, J. \BBOP 1998\BBCP.
\newblock \BBOQ The use of a psycholinguistic database in the simplification of
  text for aphasic readers\BBCQ\
\newblock In Nerbonne, J.\BED, {\Bem Linguistic Databases}, \mbox{\BPGS\
  161--173}. CSLI, New York.

\bibitem[\protect\BCAY{Eugenio, Moore, \BBA\ Paolucci}{Eugenio
  et~al.}{1997}]{Eugenio97}
Eugenio, B., Moore, J., \BBA\ Paolucci, M. \BBOP 1997\BBCP.
\newblock \BBOQ Learning Features that Predict Cue Usage\BBCQ\
\newblock In {\Bem Proc. of the 35th Annual Meeting of the Association for
  Computational Linguistics}, \mbox{\BPGS\ 80--87}.

\bibitem[\protect\BCAY{Goldberg, Driedger, \BBA\ Kittredge}{Goldberg
  et~al.}{1994}]{Goldberg94}
Goldberg, E., Driedger, N., \BBA\ Kittredge, R. \BBOP 1994\BBCP.
\newblock \BBOQ Using natural language processing to produce weather
  forecasts\BBCQ\
\newblock {\Bem IEEE Expert}, {\Bbf 9}  (2), \mbox{\BPGS\ 45--53}.

\bibitem[\protect\BCAY{Knott}{Knott}{1996}]{Knott96}
Knott, A. \BBOP 1996\BBCP.
\newblock {\Bem A Data-Driven Methodology for Motivating a Set of Coherence
  Relations}.
\newblock Ph.D.\ thesis, University of Edinburgh.

\bibitem[\protect\BCAY{Litman}{Litman}{1996}]{Litman96}
Litman, D. \BBOP 1996\BBCP.
\newblock \BBOQ Cue Phrase Classification Using Machine Learning\BBCQ\
\newblock {\Bem Artificial Intelligence Research}, {\Bbf 5}, \mbox{\BPGS\
  53--94}.

\bibitem[\protect\BCAY{Mann \BBA\ Thompson}{Mann \BBA\ Thompson}{1988}]{Mann88}
Mann, W.\BBACOMMA\ \BBA\ Thompson, S. \BBOP 1988\BBCP.
\newblock \BBOQ Rhetorical structure theory: Toward a functional theory of text
  organization\BBCQ\
\newblock {\Bem Text}, {\Bbf 8}  (3), \mbox{\BPGS\ 243--281}.

\bibitem[\protect\BCAY{McKeown}{McKeown}{1985}]{McKeown85}
McKeown, K. \BBOP 1985\BBCP.
\newblock {\Bem Text Generation}.
\newblock Cambridge University Press.

\bibitem[\protect\BCAY{Moser \BBA\ Moore}{Moser \BBA\ Moore}{1995a}]{Moser95a}
Moser, M.\BBACOMMA\ \BBA\ Moore, J. \BBOP 1995a\BBCP.
\newblock \BBOQ Investigating cue selection and placement in tutorial
  discourse\BBCQ\
\newblock In {\Bem Proc. of the 33rd Annual Meeting of the Association for
  Computational Linguistics}, \mbox{\BPGS\ 130--135}.

\bibitem[\protect\BCAY{Moser \BBA\ Moore}{Moser \BBA\ Moore}{1995b}]{Moser95b}
Moser, M.\BBACOMMA\ \BBA\ Moore, J. \BBOP 1995b\BBCP.
\newblock \BBOQ Using discourse analysis and automatic text generation to study
  discourse cue usage\BBCQ\
\newblock In {\Bem Proc. of AAAI Spring Symposium Series: Empirical Methods in
  Discourse Interpretation and Generation}, \mbox{\BPGS\ 92--98}.

\bibitem[\protect\BCAY{Quinlan}{Quinlan}{1993}]{Quinlan93}
Quinlan, R. \BBOP 1993\BBCP.
\newblock {\Bem C4.5: Programs for Machine Learning}.
\newblock Morgan Kaufmann.

\bibitem[\protect\BCAY{Quirk, Greenbaum, Leech, \BBA\ Svartvik}{Quirk
  et~al.}{1972}]{Quirk72}
Quirk, R., Greenbaum, S., Leech, G., \BBA\ Svartvik, J. \BBOP 1972\BBCP.
\newblock {\Bem A Grammar of contemporary English}.
\newblock Longman, London.

\bibitem[\protect\BCAY{Scott \BBA\ Souza}{Scott \BBA\ Souza}{1990}]{Scott90}
Scott, D.\BBACOMMA\ \BBA\ Souza, C. \BBOP 1990\BBCP.
\newblock \BBOQ Getting the Message Across in RST-based Text Generation\BBCQ\
\newblock In Dale, R., Mellish, C., \BBA\ M., Z.\BEDS, {\Bem Current Research
  in Natural Language Generation}, \mbox{\BPGS\ 47--73}. Academic Press.

\bibitem[\protect\BCAY{Underhill}{Underhill}{1988}]{Underhill88}
Underhill, R. \BBOP 1988\BBCP.
\newblock \BBOQ Like Is, Like, Focus\BBCQ\
\newblock {\Bem American Speech}, {\Bbf 63}  (3), \mbox{\BPGS\ 234--246}.

\bibitem[\protect\BCAY{Vapnik}{Vapnik}{1999}]{Vapnik99}
Vapnik, V.~N. \BBOP 1999\BBCP.
\newblock {\Bem The Nature of Statistical Learning Theory}.
\newblock Springer Verlag.

\bibitem[\protect\BCAY{Weiss \BBA\ Kulikowski}{Weiss \BBA\
  Kulikowski}{1991}]{Weiss91}
Weiss, S.\BBACOMMA\ \BBA\ Kulikowski, C. \BBOP 1991\BBCP.
\newblock {\Bem Computer Systems That Learn: Classification and Prediction
  Methods from Statistics, Neural Nets, Machine Learning, and Expert Systems}.
\newblock San Mateo, CA: Morgan Kaufmann.

\bibitem[\protect\BCAY{Williams}{Williams}{2003}]{Will03}
Williams, S. \BBOP 2003\BBCP.
\newblock \BBOQ Language choice models for microplanning and readability\BBCQ\
\newblock In {\Bem Proc. of the Student Workshop of the Human Language
  Technology and North American Chapter of the Association for Computational
  Linguistics Conference}, \mbox{\BPGS\ 13--18}.

\bibitem[\protect\BCAY{Williams}{Williams}{2004}]{Will04}
Williams, S. \BBOP 2004\BBCP.
\newblock {\Bem Natural language generation (NLG) of discourse relations for
  different reading levels}.
\newblock Ph.D.\ thesis, University of Aberdeen.

\end{thebibliography}

\begin{biography}

\bioauthor[:]{Xinyu Deng}{
Xinyu Deng received her B.E. and M.A. from Harbin Institute of Technology, 
China. From 2001 to 2003 she worked for R\&D center of Toshiba, Tokyo, Japan. 
She completed her Ph.D. course of Kyoto University in 2006. She is currently 
doing research at Kyoto University. Her research interests include natural 
language processing, machine translation and corpus linguistics. 
}

\bioauthor[:]{Jun-ichi Nakamura}{
(1956--2001) Jun-ichi Nakamura received his BSc, MSc and DSc in electrical engineering from Kyoto University in 1979, 1981, and 1989. From 1997 to 2001 he was a professor in Kyoto University.
}

\end{biography}

\biodate


\end{document}
