{\rtf1\ansi\ansicpg1252\uc1 \deff0\deflang1033\deflangfe1033{\fonttbl{\f0\froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\f1\fswiss\fcharset0\fprq2{\*\panose 020b0604020202020204}Arial;}
{\f2\fmodern\fcharset0\fprq1{\*\panose 02070309020205020404}Courier New;}{\f4\froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times;}{\f15\fswiss\fcharset0\fprq2{\*\panose 020b0604030504040204}Tahoma;}
{\f44\froman\fcharset128\fprq1{\*\panose 00000000000000000000}MS Mincho{\*\falt MS ??};}{\f46\fnil\fcharset134\fprq2{\*\panose 02010600030101010101}SimSun{\*\falt ??};}{\f47\fswiss\fcharset128\fprq2{\*\panose 020b0604020202020204}Arial Unicode MS;}
{\f48\froman\fcharset238\fprq2 Times New Roman CE;}{\f49\froman\fcharset204\fprq2 Times New Roman Cyr;}{\f51\froman\fcharset161\fprq2 Times New Roman Greek;}{\f52\froman\fcharset162\fprq2 Times New Roman Tur;}
{\f53\froman\fcharset186\fprq2 Times New Roman Baltic;}{\f54\fswiss\fcharset238\fprq2 Arial CE;}{\f55\fswiss\fcharset204\fprq2 Arial Cyr;}{\f57\fswiss\fcharset161\fprq2 Arial Greek;}{\f58\fswiss\fcharset162\fprq2 Arial Tur;}
{\f59\fswiss\fcharset186\fprq2 Arial Baltic;}{\f60\fmodern\fcharset238\fprq1 Courier New CE;}{\f61\fmodern\fcharset204\fprq1 Courier New Cyr;}{\f63\fmodern\fcharset161\fprq1 Courier New Greek;}{\f64\fmodern\fcharset162\fprq1 Courier New Tur;}
{\f65\fmodern\fcharset186\fprq1 Courier New Baltic;}{\f138\fswiss\fcharset238\fprq2 Tahoma CE;}{\f139\fswiss\fcharset204\fprq2 Tahoma Cyr;}{\f141\fswiss\fcharset161\fprq2 Tahoma Greek;}{\f142\fswiss\fcharset162\fprq2 Tahoma Tur;}
{\f143\fswiss\fcharset186\fprq2 Tahoma Baltic;}{\f326\fnil\fcharset0\fprq2 SimSun{\*\falt ??};}{\f332\fswiss\fcharset0\fprq2 Arial Unicode MS;}{\f330\fswiss\fcharset238\fprq2 Arial Unicode MS CE;}{\f331\fswiss\fcharset204\fprq2 Arial Unicode MS Cyr;}
{\f333\fswiss\fcharset161\fprq2 Arial Unicode MS Greek;}{\f334\fswiss\fcharset162\fprq2 Arial Unicode MS Tur;}{\f335\fswiss\fcharset186\fprq2 Arial Unicode MS Baltic;}}{\colortbl;\red0\green0\blue0;\red0\green0\blue255;\red0\green255\blue255;
\red0\green255\blue0;\red255\green0\blue255;\red255\green0\blue0;\red255\green255\blue0;\red255\green255\blue255;\red0\green0\blue128;\red0\green128\blue128;\red0\green128\blue0;\red128\green0\blue128;\red128\green0\blue0;\red128\green128\blue0;
\red128\green128\blue128;\red192\green192\blue192;}{\stylesheet{\qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid \snext0 Normal;}{\s1\sb120\sa120\keepn\nowidctlpar\tx540\adjustright \b\f4\cgrid \sbasedon0 \snext0 \sautoupd heading 1;}{
\s2\qc\li426\ri480\sb120\sa120\keepn\widctlpar\adjustright \b\f4\cgrid \sbasedon0 \snext0 \sautoupd heading 2;}{\s3\sb240\sa120\keepn\widctlpar\tx540\adjustright \b\f4\lang2057\cgrid \sbasedon0 \snext0 heading 3;}{\*\cs10 \additive 
Default Paragraph Font;}{\s15\qc\ri4\sb120\sa20\widctlpar\tx560\tx1134\tx2268\tqc\tx4252\tqr\tx8504\adjustright \v\f4\fs18\lang1036\cgrid \sbasedon0 \snext15 footer;}{\s16\qj\sb20\sa120\widctlpar\tx560\tx1134\tx2268\tqc\tx4252\tqr\tx9100\adjustright 
\i\v\f4\fs20\lang1036\cgrid \sbasedon0 \snext16 header;}{\*\cs17 \additive \fs14\up6 \sbasedon10 footnote reference;}{\s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid \sbasedon0 \snext18 footnote text;}{
\s19\qc\fi284\widctlpar\adjustright \f4\fs22\lang2070\cgrid \sbasedon0 \snext19 \sautoupd Bibliography;}{\*\cs20 \additive \i \sbasedon10 Emphasis;}{\s21\qc\sb20\keepn\nowidctlpar\tqc\tx1980\tqc\tx4520\tqc\tx7100\tqr\tx9000\adjustright 
\f4\fs22\lang2057\cgrid \sbasedon0 \snext21 Author;}{\s22\qc\sa300\sl280\slmult0\keepn\nowidctlpar\tx560\adjustright \b\f4\fs30\cgrid \sbasedon0 \snext22 \sautoupd Title;}{\s23\qc\ri210\widctlpar\adjustright \f2\fs18\cgrid \snext23 E-mail address;}{\*
\cs24 \additive \ul\cf2 \sbasedon10 Hyperlink;}{\s25\qj\li426\ri849\keep\nowidctlpar\adjustright \f4\fs22\cgrid \sbasedon0 \snext25 Abstract1;}{\s26\qc\sa300\sl280\slmult0\keepn\nowidctlpar\tx560\adjustright \b\f4\cgrid \sbasedon22 \snext26 Title2;}{
\s27\fi-567\li567\sb240\sa240\widctlpar\jclisttab\tx567\ls8\adjustright \b\cgrid \snext27 \sautoupd Section Heading;}{\s28\fi-567\li567\sb240\sa120\widctlpar\jclisttab\tx567\ls8\ilvl2\adjustright \b\fs22\cgrid \sbasedon27 \snext28 \sautoupd 
Subsubsection Heading;}{\s29\fi-567\li567\sb240\sa120\widctlpar\jclisttab\tx567\ls8\ilvl1\adjustright \b\fs22\cgrid \sbasedon27 \snext29 \sautoupd Subsection Heading;}{\s30\qj\nowidctlpar\adjustright \f2\fs18\cgrid \sbasedon0 \snext30 File Name;}{
\s31\qc\ri210\nowidctlpar\adjustright \f2\fs18\cgrid \sbasedon23 \snext31 URL;}{\s32\qj\li426\ri480\keep\nowidctlpar\adjustright \f4\fs22\cgrid \sbasedon0 \snext32 Abstract2;}{\s33\qj\li283\sa120\widctlpar\adjustright \f4\fs22\lang3084\cgrid 
\sbasedon0 \snext33 Body Text 2;}{\s34\qj\fi210\li283\sa120\widctlpar\adjustright \f4\fs22\lang2070\cgrid \sbasedon33 \snext34 Body Text First Indent 2;}{\s35\qj\widctlpar\adjustright \cbpat9 \f15\fs22\lang3084\cgrid \sbasedon0 \snext35 Document Map;}{
\s36\qc\fi284\nowidctlpar\adjustright \f4\fs22\cgrid \sbasedon0 \snext36 Body Text Indent 2;}{\s37\widctlpar\adjustright \fs22\cgrid \sbasedon0 \snext37 Plain Text;}{\s38\qj\li284\sb60\sa60\widctlpar\adjustright \fs18\cgrid \sbasedon37 \snext38 \sautoupd 
url;}{\s39\qj\fi-227\li227\sb120\widctlpar\adjustright \fs20\cgrid \sbasedon37 \snext39 \sautoupd References;}{\*\cs40 \additive \ul\cf12 \sbasedon10 FollowedHyperlink;}{\s41\qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid \sbasedon0 \snext41 
Body Text Indent 3;}}{\*\listtable{\list\listtemplateid431411992{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'00;}{\levelnumbers\'01;}
\b\i0\strike0\outl0\shad0\embo0\impr0\caps0\v0\f0\fs20\cf0\nosupersub\striked0\fbias0 \fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'03\'00.\'01;}{\levelnumbers\'01\'03;}
\b\i0\strike0\outl0\shad0\embo0\impr0\f0\fs20\cf0\nosupersub\striked0\fbias0 \fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'05\'00.\'01.\'02;}{\levelnumbers\'01\'03\'05;}
\b\i\f0\fs20\fbias0 \fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'08\'00.\'01.\'02.\'03.;}{\levelnumbers\'01\'03\'05\'07;}\f0\fbias0 \fi-567\li567\jclisttab\tx720 }
{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'0a\'00.\'01.\'02.\'03.\'04.;}{\levelnumbers\'01\'03\'05\'07\'09;}\f0\fbias0 \fi-567\li567\jclisttab\tx1080 }{\listlevel\levelnfc255\leveljc0\levelfollow0
\levelstartat1\levelspace0\levelindent0{\leveltext\'0c\'00.\'01.\'02.\'03.\'04.\'05.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b;}\f0\fbias0 \fi-567\li567\jclisttab\tx1080 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0
{\leveltext\'0e\'00.\'01.\'02.\'03.\'04.\'05.\'06.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d;}\f0\fbias0 \fi-567\li567\jclisttab\tx1080 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'10\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f;}\f0\fs20\fbias0 \fi-567\li567\jclisttab\tx1440 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'12\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.\'08.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f\'11;}\f0\fbias0 \fi-567\li567\jclisttab\tx1440 }{\listname ;}\listid14162699}{\list\listtemplateid1973727556{\listlevel\levelnfc0\leveljc0\levelfollow0
\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'00;}{\levelnumbers\'01;}\b\i0\strike0\outl0\shad0\embo0\impr0\caps0\v0\f0\fs24\cf0\nosupersub\striked0\fbias0 \fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1
\levelnorestart\levelspace0\levelindent0{\leveltext\'03\'00.\'01;}{\levelnumbers\'01\'03;}\b\i0\strike0\outl0\shad0\embo0\impr0\f0\fs24\cf0\nosupersub\striked0\fbias0 \fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1
\levelspace0\levelindent0{\leveltext\'05\'00.\'01.\'02;}{\levelnumbers\'01\'03\'05;}\b\i\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'08\'00.\'01.\'02.\'03.;}{\levelnumbers\'01\'03\'05\'07;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx720 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'0a\'00.\'01.\'02.\'03.\'04.;}{\levelnumbers
\'01\'03\'05\'07\'09;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx1080 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'0c\'00.\'01.\'02.\'03.\'04.\'05.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b;}
\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx1080 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'0e\'00.\'01.\'02.\'03.\'04.\'05.\'06.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d;}\f0\fs24\fbias0 
\fi-567\li567\jclisttab\tx1440 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'10\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f;}\f0\fs24\fbias0 \fi-567\li567
\jclisttab\tx1440 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'12\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.\'08.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f\'11;}\f0\fs24\fbias0 \fi-567\li567
\jclisttab\tx1800 }{\listname ;}\listid87311056}{\list\listtemplateid-968954502\listsimple{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01-;}{\levelnumbers;}\f0\fbias0 \fi-360\li644\jclisttab\tx644 }
{\listname ;}\listid208417102}{\list\listtemplateid-1565228526{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'00;}{\levelnumbers\'01;}
\b\i0\strike0\outl0\shad0\embo0\impr0\caps0\v0\f0\fs24\cf0\nosupersub\striked0\fbias0 \fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'03\'00.\'01;}{\levelnumbers\'01\'03;}
\b\i0\strike0\outl0\shad0\embo0\impr0\f0\fs24\cf0\nosupersub\striked0\fbias0 \fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'05\'00.\'01.\'02;}{\levelnumbers\'01\'03\'05;}
\b\i\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'08\'00.\'01.\'02.\'03.;}{\levelnumbers\'01\'03\'05\'07;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx720 }
{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'0a\'00.\'01.\'02.\'03.\'04.;}{\levelnumbers\'01\'03\'05\'07\'09;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx1080 }{\listlevel\levelnfc255\leveljc0\levelfollow0
\levelstartat1\levelspace0\levelindent0{\leveltext\'0c\'00.\'01.\'02.\'03.\'04.\'05.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx1080 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0
\levelindent0{\leveltext\'0e\'00.\'01.\'02.\'03.\'04.\'05.\'06.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx1440 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'10\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx1440 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'12\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.\'08.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f\'11;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx1800 }{\listname ;}\listid254753849}{\list\listtemplateid-968954502\listsimple{\listlevel\levelnfc23\leveljc0
\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01-;}{\levelnumbers;}\f0\fbias0 \fi-360\li644\jclisttab\tx644 }{\listname ;}\listid277374157}{\list\listtemplateid67567631\listsimple{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1
\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid325596128}{\list\listtemplateid-1170077030{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'01\'00;}{\levelnumbers\'01;}\fbias0 \fi-540\li540\jclisttab\tx540 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext\'03\'00.\'01;}{\levelnumbers\'01\'03;}\fbias0 \fi-540\li540\jclisttab\tx540 }
{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext\'05\'00.\'01.\'02;}{\levelnumbers\'01\'03\'05;}\fbias0 \fi-720\li720\jclisttab\tx720 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1
\levellegal\levelspace0\levelindent0{\leveltext\'07\'00.\'01.\'02.\'03;}{\levelnumbers\'01\'03\'05\'07;}\fbias0 \fi-720\li720\jclisttab\tx720 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'09\'00.\'01.\'02.\'03.\'04;}{\levelnumbers\'01\'03\'05\'07\'09;}\fbias0 \fi-720\li720\jclisttab\tx720 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'0b\'00.\'01.\'02.\'03.\'04.\'05;}{\levelnumbers\'01\'03\'05\'07\'09\'0b;}\fbias0 \fi-1080\li1080\jclisttab\tx1080 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'0d\'00.\'01.\'02.\'03.\'04.\'05.\'06;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d;}\fbias0 \fi-1080\li1080\jclisttab\tx1080 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'0f\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f;}\fbias0 \fi-1440\li1440\jclisttab\tx1440 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'11\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.\'08;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f\'11;}\fbias0 \fi-1440\li1440\jclisttab\tx1440 }{\listname ;}\listid420757796}{\list\listtemplateid930792390{\listlevel\levelnfc0\leveljc0\levelfollow0
\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'00;}{\levelnumbers\'01;}\b\i0\strike0\outl0\shad0\embo0\impr0\caps0\v0\f0\fs28\cf0\nosupersub\striked0\fbias0 \fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1
\levelspace0\levelindent0{\leveltext\'03\'00.\'01;}{\levelnumbers\'01\'03;}\b\i0\strike0\outl0\shad0\embo0\impr0\f0\fs28\cf0\nosupersub\striked0\fbias0 \fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0
\levelindent0{\leveltext\'05\'00.\'01.\'02;}{\levelnumbers\'01\'03\'05;}\b\i\f0\fs28\fbias0 \fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'08\'00.\'01.\'02.\'03.;}{\levelnumbers\'01\'03\'05\'07;}\f0\fs28\fbias0 \fi-567\li567\jclisttab\tx1080 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'0a\'00.\'01.\'02.\'03.\'04.;}{\levelnumbers
\'01\'03\'05\'07\'09;}\f0\fs28\fbias0 \fi-567\li567\jclisttab\tx1080 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'0c\'00.\'01.\'02.\'03.\'04.\'05.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b;}
\f0\fs28\fbias0 \fi-567\li567\jclisttab\tx1440 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'0e\'00.\'01.\'02.\'03.\'04.\'05.\'06.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d;}\f0\fs28\fbias0 
\fi-567\li567\jclisttab\tx1800 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'10\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f;}\f0\fs28\fbias0 \fi-567\li567
\jclisttab\tx1800 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'12\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.\'08.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f\'11;}\f0\fs28\fbias0 \fi-567\li567
\jclisttab\tx2160 }{\listname ;}\listid430975587}{\list\listtemplateid-1989921058{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'00;}{\levelnumbers\'01;}\fbias0 \fi-705\li705\jclisttab\tx705 }{\listlevel
\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'03\'00.\'01;}{\levelnumbers\'01\'03;}\fbias0 \fi-705\li705\jclisttab\tx705 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'05\'00.\'01.\'02;}{\levelnumbers\'01\'03\'05;}\fbias0 \fi-720\li720\jclisttab\tx720 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'07\'00.\'01.\'02.\'03;}{\levelnumbers\'01\'03\'05\'07;}\fbias0 
\fi-720\li720\jclisttab\tx720 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'09\'00.\'01.\'02.\'03.\'04;}{\levelnumbers\'01\'03\'05\'07\'09;}\fbias0 \fi-1080\li1080\jclisttab\tx1080 }{\listlevel\levelnfc0
\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'0b\'00.\'01.\'02.\'03.\'04.\'05;}{\levelnumbers\'01\'03\'05\'07\'09\'0b;}\fbias0 \fi-1080\li1080\jclisttab\tx1080 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1
\levelspace0\levelindent0{\leveltext\'0d\'00.\'01.\'02.\'03.\'04.\'05.\'06;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d;}\fbias0 \fi-1440\li1440\jclisttab\tx1440 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'0f\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f;}\fbias0 \fi-1440\li1440\jclisttab\tx1440 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'11\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.\'08;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f\'11;}\fbias0 \fi-1440\li1440\jclisttab\tx1440 }{\listname ;}\listid718169112}{\list\listtemplateid67698703\listsimple{\listlevel\levelnfc0\leveljc0
\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid999310495}{\list\listtemplateid67698703\listsimple{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1
\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid1041704647}{\list\listtemplateid-968954502\listsimple{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0
{\leveltext\'01-;}{\levelnumbers;}\f0\fbias0 \fi-360\li644\jclisttab\tx644 }{\listname ;}\listid1111631587}{\list\listtemplateid67698703\listsimple{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid1181704094}{\list\listtemplateid1230822526{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'00;}{\levelnumbers\'01;}
\b\i0\strike0\outl0\shad0\embo0\impr0\caps0\v0\f0\fs24\cf0\nosupersub\striked0\fbias0 \s27\fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'03\'00.\'01;}{\levelnumbers\'01\'03;}
\b\i0\strike0\outl0\shad0\embo0\impr0\f0\fs24\cf0\nosupersub\striked0\fbias0 \s29\fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'05\'00.\'01.\'02;}{\levelnumbers\'01\'03\'05;}
\b\i0\f0\fs24\fbias0 \s28\fi-567\li567\jclisttab\tx567 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'08\'00.\'01.\'02.\'03.;}{\levelnumbers\'01\'03\'05\'07;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx720 
}{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'0a\'00.\'01.\'02.\'03.\'04.;}{\levelnumbers\'01\'03\'05\'07\'09;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx1080 }{\listlevel\levelnfc255\leveljc0
\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'0c\'00.\'01.\'02.\'03.\'04.\'05.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx1080 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1
\levelspace0\levelindent0{\leveltext\'0e\'00.\'01.\'02.\'03.\'04.\'05.\'06.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx1440 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0
{\leveltext\'10\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx1440 }{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'12\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.\'08.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f\'11;}\f0\fs24\fbias0 \fi-567\li567\jclisttab\tx1800 }{\listname ;}\listid1957633416}{\list\listtemplateid-1170077030{\listlevel\levelnfc0\leveljc0
\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'00;}{\levelnumbers\'01;}\fbias0 \fi-540\li540\jclisttab\tx540 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'03\'00.\'01;}{\levelnumbers\'01\'03;}\fbias0 \fi-540\li540\jclisttab\tx540 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext\'05\'00.\'01.\'02;}{\levelnumbers\'01\'03\'05;}\fbias0 \fi-720\li720
\jclisttab\tx720 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext\'07\'00.\'01.\'02.\'03;}{\levelnumbers\'01\'03\'05\'07;}\fbias0 \fi-720\li720\jclisttab\tx720 }{\listlevel\levelnfc0\leveljc0
\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext\'09\'00.\'01.\'02.\'03.\'04;}{\levelnumbers\'01\'03\'05\'07\'09;}\fbias0 \fi-720\li720\jclisttab\tx720 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal
\levelspace0\levelindent0{\leveltext\'0b\'00.\'01.\'02.\'03.\'04.\'05;}{\levelnumbers\'01\'03\'05\'07\'09\'0b;}\fbias0 \fi-1080\li1080\jclisttab\tx1080 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0
{\leveltext\'0d\'00.\'01.\'02.\'03.\'04.\'05.\'06;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d;}\fbias0 \fi-1080\li1080\jclisttab\tx1080 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'0f\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f;}\fbias0 \fi-1440\li1440\jclisttab\tx1440 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'11\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.\'08;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f\'11;}\fbias0 \fi-1440\li1440\jclisttab\tx1440 }{\listname ;}\listid2071927807}{\list\listtemplateid67698703\listsimple{\listlevel\levelnfc0\leveljc0
\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid2093428807}}{\*\listoverridetable{\listoverride\listid718169112\listoverridecount0\ls1}
{\listoverride\listid14162699\listoverridecount0\ls2}{\listoverride\listid420757796\listoverridecount0\ls3}{\listoverride\listid2071927807\listoverridecount0\ls4}{\listoverride\listid430975587\listoverridecount0\ls5}{\listoverride\listid254753849
\listoverridecount0\ls6}{\listoverride\listid87311056\listoverridecount0\ls7}{\listoverride\listid1957633416\listoverridecount0\ls8}{\listoverride\listid325596128\listoverridecount0\ls9}{\listoverride\listid277374157\listoverridecount0\ls10}
{\listoverride\listid1111631587\listoverridecount0\ls11}{\listoverride\listid208417102\listoverridecount0\ls12}{\listoverride\listid999310495\listoverridecount0\ls13}{\listoverride\listid999310495\listoverridecount0\ls14}{\listoverride\listid999310495
\listoverridecount0\ls15}{\listoverride\listid1041704647\listoverridecount0\ls16}}{\info{\title ACL 2001 electronic submission}{\author Diana Santos}{\operator Diana Santos}{\creatim\yr2001\mo5\dy16\hr12\min32}{\revtim\yr2001\mo5\dy16\hr12\min32}
{\printim\yr2001\mo5\dy9\hr17\min36}{\version2}{\edmins0}{\nofpages6}{\nofwords4280}{\nofchars24396}{\*\company SINTEF Tele og Data}{\nofcharsws29960}{\vern89}}\paperw11907\paperh16840\margl1474\margr1474\margt1474\margb2155 
\deftab708\widowctrl\ftnbj\aenddoc\hyphhotz425\makebackup\noxlattoyen\expshrtn\noultrlspc\dntblnsbdb\nospaceforul\hyphcaps0\viewkind1\viewscale100\pgbrdrhead\pgbrdrfoot \fet0{\*\template K:\\submission\\acl2001-submission.dot}\sectd 
\psz9\linex0\headery709\footery709\colsx709\endnhere\sectunlocked1\sectdefaultcl {\header \pard\plain \s16\qj\sb20\sa120\nowidctlpar\tx560\tx1134\tx2268\tqc\tx4252\tqr\tx9100\adjustright \i\v\f4\fs20\lang1036\cgrid {
\par }}{\footer \pard\plain \s15\qc\ri4\sb120\sa20\nowidctlpar\tx560\tx1134\tx2268\tqc\tx4252\tqr\tx8504\adjustright \v\f4\fs18\lang1036\cgrid {
\par }}{\*\pnseclvl1\pnucrm\pnstart1\pnindent720\pnhang{\pntxta .}}{\*\pnseclvl2\pnucltr\pnstart1\pnindent720\pnhang{\pntxta .}}{\*\pnseclvl3\pndec\pnstart1\pnindent720\pnhang{\pntxta .}}{\*\pnseclvl4\pnlcltr\pnstart1\pnindent720\pnhang{\pntxta )}}
{\*\pnseclvl5\pndec\pnstart1\pnindent720\pnhang{\pntxtb (}{\pntxta )}}{\*\pnseclvl6\pnlcltr\pnstart1\pnindent720\pnhang{\pntxtb (}{\pntxta )}}{\*\pnseclvl7\pnlcrm\pnstart1\pnindent720\pnhang{\pntxtb (}{\pntxta )}}{\*\pnseclvl8
\pnlcltr\pnstart1\pnindent720\pnhang{\pntxtb (}{\pntxta )}}{\*\pnseclvl9\pnlcrm\pnstart1\pnindent720\pnhang{\pntxtb (}{\pntxta )}}\pard\plain \s22\qc\sa300\sl280\slmult0\keepn\nowidctlpar\tx560\adjustright \b\f4\fs30\cgrid {Evaluating CETEMP}{\lang2070 
\'fa}{blico, a free resource for Portuguese
\par }\trowd \trqc\trgaph70\trleft72 \clvertalt\cltxlrtb \cellx4321\clvertalt\cltxlrtb \cellx8857\pard\plain \s37\qc\widctlpar\intbl\adjustright \fs22\cgrid {\b\fs26\lang1031 \hich\af0\dbch\af44\loch\f0 Diana Santos}{\fs26\lang1031 
\par \hich\af0\dbch\af44\loch\f0 SINTEF Tele og Data
\par \hich\af0\dbch\af44\loch\f0 Postboks 124, Blindern
\par \hich\af0\dbch\af44\loch\f0 N-0314 Oslo, Norway
\par }\pard\plain \s38\qj\li284\sb60\sa60\widctlpar\intbl\adjustright \fs18\cgrid {\lang1031 \hich\af2\dbch\af44\loch\f2 Diana.Santos@informatics.sintef.no\cell }\pard\plain \s37\qc\widctlpar\intbl\adjustright \fs22\cgrid {\b\fs26\lang1040 
\hich\af0\dbch\af44\loch\f0 Paulo Rocha
\par }{\fs26\lang1040 \hich\af0\dbch\af44\loch\f0 Departamento de Inform}{\fs26\lang2070 \loch\af0\dbch\af44\hich\f0 \'e1\loch\f0 tica
\par }{\fs26\lang1040 \hich\af0\dbch\af44\loch\f0 Universidade do Minho
\par }{\fs26\lang2070 PT-4710-057 Braga, Portugal}{\fs26\lang1040 
\par }\pard\plain \s38\qc\li284\sb60\sa60\widctlpar\intbl\adjustright \fs18\cgrid {\lang1040 \hich\af2\dbch\af44\loch\f2 Paulo.Rocha@alfa.di.uminho.pt\cell }\pard\plain \widctlpar\intbl\adjustright \f4\fs22\lang3084\cgrid {\fs26\lang1040 \row }\pard\plain 
\s1\sb120\sa120\sl-1134\slmult0\keepn\nowidctlpar\tx540\outlinelevel0\adjustright \b\f4\cgrid {\fs22 
\par }\pard\plain \qj\sl-1134\slmult0\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 \sect }\sectd \marglsxn1418\margrsxn1418\margtsxn1531\margbsxn1701\psz9\sbknone\linex0\headery709\footery709\cols2\colsx454\endnhere\sectunlocked1\sectdefaultcl 
\pard\plain \s2\qc\li426\ri480\sb120\sa120\keepn\widctlpar\outlinelevel1\adjustright \b\f4\cgrid {Abstract
\par }\pard\plain \s32\qj\li426\ri480\keep\nowidctlpar\adjustright \f4\fs22\cgrid {In this paper we present a thorough evaluation of a corpus resource for Portuguese, CETEMP\'fa
blico, a 180-million word newspaper corpus free for R&D in Portuguese processing. We provide information that should be us
eful to those using the resource, and to considerable improvement for later versions. In addition, we think that the procedures presented can be of interest for the larger NLP community, since corpus evaluation and description is unfortunately not a commo
n exercise.
\par {\listtext\pard\plain\s27 \b\cgrid \hich\af0\dbch\af0\loch\f0 1\tab}}\pard\plain \s27\fi-567\li567\sb240\sa240\widctlpar\jclisttab\tx567\ls8\adjustright \b\cgrid {Introduction
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 CETEMP\'fablico is a large corpus of European Portuguese newspaper language, available at no cost to the community dealing with the processing of Portuguese.}{
\cs17\fs14\up6\lang1033 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ }{\i\lang2070 CETEMP\'fablico}{\lang2070  stands for }{\ldblquote }{\lang2070 
Corpus de Extractos de Textos Electr\'f3nicos MCT / P\'fablico}{\rdblquote }{\lang2070 , and its full reference is }{\field\flddirty{\*\fldinst {\lang2070  HYPERLINK "http://cgi.portugues.mct.pt/cetempublico/" }{{\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000002a00000068007400740070003a002f002f006300670069002e0070006f0072007400750067007500650073002e006d00630074002e00700074002f0063006500740065006d007000750062006c00690063006f002f000000e0c9ea79f9bace118c8200aa004b
a90b5400000068007400740070003a002f002f006300670069002e0070006f0072007400750067007500650073002e006d00630074002e00700074002f0063006500740065006d007000750062006c00690063006f002f000000}}}{\fldrslt {\cs24\f0\ul\cf2\lang2070 
http://cgi.portugues.mct.pt/cetempublico/}}}}}{\lang1033  It was created in the framework of the Computational Processing of Portuguese project, a government funded initiative to foster language engineering of the Portuguese language.}{
\cs17\fs14\up6\lang1033 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ See }{\field\flddirty{\*\fldinst { HYPERLINK "http://www.portugues.mct.pt/" }{
{\*\datafield 00d0c9ea79f9bace118c8200aa004ba90b0200000003000000e0c9ea79f9bace118c8200aa004ba90b3a00000068007400740070003a002f002f007700770077002e0070006f0072007400750067007500650073002e006d00630074002e00700074002f000000}}}{\fldrslt {\cs24\ul\cf2 
http://www.portugues.mct.pt/}}}}}{\lang1033 
\par }\pard\plain \s41\qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 Evaluating this resource, we have two main goals in mind: To contribute to improve its usefulness; and to suggest ways of going about as far as corpus eva
luation is concerned in general (noting that most corpora projects are simply described and not evaluated). 
\par }\pard\plain \qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 In fact, and despite the amount of research devoted to corpus processing nowadays, there is not much information about the actual corpora being processed, which may lead na}{
\lang1044 \'ef}{\lang1033 ve users and/or readers to conclude that this is not an interesting issue. In our opinion, that is the wrong conclusion. 
\par }\pard \qj\fi284\nowidctlpar\adjustright {\lang1033 There is, in fact, a lot to be said about any particular corpus. We believe, in addition, that such informat
ion should be available when one is buying, or even just browsing, a corpus, and it  should be taken into consideration when, in turn, systems or hypotheses are evaluated with the help of that corpus.
\par In this paper, we will solely be concerned with CETEMP\'fa
blico, but it is our belief that similar kinds of information could be published about different corpora. Our intention is to give a positive contribution both to the whole community involved in the processing of Portuguese and to the particular users of 
this corpus. At the moment of writing, 160 people have ordered (and, we assume, consequently received) it}{\cs17\fs14\up6\lang1033 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {
\cs17\fs14\up6 \chftn }{ Although we also made available a CQP (Christ et al., 1999) encoded version in March 2001, the vast majority of the users received the text-only version.}}}{\lang1033 
. There have also been more than four thousand queries via the Web site which gives access to the corpus. 
\par We want to provide evaluation data and describe how one can improve the corpus. We are genuinely interested in increasing its value, and have, since corpus release,}{\cs17\fs14\up6\lang1033 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar
\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ The corpus was ready in July 2000; the first copies were sent out in October, with the information that version 1.0 creation date was 25 July 2000.}}}{\lang1033 
 made available four patches (e-mailing this information to all \sect }\sectd \margtsxn1361\psz9\sbknone\linex0\headery709\footery709\cols2\colsx454\endnhere\sectunlocked1\sectdefaultcl \pard\plain \qj\nowidctlpar\adjustright \f4\fs22\lang3084\cgrid {
\lang1033 who ordered the corpus). We have also tried to considerably improve the Web page.
\par }\pard \qj\fi284\nowidctlpar\adjustright {\lang1033 We decided to concentrate on the evaluation of version 1.0, given that massive distribution was done of that particular version}{\cs17\fs14\up6\lang1033 \chftn {\footnote \pard\plain 
\s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ W}{\lang1033 e have no estimate
 of how many users have actually succeeded, or even tried, to apply the patches made available later on. We have just launched a Web questionnaire in order to have a better idea of our user community.}}}{\lang1033 
. Web access to the corpus (Santos and Bick, 2000) will not be dealt with here. Note that all trivial improvements described here have already been addressed in some patch.
\par {\listtext\pard\plain\s27 \b\cgrid \hich\af0\dbch\af0\loch\f0 2\tab}}\pard\plain \s27\fi-567\li567\sb240\sa240\widctlpar\jclisttab\tx567\ls8\adjustright \b\cgrid {Short technical description
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 As described in detail in Rocha and Santos (2000) and also in the FAQ at the corpus Web page, CETEMP\'fablico was built from the raw material provided by the Portuguese daily
 newspaper }{\i\lang1033 P\'fablico}{\lang1033 
: text files in Macintosh format, covering approximately the years 1991 to 1998, and including both published news articles and those created but not necessarily brought to print. These files were automatically tagged with a classificatio
n based on, but not identical to, the one used by the newspaper to identify sections, and with the semester the article was associated to. In addition, sentence separation, and title and author identification were automatically created. The texts were the
n divided in extracts with an average length of two paragraphs. These extracts were randomly shuffled (for copyright reasons) and numbered, and the final corpus was the ordered sequence of the extract numbers.
\par }\pard\plain \s41\qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid {To illustrate the corpus in text format, we present in Appendix A an extract that includes all possible tags with the exception of <marca>.
\par {\listtext\pard\plain\s27 \b\cgrid \hich\af0\dbch\af0\loch\f0 3\tab}}\pard\plain \s27\fi-567\li567\sb240\sa240\widctlpar\jclisttab\tx567\ls8\adjustright \b\cgrid {General evaluation}{
\par }\pard\plain \qj\nowidctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 We start by commenting on the distribution process, and then go on to analyse the corpus contents and the specific options chosen in its creation. 
\par }\pard \qj\fi284\nowidctlpar\adjustright {\lang1033 Let us first comment on the}{\b\lang1033  distribution options}{\lang1033 
. While this resource is entirely free (one has just to register in a Web page in order to receive the corpus at the address of one's choice), several critical remarks are not out of place:
\par First of all, when publicizing the resource, it was not clear for whom the CD distribution was actually meant: Later on, we discovered that many traditional linguists ordered it just to find out that they were much better off with the on-line version.

\par Second, more accompanying information in the CD would not hurt, instead of pointing to a Web page as the only source: In fact, the assumption that everyone has access to the Web while working with CETEMP\'fa
blico is not necessarily true in Portugal or Brazil.
\par Finally, we did not produce a medium-size technical description; in addition to the FAQ on the Web page, we provided only a full paper (Rocha and Santos, 2000) describing the whole project, arguably an overkill.
\par About the }{\b\lang1033 corpus contents}{\lang1033 , several fundamental decisions can \endash  and actually have, in previous conferences or by e-mail \endash 
 be criticized, in particular the use of a single text source and the inclusion of sentence tags (by criteria so far not yet documented). Still, we think that both are easy to defend, since 1) the 
time taken in copyright handling and contract writing with every copyright owner strongly suggests minimizing their number. And 2) although sentence separation is a controversial issue, it is straightforward to dispose of sentence separation tags. So, thi
s option cannot really be considered an obstacle to users.}{\cs17\fs14\up6\lang1033 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{
 Since extract definition is based on paragraph and not sentence boundary, the option of marking <s> boundaries has no other consequences.}}}{\lang1033 
\par We will concentrate instead on each annotation, after discussing the choice of texts and extracts.
\par {\listtext\pard\plain\s29 \b\cgrid \hich\af0\dbch\af0\loch\f0 3.1\tab}}\pard\plain \s29\fi-567\li567\sb240\sa120\widctlpar\jclisttab\tx567\ls8\ilvl1\adjustright \b\fs22\cgrid {Extract definition and choice
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 Looking at the final corpus, it is evident that many extracts should
 be discarded or, at least, rewritten. We tried to remove specific kinds of "text", namely soccer classifications, citations from other newspapers, etc., but it is still possible to detect several other objects of dubious interest in the resulting corpus.

\par }\pard \qj\fi284\widctlpar\adjustright {\lang1033 In fact, using regular expression patterns of the kind \ldblquote existence of multiple tabs in a line ending in numbers\rdblquote 
, we identified 5270 extracts having some form of classification, as well as 662 extracts with no valid content.
\par Now, it is arguable that classif
ications of other sports (e.g., athletics and motor races), solutions to crossword puzzles, film and book reviews, and TV programming tables, just to name a few, should have been extracted on the same grounds presented for removing soccer. Our decision wa
s obviously based on a question of extent. (Soccer results are much more frequent.) However, we now regret this methodological flaw and would like to clean up a little more (as done in the patches), or add back soccer results.
\par Another problem detected, conc
erning the extract structure, was our unfortunate algorithm of appending titles to the previous extract, just like authors, instead of joining them to the next extract. This means that 4.8% of the extracts end with a title in CETEMP}{\lang2070 \'fa
blico. (9.6% end with an author.)
\par {\listtext\pard\plain\s29 \b\cgrid \hich\af0\dbch\af0\loch\f0 3.2\tab}}\pard\plain \s29\fi-567\li567\sb240\sa120\widctlpar\jclisttab\tx567\ls8\ilvl1\adjustright \b\fs22\cgrid {Spurious repetitions
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 The worst problem presented by the CETEMP\'fa
blico corpus is the question of repeated material. (Incidentally, it is interesting to note that this is also a serious problem in searching the Web, as mentioned by Kobayashi and Takeda (1999).) Repeated articles}{\cs17\fs14\up6\lang1033 \chftn 
{\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ Repeated }{\b sentences}{
 can also occur in the lead and in the body of an article, and (in the opinion section) to highlight parts of an article. }}}{\lang1033  can be due to two independent factors:
\par {\pntext\pard\plain\fs22\cgrid \hich\af0\dbch\af0\loch\f0 -\tab}}\pard \qj\fi-360\li644\widctlpar\jclisttab\tx644{\*\pn \pnlvlblt\ilvl0\ls11\pnrnot0\pnstart1\pnindent644\pnhang{\pntxtb -}}\ls11\adjustright {\lang1033 
parallel editions of the local section of the newspaper in the two main cities of Portugal (Lisboa and Porto)
\par {\pntext\pard\plain\fs22\cgrid \hich\af0\dbch\af0\loch\f0 -\tab}}\pard \qj\fi-360\li644\widctlpar\jclisttab\tx644{\*\pn \pnlvlblt\ilvl0\ls11\pnrnot0\pnstart1\pnindent644\pnhang{\pntxtb -}}\ls11\adjustright {\lang1033 later publication of previously }{
\ldblquote }{\lang1033 rejected}{\rdblquote }{\lang1033  articles
\par }\pard \qj\fi284\widctlpar\adjustright {\lang1033 In addition to manuall
y inspecting rare items that one would not expect to appear more than a few times in the corpus (but which had higher frequency than expected), we used the following strategies to detect repeated extracts:
\par {\pntext\pard\plain\f4\fs22\cgrid \hich\af4\dbch\af0\loch\f4 1.\tab}}\pard \qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls16\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls16\adjustright {\lang1033 
Record the first and last 40 characters of each extract, in a hash table, as well as their size in characters. Then fully compare only the repeated extracts under this criterion.
\par {\pntext\pard\plain\f4\fs22\lang2057\cgrid \hich\af4\dbch\af0\loch\f4 2.\tab}}\pard \qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls16\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls16\adjustright {\lang2057 
Using the Perl module MD5 }{\lang2057 \hich\af4\dbch\af46\loch\f4 (useful for cryptographical purposes), we attributed to each extract a checksum of 32 bytes, and reco\hich\af4\dbch\af46\loch\f4 
rded it in a hash table. Repeated extracts have the same checksum, but it is extremely unlikely that two different ones will.}{\lang1033 
\par }\pard \qj\fi284\widctlpar\adjustright {\lang1033 The results obtained for exactly equal extracts are displayed in Table 1 for both methods.
\par Another related (and obviously more com
plicated) problem is what to do with quasi-duplicates, i.e. sentences or texts that are almost, but not, identical. An estimate of the number of approximately equal extracts, obtained with the 40 character-method but with relaxed size constraints (10%) yi
e
lds some further 15,665 possibly repeated extracts. It is not obvious whether one can automatically identify which one is the revised version, or even whether it is desirable to choose that one. We have, anyway, compiled a list of these cases, thinking th
at they might serve as raw material for studying the revision process (and to obtain a list of errors and their correction).
\par }\trowd \trgaph108\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx993
\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx2694\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx4395\pard 
\qc\widctlpar\intbl\adjustright {Kind\cell Different extracts\cell Extracts to remove\cell }\pard \widctlpar\intbl\adjustright {\row }\trowd \trgaph108\trkeep\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 
\trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx993\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 
\cltxlrtb \cellx1843\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx2694\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb 
\cellx3544\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx4395\pard \qc\widctlpar\intbl\adjustright {\cell 40chr\cell MD5\cell 40chr\cell MD5\cell }\pard \widctlpar\intbl\adjustright {
\row }\trowd \trgaph108\trkeep\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr
\brdrs\brdrw10 \cltxlrtb \cellx993\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx1843\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr
\brdrs\brdrw10 \cltxlrtb \cellx2694\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3544\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr
\brdrs\brdrw10 \cltxlrtb \cellx4395\pard \qc\widctlpar\intbl\adjustright {twice\cell 45,046 \cell 44,188\cell 45,046 \cell 44,188\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qc\widctlpar\intbl\adjustright {3 times\cell 1,493\cell 1,401\cell 
2,986\cell 2,802\cell }\pard \widctlpar\intbl\adjustright {\row }\pard\plain \s21\qc\widctlpar\intbl\adjustright \f4\fs22\lang2057\cgrid {\lang3084 4 times\cell }\pard\plain \qc\widctlpar\intbl\adjustright \f4\fs22\lang3084\cgrid {301\cell 271\cell 903
\cell 813\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qc\widctlpar\intbl\adjustright {5 times\cell 68\cell 63\cell 272\cell 252\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qc\widctlpar\intbl\adjustright {6-10\cell 83\cell 81\cell 552
\cell 548\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qc\widctlpar\intbl\adjustright {> 11\cell 31\cell 31\cell 643\cell 880\cell }\pard \widctlpar\intbl\adjustright {\row }\trowd \trgaph108\trkeep\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 
\trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx993\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl
\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx1843\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx2694\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl
\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3544\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx4395\pard \qc\widctlpar\intbl\adjustright {Total\cell 
47,022\cell 46,035\cell 50,402\cell 49,483\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qc\fi284\widctlpar\adjustright {\lang1033 Table 1. Overview of exact duplication
\par {\listtext\pard\plain\s29 \b\cgrid \hich\af0\dbch\af0\loch\f0 3.3\tab}}\pard\plain \s29\fi-567\li567\sb240\sa120\widctlpar\jclisttab\tx567\ls8\ilvl1\adjustright \b\fs22\cgrid {Title and author identification
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 In the CETEMP\'fablico corpus, newspaper titles and subtitles
, as well as author identifications, have been marked up as result of heuristic processing. In Rocha and Santos (2000), a preliminary evaluation of precision and recall for these tasks was published, but here we want to evaluate this in a different way, w
ithout making reference to the original text files. 
\par }\pard \qj\fi284\widctlpar\adjustright {\lang1033 Given the corpus, we want to address precision and error rate (i.e., of all chunks tagged as titles, how many have been rightly tagged?, and how many are wrong?). We reviewed manually the first 500 instan
ces of <t>}{\cs17\fs14\up6\lang1033 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ In the 15}{\super th}{ chunk of the corpus. }{\lang1033 
This aparently na\'efve choice of test data does not bias evaluation, since the extracts are randomly placed in the corpus and do not reflect any order of time period or kind of text.}}}{\lang1033 
, of which 427 were undoubtedly titles, a further 4 wrongly tagged authors, and at least 15 belonged to book or film reviews, indicating title, author and publisher, or director and broadcasting date, etc.
\par We then looked into the following error
-prone situation: After having noted that several paragraphs in a row including title and author tags were usually wrong (and should have been marked as list items instead), we looked for extracts containing sequences of four titles / authors and manually
 
checked 200. The precision in this case was very low: Only 38% were correctly tagged. Of the incorrect ones, as much as 34% were part of book reviews as described above. This indicates clearly that we should have processed special text formats prior to ap
plying our general heuristic rules.
\par Regarding recall, we did the following partial inspection: We noted several short sentences ending in ? or ! (a criterion to parse a text chunk as a full sentence) that should actually be tagged as titles. }{\lang2057 We therefore lo
oked at 200 paragraphs with one single sentence ending in question or exclamation mark containing less than 8 words, and concluded that 41 cases (20%) could definitively be marked as titles, while no less than 85 of these cases where questions taken from 
interviews. Most other cases were questions inside ordinary articles.
\par }{\lang1033 As far as authors are concerned, the phrase }{\i\lang1033 Leitor devidamente identificado}{\lang1033  (}{\ldblquote }{\lang1033 duly identified reader}{\rdblquote }{\lang1033 
, used to sign reader's letters where the writer does not wish to disclose his or her identity) was correctly identified only in 78% of the cases (135 in 172). In 17% of the occurrences, it was wrongly tagged as title.
\par From a list of 500 authors randomly extracted for evaluation purposes, only 395 (79%) were unambiguously so, while 8 (1.5%)
 could still be considered correct by somehow more relaxed criteria. We thus conclude that up to 21% of the author tags in the corpus may be wrongly attributed, a figure much higher than the originally estimated 4%. 
\par Among those cases, foreign names (generally in the context of film or music reviews, or book presentations) were frequently mistagged as authors of articles in P\'fa
blico, a situation highly unlikely and amenable to automatic correction. Figure 1 is an example. 
\par }\pard \qj\widctlpar\adjustright {\i\f2\fs16\lang2070 <}{\f2\fs16\lang2070 a> Contos Assombrosos </a>
\par }{\f2\fs16\lang2057 <a> Amazing Stories </a>
\par }{\f2\fs16\lang1043 <a> De Steven Spielberg </a>
\par }{\f2\fs16\lang2057 <t> Com Kevin Costner, Patrick Swayze e Sid Caesar  </t>
\par }\pard\plain \s21\qc\widctlpar\adjustright \f4\fs22\lang2057\cgrid {\lang1033 Figure 1. Wrong attribution of <a> and <t>
\par {\listtext\pard\plain\s29 \b\cgrid \hich\af0\dbch\af0\loch\f0 3.4\tab}}\pard\plain \s29\fi-567\li567\sb240\sa120\widctlpar\jclisttab\tx567\ls8\ilvl1\adjustright \b\fs22\cgrid {Sentence separation
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 In addition to paragraph separation coming from the original newspaper files, CETEMP\'fablico comes with sentence separation as an added-value feature.
\par }\pard \qj\fi284\widctlpar\adjustright {\lang1033 Now, sentence separation is obviously not a trivial question, and there are no foolproof rules for complicated cases (Nunberg, 1990; Grefenstette and Tapainanen, 1994; Santos, 1998). So, instead o
f trying to produce other subjective criteria for evaluating a particularly delicate area, we decided to look at the amount of work needed to revise the sentence separation for a given purpose, as reported in section 4.2.
\par }\pard\plain \s41\qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 But we did some complementary sear
ches for cases we would expect to be wrong whatever the sentence separation philosophy. We thus found 6,358 sentences initiated by a punctuation mark (comma, closing quotes, period, question mark and exclamation mark, respectively amounting to 4053, 410, 
1607, 227 and 61 occurrences), as well as a plethora of suspiciously small sentences, cf. Table 2.
\par }\pard\plain \qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 
\par }\trowd \trqc\trgaph70\trleft137\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 
\clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx1276\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx2595\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr
\brdrs\brdrw10 \cltxlrtb \cellx3729\pard \qc\widctlpar\intbl\adjustright {Sentence size\cell Number of sentences\cell Error estimation\cell }\pard \widctlpar\intbl\adjustright {\row }\trowd \trqc\trgaph70\trleft137\trbrdrt\brdrs\brdrw10 \trbrdrl
\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx1276\clvertalt\clbrdrt
\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx2595\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3729\pard 
\qc\widctlpar\intbl\adjustright {one\cell 14,783\cell 100%\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qc\widctlpar\intbl\adjustright {two\cell 55,121\cell 53%\cell }\pard \widctlpar\intbl\adjustright {\row }\trowd \trqc\trgaph70\trleft137
\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb 
\cellx1276\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx2595\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3729
\pard \qc\widctlpar\intbl\adjustright {three\cell 70,909\cell 20%\cell }\pard \widctlpar\intbl\adjustright {\row }\pard\plain \s21\qc\widctlpar\adjustright \f4\fs22\lang2057\cgrid {\lang1033 Table 2. Too small sentences
\par }\pard\plain \s41\qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid {Sentence separation marks
 some sentences as fragments (<s frag>); in addition, the <li> attribute was used to render list elements. We are not sure now whether it was worthwhile to have two different markup elements.
\par 
\par }\trowd \trqc\trgaph30\trrh262\trleft-30 \clvertalt\clbrdrt\brdrs\brdrw15 \clbrdrl\brdrs\brdrw15 \clbrdrb\brdrs\brdrw15 \clbrdrr\brdrs\brdrw15 \cltxlrtb \cellx980\clvertalt\clbrdrt\brdrs\brdrw15 \clbrdrl\brdrs\brdrw15 \clbrdrb\brdrs\brdrw15 \clbrdrr
\brdrs\brdrw15 \cltxlrtb \cellx1991\pard\plain \qc\widctlpar\intbl\adjustright \f4\fs22\lang3084\cgrid {\f0\cf1\lang1033\cgrid0 <s frag>\cell 63,122\cell }\pard \widctlpar\intbl\adjustright {\f0\cf1\lang1033\cgrid0 \row }\trowd 
\trqc\trgaph30\trrh262\trleft-30 \clvertalt\clbrdrt\brdrs\brdrw15 \clbrdrl\brdrs\brdrw15 \clbrdrb\brdrs\brdrw15 \clbrdrr\brdrs\brdrw15 \cltxlrtb \cellx980\clvertalt\clbrdrt\brdrs\brdrw15 \clbrdrl\brdrs\brdrw15 \clbrdrb\brdrs\brdrw15 \clbrdrr
\brdrs\brdrw15 \cltxlrtb \cellx1991\pard \qc\widctlpar\intbl\adjustright {\f0\cf1\lang1033\cgrid0 <li>\cell 113,540\cell }\pard \widctlpar\intbl\adjustright {\f0\cf1\lang1033\cgrid0 \row }\pard \qc\widctlpar\intbl\adjustright {\f0\cf1\lang1033\cgrid0 <t>
\cell 687,720\cell }\pard \widctlpar\intbl\adjustright {\f0\cf1\lang1033\cgrid0 \row }\trowd \trqc\trgaph30\trrh262\trleft-30 \clvertalt\clbrdrt\brdrs\brdrw15 \clbrdrl\brdrs\brdrw15 \clbrdrb\brdrs\brdrw15 \clbrdrr\brdrs\brdrw15 \cltxlrtb \cellx980
\clvertalt\clbrdrt\brdrs\brdrw15 \clbrdrl\brdrs\brdrw15 \clbrdrb\brdrs\brdrw15 \clbrdrr\brdrs\brdrw15 \cltxlrtb \cellx1991\pard \qc\widctlpar\intbl\adjustright {\f0\cf1\lang1033\cgrid0 <a>\cell 263,269\cell }\pard \widctlpar\intbl\adjustright {
\f0\cf1\lang1033\cgrid0 \row }\pard\plain \s21\qc\widctlpar\adjustright \f4\fs22\lang2057\cgrid {\lang1033 Table 3. Number of cases of non-standard <s>
\par }\pard\plain \s41\qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid {
Finally, the sentence separation module also introduces the <marca> tag to identify meta-characters that are used for later coreference (eg. in footnotes). The asterisk "*" was marked as such in CETEMP\'fablico, but not 
inside author or title descriptions, an undesirable inconsistency.
\par {\listtext\pard\plain\s29 \b\cgrid \hich\af0\dbch\af0\loch\f0 3.5\tab}}\pard\plain \s29\fi-567\li567\sb240\sa120\widctlpar\jclisttab\tx567\ls8\ilvl1\adjustright \b\fs22\cgrid {Extraneous characters
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 An annoying detail is the amount of strange characters that have remained in the corpus after font conversion, such as non-Portuguese characters,}{\lang1044  hyphens, bullet list 
marking, and the characters < > instead of quotes. 
\par }\pard \qj\fi284\widctlpar\adjustright {\lang1044 It is straightforward to replace these with other ISO-8859-1 characters or combinations of characters, as was done with dashes and quotes.}{\cs17\fs14\up6\lang1044 \chftn {\footnote \pard\plain 
\s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ Note that it is not always possible to have a one-to-one mapping from MacRoman into ISO-8859-1.}}}{\lang1044  }{\lang1033 
Only the last line of Table 4 requires some care, since }{\i\lang1033 \'c9}{\lang1033  is a otherwise valid Portuguese character that should only be replaced a few times.
\par }{\lang1044 
\par }\trowd \trqc\trrh255\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalb\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr
\brdrs\brdrw10 \cltxlrtb \cellx1450\clvertalb\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3293\clvertalb\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr
\brdrs\brdrw10 \cltxlrtb \cellx4253\pard \qc\widctlpar\intbl\adjustright {\fs20\lang2057 Character}{\fs20\lang2057 \cell }{\fs20\lang2057 Action}{\fs20\lang2057 \cell }{\fs20\lang2057 Number}{\fs20\lang2057 \cell }\pard \widctlpar\intbl\adjustright {
\fs16\lang2057 \row }\trowd \trqc\trrh255\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalb\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb
\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx1450\clvertalb\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3293\clvertalb\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb
\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx4253\pard \qj\widctlpar\intbl\adjustright {\fs20\lang2057 \'d0}{\fs20\lang2057 \cell }{\fs20\lang2057 non-breaking hyphen}{\fs20\lang2057 \cell }\pard \qr\widctlpar\intbl\adjustright {\fs20\lang2057 
856}{\fs20\lang2057 \cell }\pard \widctlpar\intbl\adjustright {\fs16\lang2057 \row }\pard \qj\widctlpar\intbl\adjustright {\fs20\lang2057 \'cf}{\fs20\lang2057 \cell }{\fs20\lang2057 use oe}{\fs20\lang2057 \cell }\pard \qr\widctlpar\intbl\adjustright {
\fs20\lang2057 246}{\fs20\lang2057 \cell }\pard \widctlpar\intbl\adjustright {\fs16\lang2057 \row }\pard \qj\widctlpar\intbl\adjustright {\fs20\lang2057 tab stop}{\fs20\lang2057 \cell }{\fs20\lang2057 remove/replace by " "}{\fs20\lang2057 \cell }\pard 
\qr\widctlpar\intbl\adjustright {\fs20\lang2057 50,312}{\fs20\lang2057 \cell }\pard \widctlpar\intbl\adjustright {\fs16\lang2057 \row }\pard \qj\widctlpar\intbl\adjustright {\fs20\lang2057 control character}{\fs20\lang2057 \cell }{\fs20\lang2057 
eliminate extract}{\fs20\lang2057 \cell }\pard \qr\widctlpar\intbl\adjustright {\fs20\lang2057 53,631}{\fs20\lang2057 \cell }\pard \widctlpar\intbl\adjustright {\fs16\lang2057 \row }\pard \qj\widctlpar\intbl\adjustright {\fs20\lang2057 character 0x95}{
\fs20\lang2057 \cell }{\fs20\lang2057 (?)}{\fs20\lang2057 \cell }\pard \qr\widctlpar\intbl\adjustright {\fs20\lang2057 40,665}{\fs20\lang2057 \cell }\pard \widctlpar\intbl\adjustright {\fs16\lang2057 \row }\pard \qj\widctlpar\intbl\adjustright {
\fs20\lang2057 <}{\fs20\lang2057 \cell }{\fs20\lang2057 use &lt;}{\fs20\lang2057 \cell }\pard \qr\widctlpar\intbl\adjustright {\fs20\lang2057 1,283}{\fs20\lang2057 \cell }\pard \widctlpar\intbl\adjustright {\fs16\lang2057 \row }\pard 
\qj\widctlpar\intbl\adjustright {\fs20\lang2057 >}{\fs20\lang2057 \cell }{\fs20\lang2057 use &gt;}{\fs20\lang2057 \cell }\pard \qr\widctlpar\intbl\adjustright {\fs20 1,232}{\fs20 \cell }\pard \widctlpar\intbl\adjustright {\fs16 \row }\trowd \trqc\trrh255
\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalb\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb 
\cellx1450\clvertalb\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3293\clvertalb\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx4253
\pard \qj\widctlpar\intbl\adjustright {\fs20\lang2070 \loch\af4\dbch\af47\hich\f4 \'c9\cell }{\fs20\lang2057 \hich\af4\dbch\af47\loch\f4 replace by ...\cell }\pard \qr\widctlpar\intbl\adjustright {\fs20 \hich\af4\dbch\af47\loch\f4 3,167\cell }\pard 
\widctlpar\intbl\adjustright {\fs16 \row }\pard\plain \s21\qc\widctlpar\adjustright \f4\fs22\lang2057\cgrid {\lang1044 Table 4. Occurrence of extraneous chars
\par {\listtext\pard\plain\s29 \b\cgrid \hich\af0\dbch\af0\loch\f0 3.6\tab}}\pard\plain \s29\fi-567\li567\sb240\sa120\widctlpar\jclisttab\tx567\ls8\ilvl1\adjustright \b\fs22\cgrid {Text classification
\par }\pard\plain \qj\nowidctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 CETEMP}{\lang2070 \'fa}{\lang1033 blico extracts come with a subject classification derived from (but not equal to) the original newspaper section. Due to format di
fferences of the original files, only 86% of the extracts have some classification associated. The others carry the label ND (not determined).
\par }\pard \qj\fi284\nowidctlpar\adjustright {\lang1033  We evaluate here this classification, since for half of the corpus article separation had to be carried out automatically and thus chances exist that errors may have crept in.
\par The first thing we did was to check whether repeated extracts had been attributed the same classification. Astonishingly, there were many differences: of the 47,002 cases of multiple extracts,
 10,872 (23%) had different categories, even though only in 2% of the cases none of the conflicting categories was ND.
\par Another experiment was to look at well-known polysemic or ambiguous items and see whether their meaning correlated with the kind of text 
it was purported to be in. We thus inspected manually several thousand concordances dealing with the following middle frequency words}{\cs17\fs14\up6\lang1033 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar
\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ Glosses provided are not exhaustive.}}}{\lang1033 : 201 occurrences of }{\i\lang1033 vassoura}{\lang1033  (broom; last vehicle in a bicycle race); 124 of }{
\i\lang1033 passador}{\lang1033  (sieve; drug seller; emigrant dealer); 314 of }{\i\lang1033 cunha}{\lang1033  (wooden object; corruption device); 599 of }{\i\lang1033 coxa}{\lang1033  (noun thigh; adjective lame); 205 of }{\i\lang1033 prego}{\lang1033 
 (nail; meat sandwich; pawnshop); 145 of }{\i\lang1033 garfo}{\lang1033  (fork; biking); 5505 of }{\i\lang1033 estrela}{\lang1033  (star; filmstar; success); 375 of }{\i\lang1033 dobragem}{\lang1033  (folding; dubbing; parachuting and F1 term); 573 of }{
\i\lang1033 escravatura}{\lang1033  (slavery).
\par }\pard\plain \s41\qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid {
We could only find two cases of firm disagreement with source classification (in the two last mentioned queries). This is not such a good result as it seems, though, since it can be argued that subject classi
fication is too high level (society, politics, culture) to allow for definite results.
\par {\listtext\pard\plain\s27 \b\cgrid \hich\af0\dbch\af0\loch\f0 4\tab}}\pard\plain \s27\fi-567\li567\sb240\sa240\widctlpar\jclisttab\tx567\ls8\adjustright \b\cgrid {Corpus in use
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 The best way to evaluate a corpus resource is to see how well it fares regarding the tasks it is put to. We will not evaluate concordancing for human inspe
ction, because we assume that this is a rather straightforward task for which CETEMP\'fablico is useful, especially because it requires direct scrutiny. Obviously, human inspection and judgement make the results more robust.
\par {\listtext\pard\plain\s29 \b\cgrid \hich\af0\dbch\af0\loch\f0 4.1\tab}}\pard\plain \s29\fi-567\li567\sb240\sa120\widctlpar\jclisttab\tx567\ls8\ilvl1\adjustright \b\fs22\cgrid {Proper name identification
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 One of the authors developed proper name identification tools (Santos, 1999) prior to the existence of CETEMP\'fablico. We ran them on this corpus to see how they worked.
\par }\pard \qj\fi284\widctlpar\adjustright {\lang1033 We proceeded in the following way: We inspected manually the first 1,000 proper names obtained from CETEMP\'fablico and got less then 4% wrong, i.e., over 96% precision. 
\par }\trowd \trqc\trgaph108\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr
\brdrs\brdrw10 \cltxlrtb \cellx2154\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3261\pard \qj\widctlpar\intbl\adjustright {\lang1033 Size\cell Number\cell }\pard 
\widctlpar\intbl\adjustright {\lang1033 \row }\trowd \trqc\trgaph108\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl
\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx2154\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3261\pard \qj\widctlpar\intbl\adjustright {\lang1033 
One word\cell }\pard\plain \s21\qc\widctlpar\intbl\adjustright \f4\fs22\lang2057\cgrid {\lang1033 26,518\cell }\pard\plain \widctlpar\intbl\adjustright \f4\fs22\lang3084\cgrid {\lang1033 \row }\pard \qj\widctlpar\intbl\adjustright {\lang1033 Two words
\cell }\pard \qc\widctlpar\intbl\adjustright {\lang1033 15,512\cell }\pard \widctlpar\intbl\adjustright {\lang1033 \row }\pard \qj\widctlpar\intbl\adjustright {\lang1033 Two words and }{\i\lang1033 de}{\lang1033 \cell }\pard 
\qc\widctlpar\intbl\adjustright {\lang1033 4,623\cell }\pard \widctlpar\intbl\adjustright {\lang1033 \row }\pard \qj\widctlpar\intbl\adjustright {\lang1033 Three words\cell }\pard \qc\widctlpar\intbl\adjustright {\lang1033 2,132\cell }\pard 
\widctlpar\intbl\adjustright {\lang1033 \row }\pard \qj\widctlpar\intbl\adjustright {\lang1033 Three words and }{\i\lang1033 de}{\lang1033 \cell }\pard \qc\widctlpar\intbl\adjustright {\lang1033 2,354\cell }\pard \widctlpar\intbl\adjustright {\lang1033 
\row }\pard \qj\widctlpar\intbl\adjustright {\lang1033 Four words\cell }\pard \qc\widctlpar\intbl\adjustright {\lang1033 201\cell }\pard \widctlpar\intbl\adjustright {\lang1033 \row }\pard \qj\widctlpar\intbl\adjustright {\lang1033 Four words and }{
\i\lang1033 de}{\lang1033 \cell }\pard \qc\widctlpar\intbl\adjustright {\lang1033 583\cell }\pard \widctlpar\intbl\adjustright {\lang1033 \row }\pard \qj\widctlpar\intbl\adjustright {\lang1033 >= five words\cell }\pard \qc\widctlpar\intbl\adjustright {
\lang1033 359\cell }\pard \widctlpar\intbl\adjustright {\lang1033 \row }\trowd \trqc\trgaph108\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt
\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx2154\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3261\pard 
\qj\widctlpar\intbl\adjustright {\lang1033 problems }{\cs17\fs14\up6\lang1033 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ This category encompasses 
\ldblquote deviant\rdblquote  proper names, mainly including foreign accents and numbers, irrespective of proper name length.}}}{\lang1033 \cell }\pard \qc\widctlpar\intbl\adjustright {\lang1033 383\cell }\pard \widctlpar\intbl\adjustright {\lang1033 
\row }\pard \qc\fi284\widctlpar\adjustright {\lang1033 Table 5. Size distribution of proper nouns
\par }\pard \qj\fi284\widctlpar\adjustright {Then, we computed the distribution of the 52,665 proper nouns identified by the program (23,401 types) on the first million words of the corpus as shown in Table 5, and inspected manually those 1,017 havin
g a length larger or equal than four words. Of these 88% were correct and 6.5% were plainly wrong. Cases of merging two proper names and cases where it was easy to guess one missing (preceding or following) word accounted each for approximately 5% of the 
remaining instances.}{\lang1033 
\par While use of CETEMP}{\lang2070 \'fablico allowed us to uncover cases not catered for by the program, it also illuminated some potential}{\cs17\fs14\up6\lang2070 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar
\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ Different tokenizers may have different strategies, but we assume that these will be hard cases for most.}}}{\lang2070 
 tokenization problems in the corpus, namely a large quantity of tokens ending in a dash (21,455 tokens, 6,458 types) or in a slash (7313 tokens, 4530 types), as well as up to 132,455 tokens including one single parenthesis (28,466 types).
\par {\listtext\pard\plain\s29 \b\cgrid \hich\af0\dbch\af0\loch\f0 4.2\tab}}\pard\plain \s29\fi-567\li567\sb240\sa120\widctlpar\jclisttab\tx567\ls8\ilvl1\adjustright \b\fs22\cgrid {Treebank building
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 The first million words of CETEMP\'fablico was selected for the creation of a treebank for Portuguese (Floresta Sint\'e1(c)tica}{\cs17\fs14\up6\lang1033 \chftn {\footnote 
\pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ See }{\field{\*\fldinst { HYPERLINK http://cgi.portugues.mct.pt/treebank/ }{{\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000002600000068007400740070003a002f002f006300670069002e0070006f0072007400750067007500650073002e006d00630074002e00700074002f007400720065006500620061006e006b002f000000e0c9ea79f9bace118c8200aa004ba90b4c0000006800
7400740070003a002f002f006300670069002e0070006f0072007400750067007500650073002e006d00630074002e00700074002f007400720065006500620061006e006b002f000000}}}{\fldrslt {\cs24\ul\cf2 http://cgi.portugues.mct.pt/treebank/}}}{.}}}{\lang1033 
), given that its use is copyright cleared and the corpus is free.
\par }\pard \qj\fi284\widctlpar\adjustright {\lang1033 The treebank team engaged in a manual revision of the text prior to treebank coding, refining sentence separation with the help of syntactically-based criteria (Afonso and Marchi, 
2001). We have tried to compute the amount of change produced by human intervention, which turned out to be a surprisingly complex task (Santos, 2001).
\par This one million words subcorpus contained 8,043 extracts.}{\cs17\fs14\up6\lang1033 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ N}{\lang1033 
umbered from 1 to 8067, since version 1.2 was used, and therefore 24 invalid extracts had been already removed. In addition, the treebank reviewers considered that further 129 should be taken out.}}}{\lang1033  Assuming that the first million is not diffe
rent from the rest of the corpus, the results indicate an estimate of 17% of the corpus extracts in need of improvement.
\par Looking at sentences, 2,977 sentences of the 42,026 original ones had to be re-separated into 4,304 of the resulting 43,271. Table 6 di
splays an estimate of what was actually involved in the revision of sentence tags (percentages are relative to the original number of sentences). The "Other" category includes changes among the tags <t>, <a>, <li> and <s>.
\par }\trowd \trqc\trgaph108\trleft-108\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 
\clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx1401\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx2681\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr
\brdrs\brdrw10 \cltxlrtb \cellx4034\pard \qj\widctlpar\intbl\adjustright {\lang1033 <s>-addition\cell }\pard\plain \s21\qc\widctlpar\intbl\adjustright \f4\fs22\lang2057\cgrid {\lang1033 1,481-1,872\cell 3.52-4.24%\cell }\pard\plain 
\widctlpar\intbl\adjustright \f4\fs22\lang3084\cgrid {\lang1033 \row }\trowd \trqc\trgaph108\trleft-108\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt
\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx1401\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx2681\clvertalt\clbrdrt
\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx4034\pard \qj\widctlpar\intbl\adjustright {\lang1033 <s>-removal \cell }\pard \qc\widctlpar\intbl\adjustright {\lang1033 612-115\cell 1.46-2.65%\cell 
}\pard \widctlpar\intbl\adjustright {\lang1033 \row }\trowd \trqc\trgaph108\trleft-108\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt
\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx1401\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx2681\clvertalt\clbrdrt
\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx4034\pard \qj\widctlpar\intbl\adjustright {\lang1033 Other\cell }\pard \qc\widctlpar\intbl\adjustright {\lang1033 550\cell 1.3%\cell }\pard 
\widctlpar\intbl\adjustright {\lang1033 \row }\pard\plain \s21\qc\widctlpar\adjustright \f4\fs22\lang2057\cgrid {\lang1033 Table 6. Revision of <s> tags
\par {\listtext\pard\plain\s29 \b\cgrid \hich\af0\dbch\af0\loch\f0 4.3\tab}}\pard\plain \s29\fi-567\li567\sb240\sa120\widctlpar\jclisttab\tx567\ls8\ilvl1\adjustright \b\fs22\cgrid {Spelling checker evaluation
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 One of the first and most direct uses of a large corpus is to study the coverage, evaluate, and especially improve a spelling checker and morphological analyser.
\par }\pard \qj\fi284\widctlpar\adjustright {Our preliminary results of evaluating Jspell (Almeida and Pinto, 1994) as far as type and token spelling is concerned are as follows: }{\lang2057 Among the 942,980 types of CETEMP\'fa
blico, 574,199 were not recognized by the current version of Jspell (60.4%), amounting to 3.07% of the size of the corpus. A superficial comparison showed that CETEMP}{\lang2070 \'fablico}{\lang2057 
 contains a higher percentage of unrecognized words, both types and tokens, than other Portuguese newspaper corpora. Numbers for a 1.5-million word corpus of }{\i\lang2057 Di\'e1rio do Minho}{\lang2057 
 (a regional newspaper) and for a 4-million word corpus of a political party newspaper are respectively 26.5% and 25.41% unrecognized types and 2.26% and 1.67% unrecognized tokens. These numbers may be partially explained by }{\i\lang2057 P\'fablico}{
\lang2057 's higher coverage of international affairs, together with its }{cinema and music sections}{\lang2057 , both bringing}{ an increase in foreign proper names}{\cs17\fs14\up6 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar
\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ The percentage of unrecognized tokens varies from 4.8% for culture to 2.0% for society extracts. }}}{.
\par }\trowd \trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 
\cltxlrtb \cellx2557\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3393\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb 
\cellx4308\pard \qc\widctlpar\intbl\adjustright {Description\cell Tokens\cell }\pard\plain \s21\qc\widctlpar\intbl\adjustright \f4\fs22\lang2057\cgrid {\lang3084 Types\cell }\pard\plain \widctlpar\intbl\adjustright \f4\fs22\lang3084\cgrid {\row }\trowd 
\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb 
\cellx2557\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3393\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx4308
\pard \qj\widctlpar\intbl\adjustright {Foreign first names\cell }\pard \qc\widctlpar\intbl\adjustright {130\cell 125\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qj\widctlpar\intbl\adjustright {Portuguese first names\cell }\pard 
\qc\widctlpar\intbl\adjustright {\lang2070 19\cell 16\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qj\widctlpar\intbl\adjustright {Foreign surnames\cell }\pard \qc\widctlpar\intbl\adjustright {\lang2070 216\cell 208\cell }\pard 
\widctlpar\intbl\adjustright {\row }\pard \qj\widctlpar\intbl\adjustright {Portuguese surnames\cell }\pard \qc\widctlpar\intbl\adjustright {\lang2070 35\cell 34\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qj\widctlpar\intbl\adjustright {
Foreign organizations\cell }\pard \qc\widctlpar\intbl\adjustright {50\cell 45\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qj\widctlpar\intbl\adjustright {Portuguese organizations\cell }\pard \qc\widctlpar\intbl\adjustright {26\cell 23\cell 
}\pard \widctlpar\intbl\adjustright {\row }\pard \qj\widctlpar\intbl\adjustright {Foreign geographical}{\cs17\fs14\up6 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {
\cs17\fs14\up6 \chftn }{ We classify as Portuguese or foreign the word, not the location: thus, }{\i Tanz\'e2nia}{ is a Portuguese word.}}}{\cell }\pard \qc\widctlpar\intbl\adjustright {48\cell 48\cell }\pard \widctlpar\intbl\adjustright {\row }\pard 
\qj\widctlpar\intbl\adjustright {Portuguese geographic \cell }\pard \qc\widctlpar\intbl\adjustright {28\cell 28\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qj\widctlpar\intbl\adjustright {acronyms\cell }\pard \qc\widctlpar\intbl\adjustright {81
\cell 77\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qj\widctlpar\intbl\adjustright {foreign words\cell }\pard \qc\widctlpar\intbl\adjustright {171\cell 161\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qj\widctlpar\intbl\adjustright {
Portuguese foreign words}{\cs17\fs14\up6 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{
 That is, words routinely used in Portuguese but which up to now have kept a distinctly foreign spelling, such as }{\i pullover}{.}}}{\cell }\pard \qc\widctlpar\intbl\adjustright {26\cell 25\cell }\pard \widctlpar\intbl\adjustright {\row }\pard 
\qj\widctlpar\intbl\adjustright {words missing in dict.\cell }\pard \qc\widctlpar\intbl\adjustright {101\cell 98\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qj\widctlpar\intbl\adjustright {incorrectly spelled}{\cs17\fs14\up6 \chftn {\footnote 
\pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {\cs17\fs14\up6 \chftn }{ Including one case of lack of space between two words, }{\i suacontribui\'e7\'e3o}{.}}}{\cell }\pard 
\qc\widctlpar\intbl\adjustright {36\cell 36\cell }\pard \widctlpar\intbl\adjustright {\row }\pard \qj\widctlpar\intbl\adjustright {others\cell }\pard \qc\widctlpar\intbl\adjustright {33\cell 32\cell }\pard \widctlpar\intbl\adjustright {\row }\trowd 
\trbrdrt\brdrs\brdrw10 \trbrdrl\brdrs\brdrw10 \trbrdrb\brdrs\brdrw10 \trbrdrr\brdrs\brdrw10 \trbrdrh\brdrs\brdrw10 \trbrdrv\brdrs\brdrw10 \clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb 
\cellx2557\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx3393\clvertalt\clbrdrt\brdrs\brdrw10 \clbrdrl\brdrs\brdrw10 \clbrdrb\brdrs\brdrw10 \clbrdrr\brdrs\brdrw10 \cltxlrtb \cellx4308
\pard \qj\widctlpar\intbl\adjustright {total\cell }\pard \qc\widctlpar\intbl\adjustright {1,000\cell 956\cell }\pard \widctlpar\intbl\adjustright {\row }\pard\plain \s21\qc\sb20\keepn\nowidctlpar\tqc\tx1980\tqc\tx4520\tqc\tx7100\tqr\tx9000\adjustright 
\f4\fs22\lang2057\cgrid {Table 7. Distribution of \ldblquote errors\rdblquote 
\par }\pard\plain \qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid {We investigated the \ldblquote errors\rdblquote 
 found by the system, to see how many were real and how many were due to a defficient lexical (or rule) coverage. Table 7 shows the distribution of 1,000 \ldblquote errors\rdblquote  randomly obtained from the 12}{\super th}{ corpus chunk.
\par }\pard\plain \s41\qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid {The absolute frequencies of the most common spelling errors in CETEMP\'fa
blico is another interesting evaluation parameter. Applying Jspell to types with frequency > 100 (excluding capitalized and hyphenated words), we identified manually the \ldblquote real\rdblquote 
 errors. Strikingly, all involved lack or excess of accents. The most frequent appeared 840 times (}{\i ju\'edz}{), the second one (}{\i\lang1033 sa\'edu}{\lang1033 ) 659, and the third (}{\i\lang1033 imp\'f4r}{\lang1033 
) had 637 occurrences. Their correctly spelled variants (}{\i\lang1033 juiz}{\lang1033 , }{\i\lang1033 saiu}{\lang1033 , }{\i\lang1033 impor}{\lang1033 ) appeared respectively 11896, 9892 and 5125 times.}{
\par {\listtext\pard\plain\s27 \b\cgrid \hich\af0\dbch\af0\loch\f0 5\tab}}\pard\plain \s27\fi-567\li567\sb240\sa240\widctlpar\jclisttab\tx567\ls8\adjustright \b\cgrid {Comparison with other corpora
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {One can find excellent reports on the difficulties encountered in creating corpora (see e.g. Armstrong et al. (1998) and references therein), but it is si
gnificantly rarer to get an evaluation of the resulting objects. }{\lang1033 It is thus not easy to compare CETEMP\'fablico with other corpora on the issues discussed here. }{
\par }\pard \qj\fi284\widctlpar\adjustright {For example, it was not easy to find a thorough documentation of BNC}{\cs17\fs14\up6 \chftn {\footnote \pard\plain \s18\qj\sb20\sa20\nowidctlpar\tx560\tx1134\tx1700\tx2268\adjustright \f4\fs18\lang1036\cgrid {
\cs17\fs14\up6 \chftn }{ }{\lang2070 British National Corpus. }{\field\flddirty{\*\fldinst {\lang2070  HYPERLINK "http://info.ox.ac.uk/bnc/" }{{\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b0200000003000000e0c9ea79f9bace118c8200aa004ba90b3400000068007400740070003a002f002f0069006e0066006f002e006f0078002e00610063002e0075006b002f0062006e0063002f000000000000000000000000000000000000}}}{\fldrslt {
\cs24\ul\cf2\lang2070 http://info.ox.ac.uk/bnc/}}}}}{ problems (although there is a mailing list and a specific e-mail address to report bugs), nor is similar information to be found in distribution agencies\rquote  (such as LDC or ELRA) Web sites.
\par }\pard\plain \s41\qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid {It is obviously outside the scope of the present paper to do a thorough analysis of other corpo
ra as well, but our previous experience shows that it is not at all uncommon to experience problems with characters and fonts, repeated texts or sentences, rubbish-like sections, wrong markup and/or lack of it. All this independently of corpora being paid
 and/or distributed by agencies supposed to have performed validation checks. The same happens for corpora that have been manually revised.
\par }\pard\plain \qj\fi284\widctlpar\adjustright \f4\fs22\lang3084\cgrid {As regards sentence separation, Johansson et al. (1996) mention that proofreading of the automatic insertion of <s>-u
nits was necessary for the ENPC corpus, but they do not report problems of human editors in deciding what an <s> should be. Let us, however, note that ENPC compilers were free to use an <omit> tag for complicated cases and, last but not least, were not de
aling with newspaper text.
\par {\listtext\pard\plain\s27 \b\cgrid \hich\af0\dbch\af0\loch\f0 6\tab}}\pard\plain \s27\fi-567\li567\sb240\sa240\widctlpar\jclisttab\tx567\ls8\adjustright \b\cgrid {Concluding remarks
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\lang1033 This paper can be read from a user\rquote s angle as a complement to the documentation of the CETEMP\'fa
blico corpus. In addition, by showing several simple forms of evaluating a corpus resource, we hope to have inspired others to do the same for other corpora.
\par }\pard \qj\fi284\widctlpar\adjustright {While the work described in this paper already allowed us to publish several patches, improve our corpus processing library and contribute to new versions of other people\rquote 
s programs, namely Jspell, our future plans 
are to do more extensive testing using more powerful techniques (e.g. statistical) to investigate other problems or features of the corpus.  In any case, we believe that the work reported in this paper comes logically first.
\par }\pard\plain \s1\sb120\sa120\keepn\nowidctlpar\tx540\outlinelevel0\adjustright \b\f4\cgrid {Acknowledgements
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {We are first of all grateful to the }{\i P\'fablico}{ newspaper (especially Jos\'e9 V\'edtor Malheiros, the responsible for the online edition) for making this resource possible. We thank Jos
\'e9 Jo\'e3o Dias de Almeida for several suggestions, the team of Floresta Sint\'e1(c)tica for their thorough revision of the first million words, Stefan Evert for invaluable CQP support, and Jan Engh for helpful comments.
\par }\pard\plain \s1\sb120\sa120\keepn\nowidctlpar\tx540\outlinelevel0\adjustright \b\f4\cgrid {References
\par }\pard\plain \s39\qj\fi-227\li227\sb120\widctlpar\adjustright \fs20\cgrid {\lang2070 \hich\af0\dbch\af44\loch\f0 \hich\f0 Susana Cavadas Afonso and Ana Raquel Marchi. 2001. Crit\'e9\loch\f0 \hich\f0 rios de separa\'e7\'e3\loch\f0 \hich\f0 o de senten\'e7
\loch\f0 as/frases, {\*\bkmkstart _Hlt512672580}}{\field\fldedit{\*\fldinst {\lang2070 \hich\af0\dbch\af44\loch\f0  HYPERLINK http://cgi.portugue\hich\af0\dbch\af44\loch\f0 s.mct.pt/treebank/CriteriosSeparacao.html }{\lang2070 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000003d00000068007400740070003a002f002f006300670069002e0070006f0072007400750067007500650073002e006d00630074002e00700074002f007400720065006500620061006e006b002f0043007200690074006500720069006f007300530065007000
610072006100630061006f002e00680074006d006c000000e0c9ea79f9bace118c8200aa004ba90b7a00000068007400740070003a002f002f006300670069002e0070006f0072007400750067007500650073002e006d00630074002e00700074002f007400720065006500620061006e006b002f00430072006900740065
00720069006f007300530065007000610072006100630061006f002e00680074006d006c000000}}}{\fldrslt {\cs24\ul\cf2 \hich\af0\dbch\af44\loch\f0 cgi.portugues.mct.pt/treebank/CriteriosSeparacao.html}}}{\lang2070 {\*\bkmkend _Hlt512672580}
\par \hich\af0\dbch\af44\loch\f0 J.J. Almeida and Ulisses Pinto. 1994. Jspell \endash \hich\f0  um m\'f3\loch\f0 \hich\f0 dulo para an\'e1\loch\f0 \hich\f0 lise l\'e9\loch\f0 \hich\f0 xica gen\'e9\loch\f0 rica de linguagem natural. }{\i\lang2070 
\hich\af0\dbch\af44\loch\f0 \hich\f0 Actas do Congresso da Associa\'e7\'e3\loch\f0 o Portuguesa de Li\hich\af0\dbch\af44\loch\f0 \hich\f0 ngu\'ed\loch\f0 stica}{\lang2070 \hich\af0\dbch\af44\loch\f0 \hich\f0  (\'c9\loch\f0 vora, 1994), }
{\field\flddirty{\*\fldinst {\lang2070 \hich\af0\dbch\af44\loch\f0  HYPERLINK http://www.di.uminho.pt/~jj/pln/jspell1.ps.gz }{\lang2070 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b0200000017000000270000007700770077002e00640069002e0075006d0069006e0068006f002e00700074002f007e006a006a002f0070006c006e002f006a007300700065006c006c0031002e00700073002e0067007a000000e0c9ea79f9bace118c8200aa004ba90b5c000000
68007400740070003a002f002f007700770077002e00640069002e0075006d0069006e0068006f002e00700074002f007e006a006a002f0070006c006e002f006a007300700065006c006c0031002e00700073002e0067007a000000}}}{\fldrslt {\cs24\ul\cf2 \hich\af0\dbch\af44\loch\f0 
www.di.uminho.pt/~jj/pln/jspell1.ps.gz}}}{\lang2070 \hich\af0\dbch\af44\loch\f0 .
\par }{\hich\af0\dbch\af44\loch\f0 Susan Armstrong, Masja Kempen, David McKelvie, Dominique Petitpierre, Reinhard Rapp, and Henry S. Thompson. 1998. Multilingual Cor\hich\af0\dbch\af44\loch\f0 pora for Cooperation. In Antonio Rubio et al. (eds.), }{\i 
\hich\af0\dbch\af44\loch\f0 Proceedings of The First International Conference on Language Resources and Evaluation}{\hich\af0\dbch\af44\loch\f0  (Granada, 28-30 May 1998), Vol. 2, pp.975-80.}{\lang2070 
\par }{\hich\af0\dbch\af44\loch\f0 Oliver Christ, Bruno M. Schulze, Anja Hofmann and Esther Koenig. 1999\hich\af0\dbch\af44\loch\f0 
. The IMS Corpus Workbench: Corpus Query Processor (CQP): User's Manual, Institute for Natural Language Processing, University of Stuttgart }{\field\fldedit{\*\fldinst {\hich\af0\dbch\af44\loch\f0 
 HYPERLINK http://www.ims.uni-stuttgart.de/projekte/CorpusWorkbench/CQPUserManual }{{\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000004700000068007400740070003a002f002f007700770077002e0069006d0073002e0075006e0069002d007300740075007400740067006100720074002e00640065002f00700072006f006a0065006b00740065002f0043006f00720070007500730057006f00
72006b00620065006e00630068002f0043005100500055007300650072004d0061006e00750061006c000000e0c9ea79f9bace118c8200aa004ba90b8e00000068007400740070003a002f002f007700770077002e0069006d0073002e0075006e0069002d007300740075007400740067006100720074002e00640065002f
00700072006f006a0065006b00740065002f0043006f00720070007500730057006f0072006b00620065006e00630068002f0043005100500055007300650072004d0061006e00750061006c000000}}}{\fldrslt {\cs24\ul\cf2 \hich\af0\dbch\af44\loch\f0 http://www.ims.uni-stuttgart.de
\hich\af0\dbch\af44\loch\f0 /projekte/ CorpusWorkbench/CQPUserManual}}}{
\par \hich\af0\dbch\af44\loch\f0 Gregory Grefenstette and Pasi Tapanainen. 1994. What is a word, What is a sentence? Problems of Tokenization. }{\i \hich\af0\dbch\af44\loch\f0 
Proceedings of the 3rd International Conference on Computational Lexicography (COMPLEX'94)}{\hich\af0\dbch\af44\loch\f0 , pp. 79-87}{\lang2070 
\par }{\hich\af0\dbch\af44\loch\f0 St\hich\af0\dbch\af44\loch\f0 ig Johansson, Jarle Ebeling and Knut Hofland. 1996. Coding and aligning the English-Norwegian Parallel Corpus. In Karin Aijmer, Bengt Altenberg & Mats Johansson (eds.), }{\i 
\hich\af0\dbch\af44\loch\f0 Languages in Contrast: Papers from a Symposium on Text-based Cross-linguistic Studies (\hich\af0\dbch\af44\loch\f0 Lund, 4-5 March 1994)}{\hich\af0\dbch\af44\loch\f0 , Lund University Press, pp.87-112.
\par \hich\af0\dbch\af44\loch\f0 Mei Kobayashi and Koichi Takeda. 1999. Information retrieval on the web: Selected topics. IBM Research, Tokyo Research Laboratory, IBM Japan, Dec. 16, 1999.
\par \hich\af0\dbch\af44\loch\f0 Geoffrey Nunberg. 1990. }{\i \hich\af0\dbch\af44\loch\f0 The linguistics of\hich\af0\dbch\af44\loch\f0  punctuation}{\hich\af0\dbch\af44\loch\f0 . CSLI Lecture Notes, Number 18.
\par }{\lang2070 \hich\af0\dbch\af44\loch\f0 \hich\f0 Paulo Alexandre Rocha and Diana Santos. 2000. CETEMP\'fa\loch\f0 \hich\f0 blico: Um corpus de grandes dimens\'f5\loch\f0 \hich\f0 es de linguagem jornal\'ed\loch\f0 \hich\f0 stica portuguesa. In Gra\'e7
\loch\f0 a Nunes (ed.),}{\i\lang2070 \hich\af0\dbch\af44\loch\f0 \hich\f0  Actas do V Encontro para o processamento computacional da l\'ed\hich\af0\dbch\af44\loch\f0 ngua portuguesa escrita e falada (PROPOR\rquote 2000)}{\lang2070 
\hich\af0\dbch\af44\loch\f0 \hich\f0 , (S\'e3\loch\f0 o Paulo, 19-22 November 2000), pp.131-140.
\par \hich\af0\dbch\af44\loch\f0 Diana Santos. 1998}{\hich\af0\dbch\af44\loch\f0 \hich\f0 . Punctuation and multilinguality: Reflections from a language engineering perspective. In Jo Terje Ydstie and Anne C. Wolleb\'e6\loch\f0 k (eds.), }{\i 
\hich\af0\dbch\af44\loch\f0 Working \hich\af0\dbch\af44\loch\f0 Papers in Applied Linguistics}{\hich\af0\dbch\af44\loch\f0  4/98. Oslo: Department of Linguistics, Faculty of Arts, University of Oslo, pp.138-60.
\par \hich\af0\dbch\af44\loch\f0 \hich\f0 Diana Santos. 1999. Compara\'e7\'e3\loch\f0 \hich\f0 o de corpora em portugu\'ea\loch\f0 \hich\f0 s: algumas experi\'ea\loch\f0 ncias.  }{\field{\*\fldinst {\hich\af0\dbch\af44\loch\f0 
 HYPERLINK http://www.portugues.mct.pt/Diana/download/CCP.p\hich\af0\dbch\af44\loch\f0 s }{{\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000002b0000007700770077002e0070006f0072007400750067007500650073002e006d00630074002e00700074002f004400690061006e0061002f0064006f0077006e006c006f00610064002f004300430050002e00700073000000e0c9ea79f9bace118c8200aa
004ba90b6400000068007400740070003a002f002f007700770077002e0070006f0072007400750067007500650073002e006d00630074002e00700074002f004400690061006e0061002f0064006f0077006e006c006f00610064002f004300430050002e00700073000000}}}{\fldrslt {\cs24\ul\cf2 
\hich\af0\dbch\af44\loch\f0 www.portugues.mct.pt/Diana/download/CCP.ps}}}{
\par }{\lang2070 \hich\af0\dbch\af44\loch\f0 \hich\f0 Diana Santos. 2001. Resultado da revis\'e3\loch\f0 \hich\f0 o do primeiro milh\'e3\loch\f0 \hich\f0 o de palavras do CETEMP\'fa\loch\f0 blico c }{\field\fldedit{\*\fldinst {\lang2070 
\hich\af0\dbch\af44\loch\f0  HYPERLINK http://cgi.portugues.mct.pt/treebank/CriteriosSeparacao.html }{\lang2070 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000003d00000068007400740070003a002f002f006300670069002e0070006f0072007400750067007500650073002e006d00630074002e00700074002f007400720065006500620061006e006b002f0043007200690074006500720069006f007300530065007000
610072006100630061006f002e00680074006d006c000000e0c9ea79f9bace118c8200aa004ba90b7a00000068007400740070003a002f002f006300670069002e0070006f0072007400750067007500650073002e006d00630074002e00700074002f007400720065006500620061006e006b002f00430072006900740065
00720069006f007300530065007000610072006100630061006f002e00680074006d006c000000}}}{\fldrslt {\cs24\ul\cf2 \hich\af0\dbch\af44\loch\f0 gi.portugues.mct.pt/treebank/RevisaoMilhao.\hich\af0\dbch\af44\loch\f0 html}}}{\lang2070 
\par }{\hich\af0\dbch\af44\loch\f0 Diana Santos and Eckhard Bick. 2000. }{\lang2070 \hich\af0\dbch\af44\loch\f0 Providing Internet access to Portuguese corpora: the AC/DC project. In Maria Gavriladou et al. (eds.), }{\i\lang2070 \hich\af0\dbch\af44\loch\f0 
Proceedings of the Second International Conference on Language Resources and Evaluation,}{\lang2070 \hich\af0\dbch\af44\loch\f0  }{\i\lang2070 \hich\af0\dbch\af44\loch\f0 LREC2000 }{\lang2070 \hich\af0\dbch\af44\loch\f0 (Athens, 31 
\hich\af0\dbch\af44\loch\f0 May-2 June 2000), pp.205-210.}{
\par }\pard\plain \s1\sb120\sa120\keepn\nowidctlpar\tx540\outlinelevel0\adjustright \b\f4\cgrid {Appendix A. Example of an extract
\par }\pard\plain \qj\widctlpar\adjustright \f4\fs22\lang3084\cgrid {\f2\fs20\lang2070\cgrid0 <ext n=1914 sec=nd sem=93b> 
\par <p> <s>Produ\'e7\'e3o da Hammer.</s>
\par <s>Um epis\'f3dio da II Guerra Mundial, um caso de hero\'edsmo, quando toda uma companhia \'e9 destru\'edda no Norte de \'c1frica.</s> </p>
\par <li>THE STEEL BAYONET de Michael Carreras com Leo Glenn e Kieron Moore</li> 
\par <li>Gr\'e3-Bretanha, 1957, 82 min</li> 
\par <li>Canal 1, \'e0s 15h15</li> 
\par <p><s>Um ex-presidi\'e1rio esfor\'e7adamente em busca de regenera\'e7\'e3o (Nicolas Cage) e a mulher, uma honesta e voluntariosa pol\'edcia (Holly Hunter), querem formar fam\'edlia mas descobrem que n\'e3o podem ter filhos e decidem raptar um beb\'e9
.</s> 
\par <s>O cinema dos irm\'e3os Coen sempre atraiu cr\'edticas de \'abexibicionismo\'bb e \'abfogo-de-artif\'edcio\'bb.</s> 
\par <s>Esta com\'e9dia desbragada, que de uma s\'f3 vez faz um curto-circuito com as refer\'eancias \'e0 banda desenhada, ao burlesco ou \'e0 s\'e9rie \'abMad Max\'bb, \'e9 o tipo de objecto que mais evidencia o que os detractores dos Coen considerar\'e3o um 
\'abexerc\'edcio de estilo\'bb.</s> 
\par <s>\'abArizona Junior\'bb, concorde-se, \'e9 uma obra que exibe um gozo evidente pelas proezas do trabalho de c\'e2mara e Nicolas Cage, Holly Hunter ou John Goodman t\'eam a consist\'eancia de figuras de cart\'e3o.</s> 
\par <s>Mas nem por isso se deve ignorar estarmos perante um dos universos mais paran\'f3icos do cinema actual.</s> </p>
\par <t>RAISING ARIZONA de Joel Coen com Nicolas Cage, Holly Hunter e John Goodman</t> 
\par <t>EUA, 1987, 97 min</t>
\par <a>Quatro, \'e0s 21h35</a> </ext>
\par }\pard\plain \s19\qj\widctlpar\adjustright \f4\fs22\lang2070\cgrid {
\par }}