{\rtf1\ansi\ansicpg1252\uc1 \deff4\deflang1033\deflangfe1033{\fonttbl{\f0\froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\f1\fswiss\fcharset0\fprq2{\*\panose 020b0604020202020204}Arial;}
{\f2\fmodern\fcharset0\fprq1{\*\panose 02070309020205020404}Courier New;}{\f3\froman\fcharset2\fprq2{\*\panose 05050102010706020507}Symbol;}{\f4\froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times;}
{\f6\froman\fcharset0\fprq2{\*\panose 00000000000000000000}Courier;}{\f15\fswiss\fcharset0\fprq2{\*\panose 020b0604030504040204}Tahoma;}{\f16\fswiss\fcharset0\fprq2{\*\panose 020b0604030504040204}Verdana;}
{\f81\froman\fcharset161\fprq2{\*\panose 00000000000000000000}Times Greek{\*\falt Times New Roman};}{\f84\froman\fcharset238\fprq2 Times New Roman CE;}{\f85\froman\fcharset204\fprq2 Times New Roman Cyr;}
{\f87\froman\fcharset161\fprq2 Times New Roman Greek;}{\f88\froman\fcharset162\fprq2 Times New Roman Tur;}{\f89\froman\fcharset186\fprq2 Times New Roman Baltic;}{\f90\fswiss\fcharset238\fprq2 Arial CE;}{\f91\fswiss\fcharset204\fprq2 Arial Cyr;}
{\f93\fswiss\fcharset161\fprq2 Arial Greek;}{\f94\fswiss\fcharset162\fprq2 Arial Tur;}{\f95\fswiss\fcharset186\fprq2 Arial Baltic;}{\f96\fmodern\fcharset238\fprq1 Courier New CE;}{\f97\fmodern\fcharset204\fprq1 Courier New Cyr;}
{\f99\fmodern\fcharset161\fprq1 Courier New Greek;}{\f100\fmodern\fcharset162\fprq1 Courier New Tur;}{\f101\fmodern\fcharset186\fprq1 Courier New Baltic;}{\f174\fswiss\fcharset238\fprq2 Tahoma CE;}{\f175\fswiss\fcharset204\fprq2 Tahoma Cyr;}
{\f177\fswiss\fcharset161\fprq2 Tahoma Greek;}{\f178\fswiss\fcharset162\fprq2 Tahoma Tur;}{\f179\fswiss\fcharset186\fprq2 Tahoma Baltic;}{\f180\fswiss\fcharset238\fprq2 Verdana CE;}{\f181\fswiss\fcharset204\fprq2 Verdana Cyr;}
{\f183\fswiss\fcharset161\fprq2 Verdana Greek;}{\f184\fswiss\fcharset162\fprq2 Verdana Tur;}{\f185\fswiss\fcharset186\fprq2 Verdana Baltic;}}{\colortbl;\red0\green0\blue0;\red0\green0\blue255;\red0\green255\blue255;\red0\green255\blue0;
\red255\green0\blue255;\red255\green0\blue0;\red255\green255\blue0;\red255\green255\blue255;\red0\green0\blue128;\red0\green128\blue128;\red0\green128\blue0;\red128\green0\blue128;\red128\green0\blue0;\red128\green128\blue0;\red128\green128\blue128;
\red192\green192\blue192;}{\stylesheet{\qj\fi227\widctlpar\adjustright \f4\fs20\cgrid \snext0 Normal;}{\s1\qj\sa1600\sl-320\slmult0\keep\keepn\pagebb\widctlpar\tx284\hyphpar0\outlinelevel0\adjustright \b\f4\fs28\cgrid \sbasedon0 \snext0 heading 1;}{
\s2\qj\sb520\sa280\sl-280\slmult0\keep\keepn\widctlpar\tx454\hyphpar0\outlinelevel1\adjustright \b\f4\fs20\cgrid \sbasedon0 \snext0 heading 2;}{\s3\qj\sb440\sa220\sl-240\slmult0\keep\keepn\widctlpar\tx510\hyphpar0\outlinelevel2\adjustright 
\b\f4\fs20\cgrid \sbasedon0 \snext0 heading 3;}{\s4\qj\sb240\sa60\keepn\widctlpar\ls1\ilvl3\outlinelevel3\adjustright \b\f1\cgrid \sbasedon0 \snext0 heading 4;}{\s5\qj\sb240\sa60\widctlpar\ls1\ilvl4\outlinelevel4\adjustright \f1\fs22\cgrid 
\sbasedon0 \snext0 heading 5;}{\s6\qj\sb240\sa60\widctlpar\ls1\ilvl5\outlinelevel5\adjustright \i\f4\fs22\cgrid \sbasedon0 \snext0 heading 6;}{\s7\qj\sb240\sa60\widctlpar\ls1\ilvl6\outlinelevel6\adjustright \f1\fs20\cgrid \sbasedon0 \snext0 heading 7;}{
\s8\qj\sb240\sa60\widctlpar\ls1\ilvl7\outlinelevel7\adjustright \i\f1\fs20\cgrid \sbasedon0 \snext0 heading 8;}{\s9\qj\sb240\sa60\widctlpar\ls1\ilvl8\outlinelevel8\adjustright \b\i\f1\fs18\cgrid \sbasedon0 \snext0 heading 9;}{\*\cs10 \additive 
Default Paragraph Font;}{\s15\qj\fi227\widctlpar\tqc\tx4536\tqr\tx9072\adjustright \f4\fs20\cgrid \sbasedon0 \snext15 header;}{\s16\qj\fi227\widctlpar\tqc\tx4536\tqr\tx9072\adjustright \f4\fs20\cgrid \sbasedon0 \snext16 footer;}{\*\cs17 \additive 
\sbasedon10 page number;}{\s18\qc\fi227\sa460\keep\keepn\pagebb\widctlpar\tx284\hyphpar0\adjustright \b\f4\fs28\cgrid \sbasedon0 \snext19 title;}{\s19\qc\fi227\sa220\widctlpar\adjustright \f4\fs20\cgrid \sbasedon0 \snext20 author;}{
\s20\qc\fi227\widctlpar\adjustright \f4\fs18\cgrid \sbasedon0 \snext21 address;}{\s21\qc\fi227\widctlpar\adjustright \f6\fs18\cgrid \sbasedon0 \snext28 email;}{\s22\qj\sb520\sa280\keep\keepn\widctlpar\tx454\hyphpar0\adjustright \b\f4\cgrid 
\sbasedon0 \snext29 heading1;}{\s23\qj\sb440\sa220\keep\keepn\widctlpar\tx510\hyphpar0\adjustright \b\f4\fs20\cgrid \sbasedon0 \snext29 heading2;}{\s24\qj\sb320\keep\keepn\widctlpar\tx284\hyphpar0\adjustright \b\f4\fs20\cgrid \sbasedon0 \snext29 
heading3;}{\s25\qc\fi227\li227\sb120\sa120\widctlpar\tx6237\adjustright \f4\fs20\cgrid \sbasedon0 \snext0 equation;}{\s26\qj\sb120\sa240\keep\keepn\widctlpar\adjustright \f4\fs18\cgrid \sbasedon0 \snext0 figure legend;}{
\s27\qj\sb240\sa120\keep\keepn\widctlpar\adjustright \f4\fs18\lang1031\cgrid \sbasedon0 \snext0 table title;}{\s28\qj\li567\ri567\sb600\sa120\widctlpar\adjustright \f4\fs18\cgrid \sbasedon29 \snext22 abstract;}{\s29\qj\widctlpar\adjustright 
\f4\fs20\cgrid \sbasedon0 \snext0 p1a;}{\s30\qj\fi-227\li227\widctlpar\adjustright \f4\fs18\cgrid \sbasedon0 \snext30 referenceitem;}{\*\cs31 \additive \fs12\up6\nosupersub \sbasedon10 footnote reference;}{\s32\sa240\widctlpar
\tx680\tqr\tx6237\tqr\tx6917\adjustright \f4\fs18\cgrid \sbasedon0 \snext32 Running head - left;}{\s33\qr\sa240\widctlpar\tx680\tqr\tx6237\tqr\tx6917\adjustright \f4\fs18\cgrid \sbasedon32 \snext33 Running head - right;}{\s34\qj\fi-227\li227\widctlpar
\tx227\tx454{\*\pn \pnlvlbody\ilvl11\ls2047\pnrnot0\pnf4\pnstart1\pnindent227\pnhang{\pntxtb ?}}\ls2047\ilvl11\adjustright \f4\fs20\cgrid \sbasedon35 \snext34 Bullet Item;}{\s35\qj\fi-227\li227\widctlpar\tx227\tx454{\*\pn \pnlvlbody\ilvl11\ls2047\pnrnot0
\pnf4\pnstart1\pnindent227\pnhang{\pntxtb ?}}\ls2047\ilvl11\adjustright \f4\fs20\cgrid \sbasedon0 \snext0 Item;}{\s36\qj\fi-227\li227\widctlpar\tx227\tx454{\*\pn \pnlvlbody\ilvl11\ls2047\pnrnot0\pndec\pnstart1\pnindent227\pnhang{\pntxta .}}
\ls2047\ilvl11\adjustright \f4\fs20\cgrid \sbasedon35 \snext36 Numbered Item;}{\s37\qj\fi-170\li170\sl-220\slmult0\widctlpar\tx170\adjustright \f4\fs18\cgrid \sbasedon0 \snext37 footnote text;}{\s38\li227\sb120\sa120\widctlpar
\tx1361\tx1531\tx1701\tx1871\tx2041\tx2211\tx2381\tx2552\adjustright \f6\fs20\cgrid \sbasedon0 \snext38 programcode;}{\s39\qj\fi-170\li170\widctlpar\tx170\adjustright \f4\fs18\cgrid \sbasedon0 \snext39 Fu\'dfnotentext.Footnote;}{
\s40\qj\fi227\sb120\sa120\widctlpar\adjustright \b\f4\fs20\cgrid \sbasedon0 \snext0 caption;}{\s41\qj\sb320\widctlpar\adjustright \i\f4\fs20\cgrid \sbasedon0 \snext29 heading4;}{\*\cs42 \additive \ul\cf2 \sbasedon10 Hyperlink;}{
\s43\qj\fi227\widctlpar\adjustright \f4\fs20\cgrid \sbasedon0 \snext43 Body Text 2;}{\s44\qj\fi227\widctlpar\adjustright \cbpat9 \f15\fs20\cgrid \sbasedon0 \snext44 Document Map;}{\s45\qc\sb20\keepn\nowidctlpar
\tqc\tx1980\tqc\tx4520\tqc\tx7100\tqr\tx9000\adjustright \f4\fs22\lang2057\cgrid \sbasedon0 \snext45 Author;}{\s46\qj\fi-180\li180\sa60\widctlpar\tx4000\tx10279\adjustright \f4\fs20\lang2057\cgrid \sbasedon0 \snext46 Bibliography;}{
\s47\widctlpar\adjustright \f2\fs20\lang1044\cgrid \sbasedon0 \snext47 Plain Text;}{\s48\sb100\sa100\widctlpar\adjustright \cbpat8 \f1\cf1\lang1046\cgrid \sbasedon0 \snext48 testo;}{\s49\sb100\sa100\widctlpar\adjustright \f4\cf1\lang1046\cgrid 
\sbasedon0 \snext49 Normal (Web);}{\s50\qj\sb240\widctlpar\adjustright \b\f4\fs26\cgrid \sbasedon0 \snext0 Heading;}{\*\cs51 \additive \sbasedon10 m;}{\*\cs52 \additive \b\f16\fs19\cf0\chshdng0\chcfpat0\chcbpat0 \sbasedon10 volissue;}{\*\cs53 \additive 
\ul\cf12 \sbasedon10 FollowedHyperlink;}{\s54\qj\fi227\widctlpar\adjustright \f15\fs16\cgrid \sbasedon0 \snext54 Balloon Text;}}{\*\listtable{\list\listtemplateid-1{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat0\levelspace0\levelindent0
{\leveltext\'00;}{\levelnumbers;}}{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat0\levelspace0\levelindent0{\leveltext\'00;}{\levelnumbers;}}{\listlevel\levelnfc255\leveljc0\levelfollow0\levelstartat0\levelspace0\levelindent0{\leveltext
\'00;}{\levelnumbers;}}{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat0\levelold\levelspace0\levelindent0{\leveltext\'01\'03;}{\levelnumbers\'01;}\f0\fbias0 \s4}{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat0\levelold\levelspace0
\levelindent0{\leveltext\'01\'04;}{\levelnumbers\'01;}\f0\fbias0 \s5}{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat0\levelold\levelspace0\levelindent0{\leveltext\'01\'05;}{\levelnumbers\'01;}\f0\fbias0 \s6}{\listlevel\levelnfc0\leveljc0
\levelfollow0\levelstartat0\levelold\levelspace0\levelindent0{\leveltext\'01\'06;}{\levelnumbers\'01;}\f0\fbias0 \s7}{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat0\levelold\levelspace0\levelindent0{\leveltext\'01\'07;}{\levelnumbers\'01;}
\f0\fbias0 \s8}{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat0\levelold\levelspace0\levelindent0{\leveltext\'01\'08;}{\levelnumbers\'01;}\f0\fbias0 \s9}{\listname ;}\listid-5}{\list\listtemplateid-1\listsimple{\listlevel\levelnfc0\leveljc0
\levelfollow0\levelstartat0\levelspace0\levelindent0{\leveltext\'01*;}{\levelnumbers;}}{\listname ;}\listid-2}{\list\listtemplateid2{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers
\'01;}\fi-283\li283 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'01.;}{\levelnumbers\'01;}\fi-283\li567 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext
\'02\'02.;}{\levelnumbers\'01;}\fi-283\li850 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'03.;}{\levelnumbers\'01;}\fi-283\li1134 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0
\levelindent0{\leveltext\'02\'04.;}{\levelnumbers\'01;}\fi-283\li1417 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'05.;}{\levelnumbers\'01;}\fi-283\li1701 }{\listlevel\levelnfc0\leveljc0\levelfollow2
\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'06.;}{\levelnumbers\'01;}\fi-283\li1984 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'07.;}{\levelnumbers\'01;}\fi-283\li2268 }{\listlevel
\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'08.;}{\levelnumbers\'01;}\fi-283\li2551 }{\listname ;}\listid2}{\list\listtemplateid3{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0
{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-283\li283 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'01.;}{\levelnumbers\'01;}\fi-283\li567 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1
\levelspace0\levelindent0{\leveltext\'02\'02.;}{\levelnumbers\'01;}\fi-283\li850 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'03.;}{\levelnumbers\'01;}\fi-283\li1134 }{\listlevel\levelnfc0\leveljc0
\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'04.;}{\levelnumbers\'01;}\fi-283\li1417 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'05.;}{\levelnumbers\'01;}\fi-283\li1701 }
{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'06.;}{\levelnumbers\'01;}\fi-283\li1984 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext
\'02\'07.;}{\levelnumbers\'01;}\fi-283\li2268 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'08.;}{\levelnumbers\'01;}\fi-283\li2551 }{\listname ;}\listid3}{\list\listtemplateid4{\listlevel\levelnfc0
\leveljc0\levelfollow2\levelstartat3\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-283\li283 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'03\'00.\'01;}{\levelnumbers\'01\'03;}
\fi-283\li329 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'06\'00.\'01.\'02.;}{\levelnumbers\'01\'03\'05;}\fi-283\li375 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0
{\leveltext\'08\'00.\'01.\'02.\'03.;}{\levelnumbers\'01\'03\'05\'07;}\fi-283\li421 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'0a\'00.\'01.\'02.\'03.\'04.;}{\levelnumbers\'01\'03\'05\'07\'09;}
\fi-283\li467 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'0c\'00.\'01.\'02.\'03.\'04.\'05.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b;}\fi-283\li513 }{\listlevel\levelnfc0\leveljc0\levelfollow2
\levelstartat1\levelspace0\levelindent0{\leveltext\'0e\'00.\'01.\'02.\'03.\'04.\'05.\'06.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d;}\fi-283\li559 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext
\'10\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f;}\fi-283\li605 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext
\'12\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.\'08.;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f\'11;}\fi-283\li651 }{\listname ;}\listid4}{\list\listtemplateid5{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext
\'02\'00.;}{\levelnumbers\'01;}\fi-283\li283 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'01.;}{\levelnumbers\'01;}\fi-283\li567 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0
\levelindent0{\leveltext\'02\'02.;}{\levelnumbers\'01;}\fi-283\li850 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'03.;}{\levelnumbers\'01;}\fi-283\li1134 }{\listlevel\levelnfc0\leveljc0\levelfollow2
\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'04.;}{\levelnumbers\'01;}\fi-283\li1417 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'05.;}{\levelnumbers\'01;}\fi-283\li1701 }{\listlevel
\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'06.;}{\levelnumbers\'01;}\fi-283\li1984 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'07.;}{\levelnumbers\'01;}
\fi-283\li2268 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'08.;}{\levelnumbers\'01;}\fi-283\li2551 }{\listname ;}\listid5}{\list\listtemplateid6{\listlevel\levelnfc0\leveljc0\levelfollow2
\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-283\li283 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'01.;}{\levelnumbers\'01;}\fi-283\li567 }{\listlevel
\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'02.;}{\levelnumbers\'01;}\fi-283\li850 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'03.;}{\levelnumbers\'01;}
\fi-283\li1134 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'04.;}{\levelnumbers\'01;}\fi-283\li1417 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext
\'02\'05.;}{\levelnumbers\'01;}\fi-283\li1701 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'06.;}{\levelnumbers\'01;}\fi-283\li1984 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1
\levelspace0\levelindent0{\leveltext\'02\'07.;}{\levelnumbers\'01;}\fi-283\li2268 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'08.;}{\levelnumbers\'01;}\fi-283\li2551 }{\listname ;}\listid6}
{\list\listtemplateid7{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-283\li283 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext
\'02\'01.;}{\levelnumbers\'01;}\fi-283\li567 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'02.;}{\levelnumbers\'01;}\fi-283\li850 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0
\levelindent0{\leveltext\'02\'03.;}{\levelnumbers\'01;}\fi-283\li1134 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'04.;}{\levelnumbers\'01;}\fi-283\li1417 }{\listlevel\levelnfc0\leveljc0\levelfollow2
\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'05.;}{\levelnumbers\'01;}\fi-283\li1701 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'06.;}{\levelnumbers\'01;}\fi-283\li1984 }{\listlevel
\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'07.;}{\levelnumbers\'01;}\fi-283\li2268 }{\listlevel\levelnfc0\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'08.;}{\levelnumbers\'01;}
\fi-283\li2551 }{\listname ;}\listid7}{\list\listtemplateid8{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li283 }{\listlevel\levelnfc23\leveljc0\levelfollow2
\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li567 }{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li850 }{\listlevel
\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li1134 }{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}
\fs18 \fi-283\li1417 }{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li1701 }{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0
{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li1984 }{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li2268 }{\listlevel\levelnfc23\leveljc0\levelfollow2
\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li2551 }{\listname ;}\listid8}{\list\listtemplateid9{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext
\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li283 }{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li567 }{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1
\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li850 }{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li1134 }{\listlevel\levelnfc23
\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li1417 }{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 
\fi-283\li1701 }{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li1984 }{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext
\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li2268 }{\listlevel\levelnfc23\leveljc0\levelfollow2\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\fs18 \fi-283\li2551 }{\listname ;}\listid9}{\list\listtemplateid67698703\listsimple
{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid47188797}{\list\listtemplateid67698689\listsimple{\listlevel\levelnfc23\leveljc0
\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\u-3913 ?;}{\levelnumbers;}\f3\fbias0 \fi-360\li360\jclisttab\tx360 }{\listname ;}\listid338775699}{\list\listtemplateid-1470958900{\listlevel\levelnfc23\leveljc0\levelfollow0
\levelstartat1\levelspace0\levelindent0{\leveltext\'01\u-3913 ?;}{\levelnumbers;}\f3\cf0\fbias0 \fi-360\li360\jclisttab\tx360 }{\listlevel\levelnfc4\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'01.;}{\levelnumbers\'01;}
\fi-360\li1440\jclisttab\tx1440 }{\listlevel\levelnfc2\leveljc2\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'02.;}{\levelnumbers\'01;}\fi-180\li2160\jclisttab\tx2160 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1
\levelspace0\levelindent0{\leveltext\'02\'03.;}{\levelnumbers\'01;}\fi-360\li2880\jclisttab\tx2880 }{\listlevel\levelnfc4\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'04.;}{\levelnumbers\'01;}\fi-360\li3600
\jclisttab\tx3600 }{\listlevel\levelnfc2\leveljc2\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'05.;}{\levelnumbers\'01;}\fi-180\li4320\jclisttab\tx4320 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0
\levelindent0{\leveltext\'02\'06.;}{\levelnumbers\'01;}\fi-360\li5040\jclisttab\tx5040 }{\listlevel\levelnfc4\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'07.;}{\levelnumbers\'01;}\fi-360\li5760\jclisttab\tx5760 }
{\listlevel\levelnfc2\leveljc2\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'08.;}{\levelnumbers\'01;}\fi-180\li6480\jclisttab\tx6480 }{\listname ;}\listid388119439}{\list\listtemplateid67698689\listsimple{\listlevel\levelnfc23
\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\u-3913 ?;}{\levelnumbers;}\f3\fbias0 \fi-360\li360\jclisttab\tx360 }{\listname ;}\listid448941083}{\list\listtemplateid67698703\listsimple{\listlevel\levelnfc0\leveljc0
\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid508444899}{\list\listtemplateid68419599\listsimple{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1
\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid527375506}{\list\listtemplateid67698703\listsimple{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0
{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid616714705}{\list\listtemplateid68419585\listsimple{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'01\u-3913 ?;}{\levelnumbers;}\f3\fbias0 \fi-360\li360\jclisttab\tx360 }{\listname ;}\listid671294152}{\list\listtemplateid67698689\listsimple{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'01\u-3913 ?;}{\levelnumbers;}\f3\fbias0 \fi-360\li360\jclisttab\tx360 }{\listname ;}\listid820080634}{\list\listtemplateid67698703\listsimple{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid1077173337}{\list\listtemplateid826948848{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat2\levelspace0\levelindent0{\leveltext\'01\'00;}{\levelnumbers\'01;}\fbias0 
\fi-450\li810\jclisttab\tx810 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext\'03\'00.\'01;}{\levelnumbers\'01\'03;}\fbias0 \fi-360\li720\jclisttab\tx720 }{\listlevel\levelnfc0\leveljc0
\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext\'05\'00.\'01.\'02;}{\levelnumbers\'01\'03\'05;}\fbias0 \fi-720\li1080\jclisttab\tx1080 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0
\levelindent0{\leveltext\'07\'00.\'01.\'02.\'03;}{\levelnumbers\'01\'03\'05\'07;}\fbias0 \fi-720\li1080\jclisttab\tx1080 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'09\'00.\'01.\'02.\'03.\'04;}{\levelnumbers\'01\'03\'05\'07\'09;}\fbias0 \fi-720\li1080\jclisttab\tx1080 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'0b\'00.\'01.\'02.\'03.\'04.\'05;}{\levelnumbers\'01\'03\'05\'07\'09\'0b;}\fbias0 \fi-1080\li1440\jclisttab\tx1440 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'0d\'00.\'01.\'02.\'03.\'04.\'05.\'06;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d;}\fbias0 \fi-1080\li1440\jclisttab\tx1440 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'0f\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f;}\fbias0 \fi-1440\li1800\jclisttab\tx1800 }{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levellegal\levelspace0\levelindent0{\leveltext
\'11\'00.\'01.\'02.\'03.\'04.\'05.\'06.\'07.\'08;}{\levelnumbers\'01\'03\'05\'07\'09\'0b\'0d\'0f\'11;}\fbias0 \fi-1440\li1800\jclisttab\tx1800 }{\listname ;}\listid1248729263}{\list\listtemplateid439123862{\listlevel\levelnfc23\leveljc0\levelfollow0
\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\f0\fbias0 \fi-360\li720\jclisttab\tx720 }{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat163\levelspace0\levelindent0{\leveltext\'01\'96;}{\levelnumbers;}\f0\fbias0 
\fi-360\li1440\jclisttab\tx1440 }{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\f0\fbias0 \fi-360\li2160\jclisttab\tx2160 }{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat1
\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\f0\fbias0 \fi-360\li2880\jclisttab\tx2880 }{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\f0\fbias0 \fi-360\li3600
\jclisttab\tx3600 }{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\f0\fbias0 \fi-360\li4320\jclisttab\tx4320 }{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat1\levelspace0
\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\f0\fbias0 \fi-360\li5040\jclisttab\tx5040 }{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\f0\fbias0 \fi-360\li5760
\jclisttab\tx5760 }{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\'95;}{\levelnumbers;}\f0\fbias0 \fi-360\li6480\jclisttab\tx6480 }{\listname ;}\listid1360811574}{\list\listtemplateid-1696673700
\listsimple{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat0\levelspace0\levelindent0{\leveltext\'01-;}{\levelnumbers;}\f0\fbias0 \fi-360\li360\jclisttab\tx360 }{\listname ;}\listid1563298391}{\list\listtemplateid67698703\listsimple{\listlevel
\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid1607883435}{\list\listtemplateid-539972730\listsimple{\listlevel\levelnfc23\leveljc0
\levelfollow0\levelstartat0\levelspace0\levelindent0{\leveltext\'01-;}{\levelnumbers;}\fbias0 \fi-360\li360\jclisttab\tx360 }{\listname ;}\listid1660573395}{\list\listtemplateid68419599\listsimple{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1
\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid1925333980}{\list\listtemplateid-1696673700\listsimple{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat0\levelspace0\levelindent0
{\leveltext\'01-;}{\levelnumbers;}\f0\fbias0 \fi-360\li360\jclisttab\tx360 }{\listname ;}\listid2010015627}{\list\listtemplateid67698703\listsimple{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext
\'02\'00.;}{\levelnumbers\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid2013868907}{\list\listtemplateid67698703\listsimple{\listlevel\levelnfc0\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'02\'00.;}{\levelnumbers
\'01;}\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid2057924628}{\list\listtemplateid67698689\listsimple{\listlevel\levelnfc23\leveljc0\levelfollow0\levelstartat1\levelspace0\levelindent0{\leveltext\'01\u-3913 ?;}{\levelnumbers;}\f3\fbias0 
\fi-360\li360\jclisttab\tx360 }{\listname ;}\listid2081713527}}{\*\listoverridetable{\listoverride\listid-5\listoverridecount0\ls1}{\listoverride\listid-2\listoverridecount1{\lfolevel\listoverrideformat{\listlevel\levelnfc23\leveljc0\levelfollow0
\levelstartat1\levelold\levelspace0\levelindent227{\leveltext\'01\u-4051 ?;}{\levelnumbers;}\f3\fbias0 \fi-227\li227 }}\ls2}{\listoverride\listid1563298391\listoverridecount0\ls3}{\listoverride\listid2010015627\listoverridecount0\ls4}
{\listoverride\listid1660573395\listoverridecount0\ls5}{\listoverride\listid1925333980\listoverridecount0\ls6}{\listoverride\listid1607883435\listoverridecount0\ls7}{\listoverride\listid448941083\listoverridecount0\ls8}{\listoverride\listid2081713527
\listoverridecount0\ls9}{\listoverride\listid820080634\listoverridecount0\ls10}{\listoverride\listid616714705\listoverridecount0\ls11}{\listoverride\listid2057924628\listoverridecount0\ls12}{\listoverride\listid2013868907\listoverridecount0\ls13}
{\listoverride\listid1077173337\listoverridecount0\ls14}{\listoverride\listid508444899\listoverridecount0\ls15}{\listoverride\listid338775699\listoverridecount0\ls16}{\listoverride\listid527375506\listoverridecount0\ls17}{\listoverride\listid1248729263
\listoverridecount0\ls18}{\listoverride\listid1360811574\listoverridecount0\ls19}{\listoverride\listid388119439\listoverridecount0\ls20}{\listoverride\listid671294152\listoverridecount0\ls21}{\listoverride\listid2\listoverridecount0\ls22}
{\listoverride\listid3\listoverridecount0\ls23}{\listoverride\listid4\listoverridecount0\ls24}{\listoverride\listid5\listoverridecount0\ls25}{\listoverride\listid6\listoverridecount0\ls26}{\listoverride\listid7\listoverridecount0\ls27}
{\listoverride\listid8\listoverridecount0\ls28}{\listoverride\listid9\listoverridecount0\ls29}{\listoverride\listid47188797\listoverridecount0\ls30}}{\*\revtbl {Unknown;}{Diana Santos;}}{\info{\title Lecture Notes in Computer Science:}{\author foobar}{\operator Diana Santos}
{\creatim\yr2003\mo3\dy24\hr11\min27}{\revtim\yr2003\mo3\dy24\hr11\min27}{\printim\yr2003\mo3\dy24\hr11\min27}{\version2}{\edmins0}{\nofpages8}{\nofwords3867}{\nofchars20885}{\*\company Springer Verlag}{\nofcharsws0}{\vern89}}
\paperw11907\paperh16840\margl2495\margr2495\margt2948\margb2948 
\deftab227\widowctrl\ftnbj\aenddoc\hyphhotz425\revisions\makebackup\margmirror\noxlattoyen\expshrtn\noultrlspc\dntblnsbdb\nospaceforul\lytprtmet\hyphcaps0\hyphauto1\formshade\viewkind1\viewscale100\pgbrdrhead\pgbrdrfoot \fet0\sectd 
\psz9\linex0\headery2381\footery1389\colsx709\sectdefaultcl {\*\pnseclvl1\pnucrm\pnstart1\pnindent720\pnhang{\pntxta .}}{\*\pnseclvl2\pnucltr\pnstart1\pnindent720\pnhang{\pntxta .}}{\*\pnseclvl3\pndec\pnstart1\pnindent720\pnhang{\pntxta .}}{\*\pnseclvl4
\pnlcltr\pnstart1\pnindent720\pnhang{\pntxta )}}{\*\pnseclvl5\pndec\pnstart1\pnindent720\pnhang{\pntxtb (}{\pntxta )}}{\*\pnseclvl6\pnlcltr\pnstart1\pnindent720\pnhang{\pntxtb (}{\pntxta )}}{\*\pnseclvl7\pnlcrm\pnstart1\pnindent720\pnhang{\pntxtb (}
{\pntxta )}}{\*\pnseclvl8\pnlcltr\pnstart1\pnindent720\pnhang{\pntxtb (}{\pntxta )}}{\*\pnseclvl9\pnlcrm\pnstart1\pnindent720\pnhang{\pntxtb (}{\pntxta )}}\pard\plain \s18\qc\fi227\sa460\keep\keepn\pagebb\widctlpar\tx284\hyphpar0\adjustright 
\b\f4\fs28\cgrid {An initial proposal for cooperative evaluation on information retrieval in Portuguese
\par }\pard\plain \s19\qc\fi227\sa220\widctlpar\adjustright \f4\fs20\cgrid {\lang1046 Rachel Aires}{\fs14\up6\lang1046 1,2}{\lang1046 , Sandra Alu\'edsio}{\fs14\up6\lang1046 2}{\lang1046 , Paulo Quaresma}{\lang1046\super 3}{\lang1046 ,\line Diana Santos}{
\lang1046\super 1}{\lang1046  and M\'e1rio J. Silva}{\fs14\up6\lang1046 4}{\lang1046 
\par }\pard\plain \s20\qc\fi227\widctlpar\adjustright \f4\fs18\cgrid {\super 1}{SINTEF Telecom and Informatics, Box 124 Blindern, 
\par N-0314 Oslo, Norway
\par }\pard\plain \s21\qc\fi227\widctlpar\adjustright \f6\fs18\cgrid {\{Rachel.V.Aires, Diana.Santos\}@sintef.no
\par }\pard\plain \s20\qc\fi227\widctlpar\adjustright \f4\fs18\cgrid {\lang1046\super 2}{\lang1046 ICMC-Universidade de S\'e3o Paulo, NILC, C.P. 668 
\par 13560-970, S\'e3o Carlos, SP, Brazil
\par }\pard\plain \s21\qc\fi227\widctlpar\adjustright \f6\fs18\cgrid {\lang1046 \{raires, sandra\}@icmc.usp.br
\par }\pard\plain \s20\qc\fi227\widctlpar\adjustright \f4\fs18\cgrid {\lang1046\super 3}{\lang1046 Departamento de Inform\'e1tica, Universidade de \'c9vora\line R. Rom\'e3o Ramalho, 59, 7000 \'c9vora, Portugal
\par }\pard\plain \s21\qc\fi227\widctlpar\adjustright \f6\fs18\cgrid {\lang1046 pq@di.uevora.pt
\par }\pard\plain \s20\qc\fi227\widctlpar\adjustright \f4\fs18\cgrid {\lang1046\super 4}{\lang1046 Departamento de Inform\'e1tica da Faculdade de Ci\'eancias da Universidade de Lisboa, 
\par Campo Grande 1749-016 Lisboa, Portugal
\par }\pard\plain \s21\qc\fi227\widctlpar\adjustright \f6\fs18\cgrid {\lang1046 mjs@di.fc.ul.pt
\par }\pard\plain \s28\qj\li567\ri567\sb600\sa120\widctlpar\adjustright \f4\fs18\cgrid {\b Abstract}{
. In this paper we discuss evaluation of information retrieval, Web search and question answering systems, paving the way for the organization of an evaluation contest on IR for Portuguese. Inspired by current international setu
ps, we motivate the need to study the specific problems posed by Port}{u}{guese, suggesting a collection suitable for multiple tasks.
\par }\pard\plain \s22\qj\sb520\sa280\keep\keepn\widctlpar\tx454\hyphpar0\outlinelevel0\adjustright \b\f4\cgrid {1\tab Introduction
\par }\pard\plain \qj\fi227\nowidctlpar\adjustright \f4\fs20\cgrid {\cf1 Information retrieval (IR) may be broadly defined as the process of finding info}{\cf1 r}{\cf1 mation satisfying a user's nee
d [1], with the main focus of the IR community being in text retrieval, i.e. in finding specific documents in large text bases. IR research has also been extended in recent years to information available in other media, such as video and sound, and to inf
o
rmation contained in (semi) structured forms, such as XTML and XML. An even broader definition of IR includes further processing of the information, such that more complex processes like summarization, explicit answer to a question (eventually requiring r
easoning), information extraction or mining, and identification of the gist, are seen as different facets of the same discipline.
\par }{Motivation for assessing the status of IR (in a broad sense, as defined above) in Portuguese stems from several reasons. These range from the need of obtaining an indicator of \ldblquote information society progress\rdblquote 
 for Portuguese-speaking countries and market oriented information providers, to a genuine research interest in issues co}{n}{nected with both IR and evaluation in general, and natu
ral language processing in particular. We will not dwell in this paper on the first kind of (macro-economic) mot}{i}{
vation. We shall rather discuss why the study and evaluation of IR in Portuguese can be of particular interest for both the development and imp
rovement of already existing systems dealing with Portuguese, such as Web search engines or specific database query systems, and for the NLP community that deals with Portuguese and wishes to address real world problems. 
\par For researchers of the first profile, it is advantageous to identify the challenges posed by IR in Portuguese, especially where an English-based architecture will be likely to miss the point. In addition, it is relevant to learn whether Portuguese-speakin
g users have different requirements and practice. For IR in the Web, it sounds misguided to assume that the distribution of subject matter and users\rquote 
 profiles mirrors that on the Web in general (see [2] for some comments). For a summarization task, it is al
so far from obvious that good abstracts in Portuguese should obey the same rules of their English counterparts, given the different rhetorical conventions (and practice) for the two languages. Also, the dialogue flow and implicit expectations are well kno
wn to differ, so one could, at least, hypothesize that the same would happen when interacting with a computer.
\par  For the traditional NLP community, it is high time that specialized tasks and toy-like problems were replaced by real applications that deal with text and are employed by real users. This is the case of applications such as search engines or question a}{
n}{swering systems. The aim would be not only to improve its self-confidence as an e}{n}{gineering discipline, but most of all to gain an opportunity to test systems and r}{e}{
sources against real data. Information retrieval, unlike e.g. spelling correction, gives the opportunity to test semantic and pragmatic modules and theories of the language, and thus cannot be dismissed as theoretically uninteresting. Subj
ects like sieving te}{r}{rorism information, describing commercial venues or automatically creating \ldblquote who\rquote s who\rdblquote 
 directories have been approached with a simple pattern matching approach (as shown by the MUC conferences, see [3]), but they can also resort to com
plex models of events, fine-grained representations of time, and even non-monotonic inference; together with complex NLP techniques such as anaphor resolution, sense disambigu}{a}{tion, and thesaurus induction [4]. }{\cf6 
\par }{But if we want to do information retrieval in Portuguese, and evaluate its success, we have to begin by agreeing on some subset of well-de}{\f81 fined task on whose solution we can agree on \'af
 and test the whole set of applications against some common base agreed upon. This is the evaluation contest model, which is thoroughly described elsewhere (see e.g [3], [5] and [}{6]). Basically, participants should agree on the cond}{i}{
tions of participation (e.g. the restrict use of some data collection); the results of pa}{r}{ticipant\rquote s systems are compared using the same ta
sks, the same data collection (the training data, the testing data and the answer key) and the same evaluation measures. However, this definition hides all the management work and costs involved to make the contest work from the first call for participati
o
n to the evaluation workshop. The latter serves both to present and discuss results from the experiments, including failure analyses and system comparisons (see for example, the comparative analysis of the results of CLEF 2002 campaign [7], of those of th
e Third NTCIR Web Retrieval Task [8], and the analysis of CLEF topics [9]), and to provide a fuller presentation of the systems, e.g. describing retrieval techniques used.
\par }\pard \qj\fi227\widctlpar\adjustright {One obvious approach would be to join in the existing evaluation contests such as TREC or CLEF}{\cs31\fs12\up6 \chftn {\footnote \pard\plain \s39\qj\fi-170\li170\widctlpar\tx170\adjustright \f4\fs18\cgrid {
\cs31\fs12\up6 \chftn }{ It is worth mentioning that, in the interactive CLIR track of CLEF 2002, there is a paper r}{e}{
porting a Portuguese-English experiment. It used an MT system to translate part of the initial sample into Portuguese, due to the scarce number of parallel English-Portuguese public co}{r}{pora available [10]. }}}{
, thus avoiding the huge organization efforts for the contests. Even though we have not ruled out this possibility, we are under the impression that there is very little room in these contests for expe
rimentation with language-specific problems, let alone devise collections that reflect specific problems. Furthermore, there are a}{l}{
ready efforts for evaluation of the whole field of Portuguese processing (such as the satellite Avalon'2003, the Workshop for
 Evaluation Campaigns for Portuguese). Therefore, an initiative for IR would work synergistically with those for other NLP topics in Portuguese. Another advantage of this approach is to count on a collection of texts and tasks specifically devised for Por
tuguese, and not a mere translation or a}{d}{
aptation of other language users' needs. If successful the evaluation initiatives would place researchers on Portuguese on an equal standing with organizers of international contests. This would allow contests involving Portuguese to be seamlessly incorp}
{o}{rated into international contests, thus warranting both relevance to Portuguese and more professional organization.}{\cf6 
\par }\pard\plain \s22\qj\sb520\sa280\keep\keepn\widctlpar\tx454\hyphpar0\adjustright \b\f4\cgrid {2\tab Question answering evaluation
\par }\pard\plain \qj\fi227\widctlpar\adjustright \f4\fs20\cgrid {Question answering systems aim to retrieve answers rather than docume
nts that may contain the answers. This is a more difficult goal and several constraints have been assumed in the evaluation contest models mentioned in the previous section. For instance, in TREC a special track on Question Answering (QA) has been created
 only in 1999. It has suffered some changes over the years and subdivided into different sub-tasks. As an example, the 2001 track on (QA) had the following configuration:
\par 1. The main task. This task aims to retrieve answers to specific questions, such as, \ldblquote Who discovered Azores?\rdblquote  or \ldblquote When did Vasco da Gama arrive in India?\rdblquote 
. Answers are composed by a short text (<50 chars) and by a document identifier. The document should contain text supporting the answer string. If the collection of documents has no answer
 to the question, systems should answer 'NIL'. Systems were allowed to retrieve a ranked list of up to 5 answer-pairs to each query. In 2002, this task was changed to allow only exact answers, rather than a string containing the answer.
\par 2. The list task. This task aims to obtain a set of instances as the answer for a que}{s}{tion. These instances may need to be retrieved from multiple documents and should have no duplicates. Examples are questions like: \ldblquote Name 9 EU countries
\rdblquote  or \ldblquote Name 5 Benfica football players\rdblquote 
. Answer instances may be duplicated in the collection and it is also guaranteed that the document collection has the answer to the question (it refers the adequate number of answer instances).
\par 3. The context task. In this task the goal was to track discourse objects through s}{e}{
ries of questions. The systems should be able to manage interaction contexts and to track discourse objects, solving linguistic phenomena, such as anaphora and ellipsis. As example might be the following series of questions: 1. Who wa
s the first king of Portugal? 2. When did he take the throne? 3. Where was he born?
\par In order to evaluate the performance of systems for these tasks answer pairs (a}{n}{swer-string, document-id) are required, which were provided by human referees. From experien
ce in TREC conferences, it has been concluded that one referee is sufficient for each answer pair, as relative scores remain stable despite the differences in the refereeing policy. However, in order to have absolute (rather than relative) scores it is im
portant to have more than one referee for each question and to combine them into a single judgment. 
\par In previous QA evaluation contests, the typical approach to deal with this difficult problem has been to classify the questions according to a taxonomy, as follows: a}{n}{swers to questions starting with \ldblquote who\rdblquote 
 should be a person or organization; for que}{s}{tions with \ldblquote when\rdblquote  the answer should be a time entity; and for \ldblquote where\rdblquote  a place should be given. After having classified the question, the systems commonly use info
rmation retrieval techniques and some shallow parsing techniques to obtain adequate entities and to answer the initial question.
\par Even though evaluation of IR systems for Portuguese is likely to have many fe}{a}{tures in common with that for English, different parsing strategies and named entity recognition requirements may turn specific systems into very different ones. In add}{i}{
tion, we are aware of specificities of Portuguese interrogatives that require treatment, such as }{\i O primeiro rei de Portugal tinha que nome? Em que igreja casou?}{. }{\lang2057 In these cases, }{
there are no clue words to specify the kind of information required, as demo}{n}{strated by comparing with }{\i Em que partido votou?}{
\par }\pard\plain \s22\qj\sb520\sa280\keep\keepn\widctlpar\tx454\hyphpar0\adjustright \b\f4\cgrid {3\tab Web search evaluation
\par }\pard\plain \qj\fi227\widctlpar\adjustright \f4\fs20\cgrid {According to Saracevic [11], one can conceive three ways of evalu
ating IR, namely system evaluation; user evaluation; and evaluating the system from a user point of view. We give a short overview of each paradigm in the following subsections, in what concerns Web IR.
\par Web search engines are now well established as one of
 the most fundamental and widely used components of the Internet infrastructure. Search engines use many of techniques developed over the last decades for full-text document retrieval, but are also quite different in many aspects [12]. Users interact with
 
these systems in a very different way: queries tend to be much shorter and only the first or second results pages are examined in most cases. On the other hand, the hyperlinks between pages represent a source of data that can be used to rank results effec
tively [13].
\par Web search engine evaluation has been the subject of substantial research work [14]. TREC has had a Web track in past editions, which will continue this year}{\cs31\fs12\up6 \chftn {\footnote \pard\plain \s37\qj\fi-170\li170\sl-220\slmult0\widctlpar
\tx170\adjustright \f4\fs18\cgrid {\cs31\fs12\up6 \chftn }{ }{\field{\*\fldinst { HYPERLINK "http://www.ted.cmis.csiro.au/TRECWeb/2002/index.html" }{\fs20\revised\revdttm645120730 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000003500000068007400740070003a002f002f007700770077002e007400650064002e0063006d00690073002e0063007300690072006f002e00610075002f0054005200450043005700650062002f0032003000300032002f0069006e006400650078002e006800
74006d006c000000e0c9ea79f9bace118c8200aa004ba90b6a00000068007400740070003a002f002f007700770077002e007400650064002e0063006d00690073002e0063007300690072006f002e00610075002f0054005200450043005700650062002f0032003000300032002f0069006e006400650078002e00680074
006d006c000000000000000000000000}}}{\fldrslt {\cs42\ul\cf2 http://www.ted.cmis.csiro.au/TRECWeb/2002/index.html}}}{ }}}{
. In this track, evaluations are performed against a document set that is a snapshot of the Web (in 2002, this was a collection of web pages published under the .GOV domain). In the last edition, this track focused on
 two problems: topic distillation, which i}{n}{volves finding key resources in a particular topic area, and named page finding, or locating a page which has been named by the user. There are multiple classes of alg}{o}{
rithms and approaches being used or researche
d for computing lists of resources to present in response to web search tasks, ranging from keyword matching algorithms to algorithms for finding related pages [15] (using, for instance, bibliometric techniques [16, 17]) or for clustering matched resource
s by topic [18]. 
\par Among the many possibilities to initiate evaluation of web search engines for Po}{r}{tuguese webpages, we highlight the one that consists in defining a set of tasks, execute them against several search engines and evaluate them against several
 criteria, such as:
\par {\pntext\pard\plain\f3\fs20\cgrid \loch\af3\dbch\af0\hich\f3 \'b7\tab}}\pard \qj\fi-360\li587\widctlpar\jclisttab\tx587{\*\pn \pnlvlblt\ilvl0\ls21\pnrnot0\pnf3\pnstart1\pnindent360\pnhang{\pntxtb \'b7}}\ls21
\ls21\pnrdate-1502368434\pnrstart0\pnrxst1\pnrxst0\pnrxst183\pnrxst240\pnrstop4\pnrstart1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc23\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc1\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr1\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Coverage, the amount of web pages indexed in the Portuguese language. 
\par {\pntext\pard\plain\f3\fs20\cgrid \loch\af3\dbch\af0\hich\f3 \'b7\tab}}\pard \qj\fi-360\li587\widctlpar\jclisttab\tx587{\*\pn \pnlvlblt\ilvl0\ls21\pnrnot0\pnf3\pnstart1\pnindent360\pnhang{\pntxtb \'b7}}\ls21
\ls21\pnrdate-1502368434\pnrstart0\pnrxst1\pnrxst0\pnrxst183\pnrxst240\pnrstop4\pnrstart1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc23\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr2\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Retrieval performance, or the precision of results and response times 
\par }\pard \qj\fi227\widctlpar\adjustright {To conduct this task, hundreds of search engines could be considered. A large number of search engines could be employed, including global search engines such as Google}{\cs31\fs12\up6 \chftn {\footnote \pard\plain 
\s37\qj\fi-170\li170\sl-220\slmult0\widctlpar\tx170\adjustright \f4\fs18\cgrid {\cs31\fs12\up6 \chftn }{ }{\field\flddirty{\*\fldinst { HYPERLINK "http://google.com" }{\fs20\revised\revdttm645120730 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000001200000068007400740070003a002f002f0067006f006f0067006c0065002e0063006f006d000000e0c9ea79f9bace118c8200aa004ba90b2600000068007400740070003a002f002f0067006f006f0067006c0065002e0063006f006d002f00000000000032
0000000000600000006000}}}{\fldrslt {\cs42\ul\cf2 http://google.com}}}}}{ and Alltheweb}{\cs31\fs12\up6 \chftn {\footnote \pard\plain \s37\qj\fi-170\li170\sl-220\slmult0\widctlpar\tx170\adjustright \f4\fs18\cgrid {\cs31\fs12\up6 \chftn }{ }
{\field\flddirty{\*\fldinst { HYPERLINK "http://alltheweb.com" }{\fs20\revised\revdttm645120730 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000001500000068007400740070003a002f002f0061006c006c007400680065007700650062002e0063006f006d000000e0c9ea79f9bace118c8200aa004ba90b2c00000068007400740070003a002f002f0061006c006c007400680065007700650062002e006300
6f006d002f000000000000000000000000000000000000}}}{\fldrslt {\cs42\ul\cf2 http://alltheweb.com}}}}}{, and Portuguese-specific national or cultural search engines, such as tumba!}{\cs31\fs12\up6 \chftn {\footnote \pard\plain 
\s37\qj\fi-170\li170\sl-220\slmult0\widctlpar\tx170\adjustright \f4\fs18\cgrid {\cs31\fs12\up6 \chftn }{ }{\field\flddirty{\*\fldinst { HYPERLINK "http://tumba.pt" }{\fs20\revised\revdttm645120730 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000001000000068007400740070003a002f002f00740075006d00620061002e00700074000000e0c9ea79f9bace118c8200aa004ba90b2200000068007400740070003a002f002f00740075006d00620061002e00700074002f000000000000000060000000000000
007200}}}{\fldrslt {\cs42\ul\cf2 http://tumba.pt}}}}}{ [19] and todobr}{\cs31\fs12\up6 \chftn {\footnote \pard\plain \s37\qj\fi-170\li170\sl-220\slmult0\widctlpar\tx170\adjustright \f4\fs18\cgrid {\cs31\fs12\up6 \chftn }{ }{\field\flddirty{\*\fldinst {
 HYPERLINK "http://www.todobr.com.br" }{\fs20\revised\lang2070\revdttm645120730 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000001900000068007400740070003a002f002f007700770077002e0074006f0064006f00620072002e0063006f006d002e00620072000000e0c9ea79f9bace118c8200aa004ba90b3400000068007400740070003a002f002f007700770077002e0074006f006400
6f00620072002e0063006f006d002e00620072002f000000002e00450043000000130000006000}}}{\fldrslt {\cs42\ul\cf2 http://www.todobr.com.br}}}}}{. 
\par The selection of tasks to benchmark could follow as guidelines the tasks of the TREC Web search track or the Portuguese terms most frequently searched on web search engines (using, for instance, the access logs of one or more search engines as a sample).
 However, an evaluation conducted at this level would only provide results on the performance of these systems as seen by the end users, without taking into account differences among the collections of web pages used by each search engine or add}{i}{
tional database that could be used to improve the quality of ranking algorithms. R}{e}{
searchers on Web IR algorithms demand comparisons on equal terms. To satisfy this requirement, a common, publicly available, collection of web pages extracted from a list of well-know
n sites could be considered. This collection would be annotated for evaluation purposes and given to the participants. A substantially large document collection of Portuguese web pages, such as governmental web pages, could be gat}{h}{
ered from public sources for this purpose and given in advance to participants in a web search evaluation track.
\par No matter the size and indexing capabilities, IR systems may still fall short of basic user needs, and a user-based study typically focuses on questions such as whether
 the particular needs are solved, whether the information retrieved was useful in the co}{n}{
text of use, whether the interface was friendly, what are the typical shortcomings, and so on. In addition to simple precision and recall measures (incidentally, based
 on a ill-defined and often problematic view of relevance), several other measures have been proposed to encode subjective factors: novelty (how many retrieved items were new to the user), coverage (how many were already known to the user), effort (how ma
ny irrelevant items before a relevant one), etc. (See [20]). 
\par The problem with this kind of studies is that they are extremely expensive and time consuming. Furthermore, they require users to comprise a (relatively) homogeneous set, with whom researchers can communicate directly. This is at odds with the situ}{a}{
tion on the Web, where users form a large and mostly heterogeneous group, with whom typically information providers do not have direct contact.
\par While search engine developers may be interested in compar
ing algorithms, as mentioned above, other researchers may try to assess search engine users' satisfaction and some of their problems without requiring large user-based evaluations. We may use simple usability techniques such as cognitive walkthroughs and 
questionnaires, or we can perform large-scale non-intrusive studies by looking at real users in some semi-controlled setups. Web user logs massaging [21] and click-through data [22] are two technologies that we propose to apply for Portuguese.
\par For our language there are some variables with obvious usability importance that can be studied almost independently of the evaluation setup, such as character enco}{d}{
ing, clitics handling, hyphenated words, and a particular kind of spelling mistakes produce
d by missing or wrong accents. Also, even the simple "language" variable has turned out to be defective in major search engines. For example, in December 2002 only one third of the pages in Portuguese indexed by Google have been correctly a}{s}{
signed the "Portuguese language" label!.
\par }\pard\plain \s22\qj\sb520\sa280\keep\keepn\widctlpar\tx454\hyphpar0\adjustright \b\f4\cgrid {4\tab An initial proposal for cooperative evaluation
\par }\pard\plain \qj\fi227\widctlpar\adjustright \f4\fs20\cgrid {What are we going to evaluate? Is it possible to find a subset or intersection of the above goals and ways to evaluate, discussed in the sections above, which allows a common endeav
or? No matter the final answer, one should attempt to devise a setup that could be eventually reused for the different sub-areas. It is in any case necessary to start a common reflection \endash  of which this paper represents the beginning \endash 
 around IR in Portuguese. Basically, one has to agree on what to evaluate, and how to evaluate it. Almost all questions dealing with evaluation have not yet been answered for Port}{u}{guese. Let us illustrate some relevant ones in what follows.
\par Concerning search engines: Which one behaves best for the most frequent questions in Portuguese? What is the size of the index of each engine? What subjects do they cover? What is their overlap? Concerning users: Which are the most frequent que}{s}{
tions? What kind of goals do they have? What kind of mistakes are they most likely to make? What kind of content do they search in Portuguese as opposed e.g. to doing the search in English? Concerning specific QA systems: How are question types categ}{o}{
rized? What is defined as a valid answer? What is th
e proportion of questions that have an answer in the database? Concerning QA: Which kind of questions are users interested in? What is the prior influence (if any) of their exposure to Web search? Concerning information extraction systems, again depending
 on the information to be extracted, one would like to answer a number of questions, and so on.
\par Abiding by the evaluation contest paradigm, we have in any case to define a co}{m}{mon "document collection". Probably we should have several, or at least two: a closed collection of documents (probably also taken from the Web, given its higher availabi}{
l}{ity), and a large Web collection of documents (open collection). In addition, we need to agree on an initial set of queries or topics, together with some measures of rel}{e}{vance or quality.
\par We suggest that these collections should be prepared having in mind from the start the requirement that they should be iteratively refined with judgments and annotation relative to more and more complex tasks. Thus, from basic topic de
tection (classical IR) to QA, (multi-document) summarization, and MUC-like tracks such as ENR, co-reference, etc. as well as more linguistically oriented tasks such as terminology e}{x}{
traction or thesaurus induction, or more application oriented tasks such as geograph}{i}{
cal information mining. In fact, we even propose to maximize the set of common data with other kinds of evaluation contests which are usually kept apart, since evaluation activities for Portuguese are all starting almost simultaneously. One coul
d use a subset of the texts for (machine) translation and (some subsets of) parsing. 
\par Another question to bear in mind is that, though the language is common, Brazilian and Portuguese culture and users may differ considerably.  This also applies to the cov
erage of the indexes of national search engines. Some applications may require one universe only, others may benefit from accessing both Portuguese and Brazilian i}{n}{
formation. Special care must thus be taken in order to find realistically interesting tasks 
that pertain to both variants of Portuguese if we want to cooperatively evaluate our systems. One possibility would be to have separate panels of judges for the two variants, so that they might rank differently relevance according to their national (and/o
r geographical) bias. Examples (for ease of exposition they are framed as que}{s}{tions, but of course the same would apply if they were cast as topics): }{\i Que pol\'edticos foram acusados de corrup\'e7\'e3o ultimamente? }{\i\lang1046 
Quanto ganha um jogador de futebol? Quais os restaurantes de comida japonesa mais perto? }{\i Quem foi Guimar\'e3es Rosa?}{ One would expect different information to be relevant to users from different cou}{n}{tries\'85 
\par Though we acknowledge that many hurdles will have to be overcome, we wish to start with the work to veri
fy or reject our hypotheses, by actually setting up the first collection of queries and documents for Portuguese. This process has begun, and we expect to present its first instantiation at Avalon'2003.
\par }\pard\plain \s22\qj\sb520\sa280\keep\keepn\widctlpar\tx454\hyphpar0\adjustright \b\f4\cgrid {References 
\par {\pntext\pard\plain\s30 \f4\fs18\lang2057\cgrid \hich\af4\dbch\af0\loch\f4 1.\tab}}\pard\plain \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc1\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr1\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright \f4\fs18\cgrid {\lang2057 
Voorhees, E., Harman, D.: Overview of TREC-2001. Proceedings of TREC'2001 (2001). Available at }{\field\flddirty{\*\fldinst {\lang2057  HYPERLINK "http://trec.nist.gov" }{\fs20\revised\lang2057\revdttm645120730 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000001500000068007400740070003a002f002f0074007200650063002e006e006900730074002e0067006f0076000000e0c9ea79f9bace118c8200aa004ba90b2c00000068007400740070003a002f002f0074007200650063002e006e006900730074002e006700
6f0076002f00000000000000000000720000000000000000}}}{\fldrslt {\cs42\ul\cf2\lang2057 http://trec.nist.gov}}}{\lang2057  }{
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 2.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr2\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Aires, R., Santos, D.: Measuring the Web in Portuguese. In: Brian Matthews, Bob Ho}{p}{good & Michael Wilson (eds.): Euroweb 2002 conference (2002) 198-199
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 3.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc3\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr3\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Hirschman, L.: The evolution of Evaluation: Lessons from the Message Understanding Conferences. Computer Speech and Language 12 (4) (1998) 281-305
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 4.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc4\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr4\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {Quaresma, P.,\~
 Rodrigues, I.: PGR: \~Portuguese Attorney General's Office Decisions on the Web. In:\~Bartenstein, Geske, Hannebauer, and Yoshie (eds): Web-Knowledge Manag}{e}{ment and Decision Support. LNCS/LNAI 2543 Springer-Verlag (2003) 51-61
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 5.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc5\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr5\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Grishman, R., Sundheim, B.: Message Understanding Conference - 6: A Brief History.  }{\lang1046 Proceedings of COLING-96 (1996) 466-471
\par {\pntext\pard\plain\s30 \f4\fs18\lang1046\cgrid \hich\af4\dbch\af0\loch\f4 6.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc6\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr6\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {\lang1046 
Santos, D., Rocha, P.: AvalON: uma iniciativa de avalia\'e7\'e3o conjunta para o portugu\'eas. In: Actas do XVIII Encontro da Associa\'e7\'e3o Portuguesa de Lingu\'edstica 2002 (forthcoming). 
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 7.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc7\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr7\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Peters, C.: Results of the CLEF 2002 Cross-Language System Evaluation Campaign. In: Peters, C. (Ed.): Working Notes for the CLEF 2002 (2002). Available at: }{\field\flddirty{\*\fldinst { HYPERLINK "http://clef.iei.pi.cnr.it:2002/2002WN.html" }{
\fs20\revised\revdttm645120730 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000002b00000068007400740070003a002f002f0063006c00650066002e006900650069002e00700069002e0063006e0072002e00690074003a0032003000300032002f00320030003000320057004e002e00680074006d006c000000e0c9ea79f9bace118c8200aa
004ba90b5600000068007400740070003a002f002f0063006c00650066002e006900650069002e00700069002e0063006e0072002e00690074003a0032003000300032002f00320030003000320057004e002e00680074006d006c00000000000800000300}}}{\fldrslt {\cs42\ul\cf2 
http://clef.iei.pi.cnr.it:2002/2002WN.html}}}{ 
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 8.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc8\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr8\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Eguchi, K., Oyama, K., Ishida, E., Kando, N., Kuriyama, K.: \line Overview of the Web Retrieval Task at the Third NTCIR Workshop. \line NII Technical Report, No.NII-2003-002E (2003)
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 9.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc9\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr9\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Mandl, T., Womser-Hacker, C.: Linguistic and Statistical Analysis of the CLEF Topics. In: Peters, C. (Ed.): Working Notes for the CLEF 2002 Workshop (2002)  
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 10.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc10\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr10\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Orengo, V., Huyck, C.: Portuguese-English Experiments using Latent Semantic Indexing. In: Peters, C. (Ed.): Working Notes for the CLEF 2002 Workshop (2002) 
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 11.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc11\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr11\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Saracevic, T.: Evaluation of evaluation in information retrieval. Proceedings of SIGIR 95 (1995) 138-46
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 12.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc12\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr12\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Arasu, A. Cho, J. Molina, H. Paepcke, A., Raghavan, S.: Searching the Web. ACM Tran}{s}{actions on Internet Technology 1(1) (2001) 2-43
\par {\pntext\pard\plain\s30 \cs51\f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 13.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc13\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr13\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {\cs51 
Brin, S., Page,L.: The Anatomy of a Large-Scale Hypertextual (Web) Search Engine. Computer Networks and ISDN Systems, 30(1-7) (1998) 107-117 
\par {\pntext\pard\plain\s30 \cs51\f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 14.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc14\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr14\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {\cs51 
Hawking, D., Craswell, N., Bailey, P., Griffiths,K.: Measuring Search Engine Quality. Information Retrieval l4 (1) (2001) 33-59
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 15.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc15\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr15\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Dean, J. Henzinger, M.: Finding related pages in the World Wide Web. Proceedings of the Eighth International World Wide Web Conference (1999) 389-401
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 16.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc16\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr16\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Garfield, E.: Citation Analysis as a tool in journal evaluation. Science 178 (1972) 471-479
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 17.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc17\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr17\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Small, H.: Co-citation in the scientific literature: A new measure of the relationship b}{e}{tween two documents Journal of American Society for Information Science, 24, 4, (1973) 265-269
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 18.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc18\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr18\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Zamir, O., Etzioni, O.: Web document clustering: A feasibility demonstration. Proceedings of the 21st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (1998)  46-54
\par {\*\bkmkstart evaluation_measures_www8}{\*\bkmkend evaluation_measures_www8}{\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 19.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0
\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc19\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr19\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Silva, M. J.: The Case for a Portuguese Web Search Engine. DI/FCUL Technical Report 03-3 (2003). Available at }{\field\fldedit{\*\fldinst {\cs42\ul\cf2  HYPERLINK "http://www.di.fc.ul.pt/tech-reports/03-3.pdf" }{
\fs20\ul\cf2\revised\revdttm645120730\crauth1\crdate645120730 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b0200000003000000e0c9ea79f9bace118c8200aa004ba90b5a00000068007400740070003a002f002f007700770077002e00640069002e00660063002e0075006c002e00700074002f0074006500630068002d007200650070006f007200740073002f00300033002d0033002e00
7000640066000000000000000000}}}{\fldrslt {\cs42\ul\cf2 http://www.di.fc.ul.pt/tech-reports/03-3.pdf}}}{
\par {\pntext\pard\plain\s30 \f4\fs18\cgrid \hich\af4\dbch\af0\loch\f4 20.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc20\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr20\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {
Gwizdka, J., Chignell M.: Towards Information Retrieval Measures for Evaluation of Web Search Engines. Unpublished manuscript (1999). Available at }{\field\flddirty{\*\fldinst { HYPERLINK "http://www.imedia.mie.utoronto.ca/~jacekg/pubs.html" }{
\fs20\revised\revdttm645120730 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000003400000068007400740070003a002f002f007700770077002e0069006d0065006400690061002e006d00690065002e00750074006f0072006f006e0074006f002e00630061002f007e006a006100630065006b0067002f0070007500620073002e0068007400
6d006c000000e0c9ea79f9bace118c8200aa004ba90b6800000068007400740070003a002f002f007700770077002e0069006d0065006400690061002e006d00690065002e00750074006f0072006f006e0074006f002e00630061002f007e006a006100630065006b0067002f0070007500620073002e00680074006d006c
000000000000000000000000fc36a10000}}}{\fldrslt {\cs42\ul\cf2 http://www.imedia.mie.utoronto.ca/~jacekg/pubs.html}}}{ 
\par {\pntext\pard\plain\s30 \f4\fs18 \hich\af4\dbch\af0\loch\f4 21.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc21\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr21\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {\cgrid0 
Burton, M., Walther, J.: The value of Web log data in use-based design and testing. Jou}{\cgrid0 r}{\cgrid0 nal of Computer-Mediated Communication, 6 (3) (2001). Available at }{\field\flddirty{\*\fldinst {\cgrid0 
 HYPERLINK "http://www.ascusc.org/jcmc/vol6/issue3/burton.html" }{\fs20\revised\revdttm645120730\cgrid0 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000003300000068007400740070003a002f002f007700770077002e006100730063007500730063002e006f00720067002f006a0063006d0063002f0076006f006c0036002f006900730073007500650033002f0062007500720074006f006e002e00680074006d00
6c000000e0c9ea79f9bace118c8200aa004ba90b6600000068007400740070003a002f002f007700770077002e006100730063007500730063002e006f00720067002f006a0063006d0063002f0076006f006c0036002f006900730073007500650033002f0062007500720074006f006e002e00680074006d006c00000000
000000000000}}}{\fldrslt {\cs42\ul\cf2\cgrid0 http://www.ascusc.org/jcmc/vol6/issue3/burton.html}}}{\cgrid0  }{
\par {\pntext\pard\plain\s30 \f4\fs18\lang2057\cgrid \hich\af4\dbch\af0\loch\f4 22.\tab}}\pard \s30\qj\fi-360\li360\widctlpar\jclisttab\tx360{\*\pn \pnlvlbody\ilvl0\ls30\pnrnot0\pndec\pnstart1\pnindent360\pnhang{\pntxta .}}\ls30
\ls30\pnrdate-1502368434\pnrstart0\pnrxst2\pnrxst0\pnrxst0\pnrxst0\pnrxst46\pnrxst0\pnrstop6\pnrstart1\pnrrgb1\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrrgb0\pnrstop9\pnrstart2\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0
\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc22\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrnfc0\pnrstop18\pnrstart3\pnrpnbr22\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0
\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrpnbr0\pnrstop36\adjustright {\lang2057 
Joachims, T.: Evaluating Retrieval Performance Using Clickthrough Data. Proceedings of the SIGIR Workshop on Mathematical/Formal Methods in Information Retrieval  (2002). Available at }{\field\flddirty{\*\fldinst {\cgrid0 
 HYPERLINK http://www.cs.cornell.edu/People/tj/publications/joachims_02b.pdf }{\fs20\revised\revdttm645120730\cgrid0 {\*\datafield 
00d0c9ea79f9bace118c8200aa004ba90b02000000170000004200000068007400740070003a002f002f007700770077002e00630073002e0063006f0072006e0065006c006c002e006500640075002f00500065006f0070006c0065002f0074006a002f007000750062006c00690063006100740069006f006e0073002f00
6a006f0061006300680069006d0073005f003000320062002e007000640066000000e0c9ea79f9bace118c8200aa004ba90b8400000068007400740070003a002f002f007700770077002e00630073002e0063006f0072006e0065006c006c002e006500640075002f00500065006f0070006c0065002f0074006a002f0070
00750062006c00690063006100740069006f006e0073002f006a006f0061006300680069006d0073005f003000320062002e0070006400660000000100000000000000002000000100}}}{\fldrslt {\cs42\ul\cf2 http://www.cs.cornell.edu/People/tj/publications/joachims_02b.pdf}}}{
\par }\pard \s30\qj\widctlpar\adjustright {
\par }}