Embed Size (px)
Citation preview
!"#$$%&'()*+,-&.-*/)01-23&'()*+,-&!45**1675+,-&
Zornitsa Kozareva!USC/ISI!
Marina del Rey, [email protected]!
www.isi.edu/~kozareva!
8)90(50:&;<3&<=;>&
!"#$%&'%(')%&'%*"+,%(')%"#-+%#%.)+/0',1%
2%
Q> How do you get a perfect sun tan?
Question Search Engine
!"#$%&'%(')%&'%*"+,%(')%"#-+%#%.)+/0',1%
3%
Q> How do you get a perfect sun tan?
Question Answer
A> Lie in the sun
Search Engine
• 4'&+5,%/+#56"%+,78,+/%95'-8&+%&'6):+,$%5+$58+-#;%<#68;80+/%
• =>?% @8-+,% #% &'6):+,$% 6';;+60',% #,&% #% ;8/$% '<%A+(%*'5&/B%$"+(%5+$)5,%#%5#,A+&%;8/$%'<%5+;+-#,$%&'6):+,$/%=/,899+$/?%
• =C?%D'*+-+5%%– /+#56"%+,78,+/%&'%,'$%/9+#A%')5%;#,7)#7+%– #%&'6):+,$%8/%,'$%#,%#,/*+5%$'%#%.)+/0',%– 5+;+-#,$%&'6):+,$/%:87"$%,'$%"#-+%$"+%#,/*+5%
– )/+5%/9+,&/%$''%:)6"%0:+%$'%E,&%$"+%#,/*+5%F%
!"+5+%&'%*+%/$#,&%$'&#(%
G)+/0',%H,/*+58,7%=GH?%
• H/A%$"+%6':9)$+5%#%.)+/0',%8,%,#$)5#;%;#,7)#7+%#,&%.)86A;(%5+6+8-+%$"+%+I#6$%#,/*+5%=#,&%/)J68+,$%6',$+I$%$'%-#;8&#$+%$"+%#,/*+5?%
%%%%%KIL%'()*+,-%%!"+5+%*#/%K8,/$+8,%M'5,1%
%% %%%.-*/)0%%%%%N;:%
%%%%% %%%.-*/)0&?,(-@%%O,+%&#(B%<5':%#:',7%"8/%68$(% % % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % % % % % % %-8+*/%'<%N;:B%OP'%6"'/+%#%*#$+5% % %% % % % % % %%%%%%%%%6';'5%$'%/+,&%$'%H;M+5$%K8,/$+8,%#/%#% %% % % % % %%%%%%%%%%%%%%%%%5+:+:M5#,6+%'<%K8,/$+8,%Q/%M85$"9;#6+L%
R%
S%
WILLIAM WILKINSON’S “AN ACCOUNT OF THE PRINCIPALITIES OF WALLACHIA AND MOLDOVIA” INSPIRED THIS AUTHOR’S MOST FAMOUS NOVEL%
IBM’s!Watson !
T+'9#5&(%U"#;;+,7+%*"86"%VW4X/%!#$/',%*',%on Feb 16th, 2011
Y%
Z#5#&87:/%<'5%GH%
• V[CM#/+&%#995'#6"+/%– \[KU]%%VW4%!#$/',]%@''7;+%
• ^,'*;+&7+CM#/+&%#,&%D(M58&%#995'#6"+/%– VW4%!#$/',]%H99;+%_858]%!';<5#:%H;9"#]%\5)+%^,'*;+&7+%K-8%%
`%
V[CM#/+&%a#6$'8&%GH%
• GNK_\VOb%Z[OUK__Vb@%– c+$+6$%.)+/0',%$(9+B%#,/*+5%$(9+B%<'6)/B%5+;#0',/%– a'5:);#$+%.)+58+/%$'%/+,&%$'%#%/+#56"%+,78,+%
• ZH__H@K%[K\[VKdHe%– [+$58+-+%5#,A+&%&'6):+,$/%– W5+#A%8,$'%/)8$#M;+%9#//#7+/%#,&%5+5#,A%
• Hb_!K[%Z[OUK__Vb@%– KI$5#6$%6#,&8&#$+%#,/*+5/%– [#,A%6#,&8&#$+/%%
Question Analyser
Search Engine
Paragraph Sorter
Answer Processor
WEB
Question
Answer
_8:9;8E+&%H56"8$+6$)5+%
Question Analyser Human
G)+/0',%Z5'6+//8,7%=\"8,7/%$'%+I$5#6$%<5':%$"+%.)+/0',?%• H,/*+5%\(9+%c+$+60',%
– &+68&+%$"+%-5A)@&)-+B:&=9+5/',B%9;#6+?%'<%$"+%#,/*+5&
• G)+5(%a'5:);#0',%– U"''/+%C()0:&D):/,0@*&<'5%$"+%V[%/(/$+:%
• G)+/0',%\(9+%6;#//8E6#0',%– V/%$"8/%#%&+E,80',%.)+/0',B%#%:#$"%.)+/0',B%#%;8/$%.)+/0',1%
• a'6)/%c+$+60',%– a8,&%$"+%.)+/0',%*'5&/%$"#$%#5+%5+9;#6+&%M(%$"+%#,/*+5%
• [+;#0',%KI$5#60',%– a8,&%5+;#0',/%M+$*++,%+,00+/%8,%$"+%.)+/0',%
2f%
G)+/0',%Z5'6+//8,7%
• H,/*+5%\(9+g%%N_%/$#$+%• G)+5(g%%$*'%/$#$+/B%M'5&+5B%a;'58&#B%,'5$"%• a'6)/g%$"+%$*'%/$#$+/%• [+;#0',/g%%M'5&+5/=a;'58&#B%1IB%,'5$"?%
22%
\"+(X5+% $"+% $*'% /$#$+/% (')% 6');&%M+% 5++,$+58,7% 8<%(')X5+%65'//8,7%a;'58&#X/%,'5$"+5,%M'5&+5%
Example slide from Chris Manning
4#,(%.)+/0',/%6#,%#;5+#&(%M+%#,/*+5+&%M(%*+M%/+#56"%
23%Example slide from Chris Manning
V[CM#/+&%G)+/0',%H,/*+58,7%
2F%Example slide from Chris Manning
2R%
\[KU%V[CM#/+&%GH%_(/$+:/%
• Where are zebras most likely found? — in the dictionary
• Where do lobsters like to live? — on the table
• What is an invertebrate? — Dukakis
Michael Dukakis is a member of the Democratic Party, I have long suspected that elected officials from the Democratic Party are some previously unclassified form of invertebrate, a totally spineless creature capable of great noise but no real movement or action
• Where do lobsters like to live? — on the table
• Where are zebras most likely found? — in the dictionary
• What is an invertebrate? — Dukakis
Z#5#&87:/%<'5%GH%
• EFG95*)@&5HH0,57I)*&– \[KU]%%VW4%!#$/',]%@''7;+%
• ^,'*;+&7+CM#/+&%#,&%D(M58&%#995'#6"+/%– VW4%!#$/',]%H99;+%_858]%!';<5#:%H;9"#]%\5)+%^,'*;+&7+%K-8%%
2S%Example slide from Chris Manning
^,'*;+&7+CM#/+&%#995'#6"+/%=_858?%
• W)8;&%#%/+:#,06%5+95+/+,$#0',%'<%$"+%.)+5(%– \8:+/B%&#$+/B%;'6#0',/B%+,00+/B%,):+586%.)#,00+/%
• 4#9%<5':%$"8/%/+:#,06/%$'%.)+5(%/$5)6$)5+&%&#$#%%'5%5+/')56+/%– @+'/9#0#;%&#$#M#/+/%– O,$';'78+/%=!8A89+&8#%8,<'M'I+/B%cWZ+&8#B%!'5&b+$B%h#7'?%
– [+/$#)5#,$%5+-8+*%/')56+/%#,&%5+/+5-#0',%/+5-86+/%– _68+,0E6%&#$#M#/+/%
2Y%
D(M58&%#995'#6"+/%=VW4%!#$/',?%
• W)8;&%#%/"#;;'*%/+:#,06%5+95+/+,$#0',%'<%$"+%.)+5(%
• @+,+5#$+%#,/*+5%6#,&8&#$+/%)/8,7%V[%:+$"'&/%– H)7:+,$+&%*8$"%',$';'78+/%#,&%/+:8C/$5)6$)5+&%&#$#%
• _6'5+%+#6"%6#,&8&#$+%)/8,7%586"+5%A,'*;+&7+%/')56+/%– @+'/9#0#;%&#$#M#/+/%– \+:9'5#;%5+#/',8,7%– \#I',':86#;%6;#//8E6#0',%
2`%
U'::+568#;%_(/$+:/%
!"+5+%8/%$"+%e')-5+%4)/+):%;'6#$+&1% V,%Z#58/B%a5#,6+%
!"#$X/%$"+%#MM5+-8#0',%<'5%;8:8$+&%9#5$,+5/"891%
eLZL%
!"#$%#5+%$"+%,#:+/%'<%O&8,X/%5#-+,/1% D)78,,%#,&%4),8,,%
!"#$%6)55+,6(%8/%)/+&%8,%U"8,#1% ()#,%
!"#$%A8,&%'<%,)$/%#5+%)/+&%8,%:#5i89#,1% #;:',&/%
!"#$%8,/$5):+,$%&'+/%4#I%['#6"%9;#(1% &5):/%
!"#$%8/%$"+%$+;+9"',+%,):M+5%<'5%_$#,<'5&%N,8-+5/8$(1%
YSfC`3FC3Fff%
* Handle mainly factoid questions
G)+/0',%U;#//8E6#0',%• \"+% 95'6+//% M(% *"86"% #% /(/$+:% #,#;(i+/% #%.)+/0',%#,&%;#M+;/%$"+%.)+/0',%*8$"%#%78-+,%#,/*+5%$(9+%
&&'%%j!"'%*#/%$"+%E5/$%Z58:+%48,8/$+5%'<%U#,#k%
&.-*/)0&J:H)g%Z+5/',%
• V$%8/%#,%8,$+75#;%9#5$%'<%#,(%GH%/(/$+:%– ;8:8$/%$"+%5#,7+%'<%&#$#l6'59'5#%$'%M+%/+#56"+&%
– :#A+/%$"+%GH%95'M;+:%$5#6$#M;+%
\#/A%c+E,80',%• @8-+,%#% /+$%'<%.)+/0',/B%M)8;&%#,%#)$':#$+&%/(/$+:% $"#$% 6#,% ;+#5,% $"+% :#998,7% M+$*++,%$"+%.)+/0',/%#,&%$"+%#,/*+5%$(9+/%
\#/A%c+E,80',%• @8-+,%#% /+$%'<%.)+/0',/B%M)8;&%#,%#)$':#$+&%/(/$+:% $"#$% 6#,% ;+#5,% $"+% :#998,7% M+$*++,%$"+%.)+/0',/%#,&%$"+%#,/*+5%$(9+/%
• !"#$X/%8,%#%G)+/0',1%
– G)+/0',%\(9+%
– H,/*+5%\(9+%
– G)+/0',%a'6)/%
– G)+/0',%\'986%
G)+/0',%\(9+%• K)?%% H,% 8&8':#06% 6#$+7'58i#0',% '<% .)+/0',/%<'5% $"+% 9)59'/+% '<% &8/0,7)8/"8,7% M+$*++,%&8m+5+,$% 95'6+//8,7% /$5#$+78+/% #,&l'5% #,/*+5%<'5:#$/L%
– aHU\OVcg%%%%%%%%%%%jD'*%<#5%8/%8$%<5':%K#5$"%$'%4#5/1k%
– eV_\g%%%%%%%%%%%%%%%%%%%je8/$%$"+%,#:+/%'<%6"+*8,7%7):/k%
– cKaVbV\VObg%%%%%j!"'%8/%d;#&%$"+%V:9#;+51k%
– hK_CbOg%%%%%%%%%%%%%jV/%_#&&#:%D)//+8,%#;8-+1k%– OZVbVObg%%%%%%%%%%j!"#$%&'%:'/$%H:+586#,/%$"8,A%'<%7),%6',$5';1k%– n%
Trivia-like questions where short, factual answers are expected!
H,/*+5%\(9+%• K)?%% \"+% 6;#//% '<% 'Mo+6$% =$(9+% '<% /+,$+,6+?%/')7"$%M(%$"+%.)+/0',L%
– ZK[_Ob% % % % %=<5':%j!"'%nk?%– ZeHUK% % % % %=<5':%j!"+5+%nk?%– cH\K% % % % %=<5':%j!"+,%nk?%– bN4WK[% % % %=<5':%jD'*%:#,(%nk?%
% % %M)$%#;/'%
– KpZeHbH\VOb% % %=<5':%j!"(%nk?%– 4K\DOc% % % %=<5':%jD'*%nk?%
G)+/0',%a'6)/%• K)?%% \"+% 95'9+5$(% '5% +,0$(% $"#$% 8/% M+8,7%/')7"$%M(%$"+%.)+/0',L%
– jV,%*"#$%/$#$+%8/%$"+%@5#,&%U#,(',1k%– j!"#$%8/%$"+%9'9);#0',%'<%W);7#58#1k%– j!"#$%6';'5%8/%#%9':+75#,#$+1k%
G)+/0',%a'6)/%• K)?%% \"+% 95'9+5$(% '5% +,0$(% $"#$% 8/% M+8,7%/')7"$%M(%$"+%.)+/0',L%
– jV,%*"#$%*B5B)&8/%$"+%@5#,&%U#,(',1k%– j!"#$%8/%$"+%H,H(45+,-&'<%W);7#58#1k%– j!"#$%7,4,0&8/%#%9':+75#,#$+1k%
G)+/0',%\'986%• K)?%% \"+% 'Mo+6$% =9+5/',B% 9;#6+B% n?% % '5% +-+,$%$"#$% $"+% .)+/0',% 8/% #M')$L% b'$+% $"#$% $"+%.)+/0',% :87"$% *+;;% M+% #M')$% #% 95'9+5$(% '<%$"+%$'986B%*"86"%*8;;%M+%$"+%.)+/0',%<'6)/L%
– j!"#$%8/%$"+%"+87"$%'<%4$L%K-+5+/$1k%• I)12IB&8/%$"+%<'6)/%• LBM&NO)0)*B&8/%$"+%$'986%
c#$#%U5+#0',%• !"'%65+#$+/%$"+%H,/*+5%\(9+/1%
• D'*% #,&% *"(% #5+% 6+5$#8,% H,/*+5% \(9+/%/+;+6$+&1%
- % H,/*+5% \(9+/% #5+% )/)#;;(% 0+&% $'% $"+% 6;#//+/%5+6'7,8i+&% M(% ')$9)$% '<% b#:+&% K,0$(% [+6'7,80',%_(/$+:/%- %%H,/*+5%\(9+/%&+9+,&/%',%$"+%E,#;%#99;86#0',%
\[KU%GH%c#$#%_+$/%• K#6"%(+#5%#%,+*%/+$%'<%.)+/0',/%8/%65+#$+&%
• \"+% .)+/0',/% #5+% ,'$% ;#M+;+&% *8$"% +I9+6$+&%#,/*+5%$(9+/%
• \"+%,#:+&%+,0$(%$#7/%#5+%7+,+5#$+&%M(%WWbX/%V&+,0E,&+5%,#:+&%+,0$(%$#77+5%
• \"+%5+9'/8$'5(%6',$#8,/%F2%H,/*+5%\(9+/%
KI#:9;+/%'<%H,/*+5%\(9+/%
c8/$58M)0',%'<%H,/*+5%\(9+/%
date, definition, GPE, person ~57% facility description, time are seen only 1
sparse data
H,/*+5%\(9+%\#I',':(%
• Y%6'#5/+%6;#//+/%– HWWKdVH\VObB%Kb\V\hB%cK_U[VZ\VObB%DN4HbB%eOUH\VObB%bN4K[VU%
• Sf%E,+5%6;#//+/%– eOUH\VObg%68$(B%6'),$5(B%:'),$#8,n%– DN4Hbg%75')9B%8,&8-8&)#;B%0$;+B%&+/65890',%
– Kb\V\hg%#,8:#;B%M'&(B%6';'5B%6)55+,6(n%
F2%
p8,%e8B%c#,%['$"L%3ff3L%e+#5,8,7%G)+/0',%U;#//8E+5/L%UOeVb@qf3%
Example slide from Chris Manning
F3%
e8%r%['$"X/%H,/*+5%\(9+%\#I',':(%
Example slide from Chris Manning
H,/*+5%\(9+/%8,%T+'9#5&(%
• 3Sff%#,/*+5%$(9+/%8,%3fBfff%T+'9#5&(%.)+/0',%/#:9;+%
• \"+%:'/$%<5+.)+,$%3ff%#,/*+5%$(9+/%6'-+5%s%Sft%'<%&#$#%
• \"+%Rf%:'/$%<5+.)+,$%T+'9#5&(%#,/*+5%$(9+/%"+B%6'),$5(B%68$(B%:#,B%E;:B%/$#$+B%/"+B%#)$"'5B%75')9B%"+5+B%6':9#,(B%95+/8&+,$B%6#98$#;B%/$#5B%,'-+;B%6"#5#6$+5B%*':#,B%58-+5B%8/;#,&B%A8,7B%/',7B%9#5$B%/+58+/B%/9'5$B%/8,7+5B%#6$'5B%9;#(B%$+#:B%%/"'*B%#6$5+//B%#,8:#;B%95+/8&+,0#;B%6':9'/+5B%:)/86#;B%,#0',B%M''AB%0$;+B%;+#&+5B%7#:+%
FF%
a+55)668%+$%#;L%3f2fL%W)8;&8,7%!#$/',g%H,%O-+5-8+*%'<%$"+%c++9GH%Z5'o+6$L%HV%4#7#i8,+L%a#;;%3f2fL%SuC`uL%
bC75#:%a+#$)5+/%• bC75#:/%#5+%'5&+5+&%#55#,7+:+,$%'<%!"*'5&/%
– !#$"),875#:%– ,v3%M875#:%
– n%
KIL%jD'*%<#5%8/%8$%<5':%K#5$"%$'%4#5/1k%% % %),875#:/g%w"'*B%<#5B%8/B%8$x%
% % %M875#:/g%w"'*%<#5B%<#5%8/B%8/%8$%nx%
FR%
Is it good to use higher order n-grams? Why? !
G)+/0',%!'5&%a+#$)5+%• N/+%$"+%.)+/0',%*'5&%8$/+;<%#/%#%/+9#5#$+%<+#$)5+%
KIL%jD'*%<#5%8/%8$%<5':%K#5$"%$'%4#5/1k%
% % %),875#:/g%w"'*B%<#5B%8/B%8$x%%%%%%%%%%%%%%%.)+/0',%*'5&g%w"'*x%
% % %%
FS%
KIL%j!"#$%6';'5%&'+/%;8$:)/%9#9+5%$)5,%*"+,%8$%6':+/%8,$'%6',$#6$%*8$"%#%/$5',7%#68&1k%
But isn’t this very repetitive? Would it actually help or make a difference?
avoid misclassifying the question as time
e8:8$#0',/%'<%$"+%c8/6)//+&%a+#$)5+/%• \"+(%*8;;%8,6'55+6$;(%6;#//8<(%#%.)+/0',%$"#$%"#/%:);09;+%$(9+/%
%KIL%!"#$%.)+/0',%
– !"#$%8/%$"+%95'9+5%,#:+%<'5%#%<+:#;+%%&'()*1%%– !"#$%8/%b86"';#/%U#7+X/%+(,-.**/,!1%%
– !"#$%8/%$"+%+,+)'&0,!"'<%_+#P;+1%%– !"#$%1&)*.2"$"+%e(,:')$"%y''&/1%
– !"#$%8/%$"+%3.'.+4,!."!)56.(%<'5%$"+%N,8-+5/8$(%'<%^+,$)6A(1%
– !"#$%8/%#,'$"+5%,#:+%<'5%!.&(*/743.2!.**"1%
FY%
animal
bio
cardinal
cause/effect
contact info
disease
Type
_+:#,06%a+#$)5+/%• e+$X/%8,6'59'5#$+%/':+%/+:#,06/%• !"+5+%&'+/%/+:#,06%8,<'5:#0',%6':+%<5':1%
– !"#$%8/%$"+%95'9+5%,#:+%<'5%#%<+:#;+%%&'()*1%
– !"#$%8/%$"+%+,+)'&0,!"'<%_+#P;+1%
– !"#$%8/%$"+%$#;;+/$%5,)!3&/!1%%
% % %%
F`%
animal is-a
animal
location
cardinal is-a number
is-a location
C%!"#$%8/%$"+%$#;;+/$%6)/'2/!7"8,%T#9#,1%%
is-a structure facility
Type
!'5&b+$%a+#$)5+/%D+#&*'5&%KI$5#60',%Z5'6+&)5+%a'5%+#6"%.)+/0',/%
– 5),%ZO_C$#77+5%– E,&%$"+%E5/$%,'),%9"5#/+%#,&%#//):+%8$%8/%$"+%:#8,%',+%
– +I$5#6$%$"+%587"$:'/$%*'5&%$#77+&%#/%,'),%– )/+% !'5&b+$% $'% &+$+5:8,+% #;;% "(9+5,(:/% '<% $"+%"+#&*'5&%
Fz%
works ~90%
!"#$%8/%$"+%3&''.*3"6)/'2/!7"8,%T#9#,1%%
is-a
…. unit artifact structure
!+MCM#/+&%a+#$)5+/%• a'5%+#6"%.)+/0',%#,&%8$/%"+#&*'5&B%/)M:8$%$'%$"+%!+M%;+I86'C/(,$#606%9#P+5,/%'<%$"+%<'5:g%%
% % % % %j{%/)6"%#/%4.&2%,(2k%
• [+$58+-+% #,&% 5+5#,A% #;;% /+:#,06% 6;#//+/%;+#5,+&%<5':%$"+%!+M%
!"#$%8/%$"+%95'9+5%,#:+%<'5%#%<+:#;+%%&'()*1%
O$"+5%a+#$)5+/%• Z#5$C'<C/9++6"%$#7/%• b'),%9"5#/+/%• G)+/0',%;+,7$"%• b#:+&%K,00+/%="+;9/%7+,+5#;8i+%$"+%.)+/0',?%
% % %%
Rf%
%%%%%%!"'%8/%8)9.":''/!73,!1%%%%%%%%sC|%!"'%8/%s9+5/',},#:+|1%
%!"'%8/%;4&('.*"</!26.(741%%sC|%!"'%8/%s9+5/',},#:+|1%
4#6"8,+%e+#5,8,7%U;#//8E+5%• \"+%:'/$%5'M)/$%9+5<'5:#,6+%8/%#6"8+-+&%*8$"%_)99'5$%d+6$'5%4#6"8,+/%=_d4?%
• \5#8,8,7g%– a'5%+#6"%.)+/0',%*'5&%=*"#$B%"'*B%*"+,B%n?%M)8;&%&8m+5+,$%_d4%6;#//8E+5/%
• \+/0,7g%– a'5%+#6"%),/++,%.)+/0',B%+I$5#6$%$"+%.)+/0',%*'5&%– W)8;&%#%<+#$)5+%-+6$'5%)/8,7%$"+%/#:+%<+#$)5+/%*"86"%*+5+%)/+&%8,%$5#8,8,7%
– N/+%$"+%_d4%6;#//8E+5%6'55+/9',&8,7%$'%$"+%.)+/0',%*'5&%
R2%
)P5AH4)& !5HM& 1-K17Q)0& 1-K17R02& 1-K17S,7& TQ&
2% 2% 2% f% 2%
f% f% f% 2% f%
2% 2% f% f% 2%
2% f% f% 2% 2%
2% f% 2% 2% 2%
f% 2% f% f% f%
e+#5,8,7%H;7'58$":
R3%
%#,/*+5
\5#8,+&%4#6"8,+ \5#8,%_+$%a+#$)5+%@+,+5#0', N,8-+5/8$(BfB2B2B,);;BN,8B2BRFU%%\[HVb%cH\H
\K_\%cH\H
\+/$%_+$%a+#$)5+%@+,+5#0', N_UBfB2B2B2BfB,);;%%
Q0)@17B)@&.-*/)0&
eOUH\VOb%
eOUH\VOb%
ZK[_Ob%
O[@HbV~H\VOb%
O\DK[%
O\DK[%
J0()&.-*/)0&
eOUH\VOb%
O\DK[%
ZK[_Ob%
O[@HbV~H\VOb%
O[@HbV~H\VOb%
O\DK[%
!
Precision =# correct identified NEs
# identified NEs
What is another good measure for the evaluation of question classification? !
K-#;)#0',%4+#/)5+/%• 4+#,% [+6895'6#;% [#,A% =4[[?% 8/% )/+&% $'%+-#;)#$+% #% ;8/$% '<% 9'//8M;+% 5+/9',/+/% $'% #%.)+5(B%'5&+5+&%M(%95'M#M8;8$(%'<%6'55+6$,+//L%
!
MRR =1Q
1RATi=1
Q
" Q – total number of questions to evaluate RAT – rank at which answer type is found
'()*+,-& F)*(4B&,?&Q0)@17B)@&&.-*/)0&J:H)&
!,00)7B&.-*/)0&J:H)&
F5-D& F)71H0,754&F5-D&
!"#$% 8/% b86"';#/%U#7+X/%+(,-.**/,!1%% 9+5/',B%M8'B%6',$#6$%8,<'B%&+E,80',% M8'%
!"#$%8/%$"+%$#;;+/$%6)/'2/!7"8,%T#9#,1%% +,0$(B%;'6#0',B%79+B%<#68;8$(% <#68;8$(%
K-#;)#0',%4+#/)5+/%• 4+#,% [+6895'6#;% [#,A% =4[[?% 8/% )/+&% $'%+-#;)#$+% #% ;8/$% '<% 9'//8M;+% 5+/9',/+/% $'% #%.)+5(B%'5&+5+&%M(%95'M#M8;8$(%'<%6'55+6$,+//L%
!
MRR =1Q
1RATi=1
Q
" Q – total number of questions to evaluate RAT – rank at which answer type is found
'()*+,-& F)*(4B&,?&Q0)@17B)@&&.-*/)0&J:H)&
!,00)7B&.-*/)0&J:H)&
F5-D& F)71H0,754&F5-D&
!"#$% 8/% b86"';#/%U#7+X/%+(,-.**/,!1%% 9+5/',B%91,B%6',$#6$%8,<'B%&+E,80',% M8'% 3% 2l3%
!"#$%8/%$"+%$#;;+/$%6)/'2/!7"8,%T#9#,1%% +,0$(B%;'6#0',B%79+B%?57141B:& <#68;8$(% R% 2lR%
MRR=0.375
G)+/0',%U;#//8E6#0',%Z+5<'5:#,6+%• W#/+;8,+%:'&+;%
– M#7C'<C*'5&% <+#$)5+/% =8L+L% $"+% 8,&8-8&)#;% *'5&/%$"#$%:#A+%)9%$"+%.)+/0',?%
• OM$#8,+&%5+/);$/%*8$"%&+/658M+&%<+#$)5+/%F5-D&F& V5*)41-)& ":-B5P
W91205AX&")A5-+7& ":-B5PY")A5-+7&
2% ``LSu% `zLz2% zfLFu% z2L3S%%
3% zYLR3% zYLzS% ufLRS% zzLSz%
F% zuLS2% ufLFf% uFL2f% uFLF3%
R% u2L2Y% u2L`R% uSLSS% uRLRf%
S% u3LYf% u3Lz3% u`LFR% uSLFF%
2f% uSLFF% uSLRf% uSLRf% u`L3f%
4[[% LZ$>[& LzS2`% Lz`3`% LZ[>[&WY>X&
K55'5%H,#;(/8/%• V,6',/8/$+,6(%8,%$"+%;#M+;+&%&#$#%
%!"'%8/%8)9.":''/!73,!1%
%!"'%8/%;4&('.*"</!26.(741%
% %!"#$%&'+/%;=="/$#,&%<'51%
%!"#$%&'+/%>?@?"/$#,&%<'51%
person
Type
biography
organization
definition
K55'5%H,#;(/8/%• H:M87)')/%#,/*+5%$(9+/%
%!"+5+%*#/%e8,6';,%#//#//8,#$+&1%
% %!"+5+%*#/%$"+%#6$5+//B%4#58',%c#-8+/B%M)58+&1%
facility
Type
location
location
facility
Type
K55'5%H,#;(/8/%• V,"+5+,$;(%&8J6);$%.)+/0',/%
%!"#$%8/%$"+%,#:+%'<%$"+%e8',%^8,7X/%/',%8,%$"+%%
%%%:'-8+%j\"+%e8',%^8,7k1%
%%%%!"'%&+-+;'9+&%9'$;#$6"1%
animal
Type
nationality
K55'5%H,#;(/8/%• ZO_C$#778,7%:8/6;#//8E6#0',/%%
!"#$%N_%@'-+5,:+,$%#7+,6(%5+78/$+5/%
$5#&+:#5A/1%
!"#$l!Z%N_lbbZ%@'-+5,:+,$lbb%&7.!1Albb%%
5+78/$+5/lbb_%$5#&+:#5A/lbb_%1lL%
organization
Type
Possible solution do a pull parse tree of the question!
K55'5%H,#;(/8/%• !'5&b+$%6'-+5#7+%#,&%#:M87)8$(%
!"#$%1.(.&'"7'+/%j/,#9B%65#6A;+B%9'9k1%%%
!'5&b+$%_+,/+%2%
6+5+#;B%6+5+#;%75#//%
%%%%%%%v|%75#//%
%%%%%%%%%%%v|%75#:8,+')/%9;#,$B%75#:8,#6+')/%9;#,$%
%%%%%%%%%%%%%%%v|%"+5MB%"+5M#6+')/%9;#,$%
%%%%%%%%%%%%%%%%%%%v|%-#/6);#5%9;#,$%
product
Type
U#,%(')%M)8;&%(')5%'*,%GU%/(/$+:1%
V,%*"#$%68$(%*#/%Z+$+5%M'5,1%%
!"#$%8/%Z+$+5X/%M85$"9;#6+1%%!"#$%8/%$"+%M85$"9;#6+%'<%Z+$+51%b#:+%Z+$+5X/%M85$"9;#6+L%
!"#$%8/%$"+%$#;;+/$%:'),$#8,1%!"#$%8/%$"+%"87"+/$%:'),$#8,%8,%$"+%*'5;&1%
b#:+%$"+%"87"+/$%:'),$#8,L%!"#$%8/%$"+%,#:+%'<%$"+%$#;;+/$%:'),$#8,1%
S2%
