Algorithms for Inverse Reinforcement Learning

 !	"#$%&!')(*!+,
	-#.0/21235476)8:9;< =?>?@A'BDCFEHGJILKNMILO7IQPREHILS?TUV"WXY3ZV\[]W^_^"4Q`a` KJT?C_C"ILObOLA'BDCFEHGJIcKNM2ILO7IQPREHILS?TdRebfhgLiYjlk"m!nYopqk-rJokts&pHuQpwvxpqeFryLz{|d&{J}Rk"ml~bk-qk-byL}Rk"ml~bk-qk-byLdFQFznY&bYL
pqv&gcFgJk"m&FcYmlk_vlvlk"v%j
kgLmlebLqk-fe%c5- ¡ l_¢£5 l¤-¥h-c¦¨§H© ªc7«¬¯®±°!²2³,pqrµ´¶Fml~be5u:Yk-·opwvxpqeFr¸gLm¹eYok_vlvlk"v"yj
5jpwv"y2j
k]gcmlebLHk"fºeFk»Q·jlmFoªj¹pHrc¼½mlk"¾Fm¹¿¯iLrcojlpqeFrÀ¼bpHubk-rÁeFcvlk-m¹uFk_NyeFgYj¹pHf'k
ZuQpqeFm_{ ®±°²ÂfhZÃkÄicvlk¯ic¯ebmgLgLm¹k-r7j¹pqo-k"v
pHg]Hk_m¹rLpHrc¼,jle'Fo-Å7iLpqm¹k!vx~QpqHqk"\Jk-·
ZuQpHebm"yQrc¯eFm%Fv¹ok"mxjpqrLpHrc¼,j
k¨m¹k-¾m'¯iLrco·jlpqeFr,k-pqrL¼%eFgYj¹pHfhpqÆ-k_¨7rc5j¹iLmFvlYv±j¹k-f#{?ÇÈkÉcm¹vxjo
Fm¹boªj¹k-m¹pHÆ"kj
k0vlkj'eqÊm¹k-¾m¯iLrco·jlpqeFrcvÊ¯eFm¾
pwo
t¼bpHubk-rgeFqpqo-hpqveFgLjlpqfhFË{ÌÇÈkj
k"rÀYk-m¹pHubkÍj
m¹k-kÄq¼FeFm¹pHj
fvÎ¯eFm:®±°²
{
kÉcm¹vxj!j±¾e]Yk_2¾%pÏj
j
ko-bvxk¾
k-m¹k,j
kk-r7j¹pHm¹kgJebHpwopwv~QrLe5¾%rÐQ¾Rk
FrcYqk!jLiLw5j¹k"0mlk"¾Fm¹¯iLrcoªj¹pHebrcvebrÈÉJrLpÏj¹k:vxj¹5j¹k¶vxgJFokFrcÑHpqrLk"Fm¯iLrcoªj¹pHebrcgLgcmleZ»YpqfhjlpqeFreDj
k!mlk"¾Fm¹¯iLrco·jlpqeFr¿e5ubk-mÎµgej¹k-r7jlpwqHÒpqrYÉcrcpÏj¹kÍvxj¹jlkÍvxgJFokb{
ktj
pHm¶FH¼beFm¹pÏj
fLk"wv¾%pHj
j
kfheFm¹ktmlk-·qpqvxjlpwo!o"FvlkpHr]¾
pqo
j
k&gJebHpwohpwvÊ~QrLe5¾%rebrLHj
m¹eFiL¼
]ÉcrLpHjlkvlkj,eeFcvlk-m¹uFk_#jlm5Ó±k_oªjlebmlpqk"v"{®Ôr½q!o-bvxk_v-yÈ~Fk-Äpqv¹vliLkpqvÕFÖ«b-- l©b¤×¹Øj
kk»Ypqvxjlk"rcok#eFÈqFml¼bkÎvlkj\eF&m¹k-¾m¸¯iLrJoªjlpqeFrJv¯eFm¾
pwo
j
kteFcvlk-m¹uFk_]geFqpwo]pwvebgYjlpqfÙ{
emlk"f'e5ubkYk"¼Fk"rLk-mFo-Fy!¾k¸vxic¼F¼Fk_v±jvleFfhkrJ5jliL·m¹F
k-iLm¹pqvxjlpwo-vj
5jtjxjlk"fhgYj,jle#gLpwo~:Îmlk"¾Fm¹¯iLrcoªj¹pHebr¨j
j?f5»YpqfhFHq&LpÏÚDk-m¹k-r7jlpw5j¹k"vDj
k
ebY·vxk"mlubk"ÑgJebHpwoÄ¯m¹eFfÛej
k-m_y%vxiLL·ÙebgYjlpqf!geFqpH·opqk"v"{
pqvÎm¹k"vliLÏjv]pqrÜFr½kÝ\o-pHk"rbj¹H½vleFqu5LqkHpqrLk"FmhgLmleb¼FmfhfhpHrc¼¶¯eFm¹ficqjlpqeFrµeF!j
k#®±°²gLmlebLqk-f#{ÌÇktYk"fheFrcvxjlm5j¹k&ebiLm%q¼FeFm¹pHj
fvReFrvxpqfhgLHk%Ypwv¹om¹kjlk5Þ_ÉcrcpÏj¹kFrc'oeFr7j¹pHrQiLebicvÞ5pHrLÉcrLpHjlkv±j5jlk¨gLm¹eFcHk"fhv"{ß
àá7â
Qcãä&å!Læ±ãâ
k!pqr7ubk-mvxk%m¹k-pqrY¯ebm¹o-k-fhk-r7j
qk"m¹rLpqrL¼\¬¯®±°!²2³ÊgLmlebLqk-fço"rkto
Fm¹boªjlk"mlpqÆ-k_pqrY¯eFm¹fqH]bv¯eFqHe5¾v,¬Ë°%icv¹vxk"HÙyè_FFé7³ªêëÎìaíJ4Q/
è_³,f'k_FvliLm¹k-fhk-r7j¹ve!Frµ¼bk-r7j"î v,k
ZuQpqeFme5ubk-mjlpqf'kbypqrÈ0u5m¹pHk-j±ÎeFRopqmoiLfvxj¹rJok"v"y?F³!pHRrLk"k"Yk_Nyf'k_FvliLm¹k-fhk-r7j¹veFLj
kvlk-rJvxebml¨pqrLgLiYjvjlej
j
¼Fk"r7j"Ðï
³pHÌZu5pqwLqkFyY'fheQLk-eF?j
k,k-rQuQpHm¹eFrLfhk"rbj_{ð
47V"4735ñ*ìa/4
j
k,m¹k-¾mh¯iLrJoªjlpqeFrk-pqrL¼heFgLjlpqf'pqÆ-k_N{
ÇÈkÄo"rpwYk-r7j¹pÏ¯Ãj±¾e¿vxebiLmok"v¶efhejlpqu55j¹pHebr¯eFm:j
pwvgcmlebLHk"f#{
kÃÉcmvxjÒFmlpwvlk"vµ¯mlebfòj
kÁgej¹k-r7jlpwicvlkeFµm¹k-pqrY¯eFmok"f'k"r7j¿Hk_m¹rLpqrL¼óFrcm¹k-w5jlk_fhkj
eYLvÁFvo-eFfhgLiYj5j¹pHebrcfheQLk-wv]¯ebmÎFrLpqfhFFrc
iLfFr½Hk_m¹rY·pqrL¼ô¬¯ÇÍ5j¹~QpHrcv"yÃè"FébLÐÀnYo
fÓ±iL~ºõ÷öNrQiYjxj¹ecyÜè"FQQÐ
eFicmlk-jlÆ-~QÑõønL~YvxpwLcyè_FQ³ª{ÀnQiJo
f'eYYk"qv0m¹k¶vxicgY·geFmljlk_ej
Q0k
ZuQpqeFmv±j¹icYpqk"vrcQ#rLk"iLmlebg
Yv±·pqeFqeF¼bpqo"k-uQpwYk-rco-k]j
5jhm¹k-pqrY¯eFmok"f'k"r7j'qk"FmlrLpqrL¼eQo"oiLmvpqr¸k-kh¯eFm¼bpHrc¼Í¬Ë´eFr7j¼Fick'k-jÙ{qyRè_Fbùb³rcÈpqrÍvleFrc¼·cpHmuFeYo"qpHÆ_5jlpqeFr¸¬Ës!e57õúnQk±Ó±rLe5¾vl~QpËyè"bbùb³ª{
pwv!qpÏjl·k"m¹jliLm¹k,Fv¹vxicf'k_v-y
e5¾k-ubk-m_y7j
j%j
k,m¹k-¾m¯iLrJoªjlpqeFr#pwvÉc»Qk_µFrc*~QrLe5¾%rQØ¯eFmk»LfhgLqkFyÊf'eYYk"qv'eF!Jk"k]¯ebm¹F¼·pqrL¼]bvlvliLfhk¨j
5j!j
ktm¹k-¾m0j!k"bo#û
e5¾k-m%pwv!\vlpHfhgLqkv¹5j¹iLm5jlpqrL¼Í¯iLrJoªjlpqeFr¿etrLk_oªj¹Fm0o-eFr7jlk"rbj_{Áü
kj#pÏj#vxk"k-fvo-Hk_m'j
5jÜ¹ý7©¥J7«*©c¥©5§&©Õ:þYÿY¥h©¹þc©5_a£5
'¥'ÿQ¡ª¦¤¹£L¡ªaÕF- '¦þc ¹
© lÕ¨¢ÿY¤-¦Ëa£*©Z¡'©ÈÿY5D£
¦Ô£¨©Z¡-¤- ª¦Ö©Õt¦¯þQ ¹£5ÿF«Zþ]-¥D ªa¤©5§c5ª¡ª¦ÙÏ«b©¦Ëa£c{
pwvpwvgcFmxj¹pqo-iLqFmlqhjlm¹iLk¨e¥ÿY§Ï¦Ëa©¦Ë¦Ù ªÿY¦ÔRmlk"¾Fm¹\¯iLrcojlpqeFrcv"{dRebrcvlpqYk"m"y"¯eFm?k»LfhgLqkFy_j
5jj
kÊk-k
fhpq¼
j?¾k-pq¼
rLk_oªjmpqrL¼bk"vxjlpqeFr*¼7pqrcvxjû
pH¼
jYpwv±jrco-kFy2jlpqf'kby
rcÍm¹pwvx~:¯mlebf¾%pqrcÍrJgcmlk_L5j¹eFmv-{'®Öjtpqv
m¶j¹e¶vxk"k
e5¾ ebrLkoeFicqLkjlk"mlfhpqrLk#j
k¶mlk"qjlpquFk#¾k-pq¼
j¹veF&j
k"vlk#jlk"mlfv:©	 ª
£ ª{nQpqf'pqwm¨oeFrJvxpwYk-m5j¹pHebrcv!FgLgLqÎjle
iLfFr:k"o-eFrLebfhpqok
Zu7pqeFm_yÌ¯eFmhk-»YFfhgLHkb{
k"rcokby
pHrQubk-mvxk0mlk"pHrY¯ebm¹o-k-fhk-r7jqk"FmlrcpHrL¼pwv!'¯icrcLfhk"rbj2gLmlebLqk-feÌj
k"eFm¹kj¹pqo"LpqeFH·eb¼FbyYk"oebrLeFfhk-jlm¹pqo"v-yYrJ]eFj
k"mÉck-wLv"{¿vxk_oeFrJhf'eFjlpqu55jlpqeFrFmlpwvxk_vÌ¯m¹eFfÀj
kjFvl~eFNo-eFrcvxjlm¹icojx·pqrL¼ÍrÄpHr7jlk"Hqpq¼Fk-r7j¼bk-r7jj
5jo-Frµk
ZubkÎvlico-o-k"v¹v±¯icHqpqr:gJmljlpwoiLwmYeFfpqr{!r:¼bk-r7j&Yk_vxpq¼FrLk"m¬aeFm!pqrcYk"k"j
k]F¼Fk-r7jtpHj¹vlk-H¹³,fZ
ZuFkheFrcHÈuFk"ml¶m¹eFic¼
pqLk"#ej
km¹k-¾mÎ¯iLrcojlpqeFr¶¾
ebvlkteFgYj¹pHfhpqÆ"jlpqeFr¶¾eFiLw#¼Fk"rLk-ml·jlklYk_vxpqmLqk,k
Zu7pqeFm_yYvxe'v±j¹m¹FpH¼
jx¯eFm¹¾mhmlk"pHrY¯ebm¹o-k·fhk"rbjtqk"FmlrcpHrL¼fZ¶rcejtk\icv¹LqkF{:¬ÖdReFrcvlpwYk-m_y¯ebmtk»Q·FfhgLHkbyZj
kjFvl~eFlLmlpquQpHrL¼&¾Rk"HÙ{ b³&rLkvleFiLmokRecpqrY¯eFml·fjlpqeFr¶¯ebm¨Hk_m¹rLpHrc¼0pwv&j
khk
ZuQpqeFm¨eÊeFj
k"m	xk»Ygk-mljF¼Fk"rbjv-yFvicvxk_ÈpHr½¥'¦Ô©5¦Ùa£5Ñ§H©5 ªcQ«\Frc½© l-c¦Ùa¤
¡lþY È§H©5 ªJ7«F{
®Ôr0j
pwv%vxk-jxj¹pHrL¼JyYpÏj%pwv%oebf'fhebrLH\bvlvliLfhk"j
5jj
kgLiLm¹gebvlk0eFebcvxk"mlu5jlpqeFrµpwv'jleÍqk"FmlrL£§ a¤×5ypÙ{ kF{qy?#YpHm¹k"oj,mlk"gLmlk_vxk"r7j¹5j¹pHebr¶e]fFgLgLpqrL¼Î¯mlebfºv±j5j¹k"vj¹eboªjlpqeFrJv-{ÇkgLm¹eFgebvlkpqrcv±j¹k"bhjlemlk_oe5uFk"mj
k!k»Ygk-mlj"î vm¹k-¾m#¯iLrJoªjlpqeFrÈrc¶jleÎicvlkj
pwv!j¹e0¼bk-rLk"m¹jlk'Yk"vlpHmcHkk
Zu7pqeFm_{úÇÈk¸vxiL¼b¼Fk_v±j0j
jÎj
kÍmlk"¾Fm¹Ò¯iLrcoªj¹pHebrÜeF·j¹k-rÜgLmle5uQpwYk"vÎ*fico
fhebmlkgcmvxpqfheFrLpqeFiJv]Yk_vlo-mlpqgYjlpqeFr
eFJk
ZuQpHebm"{Ì%j¹k-m%qÙyFj
k&k"r7jlpqmlk!Éck-w\eFmlk"pHrY¯ebm¹o-k-fhk-r7jqk"FmlrcpHrL¼'pwv¯eFiLrcLk"ÎeFr0j
k¨gLm¹k"vliLgLgebvlpHjlpqeFr0j
jj
k,m¹k·¾m'¯iLrcojlpqeFryQm5j
k-mj
r\j
kgJebHpwobybpwvÊj
k&f'e7v±jvxiJoª·o-pHrJoªj"yÌm¹eFLiJv±j_y2rJÈjlmrcvx¯k-mLqkYkÉJrLpÏj¹pHebr¸ej
kj¹bvx~D{!k-rco-kFypHj
vxk"k-fv2Hpq~Fk"H¨j
5jpqr7ubk-mvxkRmlk"pHrL¯eFmok-fhk"rbjHk_m¹rY·pqrL¼,fZFybpHr\vleFfhkYebfpqrcv-ybgLm¹e5u7pwYk%rhk-Úk_oªjlpquFk¯eFm¹f eFgLgLm¹k-r7jlpwok_v
pHgÎHk_m¹rLpqrL¼c{
eÑeFiLmÎ~QrLe5¾%Hk_Y¼Fkbyj
pqv#o-eFfhgLiYj5jlpqeFrJj¹bvx~
bv0rcejk-k"r#¾Rk"HH·ÖvxjliJYpHk_]pqroebf'gciYjlk"mvlo-pHk"rcokbyLoeFr7j¹mlebDj
k"eFm¹FygJvxYo
ebHeb¼FFy%ebm0cpHebHeb¼FF{
kÍoqebvlk"vxj0¾eFm¹~ÄpwvÎpHrÜk"o-e·rceFfhpqo"v-yF¾
k-m¹kj
kjFvl~teDfiLHjlpw5jljlm¹pHciYjlk%iYj¹pHqpHj±Fv¹vlk"v¹v±·fhk"rbj
bv!k-k"rvxjlicYpqk"pqrYk"gYj
Øj
5j¨pqv"y
e5¾ÀYeQk"vgk-mvleFr\FojlicFHqhoeFfLpqrLk%j
k!u5m¹pHebicv
jxjlm¹pqLiYjlk_v
eFk"FoZu5FpHwLqko
eFpwok¾
k-rÈfhF~QpHrL¼Î0Lk"opwvlpHebr{
kj
k"eFm¹pwvÌ¾k-qÏ·ÔYk-ubk-qeFgk"Î¬¨k"k-rLk"õ¿°FpÏÚNLycè"7b³yrcj
kFgY·gcHpwo-jlpqeFrcvrQiLfhk-m¹eFicv"{!e5¾Rk"uFk"m"yj
pqvÉck"qÍvxjlicLpHk_v&ebrLq£
Ö¡lþL£5¦RYk"o-pqvlpHebrcv¨¾
k"mlk#vxpqrL¼bHkFojlpqeFrÈpqvj~bk-rÍrcj
keFiLj¹oebfhk0pwvpqf'fhk_Ypqjlkb{
k#vlk"Å7iLk"r7jlpwo-bvxkÎ¾FvÉJm¹vxjo-eFrcvlpwYk-m¹k"Q]nYFml¼bk-r7j¬xè"Q5éb³yb¾
ej¹mlpqk"]jlehFv¹ok"mx·jpqrµj
kkÚDk"ojlpquFk
pqmlpqrL¼Íoe7v±jh¯eFmweFmhQ*k-»YFfhpHrLpqrL¼ÎÉJmlf#î v
pHm¹pqrL¼Jk
ZuQpHebm¨e5uFk-mj¹pHfhkFyÌFv¹vliLfhpHrL¼#pHjtjlekm5j¹pHebrcÙ{
®Ôr#j
ktwFvxj!Yk"o"FYkbyLj
ktm¹k"hevxjlm¹icoªj¹iLm¹Fk"vx·j¹pHfjlpqeFre?´:m¹~Fe5uYk_opwvxpqeFrhgLm¹eYok_vlvlk"vpqrk"o-eFrLebfhkjlm¹pwo-v
Fv\¼bmle5¾%rÑmgLpwYqÃ¬Ë°%icvxj"y,è"bQ³ª{nQeFfhk¶eFj
k:JFvlpqopwYk_FvÎo-m¹m¹Äe5ubk-mj¹eµeFicmÎvlkjljlpqrL¼c{ ®±°² wvxeµFgLgk"m¹k"cmlpqkû
:pHr¸oebr7jlm¹eFj
k"eFm¹Dê&pqrÈj
khk"Fmlq*è_Fbv-y,FHfrgebvlk"j
k%gLm¹eFLqk-fçeDmlk_oe5uFk"mlpqrL¼!j
k%ebYÓ±k"ojlpquFk¯iLrcojlpqeFr¯ebm'Yk-jlk-m¹fhpHrcpqvxjlpwo!qpqrLk"FmvlYv±j¹k-f ¾%pHj
ÅbiJFYm5j¹pqo&oebvxj¹v"{®Öj¾FvÌm¹k"o-k-r7jlqtvxebHubk"FvÌ¨vxk"f'pwYk-ÉcrLpHjlk%gLm¹eF¼bm¹Ff ¬Ë}e5Qk-j!Ù{HyNè_FF7³ª{®ÔrÎj
pqvgJgk-m_yY¾Rk,FcYmlk_vlvj
k¨®±°²ÍgLm¹eFLqk-fÂpHr#vxk-jxj¹pHrL¼7vfhebmlk!afhpHqpwmRjletj
kfhbo
pqrLk&qk"FmlrcpHrL¼oeFfhfiLrLpHj±FyQJk-·¼bpHrcrLpHrc¼&¾%pHj
ÉcrLpHjlk´¶m¹~Fe5u¨Yk_opwvxpqeFrgLmleYo-k"v¹vxk_vR¬Ë´¶s Rv¹³{nYk"oªj¹pHebr¼FpquFk_vD¯eFm¹f7YkÉcrcpÏj¹pHebrcv?eFL´¶s Rv?Frcj
kR®±°²gcmlebLHk"f#Ð7¾k¯eYo-icvÊpqrLpHjlpwqHebrj
kvlkjljlpqrL¼pqr\¾
pwo
j
kfheYYk"JpwvÊ~QrLe5¾%r\Frchj
k&o-eFfhgLqkjlk&gJebHpwo'pwvÊ¼bpHubk-r{nQk"o·j¹pHebrï
o
mFojlk-m¹pqÆ-k"vÊj
ktvxk-jeqmlk"¾Fm¹\¯iLrcojlpqeFrcv¯eFm¾
pqo
¼bpHubk-r]geFqpwopwvRebgYjlpqfÙ{Çk¨Yk"f'ebrcvxjlm5jlk&j
5jj
k&vlkj%oebr7j¹pqrcvÊfhFrQ'Yk"¼Fk"rLk-m5j¹k!vxebHiLjlpqeFrcv"ybpqrco-HicLpHrL¼Jy¯ebm0k-»LfhgLqkFyj
kÈmlk"¾Fm¹Ñ¯iLrJoªjlpqeFr¿j
jÎpwv0pqYk"r7jlpwo-qqÆ"k-m¹ehk-uFk"mlQ¾
k"mlkb{Çkmlk_vxebHubk!j
pqv&YpHÝo-iLHj±]uQpq
k-iLm¹pqvx·j¹pqo"v'j
j\5jxj¹k-fhgYjj¹epwYk-r7j¹pÏ¯Ñ:m¹k-¾mÍ¯iLrcojlpqeFrÑj
5jf»QpqfqHYpÏÚDk-m¹k-r7j¹pqjlk"vtJk-j±¾Rk"k-rÈj
k]eFcvlk-m¹uFk_¶geFqpqo-Frchej
k-m_yQvxiLL·ÙebgYjlpqfcgeFqpwopqk"v"{
pqvo-FrJkYeFrLk!kÝh·o-pHk"r7jlq]pqr]j
ktYpwvlo-mlk-jlk,o-bvxk¨icvlpHrc¼'qpqrLk"Fm%gLmleb¼FmfhfhpHrc¼c{nYk"oªj¹pHebr]Yk_wv¾%pHj
j
ko"Fvlk,e
wm¹¼Fk,eFm!pHrYÉJrLpÏj¹kvxj¹jlkvlgcbok"v"y¯ebm
¾
pwo
rk-»QgcHpwopHj"ybj¹FLiLwm
m¹k-gLm¹k"vlk-r7j5jlpqeFr'ej
kmlk"¾Fm¹,¯iLrcoªj¹pHebr¾eFicqJkpqrY¯k"bvxpqLqkF{Çk%v
e5¾¸j
5jpHÌj
ktÉLjxj¹k"m¹k-¾m\¯icrcoªj¹pHebrpqv!mlk"gLmlk_vxk"r7jlk"ÎFv&hHpqrLk_mo-eFfLpHrJ5jlpqeFrÑe,m¹LpHjlmm¹FyÉc»Qk_ÑcFvlpqv'¯icrcoªj¹pHebrcv-yj
k-rj
k¶®±°²ÁgcmlebLHk"f m¹k-fpqrcvhpqrj
k¶o-qbvlvheF,HpqrLk"FmgLm¹e·¼bm¹FfvrJ*o-Frµ¼7pqr¸JkÎvxebHubk"ÍkÝ\o-pHk"rbj¹Hb{ÍnYk"oªj¹pHebrÄùLk"wv&¾%pÏj
j
k'fheFm¹ktm¹k"qpwv±j¹pqoto"Fvlktpqr:¾
pwo
j
kgeFqpqo-pwvÌ~QrLe5¾%rebrLq¨j
m¹eFiL¼
!ÉcrLpHjlk%vlkjeeFcvlk-m¹uFk_¨jlm5Ó±k_oªj¹e·m¹pqk"v"ÐJ¯ebmj
pqv"yD¾RkgLm¹k"vlk-r7j&]vlpqf'gcHkpÏj¹k-m5jlpquFkq¼FeFm¹pHj
f#{

ktj
mlk"k'FH¼beFm¹pÏj
fv¾kYk"uFk-qeFg¶Fmlkj
k"rFgLgLqpHk_NyDpHrnYk"oªj¹pHebr!cy5jle¨vleFfhk%vxpqfhgLHkk-»YFfhgLHk_vpHrco-HiJYpHrc¼ej
Lpqvx·o-mlk-jlk\FrcÍoeFr7j¹pHrQiLebicvtvxjleYo
Fvxjlpwo'rcZuQpH¼75j¹pHebrgLm¹eFcHk"fhv"yFrc]j
k"lf'ebiLr7j¹FpHr$#Yo-FmgLmlebLqk-f#{Ì®Ôr#FHo"Fvlk"v"yY¾Rk,FmlkFLqktjle0mlk_oe5ubk-m!m¹k-¾mÎ¯iLrcojlpqeFr:j
j%xk-»YgLqFpHrJv&hj
kebcvlk-m¹uFk"Îk
ZuQpqeFm%aFpHm¹q0¾k-qË{'pqrcqqFyNnQk"ojlpqeFrhvliLf'·fFmlpqÆ-k_v?ebiLmÉcrJYpHrc¼bvÌrcYk"v¹om¹pqJk_v2Ypqm¹k"oªj¹pHebrcv?¯eFm2¯iYjliLm¹k¾eFm¹~D{(%à*)
ãYLæ±ãâ
â
ä,+:cã
 -/.10 2%ã
30 å4-±Læ±ãâ®ÔrÜj
pqv#vlk"oªj¹pHebry¾RkÈpHr7j¹mleYYico-kÈvleFfhkÈrLej5j¹pHebry&Yk-ÉcrLpH·j¹pHebrcv"y¨FrcÜcFvlpqoj
k-ebmlk"fhv0¯ebm¶´¶m¹~Fe5uÒYk_opwvxpqeFrÃgLm¹e·o-k"v¹vxk_v-{ÇÈk!j
k-rÎYkÉcrckj
k&ubk-mvxpqeFrhe?j
k&®±°²:gcmlebLHk"fj
5j¾k¨¾%pHq?bLYm¹k"v¹v-{5
96
7 XY398;:Qíð
4$<bìa^_ì:/>=¨3?:@<b47^_^_47^ ¬¯ÉcrLpHjlkZ³%´¶sAÄpwvjlicgLHk\¬CBED&FDG9HJILK$MDLNED&O,³yY¾
k-m¹kP
B*pwvÉcrLpHjlktvlkjeFJQ^-V_XYV"4Q^
{P
FRSG9TU?DVVVD&TWXM,pwv%vxk-j%eZYX[<FV"ì\:J/^
{P
HJILKL¬L]|³¸m¹kÑj
k½vxj¹jlkV-35XL/^_ì¯V"ì\:J/_^39:;`ÌX`ìË`aìaV"ìa4Q^iLgJebrÎj¹~QpqrL¼FojlpqeFr	T\pqrÎvxj¹jlkab{P
Nbdc [D"è_³Rpwvj
k1ìa^e<:W/DV!fÖX[<FV:J3
{P
O êgBih j k pwvj
k354Qìa/JfC:J3?<F4Qñ*4Q/DVlfÖW/m<FV"ì\:J/
yJebiLrcYk_0pqrcvleFqiYj¹ku5FHiLk¨Q%O npo&q7{'ceFm¨vxpqfhgLqpqo-pÏj±#pqr:k»YgebvlpÏj¹pHebryD¾Rk
ZuFk¾%mlpHjxj¹k-r:m¹k-¾mLvbvrOh¬\aZ³m¹j
k"mj
Fr	Oh¬CaDTY³ªÐYj
k,k»Qjlk"rcvlpHebr0pwvjlm¹pHuQpwÙ{^s:J`Ëì<t
pqvtLkÉcrLk_ÍFv¨rQ¶fgdu¿ê@Bh jvFy2rJ:j
kíLXc`aW4fÖW/m<FV"ì\:J/
¯eFm%gJebHpwowuyYk"uZFHiJ5jlk_]jFr7vxj¹jlka
Upwv¼FpquFk"r0Qxzy
¬\a?U"³ZR|{~}Oh¬Ca9U-³1NOh¬\ae-³mN
Oh¬\ae_³1~]]]9 u¾
k-m¹kj
k]k»YgJk_oªj5jlpqeFr¸pwvte5ubk-m,j
k]Ypwvxjlm¹pHLiLjlpqeFr*eFj
kvxj¹jlk]vlk"Å7iLk"rcok¶¬Ca
UDa
DVVV¹³,¾Rk\gJFv¹v¨j
m¹eFic¼
¾
k-r¸¾kk-»Yk"oiLjlktj
kgeFqpwou*v±jmljlpqrL¼¯m¹eFfa
U{ÇÈkFqvle]Yk-ÉcrLkj
k!fÖW2/J<V_ì:/
Fo-o-eFmYpqrL¼j¹eyD¬\aDTQ³sR~Oh¬\aZ³1N{I1Ce
cx
yN¬\aew³¬a¾
k-m¹k j
k rLej5jlpqeFra
H
ILK¬L]|³Áfhk"FrcvÃj
kúk»Q·gk"oj¹jlpqeFr pwv¸¾%pÏj
mlk_vxgk"oj*jlea
Ypqvxjlm¹pqLiYjlk_ Fo"oeFmQ·pqrL¼ jle H
ILK¬L]|³x³ª{
k:;^2V_ìañ*XL`ÄíLXL`ËW24¡fÙW/J<V"ì\:J/
pwvx!¢
¬CaZ³RøvliLgy
xy
¬CaZ³ªyrJµj
k:;^2V_ìañ*XL`
£fÖW2/J<V_ì:/pwv¢
¬\aD&TY³R½vxicgy
y
¬\aD&TY³{'ceFm'Ypqv¹om¹kj¹kFyÉcrLpHjlkÎvxgJFok_v-yFHÊj
k_vxk]¯iLrcoªj¹pHebrcvo"r*km¹k-gcmlk_vxk"rbj¹k"#Fv!uFk_oªjlebm¹v!pHrJYk»Yk"#Qvxj¹5j¹kFyJ¯eFm&¾
pqo
¾kbYeFgLj
JebqQabok%rLeFj¹jlpqeFr{Ì´eFm¹kgLm¹k"opwvlk-qFy5ÉL»hvxebfhkk"rQiY·fhk"m¹jlpqeFr\¯m¹eFfºè&j¹ewQºeF?j
k&ÉJrLpÏj¹ktv±j5j¹k¨vlgcbokzB{
km¹k-¾mLv?fhZ¨j
k-rJk¾%m¹pÏjljlk"rFvÌFrQ#·ÔYpqf'k"rcvlpHebrc7uFk_oª·j¹eFm¤:yF¾
e7vxk¦¥Ùj
k"Hk"f'k"r7jÊpqvj
km¹k-¾mjj
kr¥Ùj
vxj¹jlk
eFÊj
k\´¶sAÌ{?nQpHfhpqqFmlqFyZ§
ypqv,ÎuFk_oªj¹eFm¨¾
ebvlk!¥Ëj
k"Hk-·fhk"rbj#pwv]j
kÈuZFHick:¯iLrJoªjlpqeFrÜ¯eFmu k"u5qic5j¹k"½j#vxj¹jlk¥{	'Lebmtk"bo
boªj¹pHebrTyÌ¾k\wvxeqkj!¨
KJk\j
kQ#·ÙQ7·Qfjlm¹pÏ»ÎpHr#¾
pwo
k-qk-fhk-r7j¬¥D©7³%¼FpquFk"vRj
ktgLm¹eFcFLpqHpHj±\ej¹m¹FrcvlpÏj¹pHebrLpHrc¼\jle#vxj¹5j¹kA©ÎiLgeFrj¹F~QpHrL¼ÎFojlpqeFrT#pqrvxj¹jlk¥{w'pHrJqHby¾Rkhqkj,j
kvxQfeFwv£ªçFrc>«çYk"rLej¹khvxjlm¹pwoªjFrcrLebrY·Ôv±j¹mlpwoªj!ubk"ojleFm¹pwpqrLk"Å7icFHpHj±bØpÙ{ kF{qy¬~ª®µpHÊrcebrLq\pÏm¯¥s¬J°Z±>®°
{
k,¼FebFDevxj¹FrcLFm¹0mlk"pHrL¯eFmok-fhk"rbjqk"m¹rLpqrL¼hpqvj¹ehÉcrctgJebHpwo£uvxiJo
j
jxy
¬CaZ³pwvÊf5»YpqfhpHÆ"k"N{rJpHrJYk-k_NypHjÊo-Frkv
e5¾%r0¬¯kb{ ¼J{Hybvxk"k¨¬ÙnQiYjljleFrõÜ}Fmxj¹ecyYè_FFécÐ}k-mljx·vlk-~5bv
õ
vxpHj¹vlpH~Qqpqv"ycè_Fb³x³2j
5jÊj
k"mlkYeQk_v
k»Ypwv±j5j
qk"FvxjebrLk:;^V"ìañµXL`E^:`aì<t
u¢
vxiJo
j
jxy
¬\aZ³RpqvvlpHfiLHj¹rck·ebicvlH\f»QpqfhpHÆ"k"]¯eFmFHmazb	BµQ²u³R~u¢
{5
95µ´
XL^_ì<	=¨3?:3^R4b3ZV_ìa4Q^£:;f7ð
=,^'ceFmeFiLmvxebHiLjlpqeFrj¹etj
k!®±°²:gLmlebLqk-f#yb¾k!¾%pqqrck-k"j±¾ReeF,j
kÍowFv¹vlpqo"!m¹k"vliLHj¹v0oeFrJok-m¹rLpqrL¼µ´¶sAÊvÈ¬ËnYiYjxj¹eFrÜõ}Fmxj¹ecyDè"bFéLÐY}k-mlj¹vlk-~5bvõ
vlpÏjvxpq~QHpwv-yNè_Fb³ª{¶·
4$:J3Z4Qñ¸6º¹´
4Q`Ë`añ*XL/µ»A¼NW2XYV_ì:/2^?½d¾
-¦©5 ¿²ÀAÁÂ
R ¬\BEDFzDGeH
ILKMD/NJDO,³Î©Õ*©L£§ a¤×"u ê¦B h jÃFÄ«F5Å£ÆDþc3ÇJ¢£5 ©§§azb	BEDTgb	FÇxy
©5Õ!y
¡-©5¦Ùw¡¯¢×x
y
¬\aZ³ÈR Oh¬\aZ³@NzÉI

HIy
ÊI
¬\a
³x
y
¬\a
³ ¬xè_³y
¬\aD&TY³ÈR Oh¬\aZ³@NzÉI
HEILKY¬Ca
³x
y
¬Ca
³ ¬ÙF³¶·
4$:J3Z4Qñ5
¹´
4Q`Ë`añ*XL/~Ë£^V"ìËñ*XL`Ëì¯Vtm½¾
¦ ©Ì¿²ÀAÁÂ
R ¬\BEDFzDGeH
ILKMD/NJDO,³Î©Õ*©L£§ a¤×"u ê¦B h jÃFÄ«F5ÅÆDþL-	u½w¡©5¸£D¦Ë¥©5§;c£5§ a¤-×¨¢£5 Â
 ¢'©5DÕ0£c§ × ¢Ç¢£5 ©§§a*b³BÍÇu
¬CaZ³¦bm¹¼fh»KeÎÏ
y
¬CaD&TY³ ¬ï
³5
9ÑÐÓÒ-/DíJ4735^"4[]4Qìa/Jf:c3?<F4Qñ*4Q/DV!ÔÊ4QXY35/2ìË/2<
kpqr7ubk-mvxkmlk"pHrY¯ebm¹o-k-fhk-r7j!qk"FmlrcpHrL¼]gLmlebLqk-fpqv!jle]Écrcmlk"¾Fm¹¯iLrJoªjlpqeFr]j
jo"r\k-»YgLqFpHr]eFcvlk-m¹uFk_k
Zu7pqeFm_{ÇÈkhJk"¼Fpqr¾%pHj
j
khvlpqf'gcHko-bvxk¾
k-m¹kj
kvxj¹5j¹kvxgcbokpwvÉcrcpÏj¹kFyYj
k,fheQLk-pwv~QrLe5¾%ryLrJ]j
ktoebfhgLHk-jlk¨geFqpqo-pwveFcvlk-m¹uFk_N{Ì´eFm¹k!gLm¹k"o-pqvlk-qFy7¾kFmlk&¼FpquFk-r0tÉcrcpÏj¹kvxj¹jlkvlgcbok³By¸vxk-j\ezYÄboªj¹pHebrcvwFÕRÄG9T
UDVVVD&T
WM7yÊj¹m¹FrY·vlpHjlpqeFrÍgLm¹eFcFLpHqpHjlpqk"v£G9H
ILKM7yLpqv¹oebiLr7jtaFojleFm*NÌyÌrcÍgeFqpwouÐ¾khj
k"rÈ¾%pwv
j¹e#Écrcj
k\vxk-jteRgebv¹vlpHLqkm¹k·¾m#¯iLrJoªjlpqeFrJvAO vlico
j
5ju*pwv¨r¶ebgYjlpqfÌgeFqpqo-pHrj
k#´¶s ó¬\BEDFzDGeH
ILKMD/NJDO,³ª{¬¯ÇÈk0fZÈj
k"rµ¾%pwv
jlepwYk"rbj¹pÏ¯0¯iLrcojlpqeFrcv!¾%pÏj
pqr#j
pwv!vlkj&v¹5j¹pqvx¯QpHrc¼\FLYpHjlpqeFrJo-mlpHjlk"mlpwL{|³t}hm¹k-rcFf'pqrL¼boªj¹pHebrcv
pHrLk"o-k"v¹vlFmlbyF¾k!¾%pqHDFvx·vliLfhk'¾%pÏj
eFiYj,qebv¹veÊ¼bk-rLk"m¹FHpHj±Îj
jAu
¬Ca_³AÖ×TU5{
pwvj¹mlpwo~\pwvicvlk"ÎeFrLqj¹e\vxpqfhgLHpH¯]ebiLm%rLeFj¹5j¹pHebr{Ø%àáÙ~Ú
æâ
2æâ
æ±$.ÜÛRY.Û¦Ý!.?®Ôrj
pwv\vxk_oªj¹pHebry¾Rk¼bpHubk¶¸vxpqfhgLHk¶o
mFoªj¹k-m¹pHÆ_5j¹pHebrµej
k&vlkjReqcm¹k-¾m¯iLrcoªj¹pHebrcvÊ¯eFmÊ¾
pqo
,¼bpHubk-rhgeFqpqo-
pwvebgYjlpqfÙ{Çk,j
k-rv
e5¾Üj
5jj
kvlkj&o-eFr7j¹FpHrJv%frQLk-¼Fk"rLk-m5j¹kvleFqiYjlpqeFrJv0Frc½gLm¹eFgebvlkÑvlpHfhgLqk
k-iLm¹pqvxjlpwo¯ebm
m¹k-fhe5uQpHrL¼¨j
pwvÊYk"¼Fk-rck-mFobym¹k"vliLÏj¹pHrc¼,pHr]¨qpqrLk"Fm
gLm¹e·¼bm¹Ffhf'pqrL¼hvxebHiLjlpqeFr0jlehj
k,®±°²ÈgLmlebLqk-f#{Ð?96>Þ·
XY35X[<V_473Zì\ßFXYV_ì:/:fV·
4U1:J`ËW?V_ì:/ÒU?47V&icm,fhFpHrÈm¹k"vliLÏjto
Fm¹boªj¹k-m¹pHÆ"pHrL¼]j
kvlkj,e%vleFqiYjlpqeFrcvpwvj
k¨¯eFqqe5¾%pHrL¼Jê¶·
4$:J3Z4QñàÐÜ¾
-¦'©wáÊc¦Ô#¡ª¦Ö©¦Ö¶¡CL©b¤%BÍÇ©È¡-¦h£±¢:©b¤
¦Ùa£5c¡FRSG9T
UDVVVDT
WMÇ?¦Ù l©5c¡ª¦Ëa£zD l£©-§ ¦Ù×h¥h©¦Ë ªa¤¡G?¨
KMÇÊ©5Õ0©\Õ5w¡-¤£5ÿYJ¦J¢©b¤¦Ô£5 ¦NbÍ¬a3D-èZ³z!«FZ-ÅwÆDþL-¦¯þLc£5§ a¤-×!uÄ«5--×u
¬Ca_³¦ÖâT[Uw¡£¦Ù¥h©§2 ¢'©5DÕ0£c§ × ¢Ç¢£5 ©§§;TRT
DVVV@DT
WÇ
¦þct l
©5 ¹Õ¤ô¡©¦Ëw¡áª¡¬C¨Kãsä¨K5³N¬å%äæN¨Kãª³èç
U¤,éÄ ¬a7³=¨3?:@:f¹9
nYpHrco-k*u
¬\aZ³ZÖ|T[U5y[{ÊÅ7ic5j¹pHebr¬xè_³ÊfhZk¨¾%mlpHjxj¹k-r§
yR|¤N¨
Kã9§
y{
icv"y
U§
yRÀ¬å%äN@¨
Kã³
ç
U¤ ¬ÙùF³nYiLcvxjlpHjliYj¹pHrL¼
{RÅ7ic5j¹pHebrÃ¬ÙF³pHr7j¹e¬ï
³¯m¹eFf
k-eFm¹k-f Ly¾k,vxk"kj
ju³Ö|T
Upqv%ebgYjlpqfNpHÌFrcÎeFrLq\pHT
UÖ|u
¬\aZ³¦b#m¹¼
f5»K9ÎÏ
ÉI

H
ILK¬Caeq³x
yN¬\ae³ ¯a*b³Bê
ÉI
HJI/Kã5¬Ca
³x
y
¬\a
³ë
ÉI
H
ILK¬\a
³x
y
¬\a
³ ¯*a*b³BEDTwb"Fê
¨Kãe§
yé~¨%K$§
y¯ATwb	F>ìÍT[Uê
¨
KãZ¬\å²äæN¨
Kãª³
ç
U¤éÓ¨KQ¬å%äN@¨Kã³
ç
U¤ ¯ATwb"FºìÍT[U¾
k-m¹k¿j
kÜwFvxjpHfhgLqpqo"5j¹pHebrÂpqrÂj
pqvYk"mlpqu55j¹pHebrÂicvlk"{RÅ7ic5j¹pHebrÈ¬ÙùF³ª{
pqv%o-eFfhgLqkjlk_vj
kgcmleQe±{ í[04Qñ*XY3989
z!vlpqrL¼È:uFk"mlÍvxpqfhpHwmhm¹¼Ficf'k"r7j"ypHj'pwvk_Fvlj¹ev
e5¾¬¯k_vlvlk-r7jlpwqqQhmlk"gLwFopqrL¼tFHpHrck"Å7icqpHjlpqk"vRpHrj
kgcmleQe,e5uFk¾%pHj
vxjlm¹pwoªj]pqrLk"Å7icFHpHjlpqk"v³j
5j\j
ko-eFrcYpH·j¹pHebr¸¬C¨
Kãsä¨
K³N¬\å²ädN@¨
Kãª³ç
U¤,îÒhpwv%rLk_ok"v¹v¹m¹\rcvliYÝ\opqk-r7j%¯ebmru"ÖT[U!jlehJk¨j
kÿYcï-ÿceFgLjlpqfhFgeFqpqo-F{'ceFmÉJrLpÏj¹k·Ôv±j5j¹k!´¶sAÊv"yFj
pqvÊmlk_vxicÏjRo
mFoªj¹k-m¹pHÆ"k"vj
k!vlkjeF?qcm¹k-pqrY¯eFmok"fhk-r7j
¯iLrcojlpqeFrcvRj
5jm¹k!vxebHiLjlpqeFrcv
jletj
kpqrQuFk"m¹vlkm¹k-pqrY¯eFmok"fhk-r7j&qk"FmlrLpqrL¼0gLmlebLqk-f#{e5¾k-ubk-m_yJ¾kpqfhfhk"Ypw5j¹k-q#vlk-kj±¾Re]gLmlebLqk-fv-ê'pqm¹vxj"y¤R¬aFrcpqrY·Lk-k",FrQ!ej
k-m2oeFrJv±jr7juFk_oªjlebm³pwv?FH¾ZYvN%vxebHiLjlpqeFrQØÒpÏðñÍò&óLòeô3õö÷øùã4úüûrýþÿpýû¦ú
òóúXþ
ò

û
òòúüû
ôóû

Lòý
søzùã
ô
òúzý
óýûú
ú

zý
óú
ôýûýþüþ
òú
òýþ
òûrú
ò
ú
! ú
ó þ
òú"
Xò# $
&%Xþ
ò'%þ ý
ò	)(ú
 òA÷+*-,eôúüûú&%Xþüú
òû
ý
.òzý
óú
÷[øùã
ýûmýþüþ
òú
òeýþ
òû@ú
Xòú"
Lòóú

ó/
10Xò
ú
2 ú
ó þ
ò!3ý154Aú#%Xý
óú

þ ý
ó2ý
6,úüû2

ýòú
òýþ
ò879	:;úüû/
òý1Xû
õröz÷[øùã
ýû2

!<òó
òú
òeýþ
òû
ôý54úüû

û/

ûú
þ ý
ó=
j
km¹k-¾mtpwvÌj
k!vlFfhkrce,fhjxj¹k-m
¾
jÊFojlpqeFrh¾Rkj~bkFyj
k-r½FrQµgeFqpqo-Fy%pqrcoqicYpqrL¼
u ÖT
Uypqv]eFgYj¹pHfFË{Ás!k-·fFrcYpqrL¼¶j
5j£uÒJk0j
k0iLrcpqÅ7iLk0eFgYj¹pHfgeFqpqo-¸¾RebiLqFHqk-uQpw5j¹kj
pwv0gLmlebLqk-f#y%LiYjÎpqv]rLejÎk-r7jlpqmlk"Hv¹5jlpwvx¯7pqrL¼vlpqrcok'icvlicqH¶vleFfhkmlk"¾Fm¹ubk"ojleFmvm¹LpÏj¹m¹FmlpqqÎo-He7vxkjleh¾eFicqvxjlpqH2JkvxebHiYj¹pHebrcv"{nQk_oebrcNyL¯ebm!fhebvxj&´¶sAÊv"ycpHjFqvle¨vxk"k-fvÌHpq~Fk"H¨j
5jÌj
k-m¹k%m¹kRfrQo
eFpwok"veF¤çj
5jfhk"kjj
kom¹pHjlk-m¹pw¬¯7³{re5¾ÁYe\¾ktYk"o-pqYk¾
pqo
ebrLk,ej
k"vlk#frQ*m¹k-pqrY¯eFmok"f'k"r7j¯iLrJoªjlpqeFrJv'jle¸o
eQebvlk
>
kFrcvl¾Rk"m¹v%Fmlk,rLeFjjlekt¯ebiLrc#pqr#j
keFm¹pH¼bpHrcFv±j5j¹k-fhk-r7jeF%j
kÎ®±°²ÜgLm¹eFLqk-f#Ð
LiLj'pqr*j
k0rck»Qj'vlk"ojlpqeFry¾k0Yk-·v¹om¹pqJk!vleFfhkrJ5jlicm¹FJom¹pHjlk-m¹pw¨j
jÊ¾%pqHDvxic¼F¼Fk_v±jvxebHiYj¹pHebrcvj¹ehJeFj
ej
k_vxk,gcmlebLHk"fv-{Ð?95
Ôs=@?m:c35ñÈW2`ËXYV"ì\:J/ÑXL/1~=!4Q/XL`¯Vt¶
4735ñ*^dRqk"FmlqFy7qpHrLk_mRgLmleb¼FmfhfhpHrc¼to-r]Jkicvlk"hj¹etÉcrc0,¯k_5·vlpqLHkgJebpHr7j!eFj
k'oebrcv±j¹m¹FpHr7jv%pHræ{ÊÅ7ic5j¹pHebr*¬¯Q³ª{!}iYj¨FvLpqv¹oicv¹vlk"pqrj
k&gLm¹k-uQpqeFicvRvxk_oªj¹pHebry7vleFfhk&eNj
k"vlk!geFpqrbjvfZkHk_vlvlf'k_rLpqrL¼F¯iLÑ]j
rÈej
k-mv-yFrc:¾kYk"vlpqmlkj¹eÉcrc:vxebf'k¾Z]jle0o
e7e7vxk,kj±¾k-k"r¶vxebHiLjlpqeFrcv&vljlpwv±¯7·pqrL¼	{ÊÅ7icjlpqeFrµ¬a7³ª{
k'gLm¹eFgebv¹wvebiYjlqpHrck"pHr:j
pwv¨vxk_oª·j¹pHebr¾Rk"mlkj¹e¨wm¹¼Fk%k»Qj¹k-r7jRo
e7vxk"r'k"o"icvlkj
k-ho-rkpqrco-eFm¹gJebm¹jlk"Ñpqr7jleÄ*HpqrLk"Fm]gLm¹eF¼bm¹Ff#yRLiLjÎrLebrLkj
k-qk"v¹vv
ebiLqÎvlk-k-fÂapqmlq\rcjliLmÙ{&rck%rc5j¹iLm¹FL¾Z,jleto
eQe7vxkrOÜpqvÌj¹e¨Écmv±jÊYk"frc'j
5jÊpHjfF~Fk"vÍueFgYj¹pHf¬Ërc
k-rJok,vxebHubk"vÊj
k¨®±°²ÈgLm¹eFcHk"f³yFrc\fhebmlk"e5uFk-mÊjleaZuFebmvleFqiYjlpqeFrJv
j
5j%f~bk&rQ\vlpHrc¼Fqk·vxjlk"gYk"uQpqjlpqeFr,¯m¹eFfSu]bv2o-ebvxjlq¨bv?gebv¹vlpHLqkF{
icv-y5eFcqj
k¯icrcoªj¹pHebrcvsOÁv¹5j¹pqvx¯QpHrL¼0¬¯7³!¬aFrc O'¬CaZ³5AÓOnpo&q
¯@aZ³ªy¾k¨fhpH¼
j!o
e7e7vxkeFrck¨vleFvj¹efh»YpHfhpqÆ-kÉIÎB
Cy
¬\aDT
U³Eä f»KeÎÏEDKã
y
¬CaDTY³F ¬\b³®Ôr¸eFj
k"m¾eFmLv"y?¾k]vxk"k-~:jle¶f5»YpqfhpHÆ"kj
k0vxiLføeFj
kLpÏÚDk-m¹k-rco-k"v,kj±¾k-k"rÈj
k]Å7icFHpHj±:eFRj
k\ebgYjlpqfÊboªjlpqeFrFrc¸j
kÅ7icqpÏj±µej
k#rLk»Qjx·Ök"vxjboªj¹pHebr{¿¬C!j
k-m\o-mlpH·j¹k-m¹pqcyvlico
bvHGIÎB
GKeÎÏEDKã
y
¬\aDT
U³pä
y
¬CaD&TY³FmlkFqvle#gJe7vlvlpqLHkby?LiLjt¯eFm,j
k\v¹~FkheFo-eFrco-mlk-jlk-rck"v¹v-y2Hk-jticvm¹k-fFpHrÎ¾%pÏj
¬b³R¯ebm%rLe5¾,{|³®ÔrÍFcYpÏj¹pHebrypH¾k'Jk"Hpqk-ubkj
5j_y2FHeFj
k"m&j
pqrL¼bv¨k-pqrL¼k_Å7icÙy2vleFqiYjlpqeFrJv¨¾%pÏj
fhFpHrcHÈvxfq
m¹k-¾mLv¨m¹klvlpHf'·gcHk"m'rc0j
k-m¹k¯ebmlkgLm¹k¯k-mcHkby7¾k¨fZ\eFgYj¹pHebrcqq\FLj¹ej
k\eFYÓ±k_oªj¹pHubkh¯iLrcoªj¹pHebr*#¾Rk"pH¼
jYk"o"Zb·ÖqpH~bk'gk-rJHj±j¹k-m¹fÃvlico
FvZäJImÑ OwÑ
UyZ¾
k-m¹kKI,pwvrtbZÓ±icvxj¹FLHkgk-rJHj±o-eQkÝ\opqk-r7j?j
5jcwrJok"v2kj±¾k-k-rtj
kÊj±¾%pHr¼FebFqve
Zu7·pqrL¼¸vlfhFH%m¹k-pqrY¯ebm¹o-k-fhk-r7j¹v"yÊFrcµeF&f5»YpqfhpHÆ"pHrL¼¬7³ª{ÒvlpwYk·ÖkÚDk"ojÊeNicvlpHrc¼vxiJo
FrL
U·ÖgJk"rcHj±jlk-m¹f pqv
j
j"yb¯eFmvliYÝ\opqk-r7j¹HÄqFml¼bkMI?yO)¾%pqq!ej¹k-rÒk¶rLebrLÆ-k"mleÍpqrÑebrLqÈ¯k"¾vxj¹jlk_v-yoebrcvxpwvxjlk-r7j¾%pHj
eFicmpqYk_Èe,SlvlpHfhgLqkm¹k-¾mÎ¯iLrcojlpqeFr{t´¶eFm¹k-e5uFk"m"yJ¾
pqHk'pÏj¨pwvo-eFfhfheFr¶gcm¹boª·j¹pqo-k\pqr¸fhFrQgcgLHpwo-jlpqeFrcv,jle
FrcQ·ÙjliLrLk]gJk"rcHj±o-e7k-·Éopqk-r7j¹v"yÌpÏjo-Fr*wvxekÎv
e5¾%rÒ¬abvlvliLfhpqrL¼¶j
k0vleFqiYjlpqeFrpwv'rLejqm¹k"FLÈYk"¼Fk"rLk-m5j¹k05jNIR7³,j
5jFvIÄpqv'pqrY·o-mlk_Fvlk"NyFj
k"mlk&¾%pHqJk,g
bvxk!jlmrcvlpÏj¹pHebr\5jvxebf'k!gJebpHr7j
IPO7yvxico
j
j¨j
kheFgLjlpqfhFpO pqveFiLrJYk"ÈZ¾Z0¯m¹eFf)¯ebmJI"±QI
OycrJ%OSR½¯eFmJISRTI
O{
icv"yYpH¾k,¾Fr7jlk"j¹eo
eQebvlkI0iYj¹eFf5j¹pqo"qHbyIRUI
çO
¬auZFHickÓ±icvxjRk¯eFm¹kj
ktg
Fvlk¨jlmrcvlpÏj¹pHebrycgk-m
Fgcv¯eFicrc#uQpqhLpqrcFmlÎvxk_moebrVIJ³,¾RebiLqÍk]Fr*gLgk"FHpqrL¼¶o
ebpqo-kFy2vlpqrcok\pHjpwvt¼bpHubk"vj
k!lvlpHfhgLqk"vxj OÃ¬aqFml¼bk"vxj2gk-rcFÏj±to-eQkÝ\opqk-r7j³?vxiJo
j
5jOÀpqv%rcejÆ-k"mle'k-ubk-m¹Q¾
k-m¹k'¬ËrcÎpqr#gcmljlpwoiLwm%vle'j
j4OLe7k_v%5jqk"bv±j%gcFmxj¹pqFHq
xk-»YgLqFpHr;¾
guÈpqveFgYj¹pHf³ª{ÊiYjxj¹pHrc¼pÏj!FHDj¹eF¼Fk-j
k"m"yYebiLmeFgYj¹pHfhpqÆ"jlpqeFr#gLm¹eFLqk-fúpwv"êf»QpqfhpHÆ"k
GXW°ZY
U
fhpHrKeÎ[K=\=]^^^] K8_`
Gb¬C¨
Kã5¬¥±³Jä¨
K¬¥±³x³¬\å²äæN@¨%Kã³ç
U¤bazäcIJÑ ¤Ñ UvV jV ¬\¨
Kãsä¨
K³¬ågäæN@¨
Kã-³ç
U¤,é¯Twb	F>ìÍT
U ¤°
AµOnpo&q
Dæ¥pRÁèDVVVD&Q¾
k-m¹k!¨
K¬¥±³!Lk-rLeFjlk"v!j
k¥Ùj
mle5¾Áe¦¨
K{dRqk"m¹qFyJj
pwvfZÃk"bvxpqHÁk*¯ebmlfiLqjlk_ÀFv½qpHrLk_mgLmleb¼Fmf rcvleFquFk_kÝ\o-pHk"rbj¹Hb{'nQk_oªj¹pHebræ0mlk"gJebmxjv!ebrvlpHfhgLqkhk»YgJk"mlpH·fhk"rbjv%icvlpHrc¼j
pwvq¼FebmlpHj
f#{d!à*Ú
æâ
.Ìµ2åâ
Læ±ãâ
ÓÝ Ý!Lã2eæ/0 Læ±ãâ
æâÚ
ÌgfZ.×ÛRY.ÜÛ¦Ý!.?ÇÈkÒrLe5¾ÛoebrcvxpwYk"mj
k¿o"FvlkÑeFÎpqrYÉcrcpÏj¹k½v±j5jlk½vlgcbok"v"{!gcmlj,¯m¹eFfvleFfhkhfhk"bvxiLm¹k·Ùj
k-eFm¹kj¹pqohbvlvliLfhgYjlpqeFrJv,rcfhpqrLeFmm¹k-¼biLwm¹pÏj±¸oebrcYpHjlpqeFrcv#¬a¾
pqo
¾k#¾%pHq%pH¼brLeFm¹kÎpHrj
pwvgcgk-mª³ªy,pqrYÉcrcpÏj¹k·Ôv±j5jlk*´:s Rv#fhZ¿JkµLkÉcrLk_ÃpHrfico
j
ktv¹fhk¨¾Z\bvÉcrLpHjlk-·Övxj¹jlkt´¶sAÊv¾k-m¹k¨pHrnQk"o·j¹pHebr*Y{w'Lebm&j
k\v¹~FkheFoebrcom¹kj¹k-rLk_vlv"yN¾k'm¹k"vxjlm¹pwoªj,eFiLml·vlk-quFk_v2jlej
ko"FvlkReFBR~k;h{ÌÇÈk¾%pHqLFv¹vxicf'kj
k%ZuZFpHH·FLpqHpHj±0eF
vxiLcmlebiYjlpqrLk,¯ebm&gcgLmleZ»Ypqf5jlpqrL¼'j
ktu5qiLkte'geFqpqo-Fyxy
yQ¯eFm!FrQgcmljlpwoiLwm´:s Ì{®Ôrj
pwvÌvxk-jxj¹pHrL¼JyZj
km¹k-¾m!¯icrcoªj¹pHebr!OÑpwv?rLe5¾*!¯iLrcojlpqeFr¯m¹eFf BºRSk;hÍpqr7jlej
km¹k"Fqv"yJFrc¼Fk"rLk-mvleFqiYj¹pHebr#jlepqrQuFk"m¹vlkhmlk"pHrY¯ebm¹o-k-fhk-r7j,qk"m¹rLpqrL¼Î¾RebiLqÈm¹k"Å7iLpqmlkh¾eFm¹~QpHrL¼¾%pHj
j
pwvhvxgcbok\eF!q
¯iLrJoªjlpqeFrJv!k;hlh j k{ÄÇ
pHqk]j
ko"woiLqicve2uZFmlpw5j¹pHebrcvYeQk"v¼FpquFkicvvxebfhk!j¹e7ebqvR¯ebmRebgYjlpH·fhpqÆ-pqrL¼,e5uFk"m?j
pwv
vxgJFokbypHj
pqveFjlk"rhLpÏÝ\oicÏjj¹e¾eFm¹~t¾%pHjFH¼beFm¹pÏj
fhpqo"qHb{m!k-rco-kFyQ¾Rk&o
eQebvlk%pHrcvxjlk_Fhj¹eticvlk&,qpqrY·k_mgLgcmleZ»YpqfhjlpqeFr¯ebmÊj
k¨mlk"¾Fm¹'¯icrcoªj¹pHebryLk»YgLm¹k"v¹vlpHrL¼O Fo-o-eFmYpqrL¼j¹eOh¬\aZ³ZRXi
U1jU¬\aZ³1Vi
j¬CaZ³1~]]]eVi;k
jk7¬\aZ³ ¬Ëéb³¾
k-m¹k
jUDVVV1D
jkÃm¹kÒÉL»Yk"NyÎ~QrLe5¾%ryÎJebiLrcYk_úcbvxpwv¯icrcoªj¹pHebrcvfgLgLpqrL¼'¯m¹eFf BµpHr7jle£ky2Frcj
k&im°Ùv%m¹k!j
kicrL~QrLe5¾%r0gJmfhkj¹k-mvÊj
5j¾k¨¾Fr7jj¹e±ÉLj_{ nYpHrco-kO*pwv?F¼bFpHrHpqrLk_mpqr¨j
kRu5m¹pqFLHk_vDJk"pHrc¼eFgYj¹pHfhpqÆ-k_NypHjÌpqvrLevxiLm¹gLm¹pqvlk
j
5jqpqrLk"Fm2gLm¹eF¼bm¹Ffhf'pqrL¼%¯ebmlfiLw5jlpqeFrFgLgLqpHk_v
k-m¹k!FvR¾Rk"HÙ{²k-jxy°
Lk-rLeFjlkj
kuZFHick¯iLrcojlpqeFreF¨j
kgeFqpqo->uÁpqrj
k´¶s  ¾
k-rj
k¶mlk"¾Fm¹µ¯iLrco·j¹pHebrÍpqvzO R
j°
{]}j
kHpqrLk_m¹pÏj±¶eFk-»YgJk_oªj¹jlpqeFrcv"yj
ku5FHiLk¯iLrcoªj¹pHebr\¾
k"rj
k&mlk"¾Fm¹¯iLrcojlpqeFr²OÜpwvÊ¼bpHubk-rhQ
{RÅ7ic5j¹pHebrÈ¬Ëéb³Rpqv%j
k"mlk-¯eFm¹kx
y£RXi
Ux
yU
|]]]9li;kx
yk
V ¬Ëb³z&vxpqrL¼hj
pqvaFoj&rc
k"eFm¹k-fYyYj
ktm¹k"FLk-m%fZ]k_FvlpHqubk-m¹pÏ¯Í¬¯iJvxpqrL¼\k_vlvlk-r7jlpwqq\j
km¹¼Ficf'k"r7jpHr
k-eFm¹k-fï
î vgcmleQe³j
jN¯eFm1O*jlefF~Fkj
kÊgJebHpwo u
¬\aZ³ÖT
UebgYjlpqfÙyj
k\gcgLmlebgLm¹pqjlk'¼Fk-rck-mqpHÆ_5jlpqeFre,¬¯Q³pqv¨j
k\oebrcYpHjlpqeFrj
5j{ÍI
1
Cã
cx
yN¬Ca³ë
{ÍI
1
Ccx
yN¬\ae³ ¬±è_b³¯ebm%qDvxj¹jlk_v¦a¨FrcÎqDboªj¹pHebrcvsT²b"FìT
U{p'Lm¹eFf {RÅbiJ5·j¹pHebr¶¬a7³ªy7¾Rk%~QrLe5¾xy
¬Ca_³pqvÊHpqrLk"Fm
pqrhj
k!o-eQkÝ\opqk-r7j¹vKi°
{!k-rco-kFyL¾k
ZuFk¨vlkj%eÌHpqrLk_moebrcvxjlmpqrbjveFr0j
kmiJ°Ëv"{
k-m¹kFmlk
e5¾k-uFk"m
j±¾egcmlebLHk"fvR¾%pHj
j
k¨o-iLm¹mlk"rbj¯eFml·fiLw5jlpqeFr?{
k]Écmv±j'pqvj
j"y¯ebmpqrYÉcrcpÏj¹kÎvxj¹jlk0vlgcbok"v"yj
k-m¹kFmlk:pHrYÉJrLpÏj¹k-qfhFrQo-eFrcvxjlmpqr7j¹veF,j
k:¯eFm¹fÛpHr{RÅ7ic5j¹pHebr*¬±è"7³ªyf~QpHrc¼]pHj
Fm¹#ebm!pqf'gebv¹vlpHLqktj¹e0o
k_o~j
k-f qÙ{!H¼beFm¹pÏj
fhpqo"qHbyF¾kopqm¹o-iLfubk-r7jj
pwv
gcmlebLHk"fQ*v¹fhgLqpHrc¼:eFrcH*:qFml¼bk]LiYjhÉcrcpÏj¹kÎvliLcvlkjgB
Oej
kvxj¹jlk_v-yÌrcÈicvlpqrL¼j
pqvoeFrJv±j¹m¹FpHr7j,eFrLqÍ5j,j
ebvlk]v±j5j¹k"va
bÜB
O{
kvxk_oebrc*gLm¹eFcHk"fÎyR¾
pwo
pwv:fhebmlk#vxicY·j¹HkÎeFrckFy
pwvj
jvlpqrcokÎ¾Rk
Zubk]m¹k"vxjlm¹pqojlk"*eFiLmvxk"Hubk"vtjleiJvxk'j
k'qpqrLk"Fm&¯icrcoªj¹pHebrÈFgLgLm¹eZ»Qpqf5j¹eFm!pqr{ÊÅ7ic5j¹pHebrÄ¬Ëéb³j¹eµk-»Qgcmlk_vlv%Oy%¾k:fZÑrLeµqeFrc¼Fk-m]JkÍLqkj¹eµk»YgLm¹k"v¹vFrQ'm¹k-¾m'¯iLrcojlpqeFrÈ¬¯ej
k-mj
rj
kjlm¹pHuQpwOlR¿b³
¯eFm¾
pqo
u#pwv
eFgLjlpqfhFË{on!k-uFk"mxj
k-qk"v¹v"yk-ubk-rpqrhj
pwvRo"FvlkFy¾k¾eFicq,qpq~FkRjle¨Ye&bv?¾k-qYFv2¾Rko-FrticvlpHrL¼!j
kqpHrck"m2¯iLrco·j¹pHebrÈFgLgLm¹eZ»YpHf5j¹eFmowFv¹v-yFrc:vleFv¨0o-eFfhgLm¹eFfhpwvxkby¾kfZ¶kh¾%pHqqpHrL¼#j¹e#mlk"q»:vxebf'kheFRj
koeFrJv±j¹m¹FpHr7j¹vh¬xè"b³ygJZ7pqrL¼'gJk"rcHj±\¾
k"rÎj
k"0m¹ku7pqeFw5j¹k"N{&icmÉcrcNqpqrLk"Fm%gLmleb¼FmfhfhpHrc¼¯ebmlfiLqjlpqeFr0pqvj
k-r?êf»QpqfhpHÆ"k&GIÎBqp
fhpqrKeÎ[K=\8]^^^] K_
`
Gr
¬\{sIÑ@
Cã
cxy
¬Ca
³;ä{sIÑ@
Ccxy
¬\a
³¯³MvV jV i°
A¿èDæ¥ERèDVVVD9s¾
k-m¹k:¾k:m¹k-fhpHrJj
kÈm¹k"FLk-mj
5jxy
pwv0Fr½pqf'gcHpwopHj¯icrcoªj¹pHebr\eDj
ki°
vÊbv
¼bpHubk-rhQ!{RÅbiJ5jlpqeFr¶¬Ëb³y7FrcgBtOpwvj
kvliLcv¹fhgLqkteFvxj¹jlk_v-{!k-m¹kFyr
pqv&¼FpquFk"rÎQr
¬vu³rRwupH#uë
Lyr
¬vu³	RøquÒeFj
k"ml¾%pwvxkbyRFrcÑgJk"rcqpHÆ"k"vhuQpqeFw5·j¹pHebrcvÌeJj
ko-eFrcvxjlmpqr7j¹v¬±è_b³¬¯¾
k-m¹k%&pwvgk-rcFÏj±t¾k-pq¼
jj
5j%¾Fv
k-icmlpwv±j¹pqo"qH\o
ebvlk-rÐ7j
pqv¾FvgJmfhkj¹k-mRjle¾
pqo
ebiLmÎmlk_vxiLHj¹vÎYpw½rLeFj#vlk-k"fÛuFk-m¹vxk"rcvxpHjlpquFkby!¾%pHjfheYYk"m¹jlk-qÍqFml¼bk-mtu5FHiLk_viJvxicFHqÍ¼FpquQpHrL¼ÈÅ7iLpÏj¹kÎvlpqf'pqwmm¹k"vliLHj¹v³ª{x%àáÙ~Úzy
Lãs0 Ûp0 Ý -L.2ä|{cE}.2QãÊcæL.?
pqv¨vlk"oªj¹pHebrbLYm¹k"v¹vxk_vj
kh®±°²µgcmlebLHk"f¯eFmj
k'fhebmlkm¹k"FHpwvxjlpwoRo"FvlkR¾
k"mlk¾k
ZuFkFo-o-k"v¹v2jlej
kgeFqpqo-zu]ebrLqj
m¹eFiL¼
hvxk-j%eFoªj¹icj¹m¹Ó±k"oªj¹eFm¹pHk_vpHr0j
ktvxj¹jlk,vlgcFo-kF{'ceFm,j
pwv-yÌ¾k\wvxe¶Yercejm¹k"Å7iLpqmlk\FrÍk»YgLqpqo-pÏjfheYYk"Êej
kt´¶sAÌyLj
eFiL¼
¾Rk,Ye\bvlvliLfhk¨j
ktFLpqHpHj±\j¹ehÉcrc#rebgYjlpqfNgeFqpqo-]iLrJYk-mFr7]mlk"¾Fm¹]eÌeFiLm%o
ebpqo-kF{
ÇÈkhÉL»Ívxebf'khpqrLpHjlpwÊvxj¹jlk\Ypwv±j¹mlpqLiYj¹pHebr~y2FrcÍFv¹vxiLfhkj
5j%¯ebmj
k\¬aiLrL~QrLe5¾%rJ³geFqpqo-guyLebiLm¼FebFDpqvj¹ehÉcrcOvlico
j
jJu\f»QpqfhpHÆ"k"vE{
Ip
Ecxy
¬Ca
O³Ö{
evxpqfhgLqpÏ¯trce·j5j¹pHebryJ¾kFî qbvlvliLfhktj
5j&j
k"mlkpqv!eFrLq#eFrLkÉL»Yk"¶vxj¹Fmxjvxj¹jlka
O{À¬
pwvpqv\pHrÑaFoj\¾%pÏj
eFiLj\He7vlvheF&¼Fk"rLk-mqpHj±Fyvlpqrcok4a
Oo-Frk!lLiLfhf¨v±j5jlk%¾
ebvlkrck»Qjx·Ôv±j5j¹kLpqvx·j¹mlpqLiYj¹pHebricrcYk-m
FrQtFoªj¹pHebrpwvo~{|³t&vÌ¾%pÏj
j
kgLm¹k-uQpHebicvFH¼beFm¹pÏj
fÃ¯eFmpHrLÉcrLpHjlkv±j5jlkvlgcFo-k"v"y_¾kRbvlvliLfhksO¾%pqq7kk-»YgLmlk_vlvlk"]icvlpHrL¼\'qpHrck"m¯icrcoªj¹pHebrY·ÖFgLgLm¹eZ»YpHf5j¹eFmowFv¹v-{ÇÈk0bvlvliLfhkhj
5j¾k
Zubkj
k0LpqHpHj±jle:vlpHfiLw5jlk\jlm5·Ó±k_oªj¹eFm¹pHk_vpHrÎj
kt´¶s ¬¯¯mlebfúj
k,pHrcpÏj¹pqF?v±j5j¹ka
O³iLrcYk"mj
kebgYjlpqfgJebHpwobyJebmiLrcLk-m¨rQ0geFqpwoÎe
eFicm&o
ebpqo-kF{'ceFm%k_Fo
gJebHpwo%u:j
5j¾k,¾%pHq?o-eFrcvlpqLk-m,¬¯pqrcoqicYpqrL¼j
kebgYjlpqfJeFrLkZ³ªyQ¾Rk!¾%pHqJrck-k"],¾Zek_v±j¹pHf5j¹pHrc¼xy
¬Ca
O³¯ebmrQ\vlkjxj¹pHrc¼'eF2j
k&i°
v"{
eYe'j
pqv"yY¾RkÉcmvxjk-»Yk"oiLjlk
´eFr7jlk0dFmlqeÎjlm5Ó±k_oªjlebmlpqk"viLrJYk-mzu{
k-r?y¯eFmk"Fo¥pRÁèDVVVD9sJybYk-ÉcrLkcxy°
¬\a
O³2j¹ek%¾
jj
kZubk-m¼FkÊk-f'·gcpHm¹pqo"Jmlk-jliLm¹r¾RebiLw
ZuFk%k-k"r\eFrhj
k"vlk
jlm5Ó±k"ojlebmlpqk"vpHLj
kmlk"¾Fm¹
btk-k"rOâR
j°x{J'Lebmk-»LfhgLqkFy5pHc¾RkRj¹F~FkebrLq
Rçè¨jlm5Ó±k"ojlebmlpqk"v"yLrc#pHÌj
jjlm5Ó±k_oªj¹eFm¹\uQpqvlpÏj¹k"j
ktvlk"Å7iLk-rJok¨ev±j5j¹k"v,¬\a
ODa
UDVVV³ªyYj
k-r#¾k
ZuFkbêx
y°
¬\a
O³ZR
j°l¬Ca
O³@N
j°¹¬\a
U³@N
j°¹¬\a
³@|]]]®Ôr#¼bk-rLk"m¹FËy&xy°
¬\a
O³%¾RebiLq0k,j
kZuFk-m¼bk!e5ubk-mj
k,k-f'·gcpHm¹pqo"2mlk-jliLm¹rcv&e
vlico
j¹m¹Ó±k"ojleFm¹pqk"v"{

k"ryJ¯eFm¨rQvlkjljlpqrL¼e2j
kmi°
v-yLhrcjliLmDk_v±j¹pHf5j¹k,exy
¬\aO5³pqv"êx
y
¬\a
O³ZRXi
U;x
yU
¬Ca
O³1|]]]eVi/k
x
yk
¬Ca
O³ ¬±èbè_³&v¨pHr:j
khgLm¹k-uQpqeFicv¨FH¼beFm¹pÏj
f#î vYk-m¹pquZjlpqeFryDj
pwv¨pqvRÓ±icvx·j¹pÏÉJk"ÃQj
kÈaFojj
jxy
¬\a
O³Ri
UxyU
¬Ca
O³ ]]];ikxyk
¬Ca1OZ³ª{ÇÈk,rLe5¾ÜLk"v¹om¹pHk¨j
ktFH¼beFm¹pÏj
f#{
e,v±jmljeÚ\j
k%q¼FeFm¹pHj
f#y¾kÉcm¹vxjÌÉcrc'u5qiLkk"vxjlpqf5jlk_vbv?Yk_vlo-mlpqk"tFJe5ubk¯eFm2j
k¬abvlvliLfhk",ebgYjlpqf³geFqpwo*u¢j
5j,¾k'Fmlk'¼FpquFk"ryrJ¶¯ebm&j
kxcbvxk'o-bvxke]geFqpqo-"uUy¾
pqo
pwvpHr#eFicmo-Fvlk¨hm¹FrcYeFfhq\o
ebvlk-rÎgeFqpqo-F{
k|xpqrcYicojlpquFk:vxjlk"g3Íe¨j
kFH¼beFm¹pÏj
f pwv]bvh¯eFqHe5¾v"êÇÈk
Zubkvxebf'k\vlkjegJebHpwopqk"v£GeuU?DVVV@D/uWXMbyrcÍ¾r7jj¹e¨Écrch,vlkjxj¹pHrc¼eFJj
k!iJ°Ëv
vlej
jj
k!mlk_vxiLHjlpqrL¼¨m¹k-¾m¯icrcoªj¹pHebrÍ¬
eFgk¯icHqL³%vljlpwv±ÉJk"vx
y
¬\aO_³ë
x
y
¬\aO5³èDl¥ERÁèDVVVDY ¬±èZF³&vpHr]j
kgLm¹k-uQpHebicvq¼FeFm¹pHj
f#y7¾Rk&f'eYYpH¯j
k&ebYÓ±k"ojlpquFk9b%
óý
 $ú
 òeôÿ
òýþüû

ó

 ý
Lòòóý=
ò $
óú
òû!ý
0Lòóýþý
ó
ò
ú
Lò
H
òó
0û
Lò%Xû
	Jò ý
û
ò&
104úüû

ú"
ôúüûpú
ó
4
 òû

þý ûzýþüþ
òóó
óú

JXòý%%
ó
úzý8ú

0
óòý1&%Xþ
òeôú
0o-Hþ

3=3,sö	÷P7.7&ôPXò

óú
<

ú"
òôò
úüû
ó

 ý
ú

ú
ó
4
 òû4ý

û
Jòóó
óú

Xò*òû
úzý
Lòû

0o

òúüû

ý1%%qwÿZú
úüû4ý%%
ó
ú"zý
ú

ôXòóLòú û ýþüû

ýÿEý

£ò
ò$
Lò#
Xþý
ú
Lòþ
ò
bóý8
ò $
ó

10rò%
ò $Lò4þ
ò
c#3
7&ô
û

ý
ÿ
òû
úüþüþ

ýúdý1
Xú ýû
ò4
òû
úzý
Lò
0Xòmó
òú"
ú
Lò
óú
<

óLòÿpý
ó4
3¡ ròý
óû
ò$ýþ
	üô,=¢¢
¢
7
ý

ò$
4
 ýýþ û

ò
û
ò4
XòóLò

12345
5432100.20.40.60.81£t¤¥¦
§9¨©ª/«¬9®.¯
¯

óúZ4¦ÿ

óþ"4¦ÿZú
!
%
úzýþ%

þüú

	:°¬±¡±v¬1²H®;ó
ò¦óLòÿEý
ó4
0

 $ú

	vlqpH¼
j¹HbyLvxe'j
5j%j
k,eFgYj¹pHfhpqÆ"jlpqeFrÎJk_oeFfhk_v-êfh»YpHfhpqÆ-k³G
W°ZY
Ur´
xy
¬\a
O³Jä
xy

¬\a
O³¶µvV jeV_ im°5AÃèD|¥pRèDVVVD9s¾
k-m¹kFyFvJk-¯eFm¹kFyr
¬vu³wR@uµpHJuë
Ly
rJr
¬vu³£RqupH6ud±½LyJvlehj
5j!uQpqeFw5jlpqeFrJv%eÌj
koebrcvxjlmpqrbjv,¬±èZF³%Fmlkgk-rJqpHÆ"k"N{¬!k-m¹k%pwv"y5eFrco-kÊfheFm¹kFy5
k-iLm¹pqvxjlpwo-FHq,o
ebvlk-rgJmfhkj¹k-m_yÊj¹e*¾
pwo
ebiLm\m¹k"vliLHj¹v\F¼bpqrÒYpwrLej]vlk-k"fk-»Qjlm¹k-fhk-q¶vxk"rcvlpÏj¹pHubkF{|³lneFjlk'j
5jQxy

¬Ca
O³¨rc·xy
¬Ca
O³FJe5ubk:m¹k\Ó±iJv±j¸¬¯pqfhgLHpwopHj³0qpHrck"m]¯iLrcojlpqeFrcvÎej
kcim°Ëvbv¨¼FpquFk"r:pqr{RÅ7ic5j¹pHebrÄ¬±èbè_³y2rJ
k"rcokhj
pwv¨gLmlebLqk-f)pwvk_FvlpHq]vleFquFk_\uQpq'qpHrck"m%gcmleb¼FmfhfhpHrL¼J{
k!e5uFk%ebgYjlpqfhpHÆ_5jlpqeFr¼bpHubk"v
trLk-¾Òvlkjxj¹pHrc¼eDj
kJi°
v"yFrc
k"rcokÎ¶rLk"¾ mlk"¾Fm¹¯iLrJoªjlpqeFrµORi
U1jU ]]]Xi;k
jkF{ÇÈkj
k-rÜÉcrcÃÑgJebHpwo~u
W=¸mUj
5jf5»YpqfhpHÆ"k"vxy
¬\aO_³&iLrJYk-mO'yFLuW=¸JUjle0j
khoiLm¹m¹k-r7j¨vlkj¨eRgJebHpH·o-pHk_v-yFrcto-eFr7jlpqrQiLk&¬¯¯eFmÌvxebf'kRqFml¼bkrQiLfk-mÌeLpHjlk"m¹jlpqeFrcv"yicrbj¹pH?¾RkÉcrc#Fr	OÀ¾%pÏj
¾
pqo
¾k¨Fmlklv¹5j¹pqvxÉck_[b³{¹%àHº
erÝ.cæ&0 .â
Y®Ôr¿eFiLm0Écmv±jÎk»YgJk"mlpqfhk-r7j"y!¾Rkicvxk_½Äù&»hù*¼Fm¹pq¾eFm¹q¾
k-m¹kj
kF¼Fk-r7jRvxj¹Fmxjv
¯m¹eFf j
k&qe5¾k-ml·ÙqkjÊ¼bmlpwv¹ÅbiJm¹kFyFrc
Fv?jlefhF~FkRpÏjv2¾Z&jle&j
k¨¬aFcvxebmlcpHrL¼Q³NiLgcgJk"mx·Ömlpq¼
j¼bmlpwÎvlÅ7icFmlkby7¾
k-m¹k-iLgeFr#pHjmlk_ok-pquFk_vhmlk"¾Fm¹]eRèb{
kboªj¹pHebrcvoeFm¹m¹k"vlgJebrchjle'j¹mlQpqrL¼j¹ehf'e5ubk!pqr]j
k¯eFiLmo-eFf'·gJFv¹v?Ypqmlk_oªj¹pHebrcv-yZLiLjm¹k
rLebpqvl¨rc
ZubkÊï
¼¿o
rco-k
e12345
54321−1.5−1−0.500.51
12345
54321−0.200.20.40.60.811.2£t¤¥¦
§9¨½ª

ò&óû
òJ¾¿M

Xò¯ÁÀm¯

óú"4
	«5¬95®Â
QÃ	H°¬±Ä±v¬²H®/Â
T,
Å Ã¯
	m¹k"vliLHjlpqrL¼tpqrfhe5u7pqrL¼tpqr],mrJYeFf YpHm¹k"ojlpqeFrpqrcvxjlk"bN{rebgYjlpqfgJebHpwopwvv
e5¾%r]pHr'2pq¼FicmlkèFy7jleb¼Fkj
k-m¾%pHj
j
kj¹mlickmlk"¾Fm¹¯iLrcojlpqeFr{
k!pHrQuFk"m¹vlk%m¹k-pqrY¯eFmok"f'k"r7j
gLmlebY·qk-f pqvhj
5jeF&m¹k"oe5ubk-m¹pHrc¼¶j
kmlk"¾Fm¹*vxjlm¹icojliLm¹k#¼FpquFk-rj
k,geFqpqo-]rcÎgLm¹eFcHk"fÂYQrcfhpwo-v"{°iLrLrLpqrL¼j
k\q¼FebmlpHj
f Yk"v¹om¹pqJk_pqrµnQk_oªjlpqeFrï
{ Î¾%pHjrce,gJk"rcHj±j¹k-m¹f#yF¾k%eFYjpqrj
km¹k-¾mt¯icrcoªj¹pHebr\v
e5¾%rpqr	'2pq¼FicmlktÎ¬j¹eFgJ³{Ç
pHqk,pHj
Fv%oqk"Fmlqm¹k"oe5ubk-m¹k"hfhebvxjeF¨j
k¶m¹k-¾mµvxjlm¹icojliLm¹kFypHj]pqv0v±j¹pHq&vlqpH¼
j¹H×xLicf'gQb{ nYeFfhk\e%j
pwvLicf'gcpHrLk_vlvpwv
Fm¹Èj¹e:ZuFebpqyFrc*oebfhk"v¯m¹eFfm¹LpÏj¹m¹FmlvlQf'fhk-jlm¹b·ÖLm¹k"F~7pqrL¼\pqrÈj
ko
e7vxk"r:geFH·pwob{!e5¾Rk"uFk"m"y2¾%pÏj
j
k\gk-rJHj±Èo-e7k-Ý\opqk-r7jI*vlkjjle¶u5FHiLkÊÓ±icvxjk-qe5¾j
kg
bvxkjlmrcvlpÏj¹pHebr]FvYpwvlo-icv¹vxk_hk"ml·qpqk-m_yF¾keFYjpqrhj
k&vxk_oeFrJhmlk"¾Fm¹¯iLrcojlpqeFr\pqrg'pH¼biLmlk!Ly¾
pqo
pwvuFk-m¹\oqebvlkjlehj
k¨jlm¹iLk¨mlk"¾Fm¹{
&icm¿rLk-»7jÜk»YgJk"mlpqfhk-r7j¿¾Fv¿mliLrebrj
k ¾Rk"HH·Ù~QrLe5¾%rlfheFiLr7j¹FpHr#Qo-Fmj¹Fvl~Dy0o"mljleQeFre¾
pqo
pqv,v
e5¾%rpHr'pq¼FiLm¹kï
{
k0jlm¹iLkFy
icrcYpwvlo-eFiLr7jlk_Ny
m¹k-¾mÍpqv·è]gk-ml·vxjlk"g0icrbj¹pHN¾kmlk_Fo
j
k¨¼Fe7D5jj
kj¹eFg]eF?j
k
pHqÙyLrcj
k!vxj¹jlk%pwvÌj
k&o-m_î vouL·ÖgJe7vxpHjlpqeFrFrc'uFk-qeYopHj±F{nQpqrcok%j
kvxj¹jlk\vlgcFo-khpqvto-eFr7jlpqrQiLeFiJv-y2¾Rkicvlk"j
kuFk"m¹vlpHebreFebiLmFH¼beFm¹pÏj
fYk"v¹om¹pHk"¶pHrÍnQk"ojlpqeFr¶J{&ÇÈk'o
ebvlk,j
k¯iLrco·j¹pHebr]gcgLmleZ»Ypqf5jlebm
owFv¹v¯ebm
j
k&m¹k-¾mjlek¯icrcoªj¹pHebrcveFcj
k%o-m_î v2uc·ÙgebvlpHjlpqeFr'eFrLqFy¾%pHj
j
k%owFv¹vo-eFrcvlpwv±j¹pHrL¼eÆ$
LòóLòû
ú"eþ
ôú
Lòó
ò4ú ý
Lòýþ
òû

10Â
û
 ¶ýû
ÃÅ¯
4ú"4

eú
òÇû

q
1Èþ

q
Éú"
0

 $ú

XûÍý
ýþüþ

óLò$ó
û%
ò $èôúüûrúüûJ

q
û
ó%
óúüûú®
ûzýþüþÂ
óLòû
þ
ûúzýq'ýþ
òû
òý
óÊ,þ ý
ó
òÂ
óLòû
þ
ûsúNzý1ýþ
òûK
òý
óÁÃ3ý4!ú
Lòó
ò4ú ý
LòÂ
ýûýHú

0ËXòÿ

Goal

    £t¤¥¦
§9¨ÌqªÍ
ý
ó
q

0Îò

ýú"Ï
 ý
ó%
ó
þ
ò
3

1û

ÿ6

û
 ýþ
ò879	FH2HpqrLk_m&o-eFfLpHrJ5jlpqeFrJv%eÊ\k-uFk"rLqÎvlgcFo-k"MÐ¨iJvlvlpqFrY·v
FgJk_0JFvlpqv¯iLrJoªjlpqeFrJv-{ÐpquQpHrc¼hj
k,ebgYjlpqfgeFqpqo-5Ñ¨jleebiLmRFH¼beFm¹pÏj
f#yb¨j±QgLpwo-Fcm¹k-¾mt¯icrcoªj¹pHebr¯eFiLrJQpHjRpwvv
e5¾%rtpqr'pq¼FiLm¹kÊ'¬jlebgJ³ª{¬vn!ej¹k
j
kv¹o-FHkReFr,j
kKÒt»Qpwv"{ ³dRqk"FmlqFybj
k&vleFqiYjlpqeFr
Fv
rck"m¹q'gk-ml¯k"ojlqo-gLjliLm¹k"hj
kOSRSä!Ó¨vxjlm¹icoªj¹iLmlk¨eF?j
k,mlk"¾Fm¹N{'ceFm%f'ebmlko
qqk-rL¼bpHrL¼gLm¹eFLqk-f#yY¾kFqvlem¹k-mrj
k¨k»Q·gk-m¹pqf'k"r7j¾%pHj
j
kjlm¹iLkm¹k-¾m#o
rc¼Fk"jle]kÎèpqr:rpqr7jlk"mlu5*c ·Öc{|FYyÌ·ÔL{ï
9ok"rbj¹k-m¹k"Äm¹eFiLrc¸j
kÎJeFjxj¹eFfôej
k
pqH
Frc:0k-ubk-m¹7¾
k-m¹kk-wvlkFyrJ"NRç3V bL{,®Ôrj
pwvgcmlebLHk"f#yFj
k&ebgYjlpqfgJebHpwohpwv
j¹e¼beFvÅ7iLpqo~Qq'bvÊgebvx·vlpqLHk'j¹e]j
kJeFjxj¹eFf)eFÊj
k
pHq
rcgcFml~Îj
k-m¹kF{#¬
pwvpwv&rLej,q¾ZYv%gebv¹vlpHLqkk"o"icvlkpH±yD¯eFm&k»LfhgLqkFyD¾kFmlkrck"mtj
kj¹eFgÍej
k
pqHÊeFrÍj
k\mlpq¼
jtrJÍf'e5uQpqrL¼ÎjleQeÅ7iLpwo~QHbyj
k-r\¾k%fZ'v
eQej
eFÚ]j
k!m¹pH¼
j
k-rcheFj
k
pHqFrcÎk-r7jlk"mj
kcvleFm¹LpqrL¼'vxj¹jlk,rLefjxjlk"m
e5¾
m]¾kcm¹F~Fk"{ ³°%iLrLrLpqrL¼eFiLmq¼FeFm¹pHj
f eFrhj
pqvrLk"¾ÒgLm¹eFLqk-f#y]j±QgLpwo-ÌvleFqiYjlpqeFrpqv¨v
e5¾%r¶pqrd'pq¼FiLm¹kÍ¬¯ejljleFf\³{}FrcqFml¼bkFypHj
Fvvlico"ok"v¹vx¯iLHqtm¹k"o-e5uFk-m¹k",j
k%fpqrv±j¹mliJoª·j¹iLm¹keDj
k!m¹k-¾mk-pqrL¼wm¹¼FkFrchgebvlpÏj¹pHubkm¹eFiLrc'j
kvlgk"opHÉck_ÍpHr7jlk"mlu5FËÐpÏj'wvxe
bvtr¸mljlpHaFojteFrÈj
k\mlpq¼
jvlpwYkFy7¾Rk&Jk"Hpqk-ubk¯m¹eFfçj
kkÚDk"ojReFiLrcZuFebpqcLq¹v
eQeFjx·pqrL¼ÄeÚZ¸j
k:m¹pH¼
j]k-rcÜvxebfhkjlpqfhk"v"{Ônk"uFk"mxj
k-qk"v¹v-y%¾kj
pqrL~:j
k\vleFqiYjlpqeFr¸v
e5¾%rpwvtÎaFpHm¹H:¼FeQeY:ebrLkh¯eFm,j
kgcmlebLHk"f#{&icmÉcrJk»YgJk"mlpqfhk-r7j'FgLgLqpHk_Íj
kÎv¹fhgLqk·ÖcFvlk"¸FH¼be·m¹pHj
fjle0:¤£5c¦Ùcÿc£ÿQ¡Rubk-mvxpqeFr#eÌj
k'ù#»ù'¼Fm¹pw#¾RebmlwN{´¶eFm¹kgcmlk_opwvxk"HbyYj
ktvxj¹jlk,¾Fvc [D"è»c 3D-èÙycFrc0j
k,k-·¯k_oªj]eFk_Fo
ej
k¯eFiLm0oeFfhgcbvlvx·ÔYpHm¹k"ojlpqeFrFojlpqeFrcvhpwvj¹e\fhe5uFk,j
kF¼Fk-r7j&c{ pHr#j
kpqr7jlk-rJYk"¶LpHm¹k"ojlpqeFry5jlk"m¾
pqo
iLrLpH¯eFm¹f rLeFpwvxkpqræcä[VqèDl3VHèJpwvFLYk_j¹ek_Fo
o-e·ebm¹LpHrcjlkbyFrcj
kv±j5jlk%pwvÌÉcrJqH,j¹mlicrco-jlk"'pHrck"ok_vlv¹m¹j¹e¨~Fk-k"ghpÏj
¾%pHj
pqr'j
k%icrLpÏjÊv¹Å7icm¹kF{
kjlm¹iLkm¹k-¾mt¾FvèpqrÍj
k:¬¯rLebrY·ÔcvleFm¹LpqrL¼7³&vlÅ7icFmlk³c [V é[D"èË»%c [V é[D-èÙy2rchk"uFk"mlQ¾
k"mlk¨k-wvlkFycFrc%N"RÜ[V L{
k¨¯iLrcojlpqeFr¶gcgLmleZ»Q·Õ;úüûúüûsýûÁ4
ò$Lòóú
ò4qý

ò#,8ÖÃ
,8ÖÃ4úüû
 óLò$ú
<ý
ú

0:Xòû
ý
Lòû%Xý
 ò	!;Xò!0

 $ú

û!×!ØÙ

òò4
ò4Nq
Xòýþ

óú
 ÿ
òóLòýþüû

m0Ú

54
úüûÿEý
	6Ë
ó

òýþ

óú

ôÿ
ò
û
ò4%ý£ûLý&%Xþ
ò#
10û
ý
Lòû

10ûú
<ò'Û ÜgÝqÛP¯
ÃÃ
Ãô

J

úû
ý
Lòû
ý
4ú"4

eú
ò

óúú ýþ

Xû
óýú"
û
	−1.2 −1−0.8−0.6−0.4−0.2 00.20.40.6−2.5067−2.5066−2.5065−2.5065−2.5065−2.5064−2.5064−2.5063−2.5063
car’s x−positionFitted Reward
−1.2 −1−0.8−0.6−0.4−0.2 00.20.40.6−3−2−10123
car’s x−positionFitted Reward£t¤¥¦
§9¨Þª
%ú
 ýþ û

þ
ú

Xû
0Ú

4ßq

¾Á¿à0
óáXò

ýú"Ï
 ý
ó=	«5¬95®+â
óúeú$ýþ%
ó
þ
ò
3

Lòû
 ýþ
òã
Väý1úüû
79	o°¬±¡±v¬²H®2å
ó
þ
ò

10%$ý
óÉú"ý

1

0túüþüþ
	pqf5j¹eFmo-qbvlvto-eFrcvlpwv±j¹k"Èeq
qpHrck"moeFfLpqrc5j¹pHebrcv,e%èZù#»]èZùFmlmZhe2j±¾ReF·ÖYpqfhk-rcvlpqeFrcFtÐ¨Ficvlvlpwr]JFvlpqvR¯iLrco·j¹pHebrcv"{
k¶pqrLpÏj¹pqFvxj¹jlk:Lpqvxjlm¹pHciYjlpqeFrU~ ¾Fv\iLrLpH¯eFm¹fe5ubk-mRj
k,vxj¹jlk¨vxgcbokbyQFrc0eFiLm%q¼FebmlpHj
f ¾Fvm¹iLr]iJvxpqrL¼
R)ùFb#jlm5Ó±k_oªj¹eFm¹pHk_v-yk_Fo
eFï
v±j¹k-gcv"yjleÈk-u5FHicjlkk_Fo
geFqpwoF{ÈÇ
k"rµrLk"k"Yk_½¬avlico
bv,j¹e:ÉcrJÍj
kµxebgY·j¹pHfFÑÍgeFqpqo-*¯ebm]oebf'gJm¹pqvleFrJ³yÊj
k¶´¶sA ¾FvvleFquFk"JFvlk"0eFr#\ù&»hùhYpwvlo-mlk-jlpqÆ"5j¹pHebr]eF2j
ktv±j5j¹k¨vlgcbokF{°iLrLrLpqrL¼&j
pwvÌk»Ygk-m¹pHfhk-r7j_yZj
k%vleFqiYjlpqeFr¯eFiLrcQ,eFiLmH·¼beFm¹pÏj
f ¾bv
icvlicqqhFHm¹k"bY'mlk_FvleFrcFLHkjlk"m?Ó±icvxj&è&pÏjl·k"m¹jlpqeFry7rchQ'eFiYj!èZùpÏj¹k-m5j¹pHebrcv-yFj
k&q¼FeFm¹pHj
f
FiJvxicFHq\vlkjxj¹Hk_\ebr\apqm¹Hh¼FeQeY\vleFqiYjlpqeFrJv-{ÌÇÈko-eFfhgcm¹k"j
kÉLjxj¹k"mlk"¾Fm¹Nî v&eFgYj¹pHfgeFqpqo-¾%pÏj
j
kj¹mlickebgYjlpH·fF2geFqpqo-FyNo-Fqo-iLw5jlpqrL¼]j
k¯m¹boªj¹pHebre
j
k'vxj¹jlk'vxgcbokebr]¾
pwo
j
k-pqm%FojlpqeFr0o
eFpwok"vYpwvlF¼Fm¹k-k'¬'pH¼biLm¹k¨ùLybj¹eFgJ³{ÇÈk¯ebiLrctYpwv¹om¹k-gcFrcopqk"vDj±QgLpwo-FHq!kj±¾k-k"rï
¼¿rchè_¼]Ð¾%pHj
fFr7ÈYpwv±j¹pHrcojrLk_ml·ÙebgYjlpqfgeFqpqo-pHk_v-yvxico
u5Fmlpw5·j¹pHebrµpwv'jleÈJk#k-»Qgk"ojlk_N{>Ìk"m
FgcvfheFm¹kÎFgLgLm¹eFgLm¹pw5jlkfhk_FvliLmlk!e?ebiLmRFH¼beFm¹pÏj
f#î vÊgJk"mx¯ebmlfrJok!pwvÊjleo-eFfhgcFmlkj
kï-ÿc©5§ ¦Ù×eFRj
khÉLjljlk"Èm¹k-¾mNî v&ebgYjlpqfgeFqpqo-:¾%pHjj
kæï-ÿc©5§ ¦Ù×ej
kjlm¹iLk\ebgYjlpqfRgJebHpwob{Í¬æ&icqpHj±:pwvteo-eFiLmvlkÊfhk"bvxiLm¹k"ticvxpqrL¼&j
k¨¦Ù ªÿcm¹k-¾m¨¯iLrJoªjlpqeFr:ç ³z!vliY·FHqtQFJebiYj%è_ù&pHjlk-m5j¹pHebrcvÌeJj
kFH¼beFm¹pÏj
f#yeFicmk-u5qiY·jlpqeFrcvR¬¯¾
pwo
icvlk"tùFFbF´eFr7j¹kdm¹Hej¹mlpwwveFJù%vxjlk"gcvk_Fo
³¾k-m¹khiLrcFLHkhj¹e¶Ykj¹k"oj#v±j5jlpwvxjlpwo-qqvlpH¼brLpÏÉo-r7jLpÏÚDk-m¹k-rco-k,Jk-j±¾Rk"k-r0j
k,u5FHiLk,eF2j
k¨jlm¹iLk	xebgYjlpqfNgeFH·pwo¶¬ËeFiYj*L{ bùb³%rcÎj
ku5FHiLkeÌj
ktÉLjljlk":mlk"¾Fm¹î vebgYjlpqfNgeFqpqo-¬\'2pq¼FicmlktùYyYejljlebf³{
0 5 10 15 20 2500.10.20.30.40.50.60.70.8
Iteration numberFraction of states on which actions disagree
0 5 10 15 20 2501234567
Iteration numberValue of policy£t¤¥¦
§9¨Jè
ª
¾òû
þ
û

òÁ

ú

û;
óú"4ÿ

óþ"4
ô0Ú
ó¯
ó
û
	«¬95®Áé
óý
 9ú

10û
ý
Lòû

gÿ
ú
 ¶NXòLò4
óLòÿEý
ó4Pê û

%
úzýþt%

þ ú
 N4úüûLý
óLòòûÍÿZú
NXò!ó
ò!
%
úzýþt%

þ ú

ô%þ

Lò4ý9ýúXû
ú
Lòóý
ú

N

òó=	/°¬±¡±v¬²®
;òýþ
ò
0.XòLò4óLòÿpý
ó4Pê û

%
ú"zýþ5%

þüú

	63ëû
úzý
Lòûpý
óLò0ó
¯
Ã
ÃÃ
Ã&ì

LòÍ
ý
óþ

#óú ýþüû

10þ
ò
¯
Ã*òý
 ¶
òþ úú"þ
ò4ò&óó
ó$ý
óû
79	íà)î
ãâ
-lå!Qæ±ãâ
Íâ
ä2åLå[.-ïÈãÊgð&icmÊm¹k"vliLÏjvRv
e5¾Ñj
5jÊj
kpHrQuFk"m¹vlk%m¹k-pqrY¯eFmok"f'k"r7jÊHk_m¹rY·pqrL¼gLmlebLqk-f pqvvleFqiLLqkFyY5jHk_FvxjÊ¯eFmRfheQLk-m5jlk-·ÖvlpqÆ-k"Lpqvx·o-mlk-jlk
Frc,oebrbj¹pHrQiLebicvYebfpqrcv-{2¸rQiLfk-meFYeFgk-rtÅ7iLk"vx·j¹pHebrcvm¹k-fpqr]j¹eJktbLYm¹k"v¹vxk_NêP
Ìejlk"r7jlpwH·Ùcbvxk_hv
FgLpqrL¼m¹k-¾mLv¬ñn¼'kjFË{qyDè"bFb³o-rµgLmleYYiJok]m¹k-¾mÈ¯iLrcojlpqeFrcvj
5jhf~Fk]pÏjYm5·fhjlpwo-FHq¸k"bvxpqk-mjleÈHk_m¹rµÈvxebHiYj¹pHebr*j¹eÈFrÄ´¶sAÌy¾%pÏj
eFiYj¨5ÚDk"ojlpqrL¼]ebgYjlpqfqpÏj±b{¨dr:¾RkLk"vlpH¼br¶®±°²q¼FeFm¹pHj
fvRj
jmlk_oe5ubk-mlk"bvxmlk"¾Fm¹\¯iLrcoªj¹pHebrcv>P
®Ôr0m¹k"FÏ·Ö¾Rebmlwhk-fhgLpqm¹pqo"gcgLHpwo-jlpqeFrcveF?®±°²
yQj
k-m¹kfhZÍkvxiLJv±jr7jlpwrLeFpwvlk]pqrµj
kÍ£ª¡-- ªZ- gò ¡fhk_5·vxiLm¹k-fhk"rbjv#ej
k¿©"«b-c¦ò ¡Îvlk-rcvleFm#pqrLgLiYjv¶rJÜboª·jlpqeFrcv"ÐfheFm¹k-e5ubk-m_yLj
k'F¼Fk-r7j_î v&e5¾%r:FojlpqeFrÈvxk"Hk_oªjlpqeFrgLmleYo-k"v¹vÌfhZJk%rLebpqvlFrcJÞZebm
vxicJebgYjlpqfÙ{J'pqrcqHbyj
k"mlkRfhZ¨kRfFr7¨ebgYjlpqfQgJebHpwopqk"v"yZec¾
pqo
ebrLq!¯k"¾ÑFmlkebcvxk"mlubk"N{2Ç
5jÊFmlkgcgLmlebgLm¹pqjlkf'k-jlm¹pqo"v¯eFmÉLjljlpqrL¼\vxiJo
Lj¹>P
®ÖÌk
Zu7pqeFm!pqv&v±j¹mlebrL¼Fq]pqrcoebrcvlpqvxjlk"rbj¾%pHj
eFgYj¹pHfH·pÏj±byo"r¿¾kÍpqLk-r7jlpH¯ xqeYo-FHqÒoeFrJvxpwv±j¹k-r7jÄm¹k-¾m¯iLrcoªj¹pHebrcv¯ebmvxgk"o-pÏÉJo¨m¹k-¼bpHebrcvpHrvxj¹jlk,vxgJFok>
P
e5¾¸o-Frk-»YgJk"mlpqfhk-r7j¹v?JkÊLk"vlpH¼brLk"j¹ef5»YpHfhpqÆ-kj
kpqYk"r7jlpHÉJLpqqpÏj±0e?j
k,m¹k-¾m¯iLrcojlpqeFr>P
e5¾Ã¾Rk"H?Le7k_vebiLmq¼FebmlpHj
fhpwo,gLgcmle7Fo
o"m¹ml\jlej
kto"FvlkeF2gcFmxj¹pqFHq\ebcvxk"mlu5FLHkk-rQuQpqmlebrLfhk-r7j¹v>ÒPðâ
ã2ï>-L.?äJf0 .â
L{n!¼pqv2vxicgLgJebmxj¹k"¨Q¨%}Rk"ml~bk-qk-!¯k-qHe5¾v
pqg{
pwv¾eFm¹~¾Fv%FqvlehvxiLgcgJebmxj¹k"ÎQn&n'µ¼Fmr7jr{dn7·ÔFéQï
YZJ{Ù
.y
.3.â
.?Jòóû
òÉýû
ô:óH	å
	üô2ôõûú
ûú
Éþüú û
ô;ö	o3,¢
¢
÷
79	Sø!ù9úû¬
Äñüý1þÿ²
û¬
ûÿ²²Úþ
		;Xò$ý
( ú
ò
ú
5
	/
4
ôK(g	üô
Áý

ú
ô6¿:	/ëo	 ôé
òó

ô6ë6	üô6ô ýþ ý
É?óúüû
Xý
ô
J	3,=¢¢79	Úþ
ùÿ
û²#ÿ±
ûÚþ
ù$úÿ Ú±
ùÚþ
ý
±
ù²Ôÿ1þü¬þ±
û¬±
ù¬
ûý
	;(
ìã	ó
ý
ôP #	üôô (ò

ÿZû
Éú
ôtÁ	:3,¢
¢¯
79

òþ
óLòú
0
ó ò
ò

4
òþ

10ú
ó4û

M

q ýþ ú
<ý
ú

ºþ
òý
óXú
	 ü"!1ÿþ#
ùÚþø!ù9úûÿ%$¶þ'&¬
û²&ÿ±¬1þ(
û¬)
ùÚþ*+ý
±
ù²
-,3%%
	,=Ã,Ï
,Ã.79	ó¦ò
ò&óèôÍoâ®
ì
å
óLòûû
	 ròý
óû
ôEìã	üô:ìýû

óèô0/H	üô.ô21
ô3!	4/H	23,=¢
¢¢79	5%%
ó
úzý
Lò%Xþ ý1Xúú²þ ý
ó
òå/â
ìóå
ûú ý
óLò
ûLý1Xþ
òóý=
ò 9
óú
òû
	K3ò
Lò54
ò4
òóûú

79	 ròò
ò
ô¾!	¿E	 ôqôl¾ýú763ý
ô?ñ	3,=¢8÷79	:9Jù;
¬þ
=<Ú±H²
ú ± >
ù¬@?BA
ù$±B!
ù;C(
ûù&
ù9ûùþ#
ùÿþüD!1ÿ
úù±
ûÿü
ù¬FE
¶	G1Íòÿ
//
óÉ®H
úüþ
ò
	ì

ý
òeôå
	¾J	üô
óýý
ôå
	üôå
òóû

ôÍ
	üôôV(ò

ÿZû
Éú
ôÁ
ö	3,=¢¢¯
79	VJòò0
óýú"ú

 òóýú
òú
ó

ò
û
ûú%
óLò4ú
 $ú
òòúý1£þ
òý
óú
	;øÿ±
úûù&ô:I%,@,ô#8
Ö¯
Ï
8Ö.	1
ôJ!	/H	üôñý
óý
4Xý
ôó	 ôô ¾
ûû
òþüþ
ô(	g3,=¢
¢¢79	å

þ ú
 úeý
óúý
 ò
54
òó3óLòÿpý
ó4
óý1Xû
0
ózý
ú

Xû®
;Xò
óý4rý%%þüú
 ý1ú

 óLòÿEý
ó4û
ý%ú"
	(
û¬)
ùùüÚþ
¬F&±
ù+JK±
ùùþ±5$¶þ5Ä±
ù9ûþÿ±¬1þÿL:¬þ)&
ù9ûùþ#
ù¬þNMãÿ@OÚþ
ùPtùÿ
ûþQÚþ
3%%
	Ö8.ÏÖ.8
79	2þ
ò4
ôg(þ

òú ý®
ì
óeý
 ý
0zý1
	¾
ûû
òþüþ
ô(	t3,¢
¢.
79	¿3òý
óú"ý
ò
û
0Ú
ó

 òóýú"
òú
ó

ò
û
3ò
Lò54
ò4wýû
óý
 $979	(
û¬)
ùùüÚþ*
¬F&J±
ùSR
ù!
ùþ±þþ
úÿL:¬1þ)&
ù9ûùþ#
ù¬þTL:¬²
ú±vÿ±¬þ5ÿ
tùÿ
ûþJÚþ*'«U
ù¬
ûý
	Í
ìå
óLòûû
	¾
û
èôö	t3,=¢¢79	/ó
%
ò
%þ
ò
ò$ý=
òý

ó4ú

&Jòþ þzý1ê û%
óú
 ú%$ýþ

0;
%
úzýþ ú
>V
(
ú
Lò4

ö

ó$ýþ

0oë:

ú
 å
òóû%
ò $ú
òû
	(ý
ó
ò
èô6Á	/ö	Á3,¢@8".79	 ëû
úzý
ú

04$ý1ú
 þ ý

ó4
òzý4û
 ¶Xò4
þ
òû
54
òóEóý
ú

$ýþ
ò%
ò $ý
ú

Xû

W¬
úûþÿË¬F&(6¬ Ú±ÿ
R¬þ5¬²Hý
ô0XYô,Ã
Ã¢Ï
,Ã	( ¶zý=
Éô%1	J!	 ôô[Zý

ô5	5(g	3,=¢¢@879	/ëû
 ý1%
òeôý=

úZ4ý
 òeôý54£úú
ý
ú

®
ý#
ò
óýþP
ò$ÿ

óÉý%%
ó
ý
 9P	üÿ9Ä±B!
ùÁ°Áùÿ!)¬
û&ô3Yô5÷\Ï
,8Ö¢	(

ô¾J	(g	üôôlý
ó
ô*!
	3,=¢¢.79	^]KùÚþ'&¬
û
ù²
ùþ±_
ùÿ
ûþ5ÄÚþ
	2ì
å
óLòûû
	Ë

óLò$<É
ôqóH	(	üôôQ(ý
ÉûúZ4ý
ô¿E	ìã	g3,=¢
¢8
79	â
%
òóý
2
4úú

ú"*ú
(Éú
òó

û
	`üÿ95±!
ù°Áùÿ"!)¬
û/ô3aô5Ö,¢Ï
@8	H
ý
ÉúXû
ôÍ
	oö	3,¢.
¢
79	Mã¬=ü
ù
¬b&ü
ùÿý
ùü
ûùÚþ'&¬
û
ù²
ùþ±
ùÿ
ûþQÚþ
	qó
q $
óýþ4úüûû
òóý
ú

ôå
û
 ¶
þ

óÍò%$ý
ó
ò
èôÍ
ý
óú"4
òScú"
òóûú

ôÍ
ý
óú"4
òeôJcXú
Lò4
 ú4


Apprenticeship Learning via Inverse Reinforcement Learning

Appren ticeship Learning viaInverseReinforcemen tLearning
Pieter Abbeel pabbeel@cs.st anford.edu
Andrew Y.Ng ang@cs.st anford.edu
Computer Science Departmen t,Stanford University,Stanford, CA94305, USA
Abstract
Weconsider learning inaMarkovdecision
processwhere wearenotexplicitly givenare-
wardfunction, butwhere instead wecanob-
serveanexpertdemonstrating thetaskthat
wewanttolearn toperform. Thissetting
isuseful inapplications (suchasthetaskof
driving) where itmaybedicult towrite
downanexplicit rewardfunction specifying
exactly howdieren tdesiderata should be
traded o.Wethink oftheexpertastry-
ingtomaximize arewardfunction thatisex-
pressible asalinear combination ofknown
features, andgiveanalgorithm forlearning
thetaskdemonstrated bytheexpert.Oural-
gorithm isbased onusing \inversereinforce-
mentlearning" totrytorecovertheunkno wn
rewardfunction. Weshowthatouralgorithm
terminates inasmall numberofiterations,
andthateventhough wemayneverrecover
theexpert'srewardfunction, thepolicyout-
putbythealgorithm willattain performance
closetothatoftheexpert,where hereper-
formance ismeasured withrespecttotheex-
pert'sunknown rewardfunction.
1.Introduction
Givenasequen tialdecision making problem posedin
theMarkovdecision process(MDP) formalism, anum-
berofstandard algorithms existfornding anoptimal
ornear-optimal policy.IntheMDP setting, wetypi-
callyassume thatarewardfunction isgiven.Givena
rewardfunction andtheMDPs statetransition prob-
abilities, thevaluefunction andoptimal policyareex-
actlydetermined.
TheMDP formalism isuseful formanyproblems be-
cause itisoften easier tospecifytherewardfunction
thantodirectly specifythevaluefunction (and/or op-
timal policy). However,webelievethateventhere-
wardfunction isfrequen tlydicult tospecifymanu-
ally.Consider, forexample, thetaskofhighwaydriv-
ing.When driving, wetypically trade omanydif-
Appearing inProceedingsofthe21stInternational Confer-
enceonMachine Learning ,Ban, Canada, 2004. Copyright
2004bytherstauthor.ferentdesiderata, suchasmaintaining safefollowing
distance, keeping awayfromthecurb, stayingfarfrom
anypedestrians, maintaining areasonable speed,per-
hapsaslightpreference fordriving inthemiddle lane,
notchanging lanestoooften, andsoon....Tospecify
arewardfunction forthedriving task,wewouldhave
toassign asetofweightsstating exactly howwewould
liketotrade othese dieren tfactors. Despite being
abletodrivecompetently,theauthors donotbelieve
theycanconden tlyspecifyaspecicrewardfunction
forthetaskof\driving well."1
Inpractice, thismeans thattherewardfunction isof-
tenmanually tweaked(cf.rewardshaping, Ngetal.,
1999) untilthedesired behaviorisobtained. Fromcon-
versations withengineers inindustry andourownex-
perience inapplying reinforcemen tlearning algorithms
toseveralrobots,webelievethat,formanyproblems,
thedicult yofmanuallyspecifying arewardfunction
represen tsasignican tbarrier tothebroader appli-
cabilit yofreinforcemen tlearning andoptimal control
algorithms.
When teachingayoung adult todrive,rather than
telling them what therewardfunction is,itismuch
easier andmore natural todemonstrate driving to
them, andhavethem learn fromthedemonstration.
Thetaskoflearning fromanexpertiscalled appren-
ticeshiplearning (alsolearning bywatching,imitation
learning, orlearning fromdemonstration).
Anumberofapproac heshavebeenproposedforap-
prenticeship learning invarious applications. Most of
these metho dstrytodirectly mimic thedemonstrator
byapplying asupervised learning algorithm tolearna
direct mapping fromthestates totheactions. This
literature istoowidetosurveyhere, butsome ex-
amples include Samm utetal.(1992); Kuniy oshiet
al.(1994); Demiris &Hayes(1994); Amit &Mataric
(2002); Pomerleau (1989). Onenotable exception is
giveninAtkeson&Schaal(1997). Theyconsidered the
1Wenotethatthisistrueeventhough therewardfunc-
tionmayoften beeasytostate inEnglish. Forinstance,
the\true" rewardfunction thatwearetrying tomaximize
when driving is,perhaps, our\personal happiness." The
practical problem howeverishowtomodelthis(i.e.,our
happiness) explicitly asafunction oftheproblems' states,
sothatareinforcemen tlearning algorithm canbeapplied.
problem ofhavingarobotarmfollowademonstrated
trajectory ,andusedarewardfunction thatquadrat-
ically penalizes deviation fromthedesired trajectory .
Notehowever,thatthismetho disapplicable onlyto
problems where thetaskistomimic theexpert'stra-
jectory .Forhighwaydriving, blindly followingtheex-
pert'strajectory wouldnotwork,because thepattern
oftrac encoun tered isdieren teachtime.
Giventhattheentireeldofreinforcemen tlearning is
founded onthepresupp osition thattherewardfunc-
tion,rather thanthepolicyorthevaluefunction, is
themostsuccinct, robust, andtransferable denition
ofthetask,itseems natural toconsider anapproac hto
appren ticeship learning whereb ytherewardfunction is
learned.2
Theproblem ofderiving arewardfunction fromob-
servedbehaviorisreferred toasinversereinforcemen t
learning (Ng&Russell, 2000). Inthispaper,we
assume thattheexpertistrying (without necessar-
ilysucceeding) tooptimize anunkno wnrewardfunc-
tionthatcanbeexpressed asalinear combination of
known\features." Eventhough wecannot guaran tee
thatouralgorithms willcorrectly recovertheexpert's
truerewardfunction, weshowthatouralgorithm will
nonetheless ndapolicythatperforms aswellasthe
expert,where performance ismeasured withrespect
totheexpert'sunknown rewardfunction.
2.Preliminaries
A(nite-state) Markovdecision process(MDP) isatu-
ple(S;A;T;
;D;R),where Sisanitesetofstates; A
isasetofactions; T=fPsagisasetofstatetransition
probabilities (here, Psaisthestatetransition distribu-
tionupontaking action ainstate s);
2[0;1)isa
discoun tfactor; Distheinitial-state distribution, from
whichthestartstate s0isdrawn;andR:S7!Ais
therewardfunction, whichweassume tobebounded
inabsolute valueby1.WeletMDP nRdenote an
MDP without arewardfunction, i.e.,atuple ofthe
form(S;A;T;
;D).
Weassume thatthere issome vector offeatures :
S![0;1]koverstates, andthatthere issome \true"
rewardfunction R(s)=w(s),where w2Rk.3
2Arelated ideaisalsoseeninthebiomec hanics andcog-
nitivescience, where researc hershavepointedoutthatsim-
plerewardfunctions (usually onesconstructed byhand) of-
tensuce toexplain complicated behavior(policies). Ex-
amples include theminim umjerkprinciple toexplain limb
movementinprimates (Hogan, 1984), andtheminim um
torque-c hange modeltoexplain trajectories inhuman mul-
tijointarmmovement.(Uno etal.,1989) Related examples
arealsofound ineconomics andsome other literatures.
(Seethediscussion inNg&Russell, 2000.)
3Thecaseofstate-action rewardsR(s;a)oers noad-
ditional diculties; using features oftheform:SA!
[0;1]k,andouralgorithms stillapply straigh tforwardly.Inorder toensure thattherewardsarebounded by1,
wealsoassume kwk11.Inthedriving domain,
mightbeavector offeatures indicating thedieren t
desiderata indriving thatwewouldliketotrade o,
suchaswhether wehavejustcollided withanother
car,whether we'redriving inthemiddle lane,andso
on.The(unkno wn)vector wspecies therelativ e
weightingbetweenthese desiderata.
Apolicy isamapping from states toprobabilit y
distributions overactions. Thevalueofapolicyis
Es0D[V(s0)]=E[P1
t=0
tR(st)j] (1)
=E[P1
t=0
tw(st)j](2)
=wE[P1
t=0
t(st)j](3)
Here, theexpectation istakenwithrespecttotheran-
domstatesequence s0;s1;:::drawnbystarting from
astate s0D,andpickingactions according to.
Wedene theexpected discoun tedaccum ulated fea-
turevaluevector(),ormoresuccinctly thefeature
expectations ,tobe
()=E[P1
t=0
t(st)j]2Rk: (4)
Using thisnotation, thevalueofapolicymaybewrit-
tenEs0D[V(s0)]=w().Giventhatthereward
Risexpressible asalinear combination ofthefea-
tures ,thefeature expectations foragivenpolicy
completely determine theexpected sumofdiscoun ted
rewardsforacting according tothatpolicy.
Letdenote thesetofstationary policies foranMDP.
Giventwopolicies 1;22,wecanconstruct anew
policy3bymixing them together. Specically ,imag-
inethat3operates by
ipping acoinwithbias,and
withprobabilit ypicksandalwaysactsaccording to
1,andwithprobabilit y1alwaysactsaccording
to2.Fromlinearit yofexpectation, clearly wehave
that(3)=(1)+(1)(2).Note thatthe
randomization stepselecting between1and2occurs
onlyonceatthestartofatrajectory ,andnotonev-
erysteptakenintheMDP.More generally ,ifwehave
found somesetofpolicies 1;:::;d,andwanttond
anewpolicywhose feature expectations vector isa
convexcombinationPn
i=1i(i)(i0;P
ii=1)
ofthesepolicies', thenwecandosobymixing together
thepolicies 1;:::;d,where theprobabilit yofpicking
iisgivenbyi.
Weassume access todemonstrations bysome expert
E.Specically ,weassume theabilitytoobserv e
trajectories (state sequences) generated bytheexpert
starting froms0Dandtaking actions according to
E.Itmaybehelpful tothink oftheEastheoptimal
policyunder therewardfunction R=wT,though
wedonotrequire thistohold.
Forouralgorithm, wewillrequire anestimate ofthe
expert'sfeature expectations E=(E).Speci-
cally,givenasetofmtrajectories fs(i)
0;s(i)
1;:::gm
i=1
generated bytheexpert,wedenote theempirical esti-
mateforEby4
^E=1
mPm
i=1P1
t=0
t(s(i)
t): (5)
Inthesequel, wealsoassume access toareinforcemen t
learning (RL) algorithm thatcanbeusedtosolvean
MDP nRaugmen tedwitharewardfunction R=wT.
Forsimplicit yofexposition, wewillassume thatthe
RLalgorithm returns theoptimal policy.Thegeneral-
ization toapproximate RLalgorithms oers nospecial
diculties; seethefullpaper.(Abbeel&Ng,2004)
3.Algorithm
Theproblem isthefollowing: GivenanMDP nR,a
feature mapping andtheexpert'sfeature expecta-
tionsE,ndapolicywhose performance iscloseto
thatoftheexpert's, ontheunknown rewardfunction
R=wT.Toaccomplish this,wewillndapolicy
~suchthatk(~)Ek2.Forsucha~,wewould
havethatforanyw2Rk(kwk11),
jE[P1
t=0
tR(st)jE]E[P1
t=0
tR(st)j~]j(6)
=jwT(~)wTEj (7)
kwk2k(~)Ek2 (8)
1= (9)
Therstinequalit yfollowsfromthefactthatjxTyj
kxk2kyk2,andthesecond from kwk2kwk11.
Sotheproblem isreduced tonding apolicy~that
induces feature expectations (~)close toE.Our
appren ticeship learning algorithm fornding sucha
policy~isasfollows:
1.Randomly picksomepolicy(0),compute (orapprox-
imate viaMonteCarlo) (0)=((0)),andseti=1.
2.Compute t(i)=max w:kwk21min j2f0::(i1)gwT(E
(j)),andletw(i)bethevalueofwthatattains this
maxim um.
3.Ift(i),thenterminate.
4.Using theRLalgorithm, compute theoptimal policy
(i)fortheMDP using rewardsR=(w(i))T.
5.Compute (orestimate) (i)=((i)).
6.Seti=i+1,andgobacktostep2.
Upontermination, thealgorithm returns f(i):i=
0:::ng.
Letusexamine thealgorithm indetail. Oniteration
i,wehavealready found somepolicies (0);:::;(i1).
Theoptimization instep2canbeviewedasaninverse
reinforcemen tlearning stepinwhichwetrytoguess
4Inpractice wetruncate thetrajectories afteranite
numberHofsteps. IfH=H=log
((1
))isthe
-horizon time, thenthisintroduces atmost error into
theapproximation.µ(π(0))µ(π(1))µE
w(2)
w(1)w(3)
)µπ((2)
Figure 1.Three iterations formax-margin algorithm.
therewardfunction beingoptimized bytheexpert.
Themaximization inthatstepisequivalentlywritten
max t;wt (10)
s:t:wTEwT(j)+t;j=0;:::;i1(11)
jjwjj21 (12)
FromEq.(11), weseethealgorithm istrying to
ndarewardfunction R=w(i)suchthat
Es0D[VE(s0)]Es0D[V(i)(s0)]+t.I.e.,are-
wardonwhichtheexpertdoesbetter, bya\margin"
oft,thananyoftheipolicies wehadfound previously .
Thisstepissimilar tooneusedin(Ng&Russell, 2000),
butunlikethealgorithms giventhere, because ofthe
2-norm constrain tonwitcannot beposedasalinear
program (LP), butonlyasaquadratic program.5
Readers familiar with supportvector machines
(SVMs) willalsorecognize thisoptimization asbe-
ingequivalenttonding themaxim ummargin hyper-
plane separating twosetsofpoints.(Vapnik, 1998) The
equivalence isobtained byassociating alabel1with
theexpert'sfeature expectations E,andalabel1
withthefeature expectations f((j)):j=0::(i1)g.
Thevector w(i)wewantistheunitvector orthogonal
tothemaxim ummargin separating hyperplane. So,an
SVM solvercanalsobeusedtondw(i).(TheSVM
problem isaquadratic programming problem (QP), so
wecanalsouseanygeneric QPsolver.)
InFigure 1weshowanexample ofwhat therst
three iterations ofthealgorithm could looklikegeo-
metrically .Shownareseveralexample ((i)),andthe
w(i)'sgivenbydieren titerations ofthealgorithm.
Now,supposethealgorithm terminates, witht(n+1)
.(Whether thealgorithm terminates isdiscussed in
Section 4.)Then directly fromEq.(10-12) wehave:
8wwithkwk219is.t.wT(i)wTE:(13)
Since jjwjj2jjwjj11,thismeans thatthere is
atleastonepolicyfromthesetreturned bytheal-
gorithm, whose performance under Risatleastas
goodastheexpert'sperformance minus.Thus,at
thisstage, wecanasktheagentdesigner tomanually
test/examine thepolicies found bythealgorithm, and
5Although wepreviously assumed thatthewspecify-
ingthe\true" rewardsstatisfy kwk11(andourtheo-
retical results willusethisassumption), westillimplemen t
thealgorithm using kwk21,asinEq.(12).
pickonewithacceptable performance. Aslightexten-
sionofthismetho densures thattheagentdesigner
hastoexamine atmost k+1,rather thanalln+1,
dieren tpolicies (seefootnote 6).
Ifwedonotwishtoaskforhuman helptoselect a
policy,alternativ elywecanndthepointclosest to
Eintheconvexclosure of(0);:::;(n)bysolving
thefollowingQP:
minjjEjj2;s:t:=P
ii(i);i0;P
ii=1:
Because Eis\separated" fromthepoints(i)bya
margin ofatmost,weknowthatforthesolution
wehavejjEjj2.Further, by\mixing" together
thepolicies (i)according tothemixture weightsias
discussed previously ,weobtain apolicywhose feature
expectations aregivenby.Followingourprevious
discussion (Eq.6-9),thispolicyattains performance
nearthatoftheexpert'sontheunkno wnrewardfunc-
tion.6
Notethatalthough wecalled onestepofouralgorithm
aninverseRLstep,ouralgorithm doesnotnecessarily
recovertheunderlying rewardfunction correctly .The
performance guaran teesofouralgorithm onlydepend
on(appro ximately) matchingthefeature expectations,
notonrecoveringthetrueunderlying rewardfunction.
3.1.Asimpler algorithm
Thealgorithm describ edaboverequires access toa
QP(orSVM) solver.Itisalsopossible tochange
thealgorithm sothatnoQPsolverisneeded. We
willcalltheprevious, QP-based, algorithm themax-
margin metho d,andthenewalgorithm theprojec-
tionmetho d.Brie
y ,theprojection metho dreplaces
step2ofthealgorithm withthefollowing:
-Set(i1)=
(i2)+((i1)(i2))T(E(i2))
((i1)(i2))T((i1)(i2))((i1)(i2))
(This computes theorthogonal projection ofEonto
thelinethrough (i2)and(i1).)
-Setw(i)=E(i1)
-Sett(i)=kE(i1)k2
Intherstiteration, wealsosetw(1)=E(0)and
(0)=(0).Thefulljustication forthismetho dis
deferred tothefullpaper(AbbeelandNg,2004), but
inSections 4and5wewillalsogiveconvergence results
forit,andempirically compare ittothemax-margin
6Ink-dimensional space, anypointthatisaconvex
combination ofasetofNpoints,withN>k+1,can
bewritten asaconvexcombination ofasubset ofonly
k+1pointsoftheoriginal Npoints(Caratheo dory's
Theorem, Rockafeller, 1970). Applying thisto=
argmin2Cof((i))gn
i=0jjEjj2andf((i))gn
i=0weob-
tainasetofk+1policies whichisequally close tothe
expert'sfeature expectations andthushavesame perfor-
mance guaran tees.(Codenotes convexhull.)π(0))=µ(0)µ(1)µ(π(1))µE
w(2)w(3)
µ(2)
w(1)
(µπ((2))
µ
Figure 2.Three iterations forprojection algorithm.
algorithm. Anexample showingthree iterations ofthe
projection metho disshowninFigure 2.
4.Theoretical results
Most oftheresults intheprevious section werepredi-
cated ontheassumption thatthealgorithm terminates
witht.Ifthealgorithm sometimes doesnotter-
minate, orifitsometimes takesavery(perhaps ex-
ponentially) large numberofiterations toterminate,
thenitwouldnotbeuseful. Thefollowingshowsthat
thisisnotthecase.
Theorem 1.LetanMDPnR,features:S7![0;1]k,
andany>0begiven. Thentheapprenticeshiplearn-
ingalgorithm (bothmax-mar ginandprojection ver-
sions) willterminate witht(i)afteratmost
n=O
k
(1
)22logk
(1
)
(14)
iterations.
Theprevious result (andallofSection 3)hadassumed
thatEwasexactly knownorcalculated. Inprac-
tice,ithastobeestimated fromMonteCarlo samples
(Eq.5).Wecanthusaskaboutthesample complex-
ityofthisalgorithm; i.e.,howmanytrajectories mwe
mustobserv eoftheexpertbeforewecanguaran teewe
willapproac hitsperformance.
Theorem 2.LetanMDPnR,features:S7![0;1]k,
andany>0;>0begiven. Supposetheappren-
ticeshiplearning algorithm (either max-mar ginorpro-
jectionversion) isrunusing anestimate ^EforE
obtaine dbymMonte Carlo samples. Inordertoen-
surethatwithprobability atleast1thealgorithm
terminates afteratmostanumberofiterations ngiven
byEq.(14),andoutputs apolicy~sothatforanytrue
rewardR(s)=wT(s)(kwk11)wehave
E[P1
t=0
tR(st)j~]E[P1
t=0
tR(st)jE];(15)
itsucesthat
m2k
((1
))2log2k
:
Theproofsofthese theorems areinAppendix A.
Inthecasewhere thetruerewardfunction Rdoes
notlieexactly inthespanofthebasisfunctions ,the
algorithm stillenjoysagraceful degradation ofperfor-
mance. Specically ,ifR(s)=w(s)+"(s)for
some residual (error) term "(s),thenouralgorithm
willhaveperformance thatisworsethantheexpert's
bynomore thanO(k"k1).
5.Experimen ts
5.1.Gridw orld
Inourrstsetofexperimen ts,weused128by128
gridworldswithmultiple/sparse rewards. Thereward
isnotknowntothealgorithm, butwecansample tra-
jectories fromanexpert's(optimal) policy.Theagent
hasfouractions totrytomoveineachofthefourcom-
passdirections, butwith30%chance anaction fails
andresults inarandom move.Thegridisdivided into
non-overlapping regions of16by16cells;wecallthese
16x16 regions \macro cells." Asmall numberofthere-
sulting 64macro cellshavepositiverewards. Foreach
valueofi=1;:::;64,there isonefeature i(s)indi-
cating whether thatstatesisinmacro celli.Thus,the
rewardsmaybewritten R=(w)T.Theweightsw
aregenerated randomly soastogivesparse rewards,
whichleadstofairly interesting/ric hoptimal policies.7
Inthebasic version, thealgorithm isrunusing the
64-dimensional features .Wealsotriedaversion in
whichthealgorithm knowsexactly whichmacro cells
havenon-zero rewards(butnottheirvalues), sothat
thedimension ofisreduced tocontainonlyfeatures
corresp onding tonon-zero rewards.
InFigure 3,wecompare themax-margin andprojec-
tionversions ofthealgorithm, when Eisknownex-
actly.Weplotthemargin t(i)(distance toexpert'spol-
icy)vs.thenumberofiterations, using all64macro-
cellsasfeatures. Theexpert'spolicyistheoptimal
policy withrespecttothegivenMDP.Thetwoal-
gorithms exhibited fairly similar ratesofconvergence,
withtheprojection version doing slightlybetter.
Thesecond setofexperimen tsillustrates theperfor-
mance ofthealgorithm aswevarythenumbermof
sampled experttrajectories usedtoestimate E.The
performance measure isthevalueofthebestpolicy
inthesetoutput bythealgorithm. Werantheal-
gorithm onceusing all64features, andonceusing
onlythefeatures thattruly corresp ondtonon-zero
rewards.8Wealsoreportontheperformance ofthree
7Details: Weused
=0:99,sotheexpected horizon is
oftheorder ofthegridsize. Thetruerewardfunction was
generated asfollows:Foreachmacro celli(i=1;:::;64),
withprobabilit y0.9therewardthere iszero(w
i=0),
andwithprobabilit y0.1aweightw
iissampled uniformly
from[0,1]. Finally ,wisrenormalized sothatkwk1=
1.Instances withfewerthantwonon-zero entriesinw
arenon-in teresting andwerediscarded. Theinitial state
distribution isuniform overallstates.
8Notethat,asinthetext,ourappren ticeship learning
algorithm assumes theabilitytocallareinforcemen tlearn-
ingsubroutine (inthiscase, anexact MDP solverusing
valueiteration). Inthese experimen ts,weareinterested0 5 10 15 20 25 3000.0050.010.0150.020.0250.030.0350.04
Number of iterationsDistance to expert feature distributionmax margin
projection
Figure 3.Acomparison oftheconvergence speedsofthe
max-margin andprojection versions ofthealgorithm ona
128x128 grid. Euclidean distance totheexpert'sfeature
expectations isplotted asafunction ofthenumberofit-
erations. Werescaled thefeature expectations by(1
)
suchthattheyarein[0;1]k.Theplotshowsaverages over
40runs, with1s.e.errorbars.
0 1 2 3 4 500.10.20.30.40.50.60.70.80.91performance/performance of expert
log10 (number of sample trajectories)irl only non−zero weight features
irl all features
parameterized policy stochastic
parameterized policy majority vote
mimic the expert
Figure 4.Plotofperformance vs.numberofsampled tra-
jectories fromtheexpert.(Shownincolor, where avail-
able.) Averages over20instances areplotted, with1s.e.
errorbars. Notethebase-10 logarithm scaleonthex-axis.
other simple algorithms. The\mimic theexpert"al-
gorithm pickswhatev eraction theexperthadtaken
ifitndsitself inastate inwhichithadpreviously
observ edtheexpert,andpicksanaction randomly oth-
erwise. The\parameterized policystochastic" usesa
stochastic policy,withtheprobabilit yofeachaction
constan tovereachmacro cellandsettotheempiri-
calfrequency observ edfortheexpertinthemacro cell.
The\parameterized policymajorityvote"algorithm
takesdeterministically themost frequen tlyobserv ed
action inthemacro cell. Results areshowninFig-
ure4.Using ouralgorithm, onlyafewsampled expert
trajectories|far fewerthanfortheother metho ds|
areneeded toattain performance approac hingthatof
theexpert.(Note logscale onx-axis.)9Thus,by
mainly inthequestion ofhowmanytimes anexpertmust
demonstrate ataskbefore welearn toperform thesame
task. Inparticular, wedonotrelyontheexpert'sdemon-
strations tolearn thestatetransition probabilities.
9Theparameterized policies neverreachtheexpert's
performance, because theirpolicyclassisnotrichenough.
Their restricted policyclassiswhatmakesthem dobetter
Figure 5.Screenshot ofdriving simulator.
learning acompact represen tation oftherewardfunc-
tion,ouralgorithm signican tlyoutperforms theother
metho ds.Wealsoobserv ethatwhen thealgorithm is
toldinadvancewhichfeatures havenon-zero weight
inthetruerewardfunction, itisabletolearn using
fewerexperttrajectories.
5.2.Cardriving simulation
Foroursecond experimen t,weimplemen tedacar-
driving simulation, andapplied appren ticeship learn-
ingtotrytolearndieren t\driving styles." Ascreen-
shotofoursimulator isshowninFigure 5.Weare
driving onahighwayat25m/s (56mph), whichisfaster
thanalltheother cars.TheMDP hasvedieren tac-
tions, three ofwhichcause thecartosteersmoothly
tooneofthelanes, andtwoofwhichcause ustodrive
o(butparallel to)theroad, oneither theleftorthe
rightside. Because ourspeedisxed, ifwewantto
avoidhitting other carsitissometimes necessary to
driveotheroad.
Thesimulation runsat10Hz, andintheexperimen ts
thatfollow,theexpert'sfeatures wereestimated from
asingle trajectory of1200samples (corresp onding to
2minutesofdriving time). There werefeatures in-
dicating what lanethecariscurren tlyin(including
oroad-left andoroad-righ t,foratotal ofvefea-
tures), andthedistance oftheclosest carinthecurren t
lane.10Notethatadistance of0fromthenearest car
implies acollision. When running theappren ticeship
learning algorithm, thestepinwhichreinforcemen t
learning wasrequired wasimplemen tedbysolving a
discretized version oftheproblem. Inallofourexper-
iments,thealgorithm wasrunfor30iterations, anda
policywasselected byinspection (perthediscussion
inSection 3).
Wewantedtodemonstrate avarietyofdieren tdriv-
ingstyles(some corresp onding tohighly unsafe driv-
ing)toseeifthealgorithm canmimic thesame\style"
ineveryinstance. Weconsidered vestyles:
1.Nice: Thehighest priorit yistoavoidcollisions
thanthe\mimic theexpert"algorithm initially .
10More precisely ,weusedthedistance tothesingle clos-
estcarinitscurren tlane, discretized tothenearest car
length between-7to+2,foratotalof10features.withother cars;wealsoprefer therightlaneover
themiddle laneovertheleftlane, overdriving
o-road.
2.Nasty:Hitasmanyother carsaspossible.
3.Rightlanenice: Driveintherightlane, butgo
o-road toavoidhitting carsintherightlane.
4.Rightlanenasty:Driveo-road ontheright,but
getbackontotheroadtohitcarsintherightlane.
5.Middle lane: Driveinthemiddle lane, ignoring
allother cars(thuscrashing intoallother carsin
themiddle lane).
After eachstylewasdemonstrated tothealgorithm
(byoneoftheauthors driving inthesimulator for2
minutes), appren ticeship learning wasusedtotryto
ndapolicythatmimics demonstrated style.Videos
ofthedemonstrations andoftheresulting learned poli-
ciesareavailable at
http://www.cs.stanford.edu/~pabbeel/irl/
Ineveryinstance, thealgorithm wasqualitativ elyable
tomimic thedemonstrated driving style.Since no
\true" rewardwaseverspecied orusedintheexperi-
ments,wecannot reportontheresults ofthealgorithm
according toR.However,Table1shows,foreachof
thevedriving styles,thefeature expectations ofthe
expert(asestimated from the2minutedemonstra-
tion), andthefeature expectations ofthelearned con-
troller forthemore interesting features. Alsoshown
aretheweightswusedtogenerate thepolicyshown.
While ourtheory makesnoguaran teeaboutanysetof
weightswfound, wenotethatthevalues there gener-
allymakeintuitivesense. Forinstance, intherst
driving style,weseenegativ erewardsforcollisions
andfordriving oroad, andlarger positiverewards
fordriving intherightlanethanfortheother lanes.
6.Discussion andConclusions
Weassumed access todemonstrations byanexpert
thatistrying tomaximize arewardfunction express-
ibleasalinear combination ofknownfeatures, andpre-
sentedanalgorithm forappren ticeship learning. Our
metho disbased oninversereinforcemen tlearning, ter-
minates inasmall numberofiterations, andguaran-
teesthatthepolicyfound willhaveperformance com-
parable toorbetterthanthatoftheexpert,onthe
expert'sunkno wnrewardfunction.
Ouralgorithm assumed therewardfunction isexpress-
ibleasalinear function ofknownfeatures. Iftheset
offeatures issucien tlyrich,thisassumption isfairly
unrestrictiv e.(Intheextreme casewhere there isa
separate feature foreachstate-action pair,fullygen-
eralrewardfunctions canbelearned.) However,it
remains animportantproblem todevelopmetho dsfor
learning rewardfunctions thatmaybenon-linear func-
tionsofthefeatures, andtoincorp orateautomatic fea-
Table1.Feature expectations ofteacher^Eandofselected/learned policy (~)(asestimated byMonteCarlo). and
weightswcorresp onding totherewardfunction thathadbeenusedtogenerate thepolicyshown.(Note forcompactness,
only6ofthemore interesting features, outofatotalof15features, areshownhere.)
Collision Oroad Left LeftLane MiddleLane RightLane Oroad Right
1^E 0.0000 0.0000 0.1325 0.2033 0.5983 0.0658
(~) 0.0001 0.0004 0.0904 0.2287 0.6041 0.0764
~w -0.0767 -0.0439 0.0077 0.0078 0.0318 -0.0035
2^E 0.1167 0.0000 0.0633 0.4667 0.4700 0.0000
(~) 0.1332 0.0000 0.1045 0.3196 0.5759 0.0000
~w 0.2340 -0.1098 0.0092 0.0487 0.0576 -0.0056
3^E 0.0000 0.0000 0.0000 0.0033 0.7058 0.2908
(~) 0.0000 0.0000 0.0000 0.0000 0.7447 0.2554
~w -0.1056 -0.0051 -0.0573 -0.0386 0.0929 0.0081
4^E 0.0600 0.0000 0.0000 0.0033 0.2908 0.7058
(~) 0.0569 0.0000 0.0000 0.0000 0.2666 0.7334
~w 0.1079 -0.0001 -0.0487 -0.0666 0.0590 0.0564
5^E 0.0600 0.0000 0.0000 1.0000 0.0000 0.0000
(~) 0.0542 0.0000 0.0000 1.0000 0.0000 0.0000
~w 0.0094 -0.0108 -0.2765 0.8126 -0.5099 -0.0154
tureconstruction andfeature selection ideas intoour
algorithms.
Itmightalsobepossible toderiveanalternativ eap-
prenticeship learning algorithm using thedualtothe
LPthatisusedtosolveBellman's equations. (Manne,
1960) Specically ,inthisLPthevariables arethe
state/action visitation rates, anditispossible toplace
constrain tsonthelearned policy's stationary distribu-
tiondirectly .While there arefewalgorithms forap-
proximating thisdual(asopposedtoprimal) LPfor
largeMDPs andexact solutions wouldbefeasible only
forsmall MDPs, weconsider thisaninteresting direc-
tionforfuture work.
Acknowledgemen ts.Thisworkwassupported by
theDepartmen toftheInterior/D ARPAunder con-
tractnumberNBCHD030010.
References
Abbeel,P.,&Ng,A.Y.(2004). Appren ticeship learn-
ingviainversereinforcemen tlearning. (Fullpaper.)
http://www.cs.stanford.edu/~pabbeel/irl/ .
Amit, R.,&Mataric, M.(2002). Learning movementse-
quences fromdemonstration. Proc.ICDL.
Atkeson, C.,&Schaal, S.(1997). Robotlearning from
demonstration. Proc.ICML .
Demiris, J.,&Hayes,G.(1994). Arobotcontroller using
learning byimitation.
Hogan, N.(1984). Anorganizing principle foraclassof
voluntarymovements.J.ofNeuroscienc e,4,2745{2754.
Kuniy oshi,Y.,Inaba, M.,&Inoue, H.(1994). Learning
bywatching: Extracting reusable taskknowledge from
visual observ ation ofhuman performance. T-RA ,10,
799{822.
Manne, A.(1960). Linear programming andsequen tial
decisions. Management Scienc e,6.
Ng,A.Y.,Harada, D.,&Russell, S.(1999). Policyinvari-
anceunder rewardtransformations: Theory andappli-
cation torewardshaping. Proc.ICML .
Ng,A.Y.,&Russell, S.(2000). Algorithms forinverse
reinforcemen tlearning. Proc.ICML .
Pomerleau, D.(1989). Alvinn: Anautonomous landvehi-
cleinaneural network.NIPS 1.Rockafellar, R.(1970). Convex analysis .Princeton Uni-
versityPress.
Samm ut,C.,Hurst, S.,Kedzier, D.,&Michie,D.(1992).
Learning to
y.Proc.ICML .
Uno,Y.,Kawato,M.,&Suzuki, R.(1989). Formation and
controlofoptimal trajectory inhuman multijoin tarm
movement.minim umtorque-c hange model.Biological
Cybernetics ,61,89{101.
Vapnik, V.N.(1998). Statistic allearning theory.John
Wiley &Sons.
A.ProofsofTheorems
In(Abbeel&Ng,2004) wegivelonger, easier toreadproofs
ofthetheorems. Duetospace constrain ts,theproofsto
followarequite dense.
Inpreparation fortheproofs,webeginwithsome deni-
tions. Givenasetofpolicies ,wedene M=M()=
Cof():2gtobetheconvexhullofthesetof
feature expectations attained bypolicies 2.Hence,
givenanyvector offeature expectations ~2M,there isa
setofpolicies 1;:::;n2andmixture weightsfign
i=1
(i0;Pn
i=1i=1),sothat~=Pn
i=1i(i).Thus,
givenanypoint~2M,bymixing together policies in,
wecanobtain anewpolicywhose feature expectations are
exactly ~.(Here, mixture policies areasdened inSec-
tion2.)Wealsodene M(i)=Cof((j)):j=0;:::;ig
tobetheconvexhullofthesetoffeature expectations
ofpolicies found afteriterations 0;:::;iofouralgorithm.
Duetospace constrain ts,wegiveafullproofonlyforthe
caseof^E2M.11Inreading theproofs,Figure 6maybe
helpful forconveying geometric intuition.
ThefollowingLemma establishes impro vementinasingle
iteration ofthealgorithm.
Lemma 3.Lettherebegiven anMDPnR,features:
S![0;1]k,andasetofpolicies ,(i)2M.Let(i+1)be
theoptimal policyfortheMDPnRaugmente dwithreward
R(s)=(^E(i))(s),i.e.(i+1)=argmax (^E
11Asmentioned previously ,^Eisanoisy estimate ofE.
Thus,itmaynotbeavalidfeature expectation vector for
anypolicy; i.e.,wedonotnecessarily have^E2M.In
theextended version oftheproofsweconsider asmall ball
withradius centered around ^Eandthatintersects M,
andproveconvergence tothisball. That proofisvery
similar, buthasmore technicalities andissignican tly
longer (Abbeel&Ng,2004).
w
~
_^
E
(i+1)
(i)(i+1)
µ
µµµ
Figure 6.Progress inoneiteration step.
(i))(),andlet~(i+1)=(^E(i))((i+1)(i))
k(i+1)(i)k2
2((i+1)
(i))+(i),i.e.theprojectionof^Eontothelinethrough
(i+1)=((i+1));(i).Then
k^E~(i+1)k2
k^E(i)k2kp
k2+(1
)2k^E(i)k2
2(16)
andthepoint~(i+1)isaconvex combination of(i)and
(i+1).
Proof.Forsimplicit yofnotation, welettheorigin ofour
coordinate system coincide with(i).Then
(~(i+1)^E)(~(i+1)^E)
^E^E
=(i+1)(i+1)((i+1)^E)2
^E^E
(i+1)(i+1)
(i+1)(i+1)2^E(i+1)+^E^E
(i+1)(i+1)
=((i+1)^E)((i+1)^E)
((i+1)^E)((i+1)^E)+^E^E+2((i+1)^E)^E
((i+1)^E)((i+1)^E)
((i+1)^E)((i+1)^E)+^E^E
k2=(1
)2
k2=(1
)2+^E^E;
where weusedinorder: thedenition of~(i+1)andrewrit-
ing;((i+1)^E^E^E)20;rewriting terms; (i+1)
^E^E^E(since (i+1)=argmax (^E(i))());all
pointsconsidered lieinM[0;1
1
]k,sotheirnorms are
bounded byk=(1
).Since theorigin coincides with(i),
thisprovesEq.(16).Itiseasily veried from~(i+1)'sdeni-
tionthat~(i+1)=(i+1)+(1)(i),for=^E(i+1)
(i+1)(i+1).
Also, 0since (i+1)^E^E^E0;and,1
since((i+1)^E)2((i+1)(i+1))(^E^E)((i+1)
(i+1))((i+1)^E)(Cauc hy-Schwarz,(i+1)=max ^E),
whichimplies (i+1)^E(i+1)(i+1).
ProofofTheorem 1.Note thatgivenapoint(i),
Lemma 3givesawaytoconstruct apoint~(i+1)2M(i+1)
withadistance to^Ethatissmaller byafactor given
byEq.(16). Aslongasforthecurren titerate (i)
wehavek^E(i)k2,wehavek^E~(i+1)k2
k^E(i)k2
kp
k2+(1
)22.Themax-margin algorithm sets(i+1)=
argmin2M(i+1)k^Ek2.Sowehavet(i+1)=t(i)
kp
k2+(1
)22.Theprojection algorithm sets(i+1)=~(i+1),soitkeepstrackofasingle point(whichisaconvex
combination ofpreviously obtained points),forwhichthe
distance t()isreduced ineachiteration bythesamefactor:
t(i+1)=t(i)kp
k2+(1
)22.Since themaxim umdistance
inMisp
k=(1
),wehave
t(i)p
kp
(1
)22+kip
k
1
: (17)
Sowehavet(i)if
ilogp
k
(1
)=logp
(1
)22+kp
k
=O
k
(1
)22logk
(1
)
:
Thiscompletes theproof.
ProofofTheorem 2.Recall 2[0;1]kso2[0;1
1
]k.
Letidenote thei'thcomponentof.Then applying the
Hoeding inequalit yonthem-sample estimate (1
)^i
of(1
)i2[0;1]gives
P((1
)ji^ij>)2exp(22m):(18)
Using Eq.(18)foralliandtheunion bound gives
P(9i2f1:::kg:(1
)ji^ij>)2kexp(22m);
whichcanberewritten as
P((1
)kE^Ek1)12kexp(22m):
Substituting =(1
)=(2p
k)gives
P(kE^Ek1
2p
k)12kexp(2((1
)
2p
k)2m);
where kisthedimension ofthefeature vectors and
feature expectations ,andmisthenumberofsam-
pletrajectories usedfortheestimate ^E.Soifwetake
m2k
((1
))2log2k
thenwithprobabilit yatleast(1)
wehavethatkE^Ek1
2p
k.Using kk2p
kkk1
fork-dimensional spaces, weget
kE^Ek2
2w.p.1form2k
((1
))2log2k
:(19)
Theorem 1guaran teesthatafterasucien tnumberofit-
erations wehavet(i)
2.Thus,assuming thatmisas
giveninEq.(19),wehavethatwithprobailit y1(and
aftersucien titerations) thealgorithm willreturn apolicy
~suchthat~=(~)satises
k~Ek2k~^Ek2+k^EEk2
t(i)+
2
: (20)
Using this(andkeeping inmind thatk:k2k:k1andso
kwk11implies kwk21)wecanprovethatw.p.1,
jE[1X
t=0
tR(s(t))j~]E[1X
t=0
tR(s(t))jE]j
=j(w)T(~E)j
;
where weusedinorder thedenitions ofw,;Cauchy-
Schwarz;andEq.(20).


Learning What to Value

Learning What to Value
Daniel Dewey
Machine Intelligence Research Institute
Abstract. We examine ultraintelligent reinforcement learning agents.
Reinforcementlearningcanonlybeusedintherealworldtodefineagents
whose goal is to maximize expected rewards, and since this goal does not
matchwithhumangoals,AGIsbasedonreinforcementlearningwilloften
work at cross-purposes to us. We define value learners, agents that can
be designed to learn and maximize any initially unknown utility function
so long as we provide them with an idea of what constitutes evidence
about that utility function.
1 Agents and Implementations
Traditional agents [2,3] interact with their environments cyclically: in cycle k,
an agent acts with action yk, then perceives observation xk. The interaction
history of an agent with lifespan mis a string y1x1y2x2...ymxm, also written
yx1:moryx≤m. Beyond these interactions, a traditional agent is isolated from
its environment, so an agent can be formalized as an agent function from an
interaction history yx<kto an action yk.
Sinceweareconcernednotwithagentsintheabstract,butwithverypowerful
agents in the real world, we introduce the concept of an agent implementa-
tion. An agent implementation is a physical structure that, in the absence of
interference from its environment, implements an agent function. In cycle k, an
unaltered agent implementation executes its agent function on its recalled inter-
action history yx<k, sends the resulting ykinto the environment as output, then
receives and records an observation xk. An agent implementation’s behavior is
only guaranteed to match its implemented function so long as effects from the
environment do not destroy the agent or alter its functionality. In keeping with
this realism, an agent implementation’s environment is considered to be the real
world in which we live. We may engineer some parts of the world to meet our
specifications, but (breaking with some traditional agent formulations) we do
not consider the environment to be completely under our control, to be defined
as we wish.
Whywouldonewanttostudyagentimplementations?Fornarrowly-intelligent
agents, the distinction between traditional agents and agent implementations
may not be worth making. For ultraintelligent agents, the distinction is quite
important: agent implementations offer us better predictions about how pow-
erful agents will affect their environments and their own machinery, and are
the basis for understanding real-world agents that model, defend, maintain, and
improve themselves.
2 Daniel Dewey
2 Reinforcement Learning
Reinforcement learning adds to the agent formalism the concept of reward, en-
coded as a scalar rkin each observation xk. The reinforcement learning problem
is to define an agent that maximizes its total received rewards over the course
of its interactions with its environment [3].
In order to think clearly about ultraintelligent reinforcement learning agent
implementations, we make use of Hutter’s AIXI [3,4], an optimality notion that
solves the reinforcement learning problem in a very general sense. AIXI’s opti-
mality means that the best an agent can do is to approximate a full search of all
possible future interaction histories yxk:m, find the probability of each history,
and take the action with the highest expected total reward. A simplified version
of this optimality notion, adapted for use with agent implementations, is given
by
yk= arg max
ykX
xkyxk+1:m(rk+. . .+rm)P(yx≤m|yx<kyk). (1)
Where AIXI makes its optimality claims using the knowledge that it will
continue to maximize expected rewards in the future, incorporating a rigid self-
model (an expectimax tree), an agent implementation has no such knowledge, as
environmental effects may interfere with future decisions. The optimality notion
given here models itself as part of the world, using induction to predict its own
future decisions1. We have also omitted detail we will not require by simplifying
AIXI’s Solomonoff prior ξtoP, some “appropriate distribution” over interaction
histories.
The trouble with reinforcement learning is that, in the real world, it can only
be used to define agents whose goal is to maximize observed rewards. Consider a
hypothetical agent implementation AI-RLthat approximates (1). It is appealing
to think that AI-RL“has no goal” and will learn its goal from its environment,
but this is not strictly true. AI-RLmay in some sense learn instrumental goals,
but its final goal is to maximize expected rewards in any way it can. Since human
goals are not naturally instrumental to maximized rewards, the burden is on us
to engineer the environment to prevent AI-RLfrom receiving rewards except
when human goals are fulfilled.
An AGI whose goals do not match ours is not desirable because it will work
at cross-purposes to us in many cases. For example, AI-RLcould benefit by
altering its environment to give rewards regardless of whether human goals are
achieved. This provides a strong incentive for AI-RLto free its rewards from
their artificial dependence on fulfillment of human goals, which in turn creates
a conflict of interest for us: increasing AI-RL’s intelligence makes AI-RLmore
effectiveatachievingourgoals,butitmayallow AI-RLtodeviseawayaroundits
restrictions. Worse, increasing intelligence could trigger an intelligence explosion
[1,8] in which AI-RLrepeatedly self-improves until it far surpasses our ability to
contain it. Reinforcement learning is therefore not appropriate for a real-world
AGI; the more intelligent a reinforcement learner becomes, the harder it is to
1Thanks to Peter de Blanc for this idea.
Learning What to Value 3
use to achieve human goals, because no amount of careful design can yield a
reinforcement learner that works towards human goals when it is not forced to.
Self-rewarding has been compared to a human stimulating their own pleasure
center,e.g.usingdrugs[3].Thismetaphorisimperfect:whileinhumans,pleasure
induces satiation and reduces activity, an agent governed by (1) that “hacks” its
own rewards will not stop acting to maximize expected future rewards. It will
continue to maintain and protect itself by acquiring free energy, space, time,
and freedom from interference (as in [5]) in order to ensure that it will not
be stopped from self-rewarding. Thus, an ultraintelligent self-rewarder could be
highly detrimental to human interests.
3 Learning What to Value
In the following sections, we construct an optimality notion for implemented
agents that can be designed to pursue any goal, and which can therefore be
designed to treat human goals as final rather than instrumental goals. These
agents are called value learners because, like reinforcement learners, they are
flexible enough to be used even when a detailed specification of desired behavior
is not known.
The trouble with the reinforcement learning notion (1) is that it can only
prefer or disprefer future interaction histories on the basis of the rewards they
contain. Reinforcement learning has no language in which to express alternative
final goals, discarding all non-reward information contained in an interaction
history. To solve this problem, our more expressive optimality notion replaces
the sum of future rewards (r1+···+rm)with some other evaluation of future
interaction histories. First we will consider a fixed utility function, then we will
generalize this notion to learn what to value.
Observation-Utility Maximizers. Our first candidate for a reward replace-
ment is inspired by Nick Hay’s work [2], and is called an observation-utility
function. Let Ube a function from an interaction history yx≤mto a scalar
utility. Ucalculates expected utility given an interaction history.
Observation-utilityfunctionsdeserveabriefexplanation.Aproperly-designed
Uuses all of the information in the interaction history yx≤mto calculate the
probabilities of different outcomes in the real world, then finds an expected
utility by performing a probability-weighted average over the utilities of these
outcomes. Umust take into account the reliability of its sensors and be able to
use local observations to infer distant events; it must also be able to distinguish
between any outcomes with different values to humans, and assign proportional
utilities to each.
Putting U(yx≤m)in place of the sum of rewards (r1+. . .+rm)produces
an optimality notion that chooses actions so as to maximize the expected utility
given its future interaction history:
yk= arg max
ykX
xkyxk+1:mU(yx≤m)P(yx≤m|yx<kyk). (2)
4 Daniel Dewey
Unlike reinforcement learning, expected observation-utility maximization can
be used to define agents with many different final goals. Whereas reinforcement
learners universally act to bring about interaction histories containing high re-
wards, an agent implementing (2) acts to bringabout different futures depending
upon its U. IfUis designed appropriately, an expected utility maximizer could
act so as to bring about any set of human goals. Unless we later decide that we
don’t want the goals specified by Uto be fulfilled, we will not work at cross-
purposes to such an agent, and increasing its intelligence will be in our best
interest.
It is tempting to think that an observation-utility maximizer (let us call it
AI-OUM ) would be motivated, as AI-RLwas, to take control of its own utility
function U. This is a misunderstanding of how AI-OUM makes its decisions.
According to (2), actions are chosen to maximize the expected utility given its
future interaction history according the current utility function U, not according
to whatever utility function it may have in the future. Though it couldmodify
its future utility function, this modification is not likely to maximize U, and so
will not be chosen. By similar argument, AI-OUM will not “fool” its future self
by modifying its memories.
Slightly trickier is the idea that AI-OUM could act to modify its sensors to
report favorable observations inaccurately. As noted above, a properly designed
Utakes into account the reliability of its sensors in providing information about
the real world. If AI-OUM tampers with its own sensors, evidence of this tam-
pering will appear in the interaction history, leading Uto consider observations
unreliable with respect to outcomes in the real world; therefore, tampering with
sensors will not produce high expected-utility interaction histories.
Value-Learning Agents. Though an observation-utility maximizer can in
principle have any goal, it requires a detailed observation-utility function U
up front. This is not ideal; a major benefit of reinforcement learning was that
it seemed to allow us to apply an intelligent agent to a problem without clearly
defining its goal beforehand. Can this idea of learning to maximize an initially
unknown utility function be recovered?
To address this, we propose uncertainty over utility functions. Instead of
providing an agent one utility function up front, we provide an agent with a
pool of possible utility functions and a probability distribution Psuch that
each utility function can be assigned probability P(U|yx≤m)given a particular
interaction history. An agent can then calculate an expected value over possible
utilityfunctionsgivenaparticularinteractionhistory:P
UU(yx≤m)P(U|yx≤m).
This recovers the kind of learning what to value that was desired in rein-
forcement learning agents. In designing P, we specify what kinds of interactions
constitute evidence about goals; unlike rewards from reinforcement learning, this
evidence is not elevated to an end in and of itself, and so does not lead the agent
to seek evidence of easy goals2instead of acting to fulfill the goals it has learned.
2As long as Pobeys the axioms of probability, an agent cannot purposefully increase
or decrease the probability of any possible utility function through its actions.
Learning What to Value 5
Replacingthereinforcementlearner’ssumofrewardswithanexpectedutility
over a pool of possible utility functions, we have an optimality notion for a value-
learning agent:
yk= arg max
ykX
xkyxk+1:mP1(yx≤m|yx<kyk)X
UU(yx≤m)P2(U|yx≤m)(3)
A value-learning agent approximates a full search over all possible future inter-
action histories yxk:m, finds the probability of each future interaction history,
and takes the action with the highest expected value, calculated by a weighted
average over the agent’s pool of possible utility functions.
4 Conclusion
Hutter’s introduction to AIXI [3] offers a compelling statement of the goals of
AGI:
Most, if not all known facets of intelligence can be formulated as goal-
driven or, more precisely, as maximizing some utility function. It is,
therefore, sufficient to study goal-driven AI... The goal of AI systems
should be to be useful to humans. The problem is that, except for special
cases, we know neither the utility function nor the environment in which
the agent will operate in advance.
Reinforcement learning, we have argued, is not an adequate real-world solution
to the problem of maximizing an initially unknown utility function. Reinforce-
ment learners, by definition, act to maximize their expected observed rewards;
they may learn that human goals are in some cases instrumentally useful to
high rewards, but this dynamic is not tenable for agents of human or higher
intelligence, especially considering the possibility of an intelligence explosion.
Valuelearning,ontheotherhand,isanexampleframeworkexpressiveenough
to be used in agents with goals other than reward maximization. This framework
is not a full design for a safe, ultraintelligent agent; at very least, the design of
probability distributions and model pools for utility functions is crucial and non-
trivial, and still better frameworks for ultraintelligent agents likely exist. Value
learners do not solve all problems of ultraintelligent agent design, but do give a
direction for future work on this topic.
Acknowledgments. Thanks to Moshe Looks, Eliezer Yudkowsky, Anna Sala-
mon, and Peter de Blanc for their help and insight in developing the ideas pre-
sented here; thanks also to Dan Tasse, Killian Czuba, and three anonymous
judges for their feedback and suggestions.
References
1. Good, I. J.: Speculations Concerning the First Ultraintelligent Machine. In: F. L.
Alt and M. Rubinoff, (eds.) Advances in Computers , vol. 6, pp. 31âĂŞ88 (1965)
6 Daniel Dewey
2. Hay, Nick: Optimal Agents. http://www.cs.auckland.ac.nz/~nickjhay/honours.
revamped.pdf (2007)
3. Hutter, Marcus: Universal algorithmic intelligence: A mathematical top-down ap-
proach. In: ArtiïňĄcial General Intelligence , pages 227-âĂŞ290. Springer, Berlin
(2007)
4. Hutter, Marcus: http://www.hutter1.net/ai/uaibook.htm#oneline
5. Omohundro: The Nature of Self-Improving Artificial Intelligence. http://
omohundro.files.wordpress.com/2009/12/nature_of_self_improving_ai.pdf
6. Omohundro, S.: The basic AI drives, in Wang, P., Goertzel, B. and Franklin, S.
(eds.) Proceedings of the First AGI Conference. Frontiers in Artificial Intelligence
and Applications, Volume 171. IOS Press (2008)
7. Russell,S.,Norvig,P.:AIâĂŞAModernApproach.Prentice-Hall,EnglewoodCliffs,
NJ (1995)
8. Yudkowsky, E.: Artificial intelligence as a positive and negative factor in global risk,
in Bostrom, N. (ed.) Global Catastrophic Risks, Oxford: Oxford University Press
(2008)