85115105110103 769711010311797103101 97110100 841149711011510897116105111110 77111100101108115
116111 8310110810199116 116104101 66101115116 97109111110103 79117116112117116115 102114111109 77117108116105112108101 7784 115121115116101109115
89971151171041051141116510710598974484971141118797116971109798101 97110100 69105105991041051141118311710910511697
658482 83112111107101110 769711010311797103101 841149711011510897116105111110 821011151019711499104 76979811111497116111114105101115
5045504550 72105107971141051009710544 8310110510797459910411144 83111114971071174510311711044 75121111116111 544957454850565644 749711297110
102121971151171041051141114697107105989744 11697114111461199711697110979810144 1011051059910410511411146115117109105116971036497116114469911146106112
65981151161149799116
84104105115 11297112101114 97100100114101115115101115 116104101 11211411198108101109 111102 971171161111099711645
1059997108108121 11510110810199116105110103 116104101 98101115116 97109111110103 111117116112117116115 102114111109
109117108116105112108101 1099799104105110101 1161149711011510897116105111110 40778441 11512111511610110911546
69120105115116105110103 971121121141119799104101115 11510110810199116 116104101 111117116112117116 97115115105103110101100
116104101 104105103104101115116 11599111114101 979999111114100105110103 116111 97 11697114103101116 1089711010311797103101
10911110010110846 73110 115111109101 999711510111544 116104101 101120105115116105110103 971121121141119799104101115
100111 110111116 119111114107 11910110810846 84104105115 11297112101114 112114111112111115101115 116119111
109101116104111100115 116111 105109112114111118101 112101114102111114109971109910146 84104101 12114115116
109101116104111100 105115 9897115101100 111110 97109117108116105112108101 9911110911297114105115111110 116101115116
97110100 9910410199107115 119104101116104101114 97 11599111114101 102114111109 1089711010311797103101 97110100
1161149711011510897116105111110 109111100101108115 105115 115105103110105129997110116108121 104105103104101114 11610497110
116104101 11111610410111411546 84104101 11510199111110100 109101116104111100 105115 9897115101100 111110
112114111989798105108105116121 11610497116 97 1161149711011510897116105111110 105115 110111116 105110102101114105111114 116111
116104101 11111610410111411544 11910410599104 105115 11211410110010599116101100 102114111109 116104101 9798111118101
1159911111410111546 6912011210111410510910111011697108 114101115117108116115 115104111119 11610497116 116104101 11211411145
112111115101100 109101116104111100115 9799104105101118101 97110 10510911211411111810110910111011611110250116111
54 37 105110 112101114102111114109971109910146
49 7311011611411110011799116105111110
84104105115 11297112101114 97100100114101115115101115 116104101 9910497108108101110103105110103 11211411198108101109
111102 97117116111109971161059997108108121 11510110810199116105110103 116104101 98101115116 97109111110103 11111711645
112117116115 102114111109 109117108116105112108101 1099799104105110101 1161149711011510897116105111110 40778441
115121115116101109115 4070105103117114101 494146 73110 991111099810511097116105111110115 111102 10911710811610545
112108101 7784 11512111511610110911544 115111109101 99111109112111110101110116 7784 115121115116101109115
99971101161149711011510897116101971151111171149910111510111011610111099101119101108108119104105108101111116104101114115
9997110110111116 11910110810846 73110 11511799104 97 999711510144 9911111411410199116 11510110810199116105111110 111102
11610410198101115116999711011198118105111117115108121 98111111115116 112101114102111114109971109910146
658482 10497115 98101101110 100101118101108111112105110103 11511799104 109117108116105112108101 7784
11512111511610110911544 10511099108117100105110103 116104114101101 749711297110101115101451161114569110103108105115104
4074456941 7784 11512111511610110911558 84687784 4070117114117115101 97110100 731051009744
Outputa Outputb Outputc
The Best Output
Selection System
MTa MTb MTc
Input
70105103117114101 4958 84104101 11510110810199116105111110 11512111511610110946
495757544144 6851 40831171091051169744 504848494144 97110100 837784 408797116971109798101
101116 971084644 504848504144 97110100 116104114101101 691101031081051151044511611145749711297110101115101 406945
7441778411512111511610110911558 84687784407011711411711510197110100731051009744495757544144
72806584 4073109971091171149744 504848504144 97110100 837784 408797116971109798101
101116 971084644 504848504146 73110 111114100101114 116111 1011189710811797116101 1019799104778411512111545
11610110944 116104101 7784 111117116112117116115 119101114101 1099711011797108108121 97115115105103110101100
111110101 111102 102111117114 11497110107115
49
44 6544 6644 6744 97110100 6844 98121 1109745
116105118101 11511210197107101114115 111102 116104101 11697114103101116 108971101031179710310146 84104101 10510010197108
11510110810199116105111110 102111114 744569 7784 115121115116101109115 105115 116104101 10410510310410111511645
11497110107101100 111117116112117116115 102114111109 116104101 116104114101101 744569 7784 11512111511610110911558
8468778444 685144 97110100 83778446 84104101 10510010197108 11510110810199116105111110 102111114
694574 7784 115121115116101109115 105115 116104101 1041051031041011151164511497110107101100 111117116112117116115
102114111109116104101116104114101101694574778411512111511610110911558 84687784447280658444
97110100 83778446
70105103117114101 50 115104111119115 116104101 10511010010511810510011797108 1121011141021111141099711099101115
111102 116104101 116104114101101 744569 7784 115121115116101109115 97110100 116104101 10510010197108 11510145
10810199116105111110 115121115116101109 100101114105118101100 102114111109 116104101105114 99111109981051109711610511111046
70105103117114101 51 115104111119115 116104101 10511010010511810510011797108 1121011141021111141099711099101115 111102
116104101 116104114101101 694574 7784 115121115116101109115 97110100 116104101 10510010197108 11510145
10810199116105111110 115121115116101109 100101114105118101100 102114111109 116104101105114 99111109981051109711610511111046
84104101 1081011021164510497110100 103114111117112 111102 9897114115 1051101001059997116101115 116104101 1149745
49
84104101 102111117114 11497110107115 97114101 10010112110101100 97115 10211110810811111911558 406541 801011141021019911658
110111 11211411198108101109 105110 101105116104101114 10511010211111410997116105111110 111114 103114971091099711459 406641 709710511458
10197115121 116111 1171101001011141151169711010044 119105116104 101105116104101114 115111109101 11711010510911211111411697110116 10511045
10211111410997116105111110 109105115115105110103 111114 1397119101100 103114971091099711459 406741 659999101112116979810810158
9811411110710111044 98117116 117110100101114115116971101009798108101 119105116104 1011111111411659 406841 7811111011510111011510158
10510911211111411697110116 10511010211111410997116105111110 10497115 98101101110 1161149711011510897116101100 105110991111141141019911610812146
0
10
20
30
40
50
60
70
80
90
100
A A+B A+B+C
Performance (%)
Ideal Selection System TDMT D3 SMT
701051031171141015058 801011141021111141099711099101111102116104101105100101971081151011081019911610511111011512111545
116101109 102111114 749711297110101115101451161114569110103108105115104 7784 11111711611211711611546
116105111 111102 116104101 11011710998101114 111102 11510111011610111099101115 11497110107101100 97115 65 116111
1161041011161111169710811011710998101114111102115101110116101110991011151161149711011510897116101100981211019799104
7784 115121115116101109 401041011141019710211610111444 1121011141021111141099711099101 102111114 8297110107
654146 84104101 109105100100108101 103114111117112 111102 9897114115 1051101001059997116101115116104101 11497116105111
111102 116104101 1101171099810111411110211510111011610111099101115 11497110107101100971156511111466116111
1161041011161111169710811011710998101114111102115101110116101110991011151161149711011510897116101100981211019799104
7784 115121115116101109 401041011141019710211610111444 1121011141021111141099711099101 102111114 8297110107
6543664146 84104101 1141051031041164510497110100 103114111117112 111102 9897114115 1051101001059997116101115
116104101 11497116105111 111102 116104101 11011710998101114 111102 11510111011610111099101115 11497110107101100 97115
6544 6644 111114 67 116111 116104101 11611111697108 11011710998101114 111102 11510111011610111099101115
1161149711011510897116101100 98121 1019799104 7784 115121115116101109 401041011141019710211610111444 11210111445
1021111141099711099101 102111114 8297110107 65436643674146 84104101 981089799107 9897114115
1051101001059997116101 116104101 1121011141021111141099711099101 111102 116104101 10510010197108 11510110810199116105111110
11512111511610110946 65115 70105103117114101115 50 97110100 51 11510411111944 116104101 11210111410211111445
1099711099101 111102 116104101 10510010197108 744569 97110100 694574 11510110810199116105111110 115121115116101109
105115109117991049810111611610111411610497110116104971161111021019799104991111091121111101011101167784
11512111511610110946
6711111011810111011610511111097108 971121121141119799104101115 116111 116104101 11510110810199116105111110
11211411198108101109 10511099108117100101 109101116104111100115 406797108108105115111110456611711499104 97110100
7010811111711411011112144 5048484959 7597107105 101116 971084644 4957575741 11610497116 9711711611145
109971161059997108108121 11510110810199116 116104101 111117116112117116 97115115105103110101100 116104101 104105103104101115116
112114111989798105108105116121 804011641 401041011141019710211610111444 767745115991111141014144 97999911111410045
105110103 116111 97 1089711010311797103101 109111100101108 40767741 102111114 116104101 1161149711011545
10897116105111110 11697114103101116 108971101031179710310146 65115 97 11211410110810510910511097114121 10112045
11210111410510910111011644 116104101 97117116104111114115 97112112108105101100 116104105115 76774511599111114101 116111
11510110810199116105110103 116104101 98101115116 97109111110103 116104101 111117116112117116115 102114111109 116104101
116104114101101 744569 7784 11512111511610110911546 73110 111114100101114 116111 10997107101 97 9911110945
1129711410511511111044 116104101 97117116104111114115 97108115111 117115101100 97 11599111114101 9897115101100 111110
97 1161149711011510897116105111110 109111100101108 40847741 9997108108101100 73667752 4066114111119110
101116 971084644 4957575141 401041011141019710211610111444 8477451159911111410141 97110100 97 11599111114101
9897115101100 111110 116104101 11211411110011799116 111102 116104101 84774511599111114101 97110100 116104101
76774511599111114101 401041011141019710211610111444 847737677451159911111410141 116111 11510110810199116
116104101 98101115116 11111711611211711646 849798108101 49 115104111119115 116104101 114101115117108116115 111102
116104105115 11211410110810510910511097114121 10112011210111410510910111011646 84104101 1311197116105110103 11011710945
98101114 1051101001059997116101115 116104101 1001051110111410111099101 98101116119101101110 116104101 11210111445
1021111141099711099101 102111114 8297110107 65 111102 1019799104 11510110810199116105111110 115121115116101109
97110100 11610497116 111102 6851 40116104101 98101115116 7784 11512111511610110944 105461014644 119105116104
116104101 104105103104101115116 1121011141021111141099711099101 102111114 8297110107 654146 84104101 767745
11599111114101 97110100 84774511599111114101 9897115101100 11510110810199116105111110115 100105100 110111116
0
10
20
30
40
50
60
70
80
90
100
A A+B A+B+C
Performance (%)
Ideal Selection System TDMT HPAT SMT
701051031171141015158 801011141021111141099711099101111102116104101105100101971081151011081019911610511111011512111545
116101109 102111114 691101031081051151044511611145749711297110101115101 7784 11111711611211711611546
849798108101 4958 681051110111410111099101 105110 1121011141021111141099711099101 102111114 8297110107 65
98101116119101101110 1019799104 11510110810199116105111110 115121115116101109 97110100 685146
8399111114105110103 109101116104111100 847737677 8477 7677
681051110111410111099101
105110 1121011141021111141099711099101 524649 45494653 45484653
9811111111511647105109112114111118101 116104101 1121011141021111141099711099101 102111114 8297110107 6544
11910410111410197115 116104101 8477376774511599111114101 10010510046 84104101 11211410110810510910545
11097114121 101120112101114105109101110116 9711211210197114115 116111 1051101001059997116101 11610497116 116104101
8477376774511599111114101 119111114107115 98101116116101114 11610497110 116104101 76774511599111114101
105110 11510110810199116105110103 116104101 98101115116 11111711611211711646
65115 9997110 98101 10197115105108121 10311710111511510110044 116104101 11599111114101115 102114111109 1089711045
10311797103101 10911110010110844 1161149711011510897116105111110 10911110010110844 111114 98111116104 109111100101108115
9911110998105110101100 10497115 116119111 1121141119810810110911546 84104101 12114115116 11211411198108101109
105115 11610497116 847737677451159911111410144 8477451159911111410144 97110100 76774511599111114101
97114101 115116971161051151161059997108 11897114105979810810111546 69118101110 105110 116104101 9997115101 11610497116
8477 97110100 767797114101 11611497105110101100 111110 97 99111114112117115111102 116104101 11597109101
11510512210144 9910497110103105110103 116104101 11611497105110105110103 99111114112117115 97108115111 9910497110103101115
116104101 8477451159911111410144 116104101 7677451159911111410144 97110100 116104101 84773767745
1159911111410146 70105103117114101 52 115104111119115 116104105115 11210410111011110910111011111046 73110 116104101
1210311711410144 8482784944 8482785044 97110100 84827851 9911111411410111511211111010044
1141011151121019911610511810110812144 116111 40504144 40514144 97110100 405241 9810110811111944 11910410599104
97114101 1161149711011510897116105111110115 111102 404941 98121 10010511101114101110116 744569 7784 11512111545
11610110911546
404941 30
734511511798106
751111101121171169745110111
9911110911211711610111445111102
115104105115117116101109117
115121115116101109
10111010610511010597
101110103105110101101114
100101115117
97109
927339109 97 99111109112117116101114 1011101031051101011011144634
405041 7339109 97 99111109112117116101114 115121115116101109115 10111010310511010110111446 408482784941
405141 7339109 97 99111109112117116101114 115971081011151099711046 408482785041
405241 7311639115 9911110911211711610111446 408482785141
7677 97110100 8477 119101114101 11611497105110101100 105110 116101110 11997121115 111110 1161149710511045
105110103 11510111611544 11910410599104 119101114101 11511798115101116115 111102 658482 981141119710045
9911111810111497103101 9810510810511010311797108 989711510599 101120112114101115115105111110 40666941 99111114112117115
4084971071011229711997 101116 971084644 504848504144 979999111114100105110103 116111 11610111045102111108100
99114111115115118971081051009711610511111040771051169910410110810844 495757554146 7011111410197991041054010561
-35
-30
-25
-20
-15
-10
-5
0
12345678910
Pairs of LMs and TMs
Logarithmic scaled score
TRN1
TRN2
TRN3
70105103117114101 5258 84104101 11597109101 1161149711011510897116105111110 9997110 98101 11599111114101100
10010511101114101110116108121 98121847737677451159911111410146
849798108101 5058 84104101 50250 99111110102117115105111110 1099711611410599101115 102111114 116104114101101 744569 7784 11512111511610110911558 744569 8468778444 6851 97110100 744569 83778458
699799104 10997116114105120 115104111119115 97103114101101109101110116 97110100 10010511597103114101101109101110116 98101116119101101110 116104101 10510010197108 11510110810199116105111110 97110100 116104101 11599111114101459897115101100
11510110810199116105111110 117115105110103 116104101 847737677451159911111410146 49 97110100 48 1051101001059997116101115 921151011081019911610110034 97110100 92110111116 115101108101991161011003444 1141011151121019911610511810110812146
847737677458399111114101456697115101100 8310110810199116105111110
7310010197108 84687784 6851 837784
8310110810199116105111110 49 48 49 48 49 48
49 50504655 52524653 51554651 50514655 51494652 534653
48 494648 51494656 554649 51504648 51544655 50544653
4944 5044 514144 848278105105115 11599111114101100105110 116101110119971211159812184773767745
1159911111410146 83111109101 8477376774511599111114101115 1121089799101 848278105 40105 61 4944
5044 5141 105110 97 10010511101114101110116 11111410010111446 69118101110 105102 97 104117103101 99111114112117115
105115 11211410111297114101100 116111 11611497105110 97 103111111100 8477 97110100 767744 116104105115
112104101110111109101110111110 1141011099710511011546
73110 111114100101114 116111 115111108118101116104105115 12114115116 1121141119810810110944 116104105115 11297112101114
112114111112111115101 97 11511697116105115116105999710845116101115116459897115101100 11510110810199116105111110 11512111545
11610110946 7210111410144 116104101 115116971161051151161059997108 116101115116 117115101100 105115 97 10911710811610545
112108101991111091129711410511511111011610111511698971151011001111101161041017511411711510797108458797108108105115
116101115116 40721119910498101114103 97110100 84971091049711010144 495756514146 84104101 11211411145
112111115101100 109101116104111100 9910410199107115 119104101116104101114 116104101 104105103104101115116 11599111114101
105115 115105103110105129997110116108121 10010511101114101110116 102114111109 116104101 11111610410111411546
84104101 11510199111110100 11211411198108101109 105115 11610497116 116104101 1161149711011510897116105111110115
119105116104 116104101 104105103104101115116 8477376774511599111114101 116101110100 116111 10010511101114
102114111109 116104111115101 11497110107101100 104105103104101115116 98121 10411710997110 101118971081179745
11611111411546 849798108101 50 115104111119115 116104105115 11210410111011110910111011111046 84104101
849798108101 99111110115105115116115 111102 116104114101101 50250 99111110102117115105111110 1099711611410545
99101115 102111114 116104114101101 744569 7784 11512111511610110911558 744569 8468778444 685144
97110100 744569 83778446 699799104 10997116114105120 115104111119115 97103114101101109101110116
97110100 10010511597103114101101109101110116 98101116119101101110 116104101 10510010197108 11510110810199116105111110
98121 97 10411710997110 1011189710811797116111114 97110100 116104101 11510110810199116105111110 98121116104101
847737677451159911111410146 8410410140494449414510110810110910111011697110100116104101404844484145
101108101109101110116 1051101001059997116101 116104101 1121011149910111011697103101 111102 9710311410110110910111011644
97110100 116104101 404944484145101108101109101110116 97110100 116104101 40484449414510110810110910111011610511045
1001059997116101 116104101 1121011149910111011697103101 111102 1001051159710311410110110910111011646 73110 116104101
99111110102117115105111110 10997116114105120 102111114 744569 83778444 116104101 11011710998101114 105110
116104101404944484145101108101109101110116105115108971141031011141161049711011610497116105110116104101404844494145
10110810110910111011646 84104105115 10910197110115 11610497116 116104101 8477376774511599111114101
116101110100115 116111 103105118101 116104101 104105103104101115116 11599111114101 116111 116104101 116114971101151089745
116105111110 102114111109 744569 837784 119104101110 105116 105115 110111116 116104101 116114971101151089745
116105111110 97115115105103110101100 116104101 98101115116 1149711010746 79110 116104101 111116104101114 1049711010044
105110 116104101 99111110102117115105111110 1099711611410599101115 102111114 744569 84687784 97110100
685144 116104101 11011710998101114 105110 116104101 404844494145101108101109101110116 105115 10897114103101114
11610497110 11610497116 105110 116104101 40494448414510110810110910111011646 84104105115 10910197110115 11610497116
116104101 8477376774511599111114101 116101110100115 110111116 116111 103105118101 116104101 104105103104101115116
11599111114101 116111 116104101 1161149711011510897116105111110 102114111109 116104101 7784 11512111511610110911544
10112099101112116 102111114 744569 83778444 101118101110 105102 11610497116 1161149711011510897116105111110 105115
97115115105103110101100 116104101 98101115116 1149711010746
84111115111108118101 116104105115 11510199111110100 1121141119810810110944 116104105115 11297112101114 11211411145
112111115101115 97 11510110810199116105111110 115121115116101109 9897115101100 111110 116104101 9911111010010545
11610511111097108 112114111989798105108105116121 11610497116 97 1161149711011510897116105111110 105115 110111116 10511045
102101114105111114 116111 116104101 111116104101114 1161149711011510897116105111110115 119104101110 116104101 1161149711011545
10897116105111110 10111099111100101100 98121 117115105110103 116104101 847737677451159911111410144
116104101 8477451159911111410144 111114 116104101 7677451159911111410144 1159711610511512101115 115111109101
9911111010010511610511111011546 70111114 1019799104 7784 11512111511610110944 116104101 9911111010010545
11610511111097108 112114111989798105108105116121 105115 10810197114110101100 97115 97 114101103114101115115105111110 116114101101
40671049710998101114115 97110100 729711511610510144 4957575059 6611410110510997110 101116 971084644
4957565241 102114111109 116104101 118101991161111144510111099111100105110103 111102 116104101 116114971101151089745
116105111110115 1089798101108101100 97115 92110111116 10511010210111410511111434 111114 921051101021011141051111143446
84104101 110101120116 11510199116105111110 112114101115101110116115 111117114 116119111 112114111112111115101100
10910111610411110011546 6912011210111410510910111011697108 114101115117108116115 97114101 115104111119110 97110100
10010511599117115115101100 105110 8310199116105111110 5146 701051109710810812144 111117114 9911111099108117115105111110115
97114101 112114101115101110116101100 105110 8310199116105111110 5246
50 8011411111211111510110077101116104111100
504649 80114111112111115101100 77101116104111100 404941
841111151111081181011161041011211411511611211411198108101109100101115991141059810110010511083101991161051111104944
116104105115 831179811510199116105111110 112114111112111115101115 97 109101116104111100 11610497116 11510110810199116115
116104101 1161149711011510897116105111110 979999111114100105110103 116111 119104101116104101114 116104101 11599111114101115
111102 111117116112117116115 102114111109 1019799104 7784 115121115116101109 115105103110105129997110116108121
10010511101114 102114111109 1019799104 11111610410111446
73110 111114100101114 116111 10010111610199116 97 115105103110105129997110116 100105111011141011109910144
116104101 112114111112111115101100 109101116104111100 12114115116 11211410111297114101115 109117108116105112108101
11511798115101116115 111102 116104101 102117108108 1129711497108108101108 99111114112117115 979999111114100105110103
116111 10745102111108100 99114111115115 1189710810510097116105111110 40771051169910410110810844 4957575541 97110100
11611497105110115 98111116104 8477 97110100 7677 111110 1019799104 1151179811510111646 70111114
1011209710911210810144 116104101 102117108108 1129711497108108101108 99111114112117115 67 105115 10010545
118105100101100 105110116111 116101110 11511798115101116115 86
105
40105 61 4859 4959 11159 574146
70111114 1019799104 105 40105 61 4859 4959 11159 574144 116104101 112114111112111115101100
109101116104111100 11611497105110115 97 1161149711011510897116105111110 109111100101108 8477
105
111110
67
105
4061 67 0 86
105
41 97110100 97 1089711010311797103101 109111100101108 7677
105
111110
116104101 11697114103101116451089711010311797103101 11297114116 111102 67
105
4070105103117114101 534146
721011141019710211610111444 108101116 76
105
4011641 100101110111116101 841141054510311497109 1151169711610511511610599115
111102971161149711011510897116105111110116981211171151051101037677
105
46 6510811511144 10810111684
105
401155911641
100101110111116101
80
975083
804011559971061164144 119104101114101 115 105115 97 1161149711011510897116105111110
11511111711499101 1151011101161011109910144 116 105115 97 1161149711011510897116105111110 11697114103101116 11510111045
1161011109910144 97110100 83 105115 116104101 97108105103110109101110116 115101116
50
4066114111119110 101116
50
78111116101 11610497116 116104101 10010112110105116105111110 111102 83 105115 9910497110103101100 100101112101110100105110103
C1 C9
C
10-fold Cross Validation
…..
TM1LM1 TM9LM9
…..
C0
TM0LM0
Parallel corpus
70105103117114101 5358 84104101 109101116104111100 116111 11611497105110 109117108116105112108101 11297105114115 111102
7677 97110100 847746
971084644 4957575141 11610497116 10511099108117100101115 116104101 98101115116 9710810510311010910111011644 116104101
11010110510310498111114105110103 9710810510311010910111011611544 97110100 116104101 112101103103101100 9710810510311045
10910111011611546
84104101 112114111112111115101100 109101116104111100 11599111114101115 1019799104 111117116112117116
105110 107 1199712111546 73110 116104101 1011209710911210810144 97115 115104111119110 105110
70105103117114101 5444 116104101 112114111112111115101100 109101116104111100 11599111114101115 1161149711011545
10897116105111110 116
97
102114111109 1161149711011510897116105111110 115121115116101109 7784
97
105110
116101110 1199712111544 76
48
40116
97
4159 76
49
40116
97
4159 11159 76
57
40116
97
4144 119104101110
1161149711011510897116105111110115 97114101 11599111114101100 98121 117115105110103 97 1089711045
10311797103101 109111100101108 111102 116104101 1161149711011510897116105111110 11697114103101116 1089711045
10311797103101 4070105103117114101 544146 79110 116104101 111116104101114 1049711010044
116104101 112114111112111115101100 109101116104111100 11599111114101115 1161149711011510897116105111110 116
97
105110 116101110 1199712111544 84
48
4011559116
97
4159 84
49
4011559116
97
415911159 84
57
4011559116
97
4144
119104101110 116104101 1161149711011510897116105111110115 97114101 11599111114101100 98121 11711545
105110103 97 1161149711011510897116105111110 10911110010110846 871041011141019711544 116104101
112114111112111115101100 109101116104111100 11599111114101115 1161149711011510897116105111110 116
97
105110
116101110 1199712111544 84
48
4011559116
97
413 76
48
40116
97
4159 84
49
4011559116
97
413 76
49
40116
97
4159
11159 84
57
4011559116
97
413 76
57
40116
97
4144119104101110116104101116114971101151089711610511111011597114101
11599111114101100 98121 117115105110103 116104101 11211411110011799116115 111102 116104101 11599111114101115 111102 97
1089711010311797103101 109111100101108 97110100 97 1161149711011510897116105111110 10911110010110846
84104101110 116104101 112114111112111115101100 109101116104111100 9911110911297114101115
116104101 10910197110115 111102 116104101 1159911111410111546 73110 116104101 10112097109112108101
4070105103117114101 544144
80
57
1056148
76
105
40116
97
4161494844
80
57
1056148
76
105
40116
98
4161494844
97110100
80
57
1056148
76
105
40116
99
41614948 97114101 9911110911297114101100 119104101110
116104101 1161149711011510897116105111110115 97114101 11599111114101100 98121 117115105110103 97
1089711010311797103101 109111100101108 111102 116104101 1161149711011510897116105111110 1169711410310111646
80
57
1056148
84
105
4011559116
97
4161494844
80
57
1056148
84
105
4011559116
98
4161494844 97110100
80
57
1056148
84
105
4011559116
99
41614948 97114101 9911110911297114101100 119104101110 116104101
1161149711011510897116105111110115 97114101 11599111114101100 98121 117115105110103 97 1161149711011545
10897116105111110 10911110010110846
80
57
1056148
84
105
4011559116
97
413 76
105
40116
97
4161494844
80
57
1056148
84
105
4011559116
98
413 76
105
40116
98
4161494844 97110100
80
57
1056148
84
105
4011559116
99
413 76
105
40116
99
41614948 97114101 9911110911297114101100 119104101110
116104101 1161149711011510897116105111110115 97114101 11599111114101100 98121 117115105110103 116104101 11211411110045
11799116115 111102 116104101 11599111114101115 111102 97 1089711010311797103101 109111100101108 97110100 97
1161149711011510897116105111110 10911110010110846
84104101 112114111112111115101100 109101116104111100 9910410199107115 119104101116104101114 116104101
104105103104101115116 10910197110 105115 115105103110105129997110116108121 10010511101114101110116 102114111109 116104101
111110 116104101 84774511611497105110105110103 97108103111114105116104109115 11711510110046
(s,ta) LM0
LM1
LM9
TM0
TM1
TM9
TM0, LM0
TM1, LM1
TM9, LM9
(s,tb)
(s,tc)
…
…
…
(s,ta)
(s,tb)
(s,tc)
(s,ta)
(s,tb)
(s,tc)
(L0(ta), L1(ta),…, L9(ta))
(L0(tb), L1(tb),…, L9(tb))
(L0(tc), L1(tc),…, L9(tc))
(T0(s,ta), T1(s,ta),…, T9(s,ta))
(T0(s,tb), T1(s,tb),…, T9(s,tb))
(T0(s,tc), T1(s,tc),…, T9(s,tc))
(T0(s,ta)*L0(ta),…, T0(s,ta)*L9(ta))
(T0(s,tb)*L0(tb),…, T0(s,tb)*L9(tb))
(T0(s,tc)*L0(tc),…, T0(s,tc)*L9(tc))
#
#
#
ta: a translation of s by MTa L9(ta) : probability of ta on LM9
T9(ta) : probability of ta on TM9
Q:Which pair of averages of scores significantly different?
A:Apply Multiple Comparison Test based on Kruskal-Wallis Test (#)
70105103117114101 5458 801141011129711497116105111110 102111114 109117108116105112108101 9911110911297114105115111110
116101115116 9897115101100 111110 7511411711510797108458797108108105115 11610111511646
11111610410111411598121117115105110103971091171081161051121081019911110911297114105115111110116101115116
51
119105116104
116104101 7511411711510797108458797108108105115 116101115116
52
44 11910410599104 105115 107110111119110 97115 97
8411710710112145751149710910111445116121112101 109111100105129997116105111110 111102 116104101 68117110110
116101115116 40721119910498101114103 97110100 84971091049711010144 495756514146 73102 116104101
104105103104101115116 10910197110 105115 115105103110105129997110116108121 1001051110111410111011644 116104101 11211411145
112111115101100 109101116104111100 11510110810199116115 116104101 1161149711011510897116105111110 119105116104 116104101
104105103104101115116 1159911111410146 73102 11011111644 116104101 112114111112111115101100 109101116104111100 11510145
10810199116115 102114111109 97109111110103 116104101 1161149711011510897116105111110115 119104111115101 11599111114101115
97114101 110111116 115105103110105129997110116108121 10010511101114101110116 102114111109 116104101 104105103104101115116
11599111114101 116104101 1161149711011510897116105111110 102114111109 116104101 7784 119104111115101 11210111445
1021111141099711099101 105115 116104101 9810111511646
504650 80114111112111115101100 77101116104111100 405041
84111 115111108118101 116104101 11510199111110100 11211411198108101109 1001011159911410598101100 105110 831019945
116105111110 4944 116104105115 831179811510199116105111110 112114111112111115101115 97 11510110810199116105111110 11512111545
116101109 9897115101100 111110 116104101 9911111010010511610511111097108 112114111989798105108105116121 11610497116
97 1161149711011510897116105111110 105115 110111116 105110102101114105111114 116111 111116104101114 116114971101151089745
116105111110115 119104101110 116104101 1161149711011510897116105111110 10111099111100101100 98121 117115105110103 116104101
847737677451159911111410144 116104101 8477451159911111410144 111114 116104101 76774511599111114101
1159711610511512101115 115111109101 9911111010010511610511111011546 70111114 1019799104 7784 11512111511610110944
116104101 9911111010010511610511111097108 112114111989798105108105116121 105115 10810197114110101100 97115 97 11410145
103114101115115105111110 116114101101 102114111109 116104101 118101991161111144510111099111100105110103 111102 116104101
1161149711011510897116105111110115 1089798101108101100 97115 92110111116 10511010210111410511111434 111114 9210511010210145
1141051111143444 11910410599104 105115 116104101 99114105116101114105111110 11897114105979810810146
73110 111114100101114 116111 10810197114110 116104101 9911111010010511610511111097108 112114111989798105108105116121
109101110116105111110101100 979811111810144 1161149711011510897116105111110115 102114111109 116104101 9911110945
112111110101110116 1161149711011510897116105111110 11512111511610110911544 7784
97
44 7784
98
44 97110100
7784
99
44 97114101 11497110107101100 98121 10411710997110 1011189710811797116111114115 105110 9710045
1189711099101 4070105103117114101 554146 76101116 114
97
100101110111116101 116104101 11497110107 9711545
115105103110101100 116111 1161149711011510897116105111110 116
97
102114111109 7784 115121115116101109 7784
97
46
6510811511144 108101116114
98101115116
100101110111116101116104101981011151161149711010797109111110103114
97
44 114
98
44
51
73116 105115 119101108108 107110111119110 11610497116 11410111210197116105110103 97 115105109112108101 11645116101115116 10911710845
116105112108101 116105109101115 1051109911410197115101115 116104101 991049711099101 111102 1051109911111411410199116108121 12110100105110103 97
115105103110105129997110116 100105111011141011109910146 77117108116105112108101 9911110911297114105115111110 105115 100101115105103110101100
116111 97118111105100 11511799104 97 11210410111011110910111011111046
52
84104101 7511411711510797108458797108108105115 116101115116 105115 97 11011111045112971149710910111611410599 1111101014511997121
6511097108121115105115 111102 86971141059711099101 4065787986654146 84104105115 116101115116 100111101115 110111116 9711545
115117109101 116104101 1009711697 1001051151161141059811711610511111046
(s,ta,ra)
(s,tb,rb)
(s,tc,rc)
rbest= the best rank among ra,rb, and rc
Ri(ti) =
1 if ri=rbest
0 others
(s,ta,ra)
(s,tb,rb)
(s,tc,rc)
ra is the rank assigned 
to ta by human evaluator
(T(s,ta)* L(ta), T(s,ta), L(ta), Ra(ta))
(T(s,tb)* L(tb), T(s,tb), L(tb), Rb(tb))
(T(s,tc)* L(tc), T(s,tc), L(tc), Rc(tc))
TM, LM
TM
LM
R
{(T(s,ta)* L(ta), T(s,ta), L(ta), R(ta)),…}
{(T(s,tb)* L(tb), T(s,tb), L(tb), R(tb)),…}
{(T(s,tc)* L(tc), T(s,tc), L(tc), R(tc)),…}
RTa
RTb
RTc
RT Learner 
with pruning or shrinking
RT Learner 
with pruning or shrinking
RT Learner 
with pruning or shrinking
C
TMLM
70105103117114101 5558 801141011129711497116105111110 102111114 114101103114101115115105111110 116114101101 1081019711411045
10511010346
97110100 114
99
46 76101116 82
105
40116
105
41 98101 10010112110101100 97115 10211110810811111911558 82
105
40116
105
41
105115 10111311797108 116111 49 105102 114
105
61 114
98101115116
59 111116104101114119105115101 82
105
40116
105
41 105115
10111311797108 116111 4846 84104101114101102111114101 119104101110 82
97
40116
97
41 105115 10111311797108 116111 4944
116104101 116114971101159799116105111110 116
97
105115 115117112101114105111114 116111 111114 97115 103111111100 97115
116104101 111116104101114 116114971101151089711610511111011546
84104101 112114111112111115101100 109101116104111100 11611497105110115 97 1161149711011510897116105111110
109111100101108 8477 111110 116104101 102117108108 1129711497108108101108 99111114112117115 67 97110100 97
1089711010311797103101 109111100101108 7677 111110 116104101 1161149711011510897116105111110451169711410310111645
1089711010311797103101 11297114116 111102 67 4070105103117114101 554146
721011141019710211610111444 108101116 764011641 100101110111116101 841141054510311497109 1151169711610511511610599115
111102 97 1161149711011510897116105111110 116 98121 117115105110103 767746 6510811511144 108101116 84401155911641
100101110111116101
80
975083
804011559971061164144 119104101114101 115 105115 97 1161149711011510897116105111110
11511111711499101 1151011101161011109910144 116 105115 97 1161149711011510897116105111110 11697114103101116 11510111045
1161011109910144 97110100 83 105115 116104101 97108105103110109101110116 115101116 4066114111119110 101116 971084644
495757514146
7810112011644 116104101 112114111112111115101100 109101116104111100 10111099111100101115 116104114101101
11810199116111114115 4011559 116
97
59114
97
4144 4011559 116
98
59 114
98
4144 97110100 4011559 116
99
59 114
99
41
4070105103117114101 5541 105110116111 116104114101101 115991111141014511810199116111114115 119105116104
11011111045105110102101114105111114105116121 111114 10511010210111410511111410511612144 1141011151121019911610511810110812158
40844011559116
97
413 7640116
97
4159 844011559116
97
4159 7640116
97
415982
97
40116
97
414144
40844011559116
98
413 7640116
98
4159 844011559116
98
4159 7640116
98
415982
97
40116
98
414144 97110100
40844011559116
99
413 7640116
99
4159 844011559116
99
4159 7640116
99
415982
99
40116
99
414146
70111114 1019799104 7784
105
40105 61 9759 9859 111114 994144
116104101 112114111112111115101100 109101116104111100 10810197114110115 102114111109
10240844011559116
105
413 7640116
105
4159 8440116
105
591154159 7640116
105
4159 82
97
40116
105
4141 1154611646
116
105
105115 97 1161149711011510897116105111110 111102 115 98121 7784 115121115116101109 7784
105
103
116104101 9911111010010511610511111097108 11211411198979810510810511612144 11910410599104 105115 101120112114101115115101100
98121 116104101 114101103114101115115105111110 116114101101 40671049710998101114115 97110100 729711511610510144
4957575059 6611410110510997110 101116 971084644 4957565241 8284
105
4070105103117114101 554146
82101103114101115115105111110 116114101101 40828441 10810197114110101114 105115 107110111119110 97115 1141019911711445
115105118101 9810511097114121 1129711411610511610511111010511010346 73110 103114111119105110103 97 11611410110144 97110
8284 10810197114110101114 11410199117114115105118101108121 115112108105116115 116104101 11611497105110105110103 1009711697
105110 1019799104 110111100101 115111 97115 116111 11410110011799101 118971141059711099101 119105116104105110
11297114116105116105111110115 97115 10911799104 97115 1121111151151059810810146
73110 1031011101011149710844 116104101 10810197114110101100 82841111181011144512116115 116104101 1161149710511045
105110103 100971169746 65115 112111115116451121141119910111511510511010344 116104101 10810197114110101100 8284
105115 11510510911210810512101100 98121 117115105110103 116119111 1121141119910110011711410111558 11211411711045
105110103 97110100 115104114105110107105110103 40671049710998101114115 97110100 729711511610510144 4957575059
6611410110510997110 101116 971084644 495756524146 80114117110105110103 1151179999101115115105118101108121
115110105112115 11111 116104101 10810197115116 10510911211111411697110116 11511210810511611546 84104101 7310945
1121111141169711099101 111102 97 114111111116101100 11511798116114101101 105115 100101116101114109105110101100 98121
1161041019911111511645991111091121081011201051161211091019711511711410144 68
107
4084
48
4161684084
48
4143
107 3 1151051221014084
48
4159 119104101114101 684084
48
41 100101110111116101115 116104101 1001011181059711099101
111102 116104101 11511798116114101101 84
48
44 1151051221014084
48
41 105115 116104101 11011710998101114 111102 11610111445
10910511097108 110111100101115 111102 84
48
44 97110100 107 105115 97 991111151164599111109112108101120105116121
112971149710910111610111446 83104114105110107105110103 11410110011799101115 116104101 11011710945
(s,ta)
(s,tb)
(s,tc)
(T(s,ta)* L(ta), T(s,ta), L(ta))
(T(s,tb)* L(tb), T(s,tb), L(tb))
(T(s,tc)* L(tc), T(s,tc), L(tc))
TM, LM
TM
LM
RTa
RTb
RTc
ua
ub
uc
#
Q: Which translation is selected? 
A: The translation with the maximal score (#)
70105103117114101 5658 84104101 109101116104111100 116111 11510110810199116 97 1161149711011510897116105111110 98121
117115105110103 10810197114110101100 114101103114101115115105111110 11611410110146
98101114 111102 1011110199116105118101 110111100101115 98121 115104114105110107105110103 116104101 12116116101100
11897108117101 111102 1019799104 110111100101 11611111997114100115 105116115 11297114101110116 11011110010146
83104114117110107101110 12116116101100 1189710811710111544 102111114 97 115104114105110107105110103 112971149710910145
116101114 10744 97114101 99111109112117116101100 979999111114100105110103 116111 116104101 1141019911711411510511111044
941214011011110010141 61 107 3 12612140110111100101414340490 10741 3 9412140112971141011101164159
119104101114101 1261214011011110010141 100101110111116101115 116104101 11711511797108 12116116101100 11897108117101 102111114
97 11011110010144 94121401129711410111011641 105115 116104101 115104114117110107101110 12116116101100 11897108117101
102111114 116104101 11011110010139115 1129711410111011644 97110100 107 105115 97 115104114105110107105110103 1129745
11497109101116101114 11511799104 11610497116 48 60 107 60 4946 84104101 112971149710945
101116101114 107 105110 1019799104 111102 116104101 116119111 11211411199101100117114101115 105115 12120101100
115111 97115 116111 109105110105109105122101 99114111115115451189710810510097116105111110 10111511610510997116101115 111102
116104101 100101118105971109910146 8410410111410110211111410144 97102116101114 103114111119105110103 1019799104
8284
105
40105 61 9759 9859 111114 994144 116104101 112114111112111115101100 109101116104111100 11210111445
102111114109115 111110101 111102 116104101 11510510911210810512101100 11211411199101100117114101115 111102 11211411711045
105110103 97110100 11510411410511010710511010346
73110 116104101 11510110810199116105111110 1121049711510144 116104101 112114111112111115101100 109101116104111100
10111099111100101115 116104114101101 11297105114 111102 97 11511111711499101 11510111011610111099101 97110100 105116115
116114971101151089711610511111044 4011559116
97
4144 4011559116
98
4144 97110100 4011559116
99
41 105110116111 116104114101101
11810199116111114115 40844011559116
97
413 7640116
97
4159 844011559116
97
4159 7640116
97
414144
40844011559116
98
413 7640116
98
4159 844011559116
98
4159 7640116
98
414144 97110100
40844011559116
99
413 7640116
99
4159 844011559116
99
4159 7640116
99
414144 11410111511210199116105118101108121
4070105103117114101 564146
84104101 112114111112111115101100 109101116104111100 11211410110010599116115 116104101 9911111010010545
11610511111097108 112114111989798105108105116121 11610497116 1019799104 116
105
40105619744 9844 111114 9941 105115
110111116 105110102101114105111114 116111 116104101 111116104101114115 98121 117115105110103 8284
105
97110100 11510145
10810199116115
53
116104101 1161149711011510897116105111110 119105116104 116104101 104105103104101115116 9911111010010545
11610511111097108 11211411198979810510810511612146
51 69120112101114105109101110116971086711110911297114105115111110
514649 6912011210111410510910111011697108 77101116104111100
84104101 97117116104111114115 1011189710811797116101100 116104101 112114111112111115101100 109101116104111100115 105110
111114100101114 116111 97110115119101114 116104101 102111108108111119105110103 11311710111511610511111058 8710410599104
11510110810199116105111110 115121115116101109 105109112114111118101115 1121011141021111141099711099101 98101115116 105110
9911110911297114105115111110119105116104116104971161111021161041019810111511677841151211151161011091054610146
116104101 7784 115121115116101109115 119105116104 116104101 104105103104101115116 1121011141021111141099711099101
97115 115104111119110 105110 70105103117114101115 50 97110100 5163
73110111114100101114116111971101151191011141161041019798111118101113117101115116105111110441161041019711745
1161041111141151171151011009711510111611110211610411410110174456999111109112111110101110116778411512111545
116101109115 408468778444 685144 97110100 83778441 97110100 97 115101116 111102 116104114101101
694574 99111109112111110101110116 7784 115121115116101109115 408468778444 7280658444
971101008377844146661051081051101031179710869110103108105115104971101007497112971101011151011009711697
119101114101 102114111109 658482 9811411197100459911111810111497103101 9810510810511010311797108 989711510599
101120112114101115115105111110 40666941 99111114112117115 4084971071011229711997 101116 971084644 504848504144
11910410599104 105115 115112108105116 105110116111 116104114101101 1129711411611558 97 11611497105110105110103 115101116
111102 49505344535155 11510111011610111099101 1129710511411544 97 118101114105129997116105111110 115101116 111102
5744565550 1129710511411544 97110100 97 116101115116 115101116 111102 494844485051 1129710511411546
84104101 102117108108 99111114112117115 67 105110 11611497105110105110103 1161149711011510897116105111110 1169711445
1031011161089711010311797103101109111100101108971101001161149711011510897116105111110109111100101108 105115116104101
11611497105110105110103 11510111646 84101110 11511798115101116115 111102 116104101 102117108108 99111114112117115 119101114101
53
8011410111297114105110103 109117108116105112108101 8284115 102111114 1019799104 99111109112111110101110116 7784 11512111545
116101109 1011109798108101115 116104101 11510199111110100 109101116104111100 116111 98101 101120116101110100101100 115111 97115 116111
11510110810199116 116104101 98101115116 111117116112117116 979999111114100105110103 116111 116104101 109117108116105112108101 991111091129711445
10511511111046
117115101100 102111114 116104101 12114115116 112114111112111115101100 10910111610411110046 84104101 1161149711011545
10897116105111110 109111100101108 97110100 1089711010311797103101 109111100101108 97114101 1081019711411010110098121
117115105110103 717390654343 407999104 97110100 7810112144 5048484841 97110100 116104101
67778545679710998114105100103101 84111111108107105116 406710897114107115111110 97110100 8211111510111045
10210110810044 495757554144 1141011151121019911610511810110812146 84104101 1161149711011510897116105111110 109111100101108
105115 10810197114110101100 102114111109 736677 49 116111 5244 10511099108117100105110103 116104101 727777
10911110010110844 97115 115117103103101115116101100 98121 7999104 97110100 78101121 40504848484144
97110100 105116115 11611497105110105110103 108111111112 11997115 11610111410910511097116101100 119104101110 116104101
112101114112108101120105116121 102111114 116104101 1189710810510097116105111110 115101116 1051101001059997116101100 116104101
108111119101115116 1159911111410111546 84104101 119111114100 9910897115115101115 117115101100 105110 8477
10810197114110105110103 97114101 116104101 11297114116451111024511511210110199104 4080798341 9910897115115101115 105110
8468778446 84104101 804511897108117101 117115101100 102111114 116104101 109117108116105112108101 9911110945
11297114105115111110 116101115116 105115 4846485346
70111117114 115101116115 111102 9798111117116 12118101 104117110100114101100 11297105114115 111102 6911045
103108105115104 97110100 749711297110101115101 11510111011610111099101115 119101114101 11497110100111109108121 11510145
10810199116101100 102114111109 116104101 116101115116 11510111646 84104101 69110103108105115104 11510111011610111099101115
105110 116104101 102111117114 115101116115 119101114101 1161149711011510897116101100 98121 116104101 694574 9911110945
112111110101110116 7784 115121115116101109115 97110100 11497110107101100 98121 97 11097116105118101
11511210197107101114 111102 74971129711010111510159 108105107101119105115101 116104101 749711297110101115101 11510111045
11610111099101115105110116104101102111117114115101116115119101114101116114971101151089711610110098121116104101744569
99111109112111110101110116 7784 115121115116101109115 97110100 11497110107101100 981219711097116105118101
11511210197107101114 111102 6911010310810511510446 699799104 1121011141021111141099711099101 11997115 999710845
9911710897116101100 97115 116104101 9711810111497103101 111102 116104101 1121011141021111141099711099101 111118101114
116104101 102111117114 11510111611546 73110 11297114116105991171089711444 116104101 1121011141021111141099711099101 111102
116104101 11510199111110100 112114111112111115101100 109101116104111100 105115 99971089911710897116101100 979945
99111114100105110103 116111 10211111711445102111108100 99114111115115 1189710810510097116105111110 40771051169910410110810844
495757554146
514650 6912011210111410510910111011697108 82101115117108116115
73110 111114100101114 116111 1011189710811797116101 116104101 112111105110116 109101110116105111110101100 97116
116104101 98101103105110110105110103 111102 8310199116105111110 51464944 116104101 97117116104111114115 9911110945
11297114101100 116104101 1121011141021111141099711099101 111102 1019799104 11510110810199116105111110 115121115116101109
119105116104 11610497116 111102 116104101 98101115116 7784 11512111511610110946 65115 115104111119110 105110
70105103117114101 5044 97109111110103 116104101 744569 99111109112111110101110116 7784 11512111545
11610110911544 6851 10497100 116104101 98101115116 1121011141021111141099711099101 102111114 8297110107
6544 97110100 84687784 10497100 116104101 98101115116 1121011141021111141099711099101 102111114
98111116104829711010765436640101113117971081161111111149810111611610111411610497110664197110100
8297110107 6543664367 4010111311797108 116111 111114 98101116116101114 11610497110 674146 65115
115104111119110 105110 70105103117114101 5144 97109111110103 116104101 694574 99111109112111110101110116
7784 11512111511610110911544 84687784 10497100 116104101 98101115116 1121011141021111141099711099101
102111114 8297110107 6544 8297110107 65436644 97110100 8297110107 654366436746
70105103117114101115 5744 494844 97110100 4949 115104111119 116104101 114101115117108116115 111102 116104101
991111091129711410511511111011546 84104101 1181011141161059997108 97120105115 105110 1019799104 12103117114101
115104111119115 116104101 1001051110111410111099101 105110 116104101 112101114102111114109971109910111546
699799104 9897114 99111114114101115112111110100115 116111 97 11510110810199116105111110 11512111511610110946
84104101 12114115116 116104114101101 9897114115 105110 1081011021164511611145114105103104116 111114100101114 9911111445
114101115112111110100 116111 8477376774511599111114101459897115101100 1151011081019911610511111044 847745
11599111114101459897115101100 1151011081019911610511111044 97110100 76774511599111114101459897115101100 11510145
1081019911610511111044 11910410599104119101114101 117115101100 105110 116104101 11211410110810510910511097114121 10112045
112101114105109101110116 1001011159911410598101100 105110 8310199116105111110 4946 84104101 110101120116
116104114101101 9897114115 99111114114101115112111110100 116111 116104101 12114115116 112114111112111115101100
1091011161041111009897115101100111110847737677451159911111410144 8477451159911111410144 97110100
7677451159911111410146 84104101 110101120116 116104114101101 9897114115 99111114114101115112111110100 116111
116104101 11510199111110100 112114111112111115101100 109101116104111100 105110 11910410599104 1121141011001059945
116111114 118971141059798108101115 97114101 11410111511611410599116101100 116111 847737677451159911111410144 116111
98111116104 8477376774511599111114101 97110100 8477451159911111410144 11611197108108 1159911111410111544
767738477451159911111410144 8477451159911111410144 97110100 847737677451159911111410146
73110 116104101115101 11510110810199116105111110 10910111610411110011544 116104101 114101103114101115115105111110 116114101101115
97114101 11510510911210810512101100 98121 117115105110103 116104101 115104114105110107105110103 1121141119910110011711410146
84104101 10897115116 116104114101101 9897114115 97108115111 99111114114101115112111110100 116111 116104101 1151019945
111110100 112114111112111115101100 10910111610411110044 98117116 105110 116104101115101 11510110810199116105111110
10910111610411110011544 116104101 114101103114101115115105111110 116114101101115 97114101 11510510911210810512101100 98121
117115105110103 116104101 112114117110105110103 1121141119910110011711410146 6599991171149799121 10910197110115
116104101 1121011149910111011697103101 111102 9911111411410199116108121 11510110810199116105110103 116104101 111117116112117116
97115115105103110101100 116104101 104105103104101115116 11497110107 105110 97108108 1161141059710811546
70105103117114101 57 11510411111911511610497116 116104101 12114115116 112114111112111115101100 115121115116101109
9897115101100 111110 8477376774511599111114101 9799104105101118101100 116104101 10311410197116101115116
105109112114111118101109101110116 111102 97 108105116116108101 117110100101114 543744 105110 116104101 11210111410211111445
10997110991011021111148297110107654679110116104101111116104101114104971101004411610410110112010511511645
105110103 11510110810199116105111110 115121115116101109 115105109112108121 117115105110103 116104101 76774511599111114101
4010897110103117971031011091111001011081111021161041011161149711011510897116105111110116971141031011164199111117108100
110111116 1051091121141111181019711010010111810111010010110311497100101100 1121011141021111141099711099101 102111114
8297110107 6546
701051031171141014948115104111119115116104971161161041011151019911111010011211411111211111510110011512111545
116101109 119105116104 116104101 112114117110105110103 11211411199101100117114101 9897115101100 111110 98111116104
8477376774511599111114101 97110100 84774511599111114101 4010997114107101100 8284495045
808278 105110 116104101 1031149711210441 9799104105101118101100 116104101 10311410197116101115116 10510945
112114111118101109101110116 111102 9798111117116 533744 102111114 8297110107 654366 4010111311797108
116111 111114 104105103104101114 11610497110 664146 79110 116104101 111116104101114 1049711010044 7010510345
117114101 4948 115104111119115 11610497116 116104101 101120105115116105110103 11510110810199116105111110 115121115116101109
-16
-14
-12
-10
-8
-6
-4
-2
0
2
4
6
8
10
12
14
16
18
Difference in performance (%)
LM*TM
TM
LM
LM*TM+KWMC
TM+KWMC
LM+KWMC
RT1+SHR
RT12+SHR
RT123+SHR
RT1+PRN
RT12+PRN
RT123+PRN
A A+B A+B+C Accuracy
70105103117114101 5758 681051110111410111099101 105110 1121011141021111141099711099101 98101116119101101110
1019799104 11510110810199116105111110 115121115116101109 97110100 685146
-16
-14
-12
-10
-8
-6
-4
-2
0
2
4
6
8
10
12
14
16
18
Difference in performance (%)
LM*TM
TM
LM
LM*TM+KWMC
TM+KWMC
LM+KWMC
RT1+SHR
RT12+SHR
RT123+SHR
RT1+PRN
RT12+PRN
RT123+PRN
A A+B A+B+C Accuracy
70105103117114101 494858 681051110111410111099101 105110 1121011141021111141099711099101 98101116119101101110
1019799104 11510110810199116105111110 115121115116101109 97110100 744569 8468778446
115105109112108121 117115105110103 116104101 76774511599111114101 10497100 116104101 119111114115116 11210111445
1021111141099711099101 102111114 8297110107 65436644 119105116104 97 1001011031149710097116105111110 111102
543746
701051031171141014949115104111119115116104971161161041011151019911111010011211411111211111510110011512111545
11610110911544 119105116104 101105116104101114 116104101 112114117110105110103 11211411199101100117114101 111114 116104101
115104114105110107105110103 1121141119910110011711410144 9897115101100 111110 111110108121 84773767745
1159911111410144 111110 98111116104 8477376774511599111114101 97110100 8477451159911111410144 111114
111110 97108108 11599111114101115 9799104105101118101100 97110 1051091121141111181011091011101161111029798111117116
5037 102111114 11497110107 65 105110 97108108 999711510111546 70105103117114101 4949 97108115111
1151041111191151161049711611610410111510199111110100112114111112111115101100115121115116101109119105116104116104101
11510411410511010710511010311211411199101100117114101989711510110011111097108108 115991111141011154010997114107101100
828449505145837282 105110 116104101 1031149711210441 9799104105101118101100 97110 10510911211411111810145
109101110116 111102 97 108105116116108101 109111114101 11610497110 5037 102111114 8297110107 65436646
52 6711111099108117115105111110115
841041051151129711210111497100100114101115115101100116104101991049710810810111010310511010311211411198108101109111102
97117116111109971161059997108108121 11510110810199116105110103 116104101 98101115116 97109111110103 111117116112117116115
102114111109 109117108116105112108101 7784 115121115116101109115 116111 105109112114111118101 116114971101151089745
116105111110 1131179710810511612146 84104105115 11297112101114 112114111112111115101100 116119111 10910111610411110011546
84104101 12114115116 109101116104111100 105115 9897115101100 111110 97 109117108116105112108101 9911110945
11297114105115111110 116101115116 9897115101100 111110 116104101 7511411711510797108458797108108105115 116101115116
97110100 9910410199107115 119104101116104101114 116104101 104105103104101115116 11599111114101 102114111109 116104101
1089711010311797103101 10911110010110844 116104101 1161149711011510897116105111110 10911110010110844 111114 98111116104
109111100101108115 9911110998105110101100 105115 115105103110105129997110116108121 10010511101114101110116 102114111109
116104101 11111610410111411546 84104101 11510199111110100 109101116104111100 105115 9897115101100 111110 9911111045
10010511610511111097108 112114111989798105108105116121 11610497116 97 1161149711011510897116105111110 105115 110111116 10511045
10210111410511111411611111610410111111610410111411511910410111011610410111611497110115108971161051111101159711610511512101115
115111109101 9911111010010511610511111011546 84104101 9911111010010511610511111097108 112114111989798105108105116121105115
11211410110010599116101100 98121 97 114101103114101115115105111110 116114101101 10810197114110101100 102114111109 116104101
9798111118101 1159911111410111546 84104101 112114111112111115101100 109101116104111100115 119101114101 1011189710845
11797116101100 117115105110103 97110 658482 11611497118101108 9911111411211711546 6912011210111410510910111045
11697108 114101115117108116115 115104111119101100 11610497116 116104101 1121011141021111141099711099101 111102 116104101
112114111112111115101100 109101116104111100115 105115 10911799104 98101116116101114 11610497110 11610497116 111102
116104101 101120105115116105110103 109101116104111100115 97110100 9799104105101118101100 116104101 10510911211411111810145
109101110116 111102 50 116111 54 37 105110 112101114102111114109971109910146
6599107110111119108101100103109101110116
84104105115 1141011151019711499104 11997115 115117112112111114116101100 105110 11297114116 98121 116104101
84101108101991111091091171101059997116105111110115 651001189711099101109101110116 79114103971101051229745
116105111110 111102 74971129711046
-16
-14
-12
-10
-8
-6
-4
-2
0
2
4
6
8
10
12
14
16
18
Difference in performance (%)
LM*TM
TM
LM
LM*TM+KWMC
TM+KWMC
LM+KWMC
RT1+SHR
RT12+SHR
RT123+SHR
RT1+PRN
RT12+PRN
RT123+PRN
A A+B A+B+C Accuracy
70105103117114101 494958 681051110111410111099101 105110 1121011141021111141099711099101 98101116119101101110
1019799104 11510110810199116105111110 115121115116101109 97110100 694574 8468778446
8210110210111410111099101115
76101111 66114101105109971104474101114111109101 7246 70 11410510110010997110448210599 10497114100 6546 7910811510410111044
97110100 6710497114108101115 7446 8311611111010146 4957565246 6710897115115105129997116105111110 97110100 8210145
1031141011151151051111108411410110111546 671049711210997110 97110100 7297108108446797108105102111114110105974485836546
80101116101114 7046 661141111191104483116101112104101110 6810110810897 8010510111611497448610511099101110 116 7446 6810110810897
80105101116114974497110100 8211198101114116 7646 771011149910111446 4957575146 84104101 109971161041011099711645
10599115 111102 115116971161051151161059997108 1099799104105110101 116114971101151089711610511111058 809711497109101116101114 10111511610545
1099711610511111046 67111109112117116971161051111109710876105110103117105115116105991154449574050415850545112351494946
67104114105115 6797108108105115111110456611711499104 97110100 8297121109111110100 8346 7010811111711411011112146 5048484946
65 11211411110311497109 102111114 97117116111109971161059997108108121 11510110810199116105110103 116104101 98101115116 111117116112117116
102114111109 109117108116105112108101 1099799104105110101 1161149711011510897116105111110 10111010310511010111546 73110 8011411145
99101101100105110103115111102778483117109109105116867373734411297103101115 5451123545446
74111104110 7746 671049710998101114115 97110100 84114101118111114 7446 729711511610510146 4957575046 831169745
1161051151161059997108 77111100101108115 105110 8346 8797100115119111114116104 38 661141111111071154767111108101 6510045
1189711099101100 66111111107115 38 83111102116119971141014465 68105118105115105111110 111102 87 9710011511911111411610444
731109946446797108105102111114110105974485836546
80104105108105112 6710897114107115111110 97110100 8211111097108100 8211111510111010210110810046 4957575546 83116971161051151161059997108
1089711010311797103101 109111100101108105110103 117115105110103 116104101 67778545679710998114105100103101 11611111110810710511646
73110 8011411199101101100105110103115111102116104101691171141111121019711067111110102101114101110991011111108311210110199104
67111109109117110105999711610511111097110100841019910411011110811110312158 6985827983806969677245575544
11297103101115 505548551235055494846
7911597109117 70117114117115101 97110100 72105116111115104105 731051009746 4957575446 731109911410110910111011697108
1161149711011510897116105111110 117116105108105122105110103 99111110115116105116117101110116 9811111711010097114121 1129711611610111411011546
73110801141119910110110010511010311511110211610410149541161047311011610111411097116105111110971086711111010210111410111099101111110
671111091121171169711610511111097108 761051101031171051151161059911558 6779767378714557544411297103101115 524950123
52495546
8946 721119910498101114103 97110100 6546 6746 84971091049711010146 4957565146 77117108116105112108101 6711110945
11297114105115111110801141119910110011711410111546 871051081011214478101119 89 1111141074485836546
75101110106105 73109971091171149746 5048485046 651121121081059997116105111110 111102 1161149711011510897116105111110 10711011111910845
101100103101 9799113117105114101100 98121 10410510111497114991041059997108 11210411497115101 97108105103110109101110116 102111114
11297116116101114110459897115101100 10911646 73110 8011411199101101100105110103115 111102 116104101 57116104 6711111010210111445
101110991011111108410410111111410111610599971089711010077101116104111100111108111103105999710873115115117101115105110779745
991041051101018411497110115108971161051111104411297103101115 5552123565246
8397116111115104105 759710710544 83101116115117111 8997109971009744 97110100 6910510599104105114111 831171091051169746
4957575746 8399111114105110103 109117108116105112108101 1161149711011510897116105111110115 117115105110103 9910497114979945
116101114 110451031149710946 73110 8011411199101101100105110103115 111102 116104101 53116104 789711611711497108 769711045
10311797103101 8011411199101115115105110103 8097991051299 82105109 8312110911211111510511710958 787680828345
57574411297103101115 50575612351485046
84111109 7746 771051169910410110810846 4957575546 779799104105110101 761019711411010511010346 84104101
779971114971194572105108108 6711110911297110105101115 7311099464478101119 89 1111141074485836546
7011497110122 74111115101102 7999104 97110100 7210111410997110110 7810112146 5048484846 73109112114111118101100 1151169745
1161051151161059997108 97108105103110109101110116 10911110010110811546 73110 8011411199101101100105110103115 111102 116104101 5156116104
651101101179710877101101116105110103111102116104101651151151119910597116105111110102111114671111091121171169711610511111097108
761051101031171051151161059911558 65677648484411297103101115 52524812352525546
6910510599104105114111 831171091051169746 5048484946 6912097109112108101459897115101100 1099799104105110101 1161149711011545
10897116105111110 117115105110103 6880451099711699104105110103 98101116119101101110 119111114107 1151011131171011109910111546
73110 8011411199101101100105110103115 111102 116104101 656776 50484849 87111114107115104111112 111110 68971169745
68114105118101110 77101116104111100115 105110 779799104105110101 84114971101151089711610511111058 6868778445
504848494411297103101115 491235646
84111115104105121117107105 84971071011229711997446910510599 104105114111 83117109105116974470 11710910597107105 83117103971219744
72105114111102117109105 89971099710911111611144 97110100 8310110510599104105 89971099710911111611146 5048485046
8411111997114100 97 9811411197100459911111810111497103101 9810510810511010311797108 99111114112117115 102111114 11511210110199104
1161149711011510897116105111110 111102 11611497118101108 9911111011810111411597116105111110115 105110 116104101 11410197108 11911111410810046
73110 8011411199101101100105110103115 111102 84104105114100 731101161011141109711610511111097108 6711111010210111410111099101111110
769711010311797103101 8210111511111711499101115 97110100 69118971081179711610511111058 768269675048485044
11297103101115 49525512349535046
8497114111 879711697110979810144 75101110106105 73109971091171149744 97110100 6910510599104105114111 831171091051169746
5048485046 83116971161051151161059997108 1099799104105110101 1161149711011510897116105111110 115121115116101109 9897115101100
111110 10410510111497114991041059997108 11210411497115101 9710810510311010910111011646 73110 8011411199101101100105110103115 111102
11610410157116104671111101021011141011109910111111084104101111114101116105999710897110100771011161041111001111081111031059997108
731151151171011151051107797991041051101018411497110115108971161051111104411297103101115 49565612349575646
