ThirdParty/CharDet/chardet/jisfreq.py

changeset 3537
7662053c3906
parent 0
de9c2efb9d02
child 5714
90c57b50600f
equal deleted inserted replaced
3536:c06338ca892b 3537:7662053c3906
11 # 11 #
12 # This library is free software; you can redistribute it and/or 12 # This library is free software; you can redistribute it and/or
13 # modify it under the terms of the GNU Lesser General Public 13 # modify it under the terms of the GNU Lesser General Public
14 # License as published by the Free Software Foundation; either 14 # License as published by the Free Software Foundation; either
15 # version 2.1 of the License, or (at your option) any later version. 15 # version 2.1 of the License, or (at your option) any later version.
16 # 16 #
17 # This library is distributed in the hope that it will be useful, 17 # This library is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of 18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 # Lesser General Public License for more details. 20 # Lesser General Public License for more details.
21 # 21 #
22 # You should have received a copy of the GNU Lesser General Public 22 # You should have received a copy of the GNU Lesser General Public
23 # License along with this library; if not, write to the Free Software 23 # License along with this library; if not, write to the Free Software
24 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 24 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
25 # 02110-1301 USA 25 # 02110-1301 USA
26 ######################### END LICENSE BLOCK ######################### 26 ######################### END LICENSE BLOCK #########################
27 27
28 # Sampling from about 20M text materials include literature and computer technology 28 # Sampling from about 20M text materials include literature and computer technology
29 # 29 #
30 # Japanese frequency table, applied to both S-JIS and EUC-JP 30 # Japanese frequency table, applied to both S-JIS and EUC-JP
31 # They are sorted in order. 31 # They are sorted in order.
32 32
33 # 128 --> 0.77094 33 # 128 --> 0.77094
34 # 256 --> 0.85710 34 # 256 --> 0.85710
35 # 512 --> 0.92635 35 # 512 --> 0.92635
36 # 1024 --> 0.97130 36 # 1024 --> 0.97130
37 # 2048 --> 0.99431 37 # 2048 --> 0.99431
38 # 38 #
39 # Ideal Distribution Ratio = 0.92635 / (1-0.92635) = 12.58 39 # Ideal Distribution Ratio = 0.92635 / (1-0.92635) = 12.58
40 # Random Distribution Ration = 512 / (2965+62+83+86-512) = 0.191 40 # Random Distribution Ration = 512 / (2965+62+83+86-512) = 0.191
41 # 41 #
42 # Typical Distribution Ratio, 25% of IDR 42 # Typical Distribution Ratio, 25% of IDR
43 43
44 JIS_TYPICAL_DISTRIBUTION_RATIO = 3.0 44 JIS_TYPICAL_DISTRIBUTION_RATIO = 3.0
45 45
46 # Char to FreqOrder table , 46 # Char to FreqOrder table ,
47 JIS_TABLE_SIZE = 4368 47 JIS_TABLE_SIZE = 4368
48 48
49 JISCharToFreqOrder = ( \ 49 JISCharToFreqOrder = (
50 40, 1, 6, 182, 152, 180, 295,2127, 285, 381,3295,4304,3068,4606,3165,3510, # 16 50 40, 1, 6, 182, 152, 180, 295,2127, 285, 381,3295,4304,3068,4606,3165,3510, # 16
51 3511,1822,2785,4607,1193,2226,5070,4608, 171,2996,1247, 18, 179,5071, 856,1661, # 32 51 3511,1822,2785,4607,1193,2226,5070,4608, 171,2996,1247, 18, 179,5071, 856,1661, # 32
52 1262,5072, 619, 127,3431,3512,3230,1899,1700, 232, 228,1294,1298, 284, 283,2041, # 48 52 1262,5072, 619, 127,3431,3512,3230,1899,1700, 232, 228,1294,1298, 284, 283,2041, # 48
53 2042,1061,1062, 48, 49, 44, 45, 433, 434,1040,1041, 996, 787,2997,1255,4305, # 64 53 2042,1061,1062, 48, 49, 44, 45, 433, 434,1040,1041, 996, 787,2997,1255,4305, # 64
54 2108,4609,1684,1648,5073,5074,5075,5076,5077,5078,3687,5079,4610,5080,3927,3928, # 80 54 2108,4609,1684,1648,5073,5074,5075,5076,5077,5078,3687,5079,4610,5080,3927,3928, # 80
563 8192,8193,8194,8195,8196,8197,8198,8199,8200,8201,8202,8203,8204,8205,8206,8207, # 8208 563 8192,8193,8194,8195,8196,8197,8198,8199,8200,8201,8202,8203,8204,8205,8206,8207, # 8208
564 8208,8209,8210,8211,8212,8213,8214,8215,8216,8217,8218,8219,8220,8221,8222,8223, # 8224 564 8208,8209,8210,8211,8212,8213,8214,8215,8216,8217,8218,8219,8220,8221,8222,8223, # 8224
565 8224,8225,8226,8227,8228,8229,8230,8231,8232,8233,8234,8235,8236,8237,8238,8239, # 8240 565 8224,8225,8226,8227,8228,8229,8230,8231,8232,8233,8234,8235,8236,8237,8238,8239, # 8240
566 8240,8241,8242,8243,8244,8245,8246,8247,8248,8249,8250,8251,8252,8253,8254,8255, # 8256 566 8240,8241,8242,8243,8244,8245,8246,8247,8248,8249,8250,8251,8252,8253,8254,8255, # 8256
567 8256,8257,8258,8259,8260,8261,8262,8263,8264,8265,8266,8267,8268,8269,8270,8271) # 8272 567 8256,8257,8258,8259,8260,8261,8262,8263,8264,8265,8266,8267,8268,8269,8270,8271) # 8272
568
569 # flake8: noqa

eric ide

mercurial