Fri, 25 Apr 2014 22:07:19 +0200
updated CharDet to 2.2.1, updated changelog
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
1 | ######################## BEGIN LICENSE BLOCK ######################## |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
2 | # The Original Code is mozilla.org code. |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
3 | # |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
4 | # The Initial Developer of the Original Code is |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
5 | # Netscape Communications Corporation. |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
6 | # Portions created by the Initial Developer are Copyright (C) 1998 |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
7 | # the Initial Developer. All Rights Reserved. |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
8 | # |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
9 | # Contributor(s): |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
10 | # Mark Pilgrim - port to Python |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
11 | # |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
12 | # This library is free software; you can redistribute it and/or |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
13 | # modify it under the terms of the GNU Lesser General Public |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
14 | # License as published by the Free Software Foundation; either |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
15 | # version 2.1 of the License, or (at your option) any later version. |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
16 | # |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
17 | # This library is distributed in the hope that it will be useful, |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
20 | # Lesser General Public License for more details. |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
21 | # |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
22 | # You should have received a copy of the GNU Lesser General Public |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
23 | # License along with this library; if not, write to the Free Software |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
25 | # 02110-1301 USA |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
26 | ######################### END LICENSE BLOCK ######################### |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
27 | |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
28 | from . import constants |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
29 | from .escsm import (HZSMModel, ISO2022CNSMModel, ISO2022JPSMModel, |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
30 | ISO2022KRSMModel) |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
31 | from .charsetprober import CharSetProber |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
32 | from .codingstatemachine import CodingStateMachine |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
33 | from .compat import wrap_ord |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
34 | |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
35 | |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
36 | class EscCharSetProber(CharSetProber): |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
37 | def __init__(self): |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
38 | CharSetProber.__init__(self) |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
39 | self._mCodingSM = [ |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
40 | CodingStateMachine(HZSMModel), |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
41 | CodingStateMachine(ISO2022CNSMModel), |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
42 | CodingStateMachine(ISO2022JPSMModel), |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
43 | CodingStateMachine(ISO2022KRSMModel) |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
44 | ] |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
45 | self.reset() |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
46 | |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
47 | def reset(self): |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
48 | CharSetProber.reset(self) |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
49 | for codingSM in self._mCodingSM: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
50 | if not codingSM: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
51 | continue |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
52 | codingSM.active = True |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
53 | codingSM.reset() |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
54 | self._mActiveSM = len(self._mCodingSM) |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
55 | self._mDetectedCharset = None |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
56 | |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
57 | def get_charset_name(self): |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
58 | return self._mDetectedCharset |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
59 | |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
60 | def get_confidence(self): |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
61 | if self._mDetectedCharset: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
62 | return 0.99 |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
63 | else: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
64 | return 0.00 |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
65 | |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
66 | def feed(self, aBuf): |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
67 | for c in aBuf: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
68 | # PY3K: aBuf is a byte array, so c is an int, not a byte |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
69 | for codingSM in self._mCodingSM: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
70 | if not codingSM: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
71 | continue |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
72 | if not codingSM.active: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
73 | continue |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
74 | codingState = codingSM.next_state(wrap_ord(c)) |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
75 | if codingState == constants.eError: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
76 | codingSM.active = False |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
77 | self._mActiveSM -= 1 |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
78 | if self._mActiveSM <= 0: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
79 | self._mState = constants.eNotMe |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
80 | return self.get_state() |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
81 | elif codingState == constants.eItsMe: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
82 | self._mState = constants.eFoundIt |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
83 | self._mDetectedCharset = codingSM.get_coding_state_machine() # nopep8 |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
84 | return self.get_state() |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
85 | |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
86 | return self.get_state() |