Wed, 13 Jul 2022 15:34:50 +0200
Revisions <no_multi_processing, Variables Viewer, with_python2> closed.
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
1 | ######################## BEGIN LICENSE BLOCK ######################## |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
2 | # The Original Code is Mozilla Communicator client code. |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
3 | # |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
4 | # The Initial Developer of the Original Code is |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
5 | # Netscape Communications Corporation. |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
6 | # Portions created by the Initial Developer are Copyright (C) 1998 |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
7 | # the Initial Developer. All Rights Reserved. |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
8 | # |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
9 | # Contributor(s): |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
10 | # Mark Pilgrim - port to Python |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
11 | # |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
12 | # This library is free software; you can redistribute it and/or |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
13 | # modify it under the terms of the GNU Lesser General Public |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
14 | # License as published by the Free Software Foundation; either |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
15 | # version 2.1 of the License, or (at your option) any later version. |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
16 | # |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
17 | # This library is distributed in the hope that it will be useful, |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
20 | # Lesser General Public License for more details. |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
21 | # |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
22 | # You should have received a copy of the GNU Lesser General Public |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
23 | # License along with this library; if not, write to the Free Software |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
25 | # 02110-1301 USA |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
26 | ######################### END LICENSE BLOCK ######################### |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
27 | |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
28 | from .enums import ProbingState |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
29 | from .charsetprober import CharSetProber |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
30 | |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
31 | |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
32 | class CharSetGroupProber(CharSetProber): |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
33 | def __init__(self, lang_filter=None): |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
34 | super(CharSetGroupProber, self).__init__(lang_filter=lang_filter) |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
35 | self._active_num = 0 |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
36 | self.probers = [] |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
37 | self._best_guess_prober = None |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
38 | |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
39 | def reset(self): |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
40 | super(CharSetGroupProber, self).reset() |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
41 | self._active_num = 0 |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
42 | for prober in self.probers: |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
43 | if prober: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
44 | prober.reset() |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
45 | prober.active = True |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
46 | self._active_num += 1 |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
47 | self._best_guess_prober = None |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
48 | |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
49 | @property |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
50 | def charset_name(self): |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
51 | if not self._best_guess_prober: |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
52 | self.get_confidence() |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
53 | if not self._best_guess_prober: |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
54 | return None |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
55 | return self._best_guess_prober.charset_name |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
56 | |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
57 | @property |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
58 | def language(self): |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
59 | if not self._best_guess_prober: |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
60 | self.get_confidence() |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
61 | if not self._best_guess_prober: |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
62 | return None |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
63 | return self._best_guess_prober.language |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
64 | |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
65 | def feed(self, byte_str): |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
66 | for prober in self.probers: |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
67 | if not prober: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
68 | continue |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
69 | if not prober.active: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
70 | continue |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
71 | state = prober.feed(byte_str) |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
72 | if not state: |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
73 | continue |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
74 | if state == ProbingState.FOUND_IT: |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
75 | self._best_guess_prober = prober |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
76 | return self.state |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
77 | elif state == ProbingState.NOT_ME: |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
78 | prober.active = False |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
79 | self._active_num -= 1 |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
80 | if self._active_num <= 0: |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
81 | self._state = ProbingState.NOT_ME |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
82 | return self.state |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
83 | return self.state |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
84 | |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
85 | def get_confidence(self): |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
86 | state = self.state |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
87 | if state == ProbingState.FOUND_IT: |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
88 | return 0.99 |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
89 | elif state == ProbingState.NOT_ME: |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
90 | return 0.01 |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
91 | best_conf = 0.0 |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
92 | self._best_guess_prober = None |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
93 | for prober in self.probers: |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
94 | if not prober: |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
95 | continue |
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
96 | if not prober.active: |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
97 | self.logger.debug('%s not active', prober.charset_name) |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
98 | continue |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
99 | conf = prober.get_confidence() |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
100 | self.logger.debug('%s %s confidence = %s', prober.charset_name, prober.language, conf) |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
101 | if best_conf < conf: |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
102 | best_conf = conf |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
103 | self._best_guess_prober = prober |
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
104 | if not self._best_guess_prober: |
3537
7662053c3906
updated CharDet to 2.2.1, updated changelog
T.Rzepka <Tobias.Rzepka@gmail.com>
parents:
12
diff
changeset
|
105 | return 0.0 |
5714
90c57b50600f
Updated chardet to 3.0.2.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
3537
diff
changeset
|
106 | return best_conf |