diff -r 6762afd9f963 -r 90c57b50600f ThirdParty/CharDet/chardet/sbcharsetprober.py
--- a/ThirdParty/CharDet/chardet/sbcharsetprober.py Tue Apr 25 18:36:38 2017 +0200
+++ b/ThirdParty/CharDet/chardet/sbcharsetprober.py Tue Apr 25 18:40:46 2017 +0200
@@ -26,95 +26,107 @@
# 02110-1301 USA
######################### END LICENSE BLOCK #########################
-import sys
-from . import constants
from .charsetprober import CharSetProber
-from .compat import wrap_ord
-
-SAMPLE_SIZE = 64
-SB_ENOUGH_REL_THRESHOLD = 1024
-POSITIVE_SHORTCUT_THRESHOLD = 0.95
-NEGATIVE_SHORTCUT_THRESHOLD = 0.05
-SYMBOL_CAT_ORDER = 250
-NUMBER_OF_SEQ_CAT = 4
-POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1
-#NEGATIVE_CAT = 0
+from .enums import CharacterCategory, ProbingState, SequenceLikelihood
class SingleByteCharSetProber(CharSetProber):
- def __init__(self, model, reversed=False, nameProber=None):
- CharSetProber.__init__(self)
- self._mModel = model
+ SAMPLE_SIZE = 64
+ SB_ENOUGH_REL_THRESHOLD = 1024 # 0.25 * SAMPLE_SIZE^2
+ POSITIVE_SHORTCUT_THRESHOLD = 0.95
+ NEGATIVE_SHORTCUT_THRESHOLD = 0.05
+
+ def __init__(self, model, reversed=False, name_prober=None):
+ super(SingleByteCharSetProber, self).__init__()
+ self._model = model
# TRUE if we need to reverse every pair in the model lookup
- self._mReversed = reversed
+ self._reversed = reversed
# Optional auxiliary prober for name decision
- self._mNameProber = nameProber
+ self._name_prober = name_prober
+ self._last_order = None
+ self._seq_counters = None
+ self._total_seqs = None
+ self._total_char = None
+ self._freq_char = None
self.reset()
def reset(self):
- CharSetProber.reset(self)
+ super(SingleByteCharSetProber, self).reset()
# char order of last character
- self._mLastOrder = 255
- self._mSeqCounters = [0] * NUMBER_OF_SEQ_CAT
- self._mTotalSeqs = 0
- self._mTotalChar = 0
+ self._last_order = 255
+ self._seq_counters = [0] * SequenceLikelihood.get_num_categories()
+ self._total_seqs = 0
+ self._total_char = 0
# characters that fall in our sampling range
- self._mFreqChar = 0
+ self._freq_char = 0
- def get_charset_name(self):
- if self._mNameProber:
- return self._mNameProber.get_charset_name()
+ @property
+ def charset_name(self):
+ if self._name_prober:
+ return self._name_prober.charset_name
else:
- return self._mModel['charsetName']
+ return self._model['charset_name']
+
+ @property
+ def language(self):
+ if self._name_prober:
+ return self._name_prober.language
+ else:
+ return self._model.get('language')
- def feed(self, aBuf):
- if not self._mModel['keepEnglishLetter']:
- aBuf = self.filter_without_english_letters(aBuf)
- aLen = len(aBuf)
- if not aLen:
- return self.get_state()
- for c in aBuf:
- order = self._mModel['charToOrderMap'][wrap_ord(c)]
- if order < SYMBOL_CAT_ORDER:
- self._mTotalChar += 1
- if order < SAMPLE_SIZE:
- self._mFreqChar += 1
- if self._mLastOrder < SAMPLE_SIZE:
- self._mTotalSeqs += 1
- if not self._mReversed:
- i = (self._mLastOrder * SAMPLE_SIZE) + order
- model = self._mModel['precedenceMatrix'][i]
+ def feed(self, byte_str):
+ if not self._model['keep_english_letter']:
+ byte_str = self.filter_international_words(byte_str)
+ if not byte_str:
+ return self.state
+ char_to_order_map = self._model['char_to_order_map']
+ for i, c in enumerate(byte_str):
+ # XXX: Order is in range 1-64, so one would think we want 0-63 here,
+ # but that leads to 27 more test failures than before.
+ order = char_to_order_map[c]
+ # XXX: This was SYMBOL_CAT_ORDER before, with a value of 250, but
+ # CharacterCategory.SYMBOL is actually 253, so we use CONTROL
+ # to make it closer to the original intent. The only difference
+ # is whether or not we count digits and control characters for
+ # _total_char purposes.
+ if order < CharacterCategory.CONTROL:
+ self._total_char += 1
+ if order < self.SAMPLE_SIZE:
+ self._freq_char += 1
+ if self._last_order < self.SAMPLE_SIZE:
+ self._total_seqs += 1
+ if not self._reversed:
+ i = (self._last_order * self.SAMPLE_SIZE) + order
+ model = self._model['precedence_matrix'][i]
else: # reverse the order of the letters in the lookup
- i = (order * SAMPLE_SIZE) + self._mLastOrder
- model = self._mModel['precedenceMatrix'][i]
- self._mSeqCounters[model] += 1
- self._mLastOrder = order
+ i = (order * self.SAMPLE_SIZE) + self._last_order
+ model = self._model['precedence_matrix'][i]
+ self._seq_counters[model] += 1
+ self._last_order = order
- if self.get_state() == constants.eDetecting:
- if self._mTotalSeqs > SB_ENOUGH_REL_THRESHOLD:
- cf = self.get_confidence()
- if cf > POSITIVE_SHORTCUT_THRESHOLD:
- if constants._debug:
- sys.stderr.write('%s confidence = %s, we have a'
- 'winner\n' %
- (self._mModel['charsetName'], cf))
- self._mState = constants.eFoundIt
- elif cf < NEGATIVE_SHORTCUT_THRESHOLD:
- if constants._debug:
- sys.stderr.write('%s confidence = %s, below negative'
- 'shortcut threshhold %s\n' %
- (self._mModel['charsetName'], cf,
- NEGATIVE_SHORTCUT_THRESHOLD))
- self._mState = constants.eNotMe
+ charset_name = self._model['charset_name']
+ if self.state == ProbingState.DETECTING:
+ if self._total_seqs > self.SB_ENOUGH_REL_THRESHOLD:
+ confidence = self.get_confidence()
+ if confidence > self.POSITIVE_SHORTCUT_THRESHOLD:
+ self.logger.debug('%s confidence = %s, we have a winner',
+ charset_name, confidence)
+ self._state = ProbingState.FOUND_IT
+ elif confidence < self.NEGATIVE_SHORTCUT_THRESHOLD:
+ self.logger.debug('%s confidence = %s, below negative '
+ 'shortcut threshhold %s', charset_name,
+ confidence,
+ self.NEGATIVE_SHORTCUT_THRESHOLD)
+ self._state = ProbingState.NOT_ME
- return self.get_state()
+ return self.state
def get_confidence(self):
r = 0.01
- if self._mTotalSeqs > 0:
- r = ((1.0 * self._mSeqCounters[POSITIVE_CAT]) / self._mTotalSeqs
- / self._mModel['mTypicalPositiveRatio'])
- r = r * self._mFreqChar / self._mTotalChar
+ if self._total_seqs > 0:
+ r = ((1.0 * self._seq_counters[SequenceLikelihood.POSITIVE]) /
+ self._total_seqs / self._model['typical_positive_ratio'])
+ r = r * self._freq_char / self._total_char
if r >= 1.0:
r = 0.99
return r
