diff -r e836d196e888 -r f425b578ede7 eric6/ThirdParty/CharDet/chardet/sbcsgroupprober.py
--- a/eric6/ThirdParty/CharDet/chardet/sbcsgroupprober.py Wed Jan 13 19:02:58 2021 +0100
+++ b/eric6/ThirdParty/CharDet/chardet/sbcsgroupprober.py Wed Jan 13 19:05:48 2021 +0100
@@ -27,47 +27,57 @@
######################### END LICENSE BLOCK #########################
from .charsetgroupprober import CharSetGroupProber
+from .hebrewprober import HebrewProber
+from .langbulgarianmodel import (ISO_8859_5_BULGARIAN_MODEL,
+ WINDOWS_1251_BULGARIAN_MODEL)
+from .langgreekmodel import ISO_8859_7_GREEK_MODEL, WINDOWS_1253_GREEK_MODEL
+from .langhebrewmodel import WINDOWS_1255_HEBREW_MODEL
+# from .langhungarianmodel import (ISO_8859_2_HUNGARIAN_MODEL,
+# WINDOWS_1250_HUNGARIAN_MODEL)
+from .langrussianmodel import (IBM855_RUSSIAN_MODEL, IBM866_RUSSIAN_MODEL,
+ ISO_8859_5_RUSSIAN_MODEL, KOI8_R_RUSSIAN_MODEL,
+ MACCYRILLIC_RUSSIAN_MODEL,
+ WINDOWS_1251_RUSSIAN_MODEL)
+from .langthaimodel import TIS_620_THAI_MODEL
+from .langturkishmodel import ISO_8859_9_TURKISH_MODEL
from .sbcharsetprober import SingleByteCharSetProber
-from .langcyrillicmodel import (Win1251CyrillicModel, Koi8rModel,
- Latin5CyrillicModel, MacCyrillicModel,
- Ibm866Model, Ibm855Model)
-from .langgreekmodel import Latin7GreekModel, Win1253GreekModel
-from .langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel
-# from .langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel
-from .langthaimodel import TIS620ThaiModel
-from .langhebrewmodel import Win1255HebrewModel
-from .hebrewprober import HebrewProber
-from .langturkishmodel import Latin5TurkishModel
class SBCSGroupProber(CharSetGroupProber):
def __init__(self):
super(SBCSGroupProber, self).__init__()
+ hebrew_prober = HebrewProber()
+ logical_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
+ False, hebrew_prober)
+ # TODO: See if using ISO-8859-8 Hebrew model works better here, since
+ # it's actually the visual one
+ visual_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
+ True, hebrew_prober)
+ hebrew_prober.set_model_probers(logical_hebrew_prober,
+ visual_hebrew_prober)
+ # TODO: ORDER MATTERS HERE. I changed the order vs what was in master
+ # and several tests failed that did not before. Some thought
+ # should be put into the ordering, and we should consider making
+ # order not matter here, because that is very counter-intuitive.
self.probers = [
- SingleByteCharSetProber(Win1251CyrillicModel),
- SingleByteCharSetProber(Koi8rModel),
- SingleByteCharSetProber(Latin5CyrillicModel),
- SingleByteCharSetProber(MacCyrillicModel),
- SingleByteCharSetProber(Ibm866Model),
- SingleByteCharSetProber(Ibm855Model),
- SingleByteCharSetProber(Latin7GreekModel),
- SingleByteCharSetProber(Win1253GreekModel),
- SingleByteCharSetProber(Latin5BulgarianModel),
- SingleByteCharSetProber(Win1251BulgarianModel),
+ SingleByteCharSetProber(WINDOWS_1251_RUSSIAN_MODEL),
+ SingleByteCharSetProber(KOI8_R_RUSSIAN_MODEL),
+ SingleByteCharSetProber(ISO_8859_5_RUSSIAN_MODEL),
+ SingleByteCharSetProber(MACCYRILLIC_RUSSIAN_MODEL),
+ SingleByteCharSetProber(IBM866_RUSSIAN_MODEL),
+ SingleByteCharSetProber(IBM855_RUSSIAN_MODEL),
+ SingleByteCharSetProber(ISO_8859_7_GREEK_MODEL),
+ SingleByteCharSetProber(WINDOWS_1253_GREEK_MODEL),
+ SingleByteCharSetProber(ISO_8859_5_BULGARIAN_MODEL),
+ SingleByteCharSetProber(WINDOWS_1251_BULGARIAN_MODEL),
# TODO: Restore Hungarian encodings (iso-8859-2 and windows-1250)
# after we retrain model.
- # SingleByteCharSetProber(Latin2HungarianModel),
- # SingleByteCharSetProber(Win1250HungarianModel),
- SingleByteCharSetProber(TIS620ThaiModel),
- SingleByteCharSetProber(Latin5TurkishModel),
+ # SingleByteCharSetProber(ISO_8859_2_HUNGARIAN_MODEL),
+ # SingleByteCharSetProber(WINDOWS_1250_HUNGARIAN_MODEL),
+ SingleByteCharSetProber(TIS_620_THAI_MODEL),
+ SingleByteCharSetProber(ISO_8859_9_TURKISH_MODEL),
+ hebrew_prober,
+ logical_hebrew_prober,
+ visual_hebrew_prober,
]
- hebrew_prober = HebrewProber()
- logical_hebrew_prober = SingleByteCharSetProber(Win1255HebrewModel,
- False, hebrew_prober)
- visual_hebrew_prober = SingleByteCharSetProber(Win1255HebrewModel, True,
- hebrew_prober)
- hebrew_prober.set_model_probers(logical_hebrew_prober, visual_hebrew_prober)
- self.probers.extend([hebrew_prober, logical_hebrew_prober,
- visual_hebrew_prober])
-
self.reset()
