OLD | NEW |
| (Empty) |
1 ######################## BEGIN LICENSE BLOCK ######################## | |
2 # The Original Code is Mozilla Universal charset detector code. | |
3 # | |
4 # The Initial Developer of the Original Code is | |
5 # Netscape Communications Corporation. | |
6 # Portions created by the Initial Developer are Copyright (C) 2001 | |
7 # the Initial Developer. All Rights Reserved. | |
8 # | |
9 # Contributor(s): | |
10 # Mark Pilgrim - port to Python | |
11 # Shy Shalom - original C code | |
12 # | |
13 # This library is free software; you can redistribute it and/or | |
14 # modify it under the terms of the GNU Lesser General Public | |
15 # License as published by the Free Software Foundation; either | |
16 # version 2.1 of the License, or (at your option) any later version. | |
17 # | |
18 # This library is distributed in the hope that it will be useful, | |
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
21 # Lesser General Public License for more details. | |
22 # | |
23 # You should have received a copy of the GNU Lesser General Public | |
24 # License along with this library; if not, write to the Free Software | |
25 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA | |
26 # 02110-1301 USA | |
27 ######################### END LICENSE BLOCK ######################### | |
28 | |
29 from .charsetprober import CharSetProber | |
30 from .constants import eNotMe | |
31 from .compat import wrap_ord | |
32 | |
33 FREQ_CAT_NUM = 4 | |
34 | |
35 UDF = 0 # undefined | |
36 OTH = 1 # other | |
37 ASC = 2 # ascii capital letter | |
38 ASS = 3 # ascii small letter | |
39 ACV = 4 # accent capital vowel | |
40 ACO = 5 # accent capital other | |
41 ASV = 6 # accent small vowel | |
42 ASO = 7 # accent small other | |
43 CLASS_NUM = 8 # total classes | |
44 | |
45 Latin1_CharToClass = ( | |
46 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 00 - 07 | |
47 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 08 - 0F | |
48 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 10 - 17 | |
49 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 18 - 1F | |
50 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 20 - 27 | |
51 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 28 - 2F | |
52 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 30 - 37 | |
53 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 38 - 3F | |
54 OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 40 - 47 | |
55 ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 48 - 4F | |
56 ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 50 - 57 | |
57 ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH, # 58 - 5F | |
58 OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 60 - 67 | |
59 ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 68 - 6F | |
60 ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 70 - 77 | |
61 ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH, # 78 - 7F | |
62 OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH, # 80 - 87 | |
63 OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF, # 88 - 8F | |
64 UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 90 - 97 | |
65 OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO, # 98 - 9F | |
66 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A0 - A7 | |
67 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A8 - AF | |
68 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B0 - B7 | |
69 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B8 - BF | |
70 ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO, # C0 - C7 | |
71 ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV, # C8 - CF | |
72 ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH, # D0 - D7 | |
73 ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO, # D8 - DF | |
74 ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO, # E0 - E7 | |
75 ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV, # E8 - EF | |
76 ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH, # F0 - F7 | |
77 ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO, # F8 - FF | |
78 ) | |
79 | |
80 # 0 : illegal | |
81 # 1 : very unlikely | |
82 # 2 : normal | |
83 # 3 : very likely | |
84 Latin1ClassModel = ( | |
85 # UDF OTH ASC ASS ACV ACO ASV ASO | |
86 0, 0, 0, 0, 0, 0, 0, 0, # UDF | |
87 0, 3, 3, 3, 3, 3, 3, 3, # OTH | |
88 0, 3, 3, 3, 3, 3, 3, 3, # ASC | |
89 0, 3, 3, 3, 1, 1, 3, 3, # ASS | |
90 0, 3, 3, 3, 1, 2, 1, 2, # ACV | |
91 0, 3, 3, 3, 3, 3, 3, 3, # ACO | |
92 0, 3, 1, 3, 1, 1, 1, 3, # ASV | |
93 0, 3, 1, 3, 1, 1, 3, 3, # ASO | |
94 ) | |
95 | |
96 | |
97 class Latin1Prober(CharSetProber): | |
98 def __init__(self): | |
99 CharSetProber.__init__(self) | |
100 self.reset() | |
101 | |
102 def reset(self): | |
103 self._mLastCharClass = OTH | |
104 self._mFreqCounter = [0] * FREQ_CAT_NUM | |
105 CharSetProber.reset(self) | |
106 | |
107 def get_charset_name(self): | |
108 return "windows-1252" | |
109 | |
110 def feed(self, aBuf): | |
111 aBuf = self.filter_with_english_letters(aBuf) | |
112 for c in aBuf: | |
113 charClass = Latin1_CharToClass[wrap_ord(c)] | |
114 freq = Latin1ClassModel[(self._mLastCharClass * CLASS_NUM) | |
115 + charClass] | |
116 if freq == 0: | |
117 self._mState = eNotMe | |
118 break | |
119 self._mFreqCounter[freq] += 1 | |
120 self._mLastCharClass = charClass | |
121 | |
122 return self.get_state() | |
123 | |
124 def get_confidence(self): | |
125 if self.get_state() == eNotMe: | |
126 return 0.01 | |
127 | |
128 total = sum(self._mFreqCounter) | |
129 if total < 0.01: | |
130 confidence = 0.0 | |
131 else: | |
132 confidence = ((self._mFreqCounter[3] / total) | |
133 - (self._mFreqCounter[1] * 20.0 / total)) | |
134 if confidence < 0.0: | |
135 confidence = 0.0 | |
136 # lower the confidence of latin1 so that other more accurate | |
137 # detector can take priority. | |
138 confidence = confidence * 0.5 | |
139 return confidence | |
OLD | NEW |