swarm_client/third_party/requests/packages/charade/latin1prober.py - Issue 69143004: Delete swarm_client.

Side by Side Diff: swarm_client/third_party/requests/packages/charade/latin1prober.py

Issue 69143004: Delete swarm_client. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/

Patch Set: Created 7 years, 1 month ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View unified diff | Download patch | Annotate | Revision Log

« no previous file with comments | « swarm_client/third_party/requests/packages/charade/langthaimodel.py ('k') | swarm_client/third_party/requests/packages/charade/mbcharsetprober.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Hide Comments ('s')

OLD	NEW
	(Empty)
1 ######################## BEGIN LICENSE BLOCK ########################

2 # The Original Code is Mozilla Universal charset detector code.

3 #

4 # The Initial Developer of the Original Code is

5 # Netscape Communications Corporation.

6 # Portions created by the Initial Developer are Copyright (C) 2001

7 # the Initial Developer. All Rights Reserved.

8 #

9 # Contributor(s):

10 # Mark Pilgrim - port to Python

11 # Shy Shalom - original C code

12 #

13 # This library is free software; you can redistribute it and/or

14 # modify it under the terms of the GNU Lesser General Public

15 # License as published by the Free Software Foundation; either

16 # version 2.1 of the License, or (at your option) any later version.

17 #

18 # This library is distributed in the hope that it will be useful,

19 # but WITHOUT ANY WARRANTY; without even the implied warranty of

20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

21 # Lesser General Public License for more details.

22 #

23 # You should have received a copy of the GNU Lesser General Public

24 # License along with this library; if not, write to the Free Software

25 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA

26 # 02110-1301 USA

27 ######################### END LICENSE BLOCK #########################

28

29 from .charsetprober import CharSetProber

30 from .constants import eNotMe

31 from .compat import wrap_ord

32

33 FREQ_CAT_NUM = 4

34

35 UDF = 0 # undefined

36 OTH = 1 # other

37 ASC = 2 # ascii capital letter

38 ASS = 3 # ascii small letter

39 ACV = 4 # accent capital vowel

40 ACO = 5 # accent capital other

41 ASV = 6 # accent small vowel

42 ASO = 7 # accent small other

43 CLASS_NUM = 8 # total classes

44

45 Latin1_CharToClass = (

46 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 00 - 07

47 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 08 - 0F

48 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 10 - 17

49 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 18 - 1F

50 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 20 - 27

51 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 28 - 2F

52 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 30 - 37

53 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 38 - 3F

54 OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 40 - 47

55 ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 48 - 4F

56 ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 50 - 57

57 ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH, # 58 - 5F

58 OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 60 - 67

59 ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 68 - 6F

60 ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 70 - 77

61 ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH, # 78 - 7F

62 OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH, # 80 - 87

63 OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF, # 88 - 8F

64 UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 90 - 97

65 OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO, # 98 - 9F

66 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A0 - A7

67 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A8 - AF

68 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B0 - B7

69 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B8 - BF

70 ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO, # C0 - C7

71 ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV, # C8 - CF

72 ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH, # D0 - D7

73 ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO, # D8 - DF

74 ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO, # E0 - E7

75 ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV, # E8 - EF

76 ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH, # F0 - F7

77 ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO, # F8 - FF

78 )

79

80 # 0 : illegal

81 # 1 : very unlikely

82 # 2 : normal

83 # 3 : very likely

84 Latin1ClassModel = (

85 # UDF OTH ASC ASS ACV ACO ASV ASO

86 0, 0, 0, 0, 0, 0, 0, 0, # UDF

87 0, 3, 3, 3, 3, 3, 3, 3, # OTH

88 0, 3, 3, 3, 3, 3, 3, 3, # ASC

89 0, 3, 3, 3, 1, 1, 3, 3, # ASS

90 0, 3, 3, 3, 1, 2, 1, 2, # ACV

91 0, 3, 3, 3, 3, 3, 3, 3, # ACO

92 0, 3, 1, 3, 1, 1, 1, 3, # ASV

93 0, 3, 1, 3, 1, 1, 3, 3, # ASO

94 )

95

96

97 class Latin1Prober(CharSetProber):

98 def __init__(self):

99 CharSetProber.__init__(self)

100 self.reset()

101

102 def reset(self):

103 self._mLastCharClass = OTH

104 self._mFreqCounter = [0] * FREQ_CAT_NUM

105 CharSetProber.reset(self)

106

107 def get_charset_name(self):

108 return "windows-1252"

109

110 def feed(self, aBuf):

111 aBuf = self.filter_with_english_letters(aBuf)

112 for c in aBuf:

113 charClass = Latin1_CharToClass[wrap_ord(c)]

114 freq = Latin1ClassModel[(self._mLastCharClass * CLASS_NUM)

115 + charClass]

116 if freq == 0:

117 self._mState = eNotMe

118 break

119 self._mFreqCounter[freq] += 1

120 self._mLastCharClass = charClass

121

122 return self.get_state()

123

124 def get_confidence(self):

125 if self.get_state() == eNotMe:

126 return 0.01

127

128 total = sum(self._mFreqCounter)

129 if total < 0.01:

130 confidence = 0.0

131 else:

132 confidence = ((self._mFreqCounter[3] / total)

133 - (self._mFreqCounter[1] * 20.0 / total))

134 if confidence < 0.0:

135 confidence = 0.0

136 # lower the confidence of latin1 so that other more accurate

137 # detector can take priority.

138 confidence = confidence * 0.5

139 return confidence

OLD	NEW