OLD | NEW |
1 /* Copyright 2013 Google Inc. All Rights Reserved. | 1 /* Copyright 2013 Google Inc. All Rights Reserved. |
2 | 2 |
3 Distributed under MIT license. | 3 Distributed under MIT license. |
4 See file LICENSE for detail or copy at https://opensource.org/licenses/MIT | 4 See file LICENSE for detail or copy at https://opensource.org/licenses/MIT |
5 */ | 5 */ |
6 | 6 |
7 // Literal cost model to allow backward reference replacement to be efficient. | 7 /* Literal cost model to allow backward reference replacement to be efficient. |
| 8 */ |
8 | 9 |
9 #include "./literal_cost.h" | 10 #include "./literal_cost.h" |
10 | 11 |
11 #include <math.h> | 12 #include <brotli/types.h> |
12 #include <algorithm> | |
13 | |
14 #include "./fast_log.h" | 13 #include "./fast_log.h" |
15 #include "./types.h" | 14 #include "./port.h" |
16 #include "./utf8_util.h" | 15 #include "./utf8_util.h" |
17 | 16 |
18 namespace brotli { | 17 #if defined(__cplusplus) || defined(c_plusplus) |
| 18 extern "C" { |
| 19 #endif |
19 | 20 |
20 static size_t UTF8Position(size_t last, size_t c, size_t clamp) { | 21 static size_t UTF8Position(size_t last, size_t c, size_t clamp) { |
21 if (c < 128) { | 22 if (c < 128) { |
22 return 0; // Next one is the 'Byte 1' again. | 23 return 0; /* Next one is the 'Byte 1' again. */ |
23 } else if (c >= 192) { // Next one is the 'Byte 2' of utf-8 encoding. | 24 } else if (c >= 192) { /* Next one is the 'Byte 2' of utf-8 encoding. */ |
24 return std::min<size_t>(1, clamp); | 25 return BROTLI_MIN(size_t, 1, clamp); |
25 } else { | 26 } else { |
26 // Let's decide over the last byte if this ends the sequence. | 27 /* Let's decide over the last byte if this ends the sequence. */ |
27 if (last < 0xe0) { | 28 if (last < 0xe0) { |
28 return 0; // Completed two or three byte coding. | 29 return 0; /* Completed two or three byte coding. */ |
29 } else { // Next one is the 'Byte 3' of utf-8 encoding. | 30 } else { /* Next one is the 'Byte 3' of utf-8 encoding. */ |
30 return std::min<size_t>(2, clamp); | 31 return BROTLI_MIN(size_t, 2, clamp); |
31 } | 32 } |
32 } | 33 } |
33 } | 34 } |
34 | 35 |
35 static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask, | 36 static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask, |
36 const uint8_t *data) { | 37 const uint8_t *data) { |
37 size_t counts[3] = { 0 }; | 38 size_t counts[3] = { 0 }; |
38 size_t max_utf8 = 1; // should be 2, but 1 compresses better. | 39 size_t max_utf8 = 1; /* should be 2, but 1 compresses better. */ |
39 size_t last_c = 0; | 40 size_t last_c = 0; |
40 size_t utf8_pos = 0; | 41 size_t utf8_pos = 0; |
41 for (size_t i = 0; i < len; ++i) { | 42 size_t i; |
| 43 for (i = 0; i < len; ++i) { |
42 size_t c = data[(pos + i) & mask]; | 44 size_t c = data[(pos + i) & mask]; |
43 utf8_pos = UTF8Position(last_c, c, 2); | 45 utf8_pos = UTF8Position(last_c, c, 2); |
44 ++counts[utf8_pos]; | 46 ++counts[utf8_pos]; |
45 last_c = c; | 47 last_c = c; |
46 } | 48 } |
47 if (counts[2] < 500) { | 49 if (counts[2] < 500) { |
48 max_utf8 = 1; | 50 max_utf8 = 1; |
49 } | 51 } |
50 if (counts[1] + counts[2] < 25) { | 52 if (counts[1] + counts[2] < 25) { |
51 max_utf8 = 0; | 53 max_utf8 = 0; |
52 } | 54 } |
53 return max_utf8; | 55 return max_utf8; |
54 } | 56 } |
55 | 57 |
56 static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask, | 58 static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask, |
57 const uint8_t *data, float *cost) { | 59 const uint8_t *data, float *cost) { |
58 | 60 /* max_utf8 is 0 (normal ASCII single byte modeling), |
59 // max_utf8 is 0 (normal ascii single byte modeling), | 61 1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */ |
60 // 1 (for 2-byte utf-8 modeling), or 2 (for 3-byte utf-8 modeling). | |
61 const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data); | 62 const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data); |
62 size_t histogram[3][256] = { { 0 } }; | 63 size_t histogram[3][256] = { { 0 } }; |
63 size_t window_half = 495; | 64 size_t window_half = 495; |
64 size_t in_window = std::min(window_half, len); | 65 size_t in_window = BROTLI_MIN(size_t, window_half, len); |
65 size_t in_window_utf8[3] = { 0 }; | 66 size_t in_window_utf8[3] = { 0 }; |
66 | 67 |
67 // Bootstrap histograms. | 68 |
68 size_t last_c = 0; | 69 size_t i; |
69 size_t utf8_pos = 0; | 70 { /* Bootstrap histograms. */ |
70 for (size_t i = 0; i < in_window; ++i) { | 71 size_t last_c = 0; |
71 size_t c = data[(pos + i) & mask]; | 72 size_t utf8_pos = 0; |
72 ++histogram[utf8_pos][c]; | 73 for (i = 0; i < in_window; ++i) { |
73 ++in_window_utf8[utf8_pos]; | 74 size_t c = data[(pos + i) & mask]; |
74 utf8_pos = UTF8Position(last_c, c, max_utf8); | 75 ++histogram[utf8_pos][c]; |
75 last_c = c; | 76 ++in_window_utf8[utf8_pos]; |
| 77 utf8_pos = UTF8Position(last_c, c, max_utf8); |
| 78 last_c = c; |
| 79 } |
76 } | 80 } |
77 | 81 |
78 // Compute bit costs with sliding window. | 82 /* Compute bit costs with sliding window. */ |
79 for (size_t i = 0; i < len; ++i) { | 83 for (i = 0; i < len; ++i) { |
80 if (i >= window_half) { | 84 if (i >= window_half) { |
81 // Remove a byte in the past. | 85 /* Remove a byte in the past. */ |
82 size_t c = i < window_half + 1 ? | 86 size_t c = |
83 0 : data[(pos + i - window_half - 1) & mask]; | 87 i < window_half + 1 ? 0 : data[(pos + i - window_half - 1) & mask]; |
84 size_t last_c = i < window_half + 2 ? | 88 size_t last_c = |
85 0 : data[(pos + i - window_half - 2) & mask]; | 89 i < window_half + 2 ? 0 : data[(pos + i - window_half - 2) & mask]; |
86 size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8); | 90 size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8); |
87 --histogram[utf8_pos2][data[(pos + i - window_half) & mask]]; | 91 --histogram[utf8_pos2][data[(pos + i - window_half) & mask]]; |
88 --in_window_utf8[utf8_pos2]; | 92 --in_window_utf8[utf8_pos2]; |
89 } | 93 } |
90 if (i + window_half < len) { | 94 if (i + window_half < len) { |
91 // Add a byte in the future. | 95 /* Add a byte in the future. */ |
92 size_t c = data[(pos + i + window_half - 1) & mask]; | 96 size_t c = data[(pos + i + window_half - 1) & mask]; |
93 size_t last_c = data[(pos + i + window_half - 2) & mask]; | 97 size_t last_c = data[(pos + i + window_half - 2) & mask]; |
94 size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8); | 98 size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8); |
95 ++histogram[utf8_pos2][data[(pos + i + window_half) & mask]]; | 99 ++histogram[utf8_pos2][data[(pos + i + window_half) & mask]]; |
96 ++in_window_utf8[utf8_pos2]; | 100 ++in_window_utf8[utf8_pos2]; |
97 } | 101 } |
98 size_t c = i < 1 ? 0 : data[(pos + i - 1) & mask]; | 102 { |
99 size_t last_c = i < 2 ? 0 : data[(pos + i - 2) & mask]; | 103 size_t c = i < 1 ? 0 : data[(pos + i - 1) & mask]; |
100 size_t utf8_pos = UTF8Position(last_c, c, max_utf8); | 104 size_t last_c = i < 2 ? 0 : data[(pos + i - 2) & mask]; |
101 size_t masked_pos = (pos + i) & mask; | 105 size_t utf8_pos = UTF8Position(last_c, c, max_utf8); |
102 size_t histo = histogram[utf8_pos][data[masked_pos]]; | 106 size_t masked_pos = (pos + i) & mask; |
103 if (histo == 0) { | 107 size_t histo = histogram[utf8_pos][data[masked_pos]]; |
104 histo = 1; | 108 double lit_cost; |
| 109 if (histo == 0) { |
| 110 histo = 1; |
| 111 } |
| 112 lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo); |
| 113 lit_cost += 0.02905; |
| 114 if (lit_cost < 1.0) { |
| 115 lit_cost *= 0.5; |
| 116 lit_cost += 0.5; |
| 117 } |
| 118 /* Make the first bytes more expensive -- seems to help, not sure why. |
| 119 Perhaps because the entropy source is changing its properties |
| 120 rapidly in the beginning of the file, perhaps because the beginning |
| 121 of the data is a statistical "anomaly". */ |
| 122 if (i < 2000) { |
| 123 lit_cost += 0.7 - ((double)(2000 - i) / 2000.0 * 0.35); |
| 124 } |
| 125 cost[i] = (float)lit_cost; |
105 } | 126 } |
106 double lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo); | |
107 lit_cost += 0.02905; | |
108 if (lit_cost < 1.0) { | |
109 lit_cost *= 0.5; | |
110 lit_cost += 0.5; | |
111 } | |
112 // Make the first bytes more expensive -- seems to help, not sure why. | |
113 // Perhaps because the entropy source is changing its properties | |
114 // rapidly in the beginning of the file, perhaps because the beginning | |
115 // of the data is a statistical "anomaly". | |
116 if (i < 2000) { | |
117 lit_cost += 0.7 - (static_cast<double>(2000 - i) / 2000.0 * 0.35); | |
118 } | |
119 cost[i] = static_cast<float>(lit_cost); | |
120 } | 127 } |
121 } | 128 } |
122 | 129 |
123 void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask, | 130 void BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask, |
124 const uint8_t *data, float *cost) { | 131 const uint8_t *data, float *cost) { |
125 if (IsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) { | 132 if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) { |
126 EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost); | 133 EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost); |
127 return; | 134 return; |
128 } | 135 } else { |
129 size_t histogram[256] = { 0 }; | 136 size_t histogram[256] = { 0 }; |
130 size_t window_half = 2000; | 137 size_t window_half = 2000; |
131 size_t in_window = std::min(window_half, len); | 138 size_t in_window = BROTLI_MIN(size_t, window_half, len); |
132 | 139 |
133 // Bootstrap histogram. | 140 /* Bootstrap histogram. */ |
134 for (size_t i = 0; i < in_window; ++i) { | 141 size_t i; |
135 ++histogram[data[(pos + i) & mask]]; | 142 for (i = 0; i < in_window; ++i) { |
136 } | 143 ++histogram[data[(pos + i) & mask]]; |
| 144 } |
137 | 145 |
138 // Compute bit costs with sliding window. | 146 /* Compute bit costs with sliding window. */ |
139 for (size_t i = 0; i < len; ++i) { | 147 for (i = 0; i < len; ++i) { |
140 if (i >= window_half) { | 148 size_t histo; |
141 // Remove a byte in the past. | 149 if (i >= window_half) { |
142 --histogram[data[(pos + i - window_half) & mask]]; | 150 /* Remove a byte in the past. */ |
143 --in_window; | 151 --histogram[data[(pos + i - window_half) & mask]]; |
| 152 --in_window; |
| 153 } |
| 154 if (i + window_half < len) { |
| 155 /* Add a byte in the future. */ |
| 156 ++histogram[data[(pos + i + window_half) & mask]]; |
| 157 ++in_window; |
| 158 } |
| 159 histo = histogram[data[(pos + i) & mask]]; |
| 160 if (histo == 0) { |
| 161 histo = 1; |
| 162 } |
| 163 { |
| 164 double lit_cost = FastLog2(in_window) - FastLog2(histo); |
| 165 lit_cost += 0.029; |
| 166 if (lit_cost < 1.0) { |
| 167 lit_cost *= 0.5; |
| 168 lit_cost += 0.5; |
| 169 } |
| 170 cost[i] = (float)lit_cost; |
| 171 } |
144 } | 172 } |
145 if (i + window_half < len) { | |
146 // Add a byte in the future. | |
147 ++histogram[data[(pos + i + window_half) & mask]]; | |
148 ++in_window; | |
149 } | |
150 size_t histo = histogram[data[(pos + i) & mask]]; | |
151 if (histo == 0) { | |
152 histo = 1; | |
153 } | |
154 double lit_cost = FastLog2(in_window) - FastLog2(histo); | |
155 lit_cost += 0.029; | |
156 if (lit_cost < 1.0) { | |
157 lit_cost *= 0.5; | |
158 lit_cost += 0.5; | |
159 } | |
160 cost[i] = static_cast<float>(lit_cost); | |
161 } | 173 } |
162 } | 174 } |
163 | 175 |
164 | 176 #if defined(__cplusplus) || defined(c_plusplus) |
165 } // namespace brotli | 177 } /* extern "C" */ |
| 178 #endif |
OLD | NEW |