Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(108)

Side by Side Diff: third_party/brotli/enc/literal_cost.c

Issue 2537133002: Update brotli to v1.0.0-snapshot. (Closed)
Patch Set: Fixed typo Created 4 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « third_party/brotli/enc/literal_cost.h ('k') | third_party/brotli/enc/literal_cost.cc » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 /* Copyright 2013 Google Inc. All Rights Reserved. 1 /* Copyright 2013 Google Inc. All Rights Reserved.
2 2
3 Distributed under MIT license. 3 Distributed under MIT license.
4 See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 4 See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
5 */ 5 */
6 6
7 // Literal cost model to allow backward reference replacement to be efficient. 7 /* Literal cost model to allow backward reference replacement to be efficient.
8 */
8 9
9 #include "./literal_cost.h" 10 #include "./literal_cost.h"
10 11
11 #include <math.h> 12 #include <brotli/types.h>
12 #include <algorithm>
13
14 #include "./fast_log.h" 13 #include "./fast_log.h"
15 #include "./types.h" 14 #include "./port.h"
16 #include "./utf8_util.h" 15 #include "./utf8_util.h"
17 16
18 namespace brotli { 17 #if defined(__cplusplus) || defined(c_plusplus)
18 extern "C" {
19 #endif
19 20
20 static size_t UTF8Position(size_t last, size_t c, size_t clamp) { 21 static size_t UTF8Position(size_t last, size_t c, size_t clamp) {
21 if (c < 128) { 22 if (c < 128) {
22 return 0; // Next one is the 'Byte 1' again. 23 return 0; /* Next one is the 'Byte 1' again. */
23 } else if (c >= 192) { // Next one is the 'Byte 2' of utf-8 encoding. 24 } else if (c >= 192) { /* Next one is the 'Byte 2' of utf-8 encoding. */
24 return std::min<size_t>(1, clamp); 25 return BROTLI_MIN(size_t, 1, clamp);
25 } else { 26 } else {
26 // Let's decide over the last byte if this ends the sequence. 27 /* Let's decide over the last byte if this ends the sequence. */
27 if (last < 0xe0) { 28 if (last < 0xe0) {
28 return 0; // Completed two or three byte coding. 29 return 0; /* Completed two or three byte coding. */
29 } else { // Next one is the 'Byte 3' of utf-8 encoding. 30 } else { /* Next one is the 'Byte 3' of utf-8 encoding. */
30 return std::min<size_t>(2, clamp); 31 return BROTLI_MIN(size_t, 2, clamp);
31 } 32 }
32 } 33 }
33 } 34 }
34 35
35 static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask, 36 static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
36 const uint8_t *data) { 37 const uint8_t *data) {
37 size_t counts[3] = { 0 }; 38 size_t counts[3] = { 0 };
38 size_t max_utf8 = 1; // should be 2, but 1 compresses better. 39 size_t max_utf8 = 1; /* should be 2, but 1 compresses better. */
39 size_t last_c = 0; 40 size_t last_c = 0;
40 size_t utf8_pos = 0; 41 size_t utf8_pos = 0;
41 for (size_t i = 0; i < len; ++i) { 42 size_t i;
43 for (i = 0; i < len; ++i) {
42 size_t c = data[(pos + i) & mask]; 44 size_t c = data[(pos + i) & mask];
43 utf8_pos = UTF8Position(last_c, c, 2); 45 utf8_pos = UTF8Position(last_c, c, 2);
44 ++counts[utf8_pos]; 46 ++counts[utf8_pos];
45 last_c = c; 47 last_c = c;
46 } 48 }
47 if (counts[2] < 500) { 49 if (counts[2] < 500) {
48 max_utf8 = 1; 50 max_utf8 = 1;
49 } 51 }
50 if (counts[1] + counts[2] < 25) { 52 if (counts[1] + counts[2] < 25) {
51 max_utf8 = 0; 53 max_utf8 = 0;
52 } 54 }
53 return max_utf8; 55 return max_utf8;
54 } 56 }
55 57
56 static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask, 58 static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
57 const uint8_t *data, float *cost) { 59 const uint8_t *data, float *cost) {
58 60 /* max_utf8 is 0 (normal ASCII single byte modeling),
59 // max_utf8 is 0 (normal ascii single byte modeling), 61 1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
60 // 1 (for 2-byte utf-8 modeling), or 2 (for 3-byte utf-8 modeling).
61 const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data); 62 const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
62 size_t histogram[3][256] = { { 0 } }; 63 size_t histogram[3][256] = { { 0 } };
63 size_t window_half = 495; 64 size_t window_half = 495;
64 size_t in_window = std::min(window_half, len); 65 size_t in_window = BROTLI_MIN(size_t, window_half, len);
65 size_t in_window_utf8[3] = { 0 }; 66 size_t in_window_utf8[3] = { 0 };
66 67
67 // Bootstrap histograms. 68
68 size_t last_c = 0; 69 size_t i;
69 size_t utf8_pos = 0; 70 { /* Bootstrap histograms. */
70 for (size_t i = 0; i < in_window; ++i) { 71 size_t last_c = 0;
71 size_t c = data[(pos + i) & mask]; 72 size_t utf8_pos = 0;
72 ++histogram[utf8_pos][c]; 73 for (i = 0; i < in_window; ++i) {
73 ++in_window_utf8[utf8_pos]; 74 size_t c = data[(pos + i) & mask];
74 utf8_pos = UTF8Position(last_c, c, max_utf8); 75 ++histogram[utf8_pos][c];
75 last_c = c; 76 ++in_window_utf8[utf8_pos];
77 utf8_pos = UTF8Position(last_c, c, max_utf8);
78 last_c = c;
79 }
76 } 80 }
77 81
78 // Compute bit costs with sliding window. 82 /* Compute bit costs with sliding window. */
79 for (size_t i = 0; i < len; ++i) { 83 for (i = 0; i < len; ++i) {
80 if (i >= window_half) { 84 if (i >= window_half) {
81 // Remove a byte in the past. 85 /* Remove a byte in the past. */
82 size_t c = i < window_half + 1 ? 86 size_t c =
83 0 : data[(pos + i - window_half - 1) & mask]; 87 i < window_half + 1 ? 0 : data[(pos + i - window_half - 1) & mask];
84 size_t last_c = i < window_half + 2 ? 88 size_t last_c =
85 0 : data[(pos + i - window_half - 2) & mask]; 89 i < window_half + 2 ? 0 : data[(pos + i - window_half - 2) & mask];
86 size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8); 90 size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8);
87 --histogram[utf8_pos2][data[(pos + i - window_half) & mask]]; 91 --histogram[utf8_pos2][data[(pos + i - window_half) & mask]];
88 --in_window_utf8[utf8_pos2]; 92 --in_window_utf8[utf8_pos2];
89 } 93 }
90 if (i + window_half < len) { 94 if (i + window_half < len) {
91 // Add a byte in the future. 95 /* Add a byte in the future. */
92 size_t c = data[(pos + i + window_half - 1) & mask]; 96 size_t c = data[(pos + i + window_half - 1) & mask];
93 size_t last_c = data[(pos + i + window_half - 2) & mask]; 97 size_t last_c = data[(pos + i + window_half - 2) & mask];
94 size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8); 98 size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8);
95 ++histogram[utf8_pos2][data[(pos + i + window_half) & mask]]; 99 ++histogram[utf8_pos2][data[(pos + i + window_half) & mask]];
96 ++in_window_utf8[utf8_pos2]; 100 ++in_window_utf8[utf8_pos2];
97 } 101 }
98 size_t c = i < 1 ? 0 : data[(pos + i - 1) & mask]; 102 {
99 size_t last_c = i < 2 ? 0 : data[(pos + i - 2) & mask]; 103 size_t c = i < 1 ? 0 : data[(pos + i - 1) & mask];
100 size_t utf8_pos = UTF8Position(last_c, c, max_utf8); 104 size_t last_c = i < 2 ? 0 : data[(pos + i - 2) & mask];
101 size_t masked_pos = (pos + i) & mask; 105 size_t utf8_pos = UTF8Position(last_c, c, max_utf8);
102 size_t histo = histogram[utf8_pos][data[masked_pos]]; 106 size_t masked_pos = (pos + i) & mask;
103 if (histo == 0) { 107 size_t histo = histogram[utf8_pos][data[masked_pos]];
104 histo = 1; 108 double lit_cost;
109 if (histo == 0) {
110 histo = 1;
111 }
112 lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo);
113 lit_cost += 0.02905;
114 if (lit_cost < 1.0) {
115 lit_cost *= 0.5;
116 lit_cost += 0.5;
117 }
118 /* Make the first bytes more expensive -- seems to help, not sure why.
119 Perhaps because the entropy source is changing its properties
120 rapidly in the beginning of the file, perhaps because the beginning
121 of the data is a statistical "anomaly". */
122 if (i < 2000) {
123 lit_cost += 0.7 - ((double)(2000 - i) / 2000.0 * 0.35);
124 }
125 cost[i] = (float)lit_cost;
105 } 126 }
106 double lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo);
107 lit_cost += 0.02905;
108 if (lit_cost < 1.0) {
109 lit_cost *= 0.5;
110 lit_cost += 0.5;
111 }
112 // Make the first bytes more expensive -- seems to help, not sure why.
113 // Perhaps because the entropy source is changing its properties
114 // rapidly in the beginning of the file, perhaps because the beginning
115 // of the data is a statistical "anomaly".
116 if (i < 2000) {
117 lit_cost += 0.7 - (static_cast<double>(2000 - i) / 2000.0 * 0.35);
118 }
119 cost[i] = static_cast<float>(lit_cost);
120 } 127 }
121 } 128 }
122 129
123 void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask, 130 void BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
124 const uint8_t *data, float *cost) { 131 const uint8_t *data, float *cost) {
125 if (IsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) { 132 if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) {
126 EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost); 133 EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost);
127 return; 134 return;
128 } 135 } else {
129 size_t histogram[256] = { 0 }; 136 size_t histogram[256] = { 0 };
130 size_t window_half = 2000; 137 size_t window_half = 2000;
131 size_t in_window = std::min(window_half, len); 138 size_t in_window = BROTLI_MIN(size_t, window_half, len);
132 139
133 // Bootstrap histogram. 140 /* Bootstrap histogram. */
134 for (size_t i = 0; i < in_window; ++i) { 141 size_t i;
135 ++histogram[data[(pos + i) & mask]]; 142 for (i = 0; i < in_window; ++i) {
136 } 143 ++histogram[data[(pos + i) & mask]];
144 }
137 145
138 // Compute bit costs with sliding window. 146 /* Compute bit costs with sliding window. */
139 for (size_t i = 0; i < len; ++i) { 147 for (i = 0; i < len; ++i) {
140 if (i >= window_half) { 148 size_t histo;
141 // Remove a byte in the past. 149 if (i >= window_half) {
142 --histogram[data[(pos + i - window_half) & mask]]; 150 /* Remove a byte in the past. */
143 --in_window; 151 --histogram[data[(pos + i - window_half) & mask]];
152 --in_window;
153 }
154 if (i + window_half < len) {
155 /* Add a byte in the future. */
156 ++histogram[data[(pos + i + window_half) & mask]];
157 ++in_window;
158 }
159 histo = histogram[data[(pos + i) & mask]];
160 if (histo == 0) {
161 histo = 1;
162 }
163 {
164 double lit_cost = FastLog2(in_window) - FastLog2(histo);
165 lit_cost += 0.029;
166 if (lit_cost < 1.0) {
167 lit_cost *= 0.5;
168 lit_cost += 0.5;
169 }
170 cost[i] = (float)lit_cost;
171 }
144 } 172 }
145 if (i + window_half < len) {
146 // Add a byte in the future.
147 ++histogram[data[(pos + i + window_half) & mask]];
148 ++in_window;
149 }
150 size_t histo = histogram[data[(pos + i) & mask]];
151 if (histo == 0) {
152 histo = 1;
153 }
154 double lit_cost = FastLog2(in_window) - FastLog2(histo);
155 lit_cost += 0.029;
156 if (lit_cost < 1.0) {
157 lit_cost *= 0.5;
158 lit_cost += 0.5;
159 }
160 cost[i] = static_cast<float>(lit_cost);
161 } 173 }
162 } 174 }
163 175
164 176 #if defined(__cplusplus) || defined(c_plusplus)
165 } // namespace brotli 177 } /* extern "C" */
178 #endif
OLDNEW
« no previous file with comments | « third_party/brotli/enc/literal_cost.h ('k') | third_party/brotli/enc/literal_cost.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698