| OLD | NEW |
| 1 /* Copyright 2013 Google Inc. All Rights Reserved. | 1 /* Copyright 2013 Google Inc. All Rights Reserved. |
| 2 | 2 |
| 3 Distributed under MIT license. | 3 Distributed under MIT license. |
| 4 See file LICENSE for detail or copy at https://opensource.org/licenses/MIT | 4 See file LICENSE for detail or copy at https://opensource.org/licenses/MIT |
| 5 */ | 5 */ |
| 6 | 6 |
| 7 // Literal cost model to allow backward reference replacement to be efficient. | 7 /* Literal cost model to allow backward reference replacement to be efficient. |
| 8 */ |
| 8 | 9 |
| 9 #include "./literal_cost.h" | 10 #include "./literal_cost.h" |
| 10 | 11 |
| 11 #include <math.h> | 12 #include <brotli/types.h> |
| 12 #include <algorithm> | |
| 13 | |
| 14 #include "./fast_log.h" | 13 #include "./fast_log.h" |
| 15 #include "./types.h" | 14 #include "./port.h" |
| 16 #include "./utf8_util.h" | 15 #include "./utf8_util.h" |
| 17 | 16 |
| 18 namespace brotli { | 17 #if defined(__cplusplus) || defined(c_plusplus) |
| 18 extern "C" { |
| 19 #endif |
| 19 | 20 |
| 20 static size_t UTF8Position(size_t last, size_t c, size_t clamp) { | 21 static size_t UTF8Position(size_t last, size_t c, size_t clamp) { |
| 21 if (c < 128) { | 22 if (c < 128) { |
| 22 return 0; // Next one is the 'Byte 1' again. | 23 return 0; /* Next one is the 'Byte 1' again. */ |
| 23 } else if (c >= 192) { // Next one is the 'Byte 2' of utf-8 encoding. | 24 } else if (c >= 192) { /* Next one is the 'Byte 2' of utf-8 encoding. */ |
| 24 return std::min<size_t>(1, clamp); | 25 return BROTLI_MIN(size_t, 1, clamp); |
| 25 } else { | 26 } else { |
| 26 // Let's decide over the last byte if this ends the sequence. | 27 /* Let's decide over the last byte if this ends the sequence. */ |
| 27 if (last < 0xe0) { | 28 if (last < 0xe0) { |
| 28 return 0; // Completed two or three byte coding. | 29 return 0; /* Completed two or three byte coding. */ |
| 29 } else { // Next one is the 'Byte 3' of utf-8 encoding. | 30 } else { /* Next one is the 'Byte 3' of utf-8 encoding. */ |
| 30 return std::min<size_t>(2, clamp); | 31 return BROTLI_MIN(size_t, 2, clamp); |
| 31 } | 32 } |
| 32 } | 33 } |
| 33 } | 34 } |
| 34 | 35 |
| 35 static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask, | 36 static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask, |
| 36 const uint8_t *data) { | 37 const uint8_t *data) { |
| 37 size_t counts[3] = { 0 }; | 38 size_t counts[3] = { 0 }; |
| 38 size_t max_utf8 = 1; // should be 2, but 1 compresses better. | 39 size_t max_utf8 = 1; /* should be 2, but 1 compresses better. */ |
| 39 size_t last_c = 0; | 40 size_t last_c = 0; |
| 40 size_t utf8_pos = 0; | 41 size_t utf8_pos = 0; |
| 41 for (size_t i = 0; i < len; ++i) { | 42 size_t i; |
| 43 for (i = 0; i < len; ++i) { |
| 42 size_t c = data[(pos + i) & mask]; | 44 size_t c = data[(pos + i) & mask]; |
| 43 utf8_pos = UTF8Position(last_c, c, 2); | 45 utf8_pos = UTF8Position(last_c, c, 2); |
| 44 ++counts[utf8_pos]; | 46 ++counts[utf8_pos]; |
| 45 last_c = c; | 47 last_c = c; |
| 46 } | 48 } |
| 47 if (counts[2] < 500) { | 49 if (counts[2] < 500) { |
| 48 max_utf8 = 1; | 50 max_utf8 = 1; |
| 49 } | 51 } |
| 50 if (counts[1] + counts[2] < 25) { | 52 if (counts[1] + counts[2] < 25) { |
| 51 max_utf8 = 0; | 53 max_utf8 = 0; |
| 52 } | 54 } |
| 53 return max_utf8; | 55 return max_utf8; |
| 54 } | 56 } |
| 55 | 57 |
| 56 static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask, | 58 static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask, |
| 57 const uint8_t *data, float *cost) { | 59 const uint8_t *data, float *cost) { |
| 58 | 60 /* max_utf8 is 0 (normal ASCII single byte modeling), |
| 59 // max_utf8 is 0 (normal ascii single byte modeling), | 61 1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */ |
| 60 // 1 (for 2-byte utf-8 modeling), or 2 (for 3-byte utf-8 modeling). | |
| 61 const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data); | 62 const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data); |
| 62 size_t histogram[3][256] = { { 0 } }; | 63 size_t histogram[3][256] = { { 0 } }; |
| 63 size_t window_half = 495; | 64 size_t window_half = 495; |
| 64 size_t in_window = std::min(window_half, len); | 65 size_t in_window = BROTLI_MIN(size_t, window_half, len); |
| 65 size_t in_window_utf8[3] = { 0 }; | 66 size_t in_window_utf8[3] = { 0 }; |
| 66 | 67 |
| 67 // Bootstrap histograms. | 68 |
| 68 size_t last_c = 0; | 69 size_t i; |
| 69 size_t utf8_pos = 0; | 70 { /* Bootstrap histograms. */ |
| 70 for (size_t i = 0; i < in_window; ++i) { | 71 size_t last_c = 0; |
| 71 size_t c = data[(pos + i) & mask]; | 72 size_t utf8_pos = 0; |
| 72 ++histogram[utf8_pos][c]; | 73 for (i = 0; i < in_window; ++i) { |
| 73 ++in_window_utf8[utf8_pos]; | 74 size_t c = data[(pos + i) & mask]; |
| 74 utf8_pos = UTF8Position(last_c, c, max_utf8); | 75 ++histogram[utf8_pos][c]; |
| 75 last_c = c; | 76 ++in_window_utf8[utf8_pos]; |
| 77 utf8_pos = UTF8Position(last_c, c, max_utf8); |
| 78 last_c = c; |
| 79 } |
| 76 } | 80 } |
| 77 | 81 |
| 78 // Compute bit costs with sliding window. | 82 /* Compute bit costs with sliding window. */ |
| 79 for (size_t i = 0; i < len; ++i) { | 83 for (i = 0; i < len; ++i) { |
| 80 if (i >= window_half) { | 84 if (i >= window_half) { |
| 81 // Remove a byte in the past. | 85 /* Remove a byte in the past. */ |
| 82 size_t c = i < window_half + 1 ? | 86 size_t c = |
| 83 0 : data[(pos + i - window_half - 1) & mask]; | 87 i < window_half + 1 ? 0 : data[(pos + i - window_half - 1) & mask]; |
| 84 size_t last_c = i < window_half + 2 ? | 88 size_t last_c = |
| 85 0 : data[(pos + i - window_half - 2) & mask]; | 89 i < window_half + 2 ? 0 : data[(pos + i - window_half - 2) & mask]; |
| 86 size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8); | 90 size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8); |
| 87 --histogram[utf8_pos2][data[(pos + i - window_half) & mask]]; | 91 --histogram[utf8_pos2][data[(pos + i - window_half) & mask]]; |
| 88 --in_window_utf8[utf8_pos2]; | 92 --in_window_utf8[utf8_pos2]; |
| 89 } | 93 } |
| 90 if (i + window_half < len) { | 94 if (i + window_half < len) { |
| 91 // Add a byte in the future. | 95 /* Add a byte in the future. */ |
| 92 size_t c = data[(pos + i + window_half - 1) & mask]; | 96 size_t c = data[(pos + i + window_half - 1) & mask]; |
| 93 size_t last_c = data[(pos + i + window_half - 2) & mask]; | 97 size_t last_c = data[(pos + i + window_half - 2) & mask]; |
| 94 size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8); | 98 size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8); |
| 95 ++histogram[utf8_pos2][data[(pos + i + window_half) & mask]]; | 99 ++histogram[utf8_pos2][data[(pos + i + window_half) & mask]]; |
| 96 ++in_window_utf8[utf8_pos2]; | 100 ++in_window_utf8[utf8_pos2]; |
| 97 } | 101 } |
| 98 size_t c = i < 1 ? 0 : data[(pos + i - 1) & mask]; | 102 { |
| 99 size_t last_c = i < 2 ? 0 : data[(pos + i - 2) & mask]; | 103 size_t c = i < 1 ? 0 : data[(pos + i - 1) & mask]; |
| 100 size_t utf8_pos = UTF8Position(last_c, c, max_utf8); | 104 size_t last_c = i < 2 ? 0 : data[(pos + i - 2) & mask]; |
| 101 size_t masked_pos = (pos + i) & mask; | 105 size_t utf8_pos = UTF8Position(last_c, c, max_utf8); |
| 102 size_t histo = histogram[utf8_pos][data[masked_pos]]; | 106 size_t masked_pos = (pos + i) & mask; |
| 103 if (histo == 0) { | 107 size_t histo = histogram[utf8_pos][data[masked_pos]]; |
| 104 histo = 1; | 108 double lit_cost; |
| 109 if (histo == 0) { |
| 110 histo = 1; |
| 111 } |
| 112 lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo); |
| 113 lit_cost += 0.02905; |
| 114 if (lit_cost < 1.0) { |
| 115 lit_cost *= 0.5; |
| 116 lit_cost += 0.5; |
| 117 } |
| 118 /* Make the first bytes more expensive -- seems to help, not sure why. |
| 119 Perhaps because the entropy source is changing its properties |
| 120 rapidly in the beginning of the file, perhaps because the beginning |
| 121 of the data is a statistical "anomaly". */ |
| 122 if (i < 2000) { |
| 123 lit_cost += 0.7 - ((double)(2000 - i) / 2000.0 * 0.35); |
| 124 } |
| 125 cost[i] = (float)lit_cost; |
| 105 } | 126 } |
| 106 double lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo); | |
| 107 lit_cost += 0.02905; | |
| 108 if (lit_cost < 1.0) { | |
| 109 lit_cost *= 0.5; | |
| 110 lit_cost += 0.5; | |
| 111 } | |
| 112 // Make the first bytes more expensive -- seems to help, not sure why. | |
| 113 // Perhaps because the entropy source is changing its properties | |
| 114 // rapidly in the beginning of the file, perhaps because the beginning | |
| 115 // of the data is a statistical "anomaly". | |
| 116 if (i < 2000) { | |
| 117 lit_cost += 0.7 - (static_cast<double>(2000 - i) / 2000.0 * 0.35); | |
| 118 } | |
| 119 cost[i] = static_cast<float>(lit_cost); | |
| 120 } | 127 } |
| 121 } | 128 } |
| 122 | 129 |
| 123 void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask, | 130 void BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask, |
| 124 const uint8_t *data, float *cost) { | 131 const uint8_t *data, float *cost) { |
| 125 if (IsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) { | 132 if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) { |
| 126 EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost); | 133 EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost); |
| 127 return; | 134 return; |
| 128 } | 135 } else { |
| 129 size_t histogram[256] = { 0 }; | 136 size_t histogram[256] = { 0 }; |
| 130 size_t window_half = 2000; | 137 size_t window_half = 2000; |
| 131 size_t in_window = std::min(window_half, len); | 138 size_t in_window = BROTLI_MIN(size_t, window_half, len); |
| 132 | 139 |
| 133 // Bootstrap histogram. | 140 /* Bootstrap histogram. */ |
| 134 for (size_t i = 0; i < in_window; ++i) { | 141 size_t i; |
| 135 ++histogram[data[(pos + i) & mask]]; | 142 for (i = 0; i < in_window; ++i) { |
| 136 } | 143 ++histogram[data[(pos + i) & mask]]; |
| 144 } |
| 137 | 145 |
| 138 // Compute bit costs with sliding window. | 146 /* Compute bit costs with sliding window. */ |
| 139 for (size_t i = 0; i < len; ++i) { | 147 for (i = 0; i < len; ++i) { |
| 140 if (i >= window_half) { | 148 size_t histo; |
| 141 // Remove a byte in the past. | 149 if (i >= window_half) { |
| 142 --histogram[data[(pos + i - window_half) & mask]]; | 150 /* Remove a byte in the past. */ |
| 143 --in_window; | 151 --histogram[data[(pos + i - window_half) & mask]]; |
| 152 --in_window; |
| 153 } |
| 154 if (i + window_half < len) { |
| 155 /* Add a byte in the future. */ |
| 156 ++histogram[data[(pos + i + window_half) & mask]]; |
| 157 ++in_window; |
| 158 } |
| 159 histo = histogram[data[(pos + i) & mask]]; |
| 160 if (histo == 0) { |
| 161 histo = 1; |
| 162 } |
| 163 { |
| 164 double lit_cost = FastLog2(in_window) - FastLog2(histo); |
| 165 lit_cost += 0.029; |
| 166 if (lit_cost < 1.0) { |
| 167 lit_cost *= 0.5; |
| 168 lit_cost += 0.5; |
| 169 } |
| 170 cost[i] = (float)lit_cost; |
| 171 } |
| 144 } | 172 } |
| 145 if (i + window_half < len) { | |
| 146 // Add a byte in the future. | |
| 147 ++histogram[data[(pos + i + window_half) & mask]]; | |
| 148 ++in_window; | |
| 149 } | |
| 150 size_t histo = histogram[data[(pos + i) & mask]]; | |
| 151 if (histo == 0) { | |
| 152 histo = 1; | |
| 153 } | |
| 154 double lit_cost = FastLog2(in_window) - FastLog2(histo); | |
| 155 lit_cost += 0.029; | |
| 156 if (lit_cost < 1.0) { | |
| 157 lit_cost *= 0.5; | |
| 158 lit_cost += 0.5; | |
| 159 } | |
| 160 cost[i] = static_cast<float>(lit_cost); | |
| 161 } | 173 } |
| 162 } | 174 } |
| 163 | 175 |
| 164 | 176 #if defined(__cplusplus) || defined(c_plusplus) |
| 165 } // namespace brotli | 177 } /* extern "C" */ |
| 178 #endif |
| OLD | NEW |