| OLD | NEW |
|---|
| (Empty) | |
| 1 diff --git a/third_party/zlib/crc32.c b/third_party/zlib/crc32.c |
| 2 index 979a719..09228ed 100644 |
| 3 --- a/third_party/zlib/crc32.c |
| 4 +++ b/third_party/zlib/crc32.c |
| 5 @@ -28,6 +28,8 @@ |
| 6 # endif /* !DYNAMIC_CRC_TABLE */ |
| 7 #endif /* MAKECRCH */ |
| 8 |
| 9 +#include "deflate.h" |
| 10 +#include "x86.h" |
| 11 #include "zutil.h" /* for STDC and FAR definitions */ |
| 12 |
| 13 #define local static |
| 14 @@ -423,3 +425,28 @@ uLong ZEXPORT crc32_combine64(crc1, crc2, len2) |
| 15 { |
| 16 return crc32_combine_(crc1, crc2, len2); |
| 17 } |
| 18 + |
| 19 +ZLIB_INTERNAL void crc_reset(deflate_state *const s) |
| 20 +{ |
| 21 + if (x86_cpu_enable_simd) { |
| 22 + crc_fold_init(s); |
| 23 + return; |
| 24 + } |
| 25 + s->strm->adler = crc32(0L, Z_NULL, 0); |
| 26 +} |
| 27 + |
| 28 +ZLIB_INTERNAL void crc_finalize(deflate_state *const s) |
| 29 +{ |
| 30 + if (x86_cpu_enable_simd) |
| 31 + s->strm->adler = crc_fold_512to32(s); |
| 32 +} |
| 33 + |
| 34 +ZLIB_INTERNAL void copy_with_crc(z_streamp strm, Bytef *dst, long size) |
| 35 +{ |
| 36 + if (x86_cpu_enable_simd) { |
| 37 + crc_fold_copy(strm->state, dst, strm->next_in, size); |
| 38 + return; |
| 39 + } |
| 40 + zmemcpy(dst, strm->next_in, size); |
| 41 + strm->adler = crc32(strm->adler, dst, size); |
| 42 +} |
| 43 diff --git a/third_party/zlib/crc_folding.c b/third_party/zlib/crc_folding.c |
| 44 new file mode 100644 |
| 45 index 0000000..48d7774 |
| 46 --- /dev/null |
| 47 +++ b/third_party/zlib/crc_folding.c |
| 48 @@ -0,0 +1,493 @@ |
| 49 +/* |
| 50 + * Compute the CRC32 using a parallelized folding approach with the PCLMULQDQ |
| 51 + * instruction. |
| 52 + * |
| 53 + * A white paper describing this algorithm can be found at: |
| 54 + * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fas
t-crc-computation-generic-polynomials-pclmulqdq-paper.pdf |
| 55 + * |
| 56 + * Copyright (C) 2013 Intel Corporation. All rights reserved. |
| 57 + * Authors: |
| 58 + * Wajdi Feghali <wajdi.k.feghali@intel.com> |
| 59 + * Jim Guilford <james.guilford@intel.com> |
| 60 + * Vinodh Gopal <vinodh.gopal@intel.com> |
| 61 + * Erdinc Ozturk <erdinc.ozturk@intel.com> |
| 62 + * Jim Kukunas <james.t.kukunas@linux.intel.com> |
| 63 + * |
| 64 + * For conditions of distribution and use, see copyright notice in zlib.h |
| 65 + */ |
| 66 + |
| 67 +#include "deflate.h" |
| 68 + |
| 69 +#include <inttypes.h> |
| 70 +#include <emmintrin.h> |
| 71 +#include <immintrin.h> |
| 72 +#include <wmmintrin.h> |
| 73 + |
| 74 +#define CRC_LOAD(s) \ |
| 75 + do { \ |
| 76 + __m128i xmm_crc0 = _mm_loadu_si128((__m128i *)s->crc0 + 0);\ |
| 77 + __m128i xmm_crc1 = _mm_loadu_si128((__m128i *)s->crc0 + 1);\ |
| 78 + __m128i xmm_crc2 = _mm_loadu_si128((__m128i *)s->crc0 + 2);\ |
| 79 + __m128i xmm_crc3 = _mm_loadu_si128((__m128i *)s->crc0 + 3);\ |
| 80 + __m128i xmm_crc_part = _mm_loadu_si128((__m128i *)s->crc0 + 4); |
| 81 + |
| 82 +#define CRC_SAVE(s) \ |
| 83 + _mm_storeu_si128((__m128i *)s->crc0 + 0, xmm_crc0);\ |
| 84 + _mm_storeu_si128((__m128i *)s->crc0 + 1, xmm_crc1);\ |
| 85 + _mm_storeu_si128((__m128i *)s->crc0 + 2, xmm_crc2);\ |
| 86 + _mm_storeu_si128((__m128i *)s->crc0 + 3, xmm_crc3);\ |
| 87 + _mm_storeu_si128((__m128i *)s->crc0 + 4, xmm_crc_part);\ |
| 88 + } while (0); |
| 89 + |
| 90 +ZLIB_INTERNAL void crc_fold_init(deflate_state *const s) |
| 91 +{ |
| 92 + CRC_LOAD(s) |
| 93 + |
| 94 + xmm_crc0 = _mm_cvtsi32_si128(0x9db42487); |
| 95 + xmm_crc1 = _mm_setzero_si128(); |
| 96 + xmm_crc2 = _mm_setzero_si128(); |
| 97 + xmm_crc3 = _mm_setzero_si128(); |
| 98 + |
| 99 + CRC_SAVE(s) |
| 100 + |
| 101 + s->strm->adler = 0; |
| 102 +} |
| 103 + |
| 104 +local void fold_1(deflate_state *const s, |
| 105 + __m128i *xmm_crc0, __m128i *xmm_crc1, |
| 106 + __m128i *xmm_crc2, __m128i *xmm_crc3) |
| 107 +{ |
| 108 + const __m128i xmm_fold4 = _mm_set_epi32( |
| 109 + 0x00000001, 0x54442bd4, |
| 110 + 0x00000001, 0xc6e41596); |
| 111 + |
| 112 + __m128i x_tmp3; |
| 113 + __m128 ps_crc0, ps_crc3, ps_res; |
| 114 + |
| 115 + x_tmp3 = *xmm_crc3; |
| 116 + |
| 117 + *xmm_crc3 = *xmm_crc0; |
| 118 + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); |
| 119 + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); |
| 120 + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); |
| 121 + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); |
| 122 + ps_res = _mm_xor_ps(ps_crc0, ps_crc3); |
| 123 + |
| 124 + *xmm_crc0 = *xmm_crc1; |
| 125 + *xmm_crc1 = *xmm_crc2; |
| 126 + *xmm_crc2 = x_tmp3; |
| 127 + *xmm_crc3 = _mm_castps_si128(ps_res); |
| 128 +} |
| 129 + |
| 130 +local void fold_2(deflate_state *const s, |
| 131 + __m128i *xmm_crc0, __m128i *xmm_crc1, |
| 132 + __m128i *xmm_crc2, __m128i *xmm_crc3) |
| 133 +{ |
| 134 + const __m128i xmm_fold4 = _mm_set_epi32( |
| 135 + 0x00000001, 0x54442bd4, |
| 136 + 0x00000001, 0xc6e41596); |
| 137 + |
| 138 + __m128i x_tmp3, x_tmp2; |
| 139 + __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res31, ps_res20; |
| 140 + |
| 141 + x_tmp3 = *xmm_crc3; |
| 142 + x_tmp2 = *xmm_crc2; |
| 143 + |
| 144 + *xmm_crc3 = *xmm_crc1; |
| 145 + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); |
| 146 + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); |
| 147 + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); |
| 148 + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); |
| 149 + ps_res31= _mm_xor_ps(ps_crc3, ps_crc1); |
| 150 + |
| 151 + *xmm_crc2 = *xmm_crc0; |
| 152 + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); |
| 153 + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); |
| 154 + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); |
| 155 + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); |
| 156 + ps_res20= _mm_xor_ps(ps_crc0, ps_crc2); |
| 157 + |
| 158 + *xmm_crc0 = x_tmp2; |
| 159 + *xmm_crc1 = x_tmp3; |
| 160 + *xmm_crc2 = _mm_castps_si128(ps_res20); |
| 161 + *xmm_crc3 = _mm_castps_si128(ps_res31); |
| 162 +} |
| 163 + |
| 164 +local void fold_3(deflate_state *const s, |
| 165 + __m128i *xmm_crc0, __m128i *xmm_crc1, |
| 166 + __m128i *xmm_crc2, __m128i *xmm_crc3) |
| 167 +{ |
| 168 + const __m128i xmm_fold4 = _mm_set_epi32( |
| 169 + 0x00000001, 0x54442bd4, |
| 170 + 0x00000001, 0xc6e41596); |
| 171 + |
| 172 + __m128i x_tmp3; |
| 173 + __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res32, ps_res21, ps_res10; |
| 174 + |
| 175 + x_tmp3 = *xmm_crc3; |
| 176 + |
| 177 + *xmm_crc3 = *xmm_crc2; |
| 178 + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); |
| 179 + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); |
| 180 + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); |
| 181 + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); |
| 182 + ps_res32 = _mm_xor_ps(ps_crc2, ps_crc3); |
| 183 + |
| 184 + *xmm_crc2 = *xmm_crc1; |
| 185 + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); |
| 186 + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); |
| 187 + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); |
| 188 + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); |
| 189 + ps_res21= _mm_xor_ps(ps_crc1, ps_crc2); |
| 190 + |
| 191 + *xmm_crc1 = *xmm_crc0; |
| 192 + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); |
| 193 + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); |
| 194 + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); |
| 195 + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); |
| 196 + ps_res10= _mm_xor_ps(ps_crc0, ps_crc1); |
| 197 + |
| 198 + *xmm_crc0 = x_tmp3; |
| 199 + *xmm_crc1 = _mm_castps_si128(ps_res10); |
| 200 + *xmm_crc2 = _mm_castps_si128(ps_res21); |
| 201 + *xmm_crc3 = _mm_castps_si128(ps_res32); |
| 202 +} |
| 203 + |
| 204 +local void fold_4(deflate_state *const s, |
| 205 + __m128i *xmm_crc0, __m128i *xmm_crc1, |
| 206 + __m128i *xmm_crc2, __m128i *xmm_crc3) |
| 207 +{ |
| 208 + const __m128i xmm_fold4 = _mm_set_epi32( |
| 209 + 0x00000001, 0x54442bd4, |
| 210 + 0x00000001, 0xc6e41596); |
| 211 + |
| 212 + __m128i x_tmp0, x_tmp1, x_tmp2, x_tmp3; |
| 213 + __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3; |
| 214 + __m128 ps_t0, ps_t1, ps_t2, ps_t3; |
| 215 + __m128 ps_res0, ps_res1, ps_res2, ps_res3; |
| 216 + |
| 217 + x_tmp0 = *xmm_crc0; |
| 218 + x_tmp1 = *xmm_crc1; |
| 219 + x_tmp2 = *xmm_crc2; |
| 220 + x_tmp3 = *xmm_crc3; |
| 221 + |
| 222 + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); |
| 223 + x_tmp0 = _mm_clmulepi64_si128(x_tmp0, xmm_fold4, 0x10); |
| 224 + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); |
| 225 + ps_t0 = _mm_castsi128_ps(x_tmp0); |
| 226 + ps_res0 = _mm_xor_ps(ps_crc0, ps_t0); |
| 227 + |
| 228 + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); |
| 229 + x_tmp1 = _mm_clmulepi64_si128(x_tmp1, xmm_fold4, 0x10); |
| 230 + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); |
| 231 + ps_t1 = _mm_castsi128_ps(x_tmp1); |
| 232 + ps_res1 = _mm_xor_ps(ps_crc1, ps_t1); |
| 233 + |
| 234 + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); |
| 235 + x_tmp2 = _mm_clmulepi64_si128(x_tmp2, xmm_fold4, 0x10); |
| 236 + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); |
| 237 + ps_t2 = _mm_castsi128_ps(x_tmp2); |
| 238 + ps_res2 = _mm_xor_ps(ps_crc2, ps_t2); |
| 239 + |
| 240 + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x01); |
| 241 + x_tmp3 = _mm_clmulepi64_si128(x_tmp3, xmm_fold4, 0x10); |
| 242 + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); |
| 243 + ps_t3 = _mm_castsi128_ps(x_tmp3); |
| 244 + ps_res3 = _mm_xor_ps(ps_crc3, ps_t3); |
| 245 + |
| 246 + *xmm_crc0 = _mm_castps_si128(ps_res0); |
| 247 + *xmm_crc1 = _mm_castps_si128(ps_res1); |
| 248 + *xmm_crc2 = _mm_castps_si128(ps_res2); |
| 249 + *xmm_crc3 = _mm_castps_si128(ps_res3); |
| 250 +} |
| 251 + |
| 252 +local const unsigned zalign(32) pshufb_shf_table[60] = { |
| 253 + 0x84838281,0x88878685,0x8c8b8a89,0x008f8e8d, /* shl 15 (16 - 1)/shr1 */ |
| 254 + 0x85848382,0x89888786,0x8d8c8b8a,0x01008f8e, /* shl 14 (16 - 3)/shr2 */ |
| 255 + 0x86858483,0x8a898887,0x8e8d8c8b,0x0201008f, /* shl 13 (16 - 4)/shr3 */ |
| 256 + 0x87868584,0x8b8a8988,0x8f8e8d8c,0x03020100, /* shl 12 (16 - 4)/shr4 */ |
| 257 + 0x88878685,0x8c8b8a89,0x008f8e8d,0x04030201, /* shl 11 (16 - 5)/shr5 */ |
| 258 + 0x89888786,0x8d8c8b8a,0x01008f8e,0x05040302, /* shl 10 (16 - 6)/shr6 */ |
| 259 + 0x8a898887,0x8e8d8c8b,0x0201008f,0x06050403, /* shl 9 (16 - 7)/shr7 */ |
| 260 + 0x8b8a8988,0x8f8e8d8c,0x03020100,0x07060504, /* shl 8 (16 - 8)/shr8 */ |
| 261 + 0x8c8b8a89,0x008f8e8d,0x04030201,0x08070605, /* shl 7 (16 - 9)/shr9 */ |
| 262 + 0x8d8c8b8a,0x01008f8e,0x05040302,0x09080706, /* shl 6 (16 -10)/shr10*/ |
| 263 + 0x8e8d8c8b,0x0201008f,0x06050403,0x0a090807, /* shl 5 (16 -11)/shr11*/ |
| 264 + 0x8f8e8d8c,0x03020100,0x07060504,0x0b0a0908, /* shl 4 (16 -12)/shr12*/ |
| 265 + 0x008f8e8d,0x04030201,0x08070605,0x0c0b0a09, /* shl 3 (16 -13)/shr13*/ |
| 266 + 0x01008f8e,0x05040302,0x09080706,0x0d0c0b0a, /* shl 2 (16 -14)/shr14*/ |
| 267 + 0x0201008f,0x06050403,0x0a090807,0x0e0d0c0b /* shl 1 (16 -15)/shr15*/ |
| 268 +}; |
| 269 + |
| 270 +local void partial_fold(deflate_state *const s, const size_t len, |
| 271 + __m128i *xmm_crc0, __m128i *xmm_crc1, |
| 272 + __m128i *xmm_crc2, __m128i *xmm_crc3, |
| 273 + __m128i *xmm_crc_part) |
| 274 +{ |
| 275 + |
| 276 + const __m128i xmm_fold4 = _mm_set_epi32( |
| 277 + 0x00000001, 0x54442bd4, |
| 278 + 0x00000001, 0xc6e41596); |
| 279 + const __m128i xmm_mask3 = _mm_set1_epi32(0x80808080); |
| 280 + |
| 281 + __m128i xmm_shl, xmm_shr, xmm_tmp1, xmm_tmp2, xmm_tmp3; |
| 282 + __m128i xmm_a0_0, xmm_a0_1; |
| 283 + __m128 ps_crc3, psa0_0, psa0_1, ps_res; |
| 284 + |
| 285 + xmm_shl = _mm_load_si128((__m128i *)pshufb_shf_table + (len - 1)); |
| 286 + xmm_shr = xmm_shl; |
| 287 + xmm_shr = _mm_xor_si128(xmm_shr, xmm_mask3); |
| 288 + |
| 289 + xmm_a0_0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shl); |
| 290 + |
| 291 + *xmm_crc0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shr); |
| 292 + xmm_tmp1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shl); |
| 293 + *xmm_crc0 = _mm_or_si128(*xmm_crc0, xmm_tmp1); |
| 294 + |
| 295 + *xmm_crc1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shr); |
| 296 + xmm_tmp2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shl); |
| 297 + *xmm_crc1 = _mm_or_si128(*xmm_crc1, xmm_tmp2); |
| 298 + |
| 299 + *xmm_crc2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shr); |
| 300 + xmm_tmp3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shl); |
| 301 + *xmm_crc2 = _mm_or_si128(*xmm_crc2, xmm_tmp3); |
| 302 + |
| 303 + *xmm_crc3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shr); |
| 304 + *xmm_crc_part = _mm_shuffle_epi8(*xmm_crc_part, xmm_shl); |
| 305 + *xmm_crc3 = _mm_or_si128(*xmm_crc3, *xmm_crc_part); |
| 306 + |
| 307 + xmm_a0_1 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x10); |
| 308 + xmm_a0_0 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x01); |
| 309 + |
| 310 + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); |
| 311 + psa0_0 = _mm_castsi128_ps(xmm_a0_0); |
| 312 + psa0_1 = _mm_castsi128_ps(xmm_a0_1); |
| 313 + |
| 314 + ps_res = _mm_xor_ps(ps_crc3, psa0_0); |
| 315 + ps_res = _mm_xor_ps(ps_res, psa0_1); |
| 316 + |
| 317 + *xmm_crc3 = _mm_castps_si128(ps_res); |
| 318 +} |
| 319 + |
| 320 +ZLIB_INTERNAL void crc_fold_copy(deflate_state *const s, |
| 321 + unsigned char *dst, const unsigned char *src, long len) |
| 322 +{ |
| 323 + unsigned long algn_diff; |
| 324 + __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3; |
| 325 + |
| 326 + CRC_LOAD(s) |
| 327 + |
| 328 + if (len < 16) { |
| 329 + if (len == 0) |
| 330 + return; |
| 331 + goto partial; |
| 332 + } |
| 333 + |
| 334 + algn_diff = 0 - (uintptr_t)src & 0xF; |
| 335 + if (algn_diff) { |
| 336 + xmm_crc_part = _mm_loadu_si128((__m128i *)src); |
| 337 + _mm_storeu_si128((__m128i *)dst, xmm_crc_part); |
| 338 + |
| 339 + dst += algn_diff; |
| 340 + src += algn_diff; |
| 341 + len -= algn_diff; |
| 342 + |
| 343 + partial_fold(s, algn_diff, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, |
| 344 + &xmm_crc_part); |
| 345 + } |
| 346 + |
| 347 + while ((len -= 64) >= 0) { |
| 348 + xmm_t0 = _mm_load_si128((__m128i *)src); |
| 349 + xmm_t1 = _mm_load_si128((__m128i *)src + 1); |
| 350 + xmm_t2 = _mm_load_si128((__m128i *)src + 2); |
| 351 + xmm_t3 = _mm_load_si128((__m128i *)src + 3); |
| 352 + |
| 353 + fold_4(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); |
| 354 + |
| 355 + _mm_storeu_si128((__m128i *)dst, xmm_t0); |
| 356 + _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); |
| 357 + _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); |
| 358 + _mm_storeu_si128((__m128i *)dst + 3, xmm_t3); |
| 359 + |
| 360 + xmm_crc0 = _mm_xor_si128(xmm_crc0, xmm_t0); |
| 361 + xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t1); |
| 362 + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t2); |
| 363 + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t3); |
| 364 + |
| 365 + src += 64; |
| 366 + dst += 64; |
| 367 + } |
| 368 + |
| 369 + /* |
| 370 + * len = num bytes left - 64 |
| 371 + */ |
| 372 + if (len + 16 >= 0) { |
| 373 + len += 16; |
| 374 + |
| 375 + xmm_t0 = _mm_load_si128((__m128i *)src); |
| 376 + xmm_t1 = _mm_load_si128((__m128i *)src + 1); |
| 377 + xmm_t2 = _mm_load_si128((__m128i *)src + 2); |
| 378 + |
| 379 + fold_3(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); |
| 380 + |
| 381 + _mm_storeu_si128((__m128i *)dst, xmm_t0); |
| 382 + _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); |
| 383 + _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); |
| 384 + |
| 385 + xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0); |
| 386 + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1); |
| 387 + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t2); |
| 388 + |
| 389 + if (len == 0) |
| 390 + goto done; |
| 391 + |
| 392 + dst += 48; |
| 393 + src += 48; |
| 394 + } else if (len + 32 >= 0) { |
| 395 + len += 32; |
| 396 + |
| 397 + xmm_t0 = _mm_load_si128((__m128i *)src); |
| 398 + xmm_t1 = _mm_load_si128((__m128i *)src + 1); |
| 399 + |
| 400 + fold_2(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); |
| 401 + |
| 402 + _mm_storeu_si128((__m128i *)dst, xmm_t0); |
| 403 + _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); |
| 404 + |
| 405 + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0); |
| 406 + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t1); |
| 407 + |
| 408 + if (len == 0) |
| 409 + goto done; |
| 410 + |
| 411 + dst += 32; |
| 412 + src += 32; |
| 413 + } else if (len + 48 >= 0) { |
| 414 + len += 48; |
| 415 + |
| 416 + xmm_t0 = _mm_load_si128((__m128i *)src); |
| 417 + |
| 418 + fold_1(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); |
| 419 + |
| 420 + _mm_storeu_si128((__m128i *)dst, xmm_t0); |
| 421 + |
| 422 + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t0); |
| 423 + |
| 424 + if (len == 0) |
| 425 + goto done; |
| 426 + |
| 427 + dst += 16; |
| 428 + src += 16; |
| 429 + } else { |
| 430 + len += 64; |
| 431 + if (len == 0) |
| 432 + goto done; |
| 433 + } |
| 434 + |
| 435 +partial: |
| 436 + |
| 437 +#if defined(_MSC_VER) |
| 438 + /* VS does not permit the use of _mm_set_epi64x in 32-bit builds */ |
| 439 + { |
| 440 + int32_t parts[4] = {0, 0, 0, 0}; |
| 441 + memcpy(&parts, src, len); |
| 442 + xmm_crc_part = _mm_set_epi32(parts[3], parts[2], parts[1], parts[0]); |
| 443 + } |
| 444 +#else |
| 445 + { |
| 446 + int64_t parts[2] = {0, 0}; |
| 447 + memcpy(&parts, src, len); |
| 448 + xmm_crc_part = _mm_set_epi64x(parts[1], parts[0]); |
| 449 + } |
| 450 +#endif |
| 451 + |
| 452 + _mm_storeu_si128((__m128i *)dst, xmm_crc_part); |
| 453 + partial_fold(s, len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, |
| 454 + &xmm_crc_part); |
| 455 +done: |
| 456 + CRC_SAVE(s) |
| 457 +} |
| 458 + |
| 459 +local const unsigned zalign(16) crc_k[] = { |
| 460 + 0xccaa009e, 0x00000000, /* rk1 */ |
| 461 + 0x751997d0, 0x00000001, /* rk2 */ |
| 462 + 0xccaa009e, 0x00000000, /* rk5 */ |
| 463 + 0x63cd6124, 0x00000001, /* rk6 */ |
| 464 + 0xf7011640, 0x00000001, /* rk7 */ |
| 465 + 0xdb710640, 0x00000001 /* rk8 */ |
| 466 +}; |
| 467 + |
| 468 +local const unsigned zalign(16) crc_mask[4] = { |
| 469 + 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000 |
| 470 +}; |
| 471 + |
| 472 +local const unsigned zalign(16) crc_mask2[4] = { |
| 473 + 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF |
| 474 +}; |
| 475 + |
| 476 +unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) |
| 477 +{ |
| 478 + const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask); |
| 479 + const __m128i xmm_mask2 = _mm_load_si128((__m128i *)crc_mask2); |
| 480 + |
| 481 + unsigned crc; |
| 482 + __m128i x_tmp0, x_tmp1, x_tmp2, crc_fold; |
| 483 + |
| 484 + CRC_LOAD(s) |
| 485 + |
| 486 + /* |
| 487 + * k1 |
| 488 + */ |
| 489 + crc_fold = _mm_load_si128((__m128i *)crc_k); |
| 490 + |
| 491 + x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10); |
| 492 + xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01); |
| 493 + xmm_crc1 = _mm_xor_si128(xmm_crc1, x_tmp0); |
| 494 + xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_crc0); |
| 495 + |
| 496 + x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10); |
| 497 + xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01); |
| 498 + xmm_crc2 = _mm_xor_si128(xmm_crc2, x_tmp1); |
| 499 + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_crc1); |
| 500 + |
| 501 + x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10); |
| 502 + xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01); |
| 503 + xmm_crc3 = _mm_xor_si128(xmm_crc3, x_tmp2); |
| 504 + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); |
| 505 + |
| 506 + /* |
| 507 + * k5 |
| 508 + */ |
| 509 + crc_fold = _mm_load_si128((__m128i *)crc_k + 1); |
| 510 + |
| 511 + xmm_crc0 = xmm_crc3; |
| 512 + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); |
| 513 + xmm_crc0 = _mm_srli_si128(xmm_crc0, 8); |
| 514 + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); |
| 515 + |
| 516 + xmm_crc0 = xmm_crc3; |
| 517 + xmm_crc3 = _mm_slli_si128(xmm_crc3, 4); |
| 518 + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); |
| 519 + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); |
| 520 + xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask2); |
| 521 + |
| 522 + /* |
| 523 + * k7 |
| 524 + */ |
| 525 + xmm_crc1 = xmm_crc3; |
| 526 + xmm_crc2 = xmm_crc3; |
| 527 + crc_fold = _mm_load_si128((__m128i *)crc_k + 2); |
| 528 + |
| 529 + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); |
| 530 + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); |
| 531 + xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask); |
| 532 + |
| 533 + xmm_crc2 = xmm_crc3; |
| 534 + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); |
| 535 + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); |
| 536 + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1); |
| 537 + |
| 538 + crc = _mm_extract_epi32(xmm_crc3, 2); |
| 539 + return ~crc; |
| 540 + CRC_SAVE(s) |
| 541 +} |
| 542 diff --git a/third_party/zlib/deflate.c b/third_party/zlib/deflate.c |
| 543 index 36b26ee..3c5e576 100644 |
| 544 --- a/third_party/zlib/deflate.c |
| 545 +++ b/third_party/zlib/deflate.c |
| 546 @@ -48,8 +48,9 @@ |
| 547 */ |
| 548 |
| 549 /* @(#) $Id$ */ |
| 550 - |
| 551 +#include <assert.h> |
| 552 #include "deflate.h" |
| 553 +#include "x86.h" |
| 554 |
| 555 const char deflate_copyright[] = |
| 556 " deflate 1.2.8 Copyright 1995-2013 Jean-loup Gailly and Mark Adler "; |
| 557 @@ -85,7 +86,7 @@ local block_state deflate_huff OF((deflate_state *s, int flu
sh)); |
| 558 local void lm_init OF((deflate_state *s)); |
| 559 local void putShortMSB OF((deflate_state *s, uInt b)); |
| 560 local void flush_pending OF((z_streamp strm)); |
| 561 -local int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); |
| 562 + |
| 563 #ifdef ASMV |
| 564 void match_init OF((void)); /* asm code initialization */ |
| 565 uInt longest_match OF((deflate_state *s, IPos cur_match, int clas)); |
| 566 @@ -98,6 +99,23 @@ local void check_match OF((deflate_state *s, IPos start, IPo
s match, |
| 567 int length)); |
| 568 #endif |
| 569 |
| 570 +/* For fill_window_sse.c to use */ |
| 571 +ZLIB_INTERNAL int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); |
| 572 + |
| 573 +/* From crc32.c */ |
| 574 +extern void ZLIB_INTERNAL crc_reset(deflate_state *const s); |
| 575 +extern void ZLIB_INTERNAL crc_finalize(deflate_state *const s); |
| 576 +extern void ZLIB_INTERNAL copy_with_crc(z_streamp strm, Bytef *dst, long size); |
| 577 + |
| 578 +#ifdef _MSC_VER |
| 579 +#define INLINE __inline |
| 580 +#else |
| 581 +#define INLINE inline |
| 582 +#endif |
| 583 + |
| 584 +/* Inline optimisation */ |
| 585 +local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str); |
| 586 + |
| 587 /* =========================================================================== |
| 588 * Local data |
| 589 */ |
| 590 @@ -167,7 +185,6 @@ struct static_tree_desc_s {int dummy;}; /* for buggy compile
rs */ |
| 591 */ |
| 592 #define UPDATE_HASH(s,h,c) (h = (((h)<<s->hash_shift) ^ (c)) & s->hash_mask) |
| 593 |
| 594 - |
| 595 /* =========================================================================== |
| 596 * Insert string str in the dictionary and set match_head to the previous head |
| 597 * of the hash chain (the most recent string with same hash key). Return |
| 598 @@ -178,17 +195,28 @@ struct static_tree_desc_s {int dummy;}; /* for buggy compi
lers */ |
| 599 * input characters and the first MIN_MATCH bytes of str are valid |
| 600 * (except for the last MIN_MATCH-1 bytes of the input file). |
| 601 */ |
| 602 +local INLINE Pos insert_string_c(deflate_state *const s, const Pos str) |
| 603 +{ |
| 604 + Pos ret; |
| 605 + |
| 606 + UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]); |
| 607 #ifdef FASTEST |
| 608 -#define INSERT_STRING(s, str, match_head) \ |
| 609 - (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ |
| 610 - match_head = s->head[s->ins_h], \ |
| 611 - s->head[s->ins_h] = (Pos)(str)) |
| 612 + ret = s->head[s->ins_h]; |
| 613 #else |
| 614 -#define INSERT_STRING(s, str, match_head) \ |
| 615 - (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ |
| 616 - match_head = s->prev[(str) & s->w_mask] = s->head[s->ins_h], \ |
| 617 - s->head[s->ins_h] = (Pos)(str)) |
| 618 + ret = s->prev[str & s->w_mask] = s->head[s->ins_h]; |
| 619 #endif |
| 620 + s->head[s->ins_h] = str; |
| 621 + |
| 622 + return ret; |
| 623 +} |
| 624 + |
| 625 +local INLINE Pos insert_string(deflate_state *const s, const Pos str) |
| 626 +{ |
| 627 + if (x86_cpu_enable_simd) |
| 628 + return insert_string_sse(s, str); |
| 629 + return insert_string_c(s, str); |
| 630 +} |
| 631 + |
| 632 |
| 633 /* =========================================================================== |
| 634 * Initialize the hash table (avoiding 64K overflow for 16 bit systems). |
| 635 @@ -222,6 +250,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, m
emLevel, strategy, |
| 636 const char *version; |
| 637 int stream_size; |
| 638 { |
| 639 + unsigned window_padding = 8; |
| 640 deflate_state *s; |
| 641 int wrap = 1; |
| 642 static const char my_version[] = ZLIB_VERSION; |
| 643 @@ -231,6 +260,8 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, m
emLevel, strategy, |
| 644 * output size for (length,distance) codes is <= 24 bits. |
| 645 */ |
| 646 |
| 647 + x86_check_features(); |
| 648 + |
| 649 if (version == Z_NULL || version[0] != my_version[0] || |
| 650 stream_size != sizeof(z_stream)) { |
| 651 return Z_VERSION_ERROR; |
| 652 @@ -286,12 +317,17 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits,
memLevel, strategy, |
| 653 s->w_size = 1 << s->w_bits; |
| 654 s->w_mask = s->w_size - 1; |
| 655 |
| 656 - s->hash_bits = memLevel + 7; |
| 657 + if (x86_cpu_enable_simd) { |
| 658 + s->hash_bits = 15; |
| 659 + } else { |
| 660 + s->hash_bits = memLevel + 7; |
| 661 + } |
| 662 + |
| 663 s->hash_size = 1 << s->hash_bits; |
| 664 s->hash_mask = s->hash_size - 1; |
| 665 s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH); |
| 666 |
| 667 - s->window = (Bytef *) ZALLOC(strm, s->w_size, 2*sizeof(Byte)); |
| 668 + s->window = (Bytef *) ZALLOC(strm, s->w_size + window_padding, 2*sizeof(Byt
e)); |
| 669 s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); |
| 670 s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); |
| 671 s->class_bitmap = NULL; |
| 672 @@ -696,7 +732,7 @@ int ZEXPORT deflate (strm, flush) |
| 673 if (s->status == INIT_STATE) { |
| 674 #ifdef GZIP |
| 675 if (s->wrap == 2) { |
| 676 - strm->adler = crc32(0L, Z_NULL, 0); |
| 677 + crc_reset(s); |
| 678 put_byte(s, 31); |
| 679 put_byte(s, 139); |
| 680 put_byte(s, 8); |
| 681 @@ -975,6 +1011,7 @@ int ZEXPORT deflate (strm, flush) |
| 682 /* Write the trailer */ |
| 683 #ifdef GZIP |
| 684 if (s->wrap == 2) { |
| 685 + crc_finalize(s); |
| 686 put_byte(s, (Byte)(strm->adler & 0xff)); |
| 687 put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); |
| 688 put_byte(s, (Byte)((strm->adler >> 16) & 0xff)); |
| 689 @@ -1097,7 +1134,7 @@ int ZEXPORT deflateCopy (dest, source) |
| 690 * allocating a large strm->next_in buffer and copying from it. |
| 691 * (See also flush_pending()). |
| 692 */ |
| 693 -local int read_buf(strm, buf, size) |
| 694 +ZLIB_INTERNAL int read_buf(strm, buf, size) |
| 695 z_streamp strm; |
| 696 Bytef *buf; |
| 697 unsigned size; |
| 698 @@ -1109,15 +1146,16 @@ local int read_buf(strm, buf, size) |
| 699 |
| 700 strm->avail_in -= len; |
| 701 |
| 702 - zmemcpy(buf, strm->next_in, len); |
| 703 - if (strm->state->wrap == 1) { |
| 704 - strm->adler = adler32(strm->adler, buf, len); |
| 705 - } |
| 706 #ifdef GZIP |
| 707 - else if (strm->state->wrap == 2) { |
| 708 - strm->adler = crc32(strm->adler, buf, len); |
| 709 - } |
| 710 + if (strm->state->wrap == 2) |
| 711 + copy_with_crc(strm, buf, len); |
| 712 + else |
| 713 #endif |
| 714 + { |
| 715 + zmemcpy(buf, strm->next_in, len); |
| 716 + if (strm->state->wrap == 1) |
| 717 + strm->adler = adler32(strm->adler, buf, len); |
| 718 + } |
| 719 strm->next_in += len; |
| 720 strm->total_in += len; |
| 721 |
| 722 @@ -1530,7 +1568,19 @@ local void check_match(s, start, match, length) |
| 723 * performed for at least two bytes (required for the zip translate_eol |
| 724 * option -- not supported here). |
| 725 */ |
| 726 -local void fill_window(s) |
| 727 +local void fill_window_c(deflate_state *s); |
| 728 + |
| 729 +local void fill_window(deflate_state *s) |
| 730 +{ |
| 731 + if (x86_cpu_enable_simd) { |
| 732 + fill_window_sse(s); |
| 733 + return; |
| 734 + } |
| 735 + |
| 736 + fill_window_c(s); |
| 737 +} |
| 738 + |
| 739 +local void fill_window_c(s) |
| 740 deflate_state *s; |
| 741 { |
| 742 register unsigned n, m; |
| 743 @@ -1819,7 +1869,7 @@ local block_state deflate_fast(s, flush, clas) |
| 744 */ |
| 745 hash_head = NIL; |
| 746 if (s->lookahead >= MIN_MATCH) { |
| 747 - INSERT_STRING(s, s->strstart, hash_head); |
| 748 + hash_head = insert_string(s, s->strstart); |
| 749 } |
| 750 |
| 751 /* Find the longest match, discarding those <= prev_length. |
| 752 @@ -1850,7 +1900,7 @@ local block_state deflate_fast(s, flush, clas) |
| 753 s->match_length--; /* string at strstart already in table */ |
| 754 do { |
| 755 s->strstart++; |
| 756 - INSERT_STRING(s, s->strstart, hash_head); |
| 757 + hash_head = insert_string(s, s->strstart); |
| 758 /* strstart never exceeds WSIZE-MAX_MATCH, so there are |
| 759 * always MIN_MATCH bytes ahead. |
| 760 */ |
| 761 @@ -1935,7 +1985,7 @@ local block_state deflate_slow(s, flush, clas) |
| 762 */ |
| 763 hash_head = NIL; |
| 764 if (s->lookahead >= MIN_MATCH) { |
| 765 - INSERT_STRING(s, s->strstart, hash_head); |
| 766 + hash_head = insert_string(s, s->strstart); |
| 767 } |
| 768 |
| 769 /* Find the longest match, discarding those <= prev_length. |
| 770 @@ -2004,7 +2054,7 @@ local block_state deflate_slow(s, flush, clas) |
| 771 s->prev_length -= 2; |
| 772 do { |
| 773 if (++s->strstart <= max_insert) { |
| 774 - INSERT_STRING(s, s->strstart, hash_head); |
| 775 + hash_head = insert_string(s, s->strstart); |
| 776 } |
| 777 } while (--s->prev_length != 0); |
| 778 s->match_available = 0; |
| 779 @@ -2164,3 +2214,37 @@ local block_state deflate_huff(s, flush) |
| 780 FLUSH_BLOCK(s, 0); |
| 781 return block_done; |
| 782 } |
| 783 + |
| 784 +/* Safe to inline this as GCC/clang will use inline asm and Visual Studio will |
| 785 + * use intrinsic without extra params |
| 786 + */ |
| 787 +local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str) |
| 788 +{ |
| 789 + Pos ret; |
| 790 + unsigned *ip, val, h = 0; |
| 791 + |
| 792 + ip = (unsigned *)&s->window[str]; |
| 793 + val = *ip; |
| 794 + |
| 795 + if (s->level >= 6) |
| 796 + val &= 0xFFFFFF; |
| 797 + |
| 798 +/* Windows clang should use inline asm */ |
| 799 +#if defined(_MSC_VER) && !defined(__clang__) |
| 800 + h = _mm_crc32_u32(h, val); |
| 801 +#elif defined(__i386__) || defined(__amd64__) |
| 802 + __asm__ __volatile__ ( |
| 803 + "crc32 %1,%0\n\t" |
| 804 + : "+r" (h) |
| 805 + : "r" (val) |
| 806 + ); |
| 807 +#else |
| 808 + /* This should never happen */ |
| 809 + assert(0); |
| 810 +#endif |
| 811 + |
| 812 + ret = s->head[h & s->hash_mask]; |
| 813 + s->head[h & s->hash_mask] = str; |
| 814 + s->prev[str & s->w_mask] = ret; |
| 815 + return ret; |
| 816 +} |
| 817 diff --git a/third_party/zlib/deflate.h b/third_party/zlib/deflate.h |
| 818 index c795034..c61e4ab 100644 |
| 819 --- a/third_party/zlib/deflate.h |
| 820 +++ b/third_party/zlib/deflate.h |
| 821 @@ -109,7 +109,7 @@ typedef struct internal_state { |
| 822 uInt gzindex; /* where in extra, name, or comment */ |
| 823 Byte method; /* can only be DEFLATED */ |
| 824 int last_flush; /* value of flush param for previous deflate call */ |
| 825 - |
| 826 + unsigned zalign(16) crc0[4 * 5]; |
| 827 /* used by deflate.c: */ |
| 828 |
| 829 uInt w_size; /* LZ77 window size (32K by default) */ |
| 830 @@ -348,4 +348,14 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, c
harf *buf, |
| 831 flush = _tr_tally(s, distance, length) |
| 832 #endif |
| 833 |
| 834 +/* Functions that are SIMD optimised on x86 */ |
| 835 +void ZLIB_INTERNAL crc_fold_init(deflate_state* const s); |
| 836 +void ZLIB_INTERNAL crc_fold_copy(deflate_state* const s, |
| 837 + unsigned char* dst, |
| 838 + const unsigned char* src, |
| 839 + long len); |
| 840 +unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state* const s); |
| 841 + |
| 842 +void ZLIB_INTERNAL fill_window_sse(deflate_state* s); |
| 843 + |
| 844 #endif /* DEFLATE_H */ |
| 845 diff --git a/third_party/zlib/fill_window_sse.c b/third_party/zlib/fill_window_s
se.c |
| 846 new file mode 100644 |
| 847 index 0000000..949ccce |
| 848 --- /dev/null |
| 849 +++ b/third_party/zlib/fill_window_sse.c |
| 850 @@ -0,0 +1,175 @@ |
| 851 +/* |
| 852 + * Fill Window with SSE2-optimized hash shifting |
| 853 + * |
| 854 + * Copyright (C) 2013 Intel Corporation |
| 855 + * Authors: |
| 856 + * Arjan van de Ven <arjan@linux.intel.com> |
| 857 + * Jim Kukunas <james.t.kukunas@linux.intel.com> |
| 858 + * |
| 859 + * For conditions of distribution and use, see copyright notice in zlib.h |
| 860 + */ |
| 861 + |
| 862 +#include <immintrin.h> |
| 863 +#include "deflate.h" |
| 864 + |
| 865 +#define UPDATE_HASH(s,h,i) \ |
| 866 + {\ |
| 867 + if (s->level < 6) { \ |
| 868 + h = (3483 * (s->window[i]) +\ |
| 869 + 23081* (s->window[i+1]) +\ |
| 870 + 6954 * (s->window[i+2]) +\ |
| 871 + 20947* (s->window[i+3])) & s->hash_mask;\ |
| 872 + } else {\ |
| 873 + h = (25881* (s->window[i]) +\ |
| 874 + 24674* (s->window[i+1]) +\ |
| 875 + 25811* (s->window[i+2])) & s->hash_mask;\ |
| 876 + }\ |
| 877 + }\ |
| 878 + |
| 879 +extern int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); |
| 880 + |
| 881 +void fill_window_sse(deflate_state *s) |
| 882 +{ |
| 883 + const __m128i xmm_wsize = _mm_set1_epi16(s->w_size); |
| 884 + |
| 885 + register unsigned n; |
| 886 + register Posf *p; |
| 887 + unsigned more; /* Amount of free space at the end of the window. */ |
| 888 + uInt wsize = s->w_size; |
| 889 + |
| 890 + Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); |
| 891 + |
| 892 + do { |
| 893 + more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); |
| 894 + |
| 895 + /* Deal with !@#$% 64K limit: */ |
| 896 + if (sizeof(int) <= 2) { |
| 897 + if (more == 0 && s->strstart == 0 && s->lookahead == 0) { |
| 898 + more = wsize; |
| 899 + |
| 900 + } else if (more == (unsigned)(-1)) { |
| 901 + /* Very unlikely, but possible on 16 bit machine if |
| 902 + * strstart == 0 && lookahead == 1 (input done a byte at time) |
| 903 + */ |
| 904 + more--; |
| 905 + } |
| 906 + } |
| 907 + |
| 908 + /* If the window is almost full and there is insufficient lookahead, |
| 909 + * move the upper half to the lower one to make room in the upper half. |
| 910 + */ |
| 911 + if (s->strstart >= wsize+MAX_DIST(s)) { |
| 912 + |
| 913 + zmemcpy(s->window, s->window+wsize, (unsigned)wsize); |
| 914 + s->match_start -= wsize; |
| 915 + s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ |
| 916 + s->block_start -= (long) wsize; |
| 917 + |
| 918 + /* Slide the hash table (could be avoided with 32 bit values |
| 919 + at the expense of memory usage). We slide even when level == 0 |
| 920 + to keep the hash table consistent if we switch back to level > 0 |
| 921 + later. (Using level 0 permanently is not an optimal usage of |
| 922 + zlib, so we don't care about this pathological case.) |
| 923 + */ |
| 924 + n = s->hash_size; |
| 925 + p = &s->head[n]; |
| 926 + p -= 8; |
| 927 + do { |
| 928 + __m128i value, result; |
| 929 + |
| 930 + value = _mm_loadu_si128((__m128i *)p); |
| 931 + result = _mm_subs_epu16(value, xmm_wsize); |
| 932 + _mm_storeu_si128((__m128i *)p, result); |
| 933 + |
| 934 + p -= 8; |
| 935 + n -= 8; |
| 936 + } while (n > 0); |
| 937 + |
| 938 + n = wsize; |
| 939 +#ifndef FASTEST |
| 940 + p = &s->prev[n]; |
| 941 + p -= 8; |
| 942 + do { |
| 943 + __m128i value, result; |
| 944 + |
| 945 + value = _mm_loadu_si128((__m128i *)p); |
| 946 + result = _mm_subs_epu16(value, xmm_wsize); |
| 947 + _mm_storeu_si128((__m128i *)p, result); |
| 948 + |
| 949 + p -= 8; |
| 950 + n -= 8; |
| 951 + } while (n > 0); |
| 952 +#endif |
| 953 + more += wsize; |
| 954 + } |
| 955 + if (s->strm->avail_in == 0) break; |
| 956 + |
| 957 + /* If there was no sliding: |
| 958 + * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && |
| 959 + * more == window_size - lookahead - strstart |
| 960 + * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) |
| 961 + * => more >= window_size - 2*WSIZE + 2 |
| 962 + * In the BIG_MEM or MMAP case (not yet supported), |
| 963 + * window_size == input_size + MIN_LOOKAHEAD && |
| 964 + * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD. |
| 965 + * Otherwise, window_size == 2*WSIZE so more >= 2. |
| 966 + * If there was sliding, more >= WSIZE. So in all cases, more >= 2. |
| 967 + */ |
| 968 + Assert(more >= 2, "more < 2"); |
| 969 + |
| 970 + n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); |
| 971 + s->lookahead += n; |
| 972 + |
| 973 + /* Initialize the hash value now that we have some input: */ |
| 974 + if (s->lookahead >= MIN_MATCH) { |
| 975 + uInt str = s->strstart; |
| 976 + s->ins_h = s->window[str]; |
| 977 + if (str >= 1) |
| 978 + UPDATE_HASH(s, s->ins_h, str + 1 - (MIN_MATCH-1)); |
| 979 +#if MIN_MATCH != 3 |
| 980 + Call UPDATE_HASH() MIN_MATCH-3 more times |
| 981 +#endif |
| 982 + } |
| 983 + /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, |
| 984 + * but this is not important since only literal bytes will be emitted. |
| 985 + */ |
| 986 + |
| 987 + } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); |
| 988 + |
| 989 + /* If the WIN_INIT bytes after the end of the current data have never been |
| 990 + * written, then zero those bytes in order to avoid memory check reports of |
| 991 + * the use of uninitialized (or uninitialised as Julian writes) bytes by |
| 992 + * the longest match routines. Update the high water mark for the next |
| 993 + * time through here. WIN_INIT is set to MAX_MATCH since the longest match |
| 994 + * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. |
| 995 + */ |
| 996 + if (s->high_water < s->window_size) { |
| 997 + ulg curr = s->strstart + (ulg)(s->lookahead); |
| 998 + ulg init; |
| 999 + |
| 1000 + if (s->high_water < curr) { |
| 1001 + /* Previous high water mark below current data -- zero WIN_INIT |
| 1002 + * bytes or up to end of window, whichever is less. |
| 1003 + */ |
| 1004 + init = s->window_size - curr; |
| 1005 + if (init > WIN_INIT) |
| 1006 + init = WIN_INIT; |
| 1007 + zmemzero(s->window + curr, (unsigned)init); |
| 1008 + s->high_water = curr + init; |
| 1009 + } |
| 1010 + else if (s->high_water < (ulg)curr + WIN_INIT) { |
| 1011 + /* High water mark at or above current data, but below current data |
| 1012 + * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up |
| 1013 + * to end of window, whichever is less. |
| 1014 + */ |
| 1015 + init = (ulg)curr + WIN_INIT - s->high_water; |
| 1016 + if (init > s->window_size - s->high_water) |
| 1017 + init = s->window_size - s->high_water; |
| 1018 + zmemzero(s->window + s->high_water, (unsigned)init); |
| 1019 + s->high_water += init; |
| 1020 + } |
| 1021 + } |
| 1022 + |
| 1023 + Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, |
| 1024 + "not enough room for search"); |
| 1025 +} |
| 1026 diff --git a/third_party/zlib/simd_stub.c b/third_party/zlib/simd_stub.c |
| 1027 new file mode 100644 |
| 1028 index 0000000..796f1f6 |
| 1029 --- /dev/null |
| 1030 +++ b/third_party/zlib/simd_stub.c |
| 1031 @@ -0,0 +1,35 @@ |
| 1032 +/* simd_stub.c -- stub implementations |
| 1033 +* Copyright (C) 2014 Intel Corporation |
| 1034 +* For conditions of distribution and use, see copyright notice in zlib.h |
| 1035 +*/ |
| 1036 +#include <assert.h> |
| 1037 + |
| 1038 +#include "deflate.h" |
| 1039 +#include "x86.h" |
| 1040 + |
| 1041 +int x86_cpu_enable_simd = 0; |
| 1042 + |
| 1043 +void ZLIB_INTERNAL crc_fold_init(deflate_state *const s) { |
| 1044 + assert(0); |
| 1045 +} |
| 1046 + |
| 1047 +void ZLIB_INTERNAL crc_fold_copy(deflate_state *const s, |
| 1048 + unsigned char *dst, |
| 1049 + const unsigned char *src, |
| 1050 + long len) { |
| 1051 + assert(0); |
| 1052 +} |
| 1053 + |
| 1054 +unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) { |
| 1055 + assert(0); |
| 1056 + return 0; |
| 1057 +} |
| 1058 + |
| 1059 +void ZLIB_INTERNAL fill_window_sse(deflate_state *s) |
| 1060 +{ |
| 1061 + assert(0); |
| 1062 +} |
| 1063 + |
| 1064 +void x86_check_features(void) |
| 1065 +{ |
| 1066 +} |
| 1067 diff --git a/third_party/zlib/x86.c b/third_party/zlib/x86.c |
| 1068 new file mode 100644 |
| 1069 index 0000000..e6532fd |
| 1070 --- /dev/null |
| 1071 +++ b/third_party/zlib/x86.c |
| 1072 @@ -0,0 +1,91 @@ |
| 1073 +/* |
| 1074 + * x86 feature check |
| 1075 + * |
| 1076 + * Copyright (C) 2013 Intel Corporation. All rights reserved. |
| 1077 + * Author: |
| 1078 + * Jim Kukunas |
| 1079 + * |
| 1080 + * For conditions of distribution and use, see copyright notice in zlib.h |
| 1081 + */ |
| 1082 + |
| 1083 +#include "x86.h" |
| 1084 + |
| 1085 +int x86_cpu_enable_simd = 0; |
| 1086 + |
| 1087 +#ifndef _MSC_VER |
| 1088 +#include <pthread.h> |
| 1089 + |
| 1090 +pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; |
| 1091 +static void _x86_check_features(void); |
| 1092 + |
| 1093 +void x86_check_features(void) |
| 1094 +{ |
| 1095 + pthread_once(&cpu_check_inited_once, _x86_check_features); |
| 1096 +} |
| 1097 + |
| 1098 +static void _x86_check_features(void) |
| 1099 +{ |
| 1100 + int x86_cpu_has_sse2; |
| 1101 + int x86_cpu_has_sse42; |
| 1102 + int x86_cpu_has_pclmulqdq; |
| 1103 + unsigned eax, ebx, ecx, edx; |
| 1104 + |
| 1105 + eax = 1; |
| 1106 +#ifdef __i386__ |
| 1107 + __asm__ __volatile__ ( |
| 1108 + "xchg %%ebx, %1\n\t" |
| 1109 + "cpuid\n\t" |
| 1110 + "xchg %1, %%ebx\n\t" |
| 1111 + : "+a" (eax), "=S" (ebx), "=c" (ecx), "=d" (edx) |
| 1112 + ); |
| 1113 +#else |
| 1114 + __asm__ __volatile__ ( |
| 1115 + "cpuid\n\t" |
| 1116 + : "+a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) |
| 1117 + ); |
| 1118 +#endif /* (__i386__) */ |
| 1119 + |
| 1120 + x86_cpu_has_sse2 = edx & 0x4000000; |
| 1121 + x86_cpu_has_sse42 = ecx & 0x100000; |
| 1122 + x86_cpu_has_pclmulqdq = ecx & 0x2; |
| 1123 + |
| 1124 + x86_cpu_enable_simd = x86_cpu_has_sse2 && |
| 1125 + x86_cpu_has_sse42 && |
| 1126 + x86_cpu_has_pclmulqdq; |
| 1127 +} |
| 1128 +#else |
| 1129 +#include <intrin.h> |
| 1130 +#include <windows.h> |
| 1131 + |
| 1132 +static BOOL CALLBACK _x86_check_features(PINIT_ONCE once, |
| 1133 + PVOID param, |
| 1134 + PVOID *context); |
| 1135 +static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; |
| 1136 + |
| 1137 +void x86_check_features(void) |
| 1138 +{ |
| 1139 + InitOnceExecuteOnce(&cpu_check_inited_once, _x86_check_features, |
| 1140 + NULL, NULL); |
| 1141 +} |
| 1142 + |
| 1143 +static BOOL CALLBACK _x86_check_features(PINIT_ONCE once, |
| 1144 + PVOID param, |
| 1145 + PVOID *context) |
| 1146 +{ |
| 1147 + int x86_cpu_has_sse2; |
| 1148 + int x86_cpu_has_sse42; |
| 1149 + int x86_cpu_has_pclmulqdq; |
| 1150 + int regs[4]; |
| 1151 + |
| 1152 + __cpuid(regs, 1); |
| 1153 + |
| 1154 + x86_cpu_has_sse2 = regs[3] & 0x4000000; |
| 1155 + x86_cpu_has_sse42= regs[2] & 0x100000; |
| 1156 + x86_cpu_has_pclmulqdq = regs[2] & 0x2; |
| 1157 + |
| 1158 + x86_cpu_enable_simd = x86_cpu_has_sse2 && |
| 1159 + x86_cpu_has_sse42 && |
| 1160 + x86_cpu_has_pclmulqdq; |
| 1161 + return TRUE; |
| 1162 +} |
| 1163 +#endif /* _MSC_VER */ |
| 1164 diff --git a/third_party/zlib/x86.h b/third_party/zlib/x86.h |
| 1165 new file mode 100644 |
| 1166 index 0000000..ac3d180 |
| 1167 --- /dev/null |
| 1168 +++ b/third_party/zlib/x86.h |
| 1169 @@ -0,0 +1,13 @@ |
| 1170 +/* x86.h -- check for x86 CPU features |
| 1171 +* Copyright (C) 2013 Intel Corporation Jim Kukunas |
| 1172 +* For conditions of distribution and use, see copyright notice in zlib.h |
| 1173 +*/ |
| 1174 + |
| 1175 +#ifndef X86_H |
| 1176 +#define X86_H |
| 1177 + |
| 1178 +extern int x86_cpu_enable_simd; |
| 1179 + |
| 1180 +void x86_check_features(void); |
| 1181 + |
| 1182 +#endif /* X86_H */ |
| 1183 -- |
| 1184 2.7.4 |
| 1185 |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1955383002:
Update Zlib to version 1.2.8 (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master