NEON implementation for Adler32

third_party/zlib/neon_adler32.patch

+From d62eac3b93d61dc07c7b4c0d1738db14415b6309 Mon Sep 17 00:00:00 2001
+From: Adenilson Cavalcanti <adenilson.cavalcanti@arm.com>
+Date: Mon, 30 Jan 2017 15:30:38 -0800
+Subject: [PATCH] NEON implementation for Adler32
+
+The checksum is calculated in the uncompressed PNG data
+and can be made much faster by using SIMD.
+
+Tests in ARMv8 yielded an improvement of about 3x
+(e.g. walltime was 350ms x 125ms for a 4096x4096 bytes
+executed 30 times).
+---
+ third_party/zlib/BUILD.gn        |   5 ++
+ third_party/zlib/README.chromium |   4 +-
+ third_party/zlib/adler32.c       |   8 +++
+ third_party/zlib/neon_adler32.c  | 120 +++++++++++++++++++++++++++++++++++++++
+ third_party/zlib/neon_adler32.h  |  13 +++++
+ 5 files changed, 149 insertions(+), 1 deletion(-)
+ create mode 100644 third_party/zlib/neon_adler32.c
+ create mode 100644 third_party/zlib/neon_adler32.h
+
+diff --git a/third_party/zlib/BUILD.gn b/third_party/zlib/BUILD.gn
+index 5086563..09dfa5f 100644
+--- a/third_party/zlib/BUILD.gn
++++ b/third_party/zlib/BUILD.gn
+@@ -73,6 +73,11 @@ static_library("zlib") {
+ 
+   if (!is_ios && (current_cpu == "x86" || current_cpu == "x64")) {
+     sources += [ "x86.c" ]
++  } else if (current_cpu == "arm" || current_cpu == "arm64") {
++    sources += [
++      "neon_adler32.c",
++      "neon_adler32.h",
++    ]
+   }
+ 
+   configs -= [ "//build/config/compiler:chromium_code" ]
+diff --git a/third_party/zlib/README.chromium b/third_party/zlib/README.chromium
+index fe6bc10..166eb33 100644
+--- a/third_party/zlib/README.chromium
++++ b/third_party/zlib/README.chromium
+@@ -24,5 +24,7 @@ Local Modifications:
+    additions.
+  - google.patch contains changes from the upstream version, mostly related to
+    the build.
+- - Intel SIMD optimisations from https://github.com/jtkukunas/zlib/ have been
++ - Intel SIMD optimizations from https://github.com/jtkukunas/zlib/ have been
+    integrated. These changes are reflected in simd.patch.
++ - NEON SIMD optimizations for Adler32 checksum were integrated. Changes
++   are reflected in neon_adler32.patch.
+\ No newline at end of file
+diff --git a/third_party/zlib/adler32.c b/third_party/zlib/adler32.c
+index d0be438..0b74c72 100644
+--- a/third_party/zlib/adler32.c
++++ b/third_party/zlib/adler32.c
+@@ -6,6 +6,9 @@
+ /* @(#) $Id$ */
+ 
+ #include "zutil.h"
++#ifdef __ARM_NEON__
++#include "neon_adler32.h"
++#endif
+ 
+ local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2));
+ 
+@@ -65,6 +68,11 @@ uLong ZEXPORT adler32_z(adler, buf, len)
+     const Bytef *buf;
+     z_size_t len;
+ {
++#ifdef __ARM_NEON__
++    if (len > 31)
++        return NEON_adler32(adler, buf, len);
++#endif
++
+     unsigned long sum2;
+     unsigned n;
+ 
+diff --git a/third_party/zlib/neon_adler32.c b/third_party/zlib/neon_adler32.c
+new file mode 100644
+index 0000000..a0f95fb
+--- /dev/null
++++ b/third_party/zlib/neon_adler32.c
+@@ -0,0 +1,120 @@
++/* Copyright 2017 The Chromium Authors. All rights reserved.
++ * Use of this source code is governed by a BSD-style license that can be
++ * found in the LICENSE file.
++*/
++
++#include "neon_adler32.h"
++#ifdef __ARM_NEON__
++#include <arm_neon.h>
++
++static void NEON_accum32(uint32_t *s, const unsigned char *buf,
++                         unsigned int len)
++{
++    static const uint8_t taps[32] = {
++        32, 31, 30, 29, 28, 27, 26, 25,
++        24, 23, 22, 21, 20, 19, 18, 17,
++        16, 15, 14, 13, 12, 11, 10, 9,
++        8, 7, 6, 5, 4, 3, 2, 1 };
++
++    uint32x2_t adacc2, s2acc2, as;
++    uint8x16_t t0 = vld1q_u8(taps), t1 = vld1q_u8(taps + 16);
++
++    uint32x4_t adacc = vdupq_n_u32(0), s2acc = vdupq_n_u32(0);
++    adacc = vsetq_lane_u32(s[0], adacc, 0);
++    s2acc = vsetq_lane_u32(s[1], s2acc, 0);
++
++    while (len >= 2) {
++        uint8x16_t d0 = vld1q_u8(buf), d1 = vld1q_u8(buf + 16);
++        uint16x8_t adler, sum2;
++        s2acc = vaddq_u32(s2acc, vshlq_n_u32(adacc, 5));
++        adler = vpaddlq_u8(       d0);
++        adler = vpadalq_u8(adler, d1);
++        sum2 = vmull_u8(      vget_low_u8(t0), vget_low_u8(d0));
++        sum2 = vmlal_u8(sum2, vget_high_u8(t0), vget_high_u8(d0));
++        sum2 = vmlal_u8(sum2, vget_low_u8(t1), vget_low_u8(d1));
++        sum2 = vmlal_u8(sum2, vget_high_u8(t1), vget_high_u8(d1));
++        adacc = vpadalq_u16(adacc, adler);
++        s2acc = vpadalq_u16(s2acc, sum2);
++        len -= 2;
++        buf += 32;
++    }
++
++    while (len > 0) {
++        uint8x16_t d0 = vld1q_u8(buf);
++        uint16x8_t adler, sum2;
++        s2acc = vaddq_u32(s2acc, vshlq_n_u32(adacc, 4));
++        adler = vpaddlq_u8(d0);
++        sum2 = vmull_u8(      vget_low_u8(t1), vget_low_u8(d0));
++        sum2 = vmlal_u8(sum2, vget_high_u8(t1), vget_high_u8(d0));
++        adacc = vpadalq_u16(adacc, adler);
++        s2acc = vpadalq_u16(s2acc, sum2);
++        buf += 16;
++        len--;
++    }
++
++    adacc2 = vpadd_u32(vget_low_u32(adacc), vget_high_u32(adacc));
++    s2acc2 = vpadd_u32(vget_low_u32(s2acc), vget_high_u32(s2acc));
++    as = vpadd_u32(adacc2, s2acc2);
++    s[0] = vget_lane_u32(as, 0);
++    s[1] = vget_lane_u32(as, 1);
++}
++
++static void NEON_handle_tail(uint32_t *pair, const unsigned char *buf,
++                             unsigned int len)
++{
++    /* Oldie K&R code integration. */
++    unsigned int i;
++    for (i = 0; i < len; ++i) {
++        pair[0] += buf[i];
++        pair[1] += pair[0];
++    }
++}
++
++unsigned long NEON_adler32(unsigned long adler, const unsigned char *buf,
++                           const unsigned int len)
++{
++    /* The largest prime smaller than 65536. */
++    const uint32_t M_BASE = 65521;
++    /* This is the threshold where doing accumulation may overflow. */
++    const int M_NMAX = 5552;
++
++    unsigned long sum2;
++    uint32_t pair[2];
++    int n = M_NMAX;
++    unsigned int done = 0;
++    /* Oldie K&R code integration. */
++    unsigned int i;
++
++    /* Split Adler-32 into component sums, it can be supplied by
++     * the caller sites (e.g. in a PNG file).
++     */
++    sum2 = (adler >> 16) & 0xffff;
++    adler &= 0xffff;
++    pair[0] = adler;
++    pair[1] = sum2;
++
++    for (i = 0; i < len; i += n) {
++        if ((i + n) > len)
++            n = len - i;
++
++        if (n < 16)
++            break;
++
++        NEON_accum32(pair, buf + i, n / 16);
++        pair[0] %= M_BASE;
++        pair[1] %= M_BASE;
++
++        done += (n / 16) * 16;
++    }
++
++    /* Handle the tail elements. */
++    if (done < len) {
++        NEON_handle_tail(pair, (buf + done), len - done);
++        pair[0] %= M_BASE;
++        pair[1] %= M_BASE;
++    }
++
++    /* D = B * 65536 + A, see: https://en.wikipedia.org/wiki/Adler-32. */
++    return (pair[1] << 16) | pair[0];
++}
++#endif
+diff --git a/third_party/zlib/neon_adler32.h b/third_party/zlib/neon_adler32.h
+new file mode 100644
+index 0000000..febe9a1
+--- /dev/null
++++ b/third_party/zlib/neon_adler32.h
+@@ -0,0 +1,13 @@
++/* Copyright 2017 The Chromium Authors. All rights reserved.
++ * Use of this source code is governed by a BSD-style license that can be
++ * found in the LICENSE file.
++*/
++#ifndef __NEON_ADLER32__
++#define __NEON_ADLER32__
++
++#ifdef __ARM_NEON__
++unsigned long NEON_adler32(unsigned long adler,
++                           const unsigned char* buf,
++                           const unsigned int len);
++#endif
++#endif
+-- 
+2.7.4
+