Performance optimization 😅

benchmarks/Android.bp

@@ -33,7 +33,7 @@ license {
 cc_defaults {
     name: "bionic-benchmarks-defaults",
     cflags: [
-        "-O2",
+        "-O3",
         "-fno-builtin",
         "-Wall",
         "-Wextra",
@@ -86,6 +86,7 @@ cc_defaults {
         "-Wextra",
         "-Werror",
         "-Wunused",
+        "-O3",
     ],
 }
 

benchmarks/time_benchmark.cpp

@@ -187,3 +187,13 @@ void BM_time_localtime_r(benchmark::State& state) {
   }
 }
 BIONIC_BENCHMARK(BM_time_localtime_r);
+
+void BM_time_strftime(benchmark::State& state) {
+  char buf[128];
+  time_t t = 0;
+  struct tm* tm = gmtime(&t);
+  while (state.KeepRunning()) {
+    strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", tm);
+  }
+}
+BIONIC_BENCHMARK(BM_time_strftime);

libc/Android.bp

@@ -55,6 +55,7 @@ libc_common_flags = [
     "-Wno-deprecated-declarations",
     "-Wno-gcc-compat",
     "-Wframe-larger-than=2048",
+    "-O3",
 
     // Try to catch typical 32-bit assumptions that break with 64-bit pointers.
     "-Werror=pointer-to-int-cast",
@@ -68,6 +69,13 @@ libc_common_flags = [
 
     // GWP-ASan requires platform TLS.
     "-fno-emulated-tls",
+
+    // We know clang does a lot of harm by rewriting what we've said, and sadly
+    // never see any good it does, so let's just ask it to do what we say...
+    // (The specific motivating example was clang turning a loop that would only
+    // ever touch 0, 1, or 2 bytes into a call to memset, which was never going
+    // to amortize.)
+    "-fno-builtin",
 ]
 
 // Define some common cflags
@@ -119,9 +127,6 @@ cc_defaults {
         malloc_pattern_fill_contents: {
             cflags: ["-DSCUDO_PATTERN_FILL_CONTENTS"],
         },
-        malloc_not_svelte: {
-            cflags: ["-DUSE_SCUDO"],
-        },
     },
 }
 
@@ -150,7 +155,6 @@ cc_defaults {
         "libc_jemalloc_wrapper",
     ],
     header_libs: ["gwp_asan_headers"],
-    product_variables: libc_scudo_product_variables,
 }
 
 // Functions not implemented by jemalloc directly, or that need to
@@ -639,6 +643,7 @@ cc_library_static {
                 "upstream-openbsd/lib/libc/string/strcpy.c",
                 "upstream-openbsd/lib/libc/string/stpcpy.c",
                 "upstream-openbsd/lib/libc/string/strcat.c",
+                "upstream-openbsd/lib/libc/string/memchr.c",
             ],
         },
         arm64: {
@@ -831,6 +836,7 @@ cc_library_static {
         arm: {
             asflags: libc_common_flags + ["-mno-restrict-it"],
             srcs: [
+                "arch-arm/generic/bionic/memchr.S",
                 "arch-arm/generic/bionic/memcmp.S",
                 "arch-arm/generic/bionic/memmove.S",
                 "arch-arm/generic/bionic/memset.S",
@@ -2099,6 +2105,7 @@ cc_defaults {
         "-Wno-gcc-compat",
         "-Wall",
         "-Werror",
+        "-O3",
     ],
     sanitize: {
         never: true,

libc/arch-arm/cortex-a15/bionic/strlen.S

@@ -1,165 +1,145 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *  * Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  * Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+/* Copyright (c) 2010-2011,2013 Linaro Limited
+   All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+      * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+      * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+      * Neither the name of Linaro Limited nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+
 /*
- * Copyright (c) 2013 ARM Ltd
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the company may not be used to endorse or promote
- *    products derived from this software without specific prior written
- *    permission.
- *
- * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+   Assumes:
+   ARMv6T2, AArch32
+
+   Adapted to Bionic by Bernhard Rosenkränzer <bero@linaro.org>
  */
 
 #include <private/bionic_asm.h>
 
-    .syntax unified
+#ifdef __ARMEB__
+#define S2LO		lsl
+#define S2HI		lsr
+#else
+#define S2LO		lsr
+#define S2HI		lsl
+#endif
 
+    .text
     .thumb
-    .thumb_func
-
-ENTRY(strlen_a15)
-    pld     [r0, #0]
-    mov     r1, r0
-
-    ands    r3, r0, #7
-    beq     .L_mainloop
-
-    // Align to a double word (64 bits).
-    rsb     r3, r3, #8
-    lsls    ip, r3, #31
-    beq     .L_align_to_32
-
-    ldrb    r2, [r1], #1
-    cbz     r2, .L_update_count_and_return
-
-.L_align_to_32:
-    bcc     .L_align_to_64
-    ands    ip, r3, #2
-    beq     .L_align_to_64
-
-    ldrb    r2, [r1], #1
-    cbz     r2, .L_update_count_and_return
-    ldrb    r2, [r1], #1
-    cbz     r2, .L_update_count_and_return
-
-.L_align_to_64:
-    tst     r3, #4
-    beq     .L_mainloop
-    ldr     r3, [r1], #4
-
-    sub     ip, r3, #0x01010101
-    bic     ip, ip, r3
-    ands    ip, ip, #0x80808080
-    bne     .L_zero_in_second_register
-
-    .p2align 2
-.L_mainloop:
-    ldrd    r2, r3, [r1], #8
-
-    pld     [r1, #64]
-
-    sub     ip, r2, #0x01010101
-    bic     ip, ip, r2
-    ands    ip, ip, #0x80808080
-    bne     .L_zero_in_first_register
-
-    sub     ip, r3, #0x01010101
-    bic     ip, ip, r3
-    ands    ip, ip, #0x80808080
-    bne     .L_zero_in_second_register
-    b       .L_mainloop
-
-.L_update_count_and_return:
-    sub     r0, r1, r0
-    sub     r0, r0, #1
-    bx      lr
-
-.L_zero_in_first_register:
-    sub     r0, r1, r0
-    lsls    r3, ip, #17
-    bne     .L_sub8_and_return
-    bcs     .L_sub7_and_return
-    lsls    ip, ip, #1
-    bne     .L_sub6_and_return
-
-    sub     r0, r0, #5
-    bx      lr
-
-.L_sub8_and_return:
-    sub     r0, r0, #8
-    bx      lr
-
-.L_sub7_and_return:
-    sub     r0, r0, #7
-    bx      lr
-
-.L_sub6_and_return:
-    sub     r0, r0, #6
-    bx      lr
-
-.L_zero_in_second_register:
-    sub     r0, r1, r0
-    lsls    r3, ip, #17
-    bne     .L_sub4_and_return
-    bcs     .L_sub3_and_return
-    lsls    ip, ip, #1
-    bne     .L_sub2_and_return
+    .syntax unified
 
-    sub     r0, r0, #1
-    bx      lr
+/* Parameters and result.  */
+#define srcin		r0
+#define result		r0
 
-.L_sub4_and_return:
-    sub     r0, r0, #4
-    bx      lr
+/* Internal variables.  */
+#define src		r1
+#define data1a		r2
+#define data1b		r3
+#define const_m1	r12
+#define const_0		r4
+#define tmp1		r4		/* Overlaps const_0  */
+#define tmp2		r5
 
-.L_sub3_and_return:
-    sub     r0, r0, #3
+ENTRY(strlen_a15)
+    .p2align 6
+    pld     [srcin, #0]
+    strd    r4, r5, [sp, #-8]!
+    bic     src, srcin, #7
+    mvn     const_m1, #0
+    ands    tmp1, srcin, #7		/* (8 - bytes) to alignment.  */
+    pld     [src, #32]
+    bne.w   .L_misaligned8
+    mov     const_0, #0
+    mov     result, #-8
+.L_loop_aligned:
+	/* Bytes 0-7.  */
+    ldrd    data1a, data1b, [src]
+    pld     [src, #64]
+    add     result, result, #8
+.L_start_realigned:
+    uadd8   data1a, data1a, const_m1	/* Saturating GE<0:3> set.  */
+    sel     data1a, const_0, const_m1	/* Select based on GE<0:3>.  */
+    uadd8   data1b, data1b, const_m1
+    sel     data1b, data1a, const_m1	/* Only used if d1a == 0.  */
+    cbnz    data1b, .L_null_found
+
+	/* Bytes 8-15.  */
+    ldrd    data1a, data1b, [src, #8]
+    uadd8   data1a, data1a, const_m1	/* Saturating GE<0:3> set.  */
+    add     result, result, #8
+    sel     data1a, const_0, const_m1	/* Select based on GE<0:3>.  */
+    uadd8   data1b, data1b, const_m1
+    sel     data1b, data1a, const_m1	/* Only used if d1a == 0.  */
+    cbnz    data1b, .L_null_found
+
+	/* Bytes 16-23.  */
+    ldrd    data1a, data1b, [src, #16]
+    uadd8   data1a, data1a, const_m1	/* Saturating GE<0:3> set.  */
+    add     result, result, #8
+    sel     data1a, const_0, const_m1	/* Select based on GE<0:3>.  */
+    uadd8   data1b, data1b, const_m1
+    sel     data1b, data1a, const_m1	/* Only used if d1a == 0.  */
+    cbnz    data1b, .L_null_found
+
+	/* Bytes 24-31.  */
+    ldrd    data1a, data1b, [src, #24]
+    add     src, src, #32
+    uadd8   data1a, data1a, const_m1	/* Saturating GE<0:3> set.  */
+    add     result, result, #8
+    sel     data1a, const_0, const_m1	/* Select based on GE<0:3>.  */
+    uadd8   data1b, data1b, const_m1
+    sel     data1b, data1a, const_m1	/* Only used if d1a == 0.  */
+    cmp     data1b, #0
+    beq     .L_loop_aligned
+
+.L_null_found:
+    cmp     data1a, #0
+    itt     eq
+    addeq   result, result, #4
+    moveq   data1a, data1b
+#ifndef __ARMEB__
+    rev     data1a, data1a
+#endif
+    clz     data1a, data1a
+    ldrd    r4, r5, [sp], #8
+    add     result, result, data1a, lsr #3	/* Bits -> Bytes.  */
     bx      lr
 
-.L_sub2_and_return:
-    sub     r0, r0, #2
-    bx      lr
+.L_misaligned8:
+    ldrd    data1a, data1b, [src]
+    and     tmp2, tmp1, #3
+    rsb     result, tmp1, #0
+    lsl     tmp2, tmp2, #3			/* Bytes -> bits.  */
+    tst     tmp1, #4
+    pld     [src, #64]
+    S2HI    tmp2, const_m1, tmp2
+    orn     data1a, data1a, tmp2
+    itt     ne
+    ornne   data1b, data1b, tmp2
+    movne   data1a, const_m1
+    mov     const_0, #0
+    b       .L_start_realigned
 END(strlen_a15)