Optimize std::transform for vector<bool> (#5769)

Co-authored-by: Stephan T. Lavavej <[email protected]>

Author: AlexGuteniev

benchmarks/CMakeLists.txt

@@ -142,3 +142,4 @@ add_benchmark(vector_bool_copy src/vector_bool_copy.cpp)
 add_benchmark(vector_bool_copy_n src/vector_bool_copy_n.cpp)
 add_benchmark(vector_bool_count src/vector_bool_count.cpp)
 add_benchmark(vector_bool_move src/vector_bool_move.cpp)
+add_benchmark(vector_bool_transform src/vector_bool_transform.cpp)

benchmarks/inc/utility.hpp

@@ -10,10 +10,10 @@
 #include <type_traits>
 #include <vector>
 
-template <class Contained, template <class> class Alloc = std::allocator>
-std::vector<Contained, Alloc<Contained>> random_vector(size_t n) {
+template <class Contained, template <class> class Alloc = std::allocator, class... Seed>
+std::vector<Contained, Alloc<Contained>> random_vector(size_t n, Seed... seed) {
     std::vector<Contained, Alloc<Contained>> res(n);
-    std::mt19937_64 prng;
+    std::mt19937_64 prng{seed...};
 
     if constexpr (std::is_same_v<Contained, bool>) {
         std::generate(res.begin(), res.end(), [&prng] { return static_cast<bool>(prng() & 1); });

benchmarks/src/vector_bool_transform.cpp

@@ -0,0 +1,53 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <benchmark/benchmark.h>
+//
+#include <algorithm>
+#include <cstddef>
+#include <functional>
+#include <random>
+#include <vector>
+
+#include "skewed_allocator.hpp"
+#include "utility.hpp"
+
+using namespace std;
+
+template <class Pred>
+void transform_one_input_aligned(benchmark::State& state) {
+    const auto size = static_cast<size_t>(state.range(0));
+    auto source     = random_vector<bool, not_highly_aligned_allocator>(size);
+    vector<bool> dest(size, false);
+
+    for (auto _ : state) {
+        benchmark::DoNotOptimize(source);
+        transform(source.begin(), source.end(), dest.begin(), Pred{});
+        benchmark::DoNotOptimize(dest);
+    }
+}
+
+template <class Pred>
+void transform_two_inputs_aligned(benchmark::State& state) {
+    const auto size = static_cast<size_t>(state.range(0));
+    auto source1    = random_vector<bool, not_highly_aligned_allocator>(size);
+    auto source2    = random_vector<bool, not_highly_aligned_allocator>(size, 1729u);
+    vector<bool> dest(size, false);
+
+    for (auto _ : state) {
+        benchmark::DoNotOptimize(source1);
+        benchmark::DoNotOptimize(source2);
+        transform(source1.begin(), source1.end(), source2.begin(), dest.begin(), Pred{});
+        benchmark::DoNotOptimize(dest);
+    }
+}
+
+void common_args(auto bm) {
+    bm->RangeMultiplier(64)->Range(64, 64 << 10);
+}
+
+BENCHMARK(transform_two_inputs_aligned<logical_and<>>)->Apply(common_args);
+BENCHMARK(transform_two_inputs_aligned<logical_or<>>)->Apply(common_args);
+BENCHMARK(transform_one_input_aligned<logical_not<>>)->Apply(common_args);
+
+BENCHMARK_MAIN();

stl/inc/algorithm

@@ -4103,13 +4103,29 @@ _CONSTEXPR20 void iter_swap(_FwdIt1 _Left, _FwdIt2 _Right) { // swap *_Left and
     swap(*_Left, *_Right); // intentional ADL
 }
 
+template <class _VbIt, class _OutIt, class _Mapped_fn>
+_CONSTEXPR20 _OutIt _Transform_vbool_aligned(_VbIt _First, _VbIt _Last, _OutIt _Dest, _Mapped_fn _Mapped_func);
+
+template <class _VbIt1, class _VbIt2, class _OutIt, class _Mapped_fn>
+_CONSTEXPR20 _OutIt _Transform_vbool_aligned(
+    _VbIt1 _First1, _VbIt1 _Last1, _VbIt2 _First2, _OutIt _Dest, _Mapped_fn _Mapped_func);
+
 _EXPORT_STD template <class _InIt, class _OutIt, class _Fn>
 _CONSTEXPR20 _OutIt transform(const _InIt _First, const _InIt _Last, _OutIt _Dest, _Fn _Func) {
     // transform [_First, _Last) with _Func
     _STD _Adl_verify_range(_First, _Last);
     auto _UFirst      = _STD _Get_unwrapped(_First);
     const auto _ULast = _STD _Get_unwrapped(_Last);
     auto _UDest       = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_InIt>(_UFirst, _ULast));
+
+    if constexpr (_Is_vb_iterator<_InIt> && _Is_vb_iterator<_OutIt, true> && !is_same_v<_Map_vb_functor_t<_Fn>, void>) {
+        if (_UFirst._Myoff == 0 && _UDest._Myoff == 0) {
+            _UDest = _Transform_vbool_aligned(_UFirst, _ULast, _UDest, _Map_vb_functor_t<_Fn>{});
+            _STD _Seek_wrapped(_Dest, _UDest);
+            return _Dest;
+        }
+    }
+
     for (; _UFirst != _ULast; ++_UFirst, (void) ++_UDest) {
         *_UDest = _Func(*_UFirst);
     }
@@ -4133,6 +4149,16 @@ _CONSTEXPR20 _OutIt transform(
     const auto _Count  = _STD _Idl_distance<_InIt1>(_UFirst1, _ULast1);
     auto _UFirst2      = _STD _Get_unwrapped_n(_First2, _Count);
     auto _UDest        = _STD _Get_unwrapped_n(_Dest, _Count);
+
+    if constexpr (_Is_vb_iterator<_InIt1> && _Is_vb_iterator<_InIt2> && _Is_vb_iterator<_OutIt, true>
+                  && !is_same_v<_Map_vb_functor_t<_Fn>, void>) {
+        if (_UFirst1._Myoff == 0 && _UFirst2._Myoff == 0 && _UDest._Myoff == 0) {
+            _UDest = _Transform_vbool_aligned(_UFirst1, _ULast1, _UFirst2, _UDest, _Map_vb_functor_t<_Fn>{});
+            _STD _Seek_wrapped(_Dest, _UDest);
+            return _Dest;
+        }
+    }
+
     for (; _UFirst1 != _ULast1; ++_UFirst1, (void) ++_UFirst2, ++_UDest) {
         *_UDest = _Func(*_UFirst1, *_UFirst2);
     }

stl/inc/functional

@@ -258,6 +258,62 @@ struct bit_not<void> {
     using is_transparent = int;
 };
 
+struct _Bit_xnor {
+    template <class _Ty1, class _Ty2>
+    _NODISCARD constexpr auto operator()(_Ty1&& _Left, _Ty2&& _Right) const //
+        -> decltype(~(_STD forward<_Ty1>(_Left) ^ _STD forward<_Ty2>(_Right))) {
+        return ~(_STD forward<_Ty1>(_Left) ^ _STD forward<_Ty2>(_Right));
+    }
+
+    using is_transparent = int;
+};
+
+template <class _Ty>
+constexpr bool _Is_vbool_functor_arg = is_same_v<_Ty, void> || is_integral_v<_Ty>;
+
+template <class _Ty>
+struct _Map_vb_functor<logical_and<_Ty>> {
+    using type = conditional_t<_Is_vbool_functor_arg<_Ty>, bit_and<>, void>;
+};
+
+template <class _Ty>
+struct _Map_vb_functor<bit_and<_Ty>> {
+    using type = conditional_t<_Is_vbool_functor_arg<_Ty>, bit_and<>, void>;
+};
+
+template <class _Ty>
+struct _Map_vb_functor<logical_or<_Ty>> {
+    using type = conditional_t<_Is_vbool_functor_arg<_Ty>, bit_or<>, void>;
+};
+
+template <class _Ty>
+struct _Map_vb_functor<bit_or<_Ty>> {
+    using type = conditional_t<_Is_vbool_functor_arg<_Ty>, bit_or<>, void>;
+};
+
+template <class _Ty>
+struct _Map_vb_functor<not_equal_to<_Ty>> {
+    using type = conditional_t<_Is_vbool_functor_arg<_Ty>, bit_xor<>, void>;
+};
+
+template <class _Ty>
+struct _Map_vb_functor<bit_xor<_Ty>> {
+    using type = conditional_t<_Is_vbool_functor_arg<_Ty>, bit_xor<>, void>;
+};
+
+template <class _Ty>
+struct _Map_vb_functor<equal_to<_Ty>> {
+    using type = conditional_t<_Is_vbool_functor_arg<_Ty>, _Bit_xnor, void>;
+};
+
+template <class _Ty>
+struct _Map_vb_functor<logical_not<_Ty>> {
+    using type = conditional_t<_Is_vbool_functor_arg<_Ty>, bit_not<>, void>;
+};
+
+// bit_not isn't mapped to itself because it emits MSVC warning C4804 "'~': unsafe use of type 'bool' in operation"
+// and Clang -Wbool-operation "bitwise negation of a boolean expression; did you mean logical negation?".
+
 #if _HAS_DEPRECATED_NEGATORS
 _STL_DISABLE_DEPRECATED_WARNING
 _EXPORT_STD template <class _Fn>

stl/inc/vector

@@ -4004,6 +4004,49 @@ _CONSTEXPR20 _OutIt _Copy_vbool(_VbIt _First, _VbIt _Last, _OutIt _Dest) {
     return _DestEnd;
 }
 
+template <class _VbIt, class _OutIt, class _Mapped_fn>
+_CONSTEXPR20 _OutIt _Transform_vbool_aligned(
+    const _VbIt _First, const _VbIt _Last, _OutIt _Dest, const _Mapped_fn _Mapped_func) {
+    auto _First_ptr      = _First._Myptr;
+    const auto _Last_ptr = _Last._Myptr;
+    auto _Dest_ptr       = const_cast<_Vbase*>(_Dest._Myptr);
+
+    for (; _First_ptr != _Last_ptr; ++_First_ptr, ++_Dest_ptr) {
+        *_Dest_ptr = _Mapped_func(*_First_ptr);
+    }
+
+    if (_Last._Myoff != 0) {
+        const _Vbase _Mask = (_Vbase{1} << _Last._Myoff) - 1;
+        *_Dest_ptr         = (*_Dest_ptr & ~_Mask) | (_Mapped_func(*_First_ptr) & _Mask);
+        _Dest._Myoff       = _Last._Myoff;
+    }
+
+    _Dest._Myptr = _Dest_ptr;
+    return _Dest;
+}
+
+template <class _VbIt1, class _VbIt2, class _OutIt, class _Mapped_fn>
+_CONSTEXPR20 _OutIt _Transform_vbool_aligned(
+    const _VbIt1 _First1, const _VbIt1 _Last1, const _VbIt2 _First2, _OutIt _Dest, const _Mapped_fn _Mapped_func) {
+    auto _First1_ptr      = _First1._Myptr;
+    auto _First2_ptr      = _First2._Myptr;
+    const auto _Last1_ptr = _Last1._Myptr;
+    auto _Dest_ptr        = const_cast<_Vbase*>(_Dest._Myptr);
+
+    for (; _First1_ptr != _Last1_ptr; ++_First1_ptr, ++_First2_ptr, ++_Dest_ptr) {
+        *_Dest_ptr = _Mapped_func(*_First1_ptr, *_First2_ptr);
+    }
+
+    if (_Last1._Myoff != 0) {
+        const _Vbase _Mask = (_Vbase{1} << _Last1._Myoff) - 1;
+        *_Dest_ptr         = (*_Dest_ptr & ~_Mask) | (_Mapped_func(*_First1_ptr, *_First2_ptr) & _Mask);
+        _Dest._Myoff       = _Last1._Myoff;
+    }
+
+    _Dest._Myptr = _Dest_ptr;
+    return _Dest;
+}
+
 #undef _ASAN_VECTOR_MODIFY
 #undef _ASAN_VECTOR_REMOVE
 #undef _ASAN_VECTOR_CREATE

stl/inc/xutility

@@ -4868,6 +4868,14 @@ _OutCtgIt _Copy_memmove_n(_CtgIt _First, const size_t _Object_count, _OutCtgIt _
     return _STD _Copy_memmove_tail(_First_ch, _STD move(_Dest), _Byte_count, _Object_count);
 }
 
+template <class _Fn>
+struct _Map_vb_functor {
+    using type = void;
+};
+
+template <class _Fn>
+using _Map_vb_functor_t = typename _Map_vb_functor<_Fn>::type;
+
 template <class _It, bool _RequiresMutable = false>
 constexpr bool _Is_vb_iterator = false;