[libc++] Optimize std::find_if

Author: philnik777

libcxx/include/CMakeLists.txt

@@ -604,6 +604,7 @@ set(files
   __memory/unique_temporary_buffer.h
   __memory/uses_allocator.h
   __memory/uses_allocator_construction.h
+  __memory/valid_range.h
   __memory_resource/memory_resource.h
   __memory_resource/monotonic_buffer_resource.h
   __memory_resource/polymorphic_allocator.h
@@ -924,7 +925,6 @@ set(files
   __utility/in_place.h
   __utility/integer_sequence.h
   __utility/is_pointer_in_range.h
-  __utility/is_valid_range.h
   __utility/lazy_synth_three_way_comparator.h
   __utility/move.h
   __utility/no_destroy.h

libcxx/include/__algorithm/find_if.h

@@ -11,6 +11,7 @@
 #define _LIBCPP___ALGORITHM_FIND_IF_H
 
 #include <__config>
+#include <__memory/valid_range.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -21,6 +22,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 template <class _InputIterator, class _Predicate>
 [[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
 find_if(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
+  std::__assume_valid_range(__first, __last);
+
   for (; __first != __last; ++__first)
     if (__pred(*__first))
       break;

libcxx/include/__memory/valid_range.h

@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_VALID_RANGE_H
+#define _LIBCPP___MEMORY_VALID_RANGE_H
+
+#include <__algorithm/comp.h>
+#include <__assert>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/assume_aligned.h>
+#include <__memory/pointer_traits.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cvref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_SANITIZE("address") bool
+__is_valid_range(const _Tp* __first, const _Tp* __last) {
+  if (__libcpp_is_constant_evaluated()) {
+    // If this is not a constant during constant evaluation, that is because __first and __last are not
+    // part of the same allocation. If they are part of the same allocation, we must still make sure they
+    // are ordered properly.
+    return __builtin_constant_p(__first <= __last) && __first <= __last;
+  }
+
+  return !__less<>()(__last, __first);
+}
+
+// This functions allows the compiler to assume that [__first, __last) is a valid range to be given to an algortihm.
+// Specifically, this means that
+// - [__first, __last) is dereferenceable
+// - __first and __last are correctly aligned according to the language rules
+// This allows (curently only clang-based compilers) to auto-vectorize algorithms that contain early returns.
+template <class _Iter, class _Sent>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __assume_valid_range(_Iter&& __first, _Sent&& __last) {
+#if __has_builtin(__builtin_assume_dereferenceable) && !defined(_LIBCPP_CXX03_LANG)
+  if constexpr (__libcpp_is_contiguous_iterator<__remove_cvref_t<_Iter>>::value &&
+                is_same<__remove_cvref_t<_Iter>, __remove_cvref_t<_Sent>>::value) {
+    _LIBCPP_ASSERT_INTERNAL(std::__is_valid_range(std::__to_address(__first), std::__to_address(__last)),
+                            "Valid range assumption does not hold");
+    if (!__libcpp_is_constant_evaluated()) {
+      using __value_type = typename iterator_traits<__remove_cvref_t<_Iter>>::value_type;
+      __builtin_assume_dereferenceable(std::__to_address(__first), (__last - __first) * sizeof(__value_type));
+      (void)std::__assume_aligned<_LIBCPP_ALIGNOF(__value_type)>(std::__to_address(__first));
+      (void)std::__assume_aligned<_LIBCPP_ALIGNOF(__value_type)>(std::__to_address(__last));
+    }
+  }
+#endif
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_VALID_RANGE_H

libcxx/include/__utility/is_pointer_in_range.h

@@ -12,12 +12,12 @@
 #include <__algorithm/comp.h>
 #include <__assert>
 #include <__config>
+#include <__memory/valid_range.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_constant_evaluated.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
-#include <__utility/is_valid_range.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header

libcxx/include/__utility/is_valid_range.h

@@ -1683,6 +1683,7 @@ module std [system] {
     }
     module uses_allocator                     { header "__memory/uses_allocator.h" }
     module uses_allocator_construction        { header "__memory/uses_allocator_construction.h" }
+    module valid_range                        { header "__memory/valid_range.h" }
 
     header "memory"
     export *
@@ -2161,7 +2162,6 @@ module std [system] {
     }
     module integer_sequence                { header "__utility/integer_sequence.h" }
     module is_pointer_in_range             { header "__utility/is_pointer_in_range.h" }
-    module is_valid_range                  { header "__utility/is_valid_range.h" }
     module lazy_synth_three_way_comparator { header "__utility/lazy_synth_three_way_comparator.h" }
     module move                            { header "__utility/move.h" }
     module no_destroy                      { header "__utility/no_destroy.h" }

libcxx/include/module.modulemap.in

@@ -117,8 +117,8 @@ protected:
 #    include <__assert>
 #    include <__fwd/streambuf.h>
 #    include <__locale>
+#    include <__memory/valid_range.h>
 #    include <__type_traits/is_same.h>
-#    include <__utility/is_valid_range.h>
 #    include <__utility/scope_guard.h>
 #    include <climits>
 #    include <ios>

libcxx/include/streambuf

@@ -21,30 +21,18 @@
 int main(int argc, char** argv) {
   auto std_find    = [](auto first, auto last, auto const& value) { return std::find(first, last, value); };
   auto std_find_if = [](auto first, auto last, auto const& value) {
-    return std::find_if(first, last, [&](auto element) {
-      benchmark::DoNotOptimize(element);
-      return element == value;
-    });
+    return std::find_if(first, last, [&](auto element) { return element == value; });
   };
   auto std_find_if_not = [](auto first, auto last, auto const& value) {
-    return std::find_if_not(first, last, [&](auto element) {
-      benchmark::DoNotOptimize(element);
-      return element != value;
-    });
+    return std::find_if_not(first, last, [&](auto element) { return element != value; });
   };
 
   auto ranges_find    = [](auto first, auto last, auto const& value) { return std::ranges::find(first, last, value); };
   auto ranges_find_if = [](auto first, auto last, auto const& value) {
-    return std::ranges::find_if(first, last, [&](auto element) {
-      benchmark::DoNotOptimize(element);
-      return element == value;
-    });
+    return std::ranges::find_if(first, last, [&](auto element) { return element == value; });
   };
   auto ranges_find_if_not = [](auto first, auto last, auto const& value) {
-    return std::ranges::find_if_not(first, last, [&](auto element) {
-      benchmark::DoNotOptimize(element);
-      return element != value;
-    });
+    return std::ranges::find_if_not(first, last, [&](auto element) { return element != value; });
   };
 
   auto register_benchmarks = [&](auto bm, std::string comment) {

libcxx/test/benchmarks/algorithms/nonmodifying/find.bench.cpp

@@ -0,0 +1,51 @@
+#include <benchmark/benchmark.h>
+
+template <class Iter, class ValueT>
+Iter my_find(Iter first, Iter last, const ValueT& i) {
+  for (; first != last; ++first) {
+    if (*first == i)
+      break;
+  }
+  return first;
+}
+
+static auto bm_find_if_no_vectorization(benchmark::State& state) {
+  std::size_t const size = 8192;
+  std::vector<short> c(size, 0);
+
+  for ([[maybe_unused]] auto _ : state) {
+    benchmark::DoNotOptimize(c);
+    std::vector<short>::iterator result;
+    result = my_find(c.begin(), c.end(), 1);
+    benchmark::DoNotOptimize(result);
+  }
+}
+BENCHMARK(bm_find_if_no_vectorization);
+
+static auto bm_find_if_autovectorization(benchmark::State& state) {
+  std::size_t const size = 8192;
+  std::vector<short> c(size, 0);
+
+  for ([[maybe_unused]] auto _ : state) {
+    benchmark::DoNotOptimize(c);
+    std::vector<short>::iterator result;
+    result = find_if(c.begin(), c.end(), [](short i) { return i == 1; });
+    benchmark::DoNotOptimize(result);
+  }
+}
+BENCHMARK(bm_find_if_autovectorization);
+
+static auto bm_find_manual_vectorization(benchmark::State& state) {
+  std::size_t const size = 8192;
+  std::vector<short> c(size, 0);
+
+  for ([[maybe_unused]] auto _ : state) {
+    benchmark::DoNotOptimize(c);
+    std::vector<short>::iterator result;
+    result = find(c.begin(), c.end(), 1);
+    benchmark::DoNotOptimize(result);
+  }
+}
+BENCHMARK(bm_find_manual_vectorization);
+
+BENCHMARK_MAIN();

libcxx/test/benchmarks/algorithms/nonmodifying/find_if.bench.cpp

@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <__utility/is_valid_range.h>
+#include <__memory/valid_range.h>
 #include <cassert>
 
 #include "test_macros.h"