diff --git a/libcxx/docs/ABIGuarantees.rst b/libcxx/docs/ABIGuarantees.rst index 4d4674c7756a4..7588cf7e55453 100644 --- a/libcxx/docs/ABIGuarantees.rst +++ b/libcxx/docs/ABIGuarantees.rst @@ -114,6 +114,10 @@ hand, backwards compatibility is generally guaranteed. There are multiple ABI flags that change the symbols exported from the built library: +``_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN`` +------------------------------------------------- +This removes ``align()``. In the past ``align()`` is not an inline function, but now it changed to an inline function for performance. + ``_LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON`` ------------------------------------------------- This removes ``__basic_string_common::__throw_length_error()`` and diff --git a/libcxx/docs/ReleaseNotes/22.rst b/libcxx/docs/ReleaseNotes/22.rst index 2c19dfc57a3f8..aca7a65aaa2fc 100644 --- a/libcxx/docs/ReleaseNotes/22.rst +++ b/libcxx/docs/ReleaseNotes/22.rst @@ -83,6 +83,8 @@ Improvements and New Features iterators, resulting in a performance improvement for ``std::deque`` and ``std::join_view>>`` iterators. +- The performance of ``align`` has been improved about 2x by making it an inline function. + Deprecations and Removals ------------------------- @@ -114,5 +116,7 @@ ABI Affecting Changes potentially inheriting from the types they wrap. At this point in time we are not aware of any ABI changes caused by this. +- ``align`` has been removed from libc++ library in ABI v2. We also add a new ABI flag ``_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN``. + Build System Changes -------------------- diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h index 38b85c6ac70de..fc3729dbb02af 100644 --- a/libcxx/include/__configuration/abi.h +++ b/libcxx/include/__configuration/abi.h @@ -63,6 +63,7 @@ // These flags are documented in ABIGuarantees.rst # define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT +# define _LIBCPP_ABI_DO_NOT_EXPORT_ALIGN # define _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON # define _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON # define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10 @@ -103,6 +104,12 @@ # endif #endif +// The PE/COFF format reports a linking error when encountering multiple symbol definitions where at least one is a +// strong symbol. So we can't inlining a non-inline function without ABI break change. +#if defined(_LIBCPP_OBJECT_FORMAT_COFF) +# define _LIBCPP_DISABLE_INLINE_OPTIMIZE_BECAUSE_MULTIPLY_SYMBOLS_ERROR +#endif + // TODO(LLVM 22): Remove this check #if defined(_LIBCPP_ABI_NO_ITERATOR_BASES) && !defined(_LIBCPP_ABI_NO_REVERSE_ITERATOR_SECOND_MEMBER) # ifndef _LIBCPP_ONLY_NO_ITERATOR_BASES diff --git a/libcxx/include/__memory/align.h b/libcxx/include/__memory/align.h index 402eac3380925..c5aff63d1f5e3 100644 --- a/libcxx/include/__memory/align.h +++ b/libcxx/include/__memory/align.h @@ -11,6 +11,7 @@ #include <__config> #include <__cstddef/size_t.h> +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header @@ -18,8 +19,29 @@ _LIBCPP_BEGIN_NAMESPACE_STD +#if defined(_LIBCPP_DISABLE_INLINE_OPTIMIZE_BECAUSE_MULTIPLY_SYMBOLS_ERROR) && !defined(_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN) + _LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); +#else + +_LIBCPP_HIDE_FROM_ABI inline void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { + void* __r = nullptr; + if (__sz <= __space) { + char* __p1 = static_cast(__ptr); + char* __p2 = reinterpret_cast(reinterpret_cast(__p1 + (__align - 1)) & -__align); + size_t __d = static_cast(__p2 - __p1); + if (__d <= __space - __sz) { + __r = __p2; + __ptr = __r; + __space -= __d; + } + } + return __r; +} + +#endif + _LIBCPP_END_NAMESPACE_STD #endif // _LIBCPP___MEMORY_ALIGN_H diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt index f59fe0e08fccb..dc0ccbf5f54cb 100644 --- a/libcxx/src/CMakeLists.txt +++ b/libcxx/src/CMakeLists.txt @@ -31,6 +31,7 @@ set(LIBCXX_SOURCES include/to_chars_floating_point.h include/from_chars_floating_point.h memory.cpp + memory_align.cpp memory_resource.cpp new_handler.cpp new_helpers.cpp diff --git a/libcxx/src/memory.cpp b/libcxx/src/memory.cpp index 9be40cb9c1285..2cfbf4f7f7b30 100644 --- a/libcxx/src/memory.cpp +++ b/libcxx/src/memory.cpp @@ -132,19 +132,4 @@ __sp_mut& __get_sp_mut(const void* p) { #endif // _LIBCPP_HAS_THREADS -void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { - void* r = nullptr; - if (size <= space) { - char* p1 = static_cast(ptr); - char* p2 = reinterpret_cast(reinterpret_cast(p1 + (alignment - 1)) & -alignment); - size_t d = static_cast(p2 - p1); - if (d <= space - size) { - r = p2; - ptr = r; - space -= d; - } - } - return r; -} - _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/src/memory_align.cpp b/libcxx/src/memory_align.cpp new file mode 100644 index 0000000000000..6b70c64a505a7 --- /dev/null +++ b/libcxx/src/memory_align.cpp @@ -0,0 +1,36 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <__config> +#include <__cstddef/size_t.h> +#include + +// Don't include to avoid mulitple declartion of align() + +#if !defined(_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN) + +_LIBCPP_BEGIN_NAMESPACE_STD + +_LIBCPP_EXPORTED_FROM_ABI void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { + void* r = nullptr; + if (size <= space) { + char* p1 = static_cast(ptr); + char* p2 = reinterpret_cast(reinterpret_cast(p1 + (alignment - 1)) & -alignment); + size_t d = static_cast(p2 - p1); + if (d <= space - size) { + r = p2; + ptr = r; + space -= d; + } + } + return r; +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP_ABI_DO_NOT_EXPORT_ALIGN diff --git a/libcxx/test/benchmarks/memory/align.bench.cpp b/libcxx/test/benchmarks/memory/align.bench.cpp new file mode 100644 index 0000000000000..9fb637c21f9ac --- /dev/null +++ b/libcxx/test/benchmarks/memory/align.bench.cpp @@ -0,0 +1,39 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03 + +#include +#include + +#include "benchmark/benchmark.h" + +struct Input { + std::size_t align; + std::size_t size; + void* ptr; + std::size_t buffer_size; +}; + +static void BM_align(benchmark::State& state) { + char buffer[1024]; + Input input{}; + void* ptr = buffer + 123; + std::size_t buffer_size = sizeof(buffer) - 123; + input.align = state.range(); + input.size = state.range(); + for (auto _ : state) { + input.ptr = ptr; + input.buffer_size = buffer_size; + benchmark::DoNotOptimize(input); + benchmark::DoNotOptimize(std::align(input.align, input.size, input.ptr, input.buffer_size)); + } +} +BENCHMARK(BM_align)->Range(1, 256); + +BENCHMARK_MAIN(); diff --git a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn index 327a8ede31fff..0c1953f7a2c92 100644 --- a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn +++ b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn @@ -152,6 +152,7 @@ cxx_sources = [ "iostream.cpp", "locale.cpp", "memory.cpp", + "memory_align.cpp" "memory_resource.cpp", "mutex.cpp", "mutex_destructor.cpp",