From 615eee277a552d4211607745f8599909c7c3ddd3 Mon Sep 17 00:00:00 2001 From: "zezheng.li" Date: Tue, 11 Nov 2025 18:07:31 +0800 Subject: [PATCH 1/8] [libc++] Make std::align an inline function from v2 ABI --- libcxx/include/__memory/align.h | 19 +++++++++++++++++++ libcxx/src/memory.cpp | 3 +++ 2 files changed, 22 insertions(+) diff --git a/libcxx/include/__memory/align.h b/libcxx/include/__memory/align.h index 402eac3380925..6c8e21be03f76 100644 --- a/libcxx/include/__memory/align.h +++ b/libcxx/include/__memory/align.h @@ -11,6 +11,7 @@ #include <__config> #include <__cstddef/size_t.h> +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header @@ -18,7 +19,25 @@ _LIBCPP_BEGIN_NAMESPACE_STD +// From >=v2 ABI, std::align is an inline function. +#if _LIBCPP_ABI_VERSION >= 2 +inline _LIBCPP_HIDE_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { + void* __r = nullptr; + if (__sz <= __space) { + char* __p1 = static_cast(__ptr); + char* __p2 = reinterpret_cast(reinterpret_cast(__p1 + (__align - 1)) & -__align); + size_t __d = static_cast(__p2 - __p1); + if (__d <= __space - __sz) { + __r = __p2; + __ptr = __r; + __space -= __d; + } + } + return __r; +} +#else _LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); +#endif _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/src/memory.cpp b/libcxx/src/memory.cpp index 9be40cb9c1285..9efbb6eb8eca3 100644 --- a/libcxx/src/memory.cpp +++ b/libcxx/src/memory.cpp @@ -132,6 +132,8 @@ __sp_mut& __get_sp_mut(const void* p) { #endif // _LIBCPP_HAS_THREADS +// Remove std::align from >=v2 dylib ABI, make it an inline function. +#if _LIBCPP_ABI_VERSION == 1 void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { void* r = nullptr; if (size <= space) { @@ -146,5 +148,6 @@ void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { } return r; } +#endif _LIBCPP_END_NAMESPACE_STD From 0987145ac0269c506738aabba6cce5ace7243ff0 Mon Sep 17 00:00:00 2001 From: "zezheng.li" Date: Wed, 12 Nov 2025 18:08:21 +0800 Subject: [PATCH 2/8] also make align an inline function in v1 abi --- libcxx/docs/ReleaseNotes/22.rst | 6 ++++ libcxx/include/__memory/align.h | 13 +++++---- libcxx/src/memory.cpp | 7 +++-- libcxx/test/benchmarks/memory/align.bench.cpp | 28 +++++++++++++++++++ 4 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 libcxx/test/benchmarks/memory/align.bench.cpp diff --git a/libcxx/docs/ReleaseNotes/22.rst b/libcxx/docs/ReleaseNotes/22.rst index a6a0ac8670fb5..f1d6a34be6762 100644 --- a/libcxx/docs/ReleaseNotes/22.rst +++ b/libcxx/docs/ReleaseNotes/22.rst @@ -82,6 +82,8 @@ Improvements and New Features iterators, resulting in a performance improvement for ``std::deque`` and ``std::join_view>>`` iterators. +- The performance of ``align`` has been improved about 3x by making it an inline function. + Deprecations and Removals ------------------------- @@ -98,6 +100,8 @@ Potentially breaking changes ``_LIBCPP_ABI_NO_ITERATOR_BASES``. If you are using this flag and care about ABI stability, you should set ``_LIBCPP_ABI_NO_REVERSE_ITERATOR_SECOND_MEMBER`` as well. +- ``align`` has been removed from libc++ dynamic library in ABI v2. You could use the ABI flag ``_LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS`` to reserve it. + Announcements About Future Releases ----------------------------------- @@ -113,5 +117,7 @@ ABI Affecting Changes potentially inheriting from the types they wrap. At this point in time we are not aware of any ABI changes caused by this. +- ``align`` has been removed from libc++ dynamic library in ABI v2. You could use the ABI flag ``_LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS`` to reserve it. + Build System Changes -------------------- diff --git a/libcxx/include/__memory/align.h b/libcxx/include/__memory/align.h index 6c8e21be03f76..ce77278e99cd1 100644 --- a/libcxx/include/__memory/align.h +++ b/libcxx/include/__memory/align.h @@ -19,8 +19,12 @@ _LIBCPP_BEGIN_NAMESPACE_STD -// From >=v2 ABI, std::align is an inline function. -#if _LIBCPP_ABI_VERSION >= 2 +#ifdef _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS + +_LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); + +#else + inline _LIBCPP_HIDE_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { void* __r = nullptr; if (__sz <= __space) { @@ -35,9 +39,8 @@ inline _LIBCPP_HIDE_FROM_ABI void* align(size_t __align, size_t __sz, void*& __p } return __r; } -#else -_LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); -#endif + +#endif // _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/src/memory.cpp b/libcxx/src/memory.cpp index 9efbb6eb8eca3..56cf48a6e0cfb 100644 --- a/libcxx/src/memory.cpp +++ b/libcxx/src/memory.cpp @@ -8,6 +8,7 @@ #include <__config> #ifdef _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS +# define _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS # define _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS #endif @@ -132,8 +133,8 @@ __sp_mut& __get_sp_mut(const void* p) { #endif // _LIBCPP_HAS_THREADS -// Remove std::align from >=v2 dylib ABI, make it an inline function. -#if _LIBCPP_ABI_VERSION == 1 +#if defined(_LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS) + void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { void* r = nullptr; if (size <= space) { @@ -148,6 +149,6 @@ void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { } return r; } -#endif +#endif // _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/test/benchmarks/memory/align.bench.cpp b/libcxx/test/benchmarks/memory/align.bench.cpp new file mode 100644 index 0000000000000..ae874a6a0965e --- /dev/null +++ b/libcxx/test/benchmarks/memory/align.bench.cpp @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03 + +#include + +#include "benchmark/benchmark.h" +#include "test_macros.h" + +static void BM_align(benchmark::State& state) { + char buffer[1024]; + char* data = buffer + 123; + std::size_t sz{1024 - 123}; + void* p; + + for (auto _ : state) { + benchmark::DoNotOptimize(std::align(state.range(), state.range(), p, sz)) + } +} +BENCHMARK(BM_align)->Range(1, 256); + +BENCHMARK_MAIN(); From f52e3fa4e7de4d363e4d3cba75aa747a38c2821d Mon Sep 17 00:00:00 2001 From: "zezheng.li" Date: Wed, 12 Nov 2025 18:38:39 +0800 Subject: [PATCH 3/8] fix benchmark --- libcxx/test/benchmarks/memory/align.bench.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libcxx/test/benchmarks/memory/align.bench.cpp b/libcxx/test/benchmarks/memory/align.bench.cpp index ae874a6a0965e..74be6f7a598f7 100644 --- a/libcxx/test/benchmarks/memory/align.bench.cpp +++ b/libcxx/test/benchmarks/memory/align.bench.cpp @@ -15,12 +15,11 @@ static void BM_align(benchmark::State& state) { char buffer[1024]; - char* data = buffer + 123; - std::size_t sz{1024 - 123}; - void* p; + void* data = buffer + 123; + std::size_t sz{sizeof(buffer) - 123}; for (auto _ : state) { - benchmark::DoNotOptimize(std::align(state.range(), state.range(), p, sz)) + benchmark::DoNotOptimize(std::align(state.range(), state.range(), data, sz)); } } BENCHMARK(BM_align)->Range(1, 256); From 65e53c32333b761b29f9d9b094b9d254ea8a003b Mon Sep 17 00:00:00 2001 From: "zezheng.li" Date: Wed, 12 Nov 2025 19:01:47 +0800 Subject: [PATCH 4/8] remove duplicated code --- libcxx/include/__memory/align.h | 18 +++++++++++------- libcxx/src/memory.cpp | 13 +------------ 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/libcxx/include/__memory/align.h b/libcxx/include/__memory/align.h index ce77278e99cd1..f0a693f0360de 100644 --- a/libcxx/include/__memory/align.h +++ b/libcxx/include/__memory/align.h @@ -19,13 +19,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#ifdef _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS - -_LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); - -#else - -inline _LIBCPP_HIDE_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { +__attribute__((always_inline)) inline void* __align_impl(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { void* __r = nullptr; if (__sz <= __space) { char* __p1 = static_cast(__ptr); @@ -40,6 +34,16 @@ inline _LIBCPP_HIDE_FROM_ABI void* align(size_t __align, size_t __sz, void*& __p return __r; } +#ifdef _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS + +_LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); + +#else + +inline _LIBCPP_HIDE_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { + return __align_impl(__align, __sz, __ptr, __space); +} + #endif // _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/src/memory.cpp b/libcxx/src/memory.cpp index 56cf48a6e0cfb..bb17ecce3ebeb 100644 --- a/libcxx/src/memory.cpp +++ b/libcxx/src/memory.cpp @@ -136,18 +136,7 @@ __sp_mut& __get_sp_mut(const void* p) { #if defined(_LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS) void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { - void* r = nullptr; - if (size <= space) { - char* p1 = static_cast(ptr); - char* p2 = reinterpret_cast(reinterpret_cast(p1 + (alignment - 1)) & -alignment); - size_t d = static_cast(p2 - p1); - if (d <= space - size) { - r = p2; - ptr = r; - space -= d; - } - } - return r; + return __align_impl(alignment, size, ptr, space); } #endif // _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS From 535c59763a5ca675e9f666be0a2d5789f03730bb Mon Sep 17 00:00:00 2001 From: "zezheng.li" Date: Thu, 13 Nov 2025 10:42:15 +0800 Subject: [PATCH 5/8] remove useless decl in align.h --- libcxx/include/__memory/align.h | 9 +++------ libcxx/src/memory.cpp | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/libcxx/include/__memory/align.h b/libcxx/include/__memory/align.h index f0a693f0360de..227e8c267278c 100644 --- a/libcxx/include/__memory/align.h +++ b/libcxx/include/__memory/align.h @@ -19,7 +19,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD -__attribute__((always_inline)) inline void* __align_impl(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE inline void* +__align_impl(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { void* __r = nullptr; if (__sz <= __space) { char* __p1 = static_cast(__ptr); @@ -34,11 +35,7 @@ __attribute__((always_inline)) inline void* __align_impl(size_t __align, size_t return __r; } -#ifdef _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS - -_LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); - -#else +#ifndef _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS inline _LIBCPP_HIDE_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { return __align_impl(__align, __sz, __ptr, __space); diff --git a/libcxx/src/memory.cpp b/libcxx/src/memory.cpp index bb17ecce3ebeb..681decfb785ea 100644 --- a/libcxx/src/memory.cpp +++ b/libcxx/src/memory.cpp @@ -135,7 +135,7 @@ __sp_mut& __get_sp_mut(const void* p) { #if defined(_LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS) -void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { +_LIBCPP_EXPORTED_FROM_ABI void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { return __align_impl(alignment, size, ptr, space); } #endif // _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS From 3eb0164fd2c6886a9abdcd8d1492995dab48b32d Mon Sep 17 00:00:00 2001 From: "zezheng.li" Date: Thu, 13 Nov 2025 14:43:48 +0800 Subject: [PATCH 6/8] fix abi break change --- libcxx/docs/ABIGuarantees.rst | 4 ++++ libcxx/docs/ReleaseNotes/22.rst | 4 +--- libcxx/include/__configuration/abi.h | 1 + libcxx/include/__memory/align.h | 4 ++-- libcxx/src/memory.cpp | 8 +++++--- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/libcxx/docs/ABIGuarantees.rst b/libcxx/docs/ABIGuarantees.rst index 4d4674c7756a4..7588cf7e55453 100644 --- a/libcxx/docs/ABIGuarantees.rst +++ b/libcxx/docs/ABIGuarantees.rst @@ -114,6 +114,10 @@ hand, backwards compatibility is generally guaranteed. There are multiple ABI flags that change the symbols exported from the built library: +``_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN`` +------------------------------------------------- +This removes ``align()``. In the past ``align()`` is not an inline function, but now it changed to an inline function for performance. + ``_LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON`` ------------------------------------------------- This removes ``__basic_string_common::__throw_length_error()`` and diff --git a/libcxx/docs/ReleaseNotes/22.rst b/libcxx/docs/ReleaseNotes/22.rst index 23fe89cdbe738..670d73992e7e6 100644 --- a/libcxx/docs/ReleaseNotes/22.rst +++ b/libcxx/docs/ReleaseNotes/22.rst @@ -101,8 +101,6 @@ Potentially breaking changes ``_LIBCPP_ABI_NO_ITERATOR_BASES``. If you are using this flag and care about ABI stability, you should set ``_LIBCPP_ABI_NO_REVERSE_ITERATOR_SECOND_MEMBER`` as well. -- ``align`` has been removed from libc++ dynamic library in ABI v2. You could use the ABI flag ``_LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS`` to reserve it. - Announcements About Future Releases ----------------------------------- @@ -118,7 +116,7 @@ ABI Affecting Changes potentially inheriting from the types they wrap. At this point in time we are not aware of any ABI changes caused by this. -- ``align`` has been removed from libc++ dynamic library in ABI v2. You could use the ABI flag ``_LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS`` to reserve it. +- ``align`` has been removed from libc++ library in ABI v2. We also add a new ABI flag ``_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN``. Build System Changes -------------------- diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h index 38b85c6ac70de..f88de6442f90d 100644 --- a/libcxx/include/__configuration/abi.h +++ b/libcxx/include/__configuration/abi.h @@ -63,6 +63,7 @@ // These flags are documented in ABIGuarantees.rst # define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT +# define _LIBCPP_ABI_DO_NOT_EXPORT_ALIGN # define _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON # define _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON # define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10 diff --git a/libcxx/include/__memory/align.h b/libcxx/include/__memory/align.h index 227e8c267278c..8aaf08192f2af 100644 --- a/libcxx/include/__memory/align.h +++ b/libcxx/include/__memory/align.h @@ -35,13 +35,13 @@ __align_impl(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { return __r; } -#ifndef _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS +#ifndef _LIBCPP_EXPORT_ALIGN_SYMBOL inline _LIBCPP_HIDE_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { return __align_impl(__align, __sz, __ptr, __space); } -#endif // _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS +#endif // _LIBCPP_EXPORT_ALIGN_SYMBOL _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/src/memory.cpp b/libcxx/src/memory.cpp index 681decfb785ea..19b2894b87ec1 100644 --- a/libcxx/src/memory.cpp +++ b/libcxx/src/memory.cpp @@ -8,9 +8,11 @@ #include <__config> #ifdef _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS -# define _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS # define _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS #endif +#ifndef _LIBCPP_ABI_DO_NOT_EXPORT_ALIGN +# define _LIBCPP_EXPORT_ALIGN_SYMBOL +#endif #include <__functional/hash.h> #include @@ -133,11 +135,11 @@ __sp_mut& __get_sp_mut(const void* p) { #endif // _LIBCPP_HAS_THREADS -#if defined(_LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS) +#if defined(_LIBCPP_EXPORT_ALIGN_SYMBOL) _LIBCPP_EXPORTED_FROM_ABI void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { return __align_impl(alignment, size, ptr, space); } -#endif // _LIBCPP_ALIGN_DEFINE_LEGACY_INLINE_FUNCTIONS +#endif // _LIBCPP_EXPORT_ALIGN_SYMBOL _LIBCPP_END_NAMESPACE_STD From 783de7336501fc85deca2017208699a3d03aee85 Mon Sep 17 00:00:00 2001 From: "zezheng.li" Date: Thu, 13 Nov 2025 18:20:14 +0800 Subject: [PATCH 7/8] fix link error in PE/COFF --- libcxx/include/__configuration/abi.h | 5 +++++ libcxx/include/__memory/align.h | 7 +++++++ libcxx/src/memory.cpp | 1 + 3 files changed, 13 insertions(+) diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h index f88de6442f90d..6d519597b5a26 100644 --- a/libcxx/include/__configuration/abi.h +++ b/libcxx/include/__configuration/abi.h @@ -97,6 +97,11 @@ // so disable it. # define _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS # endif +// The PE/COFF format reports a linking error when encountering multiple symbol definitions where at least one is a +// strong symbol. So we can't inlining a function without ABI breakchange. +# if defined(_LIBCPP_OBJECT_FORMAT_COFF) +# define _LIBCPP_DISABLE_INLINE_OPTIMIZE_BECAUSE_MULTIPLY_SYMBOLS_ERROR +# endif // Feature macros for disabling pre ABI v1 features. All of these options // are deprecated. # if defined(__FreeBSD__) diff --git a/libcxx/include/__memory/align.h b/libcxx/include/__memory/align.h index 8aaf08192f2af..d987599a4cfc7 100644 --- a/libcxx/include/__memory/align.h +++ b/libcxx/include/__memory/align.h @@ -36,11 +36,18 @@ __align_impl(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { } #ifndef _LIBCPP_EXPORT_ALIGN_SYMBOL +# ifdef _LIBCPP_DISABLE_INLINE_OPTIMIZE_BECAUSE_MULTIPLY_SYMBOLS_ERROR + +_LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); + +# else inline _LIBCPP_HIDE_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { return __align_impl(__align, __sz, __ptr, __space); } +# endif // _LIBCPP_DISABLE_INLINE_OPTIMIZE_BECAUSE_MULTIPLY_SYMBOLS_ERROR + #endif // _LIBCPP_EXPORT_ALIGN_SYMBOL _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/src/memory.cpp b/libcxx/src/memory.cpp index 19b2894b87ec1..547500cf75937 100644 --- a/libcxx/src/memory.cpp +++ b/libcxx/src/memory.cpp @@ -140,6 +140,7 @@ __sp_mut& __get_sp_mut(const void* p) { _LIBCPP_EXPORTED_FROM_ABI void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { return __align_impl(alignment, size, ptr, space); } + #endif // _LIBCPP_EXPORT_ALIGN_SYMBOL _LIBCPP_END_NAMESPACE_STD From d10f92954dd2165d301c2aa2f45f05778e9db11b Mon Sep 17 00:00:00 2001 From: "zezheng.li" Date: Fri, 14 Nov 2025 17:53:10 +0800 Subject: [PATCH 8/8] fix conversation --- libcxx/docs/ReleaseNotes/22.rst | 2 +- libcxx/include/__configuration/abi.h | 11 +++--- libcxx/include/__memory/align.h | 24 +++++-------- libcxx/src/CMakeLists.txt | 1 + libcxx/src/memory.cpp | 11 ------ libcxx/src/memory_align.cpp | 36 +++++++++++++++++++ libcxx/test/benchmarks/memory/align.bench.cpp | 22 +++++++++--- llvm/utils/gn/secondary/libcxx/src/BUILD.gn | 1 + 8 files changed, 70 insertions(+), 38 deletions(-) create mode 100644 libcxx/src/memory_align.cpp diff --git a/libcxx/docs/ReleaseNotes/22.rst b/libcxx/docs/ReleaseNotes/22.rst index 670d73992e7e6..aca7a65aaa2fc 100644 --- a/libcxx/docs/ReleaseNotes/22.rst +++ b/libcxx/docs/ReleaseNotes/22.rst @@ -83,7 +83,7 @@ Improvements and New Features iterators, resulting in a performance improvement for ``std::deque`` and ``std::join_view>>`` iterators. -- The performance of ``align`` has been improved about 3x by making it an inline function. +- The performance of ``align`` has been improved about 2x by making it an inline function. Deprecations and Removals ------------------------- diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h index 6d519597b5a26..fc3729dbb02af 100644 --- a/libcxx/include/__configuration/abi.h +++ b/libcxx/include/__configuration/abi.h @@ -97,11 +97,6 @@ // so disable it. # define _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS # endif -// The PE/COFF format reports a linking error when encountering multiple symbol definitions where at least one is a -// strong symbol. So we can't inlining a function without ABI breakchange. -# if defined(_LIBCPP_OBJECT_FORMAT_COFF) -# define _LIBCPP_DISABLE_INLINE_OPTIMIZE_BECAUSE_MULTIPLY_SYMBOLS_ERROR -# endif // Feature macros for disabling pre ABI v1 features. All of these options // are deprecated. # if defined(__FreeBSD__) @@ -109,6 +104,12 @@ # endif #endif +// The PE/COFF format reports a linking error when encountering multiple symbol definitions where at least one is a +// strong symbol. So we can't inlining a non-inline function without ABI break change. +#if defined(_LIBCPP_OBJECT_FORMAT_COFF) +# define _LIBCPP_DISABLE_INLINE_OPTIMIZE_BECAUSE_MULTIPLY_SYMBOLS_ERROR +#endif + // TODO(LLVM 22): Remove this check #if defined(_LIBCPP_ABI_NO_ITERATOR_BASES) && !defined(_LIBCPP_ABI_NO_REVERSE_ITERATOR_SECOND_MEMBER) # ifndef _LIBCPP_ONLY_NO_ITERATOR_BASES diff --git a/libcxx/include/__memory/align.h b/libcxx/include/__memory/align.h index d987599a4cfc7..c5aff63d1f5e3 100644 --- a/libcxx/include/__memory/align.h +++ b/libcxx/include/__memory/align.h @@ -19,8 +19,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD -_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE inline void* -__align_impl(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { +#if defined(_LIBCPP_DISABLE_INLINE_OPTIMIZE_BECAUSE_MULTIPLY_SYMBOLS_ERROR) && !defined(_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN) + +_LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); + +#else + +_LIBCPP_HIDE_FROM_ABI inline void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { void* __r = nullptr; if (__sz <= __space) { char* __p1 = static_cast(__ptr); @@ -35,20 +40,7 @@ __align_impl(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { return __r; } -#ifndef _LIBCPP_EXPORT_ALIGN_SYMBOL -# ifdef _LIBCPP_DISABLE_INLINE_OPTIMIZE_BECAUSE_MULTIPLY_SYMBOLS_ERROR - -_LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); - -# else - -inline _LIBCPP_HIDE_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { - return __align_impl(__align, __sz, __ptr, __space); -} - -# endif // _LIBCPP_DISABLE_INLINE_OPTIMIZE_BECAUSE_MULTIPLY_SYMBOLS_ERROR - -#endif // _LIBCPP_EXPORT_ALIGN_SYMBOL +#endif _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt index f59fe0e08fccb..dc0ccbf5f54cb 100644 --- a/libcxx/src/CMakeLists.txt +++ b/libcxx/src/CMakeLists.txt @@ -31,6 +31,7 @@ set(LIBCXX_SOURCES include/to_chars_floating_point.h include/from_chars_floating_point.h memory.cpp + memory_align.cpp memory_resource.cpp new_handler.cpp new_helpers.cpp diff --git a/libcxx/src/memory.cpp b/libcxx/src/memory.cpp index 547500cf75937..2cfbf4f7f7b30 100644 --- a/libcxx/src/memory.cpp +++ b/libcxx/src/memory.cpp @@ -10,9 +10,6 @@ #ifdef _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS # define _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS #endif -#ifndef _LIBCPP_ABI_DO_NOT_EXPORT_ALIGN -# define _LIBCPP_EXPORT_ALIGN_SYMBOL -#endif #include <__functional/hash.h> #include @@ -135,12 +132,4 @@ __sp_mut& __get_sp_mut(const void* p) { #endif // _LIBCPP_HAS_THREADS -#if defined(_LIBCPP_EXPORT_ALIGN_SYMBOL) - -_LIBCPP_EXPORTED_FROM_ABI void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { - return __align_impl(alignment, size, ptr, space); -} - -#endif // _LIBCPP_EXPORT_ALIGN_SYMBOL - _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/src/memory_align.cpp b/libcxx/src/memory_align.cpp new file mode 100644 index 0000000000000..6b70c64a505a7 --- /dev/null +++ b/libcxx/src/memory_align.cpp @@ -0,0 +1,36 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <__config> +#include <__cstddef/size_t.h> +#include + +// Don't include to avoid mulitple declartion of align() + +#if !defined(_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN) + +_LIBCPP_BEGIN_NAMESPACE_STD + +_LIBCPP_EXPORTED_FROM_ABI void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { + void* r = nullptr; + if (size <= space) { + char* p1 = static_cast(ptr); + char* p2 = reinterpret_cast(reinterpret_cast(p1 + (alignment - 1)) & -alignment); + size_t d = static_cast(p2 - p1); + if (d <= space - size) { + r = p2; + ptr = r; + space -= d; + } + } + return r; +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP_ABI_DO_NOT_EXPORT_ALIGN diff --git a/libcxx/test/benchmarks/memory/align.bench.cpp b/libcxx/test/benchmarks/memory/align.bench.cpp index 74be6f7a598f7..9fb637c21f9ac 100644 --- a/libcxx/test/benchmarks/memory/align.bench.cpp +++ b/libcxx/test/benchmarks/memory/align.bench.cpp @@ -9,17 +9,29 @@ // UNSUPPORTED: c++03 #include +#include #include "benchmark/benchmark.h" -#include "test_macros.h" + +struct Input { + std::size_t align; + std::size_t size; + void* ptr; + std::size_t buffer_size; +}; static void BM_align(benchmark::State& state) { char buffer[1024]; - void* data = buffer + 123; - std::size_t sz{sizeof(buffer) - 123}; - + Input input{}; + void* ptr = buffer + 123; + std::size_t buffer_size = sizeof(buffer) - 123; + input.align = state.range(); + input.size = state.range(); for (auto _ : state) { - benchmark::DoNotOptimize(std::align(state.range(), state.range(), data, sz)); + input.ptr = ptr; + input.buffer_size = buffer_size; + benchmark::DoNotOptimize(input); + benchmark::DoNotOptimize(std::align(input.align, input.size, input.ptr, input.buffer_size)); } } BENCHMARK(BM_align)->Range(1, 256); diff --git a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn index 327a8ede31fff..0c1953f7a2c92 100644 --- a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn +++ b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn @@ -152,6 +152,7 @@ cxx_sources = [ "iostream.cpp", "locale.cpp", "memory.cpp", + "memory_align.cpp" "memory_resource.cpp", "mutex.cpp", "mutex_destructor.cpp",