[libc++] Make std::align an inline function

libcxx/docs/ABIGuarantees.rst

@@ -114,6 +114,10 @@ hand, backwards compatibility is generally guaranteed.
 
 There are multiple ABI flags that change the symbols exported from the built library:
 
+``_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN``
+-------------------------------------------------
+This removes ``align()``. In the past ``align()`` is not an inline function, but now it changed to an inline function for performance.
+
 ``_LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON``
 -------------------------------------------------
 This removes ``__basic_string_common<true>::__throw_length_error()`` and

libcxx/docs/ReleaseNotes/22.rst

@@ -83,6 +83,8 @@ Improvements and New Features
   iterators, resulting in a performance improvement for ``std::deque<short>`` and
   ``std::join_view<vector<vector<short>>>`` iterators.
 
+- The performance of ``align`` has been improved about 2x by making it an inline function.
+
 Deprecations and Removals
 -------------------------
 
@@ -114,5 +116,7 @@ ABI Affecting Changes
   potentially inheriting from the types they wrap. At this point in time we are not aware of any ABI changes caused by
   this.
 
+- ``align`` has been removed from libc++ library in ABI v2. We also add a new ABI flag ``_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN``.
+
 Build System Changes
 --------------------

libcxx/include/__configuration/abi.h

@@ -63,6 +63,7 @@
 
 // These flags are documented in ABIGuarantees.rst
 #  define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+#  define _LIBCPP_ABI_DO_NOT_EXPORT_ALIGN
 #  define _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON
 #  define _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON
 #  define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10
@@ -103,6 +104,12 @@
 #  endif
 #endif
 
+// The PE/COFF format reports a linking error when encountering multiple symbol definitions where at least one is a
+// strong symbol. So we can't inlining a non-inline function without ABI break change.
+#if defined(_LIBCPP_OBJECT_FORMAT_COFF)
+#  define _LIBCPP_DISABLE_INLINE_OPTIMIZE_BECAUSE_MULTIPLY_SYMBOLS_ERROR
+#endif
+
 // TODO(LLVM 22): Remove this check
 #if defined(_LIBCPP_ABI_NO_ITERATOR_BASES) && !defined(_LIBCPP_ABI_NO_REVERSE_ITERATOR_SECOND_MEMBER)
 #  ifndef _LIBCPP_ONLY_NO_ITERATOR_BASES

libcxx/include/__memory/align.h

@@ -11,15 +11,37 @@
 
 #include <__config>
 #include <__cstddef/size_t.h>
+#include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+#if defined(_LIBCPP_DISABLE_INLINE_OPTIMIZE_BECAUSE_MULTIPLY_SYMBOLS_ERROR) && !defined(_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN)
+
 _LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space);
 
+#else
+
+_LIBCPP_HIDE_FROM_ABI inline void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) {
+  void* __r = nullptr;
+  if (__sz <= __space) {
+    char* __p1 = static_cast<char*>(__ptr);
+    char* __p2 = reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(__p1 + (__align - 1)) & -__align);
+    size_t __d = static_cast<size_t>(__p2 - __p1);
+    if (__d <= __space - __sz) {
+      __r   = __p2;
+      __ptr = __r;
+      __space -= __d;
+    }
+  }
+  return __r;
+}
+
+#endif
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___MEMORY_ALIGN_H

libcxx/src/CMakeLists.txt

@@ -31,6 +31,7 @@ set(LIBCXX_SOURCES
   include/to_chars_floating_point.h
   include/from_chars_floating_point.h
   memory.cpp
+  memory_align.cpp
   memory_resource.cpp
   new_handler.cpp
   new_helpers.cpp

libcxx/src/memory.cpp

@@ -132,19 +132,4 @@ __sp_mut& __get_sp_mut(const void* p) {
 
 #endif // _LIBCPP_HAS_THREADS
 
-void* align(size_t alignment, size_t size, void*& ptr, size_t& space) {
-  void* r = nullptr;
-  if (size <= space) {
-    char* p1 = static_cast<char*>(ptr);
-    char* p2 = reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(p1 + (alignment - 1)) & -alignment);
-    size_t d = static_cast<size_t>(p2 - p1);
-    if (d <= space - size) {
-      r   = p2;
-      ptr = r;
-      space -= d;
-    }
-  }
-  return r;
-}
-
 _LIBCPP_END_NAMESPACE_STD

libcxx/src/memory_align.cpp

@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <__config>
+#include <__cstddef/size_t.h>
+#include <cstdint>
+
+// Don't include <memory> to avoid mulitple declartion of align()
+
+#if !defined(_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN)
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_EXPORTED_FROM_ABI void* align(size_t alignment, size_t size, void*& ptr, size_t& space) {
+  void* r = nullptr;
+  if (size <= space) {
+    char* p1 = static_cast<char*>(ptr);
+    char* p2 = reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(p1 + (alignment - 1)) & -alignment);
+    size_t d = static_cast<size_t>(p2 - p1);
+    if (d <= space - size) {
+      r   = p2;
+      ptr = r;
+      space -= d;
+    }
+  }
+  return r;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_ABI_DO_NOT_EXPORT_ALIGN

libcxx/test/benchmarks/memory/align.bench.cpp

@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03
+
+#include <memory>
+#include <iostream>
+
+#include "benchmark/benchmark.h"
+
+struct Input {
+  std::size_t align;
+  std::size_t size;
+  void* ptr;
+  std::size_t buffer_size;
+};
+
+static void BM_align(benchmark::State& state) {
+  char buffer[1024];
+  Input input{};
+  void* ptr               = buffer + 123;
+  std::size_t buffer_size = sizeof(buffer) - 123;
+  input.align             = state.range();
+  input.size              = state.range();
+  for (auto _ : state) {
+    input.ptr         = ptr;
+    input.buffer_size = buffer_size;
+    benchmark::DoNotOptimize(input);
+    benchmark::DoNotOptimize(std::align(input.align, input.size, input.ptr, input.buffer_size));
+  }
+}
+BENCHMARK(BM_align)->Range(1, 256);
+
+BENCHMARK_MAIN();

llvm/utils/gn/secondary/libcxx/src/BUILD.gn

@@ -152,6 +152,7 @@ cxx_sources = [
   "iostream.cpp",
   "locale.cpp",
   "memory.cpp",
+  "memory_align.cpp"
   "memory_resource.cpp",
   "mutex.cpp",
   "mutex_destructor.cpp",