Author: hxy9243 Date: Mon Jan 16 20:46:33 2017 New Revision: 292184 URL: http://llvm.org/viewvc/llvm-project?rev=292184&view=rev Log: [Test patch] Inline hot functions in libcxx shared_ptr
Moves hot functions such as atomic add into the memory header file so that they can be inlined, which brings performance benefits. Patch by Kevin Hu, Aditya Kumar, Sebastian Pop Differential Revision: https://reviews.llvm.org/D24991 Modified: libcxx/trunk/include/memory libcxx/trunk/src/memory.cpp Modified: libcxx/trunk/include/memory URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/memory?rev=292184&r1=292183&r2=292184&view=diff ============================================================================== --- libcxx/trunk/include/memory (original) +++ libcxx/trunk/include/memory Mon Jan 16 20:46:33 2017 @@ -3681,6 +3681,39 @@ uninitialized_move_n(_InputIt __first, _ #endif // _LIBCPP_STD_VER > 14 +// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively) +// should be sufficient for thread safety. +// See https://llvm.org/bugs/show_bug.cgi?id=22803 +#if defined(__clang__) && __has_builtin(__atomic_add_fetch) \ + && defined(__ATOMIC_RELAXED) \ + && defined(__ATOMIC_ACQ_REL) +# define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT +#elif !defined(__clang__) && defined(_GNUC_VER) && _GNUC_VER >= 407 +# define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT +#endif + +template <class _Tp> +inline _LIBCPP_INLINE_VISIBILITY _Tp +__libcpp_atomic_refcount_increment(_Tp& __t) _NOEXCEPT +{ +#if defined(_LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT) && !defined(_LIBCPP_HAS_NO_THREADS) + return __atomic_add_fetch(&__t, 1, __ATOMIC_RELAXED); +#else + return __t += 1; +#endif +} + +template <class _Tp> +inline _LIBCPP_INLINE_VISIBILITY _Tp +__libcpp_atomic_refcount_decrement(_Tp& __t) _NOEXCEPT +{ +#if defined(_LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT) && !defined(_LIBCPP_HAS_NO_THREADS) + return __atomic_add_fetch(&__t, -1, __ATOMIC_ACQ_REL); +#else + return __t -= 1; +#endif +} + class _LIBCPP_EXCEPTION_ABI bad_weak_ptr : public std::exception { @@ -3717,8 +3750,23 @@ public: explicit __shared_count(long __refs = 0) _NOEXCEPT : __shared_owners_(__refs) {} - void __add_shared() _NOEXCEPT; - bool __release_shared() _NOEXCEPT; +#ifdef _LIBCPP_BUILDING_MEMORY + _LIBCPP_FUNC_VIS void __add_shared() _NOEXCEPT; + _LIBCPP_FUNC_VIS bool __release_shared() _NOEXCEPT; +#else + _LIBCPP_INLINE_VISIBILITY + void __add_shared() _NOEXCEPT { + __libcpp_atomic_refcount_increment(__shared_owners_); + } + _LIBCPP_INLINE_VISIBILITY + bool __release_shared() _NOEXCEPT { + if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) { + __on_zero_shared(); + return true; + } + return false; + } +#endif _LIBCPP_INLINE_VISIBILITY long use_count() const _NOEXCEPT { return __libcpp_relaxed_load(&__shared_owners_) + 1; @@ -3739,9 +3787,25 @@ protected: virtual ~__shared_weak_count(); public: - void __add_shared() _NOEXCEPT; - void __add_weak() _NOEXCEPT; - void __release_shared() _NOEXCEPT; +#ifdef _LIBCPP_BUILDING_MEMORY + _LIBCPP_FUNC_VIS void __add_shared() _NOEXCEPT; + _LIBCPP_FUNC_VIS void __add_weak() _NOEXCEPT; + _LIBCPP_FUNC_VIS void __release_shared() _NOEXCEPT; +#else + _LIBCPP_INLINE_VISIBILITY + void __add_shared() _NOEXCEPT { + __shared_count::__add_shared(); + } + _LIBCPP_INLINE_VISIBILITY + void __add_weak() _NOEXCEPT { + __libcpp_atomic_refcount_increment(__shared_weak_owners_); + } + _LIBCPP_INLINE_VISIBILITY + void __release_shared() _NOEXCEPT { + if (__shared_count::__release_shared()) + __release_weak(); + } +#endif void __release_weak() _NOEXCEPT; _LIBCPP_INLINE_VISIBILITY long use_count() const _NOEXCEPT {return __shared_count::use_count();} Modified: libcxx/trunk/src/memory.cpp URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/src/memory.cpp?rev=292184&r1=292183&r2=292184&view=diff ============================================================================== --- libcxx/trunk/src/memory.cpp (original) +++ libcxx/trunk/src/memory.cpp Mon Jan 16 20:46:33 2017 @@ -17,28 +17,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace -{ - -// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively) -// should be sufficient for thread safety. -// See https://llvm.org/bugs/show_bug.cgi?id=22803 -template <class T> -inline T -increment(T& t) _NOEXCEPT -{ - return __libcpp_atomic_add(&t, 1, _AO_Relaxed); -} - -template <class T> -inline T -decrement(T& t) _NOEXCEPT -{ - return __libcpp_atomic_add(&t, -1, _AO_Acq_Rel); -} - -} // namespace - const allocator_arg_t allocator_arg = allocator_arg_t(); bad_weak_ptr::~bad_weak_ptr() _NOEXCEPT {} @@ -53,16 +31,20 @@ __shared_count::~__shared_count() { } +__shared_weak_count::~__shared_weak_count() +{ +} + void __shared_count::__add_shared() _NOEXCEPT { - increment(__shared_owners_); + __libcpp_atomic_refcount_increment(__shared_owners_); } bool __shared_count::__release_shared() _NOEXCEPT { - if (decrement(__shared_owners_) == -1) + if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) { __on_zero_shared(); return true; @@ -70,10 +52,6 @@ __shared_count::__release_shared() _NOEX return false; } -__shared_weak_count::~__shared_weak_count() -{ -} - void __shared_weak_count::__add_shared() _NOEXCEPT { @@ -83,7 +61,7 @@ __shared_weak_count::__add_shared() _NOE void __shared_weak_count::__add_weak() _NOEXCEPT { - increment(__shared_weak_owners_); + __libcpp_atomic_refcount_increment(__shared_weak_owners_); } void @@ -124,7 +102,7 @@ __shared_weak_count::__release_weak() _N //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release); __on_zero_shared_weak(); } - else if (decrement(__shared_weak_owners_) == -1) + else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1) __on_zero_shared_weak(); } _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
