[PATCH] D24991: Inline hot functions in libcxx shared_ptr implementation.

2017-01-03 Thread Kevin Hu via Phabricator via cfe-commits 
hxy9243 set the repository for this revision to rL LLVM.
hxy9243 updated this revision to Diff 82958.
hxy9243 added a comment.

Move the header back in its place, and only copy over necessary parts. Now call 
__atomic_add_fetch from inside the functions.


Repository:
  rL LLVM

https://reviews.llvm.org/D24991

Files:
  libcxx/include/memory
  libcxx/src/memory.cpp

Index: libcxx/src/memory.cpp
===
--- libcxx/src/memory.cpp
+++ libcxx/src/memory.cpp
@@ -17,28 +17,6 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-namespace
-{
-
-// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
-// should be sufficient for thread safety.
-// See https://llvm.org/bugs/show_bug.cgi?id=22803
-template 
-inline T
-increment(T& t) _NOEXCEPT
-{
-return __libcpp_atomic_add(&t, 1, _AO_Relaxed);
-}
-
-template 
-inline T
-decrement(T& t) _NOEXCEPT
-{
-return __libcpp_atomic_add(&t, -1, _AO_Acq_Rel);
-}
-
-}  // namespace
-
 const allocator_arg_t allocator_arg = allocator_arg_t();
 
 bad_weak_ptr::~bad_weak_ptr() _NOEXCEPT {}
@@ -53,27 +31,27 @@
 {
 }
 
+__shared_weak_count::~__shared_weak_count()
+{
+}
+
 void
 __shared_count::__add_shared() _NOEXCEPT
 {
-increment(__shared_owners_);
+__libcpp_atomic_increment(__shared_owners_);
 }
 
 bool
 __shared_count::__release_shared() _NOEXCEPT
 {
-if (decrement(__shared_owners_) == -1)
+if (__libcpp_atomic_decrement(__shared_owners_) == -1)
 {
 __on_zero_shared();
 return true;
 }
 return false;
 }
 
-__shared_weak_count::~__shared_weak_count()
-{
-}
-
 void
 __shared_weak_count::__add_shared() _NOEXCEPT
 {
@@ -83,7 +61,7 @@
 void
 __shared_weak_count::__add_weak() _NOEXCEPT
 {
-increment(__shared_weak_owners_);
+__libcpp_atomic_increment(__shared_weak_owners_);
 }
 
 void
@@ -124,7 +102,7 @@
 //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release);
 __on_zero_shared_weak();
 }
-else if (decrement(__shared_weak_owners_) == -1)
+else if (__libcpp_atomic_decrement(__shared_weak_owners_) == -1)
 __on_zero_shared_weak();
 }
 
Index: libcxx/include/memory
===
--- libcxx/include/memory
+++ libcxx/include/memory
@@ -3681,6 +3681,44 @@
 
 #endif // _LIBCPP_STD_VER > 14
 
+// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
+// should be sufficient for thread safety.
+// See https://llvm.org/bugs/show_bug.cgi?id=22803
+#if defined(__clang__) && __has_builtin(__atomic_load_n) \
+   && __has_builtin(__atomic_store_n)\
+   && __has_builtin(__atomic_add_fetch)  \
+   && __has_builtin(__atomic_compare_exchange_n) \
+   && defined(__ATOMIC_RELAXED)  \
+   && defined(__ATOMIC_CONSUME)  \
+   && defined(__ATOMIC_ACQUIRE)  \
+   && defined(__ATOMIC_RELEASE)  \
+   && defined(__ATOMIC_ACQ_REL)  \
+   && defined(__ATOMIC_SEQ_CST)
+#   define _LIBCPP_HAS_ATOMIC_BUILTINS
+#endif
+
+template 
+inline T
+__libcpp_atomic_increment(T& t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_ATOMIC_BUILTINS) && !defined(_LIBCPP_HAS_NO_THREADS)
+return __atomic_add_fetch(&t, 1, __ATOMIC_RELAXED);
+#else
+return t += 1;
+#endif
+}
+
+template 
+inline T
+__libcpp_atomic_decrement(T& t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_ATOMIC_BUILTINS) && !defined(_LIBCPP_HAS_NO_THREADS)
+return __atomic_add_fetch(&t, -1, __ATOMIC_ACQ_REL);
+#else
+return t -= 1;
+#endif
+}
+
 class _LIBCPP_EXCEPTION_ABI bad_weak_ptr
 : public std::exception
 {
@@ -3717,8 +3755,23 @@
 explicit __shared_count(long __refs = 0) _NOEXCEPT
 : __shared_owners_(__refs) {}
 
-void __add_shared() _NOEXCEPT;
-bool __release_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
+void _LIBCPP_FUNC_VIS __add_shared() _NOEXCEPT;
+bool _LIBCPP_FUNC_VIS __release_shared() _NOEXCEPT;
+#else
+inline _LIBCPP_INLINE_VISIBILITY
+void __add_shared() _NOEXCEPT {
+  __libcpp_atomic_increment(__shared_owners_);
+}
+inline _LIBCPP_INLINE_VISIBILITY
+bool __release_shared() _NOEXCEPT {
+  if (__libcpp_atomic_decrement(__shared_owners_) == -1) {
+__on_zero_shared();
+return true;
+  }
+  return false;
+}
+#endif
 _LIBCPP_INLINE_VISIBILITY
 long use_count() const _NOEXCEPT {
 return __libcpp_relaxed_load(&__shared_owners_) + 1;
@@ -3739,9 +3792,25 @@
 virtual ~__shared_weak_count();
 
 public:
-void __add_shared() _NOEXCEPT;
-void __add_weak() _NOEXCEPT;
-void __release_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
+void _LIBCPP_FUNC_VIS __add_shared() _NOEXCEPT;
+void _LIBCPP_FUNC_VIS _

[PATCH] D24991: Inline hot functions in libcxx shared_ptr implementation.

2017-01-03 Thread Kevin Hu via Phabricator via cfe-commits 
hxy9243 updated this revision to Diff 82962.
hxy9243 marked 4 inline comments as done.
hxy9243 added a comment.

Minor fix, remove redundant inlines.


Repository:
  rL LLVM

https://reviews.llvm.org/D24991

Files:
  libcxx/include/memory
  libcxx/src/memory.cpp

Index: libcxx/src/memory.cpp
===
--- libcxx/src/memory.cpp
+++ libcxx/src/memory.cpp
@@ -17,28 +17,6 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-namespace
-{
-
-// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
-// should be sufficient for thread safety.
-// See https://llvm.org/bugs/show_bug.cgi?id=22803
-template 
-inline T
-increment(T& t) _NOEXCEPT
-{
-return __libcpp_atomic_add(&t, 1, _AO_Relaxed);
-}
-
-template 
-inline T
-decrement(T& t) _NOEXCEPT
-{
-return __libcpp_atomic_add(&t, -1, _AO_Acq_Rel);
-}
-
-}  // namespace
-
 const allocator_arg_t allocator_arg = allocator_arg_t();
 
 bad_weak_ptr::~bad_weak_ptr() _NOEXCEPT {}
@@ -53,27 +31,27 @@
 {
 }
 
+__shared_weak_count::~__shared_weak_count()
+{
+}
+
 void
 __shared_count::__add_shared() _NOEXCEPT
 {
-increment(__shared_owners_);
+__libcpp_atomic_increment(__shared_owners_);
 }
 
 bool
 __shared_count::__release_shared() _NOEXCEPT
 {
-if (decrement(__shared_owners_) == -1)
+if (__libcpp_atomic_decrement(__shared_owners_) == -1)
 {
 __on_zero_shared();
 return true;
 }
 return false;
 }
 
-__shared_weak_count::~__shared_weak_count()
-{
-}
-
 void
 __shared_weak_count::__add_shared() _NOEXCEPT
 {
@@ -83,7 +61,7 @@
 void
 __shared_weak_count::__add_weak() _NOEXCEPT
 {
-increment(__shared_weak_owners_);
+__libcpp_atomic_increment(__shared_weak_owners_);
 }
 
 void
@@ -124,7 +102,7 @@
 //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release);
 __on_zero_shared_weak();
 }
-else if (decrement(__shared_weak_owners_) == -1)
+else if (__libcpp_atomic_decrement(__shared_weak_owners_) == -1)
 __on_zero_shared_weak();
 }
 
Index: libcxx/include/memory
===
--- libcxx/include/memory
+++ libcxx/include/memory
@@ -3681,6 +3681,44 @@
 
 #endif // _LIBCPP_STD_VER > 14
 
+// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
+// should be sufficient for thread safety.
+// See https://llvm.org/bugs/show_bug.cgi?id=22803
+#if defined(__clang__) && __has_builtin(__atomic_load_n) \
+   && __has_builtin(__atomic_store_n)\
+   && __has_builtin(__atomic_add_fetch)  \
+   && __has_builtin(__atomic_compare_exchange_n) \
+   && defined(__ATOMIC_RELAXED)  \
+   && defined(__ATOMIC_CONSUME)  \
+   && defined(__ATOMIC_ACQUIRE)  \
+   && defined(__ATOMIC_RELEASE)  \
+   && defined(__ATOMIC_ACQ_REL)  \
+   && defined(__ATOMIC_SEQ_CST)
+#   define _LIBCPP_HAS_ATOMIC_BUILTINS
+#endif
+
+template 
+inline T
+__libcpp_atomic_increment(T& t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_ATOMIC_BUILTINS) && !defined(_LIBCPP_HAS_NO_THREADS)
+return __atomic_add_fetch(&t, 1, __ATOMIC_RELAXED);
+#else
+return t += 1;
+#endif
+}
+
+template 
+inline T
+__libcpp_atomic_decrement(T& t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_ATOMIC_BUILTINS) && !defined(_LIBCPP_HAS_NO_THREADS)
+return __atomic_add_fetch(&t, -1, __ATOMIC_ACQ_REL);
+#else
+return t -= 1;
+#endif
+}
+
 class _LIBCPP_EXCEPTION_ABI bad_weak_ptr
 : public std::exception
 {
@@ -3717,8 +3755,23 @@
 explicit __shared_count(long __refs = 0) _NOEXCEPT
 : __shared_owners_(__refs) {}
 
-void __add_shared() _NOEXCEPT;
-bool __release_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
+void _LIBCPP_FUNC_VIS __add_shared() _NOEXCEPT;
+bool _LIBCPP_FUNC_VIS __release_shared() _NOEXCEPT;
+#else
+_LIBCPP_INLINE_VISIBILITY
+void __add_shared() _NOEXCEPT {
+  __libcpp_atomic_increment(__shared_owners_);
+}
+_LIBCPP_INLINE_VISIBILITY
+bool __release_shared() _NOEXCEPT {
+  if (__libcpp_atomic_decrement(__shared_owners_) == -1) {
+__on_zero_shared();
+return true;
+  }
+  return false;
+}
+#endif
 _LIBCPP_INLINE_VISIBILITY
 long use_count() const _NOEXCEPT {
 return __libcpp_relaxed_load(&__shared_owners_) + 1;
@@ -3739,9 +3792,25 @@
 virtual ~__shared_weak_count();
 
 public:
-void __add_shared() _NOEXCEPT;
-void __add_weak() _NOEXCEPT;
-void __release_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
+void _LIBCPP_FUNC_VIS __add_shared() _NOEXCEPT;
+void _LIBCPP_FUNC_VIS __add_weak() _NOEXCEPT;
+void _LIBCPP_FUNC_VIS __release_shared() _NOEXCEPT;
+#else
+_LIBCPP_INLINE_VISIBILITY
+

[PATCH] D24991: Inline hot functions in libcxx shared_ptr implementation.

2017-01-10 Thread Kevin Hu via Phabricator via cfe-commits 
hxy9243 marked an inline comment as done.
hxy9243 added a comment.

Addressed previous issues in the comments. The patch still shows consistent 
perf uplift in proprietary benchmark on shared_ptr.

@EricWF @sebpop @hiraditya Any thoughts?


Repository:
  rL LLVM

https://reviews.llvm.org/D24991



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D24991: Inline hot functions in libcxx shared_ptr implementation.

2017-01-11 Thread Kevin Hu via Phabricator via cfe-commits 
hxy9243 updated this revision to Diff 84004.
hxy9243 added a comment.

Minor changes on function names. Rename __libcpp_atomic_* to 
__libcpp_atomic_refcount_*.


Repository:
  rL LLVM

https://reviews.llvm.org/D24991

Files:
  libcxx/include/memory
  libcxx/src/memory.cpp

Index: libcxx/src/memory.cpp
===
--- libcxx/src/memory.cpp
+++ libcxx/src/memory.cpp
@@ -17,28 +17,6 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-namespace
-{
-
-// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
-// should be sufficient for thread safety.
-// See https://llvm.org/bugs/show_bug.cgi?id=22803
-template 
-inline T
-increment(T& t) _NOEXCEPT
-{
-return __libcpp_atomic_add(&t, 1, _AO_Relaxed);
-}
-
-template 
-inline T
-decrement(T& t) _NOEXCEPT
-{
-return __libcpp_atomic_add(&t, -1, _AO_Acq_Rel);
-}
-
-}  // namespace
-
 const allocator_arg_t allocator_arg = allocator_arg_t();
 
 bad_weak_ptr::~bad_weak_ptr() _NOEXCEPT {}
@@ -53,27 +31,27 @@
 {
 }
 
+__shared_weak_count::~__shared_weak_count()
+{
+}
+
 void
 __shared_count::__add_shared() _NOEXCEPT
 {
-increment(__shared_owners_);
+__libcpp_atomic_refcount_increment(__shared_owners_);
 }
 
 bool
 __shared_count::__release_shared() _NOEXCEPT
 {
-if (decrement(__shared_owners_) == -1)
+if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1)
 {
 __on_zero_shared();
 return true;
 }
 return false;
 }
 
-__shared_weak_count::~__shared_weak_count()
-{
-}
-
 void
 __shared_weak_count::__add_shared() _NOEXCEPT
 {
@@ -83,7 +61,7 @@
 void
 __shared_weak_count::__add_weak() _NOEXCEPT
 {
-increment(__shared_weak_owners_);
+__libcpp_atomic_refcount_increment(__shared_weak_owners_);
 }
 
 void
@@ -124,7 +102,7 @@
 //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release);
 __on_zero_shared_weak();
 }
-else if (decrement(__shared_weak_owners_) == -1)
+else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1)
 __on_zero_shared_weak();
 }
 
Index: libcxx/include/memory
===
--- libcxx/include/memory
+++ libcxx/include/memory
@@ -3681,6 +3681,44 @@
 
 #endif // _LIBCPP_STD_VER > 14
 
+// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
+// should be sufficient for thread safety.
+// See https://llvm.org/bugs/show_bug.cgi?id=22803
+#if defined(__clang__) && __has_builtin(__atomic_load_n) \
+   && __has_builtin(__atomic_store_n)\
+   && __has_builtin(__atomic_add_fetch)  \
+   && __has_builtin(__atomic_compare_exchange_n) \
+   && defined(__ATOMIC_RELAXED)  \
+   && defined(__ATOMIC_CONSUME)  \
+   && defined(__ATOMIC_ACQUIRE)  \
+   && defined(__ATOMIC_RELEASE)  \
+   && defined(__ATOMIC_ACQ_REL)  \
+   && defined(__ATOMIC_SEQ_CST)
+#   define _LIBCPP_HAS_ATOMIC_BUILTINS
+#endif
+
+template 
+inline T
+__libcpp_atomic_refcount_increment(T& t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_ATOMIC_BUILTINS) && !defined(_LIBCPP_HAS_NO_THREADS)
+return __atomic_add_fetch(&t, 1, __ATOMIC_RELAXED);
+#else
+return t += 1;
+#endif
+}
+
+template 
+inline T
+__libcpp_atomic_refcount_decrement(T& t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_ATOMIC_BUILTINS) && !defined(_LIBCPP_HAS_NO_THREADS)
+return __atomic_add_fetch(&t, -1, __ATOMIC_ACQ_REL);
+#else
+return t -= 1;
+#endif
+}
+
 class _LIBCPP_EXCEPTION_ABI bad_weak_ptr
 : public std::exception
 {
@@ -3717,8 +3755,23 @@
 explicit __shared_count(long __refs = 0) _NOEXCEPT
 : __shared_owners_(__refs) {}
 
-void __add_shared() _NOEXCEPT;
-bool __release_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
+void _LIBCPP_FUNC_VIS __add_shared() _NOEXCEPT;
+bool _LIBCPP_FUNC_VIS __release_shared() _NOEXCEPT;
+#else
+_LIBCPP_INLINE_VISIBILITY
+void __add_shared() _NOEXCEPT {
+  __libcpp_atomic_refcount_increment(__shared_owners_);
+}
+_LIBCPP_INLINE_VISIBILITY
+bool __release_shared() _NOEXCEPT {
+  if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) {
+__on_zero_shared();
+return true;
+  }
+  return false;
+}
+#endif
 _LIBCPP_INLINE_VISIBILITY
 long use_count() const _NOEXCEPT {
 return __libcpp_relaxed_load(&__shared_owners_) + 1;
@@ -3739,9 +3792,25 @@
 virtual ~__shared_weak_count();
 
 public:
-void __add_shared() _NOEXCEPT;
-void __add_weak() _NOEXCEPT;
-void __release_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
+void _LIBCPP_FUNC_VIS __add_shared() _NOEXCEPT;
+void _LIBCPP_FUNC_VIS __add_weak() _NOEXCEPT;
+void _LI

[PATCH] D24991: Inline hot functions in libcxx shared_ptr implementation.

2017-01-16 Thread Kevin Hu via Phabricator via cfe-commits 
hxy9243 updated this revision to Diff 84565.
hxy9243 added a comment.

Addresses comments from @EricWF.

Thanks for reviewing, I know it takes a lot of energy. It helped me learn a lot.


Repository:
  rL LLVM

https://reviews.llvm.org/D24991

Files:
  libcxx/include/memory
  libcxx/src/memory.cpp

Index: libcxx/src/memory.cpp
===
--- libcxx/src/memory.cpp
+++ libcxx/src/memory.cpp
@@ -17,28 +17,6 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-namespace
-{
-
-// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
-// should be sufficient for thread safety.
-// See https://llvm.org/bugs/show_bug.cgi?id=22803
-template 
-inline T
-increment(T& t) _NOEXCEPT
-{
-return __libcpp_atomic_add(&t, 1, _AO_Relaxed);
-}
-
-template 
-inline T
-decrement(T& t) _NOEXCEPT
-{
-return __libcpp_atomic_add(&t, -1, _AO_Acq_Rel);
-}
-
-}  // namespace
-
 const allocator_arg_t allocator_arg = allocator_arg_t();
 
 bad_weak_ptr::~bad_weak_ptr() _NOEXCEPT {}
@@ -53,27 +31,27 @@
 {
 }
 
+__shared_weak_count::~__shared_weak_count()
+{
+}
+
 void
 __shared_count::__add_shared() _NOEXCEPT
 {
-increment(__shared_owners_);
+__libcpp_atomic_refcount_increment(__shared_owners_);
 }
 
 bool
 __shared_count::__release_shared() _NOEXCEPT
 {
-if (decrement(__shared_owners_) == -1)
+if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1)
 {
 __on_zero_shared();
 return true;
 }
 return false;
 }
 
-__shared_weak_count::~__shared_weak_count()
-{
-}
-
 void
 __shared_weak_count::__add_shared() _NOEXCEPT
 {
@@ -83,7 +61,7 @@
 void
 __shared_weak_count::__add_weak() _NOEXCEPT
 {
-increment(__shared_weak_owners_);
+__libcpp_atomic_refcount_increment(__shared_weak_owners_);
 }
 
 void
@@ -124,7 +102,7 @@
 //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release);
 __on_zero_shared_weak();
 }
-else if (decrement(__shared_weak_owners_) == -1)
+else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1)
 __on_zero_shared_weak();
 }
 
Index: libcxx/include/memory
===
--- libcxx/include/memory
+++ libcxx/include/memory
@@ -3681,6 +3681,44 @@
 
 #endif // _LIBCPP_STD_VER > 14
 
+// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
+// should be sufficient for thread safety.
+// See https://llvm.org/bugs/show_bug.cgi?id=22803
+#if defined(__clang__) && __has_builtin(__atomic_load_n) \
+   && __has_builtin(__atomic_store_n)\
+   && __has_builtin(__atomic_add_fetch)  \
+   && __has_builtin(__atomic_compare_exchange_n) \
+   && defined(__ATOMIC_RELAXED)  \
+   && defined(__ATOMIC_CONSUME)  \
+   && defined(__ATOMIC_ACQUIRE)  \
+   && defined(__ATOMIC_RELEASE)  \
+   && defined(__ATOMIC_ACQ_REL)  \
+   && defined(__ATOMIC_SEQ_CST)
+#   define _LIBCPP_HAS_ATOMIC_BUILTINS
+#endif
+
+template 
+inline T
+__libcpp_atomic_refcount_increment(T& t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_ATOMIC_BUILTINS) && !defined(_LIBCPP_HAS_NO_THREADS)
+return __atomic_add_fetch(&t, 1, __ATOMIC_RELAXED);
+#else
+return t += 1;
+#endif
+}
+
+template 
+inline T
+__libcpp_atomic_refcount_decrement(T& t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_ATOMIC_BUILTINS) && !defined(_LIBCPP_HAS_NO_THREADS)
+return __atomic_add_fetch(&t, -1, __ATOMIC_ACQ_REL);
+#else
+return t -= 1;
+#endif
+}
+
 class _LIBCPP_EXCEPTION_ABI bad_weak_ptr
 : public std::exception
 {
@@ -3717,8 +3755,23 @@
 explicit __shared_count(long __refs = 0) _NOEXCEPT
 : __shared_owners_(__refs) {}
 
-void __add_shared() _NOEXCEPT;
-bool __release_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
+void _LIBCPP_FUNC_VIS __add_shared() _NOEXCEPT;
+bool _LIBCPP_FUNC_VIS __release_shared() _NOEXCEPT;
+#else
+_LIBCPP_INLINE_VISIBILITY
+void __add_shared() _NOEXCEPT {
+  __libcpp_atomic_refcount_increment(__shared_owners_);
+}
+_LIBCPP_INLINE_VISIBILITY
+bool __release_shared() _NOEXCEPT {
+  if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) {
+__on_zero_shared();
+return true;
+  }
+  return false;
+}
+#endif
 _LIBCPP_INLINE_VISIBILITY
 long use_count() const _NOEXCEPT {
 return __libcpp_relaxed_load(&__shared_owners_) + 1;
@@ -3739,9 +3792,25 @@
 virtual ~__shared_weak_count();
 
 public:
-void __add_shared() _NOEXCEPT;
-void __add_weak() _NOEXCEPT;
-void __release_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
+void _LIBCPP_FUNC_VIS __add_shared() _NOEXCEPT;
+void _LIBCPP_FUNC_VIS __add_weak()

[PATCH] D24991: Inline hot functions in libcxx shared_ptr implementation.

2017-01-16 Thread Kevin Hu via Phabricator via cfe-commits 
hxy9243 updated this revision to Diff 84569.
hxy9243 added a comment.

Addresses comments from @mclow.lists.


Repository:
  rL LLVM

https://reviews.llvm.org/D24991

Files:
  libcxx/include/memory
  libcxx/src/memory.cpp

Index: libcxx/src/memory.cpp
===
--- libcxx/src/memory.cpp
+++ libcxx/src/memory.cpp
@@ -17,28 +17,6 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-namespace
-{
-
-// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
-// should be sufficient for thread safety.
-// See https://llvm.org/bugs/show_bug.cgi?id=22803
-template 
-inline T
-increment(T& t) _NOEXCEPT
-{
-return __libcpp_atomic_add(&t, 1, _AO_Relaxed);
-}
-
-template 
-inline T
-decrement(T& t) _NOEXCEPT
-{
-return __libcpp_atomic_add(&t, -1, _AO_Acq_Rel);
-}
-
-}  // namespace
-
 const allocator_arg_t allocator_arg = allocator_arg_t();
 
 bad_weak_ptr::~bad_weak_ptr() _NOEXCEPT {}
@@ -53,27 +31,27 @@
 {
 }
 
+__shared_weak_count::~__shared_weak_count()
+{
+}
+
 void
 __shared_count::__add_shared() _NOEXCEPT
 {
-increment(__shared_owners_);
+__libcpp_atomic_refcount_increment(__shared_owners_);
 }
 
 bool
 __shared_count::__release_shared() _NOEXCEPT
 {
-if (decrement(__shared_owners_) == -1)
+if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1)
 {
 __on_zero_shared();
 return true;
 }
 return false;
 }
 
-__shared_weak_count::~__shared_weak_count()
-{
-}
-
 void
 __shared_weak_count::__add_shared() _NOEXCEPT
 {
@@ -83,7 +61,7 @@
 void
 __shared_weak_count::__add_weak() _NOEXCEPT
 {
-increment(__shared_weak_owners_);
+__libcpp_atomic_refcount_increment(__shared_weak_owners_);
 }
 
 void
@@ -124,7 +102,7 @@
 //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release);
 __on_zero_shared_weak();
 }
-else if (decrement(__shared_weak_owners_) == -1)
+else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1)
 __on_zero_shared_weak();
 }
 
Index: libcxx/include/memory
===
--- libcxx/include/memory
+++ libcxx/include/memory
@@ -3681,6 +3681,39 @@
 
 #endif // _LIBCPP_STD_VER > 14
 
+// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
+// should be sufficient for thread safety.
+// See https://llvm.org/bugs/show_bug.cgi?id=22803
+#if defined(__clang__) && __has_builtin(__atomic_add_fetch)  \
+   && defined(__ATOMIC_RELAXED)  \
+   && defined(__ATOMIC_ACQ_REL)
+#   define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT
+#elif !defined(__clang__) && defined(_GNUC_VER) && _GNUC_VER >= 407
+#   define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT
+#endif
+
+template 
+inline _LIBCPP_INLINE_VISIBILITY _Tp
+__libcpp_atomic_refcount_increment(_Tp& __t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT) && !defined(_LIBCPP_HAS_NO_THREADS)
+return __atomic_add_fetch(&__t, 1, __ATOMIC_RELAXED);
+#else
+return __t += 1;
+#endif
+}
+
+template 
+inline _LIBCPP_INLINE_VISIBILITY _Tp
+__libcpp_atomic_refcount_decrement(_Tp& __t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT) && !defined(_LIBCPP_HAS_NO_THREADS)
+return __atomic_add_fetch(&__t, -1, __ATOMIC_ACQ_REL);
+#else
+return __t -= 1;
+#endif
+}
+
 class _LIBCPP_EXCEPTION_ABI bad_weak_ptr
 : public std::exception
 {
@@ -3717,8 +3750,23 @@
 explicit __shared_count(long __refs = 0) _NOEXCEPT
 : __shared_owners_(__refs) {}
 
-void __add_shared() _NOEXCEPT;
-bool __release_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
+_LIBCPP_FUNC_VIS void __add_shared() _NOEXCEPT;
+_LIBCPP_FUNC_VIS bool __release_shared() _NOEXCEPT;
+#else
+_LIBCPP_INLINE_VISIBILITY
+void __add_shared() _NOEXCEPT {
+  __libcpp_atomic_refcount_increment(__shared_owners_);
+}
+_LIBCPP_INLINE_VISIBILITY
+bool __release_shared() _NOEXCEPT {
+  if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) {
+__on_zero_shared();
+return true;
+  }
+  return false;
+}
+#endif
 _LIBCPP_INLINE_VISIBILITY
 long use_count() const _NOEXCEPT {
 return __libcpp_relaxed_load(&__shared_owners_) + 1;
@@ -3739,9 +3787,25 @@
 virtual ~__shared_weak_count();
 
 public:
-void __add_shared() _NOEXCEPT;
-void __add_weak() _NOEXCEPT;
-void __release_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
+_LIBCPP_FUNC_VIS void __add_shared() _NOEXCEPT;
+_LIBCPP_FUNC_VIS void __add_weak() _NOEXCEPT;
+_LIBCPP_FUNC_VIS void __release_shared() _NOEXCEPT;
+#else
+_LIBCPP_INLINE_VISIBILITY
+void __add_shared() _NOEXCEPT {
+  __shared_count::__add_shared();
+}
+_LIBCPP_INLINE_VISIBILITY
+void __add_weak() _NOEXCEPT {
+  __libcpp_atomic_refcount_increment(__shared_weak_owners_);
+}
+_LIBCPP_INLINE_VISIBILITY
+void __

[PATCH] D24991: Inline hot functions in libcxx shared_ptr implementation.

2017-01-16 Thread Kevin Hu via Phabricator via cfe-commits 
This revision was automatically updated to reflect the committed changes.
Closed by commit rL292184: [Test patch] Inline hot functions in libcxx 
shared_ptr (authored by hxy9243).

Changed prior to commit:
  https://reviews.llvm.org/D24991?vs=84569&id=84622#toc

Repository:
  rL LLVM

https://reviews.llvm.org/D24991

Files:
  libcxx/trunk/include/memory
  libcxx/trunk/src/memory.cpp

Index: libcxx/trunk/src/memory.cpp
===
--- libcxx/trunk/src/memory.cpp
+++ libcxx/trunk/src/memory.cpp
@@ -17,28 +17,6 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-namespace
-{
-
-// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
-// should be sufficient for thread safety.
-// See https://llvm.org/bugs/show_bug.cgi?id=22803
-template 
-inline T
-increment(T& t) _NOEXCEPT
-{
-return __libcpp_atomic_add(&t, 1, _AO_Relaxed);
-}
-
-template 
-inline T
-decrement(T& t) _NOEXCEPT
-{
-return __libcpp_atomic_add(&t, -1, _AO_Acq_Rel);
-}
-
-}  // namespace
-
 const allocator_arg_t allocator_arg = allocator_arg_t();
 
 bad_weak_ptr::~bad_weak_ptr() _NOEXCEPT {}
@@ -53,27 +31,27 @@
 {
 }
 
+__shared_weak_count::~__shared_weak_count()
+{
+}
+
 void
 __shared_count::__add_shared() _NOEXCEPT
 {
-increment(__shared_owners_);
+__libcpp_atomic_refcount_increment(__shared_owners_);
 }
 
 bool
 __shared_count::__release_shared() _NOEXCEPT
 {
-if (decrement(__shared_owners_) == -1)
+if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1)
 {
 __on_zero_shared();
 return true;
 }
 return false;
 }
 
-__shared_weak_count::~__shared_weak_count()
-{
-}
-
 void
 __shared_weak_count::__add_shared() _NOEXCEPT
 {
@@ -83,7 +61,7 @@
 void
 __shared_weak_count::__add_weak() _NOEXCEPT
 {
-increment(__shared_weak_owners_);
+__libcpp_atomic_refcount_increment(__shared_weak_owners_);
 }
 
 void
@@ -124,7 +102,7 @@
 //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release);
 __on_zero_shared_weak();
 }
-else if (decrement(__shared_weak_owners_) == -1)
+else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1)
 __on_zero_shared_weak();
 }
 
Index: libcxx/trunk/include/memory
===
--- libcxx/trunk/include/memory
+++ libcxx/trunk/include/memory
@@ -3681,6 +3681,39 @@
 
 #endif // _LIBCPP_STD_VER > 14
 
+// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
+// should be sufficient for thread safety.
+// See https://llvm.org/bugs/show_bug.cgi?id=22803
+#if defined(__clang__) && __has_builtin(__atomic_add_fetch)  \
+   && defined(__ATOMIC_RELAXED)  \
+   && defined(__ATOMIC_ACQ_REL)
+#   define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT
+#elif !defined(__clang__) && defined(_GNUC_VER) && _GNUC_VER >= 407
+#   define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT
+#endif
+
+template 
+inline _LIBCPP_INLINE_VISIBILITY _Tp
+__libcpp_atomic_refcount_increment(_Tp& __t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT) && !defined(_LIBCPP_HAS_NO_THREADS)
+return __atomic_add_fetch(&__t, 1, __ATOMIC_RELAXED);
+#else
+return __t += 1;
+#endif
+}
+
+template 
+inline _LIBCPP_INLINE_VISIBILITY _Tp
+__libcpp_atomic_refcount_decrement(_Tp& __t) _NOEXCEPT
+{
+#if defined(_LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT) && !defined(_LIBCPP_HAS_NO_THREADS)
+return __atomic_add_fetch(&__t, -1, __ATOMIC_ACQ_REL);
+#else
+return __t -= 1;
+#endif
+}
+
 class _LIBCPP_EXCEPTION_ABI bad_weak_ptr
 : public std::exception
 {
@@ -3717,8 +3750,23 @@
 explicit __shared_count(long __refs = 0) _NOEXCEPT
 : __shared_owners_(__refs) {}
 
-void __add_shared() _NOEXCEPT;
-bool __release_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
+_LIBCPP_FUNC_VIS void __add_shared() _NOEXCEPT;
+_LIBCPP_FUNC_VIS bool __release_shared() _NOEXCEPT;
+#else
+_LIBCPP_INLINE_VISIBILITY
+void __add_shared() _NOEXCEPT {
+  __libcpp_atomic_refcount_increment(__shared_owners_);
+}
+_LIBCPP_INLINE_VISIBILITY
+bool __release_shared() _NOEXCEPT {
+  if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) {
+__on_zero_shared();
+return true;
+  }
+  return false;
+}
+#endif
 _LIBCPP_INLINE_VISIBILITY
 long use_count() const _NOEXCEPT {
 return __libcpp_relaxed_load(&__shared_owners_) + 1;
@@ -3739,9 +3787,25 @@
 virtual ~__shared_weak_count();
 
 public:
-void __add_shared() _NOEXCEPT;
-void __add_weak() _NOEXCEPT;
-void __release_shared() _NOEXCEPT;
+#ifdef _LIBCPP_BUILDING_MEMORY
+_LIBCPP_FUNC_VIS void __add_shared() _NOEXCEPT;
+_LIBCPP_FUNC_VIS void __add_weak() _NOEXCEPT;
+_LIBCPP_FUNC_VIS void __release_shared() _NOEXCEPT;
+#else
+_LIBCPP_INLINE_VISIBILITY
+void __add_shared() _NOEXCEPT {
+  __shared_count::__add