This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 5d4c50574af641061ab662d3adc1f7470922d4bc Author: Joe McDonnell <[email protected]> AuthorDate: Sat Oct 29 22:36:35 2022 -0700 IMPALA-11694: Convert gutil atomic ops to C++ atomics Impala uses the gutil's implementation of atomic ops, which has several port-specific versions. As new processors come out and make improvements, these can require maintenance over time. When testing on ARM, we noticed that a test for a spinlock was dramatically slower (> 500x) and this is likely due to improper atomics implementations. The Chromium codebase switched to using the C++ atomics package for these implementations a few years ago in https://github.com/chromium/chromium/commit/57a4e4a50c673c25e9cdaab53e32f6e53aa0b574 This seems like a much more maintainable approach. On ARM, this fixes the spinlock issue. The patch moves both the x86 and arm64 implementation to a common file yield_processor.h following the structure in the chromium project: https://github.com/chromium/chromium/blame/main/base/allocator/partition_allocator/yield_processor.h This is a prototype of adapting this change to the Impala codebase. This still needs more work, but it has passed Impala tests in the past. Known remaining work: 1. This will be taken over to the Kudu codebase's gutil as well, so we need to double-check that all APIs they use are covered. 2. Kudu has support for OS X, so this will need to be tested with x86_64 and the M1 chips. Change-Id: I3d1e3f4e71988a6c464071c1cd8bdebce622e4b8 Reviewed-on: http://gerrit.cloudera.org:8080/19185 Reviewed-by: Michael Smith <[email protected]> Tested-by: Joe McDonnell <[email protected]> --- be/src/common/init.cc | 5 - be/src/gutil/CMakeLists.txt | 6 - be/src/gutil/atomicops-internals-arm64.h | 474 ------------------- be/src/gutil/atomicops-internals-macosx.h | 406 ---------------- be/src/gutil/atomicops-internals-portable.h | 252 ++++++++++ be/src/gutil/atomicops-internals-powerpc.h | 304 ------------ be/src/gutil/atomicops-internals-tsan.h | 235 ---------- be/src/gutil/atomicops-internals-x86.cc | 112 ----- be/src/gutil/atomicops-internals-x86.h | 501 -------------------- be/src/gutil/atomicops.h | 295 +++--------- .../auxiliary/atomicops-internals-arm-generic.h | 230 ---------- .../auxiliary/atomicops-internals-arm-v6plus.h | 378 --------------- .../gutil/auxiliary/atomicops-internals-windows.h | 508 --------------------- be/src/gutil/yield_processor.h | 48 ++ 14 files changed, 368 insertions(+), 3386 deletions(-) diff --git a/be/src/common/init.cc b/be/src/common/init.cc index 86379cb15..8229f0a08 100644 --- a/be/src/common/init.cc +++ b/be/src/common/init.cc @@ -546,11 +546,6 @@ void impala::InitCommonRuntime(int argc, char** argv, bool init_jvm, if (!external_fe) { ABORT_IF_ERROR(RegisterMinidump(argv[0])); } -#ifndef THREAD_SANITIZER -#ifndef __aarch64__ - AtomicOps_x86CPUFeaturesInit(); -#endif -#endif impala::InitThreading(); impala::datetime_parse_util::SimpleDateFormatTokenizer::InitCtx(); impala::SeedOpenSSLRNG(); diff --git a/be/src/gutil/CMakeLists.txt b/be/src/gutil/CMakeLists.txt index 8ea5283bd..9e20855e6 100644 --- a/be/src/gutil/CMakeLists.txt +++ b/be/src/gutil/CMakeLists.txt @@ -53,12 +53,6 @@ set(GUTIL_SRCS utf/rune.c walltime.cc) -if (NOT CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") -set(GUTIL_SRCS - atomicops-internals-x86.cc - ${GUTIL_SRCS}) -endif() - set(GUTIL_LIBS glog protobuf) diff --git a/be/src/gutil/atomicops-internals-arm64.h b/be/src/gutil/atomicops-internals-arm64.h deleted file mode 100644 index 5111ecc4c..000000000 --- a/be/src/gutil/atomicops-internals-arm64.h +++ /dev/null @@ -1,474 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2012 Google Inc. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// This file is an internal atomic implementation, use atomicops.h instead. - -#ifndef GUTIL_ATOMICOPS_INTERNALS_ARM64_H_ -#define GUTIL_ATOMICOPS_INTERNALS_ARM64_H_ - -typedef int32_t Atomic32; -typedef int64_t Atomic64; - -namespace base { -namespace subtle { - -typedef int32_t Atomic32; -typedef int64_t Atomic64; - -inline void MemoryBarrier() { - __asm__ __volatile__ ("dmb ish" ::: "memory"); // NOLINT -} -// NoBarrier versions of the operation include "memory" in the clobber list. -// This is not required for direct usage of the NoBarrier versions of the -// operations. However this is required for correctness when they are used as -// part of the Acquire or Release versions, to ensure that nothing from outside -// the call is reordered between the operation and the memory barrier. This does -// not change the code generated, so has no or minimal impact on the -// NoBarrier operations. - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev; - int32_t temp; - - __asm__ __volatile__ ( // NOLINT - "0: \n\t" - "ldxr %w[prev], %[ptr] \n\t" // Load the previous value. - "cmp %w[prev], %w[old_value] \n\t" - "bne 1f \n\t" - "stxr %w[temp], %w[new_value], %[ptr] \n\t" // Try to store the new value. - "cbnz %w[temp], 0b \n\t" // Retry if it did not work. - "1: \n\t" - : [prev]"=&r" (prev), - [temp]"=&r" (temp), - [ptr]"+Q" (*ptr) - : [old_value]"IJr" (old_value), - [new_value]"r" (new_value) - : "cc", "memory" - ); // NOLINT - - return prev; -} - -inline Atomic32 Barrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev; - int32_t temp; - - __asm__ __volatile__ ( // NOLINT - "0: \n\t" - "ldaxr %w[prev], %[ptr] \n\t" // Load the previous value. - "cmp %w[prev], %w[old_value] \n\t" - "bne 1f \n\t" - "stlxr %w[temp], %w[new_value], %[ptr] \n\t" // Try to store the new value. - "cbnz %w[temp], 0b \n\t" // Retry if it did not work. - "1: \n\t" - : [prev]"=&r" (prev), - [temp]"=&r" (temp), - [ptr]"+Q" (*ptr) - : [old_value]"IJr" (old_value), - [new_value]"r" (new_value) - : "cc", "memory" - ); // NOLINT - - return prev; -} - - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 result; - int32_t temp; - - __asm__ __volatile__ ( // NOLINT - "0: \n\t" - "ldxr %w[result], %[ptr] \n\t" // Load the previous value. - "stxr %w[temp], %w[new_value], %[ptr] \n\t" // Try to store the new value. - "cbnz %w[temp], 0b \n\t" // Retry if it did not work. - : [result]"=&r" (result), - [temp]"=&r" (temp), - [ptr]"+Q" (*ptr) - : [new_value]"r" (new_value) - : "memory" - ); // NOLINT - - return result; -} - -inline Atomic32 Barrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 result; - int32_t temp; - - __asm__ __volatile__ ( // NOLINT - "0: \n\t" - "ldaxr %w[result], %[ptr] \n\t" // Load the previous value. - "stlxr %w[temp], %w[new_value], %[ptr] \n\t" // Try to store the new value. - "cbnz %w[temp], 0b \n\t" // Retry if it did not work. - : [result]"=&r" (result), - [temp]"=&r" (temp), - [ptr]"+Q" (*ptr) - : [new_value]"r" (new_value) - : "memory" - ); // NOLINT - - return result; -} - - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 old_val = NoBarrier_AtomicExchange(ptr, new_value); - MemoryBarrier(); - return old_val; -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - MemoryBarrier(); - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - Atomic32 result; - int32_t temp; - - __asm__ __volatile__ ( // NOLINT - "0: \n\t" - "ldxr %w[result], %[ptr] \n\t" // Load the previous value. - "add %w[result], %w[result], %w[increment]\n\t" - "stxr %w[temp], %w[result], %[ptr] \n\t" // Try to store the result. - "cbnz %w[temp], 0b \n\t" // Retry on failure. - : [result]"=&r" (result), - [temp]"=&r" (temp), - [ptr]"+Q" (*ptr) - : [increment]"IJr" (increment) - : "memory" - ); // NOLINT - - return result; -} - -inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - Atomic32 result; - int32_t temp; - - __asm__ __volatile__ ( // NOLINT - "0: \n\t" - "ldaxr %w[result], %[ptr] \n\t" // Load the previous value. - "add %w[result], %w[result], %w[increment]\n\t" - "stlxr %w[temp], %w[result], %[ptr] \n\t" // Try to store the result. - "cbnz %w[temp], 0b \n\t" // Retry on failure. - : [result]"=&r" (result), - [temp]"=&r" (temp), - [ptr]"+Q" (*ptr) - : [increment]"IJr" (increment) - : "memory" - ); - return result; -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - MemoryBarrier(); - return prev; -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - MemoryBarrier(); - Atomic32 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - return prev; -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - __asm__ __volatile__ ( // NOLINT - "stlr %w[value], %[ptr] \n\t" - : [ptr]"=Q" (*ptr) - : [value]"r" (value) - : "memory" - ); // NOLINT -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - Atomic32 value; - - __asm__ __volatile__ ( // NOLINT - "ldar %w[value], %[ptr] \n\t" - : [value]"=r" (value) - : [ptr]"Q" (*ptr) - : "memory" - ); // NOLINT - - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - MemoryBarrier(); - return *ptr; -} - -// 64-bit versions of the operations. -// See the 32-bit versions for comments. - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev; - int32_t temp; - - __asm__ __volatile__ ( // NOLINT - "0: \n\t" - "ldxr %[prev], %[ptr] \n\t" - "cmp %[prev], %[old_value] \n\t" - "bne 1f \n\t" - "stxr %w[temp], %[new_value], %[ptr] \n\t" - "cbnz %w[temp], 0b \n\t" - "1: \n\t" - : [prev]"=&r" (prev), - [temp]"=&r" (temp), - [ptr]"+Q" (*ptr) - : [old_value]"IJr" (old_value), - [new_value]"r" (new_value) - : "cc", "memory" - ); // NOLINT - - return prev; -} - - -inline Atomic64 Barrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev; - int32_t temp; - - __asm__ __volatile__ ( // NOLINT - "0: \n\t" - "ldaxr %[prev], %[ptr] \n\t" - "cmp %[prev], %[old_value] \n\t" - "bne 1f \n\t" - "stlxr %w[temp], %[new_value], %[ptr] \n\t" - "cbnz %w[temp], 0b \n\t" - "1: \n\t" - : [prev]"=&r" (prev), - [temp]"=&r" (temp), - [ptr]"+Q" (*ptr) - : [old_value]"IJr" (old_value), - [new_value]"r" (new_value) - : "cc", "memory" - ); // NOLINT - - return prev; -} - - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - Atomic64 result; - int32_t temp; - - __asm__ __volatile__ ( // NOLINT - "0: \n\t" - "ldxr %[result], %[ptr] \n\t" - "stxr %w[temp], %[new_value], %[ptr] \n\t" - "cbnz %w[temp], 0b \n\t" - : [result]"=&r" (result), - [temp]"=&r" (temp), - [ptr]"+Q" (*ptr) - : [new_value]"r" (new_value) - : "memory" - ); // NOLINT - - return result; -} - -inline Atomic64 Barrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - Atomic64 result; - int32_t temp; - - __asm__ __volatile__ ( // NOLINT - "0: \n\t" - "ldaxr %[result], %[ptr] \n\t" - "stlxr %w[temp], %[new_value], %[ptr] \n\t" - "cbnz %w[temp], 0b \n\t" - : [result]"=&r" (result), - [temp]"=&r" (temp), - [ptr]"+Q" (*ptr) - : [new_value]"r" (new_value) - : "memory" - ); // NOLINT - - return result; -} - - -inline void PauseCPU() { - __asm__ __volatile__("yield" : : : "memory"); -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - Atomic64 old_val = NoBarrier_AtomicExchange(ptr, new_value); - MemoryBarrier(); - return old_val; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - MemoryBarrier(); - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - Atomic64 result; - int32_t temp; - - __asm__ __volatile__ ( // NOLINT - "0: \n\t" - "ldxr %[result], %[ptr] \n\t" - "add %[result], %[result], %[increment] \n\t" - "stxr %w[temp], %[result], %[ptr] \n\t" - "cbnz %w[temp], 0b \n\t" - : [result]"=&r" (result), - [temp]"=&r" (temp), - [ptr]"+Q" (*ptr) - : [increment]"IJr" (increment) - : "memory" - ); // NOLINT - - return result; -} - -inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - Atomic64 result; - int32_t temp; - - __asm__ __volatile__ ( // NOLINT - "0: \n\t" - "ldaxr %[result], %[ptr] \n\t" - "add %[result], %[result], %[increment] \n\t" - "stlxr %w[temp], %[result], %[ptr] \n\t" - "cbnz %w[temp], 0b \n\t" - : [result]"=&r" (result), - [temp]"=&r" (temp), - [ptr]"+Q" (*ptr) - : [increment]"IJr" (increment) - : "memory" - ); // NOLINT - - return result; -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - MemoryBarrier(); - - return prev; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - MemoryBarrier(); - Atomic64 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - - return prev; -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - __asm__ __volatile__ ( // NOLINT - "stlr %x[value], %[ptr] \n\t" - : [ptr]"=Q" (*ptr) - : [value]"r" (value) - : "memory" - ); // NOLINT -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - return *ptr; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - Atomic64 value; - - __asm__ __volatile__ ( // NOLINT - "ldar %x[value], %[ptr] \n\t" - : [value]"=r" (value) - : [ptr]"Q" (*ptr) - : "memory" - ); // NOLINT - - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return *ptr; -} - -} // namespace subtle -} // namespace base - -#endif // GUTIL_ATOMICOPS_INTERNALS_ARM64_GCC_H_ - diff --git a/be/src/gutil/atomicops-internals-macosx.h b/be/src/gutil/atomicops-internals-macosx.h deleted file mode 100644 index 15efaef34..000000000 --- a/be/src/gutil/atomicops-internals-macosx.h +++ /dev/null @@ -1,406 +0,0 @@ -// Copyright 2006 Google Inc. -// All Rights Reserved. -// -// -// Implementation of atomic operations for Mac OS X. This file should not -// be included directly. Clients should instead include -// "base/atomicops.h". - -#ifndef BASE_AUXILIARY_ATOMICOPS_INTERNALS_MACOSX_H_ -#define BASE_AUXILIARY_ATOMICOPS_INTERNALS_MACOSX_H_ - -typedef int32_t Atomic32; -typedef int64_t Atomic64; - -// MacOS uses long for intptr_t, AtomicWord and Atomic32 are always different -// on the Mac, even when they are the same size. Similarly, on __ppc64__, -// AtomicWord and Atomic64 are always different. Thus, we need explicit -// casting. -#ifdef __LP64__ -#define AtomicWordCastType base::subtle::Atomic64 -#else -#define AtomicWordCastType Atomic32 -#endif - -#if defined(__LP64__) || defined(__i386__) -#define BASE_HAS_ATOMIC64 1 // Use only in tests and base/atomic* -#endif - -#include <libkern/OSAtomic.h> - -#if !defined(__LP64__) && defined(__ppc__) - -// The Mac 64-bit OSAtomic implementations are not available for 32-bit PowerPC, -// while the underlying assembly instructions are available only some -// implementations of PowerPC. - -// The following inline functions will fail with the error message at compile -// time ONLY IF they are called. So it is safe to use this header if user -// code only calls AtomicWord and Atomic32 operations. -// -// NOTE(user): Implementation notes to implement the atomic ops below may -// be found in "PowerPC Virtual Environment Architecture, Book II, -// Version 2.02", January 28, 2005, Appendix B, page 46. Unfortunately, -// extra care must be taken to ensure data are properly 8-byte aligned, and -// that data are returned correctly according to Mac OS X ABI specs. - -inline int64_t OSAtomicCompareAndSwap64( - int64_t oldValue, int64_t newValue, int64_t *theValue) { - __asm__ __volatile__( - "_OSAtomicCompareAndSwap64_not_supported_for_32_bit_ppc\n\t"); - return 0; -} - -inline int64_t OSAtomicAdd64(int64_t theAmount, int64_t *theValue) { - __asm__ __volatile__( - "_OSAtomicAdd64_not_supported_for_32_bit_ppc\n\t"); - return 0; -} - -inline int64_t OSAtomicCompareAndSwap64Barrier( - int64_t oldValue, int64_t newValue, int64_t *theValue) { - int64_t prev = OSAtomicCompareAndSwap64(oldValue, newValue, theValue); - OSMemoryBarrier(); - return prev; -} - -inline int64_t OSAtomicAdd64Barrier( - int64_t theAmount, int64_t *theValue) { - int64_t new_val = OSAtomicAdd64(theAmount, theValue); - OSMemoryBarrier(); - return new_val; -} -#endif - - -namespace base { -namespace subtle { - -typedef int32_t Atomic32; -typedef int64_t Atomic64; - -inline void MemoryBarrier() { - OSMemoryBarrier(); -} - -// 32-bit Versions. - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value; - do { - if (OSAtomicCompareAndSwap32(old_value, new_value, - const_cast<Atomic32*>(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32 *ptr, - Atomic32 new_value) { - Atomic32 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap32(old_value, new_value, - const_cast<Atomic32*>(ptr))); - return old_value; -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32 *ptr, - Atomic32 new_value) { - Atomic32 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap32Barrier(old_value, new_value, - const_cast<Atomic32*>(ptr))); - return old_value; -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32 *ptr, - Atomic32 new_value) { - return Acquire_AtomicExchange(ptr, new_value); -} - -inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32 *ptr, - Atomic32 increment) { - return OSAtomicAdd32(increment, const_cast<Atomic32*>(ptr)); -} - -inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32 *ptr, - Atomic32 increment) { - return OSAtomicAdd32Barrier(increment, const_cast<Atomic32*>(ptr)); -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value; - do { - if (OSAtomicCompareAndSwap32Barrier(old_value, new_value, - const_cast<Atomic32*>(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - return Acquire_CompareAndSwap(ptr, old_value, new_value); -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic32 *ptr, Atomic32 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic32 *ptr, Atomic32 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32 *ptr) { - Atomic32 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32 *ptr) { - MemoryBarrier(); - return *ptr; -} - -// 64-bit version - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev_value; - do { - if (OSAtomicCompareAndSwap64(old_value, new_value, - const_cast<Atomic64*>(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64 *ptr, - Atomic64 new_value) { - Atomic64 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap64(old_value, new_value, - const_cast<Atomic64*>(ptr))); - return old_value; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64 *ptr, - Atomic64 new_value) { - Atomic64 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap64Barrier(old_value, new_value, - const_cast<Atomic64*>(ptr))); - return old_value; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64 *ptr, - Atomic64 new_value) { - return Acquire_AtomicExchange(ptr, new_value); -} - -inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64 *ptr, - Atomic64 increment) { - return OSAtomicAdd64(increment, const_cast<Atomic64*>(ptr)); -} - -inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64 *ptr, - Atomic64 increment) { - return OSAtomicAdd64Barrier(increment, const_cast<Atomic64*>(ptr)); -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev_value; - do { - if (OSAtomicCompareAndSwap64Barrier(old_value, new_value, - const_cast<Atomic64*>(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - // The lib kern interface does not distinguish between - // Acquire and Release memory barriers; they are equivalent. - return Acquire_CompareAndSwap(ptr, old_value, new_value); -} - -#ifdef __LP64__ - -// 64-bit implementation on 64-bit platform - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; -} - -// Issue the x86 "pause" instruction, which tells the CPU that we -// are in a spinlock wait loop and should allow other hyperthreads -// to run, not speculate memory access, etc. -inline void PauseCPU() { - __asm__ __volatile__("pause" : : : "memory"); -} - -inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64 *ptr, Atomic64 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - return *ptr; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64 *ptr) { - Atomic64 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64 *ptr) { - MemoryBarrier(); - return *ptr; -} - -#else - -// 64-bit implementation on 32-bit platform - -#if defined(__ppc__) - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - __asm__ __volatile__( - "_NoBarrier_Store_not_supported_for_32_bit_ppc\n\t"); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - __asm__ __volatile__( - "_NoBarrier_Load_not_supported_for_32_bit_ppc\n\t"); - return 0; -} - -#elif defined(__i386__) - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - __asm__ __volatile__("movq %1, %%mm0\n\t" // Use mmx reg for 64-bit atomic - "movq %%mm0, %0\n\t" // moves (ptr could be read-only) - "emms\n\t" // Reset FP registers - : "=m" (*ptr) - : "m" (value) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", - "st(5)", "st(6)", "st(7)", "mm0", "mm1", - "mm2", "mm3", "mm4", "mm5", "mm6", "mm7"); - -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - Atomic64 value; - __asm__ __volatile__("movq %1, %%mm0\n\t" // Use mmx reg for 64-bit atomic - "movq %%mm0, %0\n\t" // moves (ptr could be read-only) - "emms\n\t" // Reset FP registers - : "=m" (value) - : "m" (*ptr) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", - "st(5)", "st(6)", "st(7)", "mm0", "mm1", - "mm2", "mm3", "mm4", "mm5", "mm6", "mm7"); - - return value; -} - -#elif defined(__arm__) - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - int store_failed; - Atomic64 dummy; - __asm__ __volatile__( - "1:\n" - // Dummy load to lock cache line. - "ldrexd %1, [%3]\n" - "strexd %0, %2, [%3]\n" - "teq %0, #0\n" - "bne 1b" - : "=&r" (store_failed), "=&r"(dummy) - : "r"(value), "r" (ptr) - : "cc", "memory"); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - Atomic64 res; - __asm__ __volatile__( - "ldrexd %0, [%1]\n" - "clrex\n" - : "=r" (res) - : "r"(ptr), "Q"(*ptr)); - return res; -} - -#endif - - -inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) { - NoBarrier_Store(ptr, value); - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64 *ptr, Atomic64 value) { - MemoryBarrier(); - NoBarrier_Store(ptr, value); -} - -inline Atomic64 Acquire_Load(volatile const Atomic64 *ptr) { - Atomic64 value = NoBarrier_Load(ptr); - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64 *ptr) { - MemoryBarrier(); - return NoBarrier_Load(ptr); -} -#endif // __LP64__ - -} // namespace base::subtle -} // namespace base - -// NOTE(user): The following is also deprecated. New callers should use -// the base::subtle namespace. -inline void MemoryBarrier() { - base::subtle::MemoryBarrier(); -} -#endif // BASE_AUXILIARY_ATOMICOPS_INTERNALS_MACOSX_H_ diff --git a/be/src/gutil/atomicops-internals-portable.h b/be/src/gutil/atomicops-internals-portable.h new file mode 100644 index 000000000..dbfb713a7 --- /dev/null +++ b/be/src/gutil/atomicops-internals-portable.h @@ -0,0 +1,252 @@ +// Copyright 2014 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This file is an internal atomic implementation, use atomicops.h instead. +// +// This implementation uses C++11 atomics' member functions. The code base is +// currently written assuming atomicity revolves around accesses instead of +// C++11's memory locations. The burden is on the programmer to ensure that all +// memory locations accessed atomically are never accessed non-atomically (tsan +// should help with this). +// +// TODO(jfb) Modify the atomicops.h API and user code to declare atomic +// locations as truly atomic. See the static_assert below. +// +// Of note in this implementation: +// * All NoBarrier variants are implemented as relaxed. +// * All Barrier variants are implemented as sequentially-consistent. +// * Compare exchange's failure ordering is always the same as the success one +// (except for release, which fails as relaxed): using a weaker ordering is +// only valid under certain uses of compare exchange. +// * Atomic increment is expected to return the post-incremented value, whereas +// C11 fetch add returns the previous value. The implementation therefore +// needs to increment twice (which the compiler should be able to detect and +// optimize). + +#ifndef BASE_ATOMICOPS_INTERNALS_PORTABLE_H_ +#define BASE_ATOMICOPS_INTERNALS_PORTABLE_H_ + +#include <atomic> + +namespace base { +namespace subtle { + +// This implementation is transitional and maintains the original API for +// atomicops.h. This requires casting memory locations to the atomic types, and +// assumes that the API and the C++11 implementation are layout-compatible, +// which isn't true for all implementations or hardware platforms. The static +// assertion should detect this issue, were it to fire then this header +// shouldn't be used. +// +// TODO(jfb) If this header manages to stay committed then the API should be +// modified, and all call sites updated. +typedef volatile std::atomic<Atomic32>* AtomicLocation32; +static_assert(sizeof(*(AtomicLocation32) nullptr) == sizeof(Atomic32), + "incompatible 32-bit atomic layout"); + +inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + ((AtomicLocation32)ptr) + ->compare_exchange_strong(old_value, + new_value, + std::memory_order_relaxed, + std::memory_order_relaxed); + return old_value; +} + +inline Atomic32 Barrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + ((AtomicLocation32)ptr) + ->compare_exchange_strong(old_value, + new_value); + return old_value; +} + + +inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, + Atomic32 new_value) { + return ((AtomicLocation32)ptr) + ->exchange(new_value, std::memory_order_relaxed); +} + +inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, + Atomic32 new_value) { + return ((AtomicLocation32)ptr) + ->exchange(new_value, std::memory_order_acquire); +} + +inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, + Atomic32 new_value) { + return ((AtomicLocation32)ptr) + ->exchange(new_value, std::memory_order_release); +} + +inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + return increment + + ((AtomicLocation32)ptr) + ->fetch_add(increment, std::memory_order_relaxed); +} + +inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + return increment + ((AtomicLocation32)ptr)->fetch_add(increment); +} + +inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + ((AtomicLocation32)ptr) + ->compare_exchange_strong(old_value, + new_value, + std::memory_order_acquire, + std::memory_order_acquire); + return old_value; +} + +inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + ((AtomicLocation32)ptr) + ->compare_exchange_strong(old_value, + new_value, + std::memory_order_release, + std::memory_order_relaxed); + return old_value; +} + +inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { + ((AtomicLocation32)ptr)->store(value, std::memory_order_relaxed); +} + +inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { + ((AtomicLocation32)ptr)->store(value, std::memory_order_acquire); +} + +inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { + ((AtomicLocation32)ptr)->store(value, std::memory_order_release); +} + +inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { + return ((AtomicLocation32)ptr)->load(std::memory_order_relaxed); +} + +inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { + return ((AtomicLocation32)ptr)->load(std::memory_order_acquire); +} + +inline Atomic32 Release_Load(volatile const Atomic32* ptr) { + return ((AtomicLocation32)ptr)->load(std::memory_order_release); +} + +typedef volatile std::atomic<Atomic64>* AtomicLocation64; +static_assert(sizeof(*(AtomicLocation64) nullptr) == sizeof(Atomic64), + "incompatible 64-bit atomic layout"); + +inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + ((AtomicLocation64)ptr) + ->compare_exchange_strong(old_value, + new_value, + std::memory_order_relaxed, + std::memory_order_relaxed); + return old_value; +} + +inline Atomic64 Barrier_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + ((AtomicLocation64)ptr) + ->compare_exchange_strong(old_value, + new_value); + return old_value; +} + +inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + return ((AtomicLocation64)ptr) + ->exchange(new_value, std::memory_order_relaxed); +} + +inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + return ((AtomicLocation64)ptr) + ->exchange(new_value, std::memory_order_release); +} + +inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + return ((AtomicLocation64)ptr) + ->exchange(new_value, std::memory_order_acquire); +} + +inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + return increment + + ((AtomicLocation64)ptr) + ->fetch_add(increment, std::memory_order_relaxed); +} + +inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + return increment + ((AtomicLocation64)ptr)->fetch_add(increment); +} + +inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + ((AtomicLocation64)ptr) + ->compare_exchange_strong(old_value, + new_value, + std::memory_order_acquire, + std::memory_order_acquire); + return old_value; +} + +inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + ((AtomicLocation64)ptr) + ->compare_exchange_strong(old_value, + new_value, + std::memory_order_release, + std::memory_order_relaxed); + return old_value; +} + +inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { + ((AtomicLocation64)ptr)->store(value, std::memory_order_relaxed); +} + +inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { + ((AtomicLocation64)ptr)->store(value, std::memory_order_acquire); +} + +inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { + ((AtomicLocation64)ptr)->store(value, std::memory_order_release); +} + +inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { + return ((AtomicLocation64)ptr)->load(std::memory_order_relaxed); +} + +inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { + return ((AtomicLocation64)ptr)->load(std::memory_order_acquire); +} + +inline Atomic64 Release_Load(volatile const Atomic64* ptr) { + return ((AtomicLocation64)ptr)->load(std::memory_order_release); +} + +inline void MemoryBarrier() { + std::atomic_thread_fence(std::memory_order_seq_cst); +} + +} // namespace subtle +} // namespace base + +#endif // BASE_ATOMICOPS_INTERNALS_PORTABLE_H_ diff --git a/be/src/gutil/atomicops-internals-powerpc.h b/be/src/gutil/atomicops-internals-powerpc.h deleted file mode 100644 index 0e56475bc..000000000 --- a/be/src/gutil/atomicops-internals-powerpc.h +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright 2012 Google Inc. -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// All Rights Reserved. -// -// -// Implementation of atomic operations for PowerPC. This file should not -// be included directly. Clients should instead include -// "base/atomicops.h". - -// *** WARNING EXPERIMENTAL CODE *** -// This is not tested and may contain bugs. Until we have bootstrapped -// this. - -#ifndef GUTIL_ATOMICOPS_INTERNALS_POWERPC_H_ -#define GUTIL_ATOMICOPS_INTERNALS_POWERPC_H_ - -typedef int32_t Atomic32; -#define BASE_HAS_ATOMIC64 1 // Use only in tests and base/atomic* - - -#define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") - -// 32-bit PowerPC is not supported yet. -#ifndef ARCH_POWERPC64 -#error "Only PowerPC64 is supported" -#endif - -namespace base { -namespace subtle { - -typedef int64_t Atomic64; - -// sync vs. lwsync: -// 1. lwsync only works in cache enabled memory (system memory). lwsync is -// unsuitable for memory caching disabled & guarded (device memory). -// sync can handle both system and device memory. -// 2. lwsync does not prevent reordering of a store followed by a load if they -// access different memory. sync orders all 4 kinds of memory access pairs. - -inline void MemoryBarrier() { - __asm__ __volatile__("sync" : : : "memory"); -} - -// 32-bit low-level operations. - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 value; - __asm__ __volatile__("1:\n" - " lwarx %0, 0, %2\n" - " cmpw 0, %0, %3\n" - " bne- 2f\n" - " stwcx. %4, 0, %2\n" - " bne- 1b\n" - "2:\n" - : "=&r" (value), "+m"(*ptr) - : "b"(ptr), "r"(old_value), "r"(new_value) - : "cc"); - return value; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 value; - __asm__ __volatile__("1:\n" - " lwarx %0, 0, %2\n" - " stwcx. %3, 0, %2\n" - " bne- 1b\n" - : "=&r" (value), "+m"(*ptr) - : "b"(ptr), "r"(new_value) - : "cc"); - return value; -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 value = NoBarrier_AtomicExchange(ptr, new_value); - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - MemoryBarrier(); - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - Atomic32 value; - __asm__ __volatile__("1:\n" - " lwarx %0, 0, %2\n" - " add %0, %0, %3\n" - " stwcx. %0, 0, %2\n" - " bne- 1b\n" - : "=&r" (value), "+m"(*ptr) - : "b"(ptr), "r"(increment) - : "cc"); - return value; -} - -inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - Atomic32 value; - __asm__ __volatile__(" lwsync\n" - "1:\n" - " lwarx %0, 0, %2\n" - " add %0, %0, %3\n" - " stwcx. %0, 0, %2\n" - " bne- 1b\n" - " lwsync\n" - : "=&r" (value), "+m"(*ptr) - : "b"(ptr), "r"(increment) - : "cc", "memory"); - return value; -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - MemoryBarrier(); - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - Atomic32 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - MemoryBarrier(); - return *ptr; -} - -// 64-bit low-level operations. - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 value; - __asm__ __volatile__("1:\n" - " ldarx %0, 0, %2\n" - " cmpd 0, %0, %3\n" - " bne- 2f\n" - " stdcx. %4, 0, %2\n" - " bne- 1b\n" - "2:\n" - : "=&r" (value), "+m"(*ptr) - : "b"(ptr), "r"(old_value), "r"(new_value) - : "cc"); - return value; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - Atomic64 value; - __asm__ __volatile__("1:\n" - " ldarx %0, 0, %2\n" - " stdcx. %3, 0, %2\n" - " bne- 1b\n" - : "=&r" (value), "+m"(*ptr) - : "b"(ptr), "r"(new_value) - : "cc"); - return value; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - Atomic64 value = NoBarrier_AtomicExchange(ptr, new_value); - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - MemoryBarrier(); - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - Atomic64 value; - __asm__ __volatile__("1:\n" - " ldarx %0, 0, %2\n" - " add %0, %0, %3\n" - " stdcx. %0, 0, %2\n" - " bne- 1b\n" - : "=&r" (value), "+m"(*ptr) - : "b"(ptr), "r"(increment) - : "cc"); - return value; -} - -inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - Atomic64 value; - __asm__ __volatile__(" lwsync\n" - "1:\n" - " ldarx %0, 0, %2\n" - " add %0, %0, %3\n" - " stdcx. %0, 0, %2\n" - " bne- 1b\n" - " lwsync\n" - : "=&r" (value), "+m"(*ptr) - : "b"(ptr), "r"(increment) - : "cc", "memory"); - return value; -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - return *ptr; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - Atomic64 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return *ptr; -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - MemoryBarrier(); - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -} // namespace subtle -} // namespace base - -#undef ATOMICOPS_COMPILER_BARRIER - -#endif // GUTIL_ATOMICOPS_INTERNALS_POWERPC_H_ diff --git a/be/src/gutil/atomicops-internals-tsan.h b/be/src/gutil/atomicops-internals-tsan.h deleted file mode 100644 index a1fa71bf4..000000000 --- a/be/src/gutil/atomicops-internals-tsan.h +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// This file is an internal atomic implementation for compiler-based -// ThreadSanitizer. Use base/atomicops.h instead. - -#ifndef BASE_ATOMICOPS_INTERNALS_TSAN_H_ -#define BASE_ATOMICOPS_INTERNALS_TSAN_H_ - -// Workaround for Chromium BASE_EXPORT definition -#ifndef BASE_EXPORT -#define BASE_EXPORT -#endif - -// This struct is not part of the public API of this module; clients may not -// use it. (However, it's exported via BASE_EXPORT because clients implicitly -// do use it at link time by inlining these functions.) -// Features of this x86. Values may not be correct before main() is run, -// but are set conservatively. -struct AtomicOps_x86CPUFeatureStruct { - bool has_amd_lock_mb_bug; // Processor has AMD memory-barrier bug; do lfence - // after acquire compare-and-swap. - bool has_sse2; // Processor has SSE2. -}; -BASE_EXPORT extern struct AtomicOps_x86CPUFeatureStruct - AtomicOps_Internalx86CPUFeatures; - -#define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") - -#include <sanitizer/tsan_interface_atomic.h> - -typedef int32_t Atomic32; -typedef int64_t Atomic64; - -namespace base { -namespace subtle { - -typedef int32_t Atomic32; -typedef int64_t Atomic64; - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 cmp = old_value; - __tsan_atomic32_compare_exchange_strong(ptr, &cmp, new_value, - __tsan_memory_order_relaxed, __tsan_memory_order_relaxed); - return cmp; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32 *ptr, - Atomic32 new_value) { - return __tsan_atomic32_exchange(ptr, new_value, - __tsan_memory_order_relaxed); -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32 *ptr, - Atomic32 new_value) { - return __tsan_atomic32_exchange(ptr, new_value, - __tsan_memory_order_acquire); -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32 *ptr, - Atomic32 new_value) { - return __tsan_atomic32_exchange(ptr, new_value, - __tsan_memory_order_release); -} - -inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32 *ptr, - Atomic32 increment) { - return increment + __tsan_atomic32_fetch_add(ptr, increment, - __tsan_memory_order_relaxed); -} - -inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32 *ptr, - Atomic32 increment) { - return increment + __tsan_atomic32_fetch_add(ptr, increment, - __tsan_memory_order_acq_rel); -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 cmp = old_value; - __tsan_atomic32_compare_exchange_strong(ptr, &cmp, new_value, - __tsan_memory_order_acquire, __tsan_memory_order_acquire); - return cmp; -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 cmp = old_value; - __tsan_atomic32_compare_exchange_strong(ptr, &cmp, new_value, - __tsan_memory_order_release, __tsan_memory_order_relaxed); - return cmp; -} - -inline void NoBarrier_Store(volatile Atomic32 *ptr, Atomic32 value) { - __tsan_atomic32_store(ptr, value, __tsan_memory_order_relaxed); -} - -inline void Acquire_Store(volatile Atomic32 *ptr, Atomic32 value) { - __tsan_atomic32_store(ptr, value, __tsan_memory_order_relaxed); - __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst); -} - -inline void Release_Store(volatile Atomic32 *ptr, Atomic32 value) { - __tsan_atomic32_store(ptr, value, __tsan_memory_order_release); -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32 *ptr) { - return __tsan_atomic32_load(ptr, __tsan_memory_order_relaxed); -} - -inline Atomic32 Acquire_Load(volatile const Atomic32 *ptr) { - return __tsan_atomic32_load(ptr, __tsan_memory_order_acquire); -} - -inline Atomic32 Release_Load(volatile const Atomic32 *ptr) { - __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst); - return __tsan_atomic32_load(ptr, __tsan_memory_order_relaxed); -} - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 cmp = old_value; - __tsan_atomic64_compare_exchange_strong(ptr, &cmp, new_value, - __tsan_memory_order_relaxed, __tsan_memory_order_relaxed); - return cmp; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64 *ptr, - Atomic64 new_value) { - return __tsan_atomic64_exchange(ptr, new_value, __tsan_memory_order_relaxed); -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64 *ptr, - Atomic64 new_value) { - return __tsan_atomic64_exchange(ptr, new_value, __tsan_memory_order_acquire); -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64 *ptr, - Atomic64 new_value) { - return __tsan_atomic64_exchange(ptr, new_value, __tsan_memory_order_release); -} - -inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64 *ptr, - Atomic64 increment) { - return increment + __tsan_atomic64_fetch_add(ptr, increment, - __tsan_memory_order_relaxed); -} - -inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64 *ptr, - Atomic64 increment) { - return increment + __tsan_atomic64_fetch_add(ptr, increment, - __tsan_memory_order_acq_rel); -} - -inline void NoBarrier_Store(volatile Atomic64 *ptr, Atomic64 value) { - __tsan_atomic64_store(ptr, value, __tsan_memory_order_relaxed); -} - -inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) { - __tsan_atomic64_store(ptr, value, __tsan_memory_order_relaxed); - __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst); -} - -inline void Release_Store(volatile Atomic64 *ptr, Atomic64 value) { - __tsan_atomic64_store(ptr, value, __tsan_memory_order_release); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64 *ptr) { - return __tsan_atomic64_load(ptr, __tsan_memory_order_relaxed); -} - -inline Atomic64 Acquire_Load(volatile const Atomic64 *ptr) { - return __tsan_atomic64_load(ptr, __tsan_memory_order_acquire); -} - -inline Atomic64 Release_Load(volatile const Atomic64 *ptr) { - __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst); - return __tsan_atomic64_load(ptr, __tsan_memory_order_relaxed); -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 cmp = old_value; - __tsan_atomic64_compare_exchange_strong(ptr, &cmp, new_value, - __tsan_memory_order_acquire, __tsan_memory_order_acquire); - return cmp; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 cmp = old_value; - __tsan_atomic64_compare_exchange_strong(ptr, &cmp, new_value, - __tsan_memory_order_release, __tsan_memory_order_relaxed); - return cmp; -} - -inline Atomic32 Barrier_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 cmp = old_value; - __tsan_atomic32_compare_exchange_strong(ptr, &cmp, new_value, - __tsan_memory_order_acq_rel, __tsan_memory_order_relaxed); - return cmp; -} - -inline Atomic64 Barrier_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 cmp = old_value; - __tsan_atomic64_compare_exchange_strong(ptr, &cmp, new_value, - __tsan_memory_order_acq_rel, __tsan_memory_order_relaxed); - return cmp; -} - -inline void MemoryBarrier() { - __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst); -} - -inline void PauseCPU() { -} - -} // namespace base::subtle -} // namespace base - -#undef ATOMICOPS_COMPILER_BARRIER - -#endif // BASE_ATOMICOPS_INTERNALS_TSAN_H_ diff --git a/be/src/gutil/atomicops-internals-x86.cc b/be/src/gutil/atomicops-internals-x86.cc deleted file mode 100644 index f02edc640..000000000 --- a/be/src/gutil/atomicops-internals-x86.cc +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2007 Google, Inc. -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// All rights reserved. - - -// This module gets enough CPU information to optimize the -// atomicops module on x86. - -#include "gutil/atomicops-internals-x86.h" - -#include <string.h> - -#include <common/logging.h> -#include "gutil/logging-inl.h" -#include "gutil/integral_types.h" - -// This file only makes sense with atomicops-internals-x86.h -- it -// depends on structs that are defined in that file. If atomicops.h -// doesn't sub-include that file, then we aren't needed, and shouldn't -// try to do anything. -#ifdef GUTIL_ATOMICOPS_INTERNALS_X86_H_ - -// This macro was copied from //util/cpuid/cpuid.cc -// Inline cpuid instruction. In PIC compilations, %ebx contains the address -// of the global offset table. To avoid breaking such executables, this code -// must preserve that register's value across cpuid instructions. -#if defined(__i386__) -#define cpuid(a, b, c, d, inp) \ - asm("mov %%ebx, %%edi\n" \ - "cpuid\n" \ - "xchg %%edi, %%ebx\n" \ - : "=a" (a), "=D" (b), "=c" (c), "=d" (d) : "a" (inp)) -#elif defined(__x86_64__) -#define cpuid(a, b, c, d, inp) \ - asm("mov %%rbx, %%rdi\n" \ - "cpuid\n" \ - "xchg %%rdi, %%rbx\n" \ - : "=a" (a), "=D" (b), "=c" (c), "=d" (d) : "a" (inp)) -#endif - -#if defined(cpuid) // initialize the struct only on x86 - -// Set the flags so that code will run correctly and conservatively -// until InitGoogle() is called. -struct AtomicOps_x86CPUFeatureStruct AtomicOps_Internalx86CPUFeatures = { - false, // no SSE2 - false // no cmpxchg16b -}; - -// Initialize the AtomicOps_Internalx86CPUFeatures struct. -static void AtomicOps_Internalx86CPUFeaturesInit() { - uint32 eax; - uint32 ebx; - uint32 ecx; - uint32 edx; - - // Get vendor string (issue CPUID with eax = 0) - cpuid(eax, ebx, ecx, edx, 0); - char vendor[13]; - memcpy(vendor, &ebx, 4); - memcpy(vendor + 4, &edx, 4); - memcpy(vendor + 8, &ecx, 4); - vendor[12] = 0; - - // get feature flags in ecx/edx, and family/model in eax - cpuid(eax, ebx, ecx, edx, 1); - - int family = (eax >> 8) & 0xf; // family and model fields - int model = (eax >> 4) & 0xf; - if (family == 0xf) { // use extended family and model fields - family += (eax >> 20) & 0xff; - model += ((eax >> 16) & 0xf) << 4; - } - - // edx bit 26 is SSE2 which we use to tell use whether we can use mfence - AtomicOps_Internalx86CPUFeatures.has_sse2 = ((edx >> 26) & 1); - - // ecx bit 13 indicates whether the cmpxchg16b instruction is supported - AtomicOps_Internalx86CPUFeatures.has_cmpxchg16b = ((ecx >> 13) & 1); - - VLOG(1) << "vendor " << vendor << - " family " << family << - " model " << model << - " sse2 " << AtomicOps_Internalx86CPUFeatures.has_sse2 << - " cmpxchg16b " << AtomicOps_Internalx86CPUFeatures.has_cmpxchg16b; -} - -// AtomicOps initialisation routine for external use. -void AtomicOps_x86CPUFeaturesInit() { - AtomicOps_Internalx86CPUFeaturesInit(); -} - -#endif - -#endif // GUTIL_ATOMICOPS_INTERNALS_X86_H_ diff --git a/be/src/gutil/atomicops-internals-x86.h b/be/src/gutil/atomicops-internals-x86.h deleted file mode 100644 index f9b498ec2..000000000 --- a/be/src/gutil/atomicops-internals-x86.h +++ /dev/null @@ -1,501 +0,0 @@ -// Copyright 2003 Google Inc. -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// All Rights Reserved. -// -// -// Implementation of atomic operations for x86. This file should not -// be included directly. Clients should instead include -// "base/atomicops.h". - -#ifndef GUTIL_ATOMICOPS_INTERNALS_X86_H_ -#define GUTIL_ATOMICOPS_INTERNALS_X86_H_ - -#include <stdint.h> - -#include <common/logging.h> - -#include "util/arithmetic-util.h" - -#define BASE_HAS_ATOMIC64 1 // Use only in tests and base/atomic* - - -// NOTE(user): x86 does not need to define AtomicWordCastType, because it -// already matches Atomic32 or Atomic64, depending on the platform. - - -// This struct is not part of the public API of this module; clients may not -// use it. -// Features of this x86. Values may not be correct before InitGoogle() is run, -// but are set conservatively. -struct AtomicOps_x86CPUFeatureStruct { - bool has_sse2; // Processor has SSE2. - bool has_cmpxchg16b; // Processor supports cmpxchg16b instruction. -}; -extern struct AtomicOps_x86CPUFeatureStruct AtomicOps_Internalx86CPUFeatures; - - -#define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") - -// AtomicOps initialisation for open source use. -void AtomicOps_x86CPUFeaturesInit(); - -typedef int32_t Atomic32; -typedef int64_t Atomic64; - -namespace base { -namespace subtle { - -typedef int32_t Atomic32; -typedef int64_t Atomic64; - -// These atomic primitives don't work atomically, and can cause really nasty -// hard-to-track-down bugs, if the pointer isn't naturally aligned. Check alignment -// in debug mode. -template<class T> -inline void CheckNaturalAlignment(const T *ptr) { - DCHECK_EQ(0, reinterpret_cast<const uintptr_t>(ptr) & (sizeof(T) - 1)) - << "unaligned pointer not allowed for atomics"; -} - -// 32-bit low-level operations on any platform. - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - CheckNaturalAlignment(ptr); - Atomic32 prev; - __asm__ __volatile__("lock; cmpxchgl %1,%2" - : "=a" (prev) - : "q" (new_value), "m" (*ptr), "0" (old_value) - : "memory"); - return prev; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - CheckNaturalAlignment(ptr); - __asm__ __volatile__("xchgl %1,%0" // The lock prefix is implicit for xchg. - : "=r" (new_value) - : "m" (*ptr), "0" (new_value) - : "memory"); - return new_value; // Now it's the previous value. -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - CheckNaturalAlignment(ptr); - Atomic32 old_val = NoBarrier_AtomicExchange(ptr, new_value); - return old_val; -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - CheckNaturalAlignment(ptr); - Atomic32 temp = increment; - __asm__ __volatile__("lock; xaddl %0,%1" - : "+r" (temp), "+m" (*ptr) - : : "memory"); - // temp now holds the old value of *ptr - return impala::ArithmeticUtil::AsUnsigned<std::plus>(temp, increment); -} - -inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - CheckNaturalAlignment(ptr); - Atomic32 temp = increment; - __asm__ __volatile__("lock; xaddl %0,%1" - : "+r" (temp), "+m" (*ptr) - : : "memory"); - // temp now holds the old value of *ptr - return impala::ArithmeticUtil::AsUnsigned<std::plus>(temp, increment); -} - -// On x86, the NoBarrier_CompareAndSwap() uses a locked instruction and so also -// provides both acquire and release barriers. -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline Atomic32 Barrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - CheckNaturalAlignment(ptr); - *ptr = value; -} - -// Issue the x86 "pause" instruction, which tells the CPU that we -// are in a spinlock wait loop and should allow other hyperthreads -// to run, not speculate memory access, etc. -inline void PauseCPU() { - __asm__ __volatile__("pause" : : : "memory"); -} - -#if defined(__x86_64__) - -// 64-bit implementations of memory barrier can be simpler, because it -// "mfence" is guaranteed to exist. -inline void MemoryBarrier() { - __asm__ __volatile__("mfence" : : : "memory"); -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - CheckNaturalAlignment(ptr); - *ptr = value; - MemoryBarrier(); -} - -#else - -inline void MemoryBarrier() { - if (AtomicOps_Internalx86CPUFeatures.has_sse2) { - __asm__ __volatile__("mfence" : : : "memory"); - } else { // mfence is faster but not present on PIII - Atomic32 x = 0; - Acquire_AtomicExchange(&x, 0); - } -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - if (AtomicOps_Internalx86CPUFeatures.has_sse2) { - CheckNaturalAlignment(ptr); - *ptr = value; - __asm__ __volatile__("mfence" : : : "memory"); - } else { - Acquire_AtomicExchange(ptr, value); - } -} -#endif - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - CheckNaturalAlignment(ptr); - ATOMICOPS_COMPILER_BARRIER(); - *ptr = value; // An x86 store acts as a release barrier. - // See comments in Atomic64 version of Release_Store(), below. -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - CheckNaturalAlignment(ptr); - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - CheckNaturalAlignment(ptr); - Atomic32 value = *ptr; // An x86 load acts as a acquire barrier. - // See comments in Atomic64 version of Release_Store(), below. - ATOMICOPS_COMPILER_BARRIER(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - CheckNaturalAlignment(ptr); - MemoryBarrier(); - return *ptr; -} - -#if defined(__x86_64__) - -// 64-bit low-level operations on 64-bit platform. - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev; - CheckNaturalAlignment(ptr); - __asm__ __volatile__("lock; cmpxchgq %1,%2" - : "=a" (prev) - : "q" (new_value), "m" (*ptr), "0" (old_value) - : "memory"); - return prev; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - CheckNaturalAlignment(ptr); - __asm__ __volatile__("xchgq %1,%0" // The lock prefix is implicit for xchg. - : "=r" (new_value) - : "m" (*ptr), "0" (new_value) - : "memory"); - return new_value; // Now it's the previous value. -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - Atomic64 old_val = NoBarrier_AtomicExchange(ptr, new_value); - return old_val; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - Atomic64 temp = increment; - CheckNaturalAlignment(ptr); - __asm__ __volatile__("lock; xaddq %0,%1" - : "+r" (temp), "+m" (*ptr) - : : "memory"); - // temp now contains the previous value of *ptr - return impala::ArithmeticUtil::AsUnsigned<std::plus>(temp, increment); -} - -inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - Atomic64 temp = increment; - CheckNaturalAlignment(ptr); - __asm__ __volatile__("lock; xaddq %0,%1" - : "+r" (temp), "+m" (*ptr) - : : "memory"); - // temp now contains the previous value of *ptr - return impala::ArithmeticUtil::AsUnsigned<std::plus>(temp, increment); -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - CheckNaturalAlignment(ptr); - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - CheckNaturalAlignment(ptr); - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - ATOMICOPS_COMPILER_BARRIER(); - CheckNaturalAlignment(ptr); - *ptr = value; // An x86 store acts as a release barrier - // for current AMD/Intel chips as of Jan 2008. - // See also Acquire_Load(), below. - - // When new chips come out, check: - // IA-32 Intel Architecture Software Developer's Manual, Volume 3: - // System Programming Guide, Chatper 7: Multiple-processor management, - // Section 7.2, Memory Ordering. - // Last seen at: - // http://developer.intel.com/design/pentium4/manuals/index_new.htm - // - // x86 stores/loads fail to act as barriers for a few instructions (clflush - // maskmovdqu maskmovq movntdq movnti movntpd movntps movntq) but these are - // not generated by the compiler, and are rare. Users of these instructions - // need to know about cache behaviour in any case since all of these involve - // either flushing cache lines or non-temporal cache hints. -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - CheckNaturalAlignment(ptr); - return *ptr; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - CheckNaturalAlignment(ptr); - Atomic64 value = *ptr; // An x86 load acts as a acquire barrier, - // for current AMD/Intel chips as of Jan 2008. - // See also Release_Store(), above. - ATOMICOPS_COMPILER_BARRIER(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - CheckNaturalAlignment(ptr); - MemoryBarrier(); - return *ptr; -} - -#else // defined(__x86_64__) - -// 64-bit low-level operations on 32-bit platform. - -#if !((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) -// For compilers older than gcc 4.1, we use inline asm. -// -// Potential pitfalls: -// -// 1. %ebx points to Global offset table (GOT) with -fPIC. -// We need to preserve this register. -// 2. When explicit registers are used in inline asm, the -// compiler may not be aware of it and might try to reuse -// the same register for another argument which has constraints -// that allow it ("r" for example). - -inline Atomic64 __sync_val_compare_and_swap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - CheckNaturalAlignment(ptr); - Atomic64 prev; - __asm__ __volatile__("push %%ebx\n\t" - "movl (%3), %%ebx\n\t" // Move 64-bit new_value into - "movl 4(%3), %%ecx\n\t" // ecx:ebx - "lock; cmpxchg8b (%1)\n\t" // If edx:eax (old_value) same - "pop %%ebx\n\t" - : "=A" (prev) // as contents of ptr: - : "D" (ptr), // ecx:ebx => ptr - "0" (old_value) , // else: - "S" (&new_value) // old *ptr => edx:eax - : "memory", "%ecx"); - return prev; -} -#endif // Compiler < gcc-4.1 - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_val, - Atomic64 new_val) { - CheckNaturalAlignment(ptr); - return __sync_val_compare_and_swap(ptr, old_val, new_val); -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_val) { - Atomic64 old_val; - CheckNaturalAlignment(ptr); - - do { - old_val = *ptr; - } while (__sync_val_compare_and_swap(ptr, old_val, new_val) != old_val); - - return old_val; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_val) { - CheckNaturalAlignment(ptr); - Atomic64 old_val = NoBarrier_AtomicExchange(ptr, new_val); - return old_val; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_val) { - return NoBarrier_AtomicExchange(ptr, new_val); -} - -inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - CheckNaturalAlignment(ptr); - Atomic64 old_val, new_val; - - do { - old_val = *ptr; - new_val = impala::ArithmeticUtil::AsUnsigned<std::plus>(old_val, increment); - } while (__sync_val_compare_and_swap(ptr, old_val, new_val) != old_val); - - return impala::ArithmeticUtil::AsUnsigned<std::plus>(old_val, increment); -} - -inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - CheckNaturalAlignment(ptr); - Atomic64 new_val = NoBarrier_AtomicIncrement(ptr, increment); - return new_val; -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - CheckNaturalAlignment(ptr); - __asm__ __volatile__("movq %1, %%mm0\n\t" // Use mmx reg for 64-bit atomic - "movq %%mm0, %0\n\t" // moves (ptr could be read-only) - "emms\n\t" // Empty mmx state/Reset FP regs - : "=m" (*ptr) - : "m" (value) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", - "st(5)", "st(6)", "st(7)", "mm0", "mm1", - "mm2", "mm3", "mm4", "mm5", "mm6", "mm7"); -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - NoBarrier_Store(ptr, value); - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - ATOMICOPS_COMPILER_BARRIER(); - NoBarrier_Store(ptr, value); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - CheckNaturalAlignment(ptr); - Atomic64 value; - __asm__ __volatile__("movq %1, %%mm0\n\t" // Use mmx reg for 64-bit atomic - "movq %%mm0, %0\n\t" // moves (ptr could be read-only) - "emms\n\t" // Empty mmx state/Reset FP regs - : "=m" (value) - : "m" (*ptr) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", - "st(5)", "st(6)", "st(7)", "mm0", "mm1", - "mm2", "mm3", "mm4", "mm5", "mm6", "mm7"); - return value; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - CheckNaturalAlignment(ptr); - Atomic64 value = NoBarrier_Load(ptr); - ATOMICOPS_COMPILER_BARRIER(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return NoBarrier_Load(ptr); -} - -#endif // defined(__x86_64__) - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline Atomic64 Barrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -} // namespace subtle -} // namespace base - -#undef ATOMICOPS_COMPILER_BARRIER - -#endif // GUTIL_ATOMICOPS_INTERNALS_X86_H_ diff --git a/be/src/gutil/atomicops.h b/be/src/gutil/atomicops.h index b358d6e82..eca0bfa60 100644 --- a/be/src/gutil/atomicops.h +++ b/be/src/gutil/atomicops.h @@ -1,14 +1,9 @@ -// Copyright 2003 Google Inc. -// All Rights Reserved. -// +// Copyright 2012 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. -// For atomic operations on statistics counters, see atomic_stats_counter.h. -// For atomic operations on sequence numbers, see atomic_sequence_num.h. // For atomic operations on reference counts, see atomic_refcount.h. - -// Some fast atomic operations -- typically with machine-dependent -// implementations. This file may need editing as Google code is -// ported to different architectures. +// For atomic operations on sequence numbers, see atomic_sequence_num.h. // The routines exported by this module are subtle. If you use them, even if // you get the code right, it will depend on careful reasoning about atomicity @@ -21,89 +16,56 @@ // implementations on other archtectures will cause your code to break. If you // do not know what you are doing, avoid these routines, and use a Mutex. // -// These following lower-level operations are typically useful only to people -// implementing higher-level synchronization operations like spinlocks, -// mutexes, and condition-variables. They combine CompareAndSwap(), -// addition, exchange, a load, or a store with appropriate memory-ordering -// instructions. "Acquire" operations ensure that no later memory access by -// the same thread can be reordered ahead of the operation. "Release" -// operations ensure that no previous memory access by the same thread can be -// reordered after the operation. "Barrier" operations have both "Acquire" and -// "Release" semantics. A MemoryBarrier() has "Barrier" semantics, but does no -// memory access. "NoBarrier" operations have no barrier: the CPU is -// permitted to reorder them freely (as seen by other threads), even in ways -// the appear to violate functional dependence, just as it can for any normal -// variable access. -// // It is incorrect to make direct assignments to/from an atomic variable. // You should use one of the Load or Store routines. The NoBarrier // versions are provided when no barriers are needed: // NoBarrier_Store() // NoBarrier_Load() // Although there are currently no compiler enforcement, you are encouraged -// to use these. Moreover, if you choose to use base::subtle::Atomic64 type, -// you MUST use one of the Load or Store routines to get correct behavior -// on 32-bit platforms. +// to use these. // -// The intent is eventually to put all of these routines in namespace -// base::subtle -#ifndef THREAD_ATOMICOPS_H_ -#define THREAD_ATOMICOPS_H_ +#ifndef BASE_ATOMICOPS_H_ +#define BASE_ATOMICOPS_H_ +#include <atomic> #include <stdint.h> -// ------------------------------------------------------------------------ -// Include the platform specific implementations of the types -// and operations listed below. Implementations are to provide Atomic32 -// and Atomic64 operations. If there is a mismatch between intptr_t and -// the Atomic32 or Atomic64 types for a platform, the platform-specific header -// should define the macro, AtomicWordCastType in a clause similar to the -// following: -// #if ...pointers are 64 bits... -// # define AtomicWordCastType base::subtle::Atomic64 -// #else -// # define AtomicWordCastType Atomic32 -// #endif -// ------------------------------------------------------------------------ - -#include "gutil/arm_instruction_set_select.h" +// Small C++ header which defines implementation specific macros used to +// identify the STL implementation. +// - libc++: captures __config for _LIBCPP_VERSION +// - libstdc++: captures bits/c++config.h for __GLIBCXX__ +#include <cstddef> -// ThreadSanitizer provides own implementation of atomicops. -#if defined(THREAD_SANITIZER) -#include "gutil/atomicops-internals-tsan.h" -#elif defined(__APPLE__) -#include "gutil/atomicops-internals-macosx.h" -#elif defined(__GNUC__) && defined(__aarch64__) -#include "gutil/atomicops-internals-arm64.h" -#elif defined(__GNUC__) && defined(ARMV6) -#include "gutil/atomicops-internals-arm-v6plus.h" -#elif defined(ARMV3) -#include "gutil/atomicops-internals-arm-generic.h" -#elif defined(__GNUC__) && (defined(__i386) || defined(__x86_64__)) -#include "gutil/atomicops-internals-x86.h" -#elif defined(__GNUC__) && defined(ARCH_POWERPC64) -#include "gutil/atomicops-internals-powerpc.h" -#elif defined(OS_WINDOWS) -#include "gutil/atomicops-internals-windows.h" +typedef int32_t Atomic32; +// We need to be able to go between Atomic64 and AtomicWord implicitly. This +// means Atomic64 and AtomicWord should be the same type on 64-bit. +#if defined(__ILP32__) +typedef int64_t Atomic64; #else -#error You need to implement atomic operations for this architecture +typedef intptr_t Atomic64; #endif -// Signed type that can hold a pointer and supports the atomic ops below, as -// well as atomic loads and stores. Instances must be naturally-aligned. +// Use AtomicWord for a machine-sized pointer. It will use the Atomic32 or +// Atomic64 routines below, depending on your architecture. typedef intptr_t AtomicWord; -#ifdef AtomicWordCastType -// ------------------------------------------------------------------------ -// This section is needed only when explicit type casting is required to -// cast AtomicWord to one of the basic atomic types (Atomic64 or Atomic32). -// It also serves to document the AtomicWord interface. -// ------------------------------------------------------------------------ - namespace base { namespace subtle { +typedef int32_t Atomic32; +// We need to be able to go between Atomic64 and AtomicWord implicitly. This +// means Atomic64 and AtomicWord should be the same type on 64-bit. +#if defined(__ILP32__) +typedef int64_t Atomic64; +#else +typedef intptr_t Atomic64; +#endif + +// Use AtomicWord for a machine-sized pointer. It will use the Atomic32 or +// Atomic64 routines below, depending on your architecture. +typedef intptr_t AtomicWord; + // Atomically execute: // result = *ptr; // if (*ptr == old_value) @@ -114,146 +76,58 @@ namespace subtle { // Always return the old value of "*ptr" // // This routine implies no memory barriers. -inline AtomicWord NoBarrier_CompareAndSwap(volatile AtomicWord* ptr, - AtomicWord old_value, - AtomicWord new_value) { - return NoBarrier_CompareAndSwap( - reinterpret_cast<volatile AtomicWordCastType*>(ptr), - old_value, new_value); -} +Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value); + +Atomic32 Barrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value); // Atomically store new_value into *ptr, returning the previous value held in // *ptr. This routine implies no memory barriers. -inline AtomicWord NoBarrier_AtomicExchange(volatile AtomicWord* ptr, - AtomicWord new_value) { - return NoBarrier_AtomicExchange( - reinterpret_cast<volatile AtomicWordCastType*>(ptr), new_value); -} - -inline AtomicWord Acquire_AtomicExchange(volatile AtomicWord* ptr, - AtomicWord new_value) { - return Acquire_AtomicExchange( - reinterpret_cast<volatile AtomicWordCastType*>(ptr), new_value); -} +Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, Atomic32 new_value); -inline AtomicWord Release_AtomicExchange(volatile AtomicWord* ptr, - AtomicWord new_value) { - return Release_AtomicExchange( - reinterpret_cast<volatile AtomicWordCastType*>(ptr), new_value); -} +Atomic32 Barrier_AtomicExchange(volatile Atomic32* ptr, Atomic32 new_value); +Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, Atomic32 new_value); // Atomically increment *ptr by "increment". Returns the new value of -// *ptr with the increment applied. This routine implies no memory -// barriers. -inline AtomicWord NoBarrier_AtomicIncrement(volatile AtomicWord* ptr, - AtomicWord increment) { - return NoBarrier_AtomicIncrement( - reinterpret_cast<volatile AtomicWordCastType*>(ptr), increment); -} - -inline AtomicWord Barrier_AtomicIncrement(volatile AtomicWord* ptr, - AtomicWord increment) { - return Barrier_AtomicIncrement( - reinterpret_cast<volatile AtomicWordCastType*>(ptr), increment); -} - -inline AtomicWord Acquire_CompareAndSwap(volatile AtomicWord* ptr, - AtomicWord old_value, - AtomicWord new_value) { - return base::subtle::Acquire_CompareAndSwap( - reinterpret_cast<volatile AtomicWordCastType*>(ptr), - old_value, new_value); -} - -inline AtomicWord Release_CompareAndSwap(volatile AtomicWord* ptr, - AtomicWord old_value, - AtomicWord new_value) { - return base::subtle::Release_CompareAndSwap( - reinterpret_cast<volatile AtomicWordCastType*>(ptr), - old_value, new_value); -} - -inline void NoBarrier_Store(volatile AtomicWord *ptr, AtomicWord value) { - NoBarrier_Store( - reinterpret_cast<volatile AtomicWordCastType*>(ptr), value); -} - -inline void Acquire_Store(volatile AtomicWord* ptr, AtomicWord value) { - return base::subtle::Acquire_Store( - reinterpret_cast<volatile AtomicWordCastType*>(ptr), value); -} - -inline void Release_Store(volatile AtomicWord* ptr, AtomicWord value) { - return base::subtle::Release_Store( - reinterpret_cast<volatile AtomicWordCastType*>(ptr), value); -} - -inline AtomicWord NoBarrier_Load(volatile const AtomicWord *ptr) { - return NoBarrier_Load( - reinterpret_cast<volatile const AtomicWordCastType*>(ptr)); -} - -inline AtomicWord Acquire_Load(volatile const AtomicWord* ptr) { - return base::subtle::Acquire_Load( - reinterpret_cast<volatile const AtomicWordCastType*>(ptr)); -} - -inline AtomicWord Release_Load(volatile const AtomicWord* ptr) { - return base::subtle::Release_Load( - reinterpret_cast<volatile const AtomicWordCastType*>(ptr)); -} - -} // namespace base::subtle -} // namespace base -#endif // AtomicWordCastType - -// ------------------------------------------------------------------------ -// Commented out type definitions and method declarations for documentation -// of the interface provided by this module. -// ------------------------------------------------------------------------ - -#if 0 - -// Signed 32-bit type that supports the atomic ops below, as well as atomic -// loads and stores. Instances must be naturally aligned. This type differs -// from AtomicWord in 64-bit binaries where AtomicWord is 64-bits. -typedef int32_t Atomic32; - -// Corresponding operations on Atomic32 -namespace base { -namespace subtle { - -// Signed 64-bit type that supports the atomic ops below, as well as atomic -// loads and stores. Instances must be naturally aligned. This type differs -// from AtomicWord in 32-bit binaries where AtomicWord is 32-bits. -typedef int64_t Atomic64; - -Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value); -Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, Atomic32 new_value); -Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, Atomic32 new_value); -Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, Atomic32 new_value); +// *ptr with the increment applied. This routine implies no memory barriers. Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, Atomic32 increment); + Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, Atomic32 increment); + +// These following lower-level operations are typically useful only to people +// implementing higher-level synchronization operations like spinlocks, +// mutexes, and condition-variables. They combine CompareAndSwap(), a load, or +// a store with appropriate memory-ordering instructions. "Acquire" operations +// ensure that no later memory access can be reordered ahead of the operation. +// "Release" operations ensure that no previous memory access can be reordered +// after the operation. "Barrier" operations have both "Acquire" and "Release" +// semantics. Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, Atomic32 old_value, Atomic32 new_value); Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, Atomic32 old_value, Atomic32 new_value); + void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value); -void Acquire_Store(volatile Atomic32* ptr, Atomic32 value); void Release_Store(volatile Atomic32* ptr, Atomic32 value); +void Acquire_Store(volatile Atomic32* ptr, Atomic32 value); + Atomic32 NoBarrier_Load(volatile const Atomic32* ptr); Atomic32 Acquire_Load(volatile const Atomic32* ptr); Atomic32 Release_Load(volatile const Atomic32* ptr); -// Corresponding operations on Atomic64 +// 64-bit atomic operations (only available on 64-bit processors). Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, Atomic64 old_value, Atomic64 new_value); +Atomic64 Barrier_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value); Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, Atomic64 new_value); Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, Atomic64 new_value); Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, Atomic64 new_value); @@ -272,51 +146,18 @@ void Release_Store(volatile Atomic64* ptr, Atomic64 value); Atomic64 NoBarrier_Load(volatile const Atomic64* ptr); Atomic64 Acquire_Load(volatile const Atomic64* ptr); Atomic64 Release_Load(volatile const Atomic64* ptr); -} // namespace base::subtle -} // namespace base void MemoryBarrier(); +// Platform specific implementations void PauseCPU(); -#endif // 0 - - -// ------------------------------------------------------------------------ -// The following are to be deprecated when all uses have been changed to -// use the base::subtle namespace. -// ------------------------------------------------------------------------ - -#ifdef AtomicWordCastType -// AtomicWord versions to be deprecated -inline AtomicWord Acquire_CompareAndSwap(volatile AtomicWord* ptr, - AtomicWord old_value, - AtomicWord new_value) { - return base::subtle::Acquire_CompareAndSwap(ptr, old_value, new_value); -} - -inline AtomicWord Release_CompareAndSwap(volatile AtomicWord* ptr, - AtomicWord old_value, - AtomicWord new_value) { - return base::subtle::Release_CompareAndSwap(ptr, old_value, new_value); -} - -inline void Acquire_Store(volatile AtomicWord* ptr, AtomicWord value) { - return base::subtle::Acquire_Store(ptr, value); -} - -inline void Release_Store(volatile AtomicWord* ptr, AtomicWord value) { - return base::subtle::Release_Store(ptr, value); -} +} // namespace subtle +} // namespace base -inline AtomicWord Acquire_Load(volatile const AtomicWord* ptr) { - return base::subtle::Acquire_Load(ptr); -} -inline AtomicWord Release_Load(volatile const AtomicWord* ptr) { - return base::subtle::Release_Load(ptr); -} -#endif // AtomicWordCastType +#include "gutil/atomicops-internals-portable.h" +#include "gutil/yield_processor.h" // 32-bit Acquire/Release operations to be deprecated. @@ -372,4 +213,4 @@ inline base::subtle::Atomic64 Release_Load( return base::subtle::Release_Load(ptr); } -#endif // THREAD_ATOMICOPS_H_ +#endif // BASE_ATOMICOPS_H_ diff --git a/be/src/gutil/auxiliary/atomicops-internals-arm-generic.h b/be/src/gutil/auxiliary/atomicops-internals-arm-generic.h deleted file mode 100644 index 0a73b85a7..000000000 --- a/be/src/gutil/auxiliary/atomicops-internals-arm-generic.h +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright 2003 Google Inc. -// All Rights Reserved. -// -// -// This file is an internal atomic implementation, use base/atomicops.h instead. -// -// LinuxKernelCmpxchg and Barrier_AtomicIncrement are from Google Gears. - -#ifndef BASE_AUXILIARY_ATOMICOPS_INTERNALS_ARM_GENERIC_H_ -#define BASE_AUXILIARY_ATOMICOPS_INTERNALS_ARM_GENERIC_H_ - -#include <stdio.h> -#include <stdlib.h> -#include "gutil/macros.h" // For COMPILE_ASSERT -#include "gutil/port.h" // ATTRIBUTE_WEAK - -typedef int32_t Atomic32; - -namespace base { -namespace subtle { - -typedef int64_t Atomic64; - -// 0xffff0fc0 is the hard coded address of a function provided by -// the kernel which implements an atomic compare-exchange. On older -// ARM architecture revisions (pre-v6) this may be implemented using -// a syscall. This address is stable, and in active use (hard coded) -// by at least glibc-2.7 and the Android C library. -// pLinuxKernelCmpxchg has both acquire and release barrier sematincs. -typedef Atomic32 (*LinuxKernelCmpxchgFunc)(Atomic32 old_value, - Atomic32 new_value, - volatile Atomic32* ptr); -LinuxKernelCmpxchgFunc pLinuxKernelCmpxchg ATTRIBUTE_WEAK = - (LinuxKernelCmpxchgFunc) 0xffff0fc0; - -typedef void (*LinuxKernelMemoryBarrierFunc)(void); -LinuxKernelMemoryBarrierFunc pLinuxKernelMemoryBarrier ATTRIBUTE_WEAK = - (LinuxKernelMemoryBarrierFunc) 0xffff0fa0; - - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value = *ptr; - do { - if (!pLinuxKernelCmpxchg(old_value, new_value, - const_cast<Atomic32*>(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 old_value; - do { - old_value = *ptr; - } while (pLinuxKernelCmpxchg(old_value, new_value, - const_cast<Atomic32*>(ptr))); - return old_value; -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - for (;;) { - // Atomic exchange the old value with an incremented one. - Atomic32 old_value = *ptr; - Atomic32 new_value = old_value + increment; - if (pLinuxKernelCmpxchg(old_value, new_value, - const_cast<Atomic32*>(ptr)) == 0) { - // The exchange took place as expected. - return new_value; - } - // Otherwise, *ptr changed mid-loop and we need to retry. - } -} - -inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - return Barrier_AtomicIncrement(ptr, increment); -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -inline void MemoryBarrier() { - pLinuxKernelMemoryBarrier(); -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - Atomic32 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - MemoryBarrier(); - return *ptr; -} - - -// 64-bit versions are not implemented yet. - -inline void NotImplementedFatalError(const char *function_name) { - fprintf(stderr, "64-bit %s() not implemented on this platform\n", - function_name); - abort(); -} - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - NotImplementedFatalError("NoBarrier_CompareAndSwap"); - return 0; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - NotImplementedFatalError("NoBarrier_AtomicExchange"); - return 0; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - NotImplementedFatalError("Acquire_AtomicExchange"); - return 0; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - NotImplementedFatalError("Release_AtomicExchange"); - return 0; -} - -inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - NotImplementedFatalError("NoBarrier_AtomicIncrement"); - return 0; -} - -inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - NotImplementedFatalError("Barrier_AtomicIncrement"); - return 0; -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("NoBarrier_Store"); -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("Acquire_Store64"); -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("Release_Store"); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("NoBarrier_Load"); - return 0; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("Atomic64 Acquire_Load"); - return 0; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("Atomic64 Release_Load"); - return 0; -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - NotImplementedFatalError("Atomic64 Acquire_CompareAndSwap"); - return 0; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - NotImplementedFatalError("Atomic64 Release_CompareAndSwap"); - return 0; -} - -} // namespace base::subtle -} // namespace base - -#endif // BASE_AUXILIARY_ATOMICOPS_INTERNALS_ARM_GENERIC_H_ diff --git a/be/src/gutil/auxiliary/atomicops-internals-arm-v6plus.h b/be/src/gutil/auxiliary/atomicops-internals-arm-v6plus.h deleted file mode 100644 index 631f0f0f0..000000000 --- a/be/src/gutil/auxiliary/atomicops-internals-arm-v6plus.h +++ /dev/null @@ -1,378 +0,0 @@ -// Copyright 2011 Google Inc. -// All Rights Reserved. -// -// based on atomicops-internals by Sanjay Ghemawat -// -// This file is an internal atomic implementation, use base/atomicops.h instead. -// -// This code implements ARM atomics for architectures V6 and newer. - -#ifndef BASE_AUXILIARY_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ -#define BASE_AUXILIARY_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ - -#include <stdio.h> -#include <stdlib.h> -#include "gutil/basictypes.h" // For COMPILE_ASSERT - -// The LDREXD and STREXD instructions in ARM all v7 variants or above. In v6, -// only some variants support it. For simplicity, we only use exclusive -// 64-bit load/store in V7 or above. -#if defined(ARMV7) -# define BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD -#endif - -typedef int32_t Atomic32; - -namespace base { -namespace subtle { - -typedef int64_t Atomic64; - -inline void MemoryBarrier() { - __asm__ __volatile__("dmb" : : : "memory"); -} - -// 32-bit low-level ops - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 oldval, res; - do { - __asm__ __volatile__( - "ldrex %1, [%3]\n" - "mov %0, #0\n" - "teq %1, %4\n" - // The following IT (if-then) instruction is needed for the subsequent - // conditional instruction STREXEQ when compiling in THUMB mode. - // In ARM mode, the compiler/assembler will not generate any code for it. - "it eq\n" - "strexeq %0, %5, [%3]\n" - : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) - : "r" (ptr), "Ir" (old_value), "r" (new_value) - : "cc"); - } while (res); - return oldval; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 tmp, old; - __asm__ __volatile__( - "1:\n" - "ldrex %1, [%2]\n" - "strex %0, %3, [%2]\n" - "teq %0, #0\n" - "bne 1b" - : "=&r" (tmp), "=&r" (old) - : "r" (ptr), "r" (new_value) - : "cc", "memory"); - return old; -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 old_value = NoBarrier_AtomicExchange(ptr, new_value); - MemoryBarrier(); - return old_value; -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - MemoryBarrier(); - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - Atomic32 tmp, res; - __asm__ __volatile__( - "1:\n" - "ldrex %1, [%2]\n" - "add %1, %1, %3\n" - "strex %0, %1, [%2]\n" - "teq %0, #0\n" - "bne 1b" - : "=&r" (tmp), "=&r"(res) - : "r" (ptr), "r"(increment) - : "cc", "memory"); - return res; -} - -inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - Atomic32 tmp, res; - __asm__ __volatile__( - "1:\n" - "ldrex %1, [%2]\n" - "add %1, %1, %3\n" - "dmb\n" - "strex %0, %1, [%2]\n" - "teq %0, #0\n" - "bne 1b" - : "=&r" (tmp), "=&r"(res) - : "r" (ptr), "r"(increment) - : "cc", "memory"); - return res; -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - MemoryBarrier(); - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - Atomic32 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - MemoryBarrier(); - return *ptr; -} - -// 64-bit versions are only available if LDREXD and STREXD instructions -// are available. -#ifdef BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD - -#define BASE_HAS_ATOMIC64 1 - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 oldval, res; - do { - __asm__ __volatile__( - "ldrexd %1, [%3]\n" - "mov %0, #0\n" - "teq %Q1, %Q4\n" - // The following IT (if-then) instructions are needed for the subsequent - // conditional instructions when compiling in THUMB mode. - // In ARM mode, the compiler/assembler will not generate any code for it. - "it eq\n" - "teqeq %R1, %R4\n" - "it eq\n" - "strexdeq %0, %5, [%3]\n" - : "=&r" (res), "=&r" (oldval), "+Q" (*ptr) - : "r" (ptr), "Ir" (old_value), "r" (new_value) - : "cc"); - } while (res); - return oldval; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - int store_failed; - Atomic64 old; - __asm__ __volatile__( - "1:\n" - "ldrexd %1, [%2]\n" - "strexd %0, %3, [%2]\n" - "teq %0, #0\n" - "bne 1b" - : "=&r" (store_failed), "=&r" (old) - : "r" (ptr), "r" (new_value) - : "cc", "memory"); - return old; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - Atomic64 old_value = NoBarrier_AtomicExchange(ptr, new_value); - MemoryBarrier(); - return old_value; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - MemoryBarrier(); - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - int store_failed; - Atomic64 res; - __asm__ __volatile__( - "1:\n" - "ldrexd %1, [%2]\n" - "adds %Q1, %Q1, %Q3\n" - "adc %R1, %R1, %R3\n" - "strexd %0, %1, [%2]\n" - "teq %0, #0\n" - "bne 1b" - : "=&r" (store_failed), "=&r"(res) - : "r" (ptr), "r"(increment) - : "cc", "memory"); - return res; -} - -inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - int store_failed; - Atomic64 res; - __asm__ __volatile__( - "1:\n" - "ldrexd %1, [%2]\n" - "adds %Q1, %Q1, %Q3\n" - "adc %R1, %R1, %R3\n" - "dmb\n" - "strexd %0, %1, [%2]\n" - "teq %0, #0\n" - "bne 1b" - : "=&r" (store_failed), "=&r"(res) - : "r" (ptr), "r"(increment) - : "cc", "memory"); - return res; -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - int store_failed; - Atomic64 dummy; - __asm__ __volatile__( - "1:\n" - // Dummy load to lock cache line. - "ldrexd %1, [%3]\n" - "strexd %0, %2, [%3]\n" - "teq %0, #0\n" - "bne 1b" - : "=&r" (store_failed), "=&r"(dummy) - : "r"(value), "r" (ptr) - : "cc", "memory"); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - Atomic64 res; - __asm__ __volatile__( - "ldrexd %0, [%1]\n" - "clrex\n" - : "=r" (res) - : "r"(ptr), "Q"(*ptr)); - return res; -} - -#else // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD - -inline void NotImplementedFatalError(const char *function_name) { - fprintf(stderr, "64-bit %s() not implemented on this platform\n", - function_name); - abort(); -} - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - NotImplementedFatalError("NoBarrier_CompareAndSwap"); - return 0; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - NotImplementedFatalError("NoBarrier_AtomicExchange"); - return 0; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - NotImplementedFatalError("Acquire_AtomicExchange"); - return 0; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - NotImplementedFatalError("Release_AtomicExchange"); - return 0; -} - -inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - NotImplementedFatalError("NoBarrier_AtomicIncrement"); - return 0; -} - -inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - NotImplementedFatalError("Barrier_AtomicIncrement"); - return 0; -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("NoBarrier_Store"); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("NoBarrier_Load"); - return 0; -} - -#endif // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - NoBarrier_Store(ptr, value); - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - MemoryBarrier(); - NoBarrier_Store(ptr, value); -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - Atomic64 value = NoBarrier_Load(ptr); - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return NoBarrier_Load(ptr); -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - MemoryBarrier(); - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -} // namespace subtle ends -} // namespace base ends - -#endif // BASE_AUXILIARY_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ diff --git a/be/src/gutil/auxiliary/atomicops-internals-windows.h b/be/src/gutil/auxiliary/atomicops-internals-windows.h deleted file mode 100644 index 11d7d9d5e..000000000 --- a/be/src/gutil/auxiliary/atomicops-internals-windows.h +++ /dev/null @@ -1,508 +0,0 @@ -// Copyright 2003 Google Inc. -// All Rights Reserved. -// -// -// Implementation of atomic operations using Windows API -// functions. This file should not be included directly. Clients -// should instead include "base/atomicops.h". - -#ifndef BASE_AUXILIARY_ATOMICOPS_INTERNALS_WINDOWS_H_ -#define BASE_AUXILIARY_ATOMICOPS_INTERNALS_WINDOWS_H_ - -#include <stdio.h> -#include <stdlib.h> -#include "gutil/basictypes.h" // For COMPILE_ASSERT - -typedef int32 Atomic32; - -#if defined(_WIN64) -#define BASE_HAS_ATOMIC64 1 // Use only in tests and base/atomic* -#endif - -namespace base { -namespace subtle { - -typedef int64 Atomic64; - -// On windows, we assume intrinsics and asms are compiler barriers -// This is redefined below if using gcc asms. -#define ATOMICOPS_COMPILER_BARRIER() - - -// 32-bit low-level operations on any platform - -extern "C" { -// We use windows intrinsics when we can (they seem to be supported -// well on MSVC 8.0 and above). Unfortunately, in some -// environments, <windows.h> and <intrin.h> have conflicting -// declarations of some other intrinsics, breaking compilation: -// http://connect.microsoft.com/VisualStudio/feedback/details/262047 -// Therefore, we simply declare the relevant intrinsics ourself. - -// MinGW has a bug in the header files where it doesn't indicate the -// first argument is volatile -- they're not up to date. See -// http://readlist.com/lists/lists.sourceforge.net/mingw-users/0/3861.html -// We have to const_cast away the volatile to avoid compiler warnings. -// TODO(user): remove this once MinGW has updated MinGW/include/winbase.h -#if defined(__MINGW32__) -inline LONG FastInterlockedCompareExchange(volatile LONG* ptr, - LONG newval, LONG oldval) { - return ::InterlockedCompareExchange(const_cast<LONG*>(ptr), newval, oldval); -} -inline LONG FastInterlockedExchange(volatile LONG* ptr, LONG newval) { - return ::InterlockedExchange(const_cast<LONG*>(ptr), newval); -} -inline LONG FastInterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { - return ::InterlockedExchangeAdd(const_cast<LONG*>(ptr), increment); -} - -#elif _MSC_VER >= 1400 // intrinsics didn't work so well before MSVC 8.0 -// Unfortunately, in some environments, <windows.h> and <intrin.h> -// have conflicting declarations of some intrinsics, breaking -// compilation. So we declare the intrinsics we need ourselves. See -// http://connect.microsoft.com/VisualStudio/feedback/details/262047 -LONG _InterlockedCompareExchange(volatile LONG* ptr, LONG newval, LONG oldval); -#pragma intrinsic(_InterlockedCompareExchange) -inline LONG FastInterlockedCompareExchange(volatile LONG* ptr, - LONG newval, LONG oldval) { - return _InterlockedCompareExchange(ptr, newval, oldval); -} - -LONG _InterlockedExchange(volatile LONG* ptr, LONG newval); -#pragma intrinsic(_InterlockedExchange) -inline LONG FastInterlockedExchange(volatile LONG* ptr, LONG newval) { - return _InterlockedExchange(ptr, newval); -} - -LONG _InterlockedExchangeAdd(volatile LONG* ptr, LONG increment); -#pragma intrinsic(_InterlockedExchangeAdd) -inline LONG FastInterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { - return _InterlockedExchangeAdd(ptr, increment); -} - -#else -inline LONG FastInterlockedCompareExchange(volatile LONG* ptr, - LONG newval, LONG oldval) { - return ::InterlockedCompareExchange(ptr, newval, oldval); -} -inline LONG FastInterlockedExchange(volatile LONG* ptr, LONG newval) { - return ::InterlockedExchange(ptr, newval); -} -inline LONG FastInterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { - return ::InterlockedExchangeAdd(ptr, increment); -} - -#endif // ifdef __MINGW32__ -} // extern "C" - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - LONG result = FastInterlockedCompareExchange( - reinterpret_cast<volatile LONG*>(ptr), - static_cast<LONG>(new_value), - static_cast<LONG>(old_value)); - return static_cast<Atomic32>(result); -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - LONG result = FastInterlockedExchange( - reinterpret_cast<volatile LONG*>(ptr), - static_cast<LONG>(new_value)); - return static_cast<Atomic32>(result); -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - return FastInterlockedExchangeAdd( - reinterpret_cast<volatile LONG*>(ptr), - static_cast<LONG>(increment)) + increment; -} - -inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - return Barrier_AtomicIncrement(ptr, increment); -} - -} // namespace base::subtle -} // namespace base - - -// In msvc8/vs2005, winnt.h already contains a definition for -// MemoryBarrier in the global namespace. Add it there for earlier -// versions and forward to it from within the namespace. -#if !(_MSC_VER && _MSC_VER >= 1400) -inline void MemoryBarrier() { - Atomic32 value = 0; - base::subtle::NoBarrier_AtomicExchange(&value, 0); - // actually acts as a barrier in thisd implementation -} -#endif - -namespace base { -namespace subtle { - -inline void MemoryBarrier() { - ::MemoryBarrier(); -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - Acquire_AtomicExchange(ptr, value); -} - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; // works w/o barrier for current Intel chips as of June 2005 - // See comments in Atomic64 version of Release_Store() below. -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - Atomic32 value = *ptr; - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - MemoryBarrier(); - return *ptr; -} - -// 64-bit operations - -#if defined(_WIN64) || defined(__MINGW64__) - -// 64-bit low-level operations on 64-bit platform. - -COMPILE_ASSERT(sizeof(Atomic64) == sizeof(PVOID), atomic_word_is_atomic); - -// These are the intrinsics needed for 64-bit operations. Similar to the -// 32-bit case above. - -extern "C" { -#if defined(__MINGW64__) -inline PVOID FastInterlockedCompareExchangePointer(volatile PVOID* ptr, - PVOID newval, PVOID oldval) { - return ::InterlockedCompareExchangePointer(const_cast<PVOID*>(ptr), - newval, oldval); -} -inline PVOID FastInterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { - return ::InterlockedExchangePointer(const_cast<PVOID*>(ptr), newval); -} -inline LONGLONG FastInterlockedExchangeAdd64(volatile LONGLONG* ptr, - LONGLONG increment) { - return ::InterlockedExchangeAdd64(const_cast<LONGLONG*>(ptr), increment); -} - -#elif _MSC_VER >= 1400 // intrinsics didn't work so well before MSVC 8.0 -// Like above, we need to declare the intrinsics ourselves. -PVOID _InterlockedCompareExchangePointer(volatile PVOID* ptr, - PVOID newval, PVOID oldval); -#pragma intrinsic(_InterlockedCompareExchangePointer) -inline PVOID FastInterlockedCompareExchangePointer(volatile PVOID* ptr, - PVOID newval, PVOID oldval) { - return _InterlockedCompareExchangePointer(const_cast<PVOID*>(ptr), - newval, oldval); -} - -PVOID _InterlockedExchangePointer(volatile PVOID* ptr, PVOID newval); -#pragma intrinsic(_InterlockedExchangePointer) -inline PVOID FastInterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { - return _InterlockedExchangePointer(const_cast<PVOID*>(ptr), newval); -} - -LONGLONG _InterlockedExchangeAdd64(volatile LONGLONG* ptr, LONGLONG increment); -#pragma intrinsic(_InterlockedExchangeAdd64) -inline LONGLONG FastInterlockedExchangeAdd64(volatile LONGLONG* ptr, - LONGLONG increment) { - return _InterlockedExchangeAdd64(const_cast<LONGLONG*>(ptr), increment); -} - -#else -inline PVOID FastInterlockedCompareExchangePointer(volatile PVOID* ptr, - PVOID newval, PVOID oldval) { - return ::InterlockedCompareExchangePointer(ptr, newval, oldval); -} -inline PVOID FastInterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { - return ::InterlockedExchangePointer(ptr, newval); -} -inline LONGLONG FastInterlockedExchangeAdd64(volatile LONGLONG* ptr, - LONGLONG increment) { - return ::InterlockedExchangeAdd64(ptr, increment); -} - -#endif // ifdef __MINGW64__ -} // extern "C" - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - PVOID result = FastInterlockedCompareExchangePointer( - reinterpret_cast<volatile PVOID*>(ptr), - reinterpret_cast<PVOID>(new_value), reinterpret_cast<PVOID>(old_value)); - return reinterpret_cast<Atomic64>(result); -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - PVOID result = FastInterlockedExchangePointer( - reinterpret_cast<volatile PVOID*>(ptr), - reinterpret_cast<PVOID>(new_value)); - return reinterpret_cast<Atomic64>(result); -} - -inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - return FastInterlockedExchangeAdd64( - reinterpret_cast<volatile LONGLONG*>(ptr), - static_cast<LONGLONG>(increment)) + increment; -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - NoBarrier_AtomicExchange(ptr, value); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - return *ptr; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return *ptr; -} - -#else // defined(_WIN64) || defined(__MINGW64__) - -// 64-bit low-level operations on 32-bit platform - -#if defined(_MSC_VER) -// Windows, 32-bit ABI, with MSVC compiler. - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* p) { - Atomic64 value; - __asm { - mov eax, p - movq mm0, [eax] // Use mmx reg for 64-bit atomic moves - movq value, mm0 - emms // Empty mmx state to enable FP registers - } - return value; -} - -inline void NoBarrier_Store(volatile Atomic64* p, Atomic64 value) { - __asm { - mov eax, p - movq mm0, value // Use mmx reg for 64-bit atomic moves - movq [eax], mm0 - emms // Empty mmx state to enable FP registers - } -} - -#pragma warning(push) -#pragma warning(disable : 4035) // disable the warning about no return statement - // in NoBarrier_CompareAndSwap() - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* p, - Atomic64 old_value, - Atomic64 new_value) { - __asm { - lea edi, old_value - lea esi, new_value - mov eax, [edi] - mov edx, [edi+4] - mov ebx, [esi] - mov ecx, [esi+4] - mov edi, p - lock cmpxchg8b [edi] - } - // There's no explcit return statement, so the warning is disabled above. - // The result is returned in edx,eax -} -#pragma warning(pop) - -#elif defined(__MINGW32__) -// Windows, 32-bit ABI, with GNU compiler. - -#undef ATOMICOPS_COMPILER_BARRIER -#define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - Atomic64 value; - __asm__ __volatile__("movq %1, %%mm0\n\t" // Use mmx reg for 64-bit atomic - "movq %%mm0, %0\n\t" // moves (ptr could be read-only) - "emms\n\t" // Empty mmx state/Reset FP regs - : "=m" (value) - : "m" (*ptr) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", - "st(5)", "st(6)", "st(7)", "mm0", "mm1", - "mm2", "mm3", "mm4", "mm5", "mm6", "mm7"); - return value; -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - __asm__ __volatile__("movq %1, %%mm0\n\t" // Use mmx reg for 64-bit atomic - "movq %%mm0, %0\n\t" // moves (ptr could be read-only) - "emms\n\t" // Empty mmx state/Reset FP regs - : "=m" (*ptr) - : "m" (value) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", - "st(5)", "st(6)", "st(7)", "mm0", "mm1", - "mm2", "mm3", "mm4", "mm5", "mm6", "mm7"); -} - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev; - __asm__ __volatile__("push %%ebx\n\t" - "movl (%3), %%ebx\n\t" // Move 64-bit new_value into - "movl 4(%3), %%ecx\n\t" // ecx:ebx - "lock; cmpxchg8b (%1)\n\t"// If edx:eax (old_value) same - "pop %%ebx\n\t" - : "=A" (prev) // as contents of ptr: - : "D" (ptr), // ecx:ebx => ptr - "0" (old_value), // else: - "S" (&new_value) // old *ptr => edx:eax - : "memory", "%ecx"); - return prev; -} - -#else -// Windows, 32-bit ABI, but not Microsoft compiler, or GNU compiler. - -inline void NotImplementedFatalError(const char *function_name) { - fprintf(stderr, "64-bit %s() not implemented on this platform\n", - function_name); - abort(); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("NoBarrier_Load(Atomic64 *)"); -} -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("NoBarrier_Store(Atomic64 *, ...)"); -} -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - NotImplementedFatalError("NoBarrier_CompareAndSwap(Atomic64 *, ...)"); -} -#endif - - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return NoBarrier_Load(ptr); -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - Atomic64 old_value; - do { - old_value = NoBarrier_Load(ptr); - } while (NoBarrier_CompareAndSwap(ptr, old_value, new_value) != old_value); - return old_value; -} - -inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - Atomic64 old_value; - Atomic64 new_value; - do { - old_value = NoBarrier_Load(ptr); - new_value = old_value + increment; - } while (NoBarrier_CompareAndSwap(ptr, old_value, new_value) != old_value); - return new_value; -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - // acquire and release are implicit in x86 AtomicExchange - NoBarrier_AtomicExchange(ptr, value); -} - -#endif // defined(_WIN64) || defined(__MINGW64__) - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - // acquire and release are implicit in x86 AtomicExchange - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - // acquire and release are implicit in x86 AtomicExchange - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - // acquire and release are implicit in x86 CompareAndSwap - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - // acquire and release are implicit in x86 CompareAndSwap - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - - -inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - // barriers are implicit in atomic increment on on the x86 - return NoBarrier_AtomicIncrement(ptr, increment); -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - Atomic64 result = NoBarrier_Load(ptr); // acquire is implicit in x86 loads - ATOMICOPS_COMPILER_BARRIER(); - return result; -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - ATOMICOPS_COMPILER_BARRIER(); - NoBarrier_Store(ptr, value); // release is implicit on x86 stores -} - -#undef ATOMICOPS_COMPILER_BARRIER - -} // namespace base::subtle -} // namespace base - -#endif // BASE_AUXILIARY_ATOMICOPS_INTERNALS_WINDOWS_H_ diff --git a/be/src/gutil/yield_processor.h b/be/src/gutil/yield_processor.h new file mode 100644 index 000000000..f8e24174d --- /dev/null +++ b/be/src/gutil/yield_processor.h @@ -0,0 +1,48 @@ +// Copyright 2003 Google Inc. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// All Rights Reserved. +// +// +// Implementation of PauseCPU. This file should not be included +// directly. Clients should instead include "base/atomicops.h". + +#ifndef GUTIL_YIELD_PROCESSOR_H_ +#define GUTIL_YIELD_PROCESSOR_H_ + +namespace base { +namespace subtle { + +inline void PauseCPU() { +#if defined(__x86_64__) + // Issue the x86 "pause" instruction, which tells the CPU that we + // are in a spinlock wait loop and should allow other hyperthreads + // to run, not speculate memory access, etc. + __asm__ __volatile__("pause" : : : "memory"); +#elif defined(__aarch64__) + __asm__ __volatile__("yield" : : : "memory"); +#else + // PauseCPU is not defined for other architectures. +#endif +} + +} // namespace subtle +} // namespace base + +#endif // GUTIL_YIELD_PROCESSOR_H_
