From 245525501e9a59101c4d41e9288359420bf2eb10 Mon Sep 17 00:00:00 2001 From: vng Date: Tue, 11 Sep 2012 16:10:24 +0300 Subject: [PATCH] Add boost::atomic primitive. --- 3party/boost/boost/atomic.hpp | 204 ++++++++ 3party/boost/boost/atomic/detail/base.hpp | 186 +++++++ 3party/boost/boost/atomic/detail/builder.hpp | 405 ++++++++++++++++ 3party/boost/boost/atomic/detail/fallback.hpp | 76 +++ .../boost/boost/atomic/detail/gcc-alpha.hpp | 354 ++++++++++++++ .../boost/boost/atomic/detail/gcc-armv6+.hpp | 299 ++++++++++++ 3party/boost/boost/atomic/detail/gcc-ppc.hpp | 351 ++++++++++++++ 3party/boost/boost/atomic/detail/gcc-x86.hpp | 454 ++++++++++++++++++ .../boost/boost/atomic/detail/generic-cas.hpp | 192 ++++++++ .../boost/atomic/detail/integral-casts.hpp | 296 ++++++++++++ .../boost/boost/atomic/detail/interlocked.hpp | 131 +++++ .../boost/boost/atomic/detail/linux-arm.hpp | 169 +++++++ .../atomic/detail/valid_integral_types.hpp | 37 ++ 3party/boost/boost/atomic/platform.hpp | 42 ++ std/atomic.hpp | 5 + 15 files changed, 3201 insertions(+) create mode 100644 3party/boost/boost/atomic.hpp create mode 100644 3party/boost/boost/atomic/detail/base.hpp create mode 100644 3party/boost/boost/atomic/detail/builder.hpp create mode 100644 3party/boost/boost/atomic/detail/fallback.hpp create mode 100644 3party/boost/boost/atomic/detail/gcc-alpha.hpp create mode 100644 3party/boost/boost/atomic/detail/gcc-armv6+.hpp create mode 100644 3party/boost/boost/atomic/detail/gcc-ppc.hpp create mode 100644 3party/boost/boost/atomic/detail/gcc-x86.hpp create mode 100644 3party/boost/boost/atomic/detail/generic-cas.hpp create mode 100644 3party/boost/boost/atomic/detail/integral-casts.hpp create mode 100644 3party/boost/boost/atomic/detail/interlocked.hpp create mode 100644 3party/boost/boost/atomic/detail/linux-arm.hpp create mode 100644 3party/boost/boost/atomic/detail/valid_integral_types.hpp create mode 100644 3party/boost/boost/atomic/platform.hpp create mode 100644 std/atomic.hpp diff --git a/3party/boost/boost/atomic.hpp b/3party/boost/boost/atomic.hpp new file mode 100644 index 0000000000..b2086b9f72 --- /dev/null +++ b/3party/boost/boost/atomic.hpp @@ -0,0 +1,204 @@ +#ifndef BOOST_ATOMIC_HPP +#define BOOST_ATOMIC_HPP + +// Copyright (c) 2009 Helge Bahmann +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include + +#include +#include +#include +#include + +namespace boost { + +template +class atomic : public detail::atomic::internal_atomic { +public: + typedef detail::atomic::internal_atomic super; + + atomic() {} + explicit atomic(T v) : super(v) {} +private: + atomic(const atomic &); + void operator=(const atomic &); +}; + + +template<> +class atomic : private detail::atomic::internal_atomic { +public: + typedef detail::atomic::internal_atomic super; + + atomic() {} + explicit atomic(bool v) : super(v) {} + + using super::load; + using super::store; + using super::compare_exchange_strong; + using super::compare_exchange_weak; + using super::exchange; + using super::is_lock_free; + + operator bool(void) const volatile {return load();} + bool operator=(bool v) volatile {store(v); return v;} +private: + atomic(const atomic &); + void operator=(const atomic &); +}; + +template<> +class atomic : private detail::atomic::internal_atomic { +public: + typedef detail::atomic::internal_atomic super; + + atomic() {} + explicit atomic(void * p) : super(p) {} + using super::load; + using super::store; + using super::compare_exchange_strong; + using super::compare_exchange_weak; + using super::exchange; + using super::is_lock_free; + + operator void *(void) const volatile {return load();} + void * operator=(void * v) volatile {store(v); return v;} + +private: + atomic(const atomic &); + void * operator=(const atomic &); +}; + +/* FIXME: pointer arithmetic still missing */ + +template +class atomic : private detail::atomic::internal_atomic { +public: + typedef detail::atomic::internal_atomic super; + + atomic() {} + explicit atomic(T * p) : super((intptr_t)p) {} + + T *load(memory_order order=memory_order_seq_cst) const volatile + { + return (T*)super::load(order); + } + void store(T *v, memory_order order=memory_order_seq_cst) volatile + { + super::store((intptr_t)v, order); + } + bool compare_exchange_strong( + T * &expected, + T * desired, + memory_order order=memory_order_seq_cst) volatile + { + return compare_exchange_strong(expected, desired, order, detail::atomic::calculate_failure_order(order)); + } + bool compare_exchange_weak( + T * &expected, + T *desired, + memory_order order=memory_order_seq_cst) volatile + { + return compare_exchange_weak(expected, desired, order, detail::atomic::calculate_failure_order(order)); + } + bool compare_exchange_weak( + T * &expected, + T *desired, + memory_order success_order, + memory_order failure_order) volatile + { + intptr_t expected_=(intptr_t)expected, desired_=(intptr_t)desired; + bool success=super::compare_exchange_weak(expected_, desired_, success_order, failure_order); + expected=(T*)expected_; + return success; + } + bool compare_exchange_strong( + T * &expected, + T *desired, + memory_order success_order, + memory_order failure_order) volatile + { + intptr_t expected_=(intptr_t)expected, desired_=(intptr_t)desired; + bool success=super::compare_exchange_strong(expected_, desired_, success_order, failure_order); + expected=(T*)expected_; + return success; + } + T *exchange(T * replacement, memory_order order=memory_order_seq_cst) volatile + { + return (T*)super::exchange((intptr_t)replacement, order); + } + using super::is_lock_free; + + operator T *(void) const volatile {return load();} + T * operator=(T * v) volatile {store(v); return v;} + + T * fetch_add(ptrdiff_t diff, memory_order order=memory_order_seq_cst) volatile + { + return (T*)super::fetch_add(diff*sizeof(T), order); + } + T * fetch_sub(ptrdiff_t diff, memory_order order=memory_order_seq_cst) volatile + { + return (T*)super::fetch_sub(diff*sizeof(T), order); + } + + T *operator++(void) volatile {return fetch_add(1)+1;} + T *operator++(int) volatile {return fetch_add(1);} + T *operator--(void) volatile {return fetch_sub(1)-1;} + T *operator--(int) volatile {return fetch_sub(1);} +private: + atomic(const atomic &); + T * operator=(const atomic &); +}; + +class atomic_flag : private atomic { +public: + typedef atomic super; + using super::is_lock_free; + + atomic_flag(bool initial_state) : super(initial_state?1:0) {} + atomic_flag() {} + + bool test_and_set(memory_order order=memory_order_seq_cst) + { + return super::exchange(1, order) != 0; + } + void clear(memory_order order=memory_order_seq_cst) + { + super::store(0, order); + } +}; + +typedef atomic atomic_char; +typedef atomic atomic_uchar; +typedef atomic atomic_schar; +typedef atomic atomic_uint8_t; +typedef atomic atomic_int8_t; +typedef atomic atomic_ushort; +typedef atomic atomic_short; +typedef atomic atomic_uint16_t; +typedef atomic atomic_int16_t; +typedef atomic atomic_uint; +typedef atomic atomic_int; +typedef atomic atomic_uint32_t; +typedef atomic atomic_int32_t; +typedef atomic atomic_ulong; +typedef atomic atomic_long; +typedef atomic atomic_uint64_t; +typedef atomic atomic_int64_t; +typedef atomic atomic_ullong; +typedef atomic atomic_llong; +typedef atomic atomic_address; +typedef atomic atomic_bool; + +static inline void atomic_thread_fence(memory_order order) +{ + detail::atomic::platform_atomic_thread_fence(order); +} + +} + +#endif diff --git a/3party/boost/boost/atomic/detail/base.hpp b/3party/boost/boost/atomic/detail/base.hpp new file mode 100644 index 0000000000..7de863cb3f --- /dev/null +++ b/3party/boost/boost/atomic/detail/base.hpp @@ -0,0 +1,186 @@ +#ifndef BOOST_DETAIL_ATOMIC_BASE_HPP +#define BOOST_DETAIL_ATOMIC_BASE_HPP + +// Copyright (c) 2009 Helge Bahmann +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include + +namespace boost { +namespace detail { +namespace atomic { + +static inline memory_order calculate_failure_order(memory_order order) +{ + switch(order) { + case memory_order_acq_rel: return memory_order_acquire; + case memory_order_release: return memory_order_relaxed; + default: return order; + } +} + +template +class platform_atomic : public fallback_atomic { +public: + typedef fallback_atomic super; + + explicit platform_atomic(T v) : super(v) {} + platform_atomic() {} +protected: + typedef typename super::integral_type integral_type; +}; + +template +class platform_atomic_integral : public build_atomic_from_exchange > { +public: + typedef build_atomic_from_exchange > super; + + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral() {} +protected: + typedef typename super::integral_type integral_type; +}; + +template +static inline void platform_atomic_thread_fence(T order) +{ + /* FIXME: this does not provide + sequential consistency, need one global + variable for that... */ + platform_atomic a; + a.exchange(0, order); +} + +template::test> +class internal_atomic; + +template +class internal_atomic : private detail::atomic::platform_atomic { +public: + typedef detail::atomic::platform_atomic super; + + internal_atomic() {} + explicit internal_atomic(T v) : super(v) {} + + operator T(void) const volatile {return load();} + T operator=(T v) volatile {store(v); return v;} + + using super::is_lock_free; + using super::load; + using super::store; + using super::exchange; + + bool compare_exchange_strong( + T &expected, + T desired, + memory_order order=memory_order_seq_cst) volatile + { + return super::compare_exchange_strong(expected, desired, order, calculate_failure_order(order)); + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order order=memory_order_seq_cst) volatile + { + return super::compare_exchange_strong(expected, desired, order, calculate_failure_order(order)); + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + return super::compare_exchange_strong(expected, desired, success_order, failure_order); + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + return super::compare_exchange_strong(expected, desired, success_order, failure_order); + } +private: + internal_atomic(const internal_atomic &); + void operator=(const internal_atomic &); +}; + +template +class internal_atomic : private detail::atomic::platform_atomic_integral { +public: + typedef detail::atomic::platform_atomic_integral super; + typedef typename super::integral_type integral_type; + + internal_atomic() {} + explicit internal_atomic(T v) : super(v) {} + + using super::is_lock_free; + using super::load; + using super::store; + using super::exchange; + using super::fetch_add; + using super::fetch_sub; + using super::fetch_and; + using super::fetch_or; + using super::fetch_xor; + + operator integral_type(void) const volatile {return load();} + integral_type operator=(integral_type v) volatile {store(v); return v;} + + integral_type operator&=(integral_type c) volatile {return fetch_and(c)&c;} + integral_type operator|=(integral_type c) volatile {return fetch_or(c)|c;} + integral_type operator^=(integral_type c) volatile {return fetch_xor(c)^c;} + + integral_type operator+=(integral_type c) volatile {return fetch_add(c)+c;} + integral_type operator-=(integral_type c) volatile {return fetch_sub(c)-c;} + + integral_type operator++(void) volatile {return fetch_add(1)+1;} + integral_type operator++(int) volatile {return fetch_add(1);} + integral_type operator--(void) volatile {return fetch_sub(1)-1;} + integral_type operator--(int) volatile {return fetch_sub(1);} + + bool compare_exchange_strong( + integral_type &expected, + integral_type desired, + memory_order order=memory_order_seq_cst) volatile + { + return super::compare_exchange_strong(expected, desired, order, calculate_failure_order(order)); + } + bool compare_exchange_weak( + integral_type &expected, + integral_type desired, + memory_order order=memory_order_seq_cst) volatile + { + return super::compare_exchange_strong(expected, desired, order, calculate_failure_order(order)); + } + bool compare_exchange_strong( + integral_type &expected, + integral_type desired, + memory_order success_order, + memory_order failure_order) volatile + { + return super::compare_exchange_strong(expected, desired, success_order, failure_order); + } + bool compare_exchange_weak( + integral_type &expected, + integral_type desired, + memory_order success_order, + memory_order failure_order) volatile + { + return super::compare_exchange_strong(expected, desired, success_order, failure_order); + } +private: + internal_atomic(const internal_atomic &); + void operator=(const internal_atomic &); +}; + +} +} +} + +#endif diff --git a/3party/boost/boost/atomic/detail/builder.hpp b/3party/boost/boost/atomic/detail/builder.hpp new file mode 100644 index 0000000000..545b1f521d --- /dev/null +++ b/3party/boost/boost/atomic/detail/builder.hpp @@ -0,0 +1,405 @@ +#ifndef BOOST_DETAIL_ATOMIC_BUILDER_HPP +#define BOOST_DETAIL_ATOMIC_BUILDER_HPP + +// Copyright (c) 2009 Helge Bahmann +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include +#include + +namespace boost { +namespace detail { +namespace atomic { + +/* +given a Base that implements: + +- load(memory_order order) +- compare_exchange_weak(integral_type &expected, integral_type desired, memory_order order) + +generates exchange and compare_exchange_strong +*/ +template +class build_exchange : public Base { +public: + typedef typename Base::integral_type integral_type; + + using Base::load; + using Base::compare_exchange_weak; + + bool compare_exchange_strong( + integral_type &expected, + integral_type desired, + memory_order success_order, + memory_order failure_order) volatile + { + integral_type expected_save=expected; + while(true) { + if (compare_exchange_weak(expected, desired, success_order, failure_order)) return true; + if (expected_save!=expected) return false; + expected=expected_save; + } + } + + integral_type exchange(integral_type replacement, memory_order order=memory_order_seq_cst) volatile + { + integral_type o=load(memory_order_relaxed); + do {} while(!compare_exchange_weak(o, replacement, order, memory_order_relaxed)); + return o; + } + + build_exchange() {} + explicit build_exchange(integral_type i) : Base(i) {} +}; + +/* +given a Base that implements: + +- fetch_add_var(integral_type c, memory_order order) +- fetch_inc(memory_order order) +- fetch_dec(memory_order order) + +creates a fetch_add method that delegates to fetch_inc/fetch_dec if operand +is constant +1/-1, and uses fetch_add_var otherwise + +the intention is to allow optimizing the incredibly common case of +1/-1 +*/ +template +class build_const_fetch_add : public Base { +public: + typedef typename Base::integral_type integral_type; + + integral_type fetch_add( + integral_type c, + memory_order order=memory_order_seq_cst) volatile + { + if (__builtin_constant_p(c)) { + switch(c) { + case -1: return fetch_dec(order); + case 1: return fetch_inc(order); + } + } + return fetch_add_var(c, order); + } + + build_const_fetch_add() {} + explicit build_const_fetch_add(integral_type i) : Base(i) {} +protected: + using Base::fetch_add_var; + using Base::fetch_inc; + using Base::fetch_dec; +}; + +/* +given a Base that implements: + +- load(memory_order order) +- compare_exchange_weak(integral_type &expected, integral_type desired, memory_order order) + +generates a -- not very efficient, but correct -- fetch_add operation +*/ +template +class build_fetch_add : public Base { +public: + typedef typename Base::integral_type integral_type; + + using Base::compare_exchange_weak; + + integral_type fetch_add( + integral_type c, memory_order order=memory_order_seq_cst) volatile + { + integral_type o=Base::load(memory_order_relaxed), n; + do {n=o+c;} while(!compare_exchange_weak(o, n, order, memory_order_relaxed)); + return o; + } + + build_fetch_add() {} + explicit build_fetch_add(integral_type i) : Base(i) {} +}; + +/* +given a Base that implements: + +- fetch_add(integral_type c, memory_order order) + +generates fetch_sub and post/pre- increment/decrement operators +*/ +template +class build_arithmeticops : public Base { +public: + typedef typename Base::integral_type integral_type; + + using Base::fetch_add; + + integral_type fetch_sub( + integral_type c, + memory_order order=memory_order_seq_cst) volatile + { + return fetch_add(-c, order); + } + + build_arithmeticops() {} + explicit build_arithmeticops(integral_type i) : Base(i) {} +}; + +/* +given a Base that implements: + +- load(memory_order order) +- compare_exchange_weak(integral_type &expected, integral_type desired, memory_order order) + +generates -- not very efficient, but correct -- fetch_and, fetch_or and fetch_xor operators +*/ +template +class build_logicops : public Base { +public: + typedef typename Base::integral_type integral_type; + + using Base::compare_exchange_weak; + using Base::load; + + integral_type fetch_and(integral_type c, memory_order order=memory_order_seq_cst) volatile + { + integral_type o=load(memory_order_relaxed), n; + do {n=o&c;} while(!compare_exchange_weak(o, n, order, memory_order_relaxed)); + return o; + } + integral_type fetch_or(integral_type c, memory_order order=memory_order_seq_cst) volatile + { + integral_type o=load(memory_order_relaxed), n; + do {n=o|c;} while(!compare_exchange_weak(o, n, order, memory_order_relaxed)); + return o; + } + integral_type fetch_xor(integral_type c, memory_order order=memory_order_seq_cst) volatile + { + integral_type o=load(memory_order_relaxed), n; + do {n=o^c;} while(!compare_exchange_weak(o, n, order, memory_order_relaxed)); + return o; + } + + build_logicops() {} + build_logicops(integral_type i) : Base(i) {} +}; + +/* +given a Base that implements: + +- load(memory_order order) +- store(integral_type i, memory_order order) +- compare_exchange_weak(integral_type &expected, integral_type desired, memory_order order) + +generates the full set of atomic operations for integral types +*/ +template +class build_atomic_from_minimal : public build_logicops< build_arithmeticops< build_fetch_add< build_exchange > > > { +public: + typedef build_logicops< build_arithmeticops< build_fetch_add< build_exchange > > > super; + typedef typename super::integral_type integral_type; + + build_atomic_from_minimal(void) {} + build_atomic_from_minimal(typename super::integral_type i) : super(i) {} +}; + +/* +given a Base that implements: + +- load(memory_order order) +- store(integral_type i, memory_order order) +- compare_exchange_weak(integral_type &expected, integral_type desired, memory_order order) +- compare_exchange_strong(integral_type &expected, integral_type desired, memory_order order) +- exchange(integral_type replacement, memory_order order) +- fetch_add_var(integral_type c, memory_order order) +- fetch_inc(memory_order order) +- fetch_dec(memory_order order) + +generates the full set of atomic operations for integral types +*/ +template +class build_atomic_from_typical : public build_logicops< build_arithmeticops< build_const_fetch_add > > { +public: + typedef build_logicops< build_arithmeticops< build_const_fetch_add > > super; + typedef typename super::integral_type integral_type; + + build_atomic_from_typical(void) {} + build_atomic_from_typical(typename super::integral_type i) : super(i) {} +}; + +/* +given a Base that implements: + +- load(memory_order order) +- store(integral_type i, memory_order order) +- compare_exchange_weak(integral_type &expected, integral_type desired, memory_order order) +- compare_exchange_strong(integral_type &expected, integral_type desired, memory_order order) +- exchange(integral_type replacement, memory_order order) +- fetch_add(integral_type c, memory_order order) + +generates the full set of atomic operations for integral types +*/ +template +class build_atomic_from_add : public build_logicops< build_arithmeticops > { +public: + typedef build_logicops< build_arithmeticops > super; + typedef typename super::integral_type integral_type; + + build_atomic_from_add(void) {} + build_atomic_from_add(typename super::integral_type i) : super(i) {} +}; + +/* +given a Base that implements: + +- load(memory_order order) +- store(integral_type i, memory_order order) +- compare_exchange_weak(integral_type &expected, integral_type desired, memory_order order) +- compare_exchange_strong(integral_type &expected, integral_type desired, memory_order order) +- exchange(integral_type replacement, memory_order order) + +generates the full set of atomic operations for integral types +*/ +template +class build_atomic_from_exchange : public build_logicops< build_arithmeticops< build_fetch_add > > { +public: + typedef build_logicops< build_arithmeticops< build_fetch_add > > super; + typedef typename super::integral_type integral_type; + + build_atomic_from_exchange(void) {} + build_atomic_from_exchange(typename super::integral_type i) : super(i) {} +}; + + +/* +given a Base that implements: + +- compare_exchange_weak() + +generates load, store and compare_exchange_weak for a smaller +data type (e.g. an atomic "byte" embedded into a temporary +and properly aligned atomic "int"). +*/ +template +class build_base_from_larger_type { +public: + typedef Type integral_type; + + build_base_from_larger_type() {} + build_base_from_larger_type(integral_type t) {store(t, memory_order_relaxed);} + + integral_type load(memory_order order=memory_order_seq_cst) const volatile + { + larger_integral_type v=get_base().load(order); + return extract(v); + } + bool compare_exchange_weak(integral_type &expected, + integral_type desired, + memory_order success_order, + memory_order failure_order) volatile + { + larger_integral_type expected_; + larger_integral_type desired_; + + expected_=get_base().load(memory_order_relaxed); + expected_=insert(expected_, expected); + desired_=insert(expected_, desired); + bool success=get_base().compare_exchange_weak(expected_, desired_, success_order, failure_order); + expected=extract(expected_); + return success; + } + void store(integral_type v, + memory_order order=memory_order_seq_cst) volatile + { + larger_integral_type expected, desired; + expected=get_base().load(memory_order_relaxed); + do { + desired=insert(expected, v); + } while(!get_base().compare_exchange_weak(expected, desired, order, memory_order_relaxed)); + } + + bool is_lock_free(void) + { + return get_base().is_lock_free(); + } +private: + typedef typename Base::integral_type larger_integral_type; + + const Base &get_base(void) const volatile + { + intptr_t address=(intptr_t)this; + address&=~(sizeof(larger_integral_type)-1); + return *reinterpret_cast(address); + } + Base &get_base(void) volatile + { + intptr_t address=(intptr_t)this; + address&=~(sizeof(larger_integral_type)-1); + return *reinterpret_cast(address); + } + unsigned int get_offset(void) const volatile + { + intptr_t address=(intptr_t)this; + address&=(sizeof(larger_integral_type)-1); + return address; + } + + unsigned int get_shift(void) const volatile + { +#if defined(BOOST_LITTLE_ENDIAN) + return get_offset()*8; +#elif defined(BOOST_BIG_ENDIAN) + return (sizeof(larger_integral_type)-sizeof(integral_type)-get_offset())*8; +#else + #error "Unknown endian" +#endif + } + + integral_type extract(larger_integral_type v) const volatile + { + return v>>get_shift(); + } + + larger_integral_type insert(larger_integral_type target, integral_type source) const volatile + { + larger_integral_type tmp=source; + larger_integral_type mask=(larger_integral_type)-1; + + mask=~(mask<<(8*sizeof(integral_type))); + + mask=mask< +class build_atomic_from_larger_type : public build_atomic_from_minimal< build_base_from_larger_type > { +public: + typedef build_atomic_from_minimal< build_base_from_larger_type > super; + //typedef typename super::integral_type integral_type; + typedef Type integral_type; + + build_atomic_from_larger_type() {} + build_atomic_from_larger_type(integral_type v) : super(v) {} +}; + +} +} +} + +#endif diff --git a/3party/boost/boost/atomic/detail/fallback.hpp b/3party/boost/boost/atomic/detail/fallback.hpp new file mode 100644 index 0000000000..dc2af303a6 --- /dev/null +++ b/3party/boost/boost/atomic/detail/fallback.hpp @@ -0,0 +1,76 @@ +#ifndef BOOST_DETAIL_ATOMIC_FALLBACK_HPP +#define BOOST_DETAIL_ATOMIC_FALLBACK_HPP + +// Copyright (c) 2009 Helge Bahmann +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include +#include + +namespace boost { +namespace detail { +namespace atomic { + +template +class fallback_atomic { +public: + fallback_atomic(void) {} + explicit fallback_atomic(const T &t) {memcpy(&i, &t, sizeof(T));} + + void store(const T &t, memory_order order=memory_order_seq_cst) volatile + { + detail::spinlock_pool<0>::scoped_lock guard(const_cast(&i)); + memcpy((void*)&i, &t, sizeof(T)); + } + T load(memory_order /*order*/=memory_order_seq_cst) volatile const + { + detail::spinlock_pool<0>::scoped_lock guard(const_cast(&i)); + T tmp; + memcpy(&tmp, (T*)&i, sizeof(T)); + return tmp; + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order /*success_order*/, + memory_order /*failure_order*/) volatile + { + detail::spinlock_pool<0>::scoped_lock guard(const_cast(&i)); + if (memcmp((void*)&i, &expected, sizeof(T))==0) { + memcpy((void*)&i, &desired, sizeof(T)); + return true; + } else { + memcpy(&expected, (void*)&i, sizeof(T)); + return false; + } + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + return compare_exchange_strong(expected, desired, success_order, failure_order); + } + T exchange(T replacement, memory_order /*order*/=memory_order_seq_cst) volatile + { + detail::spinlock_pool<0>::scoped_lock guard(const_cast(&i)); + T tmp; + memcpy(&tmp, (void*)&i, sizeof(T)); + memcpy((void*)&i, &replacement, sizeof(T)); + return tmp; + } + bool is_lock_free(void) const volatile {return false;} +protected: + T i; + typedef T integral_type; +}; + +} +} +} + +#endif diff --git a/3party/boost/boost/atomic/detail/gcc-alpha.hpp b/3party/boost/boost/atomic/detail/gcc-alpha.hpp new file mode 100644 index 0000000000..2d5374f765 --- /dev/null +++ b/3party/boost/boost/atomic/detail/gcc-alpha.hpp @@ -0,0 +1,354 @@ +#ifndef BOOST_DETAIL_ATOMIC_GCC_ALPHA_HPP +#define BOOST_DETAIL_ATOMIC_GCC_ALPHA_HPP + +// Copyright (c) 2009 Helge Bahmann +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include +#include + +/* + Refer to http://h71000.www7.hp.com/doc/82final/5601/5601pro_004.html + (HP OpenVMS systems documentation) and the alpha reference manual. + */ + +/* + NB: The most natural thing would be to write the increment/decrement + operators along the following lines: + + __asm__ __volatile__( + "1: ldl_l %0,%1 \n" + "addl %0,1,%0 \n" + "stl_c %0,%1 \n" + "beq %0,1b\n" + : "=&b" (tmp) + : "m" (value) + : "cc" + ); + + However according to the comments on the HP website and matching + comments in the Linux kernel sources this defies branch prediction, + as the cpu assumes that backward branches are always taken; so + instead copy the trick from the Linux kernel, introduce a forward + branch and back again. + + I have, however, had a hard time measuring the difference between + the two versions in microbenchmarks -- I am leaving it in nevertheless + as it apparently does not hurt either. +*/ + +namespace boost { +namespace detail { +namespace atomic { + +static inline void fence_before(memory_order order) +{ + switch(order) { + case memory_order_consume: + case memory_order_release: + case memory_order_acq_rel: + case memory_order_seq_cst: + __asm__ __volatile__ ("mb" ::: "memory"); + default:; + } +} + +static inline void fence_after(memory_order order) +{ + switch(order) { + case memory_order_acquire: + case memory_order_acq_rel: + case memory_order_seq_cst: + __asm__ __volatile__ ("mb" ::: "memory"); + default:; + } +} + +template<> +inline void platform_atomic_thread_fence(memory_order order) +{ + switch(order) { + case memory_order_acquire: + case memory_order_consume: + case memory_order_release: + case memory_order_acq_rel: + case memory_order_seq_cst: + __asm__ __volatile__ ("mb" ::: "memory"); + default:; + } +} + +template +class atomic_alpha_32 { +public: + typedef T integral_type; + explicit atomic_alpha_32(T v) : i(v) {} + atomic_alpha_32() {} + T load(memory_order order=memory_order_seq_cst) const volatile + { + T v=*reinterpret_cast(&i); + fence_after(order); + return v; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + fence_before(order); + *reinterpret_cast(&i)=(int)v; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + fence_before(success_order); + int current, success; + __asm__ __volatile__( + "1: ldl_l %2, %4\n" + "cmpeq %2, %0, %3\n" + "mov %2, %0\n" + "beq %3, 3f\n" + "stl_c %1, %4\n" + "2:\n" + + ".subsection 2\n" + "3: mov %3, %1\n" + "br 2b\n" + ".previous\n" + + : "+&r" (expected), "+&r" (desired), "=&r"(current), "=&r"(success) + : "m" (i) + : + ); + if (desired) fence_after(success_order); + else fence_after(failure_order); + return desired; + } + + bool is_lock_free(void) const volatile {return true;} +protected: + inline T fetch_add_var(T c, memory_order order) volatile + { + fence_before(order); + T original, modified; + __asm__ __volatile__( + "1: ldl_l %0, %2\n" + "addl %0, %3, %1\n" + "stl_c %1, %2\n" + "beq %1, 2f\n" + + ".subsection 2\n" + "2: br 1b\n" + ".previous\n" + + : "=&r" (original), "=&r" (modified) + : "m" (i), "r" (c) + : + ); + fence_after(order); + return original; + } + inline T fetch_inc(memory_order order) volatile + { + fence_before(order); + int original, modified; + __asm__ __volatile__( + "1: ldl_l %0, %2\n" + "addl %0, 1, %1\n" + "stl_c %1, %2\n" + "beq %1, 2f\n" + + ".subsection 2\n" + "2: br 1b\n" + ".previous\n" + + : "=&r" (original), "=&r" (modified) + : "m" (i) + : + ); + fence_after(order); + return original; + } + inline T fetch_dec(memory_order order) volatile + { + fence_before(order); + int original, modified; + __asm__ __volatile__( + "1: ldl_l %0, %2\n" + "subl %0, 1, %1\n" + "stl_c %1, %2\n" + "beq %1, 2f\n" + + ".subsection 2\n" + "2: br 1b\n" + ".previous\n" + + : "=&r" (original), "=&r" (modified) + : "m" (i) + : + ); + fence_after(order); + return original; + } +private: + T i; +}; + +template +class atomic_alpha_64 { +public: + typedef T integral_type; + explicit atomic_alpha_64(T v) : i(v) {} + atomic_alpha_64() {} + T load(memory_order order=memory_order_seq_cst) const volatile + { + T v=*reinterpret_cast(&i); + fence_after(order); + return v; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + fence_before(order); + *reinterpret_cast(&i)=v; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + fence_before(success_order); + int current, success; + __asm__ __volatile__( + "1: ldq_l %2, %4\n" + "cmpeq %2, %0, %3\n" + "mov %2, %0\n" + "beq %3, 3f\n" + "stq_c %1, %4\n" + "2:\n" + + ".subsection 2\n" + "3: mov %3, %1\n" + "br 2b\n" + ".previous\n" + + : "+&r" (expected), "+&r" (desired), "=&r"(current), "=&r"(success) + : "m" (i) + : + ); + if (desired) fence_after(success_order); + else fence_after(failure_order); + return desired; + } + + bool is_lock_free(void) const volatile {return true;} +protected: + inline T fetch_add_var(T c, memory_order order) volatile + { + fence_before(order); + T original, modified; + __asm__ __volatile__( + "1: ldq_l %0, %2\n" + "addq %0, %3, %1\n" + "stq_c %1, %2\n" + "beq %1, 2f\n" + + ".subsection 2\n" + "2: br 1b\n" + ".previous\n" + + : "=&r" (original), "=&r" (modified) + : "m" (i), "r" (c) + : + ); + fence_after(order); + return original; + } + inline T fetch_inc(memory_order order) volatile + { + fence_before(order); + T original, modified; + __asm__ __volatile__( + "1: ldq_l %0, %2\n" + "addq %0, 1, %1\n" + "stq_c %1, %2\n" + "beq %1, 2f\n" + + ".subsection 2\n" + "2: br 1b\n" + ".previous\n" + + : "=&r" (original), "=&r" (modified) + : "m" (i) + : + ); + fence_after(order); + return original; + } + inline T fetch_dec(memory_order order) volatile + { + fence_before(order); + T original, modified; + __asm__ __volatile__( + "1: ldq_l %0, %2\n" + "subq %0, 1, %1\n" + "stq_c %1, %2\n" + "beq %1, 2f\n" + + ".subsection 2\n" + "2: br 1b\n" + ".previous\n" + + : "=&r" (original), "=&r" (modified) + : "m" (i) + : + ); + fence_after(order); + return original; + } +private: + T i; +}; + +template +class platform_atomic_integral : public build_atomic_from_typical > > { +public: + typedef build_atomic_from_typical > > super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class platform_atomic_integral : public build_atomic_from_typical > > { +public: + typedef build_atomic_from_typical > > super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class platform_atomic_integral: public build_atomic_from_larger_type, T> { +public: + typedef build_atomic_from_larger_type, T> super; + + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class platform_atomic_integral: public build_atomic_from_larger_type, T> { +public: + typedef build_atomic_from_larger_type, T> super; + + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +} +} +} + +#endif diff --git a/3party/boost/boost/atomic/detail/gcc-armv6+.hpp b/3party/boost/boost/atomic/detail/gcc-armv6+.hpp new file mode 100644 index 0000000000..550575a26f --- /dev/null +++ b/3party/boost/boost/atomic/detail/gcc-armv6+.hpp @@ -0,0 +1,299 @@ +#ifndef BOOST_DETAIL_ATOMIC_GCC_ARMV6P_HPP +#define BOOST_DETAIL_ATOMIC_GCC_ARMV6P_HPP + +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +// +// Copyright (c) 2009 Helge Bahmann +// Copyright (c) 2009 Phil Endecott +// ARM Code by Phil Endecott, based on other architectures. + + +#include +#include +#include + +// From the ARM Architecture Reference Manual for architecture v6: +// +// LDREX{} , [] +// Specifies the destination register for the memory word addressed by +// Specifies the register containing the address. +// +// STREX{} , , [] +// Specifies the destination register for the returned status value. +// 0 if the operation updates memory +// 1 if the operation fails to update memory +// Specifies the register containing the word to be stored to memory. +// Specifies the register containing the address. +// Rd must not be the same register is Rm or Rn. +// +// ARM v7 is like ARM v6 plus: +// There are half-word and byte versions of the LDREX and STREX instructions, +// LDREXH, LDREXB, STREXH and STREXB. +// There are also double-word versions, LDREXD and STREXD. +// (Actually it looks like these are available from version 6k onwards.) +// FIXME these are not yet used; should be mostly a matter of copy-and-paste. +// I think you can supply an immediate offset to the address. +// +// A memory barrier is effected using a "co-processor 15" instruction, +// though a separate assembler mnemonic is available for it in v7. + +namespace boost { +namespace detail { +namespace atomic { + + +// "Thumb 1" is a subset of the ARM instruction set that uses a 16-bit encoding. It +// doesn't include all instructions and in particular it doesn't include the co-processor +// instruction used for the memory barrier or the load-locked/store-conditional +// instructions. So, if we're compiling in "Thumb 1" mode, we need to wrap all of our +// asm blocks with code to temporarily change to ARM mode. +// +// You can only change between ARM and Thumb modes when branching using the bx instruction. +// bx takes an address specified in a register. The least significant bit of the address +// indicates the mode, so 1 is added to indicate that the destination code is Thumb. +// A temporary register is needed for the address and is passed as an argument to these +// macros. It must be one of the "low" registers accessible to Thumb code, specified +// usng the "l" attribute in the asm statement. +// +// Architecture v7 introduces "Thumb 2", which does include (almost?) all of the ARM +// instruction set. So in v7 we don't need to change to ARM mode; we can write "universal +// assembler" which will assemble to Thumb 2 or ARM code as appropriate. The only thing +// we need to do to make this "universal" assembler mode work is to insert "IT" instructions +// to annotate the conditional instructions. These are ignored in other modes (e.g. v6), +// so they can always be present. + +#if defined(__thumb__) && !defined(__ARM_ARCH_7A__) +// FIXME also other v7 variants. +#define BOOST_ATOMIC_ARM_ASM_START(TMPREG) "adr " #TMPREG ", 1f\n" "bx " #TMPREG "\n" ".arm\n" ".align 4\n" "1: " +#define BOOST_ATOMIC_ARM_ASM_END(TMPREG) "adr " #TMPREG ", 1f + 1\n" "bx " #TMPREG "\n" ".thumb\n" ".align 2\n" "1: " + +#else +// The tmpreg is wasted in this case, which is non-optimal. +#define BOOST_ATOMIC_ARM_ASM_START(TMPREG) +#define BOOST_ATOMIC_ARM_ASM_END(TMPREG) +#endif + + +#if defined(__ARM_ARCH_7A__) +// FIXME ditto. +#define BOOST_ATOMIC_ARM_DMB "dmb\n" +#else +#define BOOST_ATOMIC_ARM_DMB "mcr\tp15, 0, r0, c7, c10, 5\n" +#endif + +// There is also a "Data Synchronisation Barrier" DSB; this exists in v6 as another co-processor +// instruction like the above. + + +static inline void fence_before(memory_order order) +{ + // FIXME I don't understand enough about barriers to know what this should do. + switch(order) { + case memory_order_release: + case memory_order_acq_rel: + case memory_order_seq_cst: + int brtmp; + __asm__ __volatile__ ( + BOOST_ATOMIC_ARM_ASM_START(%0) + BOOST_ATOMIC_ARM_DMB + BOOST_ATOMIC_ARM_ASM_END(%0) + : "=&l" (brtmp) :: "memory" + ); + default:; + } +} + +static inline void fence_after(memory_order order) +{ + // FIXME I don't understand enough about barriers to know what this should do. + switch(order) { + case memory_order_acquire: + case memory_order_acq_rel: + case memory_order_seq_cst: + int brtmp; + __asm__ __volatile__ ( + BOOST_ATOMIC_ARM_ASM_START(%0) + BOOST_ATOMIC_ARM_DMB + BOOST_ATOMIC_ARM_ASM_END(%0) + : "=&l" (brtmp) :: "memory" + ); + case memory_order_consume: + __asm__ __volatile__ ("" ::: "memory"); + default:; + } +} + +#undef BOOST_ATOMIC_ARM_DMB + + +template +class atomic_arm_4 { +public: + typedef T integral_type; + explicit atomic_arm_4(T v) : i(v) {} + atomic_arm_4() {} + T load(memory_order order=memory_order_seq_cst) const volatile + { + T v=const_cast(i); + fence_after(order); + return v; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + fence_before(order); + const_cast(i)=v; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + fence_before(success_order); + int success; + int tmp; + __asm__ __volatile__( + BOOST_ATOMIC_ARM_ASM_START(%2) + "mov %1, #0\n" // success = 0 + "ldrex %0, [%3]\n" // expected' = *(&i) + "teq %0, %4\n" // flags = expected'==expected + "ittt eq\n" + "strexeq %2, %5, [%3]\n" // if (flags.equal) *(&i) = desired, tmp = !OK + "teqeq %2, #0\n" // if (flags.equal) flags = tmp==0 + "moveq %1, #1\n" // if (flags.equal) success = 1 + BOOST_ATOMIC_ARM_ASM_END(%2) + : "=&r" (expected), // %0 + "=&r" (success), // %1 + "=&l" (tmp) // %2 + : "r" (&i), // %3 + "r" (expected), // %4 + "r" ((int)desired) // %5 + : "cc" + ); + if (success) fence_after(success_order); + else fence_after(failure_order); + return success; + } + + bool is_lock_free(void) const volatile {return true;} +protected: + inline T fetch_add_var(T c, memory_order order) volatile + { + fence_before(order); + T original, tmp; + int tmp2; + __asm__ __volatile__( + BOOST_ATOMIC_ARM_ASM_START(%2) + "1: ldrex %0, [%3]\n" // original = *(&i) + "add %1, %0, %4\n" // tmp = original + c + "strex %2, %1, [%3]\n" // *(&i) = tmp; tmp2 = !OK + "teq %2, #0\n" // flags = tmp2==0 + "it ne\n" + "bne 1b\n" // if (!flags.equal) goto 1 + BOOST_ATOMIC_ARM_ASM_END(%2) + : "=&r" (original), // %0 + "=&r" (tmp), // %1 + "=&l" (tmp2) // %2 + : "r" (&i), // %3 + "r" (c) // %4 + : "cc" + ); + fence_after(order); + return original; + } + inline T fetch_inc(memory_order order) volatile + { + fence_before(order); + T original, tmp; + int tmp2; + __asm__ __volatile__( + BOOST_ATOMIC_ARM_ASM_START(%2) + "1: ldrex %0, [%3]\n" // original = *(&i) + "add %1, %0, #1\n" // tmp = original + 1 + "strex %2, %1, [%3]\n" // *(&i) = tmp; tmp2 = !OK + "teq %2, #0\n" // flags = tmp2==0 + "it ne\n" + "bne 1b\n" // if (!flags.equal) goto 1 + BOOST_ATOMIC_ARM_ASM_END(%2) + : "=&r" (original), // %0 + "=&r" (tmp), // %1 + "=&l" (tmp2) // %2 + : "r" (&i) // %3 + : "cc" + ); + fence_after(order); + return original; + } + inline T fetch_dec(memory_order order) volatile + { + fence_before(order); + T original, tmp; + int tmp2; + __asm__ __volatile__( + BOOST_ATOMIC_ARM_ASM_START(%2) + "1: ldrex %0, [%3]\n" // original = *(&i) + "sub %1, %0, #1\n" // tmp = original - 1 + "strex %2, %1, [%3]\n" // *(&i) = tmp; tmp2 = !OK + "teq %2, #0\n" // flags = tmp2==0 + "it ne\n" + "bne 1b\n" // if (!flags.equal) goto 1 + BOOST_ATOMIC_ARM_ASM_END(%2) + : "=&r" (original), // %0 + "=&r" (tmp), // %1 + "=&l" (tmp2) // %2 + : "r" (&i) // %3 + : "cc" + ); + fence_after(order); + return original; + } +private: + T i; +}; + + +// #ifdef _ARM_ARCH_7 +// FIXME TODO can add native byte and halfword version here + + +template +class platform_atomic_integral : public build_atomic_from_typical > > { +public: + typedef build_atomic_from_typical > > super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class platform_atomic_integral: public build_atomic_from_larger_type, T> { +public: + typedef build_atomic_from_larger_type, T> super; + + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class platform_atomic_integral: public build_atomic_from_larger_type, T> { +public: + typedef build_atomic_from_larger_type, T> super; + + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + + + +typedef build_exchange > platform_atomic_address; + +} +} +} + +#undef BOOST_ATOMIC_ARM_ASM_START +#undef BOOST_ATOMIC_ARM_ASM_END + + +#endif diff --git a/3party/boost/boost/atomic/detail/gcc-ppc.hpp b/3party/boost/boost/atomic/detail/gcc-ppc.hpp new file mode 100644 index 0000000000..99b129a873 --- /dev/null +++ b/3party/boost/boost/atomic/detail/gcc-ppc.hpp @@ -0,0 +1,351 @@ +#ifndef BOOST_DETAIL_ATOMIC_GCC_PPC_HPP +#define BOOST_DETAIL_ATOMIC_GCC_PPC_HPP + +// Copyright (c) 2009 Helge Bahmann +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include +#include + +/* + Refer to: Motorola: "Programming Environments Manual for 32-Bit + Implementations of the PowerPC Architecture", Appendix E: + "Synchronization Programming Examples" for an explanation of what is + going on here (can be found on the web at various places by the + name "MPCFPE32B.pdf", Google is your friend...) + */ + +namespace boost { +namespace detail { +namespace atomic { + +static inline void fence_before(memory_order order) +{ + switch(order) { + case memory_order_release: + case memory_order_acq_rel: +#if defined(__powerpc64__) + __asm__ __volatile__ ("lwsync" ::: "memory"); + break; +#endif + case memory_order_seq_cst: + __asm__ __volatile__ ("sync" ::: "memory"); + default:; + } +} + +/* Note on the barrier instructions used by fence_after and +atomic_thread_fence: the "isync" instruction normally does +not wait for memory-accessing operations to complete, the +"trick" is to introduce a conditional branch that formally +depends on the memory-accessing instruction -- isync waits +until the branch can be resolved and thus implicitly until +the memory access completes. + +This means that the load(memory_order_relaxed) instruction +includes this branch, even though no barrier would be required +here, but as a consequence atomic_thread_fence(memory_order_acquire) +would have to be implemented using "sync" instead of "isync". +The following simple cost-analysis provides the rationale +for this decision: + +- isync: about ~12 cycles +- sync: about ~50 cycles +- "spurious" branch after load: 1-2 cycles +- making the right decision: priceless + +*/ + +static inline void fence_after(memory_order order) +{ + switch(order) { + case memory_order_acquire: + case memory_order_acq_rel: + case memory_order_seq_cst: + __asm__ __volatile__ ("isync"); + case memory_order_consume: + __asm__ __volatile__ ("" ::: "memory"); + default:; + } +} + +template<> +inline void platform_atomic_thread_fence(memory_order order) +{ + switch(order) { + case memory_order_acquire: + __asm__ __volatile__ ("isync" ::: "memory"); + break; + case memory_order_release: + case memory_order_acq_rel: +#if defined(__powerpc64__) + __asm__ __volatile__ ("lwsync" ::: "memory"); + break; +#endif + case memory_order_seq_cst: + __asm__ __volatile__ ("sync" ::: "memory"); + default:; + } +} + + +/* note: the __asm__ constraint "b" instructs gcc to use any register +except r0; this is required because r0 is not allowed in +some places. Since I am sometimes unsure if it is allowed +or not just play it safe and avoid r0 entirely -- ppc isn't +exactly register-starved, so this really should not matter :) */ + +template +class atomic_ppc_32 { +public: + typedef T integral_type; + explicit atomic_ppc_32(T v) : i(v) {} + atomic_ppc_32() {} + T load(memory_order order=memory_order_seq_cst) const volatile + { + T v=*reinterpret_cast(&i); + __asm__ __volatile__ ( + "cmpw %0, %0\n" + "bne- 1f\n" + "1f:\n" + : "+b"(v)); + fence_after(order); + return v; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + fence_before(order); + *reinterpret_cast(&i)=v; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + fence_before(success_order); + int success; + __asm__ __volatile__( + "lwarx %0,0,%2\n" + "cmpw %0, %3\n" + "bne- 2f\n" + "stwcx. %4,0,%2\n" + "bne- 2f\n" + "addi %1,0,1\n" + "1:" + + ".subsection 2\n" + "2: addi %1,0,0\n" + "b 1b\n" + ".previous\n" + : "=&b" (expected), "=&b" (success) + : "b" (&i), "b" (expected), "b" ((int)desired) + ); + if (success) fence_after(success_order); + else fence_after(failure_order); + return success; + } + + bool is_lock_free(void) const volatile {return true;} +protected: + inline T fetch_add_var(T c, memory_order order) volatile + { + fence_before(order); + T original, tmp; + __asm__ __volatile__( + "1: lwarx %0,0,%2\n" + "add %1,%0,%3\n" + "stwcx. %1,0,%2\n" + "bne- 1b\n" + : "=&b" (original), "=&b" (tmp) + : "b" (&i), "b" (c) + : "cc"); + fence_after(order); + return original; + } + inline T fetch_inc(memory_order order) volatile + { + fence_before(order); + T original, tmp; + __asm__ __volatile__( + "1: lwarx %0,0,%2\n" + "addi %1,%0,1\n" + "stwcx. %1,0,%2\n" + "bne- 1b\n" + : "=&b" (original), "=&b" (tmp) + : "b" (&i) + : "cc"); + fence_after(order); + return original; + } + inline T fetch_dec(memory_order order) volatile + { + fence_before(order); + T original, tmp; + __asm__ __volatile__( + "1: lwarx %0,0,%2\n" + "addi %1,%0,-1\n" + "stwcx. %1,0,%2\n" + "bne- 1b\n" + : "=&b" (original), "=&b" (tmp) + : "b" (&i) + : "cc"); + fence_after(order); + return original; + } +private: + T i; +}; + +#if defined(__powerpc64__) + +#warning Untested code -- please inform me if it works + +template +class atomic_ppc_64 { +public: + typedef T integral_type; + explicit atomic_ppc_64(T v) : i(v) {} + atomic_ppc_64() {} + T load(memory_order order=memory_order_seq_cst) const volatile + { + T v=*reinterpret_cast(&i); + __asm__ __volatile__ ( + "cmpw %0, %0\n" + "bne- 1f\n" + "1f:\n" + : "+b"(v)); + fence_after(order); + return v; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + fence_before(order); + *reinterpret_cast(&i)=v; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + fence_before(success_order); + int success; + __asm__ __volatile__( + "ldarx %0,0,%2\n" + "cmpw %0, %3\n" + "bne- 2f\n" + "stdcx. %4,0,%2\n" + "bne- 2f\n" + "addi %1,0,1\n" + "1:" + + ".subsection 2\n" + "2: addi %1,0,0\n" + "b 1b\n" + ".previous\n" + : "=&b" (expected), "=&b" (success) + : "b" (&i), "b" (expected), "b" ((int)desired) + ); + if (success) fence_after(success_order); + else fence_after(failure_order); + fence_after(order); + return success; + } + + bool is_lock_free(void) const volatile {return true;} +protected: + inline T fetch_add_var(T c, memory_order order) volatile + { + fence_before(order); + T original, tmp; + __asm__ __volatile__( + "1: ldarx %0,0,%2\n" + "add %1,%0,%3\n" + "stdcx. %1,0,%2\n" + "bne- 1b\n" + : "=&b" (original), "=&b" (tmp) + : "b" (&i), "b" (c) + : "cc"); + fence_after(order); + return original; + } + inline T fetch_inc(memory_order order) volatile + { + fence_before(order); + T original, tmp; + __asm__ __volatile__( + "1: ldarx %0,0,%2\n" + "addi %1,%0,1\n" + "stdcx. %1,0,%2\n" + "bne- 1b\n" + : "=&b" (original), "=&b" (tmp) + : "b" (&i) + : "cc"); + fence_after(order); + return original; + } + inline T fetch_dec(memory_order order) volatile + { + fence_before(order); + T original, tmp; + __asm__ __volatile__( + "1: ldarx %0,0,%2\n" + "addi %1,%0,-1\n" + "stdcx. %1,0,%2\n" + "bne- 1b\n" + : "=&b" (original), "=&b" (tmp) + : "b" (&i) + : "cc"); + fence_after(order); + return original; + } +private: + T i; +}; +#endif + +template +class platform_atomic_integral : public build_atomic_from_typical > > { +public: + typedef build_atomic_from_typical > > super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class platform_atomic_integral: public build_atomic_from_larger_type, T> { +public: + typedef build_atomic_from_larger_type, T> super; + + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class platform_atomic_integral: public build_atomic_from_larger_type, T> { +public: + typedef build_atomic_from_larger_type, T> super; + + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +#if defined(__powerpc64__) +template +class platform_atomic_integral : public build_atomic_from_typical > > { +public: + typedef build_atomic_from_typical > > super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; +#endif + +} +} +} + +#endif diff --git a/3party/boost/boost/atomic/detail/gcc-x86.hpp b/3party/boost/boost/atomic/detail/gcc-x86.hpp new file mode 100644 index 0000000000..a2a63139dd --- /dev/null +++ b/3party/boost/boost/atomic/detail/gcc-x86.hpp @@ -0,0 +1,454 @@ +#ifndef BOOST_DETAIL_ATOMIC_GCC_X86_HPP +#define BOOST_DETAIL_ATOMIC_GCC_X86_HPP + +// Copyright (c) 2009 Helge Bahmann +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include +#include + +namespace boost { +namespace detail { +namespace atomic { + +static inline void fence_before(memory_order order) +{ + switch(order) { + case memory_order_consume: + case memory_order_release: + case memory_order_acq_rel: + case memory_order_seq_cst: + __asm__ __volatile__ ("" ::: "memory"); + default:; + } +} + +static inline void fence_after(memory_order order) +{ + switch(order) { + case memory_order_acquire: + case memory_order_acq_rel: + case memory_order_seq_cst: + __asm__ __volatile__ ("" ::: "memory"); + default:; + } +} + +static inline void full_fence(void) +{ +#if defined(__amd64__) + __asm__ __volatile__("mfence" ::: "memory"); +#else + /* could use mfence iff i686, but it does not appear to matter much */ + __asm__ __volatile__("lock; addl $0, (%%esp)" ::: "memory"); +#endif +} + +static inline void fence_after_load(memory_order order) +{ + switch(order) { + case memory_order_seq_cst: + full_fence(); + case memory_order_acquire: + case memory_order_acq_rel: + __asm__ __volatile__ ("" ::: "memory"); + default:; + } +} + +template<> +inline void platform_atomic_thread_fence(memory_order order) +{ + switch(order) { + case memory_order_seq_cst: + full_fence(); + case memory_order_acquire: + case memory_order_consume: + case memory_order_acq_rel: + case memory_order_release: + __asm__ __volatile__ ("" ::: "memory"); + default:; + } +} + +template +class atomic_x86_8 { +public: + explicit atomic_x86_8(T v) : i(v) {} + atomic_x86_8() {} + T load(memory_order order=memory_order_seq_cst) const volatile + { + T v=*reinterpret_cast(&i); + fence_after_load(order); + return v; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + if (order!=memory_order_seq_cst) { + fence_before(order); + *reinterpret_cast(&i)=v; + } else { + exchange(v); + } + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + fence_before(success_order); + T prev=expected; + __asm__ __volatile__("lock; cmpxchgb %1, %2\n" : "=a" (prev) : "q" (desired), "m" (i), "a" (expected) : "memory"); + bool success=(prev==expected); + if (success) fence_after(success_order); + else fence_after(failure_order); + expected=prev; + return success; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + return compare_exchange_strong(expected, desired, success_order, failure_order); + } + T exchange(T r, memory_order order=memory_order_seq_cst) volatile + { + __asm__ __volatile__("xchgb %0, %1\n" : "=q" (r) : "m"(i), "0" (r) : "memory"); + return r; + } + T fetch_add(T c, memory_order order=memory_order_seq_cst) volatile + { + __asm__ __volatile__("lock; xaddb %0, %1" : "+q" (c), "+m" (i) :: "memory"); + return c; + } + + bool is_lock_free(void) const volatile {return true;} +protected: + typedef T integral_type; +private: + T i; +}; + +template +class platform_atomic_integral : public build_atomic_from_add > { +public: + typedef build_atomic_from_add > super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class atomic_x86_16 { +public: + explicit atomic_x86_16(T v) : i(v) {} + atomic_x86_16() {} + T load(memory_order order=memory_order_seq_cst) const volatile + { + T v=*reinterpret_cast(&i); + fence_after_load(order); + return v; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + if (order!=memory_order_seq_cst) { + fence_before(order); + *reinterpret_cast(&i)=v; + } else { + exchange(v); + } + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + fence_before(success_order); + T prev=expected; + __asm__ __volatile__("lock; cmpxchgw %1, %2\n" : "=a" (prev) : "q" (desired), "m" (i), "a" (expected) : "memory"); + bool success=(prev==expected); + if (success) fence_after(success_order); + else fence_after(failure_order); + expected=prev; + return success; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + return compare_exchange_strong(expected, desired, success_order, failure_order); + } + T exchange(T r, memory_order order=memory_order_seq_cst) volatile + { + __asm__ __volatile__("xchgw %0, %1\n" : "=r" (r) : "m"(i), "0" (r) : "memory"); + return r; + } + T fetch_add(T c, memory_order order=memory_order_seq_cst) volatile + { + __asm__ __volatile__("lock; xaddw %0, %1" : "+r" (c), "+m" (i) :: "memory"); + return c; + } + + bool is_lock_free(void) const volatile {return true;} +protected: + typedef T integral_type; +private: + T i; +}; + +template +class platform_atomic_integral : public build_atomic_from_add > { +public: + typedef build_atomic_from_add > super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class atomic_x86_32 { +public: + explicit atomic_x86_32(T v) : i(v) {} + atomic_x86_32() {} + T load(memory_order order=memory_order_seq_cst) const volatile + { + T v=*reinterpret_cast(&i); + fence_after_load(order); + return v; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + if (order!=memory_order_seq_cst) { + fence_before(order); + *reinterpret_cast(&i)=v; + } else { + exchange(v); + } + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + fence_before(success_order); + T prev=expected; + __asm__ __volatile__("lock; cmpxchgl %1, %2\n" : "=a" (prev) : "q" (desired), "m" (i), "a" (expected) : "memory"); + bool success=(prev==expected); + if (success) fence_after(success_order); + else fence_after(failure_order); + expected=prev; + return success; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + return compare_exchange_strong(expected, desired, success_order, failure_order); + } + T exchange(T r, memory_order order=memory_order_seq_cst) volatile + { + __asm__ __volatile__("xchgl %0, %1\n" : "=r" (r) : "m"(i), "0" (r) : "memory"); + return r; + } + T fetch_add(T c, memory_order order=memory_order_seq_cst) volatile + { + __asm__ __volatile__("lock; xaddl %0, %1" : "+r" (c), "+m" (i) :: "memory"); + return c; + } + + bool is_lock_free(void) const volatile {return true;} +protected: + typedef T integral_type; +private: + T i; +}; + +template +class platform_atomic_integral : public build_atomic_from_add > { +public: + typedef build_atomic_from_add > super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +#if defined(__amd64__) +template +class atomic_x86_64 { +public: + explicit atomic_x86_64(T v) : i(v) {} + atomic_x86_64() {} + T load(memory_order order=memory_order_seq_cst) const volatile + { + T v=*reinterpret_cast(&i); + fence_after_load(order); + return v; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + if (order!=memory_order_seq_cst) { + fence_before(order); + *reinterpret_cast(&i)=v; + } else { + exchange(v); + } + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + fence_before(success_order); + T prev=expected; + __asm__ __volatile__("lock; cmpxchgq %1, %2\n" : "=a" (prev) : "q" (desired), "m" (i), "a" (expected) : "memory"); + bool success=(prev==expected); + if (success) fence_after(success_order); + else fence_after(failure_order); + expected=prev; + return success; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + return compare_exchange_strong(expected, desired, success_order, failure_order); + } + T exchange(T r, memory_order order=memory_order_seq_cst) volatile + { + __asm__ __volatile__("xchgq %0, %1\n" : "=r" (r) : "m"(i), "0" (r) : "memory"); + return r; + } + T fetch_add(T c, memory_order order=memory_order_seq_cst) volatile + { + __asm__ __volatile__("lock; xaddq %0, %1" : "+r" (c), "+m" (i) :: "memory"); + return c; + } + + bool is_lock_free(void) const volatile {return true;} +protected: + typedef T integral_type; +private: + T i; +} __attribute__((aligned(8))); + +#elif defined(__i686__) + +template +class atomic_x86_64 { +private: + typedef atomic_x86_64 this_type; +public: + explicit atomic_x86_64(T v) : i(v) {} + atomic_x86_64() {} + + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + long scratch; + fence_before(success_order); + T prev=expected; + /* Make sure ebx is saved and restored properly in case + this object is compiled as "position independent". Since + programmers on x86 tend to forget specifying -DPIC or + similar, always assume PIC. + + To make this work uniformly even in the non-PIC case, + setup register constraints such that ebx can not be + used by accident e.g. as base address for the variable + to be modified. Accessing "scratch" should always be okay, + as it can only be placed on the stack (and therefore + accessed through ebp or esp only). + + In theory, could push/pop ebx onto/off the stack, but movs + to a prepared stack slot turn out to be faster. */ + __asm__ __volatile__( + "movl %%ebx, %1\n" + "movl %2, %%ebx\n" + "lock; cmpxchg8b 0(%4)\n" + "movl %1, %%ebx\n" + : "=A" (prev), "=m" (scratch) + : "D" ((long)desired), "c" ((long)(desired>>32)), "S" (&i), "0" (prev) + : "memory"); + bool success=(prev==expected); + if (success) fence_after(success_order); + else fence_after(failure_order); + expected=prev; + return success; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + return compare_exchange_strong(expected, desired, success_order, failure_order); + } + T exchange(T r, memory_order order=memory_order_seq_cst) volatile + { + T prev=i; + do {} while(!compare_exchange_strong(prev, r, order, memory_order_relaxed)); + return prev; + } + + T load(memory_order order=memory_order_seq_cst) const volatile + { + /* this is a bit problematic -- there is no other + way to atomically load a 64 bit value, but of course + compare_exchange requires write access to the memory + area */ + T expected=i; + do { } while(!const_cast(this)->compare_exchange_strong(expected, expected, order, memory_order_relaxed)); + return expected; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + exchange(v, order); + } + T fetch_add(T c, memory_order order=memory_order_seq_cst) volatile + { + T expected=i, desired;; + do { + desired=expected+c; + } while(!compare_exchange_strong(expected, desired, order, memory_order_relaxed)); + return expected; + } + + bool is_lock_free(void) const volatile {return true;} +protected: + typedef T integral_type; +private: + T i; +} __attribute__((aligned(8))) ; + +#endif + +#if defined(__amd64__) || defined(__i686__) +template +class platform_atomic_integral : public build_atomic_from_add >{ +public: + typedef build_atomic_from_add > super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; +#endif + +} +} +} + +#endif diff --git a/3party/boost/boost/atomic/detail/generic-cas.hpp b/3party/boost/boost/atomic/detail/generic-cas.hpp new file mode 100644 index 0000000000..7000dd2838 --- /dev/null +++ b/3party/boost/boost/atomic/detail/generic-cas.hpp @@ -0,0 +1,192 @@ +#ifndef BOOST_DETAIL_ATOMIC_GENERIC_CAS_HPP +#define BOOST_DETAIL_ATOMIC_GENERIC_CAS_HPP + +// Copyright (c) 2009 Helge Bahmann +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include + +#include +#include +#include + +/* fallback implementation for various compilation targets; +this is *not* efficient, particularly because all operations +are fully fenced (full memory barriers before and after +each operation) */ + +#if defined(__GNUC__) + namespace boost { namespace detail { namespace atomic { + static inline int32_t + fenced_compare_exchange_strong_32(volatile int32_t *ptr, int32_t expected, int32_t desired) + { + return __sync_val_compare_and_swap_4(ptr, expected, desired); + } + #define BOOST_ATOMIC_HAVE_CAS32 1 + + #if defined(__amd64__) || defined(__i686__) + static inline int64_t + fenced_compare_exchange_strong_64(int64_t *ptr, int64_t expected, int64_t desired) + { + return __sync_val_compare_and_swap_8(ptr, expected, desired); + } + #define BOOST_ATOMIC_HAVE_CAS64 1 + #endif + }}} + +#elif defined(__ICL) || defined(_MSC_VER) + + #if defined(_MSC_VER) + #include + #include + #endif + + namespace boost { namespace detail { namespace atomic { + static inline int32_t + fenced_compare_exchange_strong(int32_t *ptr, int32_t expected, int32_t desired) + { + return _InterlockedCompareExchange(reinterpret_cast(ptr), desired, expected); + } + #define BOOST_ATOMIC_HAVE_CAS32 1 + #if defined(_WIN64) + static inline int64_t + fenced_compare_exchange_strong(int64_t *ptr, int64_t expected, int64_t desired) + { + return _InterlockedCompareExchange64(ptr, desired, expected); + } + #define BOOST_ATOMIC_HAVE_CAS64 1 + #endif + }}} + +#elif (defined(__ICC) || defined(__ECC)) + namespace boost { namespace detail { namespace atomic { + static inline int32_t + fenced_compare_exchange_strong_32(int32_t *ptr, int32_t expected, int32_t desired) + { + return _InterlockedCompareExchange((void*)ptr, desired, expected); + } + #define BOOST_ATOMIC_HAVE_CAS32 1 + #if defined(__x86_64) + static inline int64_t + fenced_compare_exchange_strong(int64_t *ptr, int64_t expected, int64_t desired) + { + return cas64(ptr, expected, desired); + } + #define BOOST_ATOMIC_HAVE_CAS64 1 + #elif defined(__ECC) //IA-64 version + static inline int64_t + fenced_compare_exchange_strong(int64_t *ptr, int64_t expected, int64_t desired) + { + return _InterlockedCompareExchange64((void*)ptr, desired, expected); + } + #define BOOST_ATOMIC_HAVE_CAS64 1 + #endif + }}} + +#elif (defined(__SUNPRO_CC) && defined(__sparc)) + #include + namespace boost { namespace detail { namespace atomic { + static inline int32_t + fenced_compare_exchange_strong_32(int32_t *ptr, int32_t expected, int32_t desired) + { + return atomic_cas_32((volatile unsigned int*)ptr, expected, desired); + } + #define BOOST_ATOMIC_HAVE_CAS32 1 + + /* FIXME: check for 64 bit mode */ + static inline int64_t + fenced_compare_exchange_strong_64(int64_t *ptr, int64_t expected, int64_t desired) + { + return atomic_cas_64((volatile unsigned long long*)ptr, expected, desired); + } + #define BOOST_ATOMIC_HAVE_CAS64 1 + }}} +#endif + + +namespace boost { namespace detail { namespace atomic { + +#ifdef BOOST_ATOMIC_HAVE_CAS32 +template +class atomic_generic_cas32 { +private: + typedef atomic_generic_cas32 this_type; +public: + explicit atomic_generic_cas32(T v) : i((int32_t)v) {} + atomic_generic_cas32() {} + T load(memory_order order=memory_order_seq_cst) const volatile + { + T expected=(T)i; + do { } while(!const_cast(this)->compare_exchange_weak(expected, expected, order, memory_order_relaxed)); + return expected; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + exchange(v); + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + T found; + found=(T)fenced_compare_exchange_strong_32(&i, (int32_t)expected, (int32_t)desired); + bool success=(found==expected); + expected=found; + return success; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + return compare_exchange_strong(expected, desired, success_order, failure_order); + } + T exchange(T r, memory_order order=memory_order_seq_cst) volatile + { + T expected=(T)i; + do { } while(!compare_exchange_weak(expected, r, order, memory_order_relaxed)); + return expected; + } + + bool is_lock_free(void) const volatile {return true;} + typedef T integral_type; +private: + mutable int32_t i; +}; + +template +class platform_atomic_integral : public build_atomic_from_exchange > { +public: + typedef build_atomic_from_exchange > super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class platform_atomic_integral: public build_atomic_from_larger_type, T> { +public: + typedef build_atomic_from_larger_type, T> super; + + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class platform_atomic_integral: public build_atomic_from_larger_type, T> { +public: + typedef build_atomic_from_larger_type, T> super; + + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; +#endif + +} } } + +#endif diff --git a/3party/boost/boost/atomic/detail/integral-casts.hpp b/3party/boost/boost/atomic/detail/integral-casts.hpp new file mode 100644 index 0000000000..e5fe6eb7fc --- /dev/null +++ b/3party/boost/boost/atomic/detail/integral-casts.hpp @@ -0,0 +1,296 @@ +#ifndef BOOST_DETAIL_ATOMIC_INTEGRAL_CASTS_HPP +#define BOOST_DETAIL_ATOMIC_INTEGRAL_CASTS_HPP + +// Copyright (c) 2009 Helge Bahmann +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include + +namespace boost { namespace detail { namespace atomic { + +template +class platform_atomic : private platform_atomic_integral { +public: + typedef platform_atomic_integral super; + typedef union { T e; uint8_t i;} conv; + + platform_atomic() {} + explicit platform_atomic(T t) : super(to_integral(t)) + { + } + + void store(T t, memory_order order=memory_order_seq_cst) volatile + { + super::store(to_integral(t), order); + } + T load(memory_order order=memory_order_seq_cst) volatile const + { + return from_integral(super::load(order)); + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + uint8_t _expected, _desired; + _expected=to_integral(expected); + _desired=to_integral(desired); + bool success=super::compare_exchange_strong(_expected, _desired, success_order, failure_order); + expected=from_integral(_expected); + return success; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + uint8_t _expected, _desired; + _expected=to_integral(expected); + _desired=to_integral(desired); + bool success=super::compare_exchange_weak(_expected, _desired, success_order, failure_order); + expected=from_integral(_expected); + return success; + } + + T exchange(T replacement, memory_order order=memory_order_seq_cst) volatile + { + return from_integral(super::exchange(to_integral(replacement), order)); + } + + operator T(void) const volatile {return load();} + T operator=(T v) volatile {store(v); return v;} + + using super::is_lock_free; +protected: + static inline uint8_t to_integral(T &t) + { + uint8_t tmp; + memcpy(&tmp, &t, sizeof(t)); + return tmp; + } + static inline T from_integral(uint8_t t) + { + T tmp; + memcpy(&tmp, &t, sizeof(t)); + return tmp; + } +}; + +template +class platform_atomic : private platform_atomic_integral { +public: + typedef platform_atomic_integral super; + typedef union { T e; uint16_t i;} conv; + + platform_atomic() {} + explicit platform_atomic(T t) : super(to_integral(t)) + { + } + + void store(T t, memory_order order=memory_order_seq_cst) volatile + { + super::store(to_integral(t), order); + } + T load(memory_order order=memory_order_seq_cst) volatile const + { + return from_integral(super::load(order)); + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + uint16_t _expected, _desired; + _expected=to_integral(expected); + _desired=to_integral(desired); + bool success=super::compare_exchange_strong(_expected, _desired, success_order, failure_order); + expected=from_integral(_expected); + return success; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + uint16_t _expected, _desired; + _expected=to_integral(expected); + _desired=to_integral(desired); + bool success=super::compare_exchange_weak(_expected, _desired, success_order, failure_order); + expected=from_integral(_expected); + return success; + } + + T exchange(T replacement, memory_order order=memory_order_seq_cst) volatile + { + return from_integral(super::exchange(to_integral(replacement), order)); + } + + operator T(void) const volatile {return load();} + T operator=(T v) volatile {store(v); return v;} + + using super::is_lock_free; +protected: + static inline uint16_t to_integral(T &t) + { + uint16_t tmp; + memcpy(&tmp, &t, sizeof(t)); + return tmp; + } + static inline T from_integral(uint16_t t) + { + T tmp; + memcpy(&tmp, &t, sizeof(t)); + return tmp; + } +}; + +template +class platform_atomic : private platform_atomic_integral { +public: + typedef platform_atomic_integral super; + typedef union { T e; uint32_t i;} conv; + + platform_atomic() {} + explicit platform_atomic(T t) : super(to_integral(t)) + { + } + + void store(T t, memory_order order=memory_order_seq_cst) volatile + { + super::store(to_integral(t), order); + } + T load(memory_order order=memory_order_seq_cst) volatile const + { + return from_integral(super::load(order)); + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + uint32_t _expected, _desired; + _expected=to_integral(expected); + _desired=to_integral(desired); + bool success=super::compare_exchange_strong(_expected, _desired, success_order, failure_order); + expected=from_integral(_expected); + return success; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + uint32_t _expected, _desired; + _expected=to_integral(expected); + _desired=to_integral(desired); + bool success=super::compare_exchange_weak(_expected, _desired, success_order, failure_order); + expected=from_integral(_expected); + return success; + } + + T exchange(T replacement, memory_order order=memory_order_seq_cst) volatile + { + return from_integral(super::exchange(to_integral(replacement), order)); + } + + operator T(void) const volatile {return load();} + T operator=(T v) volatile {store(v); return v;} + + using super::is_lock_free; +protected: + static inline uint32_t to_integral(T &t) + { + uint32_t tmp; + memcpy(&tmp, &t, sizeof(t)); + return tmp; + } + static inline T from_integral(uint32_t t) + { + T tmp; + memcpy(&tmp, &t, sizeof(t)); + return tmp; + } +}; + +template +class platform_atomic : private platform_atomic_integral { +public: + typedef platform_atomic_integral super; + typedef union { T e; uint64_t i;} conv; + + platform_atomic() {} + explicit platform_atomic(T t) : super(to_integral(t)) + { + } + + void store(T t, memory_order order=memory_order_seq_cst) volatile + { + super::store(to_integral(t), order); + } + T load(memory_order order=memory_order_seq_cst) volatile const + { + return from_integral(super::load(order)); + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + uint64_t _expected, _desired; + _expected=to_integral(expected); + _desired=to_integral(desired); + bool success=super::compare_exchange_strong(_expected, _desired, success_order, failure_order); + expected=from_integral(_expected); + return success; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + uint64_t _expected, _desired; + _expected=to_integral(expected); + _desired=to_integral(desired); + bool success=super::compare_exchange_weak(_expected, _desired, success_order, failure_order); + expected=from_integral(_expected); + return success; + } + + T exchange(T replacement, memory_order order=memory_order_seq_cst) volatile + { + return from_integral(super::exchange(to_integral(replacement), order)); + } + + operator T(void) const volatile {return load();} + T operator=(T v) volatile {store(v); return v;} + + using super::is_lock_free; +protected: + static inline uint64_t to_integral(T &t) + { + uint64_t tmp; + memcpy(&tmp, &t, sizeof(t)); + return tmp; + } + static inline T from_integral(uint64_t t) + { + T tmp; + memcpy(&tmp, &t, sizeof(t)); + return tmp; + } +}; + +} } } + +#endif diff --git a/3party/boost/boost/atomic/detail/interlocked.hpp b/3party/boost/boost/atomic/detail/interlocked.hpp new file mode 100644 index 0000000000..469e5b9b98 --- /dev/null +++ b/3party/boost/boost/atomic/detail/interlocked.hpp @@ -0,0 +1,131 @@ +#ifndef BOOST_DETAIL_ATOMIC_INTERLOCKED_HPP +#define BOOST_DETAIL_ATOMIC_INTERLOCKED_HPP + +// Copyright (c) 2009 Helge Bahmann +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include + +#include +#include + +namespace boost { +namespace detail { +namespace atomic { + +static inline void full_fence(void) +{ + long tmp; + BOOST_INTERLOCKED_EXCHANGE(&tmp, 0); +} + +template<> +inline void platform_atomic_thread_fence(memory_order order) +{ + switch(order) { + case memory_order_seq_cst: + full_fence(); + default:; + } +} + +static inline void fence_after_load(memory_order order) +{ + switch(order) { + case memory_order_seq_cst: + full_fence(); + case memory_order_acquire: + case memory_order_acq_rel: + default:; + } +} + + +template +class atomic_interlocked_32 { +public: + explicit atomic_interlocked_32(T v) : i(v) {} + atomic_interlocked_32() {} + T load(memory_order order=memory_order_seq_cst) const volatile + { + T v=*reinterpret_cast(&i); + fence_after_load(order); + return v; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + if (order!=memory_order_seq_cst) { + *reinterpret_cast(&i)=v; + } else { + exchange(v); + } + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + T prev=expected; + expected=(T)BOOST_INTERLOCKED_COMPARE_EXCHANGE((long *)(&i), (long)desired, (long)expected); + bool success=(prev==expected); + return success; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + return compare_exchange_strong(expected, desired, success_order, failure_order); + } + T exchange(T r, memory_order order=memory_order_seq_cst) volatile + { + return (T)BOOST_INTERLOCKED_EXCHANGE((long *)&i, (long)r); + } + T fetch_add(T c, memory_order order=memory_order_seq_cst) volatile + { + return (T)BOOST_INTERLOCKED_EXCHANGE_ADD((long *)&i, c); + } + + bool is_lock_free(void) const volatile {return true;} + + typedef T integral_type; +private: + T i; +}; + +template +class platform_atomic_integral : public build_atomic_from_add > { +public: + typedef build_atomic_from_add > super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class platform_atomic_integral: public build_atomic_from_larger_type, T> { +public: + typedef build_atomic_from_larger_type, T> super; + + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +template +class platform_atomic_integral: public build_atomic_from_larger_type, T> { +public: + typedef build_atomic_from_larger_type, T> super; + + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + +} +} +} + +#endif diff --git a/3party/boost/boost/atomic/detail/linux-arm.hpp b/3party/boost/boost/atomic/detail/linux-arm.hpp new file mode 100644 index 0000000000..e899bf6a51 --- /dev/null +++ b/3party/boost/boost/atomic/detail/linux-arm.hpp @@ -0,0 +1,169 @@ +#ifndef BOOST_DETAIL_ATOMIC_LINUX_ARM_HPP +#define BOOST_DETAIL_ATOMIC_LINUX_ARM_HPP + +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +// +// Copyright (c) 2009 Helge Bahmann +// Copyright (c) 2009 Phil Endecott +// ARM Code by Phil Endecott, based on other architectures. + +#include +#include +#include + +namespace boost { +namespace detail { +namespace atomic { + + +// Different ARM processors have different atomic instructions. In particular, +// architecture versions before v6 (which are still in widespread use, e.g. the +// Intel/Marvell XScale chips like the one in the NSLU2) have only atomic swap. +// On Linux the kernel provides some support that lets us abstract away from +// these differences: it provides emulated CAS and barrier functions at special +// addresses that are garaunteed not to be interrupted by the kernel. Using +// this facility is slightly slower than inline assembler would be, but much +// faster than a system call. +// +// For documentation, see arch/arm/kernel/entry-armv.S in the kernel source +// (search for "User Helpers"). + + +typedef void (kernel_dmb_t)(void); +#define BOOST_ATOMIC_KERNEL_DMB (*(kernel_dmb_t *)0xffff0fa0) + +static inline void fence_before(memory_order order) +{ + switch(order) { + // FIXME I really don't know which of these cases should call + // kernel_dmb() and which shouldn't... + case memory_order_consume: + case memory_order_release: + case memory_order_acq_rel: + case memory_order_seq_cst: + BOOST_ATOMIC_KERNEL_DMB(); + default:; + } +} + +static inline void fence_after(memory_order order) +{ + switch(order) { + // FIXME I really don't know which of these cases should call + // kernel_dmb() and which shouldn't... + case memory_order_acquire: + case memory_order_acq_rel: + case memory_order_seq_cst: + BOOST_ATOMIC_KERNEL_DMB(); + default:; + } +} + +#undef BOOST_ATOMIC_KERNEL_DMB + + +template +class atomic_linux_arm_4 { + + typedef int (kernel_cmpxchg_t)(T oldval, T newval, T *ptr); +# define BOOST_ATOMIC_KERNEL_CMPXCHG (*(kernel_cmpxchg_t *)0xffff0fc0) + // Returns 0 if *ptr was changed. + +public: + explicit atomic_linux_arm_4(T v) : i(v) {} + atomic_linux_arm_4() {} + T load(memory_order order=memory_order_seq_cst) const volatile + { + T v=const_cast(i); + fence_after(order); + return v; + } + void store(T v, memory_order order=memory_order_seq_cst) volatile + { + fence_before(order); + const_cast(i)=v; + } + bool compare_exchange_strong( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + // Aparently we can consider kernel_cmpxchg to be strong if it is retried + // by the kernel after being interrupted, which I think it is. + // Also it seems that when an ll/sc implementation is used the kernel + // loops until the store succeeds. + bool success = BOOST_ATOMIC_KERNEL_CMPXCHG(expected,desired,&i)==0; + if (!success) e = load(memory_order_relaxed); + return success; + } + bool compare_exchange_weak( + T &expected, + T desired, + memory_order success_order, + memory_order failure_order) volatile + { + return compare_exchange_strong(expected, desired, success_order, failure_order); + } + T exchange(T replacement, memory_order order=memory_order_seq_cst) volatile + { + // Copied from build_exchange. + T o=load(memory_order_relaxed); + do {} while(!compare_exchange_weak(o, replacement, order)); + return o; + // Note that ARM has an atomic swap instruction that we could use here: + // T oldval; + // asm volatile ("swp\t%0, %1, [%2]" : "=&r"(oldval) : "r" (replacement), "r" (&i) : "memory"); + // return oldval; + // This instruction is deprecated in architecture >= 6. I'm unsure how inefficient + // its implementation is on those newer architectures. I don't think this would gain + // much since exchange() is not used often. + } + + bool is_lock_free(void) const volatile {return true;} +protected: + typedef T integral_type; +private: + T i; + +# undef BOOST_ATOMIC_KERNEL_CMPXCHG + +}; + +template +class platform_atomic_integral : public build_atomic_from_exchange > { +public: + typedef build_atomic_from_exchange > super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + + +template +class platform_atomic_integral : public build_atomic_from_larger_type, T > { +public: + typedef build_atomic_from_larger_type, T> super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + + +template +class platform_atomic_integral : public build_atomic_from_larger_type, T > { +public: + typedef build_atomic_from_larger_type, T> super; + explicit platform_atomic_integral(T v) : super(v) {} + platform_atomic_integral(void) {} +}; + + +typedef atomic_linux_arm_4 platform_atomic_address; + + +} +} +} + +#endif diff --git a/3party/boost/boost/atomic/detail/valid_integral_types.hpp b/3party/boost/boost/atomic/detail/valid_integral_types.hpp new file mode 100644 index 0000000000..7b0efb1d74 --- /dev/null +++ b/3party/boost/boost/atomic/detail/valid_integral_types.hpp @@ -0,0 +1,37 @@ +#ifndef BOOST_DETAIL_ATOMIC_VALID_INTEGRAL_TYPES_HPP +#define BOOST_DETAIL_ATOMIC_VALID_INTEGRAL_TYPES_HPP + +// Copyright (c) 2009 Helge Bahmann +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include + +namespace boost { +namespace detail { +namespace atomic { + +template struct is_integral_type {typedef void test;}; + +template<> struct is_integral_type {typedef int test;}; + +template<> struct is_integral_type {typedef int test;}; +template<> struct is_integral_type {typedef int test;}; +template<> struct is_integral_type {typedef int test;}; +template<> struct is_integral_type {typedef int test;}; +template<> struct is_integral_type {typedef int test;}; +template<> struct is_integral_type {typedef int test;}; +template<> struct is_integral_type {typedef int test;}; +template<> struct is_integral_type {typedef int test;}; +#ifdef BOOST_HAS_LONG_LONG +template<> struct is_integral_type {typedef int test;}; +template<> struct is_integral_type {typedef int test;}; +#endif + +} +} +} + +#endif diff --git a/3party/boost/boost/atomic/platform.hpp b/3party/boost/boost/atomic/platform.hpp new file mode 100644 index 0000000000..eacf008840 --- /dev/null +++ b/3party/boost/boost/atomic/platform.hpp @@ -0,0 +1,42 @@ +// Copyright (c) 2009 Helge Bahmann +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include + +#if defined(__GNUC__) && (defined(__i386__) || defined(__amd64__)) + + #include + +#elif defined(__GNUC__) && defined(__alpha__) + + #include + +#elif defined(__GNUC__) && (defined(__POWERPC__) || defined(__PPC__)) + + #include + +// This list of ARM architecture versions comes from Apple's arm/arch.h header. +// I don't know how complete it is. +#elif defined(__GNUC__) && (defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) \ + || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_7A__)) + + #include + +#elif defined(__linux__) && defined(__arm__) + + #include + +#elif defined(BOOST_USE_WINDOWS_H) || defined(_WIN32_CE) || defined(BOOST_MSVC) || defined(BOOST_INTEL_WIN) || defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__) + + #include + +#else + + #warning "Using slow fallback atomic implementation" + #include + +#endif diff --git a/std/atomic.hpp b/std/atomic.hpp new file mode 100644 index 0000000000..0609bd6876 --- /dev/null +++ b/std/atomic.hpp @@ -0,0 +1,5 @@ +#pragma once + +#include + +using boost::atomic;