| File: | compiler-rt/lib/asan/asan_allocator.cpp |
| Warning: | line 214, column 18 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===-- asan_allocator.cpp ------------------------------------------------===// | |||
| 2 | // | |||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | |||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
| 6 | // | |||
| 7 | //===----------------------------------------------------------------------===// | |||
| 8 | // | |||
| 9 | // This file is a part of AddressSanitizer, an address sanity checker. | |||
| 10 | // | |||
| 11 | // Implementation of ASan's memory allocator, 2-nd version. | |||
| 12 | // This variant uses the allocator from sanitizer_common, i.e. the one shared | |||
| 13 | // with ThreadSanitizer and MemorySanitizer. | |||
| 14 | // | |||
| 15 | //===----------------------------------------------------------------------===// | |||
| 16 | ||||
| 17 | #include "asan_allocator.h" | |||
| 18 | ||||
| 19 | #include "asan_mapping.h" | |||
| 20 | #include "asan_poisoning.h" | |||
| 21 | #include "asan_report.h" | |||
| 22 | #include "asan_stack.h" | |||
| 23 | #include "asan_thread.h" | |||
| 24 | #include "lsan/lsan_common.h" | |||
| 25 | #include "sanitizer_common/sanitizer_allocator_checks.h" | |||
| 26 | #include "sanitizer_common/sanitizer_allocator_interface.h" | |||
| 27 | #include "sanitizer_common/sanitizer_errno.h" | |||
| 28 | #include "sanitizer_common/sanitizer_flags.h" | |||
| 29 | #include "sanitizer_common/sanitizer_internal_defs.h" | |||
| 30 | #include "sanitizer_common/sanitizer_list.h" | |||
| 31 | #include "sanitizer_common/sanitizer_quarantine.h" | |||
| 32 | #include "sanitizer_common/sanitizer_stackdepot.h" | |||
| 33 | ||||
| 34 | namespace __asan { | |||
| 35 | ||||
| 36 | // Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. | |||
| 37 | // We use adaptive redzones: for larger allocation larger redzones are used. | |||
| 38 | static u32 RZLog2Size(u32 rz_log) { | |||
| 39 | CHECK_LT(rz_log, 8)do { __sanitizer::u64 v1 = (__sanitizer::u64)((rz_log)); __sanitizer ::u64 v2 = (__sanitizer::u64)((8)); if (__builtin_expect(!!(! (v1 < v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 39, "(" "(rz_log)" ") " "<" " (" "(8)" ")", v1, v2); } while (false); | |||
| 40 | return 16 << rz_log; | |||
| 41 | } | |||
| 42 | ||||
| 43 | static u32 RZSize2Log(u32 rz_size) { | |||
| 44 | CHECK_GE(rz_size, 16)do { __sanitizer::u64 v1 = (__sanitizer::u64)((rz_size)); __sanitizer ::u64 v2 = (__sanitizer::u64)((16)); if (__builtin_expect(!!( !(v1 >= v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 44, "(" "(rz_size)" ") " ">=" " (" "(16)" ")", v1, v2); } while (false); | |||
| 45 | CHECK_LE(rz_size, 2048)do { __sanitizer::u64 v1 = (__sanitizer::u64)((rz_size)); __sanitizer ::u64 v2 = (__sanitizer::u64)((2048)); if (__builtin_expect(! !(!(v1 <= v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 45, "(" "(rz_size)" ") " "<=" " (" "(2048)" ")", v1, v2) ; } while (false); | |||
| 46 | CHECK(IsPowerOfTwo(rz_size))do { __sanitizer::u64 v1 = (__sanitizer::u64)((IsPowerOfTwo(rz_size ))); __sanitizer::u64 v2 = (__sanitizer::u64)(0); if (__builtin_expect (!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 46, "(" "(IsPowerOfTwo(rz_size))" ") " "!=" " (" "0" ")", v1 , v2); } while (false); | |||
| 47 | u32 res = Log2(rz_size) - 4; | |||
| 48 | CHECK_EQ(rz_size, RZLog2Size(res))do { __sanitizer::u64 v1 = (__sanitizer::u64)((rz_size)); __sanitizer ::u64 v2 = (__sanitizer::u64)((RZLog2Size(res))); if (__builtin_expect (!!(!(v1 == v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 48, "(" "(rz_size)" ") " "==" " (" "(RZLog2Size(res))" ")", v1, v2); } while (false); | |||
| 49 | return res; | |||
| 50 | } | |||
| 51 | ||||
| 52 | static AsanAllocator &get_allocator(); | |||
| 53 | ||||
| 54 | static void AtomicContextStore(volatile atomic_uint64_t *atomic_context, | |||
| 55 | u32 tid, u32 stack) { | |||
| 56 | u64 context = tid; | |||
| 57 | context <<= 32; | |||
| 58 | context += stack; | |||
| 59 | atomic_store(atomic_context, context, memory_order_relaxed); | |||
| 60 | } | |||
| 61 | ||||
| 62 | static void AtomicContextLoad(const volatile atomic_uint64_t *atomic_context, | |||
| 63 | u32 &tid, u32 &stack) { | |||
| 64 | u64 context = atomic_load(atomic_context, memory_order_relaxed); | |||
| 65 | stack = context; | |||
| 66 | context >>= 32; | |||
| 67 | tid = context; | |||
| 68 | } | |||
| 69 | ||||
| 70 | // The memory chunk allocated from the underlying allocator looks like this: | |||
| 71 | // L L L L L L H H U U U U U U R R | |||
| 72 | // L -- left redzone words (0 or more bytes) | |||
| 73 | // H -- ChunkHeader (16 bytes), which is also a part of the left redzone. | |||
| 74 | // U -- user memory. | |||
| 75 | // R -- right redzone (0 or more bytes) | |||
| 76 | // ChunkBase consists of ChunkHeader and other bytes that overlap with user | |||
| 77 | // memory. | |||
| 78 | ||||
| 79 | // If the left redzone is greater than the ChunkHeader size we store a magic | |||
| 80 | // value in the first uptr word of the memory block and store the address of | |||
| 81 | // ChunkBase in the next uptr. | |||
| 82 | // M B L L L L L L L L L H H U U U U U U | |||
| 83 | // | ^ | |||
| 84 | // ---------------------| | |||
| 85 | // M -- magic value kAllocBegMagic | |||
| 86 | // B -- address of ChunkHeader pointing to the first 'H' | |||
| 87 | ||||
| 88 | class ChunkHeader { | |||
| 89 | public: | |||
| 90 | atomic_uint8_t chunk_state; | |||
| 91 | u8 alloc_type : 2; | |||
| 92 | u8 lsan_tag : 2; | |||
| 93 | ||||
| 94 | // align < 8 -> 0 | |||
| 95 | // else -> log2(min(align, 512)) - 2 | |||
| 96 | u8 user_requested_alignment_log : 3; | |||
| 97 | ||||
| 98 | private: | |||
| 99 | u16 user_requested_size_hi; | |||
| 100 | u32 user_requested_size_lo; | |||
| 101 | atomic_uint64_t alloc_context_id; | |||
| 102 | ||||
| 103 | public: | |||
| 104 | uptr UsedSize() const { | |||
| 105 | uptr R = user_requested_size_lo; | |||
| 106 | if (sizeof(uptr) > sizeof(user_requested_size_lo)) | |||
| 107 | R += (uptr)user_requested_size_hi << (8 * sizeof(user_requested_size_lo)); | |||
| 108 | return R; | |||
| 109 | } | |||
| 110 | ||||
| 111 | void SetUsedSize(uptr size) { | |||
| 112 | user_requested_size_lo = size; | |||
| 113 | if (sizeof(uptr) > sizeof(user_requested_size_lo)) { | |||
| 114 | size >>= (8 * sizeof(user_requested_size_lo)); | |||
| 115 | user_requested_size_hi = size; | |||
| 116 | CHECK_EQ(user_requested_size_hi, size)do { __sanitizer::u64 v1 = (__sanitizer::u64)((user_requested_size_hi )); __sanitizer::u64 v2 = (__sanitizer::u64)((size)); if (__builtin_expect (!!(!(v1 == v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 116, "(" "(user_requested_size_hi)" ") " "==" " (" "(size)" ")", v1, v2); } while (false); | |||
| 117 | } | |||
| 118 | } | |||
| 119 | ||||
| 120 | void SetAllocContext(u32 tid, u32 stack) { | |||
| 121 | AtomicContextStore(&alloc_context_id, tid, stack); | |||
| 122 | } | |||
| 123 | ||||
| 124 | void GetAllocContext(u32 &tid, u32 &stack) const { | |||
| 125 | AtomicContextLoad(&alloc_context_id, tid, stack); | |||
| 126 | } | |||
| 127 | }; | |||
| 128 | ||||
| 129 | class ChunkBase : public ChunkHeader { | |||
| 130 | atomic_uint64_t free_context_id; | |||
| 131 | ||||
| 132 | public: | |||
| 133 | void SetFreeContext(u32 tid, u32 stack) { | |||
| 134 | AtomicContextStore(&free_context_id, tid, stack); | |||
| 135 | } | |||
| 136 | ||||
| 137 | void GetFreeContext(u32 &tid, u32 &stack) const { | |||
| 138 | AtomicContextLoad(&free_context_id, tid, stack); | |||
| 139 | } | |||
| 140 | }; | |||
| 141 | ||||
| 142 | static const uptr kChunkHeaderSize = sizeof(ChunkHeader); | |||
| 143 | static const uptr kChunkHeader2Size = sizeof(ChunkBase) - kChunkHeaderSize; | |||
| 144 | COMPILER_CHECK(kChunkHeaderSize == 16)static_assert(kChunkHeaderSize == 16, ""); | |||
| 145 | COMPILER_CHECK(kChunkHeader2Size <= 16)static_assert(kChunkHeader2Size <= 16, ""); | |||
| 146 | ||||
| 147 | enum { | |||
| 148 | // Either just allocated by underlying allocator, but AsanChunk is not yet | |||
| 149 | // ready, or almost returned to undelying allocator and AsanChunk is already | |||
| 150 | // meaningless. | |||
| 151 | CHUNK_INVALID = 0, | |||
| 152 | // The chunk is allocated and not yet freed. | |||
| 153 | CHUNK_ALLOCATED = 2, | |||
| 154 | // The chunk was freed and put into quarantine zone. | |||
| 155 | CHUNK_QUARANTINE = 3, | |||
| 156 | }; | |||
| 157 | ||||
| 158 | class AsanChunk : public ChunkBase { | |||
| 159 | public: | |||
| 160 | uptr Beg() { return reinterpret_cast<uptr>(this) + kChunkHeaderSize; } | |||
| 161 | bool AddrIsInside(uptr addr) { | |||
| 162 | return (addr >= Beg()) && (addr < Beg() + UsedSize()); | |||
| 163 | } | |||
| 164 | }; | |||
| 165 | ||||
| 166 | class LargeChunkHeader { | |||
| 167 | static constexpr uptr kAllocBegMagic = | |||
| 168 | FIRST_32_SECOND_64(0xCC6E96B9, 0xCC6E96B9CC6E96B9ULL)(0xCC6E96B9CC6E96B9ULL); | |||
| 169 | atomic_uintptr_t magic; | |||
| 170 | AsanChunk *chunk_header; | |||
| 171 | ||||
| 172 | public: | |||
| 173 | AsanChunk *Get() const { | |||
| 174 | return atomic_load(&magic, memory_order_acquire) == kAllocBegMagic | |||
| 175 | ? chunk_header | |||
| 176 | : nullptr; | |||
| 177 | } | |||
| 178 | ||||
| 179 | void Set(AsanChunk *p) { | |||
| 180 | if (p) { | |||
| 181 | chunk_header = p; | |||
| 182 | atomic_store(&magic, kAllocBegMagic, memory_order_release); | |||
| 183 | return; | |||
| 184 | } | |||
| 185 | ||||
| 186 | uptr old = kAllocBegMagic; | |||
| 187 | if (!atomic_compare_exchange_strong(&magic, &old, 0, | |||
| 188 | memory_order_release)) { | |||
| 189 | CHECK_EQ(old, kAllocBegMagic)do { __sanitizer::u64 v1 = (__sanitizer::u64)((old)); __sanitizer ::u64 v2 = (__sanitizer::u64)((kAllocBegMagic)); if (__builtin_expect (!!(!(v1 == v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 189, "(" "(old)" ") " "==" " (" "(kAllocBegMagic)" ")", v1, v2); } while (false); | |||
| 190 | } | |||
| 191 | } | |||
| 192 | }; | |||
| 193 | ||||
| 194 | struct QuarantineCallback { | |||
| 195 | QuarantineCallback(AllocatorCache *cache, BufferedStackTrace *stack) | |||
| 196 | : cache_(cache), | |||
| 197 | stack_(stack) { | |||
| 198 | } | |||
| 199 | ||||
| 200 | void Recycle(AsanChunk *m) { | |||
| 201 | void *p = get_allocator().GetBlockBegin(m); | |||
| 202 | if (p != m) { | |||
| ||||
| 203 | // Clear the magic value, as allocator internals may overwrite the | |||
| 204 | // contents of deallocated chunk, confusing GetAsanChunk lookup. | |||
| 205 | reinterpret_cast<LargeChunkHeader *>(p)->Set(nullptr); | |||
| 206 | } | |||
| 207 | ||||
| 208 | u8 old_chunk_state = CHUNK_QUARANTINE; | |||
| 209 | if (!atomic_compare_exchange_strong(&m->chunk_state, &old_chunk_state, | |||
| 210 | CHUNK_INVALID, memory_order_acquire)) { | |||
| 211 | CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE)do { __sanitizer::u64 v1 = (__sanitizer::u64)((old_chunk_state )); __sanitizer::u64 v2 = (__sanitizer::u64)((CHUNK_QUARANTINE )); if (__builtin_expect(!!(!(v1 == v2)), 0)) __sanitizer::CheckFailed ("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 211, "(" "(old_chunk_state)" ") " "==" " (" "(CHUNK_QUARANTINE)" ")", v1, v2); } while (false); | |||
| 212 | } | |||
| 213 | ||||
| 214 | PoisonShadow(m->Beg(), | |||
| ||||
| 215 | RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY(1ULL << kDefaultShadowScale)), | |||
| 216 | kAsanHeapLeftRedzoneMagic); | |||
| 217 | ||||
| 218 | // Statistics. | |||
| 219 | AsanStats &thread_stats = GetCurrentThreadStats(); | |||
| 220 | thread_stats.real_frees++; | |||
| 221 | thread_stats.really_freed += m->UsedSize(); | |||
| 222 | ||||
| 223 | get_allocator().Deallocate(cache_, p); | |||
| 224 | } | |||
| 225 | ||||
| 226 | void *Allocate(uptr size) { | |||
| 227 | void *res = get_allocator().Allocate(cache_, size, 1); | |||
| 228 | // TODO(alekseys): Consider making quarantine OOM-friendly. | |||
| 229 | if (UNLIKELY(!res)__builtin_expect(!!(!res), 0)) | |||
| 230 | ReportOutOfMemory(size, stack_); | |||
| 231 | return res; | |||
| 232 | } | |||
| 233 | ||||
| 234 | void Deallocate(void *p) { | |||
| 235 | get_allocator().Deallocate(cache_, p); | |||
| 236 | } | |||
| 237 | ||||
| 238 | private: | |||
| 239 | AllocatorCache* const cache_; | |||
| 240 | BufferedStackTrace* const stack_; | |||
| 241 | }; | |||
| 242 | ||||
| 243 | typedef Quarantine<QuarantineCallback, AsanChunk> AsanQuarantine; | |||
| 244 | typedef AsanQuarantine::Cache QuarantineCache; | |||
| 245 | ||||
| 246 | void AsanMapUnmapCallback::OnMap(uptr p, uptr size) const { | |||
| 247 | PoisonShadow(p, size, kAsanHeapLeftRedzoneMagic); | |||
| 248 | // Statistics. | |||
| 249 | AsanStats &thread_stats = GetCurrentThreadStats(); | |||
| 250 | thread_stats.mmaps++; | |||
| 251 | thread_stats.mmaped += size; | |||
| 252 | } | |||
| 253 | void AsanMapUnmapCallback::OnUnmap(uptr p, uptr size) const { | |||
| 254 | PoisonShadow(p, size, 0); | |||
| 255 | // We are about to unmap a chunk of user memory. | |||
| 256 | // Mark the corresponding shadow memory as not needed. | |||
| 257 | FlushUnneededASanShadowMemory(p, size); | |||
| 258 | // Statistics. | |||
| 259 | AsanStats &thread_stats = GetCurrentThreadStats(); | |||
| 260 | thread_stats.munmaps++; | |||
| 261 | thread_stats.munmaped += size; | |||
| 262 | } | |||
| 263 | ||||
| 264 | // We can not use THREADLOCAL because it is not supported on some of the | |||
| 265 | // platforms we care about (OSX 10.6, Android). | |||
| 266 | // static THREADLOCAL AllocatorCache cache; | |||
| 267 | AllocatorCache *GetAllocatorCache(AsanThreadLocalMallocStorage *ms) { | |||
| 268 | CHECK(ms)do { __sanitizer::u64 v1 = (__sanitizer::u64)((ms)); __sanitizer ::u64 v2 = (__sanitizer::u64)(0); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 268, "(" "(ms)" ") " "!=" " (" "0" ")", v1, v2); } while (false ); | |||
| 269 | return &ms->allocator_cache; | |||
| 270 | } | |||
| 271 | ||||
| 272 | QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) { | |||
| 273 | CHECK(ms)do { __sanitizer::u64 v1 = (__sanitizer::u64)((ms)); __sanitizer ::u64 v2 = (__sanitizer::u64)(0); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 273, "(" "(ms)" ") " "!=" " (" "0" ")", v1, v2); } while (false ); | |||
| 274 | CHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache))do { __sanitizer::u64 v1 = (__sanitizer::u64)((sizeof(QuarantineCache ))); __sanitizer::u64 v2 = (__sanitizer::u64)((sizeof(ms-> quarantine_cache))); if (__builtin_expect(!!(!(v1 <= v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 274, "(" "(sizeof(QuarantineCache))" ") " "<=" " (" "(sizeof(ms->quarantine_cache))" ")", v1, v2); } while (false); | |||
| 275 | return reinterpret_cast<QuarantineCache *>(ms->quarantine_cache); | |||
| 276 | } | |||
| 277 | ||||
| 278 | void AllocatorOptions::SetFrom(const Flags *f, const CommonFlags *cf) { | |||
| 279 | quarantine_size_mb = f->quarantine_size_mb; | |||
| 280 | thread_local_quarantine_size_kb = f->thread_local_quarantine_size_kb; | |||
| 281 | min_redzone = f->redzone; | |||
| 282 | max_redzone = f->max_redzone; | |||
| 283 | may_return_null = cf->allocator_may_return_null; | |||
| 284 | alloc_dealloc_mismatch = f->alloc_dealloc_mismatch; | |||
| 285 | release_to_os_interval_ms = cf->allocator_release_to_os_interval_ms; | |||
| 286 | } | |||
| 287 | ||||
| 288 | void AllocatorOptions::CopyTo(Flags *f, CommonFlags *cf) { | |||
| 289 | f->quarantine_size_mb = quarantine_size_mb; | |||
| 290 | f->thread_local_quarantine_size_kb = thread_local_quarantine_size_kb; | |||
| 291 | f->redzone = min_redzone; | |||
| 292 | f->max_redzone = max_redzone; | |||
| 293 | cf->allocator_may_return_null = may_return_null; | |||
| 294 | f->alloc_dealloc_mismatch = alloc_dealloc_mismatch; | |||
| 295 | cf->allocator_release_to_os_interval_ms = release_to_os_interval_ms; | |||
| 296 | } | |||
| 297 | ||||
| 298 | struct Allocator { | |||
| 299 | static const uptr kMaxAllowedMallocSize = | |||
| 300 | FIRST_32_SECOND_64(3UL << 30, 1ULL << 40)(1ULL << 40); | |||
| 301 | ||||
| 302 | AsanAllocator allocator; | |||
| 303 | AsanQuarantine quarantine; | |||
| 304 | StaticSpinMutex fallback_mutex; | |||
| 305 | AllocatorCache fallback_allocator_cache; | |||
| 306 | QuarantineCache fallback_quarantine_cache; | |||
| 307 | ||||
| 308 | uptr max_user_defined_malloc_size; | |||
| 309 | atomic_uint8_t rss_limit_exceeded; | |||
| 310 | ||||
| 311 | // ------------------- Options -------------------------- | |||
| 312 | atomic_uint16_t min_redzone; | |||
| 313 | atomic_uint16_t max_redzone; | |||
| 314 | atomic_uint8_t alloc_dealloc_mismatch; | |||
| 315 | ||||
| 316 | // ------------------- Initialization ------------------------ | |||
| 317 | explicit Allocator(LinkerInitialized) | |||
| 318 | : quarantine(LINKER_INITIALIZED), | |||
| 319 | fallback_quarantine_cache(LINKER_INITIALIZED) {} | |||
| 320 | ||||
| 321 | void CheckOptions(const AllocatorOptions &options) const { | |||
| 322 | CHECK_GE(options.min_redzone, 16)do { __sanitizer::u64 v1 = (__sanitizer::u64)((options.min_redzone )); __sanitizer::u64 v2 = (__sanitizer::u64)((16)); if (__builtin_expect (!!(!(v1 >= v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 322, "(" "(options.min_redzone)" ") " ">=" " (" "(16)" ")" , v1, v2); } while (false); | |||
| 323 | CHECK_GE(options.max_redzone, options.min_redzone)do { __sanitizer::u64 v1 = (__sanitizer::u64)((options.max_redzone )); __sanitizer::u64 v2 = (__sanitizer::u64)((options.min_redzone )); if (__builtin_expect(!!(!(v1 >= v2)), 0)) __sanitizer:: CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 323, "(" "(options.max_redzone)" ") " ">=" " (" "(options.min_redzone)" ")", v1, v2); } while (false); | |||
| 324 | CHECK_LE(options.max_redzone, 2048)do { __sanitizer::u64 v1 = (__sanitizer::u64)((options.max_redzone )); __sanitizer::u64 v2 = (__sanitizer::u64)((2048)); if (__builtin_expect (!!(!(v1 <= v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 324, "(" "(options.max_redzone)" ") " "<=" " (" "(2048)" ")", v1, v2); } while (false); | |||
| 325 | CHECK(IsPowerOfTwo(options.min_redzone))do { __sanitizer::u64 v1 = (__sanitizer::u64)((IsPowerOfTwo(options .min_redzone))); __sanitizer::u64 v2 = (__sanitizer::u64)(0); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed ("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 325, "(" "(IsPowerOfTwo(options.min_redzone))" ") " "!=" " (" "0" ")", v1, v2); } while (false); | |||
| 326 | CHECK(IsPowerOfTwo(options.max_redzone))do { __sanitizer::u64 v1 = (__sanitizer::u64)((IsPowerOfTwo(options .max_redzone))); __sanitizer::u64 v2 = (__sanitizer::u64)(0); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed ("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 326, "(" "(IsPowerOfTwo(options.max_redzone))" ") " "!=" " (" "0" ")", v1, v2); } while (false); | |||
| 327 | } | |||
| 328 | ||||
| 329 | void SharedInitCode(const AllocatorOptions &options) { | |||
| 330 | CheckOptions(options); | |||
| 331 | quarantine.Init((uptr)options.quarantine_size_mb << 20, | |||
| 332 | (uptr)options.thread_local_quarantine_size_kb << 10); | |||
| 333 | atomic_store(&alloc_dealloc_mismatch, options.alloc_dealloc_mismatch, | |||
| 334 | memory_order_release); | |||
| 335 | atomic_store(&min_redzone, options.min_redzone, memory_order_release); | |||
| 336 | atomic_store(&max_redzone, options.max_redzone, memory_order_release); | |||
| 337 | } | |||
| 338 | ||||
| 339 | void InitLinkerInitialized(const AllocatorOptions &options) { | |||
| 340 | SetAllocatorMayReturnNull(options.may_return_null); | |||
| 341 | allocator.InitLinkerInitialized(options.release_to_os_interval_ms); | |||
| 342 | SharedInitCode(options); | |||
| 343 | max_user_defined_malloc_size = common_flags()->max_allocation_size_mb | |||
| 344 | ? common_flags()->max_allocation_size_mb | |||
| 345 | << 20 | |||
| 346 | : kMaxAllowedMallocSize; | |||
| 347 | } | |||
| 348 | ||||
| 349 | bool RssLimitExceeded() { | |||
| 350 | return atomic_load(&rss_limit_exceeded, memory_order_relaxed); | |||
| 351 | } | |||
| 352 | ||||
| 353 | void SetRssLimitExceeded(bool limit_exceeded) { | |||
| 354 | atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed); | |||
| 355 | } | |||
| 356 | ||||
| 357 | void RePoisonChunk(uptr chunk) { | |||
| 358 | // This could be a user-facing chunk (with redzones), or some internal | |||
| 359 | // housekeeping chunk, like TransferBatch. Start by assuming the former. | |||
| 360 | AsanChunk *ac = GetAsanChunk((void *)chunk); | |||
| 361 | uptr allocated_size = allocator.GetActuallyAllocatedSize((void *)chunk); | |||
| 362 | if (ac && atomic_load(&ac->chunk_state, memory_order_acquire) == | |||
| 363 | CHUNK_ALLOCATED) { | |||
| 364 | uptr beg = ac->Beg(); | |||
| 365 | uptr end = ac->Beg() + ac->UsedSize(); | |||
| 366 | uptr chunk_end = chunk + allocated_size; | |||
| 367 | if (chunk < beg && beg < end && end <= chunk_end) { | |||
| 368 | // Looks like a valid AsanChunk in use, poison redzones only. | |||
| 369 | PoisonShadow(chunk, beg - chunk, kAsanHeapLeftRedzoneMagic); | |||
| 370 | uptr end_aligned_down = RoundDownTo(end, SHADOW_GRANULARITY(1ULL << kDefaultShadowScale)); | |||
| 371 | FastPoisonShadowPartialRightRedzone( | |||
| 372 | end_aligned_down, end - end_aligned_down, | |||
| 373 | chunk_end - end_aligned_down, kAsanHeapLeftRedzoneMagic); | |||
| 374 | return; | |||
| 375 | } | |||
| 376 | } | |||
| 377 | ||||
| 378 | // This is either not an AsanChunk or freed or quarantined AsanChunk. | |||
| 379 | // In either case, poison everything. | |||
| 380 | PoisonShadow(chunk, allocated_size, kAsanHeapLeftRedzoneMagic); | |||
| 381 | } | |||
| 382 | ||||
| 383 | void ReInitialize(const AllocatorOptions &options) { | |||
| 384 | SetAllocatorMayReturnNull(options.may_return_null); | |||
| 385 | allocator.SetReleaseToOSIntervalMs(options.release_to_os_interval_ms); | |||
| 386 | SharedInitCode(options); | |||
| 387 | ||||
| 388 | // Poison all existing allocation's redzones. | |||
| 389 | if (CanPoisonMemory()) { | |||
| 390 | allocator.ForceLock(); | |||
| 391 | allocator.ForEachChunk( | |||
| 392 | [](uptr chunk, void *alloc) { | |||
| 393 | ((Allocator *)alloc)->RePoisonChunk(chunk); | |||
| 394 | }, | |||
| 395 | this); | |||
| 396 | allocator.ForceUnlock(); | |||
| 397 | } | |||
| 398 | } | |||
| 399 | ||||
| 400 | void GetOptions(AllocatorOptions *options) const { | |||
| 401 | options->quarantine_size_mb = quarantine.GetSize() >> 20; | |||
| 402 | options->thread_local_quarantine_size_kb = quarantine.GetCacheSize() >> 10; | |||
| 403 | options->min_redzone = atomic_load(&min_redzone, memory_order_acquire); | |||
| 404 | options->max_redzone = atomic_load(&max_redzone, memory_order_acquire); | |||
| 405 | options->may_return_null = AllocatorMayReturnNull(); | |||
| 406 | options->alloc_dealloc_mismatch = | |||
| 407 | atomic_load(&alloc_dealloc_mismatch, memory_order_acquire); | |||
| 408 | options->release_to_os_interval_ms = allocator.ReleaseToOSIntervalMs(); | |||
| 409 | } | |||
| 410 | ||||
| 411 | // -------------------- Helper methods. ------------------------- | |||
| 412 | uptr ComputeRZLog(uptr user_requested_size) { | |||
| 413 | u32 rz_log = user_requested_size <= 64 - 16 ? 0 | |||
| 414 | : user_requested_size <= 128 - 32 ? 1 | |||
| 415 | : user_requested_size <= 512 - 64 ? 2 | |||
| 416 | : user_requested_size <= 4096 - 128 ? 3 | |||
| 417 | : user_requested_size <= (1 << 14) - 256 ? 4 | |||
| 418 | : user_requested_size <= (1 << 15) - 512 ? 5 | |||
| 419 | : user_requested_size <= (1 << 16) - 1024 ? 6 | |||
| 420 | : 7; | |||
| 421 | u32 hdr_log = RZSize2Log(RoundUpToPowerOfTwo(sizeof(ChunkHeader))); | |||
| 422 | u32 min_log = RZSize2Log(atomic_load(&min_redzone, memory_order_acquire)); | |||
| 423 | u32 max_log = RZSize2Log(atomic_load(&max_redzone, memory_order_acquire)); | |||
| 424 | return Min(Max(rz_log, Max(min_log, hdr_log)), Max(max_log, hdr_log)); | |||
| 425 | } | |||
| 426 | ||||
| 427 | static uptr ComputeUserRequestedAlignmentLog(uptr user_requested_alignment) { | |||
| 428 | if (user_requested_alignment < 8) | |||
| 429 | return 0; | |||
| 430 | if (user_requested_alignment > 512) | |||
| 431 | user_requested_alignment = 512; | |||
| 432 | return Log2(user_requested_alignment) - 2; | |||
| 433 | } | |||
| 434 | ||||
| 435 | static uptr ComputeUserAlignment(uptr user_requested_alignment_log) { | |||
| 436 | if (user_requested_alignment_log == 0) | |||
| 437 | return 0; | |||
| 438 | return 1LL << (user_requested_alignment_log + 2); | |||
| 439 | } | |||
| 440 | ||||
| 441 | // We have an address between two chunks, and we want to report just one. | |||
| 442 | AsanChunk *ChooseChunk(uptr addr, AsanChunk *left_chunk, | |||
| 443 | AsanChunk *right_chunk) { | |||
| 444 | if (!left_chunk) | |||
| 445 | return right_chunk; | |||
| 446 | if (!right_chunk) | |||
| 447 | return left_chunk; | |||
| 448 | // Prefer an allocated chunk over freed chunk and freed chunk | |||
| 449 | // over available chunk. | |||
| 450 | u8 left_state = atomic_load(&left_chunk->chunk_state, memory_order_relaxed); | |||
| 451 | u8 right_state = | |||
| 452 | atomic_load(&right_chunk->chunk_state, memory_order_relaxed); | |||
| 453 | if (left_state != right_state) { | |||
| 454 | if (left_state == CHUNK_ALLOCATED) | |||
| 455 | return left_chunk; | |||
| 456 | if (right_state == CHUNK_ALLOCATED) | |||
| 457 | return right_chunk; | |||
| 458 | if (left_state == CHUNK_QUARANTINE) | |||
| 459 | return left_chunk; | |||
| 460 | if (right_state == CHUNK_QUARANTINE) | |||
| 461 | return right_chunk; | |||
| 462 | } | |||
| 463 | // Same chunk_state: choose based on offset. | |||
| 464 | sptr l_offset = 0, r_offset = 0; | |||
| 465 | CHECK(AsanChunkView(left_chunk).AddrIsAtRight(addr, 1, &l_offset))do { __sanitizer::u64 v1 = (__sanitizer::u64)((AsanChunkView( left_chunk).AddrIsAtRight(addr, 1, &l_offset))); __sanitizer ::u64 v2 = (__sanitizer::u64)(0); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 465, "(" "(AsanChunkView(left_chunk).AddrIsAtRight(addr, 1, &l_offset))" ") " "!=" " (" "0" ")", v1, v2); } while (false); | |||
| 466 | CHECK(AsanChunkView(right_chunk).AddrIsAtLeft(addr, 1, &r_offset))do { __sanitizer::u64 v1 = (__sanitizer::u64)((AsanChunkView( right_chunk).AddrIsAtLeft(addr, 1, &r_offset))); __sanitizer ::u64 v2 = (__sanitizer::u64)(0); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 466, "(" "(AsanChunkView(right_chunk).AddrIsAtLeft(addr, 1, &r_offset))" ") " "!=" " (" "0" ")", v1, v2); } while (false); | |||
| 467 | if (l_offset < r_offset) | |||
| 468 | return left_chunk; | |||
| 469 | return right_chunk; | |||
| 470 | } | |||
| 471 | ||||
| 472 | bool UpdateAllocationStack(uptr addr, BufferedStackTrace *stack) { | |||
| 473 | AsanChunk *m = GetAsanChunkByAddr(addr); | |||
| 474 | if (!m) return false; | |||
| 475 | if (atomic_load(&m->chunk_state, memory_order_acquire) != CHUNK_ALLOCATED) | |||
| 476 | return false; | |||
| 477 | if (m->Beg() != addr) return false; | |||
| 478 | AsanThread *t = GetCurrentThread(); | |||
| 479 | m->SetAllocContext(t ? t->tid() : kMainTid, StackDepotPut(*stack)); | |||
| 480 | return true; | |||
| 481 | } | |||
| 482 | ||||
| 483 | // -------------------- Allocation/Deallocation routines --------------- | |||
| 484 | void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack, | |||
| 485 | AllocType alloc_type, bool can_fill) { | |||
| 486 | if (UNLIKELY(!asan_inited)__builtin_expect(!!(!asan_inited), 0)) | |||
| 487 | AsanInitFromRtl(); | |||
| 488 | if (RssLimitExceeded()) { | |||
| 489 | if (AllocatorMayReturnNull()) | |||
| 490 | return nullptr; | |||
| 491 | ReportRssLimitExceeded(stack); | |||
| 492 | } | |||
| 493 | Flags &fl = *flags(); | |||
| 494 | CHECK(stack)do { __sanitizer::u64 v1 = (__sanitizer::u64)((stack)); __sanitizer ::u64 v2 = (__sanitizer::u64)(0); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 494, "(" "(stack)" ") " "!=" " (" "0" ")", v1, v2); } while (false); | |||
| 495 | const uptr min_alignment = SHADOW_GRANULARITY(1ULL << kDefaultShadowScale); | |||
| 496 | const uptr user_requested_alignment_log = | |||
| 497 | ComputeUserRequestedAlignmentLog(alignment); | |||
| 498 | if (alignment < min_alignment) | |||
| 499 | alignment = min_alignment; | |||
| 500 | if (size == 0) { | |||
| 501 | // We'd be happy to avoid allocating memory for zero-size requests, but | |||
| 502 | // some programs/tests depend on this behavior and assume that malloc | |||
| 503 | // would not return NULL even for zero-size allocations. Moreover, it | |||
| 504 | // looks like operator new should never return NULL, and results of | |||
| 505 | // consecutive "new" calls must be different even if the allocated size | |||
| 506 | // is zero. | |||
| 507 | size = 1; | |||
| 508 | } | |||
| 509 | CHECK(IsPowerOfTwo(alignment))do { __sanitizer::u64 v1 = (__sanitizer::u64)((IsPowerOfTwo(alignment ))); __sanitizer::u64 v2 = (__sanitizer::u64)(0); if (__builtin_expect (!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 509, "(" "(IsPowerOfTwo(alignment))" ") " "!=" " (" "0" ")" , v1, v2); } while (false); | |||
| 510 | uptr rz_log = ComputeRZLog(size); | |||
| 511 | uptr rz_size = RZLog2Size(rz_log); | |||
| 512 | uptr rounded_size = RoundUpTo(Max(size, kChunkHeader2Size), alignment); | |||
| 513 | uptr needed_size = rounded_size + rz_size; | |||
| 514 | if (alignment > min_alignment) | |||
| 515 | needed_size += alignment; | |||
| 516 | // If we are allocating from the secondary allocator, there will be no | |||
| 517 | // automatic right redzone, so add the right redzone manually. | |||
| 518 | if (!PrimaryAllocator::CanAllocate(needed_size, alignment)) | |||
| 519 | needed_size += rz_size; | |||
| 520 | CHECK(IsAligned(needed_size, min_alignment))do { __sanitizer::u64 v1 = (__sanitizer::u64)((IsAligned(needed_size , min_alignment))); __sanitizer::u64 v2 = (__sanitizer::u64)( 0); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed ("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 520, "(" "(IsAligned(needed_size, min_alignment))" ") " "!=" " (" "0" ")", v1, v2); } while (false); | |||
| 521 | if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize || | |||
| 522 | size > max_user_defined_malloc_size) { | |||
| 523 | if (AllocatorMayReturnNull()) { | |||
| 524 | Report("WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n", | |||
| 525 | (void*)size); | |||
| 526 | return nullptr; | |||
| 527 | } | |||
| 528 | uptr malloc_limit = | |||
| 529 | Min(kMaxAllowedMallocSize, max_user_defined_malloc_size); | |||
| 530 | ReportAllocationSizeTooBig(size, needed_size, malloc_limit, stack); | |||
| 531 | } | |||
| 532 | ||||
| 533 | AsanThread *t = GetCurrentThread(); | |||
| 534 | void *allocated; | |||
| 535 | if (t) { | |||
| 536 | AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); | |||
| 537 | allocated = allocator.Allocate(cache, needed_size, 8); | |||
| 538 | } else { | |||
| 539 | SpinMutexLock l(&fallback_mutex); | |||
| 540 | AllocatorCache *cache = &fallback_allocator_cache; | |||
| 541 | allocated = allocator.Allocate(cache, needed_size, 8); | |||
| 542 | } | |||
| 543 | if (UNLIKELY(!allocated)__builtin_expect(!!(!allocated), 0)) { | |||
| 544 | SetAllocatorOutOfMemory(); | |||
| 545 | if (AllocatorMayReturnNull()) | |||
| 546 | return nullptr; | |||
| 547 | ReportOutOfMemory(size, stack); | |||
| 548 | } | |||
| 549 | ||||
| 550 | if (*(u8 *)MEM_TO_SHADOW((uptr)allocated)((((uptr)allocated) >> kDefaultShadowScale) + (kDefaultShort64bitShadowOffset )) == 0 && CanPoisonMemory()) { | |||
| 551 | // Heap poisoning is enabled, but the allocator provides an unpoisoned | |||
| 552 | // chunk. This is possible if CanPoisonMemory() was false for some | |||
| 553 | // time, for example, due to flags()->start_disabled. | |||
| 554 | // Anyway, poison the block before using it for anything else. | |||
| 555 | uptr allocated_size = allocator.GetActuallyAllocatedSize(allocated); | |||
| 556 | PoisonShadow((uptr)allocated, allocated_size, kAsanHeapLeftRedzoneMagic); | |||
| 557 | } | |||
| 558 | ||||
| 559 | uptr alloc_beg = reinterpret_cast<uptr>(allocated); | |||
| 560 | uptr alloc_end = alloc_beg + needed_size; | |||
| 561 | uptr user_beg = alloc_beg + rz_size; | |||
| 562 | if (!IsAligned(user_beg, alignment)) | |||
| 563 | user_beg = RoundUpTo(user_beg, alignment); | |||
| 564 | uptr user_end = user_beg + size; | |||
| 565 | CHECK_LE(user_end, alloc_end)do { __sanitizer::u64 v1 = (__sanitizer::u64)((user_end)); __sanitizer ::u64 v2 = (__sanitizer::u64)((alloc_end)); if (__builtin_expect (!!(!(v1 <= v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 565, "(" "(user_end)" ") " "<=" " (" "(alloc_end)" ")", v1 , v2); } while (false); | |||
| 566 | uptr chunk_beg = user_beg - kChunkHeaderSize; | |||
| 567 | AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg); | |||
| 568 | m->alloc_type = alloc_type; | |||
| 569 | CHECK(size)do { __sanitizer::u64 v1 = (__sanitizer::u64)((size)); __sanitizer ::u64 v2 = (__sanitizer::u64)(0); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 569, "(" "(size)" ") " "!=" " (" "0" ")", v1, v2); } while ( false); | |||
| 570 | m->SetUsedSize(size); | |||
| 571 | m->user_requested_alignment_log = user_requested_alignment_log; | |||
| 572 | ||||
| 573 | m->SetAllocContext(t ? t->tid() : kMainTid, StackDepotPut(*stack)); | |||
| 574 | ||||
| 575 | uptr size_rounded_down_to_granularity = | |||
| 576 | RoundDownTo(size, SHADOW_GRANULARITY(1ULL << kDefaultShadowScale)); | |||
| 577 | // Unpoison the bulk of the memory region. | |||
| 578 | if (size_rounded_down_to_granularity) | |||
| 579 | PoisonShadow(user_beg, size_rounded_down_to_granularity, 0); | |||
| 580 | // Deal with the end of the region if size is not aligned to granularity. | |||
| 581 | if (size != size_rounded_down_to_granularity && CanPoisonMemory()) { | |||
| 582 | u8 *shadow = | |||
| 583 | (u8 *)MemToShadow(user_beg + size_rounded_down_to_granularity); | |||
| 584 | *shadow = fl.poison_partial ? (size & (SHADOW_GRANULARITY(1ULL << kDefaultShadowScale) - 1)) : 0; | |||
| 585 | } | |||
| 586 | ||||
| 587 | AsanStats &thread_stats = GetCurrentThreadStats(); | |||
| 588 | thread_stats.mallocs++; | |||
| 589 | thread_stats.malloced += size; | |||
| 590 | thread_stats.malloced_redzones += needed_size - size; | |||
| 591 | if (needed_size > SizeClassMap::kMaxSize) | |||
| 592 | thread_stats.malloc_large++; | |||
| 593 | else | |||
| 594 | thread_stats.malloced_by_size[SizeClassMap::ClassID(needed_size)]++; | |||
| 595 | ||||
| 596 | void *res = reinterpret_cast<void *>(user_beg); | |||
| 597 | if (can_fill && fl.max_malloc_fill_size) { | |||
| 598 | uptr fill_size = Min(size, (uptr)fl.max_malloc_fill_size); | |||
| 599 | REAL(memset)__interception::real_memset(res, fl.malloc_fill_byte, fill_size); | |||
| 600 | } | |||
| 601 | #if CAN_SANITIZE_LEAKS1 | |||
| 602 | m->lsan_tag = __lsan::DisabledInThisThread() ? __lsan::kIgnored | |||
| 603 | : __lsan::kDirectlyLeaked; | |||
| 604 | #endif | |||
| 605 | // Must be the last mutation of metadata in this function. | |||
| 606 | atomic_store(&m->chunk_state, CHUNK_ALLOCATED, memory_order_release); | |||
| 607 | if (alloc_beg != chunk_beg) { | |||
| 608 | CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg)do { __sanitizer::u64 v1 = (__sanitizer::u64)((alloc_beg + sizeof (LargeChunkHeader))); __sanitizer::u64 v2 = (__sanitizer::u64 )((chunk_beg)); if (__builtin_expect(!!(!(v1 <= v2)), 0)) __sanitizer ::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 608, "(" "(alloc_beg + sizeof(LargeChunkHeader))" ") " "<=" " (" "(chunk_beg)" ")", v1, v2); } while (false); | |||
| 609 | reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(m); | |||
| 610 | } | |||
| 611 | ASAN_MALLOC_HOOK(res, size)do { if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook (res, size); RunMallocHooks(res, size); } while (false); | |||
| 612 | return res; | |||
| 613 | } | |||
| 614 | ||||
| 615 | // Set quarantine flag if chunk is allocated, issue ASan error report on | |||
| 616 | // available and quarantined chunks. Return true on success, false otherwise. | |||
| 617 | bool AtomicallySetQuarantineFlagIfAllocated(AsanChunk *m, void *ptr, | |||
| 618 | BufferedStackTrace *stack) { | |||
| 619 | u8 old_chunk_state = CHUNK_ALLOCATED; | |||
| 620 | // Flip the chunk_state atomically to avoid race on double-free. | |||
| 621 | if (!atomic_compare_exchange_strong(&m->chunk_state, &old_chunk_state, | |||
| 622 | CHUNK_QUARANTINE, | |||
| 623 | memory_order_acquire)) { | |||
| 624 | ReportInvalidFree(ptr, old_chunk_state, stack); | |||
| 625 | // It's not safe to push a chunk in quarantine on invalid free. | |||
| 626 | return false; | |||
| 627 | } | |||
| 628 | CHECK_EQ(CHUNK_ALLOCATED, old_chunk_state)do { __sanitizer::u64 v1 = (__sanitizer::u64)((CHUNK_ALLOCATED )); __sanitizer::u64 v2 = (__sanitizer::u64)((old_chunk_state )); if (__builtin_expect(!!(!(v1 == v2)), 0)) __sanitizer::CheckFailed ("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 628, "(" "(CHUNK_ALLOCATED)" ") " "==" " (" "(old_chunk_state)" ")", v1, v2); } while (false); | |||
| 629 | // It was a user data. | |||
| 630 | m->SetFreeContext(kInvalidTid, 0); | |||
| 631 | return true; | |||
| 632 | } | |||
| 633 | ||||
| 634 | // Expects the chunk to already be marked as quarantined by using | |||
| 635 | // AtomicallySetQuarantineFlagIfAllocated. | |||
| 636 | void QuarantineChunk(AsanChunk *m, void *ptr, BufferedStackTrace *stack) { | |||
| 637 | CHECK_EQ(atomic_load(&m->chunk_state, memory_order_relaxed),do { __sanitizer::u64 v1 = (__sanitizer::u64)((atomic_load(& m->chunk_state, memory_order_relaxed))); __sanitizer::u64 v2 = (__sanitizer::u64)((CHUNK_QUARANTINE)); if (__builtin_expect (!!(!(v1 == v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 638, "(" "(atomic_load(&m->chunk_state, memory_order_relaxed))" ") " "==" " (" "(CHUNK_QUARANTINE)" ")", v1, v2); } while (false ) | |||
| 638 | CHUNK_QUARANTINE)do { __sanitizer::u64 v1 = (__sanitizer::u64)((atomic_load(& m->chunk_state, memory_order_relaxed))); __sanitizer::u64 v2 = (__sanitizer::u64)((CHUNK_QUARANTINE)); if (__builtin_expect (!!(!(v1 == v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 638, "(" "(atomic_load(&m->chunk_state, memory_order_relaxed))" ") " "==" " (" "(CHUNK_QUARANTINE)" ")", v1, v2); } while (false ); | |||
| 639 | AsanThread *t = GetCurrentThread(); | |||
| 640 | m->SetFreeContext(t ? t->tid() : 0, StackDepotPut(*stack)); | |||
| 641 | ||||
| 642 | Flags &fl = *flags(); | |||
| 643 | if (fl.max_free_fill_size > 0) { | |||
| 644 | // We have to skip the chunk header, it contains free_context_id. | |||
| 645 | uptr scribble_start = (uptr)m + kChunkHeaderSize + kChunkHeader2Size; | |||
| 646 | if (m->UsedSize() >= kChunkHeader2Size) { // Skip Header2 in user area. | |||
| 647 | uptr size_to_fill = m->UsedSize() - kChunkHeader2Size; | |||
| 648 | size_to_fill = Min(size_to_fill, (uptr)fl.max_free_fill_size); | |||
| 649 | REAL(memset)__interception::real_memset((void *)scribble_start, fl.free_fill_byte, size_to_fill); | |||
| 650 | } | |||
| 651 | } | |||
| 652 | ||||
| 653 | // Poison the region. | |||
| 654 | PoisonShadow(m->Beg(), | |||
| 655 | RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY(1ULL << kDefaultShadowScale)), | |||
| 656 | kAsanHeapFreeMagic); | |||
| 657 | ||||
| 658 | AsanStats &thread_stats = GetCurrentThreadStats(); | |||
| 659 | thread_stats.frees++; | |||
| 660 | thread_stats.freed += m->UsedSize(); | |||
| 661 | ||||
| 662 | // Push into quarantine. | |||
| 663 | if (t) { | |||
| 664 | AsanThreadLocalMallocStorage *ms = &t->malloc_storage(); | |||
| 665 | AllocatorCache *ac = GetAllocatorCache(ms); | |||
| 666 | quarantine.Put(GetQuarantineCache(ms), QuarantineCallback(ac, stack), m, | |||
| 667 | m->UsedSize()); | |||
| 668 | } else { | |||
| 669 | SpinMutexLock l(&fallback_mutex); | |||
| 670 | AllocatorCache *ac = &fallback_allocator_cache; | |||
| 671 | quarantine.Put(&fallback_quarantine_cache, QuarantineCallback(ac, stack), | |||
| 672 | m, m->UsedSize()); | |||
| 673 | } | |||
| 674 | } | |||
| 675 | ||||
| 676 | void Deallocate(void *ptr, uptr delete_size, uptr delete_alignment, | |||
| 677 | BufferedStackTrace *stack, AllocType alloc_type) { | |||
| 678 | uptr p = reinterpret_cast<uptr>(ptr); | |||
| 679 | if (p == 0) return; | |||
| 680 | ||||
| 681 | uptr chunk_beg = p - kChunkHeaderSize; | |||
| 682 | AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg); | |||
| 683 | ||||
| 684 | // On Windows, uninstrumented DLLs may allocate memory before ASan hooks | |||
| 685 | // malloc. Don't report an invalid free in this case. | |||
| 686 | if (SANITIZER_WINDOWS0 && | |||
| 687 | !get_allocator().PointerIsMine(ptr)) { | |||
| 688 | if (!IsSystemHeapAddress(p)) | |||
| 689 | ReportFreeNotMalloced(p, stack); | |||
| 690 | return; | |||
| 691 | } | |||
| 692 | ||||
| 693 | ASAN_FREE_HOOK(ptr)do { if (&__sanitizer_free_hook) __sanitizer_free_hook(ptr ); RunFreeHooks(ptr); } while (false); | |||
| 694 | ||||
| 695 | // Must mark the chunk as quarantined before any changes to its metadata. | |||
| 696 | // Do not quarantine given chunk if we failed to set CHUNK_QUARANTINE flag. | |||
| 697 | if (!AtomicallySetQuarantineFlagIfAllocated(m, ptr, stack)) return; | |||
| 698 | ||||
| 699 | if (m->alloc_type != alloc_type) { | |||
| 700 | if (atomic_load(&alloc_dealloc_mismatch, memory_order_acquire)) { | |||
| 701 | ReportAllocTypeMismatch((uptr)ptr, stack, (AllocType)m->alloc_type, | |||
| 702 | (AllocType)alloc_type); | |||
| 703 | } | |||
| 704 | } else { | |||
| 705 | if (flags()->new_delete_type_mismatch && | |||
| 706 | (alloc_type == FROM_NEW || alloc_type == FROM_NEW_BR) && | |||
| 707 | ((delete_size && delete_size != m->UsedSize()) || | |||
| 708 | ComputeUserRequestedAlignmentLog(delete_alignment) != | |||
| 709 | m->user_requested_alignment_log)) { | |||
| 710 | ReportNewDeleteTypeMismatch(p, delete_size, delete_alignment, stack); | |||
| 711 | } | |||
| 712 | } | |||
| 713 | ||||
| 714 | QuarantineChunk(m, ptr, stack); | |||
| 715 | } | |||
| 716 | ||||
| 717 | void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) { | |||
| 718 | CHECK(old_ptr && new_size)do { __sanitizer::u64 v1 = (__sanitizer::u64)((old_ptr && new_size)); __sanitizer::u64 v2 = (__sanitizer::u64)(0); if ( __builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed ("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 718, "(" "(old_ptr && new_size)" ") " "!=" " (" "0" ")", v1, v2); } while (false); | |||
| 719 | uptr p = reinterpret_cast<uptr>(old_ptr); | |||
| 720 | uptr chunk_beg = p - kChunkHeaderSize; | |||
| 721 | AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg); | |||
| 722 | ||||
| 723 | AsanStats &thread_stats = GetCurrentThreadStats(); | |||
| 724 | thread_stats.reallocs++; | |||
| 725 | thread_stats.realloced += new_size; | |||
| 726 | ||||
| 727 | void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, true); | |||
| 728 | if (new_ptr) { | |||
| 729 | u8 chunk_state = atomic_load(&m->chunk_state, memory_order_acquire); | |||
| 730 | if (chunk_state != CHUNK_ALLOCATED) | |||
| 731 | ReportInvalidFree(old_ptr, chunk_state, stack); | |||
| 732 | CHECK_NE(REAL(memcpy), nullptr)do { __sanitizer::u64 v1 = (__sanitizer::u64)((__interception ::real_memcpy)); __sanitizer::u64 v2 = (__sanitizer::u64)((nullptr )); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed ("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 732, "(" "(__interception::real_memcpy)" ") " "!=" " (" "(nullptr)" ")", v1, v2); } while (false); | |||
| 733 | uptr memcpy_size = Min(new_size, m->UsedSize()); | |||
| 734 | // If realloc() races with free(), we may start copying freed memory. | |||
| 735 | // However, we will report racy double-free later anyway. | |||
| 736 | REAL(memcpy)__interception::real_memcpy(new_ptr, old_ptr, memcpy_size); | |||
| 737 | Deallocate(old_ptr, 0, 0, stack, FROM_MALLOC); | |||
| 738 | } | |||
| 739 | return new_ptr; | |||
| 740 | } | |||
| 741 | ||||
| 742 | void *Calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { | |||
| 743 | if (UNLIKELY(CheckForCallocOverflow(size, nmemb))__builtin_expect(!!(CheckForCallocOverflow(size, nmemb)), 0)) { | |||
| 744 | if (AllocatorMayReturnNull()) | |||
| 745 | return nullptr; | |||
| 746 | ReportCallocOverflow(nmemb, size, stack); | |||
| 747 | } | |||
| 748 | void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC, false); | |||
| 749 | // If the memory comes from the secondary allocator no need to clear it | |||
| 750 | // as it comes directly from mmap. | |||
| 751 | if (ptr && allocator.FromPrimary(ptr)) | |||
| 752 | REAL(memset)__interception::real_memset(ptr, 0, nmemb * size); | |||
| 753 | return ptr; | |||
| 754 | } | |||
| 755 | ||||
| 756 | void ReportInvalidFree(void *ptr, u8 chunk_state, BufferedStackTrace *stack) { | |||
| 757 | if (chunk_state == CHUNK_QUARANTINE) | |||
| 758 | ReportDoubleFree((uptr)ptr, stack); | |||
| 759 | else | |||
| 760 | ReportFreeNotMalloced((uptr)ptr, stack); | |||
| 761 | } | |||
| 762 | ||||
| 763 | void CommitBack(AsanThreadLocalMallocStorage *ms, BufferedStackTrace *stack) { | |||
| 764 | AllocatorCache *ac = GetAllocatorCache(ms); | |||
| 765 | quarantine.Drain(GetQuarantineCache(ms), QuarantineCallback(ac, stack)); | |||
| 766 | allocator.SwallowCache(ac); | |||
| 767 | } | |||
| 768 | ||||
| 769 | // -------------------------- Chunk lookup ---------------------- | |||
| 770 | ||||
| 771 | // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg). | |||
| 772 | // Returns nullptr if AsanChunk is not yet initialized just after | |||
| 773 | // get_allocator().Allocate(), or is being destroyed just before | |||
| 774 | // get_allocator().Deallocate(). | |||
| 775 | AsanChunk *GetAsanChunk(void *alloc_beg) { | |||
| 776 | if (!alloc_beg) | |||
| 777 | return nullptr; | |||
| 778 | AsanChunk *p = reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Get(); | |||
| 779 | if (!p) { | |||
| 780 | if (!allocator.FromPrimary(alloc_beg)) | |||
| 781 | return nullptr; | |||
| 782 | p = reinterpret_cast<AsanChunk *>(alloc_beg); | |||
| 783 | } | |||
| 784 | u8 state = atomic_load(&p->chunk_state, memory_order_relaxed); | |||
| 785 | // It does not guaranty that Chunk is initialized, but it's | |||
| 786 | // definitely not for any other value. | |||
| 787 | if (state == CHUNK_ALLOCATED || state == CHUNK_QUARANTINE) | |||
| 788 | return p; | |||
| 789 | return nullptr; | |||
| 790 | } | |||
| 791 | ||||
| 792 | AsanChunk *GetAsanChunkByAddr(uptr p) { | |||
| 793 | void *alloc_beg = allocator.GetBlockBegin(reinterpret_cast<void *>(p)); | |||
| 794 | return GetAsanChunk(alloc_beg); | |||
| 795 | } | |||
| 796 | ||||
| 797 | // Allocator must be locked when this function is called. | |||
| 798 | AsanChunk *GetAsanChunkByAddrFastLocked(uptr p) { | |||
| 799 | void *alloc_beg = | |||
| 800 | allocator.GetBlockBeginFastLocked(reinterpret_cast<void *>(p)); | |||
| 801 | return GetAsanChunk(alloc_beg); | |||
| 802 | } | |||
| 803 | ||||
| 804 | uptr AllocationSize(uptr p) { | |||
| 805 | AsanChunk *m = GetAsanChunkByAddr(p); | |||
| 806 | if (!m) return 0; | |||
| 807 | if (atomic_load(&m->chunk_state, memory_order_acquire) != CHUNK_ALLOCATED) | |||
| 808 | return 0; | |||
| 809 | if (m->Beg() != p) return 0; | |||
| 810 | return m->UsedSize(); | |||
| 811 | } | |||
| 812 | ||||
| 813 | AsanChunkView FindHeapChunkByAddress(uptr addr) { | |||
| 814 | AsanChunk *m1 = GetAsanChunkByAddr(addr); | |||
| 815 | sptr offset = 0; | |||
| 816 | if (!m1 || AsanChunkView(m1).AddrIsAtLeft(addr, 1, &offset)) { | |||
| 817 | // The address is in the chunk's left redzone, so maybe it is actually | |||
| 818 | // a right buffer overflow from the other chunk to the left. | |||
| 819 | // Search a bit to the left to see if there is another chunk. | |||
| 820 | AsanChunk *m2 = nullptr; | |||
| 821 | for (uptr l = 1; l < GetPageSizeCached(); l++) { | |||
| 822 | m2 = GetAsanChunkByAddr(addr - l); | |||
| 823 | if (m2 == m1) continue; // Still the same chunk. | |||
| 824 | break; | |||
| 825 | } | |||
| 826 | if (m2 && AsanChunkView(m2).AddrIsAtRight(addr, 1, &offset)) | |||
| 827 | m1 = ChooseChunk(addr, m2, m1); | |||
| 828 | } | |||
| 829 | return AsanChunkView(m1); | |||
| 830 | } | |||
| 831 | ||||
| 832 | void Purge(BufferedStackTrace *stack) { | |||
| 833 | AsanThread *t = GetCurrentThread(); | |||
| 834 | if (t) { | |||
| 835 | AsanThreadLocalMallocStorage *ms = &t->malloc_storage(); | |||
| 836 | quarantine.DrainAndRecycle(GetQuarantineCache(ms), | |||
| 837 | QuarantineCallback(GetAllocatorCache(ms), | |||
| 838 | stack)); | |||
| 839 | } | |||
| 840 | { | |||
| 841 | SpinMutexLock l(&fallback_mutex); | |||
| 842 | quarantine.DrainAndRecycle(&fallback_quarantine_cache, | |||
| 843 | QuarantineCallback(&fallback_allocator_cache, | |||
| 844 | stack)); | |||
| 845 | } | |||
| 846 | ||||
| 847 | allocator.ForceReleaseToOS(); | |||
| 848 | } | |||
| 849 | ||||
| 850 | void PrintStats() { | |||
| 851 | allocator.PrintStats(); | |||
| 852 | quarantine.PrintStats(); | |||
| 853 | } | |||
| 854 | ||||
| 855 | void ForceLock() ACQUIRE(fallback_mutex)__attribute__((acquire_capability(fallback_mutex))) { | |||
| 856 | allocator.ForceLock(); | |||
| 857 | fallback_mutex.Lock(); | |||
| 858 | } | |||
| 859 | ||||
| 860 | void ForceUnlock() RELEASE(fallback_mutex)__attribute__((release_capability(fallback_mutex))) { | |||
| 861 | fallback_mutex.Unlock(); | |||
| 862 | allocator.ForceUnlock(); | |||
| 863 | } | |||
| 864 | }; | |||
| 865 | ||||
| 866 | static Allocator instance(LINKER_INITIALIZED); | |||
| 867 | ||||
| 868 | static AsanAllocator &get_allocator() { | |||
| 869 | return instance.allocator; | |||
| 870 | } | |||
| 871 | ||||
| 872 | bool AsanChunkView::IsValid() const { | |||
| 873 | return chunk_ && atomic_load(&chunk_->chunk_state, memory_order_relaxed) != | |||
| 874 | CHUNK_INVALID; | |||
| 875 | } | |||
| 876 | bool AsanChunkView::IsAllocated() const { | |||
| 877 | return chunk_ && atomic_load(&chunk_->chunk_state, memory_order_relaxed) == | |||
| 878 | CHUNK_ALLOCATED; | |||
| 879 | } | |||
| 880 | bool AsanChunkView::IsQuarantined() const { | |||
| 881 | return chunk_ && atomic_load(&chunk_->chunk_state, memory_order_relaxed) == | |||
| 882 | CHUNK_QUARANTINE; | |||
| 883 | } | |||
| 884 | uptr AsanChunkView::Beg() const { return chunk_->Beg(); } | |||
| 885 | uptr AsanChunkView::End() const { return Beg() + UsedSize(); } | |||
| 886 | uptr AsanChunkView::UsedSize() const { return chunk_->UsedSize(); } | |||
| 887 | u32 AsanChunkView::UserRequestedAlignment() const { | |||
| 888 | return Allocator::ComputeUserAlignment(chunk_->user_requested_alignment_log); | |||
| 889 | } | |||
| 890 | ||||
| 891 | uptr AsanChunkView::AllocTid() const { | |||
| 892 | u32 tid = 0; | |||
| 893 | u32 stack = 0; | |||
| 894 | chunk_->GetAllocContext(tid, stack); | |||
| 895 | return tid; | |||
| 896 | } | |||
| 897 | ||||
| 898 | uptr AsanChunkView::FreeTid() const { | |||
| 899 | if (!IsQuarantined()) | |||
| 900 | return kInvalidTid; | |||
| 901 | u32 tid = 0; | |||
| 902 | u32 stack = 0; | |||
| 903 | chunk_->GetFreeContext(tid, stack); | |||
| 904 | return tid; | |||
| 905 | } | |||
| 906 | ||||
| 907 | AllocType AsanChunkView::GetAllocType() const { | |||
| 908 | return (AllocType)chunk_->alloc_type; | |||
| 909 | } | |||
| 910 | ||||
| 911 | static StackTrace GetStackTraceFromId(u32 id) { | |||
| 912 | CHECK(id)do { __sanitizer::u64 v1 = (__sanitizer::u64)((id)); __sanitizer ::u64 v2 = (__sanitizer::u64)(0); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 912, "(" "(id)" ") " "!=" " (" "0" ")", v1, v2); } while (false ); | |||
| 913 | StackTrace res = StackDepotGet(id); | |||
| 914 | CHECK(res.trace)do { __sanitizer::u64 v1 = (__sanitizer::u64)((res.trace)); __sanitizer ::u64 v2 = (__sanitizer::u64)(0); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 914, "(" "(res.trace)" ") " "!=" " (" "0" ")", v1, v2); } while (false); | |||
| 915 | return res; | |||
| 916 | } | |||
| 917 | ||||
| 918 | u32 AsanChunkView::GetAllocStackId() const { | |||
| 919 | u32 tid = 0; | |||
| 920 | u32 stack = 0; | |||
| 921 | chunk_->GetAllocContext(tid, stack); | |||
| 922 | return stack; | |||
| 923 | } | |||
| 924 | ||||
| 925 | u32 AsanChunkView::GetFreeStackId() const { | |||
| 926 | if (!IsQuarantined()) | |||
| 927 | return 0; | |||
| 928 | u32 tid = 0; | |||
| 929 | u32 stack = 0; | |||
| 930 | chunk_->GetFreeContext(tid, stack); | |||
| 931 | return stack; | |||
| 932 | } | |||
| 933 | ||||
| 934 | StackTrace AsanChunkView::GetAllocStack() const { | |||
| 935 | return GetStackTraceFromId(GetAllocStackId()); | |||
| 936 | } | |||
| 937 | ||||
| 938 | StackTrace AsanChunkView::GetFreeStack() const { | |||
| 939 | return GetStackTraceFromId(GetFreeStackId()); | |||
| 940 | } | |||
| 941 | ||||
| 942 | void InitializeAllocator(const AllocatorOptions &options) { | |||
| 943 | instance.InitLinkerInitialized(options); | |||
| 944 | } | |||
| 945 | ||||
| 946 | void ReInitializeAllocator(const AllocatorOptions &options) { | |||
| 947 | instance.ReInitialize(options); | |||
| 948 | } | |||
| 949 | ||||
| 950 | void GetAllocatorOptions(AllocatorOptions *options) { | |||
| 951 | instance.GetOptions(options); | |||
| 952 | } | |||
| 953 | ||||
| 954 | AsanChunkView FindHeapChunkByAddress(uptr addr) { | |||
| 955 | return instance.FindHeapChunkByAddress(addr); | |||
| 956 | } | |||
| 957 | AsanChunkView FindHeapChunkByAllocBeg(uptr addr) { | |||
| 958 | return AsanChunkView(instance.GetAsanChunk(reinterpret_cast<void*>(addr))); | |||
| 959 | } | |||
| 960 | ||||
| 961 | void AsanThreadLocalMallocStorage::CommitBack() { | |||
| 962 | GET_STACK_TRACE_MALLOCBufferedStackTrace stack; if (GetMallocContextSize() <= 2) { stack.size = GetMallocContextSize(); if (GetMallocContextSize () > 0) { stack.top_frame_bp = (__sanitizer::uptr) __builtin_frame_address (0); stack.trace_buffer[0] = StackTrace::GetCurrentPc(); if ( GetMallocContextSize() > 1) stack.trace_buffer[1] = (__sanitizer ::uptr) __builtin_return_address(0); } } else { stack.Unwind( StackTrace::GetCurrentPc(), (__sanitizer::uptr) __builtin_frame_address (0), nullptr, common_flags()->fast_unwind_on_malloc, GetMallocContextSize ()); }; | |||
| 963 | instance.CommitBack(this, &stack); | |||
| 964 | } | |||
| 965 | ||||
| 966 | void PrintInternalAllocatorStats() { | |||
| 967 | instance.PrintStats(); | |||
| 968 | } | |||
| 969 | ||||
| 970 | void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) { | |||
| 971 | instance.Deallocate(ptr, 0, 0, stack, alloc_type); | |||
| 972 | } | |||
| 973 | ||||
| 974 | void asan_delete(void *ptr, uptr size, uptr alignment, | |||
| 975 | BufferedStackTrace *stack, AllocType alloc_type) { | |||
| 976 | instance.Deallocate(ptr, size, alignment, stack, alloc_type); | |||
| 977 | } | |||
| 978 | ||||
| 979 | void *asan_malloc(uptr size, BufferedStackTrace *stack) { | |||
| 980 | return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true)); | |||
| 981 | } | |||
| 982 | ||||
| 983 | void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { | |||
| 984 | return SetErrnoOnNull(instance.Calloc(nmemb, size, stack)); | |||
| 985 | } | |||
| 986 | ||||
| 987 | void *asan_reallocarray(void *p, uptr nmemb, uptr size, | |||
| 988 | BufferedStackTrace *stack) { | |||
| 989 | if (UNLIKELY(CheckForCallocOverflow(size, nmemb))__builtin_expect(!!(CheckForCallocOverflow(size, nmemb)), 0)) { | |||
| 990 | errno(*__errno_location()) = errno_ENOMEM12; | |||
| 991 | if (AllocatorMayReturnNull()) | |||
| 992 | return nullptr; | |||
| 993 | ReportReallocArrayOverflow(nmemb, size, stack); | |||
| 994 | } | |||
| 995 | return asan_realloc(p, nmemb * size, stack); | |||
| 996 | } | |||
| 997 | ||||
| 998 | void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) { | |||
| 999 | if (!p) | |||
| 1000 | return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true)); | |||
| 1001 | if (size == 0) { | |||
| 1002 | if (flags()->allocator_frees_and_returns_null_on_realloc_zero) { | |||
| 1003 | instance.Deallocate(p, 0, 0, stack, FROM_MALLOC); | |||
| 1004 | return nullptr; | |||
| 1005 | } | |||
| 1006 | // Allocate a size of 1 if we shouldn't free() on Realloc to 0 | |||
| 1007 | size = 1; | |||
| 1008 | } | |||
| 1009 | return SetErrnoOnNull(instance.Reallocate(p, size, stack)); | |||
| 1010 | } | |||
| 1011 | ||||
| 1012 | void *asan_valloc(uptr size, BufferedStackTrace *stack) { | |||
| 1013 | return SetErrnoOnNull( | |||
| 1014 | instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true)); | |||
| 1015 | } | |||
| 1016 | ||||
| 1017 | void *asan_pvalloc(uptr size, BufferedStackTrace *stack) { | |||
| 1018 | uptr PageSize = GetPageSizeCached(); | |||
| 1019 | if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))__builtin_expect(!!(CheckForPvallocOverflow(size, PageSize)), 0)) { | |||
| 1020 | errno(*__errno_location()) = errno_ENOMEM12; | |||
| 1021 | if (AllocatorMayReturnNull()) | |||
| 1022 | return nullptr; | |||
| 1023 | ReportPvallocOverflow(size, stack); | |||
| 1024 | } | |||
| 1025 | // pvalloc(0) should allocate one page. | |||
| 1026 | size = size ? RoundUpTo(size, PageSize) : PageSize; | |||
| 1027 | return SetErrnoOnNull( | |||
| 1028 | instance.Allocate(size, PageSize, stack, FROM_MALLOC, true)); | |||
| 1029 | } | |||
| 1030 | ||||
| 1031 | void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack, | |||
| 1032 | AllocType alloc_type) { | |||
| 1033 | if (UNLIKELY(!IsPowerOfTwo(alignment))__builtin_expect(!!(!IsPowerOfTwo(alignment)), 0)) { | |||
| 1034 | errno(*__errno_location()) = errno_EINVAL22; | |||
| 1035 | if (AllocatorMayReturnNull()) | |||
| 1036 | return nullptr; | |||
| 1037 | ReportInvalidAllocationAlignment(alignment, stack); | |||
| 1038 | } | |||
| 1039 | return SetErrnoOnNull( | |||
| 1040 | instance.Allocate(size, alignment, stack, alloc_type, true)); | |||
| 1041 | } | |||
| 1042 | ||||
| 1043 | void *asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace *stack) { | |||
| 1044 | if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))__builtin_expect(!!(!CheckAlignedAllocAlignmentAndSize(alignment , size)), 0)) { | |||
| 1045 | errno(*__errno_location()) = errno_EINVAL22; | |||
| 1046 | if (AllocatorMayReturnNull()) | |||
| 1047 | return nullptr; | |||
| 1048 | ReportInvalidAlignedAllocAlignment(size, alignment, stack); | |||
| 1049 | } | |||
| 1050 | return SetErrnoOnNull( | |||
| 1051 | instance.Allocate(size, alignment, stack, FROM_MALLOC, true)); | |||
| 1052 | } | |||
| 1053 | ||||
| 1054 | int asan_posix_memalign(void **memptr, uptr alignment, uptr size, | |||
| 1055 | BufferedStackTrace *stack) { | |||
| 1056 | if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))__builtin_expect(!!(!CheckPosixMemalignAlignment(alignment)), 0)) { | |||
| 1057 | if (AllocatorMayReturnNull()) | |||
| 1058 | return errno_EINVAL22; | |||
| 1059 | ReportInvalidPosixMemalignAlignment(alignment, stack); | |||
| 1060 | } | |||
| 1061 | void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC, true); | |||
| 1062 | if (UNLIKELY(!ptr)__builtin_expect(!!(!ptr), 0)) | |||
| 1063 | // OOM error is already taken care of by Allocate. | |||
| 1064 | return errno_ENOMEM12; | |||
| 1065 | CHECK(IsAligned((uptr)ptr, alignment))do { __sanitizer::u64 v1 = (__sanitizer::u64)((IsAligned((uptr )ptr, alignment))); __sanitizer::u64 v2 = (__sanitizer::u64)( 0); if (__builtin_expect(!!(!(v1 != v2)), 0)) __sanitizer::CheckFailed ("/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/compiler-rt/lib/asan/asan_allocator.cpp" , 1065, "(" "(IsAligned((uptr)ptr, alignment))" ") " "!=" " (" "0" ")", v1, v2); } while (false); | |||
| 1066 | *memptr = ptr; | |||
| 1067 | return 0; | |||
| 1068 | } | |||
| 1069 | ||||
| 1070 | uptr asan_malloc_usable_size(const void *ptr, uptr pc, uptr bp) { | |||
| 1071 | if (!ptr) return 0; | |||
| 1072 | uptr usable_size = instance.AllocationSize(reinterpret_cast<uptr>(ptr)); | |||
| 1073 | if (flags()->check_malloc_usable_size && (usable_size == 0)) { | |||
| 1074 | GET_STACK_TRACE_FATAL(pc, bp)BufferedStackTrace stack; stack.Unwind(pc, bp, nullptr, common_flags ()->fast_unwind_on_fatal); | |||
| 1075 | ReportMallocUsableSizeNotOwned((uptr)ptr, &stack); | |||
| 1076 | } | |||
| 1077 | return usable_size; | |||
| 1078 | } | |||
| 1079 | ||||
| 1080 | uptr asan_mz_size(const void *ptr) { | |||
| 1081 | return instance.AllocationSize(reinterpret_cast<uptr>(ptr)); | |||
| 1082 | } | |||
| 1083 | ||||
| 1084 | void asan_mz_force_lock() NO_THREAD_SAFETY_ANALYSIS__attribute__((no_thread_safety_analysis)) { instance.ForceLock(); } | |||
| 1085 | ||||
| 1086 | void asan_mz_force_unlock() NO_THREAD_SAFETY_ANALYSIS__attribute__((no_thread_safety_analysis)) { | |||
| 1087 | instance.ForceUnlock(); | |||
| 1088 | } | |||
| 1089 | ||||
| 1090 | void AsanSoftRssLimitExceededCallback(bool limit_exceeded) { | |||
| 1091 | instance.SetRssLimitExceeded(limit_exceeded); | |||
| 1092 | } | |||
| 1093 | ||||
| 1094 | } // namespace __asan | |||
| 1095 | ||||
| 1096 | // --- Implementation of LSan-specific functions --- {{{1 | |||
| 1097 | namespace __lsan { | |||
| 1098 | void LockAllocator() { | |||
| 1099 | __asan::get_allocator().ForceLock(); | |||
| 1100 | } | |||
| 1101 | ||||
| 1102 | void UnlockAllocator() { | |||
| 1103 | __asan::get_allocator().ForceUnlock(); | |||
| 1104 | } | |||
| 1105 | ||||
| 1106 | void GetAllocatorGlobalRange(uptr *begin, uptr *end) { | |||
| 1107 | *begin = (uptr)&__asan::get_allocator(); | |||
| 1108 | *end = *begin + sizeof(__asan::get_allocator()); | |||
| 1109 | } | |||
| 1110 | ||||
| 1111 | uptr PointsIntoChunk(void *p) { | |||
| 1112 | uptr addr = reinterpret_cast<uptr>(p); | |||
| 1113 | __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(addr); | |||
| 1114 | if (!m || atomic_load(&m->chunk_state, memory_order_acquire) != | |||
| 1115 | __asan::CHUNK_ALLOCATED) | |||
| 1116 | return 0; | |||
| 1117 | uptr chunk = m->Beg(); | |||
| 1118 | if (m->AddrIsInside(addr)) | |||
| 1119 | return chunk; | |||
| 1120 | if (IsSpecialCaseOfOperatorNew0(chunk, m->UsedSize(), addr)) | |||
| 1121 | return chunk; | |||
| 1122 | return 0; | |||
| 1123 | } | |||
| 1124 | ||||
| 1125 | uptr GetUserBegin(uptr chunk) { | |||
| 1126 | __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(chunk); | |||
| 1127 | return m ? m->Beg() : 0; | |||
| 1128 | } | |||
| 1129 | ||||
| 1130 | LsanMetadata::LsanMetadata(uptr chunk) { | |||
| 1131 | metadata_ = chunk ? reinterpret_cast<void *>(chunk - __asan::kChunkHeaderSize) | |||
| 1132 | : nullptr; | |||
| 1133 | } | |||
| 1134 | ||||
| 1135 | bool LsanMetadata::allocated() const { | |||
| 1136 | if (!metadata_) | |||
| 1137 | return false; | |||
| 1138 | __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); | |||
| 1139 | return atomic_load(&m->chunk_state, memory_order_relaxed) == | |||
| 1140 | __asan::CHUNK_ALLOCATED; | |||
| 1141 | } | |||
| 1142 | ||||
| 1143 | ChunkTag LsanMetadata::tag() const { | |||
| 1144 | __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); | |||
| 1145 | return static_cast<ChunkTag>(m->lsan_tag); | |||
| 1146 | } | |||
| 1147 | ||||
| 1148 | void LsanMetadata::set_tag(ChunkTag value) { | |||
| 1149 | __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); | |||
| 1150 | m->lsan_tag = value; | |||
| 1151 | } | |||
| 1152 | ||||
| 1153 | uptr LsanMetadata::requested_size() const { | |||
| 1154 | __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); | |||
| 1155 | return m->UsedSize(); | |||
| 1156 | } | |||
| 1157 | ||||
| 1158 | u32 LsanMetadata::stack_trace_id() const { | |||
| 1159 | __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); | |||
| 1160 | u32 tid = 0; | |||
| 1161 | u32 stack = 0; | |||
| 1162 | m->GetAllocContext(tid, stack); | |||
| 1163 | return stack; | |||
| 1164 | } | |||
| 1165 | ||||
| 1166 | void ForEachChunk(ForEachChunkCallback callback, void *arg) { | |||
| 1167 | __asan::get_allocator().ForEachChunk(callback, arg); | |||
| 1168 | } | |||
| 1169 | ||||
| 1170 | IgnoreObjectResult IgnoreObjectLocked(const void *p) { | |||
| 1171 | uptr addr = reinterpret_cast<uptr>(p); | |||
| 1172 | __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddr(addr); | |||
| 1173 | if (!m || | |||
| 1174 | (atomic_load(&m->chunk_state, memory_order_acquire) != | |||
| 1175 | __asan::CHUNK_ALLOCATED) || | |||
| 1176 | !m->AddrIsInside(addr)) { | |||
| 1177 | return kIgnoreObjectInvalid; | |||
| 1178 | } | |||
| 1179 | if (m->lsan_tag == kIgnored) | |||
| 1180 | return kIgnoreObjectAlreadyIgnored; | |||
| 1181 | m->lsan_tag = __lsan::kIgnored; | |||
| 1182 | return kIgnoreObjectSuccess; | |||
| 1183 | } | |||
| 1184 | ||||
| 1185 | void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs) { | |||
| 1186 | // Look for the arg pointer of threads that have been created or are running. | |||
| 1187 | // This is necessary to prevent false positive leaks due to the AsanThread | |||
| 1188 | // holding the only live reference to a heap object. This can happen because | |||
| 1189 | // the `pthread_create()` interceptor doesn't wait for the child thread to | |||
| 1190 | // start before returning and thus loosing the the only live reference to the | |||
| 1191 | // heap object on the stack. | |||
| 1192 | ||||
| 1193 | __asan::AsanThreadContext *atctx = | |||
| 1194 | reinterpret_cast<__asan::AsanThreadContext *>(tctx); | |||
| 1195 | __asan::AsanThread *asan_thread = atctx->thread; | |||
| 1196 | ||||
| 1197 | // Note ThreadStatusRunning is required because there is a small window where | |||
| 1198 | // the thread status switches to `ThreadStatusRunning` but the `arg` pointer | |||
| 1199 | // still isn't on the stack yet. | |||
| 1200 | if (atctx->status != ThreadStatusCreated && | |||
| 1201 | atctx->status != ThreadStatusRunning) | |||
| 1202 | return; | |||
| 1203 | ||||
| 1204 | uptr thread_arg = reinterpret_cast<uptr>(asan_thread->get_arg()); | |||
| 1205 | if (!thread_arg) | |||
| 1206 | return; | |||
| 1207 | ||||
| 1208 | auto ptrsVec = reinterpret_cast<InternalMmapVector<uptr> *>(ptrs); | |||
| 1209 | ptrsVec->push_back(thread_arg); | |||
| 1210 | } | |||
| 1211 | ||||
| 1212 | } // namespace __lsan | |||
| 1213 | ||||
| 1214 | // ---------------------- Interface ---------------- {{{1 | |||
| 1215 | using namespace __asan; | |||
| 1216 | ||||
| 1217 | // ASan allocator doesn't reserve extra bytes, so normally we would | |||
| 1218 | // just return "size". We don't want to expose our redzone sizes, etc here. | |||
| 1219 | uptr __sanitizer_get_estimated_allocated_size(uptr size) { | |||
| 1220 | return size; | |||
| 1221 | } | |||
| 1222 | ||||
| 1223 | int __sanitizer_get_ownership(const void *p) { | |||
| 1224 | uptr ptr = reinterpret_cast<uptr>(p); | |||
| 1225 | return instance.AllocationSize(ptr) > 0; | |||
| 1226 | } | |||
| 1227 | ||||
| 1228 | uptr __sanitizer_get_allocated_size(const void *p) { | |||
| 1229 | if (!p) return 0; | |||
| 1230 | uptr ptr = reinterpret_cast<uptr>(p); | |||
| 1231 | uptr allocated_size = instance.AllocationSize(ptr); | |||
| 1232 | // Die if p is not malloced or if it is already freed. | |||
| 1233 | if (allocated_size == 0) { | |||
| 1234 | GET_STACK_TRACE_FATAL_HEREBufferedStackTrace stack; if (kStackTraceMax <= 2) { stack .size = kStackTraceMax; if (kStackTraceMax > 0) { stack.top_frame_bp = (__sanitizer::uptr) __builtin_frame_address(0); stack.trace_buffer [0] = StackTrace::GetCurrentPc(); if (kStackTraceMax > 1) stack .trace_buffer[1] = (__sanitizer::uptr) __builtin_return_address (0); } } else { stack.Unwind(StackTrace::GetCurrentPc(), (__sanitizer ::uptr) __builtin_frame_address(0), nullptr, common_flags()-> fast_unwind_on_fatal, kStackTraceMax); }; | |||
| 1235 | ReportSanitizerGetAllocatedSizeNotOwned(ptr, &stack); | |||
| 1236 | } | |||
| 1237 | return allocated_size; | |||
| 1238 | } | |||
| 1239 | ||||
| 1240 | void __sanitizer_purge_allocator() { | |||
| 1241 | GET_STACK_TRACE_MALLOCBufferedStackTrace stack; if (GetMallocContextSize() <= 2) { stack.size = GetMallocContextSize(); if (GetMallocContextSize () > 0) { stack.top_frame_bp = (__sanitizer::uptr) __builtin_frame_address (0); stack.trace_buffer[0] = StackTrace::GetCurrentPc(); if ( GetMallocContextSize() > 1) stack.trace_buffer[1] = (__sanitizer ::uptr) __builtin_return_address(0); } } else { stack.Unwind( StackTrace::GetCurrentPc(), (__sanitizer::uptr) __builtin_frame_address (0), nullptr, common_flags()->fast_unwind_on_malloc, GetMallocContextSize ()); }; | |||
| 1242 | instance.Purge(&stack); | |||
| 1243 | } | |||
| 1244 | ||||
| 1245 | int __asan_update_allocation_context(void* addr) { | |||
| 1246 | GET_STACK_TRACE_MALLOCBufferedStackTrace stack; if (GetMallocContextSize() <= 2) { stack.size = GetMallocContextSize(); if (GetMallocContextSize () > 0) { stack.top_frame_bp = (__sanitizer::uptr) __builtin_frame_address (0); stack.trace_buffer[0] = StackTrace::GetCurrentPc(); if ( GetMallocContextSize() > 1) stack.trace_buffer[1] = (__sanitizer ::uptr) __builtin_return_address(0); } } else { stack.Unwind( StackTrace::GetCurrentPc(), (__sanitizer::uptr) __builtin_frame_address (0), nullptr, common_flags()->fast_unwind_on_malloc, GetMallocContextSize ()); }; | |||
| 1247 | return instance.UpdateAllocationStack((uptr)addr, &stack); | |||
| 1248 | } | |||
| 1249 | ||||
| 1250 | #if !SANITIZER_SUPPORTS_WEAK_HOOKS1 | |||
| 1251 | // Provide default (no-op) implementation of malloc hooks. | |||
| 1252 | SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook,extern "C" __attribute__((visibility("default"))) __attribute__ ((weak)) void __sanitizer_malloc_hook(void *ptr, uptr size) | |||
| 1253 | void *ptr, uptr size)extern "C" __attribute__((visibility("default"))) __attribute__ ((weak)) void __sanitizer_malloc_hook(void *ptr, uptr size) { | |||
| 1254 | (void)ptr; | |||
| 1255 | (void)size; | |||
| 1256 | } | |||
| 1257 | ||||
| 1258 | SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *ptr)extern "C" __attribute__((visibility("default"))) __attribute__ ((weak)) void __sanitizer_free_hook(void *ptr) { | |||
| 1259 | (void)ptr; | |||
| 1260 | } | |||
| 1261 | #endif |
| 1 | //===-- sanitizer_atomic_clang.h --------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file is a part of ThreadSanitizer/AddressSanitizer runtime. |
| 10 | // Not intended for direct inclusion. Include sanitizer_atomic.h. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #ifndef SANITIZER_ATOMIC_CLANG_H |
| 15 | #define SANITIZER_ATOMIC_CLANG_H |
| 16 | |
| 17 | #if defined(__i386__) || defined(__x86_64__1) |
| 18 | # include "sanitizer_atomic_clang_x86.h" |
| 19 | #else |
| 20 | # include "sanitizer_atomic_clang_other.h" |
| 21 | #endif |
| 22 | |
| 23 | namespace __sanitizer { |
| 24 | |
| 25 | // We would like to just use compiler builtin atomic operations |
| 26 | // for loads and stores, but they are mostly broken in clang: |
| 27 | // - they lead to vastly inefficient code generation |
| 28 | // (http://llvm.org/bugs/show_bug.cgi?id=17281) |
| 29 | // - 64-bit atomic operations are not implemented on x86_32 |
| 30 | // (http://llvm.org/bugs/show_bug.cgi?id=15034) |
| 31 | // - they are not implemented on ARM |
| 32 | // error: undefined reference to '__atomic_load_4' |
| 33 | |
| 34 | // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html |
| 35 | // for mappings of the memory model to different processors. |
| 36 | |
| 37 | inline void atomic_signal_fence(memory_order) { |
| 38 | __asm__ __volatile__("" ::: "memory"); |
| 39 | } |
| 40 | |
| 41 | inline void atomic_thread_fence(memory_order) { |
| 42 | __sync_synchronize(); |
| 43 | } |
| 44 | |
| 45 | template<typename T> |
| 46 | inline typename T::Type atomic_fetch_add(volatile T *a, |
| 47 | typename T::Type v, memory_order mo) { |
| 48 | (void)mo; |
| 49 | DCHECK(!((uptr)a % sizeof(*a))); |
| 50 | return __sync_fetch_and_add(&a->val_dont_use, v); |
| 51 | } |
| 52 | |
| 53 | template<typename T> |
| 54 | inline typename T::Type atomic_fetch_sub(volatile T *a, |
| 55 | typename T::Type v, memory_order mo) { |
| 56 | (void)mo; |
| 57 | DCHECK(!((uptr)a % sizeof(*a))); |
| 58 | return __sync_fetch_and_add(&a->val_dont_use, -v); |
| 59 | } |
| 60 | |
| 61 | template<typename T> |
| 62 | inline typename T::Type atomic_exchange(volatile T *a, |
| 63 | typename T::Type v, memory_order mo) { |
| 64 | DCHECK(!((uptr)a % sizeof(*a))); |
| 65 | if (mo & (memory_order_release | memory_order_acq_rel | memory_order_seq_cst)) |
| 66 | __sync_synchronize(); |
| 67 | v = __sync_lock_test_and_set(&a->val_dont_use, v); |
| 68 | if (mo == memory_order_seq_cst) |
| 69 | __sync_synchronize(); |
| 70 | return v; |
| 71 | } |
| 72 | |
| 73 | template <typename T> |
| 74 | inline bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp, |
| 75 | typename T::Type xchg, |
| 76 | memory_order mo) { |
| 77 | typedef typename T::Type Type; |
| 78 | Type cmpv = *cmp; |
| 79 | Type prev; |
| 80 | prev = __sync_val_compare_and_swap(&a->val_dont_use, cmpv, xchg); |
| 81 | if (prev == cmpv) return true; |
| 82 | *cmp = prev; |
| 83 | return false; |
| 84 | } |
| 85 | |
| 86 | template<typename T> |
| 87 | inline bool atomic_compare_exchange_weak(volatile T *a, |
| 88 | typename T::Type *cmp, |
| 89 | typename T::Type xchg, |
| 90 | memory_order mo) { |
| 91 | return atomic_compare_exchange_strong(a, cmp, xchg, mo); |
| 92 | } |
| 93 | |
| 94 | } // namespace __sanitizer |
| 95 | |
| 96 | // This include provides explicit template instantiations for atomic_uint64_t |
| 97 | // on MIPS32, which does not directly support 8 byte atomics. It has to |
| 98 | // proceed the template definitions above. |
| 99 | #if defined(_MIPS_SIM) && defined(_ABIO32) |
| 100 | #include "sanitizer_atomic_clang_mips.h" |
| 101 | #endif |
| 102 | |
| 103 | #undef ATOMIC_ORDER |
| 104 | |
| 105 | #endif // SANITIZER_ATOMIC_CLANG_H |