diff options
author | Kostya Kortchinsky <kostyak@google.com> | 2019-07-24 16:36:01 +0000 |
---|---|---|
committer | Kostya Kortchinsky <kostyak@google.com> | 2019-07-24 16:36:01 +0000 |
commit | 68316f134318c5670ceff318584a78551910035a (patch) | |
tree | 0cc374f8f65029c873c05217c8346b28487d1826 /lib/scudo/standalone/primary64.h | |
parent | 3882e027c7483086483d9cd0311c47b0140994d4 (diff) | |
download | compiler-rt-68316f134318c5670ceff318584a78551910035a.tar.gz |
[scudo][standalone] Optimization pass
Summary:
This introduces a bunch of small optimizations with the purpose of
making the fastpath tighter:
- tag more conditions as `LIKELY`/`UNLIKELY`: as a rule of thumb we
consider that every operation related to the secondary is unlikely
- attempt to reduce the number of potentially extraneous instructions
- reorganize the `Chunk` header to not straddle a word boundary and
use more appropriate types
Note that some `LIKELY`/`UNLIKELY` impact might be less obvious as
they are in slow paths (for example in `secondary.cc`), but at this
point I am throwing a pretty wide net, and it's consistant and doesn't
hurt.
This was mosly done for the benfit of Android, but other platforms
benefit from it too. An aarch64 Android benchmark gives:
- before:
```
BM_youtube/min_time:15.000/repeats:4/manual_time_mean 445244 us 659385 us 4
BM_youtube/min_time:15.000/repeats:4/manual_time_median 445007 us 658970 us 4
BM_youtube/min_time:15.000/repeats:4/manual_time_stddev 885 us 1332 us 4
```
- after:
```
BM_youtube/min_time:15.000/repeats:4/manual_time_mean 415697 us 621925 us 4
BM_youtube/min_time:15.000/repeats:4/manual_time_median 415913 us 622061 us 4
BM_youtube/min_time:15.000/repeats:4/manual_time_stddev 990 us 1163 us 4
```
Additional since `-Werror=conversion` is enabled on some platforms we
are built on, enable it upstream to catch things early: a few sign
conversions had slept through and needed additional casting.
Reviewers: hctim, morehouse, eugenis, vitalybuka
Reviewed By: vitalybuka
Subscribers: srhines, mgorny, javed.absar, kristof.beyls, delcypher, #sanitizers, llvm-commits
Tags: #llvm, #sanitizers
Differential Revision: https://reviews.llvm.org/D64664
git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@366918 91177308-0d34-0410-b5e6-96231b3b80d8
Diffstat (limited to 'lib/scudo/standalone/primary64.h')
-rw-r--r-- | lib/scudo/standalone/primary64.h | 17 |
1 files changed, 9 insertions, 8 deletions
diff --git a/lib/scudo/standalone/primary64.h b/lib/scudo/standalone/primary64.h index 035182b33..0149edc89 100644 --- a/lib/scudo/standalone/primary64.h +++ b/lib/scudo/standalone/primary64.h @@ -81,7 +81,7 @@ public: // TODO(kostyak): make the lower limit a runtime option Region->CanRelease = (ReleaseToOsInterval > 0) && (I != SizeClassMap::BatchClassId) && - (getSizeByClassId(I) >= (PageSize / 32)); + (getSizeByClassId(I) >= (PageSize / 16)); Region->RandState = getRandomU32(&Seed); } ReleaseToOsIntervalMs = ReleaseToOsInterval; @@ -102,9 +102,9 @@ public: RegionInfo *Region = getRegionInfo(ClassId); ScopedLock L(Region->Mutex); TransferBatch *B = Region->FreeList.front(); - if (B) + if (B) { Region->FreeList.pop_front(); - else { + } else { B = populateFreeList(C, ClassId, Region); if (UNLIKELY(!B)) return nullptr; @@ -131,7 +131,7 @@ public: void enable() { for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--) - getRegionInfo(I)->Mutex.unlock(); + getRegionInfo(static_cast<uptr>(I))->Mutex.unlock(); } template <typename F> void iterateOverBlocks(F Callback) const { @@ -181,7 +181,7 @@ private: static const uptr PrimarySize = RegionSize * NumClasses; // Call map for user memory with at least this size. - static const uptr MapSizeIncrement = 1UL << 16; + static const uptr MapSizeIncrement = 1UL << 17; struct RegionStats { uptr PoppedBlocks; @@ -272,7 +272,7 @@ private: } return nullptr; } - if (MappedUser == 0) + if (UNLIKELY(MappedUser == 0)) Region->Data = Data; if (UNLIKELY(!map(reinterpret_cast<void *>(RegionBeg + MappedUser), UserMapSize, "scudo:primary", @@ -307,7 +307,7 @@ private: return nullptr; } DCHECK(B); - CHECK_GT(B->getCount(), 0); + DCHECK_GT(B->getCount(), 0); Region->AllocatedUser += AllocatedUser; Region->Exhausted = false; @@ -355,7 +355,8 @@ private: const s32 IntervalMs = ReleaseToOsIntervalMs; if (IntervalMs < 0) return; - if (Region->ReleaseInfo.LastReleaseAtNs + IntervalMs * 1000000ULL > + if (Region->ReleaseInfo.LastReleaseAtNs + + static_cast<uptr>(IntervalMs) * 1000000ULL > getMonotonicTime()) { return; // Memory was returned recently. } |