summaryrefslogtreecommitdiff
path: root/lib/scudo/standalone/primary64.h
diff options
context:
space:
mode:
authorKostya Kortchinsky <kostyak@google.com>2019-07-24 16:36:01 +0000
committerKostya Kortchinsky <kostyak@google.com>2019-07-24 16:36:01 +0000
commit68316f134318c5670ceff318584a78551910035a (patch)
tree0cc374f8f65029c873c05217c8346b28487d1826 /lib/scudo/standalone/primary64.h
parent3882e027c7483086483d9cd0311c47b0140994d4 (diff)
downloadcompiler-rt-68316f134318c5670ceff318584a78551910035a.tar.gz
[scudo][standalone] Optimization pass
Summary: This introduces a bunch of small optimizations with the purpose of making the fastpath tighter: - tag more conditions as `LIKELY`/`UNLIKELY`: as a rule of thumb we consider that every operation related to the secondary is unlikely - attempt to reduce the number of potentially extraneous instructions - reorganize the `Chunk` header to not straddle a word boundary and use more appropriate types Note that some `LIKELY`/`UNLIKELY` impact might be less obvious as they are in slow paths (for example in `secondary.cc`), but at this point I am throwing a pretty wide net, and it's consistant and doesn't hurt. This was mosly done for the benfit of Android, but other platforms benefit from it too. An aarch64 Android benchmark gives: - before: ``` BM_youtube/min_time:15.000/repeats:4/manual_time_mean 445244 us 659385 us 4 BM_youtube/min_time:15.000/repeats:4/manual_time_median 445007 us 658970 us 4 BM_youtube/min_time:15.000/repeats:4/manual_time_stddev 885 us 1332 us 4 ``` - after: ``` BM_youtube/min_time:15.000/repeats:4/manual_time_mean 415697 us 621925 us 4 BM_youtube/min_time:15.000/repeats:4/manual_time_median 415913 us 622061 us 4 BM_youtube/min_time:15.000/repeats:4/manual_time_stddev 990 us 1163 us 4 ``` Additional since `-Werror=conversion` is enabled on some platforms we are built on, enable it upstream to catch things early: a few sign conversions had slept through and needed additional casting. Reviewers: hctim, morehouse, eugenis, vitalybuka Reviewed By: vitalybuka Subscribers: srhines, mgorny, javed.absar, kristof.beyls, delcypher, #sanitizers, llvm-commits Tags: #llvm, #sanitizers Differential Revision: https://reviews.llvm.org/D64664 git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@366918 91177308-0d34-0410-b5e6-96231b3b80d8
Diffstat (limited to 'lib/scudo/standalone/primary64.h')
-rw-r--r--lib/scudo/standalone/primary64.h17
1 files changed, 9 insertions, 8 deletions
diff --git a/lib/scudo/standalone/primary64.h b/lib/scudo/standalone/primary64.h
index 035182b33..0149edc89 100644
--- a/lib/scudo/standalone/primary64.h
+++ b/lib/scudo/standalone/primary64.h
@@ -81,7 +81,7 @@ public:
// TODO(kostyak): make the lower limit a runtime option
Region->CanRelease = (ReleaseToOsInterval > 0) &&
(I != SizeClassMap::BatchClassId) &&
- (getSizeByClassId(I) >= (PageSize / 32));
+ (getSizeByClassId(I) >= (PageSize / 16));
Region->RandState = getRandomU32(&Seed);
}
ReleaseToOsIntervalMs = ReleaseToOsInterval;
@@ -102,9 +102,9 @@ public:
RegionInfo *Region = getRegionInfo(ClassId);
ScopedLock L(Region->Mutex);
TransferBatch *B = Region->FreeList.front();
- if (B)
+ if (B) {
Region->FreeList.pop_front();
- else {
+ } else {
B = populateFreeList(C, ClassId, Region);
if (UNLIKELY(!B))
return nullptr;
@@ -131,7 +131,7 @@ public:
void enable() {
for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--)
- getRegionInfo(I)->Mutex.unlock();
+ getRegionInfo(static_cast<uptr>(I))->Mutex.unlock();
}
template <typename F> void iterateOverBlocks(F Callback) const {
@@ -181,7 +181,7 @@ private:
static const uptr PrimarySize = RegionSize * NumClasses;
// Call map for user memory with at least this size.
- static const uptr MapSizeIncrement = 1UL << 16;
+ static const uptr MapSizeIncrement = 1UL << 17;
struct RegionStats {
uptr PoppedBlocks;
@@ -272,7 +272,7 @@ private:
}
return nullptr;
}
- if (MappedUser == 0)
+ if (UNLIKELY(MappedUser == 0))
Region->Data = Data;
if (UNLIKELY(!map(reinterpret_cast<void *>(RegionBeg + MappedUser),
UserMapSize, "scudo:primary",
@@ -307,7 +307,7 @@ private:
return nullptr;
}
DCHECK(B);
- CHECK_GT(B->getCount(), 0);
+ DCHECK_GT(B->getCount(), 0);
Region->AllocatedUser += AllocatedUser;
Region->Exhausted = false;
@@ -355,7 +355,8 @@ private:
const s32 IntervalMs = ReleaseToOsIntervalMs;
if (IntervalMs < 0)
return;
- if (Region->ReleaseInfo.LastReleaseAtNs + IntervalMs * 1000000ULL >
+ if (Region->ReleaseInfo.LastReleaseAtNs +
+ static_cast<uptr>(IntervalMs) * 1000000ULL >
getMonotonicTime()) {
return; // Memory was returned recently.
}