[compiler-rt] Enhance function padding detection for function interception

Summary:
Many CRT (64-bits) functions contains a "hint-nop". The current padding
detection is not able to recognize the 10-bytes padding and the HotPatch
hooking technique cannot be used.

Other patterns may be discover and may be added later.

Reviewers: rnk

Subscribers: llvm-commits, wang0109, chrisha

Differential Revision: http://reviews.llvm.org/D22258

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@275180 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 24 insertions, 1 deletions

diff --git a/lib/interception/interception_win.cc b/lib/interception/interception_win.cc
index 96b567707..994961adc 100644
--- a/lib/interception/interception_win.cc
+++ b/lib/interception/interception_win.cc
@@ -202,6 +202,29 @@ static bool IsMemoryPadding(uptr address, uptr size) {
   return true;
 }
 
+static const u8 kHintNop10Bytes[] = {
+  0x66, 0x66, 0x0F, 0x1F, 0x84,
+  0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+template<class T>
+static bool FunctionHasPrefix(uptr address, const T &pattern) {
+  u8* function = (u8*)address - sizeof(pattern);
+  for (size_t i = 0; i < sizeof(pattern); ++i)
+    if (function[i] != pattern[i])
+      return false;
+  return true;
+}
+
+static bool FunctionHasPadding(uptr address, uptr size) {
+  if (IsMemoryPadding(address - size, size))
+    return true;
+  if (size <= sizeof(kHintNop10Bytes) &&
+      FunctionHasPrefix(address, kHintNop10Bytes))
+    return true;
+  return false;
+}
+
 static void WritePadding(uptr from, uptr size) {
   _memset((void*)from, 0xCC, (size_t)size);
 }
@@ -617,7 +640,7 @@ bool OverrideFunctionWithHotPatch(
   // Validate that the function is hot patchable.
   size_t instruction_size = GetInstructionSize(old_func);
   if (instruction_size < kShortJumpInstructionLength ||
-      !IsMemoryPadding(header, kHotPatchHeaderLen))
+      !FunctionHasPadding(old_func, kHotPatchHeaderLen))
     return false;
 
   if (orig_old_func) {