1 files changed, 17 insertions, 14 deletions
diff --git a/lib/tsan/rtl/tsan_update_shadow_word_inl.h b/lib/tsan/rtl/tsan_update_shadow_word_inl.h
index 6e3ac2fa1..056c3aa20 100644
--- a/lib/tsan/rtl/tsan_update_shadow_word_inl.h
+++ b/lib/tsan/rtl/tsan_update_shadow_word_inl.h
@@ -1,9 +1,8 @@
 //===-- tsan_update_shadow_word_inl.h ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -18,31 +17,35 @@ do {
   const unsigned kAccessSize = 1 << kAccessSizeLog;
   u64 *sp = &shadow_mem[idx];
   old = LoadShadow(sp);
-  if (old.IsZero()) {
+  if (LIKELY(old.IsZero())) {
     StatInc(thr, StatShadowZero);
-    if (store_word)
+    if (!stored) {
       StoreIfNotYetStored(sp, &store_word);
-    // The above StoreIfNotYetStored could be done unconditionally
-    // and it even shows 4% gain on synthetic benchmarks (r4307).
+      stored = true;
+    }
     break;
   }
   // is the memory access equal to the previous?
-  if (Shadow::Addr0AndSizeAreEqual(cur, old)) {
+  if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) {
     StatInc(thr, StatShadowSameSize);
     // same thread?
-    if (Shadow::TidsAreEqual(old, cur)) {
+    if (LIKELY(Shadow::TidsAreEqual(old, cur))) {
       StatInc(thr, StatShadowSameThread);
-      if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))
+      if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) {
         StoreIfNotYetStored(sp, &store_word);
+        stored = true;
+      }
       break;
     }
     StatInc(thr, StatShadowAnotherThread);
     if (HappensBefore(old, thr)) {
-      if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))
+      if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) {
         StoreIfNotYetStored(sp, &store_word);
+        stored = true;
+      }
       break;
     }
-    if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
+    if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)))
       break;
     goto RACE;
   }
@@ -56,7 +59,7 @@ do {
     StatInc(thr, StatShadowAnotherThread);
     if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
       break;
-    if (HappensBefore(old, thr))
+    if (LIKELY(HappensBefore(old, thr)))
       break;
     goto RACE;
   }