From 0eb679f70c6066b85251a8324ba69bf60d25b8dd Mon Sep 17 00:00:00 2001
From: Nobuyoshi Nakada <nobu@ruby-lang.org>
Date: Fri, 24 Sep 2021 11:36:42 +0900
Subject: HCRYPTPROV is not a HANDLE

Even though it is called "handle" and prefixed with "H", and its
value looks like a handle.
---
 random.c | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

(limited to 'random.c')

diff --git a/random.c b/random.c
index eff8b27acd..fd9db0cf75 100644
--- a/random.c
+++ b/random.c
@@ -544,11 +544,18 @@ fill_random_bytes_syscall(void *buf, size_t size, int unused)
 #endif
 }
 #elif defined(_WIN32)
+STATIC_ASSERT(sizeof_HCRYPTPROV, sizeof(HCRYPTPROV) == sizeof(size_t));
+
+/* Although HCRYPTPROV is not a HANDLE, it looks like
+ * INVALID_HANDLE_VALUE is not a valid value */
+static const HCRYPTPROV INVALID_HCRYPTPROV = (HCRYPTPROV)INVALID_HANDLE_VALUE;
+
 static void
 release_crypt(void *p)
 {
-    HCRYPTPROV prov = (HCRYPTPROV)ATOMIC_PTR_EXCHANGE(*(HCRYPTPROV *)p, INVALID_HANDLE_VALUE);
-    if (prov && prov != (HCRYPTPROV)INVALID_HANDLE_VALUE) {
+    HCRYPTPROV *ptr = p;
+    HCRYPTPROV prov = (HCRYPTPROV)ATOMIC_SIZE_EXCHANGE(*ptr, INVALID_HCRYPTPROV);
+    if (prov && prov != INVALID_HCRYPTPROV) {
 	CryptReleaseContext(prov, 0);
     }
 }
@@ -560,24 +567,24 @@ fill_random_bytes_syscall(void *seed, size_t size, int unused)
     HCRYPTPROV prov = perm_prov, old_prov;
     if (!prov) {
 	if (!CryptAcquireContext(&prov, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT)) {
-	    prov = (HCRYPTPROV)INVALID_HANDLE_VALUE;
+	    prov = INVALID_HCRYPTPROV;
 	}
-	old_prov = (HCRYPTPROV)ATOMIC_PTR_CAS(perm_prov, 0, prov);
+	old_prov = (HCRYPTPROV)ATOMIC_SIZE_CAS(perm_prov, 0, prov);
 	if (LIKELY(!old_prov)) { /* no other threads acquired */
-	    if (prov != (HCRYPTPROV)INVALID_HANDLE_VALUE) {
+	    if (prov != INVALID_HCRYPTPROV) {
 #undef RUBY_UNTYPED_DATA_WARNING
 #define RUBY_UNTYPED_DATA_WARNING 0
 		rb_gc_register_mark_object(Data_Wrap_Struct(0, 0, release_crypt, &perm_prov));
 	    }
 	}
 	else {			/* another thread acquired */
-	    if (prov != (HCRYPTPROV)INVALID_HANDLE_VALUE) {
+	    if (prov != INVALID_HCRYPTPROV) {
 		CryptReleaseContext(prov, 0);
 	    }
 	    prov = old_prov;
 	}
     }
-    if (prov == (HCRYPTPROV)INVALID_HANDLE_VALUE) return -1;
+    if (prov == INVALID_HCRYPTPROV) return -1;
     CryptGenRandom(prov, size, seed);
     return 0;
 }
-- 
cgit v1.2.1