tree-wide: fix return value handling of base64mem()

This returns an ssize_t, not an int. On populare archs that's the
difference between 64bit and 32bit. hence, let's be more careful here,
and not silently drop half the bits on the ground by assigning the
return value to "int".

As noticed by @malikabhi05:

https://github.com/systemd/systemd/pull/24754#discussion_r1062903159

4 files changed, 21 insertions, 16 deletions

diff --git a/src/home/homectl-fido2.c b/src/home/homectl-fido2.c
index 61f0d081a3..3cbdf912aa 100644
--- a/src/home/homectl-fido2.c
+++ b/src/home/homectl-fido2.c
@@ -26,14 +26,15 @@ static int add_fido2_credential_id(
         _cleanup_(json_variant_unrefp) JsonVariant *w = NULL;
         _cleanup_strv_free_ char **l = NULL;
         _cleanup_free_ char *escaped = NULL;
+        ssize_t escaped_size;
         int r;
 
         assert(v);
         assert(cid);
 
-        r = base64mem(cid, cid_size, &escaped);
-        if (r < 0)
-                return log_error_errno(r, "Failed to base64 encode FIDO2 credential ID: %m");
+        escaped_size = base64mem(cid, cid_size, &escaped);
+        if (escaped_size < 0)
+                return log_error_errno(escaped_size, "Failed to base64 encode FIDO2 credential ID: %m");
 
         w = json_variant_ref(json_variant_by_key(*v, "fido2HmacCredential"));
         if (w) {
@@ -73,13 +74,14 @@ static int add_fido2_salt(
 
         _cleanup_(json_variant_unrefp) JsonVariant *l = NULL, *w = NULL, *e = NULL;
         _cleanup_(erase_and_freep) char *base64_encoded = NULL, *hashed = NULL;
+        ssize_t base64_encoded_size;
         int r;
 
         /* Before using UNIX hashing on the supplied key we base64 encode it, since crypt_r() and friends
          * expect a NUL terminated string, and we use a binary key */
-        r = base64mem(secret, secret_size, &base64_encoded);
-        if (r < 0)
-                return log_error_errno(r, "Failed to base64 encode secret key: %m");
+        base64_encoded_size = base64mem(secret, secret_size, &base64_encoded);
+        if (base64_encoded_size < 0)
+                return log_error_errno(base64_encoded_size, "Failed to base64 encode secret key: %m");
 
         r = hash_password(base64_encoded, &hashed);
         if (r < 0)
diff --git a/src/home/homectl-pkcs11.c b/src/home/homectl-pkcs11.c
index 69c9d97aca..dc6ecf1665 100644
--- a/src/home/homectl-pkcs11.c
+++ b/src/home/homectl-pkcs11.c
@@ -19,6 +19,7 @@ static int add_pkcs11_encrypted_key(
 
         _cleanup_(json_variant_unrefp) JsonVariant *l = NULL, *w = NULL, *e = NULL;
         _cleanup_(erase_and_freep) char *base64_encoded = NULL, *hashed = NULL;
+        ssize_t base64_encoded_size;
         int r;
 
         assert(v);
@@ -30,9 +31,9 @@ static int add_pkcs11_encrypted_key(
 
         /* Before using UNIX hashing on the supplied key we base64 encode it, since crypt_r() and friends
          * expect a NUL terminated string, and we use a binary key */
-        r = base64mem(decrypted_key, decrypted_key_size, &base64_encoded);
-        if (r < 0)
-                return log_error_errno(r, "Failed to base64 encode secret key: %m");
+        base64_encoded_size = base64mem(decrypted_key, decrypted_key_size, &base64_encoded);
+        if (base64_encoded_size < 0)
+                return log_error_errno(base64_encoded_size, "Failed to base64 encode secret key: %m");
 
         r = hash_password(base64_encoded, &hashed);
         if (r < 0)
diff --git a/src/home/homework-fido2.c b/src/home/homework-fido2.c
index 23fda4a355..5c7cd52e1b 100644
--- a/src/home/homework-fido2.c
+++ b/src/home/homework-fido2.c
@@ -17,6 +17,7 @@ int fido2_use_token(
         _cleanup_(erase_and_freep) void *hmac = NULL;
         size_t hmac_size;
         Fido2EnrollFlags flags = 0;
+        ssize_t ss;
         int r;
 
         assert(h);
@@ -65,9 +66,9 @@ int fido2_use_token(
         if (r < 0)
                 return r;
 
-        r = base64mem(hmac, hmac_size, ret);
-        if (r < 0)
-                return log_error_errno(r, "Failed to base64 encode HMAC secret: %m");
+        ss = base64mem(hmac, hmac_size, ret);
+        if (ss < 0)
+                return log_error_errno(ss, "Failed to base64 encode HMAC secret: %m");
 
         return 0;
 }
diff --git a/src/home/homework-fscrypt.c b/src/home/homework-fscrypt.c
index f8f9f462bb..afe3447d62 100644
--- a/src/home/homework-fscrypt.c
+++ b/src/home/homework-fscrypt.c
@@ -407,6 +407,7 @@ static int fscrypt_slot_set(
         _cleanup_free_ void *encrypted = NULL;
         const EVP_CIPHER *cc;
         size_t encrypted_size;
+        ssize_t ss;
 
         r = crypto_random_bytes(salt, sizeof(salt));
         if (r < 0)
@@ -457,12 +458,12 @@ static int fscrypt_slot_set(
         assert((size_t) encrypted_size_out1 + (size_t) encrypted_size_out2 < encrypted_size);
         encrypted_size = (size_t) encrypted_size_out1 + (size_t) encrypted_size_out2;
 
-        r = base64mem(salt, sizeof(salt), &salt_base64);
-        if (r < 0)
+        ss = base64mem(salt, sizeof(salt), &salt_base64);
+        if (ss < 0)
                 return log_oom();
 
-        r = base64mem(encrypted, encrypted_size, &encrypted_base64);
-        if (r < 0)
+        ss = base64mem(encrypted, encrypted_size, &encrypted_base64);
+        if (ss < 0)
                 return log_oom();
 
         joined = strjoin(salt_base64, ":", encrypted_base64);