Address review comments on Fenrir zeroization fixes

Two follow-ups raised by Copilot review on PR #10247:

src/pk_rsa.c: Make derAllocSz a word32 instead of int and only assign
it after a successful XMALLOC, so the cleanup path can never call
ForceZero with a wrapped-around size derived from a negative derSz.

src/pk.c: Capture allocSz at the XMALLOC call site (and clear it back
to 0 on allocation failure) so the relationship between the buffer
allocation and the recorded size is explicit and cannot drift if the
surrounding control flow changes.

Author: julek-wolfssl

src/pk.c

@@ -7150,16 +7150,14 @@ static int pem_write_mem_pkcs8privatekey(byte** pem, int* pemSz,
             *pemSz += 54;
         }
 
+        allocSz = (word32)*pemSz;
         /* Allocate enough memory to hold PEM encoded encrypted key. */
-        *pem = (byte*)XMALLOC((size_t)*pemSz, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+        *pem = (byte*)XMALLOC((size_t)allocSz, NULL, DYNAMIC_TYPE_TMP_BUFFER);
         if (*pem == NULL) {
+            allocSz = 0;
             res = 0;
         }
         else {
-            /* Remember the allocation size before *pemSz is updated to the
-             * actual PEM output length, so we can zero any unused tail that
-             * held the DER staging area. */
-            allocSz = (word32)*pemSz;
             /* Use end of PEM buffer for key data. */
             key = *pem + *pemSz - keySz;
         }

src/pk_rsa.c

@@ -782,7 +782,7 @@ static int wolfSSL_RSA_To_Der_ex(WOLFSSL_RSA* rsa, byte** outBuf, int publicKey,
 {
     int ret = 1;
     int derSz = 0;
-    int derAllocSz = 0;
+    word32 derAllocSz = 0;
     byte* derBuf = NULL;
 
     WOLFSSL_ENTER("wolfSSL_RSA_To_Der");
@@ -824,7 +824,6 @@ static int wolfSSL_RSA_To_Der_ex(WOLFSSL_RSA* rsa, byte** outBuf, int publicKey,
         }
     }
 
-    derAllocSz = derSz;
     if ((ret == 1) && (outBuf != NULL)) {
         derBuf = *outBuf;
         if (derBuf == NULL) {
@@ -835,6 +834,9 @@ static int wolfSSL_RSA_To_Der_ex(WOLFSSL_RSA* rsa, byte** outBuf, int publicKey,
                 WOLFSSL_ERROR_MSG("Memory allocation failed");
                 ret = MEMORY_ERROR;
             }
+            else {
+                derAllocSz = (word32)derSz;
+            }
         }
     }
     if ((ret == 1) && (outBuf != NULL)) {
@@ -868,8 +870,8 @@ static int wolfSSL_RSA_To_Der_ex(WOLFSSL_RSA* rsa, byte** outBuf, int publicKey,
 
     if ((outBuf != NULL) && (*outBuf != derBuf)) {
         /* Not returning buffer, needs to be disposed of. */
-        if ((derBuf != NULL) && (publicKey == 0)) {
-            ForceZero(derBuf, (word32)derAllocSz);
+        if ((derBuf != NULL) && (publicKey == 0) && (derAllocSz > 0)) {
+            ForceZero(derBuf, derAllocSz);
         }
         XFREE(derBuf, heap, DYNAMIC_TYPE_TMP_BUFFER);
     }