Fix the return check of OBJ_obj2txt

Also update OBJ_nid2obj.pod to document the possible return values.

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/17005)

crypto/cms/cms_dh.c

@@ -118,7 +118,7 @@ static int dh_cms_set_shared_info(EVP_PKEY_CTX *pctx, CMS_RecipientInfo *ri)
     if (kekctx == NULL)
         goto err;
 
-    if (!OBJ_obj2txt(name, sizeof(name), kekalg->algorithm, 0))
+    if (OBJ_obj2txt(name, sizeof(name), kekalg->algorithm, 0) <= 0)
         goto err;
 
     kekcipher = EVP_CIPHER_fetch(pctx->libctx, name, pctx->propquery);

crypto/cms/cms_ec.c

@@ -47,7 +47,7 @@ static EVP_PKEY *pkey_type2param(int ptype, const void *pval,
         pctx = EVP_PKEY_CTX_new_from_name(libctx, "EC", propq);
         if (pctx == NULL || EVP_PKEY_paramgen_init(pctx) <= 0)
             goto err;
-        if (!OBJ_obj2txt(groupname, sizeof(groupname), poid, 0)
+        if (OBJ_obj2txt(groupname, sizeof(groupname), poid, 0) <= 0
                 || !EVP_PKEY_CTX_set_group_name(pctx, groupname)) {
             ERR_raise(ERR_LIB_CMS, CMS_R_DECODE_ERROR);
             goto err;

crypto/cms/cms_sd.c

@@ -784,8 +784,8 @@ int CMS_SignerInfo_sign(CMS_SignerInfo *si)
     const CMS_CTX *ctx = si->cms_ctx;
     char md_name[OSSL_MAX_NAME_SIZE];
 
-    if (!OBJ_obj2txt(md_name, sizeof(md_name),
+    if (OBJ_obj2txt(md_name, sizeof(md_name),
-                     si->digestAlgorithm->algorithm, 0))
+                     si->digestAlgorithm->algorithm, 0) <= 0)
         return 0;
 
     if (CMS_signed_get_attr_by_NID(si, NID_pkcs9_signingTime, -1) < 0) {

crypto/core_namemap.c

@@ -397,7 +397,7 @@ static void get_legacy_evp_names(int base_nid, int nid, const char *pem_name,
         if ((obj = OBJ_nid2obj(nid)) != NULL) {
             char txtoid[OSSL_MAX_NAME_SIZE];
 
-            if (OBJ_obj2txt(txtoid, sizeof(txtoid), obj, 1))
+            if (OBJ_obj2txt(txtoid, sizeof(txtoid), obj, 1) > 0)
                 num = ossl_namemap_add_name(arg, num, txtoid);
         }
     }

crypto/crmf/crmf_pbm.c

@@ -200,8 +200,8 @@ int OSSL_CRMF_pbm_new(OSSL_LIB_CTX *libctx, const char *propq,
     mac_nid = OBJ_obj2nid(pbmp->mac->algorithm);
 
     if (!EVP_PBE_find(EVP_PBE_TYPE_PRF, mac_nid, NULL, &hmac_md_nid, NULL)
-        || !OBJ_obj2txt(hmac_mdname, sizeof(hmac_mdname),
+        || OBJ_obj2txt(hmac_mdname, sizeof(hmac_mdname),
-                        OBJ_nid2obj(hmac_md_nid), 0)) {
+                        OBJ_nid2obj(hmac_md_nid), 0) <= 0) {
         ERR_raise(ERR_LIB_CRMF, CRMF_R_UNSUPPORTED_ALGORITHM);
         goto err;
     }

crypto/dh/dh_kdf.c

@@ -70,7 +70,7 @@ int DH_KDF_X9_42(unsigned char *out, size_t outlen,
     const OSSL_PROVIDER *prov = EVP_MD_get0_provider(md);
     OSSL_LIB_CTX *libctx = ossl_provider_libctx(prov);
 
-    if (!OBJ_obj2txt(key_alg, sizeof(key_alg), key_oid, 0))
+    if (OBJ_obj2txt(key_alg, sizeof(key_alg), key_oid, 0) <= 0)
         return 0;
 
     return ossl_dh_kdf_X9_42_asn1(out, outlen, Z, Zlen, key_alg,

doc/man3/OBJ_nid2obj.pod

@@ -155,6 +155,10 @@ a NID or B<NID_undef> on error.
 
 OBJ_add_sigid() returns 1 on success or 0 on error.
 
+OBJ_obj2txt() returns the number of bytes written to I<buf> if I<buf_len> is big enough. 
+Otherwise, the result is truncated and the total amount of space required is returned. 
+It also returns -1 on error.
+
 =head1 EXAMPLES
 
 Create an object for B<commonName>:

providers/implementations/encode_decode/decode_spki2typespki.c

@@ -87,7 +87,7 @@ static int spki2typespki_decode(void *vctx, OSSL_CORE_BIO *cin, int selection,
         strcpy(dataname, "SM2");
     else
 #endif
-    if (!OBJ_obj2txt(dataname, sizeof(dataname), oid, 0))
+    if (OBJ_obj2txt(dataname, sizeof(dataname), oid, 0) <= 0)
         goto end;
 
     ossl_X509_PUBKEY_INTERNAL_free(xpub);

test/algorithmid_test.c

@@ -48,7 +48,7 @@ static int test_spki_aid(X509_PUBKEY *pubkey, const char *filename)
         goto end;
 
     X509_ALGOR_get0(&oid, NULL, NULL, alg);
-    if (!TEST_true(OBJ_obj2txt(name, sizeof(name), oid, 0)))
+    if (!TEST_int_gt(OBJ_obj2txt(name, sizeof(name), oid, 0), 0))
         goto end;
 
     /*

test/evp_fetch_prov_test.c

@@ -220,11 +220,11 @@ static int test_explicit_EVP_MD_fetch_by_X509_ALGOR(int idx)
     X509_ALGOR_get0(&obj, NULL, NULL, algor);
     switch (idx) {
     case 0:
-        if (!TEST_true(OBJ_obj2txt(id, sizeof(id), obj, 0)))
+        if (!TEST_int_gt(OBJ_obj2txt(id, sizeof(id), obj, 0), 0))
             goto end;
         break;
     case 1:
-        if (!TEST_true(OBJ_obj2txt(id, sizeof(id), obj, 1)))
+        if (!TEST_int_gt(OBJ_obj2txt(id, sizeof(id), obj, 1), 0))
             goto end;
         break;
     }
@@ -336,11 +336,11 @@ static int test_explicit_EVP_CIPHER_fetch_by_X509_ALGOR(int idx)
     X509_ALGOR_get0(&obj, NULL, NULL, algor);
     switch (idx) {
     case 0:
-        if (!TEST_true(OBJ_obj2txt(id, sizeof(id), obj, 0)))
+        if (!TEST_int_gt(OBJ_obj2txt(id, sizeof(id), obj, 0), 0))
             goto end;
         break;
     case 1:
-        if (!TEST_true(OBJ_obj2txt(id, sizeof(id), obj, 1)))
+        if (!TEST_int_gt(OBJ_obj2txt(id, sizeof(id), obj, 1), 0))
             goto end;
         break;
     }