Add some tests for -inform/keyform enforcement

Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15100)

test/recipes/20-test_pkeyutl.t

@@ -80,7 +80,7 @@ sub tsignverify {
     my $sigfile = basename($privkey, '.pem') . '.sig';
 
     my @args = ();
-    plan tests => 4;
+    plan tests => 5;
 
     @args = ('openssl', 'pkeyutl', '-sign',
              '-inkey', $privkey,
@@ -90,6 +90,15 @@ sub tsignverify {
     ok(run(app([@args])),
        $testtext.": Generating signature");
 
+    @args = ('openssl', 'pkeyutl', '-sign',
+             '-inkey', $privkey,
+             '-keyform', 'DER',
+             '-out', $sigfile,
+             '-in', $data_to_sign);
+    push(@args, @extraopts);
+    ok(!run(app([@args])),
+       $testtext.": Checking that mismatching keyform fails");
+
     @args = ('openssl', 'pkeyutl', '-verify',
              '-inkey', $privkey,
              '-sigfile', $sigfile,
@@ -99,6 +108,7 @@ sub tsignverify {
        $testtext.": Verify signature with private key");
 
     @args = ('openssl', 'pkeyutl', '-verify',
+             '-keyform', 'PEM',
              '-inkey', $pubkey, '-pubin',
              '-sigfile', $sigfile,
              '-in', $data_to_sign);

test/recipes/25-test_crl.t

@@ -15,7 +15,7 @@ use OpenSSL::Test qw/:DEFAULT srctop_file/;
 
 setup("test_crl");
 
-plan tests => 9;
+plan tests => 10;
 
 require_ok(srctop_file('test','recipes','tconversion.pl'));
 
@@ -44,8 +44,10 @@ ok(compare1stline_stdin([qw{openssl crl -hash -noout}],
                         '106cd822'),
    "crl piped input test");
 
-ok(run(app(["openssl", "crl", "-text", "-in", $pem, "-out", $out,
+ok(!run(app(["openssl", "crl", "-text", "-in", $pem, "-inform", "DER",
-            "-nameopt", "utf8"])));
+             "-out", $out, "-nameopt", "utf8"])));
+ok(run(app(["openssl", "crl", "-text", "-in", $pem, "-inform", "PEM",
+            "-out", $out, "-nameopt", "utf8"])));
 is(cmp_text($out, srctop_file("test/certs", "cyrillic_crl.utf8")),
    0, 'Comparing utf8 output');
 

test/recipes/25-test_req.t

@@ -73,16 +73,24 @@ subtest "generating alt certificate requests with RSA" => sub {
 
 
 subtest "generating certificate requests with RSA" => sub {
-    plan tests => 2;
+    plan tests => 3;
 
     SKIP: {
         skip "RSA is not supported by this OpenSSL build", 2
             if disabled("rsa");
 
+        ok(!run(app(["openssl", "req",
+                     "-config", srctop_file("test", "test.cnf"),
+                     "-new", "-out", "testreq-rsa.pem", "-utf8",
+                     "-key", srctop_file("test", "testrsa.pem"),
+                     "-keyform", "DER"])),
+           "Checking that mismatching keyform fails");
+
         ok(run(app(["openssl", "req",
                     "-config", srctop_file("test", "test.cnf"),
                     "-new", "-out", "testreq-rsa.pem", "-utf8",
-                    "-key", srctop_file("test", "testrsa.pem")])),
+                    "-key", srctop_file("test", "testrsa.pem"),
+                    "-keyform", "PEM"])),
            "Generating request");
 
         ok(run(app(["openssl", "req",

test/recipes/25-test_x509.t

@@ -16,7 +16,7 @@ use OpenSSL::Test qw/:DEFAULT srctop_file/;
 
 setup("test_x509");
 
-plan tests => 15;
+plan tests => 18;
 
 require_ok(srctop_file("test", "recipes", "tconversion.pl"));
 
@@ -24,6 +24,8 @@ my @certs = qw(test certs);
 my $pem = srctop_file(@certs, "cyrillic.pem");
 my $out_msb = "out-cyrillic.msb";
 my $out_utf8 = "out-cyrillic.utf8";
+my $der = "cyrillic.der";
+my $der2 = "cyrillic.der";
 my $msb = srctop_file(@certs, "cyrillic.msb");
 my $utf = srctop_file(@certs, "cyrillic.utf8");
 
@@ -36,7 +38,7 @@ ok(run(app(["openssl", "x509", "-text", "-in", $pem, "-out", $out_utf8,
 is(cmp_text($out_utf8, $utf),
    0, 'Comparing utf8 output with cyrillic.utf8');
 
- SKIP: {
+SKIP: {
     skip "DES disabled", 1 if disabled("des");
 
     my $p12 = srctop_file("test", "shibboleth.pfx");
@@ -47,6 +49,16 @@ is(cmp_text($out_utf8, $utf),
     # not unlinking $out_pem
 }
 
+ok(!run(app(["openssl", "x509", "-in", $pem, "-inform", "DER",
+             "-out", $der, "-outform", "DER"])),
+   "Checking failure of mismatching -inform DER");
+ok(run(app(["openssl", "x509", "-in", $pem, "-inform", "PEM",
+            "-out", $der, "-outform", "DER"])),
+   "Conversion to DER");
+ok(!run(app(["openssl", "x509", "-in", $der, "-inform", "PEM",
+             "-out", $der2, "-outform", "DER"])),
+   "Checking failure of mismatching -inform PEM");
+
 # producing and checking self-issued (but not self-signed) cert
 my $subj = "/CN=CA"; # using same DN as in issuer of ee-cert.pem
 my $extfile = srctop_file("test", "v3_ca_exts.cnf");