Clean up signature hash calculation code

Use the defined digest lengths for MD5/SHA1/SHA1X rather than magic
numbers, clear the hash type at the start and then only set it if we
know it.

diff --git a/sigcheck.c b/sigcheck.c
index 0f93a31009c123e77fa0660eba76167bc7b72be7..89b3f92970205df3ff70f629ef201f6bb839c705 100644 (file)
--- a/sigcheck.c
+++ b/sigcheck.c
@@ -63,6 +63,9 @@ onak_status_t calculate_packet_sighash(struct openpgp_publickey *key,
        uint64_t keyid;
        onak_status_t res;
 
+       *hashtype = 0;
+       *sighash = NULL;
+
        switch (sig->data[0]) {
        case 2:
        case 3:
@@ -108,8 +111,6 @@ onak_status_t calculate_packet_sighash(struct openpgp_publickey *key,
                hashlen[1] = key->publickey->length;
                chunks = 2;
 
-               *hashtype = sig->data[3];
-
                /* Check to see if this is an X509 based signature */
                if (sig->data[2] == 0 || sig->data[2] == 100) {
                        size_t len;
@@ -136,6 +137,8 @@ onak_status_t calculate_packet_sighash(struct openpgp_publickey *key,
                        }
                }
 
+               *hashtype = sig->data[3];
+
                if (packet != NULL) {
                        if (packet->tag == OPENPGP_PACKET_PUBLICSUBKEY) {
                                packetheader[0] = 0x99;
@@ -262,21 +265,21 @@ onak_status_t calculate_packet_sighash(struct openpgp_publickey *key,
                for (i = 0; i < chunks; i++) {
                        md5_update(&md5_context, hashlen[i], hashdata[i]);
                }
-               md5_digest(&md5_context, 16, hash);
+               md5_digest(&md5_context, MD5_DIGEST_SIZE, hash);
                break;
        case OPENPGP_HASH_SHA1:
                sha1_init(&sha1_context);
                for (i = 0; i < chunks; i++) {
                        sha1_update(&sha1_context, hashlen[i], hashdata[i]);
                }
-               sha1_digest(&sha1_context, 20, hash);
+               sha1_digest(&sha1_context, SHA1_DIGEST_SIZE, hash);
                break;
        case OPENPGP_HASH_SHA1X:
                sha1x_init(&sha1x_context);
                for (i = 0; i < chunks; i++) {
                        sha1x_update(&sha1x_context, hashlen[i], hashdata[i]);
                }
-               sha1x_digest(&sha1x_context, 20, hash);
+               sha1x_digest(&sha1x_context, SHA1X_DIGEST_SIZE, hash);
                break;
 #ifdef HAVE_NETTLE
        case OPENPGP_HASH_RIPEMD160: