VirtualBox

source: vbox/trunk/src/VBox/Runtime/common/crypto/pkix-signature-rsa.cpp@ 73665

Last change on this file since 73665 was 73665, checked in by vboxsync, 7 years ago

IPRT,SUP,Main: Working on new crypto key handling and rsa signing. bugref:9152
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 18.5 KB
Line 
1/* $Id: pkix-signature-rsa.cpp 73665 2018-08-14 17:49:23Z vboxsync $ */
2/** @file
3 * IPRT - Crypto - Public Key Signature Schema Algorithm, RSA Providers.
4 */
5
6/*
7 * Copyright (C) 2006-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*********************************************************************************************************************************
29* Header Files *
30*********************************************************************************************************************************/
31#include "internal/iprt.h"
32#include <iprt/crypto/rsa.h>
33
34#include <iprt/bignum.h>
35#include <iprt/err.h>
36#include <iprt/mem.h>
37#include <iprt/string.h>
38#include <iprt/crypto/digest.h>
39#include <iprt/crypto/pkix.h>
40
41#include "rsa-internal.h"
42#include "pkix-signature-builtin.h"
43#include "key-internal.h"
44
45
46/*********************************************************************************************************************************
47* Structures and Typedefs *
48*********************************************************************************************************************************/
49/**
50 * RSA signature provider instance.
51 */
52typedef struct RTCRPKIXSIGNATURERSA
53{
54 /** Set if we're signing, clear if verifying. */
55 bool fSigning;
56
57 /** Temporary big number for use when signing or verifiying. */
58 RTBIGNUM TmpBigNum1;
59 /** Temporary big number for use when signing or verifiying. */
60 RTBIGNUM TmpBigNum2;
61
62 /** Scratch space for decoding the key. */
63 union
64 {
65 /** Public key. */
66 RTCRRSAPUBLICKEY PublicKey;
67 /** Private key. */
68 RTCRRSAPRIVATEKEY PrivateKey;
69 /** Scratch area where we assemble the signature. */
70 uint8_t abSignature[RTCRRSA_MAX_MODULUS_BITS / 8 * 2];
71 } Scratch;
72} RTCRPKIXSIGNATURERSA;
73/** Pointer to an RSA signature provider instance. */
74typedef RTCRPKIXSIGNATURERSA *PRTCRPKIXSIGNATURERSA;
75
76
77/*********************************************************************************************************************************
78* Global Variables *
79*********************************************************************************************************************************/
80/** @name Pre-encoded DigestInfo DER sequences.
81 * @{ */
82static const uint8_t g_abMd2[] =
83{/* { { 1.2.840.113549.2.2 (MD2), NULL }, hash octet-string } */
84 0x30,0x20, 0x30,0x0c, 0x06,0x08,0x2a,0x86,0x48,0x86,0xf7,0x0d,0x02,0x02, 0x05,0x00, 0x04,0x10
85};
86static const uint8_t g_abMd4[] =
87{/* { { 1.2.840.113549.2.4 (MD4), NULL }, hash octet-string } */
88 0x30,0x20, 0x30,0x0c, 0x06,0x08,0x2a,0x86,0x48,0x86,0xf7,0x0d,0x02,0x04, 0x05,0x00, 0x04,0x10
89};
90static const uint8_t g_abMd5[] =
91{/* { { 1.2.840.113549.2.5 (MD5), NULL }, hash octet-string } */
92 0x30,0x20, 0x30,0x0c, 0x06,0x08,0x2a,0x86,0x48,0x86,0xf7,0x0d,0x02,0x05, 0x05,0x00, 0x04,0x10
93};
94static const uint8_t g_abSha1[] =
95{/* { { 1.3.14.3.2.26 (SHA-1), NULL }, hash octet-string } */
96 0x30,0x21, 0x30,0x09, 0x06,0x05,0x2b,0x0e,0x03,0x02,0x1a, 0x05,0x00, 0x04,0x14
97};
98static const uint8_t g_abSha256[] =
99{/* { { 2.16.840.1.101.3.4.2.1 (SHA-256), NULL }, hash octet-string } */
100 0x30,0x31, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x01, 0x05,0x00, 0x04,0x20
101};
102static const uint8_t g_abSha384[] =
103{/* { { 2.16.840.1.101.3.4.2.2 (SHA-384), NULL }, hash octet-string } */
104 0x30,0x41, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x02, 0x05,0x00, 0x04,0x30
105};
106static const uint8_t g_abSha512[] =
107{/* { { 2.16.840.1.101.3.4.2.3 (SHA-512), NULL }, hash octet-string } */
108 0x30,0x51, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x03, 0x05,0x00, 0x04,0x40
109};
110static const uint8_t g_abSha224[] =
111{/* { { 2.16.840.1.101.3.4.2.4 (SHA-224), NULL }, hash octet-string } */
112 0x30,0x2d, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x04, 0x05,0x00, 0x04,0x1c
113};
114/** @} */
115
116/** Lookup array for the pre-encoded DigestInfo DER sequences. */
117static struct
118{
119 RTDIGESTTYPE enmDigest;
120 const uint8_t *pb;
121 size_t cb;
122} const g_aDigestInfos[] =
123{
124 { RTDIGESTTYPE_MD2, g_abMd2, sizeof(g_abMd2) },
125 { RTDIGESTTYPE_MD4, g_abMd4, sizeof(g_abMd4) },
126 { RTDIGESTTYPE_MD5, g_abMd5, sizeof(g_abMd5) },
127 { RTDIGESTTYPE_SHA1, g_abSha1, sizeof(g_abSha1) },
128 { RTDIGESTTYPE_SHA256, g_abSha256, sizeof(g_abSha256) },
129 { RTDIGESTTYPE_SHA384, g_abSha384, sizeof(g_abSha384) },
130 { RTDIGESTTYPE_SHA512, g_abSha512, sizeof(g_abSha512) },
131 { RTDIGESTTYPE_SHA224, g_abSha224, sizeof(g_abSha224) },
132};
133
134
135/** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnInit} */
136static DECLCALLBACK(int) rtCrPkixSignatureRsa_Init(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, void *pvOpaque,
137 bool fSigning, RTCRKEY hKey, PCRTASN1DYNTYPE pParams)
138{
139 RT_NOREF_PV(pDesc); RT_NOREF_PV(pvState); RT_NOREF_PV(pvOpaque);
140
141 if (pParams)
142 return VERR_CR_PKIX_SIGNATURE_TAKES_NO_PARAMETERS;
143
144 RTCRKEYTYPE enmKeyType = RTCrKeyGetType(hKey);
145 if (fSigning)
146 AssertReturn(enmKeyType == RTCRKEYTYPE_RSA_PRIVATE, VERR_CR_PKIX_NOT_RSA_PRIVATE_KEY);
147 else
148 AssertReturn(enmKeyType == RTCRKEYTYPE_RSA_PUBLIC, VERR_CR_PKIX_NOT_RSA_PUBLIC_KEY);
149
150 PRTCRPKIXSIGNATURERSA pThis = (PRTCRPKIXSIGNATURERSA)pvState;
151 pThis->fSigning = fSigning;
152
153 return VINF_SUCCESS;
154}
155
156
157/** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnReset} */
158static DECLCALLBACK(int) rtCrPkixSignatureRsa_Reset(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, bool fSigning)
159{
160 PRTCRPKIXSIGNATURERSA pThis = (PRTCRPKIXSIGNATURERSA)pvState;
161 RT_NOREF_PV(fSigning); RT_NOREF_PV(pDesc);
162 Assert(pThis->fSigning == fSigning); NOREF(pThis);
163 return VINF_SUCCESS;
164}
165
166
167/** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnDelete} */
168static DECLCALLBACK(void) rtCrPkixSignatureRsa_Delete(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, bool fSigning)
169{
170 PRTCRPKIXSIGNATURERSA pThis = (PRTCRPKIXSIGNATURERSA)pvState;
171 RT_NOREF_PV(fSigning); RT_NOREF_PV(pDesc);
172 Assert(pThis->fSigning == fSigning);
173}
174
175
176/**
177 * Common worker for rtCrPkixSignatureRsa_Verify and
178 * rtCrPkixSignatureRsa_Sign that encodes an EMSA-PKCS1-V1_5 signature in
179 * the scratch area.
180 *
181 * This function is referred to as EMSA-PKCS1-v1_5-ENCODE(M,k) in RFC-3447 and
182 * is described in section 9.2
183 *
184 * @returns IPRT status code.
185 * @param pThis The RSA signature provider instance.
186 * @param hDigest The digest which hash to turn into a signature.
187 * @param cbEncodedMsg The desired encoded message length.
188 * @param fNoDigestInfo If true, skip the DigestInfo and encode the digest
189 * without any prefix like described in v1.5 (RFC-2313)
190 * and observed with RSA+MD5 signed timestamps. If
191 * false, include the prefix like v2.0 (RFC-2437)
192 * describes in step in section 9.2.1
193 * (EMSA-PKCS1-v1_5)
194 */
195static int rtCrPkixSignatureRsa_EmsaPkcs1V15Encode(PRTCRPKIXSIGNATURERSA pThis, RTCRDIGEST hDigest, size_t cbEncodedMsg,
196 bool fNoDigestInfo)
197{
198 AssertReturn(cbEncodedMsg * 2 <= sizeof(pThis->Scratch), VERR_CR_PKIX_INTERNAL_ERROR);
199
200 /*
201 * Figure out which hash and select the associate prebaked DigestInfo.
202 */
203 RTDIGESTTYPE const enmDigest = RTCrDigestGetType(hDigest);
204 AssertReturn(enmDigest != RTDIGESTTYPE_INVALID && enmDigest != RTDIGESTTYPE_UNKNOWN, VERR_CR_PKIX_UNKNOWN_DIGEST_TYPE);
205 uint8_t const *pbDigestInfoStart = NULL;
206 size_t cbDigestInfoStart = 0;
207 for (uint32_t i = 0; i < RT_ELEMENTS(g_aDigestInfos); i++)
208 if (g_aDigestInfos[i].enmDigest == enmDigest)
209 {
210 pbDigestInfoStart = g_aDigestInfos[i].pb;
211 cbDigestInfoStart = g_aDigestInfos[i].cb;
212 break;
213 }
214 if (!pbDigestInfoStart)
215 return VERR_CR_PKIX_UNKNOWN_DIGEST_TYPE;
216
217 /*
218 * Get the hash size and verify that it matches what we've got in the
219 * precooked DigestInfo. ASSUMES less that 256 bytes of hash.
220 */
221 uint32_t const cbHash = RTCrDigestGetHashSize(hDigest);
222 AssertReturn(cbHash > 0 && cbHash < _16K, VERR_OUT_OF_RANGE);
223 AssertReturn(cbHash == pbDigestInfoStart[cbDigestInfoStart - 1], VERR_CR_PKIX_INTERNAL_ERROR);
224
225 if (fNoDigestInfo)
226 cbDigestInfoStart = 0;
227
228 if (cbDigestInfoStart + cbHash + 11 > cbEncodedMsg)
229 return VERR_CR_PKIX_HASH_TOO_LONG_FOR_KEY;
230
231 /*
232 * Encode the message the first part of the scratch area.
233 */
234 uint8_t *pbDst = &pThis->Scratch.abSignature[0];
235 pbDst[0] = 0x00;
236 pbDst[1] = 0x01; /* BT - block type, see RFC-2313. */
237 size_t cbFFs = cbEncodedMsg - cbHash - cbDigestInfoStart - 3;
238 memset(&pbDst[2], 0xff, cbFFs);
239 pbDst += cbFFs + 2;
240 *pbDst++ = 0x00;
241 memcpy(pbDst, pbDigestInfoStart, cbDigestInfoStart);
242 pbDst += cbDigestInfoStart;
243 int rc = RTCrDigestFinal(hDigest, pbDst, cbHash);
244 if (RT_FAILURE(rc))
245 return rc;
246 pbDst += cbHash;
247 Assert((size_t)(pbDst - &pThis->Scratch.abSignature[0]) == cbEncodedMsg);
248 return VINF_SUCCESS;
249}
250
251
252
253/** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnVerify} */
254static DECLCALLBACK(int) rtCrPkixSignatureRsa_Verify(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, RTCRKEY hKey,
255 RTCRDIGEST hDigest, void const *pvSignature, size_t cbSignature)
256{
257 PRTCRPKIXSIGNATURERSA pThis = (PRTCRPKIXSIGNATURERSA)pvState;
258 RT_NOREF_PV(pDesc);
259 Assert(!pThis->fSigning);
260 if (cbSignature > sizeof(pThis->Scratch) / 2)
261 return VERR_CR_PKIX_SIGNATURE_TOO_LONG;
262
263 /*
264 * Get the key bits we need.
265 */
266 Assert(RTCrKeyGetType(hKey) == RTCRKEYTYPE_RSA_PUBLIC);
267 PRTBIGNUM pModulus = &hKey->u.RsaPublic.Modulus;
268 PRTBIGNUM pExponent = &hKey->u.RsaPublic.Exponent;
269
270 /*
271 * 8.2.2.1 - Length check. (RFC-3447)
272 */
273 if (cbSignature != RTBigNumByteWidth(pModulus))
274 return VERR_CR_PKIX_INVALID_SIGNATURE_LENGTH;
275
276 /*
277 * 8.2.2.2 - RSA verification / Decrypt the signature.
278 */
279 /* a) s = OS2IP(S) -- Convert signature to integer. */
280 int rc = RTBigNumInit(&pThis->TmpBigNum1, RTBIGNUMINIT_F_ENDIAN_BIG | RTBIGNUMINIT_F_UNSIGNED,
281 pvSignature, cbSignature);
282 if (RT_FAILURE(rc))
283 return rc;
284 /* b) RSAVP1 - 5.2.2.2: Range check (0 <= s < n). */
285 if (RTBigNumCompare(&pThis->TmpBigNum1, pModulus) < 0)
286 {
287 if (RTBigNumCompareWithU64(&pThis->TmpBigNum1, 0) >= 0)
288 {
289 /* b) RSAVP1 - 5.2.2.3: s^e mod n */
290 rc = RTBigNumInitZero(&pThis->TmpBigNum2, 0);
291 if (RT_SUCCESS(rc))
292 {
293 rc = RTBigNumModExp(&pThis->TmpBigNum2, &pThis->TmpBigNum1, pExponent, pModulus);
294 if (RT_SUCCESS(rc))
295 {
296 /* c) EM' = I2OSP(m, k) -- Convert the result to bytes. */
297 uint32_t cbDecrypted = RTBigNumByteWidth(&pThis->TmpBigNum2) + 1; /* 1 = leading zero byte */
298 if (cbDecrypted <= sizeof(pThis->Scratch) / 2)
299 {
300 uint8_t *pbDecrypted = &pThis->Scratch.abSignature[sizeof(pThis->Scratch) / 2];
301 rc = RTBigNumToBytesBigEndian(&pThis->TmpBigNum2, pbDecrypted, cbDecrypted);
302 if (RT_SUCCESS(rc))
303 {
304 /*
305 * 8.2.2.3 - Build a hopefully identical signature using hDigest.
306 */
307 rc = rtCrPkixSignatureRsa_EmsaPkcs1V15Encode(pThis, hDigest, cbDecrypted, false /* fNoDigestInfo */);
308 if (RT_SUCCESS(rc))
309 {
310 /*
311 * 8.2.2.4 - Compare the two.
312 */
313 if (memcmp(&pThis->Scratch.abSignature[0], pbDecrypted, cbDecrypted) == 0)
314 rc = VINF_SUCCESS;
315 else
316 {
317 /*
318 * Try again without digestinfo. This style signing has been
319 * observed in Vista timestamp counter signatures (Thawte).
320 */
321 rc = rtCrPkixSignatureRsa_EmsaPkcs1V15Encode(pThis, hDigest, cbDecrypted,
322 true /* fNoDigestInfo */);
323 if (RT_SUCCESS(rc))
324 {
325 if (memcmp(&pThis->Scratch.abSignature[0], pbDecrypted, cbDecrypted) == 0)
326 rc = VINF_SUCCESS;
327 else
328 rc = VERR_CR_PKIX_SIGNATURE_MISMATCH;
329 }
330 }
331 }
332 }
333 }
334 else
335 rc = VERR_CR_PKIX_SIGNATURE_TOO_LONG;
336 }
337 RTBigNumDestroy(&pThis->TmpBigNum2);
338 }
339 }
340 else
341 rc = VERR_CR_PKIX_SIGNATURE_NEGATIVE;
342 }
343 else
344 rc = VERR_CR_PKIX_SIGNATURE_GE_KEY;
345 RTBigNumDestroy(&pThis->TmpBigNum1);
346 return rc;
347}
348
349
350/** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnSign} */
351static DECLCALLBACK(int) rtCrPkixSignatureRsa_Sign(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, RTCRKEY hKey,
352 RTCRDIGEST hDigest, void *pvSignature, size_t *pcbSignature)
353{
354 PRTCRPKIXSIGNATURERSA pThis = (PRTCRPKIXSIGNATURERSA)pvState;
355 RT_NOREF_PV(pDesc);
356 Assert(pThis->fSigning);
357
358 /*
359 * Get the key bits we need.
360 */
361 Assert(RTCrKeyGetType(hKey) == RTCRKEYTYPE_RSA_PRIVATE);
362 PRTBIGNUM pModulus = &hKey->u.RsaPrivate.Modulus;
363 PRTBIGNUM pExponent = &hKey->u.RsaPrivate.PrivateExponent;
364
365 /*
366 * Calc signature length and return if destination buffer isn't big enough.
367 */
368 size_t const cbDst = *pcbSignature;
369 size_t const cbEncodedMsg = RTBigNumByteWidth(pModulus);
370 *pcbSignature = cbEncodedMsg;
371 if (cbEncodedMsg > sizeof(pThis->Scratch) / 2)
372 return VERR_CR_PKIX_SIGNATURE_TOO_LONG;
373 if (!pvSignature || cbDst < cbEncodedMsg)
374 return VERR_BUFFER_OVERFLOW;
375
376 /*
377 * 8.1.1.1 - EMSA-PSS encoding. (RFC-3447)
378 */
379 int rc = rtCrPkixSignatureRsa_EmsaPkcs1V15Encode(pThis, hDigest, cbEncodedMsg, false /* fNoDigestInfo */);
380 if (RT_FAILURE(rc))
381 return rc;
382
383 /*
384 * 8.1.1.2 - RSA signature.
385 */
386 /* a) m = OS2IP(EM) -- Convert the encoded message (EM) to integer. */
387 rc = RTBigNumInit(&pThis->TmpBigNum1, RTBIGNUMINIT_F_ENDIAN_BIG | RTBIGNUMINIT_F_UNSIGNED,
388 pThis->Scratch.abSignature, cbEncodedMsg);
389 if (RT_FAILURE(rc))
390 return rc;
391
392 /* b) s = RSASP1(K, m = EM) - 5.2.1.1: Range check (0 <= m < n). */
393 if (RTBigNumCompare(&pThis->TmpBigNum1, pModulus) < 0)
394 {
395 /* b) s = RSAVP1(K, m = EM) - 5.2.1.2.a: s = m^d mod n */
396 rc = RTBigNumInitZero(&pThis->TmpBigNum2, 0);
397 if (RT_SUCCESS(rc))
398 {
399 rc = RTBigNumModExp(&pThis->TmpBigNum2, &pThis->TmpBigNum1, pExponent, pModulus);
400 if (RT_SUCCESS(rc))
401 {
402 /* c) S = I2OSP(s, k) -- Convert the result to bytes. */
403 rc = RTBigNumToBytesBigEndian(&pThis->TmpBigNum2, pvSignature, cbEncodedMsg);
404 AssertStmt(RT_SUCCESS(rc) || rc != VERR_BUFFER_OVERFLOW, rc = VERR_CR_PKIX_INTERNAL_ERROR);
405 }
406 RTBigNumDestroy(&pThis->TmpBigNum2);
407 }
408 }
409 else
410 rc = VERR_CR_PKIX_SIGNATURE_GE_KEY;
411 RTBigNumDestroy(&pThis->TmpBigNum1);
412 return rc;
413}
414
415
416
417
418/** RSA alias ODIs. */
419static const char * const g_apszHashWithRsaAliases[] =
420{
421 RTCR_PKCS1_MD2_WITH_RSA_OID,
422 RTCR_PKCS1_MD4_WITH_RSA_OID,
423 RTCR_PKCS1_MD5_WITH_RSA_OID,
424 RTCR_PKCS1_SHA1_WITH_RSA_OID,
425 RTCR_PKCS1_SHA256_WITH_RSA_OID,
426 RTCR_PKCS1_SHA384_WITH_RSA_OID,
427 RTCR_PKCS1_SHA512_WITH_RSA_OID,
428 RTCR_PKCS1_SHA224_WITH_RSA_OID,
429 /* Note: Note quite sure about these OIW oddballs. */
430 "1.3.14.3.2.11" /* OIW rsaSignature */,
431 "1.3.14.3.2.14" /* OIW mdc2WithRSASignature */,
432 "1.3.14.3.2.15" /* OIW shaWithRSASignature */,
433 "1.3.14.3.2.24" /* OIW md2WithRSASignature */,
434 "1.3.14.3.2.25" /* OIW md5WithRSASignature */,
435 "1.3.14.3.2.29" /* OIW sha1WithRSASignature */,
436 NULL
437};
438
439
440/** RSA descriptor. */
441DECL_HIDDEN_CONST(RTCRPKIXSIGNATUREDESC const) g_rtCrPkixSigningHashWithRsaDesc =
442{
443 "RSASSA-PKCS1-v1_5",
444 RTCR_PKCS1_RSA_OID,
445 g_apszHashWithRsaAliases,
446 sizeof(RTCRPKIXSIGNATURERSA),
447 0,
448 0,
449 rtCrPkixSignatureRsa_Init,
450 rtCrPkixSignatureRsa_Reset,
451 rtCrPkixSignatureRsa_Delete,
452 rtCrPkixSignatureRsa_Verify,
453 rtCrPkixSignatureRsa_Sign,
454};
455
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette