CODEWITHSUNDEEP
c++winapicryptoapicng
David
David

Reputation: 108

NCryptSignHash returns NTE_INVALID_PARAMETER (0x80090027)

I'm working with some old code that was signing data using an installed certificate and the CryptoAPI, and I need to migrate it to use CNG. I've tried two different approaches, one using CryptAcquireCertificatePrivateKey() and another using NCryptOpenStorageProvider() and NCryptOpenKey(). Both approaches open the CNG key, and the first call to NCryptSignHash() correctly returns the signature length as 256 bytes.

The second call to NCryptOpenKey(), for creating the signature data, always fails with NTE_INVALID_PARAMETER (0x80090027).

I've condensed the problem into this sample:

#define MY_TYPE  (X509_ASN_ENCODING | PKCS_7_ASN_ENCODING)

HRESULT AcquireCertificatePrivateKey (PCCERT_CONTEXT pSignerCert, NCRYPT_PROV_HANDLE* phProvider, NCRYPT_KEY_HANDLE* phKey, DWORD dwFlags)
{
    HRESULT hr;
    DWORD dwSize = 0;
    CRYPT_KEY_PROV_INFO* pKeyProvInfo = NULL;
    NCRYPT_PROV_HANDLE hProvider = NULL;

    CheckIf(NULL == pSignerCert || NULL == phKey, E_INVALIDARG);

    CheckIfGetLastError(!CertGetCertificateContextProperty(pSignerCert, CERT_KEY_PROV_INFO_PROP_ID, NULL, &dwSize));

    pKeyProvInfo = (CRYPT_KEY_PROV_INFO*)malloc(dwSize);
    CheckAlloc(pKeyProvInfo);

    CheckIfGetLastError(!CertGetCertificateContextProperty(pSignerCert, CERT_KEY_PROV_INFO_PROP_ID, pKeyProvInfo, &dwSize));

    Check(NCryptOpenStorageProvider(&hProvider, pKeyProvInfo->pwszProvName, 0));
    Check(NCryptOpenKey(
        hProvider,
        phKey,
        pKeyProvInfo->pwszContainerName,
        pKeyProvInfo->dwKeySpec,
        dwFlags));
    if(phProvider)
    {
        *phProvider = hProvider;
        hProvider = NULL;
    }

Cleanup:
    if(hProvider)
        NCryptFreeObject(hProvider);
    free(pKeyProvInfo);
    return hr;
}

INT main (INT cArgs, __in_ecount(cArgs) PCSTR* ppcszArgs)
{
    HRESULT hr;
    HCERTSTORE hStoreHandle = NULL;
    PCCERT_CONTEXT pSignerCert = NULL;
    PCWSTR pcwzSigner = L"name of my certificate";
    DWORD dwKeySpec = 0, dwSigLen = 0;
    BOOL fCallerFree = FALSE;
    NCRYPT_PROV_HANDLE hProvider = NULL;
    NCRYPT_KEY_HANDLE hKey = NULL;
    BYTE bData[20], *pbSignature = NULL;

    hStoreHandle = CertOpenStore(CERT_STORE_PROV_SYSTEM, 0, NULL, CERT_SYSTEM_STORE_LOCAL_MACHINE | CERT_STORE_READONLY_FLAG | CERT_STORE_OPEN_EXISTING_FLAG /*CERT_SYSTEM_STORE_CURRENT_USER*/, L"Root");
    CheckIfGetLastError(NULL == hStoreHandle);

    pSignerCert = CertFindCertificateInStore(hStoreHandle, MY_TYPE, 0, CERT_FIND_SUBJECT_STR, pcwzSigner, pSignerCert);
    CheckIfGetLastError(NULL == pSignerCert);

    /*
    CheckIfGetLastError(!CryptAcquireCertificatePrivateKey(pSignerCert,
        CRYPT_ACQUIRE_ONLY_NCRYPT_KEY_FLAG, NULL,
        (HCRYPTPROV_OR_NCRYPT_KEY_HANDLE*)&hKey,
        &dwKeySpec, &fCallerFree));
    CheckIf(0 == (dwKeySpec & CERT_NCRYPT_KEY_SPEC), E_FAIL);
    CheckIf(!fCallerFree, E_FAIL);
    */
    Check(AcquireCertificatePrivateKey(pSignerCert, &hProvider, &hKey, NCRYPT_MACHINE_KEY_FLAG));
    fCallerFree = TRUE;

    // The real data is a SHA-1 hash, but this should be fine for testing
    for(int i = 0; i < sizeof(bData); i++)
        bData[i] = i;
    Check(NCryptSignHash(hKey, NULL, bData, sizeof(bData), NULL, 0, &dwSigLen, 0));

    pbSignature = (BYTE*)malloc(dwSigLen);
    CheckAlloc(pbSignature);

    Check(NCryptSignHash(hKey, NULL, bData, sizeof(bData), pbSignature, dwSigLen, &dwSigLen, 0));

Cleanup:
    if(pbSignature)
        free(pbSignature);
    if(hKey && fCallerFree)
        NCryptFreeObject(hKey);
    if(hProvider)
        NCryptFreeObject(hProvider);
    if(hStoreHandle)
        CertCloseStore(hStoreHandle, CERT_CLOSE_STORE_CHECK_FLAG);
    return hr;
}

When I tried AcquireCertificatePrivateKey(), I did confirm that pKeyProvInfo->dwKeySpec contained AT_KEYEXCHANGE, which is also what the original code was using. I am passing NCRYPT_MACHINE_KEY_FLAG because of how the certificate is setup, and I do run Visual Studio and Embarcadero as administrator while testing this.

Any ideas?

Thanks!

Upvotes: 0

Views: 265

Answers (1)

RbMm
RbMm

Reputation: 33706

you forget about pPaddingInfo and dwFlags parameters of NCryptSignHash. it not always can be 0. it can be 0 only for ECDSA but not for RSA. you can query type of key, by NCryptGetProperty(hKey, NCRYPT_ALGORITHM_GROUP_PROPERTY,) and compare it with NCRYPT_RSA_ALGORITHM_GROUP, NCRYPT_ECDSA_ALGORITHM_GROUP, etc.

union {
    WCHAR Group[16];
    UCHAR bData[32];
};

HRESULT hr = NCryptGetProperty(hKey, NCRYPT_ALGORITHM_GROUP_PROPERTY, (PBYTE)Group, sizeof(Group), &cb, 0);

if (hr)
{
    // hr;
}

BCRYPT_PKCS1_PADDING_INFO pi, *ppi = 0;
DWORD dwFlags = 0;

if (!wcscmp(NCRYPT_RSA_ALGORITHM_GROUP, Group))
{
    pi.pszAlgId = BCRYPT_SHA256_ALGORITHM;
    ppi = &pi;
    dwFlags = BCRYPT_PAD_PKCS1;
}
else if (wcscmp(NCRYPT_ECDSA_ALGORITHM_GROUP, Group))
{
    // hr = NTE_NOT_SUPPORTED;
}

ULONG cb = sizeof(bData);
if (!CryptHashCertificate2(BCRYPT_SHA256_ALGORITHM, 0, 0, *, *, bData, &cb))
{
    // hr = GetLastError();
}

PBYTE pbSignature = 0;
ULONG dwSigLen = 0;

while (NOERROR == (hr = NCryptSignHash(hKey, ppi, bData, cb, pbSignature, dwSigLen, &dwSigLen, dwFlags)))
{
    if (pbSignature)
    {
        break;
    }
    
    pbSignature = (PBYTE)alloca(dwSigLen);
}

note also that you not need call NCryptSignHash 2 time in src code. call it only once

Upvotes: 0

Related Questions