AS2 file decryption issue with windows CertGetSubjectCertificateFromStore api

Question

We have an EDI application which exchange documents using AS2 protocol where when we receive documents from one of our business partner and try to decrypt it ,we are getting error - Error = 0x8009200C (Cannot find the certificate and private key to use for decryption.) We are reading the certificate from windows certificate store using CertGetSubjectCertificateFromStore api

ptrEnvelopeCert = CertGetSubjectCertificateFromStore(dwCertStoreIdx, X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, ptrCertInfo);

Same solutions are working with all our partners.

Can someone please help what could be the problem. we have verify all the details with regards to certificate configurations, public and private key with our partners but not able to identify the issue.

Answer 1

CRYPT_E_NO_DECRYPT_CERT mean that cert exist but not exist private key for it. the call to CryptAcquireCertificatePrivateKey is fail

here can be 3 variants:

1. no CERT_KEY_PROV_INFO_PROP_ID on cert
2. no container. if key not on smartcard, it usually was in file
3. private key exist but not match to public key from cert

we can test it with next code:

template <typename T> 
T HR(HRESULT& hr, T t)
{
    hr = t ? NOERROR : GetLastError();
    return t;
}

HRESULT Compare(PCCERT_CONTEXT pCertContext, 
                HCRYPTPROV_OR_NCRYPT_KEY_HANDLE hCryptProvOrNCryptKey,
                ULONG dwKeySpec)
{
    HRESULT hr;

    PCERT_PUBLIC_KEY_INFO pInfo = 0;
    ULONG cb = 0;
    while (HR(hr, CryptExportPublicKeyInfo(hCryptProvOrNCryptKey, dwKeySpec, X509_ASN_ENCODING, pInfo, &cb)))
    {
        if (pInfo)
        {
            if (!CertComparePublicKeyInfo(X509_ASN_ENCODING, pInfo, &pCertContext->pCertInfo->SubjectPublicKeyInfo))
            {
                hr = NTE_BAD_PUBLIC_KEY;
            }
            break;
        }

        pInfo = (PCERT_PUBLIC_KEY_INFO)alloca(cb);
    }

    return hr;
}

HRESULT TestPrivKey(PCCERT_CONTEXT pCertContext)
{
    union {
        PCRYPT_KEY_PROV_INFO pckpi;
        PVOID buf = 0;
    };

    union {
        HCRYPTPROV hProv;
        NCRYPT_PROV_HANDLE hProvider;
    };

    union {
        HCRYPTKEY hKey;
        NCRYPT_KEY_HANDLE hNCryptKey;
    };

    HRESULT hr;
    ULONG cb = 0;
    while(HR(hr, CertGetCertificateContextProperty(pCertContext, CERT_KEY_PROV_INFO_PROP_ID, buf, &cb)))
    {
        if (buf)
        {
            union {
                CHAR aFileName[0x100];
                WCHAR wFileName[0x100];
            };

            if (pckpi->dwProvType)
            {
                if (HR(hr, CryptAcquireContextW(&hProv, pckpi->pwszContainerName,
                    pckpi->pwszProvName, pckpi->dwProvType, pckpi->dwFlags & CRYPT_MACHINE_KEYSET)))
                {
                    if (CryptGetProvParam(hProv, PP_UNIQUE_CONTAINER, (PBYTE)aFileName, &(cb = sizeof(aFileName)), 0))
                    {
                        DbgPrint("\"%hs\"\n", aFileName);
                    }
                    hr = Compare(pCertContext, hProv, pckpi->dwKeySpec);
                    CryptReleaseContext(hProv, 0);
                }
            }
            else
            {
                if (NOERROR == (hr = NCryptOpenStorageProvider(&hProvider, pckpi->pwszProvName, 0)))
                {
                    hr = NCryptOpenKey(hProvider, &hNCryptKey, 
                        pckpi->pwszContainerName, 
                        pckpi->dwKeySpec, pckpi->dwFlags & NCRYPT_MACHINE_KEY_FLAG);

                    NCryptFreeObject(hProvider);

                    if (NOERROR == hr)
                    {
                        if (!NCryptGetProperty(hNCryptKey, NCRYPT_UNIQUE_NAME_PROPERTY, (PBYTE)wFileName, sizeof(wFileName), &cb, 0))
                        {
                            DbgPrint("\"%ws\"\n", wFileName);
                        }
                        
                        hr = Compare(pCertContext, hNCryptKey, pckpi->dwKeySpec);

                        NCryptFreeObject(hNCryptKey);
                    }
                }
            }
            break;
        }

        buf = alloca(cb);
    }

    return hr;
}