Writng PKCS12 file c++ i2d_PKCS12_bio and SN1_item_i2d_fp and ASN1_item_i2d_bio giving different results

Question

I am trying write a PKCS12 structure to a file. If I use any of the function ending in _fp( and prvoide a file pointer, it all works fine. But if I use the BIO functions, or read to memory with i2d functions then write to a file, the resulting p12 file is invalid. I am am sure I am missing something here and help would be much appreciated -

p12 is a populated PKCS12 structure.

    .....
    fnRet = fopen_s(&fp, "pkcs12.p12", "w");
    if(fnRet == 0) {                                         // doesn't work
        unsigned char *buf, *buf1;
        int len=i2d_PKCS12(p12,NULL);
        buf=(unsigned char *)malloc(len);
        buf1=buf;
        i2d_PKCS12(p12,&buf);
        fwrite(buf1,1,len,fp);
        fclose(fp);
        qDebug("Keystore PKCS12 written\n");
        free(buf1);
    }
    else { qDebug("Error Creating Keystore filen"); break; }  

    fnRet = fopen_s(&fp, "pkcs12_i2d_mem.p12", "w");
    if(fnRet == 0) {                                         // doesn't work
        unsigned char *buf, *buf1;
        ASN1_item_i2d(ASN1_ITEM_rptr(PKCS12),);
        buf=(unsigned char *)malloc(len);
        buf1=buf;
        i2d_PKCS12(p12,&buf);
        fwrite(buf1,1,len,fp);
        fclose(fp);
        qDebug("Keystore PKCS12 written\n");
    }
    else { qDebug("Error Creating Keystore filen"); break; }    

            fnRet = fopen_s(&fp, "pkcs12_ASN1_fp.p12", "w");          // works
    if(fnRet == 0) {
        ASN1_item_i2d_fp(ASN1_ITEM_rptr(PKCS12), fp, p12);
        fclose(fp);
        qDebug("Keystore fp written\n");
    }
    else { qDebug("Error Creating Keystore filen"); break; }

If I also use the i2d_PKCS12_fp(fp,p12); it works as well. I am trying to avoid the including the applink.c from openssl.

Writing of the private key and certificates (X509) all work find using the bio or memory functions.

When I looked at the source in the Openssl library, both the fp and bio seem to call the same underlying functions

from p12.util.c

int i2d_PKCS12_bio(BIO *bp, const PKCS12 *p12) { 
    return ASN1_item_i2d_bio(ASN1_ITEM_rptr(PKCS12), bp, p12); 
}

#ifndef OPENSSL_NO_STDIO 

int i2d_PKCS12_fp(FILE *fp, const PKCS12 *p12) { 
return ASN1_item_i2d_fp(ASN1_ITEM_rptr(PKCS12), fp, p12); 
} 

#endif

from a_i2d_fp.c

#ifndef OPENSSL_NO_STDIO 

int ASN1_item_i2d_fp(const ASN1_ITEM *it, FILE *out, const void *x) { 
BIO *b; int ret;

if ((b = BIO_new(BIO_s_file())) == NULL) {
    ERR_raise(ERR_LIB_ASN1, ERR_R_BUF_LIB);
    return 0;
}
BIO_set_fp(b, out, BIO_NOCLOSE);
ret = ASN1_item_i2d_bio(it, b, x);
BIO_free(b);
return ret;
}

 #endif


int ASN1_item_i2d_bio(const ASN1_ITEM *it, BIO *out, const void *x) { 
    unsigned char *b = NULL; int i, j = 0, n, ret = 1;
    n = ASN1_item_i2d(x, &b, it);
    if (b == NULL) {
        ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE);
        return 0;
    }

    for (;;) {
        i = BIO_write(out, &(b[j]), n);
        if (i == n)
            break;
        if (i <= 0) {
            ret = 0;
            break;
        }
        j += i;
        n -= i;
    }
    OPENSSL_free(b);
    return ret;
}

BIO *ASN1_item_i2d_mem_bio(const ASN1_ITEM *it, const ASN1_VALUE *val)
{
    BIO *res;

    if (it == NULL || val == NULL) {
        ERR_raise(ERR_LIB_ASN1, ERR_R_PASSED_NULL_PARAMETER);
        return NULL;
    }

    if ((res = BIO_new(BIO_s_mem())) == NULL)
        return NULL;
    if (ASN1_item_i2d_bio(it, res, val) <= 0) {
        BIO_free(res);
        res = NULL;
    }
    return res;
}

I have the same structure and written it our using the functions, only one the i2d_PKCS12_fp(fp,p12) and the ASN1_item_i2d_fp(ASN1_ITEM_rptr(PKCS12), fp, p12) functions seem to work and write a valid file.

Answer 1

I eventually found that I had to add 'b' to the mode for binary. I suspect it will also work for the other methods that use BIO. #$#$%^& windows OS.

    bio=BIO_new_file(pkcs12Name,"wb");
    if(bio!=NULL) {
        //ASN1_item_i2d_bio(ASN1_ITEM_rptr(PKCS12),bio,p12);
        if(i2d_PKCS12_bio(bio,p12)<0) {
            printOpensslError("Error BIO Writing Keystore File");
            BIO_free(bio); bio=NULL;
            break;
        }
        else retVal=true;
    }
    else {
        printOpensslError("Error BIO Opening Keystore File");
        break;
    }
    BIO_free(bio); bio=NULL;