Call to GetOpenFileNameA causes common dialog error 2

Question

I'm trying to open a dialog box using GetOpenFileNameA. However, the dialog does not open. Instead, I get a nice CommDlgError 2. Searching Google and StackOverflow for this error did not produce any helpful results.

Confusingly, this code works on a school computer that also uses Visual Studio, albeit a different version.

Note: All variables not declared in this block of code are "global" variables that are only accessible within the main code module.

void GetInputFile()
{
    char szFileNameIN[MAX_PATH];
    char szFileNameOUT[MAX_PATH];

    // get the input file name
    OPENFILENAME ofn;
    ZeroMemory(&fInputPath, sizeof(fInputPath));
    ZeroMemory(&ofn, sizeof(ofn));
    ofn.lStructSize = sizeof(ofn);
    ofn.hwndOwner = NULL;
    ofn.lpstrFilter = LPWSTR("Any File\0*.*\0");
    ofn.lpstrFile = LPWSTR(fInputPath);
    ofn.nMaxFile = MAX_PATH;
    ofn.lpstrFileTitle = LPWSTR(szFileNameIN);
    ofn.nMaxFileTitle = MAX_PATH;
    ofn.lpstrTitle = LPWSTR("Select an input File");
    ofn.Flags = OFN_DONTADDTORECENT | OFN_FILEMUSTEXIST;
    if (GetOpenFileNameA(LPOPENFILENAMEA(&ofn))) // user selected an input file
    {
    }
    else {
        // Get error
        TCHAR error[MAX_LOADSTRING];
        wsprintf(error,TEXT("%i"),CommDlgExtendedError());
        MessageBox(NULL,error,TEXT("ERROR"),MB_OK);
    }

}

Answer 1

Those awful (LPWSTR) casts tell me that you are compiling with UNICODE defined, so the OPENFILENAME struct you are using is actually OPENFILENAMEW; given that you are using GetOpenFileNameA, you have to use OPENFILENAMEA (or use straight GetOpenFileName with wide strings).

(and remember that, as a rule of thumb, if you have to cast pointers to/from anything different than void * and similar, you are probably doing something wrong; adding pointer casts will only silence the compiler, not make the error go away)

Answer 2

You are using the TCHAR version of OPENFILENAME. Since you are assigning Unicode string pointers to its fields, that means your project is being compiled with UNICODE defined, so TCHAR maps to wchar_t and OPENFILENAME maps to OPENFILENAMEW. But you are using ANSI character buffers and the ANSI version of GetOpenFileName(), and using incorrect type-casts all over the place.

So get rid of all the type-casts, and then either:

1. use proper TCHAR types and APIs for everything:

   void GetInputFile()
   {
       TCHAR szFileNameIN[MAX_PATH];
       TCHAR szFileNameOUT[MAX_PATH];
   
       // get the input file name
       OPENFILENAME ofn;
       ZeroMemory(&fInputPath, sizeof(fInputPath));
       ZeroMemory(&ofn, sizeof(ofn));
       ofn.lStructSize = sizeof(ofn);
       ofn.hwndOwner = NULL;
       ofn.lpstrFilter = TEXT("Any File\0*.*\0");
       ofn.lpstrFile = fInputPath; // must be TCHAR[]...
       ofn.nMaxFile = MAX_PATH;
       ofn.lpstrFileTitle = szFileNameIN;
       ofn.nMaxFileTitle = MAX_PATH;
       ofn.lpstrTitle = TEXT("Select an input File");
       ofn.Flags = OFN_DONTADDTORECENT | OFN_FILEMUSTEXIST;
       if (GetOpenFileName(&ofn)) // user selected an input file
       {
       }
       else
       {
           // Get error
           TCHAR error[MAX_LOADSTRING];
           wsprintf(error, TEXT("%i"), CommDlgExtendedError());
           MessageBox(NULL, error, TEXT("ERROR"), MB_OK);
       }
   }
   

2. use proper CHAR/WCHAR types and APIs for everything:

   void GetInputFile()
   {
       WCHAR szFileNameIN[MAX_PATH];
       WCHAR szFileNameOUT[MAX_PATH];
   
       // get the input file name
       OPENFILENAMEW ofn;
       ZeroMemory(&fInputPath, sizeof(fInputPath));
       ZeroMemory(&ofn, sizeof(ofn));
       ofn.lStructSize = sizeof(ofn);
       ofn.hwndOwner = NULL;
       ofn.lpstrFilter = L"Any File\0*.*\0";
       ofn.lpstrFile = fInputPath; // must be WCHAR[]...
       ofn.nMaxFile = MAX_PATH;
       ofn.lpstrFileTitle = szFileNameIN;
       ofn.nMaxFileTitle = MAX_PATH;
       ofn.lpstrTitle = L"Select an input File";
       ofn.Flags = OFN_DONTADDTORECENT | OFN_FILEMUSTEXIST;
       if (GetOpenFileNameW(&ofn)) // user selected an input file
       {
       }
       else
       {
           // Get error
           WCHAR error[MAX_LOADSTRING];
           wsprintfW(error, L"%i", CommDlgExtendedError());
           MessageBoxW(NULL, error, L"ERROR", MB_OK);
       }
   }
   

   void GetInputFile()
   {
       CHAR szFileNameIN[MAX_PATH];
       CHAR szFileNameOUT[MAX_PATH];
   
       // get the input file name
       OPENFILENAMEA ofn;
       ZeroMemory(&fInputPath, sizeof(fInputPath));
       ZeroMemory(&ofn, sizeof(ofn));
       ofn.lStructSize = sizeof(ofn);
       ofn.hwndOwner = NULL;
       ofn.lpstrFilter = "Any File\0*.*\0";
       ofn.lpstrFile = fInputPath; // must be CHAR[]...
       ofn.nMaxFile = MAX_PATH;
       ofn.lpstrFileTitle = szFileNameIN;
       ofn.nMaxFileTitle = MAX_PATH;
       ofn.lpstrTitle = "Select an input File";
       ofn.Flags = OFN_DONTADDTORECENT | OFN_FILEMUSTEXIST;
       if (GetOpenFileNameA(&ofn)) // user selected an input file
       {
       }
       else
       {
           // Get error
           CHAR error[MAX_LOADSTRING];
           wsprintfA(error, "%i", CommDlgExtendedError());
           MessageBoxA(NULL, error, "ERROR", MB_OK);
       }
   }