Asynchronous NtQueryDirectoryFile?

Question

Has anyone managed to figure out how asynchronous calls to NtQueryDirectoryFile work?

By an asynchronous call, I mean calling NtQueryDirectoryFile on directories not opened with FILE_SYNCHRONOUS_IO_ALERT or with FILE_SYNCHRONOUS_IO_NONALERT.

For me, it seems to return STATUS_PENDING just like a normal NtReadFile request does on a normal file, but when I tried using NtWaitForSingleObject on the directory, it didn't end properly, and I still don't get all the data... why does this happen?

Answer 1

NtQueryDirectoryFile works well in asynchronous!

pass callback in ApcRoutine, and callback data in ApcContext

asynchronous procedure calls only call when the thread is in alertable state(for example: calling SleepEx(INFINITE, TRUE), WSAaccept)

this program shows how asynchronous NtQueryDirectoryFile work.

#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <stdio.h>
#include <winternl.h>
#include <winnt.h>

#define LIST_DIR_SIZE 2000
#define STATUS_NO_MORE_FILES ((NTSTATUS)80000006)

typedef struct _FILE_NAMES_INFORMATION {
    ULONG NextEntryOffset;
    ULONG FileIndex;
    ULONG FileNameLength;
    WCHAR FileName[1];
} FILE_NAMES_INFORMATION, * PFILE_NAMES_INFORMATION;

typedef struct {
    HANDLE hFile;
    OVERLAPPED ol;
    DECLSPEC_ALIGN(4) FILE_NAMES_INFORMATION buf[LIST_DIR_SIZE];
    IO_STATUS_BLOCK iob;
    bool finished;
} LIST_DIR_DATA, * PLIST_DIR_DATA; // my private data


__kernel_entry NTSYSCALLAPI
NTSTATUS
NTAPI
NtQueryDirectoryFile(
    _In_ HANDLE FileHandle,
    _In_opt_ HANDLE Event,
    _In_opt_ PIO_APC_ROUTINE ApcRoutine,
    _In_opt_ PVOID ApcContext,
    _Out_ PIO_STATUS_BLOCK IoStatusBlock,
    _Out_writes_bytes_(Length) PVOID FileInformation,
    _In_ ULONG Length,
    _In_ FILE_INFORMATION_CLASS FileInformationClass,
    _In_ BOOLEAN ReturnSingleEntry,
    _In_opt_ PUNICODE_STRING FileName,
    _In_ BOOLEAN RestartScan
);

#define NTDLL_extern(s) typedef decltype(&s) s##T;s##T s##F;
#define NTDLL_import(s) s##F = (s##T)GetProcAddress(ntdll, #s);

NTDLL_extern(NtOpenFile);
NTDLL_extern(NtQueryDirectoryFile);
NTDLL_extern(NtClose);
NTDLL_extern(RtlInitUnicodeString);

HMODULE ntdll;

VOID NTAPI callback(
    IN PVOID ApcContext,
    IN PIO_STATUS_BLOCK IoStatusBlock,
    IN ULONG Reserved) {

    UNREFERENCED_PARAMETER(Reserved);
    PFILE_NAMES_INFORMATION file_info = ((PLIST_DIR_DATA)ApcContext)->buf;
    do {
        fputws(file_info->FileName, stdout);
        putwchar(L'\t');
        file_info = (PFILE_NAMES_INFORMATION)((char*)file_info + file_info->NextEntryOffset);
    } while (file_info->NextEntryOffset);
    fputws(file_info->FileName, stdout);
    putwchar(L'\t');
    PLIST_DIR_DATA c = (PLIST_DIR_DATA)ApcContext;
    if (IoStatusBlock->Information != 0) {
        NTSTATUS status = NtQueryDirectoryFileF(
            c->hFile,
            NULL,
            callback,
            ApcContext,
            &c->iob,
            c->buf,
            sizeof(c->buf),
            FILE_INFORMATION_CLASS(12),
            FALSE, NULL, FALSE);
        switch (status) {
        case STATUS_PENDING:
            break;
        default:
            fputs("warning: status != STATUS_PENDING", stderr);
        }
    }
    else {
        c->finished = true;
    }
}
BOOL init() {
    ntdll = LoadLibraryW(L"NtDLL.dll");
    if (ntdll == NULL) {
        fputs("LoadLibraryW", stderr);
        return FALSE;
    }
    NTDLL_import(NtQueryDirectoryFile);
    NTDLL_import(NtOpenFile);
    NTDLL_import(NtClose);
    NTDLL_import(RtlInitUnicodeString);
    if (NtCloseF != NULL && NtOpenFileF != NULL && NtCloseF != NULL) {
        return TRUE;
    }
    else {
        fputs("GetProcAddress", stderr);
        return FALSE;
    }
}

int main() {
    if (init() == FALSE) {
        fputs("error: init() failed!", stderr);
        return -1;
    }
    NTSTATUS status;
    PLIST_DIR_DATA data = new LIST_DIR_DATA{};
    {
        OBJECT_ATTRIBUTES ObjectAttributes;
        UNICODE_STRING s;
        RtlInitUnicodeStringF(&s, L"\\??\\c:\\Windows\\System32");
        InitializeObjectAttributes(
            &ObjectAttributes,
            &s,
            OBJ_CASE_INSENSITIVE,
            NULL,
            NULL);
        status = NtOpenFileF(
            &data->hFile,
            FILE_READ_DATA | FILE_LIST_DIRECTORY, // | FILE_TRAVERSE | SYNCHRONIZE
            &ObjectAttributes,
            &data->iob,
            FILE_SHARE_READ,
            FILE_DIRECTORY_FILE); // | FILE_SYNCHRONOUS_IO_NONALERT
    }
    if (status < 0 || data->hFile == NULL) {
        fputs("error: NtOpenFile failed", stderr);
        return -2;
    }
    status = NtQueryDirectoryFileF(
        data->hFile,
        NULL,
        callback,
        data,
        &data->iob,
        data->buf,
        sizeof(data->buf),
        FILE_INFORMATION_CLASS(12),
        FALSE, NULL, FALSE);
    switch (status) {
        case STATUS_PENDING: 
            break;
        default:
            fputs("warning: status != STATUS_PENDING", stderr);
    }
    for (;data->finished==false;) SleepEx(INFINITE, TRUE); // put main thread into alertable wait

    NtCloseF(data->hFile);
    FreeLibrary(ntdll);
    return 0;
}

if you want UTF-8 output, try this (note: recommand use support UTF-8 terminal)

#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <winternl.h>
#include <winnt.h>
#include <crtdbg.h>
#include <cstdio>

#define LIST_DIR_SIZE 200
#define STATUS_NO_MORE_FILES ((NTSTATUS)80000006)

typedef struct _FILE_NAMES_INFORMATION {
    ULONG NextEntryOffset;
    ULONG FileIndex;
    ULONG FileNameLength;
    WCHAR FileName[1];
} FILE_NAMES_INFORMATION, * PFILE_NAMES_INFORMATION;

typedef struct {
    HANDLE hFile;
    OVERLAPPED ol;
    DECLSPEC_ALIGN(4) FILE_NAMES_INFORMATION buf[LIST_DIR_SIZE];
    IO_STATUS_BLOCK iob;
    bool finished;
} LIST_DIR_DATA, * PLIST_DIR_DATA; // my private data


__kernel_entry NTSYSCALLAPI
NTSTATUS
NTAPI
NtQueryDirectoryFile(
    _In_ HANDLE FileHandle,
    _In_opt_ HANDLE Event,
    _In_opt_ PIO_APC_ROUTINE ApcRoutine,
    _In_opt_ PVOID ApcContext,
    _Out_ PIO_STATUS_BLOCK IoStatusBlock,
    _Out_writes_bytes_(Length) PVOID FileInformation,
    _In_ ULONG Length,
    _In_ FILE_INFORMATION_CLASS FileInformationClass,
    _In_ BOOLEAN ReturnSingleEntry,
    _In_opt_ PUNICODE_STRING FileName,
    _In_ BOOLEAN RestartScan
);

#define NTDLL_extern(s) typedef decltype(&s) s##T;s##T s##F;
#define NTDLL_init(s) s##F = (s##T)GetProcAddress(ntdll, #s);

NTDLL_extern(NtOpenFile);
NTDLL_extern(NtQueryDirectoryFile);
NTDLL_extern(NtClose);
NTDLL_extern(RtlInitUnicodeString);

HMODULE ntdll;
HANDLE heap;

VOID NTAPI callback(
    IN PVOID ApcContext,
    IN PIO_STATUS_BLOCK IoStatusBlock,
    IN ULONG Reserved) {
    UNREFERENCED_PARAMETER(Reserved);
    PLIST_DIR_DATA c = (PLIST_DIR_DATA)ApcContext;
    if (IoStatusBlock->Information){
        PFILE_NAMES_INFORMATION file_info = c->buf;
        ULONG_PTR length = 0;
        ULONG last;
        do {
            last = file_info->NextEntryOffset;
            file_info->FileNameLength /= 2; // wide char length always base of 2 in bytes
            length += (
                file_info->FileIndex=WideCharToMultiByte(
                    CP_UTF8, WC_ERR_INVALID_CHARS, 
                    file_info->FileName, file_info->FileNameLength, 
                    NULL, 0, 
                    NULL, NULL)
                )+1;
            if (file_info->FileIndex == 0) { // FileIndex is how many byte is the UTF-8 string
                _RPTF0(_CRT_WARN, "WideCharToMultiByte failed!");
            }
            file_info = (PFILE_NAMES_INFORMATION)((char*)file_info + file_info->NextEntryOffset);
        } while (last);
        LPSTR pData = (LPSTR)HeapAlloc(heap, HEAP_NO_SERIALIZE, length), ptr=pData;
        if (ptr == NULL) {
            _RPTF0(_CRT_ERROR, "HeapAlloc failed!");
            return;
        }
        file_info = c->buf;
        do {
            last = file_info->NextEntryOffset;
            if (WideCharToMultiByte(
                CP_UTF8, WC_ERR_INVALID_CHARS,
                file_info->FileName, file_info->FileNameLength, 
                pData, file_info->FileIndex, 
                NULL, NULL)==0) {
                _RPTF0(_CRT_WARN, "WideCharToMultiByte failed!");
            }
            pData += file_info->FileIndex;
            *pData++ = '\n';
            file_info = (PFILE_NAMES_INFORMATION)((char*)file_info + file_info->NextEntryOffset);
        } while (last);

        // use data here
        fwrite(ptr, length, 1, stdout);
        // use data here

        HeapFree(heap, HEAP_NO_SERIALIZE, ptr);
        NTSTATUS status = NtQueryDirectoryFileF(
            c->hFile,
            NULL,
            callback,
            ApcContext,
            &c->iob,
            c->buf,
            sizeof(c->buf),
            FILE_INFORMATION_CLASS(12),
            FALSE, NULL, FALSE);

        switch (status) {
        case STATUS_PENDING:
            break;
        default:
            _RPTF0(_CRT_WARN, "status != STATUS_PENDING");
        }
    }else{
        c->finished = true;
    }
}
BOOL init() {
    ntdll = LoadLibraryW(L"NtDLL.dll");
    if (ntdll == NULL) {
        _RPTF0(_CRT_ERROR, "fail to load NtDLL.dll");
        return FALSE;
    }
    NTDLL_init(NtQueryDirectoryFile);
    NTDLL_init(NtOpenFile);
    NTDLL_init(NtClose);
    NTDLL_init(RtlInitUnicodeString);
    if (NtCloseF != NULL && 
        NtOpenFileF != NULL && 
        NtCloseF != NULL && 
        (heap = HeapCreate(HEAP_NO_SERIALIZE, 4096,0))!=NULL
        ){
        return TRUE;
    }
    else {
        _RPTF0(_CRT_ERROR, "failed to load function and create heap");
        return FALSE;
    }
}

int main() {
    if (init() == FALSE) {
        _RPTF0(_CRT_ERROR, "init failed");
        return -1;
    }
    SetConsoleCP(CP_UTF8);
    NTSTATUS status;
    PLIST_DIR_DATA data = new LIST_DIR_DATA{};
    {
        OBJECT_ATTRIBUTES ObjectAttributes;
        UNICODE_STRING s;
        RtlInitUnicodeStringF(&s, L"\\??\\c:\\Users");
        InitializeObjectAttributes(
            &ObjectAttributes,
            &s,
            OBJ_CASE_INSENSITIVE,
            NULL,
            NULL);
        status = NtOpenFileF(
            &data->hFile,
            FILE_READ_DATA | FILE_LIST_DIRECTORY, // | FILE_TRAVERSE | SYNCHRONIZE
            &ObjectAttributes,
            &data->iob,
            FILE_SHARE_READ,
            FILE_DIRECTORY_FILE); // | FILE_SYNCHRONOUS_IO_NONALERT
    }
    if (status < 0 || data->hFile == NULL) {
        _RPTF0(_CRT_ERROR, "NtOpenFile failed!");
        return -2;
    }
    status = NtQueryDirectoryFileF(
        data->hFile,
        NULL,
        callback,
        data,
        &data->iob,
        data->buf,
        sizeof(data->buf),
        FILE_INFORMATION_CLASS(12),
        FALSE, NULL, FALSE);
    switch (status) {
        case STATUS_PENDING: 
            break;
        default:
            _RPTF0(_CRT_WARN, "status != STATUS_PENDING");
    }
    for (;data->finished==false;) SleepEx(INFINITE, TRUE); // put main thread into alertable wait

    if (NtCloseF(data->hFile)<0) {
        _RPTF0(_CRT_ERROR, "NtClose failed!");
    }
    if (FreeLibrary(ntdll) == FALSE) {
        _RPTF0(_CRT_WARN, "failed to Free libary");
    }
    if (HeapDestroy(heap) == FALSE) {
        _RPTF0(_CRT_WARN, "fail to destroy heap");
    }
}

Answer 2

As far as I know, none of the Windows filesystems support asynchronous query directory calls. The Win32 APIs never call NtQueryDirectoryFile asnchronously, so support for it is hit-or-miss.

NTFS theoretically supports asynchronous NtQueryDirectoryFile but (as I mentioned) it is not extensively tested so it may not work.

You response indicated that you called WaitForSingleObject on the directory - that's not how the async pattern works in NT - you need to call WaitForSingleObject on the event handle provided as a parameter to NtQueryDirectoryFile.

This update is a result of asking the NTFS developer for more information, he tested this scenario on his machine and it worked for him (on Windows 7).