Find Two-byte Illegal Opcodes for x86-64

Question

My goal is to find what two-byte opcodes generate an illegal instruction exception. For example, opcodes 0F 0B UD2 raises an invalid opcode exception. The UD2 instruction is provided for software testing to explicitly generate an invalid opcode.

Warning Snake oil code ahead as I'm not familiar with Windows internals.

The code below allocates a 4K page with read/write/execute permissions and using UD2 as a starting point it tries to determine all the possible two-byte opcodes.

First, it copies the two-byte opcodes to the last two bytes of the 4K page

then executes them and checks for the exception code.

I figured that executing the last two page bytes would either

1. Generate an illegal exception EXCEPTION_ILLEGAL_INSTRUCTION with exactly two bytes.
2. Generate an access violation EXCEPTION_ACCESS_VIOLATION when extending beyond the 4K page.

Running the code below shows interesting instructions plus many unknowns too:

Illegal opcodes 0x0f 0x0b (error 0xc000001d)
ud2 - Generates an invalid opcode. 
   
Illegal opcodes 0x0f 0x37 (error 0xc000001d)
getsec - Exit authenticated code execution mode.

Illegal opcodes 0x0f 0xaa (error 0xc000001d)
rsm - Resume operation of interrupted program.

Question

The hack'ish code runs fine in this opcode range

Executing opcodes 0x0f 0x0b ... Executing opcodes 0x0f 0xcb

until it encounters these two opcodes

0x0f 0xcc bswap esp

It seems anything that manipulates the stack pointer causes issues whereby it's stuck at this point (clicking Continue just repeats the message)

I've tried moving the opcode execution into its own thread since they have their own stack, but that didn't help!

Is there a way to preserve the stack pointers RSP and RBP or maybe there's a simple fix to resolve it?

(Built using M$ Visual C++ 2019)

#include <windows.h>
#include <stdio.h>
#include <string.h>
#include <string.h>
#include <intrin.h>

// The UD2 (0x0F, 0x0B) instruction is guaranteed to generate an invalid opcode exception.

DWORD InstructionResult;

void ExecuteOpcodes(LPVOID mem)
{
    __try
    {
        // Execute opcodes...
        ((void(*)())((unsigned char*)mem + 0xFFE))();
    }
    __except (EXCEPTION_EXECUTE_HANDLER)
    {
        InstructionResult = GetExceptionCode();
    }
}

int main()
{
    LPVOID mem = VirtualAlloc(NULL, 2, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE);
    DWORD oldProtect = VirtualProtect(mem, 2, PAGE_EXECUTE_READWRITE, &oldProtect);

    // Start searching at the UD2 (0x0F, 0x0B) instruction which is guaranteed to generate an invalid opcode exception.
    for (int i = 15; i <= 255; i++)
    {
        for (int j = 11; j <= 255; j++)
        {
            // Write two byte opcodes at the 4K page end.
            *((unsigned char*)mem + 0xFFE) = i;
            *((unsigned char*)mem + 0xFFF) = j;

            printf("Executing opcodes 0x%02x 0x%02x\n",i,j);
            HANDLE hThread = CreateThread(0, 0, (LPTHREAD_START_ROUTINE)ExecuteOpcodes, mem, 0, 0);
            WaitForSingleObject(hThread, INFINITE);
            CloseHandle(hThread);
            if (InstructionResult == EXCEPTION_ILLEGAL_INSTRUCTION)
            {
                printf("Illegal opcodes 0x%02x 0x%02x (error 0x%08x)\n", i, j, InstructionResult);
            }
        }
    }

    VirtualFree(mem, 0, MEM_RELEASE);

    return 0;
}

UPDATE

Based upon the good answer about creating a child process, I've pasted the updated code here: pastebin.com/j3NkL44q

#define _CRT_SECURE_NO_WARNINGS
#include <windows.h>
#include <stdio.h>

// Array of illegal single-byte opcodes
int IllegalSingleByteOpcodes[] = { 0x06 ,0x07, 0x0e, 0x16, 0x17, 0x1e, 0x1f, 0x27, 0x2f, 0x37, 0x3f, 0x60, 0x61, 0xce, 0xd6 };

// Check if a given opcode is illegal
bool is_illegal_opcode(int opcode) {
    for (size_t i = 0; i < sizeof(IllegalSingleByteOpcodes) / sizeof(IllegalSingleByteOpcodes[0]); i++) {
        if (IllegalSingleByteOpcodes[i] == opcode) {
            return true;
        }
    }
    return false;
}

// Create and wait for a process with the given opcodes
void create_and_wait_for_process(int opcode1, int opcode2) {
    // Set up startup and process info
    STARTUPINFO si;
    PROCESS_INFORMATION pi;
    ZeroMemory(&si, sizeof(si));
    si.cb = sizeof(si);
    ZeroMemory(&pi, sizeof(pi));

    // Create command line string
    char cmdline[256];
    snprintf(cmdline, sizeof(cmdline), "IllegalOpcodes.exe %d %d", opcode1, opcode2);

    // Create process
    if (!CreateProcess(NULL, cmdline, NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) {
        printf("CreateProcess failed (%d).\n", GetLastError());
        exit(1);
    }

    // Wait for process to finish
    if (WaitForSingleObject(pi.hProcess, 1000) != WAIT_OBJECT_0) {
        TerminateProcess(pi.hProcess, 0);
    }

    // Clean up handles
    CloseHandle(pi.hProcess);
    CloseHandle(pi.hThread);
}

int main() {
    // Iterate through all possible opcode pairs
    for (int opcode1 = 0; opcode1 <= 255; opcode1++) {
        // Skip illegal opcodes
        if (is_illegal_opcode(opcode1)) {
            printf("\nSkipping Illegal Opcode 0x%02x ...\n", opcode1);
            continue;
        }
        printf("\nChecking Opcode 0x%02x ...\n", opcode1);

        for (int opcode2 = 0; opcode2 <= 255; opcode2++) {
            create_and_wait_for_process(opcode1, opcode2);
        }
    }

    return 0;
}


------------------- IllegalOpcodes.cpp -------------------


#include <windows.h>
#include <stdio.h>

// Offset to write opcodes at the 4K page end
#define CODE_PAGE_END_OFFSET 0xFFE

// Write two byte opcodes at the 4K page end and execute them
void write_and_execute_opcodes(LPVOID code_page_mem, int opcode1, int opcode2)
{
    *((unsigned char*)code_page_mem + CODE_PAGE_END_OFFSET) = opcode1;
    *((unsigned char*)code_page_mem + CODE_PAGE_END_OFFSET + 1) = opcode2;

    __try
    {
        // Execute opcodes...
        ((void(*)())((unsigned char*)code_page_mem + CODE_PAGE_END_OFFSET))();
    }
    __except (EXCEPTION_EXECUTE_HANDLER)
    {
        switch (GetExceptionCode()) {
            case EXCEPTION_ILLEGAL_INSTRUCTION:
                printf("{0x%02x,0x%02x},", opcode1, opcode2);
                break;
            default:
                // Ignore other exceptions
                break;
        }
    }
}

int main(int argc, char* const argv[])
{
    int opcode1 = atoi(argv[1]);
    int opcode2 = atoi(argv[2]);

    LPVOID code_page_mem = VirtualAlloc(NULL, 2, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE);

    DWORD old_protect = VirtualProtect(code_page_mem, 2, PAGE_EXECUTE_READWRITE, &old_protect);

    write_and_execute_opcodes(code_page_mem, opcode1, opcode2);

    VirtualFree(code_page_mem, 0, MEM_RELEASE);

    return 0;
}

It now includes checks to avoid these single byte illegal opcodes

32-bit Legal

06              push    es
07              pop     es
0e              push    cs
16              push    ss
17              pop     ss
1e              push    ds
1f              pop     ds
27              daa
2f              das
37              aaa
3f              aas
60              pushad
61              popad
ce              into
d6              ???  <--- http://ref.x86asm.net/coder64.html#xD6

64-bit Illegal

06              ???
07              ???
0e              ???
16              ???
17              ???
1e              ???
1f              ???
27              ???
2f              ???
37              ???
3f              ???
60              ???
61              ???
ce              ???
d6              ???

Answer 1

maybe there's a simple fix to resolve it?

The UNIX-standard way to resolve this is to do all the test execution in a child process.

When I last worked on Windows 15 years ago, creating a child process was very expensive (slow). But since you have fewer that 64K byte combinations to try, even a slow mechanism will get you all the answers in at most a few hours.