ExecuteIndirect crashed on AMD GPU when expanding indirect command buffer

Question

I try to use ExecuteIndirect for drawing my instances.

Here is my code:

struct IndirectCommand
{
    D3D12_GPU_VIRTUAL_ADDRESS materialBufferAddress;
    D3D12_GPU_VIRTUAL_ADDRESS instanceBufferAddress;
    D3D12_DRAW_INDEXED_ARGUMENTS drawArguments;
};  // byte stride: 40

// code for initializing command signature
void InstanceManager::InitIndirectBuffer()
{
    D3D12_INDIRECT_ARGUMENT_DESC indirectDescs[3] = {};
    indirectDescs[0].Type = D3D12_INDIRECT_ARGUMENT_TYPE_CONSTANT_BUFFER_VIEW;
    indirectDescs[0].ConstantBufferView.RootParameterIndex = cMaterialPass;
    indirectDescs[1].Type = D3D12_INDIRECT_ARGUMENT_TYPE_SHADER_RESOURCE_VIEW;
    indirectDescs[1].ShaderResourceView.RootParameterIndex = cInstancePass;
    indirectDescs[2].Type = D3D12_INDIRECT_ARGUMENT_TYPE_DRAW_INDEXED;

    D3D12_COMMAND_SIGNATURE_DESC commandSignatureDesc = {};
    commandSignatureDesc.pArgumentDescs = indirectDescs;
    commandSignatureDesc.NumArgumentDescs = _countof(indirectDescs);
    commandSignatureDesc.ByteStride = sizeof(IndirectCommand);

    ThrowIfFailed(Engine::GetApp()->GetDevice()->CreateCommandSignature(&commandSignatureDesc, Engine::GetApp()->GetRootSignature(), IID_PPV_ARGS(&mCommandSignature)));
}

Appending indirect command buffer:

void AppendIndirectCommandBuffer()
{
    // wait for gpu
    Engine::GetApp()->ResetCommandList();
    Engine::GetApp()->ExecuteCommand();
    Engine::GetApp()->FlushCommandQueue();

    for (int i = 0; i < gNumFrameResources; i++)
    {
        // alloc upload heap
        Engine::GetApp()->GetFrameManager().GetFrameResource(i)->AppendUploadBuffer<IndirectCommand>(mIndirectBufferUpload[i]
            , mLastIndirectUploadCount[i]
            , mIndirectCount
            , false);

        // alloc default heap
        Engine::GetApp()->GetFrameManager().GetFrameResource(i)->AppendDefaultBuffer<IndirectCommand>(mIndirectBufferDefault[i]
            , mLastIndirectDefaultCount[i]
            , mIndirectCount++
            , false);
    }
    UpdateIndirectData(_obj);
    // code for appending indirect command buffer 
}

Update indirect command buffer data:

void UpdateIndirectData(GameObject _obj)
{
    UINT matCBByteSize = d3dUtil::CalcConstantBufferByteSize(sizeof(MaterialData));
    UINT insCBByteSize = sizeof(InstanceData);

    for (int i = 0; i < gNumFrameResources; i++)
    {
        if (Engine::GetApp()->GetFrameManager().GetFrameResource(i) != NULL)
        {
            string _matName = _obj->GetComponent<RenderObject>()->GetMaterialName();
            string _geoName = _obj->GetComponent<RenderObject>()->GetGeometryName();
            Material *mat = Engine::GetApp()->GetMaterialManager().GetMaterial(_matName);

            if (mat != nullptr)
            {
                IndirectCommand data;

                data.materialBufferAddress = Engine::GetApp()->GetMaterialManager().GetMaterialBuffer(i)->Resource()->GetGPUVirtualAddress()
                + matCBByteSize*mat->GetMatBufferIndex();

                data.instanceBufferAddress = mInstanceBuffer[i]->Resource()->GetGPUVirtualAddress() + mInstanceIndex[_obj->GetID()] * insCBByteSize;

                data.drawArguments.BaseVertexLocation = mDrawArgs[_geoName].BaseVertexLocation;
                data.drawArguments.IndexCountPerInstance = mDrawArgs[_geoName].IndexCount;
                data.drawArguments.StartIndexLocation = mDrawArgs[_geoName].StartIndexLocation;
                data.drawArguments.StartInstanceLocation = 0;
                data.drawArguments.InstanceCount = 1;

                int indirectIndex = mIndirectIndex[_obj->GetID()];
                mIndirectCommand[indirectIndex] = data;    // an array of indirect command, size is set to 1000 temporarily

                // copy to default heap
                D3D12_SUBRESOURCE_DATA commandData = {};
                commandData.pData = reinterpret_cast<UINT8*>(&mIndirectCommand[0]);
                commandData.RowPitch = sizeof(IndirectCommand) * mIndirectCount;
                commandData.SlicePitch = commandData.RowPitch;
                UpdateSubresources<1U>(Engine::GetApp()->GetCommandList(), mIndirectBufferDefault[i]->Resource(), mIndirectBufferUpload[i]->Resource(), 0, 0, 1, &commandData);
            }
        }
    }
}

Last, executeindirect

void DrawIndirectInstance()
{
    Engine::GetApp()->GetCommandList()->IASetVertexBuffers(0, 1, &mGeometry->VertexBufferView());
    Engine::GetApp()->GetCommandList()->IASetIndexBuffer(&mGeometry->IndexBufferView16());
    Engine::GetApp()->GetCommandList()->IASetPrimitiveTopology(D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST);

    // just set a pso for testing, I haven't group my instances by material yet
    ID3D12PipelineState *pso = Engine::GetApp()->GetMaterialManager().GetMaterialPSO("test");
    Engine::GetApp()->GetCommandList()->SetPipelineState(pso);

    int frameIndex = Engine::GetApp()->GetFrameManager().GetCurrFrameIndex();
    auto indirectBuffer = mIndirectBufferUpload[frameIndex].get();

    auto materialBuffer = Engine::GetApp()->GetMaterialManager().GetMaterialBuffer(frameIndex);
    Engine::GetApp()->GetCommandList()->SetGraphicsRootConstantBufferView(cMaterialPass, materialBuffer->Resource()->GetGPUVirtualAddress());
    Engine::GetApp()->GetCommandList()->SetGraphicsRootShaderResourceView(cInstancePass, mInstanceBuffer[frameIndex]->Resource()->GetGPUVirtualAddress());

    Engine::GetApp()->GetCommandList()->ResourceBarrier(1
    , &CD3DX12_RESOURCE_BARRIER::Transition(indirectBuffer->Resource(), D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_INDIRECT_ARGUMENT));

    Engine::GetApp()->GetCommandList()->ExecuteIndirect(mCommandSignature.Get()
    , mIndirectCount
    , indirectBuffer->Resource()
    , 0
    , nullptr
    , 0);

    Engine::GetApp()->GetCommandList()->ResourceBarrier(1
    , &CD3DX12_RESOURCE_BARRIER::Transition(indirectBuffer->Resource(), D3D12_RESOURCE_STATE_INDIRECT_ARGUMENT, D3D12_RESOURCE_STATE_COPY_DEST));
}

I use my SceneManager to load a scene. If a gameobject is a render object. My system will call AppendIndirectCommandBuffer() and copy data into indirect command buffer.

ExecuteIndirect() will work great if I do not clone my gameobject after initialization of my scene.

And I try to clone my gameobject at runtime(in my Update()).

My system calls AppendIndirectCommandBuffer() again to resize indirect command buffer and copies new data to buffer.

After cloning gameobject for a few second, my system starts to glitch and result in TDR on my R9 380.

But works properly on both WARP and intel gpu.

And I don't know why :(.

How to solve this problem? I really want to make good use of indirect drawing.

Thanks!

Update

A upload heap resource can't transit to D3D12_RESOURCE_STATE_INDIRECT_ARGUMENT.

I create a default type heap, and use upload heap to copy data to it.

I tried debug layer, no errors are returned from it. I tried GBV and only got two errors:

D3D12 ERROR: GPU-BASED VALIDATION: Draw, Uninitialized root argument accessed. Shader Stage: VERTEX, Root Parameter Index: [2], Draw Index: [0], Shader Code: Forward.hlsl(21,2-42), Asm Instruction Range: [0x114-0x13b], Asm Operand Index: [3], Command List: 0x00000134BB3C1EE0:'Unnamed ID3D12GraphicsCommandList Object', SRV/UAV/CBV Descriptor Heap: 0x00000134AE6DD020:'Unnamed ID3D12DescriptorHeap Object', Sampler Descriptor Heap: <not set>, Pipeline State: 0x00000134BB5CB540:'Unnamed ID3D12PipelineState Object',  [ EXECUTION ERROR #935: GPU_BASED_VALIDATION_ROOT_ARGUMENT_UNINITIALIZED]
D3D12 ERROR: GPU-BASED VALIDATION: Draw, Uninitialized root argument accessed. Shader Stage: VERTEX, Root Parameter Index: [1], Draw Index: [0], Shader Code: Forward.hlsl(37,2-41), Asm Instruction Range: [0x8c0-0x8e3], Asm Operand Index: [2], Command List: 0x000001D0D7B70860:'Unnamed ID3D12GraphicsCommandList Object', SRV/UAV/CBV Descriptor Heap: 0x000001D0D7AC6C80:'Unnamed ID3D12DescriptorHeap Object', Sampler Descriptor Heap: <not set>, Pipeline State: 0x000001D0D7BBF450:'Unnamed ID3D12PipelineState Object',  [ EXECUTION ERROR #935: GPU_BASED_VALIDATION_ROOT_ARGUMENT_UNINITIALIZED]

And a warning:

D3D12 WARNING: ID3D12CommandList::ExecuteIndirect: GPU-based validation is not supported for ExecuteIndirect that changes root bindings. All further GPU-based validation output may not be reliable. [ EXECUTION WARNING #1000: GPU_BASED_VALIDATION_UNSUPPORTED]

After I call the following functions, these two errors are gone.

Engine::GetApp()->GetCommandList()->SetGraphicsRootConstantBufferView(cMaterialPass, materialBuffer->Resource()->GetGPUVirtualAddress());
Engine::GetApp()->GetCommandList()->SetGraphicsRootShaderResourceView(cInstancePass, mInstanceBuffer[frameIndex]->Resource()->GetGPUVirtualAddress());

Despite making these modification, it still doesn't work properly.

Answer 1

Finally, I find the reason!

Bug is from my instance data buffer.

Engine::GetApp()->GetFrameManager().GetFrameResource(i)->AppendUploadBuffer<InstanceData>(mInstanceBuffer[i]
            , mLastInstanceCount[i]
            , mInstanceCount
            , false);

After calling this, memory address is changed. If I set indirect command buffer data with the following code, GPU will crash.

data.instanceBufferAddress = mInstanceBuffer[i]->Resource()->GetGPUVirtualAddress() + mInstanceIndex[_obj->GetID()] * insCBByteSize;

So I need to either create a fixed-size instance buffer at initialization (address won't change) or update all previous indirect command data after cloning my gameobject (would kill performance).

Sorry for asking a stupid question :$.