/*========================== begin_copyright_notice ============================

Copyright (C) 2019-2021 Intel Corporation

SPDX-License-Identifier: MIT

============================= end_copyright_notice ===========================*/

//===----------------------------------------------------------------------===//
///
/// This file contains a definition of the structure of the ray tracing
/// stack as seen by a ray.
///
//===----------------------------------------------------------------------===//
#pragma once

#include "API/RayDispatchGlobalData.h"

#include <stdint.h>
#include <stddef.h>


// This code defines a set of macros, that allows us to intentionally disable certain warnings
// in a way that is both concise and portable across compilers.
// We need to disable the warnings in such way because this C++ file may be compiled with Clang, GCC, and Microsoft Visual Studio Compiler
// This works around (at least) two known problems:
//  - QuickBuild treats compiler warnings as errors
//  - RTStackReflection.exe generates warnings when compiling this file
//
// The only macros that are used by the main code are:
//     DISABLE_WARNING_PUSH
//     DISABLE_WARNING_POP
//     and the macros for the individual warnings
// The others are lower level internal utilities, used only by higher level macros.
//
// Explanation of code in original resource : https://www.fluentcpp.com/2019/08/30/how-to-disable-a-warning-in-cpp/
#if defined(__clang__)
    #define DO_PRAGMA_DIAG(X) _Pragma(#X)                                                     // internal utility
    #define DISABLE_WARNING_PUSH         DO_PRAGMA_DIAG(GCC diagnostic push)
    #define DISABLE_WARNING_POP          DO_PRAGMA_DIAG(GCC diagnostic pop)
    #define DISABLE_WARNING(warningName) DO_PRAGMA_DIAG(GCC diagnostic ignored #warningName)  // internal utility
    // Add warnings that you want to disable here:
    #define DISABLE_WARNING_ANONYMOUS_STRUCT_UNION   \
        DISABLE_WARNING(-Wgnu-anonymous-struct)  // anonymous structs are a GNU extension
    #define DISABLE_WARNING_PADDING_AT_END_OF_STRUCT
    #define DISABLE_WARNING_ANON_TYPES_IN_ANON_UNION \
        DISABLE_WARNING(-Wnested-anon-types)     // anonymous types declared in an anonymous union are an extension
#elif defined(__GNUC__)
    #define DO_PRAGMA_DIAG(X) _Pragma(#X)                                                     // internal utility
    #define DISABLE_WARNING_PUSH         DO_PRAGMA_DIAG(GCC diagnostic push)
    #define DISABLE_WARNING_POP          DO_PRAGMA_DIAG(GCC diagnostic pop)
    // Treats only this header file as a system header, AKA disables all warnings in this header file only.
    // The effect does not extend into any file that includes this header file.
    #pragma GCC system_header
    // define the macros that are used in the code down below to prevent compiler failure
    // NOTE: Apparently GCC compiler doesn't support disabling individual warnings.
    #define DISABLE_WARNING_ANONYMOUS_STRUCT_UNION
    #define DISABLE_WARNING_PADDING_AT_END_OF_STRUCT
    #define DISABLE_WARNING_ANON_TYPES_IN_ANON_UNION
#elif defined(_MSC_VER)
    #define DISABLE_WARNING_PUSH           __pragma(warning( push ))
    #define DISABLE_WARNING_POP            __pragma(warning( pop ))
    #define DISABLE_WARNING(warningNumber) __pragma(warning( disable : warningNumber ))  // internal utility
    // Add warnings that you want to disable here:
    #define DISABLE_WARNING_ANONYMOUS_STRUCT_UNION   \
        DISABLE_WARNING(4201)  // nonstandard extension used: nameless struct/union
    #define DISABLE_WARNING_PADDING_AT_END_OF_STRUCT \
        DISABLE_WARNING(4820)  // 'MemTravStack::<unnamed-tag>': '4' bytes padding added after data member 'MemTravStack::<unnamed-tag>::offset'
    #define DISABLE_WARNING_ANON_TYPES_IN_ANON_UNION
#else
    // define the macros that are used in the code down below to prevent compiler failure
    // NOTE: internal utility macros should not be defined here
    #define DISABLE_WARNING_PUSH
    #define DISABLE_WARNING_POP
    // Add warnings that you want to disable here:
    #define DISABLE_WARNING_ANONYMOUS_STRUCT_UNION
    #define DISABLE_WARNING_PADDING_AT_END_OF_STRUCT
    #define DISABLE_WARNING_ANON_TYPES_IN_ANON_UNION
#endif


DISABLE_WARNING_PUSH       // save the current pragma state, save the current compiler settings
// select the warnings that we want to disable, for this file only
DISABLE_WARNING_ANONYMOUS_STRUCT_UNION
DISABLE_WARNING_PADDING_AT_END_OF_STRUCT
DISABLE_WARNING_ANON_TYPES_IN_ANON_UNION


namespace IGC {

// will be patched with the global root signature at compile time
struct alignas(8) TypeHoleGlobalRootSig
{
    char __Padding[16];
};

// This is currently all of the cross-thread constant data that will be populated
// in the indirect data by the UMD (See D3D12RaytracingDispatch.h for more info).
struct RTGlobalsAndRootSig
{
    RayDispatchGlobalData RTGlobals;
    TypeHoleGlobalRootSig GlobalRootSig;
};

} // namespace IGC

namespace RTStackFormat {

uint32_t getRTStackHeaderSize(uint32_t MaxBVHLevels);

constexpr static uint32_t MAX_BVH_LEVELS = 2;
constexpr static uint32_t MEM_STACK_SIZE = 4;

// DXR uses two BVH levels: a top-level acceleration structure and a
// bottom-level acceleration structure.
constexpr static uint32_t TOP_LEVEL_BVH    = 0;
constexpr static uint32_t BOTTOM_LEVEL_BVH = 1;

// Auxiliary types to make update of the structs easier.
using uint3  = uint32_t[3];
using uint4  = uint32_t[4];
using Vec3f  = float[3];
using float4 = float[4];

static_assert(sizeof(uint3) == 12, "size mismatch");
static_assert(sizeof(uint4) == 16, "size mismatch");
static_assert(sizeof(Vec3f) == 12, "size mismatch");
static_assert(sizeof(float4) == 16, "size mismatch");

// This is taken from the functional model.  It's not clear that we have an
// immediate use to read these fields but it is useful to have here for
// documentation purposes.
struct KSP
{
    /* returns the base address of the shader record this KSP is part of */
    void* getShaderRecord() {
        return (char*)this + 8 * _offset - 32; // ShaderIdentifier is 32 bytes large
    }

    /* checks if this KSP is a NULL KSP */
    bool isNull() const {
        return *(uint64_t*)this == NullValue;
    }

    static constexpr uint64_t NullValue = 0;

public:
    uint64_t _offset               : 3;   // offset in 8-byte blocks to the start of shader parameters
    uint64_t _push_constant_enable : 1;   // If this field is enabled,
                                          // 8DWs are pushed from the Local
                                          // Arguments at 32B offset from the
                                          // start of the shader record address.

    uint64_t _dispatch_mode        : 1;   // 0 = SIMD16, 1 = SIMD8
    uint64_t _reserved0            : 1;
    uint64_t _shader               : 26;  // shader function pointer
    uint64_t _reserved1            : 32;  // unused padding bytes
};

static_assert(sizeof(KSP) == 8, "changed?");


struct NodeInfo
{
    enum class Bits : uint8_t
    {
        type     = 3,
        parent   = 1,

        cur_prim = 4,
    };

    using T = uint32_t;

    /* NodeType */
    uint8_t  type     : (T)Bits::type;
    uint8_t  parent   : (T)Bits::parent;   // Indicates a culled stack entry where a single parent node
                                           // is stored in place of multiple child nodes
    uint8_t  cur_prim : (T)Bits::cur_prim;
};

/**
 * MemTravStack originally used to have two different definitions in two different files.
 * Here these two definitions are merged into one for preserving backwards compatibility.
 *
 * Basically these two definitions have different internal layouts, different fields, but they have the same size.
 * So we can put these two definitions inside an anonymous union inside the main structure definition.
 * The anonymous union allows both these definitions to be accessed at once.
 * And the anonymous structs force all the fields in a particular definition to be non-overlapping.
 * Since the NoneInfo nodeInfo[MEM_STACK_SIZE] cannot be defined in both these anonymous structs, it doesn't matter which one we put that field in.
 *
 * Any one of these fields can be used, as long as one is consistent with which "overlapping definition" is being used.
 * Or both can be used at once for type punning, to set the value of the full variable, and then get a particular sub-section of that value as a bit field.
 * We maybe having code which accesses the fields in the first definition, and also code somewhere else in the project which accesses the fields in the second definition.
 */
struct StackEntry
{
    enum class Bits : uint8_t
    {
        offset    = 31,
        lastChild = 1,
    };

    using T = uint32_t;

    uint32_t offset    : (T)Bits::offset;    // in multiples of 64B. max 2^29 prims i.e. max 2^30 nodes. one extra bit
    uint32_t lastChild : (T)Bits::lastChild;
};

struct MemTravStack
{
    union {
        ///  The original definition from RTStackFormat.h
        struct {
            uint32_t indexArray0;   // Restart information for the stack. Described in the background document.
            uint32_t indexArray1;
            uint32_t indexArray2;
            StackEntry stackEntry[MEM_STACK_SIZE];
            NodeInfo nodeInfo[MEM_STACK_SIZE];
        };

        ///  The original definition from ocl_raytracing_structures.h
        struct {
            uint64_t curDepth      : 5;        // current depth in the restart trail
            uint64_t restartTrail0 : 29;       // lower bits of restart trail
            uint64_t restartTrail1 : 29;       // higher bits of restart trail
            uint64_t lastChild0    : 1;        // last child bit for node 0
            uint32_t restartTrail2 : 29;       // highest bits of restart trail
            uint32_t lastChild123  : 3;        // last child bit for nodes 1/2/3

            int32_t offset[MEM_STACK_SIZE];    // Signed offset relative to BVH root in multiples of 64B.
            NodeInfo pad[MEM_STACK_SIZE];      // padding to make sure two structs of this union are the same size
        };
    };
};

// Define structs Xe, used to specialize other template structs, like MemHit or RTStack, via GenT arg
#define STYLE(X) struct X;
#include "RayTracingMemoryStyle.h"
#undef STYLE

template <typename GenT> struct MemHit;
template <typename GenT> struct MemRay;
template <typename GenT> struct PrimLeafDesc;
template <typename GenT> struct QuadLeaf;
template <typename GenT> struct InstanceLeaf;

template <>
struct MemHit<Xe>
{
    float    t;                    // hit distance of current hit (or initial traversal distance)
    float    u, v;                 // barycentric hit coordinates

    enum class Bits : uint8_t
    {
        primIndexDelta  = 16,
        valid           = 1,
        leafType        = 3,
        primLeafIndex   = 4,
        bvhLevel        = 3,
        frontFace       = 1,
        done            = 1,
        pad0            = 3,

        primLeafPtr     = 42,
        hitGroupRecPtr0 = 22,

        instLeafPtr     = 42,
        hitGroupRecPtr1 = 22,
    };

    // This is the offset within the bitfield
    enum class Offset : uint8_t
    {
        // Add as needed
        done = 28,
    };

    using T = uint32_t;

    uint32_t primIndexDelta : (T)Bits::primIndexDelta; // prim index delta for compressed meshlets and quads
    uint32_t valid          : (T)Bits::valid;          // set if there is a hit
    uint32_t leafType       : (T)Bits::leafType;       // type of node primLeafPtr is pointing to
    uint32_t primLeafIndex  : (T)Bits::primLeafIndex;  // index of the hit primitive inside the leaf
    uint32_t bvhLevel       : (T)Bits::bvhLevel;       // the instancing level at which the hit occured
    uint32_t frontFace      : (T)Bits::frontFace;      // whether we hit the front-facing side of a triangle (also used to pass opaque flag when calling intersection shaders)
    uint32_t done           : (T)Bits::done;           // used in sync mode to indicate that traversal is done (HW will only set this to 0)
    uint32_t pad0           : (T)Bits::pad0;           // unused bits (explicit padding)

    uint64_t primLeafPtr     : (T)Bits::primLeafPtr;     // pointer to BVH leaf node (multiple of 64 bytes)
    uint64_t hitGroupRecPtr0 : (T)Bits::hitGroupRecPtr0; // LSB of hit group record of the hit triangle (multiple of 16 bytes)

    uint64_t instLeafPtr     : (T)Bits::instLeafPtr;     // pointer to BVH instance leaf node (in multiple of 64 bytes)
    uint64_t hitGroupRecPtr1 : (T)Bits::hitGroupRecPtr1; // MSB of hit group record of the hit triangle (multiple of 16 bytes)

    static_assert((uint32_t)Bits::primLeafPtr == (uint32_t)Bits::instLeafPtr,
        "Size changed?");
};
static_assert(sizeof(MemHit<Xe>) == 32, "MemHit has to be 32 bytes large");

template <>
struct MemRay<Xe>
{
    // 32 B
    Vec3f org;         // the origin of the ray
    Vec3f dir;         // the direction of the ray
    float tnear;       // the start of the ray
    float tfar;        // the end of the ray

    using T = uint32_t;
    enum class Bits : uint8_t
    {
        rootNodePtr           = 48,
        rayFlags              = 16,

        hitGroupSRBasePtr     = 48,
        hitGroupSRStride      = 16,

        missSRPtr             = 48,
        pad                   = 1,
        shaderIndexMultiplier = 8,

        instLeafPtr           = 48,
        rayMask               = 8,

        ComparisonValue       = 7,
        pad2                  = 8,
    };

    // 32 B

    // RayFlags:
    //
    // There are 3 sets of flags, from the application point of view,
    // available during the ray traversal:
    // 1. RayFlags set in the shader before the TraceRay functions is called.
    // 2. Flags applied to the entire Pipeline.
    // 3. Flags applied per instances and geometry in the BVH
    //
    // All the Ray data is stored in the MemRayStructures which have two instances:
    // 1. TopLevel (Ray0)
    // 2. BottomLevel (Ray1)
    //
    // TopLevel is written by SW when the Ray is cast (before calling TraceRay).
    // When the traversal reaches the instance and enters related BottomLevel BVH,
    // TopLevel MemRay is copied by HW. Flags from entered instance are applied to
    // the RayFlags.
    // The BottomLevel MemRay is written only by HW, when AnyHit or Intersection
    // Shaders are called.
    //
    // As HW can only apply flags from TracRay (1) and the BVH (3), flags from
    // the Pipeline (2) has to applied by SW. It is done by applying them to
    // the flags set before the TraceRay (1). But the application can read
    // both the Pipeline flags (2) and the RayFlags (1) independently.
    // So the original RayFlags (1) are additionally stored in the MemRay memory.
    // As there is no dedicated space for them, the instLeafPtr from TopLevel MemRay
    // is used, as it is irrelevant to the traversal at the TopLevel.
    //
    // When the AnyHit or Intersection shader are called, there are 3 sets of flags
    // available:
    // 1. RayFlags combined with Pipeline Flags - Ray0.rayFlags
    // 2. RayFlags - Ray0.instLeafPtr aka Ray0.flagsFromTraceRay.rayFlags
    // 3. RayFlags + Pipeline Flags + Instance/GeometryFlags - Ray1.rayFlags
    //
    // rayFlags defined here is used to access:
    // 1. Ray0.rayFlags
    // 2. Ray1.rayFlags
    //
    // Additionally an alias for 16-bit access is added.
    // As we must avoid structure members alignment, to properly generate
    // offsets in MemRayStructure, rayFlags has 64bit type. To simplify the
    // access, a 16bit alias is created (rayFlags16BitAccessAlias.rayFlags).
    // It points to same part of MemHit structure as just rayFlags.
    union
    {
        struct
        {
            uint16_t uw0;
            uint16_t uw1;
            uint16_t uw2;
            uint16_t rayFlags;
        } rayFlags16BitTypeAlias;

        struct
        {
            uint64_t rootNodePtr : (T)Bits::rootNodePtr; // root node to start traversal at
            uint64_t rayFlags : (T)Bits::rayFlags;    // ray flags (see RayFlags structure)
        };
    };

    uint64_t hitGroupSRBasePtr : (T)Bits::hitGroupSRBasePtr; // base of hit group shader record array (16-bytes alignment)
    uint64_t hitGroupSRStride  : (T)Bits::hitGroupSRStride;  // stride of hit group shader record array (16-bytes alignment)

    uint64_t missSRPtr             : (T)Bits::missSRPtr;             // pointer to miss shader record to invoke on a miss (8-bytes alignment)
    // DG2
    //uint64_t pad                 : 8;                              // explicit padding bits
    uint64_t pad                   : (T)Bits::pad;                   // explicit padding bits
    uint64_t ComparisonValue       : (T)Bits::ComparisonValue;       // to be compared with Instance.ComparisonValue
    uint64_t shaderIndexMultiplier : (T)Bits::shaderIndexMultiplier; // shader index multiplier

    union
    {
        // This just an alias for 16-bit type.
        // As we must avoid structure members alignment, to properly generate
        // offsets in MemRayStructure, rayFlags has 64bit type. To simplify the
        // access, a 16bit alias is created (Ray0.flagsFromTraceRay16BitTypeAlias.rayFlags).
        // It points to same part of MemHit structure as just Ray0.flagsFromTraceRay.rayFlags.
        struct
        {
            uint16_t rayFlags;
            uint16_t uw1;
            uint16_t uw2;
            uint16_t uw3;
        } flagsFromTraceRay16BitTypeAlias;

        struct
        {
            // This is just an alias for instLeafPtr used to access
            // RayFlags written by the application in the shader, before the TraceRay.
            // This structure is used to access Ray0.flagsFromTraceRay.rayFlags
            // from the above RayFlags description.
            uint64_t rayFlags : (T)Bits::instLeafPtr;
            uint64_t pad          : 16;
        } flagsFromTraceRay;

        struct {
            uint64_t instLeafPtr : (T)Bits::instLeafPtr;  // the pointer to instance leaf in case we traverse an instance (64-bytes alignment)
            uint64_t rayMask     : (T)Bits::rayMask;      // ray mask used for ray masking
            uint64_t pad2        : (T)Bits::pad2;
        };
    };
};
static_assert(sizeof(MemRay<Xe>) == 64, "MemRay has to be 64 bytes large");

template <>
struct PrimLeafDesc<Xe>
{
    enum Type : uint32_t
    {
        TYPE_NONE = 0,

        /* For a node type of NODE_TYPE_MESHLET, the referenced leaf may
         * still be a QuadLeaf or a Meshlet. We need this as we produce
         * two quads instead of one meshlet when meshlet compression does
         * not work well. */

         TYPE_QUAD    = 0,
         TYPE_MESHLET = 1,

         /* For a node type of NODE_TYPE_PROCEDURAL we support enabling
          * and disabling the opaque/non_opaque culling. */

          TYPE_OPACITY_CULLING_ENABLED  = 0,
          TYPE_OPACITY_CULLING_DISABLED = 1
    };

    enum class Bits : uint8_t
    {
        shaderIndex = 24,
        geomMask    = 8,
        geomIndex   = 29,
        type        = 1,
        geomFlags   = 2,
    };

    using T = uint32_t;

    uint32_t shaderIndex : (T)Bits::shaderIndex; // shader index used for shader record calculations
    uint32_t geomMask    : (T)Bits::geomMask;    // geometry mask used for ray masking

    uint32_t geomIndex : (T)Bits::geomIndex; // the geometry index specifies the n'th geometry of the scene
    uint32_t type      : (T)Bits::type;      // distinguish between QuadLeaves and Meshlets or enable/disable culling for procedurals and instances
    uint32_t geomFlags : (T)Bits::geomFlags; // geometry flags of this geometry
};
static_assert(sizeof(PrimLeafDesc<Xe>) == 8, "PrimLeafDesc must be 8 bytes large");

template <>
struct QuadLeaf<Xe>
{
    PrimLeafDesc<Xe> leafDesc;  // the leaf header

    uint32_t primIndex0;    // primitive index of first triangle (has to be at same offset as for CompressedMeshlet!)

    enum class Bits : uint8_t
    {
        primIndex1Delta = 16,
        j0              = 2,
        j1              = 2,
        j2              = 2,
        last            = 1,
        pad             = 9,
    };

    using T = uint32_t;

    uint32_t primIndex1Delta : (T)Bits::primIndex1Delta;  // delta encoded primitive index of second triangle
    uint32_t j0              : (T)Bits::j0;               // specifies first vertex of second triangle
    uint32_t j1              : (T)Bits::j1;               // specified second vertex of second triangle
    uint32_t j2              : (T)Bits::j2;               // specified third vertex of second triangle
    uint32_t last            : (T)Bits::last;             // true if the second triangle is the last triangle in a leaf list
    uint32_t pad             : (T)Bits::pad;              // unused bits

    Vec3f v0;  // first vertex of first triangle
    Vec3f v1;  // second vertex of first triangle
    Vec3f v2;  // third vertex of first triangle
    Vec3f v3;  // additional vertex only used for second triangle
};
static_assert(sizeof(QuadLeaf<Xe>) == 64, "QuadLeaf must be 64 bytes large");

template <>
struct InstanceLeaf<Xe>
{
    /* first 64 bytes accessed during traversal by hardware */
    struct Part0
    {
        using T = uint32_t;
        enum class Bits : uint8_t
        {
            shaderIndex           = 24,
            geomMask              = 8,
            instContToHitGrpIndex = 24,
            pad0                  = 5,
            type                  = 1,
            geomFlags             = 2,
            startNodePtr          = 48,
            instFlags             = 8,
            ComparisonMode        = 1,
            ComparisonValue       = 7,
        };

        uint32_t shaderIndex : (T)Bits::shaderIndex;  // shader index used to calculate instancing shader in case of software instancing
        uint32_t geomMask    : (T)Bits::geomMask;     // geometry mask used for ray masking

        uint32_t instanceContributionToHitGroupIndex : (T)Bits::instContToHitGrpIndex;  // TODO: add description
        uint32_t pad0                                : (T)Bits::pad0;                   // explicit padding bits
        /* PrimLeafDesc */
        uint32_t type                                : (T)Bits::type;                   // enables/disables opaque culling
        uint32_t geomFlags                           : (T)Bits::geomFlags;              // geometry flags are not used for instances

        uint64_t startNodePtr : (T)Bits::startNodePtr;  // start node where to continue traversal of the instanced object
        uint64_t instFlags    : (T)Bits::instFlags;     // flags for the instance (see InstanceFlags)

        // Xe2+
        uint64_t ComparisonMode  : (T)Bits::ComparisonMode;  // 0 for less than or equal, 1 for greater
        uint64_t ComparisonValue : (T)Bits::ComparisonValue; // to be compared with ray.ComparisonValue


        // Note that the hardware swaps the translation components of the
        // world2obj and obj2world matrices, and uses column-major instead of row-major.

        // DXR and Vulkan specify transform matrices in row-major order.
        // A 3x4 row-major matrix from the API maps to HWInstanceLeaf layout as shown:
        //     | vx[0] vy[0] vz[0] p[0] |
        // M = | vx[1] vy[1] vz[1] p[1] |
        //     | vx[2] vy[2] vz[2] p[2] |

        Vec3f world2obj_vx;   // 1st col of Worl2Obj transform
        Vec3f world2obj_vy;   // 2nd col of Worl2Obj transform
        Vec3f world2obj_vz;   // 3rd col of Worl2Obj transform
        Vec3f obj2world_p;    // translation of Obj2World transform (on purpose in first 64 bytes)
    } part0;

    /* second 64 bytes accessed during shading */
    struct Part1
    {
        uint64_t bvhPtr : 48;   // pointer to BVH where start node belongs too
        uint64_t pad    : 16;   // unused bits (explicit padding)

        uint32_t instanceID;    // user defined value per DXR spec
        uint32_t instanceIndex; // geometry index of the instance (n'th geometry in scene)

        Vec3f obj2world_vx;     // 1st col of Obj2World transform
        Vec3f obj2world_vy;     // 2nd col of Obj2World transform
        Vec3f obj2world_vz;     // 3rd col of Obj2World transform
        Vec3f world2obj_p;      // translation of World2Obj transform
    } part1;
};

template <>
struct MemHit<Xe3>
{
    enum class Bits : uint8_t
    {
        u = 24,
        v = 24,
        hitGroupIndex0 = 8,
        hitGroupIndex1 = 8,
        hitGroupIndex2 = 6,
        hitGroupIndex3 = 6,
        primIndexDelta = 5,
        pad1 = 7,
        leafNodeSubType = 4,
        valid = 1,
        leafType = 3,
        primLeafIndex = 4,
        bvhLevel = 3,
        frontFace = 1,
        done = 1,
        needSWSTOC = 1,
        reserved = 2,

        primLeafPtr = 58,
        instLeafPtr = 58,
    };

    enum class Offset : uint8_t
    {
        done = 28,
    };

    using T = uint32_t;

    float    t;                                        // hit distance of current hit (or initial traversal distance)

    uint32_t u              : (T)Bits::u;              // barycentric u hit coordinate stored as 24 bit unorm
    uint32_t hitGroupIndex0 : (T)Bits::hitGroupIndex0; // 1st bits of hitGroupIndex

    uint32_t v              : (T)Bits::v;              // barycentric v hit coordinate stored as 24 bit unorm
    uint32_t hitGroupIndex1 : (T)Bits::hitGroupIndex1; // 2nd bits of hitGroupIndex

    uint32_t primIndexDelta  : (T)Bits::primIndexDelta;  // prim index delta for compressed meshlets and quads
    uint32_t pad1            : (T)Bits::pad1;            // MBZ
    uint32_t leafNodeSubType : (T)Bits::leafNodeSubType; // sub-type of leaf node
    uint32_t valid           : (T)Bits::valid;           // set if there is a hit
    uint32_t leafType        : (T)Bits::leafType;        // type of node primLeafPtr is pointing to
    uint32_t primLeafIndex   : (T)Bits::primLeafIndex;   // index of the hit primitive inside the leaf
    uint32_t bvhLevel        : (T)Bits::bvhLevel;        // the instancing level at which the hit occured
    uint32_t frontFace       : (T)Bits::frontFace;       // whether we hit the front-facing side of a triangle (also used to pass opaque flag when calling intersection shaders)
    uint32_t done            : (T)Bits::done;            // used in sync mode to indicate that traversal is done
    uint32_t needSWSTOC      : (T)Bits::needSWSTOC;      // If set, any-hit shader must perform a SW fallback STOC test
    uint32_t reserved        : (T)Bits::reserved;        // unused bit

    uint64_t hitGroupIndex2 : (T)Bits::hitGroupIndex2; // 3rd bits of hitGroupIndex
    uint64_t primLeafPtr    : (T)Bits::primLeafPtr;    // pointer to BVH leaf node (MSBs of 64b pointer aligned to 64B)

    uint64_t hitGroupIndex3 : (T)Bits::hitGroupIndex3; // 4th bits of hit group index
    uint64_t instLeafPtr    : (T)Bits::instLeafPtr;    // pointer to BVH instance leaf node (MSBs of 64b pointer aligned to 64B)
};
static_assert(sizeof(MemHit<Xe3>) == 32, "MemHit has to be 32 bytes large");

template <>
struct MemRay<Xe3>
{
    // 32 B
    Vec3f org;         // the origin of the ray
    Vec3f dir;         // the direction of the ray
    float tnear;       // the start of the ray
    float tfar;        // the end of the ray

    using T = uint32_t;
    enum class Bits : uint8_t
    {
        rootNodePtr           = 64,
        instLeafPtr           = 64,
        rayFlags              = 16,
        rayMask               = 8,
        ComparisonValue       = 7,
        pad1                  = 1,
        missShaderIndex       = 16,
        shaderIndexMultiplier = 4,
        pad2                  = 4,
        internalRayFlags      = 8,
    };

    uint64_t rootNodePtr : (T)Bits::rootNodePtr;  // root node to start traversal at (64-byte alignment)
    union
    {
        // This just an alias for 16-bit type.
        // As we must avoid structure members alignment, to properly generate
        // offsets in MemRayStructure, rayFlags has 64bit type. To simplify the
        // access, a 16bit alias is created (Ray0.flagsFromTraceRay16BitTypeAlias.rayFlags).
        // It points to same part of MemHit structure as just Ray0.flagsFromTraceRay.rayFlags.
        struct
        {
            uint16_t rayFlags;
            uint16_t uw1;
            uint16_t uw2;
            uint16_t uw3;
        } flagsFromTraceRay16BitTypeAlias;

        struct
        {
           // This is just an alias for instLeafPtr used to access
           // RayFlags written by the application in the shader, before the TraceRay.
           // This structure is used to access Ray0.flagsFromTraceRay.rayFlags
           // from the below RayFlags description.
            uint64_t rayFlags;
        } flagsFromTraceRay;

        uint64_t instLeafPtr : (T)Bits::instLeafPtr;  // the pointer to instance leaf in case we traverse an instance (64-bytes alignment)
    };

    // RayFlags:
    //
    // There are 3 sets of flags, from the application point of view,
    // available during the ray traversal:
    // 1. RayFlags set in the shader before the TraceRay functions is called.
    // 2. Flags applied to the entire Pipeline.
    // 3. Flags applied per instances and geometry in the BVH
    //
    // All the Ray data is stored in the MemRayStructures which have two instances:
    // 1. TopLevel (Ray0)
    // 2. BottomLevel (Ray1)
    //
    // TopLevel is written by SW when the Ray is cast (before calling TraceRay).
    // When the traversal reaches the instance and enters related BottomLevel BVH,
    // TopLevel MemRay is copied by HW. Flags from entered instance are applied to
    // the RayFlags.
    // The BottomLevel MemRay is written only by HW, when AnyHit or Intersection
    // Shaders are called.
    //
    // As HW can only apply flags from TracRay (1) and the BVH (3), flags from
    // the Pipeline (2) has to applied by SW. It is done by applying them to
    // the flags set before the TraceRay (1). But the application can read
    // both the Pipeline flags (2) and the RayFlags (1) independently.
    // So the original RayFlags (1) are additionally stored in the MemRay memory.
    // As there is no dedicated space for them, the instLeafPtr from TopLevel MemRay
    // is used, as it is irrelevant to the traversal at the TopLevel.
    //
    // When the AnyHit or Intersection shader are called, there are 3 sets of flags
    // available:
    // 1. RayFlags combined with Pipeline Flags - Ray0.rayFlags
    // 2. RayFlags - Ray0.instLeafPtr aka Ray0.flagsFromTraceRay.rayFlags
    // 3. RayFlags + Pipeline Flags + Instance/GeometryFlags - Ray1.rayFlags
    //
    // rayFlags defined here is used to access:
    // 1. Ray0.rayFlags
    // 2. Ray1.rayFlags
    //
    // Additionally an alias for 16-bit access is added.
    // As we must avoid structure members alignment, to properly generate
    // offsets in MemRayStructure, rayFlags has 64bit type. To simplify the
    // access, a 16bit alias is created (rayFlags16BitAccessAlias.rayFlags).
    // It points to same part of MemHit structure as just rayFlags.
    union
    {
        struct
        {
            uint16_t rayFlags;
            uint16_t uw1;
        } rayFlags16BitTypeAlias;

        struct
        {
            uint32_t rayFlags : (T)Bits::rayFlags;        // ray flags (see RayFlag structure)
            uint32_t rayMask : (T)Bits::rayMask;         // ray mask used for ray masking
            uint32_t ComparisonValue : (T)Bits::ComparisonValue; // to be compared with Instance.ComparisonMask
            uint32_t pad1 : (T)Bits::pad1;
        };
    };

    uint32_t hitGroupIndex;                              // hit group shader index

    uint32_t missShaderIndex       : (T)Bits::missShaderIndex;       // index of miss shader to invoke on a miss
    uint32_t shaderIndexMultiplier : (T)Bits::shaderIndexMultiplier; // shader index multiplier
    uint32_t pad2                  : (T)Bits::pad2;
    uint32_t internalRayFlags      : (T)Bits::internalRayFlags;      // Xe3: internal ray flags (see InternalRayFlags enum)

    float time;            // ray time in range [0,1] (force to 0 for now)
};
static_assert(sizeof(MemRay<Xe3>) == 64, "MemRay has to be 64 bytes large");

enum class InternalRayFlags : uint16_t
{
    NONE                            = 0x0,
    TRIANGLE_FRONT_COUNTERCLOCKWISE = 0x1, // Xe3: switch front and back-facing triangles
    LEVEL_ASCEND_DISABLED           = 0x2, // Xe3: disables the automatic level ascend for this level
    SKIP_MISS_SHADER                = 0x4, // Xe3: skip execution of miss shader
    DISABLE_STOC                    = 0x8, // Xe3: disables sub triangle opacity culling
};

template <>
struct PrimLeafDesc<Xe3>
{
    enum Type : uint32_t
    {
        TYPE_NONE = 0,

        /* For a node type of NODE_TYPE_MESHLET, the referenced leaf may
         * still be a QuadLeaf or a Meshlet. We need this as we produce
         * two quads instead of one meshlet when meshlet compression does
         * not work well. */

         TYPE_QUAD    = 0,
         TYPE_MESHLET = 1,

         /* For a node type of NODE_TYPE_PROCEDURAL we support enabling
          * and disabling the opaque/non_opaque culling. */

          TYPE_OPACITY_CULLING_ENABLED  = 0,
          TYPE_OPACITY_CULLING_DISABLED = 1
    };

    enum class Bits : uint8_t
    {
        shaderIndex         = 24,
        geomMask            = 8,

        geomIndex           = 24,
        subType             = 4,
        reserved0           = 1,
        DisableOpacityCull  = 1,
        OpaqueGeometry      = 1,
        IgnoreRayMultiplier = 1,
    };

    using T = uint32_t;

    uint32_t shaderIndex         : (T)Bits::shaderIndex;         // shader index used for shader record calculations
    uint32_t geomMask            : (T)Bits::geomMask;            // geometry mask used for ray masking

    uint32_t geomIndex           : (T)Bits::geomIndex;           // Xe1+: the geometry index specifies the n'th geometry of the scene
    uint32_t subType             : (T)Bits::subType;             // Xe3: geometry sub-type
    uint32_t reserved0           : (T)Bits::reserved0;           // Xe1+: reserved bit (MBZ)
    uint32_t DisableOpacityCull  : (T)Bits::DisableOpacityCull;  // Xe1+: disables opacity culling
    uint32_t OpaqueGeometry      : (T)Bits::OpaqueGeometry;      // Xe1+: determines if geometry is opaque
    uint32_t IgnoreRayMultiplier : (T)Bits::IgnoreRayMultiplier; // Xe3: ignores ray geometry multiplier
};
static_assert(sizeof(PrimLeafDesc<Xe3>) == 8, "PrimLeafDesc must be 8 bytes large");

template <>
struct QuadLeaf<Xe3>
{
    PrimLeafDesc<Xe3> leafDesc;  // the leaf header

    uint32_t primIndex0;         // primitive index of first triangle (has to be at same offset as for CompressedMeshlet!)

    enum class Bits : uint8_t
    {
        primIndex1Delta   = 5,
        stoc1_tri0_swstoc = 1,
        stoc1_tri1_swstoc = 1,
        pad               = 1,
        stoc1_tri0_opaque = 4,
        stoc1_tri0_transp = 4,
        j0                = 2,
        j1                = 2,
        j2                = 2,
        last              = 1,
        lastInMeshlet     = 1,
        stoc1_tri1_opaque = 4,
        stoc1_tri1_transp = 4,
    };

    using T = uint32_t;

    uint32_t primIndex1Delta   : (T)Bits::primIndex1Delta;   // offset of primID of second triangle
    uint32_t stoc1_tri0_swstoc : (T)Bits::stoc1_tri0_swstoc; // indicates that software STOC emulation for triangle 0 is required in AHS
    uint32_t stoc1_tri1_swstoc : (T)Bits::stoc1_tri1_swstoc; // indicates that software STOC emulation for triangle 1 is required in AHS
    uint32_t pad               : (T)Bits::pad;               // reserved (MBZ)
    uint32_t stoc1_tri0_opaque : (T)Bits::stoc1_tri0_opaque; // STOC level 1 sub-triangle opaque      bits for triangle 0
    uint32_t stoc1_tri0_transp : (T)Bits::stoc1_tri0_transp; // STOC level 1 sub-triangle transparent bits for triangle 0
    uint32_t j0                : (T)Bits::j0;
    uint32_t j1                : (T)Bits::j1;
    uint32_t j2                : (T)Bits::j2;
    uint32_t last              : (T)Bits::last;              // last quad in BVH leaf
    uint32_t lastInMeshlet     : (T)Bits::lastInMeshlet;     // last quad in meshlet
    uint32_t stoc1_tri1_opaque : (T)Bits::stoc1_tri1_opaque; // STOC level 1 sub-triangle opaque      bit for triangle 1
    uint32_t stoc1_tri1_transp : (T)Bits::stoc1_tri1_transp; // STOC level 1 sub-triangle transparent bit for triangle 1

    Vec3f v0;  // first vertex of first triangle
    Vec3f v1;  // second vertex of first triangle
    Vec3f v2;  // third vertex of first triangle
    Vec3f v3;  // additional vertex only used for second triangle
};
static_assert(sizeof(QuadLeaf<Xe3>) == 64, "QuadLeaf must be 64 bytes large");

struct QuadLeaf_STOC3
{
    QuadLeaf<Xe3> quad;
    uint64_t fullyOpaqueBits0;       // fully opaque sub-triangle bits for triangle 0
    uint64_t fullyTransparentBits0;  // fully transparent sub-triangle bits for triangle 0
    uint64_t fullyOpaqueBits1;       // fully opaque sub-triangle bits for triangle 1
    uint64_t fullyTransparentBits1;  // fully transparent sub-triangle bits for triangle 1

    uint64_t stoc3_tri0_addr;      // Xe3: address of full STOC bits for triangle 0
    uint8_t  stoc3_tri0_level;     // Xe3: the recursion level for STOC bits for triangle 0 (4-12)

    uint8_t  stoc3_tri0_swstoc : 1;// Xe3: indicates that software STOC emulation for triangle 0 is required in AHS
    uint8_t  _reserved0        : 7;

    uint8_t  reserved0[6];

    uint64_t stoc3_tri1_addr;      // Xe3: address of full STOC bits for triangle 1
    uint8_t  stoc3_tri1_level;     // Xe3: the recursion level for STOC bits for triangle 1 (4-12)

    uint8_t  stoc3_tri1_swstoc : 1;// Xe3: indicates that software STOC emulation for triangle 1 is required in AHS
    uint8_t  _reserved1        : 7;

    uint8_t  reserved1[6];
};
static_assert(sizeof(QuadLeaf_STOC3) == 128, "QuadLeaf_STOC3 must be 128 bytes large");

template <>
struct InstanceLeaf<Xe3>
{
    /* first 64 bytes accessed during traversal by hardware */
    struct Part0
    {
        using T = uint32_t;
        enum class Bits : uint8_t
        {
            instContToHitGrpIndex = 24,
            geomMask              = 8,
            instFlags             = 8,
            ComparisonMode        = 1,
            ComparisonValue       = 7,
            pad0                  = 8,
            subType               = 3,
            pad1                  = 2,
            DisableOpacityCull    = 1,
            OpaqueGeometry        = 1,
            IgnoreRayMultiplier   = 1,
        };

        uint32_t instanceContributionToHitGroupIndex : (T)Bits::instContToHitGrpIndex; // Xe3: instance contribution to hit group index
        uint32_t geomMask                            : (T)Bits::geomMask;             // Xe1+: geometry mask used for ray masking

        uint32_t instFlags            : (T)Bits::instFlags;           // Xe3: flags for the instance (see InstanceFlags)
        uint32_t ComparisonMode       : (T)Bits::ComparisonMode;      // Xe3: 0 for <=, 1 for > comparison
        uint32_t ComparisonValue      : (T)Bits::ComparisonValue;     // Xe3: to be compared with ray.ComparionMask
        uint32_t pad0                 : (T)Bits::pad0;                // reserved (MBZ)
        uint32_t subType              : (T)Bits::subType;             // Xe3: geometry sub-type
        uint32_t pad1                 : (T)Bits::pad1;                // reserved (MBZ)
        uint32_t DisableOpacityCull   : (T)Bits::DisableOpacityCull;  // Xe1+: disables opacity culling
        uint32_t OpaqueGeometry       : (T)Bits::OpaqueGeometry;      // Xe1+: determines if geometry is opaque
        uint32_t IgnoreRayMultiplier  : (T)Bits::IgnoreRayMultiplier; // Xe3: ignores ray geometry multiplier

        uint64_t startNodePtr;                                        // Xe3: 64 bit start node of instanced object


        // Note that the hardware swaps the translation components of the
        // world2obj and obj2world matrices, and uses column-major instead of row-major.

        // DXR and Vulkan specify transform matrices in row-major order.
        // A 3x4 row-major matrix from the API maps to HWInstanceLeaf layout as shown:
        //     | vx[0] vy[0] vz[0] p[0] |
        // M = | vx[1] vy[1] vz[1] p[1] |
        //     | vx[2] vy[2] vz[2] p[2] |

        Vec3f world2obj_vx;   // 1st col of Worl2Obj transform
        Vec3f world2obj_vy;   // 2nd col of Worl2Obj transform
        Vec3f world2obj_vz;   // 3rd col of Worl2Obj transform
        Vec3f obj2world_p;    // translation of Obj2World transform (on purpose in first 64 bytes)
    } part0;

    /* second 64 bytes accessed during shading */
    struct Part1
    {
        uint64_t bvhPtr : 48;   // pointer to BVH where start node belongs too
        uint64_t pad    : 16;   // unused bits (explicit padding)

        uint32_t instanceID;    // user defined value per DXR spec
        uint32_t instanceIndex; // geometry index of the instance (n'th geometry in scene)

        Vec3f obj2world_vx;     // 1st col of Obj2World transform
        Vec3f obj2world_vy;     // 2nd col of Obj2World transform
        Vec3f obj2world_vz;     // 3rd col of Obj2World transform
        Vec3f world2obj_p;      // translation of World2Obj transform
    } part1;
};
static_assert(sizeof(InstanceLeaf<Xe3>) == 128, "InstanceLeaf must be 128 bytes large");


// HW stack
template <typename GenT, uint32_t MaxBVHLevels>
struct RTStack
{
    MemHit<GenT> committedHit;    // stores committed hit
    MemHit<GenT> potentialHit;    // stores potential hit that is passed to any hit shader

    MemRay<GenT> ray0;
    MemRay<GenT> ray1;
    MemRay<GenT> ray[MaxBVHLevels - 2];       // stores a ray for each instancing level
    MemTravStack travStack[MaxBVHLevels]; // spill location for the internal stack state per instancing level

    static_assert(MaxBVHLevels > 2);
};

template <typename GenT>
struct RTStack<GenT, MAX_BVH_LEVELS>
{
    MemHit<GenT> committedHit;    // stores committed hit
    MemHit<GenT> potentialHit;    // stores potential hit that is passed to any hit shader

    MemRay<GenT> ray0;
    MemRay<GenT> ray1;
    MemTravStack travStack[MAX_BVH_LEVELS]; // spill location for the internal stack state per instancing level
};


// This is the ShadowMemory that we maintain if we DONOT need spill/fill it to/from HW RTStack.
// We keep this data structure as small as possible to reduce the size.
template <typename GenT, uint32_t MaxBVHLevels>
struct SMStack
{
    MemHit<GenT> committedHit;    // stores committed hit
    MemHit<GenT> potentialHit;    // stores potential hit that is passed to any hit shader

    MemRay<GenT> ray0;
    MemRay<GenT> ray1;
    MemRay<GenT> ray[MaxBVHLevels - 2];       // stores a ray for each instancing level
    //MemTravStack travStack[MaxBVHLevels]; // spill location for the internal stack state per instancing level

    static_assert(MaxBVHLevels > 2);
};

template <typename GenT>
struct SMStack<GenT, MAX_BVH_LEVELS>
{
    MemHit<GenT> committedHit;    // stores committed hit
    MemHit<GenT> potentialHit;    // stores potential hit that is passed to any hit shader

    MemRay<GenT> ray0;
    MemRay<GenT> ray1;
    //MemTravStack travStack[MaxBVHLevels]; // spill location for the internal stack state per instancing level
};

// For now, we're just defaulting to using an RTStack assuming
// MAX_BVH_LEVELS == 2.
template <typename GenT>
using RTStack2 = RTStack<GenT, MAX_BVH_LEVELS>;
template <typename GenT>
using SMStack2 = SMStack<GenT, MAX_BVH_LEVELS>;

template <typename RTStack2T>
constexpr size_t sizeofRTStack2() { return sizeof(RTStack2T); }

#ifdef HAS_INCLUDE_TYPE_TRAITS
static_assert(std::is_standard_layout_v<RTStack2<Xe>>);
static_assert(std::is_standard_layout_v<SMStack2<Xe>>);
#endif // HAS_INCLUDE_TYPE_TRAITS

// Makes sure that important fields are at their proper offset from the start
// of the structure. Update this if the structure changes.  This is just for
// documentation purposes.
static_assert(offsetof(RTStack2<Xe>, committedHit) == 0,   "unexpected offset!");
static_assert(offsetof(RTStack2<Xe>, potentialHit) == 32,  "unexpected offset!");
static_assert(offsetof(RTStack2<Xe>, ray0)         == 64,  "unexpected offset!");
static_assert(offsetof(RTStack2<Xe>, ray1)         == 64 + 64, "unexpected offset!");
static_assert(offsetof(RTStack2<Xe>, travStack)    == 192, "unexpected offset!");


/////////////// BVH structures ///////////////

enum NodeType : uint8_t
{
    NODE_TYPE_MIXED = 0x0,        // identifies a mixed internal node where each child can have a different type
    NODE_TYPE_INTERNAL = 0x0,     // internal BVH node with 6 children
    NODE_TYPE_INSTANCE = 0x1,     // instance leaf
    NODE_TYPE_PROCEDURAL = 0x3,   // procedural leaf
    NODE_TYPE_QUAD = 0x4,         // quad leaf
    NODE_TYPE_MESHLET = 0x6,      // meshlet leaf
    NODE_TYPE_INVALID = 0x7       // indicates invalid node
};

struct InternalNode
{
    static constexpr const uint32_t NUM_CHILDREN = 6;

    Vec3f lower;          // world space origin of quantization grid
    int32_t childOffset;  // offset to all children in 64B multiples

    NodeType nodeType;    // the type of the node
    uint8_t pad;          // unused byte

    int8_t exp_x;         // 2^exp_x is the size of the grid in x dimension
    int8_t exp_y;         // 2^exp_y is the size of the grid in y dimension
    int8_t exp_z;         // 2^exp_z is the size of the grid in z dimension
    uint8_t nodeMask;     // mask used for ray filtering

    struct ChildData
    {
        uint8_t blockIncr : 2;  // size of child in 64 byte blocks
        uint8_t startPrim : 4;  // start primitive in fat leaf mode or child type in mixed mode
        uint8_t pad       : 2;  // unused bits
    } childData[NUM_CHILDREN];

    uint8_t lower_x[NUM_CHILDREN];  // the quantized lower bounds in x-dimension
    uint8_t upper_x[NUM_CHILDREN];  // the quantized upper bounds in x-dimension
    uint8_t lower_y[NUM_CHILDREN];  // the quantized lower bounds in y-dimension
    uint8_t upper_y[NUM_CHILDREN];  // the quantized upper bounds in y-dimension
    uint8_t lower_z[NUM_CHILDREN];  // the quantized lower bounds in z-dimension
    uint8_t upper_z[NUM_CHILDREN];  // the quantized upper bounds in z-dimension
};

enum class GeometryFlags : uint32_t
{
    NONE       = 0x0,
    RTX_OPAQUE = 0x1
};

template <typename GenT>
struct ProceduralLeaf
{
    static const uint32_t N = 13;

    enum class Bits : uint8_t
    {
        numPrimitives = 4,
        pad           = 32 - numPrimitives - N,
        last          = N,
    };

    using T = uint32_t;

    PrimLeafDesc<GenT> leafDesc;             // leaf header identifying the geometry
    uint32_t numPrimitives : (T)Bits::numPrimitives;  // number of stored primitives
    uint32_t pad           : (T)Bits::pad;            // explicit padding bits
    uint32_t last          : (T)Bits::last;           // bit vector with a last bit per primitive
    uint32_t _primIndex[N];                           // primitive indices of all primitives stored inside the leaf
};

static_assert(sizeof(ProceduralLeaf<Xe>) == 64, "ProceduralLeaf must be 64 bytes large");

static_assert(sizeof(QuadLeaf<Xe>) == sizeof(ProceduralLeaf<Xe>),
    "Leaves must be same size");
static_assert(sizeof(QuadLeaf<Xe3>) == sizeof(ProceduralLeaf<Xe3>),
    "Leaves must be same size");
static_assert(sizeof(QuadLeaf<Xe>) == sizeof(QuadLeaf<Xe3>));

constexpr uint32_t LeafSize = sizeof(QuadLeaf<Xe>);

/*
  The CompressedMeshlet structure stores triangles compressed
  losslessly. Between 1 to 16 triangles can get stored that index
  into a vertex array with up to 16 vertices.

  The structure contains a header, a list of delta compressed
  indices (allocated front to back), and a list of compressed
  vertices (allocated back to front).

 */
template <typename GenT>
struct CompressedMeshlet
{
    PrimLeafDesc<GenT> leafDesc;
    uint32_t first_triangle_primID;      // has to be at same offset as for QuadLeaf!

    uint32_t num_triangles         : 4;  // number of stored triangles (0 -> 1 triangle, 1 -> 2 triangles, etc..)
    uint32_t num_position_bits_x   : 4;  // number of bit pairs for x coordinate (0 -> 2 bits, 1 -> 4 bits, etc.)
    uint32_t num_position_bits_y   : 4;  // number of bit pairs for y coordinate (0 -> 2 bits, 1 -> 4 bits, etc.)
    uint32_t num_position_bits_z   : 4;  // number of bit pairs for z coordinate (0 -> 2 bits, 1 -> 4 bits, etc.)
    uint32_t num_primID_delta_bits : 4;  // number of bits for primIndex delta encoding (0 -> 1 bit, 1 -> 2 bits, etc.)
    uint32_t first_triangle_islast : 1;  // last bit for first triangle
    uint32_t pad                   : 8;  // explicit padding bits

    /* in this array triangles are stored front to back and vertex
     * positions back to front. */
    char data[100];

    Vec3f first_position;
};

static_assert(sizeof(CompressedMeshlet<Xe>) == 128,
    "CompressedMeshlet has to be 128 bytes large");

enum class InstanceFlags : uint8_t
{
    NONE = 0x0,
    TRIANGLE_CULL_DISABLE = 0x1,
    TRIANGLE_FRONT_COUNTERCLOCKWISE = 0x2,
    FORCE_OPAQUE = 0x4,
    FORCE_NON_OPAQUE = 0x8,
    FORCE_2STATE_STOC = 0x10,
    DISABLE_STOC = 0x20,
};

template <typename GenT>
struct alignas(256) BVH
{
    uint64_t rootNodeOffset;        // root node offset

    Vec3f   bounds_min;             // bounds of the BVH
    Vec3f   bounds_max;

    uint32_t nodeDataStart;         // first 64 byte block of node data
    uint32_t nodeDataCur;           // next free 64 byte block for node allocations
    uint32_t leafDataStart;         // first 64 byte block of leaf data
    uint32_t leafDataCur;           // next free 64 byte block for leaf allocations
    uint32_t proceduralDataStart;   // first 64 byte block for procedural leaf data
    uint32_t proceduralDataCur;     // next free 64 byte block for procedural leaf allocations
    uint32_t backPointerDataStart;  // first 64 byte block for back pointers
    uint32_t backPointerDataEnd;    // end of back pointer array

    // miscellaneous header information
    // ...

    // node data

    // These are really variable length arrays in memory but are here for
    // documentation purposes.
    InternalNode         innerNode[1];
    QuadLeaf<GenT>       geomLeaf[1];
    ProceduralLeaf<GenT> proceduralLeaf[1];
    InstanceLeaf<GenT>   instLeaf[1];
    uint32_t             backPointers[1];

    // array size = 256 - (sizeOfStructMembers % 256)
    // sizeOfStructMembers = 8          //uint64_t rootNodeOffset
    //                     + 2*12       // Vec3f   bounds_min; Vec3f   bounds_max;
    //                     + 9 * 4      // 9 * uint32_t
    //                     + 64         // InternalNode
    //                     + 64         // QuadLeaf
    //                     + 64         // ProceduralLeaf
    //                     + 128        // InstanceLeaf
    char __Padding[124];
};

constexpr uint32_t getRayFlagMask(uint32_t Val)
{
    return (Val << 1) - 1;
}

enum class RAYTRACING_PIPELINE_FLAGS : uint16_t
{
    NONE = 0x0,
    SKIP_TRIANGLES = 0x100,
    SKIP_PROCEDURAL_PRIMITIVES = 0x200,
};

enum class RayFlags : uint16_t
{
    NONE = 0x00,
    FORCE_OPAQUE = 0x01,                        // forces geometry to be opaque (no anyhit shader invokation)
    FORCE_NON_OPAQUE = 0x02,                    // forces geometry to be non-opqaue (invoke anyhit shader)
    ACCEPT_FIRST_HIT_AND_END_SEARCH = 0x04,     // terminates traversal on the first hit found (shadow rays)
    SKIP_CLOSEST_HIT_SHADER = 0x08,             // skip execution of the closest hit shader
    CULL_BACK_FACING_TRIANGLES = 0x10,          // back facing triangles to not produce a hit
    CULL_FRONT_FACING_TRIANGLES = 0x20,         // front facing triangles do not produce a hit
    CULL_OPAQUE = 0x40,                         // opaque geometry does not produce a hit
    CULL_NON_OPAQUE = 0x80,                     // non-opaque geometry does not produce a hit

    SKIP_TRIANGLES = 0x100,                     // Skip all triangle intersections and consider them as misses
    SKIP_PROCEDURAL_PRIMITIVES = 0x200,         // Skip execution of intersection shaders for procedural primitives

    // Anything below this point should not be propagated to RayFlags()

    //TRIANGLE_FRONT_COUNTERCLOCKWISE = 0x4000, // This value MUST not be programmed by SW but used internally by HW only.

    LEVEL_ASCEND_DISABLED = 0x8000,             // disables the automatic level ascend for this level,
                                                // thus traversal will terminate when BVH at this level is done
};

// DXR spec:
// "Only defined ray flags are propagated by the system, e.g. visible to the RayFlags() shader intrinsic."
constexpr uint32_t RayFlagsMask = getRayFlagMask((uint32_t)RayFlags::SKIP_PROCEDURAL_PRIMITIVES);

enum class RayFlags_Xe3 : uint16_t
{
    NONE = 0x00,
    FORCE_OPAQUE = 0x01,                        // forces geometry to be opaque (no anyhit shader invokation)
    FORCE_NON_OPAQUE = 0x02,                    // forces geometry to be non-opqaue (invoke anyhit shader)
    ACCEPT_FIRST_HIT_AND_END_SEARCH = 0x04,     // terminates traversal on the first hit found (shadow rays)
    SKIP_CLOSEST_HIT_SHADER = 0x08,             // skip execution of the closest hit shader
    CULL_BACK_FACING_TRIANGLES = 0x10,          // back facing triangles to not produce a hit
    CULL_FRONT_FACING_TRIANGLES = 0x20,         // front facing triangles do not produce a hit
    CULL_OPAQUE = 0x40,                         // opaque geometry does not produce a hit
    CULL_NON_OPAQUE = 0x80,                     // non-opaque geometry does not produce a hit

    SKIP_TRIANGLES = 0x100,                     // Skip all triangle intersections and consider them as misses
    SKIP_PROCEDURAL_PRIMITIVES = 0x200,         // Skip execution of intersection shaders for procedural primitives

    FORCE_2STATE_STOC = 0x400,                  // Force 2 state sub triangle opacity mode.
};

// DXR spec:
// "Only defined ray flags are propagated by the system, e.g. visible to the RayFlags() shader intrinsic."
constexpr uint32_t RayFlagsMask_Xe3 = getRayFlagMask((uint32_t)RayFlags_Xe3::FORCE_2STATE_STOC);
static_assert(sizeof(MemTravStack) == 32, "MemTravStack has to be 32 bytes large");


// On DG2, writes will not go to the L1$ unless they are 16-byte aligned
// and at least 16 bytes in size.
constexpr uint32_t LSC_WRITE_GRANULARITY = 16;

struct StackPtrAndBudges
{
    enum class Bits : uint8_t
    {
        DimBits = 5,
    };

    using T = uint32_t;

    // This tracks the offset from the base of the HW portion of the stack
    // to wherever we currently point in the SW portion of the stack.
    uint16_t StackOffset;

    union {
        uint16_t BudgeBits;

        struct {
            uint16_t XSize : (T)Bits::DimBits;  // TODO: add description
            uint16_t YSize : (T)Bits::DimBits;  // TODO: add description
            uint16_t ZSize : (T)Bits::DimBits;  // TODO: add description
            uint16_t Pad   : 1;                 // explicit padding bit to get 16 bits in the union
        };
    };
};

static_assert(sizeof(StackPtrAndBudges) == 4, "wrong size?");

// The goal is to minimize the amount of data we put into the L1$.  Rather than
// storing a full a64 pointer and 3 dwords for the x,y,z DispatchRayIndex()
// values (2 + 3 = 5 dwords per ray), we note that the DXR spec says:
//
// "There are 3 dimensions passed in to set the grid size: width/height/depth.
//  These dimensions are constrained such that width*height*depth <= 2^30.
//  Exceeding this produces undefined behavior."
//
// This means we can compress the three dims into 30 bits and use the upper
// bits of the stack pointer dword to track how many bits are needed by
// each dimension
struct StackPtrDRIEncoding
{
    StackPtrAndBudges PtrAndBudges;
    uint32_t CompressedDispatchRayIndices;
};

static_assert(sizeof(StackPtrDRIEncoding) == 8, "wrong size?");

struct alignas(LSC_WRITE_GRANULARITY) SWHotZone_v1
{
    StackPtrDRIEncoding Encoding;

    // Compiler may elect to enlarge the hot-zone.
    //uint8_t additional_bytes[];

    // pad to LSC write granularity (16B on Gen12)
    char __Padding[LSC_WRITE_GRANULARITY - sizeof(StackPtrDRIEncoding)];
};

// We currently default to this encoding.  Set the 'EnableCompressedRayIndices'
// regkey to switch to the "v1" encoding above.
struct alignas(LSC_WRITE_GRANULARITY) SWHotZone_v2
{
    uint32_t StackOffset;
    uint32_t DispatchRaysIndex[3];

    // Compiler may elect to enlarge the hot-zone.
    //uint8_t additional_bytes[];

    // pad to LSC write granularity (16B on Gen12)
};

struct alignas(LSC_WRITE_GRANULARITY) SWHotZone_v3
{
    uint32_t StackOffset;
    uint32_t DispatchRaysIndex[3];

    uint32_t Complete;

    // pad to LSC write granularity (16B on Gen12)
    uint32_t Pad[3];
};

constexpr uint32_t StackFrameAlign = 16;
static_assert(IGC::RTStackAlign % LSC_WRITE_GRANULARITY == 0, "not aligned to write granularity?");

// This is the structure of memory that will be allocated by the UMD:
template <typename HotZoneTy, typename SyncStackTy, typename AsyncStackTy, typename SWStackTy,
          uint32_t RTStackAlign, uint64_t DSS_COUNT, uint64_t NumDSSRTStacks, uint64_t SIMD_LANES_PER_DSS>
struct RTMemory
{
    // Packed SW hot-zones
    alignas(IGC::RTStackAlign) HotZoneTy HotZones[DSS_COUNT * NumDSSRTStacks];

    // sync stacks for synchronous ray tracing
    alignas(RTStackAlign) SyncStackTy SyncStacks[DSS_COUNT * SIMD_LANES_PER_DSS];

    // RTMemBasePointer points here <----
    alignas(RTStackAlign) AsyncStackTy AsyncStacks[DSS_COUNT * NumDSSRTStacks];

    // Align to L3 sector size, or LSC sector size if stack is LSC-cached
    alignas(IGC::RTStackAlign) SWStackTy SWStacks[DSS_COUNT * NumDSSRTStacks];
};

// This will be invoked by the UMD to allocate stack space.
constexpr uint64_t calcRTMemoryAllocSize(
    uint64_t SWHotZoneSize,
    uint64_t SyncStackSize,
    uint64_t AsyncStackSize,
    uint64_t SWStackSize,
    uint64_t DSS_COUNT,
    uint64_t NumDSSRTStacks,
    uint64_t SIMD_LANES_PER_DSS)
{
    // SIMD_LANES_PER_DSS = EUCount * ThreadCount * SIMD16
    return IGC::Align(SWHotZoneSize * DSS_COUNT * NumDSSRTStacks, IGC::RTStackAlign) +
        IGC::Align(SyncStackSize * DSS_COUNT * SIMD_LANES_PER_DSS, IGC::RTStackAlign) +
        IGC::Align(AsyncStackSize * DSS_COUNT * NumDSSRTStacks, IGC::RTStackAlign) +
        IGC::Align(SWStackSize * DSS_COUNT * NumDSSRTStacks, IGC::RTStackAlign);
}


constexpr uint32_t getSyncStackSize() {
#define STYLE(X) static_assert(sizeofRTStack2<RTStack2<Xe>>() == sizeofRTStack2<RTStack2<X>>());
#include "RayTracingMemoryStyle.h"
#undef STYLE
    return sizeof(RTStack2<Xe>);
}

// As per this:
// The rtMemBasePtr points to the top of the async stacks.  After having computed
// the full allocation with 'calcRTMemoryAllocSize()', this returns the offset
// of the async stacks, hot zone, and sw stack from the base of the allocation.
constexpr uint64_t calcRTMemoryOffsets(
    uint64_t SWHotZoneSize,
    uint64_t SyncStackSize,
    uint64_t AsyncStackSize,
    uint64_t &SWHotZoneOffset,
    uint64_t &SWStackOffset,
    uint64_t DSS_COUNT,
    uint64_t NumDSSRTStacks,
    uint64_t SIMD_LANES_PER_DSS)
{
    uint64_t AsyncOffset =
        IGC::Align(SWHotZoneSize * DSS_COUNT * NumDSSRTStacks, IGC::RTStackAlign) +
        IGC::Align(SyncStackSize * DSS_COUNT * SIMD_LANES_PER_DSS, IGC::RTStackAlign);

    // The hot zone is the base of the allocation
    SWHotZoneOffset = 0;
    SWStackOffset =
        AsyncOffset +
        IGC::Align(AsyncStackSize * DSS_COUNT * NumDSSRTStacks, IGC::RTStackAlign);

    return AsyncOffset;
}


// unit tests:
static_assert(
    calcRTMemoryAllocSize(
        sizeof(SWHotZone_v1), sizeof(RTStack2<Xe>), sizeof(RTStack2<Xe>), 128,
        32, 2048, 16 * 8 * 16) ==
    sizeof(RTMemory<SWHotZone_v1, RTStack2<Xe>, RTStack2<Xe>, uint8_t[128], IGC::RTStackAlign,
        32, 2048, 16 * 8 * 16>), "mismatch?");

static_assert(
    calcRTMemoryAllocSize(
        8, sizeof(RTStack2<Xe>), sizeof(RTStack2<Xe>), 136,
        32, 2048, 16 * 8 * 16) ==
    sizeof(RTMemory<uint8_t[8], RTStack2<Xe>, RTStack2<Xe>, uint8_t[136], IGC::RTStackAlign,
        32, 2048, 16 * 8 * 16>), "mismatch?");


/* a list of commands for the ray tracing hardware */
enum class TraceRayCtrl : uint8_t
{
    TRACE_RAY_INITIAL = 0,              // Initializes hit and initializes traversal state
    TRACE_RAY_INSTANCE = 1,             // Loads committed hit and initializes traversal state
    TRACE_RAY_COMMIT = 2,               // Loads potential hit and loads traversal state
    TRACE_RAY_CONTINUE = 3,             // Loads committed hit and loads traversal state
    TRACE_RAY_NONE = 5                  // Illegal!
};

struct TraceRayPayload
{
    enum class PayloadBits : uint8_t
    {
        bvhLevel     = 3,
        reserved     = 5,
        traceRayCtrl = 3,
        reserved_1   = 5,
        stackID      = 12,
        reserved_2   = 4,
    };

    // This is the offset within the bitfield
    enum class PayloadOffsets : uint8_t
    {
        bvhLevel     = 0,
        reserved     = (uint8_t)PayloadBits::bvhLevel,
        traceRayCtrl = PayloadOffsets::reserved + (uint8_t)PayloadBits::reserved,
        reserved_1   = PayloadOffsets::traceRayCtrl + (uint8_t)PayloadBits::traceRayCtrl,
        stackID      = PayloadOffsets::reserved_1 + (uint8_t)PayloadBits::reserved_1,
    };
    // Trace Ray Payload (per SIMD lane)
    struct TraceRayPayloadData
    {
        uint32_t        bvhLevel     : (uint32_t)PayloadBits::bvhLevel;     // bits[  2:0]: tells the hardware which ray to process
        uint32_t        reserved     : (uint32_t)PayloadBits::reserved;     // bits[  7:3]: reserved MBZ
        uint32_t        traceRayCtrl : (uint32_t)PayloadBits::traceRayCtrl; // bits[ 10:8]: trace ray operation to perform
        uint32_t        reserved_1   : (uint32_t)PayloadBits::reserved_1;   // bits[15:11]: reserved MBZ
        uint32_t        stackID      : (uint32_t)PayloadBits::stackID;      // bits[27:16]: the ID of the stack of this thread
        uint32_t        reserved_2   : (uint32_t)PayloadBits::reserved_2;   // bits[31:28]: reserved MBZ
    } payload;
};

struct TraceRayMessage
{
    // This data is sent per message and is uniform across all rays in a dispatch.
    struct Header
    {
        // bits [63:0]
        uint64_t rtDispatchGlobalsPtr : 48;  // pointer to dispatch globals
        uint64_t padding;
        union {
            uint64_t rayQueryLocation;
            // bit 128
            uint64_t rayQuery         : 1;   // indicates a ray query message
            uint64_t rayQueryCheck    : 1;   // indicates a ray query check message
            uint64_t rayQueryRelease  : 1;   // indicates a ray query release message
        };
    } header;

    TraceRayPayload payload;

    static_assert(sizeof(TraceRayPayload::payload) == 4, "Payload must be 4 bytes");
};

// RayQuery enums

enum COMMITTED_STATUS : uint32_t
{
    COMMITTED_NOTHING,
    COMMITTED_TRIANGLE_HIT,
    COMMITTED_PROCEDURAL_PRIMITIVE_HIT
};

enum CANDIDATE_TYPE : uint32_t
{
    CANDIDATE_NON_OPAQUE_TRIANGLE,
    CANDIDATE_PROCEDURAL_PRIMITIVE
};

struct RayQueryReturnData
{
    enum class Bits : uint8_t
    {
        proceed_further = 1,
        committedStatus = 2,
        candidateType = 1,
        reserved = 28,
    };

    // This union is defined for futures use.
    // It will replace writing the RayQuery Return Value
    // to the MemHit in ShadowMemory.
    union
    {
        uint32_t rayQueryReturnDWord;

        struct
        {
            uint32_t PROCEED_FURTHER : (uint32_t)Bits::proceed_further; // This bit when set indicates that traversal is not complete i.e. all candidates have not been searched yet.
            COMMITTED_STATUS committedStatus : (uint32_t)Bits::committedStatus; // This bit field provides the Committed Hit's Status / Type.
            CANDIDATE_TYPE candidateType : (uint32_t)Bits::candidateType;   // This bit indicates the candidate type for a potential hit.
            uint32_t reserved : (uint32_t)Bits::reserved;        // Reserved
        };
    };
};

} // namespace RTStackFormat


DISABLE_WARNING_POP  // restore the previously saved pragma state, restore former compiler settings
