path: root/zluda/src/impl/device.rs

use super::{context, transmute_lifetime, transmute_lifetime_mut, CUresult, GlobalState};
use crate::cuda;
use cuda::{CUdevice_attribute, CUuuid_st};
use ocl_core::{ClDeviceIdPtr, ContextProperties, DeviceType};
use std::{
    cmp,
    ffi::c_void,
    mem,
    os::raw::{c_char, c_int, c_uint},
    ptr,
    sync::atomic::{AtomicU32, Ordering},
};

const PROJECT_URL_SUFFIX_SHORT: &'static str = " [ZLUDA]";
const PROJECT_URL_SUFFIX_LONG: &'static str = " [github.com/vosen/ZLUDA]";

#[repr(transparent)]
#[derive(Clone, Copy, Eq, PartialEq, Hash)]
pub struct Index(pub c_int);

pub struct Device {
    pub index: Index,
    pub base: l0::Device,
    pub ocl_base: ocl_core::DeviceId,
    pub default_queue: ocl_core::CommandQueue,
    pub ocl_context: ocl_core::Context,
    pub(crate) ocl_ext: OpenCLExtensions,
    pub primary_context: context::Context,
    properties: Option<Box<l0::sys::ze_device_properties_t>>,
    image_properties: Option<Box<l0::sys::ze_device_image_properties_t>>,
    memory_properties: Option<Vec<l0::sys::ze_device_memory_properties_t>>,
    compute_properties: Option<Box<l0::sys::ze_device_compute_properties_t>>,
}

type cl_mem_properties_intel = ocl_core::ffi::cl_bitfield;

pub(crate) struct OpenCLExtensions {
    pub clDeviceMemAllocINTEL: unsafe extern "system" fn(
        ocl_core::ffi::cl_context,
        ocl_core::ffi::cl_device_id,
        *const cl_mem_properties_intel,
        usize,
        ocl_core::ffi::cl_uint,
        *mut ocl_core::ffi::cl_int,
    ) -> *mut c_void,
    pub clEnqueueMemcpyINTEL: unsafe extern "system" fn(
        ocl_core::ffi::cl_command_queue,
        ocl_core::ffi::cl_bool,
        *mut c_void,
        *const c_void,
        usize,
        ocl_core::ffi::cl_uint,
        *const ocl_core::ffi::cl_event,
        *mut ocl_core::ffi::cl_event,
    ) -> ocl_core::ffi::cl_int,
    pub clMemBlockingFreeINTEL:
        unsafe extern "system" fn(ocl_core::ffi::cl_context, *mut c_void) -> ocl_core::ffi::cl_int,
    pub clEnqueueMemFillINTEL: unsafe extern "system" fn(
        ocl_core::ffi::cl_command_queue,
        *mut c_void,
        *const c_void,
        usize,
        usize,
        ocl_core::ffi::cl_uint,
        *const ocl_core::ffi::cl_event,
        *mut ocl_core::ffi::cl_event,
    ) -> ocl_core::ffi::cl_int,
}

impl OpenCLExtensions {
    fn new(plat: &ocl_core::PlatformId) -> Result<Self, CUresult> {
        let clDeviceMemAllocINTEL = unsafe {
            ocl_core::get_extension_function_address_for_platform(
                plat,
                "clDeviceMemAllocINTEL",
                None,
            )?
        };
        let clEnqueueMemcpyINTEL = unsafe {
            ocl_core::get_extension_function_address_for_platform(
                plat,
                "clEnqueueMemcpyINTEL",
                None,
            )?
        };
        let clMemBlockingFreeINTEL = unsafe {
            ocl_core::get_extension_function_address_for_platform(
                plat,
                "clMemBlockingFreeINTEL",
                None,
            )?
        };
        let clEnqueueMemFillINTEL = unsafe {
            ocl_core::get_extension_function_address_for_platform(
                plat,
                "clEnqueueMemFillINTEL",
                None,
            )?
        };
        Ok(Self {
            clDeviceMemAllocINTEL: unsafe { mem::transmute(clDeviceMemAllocINTEL) },
            clEnqueueMemcpyINTEL: unsafe { mem::transmute(clEnqueueMemcpyINTEL) },
            clMemBlockingFreeINTEL: unsafe { mem::transmute(clMemBlockingFreeINTEL) },
            clEnqueueMemFillINTEL: unsafe { mem::transmute(clEnqueueMemFillINTEL) },
        })
    }

    pub unsafe fn device_mem_alloc(
        &self,
        ctx: &ocl_core::Context,
        device: &ocl_core::DeviceId,
        size: usize,
        alignment: ocl_core::ffi::cl_uint,
    ) -> Result<*mut c_void, CUresult> {
        let mut error = 0;
        let result = (self.clDeviceMemAllocINTEL)(
            ctx.as_ptr(),
            device.as_ptr(),
            ptr::null(),
            size,
            alignment,
            &mut error,
        );
        if error == 0 {
            Ok(result)
        } else {
            Err(CUresult::CUDA_ERROR_UNKNOWN)
        }
    }

    pub unsafe fn enqueue_memcpy(
        &self,
        queue: &ocl_core::CommandQueue,
        blocking: bool,
        dst: *mut c_void,
        src: *const c_void,
        size: usize,
    ) -> Result<(), CUresult> {
        let error = (self.clEnqueueMemcpyINTEL)(
            queue.as_ptr(),
            if blocking { 1 } else { 0 },
            dst,
            src,
            size,
            0,
            ptr::null(),
            ptr::null_mut(),
        );
        if error == 0 {
            Ok(())
        } else {
            Err(CUresult::CUDA_ERROR_UNKNOWN)
        }
    }

    pub unsafe fn mem_blocking_free(
        &self,
        ctx: &ocl_core::Context,
        mem_ptr: *mut c_void,
    ) -> Result<(), CUresult> {
        let error = (self.clMemBlockingFreeINTEL)(ctx.as_ptr(), mem_ptr);
        if error == 0 {
            Ok(())
        } else {
            Err(CUresult::CUDA_ERROR_UNKNOWN)
        }
    }

    pub unsafe fn enqueue_memfill(
        &self,
        queue: &ocl_core::CommandQueue,
        dst: *mut c_void,
        pattern: *const c_void,
        patternSize: usize,
        size: usize,
    ) -> Result<ocl_core::Event, CUresult> {
        let mut signal: ocl_core::ffi::cl_event = ptr::null_mut();
        let error = (self.clEnqueueMemFillINTEL)(
            queue.as_ptr(),
            dst,
            pattern,
            patternSize,
            size,
            0,
            ptr::null(),
            &mut signal,
        );
        if error == 0 {
            Ok(ocl_core::Event::from_raw(signal))
        } else {
            Err(CUresult::CUDA_ERROR_UNKNOWN)
        }
    }
}

unsafe impl Send for Device {}

impl Device {
    pub fn new(
        l0_dev: l0::Device,
        platform: ocl_core::PlatformId,
        ocl_dev: ocl_core::DeviceId,
        idx: usize,
    ) -> Result<Self, CUresult> {
        let ocl_ext = OpenCLExtensions::new(&platform)?;
        let mut props = ocl_core::ContextProperties::new();
        props.set_platform(platform);
        let ctx = ocl_core::create_context(Some(&props), &[ocl_dev], None, None)?;
        let queue = ocl_core::create_command_queue(&ctx, ocl_dev, None)?;
        let primary_context =
            context::Context::new(context::ContextData::new(0, true, ptr::null_mut())?);
        Ok(Self {
            ocl_ext,
            index: Index(idx as c_int),
            base: l0_dev,
            ocl_base: ocl_dev,
            default_queue: queue,
            ocl_context: ctx,
            primary_context,
            properties: None,
            image_properties: None,
            memory_properties: None,
            compute_properties: None,
        })
    }

    pub fn late_init(&mut self) {
        self.primary_context.as_option_mut().unwrap().device = self as *mut _;
    }

    fn get_properties<'a>(&'a mut self) -> l0::Result<&'a l0::sys::ze_device_properties_t> {
        if let Some(ref prop) = self.properties {
            return Ok(prop);
        }
        let mut props = Default::default();
        self.base.get_properties(&mut props)?;
        Ok(self.properties.get_or_insert(Box::new(props)))
    }

    fn get_image_properties(&mut self) -> l0::Result<&l0::sys::ze_device_image_properties_t> {
        if let Some(ref prop) = self.image_properties {
            return Ok(prop);
        }
        let mut props = Default::default();
        self.base.get_image_properties(&mut props)?;
        Ok(self.image_properties.get_or_insert(Box::new(props)))
    }

    fn get_memory_properties(&mut self) -> l0::Result<&[l0::sys::ze_device_memory_properties_t]> {
        if let Some(ref prop) = self.memory_properties {
            return Ok(prop);
        }
        match self.base.get_memory_properties() {
            Ok(prop) => Ok(self.memory_properties.get_or_insert(prop)),
            Err(e) => Err(e),
        }
    }

    fn get_compute_properties(&mut self) -> l0::Result<&l0::sys::ze_device_compute_properties_t> {
        if let Some(ref prop) = self.compute_properties {
            return Ok(prop);
        }
        let mut props = Default::default();
        self.base.get_compute_properties(&mut props)?;
        Ok(self.compute_properties.get_or_insert(Box::new(props)))
    }

    fn get_max_simd(&mut self) -> l0::Result<u32> {
        let props = self.get_compute_properties()?;
        Ok(*props.subGroupSizes[0..props.numSubGroupSizes as usize]
            .iter()
            .max()
            .unwrap())
    }
}

pub fn get_count(count: *mut c_int) -> Result<(), CUresult> {
    let len = GlobalState::lock(|state| state.devices.len())?;
    unsafe { *count = len as c_int };
    Ok(())
}

pub fn get(device: *mut Index, ordinal: c_int) -> Result<(), CUresult> {
    if device == ptr::null_mut() || ordinal < 0 {
        return Err(CUresult::CUDA_ERROR_INVALID_VALUE);
    }
    let len = GlobalState::lock(|state| state.devices.len())?;
    if ordinal < (len as i32) {
        unsafe { *device = Index(ordinal) };
        Ok(())
    } else {
        Err(CUresult::CUDA_ERROR_INVALID_VALUE)
    }
}

pub fn get_name(name: *mut c_char, len: i32, dev_idx: Index) -> Result<(), CUresult> {
    if name == ptr::null_mut() || len < 0 {
        return Err(CUresult::CUDA_ERROR_INVALID_VALUE);
    }
    let name_ptr = GlobalState::lock_device(dev_idx, |dev| {
        let props = dev.get_properties()?;
        Ok::<_, l0::sys::ze_result_t>(props.name.as_ptr())
    })??;
    let name_len = (0..256)
        .position(|i| unsafe { *name_ptr.add(i) } == 0)
        .unwrap_or(256);
    let mut dst_null_pos = cmp::min((len - 1) as usize, name_len);
    unsafe { std::ptr::copy_nonoverlapping(name_ptr, name, dst_null_pos) };
    if name_len + PROJECT_URL_SUFFIX_LONG.len() < (len as usize) {
        unsafe {
            std::ptr::copy_nonoverlapping(
                PROJECT_URL_SUFFIX_LONG.as_ptr(),
                name.add(name_len) as *mut _,
                PROJECT_URL_SUFFIX_LONG.len(),
            )
        };
        dst_null_pos += PROJECT_URL_SUFFIX_LONG.len();
    } else if name_len + PROJECT_URL_SUFFIX_SHORT.len() < (len as usize) {
        unsafe {
            std::ptr::copy_nonoverlapping(
                PROJECT_URL_SUFFIX_SHORT.as_ptr(),
                name.add(name_len) as *mut _,
                PROJECT_URL_SUFFIX_SHORT.len(),
            )
        };
        dst_null_pos += PROJECT_URL_SUFFIX_SHORT.len();
    }
    unsafe { *(name.add(dst_null_pos)) = 0 };
    Ok(())
}

pub fn total_mem_v2(bytes: *mut usize, dev_idx: Index) -> Result<(), CUresult> {
    if bytes == ptr::null_mut() {
        return Err(CUresult::CUDA_ERROR_INVALID_VALUE);
    }
    let mem_props = GlobalState::lock_device(dev_idx, |dev| {
        let mem_props = dev.get_memory_properties()?;
        Ok::<_, l0::sys::ze_result_t>(mem_props)
    })??;
    let max_mem = mem_props
        .iter()
        .map(|p| p.totalSize)
        .max()
        .ok_or(CUresult::CUDA_ERROR_ILLEGAL_STATE)?;
    unsafe { *bytes = max_mem as usize };
    Ok(())
}

impl CUdevice_attribute {
    fn get_static_value(self) -> Option<i32> {
        match self {
            CUdevice_attribute::CU_DEVICE_ATTRIBUTE_GPU_OVERLAP => Some(1),
            CUdevice_attribute::CU_DEVICE_ATTRIBUTE_KERNEL_EXEC_TIMEOUT => Some(1),
            // TODO: go back to this once we have more funcitonality implemented
            CUdevice_attribute::CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR => Some(8),
            CUdevice_attribute::CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR => Some(0),
            CUdevice_attribute::CU_DEVICE_ATTRIBUTE_CAN_MAP_HOST_MEMORY => Some(1),
            _ => None,
        }
    }
}

pub fn get_attribute(
    pi: *mut i32,
    attrib: CUdevice_attribute,
    dev_idx: Index,
) -> Result<(), CUresult> {
    if pi == ptr::null_mut() {
        return Err(CUresult::CUDA_ERROR_INVALID_VALUE);
    }
    if let Some(value) = attrib.get_static_value() {
        unsafe { *pi = value };
        return Ok(());
    }
    let value = match attrib {
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_INTEGRATED => {
            GlobalState::lock_device(dev_idx, |dev| {
                let props = dev.get_properties()?;
                if (props.flags
                    & l0::sys::ze_device_property_flags_t::ZE_DEVICE_PROPERTY_FLAG_INTEGRATED)
                    == l0::sys::ze_device_property_flags_t::ZE_DEVICE_PROPERTY_FLAG_INTEGRATED
                {
                    Ok::<_, CUresult>(1)
                } else {
                    Ok(0)
                }
            })??
        }
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_ASYNC_ENGINE_COUNT => {
            GlobalState::lock_device(dev_idx, |dev| {
                let props = dev.get_properties()?;
                Ok::<_, l0::sys::ze_result_t>(props.maxHardwareContexts as i32)
            })??
        }
        // Streaming Multiprocessor corresponds roughly to a sub-slice (thread group can't cross either)
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT => {
            GlobalState::lock_device(dev_idx, |dev| {
                let props = dev.get_properties()?;
                Ok::<_, l0::sys::ze_result_t>((props.numSlices * props.numSubslicesPerSlice) as i32)
            })??
        }
        // I honestly don't know how to answer this query
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR => {
            GlobalState::lock_device(dev_idx, |dev| {
                let max_simd = dev.get_max_simd()?;
                let props = dev.get_properties()?;
                Ok::<_, l0::sys::ze_result_t>(
                    (props.numEUsPerSubslice * props.numThreadsPerEU * max_simd) as i32,
                )
            })??
        }
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK => {
            GlobalState::lock_device(dev_idx, |dev| {
                let props = dev.get_compute_properties()?;
                Ok::<_, l0::sys::ze_result_t>(cmp::min(
                    i32::max_value() as u32,
                    props.maxTotalGroupSize,
                ) as i32)
            })??
        }
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE1D_WIDTH => {
            GlobalState::lock_device(dev_idx, |dev| {
                let props = dev.get_image_properties()?;
                Ok::<_, l0::sys::ze_result_t>(cmp::min(
                    props.maxImageDims1D,
                    c_int::max_value() as u32,
                ) as c_int)
            })??
        }
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X => {
            GlobalState::lock_device(dev_idx, |dev| {
                let props = dev.get_compute_properties()?;
                Ok::<_, l0::sys::ze_result_t>(cmp::min(
                    i32::max_value() as u32,
                    props.maxGroupCountX,
                ) as i32)
            })??
        }
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Y => {
            GlobalState::lock_device(dev_idx, |dev| {
                let props = dev.get_compute_properties()?;
                Ok::<_, l0::sys::ze_result_t>(cmp::min(
                    i32::max_value() as u32,
                    props.maxGroupCountY,
                ) as i32)
            })??
        }
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Z => {
            GlobalState::lock_device(dev_idx, |dev| {
                let props = dev.get_compute_properties()?;
                Ok::<_, l0::sys::ze_result_t>(cmp::min(
                    i32::max_value() as u32,
                    props.maxGroupCountZ,
                ) as i32)
            })??
        }
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X => {
            GlobalState::lock_device(dev_idx, |dev| {
                let props = dev.get_compute_properties()?;
                Ok::<_, l0::sys::ze_result_t>(
                    cmp::min(i32::max_value() as u32, props.maxGroupSizeX) as i32,
                )
            })??
        }
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Y => {
            GlobalState::lock_device(dev_idx, |dev| {
                let props = dev.get_compute_properties()?;
                Ok::<_, l0::sys::ze_result_t>(
                    cmp::min(i32::max_value() as u32, props.maxGroupSizeY) as i32,
                )
            })??
        }
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Z => {
            GlobalState::lock_device(dev_idx, |dev| {
                let props = dev.get_compute_properties()?;
                Ok::<_, l0::sys::ze_result_t>(
                    cmp::min(i32::max_value() as u32, props.maxGroupSizeZ) as i32,
                )
            })??
        }
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK => {
            GlobalState::lock_device(dev_idx, |dev| {
                let props = dev.get_compute_properties()?;
                Ok::<_, l0::sys::ze_result_t>(props.maxSharedLocalMemory as i32)
            })??
        }
        CUdevice_attribute::CU_DEVICE_ATTRIBUTE_WARP_SIZE => {
            GlobalState::lock_device(dev_idx, |dev| Ok::<_, CUresult>(dev.get_max_simd()? as i32))??
        }
        _ => {
            // TODO: support more attributes for CUDA runtime
            /*
            return Err(l0::Error(
                l0::sys::ze_result_t::ZE_RESULT_ERROR_UNSUPPORTED_FEATURE,
            ))
            */
            return Ok(());
        }
    };
    unsafe { *pi = value };
    Ok(())
}

pub fn get_uuid(uuid: *mut CUuuid_st, dev_idx: Index) -> Result<(), CUresult> {
    let ze_uuid = GlobalState::lock_device(dev_idx, |dev| {
        let props = dev.get_properties()?;
        Ok::<_, l0::sys::ze_result_t>(props.uuid)
    })??;
    unsafe {
        *uuid = CUuuid_st {
            bytes: mem::transmute(ze_uuid.id),
        }
    };
    Ok(())
}

// TODO: add support if Level 0 exposes it
pub fn get_luid(
    luid: *mut c_char,
    dev_node_mask: *mut c_uint,
    _dev_idx: Index,
) -> Result<(), CUresult> {
    unsafe { ptr::write_bytes(luid, 0u8, 8) };
    unsafe { *dev_node_mask = 0 };
    Ok(())
}

pub fn primary_ctx_get_state(
    dev_idx: Index,
    flags: *mut u32,
    active: *mut i32,
) -> Result<(), CUresult> {
    let (is_active, flags_value) = GlobalState::lock_device(dev_idx, |dev| {
        // This is safe because primary context can't be dropped
        let ctx_ptr = &mut dev.primary_context as *mut _;
        let flags_ptr =
            (&unsafe { dev.primary_context.as_ref_unchecked() }.flags) as *const AtomicU32;
        let is_active = context::CONTEXT_STACK
            .with(|stack| stack.borrow().last().map(|x| *x))
            .map(|current| current == ctx_ptr)
            .unwrap_or(false);
        let flags_value = unsafe { &*flags_ptr }.load(Ordering::Relaxed);
        Ok::<_, l0::sys::ze_result_t>((is_active, flags_value))
    })??;
    unsafe { *active = if is_active { 1 } else { 0 } };
    unsafe { *flags = flags_value };
    Ok(())
}

pub fn primary_ctx_retain(
    pctx: *mut *mut context::Context,
    dev_idx: Index,
) -> Result<(), CUresult> {
    let ctx_ptr = GlobalState::lock_device(dev_idx, |dev| &mut dev.primary_context as *mut _)?;
    unsafe { *pctx = ctx_ptr };
    Ok(())
}

// TODO: allow for retain/reset/release of primary context
pub(crate) fn primary_ctx_release_v2(_dev_idx: Index) -> CUresult {
    CUresult::CUDA_SUCCESS
}

pub struct DynamicEventPool {
    count: usize,
    pool_flags: l0::sys::ze_event_pool_flags_t,
    signal_flags: l0::sys::ze_event_scope_flags_t,
    events: Vec<DynamicEventPoolEntry>,
}

impl DynamicEventPool {
    fn new(
        dev: l0::Device,
        ctx: &'static l0::Context,
        pool_flags: l0::sys::ze_event_pool_flags_t,
        signal_flags: l0::sys::ze_event_scope_flags_t,
    ) -> l0::Result<Self> {
        Ok(DynamicEventPool {
            count: 0,
            pool_flags,
            signal_flags,
            events: vec![DynamicEventPoolEntry::new(dev, ctx, pool_flags)?],
        })
    }

    pub fn get(
        &'static mut self,
        dev: l0::Device,
        ctx: &'static l0::Context,
    ) -> l0::Result<(l0::Event<'static>, u64)> {
        self.count += 1;
        let events = unsafe { transmute_lifetime_mut(&mut self.events) };
        let (global_idx, (ev, local_idx)) = {
            for (idx, entry) in self.events.iter_mut().enumerate() {
                if let Some((ev, local_idx)) = entry.get(self.signal_flags)? {
                    let marker = (idx << 32) as u64 | local_idx as u64;
                    return Ok((ev, marker));
                }
            }
            events.push(DynamicEventPoolEntry::new(dev, ctx, self.pool_flags)?);
            let global_idx = (events.len() - 1) as u64;
            (
                global_idx,
                events.last_mut().unwrap().get(self.signal_flags)?.unwrap(),
            )
        };
        let marker = (global_idx << 32) | local_idx as u64;
        Ok((ev, marker))
    }

    pub fn mark_as_free(&mut self, marker: u64) {
        let global_idx = (marker >> 32) as u32;
        self.events[global_idx as usize].mark_as_free(marker as u32);
        self.count -= 1;
        // TODO: clean up empty entries
    }
}

const DYNAMIC_EVENT_POOL_ENTRY_SIZE: usize = 448;
const DYNAMIC_EVENT_POOL_ENTRY_BITMAP_SIZE: usize =
    DYNAMIC_EVENT_POOL_ENTRY_SIZE / (mem::size_of::<u64>() * 8);
#[repr(C)]
#[repr(align(64))]
struct DynamicEventPoolEntry {
    event_pool: l0::EventPool<'static>,
    bit_map: [u64; DYNAMIC_EVENT_POOL_ENTRY_BITMAP_SIZE],
}

impl DynamicEventPoolEntry {
    fn new(
        dev: l0::Device,
        ctx: &'static l0::Context,
        flags: l0::sys::ze_event_pool_flags_t,
    ) -> l0::Result<Self> {
        Ok(DynamicEventPoolEntry {
            event_pool: l0::EventPool::new(
                ctx,
                flags,
                DYNAMIC_EVENT_POOL_ENTRY_SIZE as u32,
                Some(&[dev]),
            )?,
            bit_map: [0; DYNAMIC_EVENT_POOL_ENTRY_BITMAP_SIZE],
        })
    }

    fn get(
        &'static mut self,
        signal: l0::sys::ze_event_scope_flags_t,
    ) -> l0::Result<Option<(l0::Event<'static>, u32)>> {
        for (idx, value) in self.bit_map.iter_mut().enumerate() {
            let shift = first_index_of_zero_u64(*value);
            if shift == 64 {
                continue;
            }
            *value = *value | (1u64 << shift);
            let entry_index = (idx as u32 * 64u32) + shift;
            let event = l0::Event::new(
                &self.event_pool,
                entry_index,
                signal,
                l0::sys::ze_event_scope_flags_t(0),
            )?;
            return Ok(Some((event, entry_index)));
        }
        Ok(None)
    }

    fn mark_as_free(&mut self, idx: u32) {
        let value = &mut self.bit_map[idx as usize / 64];
        let shift = idx % 64;
        *value = *value & !(1 << shift);
    }
}

fn first_index_of_zero_u64(x: u64) -> u32 {
    let x = !x;
    (x & x.wrapping_neg()).trailing_zeros()
}

#[cfg(test)]
mod test {
    use std::mem;

    use super::DynamicEventPoolEntry;

    use super::super::test::CudaDriverFns;
    use super::super::CUresult;

    cuda_driver_test!(primary_ctx_default_inactive);

    fn primary_ctx_default_inactive<T: CudaDriverFns>() {
        assert_eq!(T::cuInit(0), CUresult::CUDA_SUCCESS);
        let mut flags = u32::max_value();
        let mut active = i32::max_value();
        assert_eq!(
            T::cuDevicePrimaryCtxGetState(0, &mut flags, &mut active),
            CUresult::CUDA_SUCCESS
        );
        assert_eq!(flags, 0);
        assert_eq!(active, 0);
    }

    #[test]
    pub fn dynamic_event_pool_page_is_64b() {
        assert_eq!(mem::size_of::<DynamicEventPoolEntry>(), 64);
        assert_eq!(mem::align_of::<DynamicEventPoolEntry>(), 64);
    }
}