From: Zhi Wang <[email protected]> The GSP plugin needs instance identity, channel allocation and host parameters before it can serve guest requests.
Encode the configuration as an NVKV stream, prefix it with its word count and send SETUP_CONFIG_PARAMS_AND_INIT after protocol negotiation through the instance's PluginRpc channel. Derive the host's opaque device identity from the VF DBDF within the supported 16-bit PCI segment namespace. This identifies the local device without imposing a migration-persistent identity. Describe the supported whole-GPU configuration and distinguish the optional VMM capability bitmap from the separately reported migration features. Encode the host page size with the kernel's infallible conversion. Co-developed-by: Eliot Courtney <[email protected]> Signed-off-by: Eliot Courtney <[email protected]> Signed-off-by: Zhi Wang <[email protected]> Signed-off-by: Alok Kumar <[email protected]> --- drivers/gpu/nova-core/vgpu/commands.rs | 9 ++ drivers/gpu/nova-core/vgpu/fw/commands.rs | 135 +++++++++++++++++++ drivers/gpu/nova-core/vgpu/gsp_plugin_rpc.rs | 17 +++ drivers/gpu/nova-core/vgpu/instance.rs | 16 +++ 4 files changed, 177 insertions(+) diff --git a/drivers/gpu/nova-core/vgpu/commands.rs b/drivers/gpu/nova-core/vgpu/commands.rs index a68841fe4019..968942273b9e 100644 --- a/drivers/gpu/nova-core/vgpu/commands.rs +++ b/drivers/gpu/nova-core/vgpu/commands.rs @@ -170,3 +170,12 @@ pub(super) fn negotiate_plugin_version( ) -> Result { rpc.rpc_call(dev, RpcMessage::VersionNegotiation, &[]) } + +/// Send an instance's encoded configuration to the GSP plugin. +pub(super) fn send_plugin_config( + dev: &device::Device<device::Bound>, + rpc: &mut PluginRpc<'_, '_>, + config: &[u64], +) -> Result { + rpc.rpc_call_nvkv(dev, RpcMessage::SetupConfigParamsAndInit, config) +} diff --git a/drivers/gpu/nova-core/vgpu/fw/commands.rs b/drivers/gpu/nova-core/vgpu/fw/commands.rs index c3d0acece2eb..3f94435e7a59 100644 --- a/drivers/gpu/nova-core/vgpu/fw/commands.rs +++ b/drivers/gpu/nova-core/vgpu/fw/commands.rs @@ -9,6 +9,7 @@ use kernel::{ alloc::ArrayVec, bitfield, + num::casts::usize_as_u64, prelude::*, // }; @@ -34,6 +35,7 @@ #[repr(u32)] pub(crate) enum RpcMessage { VersionNegotiation = bindings::MESSAGE_NV_VGPU_CPU_RPC_MSG_VERSION_NEGOTIATION, + SetupConfigParamsAndInit = bindings::MESSAGE_NV_VGPU_CPU_RPC_MSG_SETUP_CONFIG_PARAMS_AND_INIT, } bitfield! { @@ -262,3 +264,136 @@ pub(crate) struct VgpuProperties { impl VgpuProperties { const STRING_LEN: usize = 64; } + +#[derive(Clone, Copy)] +#[repr(u32)] +enum HypervisorType { + Unknown = 4, +} + +impl From<HypervisorType> for u32 { + fn from(value: HypervisorType) -> Self { + // CAST: `HypervisorType` uses the wire field's `u32` representation. + value as u32 + } +} + +#[derive(Clone, Copy)] +#[repr(u32)] +enum CpuArch { + Aarch64 = 1, + X86_64 = 2, +} + +impl CpuArch { + fn host() -> Result<Self> { + if cfg!(target_arch = "x86_64") { + Ok(Self::X86_64) + } else if cfg!(target_arch = "aarch64") { + Ok(Self::Aarch64) + } else { + Err(EOPNOTSUPP) + } + } +} + +impl From<CpuArch> for u32 { + fn from(value: CpuArch) -> Self { + // CAST: `CpuArch` uses the wire field's `u32` representation. + value as u32 + } +} + +#[derive(Clone, Copy)] +struct MigrationFeature(u32); + +impl MigrationFeature { + const PRESERVE_CTX_BUF: Self = Self(0x4000); +} + +impl From<MigrationFeature> for u32 { + fn from(value: MigrationFeature) -> Self { + value.0 + } +} + +bitfield! { + struct FeatureFlags(u64) { + 3:3 enable_uvm => bool; + 5:5 vmm_migration => bool; + } +} + +nvkv_encode! { + struct PluginConfigParamsRequest { + uuid: Key<[u8; 16], { Self::UUID_KEY }>, + dbdf: Key<Dbdf, { Self::DBDF_KEY }, u32>, + device_instance_id: Key<u32, { Self::DEVICE_INSTANCE_ID_KEY }>, + vgpu_type: Key<u32, { Self::VGPU_TYPE_KEY }>, + vm_pid: Key<u32, { Self::VM_PID_KEY }>, + swizz_id: Key<SwizzId, { Self::SWIZZ_ID_KEY }, u32>, + num_channels: Key<u32, { Self::NUM_CHANNELS_KEY }>, + num_plugin_channels: Key<u32, { Self::NUM_PLUGIN_CHANNELS_KEY }>, + vmm_cap: Key<u32, { Self::VMM_CAP_KEY }>, + migration_feature: Key<MigrationFeature, { Self::MIGRATION_FEATURE_KEY }, u32>, + hypervisor_type: Key<HypervisorType, { Self::HYPERVISOR_TYPE_KEY }, u32>, + cpu_arch: Key<CpuArch, { Self::CPU_ARCH_KEY }, u32>, + page_size: Key<u64, { Self::PAGE_SIZE_KEY }>, + feature_flags: Key<FeatureFlags, { Self::FEATURE_FLAGS_KEY }, u64>, + } +} + +impl PluginConfigParamsRequest { + const UUID_KEY: KeyId = 0x0001; + const DBDF_KEY: KeyId = 0x0002; + const DEVICE_INSTANCE_ID_KEY: KeyId = 0x0004; + const VGPU_TYPE_KEY: KeyId = 0x0005; + const VM_PID_KEY: KeyId = 0x0006; + const SWIZZ_ID_KEY: KeyId = 0x0010; + const NUM_CHANNELS_KEY: KeyId = 0x0011; + const NUM_PLUGIN_CHANNELS_KEY: KeyId = 0x0012; + const VMM_CAP_KEY: KeyId = 0x0020; + const MIGRATION_FEATURE_KEY: KeyId = 0x0021; + const HYPERVISOR_TYPE_KEY: KeyId = 0x0022; + const CPU_ARCH_KEY: KeyId = 0x0023; + const PAGE_SIZE_KEY: KeyId = 0x0024; + const FEATURE_FLAGS_KEY: KeyId = 0x0030; +} + +/// Encodes plugin configuration parameters using the typed NVKV schema. +pub(crate) fn encode_plugin_config_params( + uuid: [u8; 16], + dbdf: Dbdf, + vgpu_type: u32, + vm_pid: u32, + num_channels: u32, + num_plugin_channels: u32, +) -> Result<KVVec<u64>> { + let request = PluginConfigParamsRequest { + uuid: uuid.into(), + dbdf: dbdf.into(), + // The full host PCI address distinguishes VFs within a VM, including VFs + // from different PFs. This opaque identity lasts for the host device lifetime. + device_instance_id: dbdf.into_raw().into(), + vgpu_type: vgpu_type.into(), + vm_pid: vm_pid.into(), + swizz_id: SwizzId::WHOLE_GPU.into(), + num_channels: num_channels.into(), + num_plugin_channels: num_plugin_channels.into(), + // Advertise no optional VMM capabilities (vmioplugin.h VMM_CAP_*), + // independently of the plugin feature_flags and migration_feature below. + vmm_cap: 0.into(), + migration_feature: MigrationFeature::PRESERVE_CTX_BUF.into(), + hypervisor_type: HypervisorType::Unknown.into(), + cpu_arch: CpuArch::host()?.into(), + page_size: usize_as_u64(kernel::page::PAGE_SIZE).into(), + feature_flags: FeatureFlags::zeroed() + .with_enable_uvm(false) + .with_vmm_migration(true) + .into(), + }; + + let mut encoder = Encoder::new(); + request.encode(&mut encoder)?; + Ok(encoder.finish()) +} diff --git a/drivers/gpu/nova-core/vgpu/gsp_plugin_rpc.rs b/drivers/gpu/nova-core/vgpu/gsp_plugin_rpc.rs index 696127c7169f..48977cd67792 100644 --- a/drivers/gpu/nova-core/vgpu/gsp_plugin_rpc.rs +++ b/drivers/gpu/nova-core/vgpu/gsp_plugin_rpc.rs @@ -19,6 +19,7 @@ use kernel::{ device, + num::casts::usize_as_u64, prelude::*, time::{ delay::fsleep, @@ -26,6 +27,7 @@ Instant, Monotonic, // }, + transmute::AsBytes, // }; use crate::{ @@ -105,6 +107,21 @@ pub(super) fn rpc_call( self.wait_response(dev, sequence) } + /// Send an NVKV stream prefixed by its word count. + pub(super) fn rpc_call_nvkv( + &mut self, + dev: &device::Device<device::Bound>, + message_type: RpcMessage, + encoded: &[u64], + ) -> Result { + let word_count = usize_as_u64(encoded.len()); + let mut payload = KVec::new(); + payload.extend_from_slice(&word_count.to_le_bytes(), GFP_KERNEL)?; + payload.extend_from_slice(AsBytes::as_bytes(encoded), GFP_KERNEL)?; + + self.rpc_call(dev, message_type, &payload) + } + fn wait_response(&self, dev: &device::Device<device::Bound>, expected_sequence: u32) -> Result { let start = Instant::<Monotonic>::now(); let timeout = Delta::from_secs(120); diff --git a/drivers/gpu/nova-core/vgpu/instance.rs b/drivers/gpu/nova-core/vgpu/instance.rs index 445d7f513fdc..247e88bdc82f 100644 --- a/drivers/gpu/nova-core/vgpu/instance.rs +++ b/drivers/gpu/nova-core/vgpu/instance.rs @@ -34,10 +34,12 @@ negotiate_plugin_version, send_bootload, send_cleanup, + send_plugin_config, send_shutdown, Dbdf, // }, fw::commands::{ + encode_plugin_config_params, encode_vgpu_bootload, BootloadInfo, ChannelMapEntry, // @@ -159,6 +161,7 @@ fn activate(&mut self, vgpu: &VgpuManager<'gpu>) -> Result { self.bootload(dev, vgpu.cmdq, &vgpu.fifo_engine_list)?; self.plugin_rpc.init_rpc()?; negotiate_plugin_version(dev, &mut self.plugin_rpc)?; + self.configure_plugin(dev)?; Ok(()) } @@ -230,6 +233,19 @@ fn bootload( Ok(()) } + fn configure_plugin(&mut self, dev: &device::Device<device::Bound>) -> Result { + let config = encode_plugin_config_params( + [0; 16], + self.dbdf, + self.vgpu_type.vgpu_type_id, + self.vm_pid, + u32::try_from(self.chids.len()).map_err(|_| EOVERFLOW)?, + self.num_plugin_channels, + )?; + + send_plugin_config(dev, &mut self.plugin_rpc, &config) + } + /// Stop the plugin when firmware may own instance resources. fn shutdown(&mut self, dev: &device::Device<device::Bound>, cmdq: &Cmdq<'_>) -> Result { if self.needs_teardown { -- 2.53.0
