diff --git a/flowey/flowey_hvlite/src/pipelines/build_igvm.rs b/flowey/flowey_hvlite/src/pipelines/build_igvm.rs index 0f5ba7345f..518716fab9 100644 --- a/flowey/flowey_hvlite/src/pipelines/build_igvm.rs +++ b/flowey/flowey_hvlite/src/pipelines/build_igvm.rs @@ -22,6 +22,8 @@ pub enum OpenhclRecipeCli { X64Cvm, /// X64 OpenHCL, with CVM support using the dev kernel in VTL2 X64CvmDevkern, + /// X64 OpenHCL, with CVM support, plus NVIDIA GPU VPCI relay support. + X64CvmNvidiaVpciRelay, /// X64 OpenHCL booting VTL0 using a test linux-direct kernel + initrd (no /// UEFI). X64TestLinuxDirect, @@ -348,6 +350,7 @@ impl IntoPipeline for BuildIgvmCli { | OpenhclRecipeCli::X64Devkern | OpenhclRecipeCli::X64Cvm | OpenhclRecipeCli::X64CvmDevkern + | OpenhclRecipeCli::X64CvmNvidiaVpciRelay | OpenhclRecipeCli::X64TestLinuxDirect | OpenhclRecipeCli::X64TestLinuxDirectDevkern => CommonArch::X86_64, OpenhclRecipeCli::Aarch64 | OpenhclRecipeCli::Aarch64Devkern => CommonArch::Aarch64, @@ -438,6 +441,7 @@ impl IntoPipeline for BuildIgvmCli { } OpenhclRecipeCli::X64Cvm => OpenhclIgvmRecipe::X64Cvm, OpenhclRecipeCli::X64CvmDevkern => OpenhclIgvmRecipe::X64CvmDevkern, + OpenhclRecipeCli::X64CvmNvidiaVpciRelay => OpenhclIgvmRecipe::X64CvmNvidiaVpciRelay, OpenhclRecipeCli::Aarch64 => OpenhclIgvmRecipe::Aarch64, OpenhclRecipeCli::Aarch64Devkern => OpenhclIgvmRecipe::Aarch64Devkern, }, diff --git a/flowey/flowey_lib_hvlite/src/build_openhcl_igvm_from_recipe.rs b/flowey/flowey_lib_hvlite/src/build_openhcl_igvm_from_recipe.rs index 02276470bc..bcb9ce35b7 100644 --- a/flowey/flowey_lib_hvlite/src/build_openhcl_igvm_from_recipe.rs +++ b/flowey/flowey_lib_hvlite/src/build_openhcl_igvm_from_recipe.rs @@ -65,6 +65,12 @@ pub enum OpenhclIgvmOutput { #[serde(flatten)] endorsements: OpenhclIgvmEndorsements, }, + X64CvmNvidiaVpciRelay { + #[serde(rename = "openhcl-x64-cvm-nvidia-vpci-relay.bin")] + igvm_bin: PathBuf, + #[serde(flatten)] + endorsements: OpenhclIgvmEndorsements, + }, Aarch64 { #[serde(rename = "openhcl-aarch64.bin")] igvm_bin: PathBuf, @@ -115,6 +121,7 @@ impl OpenhclIgvmOutput { | OpenhclIgvmOutput::X64TestLinuxDirectDevkern { igvm_bin } | OpenhclIgvmOutput::X64Cvm { igvm_bin, .. } | OpenhclIgvmOutput::X64CvmDevkern { igvm_bin, .. } + | OpenhclIgvmOutput::X64CvmNvidiaVpciRelay { igvm_bin, .. } | OpenhclIgvmOutput::Aarch64 { igvm_bin } | OpenhclIgvmOutput::Aarch64Devkern { igvm_bin } => igvm_bin, } @@ -124,7 +131,8 @@ impl OpenhclIgvmOutput { match self { OpenhclIgvmOutput::LocalOnlyCustom { endorsements, .. } => endorsements.as_ref(), OpenhclIgvmOutput::X64Cvm { endorsements, .. } - | OpenhclIgvmOutput::X64CvmDevkern { endorsements, .. } => Some(endorsements), + | OpenhclIgvmOutput::X64CvmDevkern { endorsements, .. } + | OpenhclIgvmOutput::X64CvmNvidiaVpciRelay { endorsements, .. } => Some(endorsements), _ => None, } } @@ -142,6 +150,9 @@ impl OpenhclIgvmOutput { } OpenhclIgvmOutput::X64Cvm { .. } => Some(OpenhclIgvmRecipe::X64Cvm), OpenhclIgvmOutput::X64CvmDevkern { .. } => Some(OpenhclIgvmRecipe::X64CvmDevkern), + OpenhclIgvmOutput::X64CvmNvidiaVpciRelay { .. } => { + Some(OpenhclIgvmRecipe::X64CvmNvidiaVpciRelay) + } OpenhclIgvmOutput::Aarch64 { .. } => Some(OpenhclIgvmRecipe::Aarch64), OpenhclIgvmOutput::Aarch64Devkern { .. } => Some(OpenhclIgvmRecipe::Aarch64Devkern), } @@ -194,6 +205,15 @@ impl OpenhclIgvmOutput { .filter(OpenhclIgvmEndorsements::is_complete) .expect("missing endorsements"), }, + OpenhclIgvmRecipe::X64CvmNvidiaVpciRelay => { + OpenhclIgvmOutput::X64CvmNvidiaVpciRelay { + igvm_bin, + endorsements: endorsements + .take() + .filter(OpenhclIgvmEndorsements::is_complete) + .expect("missing endorsements"), + } + } OpenhclIgvmRecipe::Aarch64 => OpenhclIgvmOutput::Aarch64 { igvm_bin }, OpenhclIgvmRecipe::Aarch64Devkern => { OpenhclIgvmOutput::Aarch64Devkern { igvm_bin } @@ -291,6 +311,7 @@ pub enum OpenhclIgvmRecipe { X64TestLinuxDirectDevkern, X64Cvm, X64CvmDevkern, + X64CvmNvidiaVpciRelay, Aarch64, Aarch64Devkern, } @@ -334,6 +355,7 @@ impl OpenhclIgvmRecipe { OpenhclIgvmRecipe::X64TestLinuxDirectDevkern => Some("test-linux-direct-devkern"), OpenhclIgvmRecipe::X64Cvm => Some("cvm"), OpenhclIgvmRecipe::X64CvmDevkern => Some("cvm-devkern"), + OpenhclIgvmRecipe::X64CvmNvidiaVpciRelay => Some("cvm-nvidia-vpci-relay"), } } @@ -344,7 +366,8 @@ impl OpenhclIgvmRecipe { | OpenhclIgvmRecipe::X64TestLinuxDirect | OpenhclIgvmRecipe::X64TestLinuxDirectDevkern | OpenhclIgvmRecipe::X64Cvm - | OpenhclIgvmRecipe::X64CvmDevkern => "x64", + | OpenhclIgvmRecipe::X64CvmDevkern + | OpenhclIgvmRecipe::X64CvmNvidiaVpciRelay => "x64", OpenhclIgvmRecipe::Aarch64 | OpenhclIgvmRecipe::Aarch64Devkern => "aarch64", } } @@ -484,6 +507,25 @@ impl OpenhclIgvmRecipe { with_sidecar: false, max_trace_level, }, + Self::X64CvmNvidiaVpciRelay => OpenhclIgvmRecipeDetails { + local_only: None, + igvm_manifest: in_repo_template( + "openhcl-x64-cvm-dev.json", + "openhcl-x64-cvm-release.json", + ), + openhcl_kernel_package: OpenhclKernelPackage::Cvm, + openvmm_hcl_features: { + let mut f = base_openvmm_hcl_features(); + f.insert(OpenvmmHclFeature::NvidiaVpciRelayAllowed); + f + }, + target: CommonTriple::X86_64_LINUX_MUSL, + vtl0_kernel_type: None, + with_uefi: true, + with_interactive, + with_sidecar: false, + max_trace_level, + }, Self::Aarch64 => OpenhclIgvmRecipeDetails { local_only: None, igvm_manifest: in_repo_template( diff --git a/flowey/flowey_lib_hvlite/src/build_openvmm_hcl.rs b/flowey/flowey_lib_hvlite/src/build_openvmm_hcl.rs index 756bfac17c..5f5e078ca9 100644 --- a/flowey/flowey_lib_hvlite/src/build_openvmm_hcl.rs +++ b/flowey/flowey_lib_hvlite/src/build_openvmm_hcl.rs @@ -15,6 +15,7 @@ pub enum OpenvmmHclFeature { Gdb, MiSecure, Tpm, + NvidiaVpciRelayAllowed, LocalOnlyCustom(String), } @@ -135,6 +136,7 @@ impl FlowNode for Node { OpenvmmHclFeature::Gdb => "gdb".into(), OpenvmmHclFeature::MiSecure => "mi-secure".into(), OpenvmmHclFeature::Tpm => "tpm".into(), + OpenvmmHclFeature::NvidiaVpciRelayAllowed => "nvidia_vpci_relay_allowed".into(), OpenvmmHclFeature::LocalOnlyCustom(s) => s, }) .collect::>(); diff --git a/openhcl/openvmm_hcl/Cargo.toml b/openhcl/openvmm_hcl/Cargo.toml index 751ebc6c27..16193549d1 100644 --- a/openhcl/openvmm_hcl/Cargo.toml +++ b/openhcl/openvmm_hcl/Cargo.toml @@ -27,6 +27,9 @@ uidevices = ["openvmm_hcl_resources/uidevices", "openvmm_hcl_resources/vnc_worke nvme = ["openvmm_hcl_resources/nvme", "vpci"] vpci = ["underhill_entry/vpci"] +# Allow NVIDIA GPUs and their NVLink/NVSwitch fabric devices through the VPCI relay. +nvidia_vpci_relay_allowed = ["underhill_entry/nvidia_vpci_relay_allowed"] + # Enable TIO support for SEV-SNP dev_snp_ohcl_tio_support = ["underhill_entry/dev_snp_ohcl_tio_support"] diff --git a/openhcl/underhill_core/Cargo.toml b/openhcl/underhill_core/Cargo.toml index d198be0232..ee403a33ce 100644 --- a/openhcl/underhill_core/Cargo.toml +++ b/openhcl/underhill_core/Cargo.toml @@ -27,6 +27,9 @@ dev_snp_ohcl_tio_support = ["tee_call/dev_snp_ohcl_tio_support"] # Enable VPCI device support vpci = [] +# Allow NVIDIA GPUs and their NVLink/NVSwitch fabric devices through the VPCI relay. +nvidia_vpci_relay_allowed = [] + [target.'cfg(target_os = "linux")'.dependencies] vmotherboard = { workspace = true, features = [ "dev_underhill_vga_proxy", diff --git a/openhcl/underhill_core/src/worker.rs b/openhcl/underhill_core/src/worker.rs index 19ad0ee0f0..183aed3123 100644 --- a/openhcl/underhill_core/src/worker.rs +++ b/openhcl/underhill_core/src/worker.rs @@ -197,6 +197,11 @@ pub const UNDERHILL_WORKER: WorkerId = WorkerId::new( const MAX_SUBCHANNELS_PER_VNIC: u16 = 32; +// NVIDIA PCI vendor ID (used to relay NVIDIA GPUs and NVLink/NVSwitch fabric +// devices to Azure Local confidential guests). +#[cfg(feature = "nvidia_vpci_relay_allowed")] +const NVIDIA_VPCI_VENDOR_ID: u16 = 0x10DE; + struct GuestEmulationTransportInfra { get_thread: JoinHandle<()>, get_spawner: DefaultDriver, @@ -3399,6 +3404,36 @@ async fn new_underhill_vm( sub_system_id: None, }); + #[cfg(feature = "nvidia_vpci_relay_allowed")] + { + // Datacenter GPUs (e.g. H100/H200/B200/B300) are headless and + // enumerate as a 3D controller (class 0x0302), not VGA. + relay.add_allowed_device(AllowedDevice { + vendor_id: Some(NVIDIA_VPCI_VENDOR_ID), + device_id: None, + revision_id: None, + prog_if: None, + sub_class: Some(Subclass::DISPLAY_CONTROLLER_3D), + base_class: Some(ClassCode::DISPLAY_CONTROLLER), + sub_vendor_id: None, + sub_system_id: None, + }); + + // HGX clusters additionally expose NVSwitch NVLink-fabric + // devices, which enumerate as an "other" PCI bridge (class + // 0x0680) rather than a display controller. + relay.add_allowed_device(AllowedDevice { + vendor_id: Some(NVIDIA_VPCI_VENDOR_ID), + device_id: None, + revision_id: None, + prog_if: None, + sub_class: Some(Subclass::BRIDGE_OTHER), + base_class: Some(ClassCode::BRIDGE), + sub_vendor_id: None, + sub_system_id: None, + }); + } + vpci_relay = Some(relay); } diff --git a/openhcl/underhill_entry/Cargo.toml b/openhcl/underhill_entry/Cargo.toml index 27dac3d6f1..024d870304 100644 --- a/openhcl/underhill_entry/Cargo.toml +++ b/openhcl/underhill_entry/Cargo.toml @@ -25,6 +25,9 @@ disable_secure_avic = ["underhill_core/disable_secure_avic"] # Enable vpci support. vpci = ["underhill_core/vpci"] +# Allow NVIDIA GPUs and their NVLink/NVSwitch fabric devices through the VPCI relay. +nvidia_vpci_relay_allowed = ["underhill_core/nvidia_vpci_relay_allowed"] + # Enable TIO support for SEV-SNP dev_snp_ohcl_tio_support = ["underhill_core/dev_snp_ohcl_tio_support"] diff --git a/vm/devices/pci/pci_core/src/spec.rs b/vm/devices/pci/pci_core/src/spec.rs index cc590bc78a..ae27ffae15 100644 --- a/vm/devices/pci/pci_core/src/spec.rs +++ b/vm/devices/pci/pci_core/src/spec.rs @@ -124,6 +124,10 @@ pub mod hwid { // Other values: 0x01 - 0x08, 0x80 NETWORK_CONTROLLER_ETHERNET = 0x00, + // Display Controller (Class code: 0x03) + // Other values: 0x00 (VGA), 0x01 (XGA), 0x80 (Other) + DISPLAY_CONTROLLER_3D = 0x02, + // Simple Communication Controller (Class code: 0x07) // Other values: 0x00 - 0x07 SIMPLE_COMMUNICATION_CONTROLLER_OTHER = 0x80,