openhcl: add build that relays NVIDIA GPUs to CVM guests

flowey/flowey_hvlite/src/pipelines/build_igvm.rs

@@ -22,6 +22,8 @@ pub enum OpenhclRecipeCli {
     X64Cvm,
     /// X64 OpenHCL, with CVM support using the dev kernel in VTL2
     X64CvmDevkern,
+    /// X64 OpenHCL, with CVM support, plus NVIDIA GPU VPCI relay support.
+    X64CvmNvidiaVpciRelay,
     /// X64 OpenHCL booting VTL0 using a test linux-direct kernel + initrd (no
     /// UEFI).
     X64TestLinuxDirect,
@@ -348,6 +350,7 @@ impl IntoPipeline for BuildIgvmCli {
             | OpenhclRecipeCli::X64Devkern
             | OpenhclRecipeCli::X64Cvm
             | OpenhclRecipeCli::X64CvmDevkern
+            | OpenhclRecipeCli::X64CvmNvidiaVpciRelay
             | OpenhclRecipeCli::X64TestLinuxDirect
             | OpenhclRecipeCli::X64TestLinuxDirectDevkern => CommonArch::X86_64,
             OpenhclRecipeCli::Aarch64 | OpenhclRecipeCli::Aarch64Devkern => CommonArch::Aarch64,
@@ -438,6 +441,7 @@ impl IntoPipeline for BuildIgvmCli {
                 }
                 OpenhclRecipeCli::X64Cvm => OpenhclIgvmRecipe::X64Cvm,
                 OpenhclRecipeCli::X64CvmDevkern => OpenhclIgvmRecipe::X64CvmDevkern,
+                OpenhclRecipeCli::X64CvmNvidiaVpciRelay => OpenhclIgvmRecipe::X64CvmNvidiaVpciRelay,
                 OpenhclRecipeCli::Aarch64 => OpenhclIgvmRecipe::Aarch64,
                 OpenhclRecipeCli::Aarch64Devkern => OpenhclIgvmRecipe::Aarch64Devkern,
             },

flowey/flowey_lib_hvlite/src/build_openhcl_igvm_from_recipe.rs

@@ -65,6 +65,12 @@ pub enum OpenhclIgvmOutput {
         #[serde(flatten)]
         endorsements: OpenhclIgvmEndorsements,
     },
+    X64CvmNvidiaVpciRelay {
+        #[serde(rename = "openhcl-x64-cvm-nvidia-vpci-relay.bin")]
+        igvm_bin: PathBuf,
+        #[serde(flatten)]
+        endorsements: OpenhclIgvmEndorsements,
+    },
     Aarch64 {
         #[serde(rename = "openhcl-aarch64.bin")]
         igvm_bin: PathBuf,
@@ -115,6 +121,7 @@ impl OpenhclIgvmOutput {
             | OpenhclIgvmOutput::X64TestLinuxDirectDevkern { igvm_bin }
             | OpenhclIgvmOutput::X64Cvm { igvm_bin, .. }
             | OpenhclIgvmOutput::X64CvmDevkern { igvm_bin, .. }
+            | OpenhclIgvmOutput::X64CvmNvidiaVpciRelay { igvm_bin, .. }
             | OpenhclIgvmOutput::Aarch64 { igvm_bin }
             | OpenhclIgvmOutput::Aarch64Devkern { igvm_bin } => igvm_bin,
         }
@@ -124,7 +131,8 @@ impl OpenhclIgvmOutput {
         match self {
             OpenhclIgvmOutput::LocalOnlyCustom { endorsements, .. } => endorsements.as_ref(),
             OpenhclIgvmOutput::X64Cvm { endorsements, .. }
-            | OpenhclIgvmOutput::X64CvmDevkern { endorsements, .. } => Some(endorsements),
+            | OpenhclIgvmOutput::X64CvmDevkern { endorsements, .. }
+            | OpenhclIgvmOutput::X64CvmNvidiaVpciRelay { endorsements, .. } => Some(endorsements),
             _ => None,
         }
     }
@@ -142,6 +150,9 @@ impl OpenhclIgvmOutput {
             }
             OpenhclIgvmOutput::X64Cvm { .. } => Some(OpenhclIgvmRecipe::X64Cvm),
             OpenhclIgvmOutput::X64CvmDevkern { .. } => Some(OpenhclIgvmRecipe::X64CvmDevkern),
+            OpenhclIgvmOutput::X64CvmNvidiaVpciRelay { .. } => {
+                Some(OpenhclIgvmRecipe::X64CvmNvidiaVpciRelay)
+            }
             OpenhclIgvmOutput::Aarch64 { .. } => Some(OpenhclIgvmRecipe::Aarch64),
             OpenhclIgvmOutput::Aarch64Devkern { .. } => Some(OpenhclIgvmRecipe::Aarch64Devkern),
         }
@@ -194,6 +205,15 @@ impl OpenhclIgvmOutput {
                             .filter(OpenhclIgvmEndorsements::is_complete)
                             .expect("missing endorsements"),
                     },
+                    OpenhclIgvmRecipe::X64CvmNvidiaVpciRelay => {
+                        OpenhclIgvmOutput::X64CvmNvidiaVpciRelay {
+                            igvm_bin,
+                            endorsements: endorsements
+                                .take()
+                                .filter(OpenhclIgvmEndorsements::is_complete)
+                                .expect("missing endorsements"),
+                        }
+                    }
                     OpenhclIgvmRecipe::Aarch64 => OpenhclIgvmOutput::Aarch64 { igvm_bin },
                     OpenhclIgvmRecipe::Aarch64Devkern => {
                         OpenhclIgvmOutput::Aarch64Devkern { igvm_bin }
@@ -291,6 +311,7 @@ pub enum OpenhclIgvmRecipe {
     X64TestLinuxDirectDevkern,
     X64Cvm,
     X64CvmDevkern,
+    X64CvmNvidiaVpciRelay,
     Aarch64,
     Aarch64Devkern,
 }
@@ -334,6 +355,7 @@ impl OpenhclIgvmRecipe {
             OpenhclIgvmRecipe::X64TestLinuxDirectDevkern => Some("test-linux-direct-devkern"),
             OpenhclIgvmRecipe::X64Cvm => Some("cvm"),
             OpenhclIgvmRecipe::X64CvmDevkern => Some("cvm-devkern"),
+            OpenhclIgvmRecipe::X64CvmNvidiaVpciRelay => Some("cvm-nvidia-vpci-relay"),
         }
     }
 
@@ -344,7 +366,8 @@ impl OpenhclIgvmRecipe {
             | OpenhclIgvmRecipe::X64TestLinuxDirect
             | OpenhclIgvmRecipe::X64TestLinuxDirectDevkern
             | OpenhclIgvmRecipe::X64Cvm
-            | OpenhclIgvmRecipe::X64CvmDevkern => "x64",
+            | OpenhclIgvmRecipe::X64CvmDevkern
+            | OpenhclIgvmRecipe::X64CvmNvidiaVpciRelay => "x64",
             OpenhclIgvmRecipe::Aarch64 | OpenhclIgvmRecipe::Aarch64Devkern => "aarch64",
         }
     }
@@ -484,6 +507,25 @@ impl OpenhclIgvmRecipe {
                 with_sidecar: false,
                 max_trace_level,
             },
+            Self::X64CvmNvidiaVpciRelay => OpenhclIgvmRecipeDetails {
+                local_only: None,
+                igvm_manifest: in_repo_template(
+                    "openhcl-x64-cvm-dev.json",
+                    "openhcl-x64-cvm-release.json",
+                ),
+                openhcl_kernel_package: OpenhclKernelPackage::Cvm,
+                openvmm_hcl_features: {
+                    let mut f = base_openvmm_hcl_features();
+                    f.insert(OpenvmmHclFeature::NvidiaVpciRelayAllowed);
+                    f
+                },
+                target: CommonTriple::X86_64_LINUX_MUSL,
+                vtl0_kernel_type: None,
+                with_uefi: true,
+                with_interactive,
+                with_sidecar: false,
+                max_trace_level,
+            },
             Self::Aarch64 => OpenhclIgvmRecipeDetails {
                 local_only: None,
                 igvm_manifest: in_repo_template(

flowey/flowey_lib_hvlite/src/build_openvmm_hcl.rs

@@ -15,6 +15,7 @@ pub enum OpenvmmHclFeature {
     Gdb,
     MiSecure,
     Tpm,
+    NvidiaVpciRelayAllowed,
     LocalOnlyCustom(String),
 }
 
@@ -135,6 +136,7 @@ impl FlowNode for Node {
                     OpenvmmHclFeature::Gdb => "gdb".into(),
                     OpenvmmHclFeature::MiSecure => "mi-secure".into(),
                     OpenvmmHclFeature::Tpm => "tpm".into(),
+                    OpenvmmHclFeature::NvidiaVpciRelayAllowed => "nvidia_vpci_relay_allowed".into(),
                     OpenvmmHclFeature::LocalOnlyCustom(s) => s,
                 })
                 .collect::<Vec<String>>();

openhcl/openvmm_hcl/Cargo.toml

@@ -27,6 +27,9 @@ uidevices = ["openvmm_hcl_resources/uidevices", "openvmm_hcl_resources/vnc_worke
 nvme = ["openvmm_hcl_resources/nvme", "vpci"]
 vpci = ["underhill_entry/vpci"]
 
+# Allow NVIDIA GPUs and their NVLink/NVSwitch fabric devices through the VPCI relay.
+nvidia_vpci_relay_allowed = ["underhill_entry/nvidia_vpci_relay_allowed"]
+
 # Enable TIO support for SEV-SNP
 dev_snp_ohcl_tio_support = ["underhill_entry/dev_snp_ohcl_tio_support"]
 

openhcl/underhill_core/Cargo.toml

@@ -27,6 +27,9 @@ dev_snp_ohcl_tio_support = ["tee_call/dev_snp_ohcl_tio_support"]
 # Enable VPCI device support
 vpci = []
 
+# Allow NVIDIA GPUs and their NVLink/NVSwitch fabric devices through the VPCI relay.
+nvidia_vpci_relay_allowed = []
+
 [target.'cfg(target_os = "linux")'.dependencies]
 vmotherboard = { workspace = true, features = [
     "dev_underhill_vga_proxy",

openhcl/underhill_core/src/worker.rs

@@ -197,6 +197,11 @@ pub const UNDERHILL_WORKER: WorkerId<UnderhillWorkerParameters> = WorkerId::new(
 
 const MAX_SUBCHANNELS_PER_VNIC: u16 = 32;
 
+// NVIDIA PCI vendor ID (used to relay NVIDIA GPUs and NVLink/NVSwitch fabric
+// devices to Azure Local confidential guests).
+#[cfg(feature = "nvidia_vpci_relay_allowed")]
+const NVIDIA_VPCI_VENDOR_ID: u16 = 0x10DE;
+
 struct GuestEmulationTransportInfra {
     get_thread: JoinHandle<()>,
     get_spawner: DefaultDriver,
@@ -3399,6 +3404,36 @@ async fn new_underhill_vm(
                     sub_system_id: None,
                 });
 
+                #[cfg(feature = "nvidia_vpci_relay_allowed")]
+                {
+                    // Datacenter GPUs (e.g. H100/H200/B200/B300) are headless and
+                    // enumerate as a 3D controller (class 0x0302), not VGA.
+                    relay.add_allowed_device(AllowedDevice {
+                        vendor_id: Some(NVIDIA_VPCI_VENDOR_ID),
+                        device_id: None,
+                        revision_id: None,
+                        prog_if: None,
+                        sub_class: Some(Subclass::DISPLAY_CONTROLLER_3D),
+                        base_class: Some(ClassCode::DISPLAY_CONTROLLER),
+                        sub_vendor_id: None,
+                        sub_system_id: None,
+                    });
+
+                    // HGX clusters additionally expose NVSwitch NVLink-fabric
+                    // devices, which enumerate as an "other" PCI bridge (class
+                    // 0x0680) rather than a display controller.
+                    relay.add_allowed_device(AllowedDevice {
+                        vendor_id: Some(NVIDIA_VPCI_VENDOR_ID),
+                        device_id: None,
+                        revision_id: None,
+                        prog_if: None,
+                        sub_class: Some(Subclass::BRIDGE_OTHER),
+                        base_class: Some(ClassCode::BRIDGE),
+                        sub_vendor_id: None,
+                        sub_system_id: None,
+                    });
+                }
+
                 vpci_relay = Some(relay);
             }
 

openhcl/underhill_entry/Cargo.toml

@@ -25,6 +25,9 @@ disable_secure_avic = ["underhill_core/disable_secure_avic"]
 # Enable vpci support.
 vpci = ["underhill_core/vpci"]
 
+# Allow NVIDIA GPUs and their NVLink/NVSwitch fabric devices through the VPCI relay.
+nvidia_vpci_relay_allowed = ["underhill_core/nvidia_vpci_relay_allowed"]
+
 # Enable TIO support for SEV-SNP
 dev_snp_ohcl_tio_support = ["underhill_core/dev_snp_ohcl_tio_support"]
 

vm/devices/pci/pci_core/src/spec.rs

@@ -124,6 +124,10 @@ pub mod hwid {
             // Other values: 0x01 - 0x08, 0x80
             NETWORK_CONTROLLER_ETHERNET = 0x00,
 
+            // Display Controller (Class code: 0x03)
+            // Other values: 0x00 (VGA), 0x01 (XGA), 0x80 (Other)
+            DISPLAY_CONTROLLER_3D = 0x02,
+
             // Simple Communication Controller (Class code: 0x07)
             // Other values: 0x00 - 0x07
             SIMPLE_COMMUNICATION_CONTROLLER_OTHER = 0x80,