From 0d51b7b357cec267e82fec8fd1a6f5819a7b8f3b Mon Sep 17 00:00:00 2001 From: wukent Date: Sat, 13 Jul 2019 19:35:57 +0000 Subject: [PATCH] openstack-crowbar: Add pci passthough option Add Jenkins parameter 'want_pci_passthrough' to allow running tests to validate pci passthough. To test pci passthrough locally with mkcloud, simply just export 'want_pci_passthrouh=1' and append 'rebootcloud testpcipassthru' to 'plain' step, e.g. your_mkcloud.sh plain rebootcloud testpcipassthru The rebootcloud step will ensure intel_iommu=on kernel param enabled. The testpcipassthru step will prepare the environment and spin up a nested vm (L1) on compute (L0) with a virtio disk passing from compute vm (L0). Noted that this is not passing any device from the host itself. 1. add environment prep and verify function in qa_crowbarsetup.sh 2. create ci job template --- ...cloud-mkcloud-job-pcipassthru-template.yml | 29 +++ .../jenkins/cloud/ansible/deploy-crowbar.yml | 7 + scripts/mkcloud | 12 +- scripts/qa_crowbarsetup.sh | 204 ++++++++++++++++++ 4 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml diff --git a/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml b/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml new file mode 100644 index 0000000000..5a0ba0d61a --- /dev/null +++ b/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml @@ -0,0 +1,29 @@ +- job-template: + name: 'cloud-mkcloud{version}-job-pcipassthru-{arch}' + node: cloud-trigger + disabled: '{obj:disabled}' + + triggers: + - timed: 'H 20 * * *' + + logrotate: + numToKeep: -1 + daysToKeep: 7 + + builders: + - trigger-builds: + - project: openstack-mkcloud + condition: SUCCESS + block: true + current-parameters: true + predefined-parameters: | + TESTHEAD=1 + cloudsource=develcloud{version} + libvirt_type=kvm + nodenumber=2 + mkcloudtarget=all + want_pci_passthrough=1 + vcpus=4 + label={label} + job_name=cloud-mkcloud{version}-job-pcipassthru-{arch} + diff --git a/scripts/jenkins/cloud/ansible/deploy-crowbar.yml b/scripts/jenkins/cloud/ansible/deploy-crowbar.yml index c7928fa2c9..260946e523 100644 --- a/scripts/jenkins/cloud/ansible/deploy-crowbar.yml +++ b/scripts/jenkins/cloud/ansible/deploy-crowbar.yml @@ -47,6 +47,13 @@ - reboot_after_deploy - not update_after_deploy + - include_role: + name: crowbar_setup + vars: + qa_crowbarsetup_cmd: onadmin_verify_pci_passthru + when: + - want_pci_passthrough|int == 1 + rescue: - include_role: name: rocketchat_notify diff --git a/scripts/mkcloud b/scripts/mkcloud index f1e009e5a9..cb4e04058a 100755 --- a/scripts/mkcloud +++ b/scripts/mkcloud @@ -1029,6 +1029,14 @@ function rebootcrowbar return $? } +# Need to run rebootcloud step to enable iommu (e.g., plain rebootcloud +# testpcipassthru) +function testpcipassthru +{ + onadmin verify_pci_passthru + return $? +} + function rebootcloud { # reboot compute nodes @@ -1455,6 +1463,8 @@ Optional If set, does not use the --insecure flag in openstack CLI commands. want_monasca_tsdb (Cloud9+ only) Allows setting time-series DB used for Monasca [influxdb|cassandra]. + want_pci_passthrough (default=0) + Deploy node with an extra disk to test pci passthrough. Cloud9+ only. want_ipv6 (Currently in development) Prepare crowbar to use a full IPv6 single stack control plane. Enabling will also set the firmware_type to UEFI as it's required to netboot nodes over IPv6. @@ -1619,7 +1629,7 @@ allcmds="$step_aliases _test_setuphost \ lonelynode_nfs_server setupironicnodes\ restoreadminfromsnapshot createcloudsnapshot restorecloudfromsnapshot \ cct steps batch setup_aliases onadmin onhost devsetup plain_with_upgrade_test \ - testpreupgrade testpostupgrade deployexternalceph" + testpreupgrade testpostupgrade deployexternalceph testpcipassthru" wantedcmds=$@ function expand_steps diff --git a/scripts/qa_crowbarsetup.sh b/scripts/qa_crowbarsetup.sh index a8d0bad121..1469be10fe 100644 --- a/scripts/qa_crowbarsetup.sh +++ b/scripts/qa_crowbarsetup.sh @@ -57,10 +57,12 @@ fi # global variables that are set within this script novacontroller= +novacompute_kvm= horizonserver= horizonservice= manila_service_vm_uuid= manila_tenant_vm_ip= +pci_passthru_vm_name="pci-passthru-instance" clusternodesdrbd= clusternodesdata= clusternodesnetwork= @@ -3315,6 +3317,37 @@ function onadmin_proposal done set_dashboard_alias + + prepare_pci_passthru_env +} + +function prepare_pci_passthru_env() +{ + if iscloudver 9plus && [[ $want_pci_passthrough = 1 ]] ; then + # Place nova configuration to verify pci_passthru + get_novacontroller + safely oncontroller create_pci_passthru_conf + append_pci_passthru_filter + get_novacompute_kvm + safely oncompute create_pci_passthru_conf + safely oncompute load_vfio_module_onboot + safely oncompute bind_vfio_module_onboot + # TODO: need support for amd param + safely oncompute add_intel_iommu_param + fi +} + +function onadmin_verify_pci_passthru() +{ + # spin up vm and pass device from compute host to vm to verify pci + # passthru functionality. + get_novacontroller + safely oncontroller setup_pci_passthru_vm + # Get instance name (e.g.,instance-00000001) + pci_passthru_instance_name=$(ssh "$novacontroller" "source /root/.openrc; + openstack server show -c OS-EXT-SRV-ATTR:instance_name -f value $pci_passthru_vm_name") + get_novacompute_kvm + safely oncompute parse_vm_xml "$pci_passthru_instance_name" } function set_node_alias @@ -3417,6 +3450,15 @@ function get_novacontroller novacontroller=`resolve_element_to_hostname "$element"` } +function get_novacompute_kvm +{ + local element=`crowbar nova proposal show default | \ + rubyjsonparse " + puts j['deployment']['nova']\ + ['elements']['nova-compute-kvm']"` + novacompute_kvm=`resolve_element_to_hostname "$element"` +} + function get_horizon { local element=`crowbar horizon proposal show default | \ @@ -3860,6 +3902,81 @@ function oncontroller_heat_image_setup() fi } +function oncompute_parse_vm_xml() +{ + local instance_name=$1 + echo $instance_name + virsh list --all | grep $instance_name + [ $? != 0 ] && complain 53 "failed to find pci-passthru-instance" + + # verify node xml to see if it has pci passthru section which has the + # specific address passing from compute host + virsh dumpxml $instance_name | grep -E "&1 >/dev/null) + if [[ $ret =~ "200 OK" ]]; then + echo $ret + elif [[ $ret =~ "Not Found" ]]; then + complain 73 "cirros image not found: $ret" + else + complain 74 "failed to retrieve cirros image: $ret" + fi + fi + + . .openrc + + # using list subcommand because show requires an ID + if ! openstack image list --format value -c Name | grep -q "^cirros-0.4.0-x86_64-disk$"; then + openstack image create --file $cirros_image_name \ + $cirros_image_params --container-format bare --public \ + cirros-0.4.0-x86_64-disk + fi + + # create flavor with pci-passthru property so that the vm using this flavor + # will aquire the pci device passing from the host (compute) + openstack flavor create --id 200 --ram 512 --ephemeral 0 \ + --vcpus 1 --property "pci_passthrough:alias"="a1:1" \ + pci-passthru-flavor + + if iscloudver 9plus && ! openstack security group show $sec_group 2>/dev/null ; then + openstack security group create --description "$sec_group description" $sec_group + openstack security group rule create --protocol icmp $sec_group + openstack security group rule create --protocol tcp --dst-port 22 $sec_group + fi + + fixed_net_id=`neutron net-show fixed -f value -c id` + timeout 10m openstack server create --flavor 200 \ + --image cirros-0.4.0-x86_64-disk \ + --security-group $sec_group \ + --nic net-id=$fixed_net_id $pci_passthru_vm_name + + [ $? != 0 ] && complain 43 "nova boot pci-passthru-instance failed" + + # Check status of the vm + wait_for 60 1 "openstack server show -c status -f value pci-passthru-instance | grep '^ACTIVE$'" \ + "pci passthru VM booted and is in ACTIVE state" \ + "echo \"ERROR: pci passthru VM is not in ACTIVE state.\"" +} + function oncontroller_manila_generic_driver_setup() { if [[ $wantxenpv ]] ; then @@ -4478,6 +4595,93 @@ function oncontroller run_on "$novacontroller" "oncontroller_$func $@" } +function oncompute +{ + local func=$1 ; shift + run_on "$novacompute_kvm" "oncompute_$func $@" +} + +function oncompute_add_intel_iommu_param +{ + # enabled intel_iommu in kernel param + sed -in "s/\(^GRUB_CMDLINE_LINUX_DEFAULT=.*\)\"$/\1 intel_iommu=on\"/g" /etc/default/grub + update-bootloader +} + +function oncompute_load_vfio_module_onboot +{ + # vfio-pci module is not loaded by default, however, we need it to bind to + # our candidate device to verify pci passthru + cat > /etc/modules-load.d/pci-passthru.conf < /etc/udev/rules.d/99-pci-passthru.rules < /etc/nova/nova.conf.d/200-nova-pci-passthru.conf < /etc/nova/nova.conf.d/200-nova-pci-passthru.conf <