diff --git a/ods_ci/tasks/Resources/Provisioning/GPU/AMD/amd_operator.sh b/ods_ci/tasks/Resources/Provisioning/GPU/AMD/amd_operator.sh index 4ff4b4d88..1f5c2e95c 100755 --- a/ods_ci/tasks/Resources/Provisioning/GPU/AMD/amd_operator.sh +++ b/ods_ci/tasks/Resources/Provisioning/GPU/AMD/amd_operator.sh @@ -135,6 +135,13 @@ function wait_until_driver_image_is_built() { } function create_acceleratorprofile() { + echo "Creating AMD Accelerator Profile" + rhoai_ns=$(oc get namespace redhat-ods-applications --ignore-not-found -oname) + if [ -z $rhoai_ns ]; + then + echo "redhat-ods-applications namespace not found. Is RHOAI Installed? NVIDIA Accelerator Profile creation SKIPPED." + return 0 + fi echo "Creating an Accelerator Profile for Dashboard" oc apply -f - < "Log Out" to actually close the login session @@ -362,7 +371,7 @@ Verify User Can Log Out And Return To Project From Jupyter Notebook # robocop ... project_title=${PRJ_TITLE} Verify Event Log Is Accessible While Starting A Workbench - [Tags] Tier1 Sanity + [Tags] Sanity ... ODS-1970 [Documentation] Verify user can access event log while starting a workbench Create Workbench workbench_title=${WORKBENCH_6_TITLE} workbench_description=${WORKBENCH_6_DESCRIPTION} @@ -385,7 +394,8 @@ Verify Event Log Is Accessible While Starting A Workbench ... project_title=${PRJ_TITLE} Verify User Can Cancel Workbench Start From Event Log - [Tags] Sanity Tier1 ODS-1975 + [Tags] Sanity + ... ODS-1975 [Documentation] Verify user can cancel workbench start from event log Create Workbench workbench_title=${WORKBENCH_TITLE} workbench_description=${WORKBENCH_DESCRIPTION} @@ -410,7 +420,7 @@ Verify User Can Cancel Workbench Start From Event Log [Teardown] Clean Project From Workbench Resources workbench_title=${WORKBENCH_TITLE} project_title=${PRJ_TITLE} Verify Error Is Reported When Workbench Fails To Start # robocop: disable - [Tags] Tier1 Sanity + [Tags] Sanity ... ODS-1973 [Documentation] Verify UI informs users about workbenches failed to start. ... At the moment the test is considering only the scenario where @@ -432,7 +442,7 @@ Verify Error Is Reported When Workbench Fails To Start # robocop: disable Wait Until Project Is Open project_title=${PRJ_TITLE} Verify Users Can Start, Stop, Launch And Delete A Workbench - [Tags] Smoke Tier1 + [Tags] Smoke ... ODS-1813 ODS-1815 ODS-1817 [Documentation] Verifies users can start, stop, launch and delete a running workbench from project details page [Setup] Run Keywords @@ -453,7 +463,8 @@ Verify Users Can Start, Stop, Launch And Delete A Workbench Launch And Access Workbench workbench_title=${WORKBENCH_TITLE} ... username=${TEST_USER_3.USERNAME} password=${TEST_USER_3.PASSWORD} ... auth_type=${TEST_USER_3.AUTH_TYPE} - Check Launched Workbench Is The Correct One workbench_title=${WORKBENCH_TITLE} + # TODO: Following Keyword should rather be called within "JupyterHub" Suite + Verify Workbench Image In Jupyter Lab workbench_title=${WORKBENCH_TITLE} ... image=${NB_IMAGE} project_title=${PRJ_TITLE} SeleniumLibrary.Switch Window title=${ODH_DASHBOARD_PROJECT_NAME} Wait Until Project Is Open project_title=${PRJ_TITLE} @@ -467,9 +478,8 @@ Verify Users Can Start, Stop, Launch And Delete A Workbench ... SeleniumLibrary.Close Browser Verify Users Can Start, Stop And Launch A Workbench From DS Projects Home Page - [Tags] Smoke Sanity Tier1 + [Tags] Sanity ... ODS-1818 ODS-1823 - ... ProductBug RHOAIENG-5084 [Documentation] Verifies users can start, stop, launch and delete a running workbench from project details page [Setup] Run Keywords ... Launch Data Science Project Main Page @@ -481,20 +491,16 @@ Verify Users Can Start, Stop And Launch A Workbench From DS Projects Home Page ... press_cancel=${FALSE} envs=${NONE} ... AND ... Wait Until Workbench Is Started workbench_title=${WORKBENCH_TITLE} - # ${ns_name}= Get Openshift Namespace From Data Science Project project_title=${PRJ_TITLE} - # ${_} ${workbench_cr_name}= Get Openshift Notebook CR From Workbench workbench_title=${WORKBENCH_TITLE} - # ... namespace=${ns_name} Open Data Science Projects Home Page Wait Until Project Is Listed project_title=${PRJ_TITLE} Stop Workbench From Projects Home Page workbench_title=${WORKBENCH_TITLE} project_title=${PRJ_TITLE} - # ... workbench_cr_name=${workbench_cr_name} namespace=${ns_name} Workbench Launch Link Should Be Disabled workbench_title=${WORKBENCH_TITLE} project_title=${PRJ_TITLE} Start Workbench From Projects Home Page workbench_title=${WORKBENCH_TITLE} project_title=${PRJ_TITLE} - # ... workbench_cr_name=${workbench_cr_name} namespace=${ns_name} Launch And Access Workbench From Projects Home Page workbench_title=${WORKBENCH_TITLE} ... project_title=${PRJ_TITLE} username=${TEST_USER_3.USERNAME} ... password=${TEST_USER_3.PASSWORD} auth_type=${TEST_USER_3.AUTH_TYPE} - Check Launched Workbench Is The Correct One workbench_title=${WORKBENCH_TITLE} + # TODO: Following Keyword should rather be called within "JupyterHub" Suite + Verify Workbench Image In Jupyter Lab workbench_title=${WORKBENCH_TITLE} ... image=${NB_IMAGE} project_title=${PRJ_TITLE} [Teardown] Run Keywords ... Clean Project From Workbench Resources workbench_title=${WORKBENCH_TITLE} project_title=${PRJ_TITLE} @@ -502,7 +508,7 @@ Verify Users Can Start, Stop And Launch A Workbench From DS Projects Home Page ... SeleniumLibrary.Close Browser Verify User Can Delete A Data Science Project - [Tags] Smoke Tier1 + [Tags] Smoke ... ODS-1784 [Documentation] Verifies users can delete a Data Science project Open Data Science Projects Home Page @@ -511,7 +517,8 @@ Verify User Can Delete A Data Science Project # check workbenches and resources get deleted too Verify User Can Access Only Its Owned Projects - [Tags] Sanity Tier1 ODS-1868 + [Tags] Sanity + ... ODS-1868 [Documentation] Verifies each user can access only they owned projects. Except for ... cluster and dedicated admins which should be able to fetch all the DS Projects [Setup] Run Keywords @@ -576,7 +583,7 @@ Project Suite Setup Delete List Of Projects Via CLI ocp_projects=${PROJECTS_TO_DELETE} Launch Data Science Project Main Page Create Data Science Project title=${PRJ_TITLE} description=${PRJ_DESCRIPTION} - ... resource_name=${PRJ_RESOURCE_NAME} + ... existing_project=${TRUE} Project Suite Teardown [Documentation] Suite teardown steps after testing DS Projects. It Deletes diff --git a/ods_ci/tests/Tests/0600__distributed_workloads/0601__workloads_orchestration/test-run-codeflare-sdk-e2e-tests.robot b/ods_ci/tests/Tests/0600__distributed_workloads/0601__workloads_orchestration/test-run-codeflare-sdk-e2e-tests.robot index 9b31c3ba5..69da96099 100644 --- a/ods_ci/tests/Tests/0600__distributed_workloads/0601__workloads_orchestration/test-run-codeflare-sdk-e2e-tests.robot +++ b/ods_ci/tests/Tests/0600__distributed_workloads/0601__workloads_orchestration/test-run-codeflare-sdk-e2e-tests.robot @@ -8,25 +8,76 @@ Resource ../../../../tasks/Resources/RHODS_OLM/install/oc_install.robot Resource ../../../Resources/RHOSi.resource Resource ../../../../tests/Resources/Page/DistributedWorkloads/DistributedWorkloads.resource +*** Variables *** +${CODEFLARE-SDK_HTC_REPO_URL} https://github.com/Ygnas/codeflare-sdk.git +${CODEFLARE-SDK-HTC-RELEASE-TAG} heterogeneous-clusters +${CODEFLARE-SDK_HTC_DIR} codeflare-sdk-htc *** Test Cases *** -Run TestRayClusterSDKOauth test +Run TestRayClusterSDKOauth test with Python 3.9 [Documentation] Run Python E2E test: TestRayClusterSDKOauth [Tags] ... Tier1 ... DistributedWorkloads ... WorkloadsOrchestration ... Codeflare-sdk - Run Codeflare-SDK Test e2e mnist_raycluster_sdk_oauth_test.py + Run Codeflare-SDK Test e2e mnist_raycluster_sdk_oauth_test.py 3.9 ${RAY_IMAGE_3.9} -Run TestRayLocalInteractiveOauth test +Run TestRayClusterSDKOauth test with Python 3.11 + [Documentation] Run Python E2E test: TestRayClusterSDKOauth + [Tags] + ... Tier1 + ... DistributedWorkloads + ... WorkloadsOrchestration + ... Codeflare-sdk + Run Codeflare-SDK Test e2e mnist_raycluster_sdk_oauth_test.py 3.11 ${RAY_IMAGE_3.11} + +Run TestRayLocalInteractiveOauth test with Python 3.9 + [Documentation] Run Python E2E test: TestRayLocalInteractiveOauth + [Tags] + ... Tier1 + ... DistributedWorkloads + ... WorkloadsOrchestration + ... Codeflare-sdk + Run Codeflare-SDK Test e2e local_interactive_sdk_oauth_test.py 3.9 ${RAY_IMAGE_3.9} + +Run TestRayLocalInteractiveOauth test with Python 3.11 [Documentation] Run Python E2E test: TestRayLocalInteractiveOauth [Tags] ... Tier1 ... DistributedWorkloads ... WorkloadsOrchestration ... Codeflare-sdk - Run Codeflare-SDK Test e2e local_interactive_sdk_oauth_test.py + Run Codeflare-SDK Test e2e local_interactive_sdk_oauth_test.py 3.11 ${RAY_IMAGE_3.11} + +Run TestHeterogenousClustersOauth + [Documentation] Run Python E2E test: TestHeterogenousClustersOauth (workaround for 2.15) + [Tags] + ... Tier1 + ... DistributedWorkloads + ... WorkloadsOrchestration + ... HeterogeneousCluster + ... Codeflare-sdk + DistributedWorkloads.Clone Git Repository ${CODEFLARE-SDK_HTC_REPO_URL} ${CODEFLARE-SDK-HTC-RELEASE-TAG} ${CODEFLARE-SDK_HTC_DIR} + Log To Console "Running codeflare-sdk test: ${TEST_NAME}" + ${result} = Run Process source ${VIRTUAL_ENV_NAME}/bin/activate && cd ${CODEFLARE-SDK_HTC_DIR} && poetry env use 3.11 && poetry install --with test,docs && poetry run pytest -v -s ./tests/e2e/heterogeneous_clusters_oauth_test.py --timeout\=600 && deactivate + ... env:RAY_IMAGE=${RAY_IMAGE} + ... env:AWS_DEFAULT_ENDPOINT=${AWS_DEFAULT_ENDPOINT} + ... env:AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + ... env:AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + ... env:AWS_STORAGE_BUCKET=${AWS_STORAGE_BUCKET} + ... env:AWS_STORAGE_BUCKET_MNIST_DIR=${AWS_STORAGE_BUCKET_MNIST_DIR} + ... env:PIP_INDEX_URL=${PIP_INDEX_URL} + ... env:PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST} + ... env:CONTROL_LABEL=node-role.kubernetes.io/control-plane= + ... env:WORKER_LABEL=node-role.kubernetes.io/worker= + ... env:TOLERATION_KEY=node-role.kubernetes.io/master + ... shell=true + ... stderr=STDOUT + Log To Console ${result.stdout} + IF ${result.rc} != 0 + FAIL Running test ${TEST_NAME} failed + END *** Keywords *** Prepare Codeflare-sdk E2E Test Suite diff --git a/ods_ci/tests/Tests/0600__distributed_workloads/test-run-distributed-workloads-tests.robot b/ods_ci/tests/Tests/0600__distributed_workloads/test-run-distributed-workloads-tests.robot index 586afd0bc..630e4073a 100644 --- a/ods_ci/tests/Tests/0600__distributed_workloads/test-run-distributed-workloads-tests.robot +++ b/ods_ci/tests/Tests/0600__distributed_workloads/test-run-distributed-workloads-tests.robot @@ -11,6 +11,7 @@ Resource ../../Resources/Page/DistributedWorkloads/DistributedWorkloads *** Variables *** ${RAY_CUDA_IMAGE} quay.io/modh/ray@sha256:db667df1bc437a7b0965e8031e905d3ab04b86390d764d120e05ea5a5c18d1b4 ${RAY_TORCH_CUDA_IMAGE} quay.io/rhoai/ray@sha256:5077f9bb230dfa88f34089fecdfcdaa8abc6964716a8a8325c7f9dcdf11bbbb3 +${RAY_ROCM_IMAGE} quay.io/modh/ray@sha256:f8b4f2b1c954187753c1f5254f7bb6a4286cec5a4f1b43def7ef4e009f2d28cb *** Test Cases *** @@ -23,8 +24,8 @@ Run TestKueueRayCpu ODH test ... WorkloadsOrchestration Run DistributedWorkloads ODH Test TestMnistRayCpu ${RAY_CUDA_IMAGE} -Run TestKueueRayGpu ODH test - [Documentation] Run Go ODH test: TestKueueRayGpu +Run TestKueueRayCudaGpu ODH test + [Documentation] Run Go ODH test: TestKueueRayCudaGpu [Tags] Resources-GPU NVIDIA-GPUs ... Tier1 ... DistributedWorkloads @@ -32,6 +33,15 @@ Run TestKueueRayGpu ODH test ... WorkloadsOrchestration Run DistributedWorkloads ODH Test TestMnistRayCudaGpu ${RAY_CUDA_IMAGE} +Run TestKueueRayROCmGpu ODH test + [Documentation] Run Go ODH test: TestKueueRayROCmGpu + [Tags] Resources-GPU AMD-GPUs ROCm + ... Tier1 + ... DistributedWorkloads + ... Training + ... WorkloadsOrchestration + Run DistributedWorkloads ODH Test TestMnistRayROCmGpu ${RAY_ROCM_IMAGE} + Run TestRayTuneHPOCpu ODH test [Documentation] Run Go ODH test: TestMnistRayTuneHpoCpu [Tags] RHOAIENG-10004 diff --git a/ods_ci/tests/Tests/1000__model_serving/1007__model_serving_llm/1007__model_serving_llm_models.robot b/ods_ci/tests/Tests/1000__model_serving/1007__model_serving_llm/1007__model_serving_llm_models.robot index 97e2ea4f6..39733f9b4 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1007__model_serving_llm/1007__model_serving_llm_models.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1007__model_serving_llm/1007__model_serving_llm_models.robot @@ -1347,7 +1347,7 @@ Set Runtime Image [Arguments] ${gpu_type} IF "${RUNTIME_IMAGE}" == "${EMPTY}" IF "${gpu_type}" == "nvidia" - Set Test Variable ${runtime_image} quay.io/modh/vllm@sha256:94e2d256da29891a865103f7e92a1713f0fd385ef611c6162526f4a297e70916 + Set Test Variable ${runtime_image} quay.io/modh/vllm@sha256:a8ba53e1b12309913cd958331dd8dda7f2b1fad39f5350d3c722608835e14512 ELSE IF "${gpu_type}" == "amd" Set Test Variable ${runtime_image} quay.io/modh/vllm@sha256:9969e5273a492132b39ce25165c94480393bb87628f50c30d4de26a0afa56abd ELSE diff --git a/ods_ci/tests/Tests/1000__model_serving/1009__model_serving_triton_on_kserve/1009__model_serving_triton_on_kserve.robot b/ods_ci/tests/Tests/1000__model_serving/1009__model_serving_triton_on_kserve/1009__model_serving_triton_on_kserve.robot index 5abb2c489..e449146f6 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1009__model_serving_triton_on_kserve/1009__model_serving_triton_on_kserve.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1009__model_serving_triton_on_kserve/1009__model_serving_triton_on_kserve.robot @@ -45,6 +45,11 @@ ${TENSORFLOW_RUNTIME_NAME}= triton-tensorflow-grpc ${TENSORFLOW_GRPC_RUNTIME_NAME}= triton-tensorflow-grpc ${TENSORFLOW_RUNTIME_FILEPATH}= ${RESOURCES_DIRPATH}/triton_tensorflow_gRPC_servingruntime.yaml ${EXPECTED_INFERENCE_GRPC_OUTPUT_FILE_TENSORFLOW}= tests/Resources/Files/triton/kserve-triton-inception_graphdef-gRPC-output.json +${KERAS_RUNTIME_NAME}= triton-keras-rest +${KERAS_MODEL_NAME}= resnet50 +${KERAS_RUNTIME_FILEPATH}= ${RESOURCES_DIRPATH}/triton_keras_rest_servingruntime.yaml +${INFERENCE_REST_INPUT_KERAS}= @tests/Resources/Files/triton/kserve-triton-keras-rest-input.json +${EXPECTED_INFERENCE_REST_OUTPUT_FILE_KERAS}= tests/Resources/Files/triton/kserve-triton-keras-rest-output.json @@ -203,6 +208,38 @@ Test Tensorflow Model Grpc Inference Via UI (Triton on Kserve) # robocop: off ... AND ... Delete Serving Runtime Template From CLI displayed_name=triton-tensorflow-grpc +Test KERAS Model Inference Via UI(Triton on Kserve) + [Documentation] Test the deployment of an keras model in Kserve using Triton + [Tags] Sanity RHOAIENG-10328 + + Open Data Science Projects Home Page + Create Data Science Project title=${PRJ_TITLE} description=${PRJ_DESCRIPTION} + ... existing_project=${FALSE} + Open Dashboard Settings settings_page=Serving runtimes + Upload Serving Runtime Template runtime_filepath=${KERAS_RUNTIME_FILEPATH} + ... serving_platform=single runtime_protocol=REST + Serving Runtime Template Should Be Listed displayed_name=${KERAS_RUNTIME_NAME} + ... serving_platform=single + Recreate S3 Data Connection project_title=${PRJ_TITLE} dc_name=model-serving-connection + ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} + ... aws_bucket_name=ods-ci-s3 + Deploy Kserve Model Via UI model_name=${KERAS_MODEL_NAME} serving_runtime=triton-keras-rest + ... data_connection=model-serving-connection path=tritonkeras/model_repository/ model_framework=tensorflow - 2 + Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${KERAS_MODEL_NAME} + ... namespace=${PRJ_TITLE} timeout=180s + ${EXPECTED_INFERENCE_REST_OUTPUT_KERAS}= Load Json File + ... file_path=${EXPECTED_INFERENCE_REST_OUTPUT_FILE_KERAS} as_string=${TRUE} + Log ${EXPECTED_INFERENCE_REST_OUTPUT_KERAS} + Run Keyword And Continue On Failure Verify Model Inference With Retries + ... ${KERAS_MODEL_NAME} ${INFERENCE_REST_INPUT_KERAS} ${EXPECTED_INFERENCE_REST_OUTPUT_KERAS} + ... token_auth=${FALSE} project_title=${PRJ_TITLE} + [Teardown] Run Keywords Get Kserve Events And Logs model_name=${KERAS_MODEL_NAME} + ... project_title=${PRJ_TITLE} + ... AND + ... Clean All Models Of Current User + ... AND + ... Delete Serving Runtime Template From CLI displayed_name=triton-keras-rest + *** Keywords *** Triton On Kserve Suite Setup [Documentation] Suite setup steps for testing Triton. It creates some test variables diff --git a/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_smoke.robot b/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_smoke.robot index 466bb6a1c..132136d10 100644 --- a/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_smoke.robot +++ b/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_smoke.robot @@ -1,15 +1,18 @@ *** Settings *** Documentation Smoke Test for Model Registry Deployment -Suite Setup Setup Test Environment -Suite Teardown Teardown Model Registry Test Setup +Suite Setup Setup Test Environment Non UI +Suite Teardown Teardown Model Registry Test Setup Non UI Library Collections Library OperatingSystem Library Process Library OpenShiftLibrary Library RequestsLibrary +Library BuiltIn Resource ../../Resources/Page/ODH/JupyterHub/HighAvailability.robot Resource ../../Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource Resource ../../Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/DataConnections.resource +Resource ../../Resources/Page/ModelRegistry/ModelRegistry.resource +Resource ../../Resources/Page/Components/Components.resource Resource ../../Resources/OCP.resource Resource ../../Resources/Common.robot @@ -17,7 +20,11 @@ Resource ../../Resources/Common.robot *** Variables *** ${MODELREGISTRY_BASE_FOLDER}= tests/Resources/CLI/ModelRegistry ${MODEL_REGISTRY_DB_SAMPLES}= ${MODELREGISTRY_BASE_FOLDER}/samples/istio/mysql -${DISABLE_COMPONENT}= ${False} +${OPERATOR_NS} ${OPERATOR_NAMESPACE} +${APPLICATIONS_NS} ${APPLICATIONS_NAMESPACE} +${DSC_NAME} default-dsc + +@{REDHATIO_PATH_CHECK_EXCLUSTION_LIST} model-registry-operator-controller-manager *** Test Cases *** @@ -27,9 +34,13 @@ Deploy Model Registry Set Library Search Order SeleniumLibrary RHOSi Setup Enable Model Registry If Needed + Set DSC Component Managed State And Wait For Completion modelregistry + ... model-registry-operator-controller-manager + ... control-plane=model-registry-operator Component Should Be Enabled modelregistry Sleep 60s reason=Wait for webhook endpoint - Apply Db Config Samples + Apply Db Config Samples namespace=${NAMESPACE_MODEL_REGISTRY} samples=${MODEL_REGISTRY_DB_SAMPLES} + Wait Until Keyword Succeeds 10 s 2 s Verify Model Registry Can Accept Requests Registering A Model In The Registry [Documentation] Registers a model in the model registry @@ -38,14 +49,15 @@ Registering A Model In The Registry Register A Model ${URL} Verify Model Registry - [Documentation] Deploy Python Client And Register Model. + [Documentation] Verify the registered model. [Tags] Smoke MR1302 ModelRegistry Depends On Test Registering A Model In The Registry - Run Curl Command And Verify Response ${URL} + Log Attempting to verify Model Registry + Wait Until Keyword Succeeds 10 s 2 s Run Curl Command And Verify Response *** Keywords *** -Setup Test Environment +Setup Test Environment Non UI [Documentation] Set Model Regisry Test Suite ${NAMESPACE_MODEL_REGISTRY}= Get Model Registry Namespace From DSC Log Set namespace to: ${NAMESPACE_MODEL_REGISTRY} @@ -54,100 +66,21 @@ Setup Test Environment Get Cluster Domain And Token Set Suite Variable ${URL} http://modelregistry-sample-rest.${DOMAIN}/api/model_registry/v1alpha3/registered_models -Teardown Model Registry Test Setup +Teardown Model Registry Test Setup Non UI [Documentation] Teardown Model Registry Suite - Remove Model Registry + Remove Model Registry Non UI Disable Model Registry If Needed RHOSi Teardown -Apply Db Config Samples - [Documentation] Applying the db config samples from https://github.com/opendatahub-io/model-registry-operator - ${rc} ${out}= Run And Return Rc And Output - ... oc apply -k ${MODEL_REGISTRY_DB_SAMPLES} -n ${NAMESPACE_MODEL_REGISTRY} - Should Be Equal As Integers ${rc} 0 msg=${out} - Wait For Model Registry Containers To Be Ready - -Wait For Model Registry Containers To Be Ready - [Documentation] Wait for model-registry-deployment to be ready - ${result}= Run Process - ... oc wait --for\=condition\=Available --timeout\=5m -n ${NAMESPACE_MODEL_REGISTRY} deployment/model-registry-db # robocop: disable:line-too-long - ... shell=true stderr=STDOUT - Log To Console ${result.stdout} - ${result}= Run Process - ... oc wait --for\=condition\=Available --timeout\=5m -n ${NAMESPACE_MODEL_REGISTRY} deployment/modelregistry-sample # robocop: disable:line-too-long - ... shell=true stderr=STDOUT - Log To Console ${result.stdout} - -Remove Model Registry +Remove Model Registry Non UI [Documentation] Run multiple oc delete commands to remove model registry components # We don't want to stop the teardown if any of these resources are not found Run Keyword And Continue On Failure ... Run And Verify Command ... oc delete -k ${MODELREGISTRY_BASE_FOLDER}/samples/istio/mysql -n ${NAMESPACE_MODEL_REGISTRY} -Get Model Registry Namespace From DSC - [Documentation] Fetches the namespace defined for model registry in the DSC - ${rc} ${ns}= Run And Return Rc And Output - ... oc get dsc default-dsc -o json | jq '.spec.components.modelregistry.registriesNamespace' - Should Be Equal As Integers ${rc} 0 - Log ${ns} - # Remove double quotes - ${ns}= Get Substring ${ns} 1 -1 - Log ${ns} - RETURN ${ns} - -Enable Model Registry If Needed - [Documentation] While in tech preview the component will not be enabled by default. This keyword enables it. - ${management_state}= Get DSC Component State default-dsc modelregistry ${OPERATOR_NAMESPACE} - IF "${management_state}" != "Managed" - Set Component State modelregistry Managed - Set Suite Variable ${DISABLE_COMPONENT} ${True} - Wait For Namespace To Be Active ${NAMESPACE_MODEL_REGISTRY} timeout=5m - END - -Disable Model Registry If Needed - [Documentation] If we had to enable the component before the test run, let's also disable it at the end to leave - ... the cluster in the same state we found it in - IF ${DISABLE_COMPONENT}==${True} - Set Component State modelregistry Removed - Run And Verify Command oc delete namespace ${NAMESPACE_MODEL_REGISTRY} --force - END - -Get Cluster Domain And Token - [Documentation] Logs the Domain and Token capture. - ${domain}= Get Domain - ${token}= Get Token - Set Suite Variable ${DOMAIN} ${domain} - Set Suite Variable ${TOKEN} ${token} - Log Domain: ${DOMAIN} - -Get Domain - [Documentation] Gets the Domain and returns it to 'Get Cluster Domain And Token'. - # Run the command to get the ingress domain - ${domain_result}= Run Process oc get ingresses.config/cluster - ... -o yaml stdout=PIPE stderr=PIPE - ${rc}= Set Variable ${domain_result.rc} - IF $rc > 0 Fail Command 'oc whoami -t' returned non-zero exit code: ${rc} - ${domain_yaml_output}= Set Variable ${domain_result.stdout} - - # Return the domain from stdout - ${domain_parsed_yaml}= Evaluate yaml.load('''${domain_yaml_output}''', Loader=yaml.FullLoader) - ${ingress_domain}= Set Variable ${domain_parsed_yaml['spec']['domain']} - - # Return both results - RETURN ${ingress_domain} - -Get Token - [Documentation] Gets the Token and returns it to 'Get Cluster Domain And Token'. - ${token_result}= Run Process oc whoami -t stdout=YES - ${rc}= Set Variable ${token_result.rc} - IF ${rc} > 0 Fail Command 'oc whoami -t' returned non-zero exit code: ${rc} - ${token}= Set Variable ${token_result.stdout} - RETURN ${token} - Run Curl Command And Verify Response [Documentation] Runs a curl command to verify response from server - [Arguments] ${URL} ${result}= Run Process curl -H Authorization: Bearer ${TOKEN} ... ${URL} stdout=stdout stderr=stderr Log ${result.stderr} @@ -159,6 +92,17 @@ Run Curl Command And Verify Response Should Contain ${result.stdout} model-name Should Not Contain ${result.stdout} error +Verify Model Registry Can Accept Requests + [Documentation] Runs a curl command to verify response from server + ${result}= Run Process curl -H Authorization: Bearer ${TOKEN} + ... ${URL} stdout=stdout stderr=stderr + Log ${result.stderr} + Log ${result.stdout} + Should Contain ${result.stdout} items + Should Contain ${result.stdout} nextPageToken + Should Contain ${result.stdout} pageSize + Should Not Contain ${result.stdout} error + Register A Model [Documentation] Registers a test model in the model-registry [Arguments] ${URL} diff --git a/ods_ci/tests/Tests/2001__disruptive_tests/2003__smcp_already_created.robot b/ods_ci/tests/Tests/2001__disruptive_tests/2003__smcp_already_created.robot index 20bcca38a..aea001494 100644 --- a/ods_ci/tests/Tests/2001__disruptive_tests/2003__smcp_already_created.robot +++ b/ods_ci/tests/Tests/2001__disruptive_tests/2003__smcp_already_created.robot @@ -23,13 +23,13 @@ ${INSTALL_TYPE} CLi ${TEST_ENV} PSI ${IS_PRESENT} 0 ${IS_NOT_PRESENT} 1 -${MSSG_REGEX} denied the request: only one service mesh may be installed per project/namespace +${MSG_REGEX} denied the request: only one service mesh may be installed per project/namespace *** Test Cases *** Validate Service Mesh Control Plane Already Created [Documentation] This Test Case validates that only one ServiceMeshControlPlane is allowed to be installed per project/namespace - [Tags] RHOAIENG-2517 + [Tags] RHOAIENG-2517 Operator Fetch Image Url And Update Channel Check Whether DSC Exists And Save Component Statuses Fetch Cluster Type By Domain @@ -142,17 +142,20 @@ Operator Deployment Should Be Ready Verify Pod Logs Contain Error [Documentation] Checks whether there is a SMCP related error on the Pod Logs ${pod_name}= Get Pod Name ${OPERATOR_NAMESPACE} ${OPERATOR_LABEL_SELECTOR} - ${pod_logs}= Oc Get Pod Logs - ... name=${pod_name} - ... namespace=${OPERATOR_NAMESPACE} - ... container=${OPERATOR_POD_CONTAINER_NAME} - ${match_list}= Get Regexp Matches ${pod_logs} ${MSSG_REGEX} - ${entry_msg}= Remove Duplicates ${match_list} - ${length}= Get Length ${entry_msg} - IF ${length} == ${0} - FAIL Pod ${pod_name} logs should contain message ${MSSG_REGEX}. - ELSE - Log message=Pod ${pod_name} logs contain error message '${MSSG_REGEX}' level=INFO + ${length}= Set Variable 0 + TRY + WHILE ${length} == 0 limit=5m + ${pod_logs}= Oc Get Pod Logs + ... name=${pod_name} + ... namespace=${OPERATOR_NAMESPACE} + ... container=${OPERATOR_POD_CONTAINER_NAME} + ${match_list}= Get Regexp Matches ${pod_logs} ${MSG_REGEX} + ${entry_msg}= Remove Duplicates ${match_list} + ${length}= Get Length ${entry_msg} + Sleep 10s + END + EXCEPT + Fail msg=Pod ${pod_name} logs should contain message '${MSG_REGEX}' END Cleanup Olm Install Dir diff --git a/ods_ci/utils/scripts/ocm/ocm.py b/ods_ci/utils/scripts/ocm/ocm.py index f25845f36..f1d390b2a 100644 --- a/ods_ci/utils/scripts/ocm/ocm.py +++ b/ods_ci/utils/scripts/ocm/ocm.py @@ -147,8 +147,11 @@ def osd_cluster_create(self): version_cmd = f'ocm list versions {chan_grp} | grep -w "{re.escape(version)}*"' versions = execute_command(version_cmd) - if versions is not None: + if versions: version = [ver for ver in versions.split("\n") if ver][-1] + else: + log.error(f"No supported versions found for {version} in OCM") + sys.exit(1) self.openshift_version = version else: log.info("Using the osd version given by user as it is...")