diff --git a/scripts/fetch_tools.sh b/scripts/fetch_tools.sh index 377c093591..c7b27e92ca 100755 --- a/scripts/fetch_tools.sh +++ b/scripts/fetch_tools.sh @@ -282,6 +282,53 @@ EOF fi } +gettool_omc() { + local ver="3.15.0" + declare -A checksums=( + ["x86_64"]="201b78b0f6b960e6243fe8dbc716c13d6eb28a18122473cb705afff454d77eba" + ["aarch64"]="538c9ced0b2faace26e8cd426d5f1e56e50939b0a6d823125065825bf8ca2d2a") + + local checksum="${checksums[${ARCH}]}" + local filename="omc" + local dest="${DEST_DIR}/${filename}" + + [[ -e "${dest}" ]] && return 0 + echo "Installing ${filename} to ${DEST_DIR}" + + if [[ "${ARCH}" == "aarch64" ]]; then + local url="https://github.com/gmeghnag/omc/releases/download/v${ver}/omc_Linux_aarch64.tar.gz" + local tarball="omc_Linux_aarch64.tar.gz" + + curl -sSfL --retry 5 --retry-delay 3 -o "${WORK_DIR}/${tarball}" "${url}" + echo -n "${checksum} -" >"${WORK_DIR}/checksum.txt" + if ! sha256sum -c "${WORK_DIR}/checksum.txt" < "${WORK_DIR}/${tarball}" &>/dev/null; then + echo " Checksum for ${tarball} doesn't match" + return 1 + fi + (cd "${WORK_DIR}" && tar xzf "${tarball}") + chmod +x "${WORK_DIR}/${filename}" + mkdir -p "$(dirname "${dest}")" + mv "${WORK_DIR}/${filename}" "${dest}" + else + local url="https://github.com/gmeghnag/omc/releases/download/v${ver}/omc_Linux_x86_64" + _install "${url}" "${checksum}" "${filename}" "omc_Linux_x86_64" + fi +} + +gettool_omg() { + local ver="1.2.6" + local venv="${DEST_DIR}/omg-venv" + + if "${venv}/bin/omg" --help &>/dev/null; then + return 0 + fi + echo "Installing omg ${ver} to ${venv}" + + python3 -m venv "${venv}" + "${venv}/bin/python3" -m pip install --upgrade pip + "${venv}/bin/pip" install "o-must-gather==${ver}" +} + gettool_tar-diff() { # See https://github.com/containers/tar-diff local ver="v0.1.2" diff --git a/test/resources/sos-report.resource b/test/resources/sos-report.resource new file mode 100644 index 0000000000..5d3e87139f --- /dev/null +++ b/test/resources/sos-report.resource @@ -0,0 +1,53 @@ +*** Settings *** +Documentation Keywords for creating, extracting, and cleaning up +... MicroShift SOS reports. + +Resource common.resource +Resource microshift-host.resource + + +*** Keywords *** +Create Sos Report + [Documentation] Create a MicroShift SOS Report using microshift-sos-report + ... and return the tar file path. + ... Requires ${SOS_REPORT_BASE_DIR} to be set in the importing suite. + ${rand_str}= Generate Random String 4 [NUMBERS] + ${sos_report_dir}= Catenate SEPARATOR=${EMPTY} ${SOS_REPORT_BASE_DIR}_ ${rand_str} + Command Should Work mkdir -p ${sos_report_dir} + ${output}= Command Should Work + ... /usr/bin/microshift-sos-report --tmp-dir ${sos_report_dir} + Should Contain ${output} Your sos report has been generated and saved in + ${sos_report_tarfile}= Command Should Work + ... find ${sos_report_dir} -type f -name "sosreport-*.tar.xz" + Should Not Be Empty ${sos_report_tarfile} + RETURN ${sos_report_tarfile} + +Create Sos Report With Profile + [Documentation] Create a SOS report with the specified profile and return the tar file path. + ... Requires ${SOS_REPORT_BASE_DIR} to be set in the importing suite. + [Arguments] ${profile} + ${rand_str}= Generate Random String 4 [NUMBERS] + ${sos_report_dir}= Catenate SEPARATOR=${EMPTY} ${SOS_REPORT_BASE_DIR}_ ${rand_str} + Command Should Work mkdir -p ${sos_report_dir} + Command Should Work + ... sos report --batch --clean --all-logs --profile ${profile} --tmp-dir ${sos_report_dir} + ${sos_report_tarfile}= Command Should Work + ... find ${sos_report_dir} -type f -name "sosreport-*.tar.xz" + Should Not Be Empty ${sos_report_tarfile} + RETURN ${sos_report_tarfile} + +Extract Sos Report + [Documentation] Extract SOS report from the tar file and return the extracted directory. + [Arguments] ${sos_report_tarfile} + ${sos_report_dir}= Command Should Work dirname ${sos_report_tarfile} + Command Should Work tar xf ${sos_report_tarfile} -C ${sos_report_dir} + ${sos_report_untared}= Command Should Work + ... find ${sos_report_dir} -maxdepth 1 -type d -name "sosreport-*" + Should Not Be Empty ${sos_report_untared} + RETURN ${sos_report_untared} + +Cleanup Sos Report Directory + [Documentation] Remove temporary SOS report files created during tests. + ... Requires ${SOS_REPORT_BASE_DIR} to be set in the importing suite. + Command Should Work rm -rf ${SOS_REPORT_BASE_DIR}_* + Command Should Work rm -rf /var/tmp/sosreport-* diff --git a/test/scenarios-bootc/el10/releases/el102-lrel@otp-workloads.sh b/test/scenarios-bootc/el10/releases/el102-lrel@otp-workloads.sh index 2aadbc69c9..447c39258e 100644 --- a/test/scenarios-bootc/el10/releases/el102-lrel@otp-workloads.sh +++ b/test/scenarios-bootc/el10/releases/el102-lrel@otp-workloads.sh @@ -20,7 +20,14 @@ scenario_remove_vms() { scenario_run_tests() { exit_if_image_not_found "${start_image}" + DEST_DIR="${RF_VENV}" "${ROOTDIR}/scripts/fetch_tools.sh" omc omg || { + record_junit "host1" "support_tools_installed" "FAILED" + exit 1 + } + record_junit "host1" "support_tools_installed" "OK" + run_tests host1 \ suites/otp-workloads/sos-report-plugins.robot \ + suites/otp-workloads/sos-report-support-tools.robot \ suites/otp-workloads/kcm-flags.robot } diff --git a/test/scenarios-bootc/el9/releases/el98-lrel@otp-workloads.sh b/test/scenarios-bootc/el9/releases/el98-lrel@otp-workloads.sh index de9916ef8e..bfc1b9babb 100644 --- a/test/scenarios-bootc/el9/releases/el98-lrel@otp-workloads.sh +++ b/test/scenarios-bootc/el9/releases/el98-lrel@otp-workloads.sh @@ -20,7 +20,14 @@ scenario_remove_vms() { scenario_run_tests() { exit_if_image_not_found "${start_image}" + DEST_DIR="${RF_VENV}" "${ROOTDIR}/scripts/fetch_tools.sh" omc omg || { + record_junit "host1" "support_tools_installed" "FAILED" + exit 1 + } + record_junit "host1" "support_tools_installed" "OK" + run_tests host1 \ suites/otp-workloads/sos-report-plugins.robot \ + suites/otp-workloads/sos-report-support-tools.robot \ suites/otp-workloads/kcm-flags.robot } diff --git a/test/scenarios/releases/el98-lrel@otp-workloads.sh b/test/scenarios/releases/el98-lrel@otp-workloads.sh index 5b7696dcd5..f452f74cae 100644 --- a/test/scenarios/releases/el98-lrel@otp-workloads.sh +++ b/test/scenarios/releases/el98-lrel@otp-workloads.sh @@ -20,7 +20,14 @@ scenario_remove_vms() { scenario_run_tests() { exit_if_commit_not_found "${start_image}" + DEST_DIR="${RF_VENV}" "${ROOTDIR}/scripts/fetch_tools.sh" omc omg || { + record_junit "host1" "support_tools_installed" "FAILED" + exit 1 + } + record_junit "host1" "support_tools_installed" "OK" + run_tests host1 \ suites/otp-workloads/sos-report-plugins.robot \ + suites/otp-workloads/sos-report-support-tools.robot \ suites/otp-workloads/kcm-flags.robot } diff --git a/test/suites/otp-workloads/sos-report-plugins.robot b/test/suites/otp-workloads/sos-report-plugins.robot index 671d45d06e..a5a74a9518 100644 --- a/test/suites/otp-workloads/sos-report-plugins.robot +++ b/test/suites/otp-workloads/sos-report-plugins.robot @@ -9,6 +9,7 @@ Documentation Tests verifying MicroShift SOS report plugins are listed and Resource ../../resources/microshift-host.resource Resource ../../resources/microshift-process.resource Resource ../../resources/common.resource +Resource ../../resources/sos-report.resource Suite Setup Setup Suite Teardown Teardown @@ -134,45 +135,3 @@ Plugin Should Be Enabled [Documentation] Verify that the named plugin is not disabled in the sos report -l output. [Arguments] ${output} ${plugin_name} Should Not Match Regexp ${output} ${plugin_name}\\s+.*disabled - -Create Sos Report - [Documentation] Create a MicroShift SOS Report using microshift-sos-report - ... and return the tar file path. - ${rand_str}= Generate Random String 4 [NUMBERS] - ${sos_report_dir}= Catenate SEPARATOR=${EMPTY} ${SOS_REPORT_BASE_DIR}_ ${rand_str} - Command Should Work mkdir -p ${sos_report_dir} - ${output}= Command Should Work - ... /usr/bin/microshift-sos-report --tmp-dir ${sos_report_dir} - Should Contain ${output} Your sos report has been generated and saved in - ${sos_report_tarfile}= Command Should Work - ... find ${sos_report_dir} -type f -name "sosreport-*.tar.xz" - Should Not Be Empty ${sos_report_tarfile} - RETURN ${sos_report_tarfile} - -Create Sos Report With Profile - [Documentation] Create a SOS report with the specified profile and return the tar file path. - [Arguments] ${profile} - ${rand_str}= Generate Random String 4 [NUMBERS] - ${sos_report_dir}= Catenate SEPARATOR=${EMPTY} ${SOS_REPORT_BASE_DIR}_ ${rand_str} - Command Should Work mkdir -p ${sos_report_dir} - Command Should Work - ... sos report --batch --clean --all-logs --profile ${profile} --tmp-dir ${sos_report_dir} - ${sos_report_tarfile}= Command Should Work - ... find ${sos_report_dir} -type f -name "sosreport-*.tar.xz" - Should Not Be Empty ${sos_report_tarfile} - RETURN ${sos_report_tarfile} - -Extract Sos Report - [Documentation] Extract SOS report from the tar file and return the extracted directory. - [Arguments] ${sos_report_tarfile} - ${sos_report_dir}= Command Should Work dirname ${sos_report_tarfile} - Command Should Work tar xf ${sos_report_tarfile} -C ${sos_report_dir} - ${sos_report_untared}= Command Should Work - ... find ${sos_report_dir} -maxdepth 1 -type d -name "sosreport-*" - Should Not Be Empty ${sos_report_untared} - RETURN ${sos_report_untared} - -Cleanup Sos Report Directory - [Documentation] Remove temporary SOS report files created during tests. - Command Should Work rm -rf ${SOS_REPORT_BASE_DIR}_* - Command Should Work rm -rf /var/tmp/sosreport-* diff --git a/test/suites/otp-workloads/sos-report-support-tools.robot b/test/suites/otp-workloads/sos-report-support-tools.robot new file mode 100644 index 0000000000..1b9a9c49ca --- /dev/null +++ b/test/suites/otp-workloads/sos-report-support-tools.robot @@ -0,0 +1,124 @@ +*** Settings *** +Documentation Tests verifying MicroShift SOS report output can be consumed +... by the omc and omg support tools. Both tools are pre-installed +... locally by fetch_tools.sh (omc and omg) and run against +... a sosreport downloaded from the remote host. +... +... Ported from Polarion: OCP-61971 +... USHIFT-7221 + +Resource ../../resources/microshift-host.resource +Resource ../../resources/common.resource +Resource ../../resources/sos-report.resource + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags slow + + +*** Variables *** +${SOS_REPORT_BASE_DIR} /tmp/rf-test/sos-report-support-tools +${OMC_BINARY} ${EXECDIR}/../_output/robotenv/omc +${OMG_BINARY} ${EXECDIR}/../_output/robotenv/omg-venv/bin/omg +${MUST_GATHER_PATH} ${EMPTY} +${LOCAL_SOS_DIR} ${EMPTY} + + +*** Test Cases *** +Verify Omc Can Consume SOS Report + [Documentation] Verify that the omc tool can consume the SOS report + ... output under sos_commands/microshift and list pods, nodes, + ... projects, and retrieve logs. + ... OCP-61971 + Verify Support Tool Can Consume Sos Report ${OMC_BINARY} + +Verify Omg Can Consume SOS Report + [Documentation] Verify that the omg tool can consume the SOS report + ... output under sos_commands/microshift and list pods, nodes, + ... projects, and retrieve logs. + ... OCP-61971 + Verify Support Tool Can Consume Sos Report ${OMG_BINARY} + + +*** Keywords *** +Setup + [Documentation] Set up all of the tests in this suite + Check Required Env Variables + OperatingSystem.File Should Exist ${OMC_BINARY} omc not found - run fetch_tools.sh omc + OperatingSystem.File Should Exist ${OMG_BINARY} omg not found - run fetch_tools.sh omg + Login MicroShift Host + ${sos_report_tarfile}= Create Sos Report + ${sos_report_extracted}= Extract Sos Report ${sos_report_tarfile} + ${local_dir}= Download Must Gather Data ${sos_report_extracted} + VAR ${LOCAL_SOS_DIR}= ${local_dir} scope=SUITE + VAR ${MUST_GATHER_PATH}= ${local_dir}/sos_commands/microshift scope=SUITE + OperatingSystem.Directory Should Exist ${MUST_GATHER_PATH} + +Teardown + [Documentation] Test suite teardown + Run Keyword And Ignore Error + ... OperatingSystem.Remove Directory ${LOCAL_SOS_DIR} recursive=True + Cleanup Sos Report Directory + Logout MicroShift Host + +Download Must Gather Data + [Documentation] Download the microshift sos_commands directory from the remote host + ... to a local temp directory and return the local path. + [Arguments] ${remote_extracted_dir} + Command Should Work chmod -R a+rX ${remote_extracted_dir} + ${local_dir}= Create Random Temp Directory + VAR ${local_must_gather}= ${local_dir}/sos_commands/microshift + OperatingSystem.Create Directory ${local_must_gather} + SSHLibrary.Get Directory + ... ${remote_extracted_dir}/sos_commands/microshift ${local_must_gather} recursive=True + OperatingSystem.Directory Should Not Be Empty ${local_must_gather} + RETURN ${local_dir} + +Verify Support Tool Can Consume Sos Report + [Documentation] Verify that the given support tool can consume the SOS report + ... and list pods, nodes, projects, and retrieve logs. + [Arguments] ${tool} + Local Command Should Work ${tool} use ${MUST_GATHER_PATH} + Verify Tool Can List Resources ${tool} + Verify Tool Can Retrieve Logs ${tool} openshift-dns + ${projects_output}= Local Command Should Work ${tool} projects + Should Not Be Empty ${projects_output} + +Verify Tool Can List Resources + [Documentation] Verify the tool can list pods, nodes, and projects. + [Arguments] ${tool} + Local Command Should Work ${tool} project + ${pods_output}= Local Command Should Work ${tool} get pods -A + Should Contain ${pods_output} openshift-dns + ${nodes_output}= Local Command Should Work ${tool} get nodes + Should Not Be Empty ${nodes_output} + Local Command Should Work ${tool} get node --show-labels + Local Command Should Work ${tool} get node -o wide + ${pods_json}= Local Command Should Work ${tool} get pods -A -o json + Should Contain ${pods_json} "items" + +Verify Tool Can Retrieve Logs + [Documentation] Verify the tool can retrieve logs for a pod in the given namespace. + [Arguments] ${tool} ${namespace} + ${pod_name}= Get Pod Name From Tool ${tool} ${namespace} + ${container_name}= Get Container Name From Tool ${tool} ${namespace} + ${logs_output}= Local Command Should Work + ... ${tool} logs ${pod_name} -c ${container_name} -n ${namespace} + Should Not Be Empty ${logs_output} + +Get Pod Name From Tool + [Documentation] Get the first pod name from the specified namespace using the given tool. + [Arguments] ${tool} ${namespace} + ${output}= Local Command Should Work + ... ${tool} get pods -n ${namespace} | tail -n +2 | head -1 | awk '{print $1}' + Should Not Be Empty ${output} + RETURN ${output} + +Get Container Name From Tool + [Documentation] Get the first container name from the first pod in the specified namespace. + [Arguments] ${tool} ${namespace} + ${output}= Local Command Should Work + ... ${tool} get pods -n ${namespace} -o json | python3 -c "import sys,json; print(json.load(sys.stdin)['items'][0]['spec']['containers'][0]['name'])" + Should Not Be Empty ${output} + RETURN ${output}