From 6a6e86829071d20a7f7fc30d7346c95a6fd3927a Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 26 Jan 2021 15:05:00 -0500 Subject: Cirrus: Collect ginkgo node logs artifacts In rare cases, it's possible for one of the ginkgo processes to "hang". When this occurs, the main output will contain this message: ``Ginkgo timed out waiting for all parallel nodes to report`` The only way to debug this was to look through concatenated printing of the ginkgo node logs. This is a tedious and daunting task, requiring special search knowledge, facing a "wall of text". Simplify the situation by collecting the node logs separately, as individual files in a cirrus-artifact. In this way, it's faster to figure out which test "hung" by examining each log individually. The log file which does not have a pass/fail summary at the end, indicates the last test hung (for whatever reason), and includes it's output (if any). Signed-off-by: Chris Evich --- .cirrus.yml | 11 +++++++---- contrib/cirrus/logcollector.sh | 1 - 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index eda03bf23..a23595712 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -433,7 +433,6 @@ apiv2_test_task: path: ./*.html type: text/html package_versions_script: '$SCRIPT_BASE/logcollector.sh packages' - ginkgo_node_logs_script: '$SCRIPT_BASE/logcollector.sh ginkgo' df_script: '$SCRIPT_BASE/logcollector.sh df' audit_log_script: '$SCRIPT_BASE/logcollector.sh audit' journal_script: '$SCRIPT_BASE/logcollector.sh journal' @@ -478,7 +477,11 @@ local_integration_test_task: &local_integration_test_task gopath_cache: *ro_gopath_cache setup_script: *setup main_script: *main - always: *logs_artifacts + always: &int_logs_artifacts + <<: *logs_artifacts + ginkgo_node_logs_artifacts: + path: ./test/e2e/ginkgo-node-*.log + type: text/plain # Nearly identical to `local_integration_test` except all operations @@ -521,7 +524,7 @@ container_integration_test_task: gopath_cache: *ro_gopath_cache setup_script: *setup main_script: *main - always: *logs_artifacts + always: *int_logs_artifacts # Execute most integration tests as a regular (non-root) user. @@ -542,7 +545,7 @@ rootless_integration_test_task: gopath_cache: *ro_gopath_cache setup_script: *setup main_script: *main - always: *logs_artifacts + always: *int_logs_artifacts # Always run subsequent to integration tests. While parallelism is lost diff --git a/contrib/cirrus/logcollector.sh b/contrib/cirrus/logcollector.sh index 323015cef..38a15ded1 100755 --- a/contrib/cirrus/logcollector.sh +++ b/contrib/cirrus/logcollector.sh @@ -31,7 +31,6 @@ case $1 in esac ;; df) showrun df -lhTx tmpfs ;; - ginkgo) showrun cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log ;; journal) showrun journalctl -b ;; podman) showrun ./bin/podman system info ;; server) -- cgit v1.2.3-54-g00ecf