Skip to content

Commit

Permalink
test: filter "go test" output with gotestsum instead of grep
Browse files Browse the repository at this point in the history 
Filtering the output with grep leads to hard to read log output, e.g. from
pull-kubernetes-unit:

        [0613 15:32:48] Running tests without code coverage and with -race
    {"Time":"2024-06-13T15:33:47.845457374Z","Action":"output","Package":"k8s.io/kubernetes/cluster/gce/cos","Test":"TestCreateMasterAuditPolicy","Output":"        /tmp/configure-helper-test47992121/kube-env: line 1: `}'\n"}
    {"Time":"2024-06-13T15:33:49.053732803Z","Action":"output","Package":"k8s.io/kubernetes/cluster/gce/cos","Output":"ok  \tk8s.io/kubernetes/cluster/gce/cos\t2.906s\n"}

We can do better than that. When feeding the output of the "go test" command(s)
into gotestsum *while it runs*, we can use --format=standard-quiet (= normal go
test output) or --format=standard-verbose (= `go test -v`) when FULL_LOG is
requested to get nicer output.

This works when testing everything at once. This was said to be not possible
when doing coverage profiling. But recent Go no longer has that limitation, so
the xargs trick gets removed. All that we need to do for coverage profiling is
to add some additional parameters and the conversion to HTML.
  • Loading branch information
@pohly
pohly committed Aug 19, 2024
1 parent cb7b4ea commit a7da865
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 127 deletions.
161 changes: 59 additions & 102 deletions hack/make-rules/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 66,6 @@ KUBE_COVERMODE=${KUBE_COVERMODE:-atomic}
# The directory to save test coverage reports to, if generating them. If unset,
# a semi-predictable temporary directory will be used.
KUBE_COVER_REPORT_DIR="${KUBE_COVER_REPORT_DIR:-}"
# How many 'go test' instances to run simultaneously when running tests in
# coverage mode.
KUBE_COVERPROCS=${KUBE_COVERPROCS:-4}
# use KUBE_RACE="" to disable the race detector
# this is defaulted to "-race" in make test as well
# NOTE: DO NOT ADD A COLON HERE. KUBE_RACE="" is meaningful!
Expand Down Expand Up @@ -133,26 130,21 @@ done
shift $((OPTIND - 1))

# Use eval to preserve embedded quoted strings.
#
# KUBE_TEST_ARGS contains arguments for `go test` (like -short)
# and may end with `-args <arguments for test binary>`, so it
# has to be passed to `go test` at the end of the invocation.
testargs=()
eval "testargs=(${KUBE_TEST_ARGS:-})"

# Used to filter verbose test output.
go_test_grep_pattern=".*"

goflags=()
# The junit report tool needs full test case information to produce a
# meaningful report.
if [[ -n "${KUBE_JUNIT_REPORT_DIR}" ]] ; then
goflags =(-v)
goflags =(-json)
# Show only summary lines by matching lines like "status package/test"
go_test_grep_pattern="^[^[:space:]]\ [[:space:]]\ [^[:space:]]\ /[^[[:space:]]\ "
fi

# gotestsum --format value
gotestsum_format=standard-quiet
if [[ -n "${FULL_LOG:-}" ]] ; then
go_test_grep_pattern=".*"
gotestsum_format=standard-verbose
fi

goflags=()

# Filter out arguments that start with "-" and move them to goflags.
testcases=()
for arg; do
Expand Down Expand Up @@ -180,113 172,78 @@ junitFilenamePrefix() {
echo "${KUBE_JUNIT_REPORT_DIR}/junit_$(kube::util::sortable_date)"
}

produceJUnitXMLReport() {
local -r junit_filename_prefix=$1
if [[ -z "${junit_filename_prefix}" ]]; then
return
fi

local junit_xml_filename
junit_xml_filename="${junit_filename_prefix}.xml"

installTools() {
if ! command -v gotestsum >/dev/null 2>&1; then
kube::log::status "gotestsum not found; installing from ./hack/tools"
go -C "${KUBE_ROOT}/hack/tools" install gotest.tools/gotestsum
fi
gotestsum --junitfile "${junit_xml_filename}" --raw-command cat "${junit_filename_prefix}"*.stdout
if [[ ! ${KUBE_KEEP_VERBOSE_TEST_OUTPUT} =~ ^[yY]$ ]]; then
rm "${junit_filename_prefix}"*.stdout
fi

if ! command -v prune-junit-xml >/dev/null 2>&1; then
kube::log::status "prune-junit-xml not found; installing from ./cmd"
go -C "${KUBE_ROOT}/cmd/prune-junit-xml" install .
fi
prune-junit-xml "${junit_xml_filename}"

kube::log::status "Saved JUnit XML test report to ${junit_xml_filename}"
}

runTests() {
local junit_filename_prefix
junit_filename_prefix=$(junitFilenamePrefix)

# Try to normalize input names.
installTools

# Try to normalize input names. This is slow!
local -a targets
kube::log::status "Normalizing Go targets"
kube::util::read-array targets < <(kube::golang::normalize_go_targets "$@")

# If we're not collecting coverage, run all requested tests with one 'go test'
# command, which is much faster.
if [[ ! ${KUBE_COVER} =~ ^[yY]$ ]]; then
kube::log::status "Running tests without code coverage ${KUBE_RACE: "and with ${KUBE_RACE}"}"
# shellcheck disable=SC2031
go test "${goflags[@]: ${goflags[@]}}" \
"${KUBE_TIMEOUT}" "${targets[@]}" \
"${testargs[@]: ${testargs[@]}}" \
| tee ${junit_filename_prefix: "${junit_filename_prefix}.stdout"} \
| grep --binary-files=text "${go_test_grep_pattern}" && rc=$? || rc=$?
produceJUnitXMLReport "${junit_filename_prefix}"
return "${rc}"
# Enable coverage data collection?
local cover_msg
local COMBINED_COVER_PROFILE

if [[ ${KUBE_COVER} =~ ^[yY]$ ]]; then
cover_msg="with code coverage"
if [[ -z "${KUBE_COVER_REPORT_DIR}" ]]; then
cover_report_dir="/tmp/k8s_coverage/$(kube::util::sortable_date)"
else
cover_report_dir="${KUBE_COVER_REPORT_DIR}"
fi
kube::log::status "Saving coverage output in '${cover_report_dir}'"
mkdir -p "${@ ${@/#/${cover_report_dir}/}}"
COMBINED_COVER_PROFILE="${cover_report_dir}/combined-coverage.out"
goflags =(-cover -covermode="${KUBE_COVERMODE}" -coverprofile="${COMBINED_COVER_PROFILE}")
else
cover_msg="without code coverage"
fi

# Keep the raw JSON output in addition to the JUnit file?
local jsonfile=""
if [[ -n "${junit_filename_prefix}" ]] && [[ ${KUBE_KEEP_VERBOSE_TEST_OUTPUT} =~ ^[yY]$ ]]; then
jsonfile="${junit_filename_prefix}.stdout"
fi

kube::log::status "Running tests with code coverage ${KUBE_RACE: "and with ${KUBE_RACE}"}"
kube::log::status "Running tests ${cover_msg} ${KUBE_RACE: "and with ${KUBE_RACE}"}"
gotestsum --format="${gotestsum_format}" \
--jsonfile="${jsonfile}" \
--junitfile="${junit_filename_prefix: "${junit_filename_prefix}.xml"}" \
--raw-command \
-- \
go test -json \
"${goflags[@]: ${goflags[@]}}" \
"${KUBE_TIMEOUT}" \
"${targets[@]}" \
"${testargs[@]: ${testargs[@]}}" \
&& rc=$? || rc=$?

if [[ -n "${junit_filename_prefix}" ]]; then
prune-junit-xml "${junit_filename_prefix}.xml"
fi

# Create coverage report directories.
if [[ -z "${KUBE_COVER_REPORT_DIR}" ]]; then
cover_report_dir="/tmp/k8s_coverage/$(kube::util::sortable_date)"
else
cover_report_dir="${KUBE_COVER_REPORT_DIR}"
if [[ ${KUBE_COVER} =~ ^[yY]$ ]]; then
coverage_html_file="${cover_report_dir}/combined-coverage.html"
go tool cover -html="${COMBINED_COVER_PROFILE}" -o="${coverage_html_file}"
kube::log::status "Combined coverage report: ${coverage_html_file}"
fi
cover_profile="coverage.out" # Name for each individual coverage profile
kube::log::status "Saving coverage output in '${cover_report_dir}'"
mkdir -p "${@ ${@/#/${cover_report_dir}/}}"

# Run all specified tests, collecting coverage results. Go currently doesn't
# support collecting coverage across multiple packages at once, so we must issue
# separate 'go test' commands for each package and then combine at the end.
# To speed things up considerably, we can at least use xargs -P to run multiple
# 'go test' commands at once.
# To properly parse the test results if generating a JUnit test report, we
# must make sure the output from PARALLEL runs is not mixed. To achieve this,
# we spawn a subshell for each PARALLEL process, redirecting the output to
# separate files.

printf "%s\n" "${@}" \
| xargs -I{} -n 1 -P "${KUBE_COVERPROCS}" \
bash -c "set -o pipefail; _pkg=\"\$0\"; _pkg_out=\${_pkg//\//_}; \
go test ${goflags[*]: ${goflags[*]}} \
${KUBE_TIMEOUT} \
-cover -covermode=\"${KUBE_COVERMODE}\" \
-coverprofile=\"${cover_report_dir}/\${_pkg}/${cover_profile}\" \
\"\${_pkg}\" \
${testargs[*]: ${testargs[*]}} \
| tee ${junit_filename_prefix: \"${junit_filename_prefix}-\$_pkg_out.stdout\"} \
| grep \"${go_test_grep_pattern}\"" \
{} \
&& test_result=$? || test_result=$?

produceJUnitXMLReport "${junit_filename_prefix}"

COMBINED_COVER_PROFILE="${cover_report_dir}/combined-coverage.out"
{
# The combined coverage profile needs to start with a line indicating which
# coverage mode was used (set, count, or atomic). This line is included in
# each of the coverage profiles generated when running 'go test -cover', but
# we strip these lines out when combining so that there's only one.
echo "mode: ${KUBE_COVERMODE}"

# Include all coverage reach data in the combined profile, but exclude the
# 'mode' lines, as there should be only one.
while IFS='' read -r x; do
grep -h -v "^mode:" < "${x}" || true
done < <(find "${cover_report_dir}" -name "${cover_profile}")
} >"${COMBINED_COVER_PROFILE}"

coverage_html_file="${cover_report_dir}/combined-coverage.html"
go tool cover -html="${COMBINED_COVER_PROFILE}" -o="${coverage_html_file}"
kube::log::status "Combined coverage report: ${coverage_html_file}"

return "${test_result}"

return "${rc}"
}

reportCoverageToCoveralls() {
Expand Down
10 changes: 4 additions & 6 deletions hack/tools/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 10,7 @@ require (
github.com/jcchavezs/porto v0.6.0
github.com/vektra/mockery/v2 v2.40.3
go.uber.org/automaxprocs v1.5.2
gotest.tools/gotestsum v1.6.4
gotest.tools/gotestsum v1.12.0
honnef.co/go/tools v0.5.1
sigs.k8s.io/logtools v0.8.1
)
Expand All @@ -35,6 35,7 @@ require (
github.com/ashanbrown/forbidigo v1.6.0 // indirect
github.com/ashanbrown/makezero v1.1.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bitfield/gotestdox v0.2.2 // indirect
github.com/bkielbasa/cyclop v1.2.1 // indirect
github.com/blizzy78/varnamelen v0.8.0 // indirect
github.com/bombsimon/wsl/v4 v4.2.1 // indirect
Expand All @@ -58,7 59,7 @@ require (
github.com/fatih/color v1.16.0 // indirect
github.com/fatih/structtag v1.2.0 // indirect
github.com/firefart/nonamedreturns v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/fzipp/gocyclo v0.6.0 // indirect
github.com/ghostiam/protogetter v0.3.4 // indirect
github.com/go-critic/go-critic v0.11.1 // indirect
Expand Down Expand Up @@ -101,7 102,6 @@ require (
github.com/jinzhu/copier v0.3.5 // indirect
github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
github.com/jjti/go-spancheck v0.5.2 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/julz/importas v0.1.0 // indirect
github.com/kisielk/errcheck v1.7.0 // indirect
github.com/kisielk/gotool v1.0.0 // indirect
Expand Down Expand Up @@ -133,7 133,6 @@ require (
github.com/nunnatsa/ginkgolinter v0.15.2 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polyfloyd/go-errorlint v1.4.8 // indirect
github.com/prometheus/client_golang v1.12.1 // indirect
Expand Down Expand Up @@ -189,13 188,12 @@ require (
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/crypto v0.19.0 // indirect
golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
golang.org/x/exp/typeparams v0.0.0-20231219180239-dc181d75b848 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/term v0.17.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.21.1-0.20240531212143-b6235391adb3 // indirect
google.golang.org/protobuf v1.34.2 // indirect
Expand Down
Loading

0 comments on commit a7da865

Please sign in to comment.