Skip to content

Commit 7802596

Browse files
HTRamseyHTRamsey
committed
feat(test): Add CTest infrastructure and improve UnitTestFramework
Comprehensive test infrastructure improvements for parallel test execution: ## CTest Integration - Add cmake/QGCTest.cmake with add_qgc_test() function - Support labels (Unit, Integration, Slow, Network, etc.) - Configurable timeouts per test category - RESOURCE_LOCK support for tests that can't run in parallel - Convenience targets: check, check-unit, check-integration, check-fast ## Parallel Test Fixes - Add --allow-multiple flag to bypass RunGuard single-instance check - Use unique application name per test (includes test name) - Isolate parameter cache directory per test process - Fix VehicleLinkManagerTest race conditions in signal handling ## New CMake Modules - cmake/Coverage.cmake: gcov/lcov integration with coverage-report target - cmake/Sanitizers.cmake: ASan/TSan/UBSan/MSan support ## UnitTestFramework Improvements - Enhanced MultiSignalSpy with fluent API and better diagnostics - TestContext for hierarchical test state tracking - TestDebug for capturing debug output during failures - RAII fixtures for common test patterns - Base test classes: CommsTest, ParameterTest, TerrainTest, FTPTest
1 parent cd0ead7 commit 7802596

File tree

242 files changed

+13147
-7989
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

242 files changed

+13147
-7989
lines changed

‎.clang-format‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ IndentWidth: 4
1111
TabWidth: 4
1212
UseTab: Never
1313
ContinuationIndentWidth: 4
14+
AccessModifierOffset: -4
1415

1516
# Bracing - Custom QGC style
1617
# Functions/Classes: New line (Allman)
@@ -73,7 +74,7 @@ AlignTrailingComments:
7374

7475
# Line breaks
7576
AlwaysBreakAfterReturnType: None
76-
AlwaysBreakTemplateDeclarations: Yes
77+
AlwaysBreakTemplateDeclarations: true
7778
BreakBeforeBinaryOperators: None
7879
BreakBeforeTernaryOperators: true
7980
BreakConstructorInitializers: BeforeColon
@@ -90,4 +91,5 @@ BinPackParameters: true
9091
KeepEmptyLinesAtTheStartOfBlocks: false
9192
MaxEmptyLinesToKeep: 1
9293
ReflowComments: true
94+
SeparateDefinitionBlocks: Always
9395
Standard: c++20

‎.github/workflows/linux.yml‎

Lines changed: 23 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -140,46 +140,47 @@ jobs:
140140
# with:
141141
# category: "/language:c-cpp"
142142

143-
# NOTE: QGC's command line parser doesn't support Qt Test -o options.
144-
# The test runner (UnitTest::run) uses hardcoded args via QTest::qExec().
145-
# JUnit XML output would require modifying src/Utilities/QGCCommandLineParser.cc
146-
# to forward -o options to the test framework.
147-
# xvfb-run: Qt GUI apps require a display server; -a auto-selects free display
143+
# Run tests via CTest for timeouts, parallel execution, and failure detection
144+
# CTest sets QT_QPA_PLATFORM=offscreen automatically via test properties
148145
- name: Run Unit Tests
149146
if: matrix.build_type == 'Debug'
150147
id: tests
151-
working-directory: ${{ runner.temp }}/build/Debug
148+
working-directory: ${{ runner.temp }}/build
152149
run: |
153150
set +e
154-
# TODO: Add JUnit output when QGC supports Qt Test -o options (see note above)
155-
# xvfb-run -a ./QGroundControl --unittest \
156-
# -o ../junit-results.xml,junitxml \
157-
# -o -,txt 2>&1 | tee ../test-output.txt
158-
xvfb-run -a ./QGroundControl --unittest 2>&1 | tee ../test-output.txt
151+
ctest --output-on-failure --parallel $(nproc) -LE Flaky 2>&1 | tee test-output.txt
159152
TEST_EXIT_CODE=${PIPESTATUS[0]}
160153
echo "exit_code=$TEST_EXIT_CODE" >> $GITHUB_OUTPUT
161154
exit $TEST_EXIT_CODE
162155
156+
# Generate JUnit XML for CI integration (runs all tests sequentially)
157+
- name: Generate JUnit XML Results
158+
if: matrix.build_type == 'Debug' && always()
159+
working-directory: ${{ runner.temp }}/build
160+
run: |
161+
export QT_QPA_PLATFORM=offscreen
162+
export QT_LOGGING_RULES="*=false"
163+
./QGroundControl --unittest --unittest-output:junit-results.xml || true
164+
continue-on-error: true
165+
163166
- name: Upload Test Results
164167
if: matrix.build_type == 'Debug' && always()
165168
uses: actions/upload-artifact@v6
166169
with:
167170
name: test-results-${{ matrix.arch }}
168-
# TODO: Add junit-results.xml when QGC supports Qt Test -o options
169171
path: |
170172
${{ runner.temp }}/build/test-output.txt
171-
# ${{ runner.temp }}/build/junit-results.xml
173+
${{ runner.temp }}/build/junit-results-*.xml
172174
retention-days: 7
173175

174-
# TODO: Enable when JUnit XML output is supported (see note above)
175-
# - name: Upload to Trunk Flaky Tests
176-
# if: matrix.build_type == 'Debug' && always() && github.event_name != 'pull_request'
177-
# continue-on-error: true
178-
# uses: trunk-io/analytics-uploader@v1
179-
# with:
180-
# junit-paths: ${{ runner.temp }}/build/junit-results.xml
181-
# org-slug: ${{ vars.TRUNK_ORG_SLUG }}
182-
# token: ${{ secrets.TRUNK_TOKEN }}
176+
- name: Upload to Trunk Flaky Tests
177+
if: matrix.build_type == 'Debug' && always() && github.event_name != 'pull_request'
178+
continue-on-error: true
179+
uses: trunk-io/analytics-uploader@v1
180+
with:
181+
junit-paths: ${{ runner.temp }}/build/junit-results-*.xml
182+
org-slug: ${{ vars.TRUNK_ORG_SLUG }}
183+
token: ${{ secrets.TRUNK_TOKEN }}
183184

184185
- name: Coverage Report
185186
if: matrix.build_type == 'Debug'

‎CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,9 @@ project(${QGC_APP_NAME}
100100
include(GNUInstallDirs)
101101
include(FetchContent)
102102
include(CMakePrintHelpers)
103+
if(QGC_BUILD_TESTING)
104+
include(CTest)
105+
endif()
103106

104107
# ----------------------------------------------------------------------------
105108
# CPM (CMake Package Manager) Configuration

‎CTestCustom.cmake.in‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# ============================================================================
2+
# CTest Custom Configuration
3+
# ============================================================================
4+
# This file is configured by CMake and copied to the build directory.
5+
# It configures memory checking (Valgrind) and other CTest options.
6+
7+
# ----------------------------------------------------------------------------
8+
# Memory Checking Configuration
9+
# ----------------------------------------------------------------------------
10+
set(CTEST_MEMORYCHECK_COMMAND "@VALGRIND_EXECUTABLE@")
11+
set(CTEST_MEMORYCHECK_TYPE "Valgrind")
12+
13+
# Command options - use semicolons to create a proper CMake list
14+
set(CTEST_MEMORYCHECK_COMMAND_OPTIONS
15+
"--leak-check=full"
16+
"--show-leak-kinds=definite,possible"
17+
"--track-origins=yes"
18+
"--trace-children=yes"
19+
"--error-exitcode=0"
20+
"--num-callers=50"
21+
"--suppressions=@CMAKE_SOURCE_DIR@/tools/debuggers/valgrind.supp"
22+
)
23+
24+
# Suppressions file (also set for some CTest versions)
25+
set(CTEST_MEMORYCHECK_SUPPRESSIONS_FILE "@CMAKE_SOURCE_DIR@/tools/debuggers/valgrind.supp")
26+
27+
# ----------------------------------------------------------------------------
28+
# Test Filtering
29+
# ----------------------------------------------------------------------------
30+
# Tests to skip entirely
31+
set(CTEST_CUSTOM_TESTS_IGNORE
32+
# Add test names to skip here
33+
)
34+
35+
# Tests that are expected to fail (won't count as failures)
36+
set(CTEST_CUSTOM_MEMCHECK_IGNORE
37+
# Tests with known memory issues in third-party code
38+
)
39+
40+
# ----------------------------------------------------------------------------
41+
# Output Filtering
42+
# ----------------------------------------------------------------------------
43+
# Patterns that indicate warnings (but not failures)
44+
set(CTEST_CUSTOM_WARNING_MATCH
45+
"warning:"
46+
"Warning:"
47+
"WARNING:"
48+
)
49+
50+
# Patterns to ignore in warnings
51+
set(CTEST_CUSTOM_WARNING_EXCEPTION
52+
".*Qt.*"
53+
".*_deps.*"
54+
".*third_party.*"
55+
)
56+
57+
# Error patterns
58+
set(CTEST_CUSTOM_ERROR_MATCH
59+
"error:"
60+
"Error:"
61+
"ERROR:"
62+
"FAIL!"
63+
"Segmentation fault"
64+
"ASSERT"
65+
)
66+
67+
# Patterns to ignore in errors
68+
set(CTEST_CUSTOM_ERROR_EXCEPTION
69+
".*_deps.*"
70+
)
71+
72+
# ----------------------------------------------------------------------------
73+
# Coverage Configuration
74+
# ----------------------------------------------------------------------------
75+
set(CTEST_CUSTOM_COVERAGE_EXCLUDE
76+
".*_deps/.*"
77+
".*/test/.*"
78+
".*/build/.*"
79+
".*/Qt/.*"
80+
".*moc_.*"
81+
".*_autogen/.*"
82+
)

‎cmake/Coverage.cmake‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# ============================================================================
2+
# Code Coverage Configuration
3+
# ============================================================================
4+
# Provides coverage-report target for CI integration.
5+
# Requires gcov/lcov and debug build with coverage flags.
6+
7+
if(NOT QGC_BUILD_TESTING)
8+
return()
9+
endif()
10+
11+
# Check if coverage tools are available
12+
find_program(GCOV_PATH gcov)
13+
find_program(LCOV_PATH lcov)
14+
find_program(GENHTML_PATH genhtml)
15+
16+
# Only enable coverage on Debug builds with GCC/Clang
17+
set(_coverage_supported FALSE)
18+
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
19+
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
20+
if(GCOV_PATH AND LCOV_PATH AND GENHTML_PATH)
21+
set(_coverage_supported TRUE)
22+
endif()
23+
endif()
24+
endif()
25+
26+
if(_coverage_supported)
27+
message(STATUS "Code coverage: enabled (gcov + lcov)")
28+
29+
# Add coverage flags
30+
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} --coverage -fprofile-arcs -ftest-coverage")
31+
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} --coverage -fprofile-arcs -ftest-coverage")
32+
set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} --coverage")
33+
34+
# Coverage report target
35+
add_custom_target(coverage-report
36+
COMMAND ${LCOV_PATH} --capture --directory . --output-file coverage.info --ignore-errors mismatch
37+
COMMAND ${LCOV_PATH} --remove coverage.info '/usr/*' '*/Qt/*' '*/test/*' '*/build/*' --output-file coverage.info
38+
COMMAND ${GENHTML_PATH} coverage.info --output-directory coverage-report
39+
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
40+
COMMENT "Generating code coverage report..."
41+
VERBATIM
42+
)
43+
else()
44+
message(STATUS "Code coverage: disabled (missing tools or not Debug build)")
45+
46+
# Provide a no-op target so CI doesn't fail
47+
add_custom_target(coverage-report
48+
COMMAND ${CMAKE_COMMAND} -E echo "Coverage report skipped - not configured"
49+
COMMENT "Coverage report not available"
50+
VERBATIM
51+
)
52+
endif()

‎cmake/QGCTest.cmake‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
# ============================================================================
2+
# QGroundControl Test Infrastructure
3+
# ============================================================================
4+
# CTest integration with labels, timeouts, and resource locking.
5+
#
6+
# Usage:
7+
# include(QGCTest)
8+
# add_qgc_test(MyTest LABELS Unit)
9+
# add_qgc_test(MyIntegrationTest LABELS Integration RESOURCE_LOCK MockLink)
10+
11+
if(NOT QGC_BUILD_TESTING)
12+
return()
13+
endif()
14+
15+
# ----------------------------------------------------------------------------
16+
# Configuration
17+
# ----------------------------------------------------------------------------
18+
19+
cmake_host_system_information(RESULT _num_cores QUERY NUMBER_OF_LOGICAL_CORES)
20+
set(QGC_TEST_PARALLEL_LEVEL ${_num_cores} CACHE STRING "Number of parallel test jobs")
21+
22+
set(QGC_TEST_TIMEOUT_UNIT 60 CACHE STRING "Timeout for unit tests (seconds)")
23+
set(QGC_TEST_TIMEOUT_INTEGRATION 120 CACHE STRING "Timeout for integration tests (seconds)")
24+
set(QGC_TEST_TIMEOUT_SLOW 180 CACHE STRING "Timeout for slow tests (seconds)")
25+
set(QGC_TEST_TIMEOUT_DEFAULT 90 CACHE STRING "Default test timeout (seconds)")
26+
27+
# ----------------------------------------------------------------------------
28+
# Convenience Targets
29+
# ----------------------------------------------------------------------------
30+
31+
add_custom_target(check
32+
COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure --parallel ${QGC_TEST_PARALLEL_LEVEL}
33+
USES_TERMINAL
34+
COMMENT "Running all tests"
35+
)
36+
37+
add_custom_target(check-unit
38+
COMMAND ${CMAKE_CTEST_COMMAND} -L Unit --output-on-failure --parallel ${QGC_TEST_PARALLEL_LEVEL}
39+
USES_TERMINAL
40+
COMMENT "Running unit tests"
41+
)
42+
43+
add_custom_target(check-integration
44+
COMMAND ${CMAKE_CTEST_COMMAND} -L Integration --output-on-failure --parallel ${QGC_TEST_PARALLEL_LEVEL}
45+
USES_TERMINAL
46+
COMMENT "Running integration tests"
47+
)
48+
49+
add_custom_target(check-fast
50+
COMMAND ${CMAKE_CTEST_COMMAND} -LE Slow --output-on-failure --parallel ${QGC_TEST_PARALLEL_LEVEL}
51+
USES_TERMINAL
52+
COMMENT "Running fast tests (excluding Slow)"
53+
)
54+
55+
add_custom_target(check-ci
56+
COMMAND ${CMAKE_CTEST_COMMAND} -LE "Flaky|Network" --output-on-failure --parallel ${QGC_TEST_PARALLEL_LEVEL}
57+
USES_TERMINAL
58+
COMMENT "Running CI-safe tests"
59+
VERBATIM
60+
)
61+
62+
# Category-specific targets
63+
foreach(_category MissionManager Vehicle Utilities MAVLink Comms)
64+
string(TOLOWER ${_category} _target_suffix)
65+
add_custom_target(check-${_target_suffix}
66+
COMMAND ${CMAKE_CTEST_COMMAND} -L ${_category} --output-on-failure --parallel ${QGC_TEST_PARALLEL_LEVEL}
67+
USES_TERMINAL
68+
COMMENT "Running ${_category} tests"
69+
)
70+
endforeach()
71+
72+
# ----------------------------------------------------------------------------
73+
# add_qgc_test()
74+
# ----------------------------------------------------------------------------
75+
# Registers a test with CTest and configures properties.
76+
#
77+
# Arguments:
78+
# test_name - Name of the test (must match UnitTestList registration)
79+
# LABELS label... - Test labels for filtering (Unit, Integration, Slow, etc.)
80+
# TIMEOUT seconds - Override default timeout
81+
# RESOURCE_LOCK res.. - Resources that prevent parallel execution
82+
# SERIAL - Shorthand for locking all shared resources
83+
#
84+
# Labels:
85+
# Unit - Fast, isolated tests (~30s timeout)
86+
# Integration - Tests requiring MockLink/Vehicle (~60s timeout)
87+
# Slow - Long-running tests (~120s timeout)
88+
# Flaky - Tests with intermittent failures (excluded from CI)
89+
# Network - Tests requiring network access (excluded from CI)
90+
#
91+
# Example:
92+
# add_qgc_test(ParameterManagerTest LABELS Integration Vehicle SERIAL)
93+
94+
function(add_qgc_test test_name)
95+
cmake_parse_arguments(ARG "SERIAL" "TIMEOUT" "LABELS;RESOURCE_LOCK" ${ARGN})
96+
97+
add_test(
98+
NAME ${test_name}
99+
COMMAND $<TARGET_FILE:${PROJECT_NAME}> --unittest:${test_name} --allow-multiple
100+
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
101+
)
102+
103+
# Determine timeout based on labels or explicit value
104+
if(ARG_TIMEOUT)
105+
set(_timeout ${ARG_TIMEOUT})
106+
elseif("Slow" IN_LIST ARG_LABELS)
107+
set(_timeout ${QGC_TEST_TIMEOUT_SLOW})
108+
elseif("Integration" IN_LIST ARG_LABELS)
109+
set(_timeout ${QGC_TEST_TIMEOUT_INTEGRATION})
110+
elseif("Unit" IN_LIST ARG_LABELS)
111+
set(_timeout ${QGC_TEST_TIMEOUT_UNIT})
112+
else()
113+
set(_timeout ${QGC_TEST_TIMEOUT_DEFAULT})
114+
endif()
115+
116+
set_tests_properties(${test_name} PROPERTIES
117+
TIMEOUT ${_timeout}
118+
ENVIRONMENT "QT_QPA_PLATFORM=offscreen;QT_LOGGING_RULES=*.debug=false"
119+
FAIL_REGULAR_EXPRESSION "FAIL!;Segmentation fault;ASSERT"
120+
)
121+
122+
if(ARG_LABELS)
123+
set_tests_properties(${test_name} PROPERTIES LABELS "${ARG_LABELS}")
124+
endif()
125+
126+
# Resource locking for tests that can't run in parallel
127+
if(ARG_SERIAL)
128+
set_tests_properties(${test_name} PROPERTIES
129+
RESOURCE_LOCK "MockLink;Vehicle;ParameterManager;MissionController"
130+
RUN_SERIAL TRUE
131+
)
132+
elseif(ARG_RESOURCE_LOCK)
133+
set_tests_properties(${test_name} PROPERTIES RESOURCE_LOCK "${ARG_RESOURCE_LOCK}")
134+
elseif("Integration" IN_LIST ARG_LABELS)
135+
set_tests_properties(${test_name} PROPERTIES RESOURCE_LOCK "MockLink")
136+
endif()
137+
138+
# Add dependency so 'check' target builds first
139+
add_dependencies(check ${PROJECT_NAME})
140+
endfunction()

0 commit comments

Comments
 (0)