5794821592
With these changes, the flaky/known-failing tests are no longer installed as installed-tests at all, so remove them from the autopkgtest metadata.
94 lines
3.2 KiB
Diff
94 lines
3.2 KiB
Diff
From: Simon McVittie <smcv@debian.org>
|
|
Date: Wed, 23 Nov 2022 21:26:50 +0000
|
|
Subject: testsuite: Don't create .test files for flaky or failing tests
|
|
|
|
These tests can be run manually, but are not suitable for use as an
|
|
acceptance test, so let's not make frameworks like Debian's autopkgtest
|
|
run these when they run ginsttest-runner in the most obvious way.
|
|
|
|
a11ytests.test doesn't seem to be reliable enough to be used as a QA
|
|
acceptance criterion, and has been disabled as a build-time test in both
|
|
Gitlab-CI and Debian since 2019. a11ystate.test is not set up to be run
|
|
at build time at all, and has been marked as flaky on ci.debian.net
|
|
since 2018.
|
|
|
|
The rest of the testsuite/a11y directory seems to have been
|
|
reliable in practice, at least on ci.debian.net, so try leaving them
|
|
enabled as installed-tests.
|
|
|
|
In principle this could be made finer-grained by having a separate .test
|
|
file and a separate Meson test() for each .ui file, but that would
|
|
require more active maintenance of GTK 3.
|
|
|
|
Signed-off-by: Simon McVittie <smcv@debian.org>
|
|
---
|
|
testsuite/a11y/meson.build | 7 ++++++-
|
|
testsuite/a11y/state/meson.build | 7 +++++--
|
|
testsuite/gtk/meson.build | 7 ++++++-
|
|
3 files changed, 17 insertions(+), 4 deletions(-)
|
|
|
|
diff --git a/testsuite/a11y/meson.build b/testsuite/a11y/meson.build
|
|
index ea6348c..2613e3c 100644
|
|
--- a/testsuite/a11y/meson.build
|
|
+++ b/testsuite/a11y/meson.build
|
|
@@ -181,7 +181,12 @@ installed_test_data = [
|
|
]
|
|
|
|
a11y_installed_tests = [
|
|
- 'a11ytests.test',
|
|
+ # This is the equivalent of a11y_state_tests above, and does not seem
|
|
+ # to be reliable enough to act as a QA gate in practice. We install the
|
|
+ # test executable and the data needed to run it, but don't hook it up
|
|
+ # to ginsttest-runner.
|
|
+ #'a11ytests.test',
|
|
+
|
|
'a11ychildren.test',
|
|
'a11ytree.test',
|
|
'a11yvalue.test',
|
|
diff --git a/testsuite/a11y/state/meson.build b/testsuite/a11y/state/meson.build
|
|
index 782649e..aa18a84 100644
|
|
--- a/testsuite/a11y/state/meson.build
|
|
+++ b/testsuite/a11y/state/meson.build
|
|
@@ -16,7 +16,10 @@ test_data = [
|
|
|
|
|
|
a11y_installed_tests = [
|
|
- 'a11ystate.test',
|
|
+ # This is not run at build time at all, and consistently fails on
|
|
+ # Debian's CI infrastructure, so don't set it up to be run by
|
|
+ # ginsttest-runner.
|
|
+ #'a11ystate.test',
|
|
]
|
|
|
|
if get_option('installed_tests')
|
|
@@ -32,4 +35,4 @@ if get_option('installed_tests')
|
|
endforeach
|
|
|
|
install_data(test_data, install_dir: join_paths(installed_test_bindir, 'state'))
|
|
-endif
|
|
\ No newline at end of file
|
|
+endif
|
|
diff --git a/testsuite/gtk/meson.build b/testsuite/gtk/meson.build
|
|
index 137f93a..84a8cfc 100644
|
|
--- a/testsuite/gtk/meson.build
|
|
+++ b/testsuite/gtk/meson.build
|
|
@@ -146,6 +146,11 @@ endif
|
|
if get_option('installed_tests')
|
|
foreach t : tests
|
|
test_name = t.get(0)
|
|
+
|
|
+ if flaky.contains(test_name) or xfail.contains(test_name)
|
|
+ continue
|
|
+ endif
|
|
+
|
|
conf = configuration_data()
|
|
conf.set('testexecdir', installed_test_bindir)
|
|
conf.set('test', test_name)
|
|
@@ -158,4 +163,4 @@ if get_option('installed_tests')
|
|
install_subdir('icons', install_dir: installed_test_bindir)
|
|
install_subdir('icons2', install_dir: installed_test_bindir)
|
|
install_subdir('ui', install_dir: installed_test_bindir)
|
|
-endif
|
|
\ No newline at end of file
|
|
+endif
|