diff --git a/.gitlab-ci/test-docker-meson.sh b/.gitlab-ci/test-docker-meson.sh
index a116839a9d..26abe0fd86 100755
--- a/.gitlab-ci/test-docker-meson.sh
+++ b/.gitlab-ci/test-docker-meson.sh
@@ -22,9 +22,22 @@ meson \
 cd _build
 ninja
 
+# Meson < 0.57 can't exclude suites in a test_setup() so we have to
+# explicitly leave out the failing and flaky suites.
 xvfb-run -a -s "-screen 0 1024x768x24" \
     meson test \
         --timeout-multiplier 4 \
         --print-errorlogs \
         --suite=gtk+-3.0 \
-        --no-suite=gtk+-3.0:a11y
+        --no-suite=flaky \
+        --no-suite=failing
+
+# We run the flaky and failing tests to get them reported in the CI logs,
+# but if they fail (which we expect they often will), that isn't an error.
+xvfb-run -a -s "-screen 0 1024x768x24" \
+    meson test \
+        --timeout-multiplier 4 \
+        --print-errorlogs \
+        --suite=flaky \
+        --suite=failing \
+    || true
diff --git a/testsuite/a11y/meson.build b/testsuite/a11y/meson.build
index fa8b045030..85d0f5d467 100644
--- a/testsuite/a11y/meson.build
+++ b/testsuite/a11y/meson.build
@@ -66,7 +66,7 @@ foreach t: a11y_state_tests
         'GSETTINGS_SCHEMA_DIR=@0@'.format(gtk_schema_build_dir),
         'GTK_TEST_MESON=1',
       ],
-      suite: 'a11y',
+      suite: ['a11y', 'flaky'],
     )
   endif
 endforeach
@@ -100,7 +100,7 @@ foreach t: a11y_tests
                'G_TEST_BUILDDIR=@0@'.format(meson.current_build_dir()),
                'GSETTINGS_SCHEMA_DIR=@0@'.format(gtk_schema_build_dir),
              ],
-        suite: 'a11y')
+        suite: ['a11y', 'flaky'])
 endforeach
 
 installed_test_data = [
diff --git a/testsuite/gtk/meson.build b/testsuite/gtk/meson.build
index 206af0dc2c..137f93a626 100644
--- a/testsuite/gtk/meson.build
+++ b/testsuite/gtk/meson.build
@@ -50,7 +50,9 @@ tests = [
   ['revealer-size'],
 ]
 
-# Tests that are expected to fail
+# Tests that are expected to fail, sometimes or always
+flaky = [
+]
 xfail = [
 ]
 
@@ -83,7 +85,15 @@ foreach t : tests
     install: get_option('installed_tests'),
     install_dir: installed_test_bindir)
 
-  expect_fail = xfail.contains(test_name)
+  suites = ['gtk']
+
+  if flaky.contains(test_name)
+    suites += 'flaky'
+  endif
+
+  if xfail.contains(test_name)
+    suites += 'failing'
+  endif
 
   test(test_name, test_exe,
        args: [ '--tap', '-k' ],
@@ -97,8 +107,7 @@ foreach t : tests
               'GSETTINGS_SCHEMA_DIR=@0@'.format(gtk_schema_build_dir),
               'GTK_TEST_MESON=1',
             ],
-       suite: 'gtk',
-       should_fail: expect_fail,
+       suite: suites,
   )
 endforeach
 
diff --git a/testsuite/meson.build b/testsuite/meson.build
index 289f2725e1..032baf2979 100644
--- a/testsuite/meson.build
+++ b/testsuite/meson.build
@@ -2,6 +2,13 @@ gtk_libexecdir = join_paths(gtk_prefix, get_option('libexecdir'))
 installed_test_bindir = join_paths(gtk_libexecdir, 'installed-tests', 'gtk+')
 installed_test_datadir = join_paths(gtk_datadir, 'installed-tests', 'gtk+')
 
+if meson.version().version_compare('>=0.57.0')
+  add_test_setup('default',
+    is_default: true,
+    exclude_suites: ['flaky', 'failing'],
+  )
+endif
+
 subdir('gtk')
 subdir('gdk')
 subdir('css')
diff --git a/testsuite/reftests/meson.build b/testsuite/reftests/meson.build
index 2135ebb434..8c3c3e8215 100644
--- a/testsuite/reftests/meson.build
+++ b/testsuite/reftests/meson.build
@@ -419,7 +419,7 @@ test_data = [
 ]
 
 # Depending on the environment these fail, feel free to fix them
-somehow_broken = [
+flaky = [
   'button-wrapping.ui',
   'cellrenderer-pixbuf-stock-rtl.ui',
   'label-sizing.ui',
@@ -428,9 +428,21 @@ somehow_broken = [
   'symbolic-icon-translucent-color.ui',
   'window-height-for-width.ui',
 ]
+xfail = [
+]
 
 foreach testname : test_data
   if testname.endswith('.ui') and not testname.endswith('.ref.ui')
+    suites = ['reftest']
+
+    if flaky.contains(testname)
+      suites += 'flaky'
+    endif
+
+    if xfail.contains(testname)
+      suites += 'failing'
+    endif
+
     # reftests fail when multiple windows open at the same time stealing the focus,
     # so set is_parallel to false
     test('reftest ' + testname, gtk_reftest,
@@ -450,8 +462,7 @@ foreach testname : test_data
                 'G_TEST_BUILDDIR=@0@'.format(meson.current_build_dir()),
                 'REFTEST_MODULE_DIR=@0@'.format(meson.current_build_dir()),
               ],
-         suite: 'reftest',
-         should_fail: somehow_broken.contains(testname),
+         suite: suites,
          is_parallel: false)
   endif
 endforeach