gitlab: add optional job to run flaky avocado tests

One problem with flaky tests is they often only fail under CI conditions which makes it hard to debug. We add an optional allow_fail job so developers can trigger the only the flaky tests in the CI environment if they are debugging. Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Acked-by: Stefan Hajnoczi <stefanha@redhat.com> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Message-Id: <20231201093633.2551497-8-alex.bennee@linaro.org>
3 years ago · 5d25fcb702
14 changed files with 57 additions and 7 deletions
--- a/.gitlab-ci.d/buildtest.yml
+++ b/.gitlab-ci.d/buildtest.yml
@ -217,6 +217,36 @@ avocado-system-opensuse:
    MAKE_CHECK_ARGS: check-avocado
    AVOCADO_TAGS: arch:s390x arch:x86_64 arch:aarch64

+#
+# Flaky tests. We don't run these by default and they are allow fail
+# but often the CI system is the only way to trigger the failures.
+#
+
+build-system-flaky:
+  extends:
+    - .native_build_job_template
+    - .native_build_artifact_template
+  needs:
+    job: amd64-debian-container
+  variables:
+    IMAGE: debian
+    QEMU_JOB_OPTIONAL: 1
+    TARGETS: aarch64-softmmu arm-softmmu mips64el-softmmu
+      ppc64-softmmu rx-softmmu s390x-softmmu sh4-softmmu x86_64-softmmu
+    MAKE_CHECK_ARGS: check-build
+
+avocado-system-flaky:
+  extends: .avocado_test_job_template
+  needs:
+    - job: build-system-flaky
+      artifacts: true
+  allow_failure: true
+  variables:
+    IMAGE: debian
+    MAKE_CHECK_ARGS: check-avocado
+    QEMU_JOB_OPTIONAL: 1
+    QEMU_TEST_FLAKY_TESTS: 1
+    AVOCADO_TAGS: flaky

 # This jobs explicitly disable TCG (--disable-tcg), KVM is detected by
 # the configure script. The container doesn't contain Xen headers so
--- a/docs/devel/testing.rst
+++ b/docs/devel/testing.rst
@ -1387,16 +1387,17 @@ the code snippet below:
  def test(self):
      do_something()

-Tests should not live in this state forever and should either be fixed
-or eventually removed.
-
-To run such tests locally you will need to set the environment
-variable. For example:
+You can also add ``:avocado: tags=flaky`` to the test meta-data so
+only the flaky tests can be run as a group:

 .. code::

-   env QEMU_TEST_FLAKY_TESTS=1 ./pyvenv/bin/avocado run \
-      tests/avocado/boot_linux.py:BootLinuxPPC64.test_pseries_tcg
+   env QEMU_TEST_FLAKY_TESTS=1 ./pyvenv/bin/avocado \
+      run tests/avocado -filter-by-tags=flaky
+
+Tests should not live in this state forever and should either be fixed
+or eventually removed.
+

 Uninstalling Avocado
 ~~~~~~~~~~~~~~~~~~~~
--- a/tests/avocado/boot_linux.py
+++ b/tests/avocado/boot_linux.py
@ -99,6 +99,7 @@ class BootLinuxPPC64(LinuxTest):
        """
        :avocado: tags=machine:pseries
        :avocado: tags=accel:tcg
+        :avocado: tags=flaky
        """
        self.require_accelerator("tcg")
        self.vm.add_args("-accel", "tcg")
@ -118,6 +119,7 @@ class BootLinuxS390X(LinuxTest):
        """
        :avocado: tags=machine:s390-ccw-virtio
        :avocado: tags=accel:tcg
+        :avocado: tags=flaky
        """
        self.require_accelerator("tcg")
        self.vm.add_args("-accel", "tcg")
--- a/tests/avocado/boot_linux_console.py
+++ b/tests/avocado/boot_linux_console.py
@ -1425,6 +1425,7 @@ class BootLinuxConsole(LinuxKernelTest):
        """
        :avocado: tags=arch:sh4
        :avocado: tags=machine:r2d
+        :avocado: tags=flaky
        """
        tar_hash = 'fe06a4fd8ccbf2e27928d64472939d47829d4c7e'
        self.vm.add_args('-append', 'console=ttySC1')
--- a/tests/avocado/intel_iommu.py
+++ b/tests/avocado/intel_iommu.py
@ -22,6 +22,7 @@ class IntelIOMMU(LinuxTest):
    :avocado: tags=machine:q35
    :avocado: tags=accel:kvm
    :avocado: tags=intel_iommu
+    :avocado: tags=flaky
    """

    IOMMU_ADDON = ',iommu_platform=on,disable-modern=off,disable-legacy=on'
--- a/tests/avocado/linux_initrd.py
+++ b/tests/avocado/linux_initrd.py
@ -57,6 +57,8 @@ class LinuxInitrd(QemuSystemTest):

    def test_with_2gib_file_should_work_with_linux_v4_16(self):
        """
+        :avocado: tags=flaky
+
        QEMU has supported up to 4 GiB initrd for recent kernel
        Expect guest can reach 'Unpacking initramfs...'
        """
--- a/tests/avocado/machine_aspeed.py
+++ b/tests/avocado/machine_aspeed.py
@ -317,6 +317,7 @@ class AST2x00MachineSDK(QemuSystemTest, LinuxSSHMixIn):
        """
        :avocado: tags=arch:arm
        :avocado: tags=machine:ast2500-evb
+        :avocado: tags=flaky
        """

        image_url = ('https://github.com/AspeedTech-BMC/openbmc/releases/'
@ -336,6 +337,7 @@ class AST2x00MachineSDK(QemuSystemTest, LinuxSSHMixIn):
        """
        :avocado: tags=arch:arm
        :avocado: tags=machine:ast2600-evb
+        :avocado: tags=flaky
        """

        image_url = ('https://github.com/AspeedTech-BMC/openbmc/releases/'
--- a/tests/avocado/machine_mips_malta.py
+++ b/tests/avocado/machine_mips_malta.py
@ -109,6 +109,7 @@ class MaltaMachineFramebuffer(QemuSystemTest):
        :avocado: tags=machine:malta
        :avocado: tags=cpu:I6400
        :avocado: tags=mips:smp
+        :avocado: tags=flaky
        """
        self.do_test_i6400_framebuffer_logo(7)

@ -120,6 +121,7 @@ class MaltaMachineFramebuffer(QemuSystemTest):
        :avocado: tags=machine:malta
        :avocado: tags=cpu:I6400
        :avocado: tags=mips:smp
+        :avocado: tags=flaky
        """
        self.do_test_i6400_framebuffer_logo(8)

--- a/tests/avocado/machine_rx_gdbsim.py
+++ b/tests/avocado/machine_rx_gdbsim.py
@ -31,6 +31,7 @@ class RxGdbSimMachine(QemuSystemTest):
        :avocado: tags=arch:rx
        :avocado: tags=machine:gdbsim-r5f562n8
        :avocado: tags=endian:little
+        :avocado: tags=flaky
        """
        uboot_url = ('https://acc.dl.osdn.jp/users/23/23888/u-boot.bin.gz')
        uboot_hash = '9b78dbd43b40b2526848c0b1ce9de02c24f4dcdb'
@ -56,6 +57,7 @@ class RxGdbSimMachine(QemuSystemTest):
        :avocado: tags=arch:rx
        :avocado: tags=machine:gdbsim-r5f562n7
        :avocado: tags=endian:little
+        :avocado: tags=flaky
        """
        dtb_url = ('https://acc.dl.osdn.jp/users/23/23887/rx-virt.dtb')
        dtb_hash = '7b4e4e2c71905da44e86ce47adee2210b026ac18'
--- a/tests/avocado/machine_s390_ccw_virtio.py
+++ b/tests/avocado/machine_s390_ccw_virtio.py
@ -167,6 +167,7 @@ class S390CCWVirtioMachine(QemuSystemTest):
        :avocado: tags=device:virtio-gpu
        :avocado: tags=device:virtio-crypto
        :avocado: tags=device:virtio-net
+        :avocado: tags=flaky
        """

        kernel_url = ('https://archives.fedoraproject.org/pub/archive'
--- a/tests/avocado/replay_kernel.py
+++ b/tests/avocado/replay_kernel.py
@ -88,6 +88,7 @@ class ReplayKernelNormal(ReplayKernelBase):
        """
        :avocado: tags=arch:x86_64
        :avocado: tags=machine:pc
+        :avocado: tags=flaky
        """
        kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
                      '/linux/releases/29/Everything/x86_64/os/images/pxeboot'
@ -186,6 +187,7 @@ class ReplayKernelNormal(ReplayKernelBase):
        """
        :avocado: tags=arch:arm
        :avocado: tags=machine:cubieboard
+        :avocado: tags=flaky
        """
        deb_url = ('https://apt.armbian.com/pool/main/l/'
                   'linux-5.10.16-sunxi/linux-image-current-sunxi_21.02.2_armhf.deb')
--- a/tests/avocado/reverse_debugging.py
+++ b/tests/avocado/reverse_debugging.py
@ -255,6 +255,7 @@ class ReverseDebugging_ppc64(ReverseDebugging):
        """
        :avocado: tags=arch:ppc64
        :avocado: tags=machine:pseries
+        :avocado: tags=flaky
        """
        # SLOF branches back to its entry point, which causes this test
        # to take the 'hit a breakpoint again' path. That's not a problem,
@ -269,6 +270,7 @@ class ReverseDebugging_ppc64(ReverseDebugging):
        """
        :avocado: tags=arch:ppc64
        :avocado: tags=machine:powernv
+        :avocado: tags=flaky
        """
        self.endian_is_le = False
        self.reverse_debugging()
--- a/tests/avocado/smmu.py
+++ b/tests/avocado/smmu.py
@ -22,6 +22,7 @@ class SMMU(LinuxTest):
    :avocado: tags=machine:virt
    :avocado: tags=distro:fedora
    :avocado: tags=smmu
+    :avocado: tags=flaky
    """

    IOMMU_ADDON = ',iommu_platform=on,disable-modern=off,disable-legacy=on'
--- a/tests/avocado/tuxrun_baselines.py
+++ b/tests/avocado/tuxrun_baselines.py
@ -561,6 +561,7 @@ class TuxRunBaselineTest(QemuSystemTest):
        :avocado: tags=image:zImage
        :avocado: tags=root:sda
        :avocado: tags=console:ttySC1
+        :avocado: tags=flaky
        """
        sums = { "rootfs.ext4.zst" :
                 "3592a7a3d5a641e8b9821449e77bc43c9904a56c30d45da0694349cfd86743fd",