diff --git a/tests/.fmf/version b/.fmf/version
similarity index 100%
rename from tests/.fmf/version
rename to .fmf/version
diff --git a/llvm.spec b/llvm.spec
index 25e8ad0..2cb4b6a 100644
--- a/llvm.spec
+++ b/llvm.spec
@@ -243,7 +243,7 @@ pathfix.py -i %{__python3} -pn \
 	\
 	-DLLVM_INCLUDE_TESTS:BOOL=ON \
 	-DLLVM_BUILD_TESTS:BOOL=ON \
-    -DLLVM_LIT_EXTRA_ARGS=-v \
+	-DLLVM_LIT_EXTRA_ARGS=-v \
 	\
 	-DLLVM_INCLUDE_EXAMPLES:BOOL=ON \
 	-DLLVM_BUILD_EXAMPLES:BOOL=OFF \
diff --git a/tests/binutils-plugin-ar/main.fmf b/tests/binutils-plugin-ar/main.fmf
new file mode 100644
index 0000000..a8b6ebf
--- /dev/null
+++ b/tests/binutils-plugin-ar/main.fmf
@@ -0,0 +1,2 @@
+summary: Make sure the the LLVM plugin for binutils' ar works
+test: ./test.sh
diff --git a/tests/binutils-plugin-ar/runtest.sh b/tests/binutils-plugin-ar/test.sh
similarity index 80%
rename from tests/binutils-plugin-ar/runtest.sh
rename to tests/binutils-plugin-ar/test.sh
index d18ab53..5dbacec 100755
--- a/tests/binutils-plugin-ar/runtest.sh
+++ b/tests/binutils-plugin-ar/test.sh
@@ -1,4 +1,6 @@
-set -exo pipefail
+#!/bin/sh -eux
+
+set pipefail
 
 echo "void lto_function(){}" | clang -flto -O2 -c -x c -o foo.o -
 ar crs foo.a foo.o
diff --git a/tests/build-compat/main.fmf b/tests/build-compat/main.fmf
new file mode 100644
index 0000000..fe61457
--- /dev/null
+++ b/tests/build-compat/main.fmf
@@ -0,0 +1,3 @@
+summary: Make sure rebuilding LLVM still works with the new LLVM
+test: ./test.sh
+enabled: false
diff --git a/tests/build-compat/runtest.sh b/tests/build-compat/test.sh
similarity index 93%
rename from tests/build-compat/runtest.sh
rename to tests/build-compat/test.sh
index 9d0a02d..04a97b2 100755
--- a/tests/build-compat/runtest.sh
+++ b/tests/build-compat/test.sh
@@ -1,6 +1,4 @@
-#!/bin/bash
-
-set -ex
+#!/bin/sh -eux
 
 dnf download --disablerepo=* --enablerepo=test-llvm --source llvm
 
diff --git a/tests/default.fmf b/tests/default.fmf
new file mode 100644
index 0000000..9f30dd3
--- /dev/null
+++ b/tests/default.fmf
@@ -0,0 +1,9 @@
+summary: All available LLVM tests
+discover:
+    how: fmf
+execute:
+    how: tmt
+prepare:
+    - name: packages
+      how: install
+      package: llvm
diff --git a/tests/integration-test-suite/main.fmf b/tests/integration-test-suite/main.fmf
new file mode 100644
index 0000000..3e6fc4d
--- /dev/null
+++ b/tests/integration-test-suite/main.fmf
@@ -0,0 +1,19 @@
+summary: Run the upstream LLVM integration test suite
+test: ./test.sh
+require:
+  - cmake
+  - llvm-devel
+  - clang
+  - clang-analyzer
+  - clang-tools-extra
+  - compiler-rt
+  - ninja-build
+  - libcxx-devel
+  - libomp-devel
+  - python-lit
+  - lld
+  - lldb
+  - git
+  - make
+  - libstdc++-static
+  - clang-devel
diff --git a/tests/integration-test-suite/runtest.sh b/tests/integration-test-suite/test.sh
similarity index 92%
rename from tests/integration-test-suite/runtest.sh
rename to tests/integration-test-suite/test.sh
index 1af4e4a..682e4f0 100755
--- a/tests/integration-test-suite/runtest.sh
+++ b/tests/integration-test-suite/test.sh
@@ -1,10 +1,11 @@
-#!/bin/bash
+#!/bin/sh -eux
 
 usage() {
     echo "usage `basename $0` [OPTIONS]"
     echo "  --threads NUM       The number of threads to use for running tests."
 }
 
+threads=0
 thread_args=""
 repo_url=https://github.com/opencollab/llvm-toolchain-integration-test-suite
 
@@ -24,12 +25,10 @@ while [ $# -gt 0 ]; do
     shift
 done
 
-if [ -n "$threads" ]; then
+if [ "$threads" -ge 1 ]; then
   thread_args="-j$threads"
 fi
 
-set -xe
-
 cd $(mktemp -d -p /var/tmp)
 git clone $repo_url
 cd llvm-toolchain-integration-test-suite
diff --git a/tests/libllvm-size/main.fmf b/tests/libllvm-size/main.fmf
new file mode 100644
index 0000000..f03b183
--- /dev/null
+++ b/tests/libllvm-size/main.fmf
@@ -0,0 +1,2 @@
+summary: Check that libLLVM.so has been properly stripped
+test: ./test.sh
diff --git a/tests/libllvm-size/test.sh b/tests/libllvm-size/test.sh
new file mode 100755
index 0000000..7ceb8cb
--- /dev/null
+++ b/tests/libllvm-size/test.sh
@@ -0,0 +1,10 @@
+#!/bin/sh -eux
+
+# There is a bug in the build process when it runs out of disk space
+# while stripping binaries, which causes the strip to fail, but does
+# not fail the build.  This results in a libLLVM.so that is over 2GB
+# which breaks the nightly compose.  So this test checks that libLLVM.so
+# is less than 100MB to ensure it was successfully stripped.
+# https://bugzilla.redhat.com/show_bug.cgi?id=1793250
+
+test $(stat -L -c %s /usr/lib64/libLLVM.so) -lt 104857600
diff --git a/tests/llvm-config/main.fmf b/tests/llvm-config/main.fmf
new file mode 100644
index 0000000..6c36dd6
--- /dev/null
+++ b/tests/llvm-config/main.fmf
@@ -0,0 +1,2 @@
+summary: Make sure llvm-config symlink is properly set up
+test: ./test.sh
diff --git a/tests/llvm-config/test.sh b/tests/llvm-config/test.sh
new file mode 100755
index 0000000..09a0286
--- /dev/null
+++ b/tests/llvm-config/test.sh
@@ -0,0 +1,3 @@
+#!/bin/sh -eux
+
+llvm-config --version
diff --git a/tests/provision.fmf b/tests/provision.fmf
deleted file mode 100644
index dd69f34..0000000
--- a/tests/provision.fmf
+++ /dev/null
@@ -1,5 +0,0 @@
----
-
-standard-inventory-qcow2:
-  qemu:
-    m: 4G
diff --git a/tests/rust-sanity/main.fmf b/tests/rust-sanity/main.fmf
new file mode 100644
index 0000000..e4f107d
--- /dev/null
+++ b/tests/rust-sanity/main.fmf
@@ -0,0 +1,5 @@
+summary: Test basic cargo usage
+test: ./test.sh
+require:
+  - cargo
+  - rust
diff --git a/tests/rust-sanity/test.sh b/tests/rust-sanity/test.sh
new file mode 100755
index 0000000..fa15399
--- /dev/null
+++ b/tests/rust-sanity/test.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -eux
+
+cargo new hello
+cd hello
+cargo run
diff --git a/tests/tests-llvm.yml b/tests/tests-llvm.yml
deleted file mode 100644
index 3afd361..0000000
--- a/tests/tests-llvm.yml
+++ /dev/null
@@ -1,73 +0,0 @@
-- hosts: localhost
-  roles:
-  - role: standard-test-basic
-    tags:
-      - classic
-    required_packages:
-      # Required for rust and possibly other dependant packages
-      #
-      # NOTE #1: drop the requirement once the transitional period between releases is gone
-      # and llvm-compat is no longer a) needed, and b) part of the build.
-      #
-      # NOTE #2: keep it the first requirement. Depending on the order of requirements, placing it somewhere
-      # else in the list may lead to harness uninstalling and re-installing LLVM packages. That a) would be
-      # a waste of time, b) could potentially hide issues, leading to testing unexpected NVRs.
-      #
-      # - llvm-compat
-
-      - rust
-      - cargo
-
-      # the requirements below are for the integration suite
-      - binutils
-      - cmake
-      - llvm-devel
-      - clang
-      - clang-analyzer
-      - clang-tools-extra
-      - compiler-rt
-      - ninja-build
-      - libomp-devel
-      - lld
-      - lldb
-      - git
-      - make
-      - libstdc++-static
-      - clang-devel
-
-      # In Fedora, python-lit is valid. In RHEL, however, the actual package is python3-lit.
-      - python-lit
-      # - python3-lit
-
-      # Required in Fedora, but not shipped with RHEL. For future reference, comment the libcxx out as needed.
-      - libcxx-devel
-
-    tests:
-      - rust-sanity:
-          dir: ./
-          run: cargo new hello && cd hello && cargo run
-      # There is a bug in the build process when it runs out of disk space
-      # while stripping binaries, which causes the strip to fail, but does
-      # not fail the build.  This results in a libLLVM.so that is over 2GB
-      # which breaks the nightly compose.  So this test checks that libLLVM.so
-      # is less than 100MB to ensure it was successfully stripped.
-      # https://bugzilla.redhat.com/show_bug.cgi?id=1793250
-      - libllvm-size:
-          dir: ./
-          run: test `stat -L -c %s /usr/lib64/libLLVM.so` -lt 104857600
-      # This test ensures that the spec file still builds correctly with
-      # %global compat_build 1
-      # FIXME: This fails, because the CI system has a hard-coded timeout of 4
-      # hours.
-      #- build-compat
-      - binutils-plugin-ar
-      # make sure llvm-config symlink is properly setup
-      - llvm-config:
-          dir: ./
-          run: llvm-config --version
-      # make sure versioned llvm-config symlink is properly setup
-      - versioned-llvm-config:
-          dir: ./
-          run: llvm-config-13 --version
-      # NOTE: don't forget to disable libcxx tests when libcxx is not available, by adding -DENABLE_LIBCXX=OFF.
-      - integration-test-suite
diff --git a/tests/versioned-llvm-config/main.fmf b/tests/versioned-llvm-config/main.fmf
new file mode 100644
index 0000000..eddb736
--- /dev/null
+++ b/tests/versioned-llvm-config/main.fmf
@@ -0,0 +1,2 @@
+summary: Make sure the versioned llvm-config symlink is properly set up
+test: ./test.sh
diff --git a/tests/versioned-llvm-config/test.sh b/tests/versioned-llvm-config/test.sh
new file mode 100755
index 0000000..cfe72cf
--- /dev/null
+++ b/tests/versioned-llvm-config/test.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -eux
+
+major=$(llvm-config --version | cut -d '.' -f1)
+
+llvm-config-$major --version