kbuild: upload make backend dtbs from an archive

Archive uploads of dtbs are only used when both the extracted dtb files
and a dtbs.tar.xz artifact are present. The tuxmake backend gets the
archive for free, but _package_dtbs() only ever copies the dtbs_install
output into the artifacts directory, so make backend builds upload every
dtb as a separate request. That is 36 of the currently scheduled kbuild
jobs, including the whole CIP set and the arm64 chromebook build.

Pack the installed dtbs into dtbs.tar.xz with the same dtbs/ prefix
inside as the tuxmake archive, so upload_artifacts() picks it up with no
change. The archive is only created when at least one dtb was built,
which leaves verify_build() to drop the artifact and the upload to fall
back to individual files otherwise.

Signed-off-by: Denys Fedoryshchenko <denys.f@collabora.com>

Author: nuclearcat

kernelci/kbuild.py

@@ -1249,7 +1249,20 @@ def _package_dtbs(self):
         self.addcmd(f"mkdir -p {self._af_dir}/dtbs")
         # copy dtbs to artifacts
         self.addcmd(f"cp -r _dtbs_/* {self._af_dir}/dtbs", False)
+        # Also pack them into a single archive, with the same dtbs/ prefix
+        # inside as the one tuxmake produces, so that upload_artifacts() can
+        # upload the whole set in one request instead of one per dtb file.
+        # Skipped when no dtb was built, so that verify_build() drops the
+        # artifact and the upload falls back to individual files.
+        # -print -quit stops at the first dtb found, no full traversal
+        find_dtb = f"find {self._af_dir}/dtbs -name '*.dtb' -print -quit"
+        self.addcmd(
+            f'if [ -n "$({find_dtb})" ]; then '
+            f"tar -C {self._af_dir} -cJf {self._af_dir}/dtbs.tar.xz dtbs; "
+            "fi"
+        )
         self.addcmd("cd ..")
+        self._artifacts.append("dtbs.tar.xz")
 
     def _write_metadata(self):
         """

tests/test_kbuild.py

@@ -163,3 +163,54 @@ def test_tuxmake_dtbs_use_archive_upload(self, tmp_path):
         assert "dtbs/board-a.dtb" in kbuild._full_artifacts
         assert "dtbs/nested/board-b.dtb" in kbuild._full_artifacts
         assert node_af["dtbs/board-a_dtb"].endswith("dtbs/board-a.dtb")
+
+    def test_make_dtbs_use_archive_upload(self, tmp_path):
+        kbuild = _kbuild(tmp_path, arch="arm64")
+        kbuild._backend = "make"
+        kbuild._dtbs_check = False
+        af_dir = tmp_path / "artifacts"
+        (af_dir / "dtbs" / "nested").mkdir(parents=True)
+        (af_dir / "dtbs" / "board-a.dtb").write_bytes(b"dtb-a")
+        (af_dir / "dtbs" / "nested" / "board-b.dtb").write_bytes(b"dtb-b")
+        (af_dir / "dtbs.tar.xz").write_bytes(b"archive")
+        kbuild._artifacts = ["dtbs.tar.xz"]
+        kbuild.verify_build()
+
+        storage = FakeStorage()
+        kbuild._get_storage = lambda: storage
+        kbuild._apijobname = "kbuild-gcc-arm64"
+        kbuild._node = {"id": "node123", "data": {}}
+        kbuild._full_artifacts = {}
+
+        node_af = kbuild.upload_artifacts()
+
+        assert storage.single_uploads == []
+        assert len(storage.archive_uploads) == 1
+        archive_path, file_paths, _dest_path, archive_name = (
+            storage.archive_uploads[0]
+        )
+        assert archive_path == str(af_dir / "dtbs.tar.xz")
+        assert archive_name == "dtbs.tar.xz"
+        assert sorted(file_dst for _file_src, file_dst in file_paths) == [
+            "dtbs/board-a.dtb",
+            "dtbs/nested/board-b.dtb",
+        ]
+        assert node_af["dtbs/board-a_dtb"].endswith("dtbs/board-a.dtb")
+
+
+class TestPackageDtbs:
+    def test_dtbs_are_packed_into_archive(self, tmp_path):
+        kbuild = _kbuild(tmp_path, arch="arm64")
+        kbuild._package_dtbs()
+        steps = "\n".join(kbuild._steps)
+        af_dir = kbuild._af_dir
+        assert f"tar -C {af_dir} -cJf {af_dir}/dtbs.tar.xz dtbs" in steps
+        # the archive is only built when at least one dtb was produced
+        assert "-print -quit" in steps
+        assert "dtbs.tar.xz" in kbuild._artifacts
+
+    def test_archive_dropped_when_no_dtbs_built(self, tmp_path):
+        kbuild = _kbuild(tmp_path, arch="arm64")
+        kbuild._package_dtbs()
+        kbuild.verify_build()
+        assert "dtbs.tar.xz" not in kbuild._artifacts