From 7f4bc96c8437aa7b7491116a9a0f8eb08c9e083f Mon Sep 17 00:00:00 2001 From: Hanabi <317387557+Hanabi9248@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:37:52 +0800 Subject: [PATCH] fix(buildroot): honor fs2json exclusions for tar input Pass the public CLI exclusions into the tar reader and prune matching paths and descendants without dropping prefix siblings. Preserve default archive output and validate the complete CLI using genuine tar fixtures. Signed-off-by: Hanabi <317387557+Hanabi9248@users.noreply.github.com> --- packages/buildroot/tools/fs2json.py | 12 ++- tests/test_fs2json_tar_excludes.py | 155 ++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 tests/test_fs2json_tar_excludes.py diff --git a/packages/buildroot/tools/fs2json.py b/packages/buildroot/tools/fs2json.py index 1a78651..ced1d58 100755 --- a/packages/buildroot/tools/fs2json.py +++ b/packages/buildroot/tools/fs2json.py @@ -8,6 +8,7 @@ import argparse import json import os +import posixpath import stat import sys import itertools @@ -79,7 +80,7 @@ def main(): tar = None if tar: - (root, total_size) = handle_tar(logger, tar) + (root, total_size) = handle_tar(logger, tar, args.exclude) else: (root, total_size) = handle_dir(logger, path, args.exclude) @@ -212,12 +213,19 @@ def make_node(st, name): return (mainroot, total_size) -def handle_tar(logger, tar): +def handle_tar(logger, tar, exclude=None): mainroot = [] filename_to_hash = {} total_size = 0 + exclude = {posixpath.normpath("/" + p.lstrip("/")) for p in (exclude or [])} + for member in tar.getmembers(): + member_path = posixpath.normpath("/" + member.name.lstrip("/")) + if any(member_path == p or member_path.startswith(p.rstrip("/") + "/") + for p in exclude): + continue + parts = member.name.split("/") name = parts.pop() diff --git a/tests/test_fs2json_tar_excludes.py b/tests/test_fs2json_tar_excludes.py new file mode 100644 index 0000000..e95cabe --- /dev/null +++ b/tests/test_fs2json_tar_excludes.py @@ -0,0 +1,155 @@ +"""Run the complete public CLI against real directories and tar archives. + +Set IVORYSQL_WASM_SOURCE_ROOT to run these same tests against another tree. +Directory CLI coverage requires a host where tarfile.open(directory) raises +IsADirectoryError, as required by the unchanged upstream directory dispatch. +""" + +import hashlib +import io +import json +import os +import stat +import subprocess +import sys +import tarfile +import tempfile +import unittest +from pathlib import Path + +SOURCE_ROOT = Path( + os.environ.get("IVORYSQL_WASM_SOURCE_ROOT", str(Path(__file__).resolve().parents[1])) +) +SCRIPT = SOURCE_ROOT / "packages/buildroot/tools/fs2json.py" +DIRECTORIES = ("cache", "cache/nested", "cache-old", "etc", "var", "var/log") +FILES = { + "cache/secret.txt": b"excluded cache\n", + "cache/nested/secret.txt": b"excluded descendant\n", + "cache-old/keep.txt": b"keep prefix sibling\n", + "etc/config": b"keep configuration\n", + "var/log/app.log": b"exclude separately\n", +} + + +def flatten(nodes, prefix=""): + result = {} + for node in nodes: + path = prefix + node[0] + result[path] = node + if stat.S_ISDIR(node[3]): + result.update(flatten(node[6], path + "/")) + return result + + +class Fs2jsonExclusionTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="fs2json-excludes-") + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.directory = self.root / "input" + self.directory.mkdir() + for name in DIRECTORIES: + (self.directory / name).mkdir() + for name, data in FILES.items(): + (self.directory / name).write_bytes(data) + self.archive = self.root / "input.tar" + with tarfile.open(self.archive, "w") as archive: + for name in DIRECTORIES: + member = self.member(name, 0o755) + member.type = tarfile.DIRTYPE + archive.addfile(member) + for name, data in FILES.items(): + member = self.member(name, 0o644) + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + member = self.member("etc/config-link", 0o777) + member.type = tarfile.SYMTYPE + member.linkname = "config" + archive.addfile(member) + + @staticmethod + def member(name, mode): + member = tarfile.TarInfo(name) + member.mode = mode + member.uid = 1000 + member.gid = 1000 + member.mtime = 1700000000 + return member + + def run_cli(self, source, *excludes): + output = self.root / "output.json" + command = [sys.executable, str(SCRIPT), "--out", str(output)] + for exclude in excludes: + command.extend(["--exclude", exclude]) + command.append(str(source)) + completed = subprocess.run( + command, cwd=self.root, capture_output=True, text=True, timeout=30 + ) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + return json.loads(output.read_text()) + + def assert_retained_file(self, nodes, name): + node = nodes[name] + self.assertTrue(stat.S_ISREG(node[3])) + self.assertEqual(node[1], len(FILES[name])) + self.assertEqual(node[6], hashlib.sha256(FILES[name]).hexdigest()[:8] + ".bin") + + def test_tar_excludes_directory_and_descendants_but_not_prefix_sibling(self): + result = self.run_cli(self.archive, "cache") + nodes = flatten(result["fsroot"]) + self.assertNotIn("cache", nodes) + self.assertFalse(any(name.startswith("cache/") for name in nodes)) + self.assert_retained_file(nodes, "cache-old/keep.txt") + self.assert_retained_file(nodes, "etc/config") + self.assertEqual( + result["size"], + sum(len(data) for name, data in FILES.items() if not name.startswith("cache/")), + ) + + def test_tar_repeated_normalized_excludes(self): + result = self.run_cli(self.archive, "/cache/", "./var/log") + nodes = flatten(result["fsroot"]) + self.assertNotIn("cache", nodes) + self.assertNotIn("var/log", nodes) + self.assertIn("var", nodes) + self.assertEqual(nodes["var"][6], []) + self.assert_retained_file(nodes, "cache-old/keep.txt") + self.assertEqual( + result["size"], len(FILES["cache-old/keep.txt"]) + len(FILES["etc/config"]) + ) + + def test_tar_default_preserves_layout_metadata_hashes_and_symlink(self): + result = self.run_cli(self.archive) + nodes = flatten(result["fsroot"]) + self.assertEqual(result["version"], 3) + self.assertEqual(set(nodes), set(DIRECTORIES) | set(FILES) | {"etc/config-link"}) + self.assertEqual(result["size"], sum(map(len, FILES.values()))) + for name in FILES: + self.assert_retained_file(nodes, name) + self.assertEqual(nodes[name][2:3], [1700000000]) + self.assertEqual(nodes[name][4:6], [1000, 1000]) + self.assertTrue(stat.S_ISLNK(nodes["etc/config-link"][3])) + self.assertEqual(nodes["etc/config-link"][6], "config") + + def test_tar_nonmatching_exclude_is_identical_to_default(self): + self.assertEqual(self.run_cli(self.archive, "absent"), self.run_cli(self.archive)) + + @unittest.skipIf(os.name == "nt", "unchanged directory CLI dispatch requires IsADirectoryError") + def test_directory_excludes_still_prune_subtrees(self): + result = self.run_cli(self.directory, "cache", "var/log") + nodes = flatten(result["fsroot"]) + self.assertNotIn("cache", nodes) + self.assertNotIn("var/log", nodes) + self.assert_retained_file(nodes, "cache-old/keep.txt") + self.assert_retained_file(nodes, "etc/config") + + @unittest.skipIf(os.name == "nt", "unchanged directory CLI dispatch requires IsADirectoryError") + def test_directory_default_still_includes_all_files(self): + nodes = flatten(self.run_cli(self.directory)["fsroot"]) + self.assertEqual(set(nodes), set(DIRECTORIES) | set(FILES)) + for name in FILES: + self.assert_retained_file(nodes, name) + + +if __name__ == "__main__": + unittest.main()