From 6d366e0d3b4c2757da3dcd77f585ade7dc1d9928 Mon Sep 17 00:00:00 2001 From: pyrco <105293448+pyrco@users.noreply.github.com> Date: Mon, 27 Nov 2023 13:11:35 +0100 Subject: [PATCH] Replace the internal walkfs_ext plugin with Target.fs.walk_ext DIS-2251 --- .../plugins/filesystem/unix/capability.py | 143 ++++++++++-------- dissect/target/plugins/filesystem/walkfs.py | 42 ++--- dissect/target/plugins/filesystem/yara.py | 39 ++--- .../loaders/tar/test-archive-dot-folder.tgz | Bin 156 -> 0 bytes tests/plugins/filesystem/test_walkfs.py | 69 +-------- tests/plugins/filesystem/test_yara.py | 2 +- tests/tools/test_query.py | 12 +- 7 files changed, 134 insertions(+), 173 deletions(-) delete mode 100644 tests/_data/loaders/tar/test-archive-dot-folder.tgz diff --git a/dissect/target/plugins/filesystem/unix/capability.py b/dissect/target/plugins/filesystem/unix/capability.py index 4af991ba6..b9a9322d3 100644 --- a/dissect/target/plugins/filesystem/unix/capability.py +++ b/dissect/target/plugins/filesystem/unix/capability.py @@ -2,9 +2,10 @@ from enum import IntEnum from io import BytesIO -from dissect.target.exceptions import UnsupportedPluginError +from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError from dissect.target.helpers.record import TargetRecordDescriptor from dissect.target.plugin import Plugin, export +from dissect.target.plugins.filesystem.walkfs import generate_record CapabilityRecord = TargetRecordDescriptor( "filesystem/unix/capability", @@ -87,72 +88,82 @@ def check_compatible(self) -> None: @export(record=CapabilityRecord) def capability_binaries(self): """Find all files that have capabilities set.""" - for entry, record in self.target.walkfs_ext(): - try: - attrs = entry.get().lattr() - except Exception: - self.target.log.exception("Failed to get attrs for entry %s", entry) - continue - - for attr in attrs: - if attr.name != "security.capability": + for path_entries, _, files in self.target.fs.walk_ext("/"): + entries = [path_entries[-1]] + files + for entry in entries: + path = self.target.fs.path(entry.path) + try: + record = generate_record(self.target, path) + except FileNotFoundError: continue - - buf = BytesIO(attr.value) - - # Reference: https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h - # The struct is small enough we can just use struct - magic_etc = struct.unpack("> 5 - if cap_index >= len(caps): - # We loop over all capabilities, but might only have a version 1 caps list - continue - - if caps[cap_index] & (1 << (capability.value & 31)) != 0: - results.append(capability.name) - - yield CapabilityRecord( - record=record, - permitted=permitted, - inheritable=inheritable, - effective=magic_etc & VFS_CAP_FLAGS_EFFECTIVE != 0, - rootid=rootid, - _target=self.target, - ) + for attr in attrs: + if attr.name != "security.capability": + continue + + buf = BytesIO(attr.value) + + # Reference: https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h + # The struct is small enough we can just use struct + magic_etc = struct.unpack("> 5 + if cap_index >= len(caps): + # We loop over all capabilities, but might only have a version 1 caps list + continue + + if caps[cap_index] & (1 << (capability.value & 31)) != 0: + results.append(capability.name) + + yield CapabilityRecord( + record=record, + permitted=permitted, + inheritable=inheritable, + effective=magic_etc & VFS_CAP_FLAGS_EFFECTIVE != 0, + rootid=rootid, + _target=self.target, + ) diff --git a/dissect/target/plugins/filesystem/walkfs.py b/dissect/target/plugins/filesystem/walkfs.py index a56ee728f..47c9e40bf 100644 --- a/dissect/target/plugins/filesystem/walkfs.py +++ b/dissect/target/plugins/filesystem/walkfs.py @@ -1,8 +1,13 @@ +from typing import Iterable + from dissect.util.ts import from_unix from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError +from dissect.target.filesystem import RootFilesystemEntry +from dissect.target.helpers.fsutil import TargetPath from dissect.target.helpers.record import TargetRecordDescriptor -from dissect.target.plugin import Plugin, export, internal +from dissect.target.plugin import Plugin, export +from dissect.target.target import Target FilesystemRecord = TargetRecordDescriptor( "filesystem/entry", @@ -17,8 +22,7 @@ ("uint32", "mode"), ("uint32", "uid"), ("uint32", "gid"), - ("string", "fstype"), - ("uint32", "fsidx"), + ("string[]", "fstypes"), ], ) @@ -29,36 +33,36 @@ def check_compatible(self) -> None: raise UnsupportedPluginError("No filesystems found") @export(record=FilesystemRecord) - def walkfs(self): + def walkfs(self) -> Iterable[FilesystemRecord]: """Walk a target's filesystem and return all filesystem entries.""" - for _, record in self.walkfs_ext(): - yield record - - @internal - def walkfs_ext(self, root="/", pattern="*"): - for idx, fs in enumerate(self.target.filesystems): - for entry in fs.path(root).rglob(pattern): + for path_entries, _, files in self.target.fs.walk_ext("/"): + entries = [path_entries[-1]] + files + for entry in entries: + path = self.target.fs.path(entry.path) try: - yield entry, generate_record(self.target, entry, idx) + record = generate_record(self.target, path) except FileNotFoundError: continue - except Exception: - self.target.log.exception("Failed to generate record from entry %s", entry) + yield record -def generate_record(target, entry, idx): - stat = entry.lstat() +def generate_record(target: Target, path: TargetPath) -> FilesystemRecord: + stat = path.lstat() + entry = path.get() + if isinstance(entry, RootFilesystemEntry): + fs_types = [sub_entry.fs.__type__ for sub_entry in entry.entries] + else: + fs_types = [entry.fs.__type__] return FilesystemRecord( atime=from_unix(stat.st_atime), mtime=from_unix(stat.st_mtime), ctime=from_unix(stat.st_ctime), ino=stat.st_ino, - path=entry, + path=path, size=stat.st_size, mode=stat.st_mode, uid=stat.st_uid, gid=stat.st_gid, - fstype=entry.get().fs.__type__, - fsidx=idx, + fstypes=fs_types, _target=target, ) diff --git a/dissect/target/plugins/filesystem/yara.py b/dissect/target/plugins/filesystem/yara.py index 7240bd8b9..bc348d47f 100644 --- a/dissect/target/plugins/filesystem/yara.py +++ b/dissect/target/plugins/filesystem/yara.py @@ -5,7 +5,7 @@ except ImportError: raise ImportError("Please install 'yara-python' to use 'target-query -f yara'.") -from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError +from dissect.target.exceptions import FileNotFoundError from dissect.target.helpers.record import TargetRecordDescriptor from dissect.target.plugin import Plugin, arg, export @@ -26,8 +26,7 @@ class YaraPlugin(Plugin): DEFAULT_MAX_SIZE = 10 * 1024 * 1024 def check_compatible(self) -> None: - if not self.target.has_function("walkfs"): - raise UnsupportedPluginError("No walkfs plugin found") + pass @arg("--rule-files", "-r", type=Path, nargs="+", required=True, help="path to YARA rule file") @arg("--scan-path", default="/", help="path to recursively scan") @@ -43,20 +42,22 @@ def yara(self, rule_files, scan_path="/", max_size=DEFAULT_MAX_SIZE): rule_data = "\n".join([rule_file.read_text() for rule_file in rule_files]) rules = yara.compile(source=rule_data) - for entry, _ in self.target.walkfs_ext(scan_path): - try: - if not entry.is_file() or entry.stat().st_size > max_size: + for _, _, files in self.target.fs.walk_ext(scan_path): + for file_entry in files: + path = self.target.fs.path(file_entry.path) + try: + if path.stat().st_size > max_size: + continue + + for match in rules.match(data=path.read_bytes()): + yield YaraMatchRecord( + path=path, + digest=path.get().hash(), + rule=match.rule, + tags=match.tags, + _target=self.target, + ) + except FileNotFoundError: continue - - for match in rules.match(data=entry.read_bytes()): - yield YaraMatchRecord( - path=entry, - digest=entry.get().hash(), - rule=match.rule, - tags=match.tags, - _target=self.target, - ) - except FileNotFoundError: - continue - except Exception: - self.target.log.exception("Error scanning file: %s", entry) + except Exception: + self.target.log.exception("Error scanning file: %s", path) diff --git a/tests/_data/loaders/tar/test-archive-dot-folder.tgz b/tests/_data/loaders/tar/test-archive-dot-folder.tgz deleted file mode 100644 index 7108a1980deae74cef5f1646ae076a4e39ed1670..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 156 zcmb2|=3p?5W=~;YetXf8>yUv!>%%p+u6`5L%9dI7du-?3qFhmPInU)_SGU5W|G8=^ zk$WDto--9+y=K?hSCx6sRfH}t(3txPrUwDz4nd0>dVcaH^wg6ZU4SL_x}9TDUVay^V=C2kidtD(oaI&bQv@l F7y$3mNmu{? diff --git a/tests/plugins/filesystem/test_walkfs.py b/tests/plugins/filesystem/test_walkfs.py index 0bb5b1d90..beb5dc986 100644 --- a/tests/plugins/filesystem/test_walkfs.py +++ b/tests/plugins/filesystem/test_walkfs.py @@ -1,7 +1,5 @@ from dissect.target.filesystem import VirtualFile -from dissect.target.loaders.tar import TarLoader from dissect.target.plugins.filesystem.walkfs import WalkFSPlugin -from tests._utils import absolute_path def test_walkfs_plugin(target_unix, fs_unix): @@ -9,12 +7,18 @@ def test_walkfs_plugin(target_unix, fs_unix): fs_unix.map_file_entry("/path/to/some/other/file.ext", VirtualFile(fs_unix, "file.ext", None)) fs_unix.map_file_entry("/root_file", VirtualFile(fs_unix, "root_file", None)) fs_unix.map_file_entry("/other_root_file.ext", VirtualFile(fs_unix, "other_root_file.ext", None)) + fs_unix.map_file_entry("/.test/test.txt", VirtualFile(fs_unix, "test.txt", None)) + fs_unix.map_file_entry("/.test/.more.test.txt", VirtualFile(fs_unix, ".more.test.txt", None)) target_unix.add_plugin(WalkFSPlugin) results = list(target_unix.walkfs()) - assert len(results) == 10 + assert len(results) == 14 assert sorted([r.path for r in results]) == [ + "/", + "/.test", + "/.test/.more.test.txt", + "/.test/test.txt", "/etc", "/other_root_file.ext", "/path", @@ -26,62 +30,3 @@ def test_walkfs_plugin(target_unix, fs_unix): "/root_file", "/var", ] - - -def test_walkfs_ext_internal(target_unix, fs_unix): - fs_unix.map_file_entry("/path/to/some/file", VirtualFile(fs_unix, "file", None)) - fs_unix.map_file_entry("/path/to/some/other/file.ext", VirtualFile(fs_unix, "file.ext", None)) - fs_unix.map_file_entry("/root_file", VirtualFile(fs_unix, "root_file", None)) - fs_unix.map_file_entry("/other_root_file.ext", VirtualFile(fs_unix, "other_root_file.ext", None)) - - target_unix.add_plugin(WalkFSPlugin) - - results = list(target_unix.walkfs_ext()) - assert len(results) == 10 - assert sorted([r.path for _, r in results]) == [ - "/etc", - "/other_root_file.ext", - "/path", - "/path/to", - "/path/to/some", - "/path/to/some/file", - "/path/to/some/other", - "/path/to/some/other/file.ext", - "/root_file", - "/var", - ] - - results = list(target_unix.walkfs_ext(root="/path")) - assert len(results) == 5 - assert sorted([r.path for _, r in results]) == [ - "/path/to", - "/path/to/some", - "/path/to/some/file", - "/path/to/some/other", - "/path/to/some/other/file.ext", - ] - - results = list(target_unix.walkfs_ext(pattern="*.ext")) - assert len(results) == 2 - assert sorted([r.path for _, r in results]) == [ - "/other_root_file.ext", - "/path/to/some/other/file.ext", - ] - - results = list(target_unix.walkfs_ext(root="/path", pattern="*.ext")) - assert len(results) == 1 - assert sorted([r.path for _, r in results]) == [ - "/path/to/some/other/file.ext", - ] - - -def test_walkfs_dot_folder(target_unix): - archive_path = absolute_path("_data/loaders/tar/test-archive-dot-folder.tgz") - loader = TarLoader(archive_path) - loader.map(target_unix) - target_unix.add_plugin(WalkFSPlugin) - results = list(target_unix.walkfs()) - - assert len(target_unix.filesystems) == 2 - assert len(results) == 3 - assert target_unix.fs.path("/test.txt").exists() diff --git a/tests/plugins/filesystem/test_yara.py b/tests/plugins/filesystem/test_yara.py index 922f82540..5bb050f3e 100644 --- a/tests/plugins/filesystem/test_yara.py +++ b/tests/plugins/filesystem/test_yara.py @@ -23,7 +23,7 @@ def test_yara_plugin(tmp_path, target_default): vfs.map_file_fh("test_file", BytesIO(b"test string")) vfs.map_file_fh("/test/dir/to/test_file", BytesIO(b"test string")) - target_default.filesystems.add(vfs) + target_default.fs.mount("/", vfs) with tempfile.NamedTemporaryFile(mode="w+t", dir=tmp_path, delete=False) as tmp_file: tmp_file.write(test_rule) diff --git a/tests/tools/test_query.py b/tests/tools/test_query.py index e1a25841e..f7c31a450 100644 --- a/tests/tools/test_query.py +++ b/tests/tools/test_query.py @@ -56,7 +56,7 @@ def test_target_query_invalid_functions( with monkeypatch.context() as m: m.setattr( "sys.argv", - ["target-query", "-f", ",".join(given_funcs), "tests/_data/loaders/tar/test-archive-dot-folder.tgz"], + ["target-query", "-f", ",".join(given_funcs), "tests/_data/loaders/tar/test-archive.tar.gz"], ) with pytest.raises((SystemExit)): @@ -115,7 +115,7 @@ def test_target_query_invalid_excluded_functions( "hostname", "-xf", ",".join(given_funcs), - "tests/_data/loaders/tar/test-archive-dot-folder.tgz", + "tests/_data/loaders/tar/test-archive.tar.gz", ], ) @@ -139,7 +139,7 @@ def test_target_query_unsupported_plugin_log(capsys: pytest.CaptureFixture, monk with monkeypatch.context() as m: m.setattr( "sys.argv", - ["target-query", "-f", "regf", "tests/_data/loaders/tar/test-archive-dot-folder.tgz"], + ["target-query", "-f", "regf", "tests/_data/loaders/tar/test-archive.tar.gz"], ) target_query() @@ -188,7 +188,7 @@ def test_target_query_filtered_functions(monkeypatch: pytest.MonkeyPatch) -> Non "foo,bar,bla,foo", "-xf", "bla", - "tests/_data/loaders/tar/test-archive-dot-folder.tgz", + "tests/_data/loaders/tar/test-archive.tar.gz", ], ) @@ -220,9 +220,9 @@ def test_target_query_filtered_functions(monkeypatch: pytest.MonkeyPatch) -> Non def test_target_query_dry_run(capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch) -> None: if os.sep == "\\": - target_file = "tests\\_data\\loaders\\tar\\test-archive-dot-folder.tgz" + target_file = "tests\\_data\\loaders\\tar\\test-archive.tar.gz" else: - target_file = "tests/_data/loaders/tar/test-archive-dot-folder.tgz" + target_file = "tests/_data/loaders/tar/test-archive.tar.gz" with monkeypatch.context() as m: m.setattr(