Replace the internal walkfs_ext plugin with Target.fs.walk_ext

dissect/target/plugins/filesystem/unix/capability.py

@@ -2,9 +2,10 @@
 from enum import IntEnum
 from io import BytesIO
 
-from dissect.target.exceptions import UnsupportedPluginError
+from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
 from dissect.target.helpers.record import TargetRecordDescriptor
 from dissect.target.plugin import Plugin, export
+from dissect.target.plugins.filesystem.walkfs import generate_record
 
 CapabilityRecord = TargetRecordDescriptor(
     "filesystem/unix/capability",
@@ -87,72 +88,82 @@ def check_compatible(self) -> None:
     @export(record=CapabilityRecord)
     def capability_binaries(self):
         """Find all files that have capabilities set."""
-        for entry, record in self.target.walkfs_ext():
-            try:
-                attrs = entry.get().lattr()
-            except Exception:
-                self.target.log.exception("Failed to get attrs for entry %s", entry)
-                continue
-
-            for attr in attrs:
-                if attr.name != "security.capability":
+        for path_entries, _, files in self.target.fs.walk_ext("/"):
+            entries = [path_entries[-1]] + files
+            for entry in entries:
+                path = self.target.fs.path(entry.path)
+                try:
+                    record = generate_record(self.target, path)
+                except FileNotFoundError:
                     continue
-
-                buf = BytesIO(attr.value)
-
-                # Reference: https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h
-                # The struct is small enough we can just use struct
-                magic_etc = struct.unpack("<I", buf.read(4))[0]
-                cap_revision = magic_etc & VFS_CAP_REVISION_MASK
-
-                permitted_caps = []
-                inheritable_caps = []
-                rootid = None
-
-                if cap_revision == VFS_CAP_REVISION_1:
-                    num_caps = VFS_CAP_U32_1
-                    data_len = (1 + 2 * VFS_CAP_U32_1) * 4
-                elif cap_revision == VFS_CAP_REVISION_2:
-                    num_caps = VFS_CAP_U32_2
-                    data_len = (1 + 2 * VFS_CAP_U32_2) * 4
-                elif cap_revision == VFS_CAP_REVISION_3:
-                    num_caps = VFS_CAP_U32_3
-                    data_len = (2 + 2 * VFS_CAP_U32_2) * 4
-                else:
-                    self.target.log.error("Unexpected capability revision: %s", entry)
+                try:
+                    attrs = path.get().lattr()
+                except TypeError:
+                    # Virtual(File|Directory|Symlink) instances don't have a functional lattr()
                     continue
-
-                if data_len != len(attr.value):
-                    self.target.log.error("Unexpected capability length: %s", entry)
+                except Exception:
+                    self.target.log.exception("Failed to get attrs for entry %s", entry)
                     continue
 
-                for _ in range(num_caps):
-                    permitted_val, inheritable_val = struct.unpack("<2I", buf.read(8))
-                    permitted_caps.append(permitted_val)
-                    inheritable_caps.append(inheritable_val)
-
-                if cap_revision == VFS_CAP_REVISION_3:
-                    rootid = struct.unpack("<I", buf.read(4))[0]
-
-                permitted = []
-                inheritable = []
-
-                for capability in Capabilities:
-                    for caps, results in [(permitted_caps, permitted), (inheritable_caps, inheritable)]:
-                        # CAP_TO_INDEX
-                        cap_index = capability.value >> 5
-                        if cap_index >= len(caps):
-                            # We loop over all capabilities, but might only have a version 1 caps list
-                            continue
-
-                        if caps[cap_index] & (1 << (capability.value & 31)) != 0:
-                            results.append(capability.name)
-
-                yield CapabilityRecord(
-                    record=record,
-                    permitted=permitted,
-                    inheritable=inheritable,
-                    effective=magic_etc & VFS_CAP_FLAGS_EFFECTIVE != 0,
-                    rootid=rootid,
-                    _target=self.target,
-                )
+                for attr in attrs:
+                    if attr.name != "security.capability":
+                        continue
+
+                    buf = BytesIO(attr.value)
+
+                    # Reference: https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h
+                    # The struct is small enough we can just use struct
+                    magic_etc = struct.unpack("<I", buf.read(4))[0]
+                    cap_revision = magic_etc & VFS_CAP_REVISION_MASK
+
+                    permitted_caps = []
+                    inheritable_caps = []
+                    rootid = None
+
+                    if cap_revision == VFS_CAP_REVISION_1:
+                        num_caps = VFS_CAP_U32_1
+                        data_len = (1 + 2 * VFS_CAP_U32_1) * 4
+                    elif cap_revision == VFS_CAP_REVISION_2:
+                        num_caps = VFS_CAP_U32_2
+                        data_len = (1 + 2 * VFS_CAP_U32_2) * 4
+                    elif cap_revision == VFS_CAP_REVISION_3:
+                        num_caps = VFS_CAP_U32_3
+                        data_len = (2 + 2 * VFS_CAP_U32_2) * 4
+                    else:
+                        self.target.log.error("Unexpected capability revision: %s", entry)
+                        continue
+
+                    if data_len != len(attr.value):
+                        self.target.log.error("Unexpected capability length: %s", entry)
+                        continue
+
+                    for _ in range(num_caps):
+                        permitted_val, inheritable_val = struct.unpack("<2I", buf.read(8))
+                        permitted_caps.append(permitted_val)
+                        inheritable_caps.append(inheritable_val)
+
+                    if cap_revision == VFS_CAP_REVISION_3:
+                        rootid = struct.unpack("<I", buf.read(4))[0]
+
+                    permitted = []
+                    inheritable = []
+
+                    for capability in Capabilities:
+                        for caps, results in [(permitted_caps, permitted), (inheritable_caps, inheritable)]:
+                            # CAP_TO_INDEX
+                            cap_index = capability.value >> 5
+                            if cap_index >= len(caps):
+                                # We loop over all capabilities, but might only have a version 1 caps list
+                                continue
+
+                            if caps[cap_index] & (1 << (capability.value & 31)) != 0:
+                                results.append(capability.name)
+
+                    yield CapabilityRecord(
+                        record=record,
+                        permitted=permitted,
+                        inheritable=inheritable,
+                        effective=magic_etc & VFS_CAP_FLAGS_EFFECTIVE != 0,
+                        rootid=rootid,
+                        _target=self.target,
+                    )

dissect/target/plugins/filesystem/walkfs.py

@@ -1,8 +1,13 @@
+from typing import Iterable
+
 from dissect.util.ts import from_unix
 
 from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
+from dissect.target.filesystem import RootFilesystemEntry
+from dissect.target.helpers.fsutil import TargetPath
 from dissect.target.helpers.record import TargetRecordDescriptor
-from dissect.target.plugin import Plugin, export, internal
+from dissect.target.plugin import Plugin, export
+from dissect.target.target import Target
 
 FilesystemRecord = TargetRecordDescriptor(
     "filesystem/entry",
@@ -17,8 +22,7 @@
         ("uint32", "mode"),
         ("uint32", "uid"),
         ("uint32", "gid"),
-        ("string", "fstype"),
-        ("uint32", "fsidx"),
+        ("string[]", "fstypes"),
     ],
 )
 
@@ -29,36 +33,36 @@ def check_compatible(self) -> None:
             raise UnsupportedPluginError("No filesystems found")
 
     @export(record=FilesystemRecord)
-    def walkfs(self):
+    def walkfs(self) -> Iterable[FilesystemRecord]:
         """Walk a target's filesystem and return all filesystem entries."""
-        for _, record in self.walkfs_ext():
-            yield record
-
-    @internal
-    def walkfs_ext(self, root="/", pattern="*"):
-        for idx, fs in enumerate(self.target.filesystems):
-            for entry in fs.path(root).rglob(pattern):
+        for path_entries, _, files in self.target.fs.walk_ext("/"):
+            entries = [path_entries[-1]] + files
+            for entry in entries:
+                path = self.target.fs.path(entry.path)
                 try:
-                    yield entry, generate_record(self.target, entry, idx)
+                    record = generate_record(self.target, path)
                 except FileNotFoundError:
                     continue
-                except Exception:
-                    self.target.log.exception("Failed to generate record from entry %s", entry)
+                yield record
 
 
-def generate_record(target, entry, idx):
-    stat = entry.lstat()
+def generate_record(target: Target, path: TargetPath) -> FilesystemRecord:
+    stat = path.lstat()
+    entry = path.get()
+    if isinstance(entry, RootFilesystemEntry):
+        fs_types = [sub_entry.fs.__type__ for sub_entry in entry.entries]
+    else:
+        fs_types = [entry.fs.__type__]
     return FilesystemRecord(
         atime=from_unix(stat.st_atime),
         mtime=from_unix(stat.st_mtime),
         ctime=from_unix(stat.st_ctime),
         ino=stat.st_ino,
-        path=entry,
+        path=path,
         size=stat.st_size,
         mode=stat.st_mode,
         uid=stat.st_uid,
         gid=stat.st_gid,
-        fstype=entry.get().fs.__type__,
-        fsidx=idx,
+        fstypes=fs_types,
         _target=target,
     )

dissect/target/plugins/filesystem/yara.py

@@ -5,7 +5,7 @@
 except ImportError:
     raise ImportError("Please install 'yara-python' to use 'target-query -f yara'.")
 
-from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
+from dissect.target.exceptions import FileNotFoundError
 from dissect.target.helpers.record import TargetRecordDescriptor
 from dissect.target.plugin import Plugin, arg, export
 
@@ -26,8 +26,7 @@ class YaraPlugin(Plugin):
     DEFAULT_MAX_SIZE = 10 * 1024 * 1024
 
     def check_compatible(self) -> None:
-        if not self.target.has_function("walkfs"):
-            raise UnsupportedPluginError("No walkfs plugin found")
+        pass
 
     @arg("--rule-files", "-r", type=Path, nargs="+", required=True, help="path to YARA rule file")
     @arg("--scan-path", default="/", help="path to recursively scan")
@@ -43,20 +42,22 @@ def yara(self, rule_files, scan_path="/", max_size=DEFAULT_MAX_SIZE):
         rule_data = "\n".join([rule_file.read_text() for rule_file in rule_files])
 
         rules = yara.compile(source=rule_data)
-        for entry, _ in self.target.walkfs_ext(scan_path):
-            try:
-                if not entry.is_file() or entry.stat().st_size > max_size:
+        for _, _, files in self.target.fs.walk_ext(scan_path):
+            for file_entry in files:
+                path = self.target.fs.path(file_entry.path)
+                try:
+                    if path.stat().st_size > max_size:
+                        continue
+
+                    for match in rules.match(data=path.read_bytes()):
+                        yield YaraMatchRecord(
+                            path=path,
+                            digest=path.get().hash(),
+                            rule=match.rule,
+                            tags=match.tags,
+                            _target=self.target,
+                        )
+                except FileNotFoundError:
                     continue
-
-                for match in rules.match(data=entry.read_bytes()):
-                    yield YaraMatchRecord(
-                        path=entry,
-                        digest=entry.get().hash(),
-                        rule=match.rule,
-                        tags=match.tags,
-                        _target=self.target,
-                    )
-            except FileNotFoundError:
-                continue
-            except Exception:
-                self.target.log.exception("Error scanning file: %s", entry)
+                except Exception:
+                    self.target.log.exception("Error scanning file: %s", path)

tests/plugins/filesystem/test_walkfs.py

@@ -1,20 +1,24 @@
 from dissect.target.filesystem import VirtualFile
-from dissect.target.loaders.tar import TarLoader
 from dissect.target.plugins.filesystem.walkfs import WalkFSPlugin
-from tests._utils import absolute_path
 
 
 def test_walkfs_plugin(target_unix, fs_unix):
     fs_unix.map_file_entry("/path/to/some/file", VirtualFile(fs_unix, "file", None))
     fs_unix.map_file_entry("/path/to/some/other/file.ext", VirtualFile(fs_unix, "file.ext", None))
     fs_unix.map_file_entry("/root_file", VirtualFile(fs_unix, "root_file", None))
     fs_unix.map_file_entry("/other_root_file.ext", VirtualFile(fs_unix, "other_root_file.ext", None))
+    fs_unix.map_file_entry("/.test/test.txt", VirtualFile(fs_unix, "test.txt", None))
+    fs_unix.map_file_entry("/.test/.more.test.txt", VirtualFile(fs_unix, ".more.test.txt", None))
 
     target_unix.add_plugin(WalkFSPlugin)
 
     results = list(target_unix.walkfs())
-    assert len(results) == 10
+    assert len(results) == 14
     assert sorted([r.path for r in results]) == [
+        "/",
+        "/.test",
+        "/.test/.more.test.txt",
+        "/.test/test.txt",
         "/etc",
         "/other_root_file.ext",
         "/path",
@@ -26,62 +30,3 @@ def test_walkfs_plugin(target_unix, fs_unix):
         "/root_file",
         "/var",
     ]
-
-
-def test_walkfs_ext_internal(target_unix, fs_unix):
-    fs_unix.map_file_entry("/path/to/some/file", VirtualFile(fs_unix, "file", None))
-    fs_unix.map_file_entry("/path/to/some/other/file.ext", VirtualFile(fs_unix, "file.ext", None))
-    fs_unix.map_file_entry("/root_file", VirtualFile(fs_unix, "root_file", None))
-    fs_unix.map_file_entry("/other_root_file.ext", VirtualFile(fs_unix, "other_root_file.ext", None))
-
-    target_unix.add_plugin(WalkFSPlugin)
-
-    results = list(target_unix.walkfs_ext())
-    assert len(results) == 10
-    assert sorted([r.path for _, r in results]) == [
-        "/etc",
-        "/other_root_file.ext",
-        "/path",
-        "/path/to",
-        "/path/to/some",
-        "/path/to/some/file",
-        "/path/to/some/other",
-        "/path/to/some/other/file.ext",
-        "/root_file",
-        "/var",
-    ]
-
-    results = list(target_unix.walkfs_ext(root="/path"))
-    assert len(results) == 5
-    assert sorted([r.path for _, r in results]) == [
-        "/path/to",
-        "/path/to/some",
-        "/path/to/some/file",
-        "/path/to/some/other",
-        "/path/to/some/other/file.ext",
-    ]
-
-    results = list(target_unix.walkfs_ext(pattern="*.ext"))
-    assert len(results) == 2
-    assert sorted([r.path for _, r in results]) == [
-        "/other_root_file.ext",
-        "/path/to/some/other/file.ext",
-    ]
-
-    results = list(target_unix.walkfs_ext(root="/path", pattern="*.ext"))
-    assert len(results) == 1
-    assert sorted([r.path for _, r in results]) == [
-        "/path/to/some/other/file.ext",
-    ]
-
-
-def test_walkfs_dot_folder(target_unix):
-    archive_path = absolute_path("_data/loaders/tar/test-archive-dot-folder.tgz")
-    loader = TarLoader(archive_path)
-    loader.map(target_unix)
-    target_unix.add_plugin(WalkFSPlugin)
-    results = list(target_unix.walkfs())
-
-    assert len(target_unix.filesystems) == 2
-    assert len(results) == 3
-    assert target_unix.fs.path("/test.txt").exists()

tests/plugins/filesystem/test_yara.py

@@ -23,7 +23,7 @@ def test_yara_plugin(tmp_path, target_default):
     vfs.map_file_fh("test_file", BytesIO(b"test string"))
     vfs.map_file_fh("/test/dir/to/test_file", BytesIO(b"test string"))
 
-    target_default.filesystems.add(vfs)
+    target_default.fs.mount("/", vfs)
 
     with tempfile.NamedTemporaryFile(mode="w+t", dir=tmp_path, delete=False) as tmp_file:
         tmp_file.write(test_rule)