Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace the internal walkfs_ext plugin with Target.fs.walk_ext #459

Merged
merged 1 commit into from
Nov 30, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 77 additions & 66 deletions dissect/target/plugins/filesystem/unix/capability.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
from enum import IntEnum
from io import BytesIO

from dissect.target.exceptions import UnsupportedPluginError
from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
from dissect.target.helpers.record import TargetRecordDescriptor
from dissect.target.plugin import Plugin, export
from dissect.target.plugins.filesystem.walkfs import generate_record

CapabilityRecord = TargetRecordDescriptor(
"filesystem/unix/capability",
Expand Down Expand Up @@ -87,72 +88,82 @@ def check_compatible(self) -> None:
@export(record=CapabilityRecord)
def capability_binaries(self):
"""Find all files that have capabilities set."""
for entry, record in self.target.walkfs_ext():
try:
attrs = entry.get().lattr()
except Exception:
self.target.log.exception("Failed to get attrs for entry %s", entry)
continue

for attr in attrs:
if attr.name != "security.capability":
for path_entries, _, files in self.target.fs.walk_ext("/"):
entries = [path_entries[-1]] + files
for entry in entries:
path = self.target.fs.path(entry.path)
try:
record = generate_record(self.target, path)
except FileNotFoundError:
continue

buf = BytesIO(attr.value)

# Reference: https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h
# The struct is small enough we can just use struct
magic_etc = struct.unpack("<I", buf.read(4))[0]
cap_revision = magic_etc & VFS_CAP_REVISION_MASK

permitted_caps = []
inheritable_caps = []
rootid = None

if cap_revision == VFS_CAP_REVISION_1:
num_caps = VFS_CAP_U32_1
data_len = (1 + 2 * VFS_CAP_U32_1) * 4
elif cap_revision == VFS_CAP_REVISION_2:
num_caps = VFS_CAP_U32_2
data_len = (1 + 2 * VFS_CAP_U32_2) * 4
elif cap_revision == VFS_CAP_REVISION_3:
num_caps = VFS_CAP_U32_3
data_len = (2 + 2 * VFS_CAP_U32_2) * 4
else:
self.target.log.error("Unexpected capability revision: %s", entry)
try:
attrs = path.get().lattr()
except TypeError:
# Virtual(File|Directory|Symlink) instances don't have a functional lattr()
continue

if data_len != len(attr.value):
self.target.log.error("Unexpected capability length: %s", entry)
except Exception:
self.target.log.exception("Failed to get attrs for entry %s", entry)
continue

for _ in range(num_caps):
permitted_val, inheritable_val = struct.unpack("<2I", buf.read(8))
permitted_caps.append(permitted_val)
inheritable_caps.append(inheritable_val)

if cap_revision == VFS_CAP_REVISION_3:
rootid = struct.unpack("<I", buf.read(4))[0]

permitted = []
inheritable = []

for capability in Capabilities:
for caps, results in [(permitted_caps, permitted), (inheritable_caps, inheritable)]:
# CAP_TO_INDEX
cap_index = capability.value >> 5
if cap_index >= len(caps):
# We loop over all capabilities, but might only have a version 1 caps list
continue

if caps[cap_index] & (1 << (capability.value & 31)) != 0:
results.append(capability.name)

yield CapabilityRecord(
record=record,
permitted=permitted,
inheritable=inheritable,
effective=magic_etc & VFS_CAP_FLAGS_EFFECTIVE != 0,
rootid=rootid,
_target=self.target,
)
for attr in attrs:
if attr.name != "security.capability":
continue

buf = BytesIO(attr.value)

# Reference: https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h
# The struct is small enough we can just use struct
magic_etc = struct.unpack("<I", buf.read(4))[0]
cap_revision = magic_etc & VFS_CAP_REVISION_MASK

permitted_caps = []
inheritable_caps = []
rootid = None

if cap_revision == VFS_CAP_REVISION_1:
num_caps = VFS_CAP_U32_1
data_len = (1 + 2 * VFS_CAP_U32_1) * 4
elif cap_revision == VFS_CAP_REVISION_2:
num_caps = VFS_CAP_U32_2
data_len = (1 + 2 * VFS_CAP_U32_2) * 4
elif cap_revision == VFS_CAP_REVISION_3:
num_caps = VFS_CAP_U32_3
data_len = (2 + 2 * VFS_CAP_U32_2) * 4
else:
self.target.log.error("Unexpected capability revision: %s", entry)
continue

if data_len != len(attr.value):
self.target.log.error("Unexpected capability length: %s", entry)
continue

for _ in range(num_caps):
permitted_val, inheritable_val = struct.unpack("<2I", buf.read(8))
permitted_caps.append(permitted_val)
inheritable_caps.append(inheritable_val)

if cap_revision == VFS_CAP_REVISION_3:
rootid = struct.unpack("<I", buf.read(4))[0]

permitted = []
inheritable = []

for capability in Capabilities:
for caps, results in [(permitted_caps, permitted), (inheritable_caps, inheritable)]:
# CAP_TO_INDEX
cap_index = capability.value >> 5
if cap_index >= len(caps):
# We loop over all capabilities, but might only have a version 1 caps list
continue

if caps[cap_index] & (1 << (capability.value & 31)) != 0:
results.append(capability.name)

yield CapabilityRecord(
record=record,
permitted=permitted,
inheritable=inheritable,
effective=magic_etc & VFS_CAP_FLAGS_EFFECTIVE != 0,
rootid=rootid,
_target=self.target,
)
42 changes: 23 additions & 19 deletions dissect/target/plugins/filesystem/walkfs.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
from typing import Iterable

from dissect.util.ts import from_unix

from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
from dissect.target.filesystem import RootFilesystemEntry
from dissect.target.helpers.fsutil import TargetPath
from dissect.target.helpers.record import TargetRecordDescriptor
from dissect.target.plugin import Plugin, export, internal
from dissect.target.plugin import Plugin, export
from dissect.target.target import Target

FilesystemRecord = TargetRecordDescriptor(
"filesystem/entry",
Expand All @@ -17,8 +22,7 @@
("uint32", "mode"),
("uint32", "uid"),
("uint32", "gid"),
("string", "fstype"),
("uint32", "fsidx"),
("string[]", "fstypes"),
],
)

Expand All @@ -29,36 +33,36 @@ def check_compatible(self) -> None:
raise UnsupportedPluginError("No filesystems found")

@export(record=FilesystemRecord)
def walkfs(self):
def walkfs(self) -> Iterable[FilesystemRecord]:
"""Walk a target's filesystem and return all filesystem entries."""
for _, record in self.walkfs_ext():
yield record

@internal
def walkfs_ext(self, root="/", pattern="*"):
for idx, fs in enumerate(self.target.filesystems):
for entry in fs.path(root).rglob(pattern):
for path_entries, _, files in self.target.fs.walk_ext("/"):
entries = [path_entries[-1]] + files
for entry in entries:
path = self.target.fs.path(entry.path)
try:
yield entry, generate_record(self.target, entry, idx)
record = generate_record(self.target, path)
except FileNotFoundError:
continue
except Exception:
self.target.log.exception("Failed to generate record from entry %s", entry)
yield record


def generate_record(target, entry, idx):
stat = entry.lstat()
def generate_record(target: Target, path: TargetPath) -> FilesystemRecord:
stat = path.lstat()
entry = path.get()
if isinstance(entry, RootFilesystemEntry):
fs_types = [sub_entry.fs.__type__ for sub_entry in entry.entries]
else:
fs_types = [entry.fs.__type__]
return FilesystemRecord(
atime=from_unix(stat.st_atime),
mtime=from_unix(stat.st_mtime),
ctime=from_unix(stat.st_ctime),
ino=stat.st_ino,
path=entry,
path=path,
size=stat.st_size,
mode=stat.st_mode,
uid=stat.st_uid,
gid=stat.st_gid,
fstype=entry.get().fs.__type__,
fsidx=idx,
fstypes=fs_types,
_target=target,
)
39 changes: 20 additions & 19 deletions dissect/target/plugins/filesystem/yara.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
except ImportError:
raise ImportError("Please install 'yara-python' to use 'target-query -f yara'.")

from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
from dissect.target.exceptions import FileNotFoundError
from dissect.target.helpers.record import TargetRecordDescriptor
from dissect.target.plugin import Plugin, arg, export

Expand All @@ -26,8 +26,7 @@ class YaraPlugin(Plugin):
DEFAULT_MAX_SIZE = 10 * 1024 * 1024

def check_compatible(self) -> None:
if not self.target.has_function("walkfs"):
raise UnsupportedPluginError("No walkfs plugin found")
pass

@arg("--rule-files", "-r", type=Path, nargs="+", required=True, help="path to YARA rule file")
@arg("--scan-path", default="/", help="path to recursively scan")
Expand All @@ -43,20 +42,22 @@ def yara(self, rule_files, scan_path="/", max_size=DEFAULT_MAX_SIZE):
rule_data = "\n".join([rule_file.read_text() for rule_file in rule_files])

rules = yara.compile(source=rule_data)
for entry, _ in self.target.walkfs_ext(scan_path):
try:
if not entry.is_file() or entry.stat().st_size > max_size:
for _, _, files in self.target.fs.walk_ext(scan_path):
for file_entry in files:
path = self.target.fs.path(file_entry.path)
try:
if path.stat().st_size > max_size:
continue

for match in rules.match(data=path.read_bytes()):
yield YaraMatchRecord(
path=path,
digest=path.get().hash(),
rule=match.rule,
tags=match.tags,
_target=self.target,
)
except FileNotFoundError:
continue

for match in rules.match(data=entry.read_bytes()):
yield YaraMatchRecord(
path=entry,
digest=entry.get().hash(),
rule=match.rule,
tags=match.tags,
_target=self.target,
)
except FileNotFoundError:
continue
except Exception:
self.target.log.exception("Error scanning file: %s", entry)
except Exception:
self.target.log.exception("Error scanning file: %s", path)
Binary file removed tests/_data/loaders/tar/test-archive-dot-folder.tgz
Binary file not shown.
69 changes: 7 additions & 62 deletions tests/plugins/filesystem/test_walkfs.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,24 @@
from dissect.target.filesystem import VirtualFile
from dissect.target.loaders.tar import TarLoader
from dissect.target.plugins.filesystem.walkfs import WalkFSPlugin
from tests._utils import absolute_path


def test_walkfs_plugin(target_unix, fs_unix):
fs_unix.map_file_entry("/path/to/some/file", VirtualFile(fs_unix, "file", None))
fs_unix.map_file_entry("/path/to/some/other/file.ext", VirtualFile(fs_unix, "file.ext", None))
fs_unix.map_file_entry("/root_file", VirtualFile(fs_unix, "root_file", None))
fs_unix.map_file_entry("/other_root_file.ext", VirtualFile(fs_unix, "other_root_file.ext", None))
fs_unix.map_file_entry("/.test/test.txt", VirtualFile(fs_unix, "test.txt", None))
fs_unix.map_file_entry("/.test/.more.test.txt", VirtualFile(fs_unix, ".more.test.txt", None))

target_unix.add_plugin(WalkFSPlugin)

results = list(target_unix.walkfs())
assert len(results) == 10
assert len(results) == 14
assert sorted([r.path for r in results]) == [
"/",
"/.test",
"/.test/.more.test.txt",
"/.test/test.txt",
"/etc",
"/other_root_file.ext",
"/path",
Expand All @@ -26,62 +30,3 @@ def test_walkfs_plugin(target_unix, fs_unix):
"/root_file",
"/var",
]


def test_walkfs_ext_internal(target_unix, fs_unix):
fs_unix.map_file_entry("/path/to/some/file", VirtualFile(fs_unix, "file", None))
fs_unix.map_file_entry("/path/to/some/other/file.ext", VirtualFile(fs_unix, "file.ext", None))
fs_unix.map_file_entry("/root_file", VirtualFile(fs_unix, "root_file", None))
fs_unix.map_file_entry("/other_root_file.ext", VirtualFile(fs_unix, "other_root_file.ext", None))

target_unix.add_plugin(WalkFSPlugin)

results = list(target_unix.walkfs_ext())
assert len(results) == 10
assert sorted([r.path for _, r in results]) == [
"/etc",
"/other_root_file.ext",
"/path",
"/path/to",
"/path/to/some",
"/path/to/some/file",
"/path/to/some/other",
"/path/to/some/other/file.ext",
"/root_file",
"/var",
]

results = list(target_unix.walkfs_ext(root="/path"))
assert len(results) == 5
assert sorted([r.path for _, r in results]) == [
"/path/to",
"/path/to/some",
"/path/to/some/file",
"/path/to/some/other",
"/path/to/some/other/file.ext",
]

results = list(target_unix.walkfs_ext(pattern="*.ext"))
assert len(results) == 2
assert sorted([r.path for _, r in results]) == [
"/other_root_file.ext",
"/path/to/some/other/file.ext",
]

results = list(target_unix.walkfs_ext(root="/path", pattern="*.ext"))
assert len(results) == 1
assert sorted([r.path for _, r in results]) == [
"/path/to/some/other/file.ext",
]


def test_walkfs_dot_folder(target_unix):
archive_path = absolute_path("_data/loaders/tar/test-archive-dot-folder.tgz")
loader = TarLoader(archive_path)
loader.map(target_unix)
target_unix.add_plugin(WalkFSPlugin)
results = list(target_unix.walkfs())

assert len(target_unix.filesystems) == 2
assert len(results) == 3
assert target_unix.fs.path("/test.txt").exists()
2 changes: 1 addition & 1 deletion tests/plugins/filesystem/test_yara.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def test_yara_plugin(tmp_path, target_default):
vfs.map_file_fh("test_file", BytesIO(b"test string"))
vfs.map_file_fh("/test/dir/to/test_file", BytesIO(b"test string"))

target_default.filesystems.add(vfs)
target_default.fs.mount("/", vfs)

with tempfile.NamedTemporaryFile(mode="w+t", dir=tmp_path, delete=False) as tmp_file:
tmp_file.write(test_rule)
Expand Down
Loading