diff --git a/.github/workflows/dissect-ci.yml b/.github/workflows/dissect-ci.yml index abba3544d..a0b36ec93 100644 --- a/.github/workflows/dissect-ci.yml +++ b/.github/workflows/dissect-ci.yml @@ -11,6 +11,8 @@ on: jobs: ci: uses: fox-it/dissect-workflow-templates/.github/workflows/dissect-ci-template.yml@main + with: + deb-packages: 'liblzo2-dev' secrets: inherit publish: diff --git a/.github/workflows/on-demand-test.yml b/.github/workflows/on-demand-test.yml index 133df0aa4..ae039e936 100644 --- a/.github/workflows/on-demand-test.yml +++ b/.github/workflows/on-demand-test.yml @@ -16,6 +16,7 @@ jobs: python-include: "pypy3.9" tox-env: "pypy39" steps: + - run: sudo apt-get install -qq liblzo2-dev - uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/dissect/target/helpers/protobuf.py b/dissect/target/helpers/protobuf.py index 2cfac0923..a6d4b26fd 100644 --- a/dissect/target/helpers/protobuf.py +++ b/dissect/target/helpers/protobuf.py @@ -3,30 +3,26 @@ from typing import Any, BinaryIO from dissect.cstruct.types.base import BaseType -from dissect.cstruct.types.bytesinteger import BytesInteger -class ProtobufVarint(BytesInteger): +class ProtobufVarint(BaseType): """Implements a protobuf integer type for dissect.cstruct that can span a variable amount of bytes. - Mainly follows the cstruct BytesInteger implementation with minor tweaks - to support protobuf's msb varint implementation. + Supports protobuf's msb varint implementation. Resources: - https://protobuf.dev/programming-guides/encoding/ - https://github.com/protocolbuffers/protobuf/blob/main/python/google/protobuf/internal/decoder.py """ - def _read(self, stream: BinaryIO, context: dict[str, Any] = None) -> int: + @classmethod + def _read(cls, stream: BinaryIO, context: dict[str, Any] = None) -> int: return decode_varint(stream) - def _write(self, stream: BinaryIO, data: int) -> int: + @classmethod + def _write(cls, stream: BinaryIO, data: int) -> int: return stream.write(encode_varint(data)) - _read_array = BaseType._read_array - - _write_array = BaseType._write_array - def decode_varint(stream: BinaryIO) -> int: """Reads a varint from the provided buffer stream. diff --git a/dissect/target/helpers/ssh.py b/dissect/target/helpers/ssh.py index 88d07e5ef..34e26b7a5 100644 --- a/dissect/target/helpers/ssh.py +++ b/dissect/target/helpers/ssh.py @@ -1,9 +1,9 @@ import base64 import binascii -from dissect import cstruct +from dissect.cstruct import cstruct -c_rfc4716_def = """ +rfc4716_def = """ struct ssh_string { uint32 length; char value[length]; @@ -23,8 +23,7 @@ } """ -c_rfc4716 = cstruct.cstruct(endian=">") -c_rfc4716.load(c_rfc4716_def) +c_rfc4716 = cstruct(endian=">").load(rfc4716_def) RFC4716_MARKER_START = b"-----BEGIN OPENSSH PRIVATE KEY-----" RFC4716_MARKER_END = b"-----END OPENSSH PRIVATE KEY-----" diff --git a/dissect/target/plugins/apps/av/trendmicro.py b/dissect/target/plugins/apps/av/trendmicro.py index dcef53d16..0bbeff59a 100644 --- a/dissect/target/plugins/apps/av/trendmicro.py +++ b/dissect/target/plugins/apps/av/trendmicro.py @@ -1,6 +1,6 @@ from typing import Iterator -from dissect import cstruct +from dissect.cstruct import cstruct from dissect.util.ts import from_unix from dissect.target import Target @@ -47,8 +47,7 @@ char _pad3[10]; }; """ -c_pfwlog = cstruct.cstruct() -c_pfwlog.load(pfwlog_def) +c_pfwlog = cstruct().load(pfwlog_def) class TrendMicroPlugin(Plugin): diff --git a/dissect/target/plugins/apps/container/docker.py b/dissect/target/plugins/apps/container/docker.py index 881b75e29..ff360ed7c 100644 --- a/dissect/target/plugins/apps/container/docker.py +++ b/dissect/target/plugins/apps/container/docker.py @@ -88,7 +88,7 @@ """ c_local = cstruct(endian=">") -c_local.addtype("varint", ProtobufVarint(c_local, "varint", size=None, signed=False, alignment=1)) +c_local.add_custom_type("varint", ProtobufVarint, size=None, alignment=1, signed=False) c_local.load(local_def, compiled=False) RE_DOCKER_NS = re.compile(r"\.(?P\d{7,})(?PZ|\+\d{2}:\d{2})") diff --git a/dissect/target/plugins/os/unix/locate/gnulocate.py b/dissect/target/plugins/os/unix/locate/gnulocate.py index 2d69f530c..413dd00fd 100644 --- a/dissect/target/plugins/os/unix/locate/gnulocate.py +++ b/dissect/target/plugins/os/unix/locate/gnulocate.py @@ -26,8 +26,7 @@ ], ) -c_gnulocate = cstruct() -c_gnulocate.load(gnulocate_def) +c_gnulocate = cstruct().load(gnulocate_def) class GNULocateFile: diff --git a/dissect/target/plugins/os/unix/locate/mlocate.py b/dissect/target/plugins/os/unix/locate/mlocate.py index cc31d8a7e..6cfbaddea 100644 --- a/dissect/target/plugins/os/unix/locate/mlocate.py +++ b/dissect/target/plugins/os/unix/locate/mlocate.py @@ -20,10 +20,10 @@ int32 conf_size; int8 version; /* file format version */ int8 require_visibility; - int8 pad[2]; /* 32-bit total alignment */ + int8 pad0[2]; /* 32-bit total alignment */ char root_database; char config_block[conf_size]; - int8 pad; + int8 pad1; }; enum DBE_TYPE: uint8 { /* database entry type */ @@ -68,8 +68,7 @@ class MLocate: ], ) -c_mlocate = cstruct(endian=">") -c_mlocate.load(mlocate_def) +c_mlocate = cstruct(endian=">").load(mlocate_def) class MLocateFile: diff --git a/dissect/target/plugins/os/unix/locate/plocate.py b/dissect/target/plugins/os/unix/locate/plocate.py index 9b70395fa..d878e4f99 100644 --- a/dissect/target/plugins/os/unix/locate/plocate.py +++ b/dissect/target/plugins/os/unix/locate/plocate.py @@ -65,8 +65,7 @@ ], ) -c_plocate = cstruct() -c_plocate.load(plocate_def) +c_plocate = cstruct().load(plocate_def) class PLocateFile: diff --git a/dissect/target/plugins/os/unix/log/atop.py b/dissect/target/plugins/os/unix/log/atop.py index 2a3a392a6..a7bd07b41 100644 --- a/dissect/target/plugins/os/unix/log/atop.py +++ b/dissect/target/plugins/os/unix/log/atop.py @@ -2,7 +2,7 @@ from io import BytesIO from typing import BinaryIO, Iterator -from dissect.cstruct import Instance, cstruct +from dissect.cstruct import cstruct from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.record import TargetRecordDescriptor @@ -178,8 +178,7 @@ }; """ # noqa: E501 -c_atop = cstruct() -c_atop.load(atop_def) +c_atop = cstruct().load(atop_def) c_atop.load(atop_tstat_def, align=True) AtopRecord = TargetRecordDescriptor( @@ -226,7 +225,7 @@ def __init__(self, fh: BinaryIO): self.header = c_atop.rawheader(self.fh) self.version = self.version() - def __iter__(self) -> Iterator[Instance]: + def __iter__(self) -> Iterator[c_atop.tstat]: while True: try: record = c_atop.rawrecord(self.fh) diff --git a/dissect/target/plugins/os/unix/log/journal.py b/dissect/target/plugins/os/unix/log/journal.py index ab9ca91fe..4c9e3b05e 100644 --- a/dissect/target/plugins/os/unix/log/journal.py +++ b/dissect/target/plugins/os/unix/log/journal.py @@ -1,8 +1,10 @@ +from __future__ import annotations + import lzma from typing import BinaryIO, Callable, Iterator import zstandard -from dissect.cstruct import Instance, cstruct +from dissect.cstruct import cstruct from dissect.util import ts from dissect.util.compression import lz4 @@ -252,8 +254,7 @@ }; """ # noqa: E501 -c_journal = cstruct() -c_journal.load(journal_def) +c_journal = cstruct().load(journal_def) def get_optional(value: str, to_type: Callable): @@ -314,7 +315,7 @@ def decode_value(self, value: bytes) -> tuple[str, str]: return key, value - def __iter__(self) -> Iterator[Instance]: + def __iter__(self) -> Iterator[dict[str, int | str]]: "Iterate over the entry objects to read payloads." for offset in self.entry_object_offsets(): diff --git a/dissect/target/plugins/os/unix/log/lastlog.py b/dissect/target/plugins/os/unix/log/lastlog.py index 4da06e7ad..3970a908b 100644 --- a/dissect/target/plugins/os/unix/log/lastlog.py +++ b/dissect/target/plugins/os/unix/log/lastlog.py @@ -1,6 +1,6 @@ from typing import BinaryIO -from dissect import cstruct +from dissect.cstruct import cstruct from dissect.util import ts from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError @@ -36,8 +36,7 @@ }; """ -c_lastlog = cstruct.cstruct() -c_lastlog.load(lastlog_def) +c_lastlog = cstruct().load(lastlog_def) class LastLogFile: diff --git a/dissect/target/plugins/os/unix/log/utmp.py b/dissect/target/plugins/os/unix/log/utmp.py index 76a9a1abb..9556fadba 100644 --- a/dissect/target/plugins/os/unix/log/utmp.py +++ b/dissect/target/plugins/os/unix/log/utmp.py @@ -39,14 +39,14 @@ ], ) -c_utmp = """ +utmp_def = """ #define UT_LINESIZE 32 #define UT_NAMESIZE 32 #define UT_HOSTSIZE 256 typedef uint32 pid_t; -enum Type : char { +enum Type : uint8_t { EMPTY = 0x0, RUN_LVL = 0x1, BOOT_TIME = 0x2, @@ -84,8 +84,7 @@ }; """ # noqa: E501 -utmp = cstruct() -utmp.load(c_utmp) +c_utmp = cstruct().load(utmp_def) UTMP_ENTRY = namedtuple( "UTMPRecord", @@ -122,11 +121,11 @@ def __iter__(self): while True: try: - entry = utmp.entry(byte_stream) + entry = c_utmp.entry(byte_stream) r_type = "" - if entry.ut_type in utmp.Type.reverse: - r_type = utmp.Type.reverse[entry.ut_type] + if entry.ut_type in c_utmp.Type: + r_type = c_utmp.Type(entry.ut_type).name ut_host = entry.ut_host.decode(errors="surrogateescape").strip("\x00") ut_addr = None diff --git a/dissect/target/plugins/os/windows/adpolicy.py b/dissect/target/plugins/os/windows/adpolicy.py index 64e3ef384..dc4424cd2 100644 --- a/dissect/target/plugins/os/windows/adpolicy.py +++ b/dissect/target/plugins/os/windows/adpolicy.py @@ -1,7 +1,7 @@ from struct import unpack from defusedxml import ElementTree -from dissect import cstruct +from dissect.cstruct import cstruct from dissect.regf.c_regf import ( REG_BINARY, REG_DWORD, @@ -18,14 +18,13 @@ from dissect.target.helpers.record import TargetRecordDescriptor from dissect.target.plugin import Plugin, export -c_def = """ +policy_def = """ struct registry_policy_header { uint32 signature; uint32 version; }; """ -c_adpolicy = cstruct.cstruct() -c_adpolicy.load(c_def) +c_adpolicy = cstruct().load(policy_def) ADPolicyRecord = TargetRecordDescriptor( "windows/adpolicy", diff --git a/dissect/target/plugins/os/windows/credhist.py b/dissect/target/plugins/os/windows/credhist.py index 875033bb8..719f177d6 100644 --- a/dissect/target/plugins/os/windows/credhist.py +++ b/dissect/target/plugins/os/windows/credhist.py @@ -53,8 +53,7 @@ }; """ -c_credhist = cstruct() -c_credhist.load(credhist_def) +c_credhist = cstruct().load(credhist_def) @dataclass diff --git a/dissect/target/plugins/os/windows/datetime.py b/dissect/target/plugins/os/windows/datetime.py index 7c772693d..55d7c9967 100644 --- a/dissect/target/plugins/os/windows/datetime.py +++ b/dissect/target/plugins/os/windows/datetime.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta, timezone, tzinfo from typing import Dict, Tuple -from dissect import cstruct +from dissect.cstruct import cstruct from dissect.target.exceptions import ( RegistryError, @@ -34,8 +34,7 @@ SYSTEMTIME DaylightDate; } REG_TZI_FORMAT; """ -c_tz = cstruct.cstruct() -c_tz.load(tz_def) +c_tz = cstruct().load(tz_def) # Althoug calendar.SUNDAY is only officially documented since Python 3.10, it @@ -63,7 +62,7 @@ HOUR = timedelta(hours=1) -def parse_systemtime_transition(systemtime: cstruct.Instance, year: int) -> datetime: +def parse_systemtime_transition(systemtime: c_tz._SYSTEMTIME, year: int) -> datetime: """Return the transition datetime for a given year using the SYSTEMTIME of a STD or DST transition date. The SYSTEMTIME date of a TZI structure needs to be used to calculate the actual date for a given year. diff --git a/dissect/target/plugins/os/windows/defender.py b/dissect/target/plugins/os/windows/defender.py index 243d82d89..63f0b4b16 100644 --- a/dissect/target/plugins/os/windows/defender.py +++ b/dissect/target/plugins/os/windows/defender.py @@ -237,8 +237,7 @@ }; """ -c_defender = cstruct() -c_defender.load(defender_def) +c_defender = cstruct().load(defender_def) STREAM_ID = c_defender.STREAM_ID STREAM_ATTRIBUTES = c_defender.STREAM_ATTRIBUTES @@ -381,7 +380,7 @@ def _add_field(self, field: Structure): self.last_access_time = ts.wintimestamp(int.from_bytes(field.Data, "little")) elif field.Identifier == FIELD_IDENTIFIER.LastWriteTime: self.last_write_time = ts.wintimestamp(int.from_bytes(field.Data, "little")) - elif field.Identifier not in FIELD_IDENTIFIER.values.values(): + elif field.Identifier not in FIELD_IDENTIFIER: self.unknown_fields.append(field) @@ -526,7 +525,7 @@ def recover(self, output_dir: Path) -> None: subdir = resource.resource_id[0:2] resourcedata_location = resourcedata_directory.joinpath(subdir).joinpath(resource.resource_id) if not resourcedata_location.exists(): - self.target.log.warning(f"Could not find a ResourceData file for {entry.resource_id}.") + self.target.log.warning(f"Could not find a ResourceData file for {resource.resource_id}.") continue if not resourcedata_location.is_file(): self.target.log.warning(f"{resourcedata_location} is not a file!") diff --git a/dissect/target/plugins/os/windows/dpapi/blob.py b/dissect/target/plugins/os/windows/dpapi/blob.py index fd3e37923..78229df9e 100644 --- a/dissect/target/plugins/os/windows/dpapi/blob.py +++ b/dissect/target/plugins/os/windows/dpapi/blob.py @@ -36,8 +36,7 @@ }; """ -c_blob = cstruct() -c_blob.load(blob_def) +c_blob = cstruct().load(blob_def) class Blob: diff --git a/dissect/target/plugins/os/windows/dpapi/master_key.py b/dissect/target/plugins/os/windows/dpapi/master_key.py index 061f517bc..1875882dc 100644 --- a/dissect/target/plugins/os/windows/dpapi/master_key.py +++ b/dissect/target/plugins/os/windows/dpapi/master_key.py @@ -29,7 +29,7 @@ DWORD accessCheckLen; char guid[16]; char encryptedSecret[secretLen]; - char accessCheckLen[accessCheckLen]; + char accessCheck[accessCheckLen]; }; struct CredHist { @@ -66,8 +66,7 @@ QWORD qwDomainKeySize; }; """ -c_master_key = cstruct() -c_master_key.load(master_key_def) +c_master_key = cstruct().load(master_key_def) class MasterKey: diff --git a/dissect/target/plugins/os/windows/notifications.py b/dissect/target/plugins/os/windows/notifications.py index 4bbaf5188..f932f5c0e 100644 --- a/dissect/target/plugins/os/windows/notifications.py +++ b/dissect/target/plugins/os/windows/notifications.py @@ -91,8 +91,7 @@ } Chunk; // size: 0x23810 """ -c_appdb = cstruct(endian="<") -c_appdb.load(appdb_def) +c_appdb = cstruct(endian="<").load(appdb_def) APPDB_MAGIC = b"DNPW" NUM_APPDB_CHUNKS = 256 diff --git a/dissect/target/plugins/os/windows/prefetch.py b/dissect/target/plugins/os/windows/prefetch.py index c298e3ae0..e43549e87 100644 --- a/dissect/target/plugins/os/windows/prefetch.py +++ b/dissect/target/plugins/os/windows/prefetch.py @@ -1,6 +1,6 @@ from io import BytesIO -from dissect import cstruct +from dissect.cstruct import cstruct from dissect.util import lzxpress_huffman from dissect.util.ts import wintimestamp @@ -33,7 +33,7 @@ ) -c_prefetch = """ +prefetch_def = """ struct PREFETCH_HEADER_DETECT { char signature[4]; uint32 size; @@ -59,14 +59,14 @@ uint32 volumes_information_offset; uint32 number_of_volumes; uint32 volumes_information_size; - uint32 unknown[2]; + uint32 unknown0[2]; uint64 last_run_time; uint64 last_run_remains[7]; - uint64 unknown[2]; + uint64 unknown1[2]; uint32 run_count; - uint32 unknown; - uint32 unknown; - char unknown[88]; + uint32 unknown2; + uint32 unknown3; + char unknown4[88]; }; struct FILE_INFORMATION_17 { @@ -80,9 +80,9 @@ uint32 number_of_volumes; uint32 volumes_information_size; uint32 last_run_time; - uint32 unknown; + uint32 unknown0; uint32 run_count; - uint32 unknown; + uint32 unknown1; }; struct FILE_INFORMATION_23 { @@ -99,9 +99,9 @@ uint64 last_run_time; uint64 last_run_remains[2]; uint32 run_count; - uint32 unknown; - uint32 unknown; - char unknown[80]; + uint32 unknown0; + uint32 unknown1; + char unknown2[80]; }; struct VOLUME_INFORMATION_17 { @@ -125,19 +125,19 @@ uint32 file_reference_size; uint32 directory_strings_array_offset; uint32 number_of_directory_strings; - char unknown[4]; - char unknown[24]; - char unknown[4]; - char unknown[24]; - char unknown[4]; + char unknown0[4]; + char unknown1[24]; + char unknown2[4]; + char unknown3[24]; + char unknown4[4]; }; struct TRACE_CHAIN_ARRAY_ENTRY_17 { uint32 next_array_entry_index; uint32 total_block_load_count; - uint32 unknown; - uint32 unknown; - uint32 unknown; + uint32 unknown0; + uint32 unknown1; + uint32 unknown2; }; struct FILE_METRICS_ARRAY_ENTRY_17 { @@ -158,25 +158,24 @@ uint64 ntfs_reference; }; """ -prefetch = cstruct.cstruct() -prefetch.load(c_prefetch) +c_prefetch = cstruct().load(prefetch_def) prefetch_version_structs = { - 17: (prefetch.FILE_INFORMATION_17, prefetch.FILE_METRICS_ARRAY_ENTRY_17), - 23: (prefetch.FILE_INFORMATION_23, prefetch.FILE_METRICS_ARRAY_ENTRY_23), - 30: (prefetch.FILE_INFORMATION_26, prefetch.FILE_METRICS_ARRAY_ENTRY_23), + 17: (c_prefetch.FILE_INFORMATION_17, c_prefetch.FILE_METRICS_ARRAY_ENTRY_17), + 23: (c_prefetch.FILE_INFORMATION_23, c_prefetch.FILE_METRICS_ARRAY_ENTRY_23), + 30: (c_prefetch.FILE_INFORMATION_26, c_prefetch.FILE_METRICS_ARRAY_ENTRY_23), } class Prefetch: def __init__(self, fh): - header_detect = prefetch.PREFETCH_HEADER_DETECT(fh.read(8)) + header_detect = c_prefetch.PREFETCH_HEADER_DETECT(fh.read(8)) if header_detect.signature == b"MAM\x04": fh = BytesIO(lzxpress_huffman.decompress(fh)) self.fh = fh self.fh.seek(0) - self.header = prefetch.PREFETCH_HEADER(self.fh) + self.header = c_prefetch.PREFETCH_HEADER(self.fh) self.version = self.identify() self.volumes = None self.metrics = None diff --git a/dissect/target/plugins/os/windows/recyclebin.py b/dissect/target/plugins/os/windows/recyclebin.py index b0d35c1bb..3dea8254f 100644 --- a/dissect/target/plugins/os/windows/recyclebin.py +++ b/dissect/target/plugins/os/windows/recyclebin.py @@ -1,6 +1,8 @@ +from __future__ import annotations + from typing import Generator -from dissect import cstruct +from dissect.cstruct import cstruct from dissect.util.ts import wintimestamp from dissect.target import Target @@ -21,7 +23,7 @@ ], ) -c_recyclebin_i = """ +recyclebin_def = """ struct header_v1 { int64 version; int64 file_size; @@ -37,14 +39,14 @@ }; """ +c_recyclebin = cstruct().load(recyclebin_def) + class RecyclebinPlugin(Plugin): """Recyclebin plugin.""" def __init__(self, target: Target) -> None: super().__init__(target) - self.recyclebin_parser = cstruct.cstruct() - self.recyclebin_parser.load(c_recyclebin_i) def check_compatible(self) -> None: for fs_entry in self.target.fs.path("/").iterdir(): @@ -131,11 +133,11 @@ def find_sid(self, path: TargetPath) -> str: return "unknown" return parent_path.name - def select_header(self, data: bytes) -> cstruct.Structure: + def select_header(self, data: bytes) -> c_recyclebin.header_v1 | c_recyclebin.header_v2: """Selects the correct header based on the version field in the header""" - header_version = self.recyclebin_parser.uint64(data[:8]) + header_version = c_recyclebin.uint64(data[:8]) if header_version == 2: - return self.recyclebin_parser.header_v2 + return c_recyclebin.header_v2 else: - return self.recyclebin_parser.header_v1 + return c_recyclebin.header_v1 diff --git a/dissect/target/plugins/os/windows/regf/auditpol.py b/dissect/target/plugins/os/windows/regf/auditpol.py index 0a007088b..84911930f 100644 --- a/dissect/target/plugins/os/windows/regf/auditpol.py +++ b/dissect/target/plugins/os/windows/regf/auditpol.py @@ -1,14 +1,12 @@ import io -from dissect import cstruct +from dissect.cstruct import cstruct from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.record import TargetRecordDescriptor from dissect.target.plugin import Plugin, export -c_adtev = cstruct.cstruct() -c_adtev.load( - """ +adtev_def = """ struct header { uint16 unk0; uint16 unk1; @@ -18,7 +16,8 @@ uint16 unk3; }; """ -) + +c_adtev = cstruct().load(adtev_def) POLICY_CATEGORIES = [ "System", diff --git a/dissect/target/plugins/os/windows/regf/bam.py b/dissect/target/plugins/os/windows/regf/bam.py index 3deae0c80..8191ea443 100644 --- a/dissect/target/plugins/os/windows/regf/bam.py +++ b/dissect/target/plugins/os/windows/regf/bam.py @@ -5,13 +5,12 @@ from dissect.target.helpers.record import TargetRecordDescriptor from dissect.target.plugin import Plugin, export -c_bamdef = """ +bam_def = """ struct entry { uint64 ts; }; """ -c_bam = cstruct() -c_bam.load(c_bamdef) +c_bam = cstruct().load(bam_def) BamDamRecord = TargetRecordDescriptor( "windows/registry/bam", diff --git a/dissect/target/plugins/os/windows/regf/cit.py b/dissect/target/plugins/os/windows/regf/cit.py index ece42189b..f4ab41ef0 100644 --- a/dissect/target/plugins/os/windows/regf/cit.py +++ b/dissect/target/plugins/os/windows/regf/cit.py @@ -212,8 +212,7 @@ } CIT_DP_DATA; """ -c_cit = cstruct() -c_cit.load(cit_def) +c_cit = cstruct().load(cit_def) CITSystemRecord = TargetRecordDescriptor( diff --git a/dissect/target/plugins/os/windows/regf/recentfilecache.py b/dissect/target/plugins/os/windows/regf/recentfilecache.py index 58db3be01..fc4dd5d64 100644 --- a/dissect/target/plugins/os/windows/regf/recentfilecache.py +++ b/dissect/target/plugins/os/windows/regf/recentfilecache.py @@ -1,10 +1,10 @@ -from dissect import cstruct +from dissect.cstruct import cstruct from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.record import TargetRecordDescriptor from dissect.target.plugin import Plugin, export -c_recent_files_def = """ +recent_files_def = """ struct header { uint32 magic; uint32 unk0; @@ -18,8 +18,7 @@ wchar path[length + 1]; }; """ -c_recent_files = cstruct.cstruct() -c_recent_files.load(c_recent_files_def) +c_recent_files = cstruct().load(recent_files_def) RecentFileCacheRecord = TargetRecordDescriptor( "windows/recentfilecache", diff --git a/dissect/target/plugins/os/windows/regf/shellbags.py b/dissect/target/plugins/os/windows/regf/shellbags.py index 89d35a279..0dd648b9c 100644 --- a/dissect/target/plugins/os/windows/regf/shellbags.py +++ b/dissect/target/plugins/os/windows/regf/shellbags.py @@ -243,8 +243,7 @@ uint32 signature; }; """ -c_bag = cstruct() -c_bag.load(bag_def) +c_bag = cstruct().load(bag_def) DELEGATE_ITEM_IDENTIFIER = b"\x74\x1a\x59\x5e\x96\xdf\xd3\x48\x8d\x67\x17\x33\xbc\xee\x28\xba" diff --git a/dissect/target/plugins/os/windows/regf/shimcache.py b/dissect/target/plugins/os/windows/regf/shimcache.py index 56ea8cef6..8b6d785f2 100644 --- a/dissect/target/plugins/os/windows/regf/shimcache.py +++ b/dissect/target/plugins/os/windows/regf/shimcache.py @@ -21,7 +21,7 @@ ], ) -c_shimdef = """ +shim_def = """ struct NT61_HEADER { uint32 magic; uint32 num_entries; @@ -99,8 +99,7 @@ uint64 ts; }; """ -c_shim = cstruct() -c_shim.load(c_shimdef) +c_shim = cstruct().load(shim_def) MAGIC_NT61 = 0xBADC0FEE MAGIC_NT52 = 0xBADC0FFE diff --git a/dissect/target/plugins/os/windows/regf/userassist.py b/dissect/target/plugins/os/windows/regf/userassist.py index 2dcc448f4..59e62b001 100644 --- a/dissect/target/plugins/os/windows/regf/userassist.py +++ b/dissect/target/plugins/os/windows/regf/userassist.py @@ -1,6 +1,6 @@ import codecs -from dissect import cstruct +from dissect.cstruct import cstruct from dissect.util.ts import wintimestamp from dissect.target.exceptions import RegistryValueNotFoundError, UnsupportedPluginError @@ -14,13 +14,13 @@ userassist_def = """ struct VERSION5_ENTRY { - char padding[4]; + char padding0[4]; uint32 number_of_executions; uint32 application_focus_count; uint32 application_focus_duration; - char padding[44]; + char padding1[44]; uint64 timestamp; - char padding[4]; + char padding2[4]; }; struct VERSION3_ENTRY { @@ -29,8 +29,7 @@ uint64 timestamp; }; """ -c_userassist = cstruct.cstruct() -c_userassist.load(userassist_def) +c_userassist = cstruct().load(userassist_def) UserAssistRecordDescriptor = create_extended_descriptor( [ diff --git a/dissect/target/plugins/os/windows/sam.py b/dissect/target/plugins/os/windows/sam.py index e8d139466..6455b318f 100644 --- a/dissect/target/plugins/os/windows/sam.py +++ b/dissect/target/plugins/os/windows/sam.py @@ -9,14 +9,14 @@ except ImportError: HAS_CRYPTO = False -from dissect import cstruct +from dissect.cstruct import cstruct from dissect.util import ts from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.record import TargetRecordDescriptor from dissect.target.plugin import Plugin, export -c_sam_def = """ +sam_def = """ struct user_F { char unknown1[8]; uint64 t_last_login; /* Time of last login */ @@ -166,7 +166,7 @@ uint16 min_password_length; /* 0x50 */ uint16 password_history_length; /* 0x52 */ uint16 lockout_threshold; /* 0x54 */ - uint16 unknown1_1; /* 0x56 */ + uint16 unknown1_3; /* 0x56 */ uint32 server_state; /* 0x58 */ uint16 server_role; /* 0x5c */ uint16 uas_compability_required; /* 0x5e */ @@ -207,8 +207,7 @@ }; """ -c_sam = cstruct.cstruct() -c_sam.load(c_sam_def) +c_sam = cstruct().load(sam_def) SamRecord = TargetRecordDescriptor( "windows/registry/sam", diff --git a/dissect/target/plugins/os/windows/task_helpers/tasks_job.py b/dissect/target/plugins/os/windows/task_helpers/tasks_job.py index 7e5111358..358d8dc4a 100644 --- a/dissect/target/plugins/os/windows/task_helpers/tasks_job.py +++ b/dissect/target/plugins/os/windows/task_helpers/tasks_job.py @@ -2,7 +2,7 @@ import warnings from typing import Iterator, Optional -from dissect import cstruct +from dissect.cstruct import cstruct from flow.record import GroupedRecord from dissect.target.exceptions import InvalidTaskError @@ -135,8 +135,7 @@ // uint8 job_signature[64 * s_ver * c_ver]; /* - calculated job signature. */ }; """ -atjob = cstruct.cstruct() -atjob.load(atjob_def) +c_atjob = cstruct().load(atjob_def) class AtTask: @@ -149,7 +148,7 @@ class AtTask: def __init__(self, job_file: TargetPath, target: Target): try: - self.at_data = atjob.ATJOB_DATA(job_file.open()) + self.at_data = c_atjob.ATJOB_DATA(job_file.open()) except Exception as e: raise InvalidTaskError(e) diff --git a/pyproject.toml b/pyproject.toml index f0a627b09..c907b8e1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,14 +26,14 @@ classifiers = [ ] dependencies = [ "defusedxml", - "dissect.cstruct>=3.0.dev,<4.0.dev", - "dissect.eventlog>=3.0.dev,<4.0.dev", - "dissect.evidence>=3.0.dev,<4.0.dev", - "dissect.hypervisor>=3.0.dev,<4.0.dev", - "dissect.ntfs>=3.4.dev,<4.0.dev", - "dissect.regf>=3.3.dev,<4.0.dev", - "dissect.util>=3.0.dev,<4.0.dev", - "dissect.volume>=3.0.dev,<4.0.dev", + "dissect.cstruct>=4.dev,<5", + "dissect.eventlog>=3,<4", + "dissect.evidence>=3,<4", + "dissect.hypervisor>=3,<4", + "dissect.ntfs>=3.4,<4", + "dissect.regf>=3.3,<4", + "dissect.util>=3,<4", + "dissect.volume>=2,<4", "flow.record~=3.15.0", "structlog", ] @@ -47,21 +47,21 @@ repository = "https://github.com/fox-it/dissect.target" [project.optional-dependencies] full = [ "asn1crypto", - "dissect.btrfs>=1.0.dev,<2.0.dev", - "dissect.cim>=3.0.dev,<4.0.dev", - "dissect.clfs>=1.0.dev,<2.0.dev", - "dissect.esedb>=3.0.dev,<4.0.dev", - "dissect.etl>=3.0.dev,<4.0.dev", - "dissect.extfs>=3.0.dev,<4.0.dev", - "dissect.fat>=3.0.dev,<4.0.dev", - "dissect.ffs>=3.0.dev,<4.0.dev", - "dissect.jffs>=1.0.dev,<2.0.dev", - "dissect.shellitem>=3.0.dev,<4.0.dev", - "dissect.squashfs>=1.0.dev,<2.0.dev", - "dissect.sql>=3.0.dev,<4.0.dev", - "dissect.thumbcache>=1.0.dev,<2.0.dev", - "dissect.vmfs>=3.0.dev,<4.0.dev", - "dissect.xfs>=3.0.dev,<4.0.dev", + "dissect.btrfs>=1,<2", + "dissect.cim>=3,<4", + "dissect.clfs>=1,<2", + "dissect.esedb>=3,<4", + "dissect.etl>=3,<4", + "dissect.extfs>=3,<4", + "dissect.fat>=3,<4", + "dissect.ffs>=3,<4", + "dissect.jffs>=1,<2", + "dissect.shellitem>=3,<4", + "dissect.squashfs>=1,<2", + "dissect.sql>=3,<4", + "dissect.thumbcache>=1,<2", + "dissect.vmfs>=3,<4", + "dissect.xfs>=3,<4", "ipython", "fusepy", "pycryptodome", @@ -76,6 +76,32 @@ full = [ # flow.record, we define it as a dependency of dissect.target. "zstandard", ] +dev = [ + "dissect.target[full,mqtt,yara]", + "dissect.btrfs[dev]>=1.0.dev,<2.0.dev", + "dissect.cim[dev]>=3.0.dev,<4.0.dev", + "dissect.clfs[dev]>=1.0.dev,<2.0.dev", + "dissect.cstruct>=4.0.dev,<5.0.dev", + "dissect.esedb[dev]>=3.0.dev,<4.0.dev", + "dissect.etl[dev]>=3.0.dev,<4.0.dev", + "dissect.eventlog[dev]>=3.0.dev,<4.0.dev", + "dissect.evidence[dev]>=3.0.dev,<4.0.dev", + "dissect.extfs[dev]>=3.0.dev,<4.0.dev", + "dissect.fat[dev]>=3.0.dev,<4.0.dev", + "dissect.ffs[dev]>=3.0.dev,<4.0.dev", + "dissect.hypervisor[dev]>=3.0.dev,<4.0.dev", + "dissect.jffs[dev]>=1.0.dev,<2.0.dev", + "dissect.ntfs[dev]>=3.4.dev,<4.0.dev", + "dissect.regf[dev]>=3.3.dev,<4.0.dev", + "dissect.shellitem[dev]>=3.0.dev,<4.0.dev", + "dissect.sql[dev]>=3.0.dev,<4.0.dev", + "dissect.squashfs[dev]>=1.0.dev,<2.0.dev", + "dissect.thumbcache[dev]>=1.0.dev,<2.0.dev", + "dissect.util>=3.0.dev,<4.0.dev", + "dissect.vmfs[dev]>=3.0.dev,<4.0.dev", + "dissect.volume[dev]>=3.0.dev,<4.0.dev", + "dissect.xfs[dev]>=3.0.dev,<4.0.dev", +] yara = [ # Grab the dependencies for dissect.target "dissect.target[full]", diff --git a/tests/helpers/test_protobuf.py b/tests/helpers/test_protobuf.py index e52bc4ca7..609ce7219 100644 --- a/tests/helpers/test_protobuf.py +++ b/tests/helpers/test_protobuf.py @@ -37,7 +37,7 @@ def test_protobuf_varint_cstruct() -> None: }; """ cs = cstruct(endian=">") - cs.addtype("varint", ProtobufVarint(cs, "varint", size=None, signed=False, alignment=1)) + cs.add_custom_type("varint", ProtobufVarint, size=None, alignment=1, signed=False) cs.load(struct_def, compiled=False) aaa = b"a" * 123456 diff --git a/tests/plugins/os/windows/regf/test_cit.py b/tests/plugins/os/windows/regf/test_cit.py index 7d4d19c7a..772081767 100644 --- a/tests/plugins/os/windows/regf/test_cit.py +++ b/tests/plugins/os/windows/regf/test_cit.py @@ -174,7 +174,10 @@ def test_cit_telemetry_plugin(target_win, hive_hklm): assert results[0].version == 1705 assert results[0].path == "\\Device\\HarddiskVolume2\\Windows\\System32\\taskhost.exe" assert results[0].value == "DEVICECHANGE" - assert results[1].value == "POWERBROADCAST|DEVICECHANGE" + + output_value = results[1].value.split("|") + output_value = sorted(output_value) + assert output_value == ["DEVICECHANGE", "POWERBROADCAST"] def test_cit_modules_plugin(target_win, hive_hklm): diff --git a/tests/plugins/os/windows/test_prefetch.py b/tests/plugins/os/windows/test_prefetch.py index c1fb982a6..b43ca77de 100644 --- a/tests/plugins/os/windows/test_prefetch.py +++ b/tests/plugins/os/windows/test_prefetch.py @@ -7,14 +7,14 @@ @pytest.fixture() def mocked_cstruct(version): - with patch.object(prefetch, "prefetch") as mocked_cstruct: + with patch.object(prefetch, "c_prefetch") as mocked_cstruct: mocked_cstruct.PREFETCH_HEADER.return_value.version = version yield mocked_cstruct @pytest.fixture() def mocked_prefetch(): - with patch.object(prefetch, "prefetch"): + with patch.object(prefetch, "c_prefetch"): with patch.multiple(prefetch.Prefetch, identify=Mock(), parse=Mock()): return prefetch.Prefetch(Mock()) diff --git a/tests/plugins/os/windows/test_recyclebin.py b/tests/plugins/os/windows/test_recyclebin.py index f5ad61174..733a9198f 100644 --- a/tests/plugins/os/windows/test_recyclebin.py +++ b/tests/plugins/os/windows/test_recyclebin.py @@ -6,7 +6,7 @@ from dissect.target.exceptions import UnsupportedPluginError from dissect.target.filesystem import VirtualFilesystem -from dissect.target.plugins.os.windows.recyclebin import RecyclebinPlugin +from dissect.target.plugins.os.windows.recyclebin import RecyclebinPlugin, c_recyclebin @pytest.fixture @@ -78,7 +78,7 @@ def test_parse_header(target_win, version_number, expected_header): recycle_plugin = RecyclebinPlugin(target_win) header = recycle_plugin.select_header(version_number) - assert header is recycle_plugin.recyclebin_parser.__getattr__(expected_header) + assert header is getattr(c_recyclebin, expected_header) @pytest.mark.parametrize( @@ -91,9 +91,7 @@ def test_parse_header(target_win, version_number, expected_header): def test_read_bin_file_unknown(target_win, path): recycle_plugin = RecyclebinPlugin(target_win) - header_1 = recycle_plugin.recyclebin_parser.header_v1( - version=0, file_size=0x20, timestamp=0x20, filename="hello_world" + "\x00" * 249 - ) + header_1 = c_recyclebin.header_v1(version=0, file_size=0x20, timestamp=0x20, filename="hello_world" + "\x00" * 249) with patch.object(Path, "open", mock_open(read_data=header_1.dumps())): normal_path = Path(path) diff --git a/tox.ini b/tox.ini index 5921b3ace..895ef09cd 100644 --- a/tox.ini +++ b/tox.ini @@ -11,9 +11,7 @@ minversion = 4.4.3 requires = virtualenv>=20.16.6 [testenv] -extras = - yara - mqtt +extras = dev deps = pytest pytest-cov