Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Compatibility with cstruct v4 #717

Merged
merged 18 commits into from
Jun 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/dissect-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ on:
jobs:
ci:
uses: fox-it/dissect-workflow-templates/.github/workflows/dissect-ci-template.yml@main
with:
deb-packages: 'liblzo2-dev'
secrets: inherit

publish:
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/on-demand-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ jobs:
python-include: "pypy3.9"
tox-env: "pypy39"
steps:
- run: sudo apt-get install -qq liblzo2-dev
- uses: actions/checkout@v3
with:
fetch-depth: 0
Expand Down
16 changes: 6 additions & 10 deletions dissect/target/helpers/protobuf.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,30 +3,26 @@
from typing import Any, BinaryIO

from dissect.cstruct.types.base import BaseType
from dissect.cstruct.types.bytesinteger import BytesInteger


class ProtobufVarint(BytesInteger):
class ProtobufVarint(BaseType):
"""Implements a protobuf integer type for dissect.cstruct that can span a variable amount of bytes.

Mainly follows the cstruct BytesInteger implementation with minor tweaks
to support protobuf's msb varint implementation.
Supports protobuf's msb varint implementation.

Resources:
- https://protobuf.dev/programming-guides/encoding/
- https://github.com/protocolbuffers/protobuf/blob/main/python/google/protobuf/internal/decoder.py
"""

def _read(self, stream: BinaryIO, context: dict[str, Any] = None) -> int:
@classmethod
def _read(cls, stream: BinaryIO, context: dict[str, Any] = None) -> int:
return decode_varint(stream)

def _write(self, stream: BinaryIO, data: int) -> int:
@classmethod
def _write(cls, stream: BinaryIO, data: int) -> int:
return stream.write(encode_varint(data))

_read_array = BaseType._read_array

_write_array = BaseType._write_array


def decode_varint(stream: BinaryIO) -> int:
"""Reads a varint from the provided buffer stream.
Expand Down
7 changes: 3 additions & 4 deletions dissect/target/helpers/ssh.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import base64
import binascii

from dissect import cstruct
from dissect.cstruct import cstruct

c_rfc4716_def = """
rfc4716_def = """
struct ssh_string {
uint32 length;
char value[length];
Expand All @@ -23,8 +23,7 @@
}
"""

c_rfc4716 = cstruct.cstruct(endian=">")
c_rfc4716.load(c_rfc4716_def)
c_rfc4716 = cstruct(endian=">").load(rfc4716_def)

RFC4716_MARKER_START = b"-----BEGIN OPENSSH PRIVATE KEY-----"
RFC4716_MARKER_END = b"-----END OPENSSH PRIVATE KEY-----"
Expand Down
5 changes: 2 additions & 3 deletions dissect/target/plugins/apps/av/trendmicro.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from typing import Iterator

from dissect import cstruct
from dissect.cstruct import cstruct
from dissect.util.ts import from_unix

from dissect.target import Target
Expand Down Expand Up @@ -47,8 +47,7 @@
char _pad3[10];
};
"""
c_pfwlog = cstruct.cstruct()
c_pfwlog.load(pfwlog_def)
c_pfwlog = cstruct().load(pfwlog_def)


class TrendMicroPlugin(Plugin):
Expand Down
2 changes: 1 addition & 1 deletion dissect/target/plugins/apps/container/docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@
"""

c_local = cstruct(endian=">")
c_local.addtype("varint", ProtobufVarint(c_local, "varint", size=None, signed=False, alignment=1))
c_local.add_custom_type("varint", ProtobufVarint, size=None, alignment=1, signed=False)
c_local.load(local_def, compiled=False)

RE_DOCKER_NS = re.compile(r"\.(?P<nanoseconds>\d{7,})(?P<postfix>Z|\+\d{2}:\d{2})")
Expand Down
3 changes: 1 addition & 2 deletions dissect/target/plugins/os/unix/locate/gnulocate.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@
],
)

c_gnulocate = cstruct()
c_gnulocate.load(gnulocate_def)
c_gnulocate = cstruct().load(gnulocate_def)


class GNULocateFile:
Expand Down
7 changes: 3 additions & 4 deletions dissect/target/plugins/os/unix/locate/mlocate.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@
int32 conf_size;
int8 version; /* file format version */
int8 require_visibility;
int8 pad[2]; /* 32-bit total alignment */
int8 pad0[2]; /* 32-bit total alignment */
char root_database;
char config_block[conf_size];
int8 pad;
int8 pad1;
};

enum DBE_TYPE: uint8 { /* database entry type */
Expand Down Expand Up @@ -68,8 +68,7 @@ class MLocate:
],
)

c_mlocate = cstruct(endian=">")
c_mlocate.load(mlocate_def)
c_mlocate = cstruct(endian=">").load(mlocate_def)


class MLocateFile:
Expand Down
3 changes: 1 addition & 2 deletions dissect/target/plugins/os/unix/locate/plocate.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@
],
)

c_plocate = cstruct()
c_plocate.load(plocate_def)
c_plocate = cstruct().load(plocate_def)


class PLocateFile:
Expand Down
7 changes: 3 additions & 4 deletions dissect/target/plugins/os/unix/log/atop.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from io import BytesIO
from typing import BinaryIO, Iterator

from dissect.cstruct import Instance, cstruct
from dissect.cstruct import cstruct

from dissect.target.exceptions import UnsupportedPluginError
from dissect.target.helpers.record import TargetRecordDescriptor
Expand Down Expand Up @@ -178,8 +178,7 @@
};
""" # noqa: E501

c_atop = cstruct()
c_atop.load(atop_def)
c_atop = cstruct().load(atop_def)
c_atop.load(atop_tstat_def, align=True)

AtopRecord = TargetRecordDescriptor(
Expand Down Expand Up @@ -226,7 +225,7 @@ def __init__(self, fh: BinaryIO):
self.header = c_atop.rawheader(self.fh)
self.version = self.version()

def __iter__(self) -> Iterator[Instance]:
def __iter__(self) -> Iterator[c_atop.tstat]:
while True:
try:
record = c_atop.rawrecord(self.fh)
Expand Down
9 changes: 5 additions & 4 deletions dissect/target/plugins/os/unix/log/journal.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
from __future__ import annotations

import lzma
from typing import BinaryIO, Callable, Iterator

import zstandard
from dissect.cstruct import Instance, cstruct
from dissect.cstruct import cstruct
from dissect.util import ts
from dissect.util.compression import lz4

Expand Down Expand Up @@ -252,8 +254,7 @@
};
""" # noqa: E501

c_journal = cstruct()
c_journal.load(journal_def)
c_journal = cstruct().load(journal_def)


def get_optional(value: str, to_type: Callable):
Expand Down Expand Up @@ -314,7 +315,7 @@ def decode_value(self, value: bytes) -> tuple[str, str]:

return key, value

def __iter__(self) -> Iterator[Instance]:
def __iter__(self) -> Iterator[dict[str, int | str]]:
"Iterate over the entry objects to read payloads."

for offset in self.entry_object_offsets():
Expand Down
5 changes: 2 additions & 3 deletions dissect/target/plugins/os/unix/log/lastlog.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from typing import BinaryIO

from dissect import cstruct
from dissect.cstruct import cstruct
from dissect.util import ts

from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
Expand Down Expand Up @@ -36,8 +36,7 @@
};
"""

c_lastlog = cstruct.cstruct()
c_lastlog.load(lastlog_def)
c_lastlog = cstruct().load(lastlog_def)


class LastLogFile:
Expand Down
13 changes: 6 additions & 7 deletions dissect/target/plugins/os/unix/log/utmp.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,14 @@
],
)

c_utmp = """
utmp_def = """
#define UT_LINESIZE 32
#define UT_NAMESIZE 32
#define UT_HOSTSIZE 256

typedef uint32 pid_t;

enum Type : char {
enum Type : uint8_t {
EMPTY = 0x0,
RUN_LVL = 0x1,
BOOT_TIME = 0x2,
Expand Down Expand Up @@ -84,8 +84,7 @@
};
""" # noqa: E501

utmp = cstruct()
utmp.load(c_utmp)
c_utmp = cstruct().load(utmp_def)

UTMP_ENTRY = namedtuple(
"UTMPRecord",
Expand Down Expand Up @@ -122,11 +121,11 @@ def __iter__(self):

while True:
try:
entry = utmp.entry(byte_stream)
entry = c_utmp.entry(byte_stream)

r_type = ""
if entry.ut_type in utmp.Type.reverse:
r_type = utmp.Type.reverse[entry.ut_type]
if entry.ut_type in c_utmp.Type:
r_type = c_utmp.Type(entry.ut_type).name

ut_host = entry.ut_host.decode(errors="surrogateescape").strip("\x00")
ut_addr = None
Expand Down
7 changes: 3 additions & 4 deletions dissect/target/plugins/os/windows/adpolicy.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from struct import unpack

from defusedxml import ElementTree
from dissect import cstruct
from dissect.cstruct import cstruct
from dissect.regf.c_regf import (
REG_BINARY,
REG_DWORD,
Expand All @@ -18,14 +18,13 @@
from dissect.target.helpers.record import TargetRecordDescriptor
from dissect.target.plugin import Plugin, export

c_def = """
policy_def = """
struct registry_policy_header {
uint32 signature;
uint32 version;
};
"""
c_adpolicy = cstruct.cstruct()
c_adpolicy.load(c_def)
c_adpolicy = cstruct().load(policy_def)

ADPolicyRecord = TargetRecordDescriptor(
"windows/adpolicy",
Expand Down
3 changes: 1 addition & 2 deletions dissect/target/plugins/os/windows/credhist.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,7 @@
};
"""

c_credhist = cstruct()
c_credhist.load(credhist_def)
c_credhist = cstruct().load(credhist_def)


@dataclass
Expand Down
7 changes: 3 additions & 4 deletions dissect/target/plugins/os/windows/datetime.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from datetime import datetime, timedelta, timezone, tzinfo
from typing import Dict, Tuple

from dissect import cstruct
from dissect.cstruct import cstruct

from dissect.target.exceptions import (
RegistryError,
Expand Down Expand Up @@ -34,8 +34,7 @@
SYSTEMTIME DaylightDate;
} REG_TZI_FORMAT;
"""
c_tz = cstruct.cstruct()
c_tz.load(tz_def)
c_tz = cstruct().load(tz_def)


# Althoug calendar.SUNDAY is only officially documented since Python 3.10, it
Expand Down Expand Up @@ -63,7 +62,7 @@
HOUR = timedelta(hours=1)


def parse_systemtime_transition(systemtime: cstruct.Instance, year: int) -> datetime:
def parse_systemtime_transition(systemtime: c_tz._SYSTEMTIME, year: int) -> datetime:
"""Return the transition datetime for a given year using the SYSTEMTIME of a STD or DST transition date.

The SYSTEMTIME date of a TZI structure needs to be used to calculate the actual date for a given year.
Expand Down
7 changes: 3 additions & 4 deletions dissect/target/plugins/os/windows/defender.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,8 +237,7 @@
};
"""

c_defender = cstruct()
c_defender.load(defender_def)
c_defender = cstruct().load(defender_def)

STREAM_ID = c_defender.STREAM_ID
STREAM_ATTRIBUTES = c_defender.STREAM_ATTRIBUTES
Expand Down Expand Up @@ -381,7 +380,7 @@
self.last_access_time = ts.wintimestamp(int.from_bytes(field.Data, "little"))
elif field.Identifier == FIELD_IDENTIFIER.LastWriteTime:
self.last_write_time = ts.wintimestamp(int.from_bytes(field.Data, "little"))
elif field.Identifier not in FIELD_IDENTIFIER.values.values():
elif field.Identifier not in FIELD_IDENTIFIER:
self.unknown_fields.append(field)


Expand Down Expand Up @@ -526,7 +525,7 @@
subdir = resource.resource_id[0:2]
resourcedata_location = resourcedata_directory.joinpath(subdir).joinpath(resource.resource_id)
if not resourcedata_location.exists():
self.target.log.warning(f"Could not find a ResourceData file for {entry.resource_id}.")
self.target.log.warning(f"Could not find a ResourceData file for {resource.resource_id}.")

Check warning on line 528 in dissect/target/plugins/os/windows/defender.py

View check run for this annotation

Codecov / codecov/patch

dissect/target/plugins/os/windows/defender.py#L528

Added line #L528 was not covered by tests
continue
if not resourcedata_location.is_file():
self.target.log.warning(f"{resourcedata_location} is not a file!")
Expand Down
3 changes: 1 addition & 2 deletions dissect/target/plugins/os/windows/dpapi/blob.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@
};
"""

c_blob = cstruct()
c_blob.load(blob_def)
c_blob = cstruct().load(blob_def)


class Blob:
Expand Down
5 changes: 2 additions & 3 deletions dissect/target/plugins/os/windows/dpapi/master_key.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
DWORD accessCheckLen;
char guid[16];
char encryptedSecret[secretLen];
char accessCheckLen[accessCheckLen];
char accessCheck[accessCheckLen];
};

struct CredHist {
Expand Down Expand Up @@ -66,8 +66,7 @@
QWORD qwDomainKeySize;
};
"""
c_master_key = cstruct()
c_master_key.load(master_key_def)
c_master_key = cstruct().load(master_key_def)


class MasterKey:
Expand Down
3 changes: 1 addition & 2 deletions dissect/target/plugins/os/windows/notifications.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,7 @@
} Chunk; // size: 0x23810
"""

c_appdb = cstruct(endian="<")
c_appdb.load(appdb_def)
c_appdb = cstruct(endian="<").load(appdb_def)

APPDB_MAGIC = b"DNPW"
NUM_APPDB_CHUNKS = 256
Expand Down
Loading
Loading