From b804716cdf0e467c3b15d7a0cb10bb634377d860 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 17 Apr 2024 17:38:40 +0200 Subject: [PATCH 1/7] simplify dhcp ip parsing --- dissect/target/helpers/network_managers.py | 23 ++++++++++++++------- dissect/target/plugins/os/unix/linux/_os.py | 7 ++++--- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/dissect/target/helpers/network_managers.py b/dissect/target/helpers/network_managers.py index 9f776a673..bc8aa8282 100644 --- a/dissect/target/helpers/network_managers.py +++ b/dissect/target/helpers/network_managers.py @@ -7,7 +7,7 @@ from io import StringIO from itertools import chain from re import compile, sub -from typing import Any, Callable, Iterable, Match, Optional +from typing import Any, Callable, Iterable, Iterator, Match, Optional from defusedxml import ElementTree @@ -509,14 +509,15 @@ def get_config_value(self, attr: str) -> list[set]: return values -def parse_unix_dhcp_log_messages(target) -> list[str]: +def parse_unix_dhcp_log_messages(target, iter_all: bool = False) -> list[str]: """Parse local syslog, journal and cloud init-log files for DHCP lease IPs. Args: target: Target to discover and obtain network information from. + iter_all: Parse limited amount of journal messages (first 10000) or all of them. Returns: - List of DHCP ip addresses. + List of found DHCP IP addresses. """ ips = set() messages = set() @@ -530,7 +531,14 @@ def parse_unix_dhcp_log_messages(target) -> list[str]: if not messages: target.log.warning(f"Could not search for DHCP leases using {log_func}: No log entries found.") - for record in messages: + def messages_enumerate(iterable: Iterable) -> Iterator[tuple[int, Any]]: + n = 0 + for rec in iterable: + if rec._desc.name == "linux/log/journal": + n += 1 + yield n, rec + + for count, record in messages_enumerate(messages): line = record.message # Ubuntu cloud-init @@ -576,9 +584,10 @@ def parse_unix_dhcp_log_messages(target) -> list[str]: ips.add(ip) continue - # Journals and syslogs can be large and slow to iterate, - # so we stop if we have some results and have reached the journal plugin. - if len(ips) >= 2 and record._desc.name == "linux/log/journal": + # The journal parser is relatively slow, so we stop when we have read 10000 journal entries. + if not iter_all and (ips or count > 10_000): + if not ips: + log.warning("No DHCP IP addresses found in first 10000 journal entries.") break return ips diff --git a/dissect/target/plugins/os/unix/linux/_os.py b/dissect/target/plugins/os/unix/linux/_os.py index 8b8d96751..4e11867c3 100644 --- a/dissect/target/plugins/os/unix/linux/_os.py +++ b/dissect/target/plugins/os/unix/linux/_os.py @@ -6,7 +6,7 @@ LinuxNetworkManager, parse_unix_dhcp_log_messages, ) -from dissect.target.plugin import OperatingSystem, export +from dissect.target.plugin import OperatingSystem, arg, export from dissect.target.plugins.os.unix._os import UnixPlugin from dissect.target.target import Target @@ -33,7 +33,8 @@ def detect(cls, target: Target) -> Optional[Filesystem]: return None @export(property=True) - def ips(self) -> list[str]: + @arg("--dhcp-all", action="store_true", help="parse all syslog, messages and journal files for DHCP IP addresses") + def ips(self, dhcp_all: bool = False) -> list[str]: """Returns a list of static IP addresses and DHCP lease IP addresses found on the host system.""" ips = [] @@ -41,7 +42,7 @@ def ips(self) -> list[str]: for ip in ip_set: ips.append(ip) - for ip in parse_unix_dhcp_log_messages(self.target): + for ip in parse_unix_dhcp_log_messages(self.target, dhcp_all): if ip not in ips: ips.append(ip) From 9d4ecbaea40fc066f825e078c582f87e691c4200 Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:40:31 +0200 Subject: [PATCH 2/7] check if record.message is not NoneType --- dissect/target/helpers/network_managers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dissect/target/helpers/network_managers.py b/dissect/target/helpers/network_managers.py index bc8aa8282..e7cf2e2d4 100644 --- a/dissect/target/helpers/network_managers.py +++ b/dissect/target/helpers/network_managers.py @@ -541,6 +541,9 @@ def messages_enumerate(iterable: Iterable) -> Iterator[tuple[int, Any]]: for count, record in messages_enumerate(messages): line = record.message + if not line: + continue + # Ubuntu cloud-init if "Received dhcp lease on" in line: interface, ip, netmask = re.search(r"Received dhcp lease on (\w{0,}) for (\S+)\/(\S+)", line).groups() From 8964383bc6a9a8c277e7c83e22ff24bd1afce405 Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 24 Jul 2024 11:27:44 +0200 Subject: [PATCH 3/7] Apply suggestions from code review Co-authored-by: Stefan de Reuver <9864602+Horofic@users.noreply.github.com> --- dissect/target/helpers/network_managers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dissect/target/helpers/network_managers.py b/dissect/target/helpers/network_managers.py index e7cf2e2d4..635e953b3 100644 --- a/dissect/target/helpers/network_managers.py +++ b/dissect/target/helpers/network_managers.py @@ -509,7 +509,7 @@ def get_config_value(self, attr: str) -> list[set]: return values -def parse_unix_dhcp_log_messages(target, iter_all: bool = False) -> list[str]: +def parse_unix_dhcp_log_messages(target: Target, iter_all: bool = False) -> set[str]: """Parse local syslog, journal and cloud init-log files for DHCP lease IPs. Args: @@ -517,7 +517,7 @@ def parse_unix_dhcp_log_messages(target, iter_all: bool = False) -> list[str]: iter_all: Parse limited amount of journal messages (first 10000) or all of them. Returns: - List of found DHCP IP addresses. + A set of found DHCP IP addresses. """ ips = set() messages = set() @@ -531,14 +531,14 @@ def parse_unix_dhcp_log_messages(target, iter_all: bool = False) -> list[str]: if not messages: target.log.warning(f"Could not search for DHCP leases using {log_func}: No log entries found.") - def messages_enumerate(iterable: Iterable) -> Iterator[tuple[int, Any]]: - n = 0 + def records_enumerate(iterable: Iterable) -> Iterator[tuple[int, JournalRecord | MessagesRecord]]: + count = 0 for rec in iterable: if rec._desc.name == "linux/log/journal": - n += 1 - yield n, rec + count += 1 + yield count, rec - for count, record in messages_enumerate(messages): + for count, record in records_enumerate(messages): line = record.message if not line: From ab47074858c5feaf3b17a6dbd243ae41dd32bf9d Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 24 Jul 2024 12:04:53 +0200 Subject: [PATCH 4/7] add dhcp_all test --- dissect/target/helpers/network_managers.py | 5 +- tests/plugins/os/unix/test_ips.py | 87 +++++++++++++++++++--- 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/dissect/target/helpers/network_managers.py b/dissect/target/helpers/network_managers.py index 635e953b3..897ef1060 100644 --- a/dissect/target/helpers/network_managers.py +++ b/dissect/target/helpers/network_managers.py @@ -13,6 +13,8 @@ from dissect.target.exceptions import PluginError from dissect.target.helpers.fsutil import TargetPath +from dissect.target.plugins.os.unix.log.journal import JournalRecord +from dissect.target.plugins.os.unix.log.messages import MessagesRecord from dissect.target.target import Target log = logging.getLogger(__name__) @@ -587,7 +589,8 @@ def records_enumerate(iterable: Iterable) -> Iterator[tuple[int, JournalRecord | ips.add(ip) continue - # The journal parser is relatively slow, so we stop when we have read 10000 journal entries. + # The journal parser is relatively slow, so we stop when we have read 10000 journal entries, + # or if we have found at least one ip address. When `iter_all` is `True` we continue searching. if not iter_all and (ips or count > 10_000): if not ips: log.warning("No DHCP IP addresses found in first 10000 journal entries.") diff --git a/tests/plugins/os/unix/test_ips.py b/tests/plugins/os/unix/test_ips.py index 4c85ea032..2899b99e9 100644 --- a/tests/plugins/os/unix/test_ips.py +++ b/tests/plugins/os/unix/test_ips.py @@ -1,5 +1,6 @@ import textwrap from io import BytesIO +from unittest.mock import patch import pytest @@ -7,21 +8,40 @@ from dissect.target.filesystem import VirtualFilesystem from dissect.target.helpers.network_managers import NetworkManager from dissect.target.plugins.os.unix.linux._os import LinuxPlugin +from dissect.target.tools.query import main as target_query from tests._utils import absolute_path -def test_ips_dhcp(target_unix_users: Target, fs_unix: VirtualFilesystem) -> None: +@pytest.mark.parametrize( + "expected_ips, messages", + [ + ( + ["10.13.37.1"], + "Jan 1 13:37:01 hostname NetworkManager[1]: [1600000000.0000] dhcp4 (eth0): option ip_address => '10.13.37.1'", # noqa: E501 + ), + (["10.13.37.2"], "Feb 2 13:37:02 test systemd-networkd[2]: eth0: DHCPv4 address 10.13.37.2/24 via 10.13.37.0"), + ( + ["10.13.37.3"], + "Mar 3 13:37:03 localhost NetworkManager[3]: [1600000000.0003] dhcp4 (eth0): address 10.13.37.3", + ), + ( + ["10.13.37.4"], + "Apr 4 13:37:04 localhost dhclient[4]: bound to 10.13.37.4 -- renewal in 1337 seconds.", + ), + ( + ["2001:db8::"], + ( + "Jun 6 13:37:06 test systemd-networkd[5]: eth0: DHCPv6 address 2001:db8::/64 via 2001:db8:ffff:ffff:ffff:ffff:ffff:ffff\n" # noqa: E501 + "May 5 13:37:05 test systemd-networkd[5]: eth0: DHCPv6 lease lost\n" + ), + ), + ], +) +def test_ips_dhcp( + target_unix_users: Target, fs_unix: VirtualFilesystem, expected_ips: list[str], messages: str +) -> None: """Test DHCP lease messages from /var/log/syslog.""" - messages = """ - Jan 1 13:37:01 hostname NetworkManager[1]: [1600000000.0000] dhcp4 (eth0): option ip_address => '10.13.37.1' - Feb 2 13:37:02 test systemd-networkd[2]: eth0: DHCPv4 address 10.13.37.2/24 via 10.13.37.0 - Mar 3 13:37:03 localhost NetworkManager[3]: [1600000000.0003] dhcp4 (eth0): address 10.13.37.3 - Apr 4 13:37:04 localhost dhclient[4]: bound to 10.13.37.4 -- renewal in 1337 seconds. - May 5 13:37:05 test systemd-networkd[5]: eth0: DHCPv6 lease lost - Jun 6 13:37:06 test systemd-networkd[5]: eth0: DHCPv6 address 2001:db8::/64 via 2001:db8:ffff:ffff:ffff:ffff:ffff:ffff - """ # noqa E501 - fs_unix.map_file_fh( "/var/log/syslog", BytesIO(textwrap.dedent(messages).encode()), @@ -30,8 +50,51 @@ def test_ips_dhcp(target_unix_users: Target, fs_unix: VirtualFilesystem) -> None target_unix_users.add_plugin(LinuxPlugin) results = target_unix_users.ips results.reverse() - assert len(results) == 5 - assert sorted(results) == ["10.13.37.1", "10.13.37.2", "10.13.37.3", "10.13.37.4", "2001:db8::"] + assert len(results) == len(expected_ips) + assert sorted(results) == expected_ips + + +@pytest.mark.parametrize( + "flag, expected_out", + [ + (None, "['10.13.37.2']"), + ("--dhcp-all", "['10.13.37.2', '10.13.37.1']"), + ], +) +def test_ips_dhcp_arg( + target_unix: Target, + fs_unix: VirtualFilesystem, + flag: str, + expected_out: str, + capsys: pytest.CaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test --dhcp-all flag behaviour""" + + fs_unix.map_file_fh("/etc/timezone", BytesIO(b"Europe/Amsterdam")) + + messages = """ + Apr 1 13:37:01 localhost dhclient[4]: bound to 10.13.37.1 -- renewal in 1337 seconds. + Apr 2 13:37:02 localhost foo[1]: some other message. + Apr 3 13:37:03 localhost dhclient[4]: bound to 10.13.37.2 -- renewal in 1337 seconds. + """ + + fs_unix.map_file_fh( + "/var/log/syslog", + BytesIO(textwrap.dedent(messages).encode()), + ) + target_unix.add_plugin(LinuxPlugin) + + argv = ["target-query", "foo", "-f", "ips"] + if flag: + argv.append(flag) + + with patch("dissect.target.Target.open_all", return_value=[target_unix]): + with monkeypatch.context() as m: + m.setattr("sys.argv", argv) + target_query() + out, _ = capsys.readouterr() + assert expected_out in out def test_ips_cloud_init(target_unix_users: Target, fs_unix: VirtualFilesystem) -> None: From a862b2ea5ca65841d7abbb4c4b44a8b597ce865b Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:18:09 +0200 Subject: [PATCH 5/7] temporarily disable dhcp-all feature --- dissect/target/plugins/os/unix/linux/_os.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dissect/target/plugins/os/unix/linux/_os.py b/dissect/target/plugins/os/unix/linux/_os.py index 4e11867c3..92731bf8e 100644 --- a/dissect/target/plugins/os/unix/linux/_os.py +++ b/dissect/target/plugins/os/unix/linux/_os.py @@ -6,7 +6,7 @@ LinuxNetworkManager, parse_unix_dhcp_log_messages, ) -from dissect.target.plugin import OperatingSystem, arg, export +from dissect.target.plugin import OperatingSystem, export from dissect.target.plugins.os.unix._os import UnixPlugin from dissect.target.target import Target @@ -33,8 +33,7 @@ def detect(cls, target: Target) -> Optional[Filesystem]: return None @export(property=True) - @arg("--dhcp-all", action="store_true", help="parse all syslog, messages and journal files for DHCP IP addresses") - def ips(self, dhcp_all: bool = False) -> list[str]: + def ips(self) -> list[str]: """Returns a list of static IP addresses and DHCP lease IP addresses found on the host system.""" ips = [] @@ -42,7 +41,7 @@ def ips(self, dhcp_all: bool = False) -> list[str]: for ip in ip_set: ips.append(ip) - for ip in parse_unix_dhcp_log_messages(self.target, dhcp_all): + for ip in parse_unix_dhcp_log_messages(self.target, dhcp_all=False): if ip not in ips: ips.append(ip) From 04878f99ebebd6e76b359dd2791ad4407375c3db Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:21:30 +0200 Subject: [PATCH 6/7] fix tests --- dissect/target/plugins/os/unix/linux/_os.py | 2 +- tests/plugins/os/unix/test_ips.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dissect/target/plugins/os/unix/linux/_os.py b/dissect/target/plugins/os/unix/linux/_os.py index 92731bf8e..0b22cc309 100644 --- a/dissect/target/plugins/os/unix/linux/_os.py +++ b/dissect/target/plugins/os/unix/linux/_os.py @@ -41,7 +41,7 @@ def ips(self) -> list[str]: for ip in ip_set: ips.append(ip) - for ip in parse_unix_dhcp_log_messages(self.target, dhcp_all=False): + for ip in parse_unix_dhcp_log_messages(self.target, iter_all=False): if ip not in ips: ips.append(ip) diff --git a/tests/plugins/os/unix/test_ips.py b/tests/plugins/os/unix/test_ips.py index 2899b99e9..2da71bb4d 100644 --- a/tests/plugins/os/unix/test_ips.py +++ b/tests/plugins/os/unix/test_ips.py @@ -58,7 +58,9 @@ def test_ips_dhcp( "flag, expected_out", [ (None, "['10.13.37.2']"), - ("--dhcp-all", "['10.13.37.2', '10.13.37.1']"), + # ("--dhcp-all", "['10.13.37.2', '10.13.37.1']"), + # Temporarily disabled behaviour, for discussion see: + # https://github.com/fox-it/dissect.target/pull/687#discussion_r1698515269 ], ) def test_ips_dhcp_arg( From 8851f13a518ec9f3b696c6504a496746b36dcfcd Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Thu, 1 Aug 2024 10:09:49 +0200 Subject: [PATCH 7/7] Update dissect/target/helpers/network_managers.py Co-authored-by: Stefan de Reuver <9864602+Horofic@users.noreply.github.com> --- dissect/target/helpers/network_managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dissect/target/helpers/network_managers.py b/dissect/target/helpers/network_managers.py index 897ef1060..0c04f75a3 100644 --- a/dissect/target/helpers/network_managers.py +++ b/dissect/target/helpers/network_managers.py @@ -593,7 +593,7 @@ def records_enumerate(iterable: Iterable) -> Iterator[tuple[int, JournalRecord | # or if we have found at least one ip address. When `iter_all` is `True` we continue searching. if not iter_all and (ips or count > 10_000): if not ips: - log.warning("No DHCP IP addresses found in first 10000 journal entries.") + target.log.warning("No DHCP IP addresses found in first 10000 journal entries.") break return ips