Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Guarddog integration #2499

Closed
wants to merge 4 commits into from
Closed

Guarddog integration #2499

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b19c66e
Update Dockerfile for malware tools analyzers to integrate guarddog, …
hugo-glez Sep 6, 2024
e581efb
Update app.py to add guarddog command
hugo-glez Sep 6, 2024
afad12e
Added classes to handle files and observables for guarddog
hugo-glez Sep 6, 2024
ea3bbac
Add migrations for guarddog and migrations merges
hugo-glez Sep 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions api_app/analyzers_manager/file_analyzers/guarddog_file.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import logging

from api_app.analyzers_manager.classes import DockerBasedAnalyzer, FileAnalyzer
from tests.mock_utils import MockUpResponse

logger = logging.getLogger(__name__)


class GuarddogFile(FileAnalyzer, DockerBasedAnalyzer):
name: str = "Guarddog"
url: str = "http://malware_tools_analyzers:4002/guarddog"
# http request polling max number of tries
max_tries: int = 15
# interval between http request polling (in secs)
poll_distance: int = 30
Comment on lines +14 to +15
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the polling between this script in the main docker container and the malware docker container. The problem about this number is that if the analyzers finishes its computation in the middle of those 30 seconds, the client would get the results a lot later than intended. Because of this, this value can usually reduced to a lower number.
How much time does this analyzer require to be completely executed in your experience?


scan_type: str

def run(self):
binary = self.read_file_bytes()
fname = str(self.filename).replace("/", "_").replace(" ", "_")
args = [
self.scan_type,
"scan",
f"@{fname}",
]
req_data = {"args": args}
req_files = {fname: binary}
logger.info(
f"Running {self.analyzer_name} on {self.filename} with args: {args}"
)
result = self._docker_run(req_data, req_files, analyzer_name=self.analyzer_name)
return result

@staticmethod
def mocked_docker_analyzer_get(*args, **kwargs):
return MockUpResponse(
{
"key": "test",
"returncode": 0,
"report": "Found 0 potentially malicious indicators scanning ... \n",
},
200,
)
152 changes: 152 additions & 0 deletions api_app/analyzers_manager/migrations/0121_analyzer_config_guarddogobservable_go.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
from django.db import migrations
from django.db.models.fields.related_descriptors import (
ForwardManyToOneDescriptor,
ForwardOneToOneDescriptor,
ManyToManyDescriptor,
)

plugin = {
"python_module": {
"health_check_schedule": None,
"update_schedule": None,
"module": "guarddog_observable.GuarddogObservable",
"base_path": "api_app.analyzers_manager.observable_analyzers",
},
"name": "GuarddogObservable_go",
"description": "GuardDog is a tool that allows to identify malicious PyPI and npm packages or Go modules. It runs a set of heuristics on the package source code (through Semgrep rules) and on the package metadata.\r\n\r\ngo modules observable. Just give the name of the package",
Comment on lines +15 to +16
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's enough a single migration for all the 3 cases (npm,pypi, org) and you can call it GuarddogObservable generically

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you can remove the other 2 and the merge migration

"disabled": False,
"soft_time_limit": 600,
"routing_key": "default",
"health_check_status": True,
"type": "observable",
"docker_based": True,
"maximum_tlp": "RED",
"observable_supported": ["generic"],
"supported_filetypes": [],
"run_hash": False,
"run_hash_type": "",
"not_supported_filetypes": [],
"model": "analyzers_manager.AnalyzerConfig",
}

params = [
{
"python_module": {
"module": "guarddog_observable.GuarddogObservable",
"base_path": "api_app.analyzers_manager.observable_analyzers",
},
"name": "scan_type",
"type": "str",
"description": "npm, pypi, go",
"is_secret": False,
"required": False,
}
]

values = [
{
"parameter": {
"python_module": {
"module": "guarddog_observable.GuarddogObservable",
"base_path": "api_app.analyzers_manager.observable_analyzers",
},
"name": "scan_type",
"type": "str",
"description": "npm, pypi, go",
"is_secret": False,
"required": False,
},
"analyzer_config": "GuarddogObservable_go",
"connector_config": None,
"visualizer_config": None,
"ingestor_config": None,
"pivot_config": None,
"for_organization": False,
"value": "go",
"updated_at": "2024-09-06T17:45:10.584302Z",
"owner": None,
}
]


def _get_real_obj(Model, field, value):
def _get_obj(Model, other_model, value):
if isinstance(value, dict):
real_vals = {}
for key, real_val in value.items():
real_vals[key] = _get_real_obj(other_model, key, real_val)
value = other_model.objects.get_or_create(**real_vals)[0]
# it is just the primary key serialized
else:
if isinstance(value, int):
if Model.__name__ == "PluginConfig":
value = other_model.objects.get(name=plugin["name"])
else:
value = other_model.objects.get(pk=value)
else:
value = other_model.objects.get(name=value)
return value

if (
type(getattr(Model, field))
in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor]
and value
):
other_model = getattr(Model, field).get_queryset().model
value = _get_obj(Model, other_model, value)
elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
other_model = getattr(Model, field).rel.model
value = [_get_obj(Model, other_model, val) for val in value]
return value


def _create_object(Model, data):
mtm, no_mtm = {}, {}
for field, value in data.items():
value = _get_real_obj(Model, field, value)
if type(getattr(Model, field)) is ManyToManyDescriptor:
mtm[field] = value
else:
no_mtm[field] = value
try:
o = Model.objects.get(**no_mtm)
except Model.DoesNotExist:
o = Model(**no_mtm)
o.full_clean()
o.save()
for field, value in mtm.items():
attribute = getattr(o, field)
if value is not None:
attribute.set(value)
return False
return True


def migrate(apps, schema_editor):
Parameter = apps.get_model("api_app", "Parameter")
PluginConfig = apps.get_model("api_app", "PluginConfig")
python_path = plugin.pop("model")
Model = apps.get_model(*python_path.split("."))
if not Model.objects.filter(name=plugin["name"]).exists():
exists = _create_object(Model, plugin)
if not exists:
for param in params:
_create_object(Parameter, param)
for value in values:
_create_object(PluginConfig, value)


def reverse_migrate(apps, schema_editor):
python_path = plugin.pop("model")
Model = apps.get_model(*python_path.split("."))
Model.objects.get(name=plugin["name"]).delete()


class Migration(migrations.Migration):
atomic = False
dependencies = [
("api_app", "0062_alter_parameter_python_module"),
("analyzers_manager", "0120_analyzer_config_guarddogobservable_pypi"),
]

operations = [migrations.RunPython(migrate, reverse_migrate)]
152 changes: 152 additions & 0 deletions api_app/analyzers_manager/migrations/0121_analyzer_config_guarddogobservable_npm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
from django.db import migrations
from django.db.models.fields.related_descriptors import (
ForwardManyToOneDescriptor,
ForwardOneToOneDescriptor,
ManyToManyDescriptor,
)

plugin = {
"python_module": {
"health_check_schedule": None,
"update_schedule": None,
"module": "guarddog_observable.GuarddogObservable",
"base_path": "api_app.analyzers_manager.observable_analyzers",
},
"name": "GuarddogObservable_npm",
"description": "GuardDog is a tool that allows to identify malicious PyPI and npm packages or Go modules. It runs a set of heuristics on the package source code (through Semgrep rules) and on the package metadata.\r\n\r\nnpm packages observable. Just give the name of the package",
"disabled": False,
"soft_time_limit": 600,
"routing_key": "default",
"health_check_status": True,
"type": "observable",
"docker_based": True,
"maximum_tlp": "RED",
"observable_supported": ["generic"],
"supported_filetypes": [],
"run_hash": False,
"run_hash_type": "",
"not_supported_filetypes": [],
"model": "analyzers_manager.AnalyzerConfig",
}

params = [
{
"python_module": {
"module": "guarddog_observable.GuarddogObservable",
"base_path": "api_app.analyzers_manager.observable_analyzers",
},
"name": "scan_type",
"type": "str",
"description": "npm, pypi, go",
"is_secret": False,
"required": False,
}
]

values = [
{
"parameter": {
"python_module": {
"module": "guarddog_observable.GuarddogObservable",
"base_path": "api_app.analyzers_manager.observable_analyzers",
},
"name": "scan_type",
"type": "str",
"description": "npm, pypi, go",
"is_secret": False,
"required": False,
},
"analyzer_config": "GuarddogObservable_npm",
"connector_config": None,
"visualizer_config": None,
"ingestor_config": None,
"pivot_config": None,
"for_organization": False,
"value": "npm",
"updated_at": "2024-09-06T17:44:46.967434Z",
"owner": None,
}
]


def _get_real_obj(Model, field, value):
def _get_obj(Model, other_model, value):
if isinstance(value, dict):
real_vals = {}
for key, real_val in value.items():
real_vals[key] = _get_real_obj(other_model, key, real_val)
value = other_model.objects.get_or_create(**real_vals)[0]
# it is just the primary key serialized
else:
if isinstance(value, int):
if Model.__name__ == "PluginConfig":
value = other_model.objects.get(name=plugin["name"])
else:
value = other_model.objects.get(pk=value)
else:
value = other_model.objects.get(name=value)
return value

if (
type(getattr(Model, field))
in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor]
and value
):
other_model = getattr(Model, field).get_queryset().model
value = _get_obj(Model, other_model, value)
elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
other_model = getattr(Model, field).rel.model
value = [_get_obj(Model, other_model, val) for val in value]
return value


def _create_object(Model, data):
mtm, no_mtm = {}, {}
for field, value in data.items():
value = _get_real_obj(Model, field, value)
if type(getattr(Model, field)) is ManyToManyDescriptor:
mtm[field] = value
else:
no_mtm[field] = value
try:
o = Model.objects.get(**no_mtm)
except Model.DoesNotExist:
o = Model(**no_mtm)
o.full_clean()
o.save()
for field, value in mtm.items():
attribute = getattr(o, field)
if value is not None:
attribute.set(value)
return False
return True


def migrate(apps, schema_editor):
Parameter = apps.get_model("api_app", "Parameter")
PluginConfig = apps.get_model("api_app", "PluginConfig")
python_path = plugin.pop("model")
Model = apps.get_model(*python_path.split("."))
if not Model.objects.filter(name=plugin["name"]).exists():
exists = _create_object(Model, plugin)
if not exists:
for param in params:
_create_object(Parameter, param)
for value in values:
_create_object(PluginConfig, value)


def reverse_migrate(apps, schema_editor):
python_path = plugin.pop("model")
Model = apps.get_model(*python_path.split("."))
Model.objects.get(name=plugin["name"]).delete()


class Migration(migrations.Migration):
atomic = False
dependencies = [
("api_app", "0062_alter_parameter_python_module"),
("analyzers_manager", "0120_analyzer_config_guarddogobservable_pypi"),
]

operations = [migrations.RunPython(migrate, reverse_migrate)]
Loading
Loading