Update variables from PRE_COMMANDS / POST_COMMANDS

.automation/test/pre-post-test/.mega-linter.yml

@@ -17,12 +17,18 @@ PRE_COMMANDS:
     cwd: "workspace"
   - command: pip install flake8-cognitive-complexity
     venv: flake8
+  - command: export MY_OUTPUT_VARIABLE="my output variable value" && export MY_OUTPUT_VARIABLE2="my output variable value2"
+    output_variables: ["MY_OUTPUT_VARIABLE", "MY_OUTPUT_VARIABLE2"]
+    cwd: "root"
 POST_COMMANDS:
   - command: npm run test
     cwd: "workspace"
 MARKDOWN_PRE_COMMANDS:
   - command: echo 'descriptor pre-command has been run'
     cwd: "root"
+  - command: export MY_OUTPUT_LINTER_VARIABLE="my output linter variable value" && export MY_OUTPUT_LINTER_VARIABLE2="my output linter variable value2"
+    output_variables: ["MY_OUTPUT_LINTER_VARIABLE", "MY_OUTPUT_LINTER_VARIABLE2"]
+    cwd: "root"
 MARKDOWN_POST_COMMANDS:
   - command: echo 'descriptor post-command has been run'
     cwd: "root"

CHANGELOG.md

@@ -18,6 +18,7 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l
 - Core
   - Allow to override CLI_LINT_MODE when defined as project
   - Allow to use absolute paths for LINTER_RULES_PATH
+  - Allow to update variables from [PRE/POST Commands](https://megalinter.io/latest/config-precommands/) using `output_variables` property
 
 - Media
   - [MegaLinter: un linter pour les gouverner tous](https://blog.wescale.fr/megalinter-un-linter-pour-les-gouverner-tous) (FR), by [Guillaume Arnaud](https://www.linkedin.com/in/guillaume-arnaud/) from [WeScale](https://www.wescale.fr/)

README.md

@@ -1141,6 +1141,8 @@ PRE_COMMANDS:
     continue_if_failed: False  # Will stop the process if command is failed (return code > 0)
   - command: pip install flake8-cognitive-complexity
     venv: flake8 # Will be run within flake8 python virtualenv. There is one virtualenv per python-based linter, with the same name
+  - command: export MY_OUTPUT_VAR="my output var" && export MY_OUTPUT_VAR2="my output var2"
+    output_variables: ["MY_OUTPUT_VAR","MY_OUTPUT_VAR2"] # Will collect the values of output variables and update MegaLinter own ENV context
 ```
 
 <!-- config-precommands-section-end -->

megalinter/config.py

@@ -10,6 +10,7 @@
 import yaml
 
 RUN_CONFIGS = {}  # type: ignore[var-annotated]
+SKIP_DELETE_CONFIG = False
 
 
 def init_config(request_id, workspace=None, params={}):
@@ -264,13 +265,15 @@ def copy(request_id):
 
 def delete(request_id=None, key=None):
     global RUN_CONFIGS
+    global SKIP_DELETE_CONFIG
     # Global delete (used for tests)
     if request_id is None:
         RUN_CONFIGS = {}
         return
     if key is None:
-        del RUN_CONFIGS[request_id]
-        logging.debug("Cleared MegaLinter runtime config for request " + request_id)
+        if SKIP_DELETE_CONFIG is not True:
+            del RUN_CONFIGS[request_id]
+            logging.debug("Cleared MegaLinter runtime config for request " + request_id)
         return
     config = get_config(request_id)
     if key in config:

megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json

@@ -15,7 +15,7 @@
           "type": "string"
         },
         "continue_if_failed": {
-          "Description": "If command fails, continue MegaLinter process or not",
+          "description": "If command fails, continue MegaLinter process or not",
           "default": true,
           "title": "Continue if failed",
           "type": "boolean"
@@ -31,6 +31,27 @@
           ],
           "title": "Folder where to run the command",
           "type": "string"
+        },
+        "output_variables": {
+          "description": "ENV variables to get from output after running the commands, and store in MegaLinter ENV context",
+          "title": "Output ENV variables",
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "secured_env": {
+          "description": "Apply filter of secured env variables before calling the command (default true). Be careful if you disable it.",
+          "default": true,
+          "title": "Secured ENV variables",
+          "type": "boolean"
+        },
+        "venv": {
+          "examples": [
+            "flake8"
+          ],
+          "title": "Name of the python venv to run the command in.",
+          "type": "string"
         }
       },
       "required": [],

megalinter/pre_post_factory.py

@@ -1,6 +1,7 @@
 # Class to manage MegaLinter plugins
 import logging
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -111,6 +112,31 @@ def run_command(command_info, log_key, mega_linter, linter=None):
         raise Exception(
             f"{log_key}: User command failed, stop running MegaLinter\n{return_stdout}"
         )
+    # Get output variables if defined
+    if command_info.get("output_variables", None) is not None:
+        for output_variable in command_info["output_variables"]:
+            regex_start = f"output of {output_variable}:"
+            if regex_start in return_stdout:
+                match = re.search(rf"{regex_start}(.+)", return_stdout)
+                if match:
+                    output_variable_value = match.group(1).strip()
+                    if (
+                        config.get(mega_linter.request_id, output_variable, None)
+                        != output_variable_value
+                    ):
+                        config.set_value(
+                            mega_linter.request_id,
+                            output_variable,
+                            output_variable_value,
+                        )
+                        add_in_logs(
+                            linter,
+                            log_key,
+                            [
+                                f"{log_key} updated ENV var {output_variable}] from command output"
+                            ],
+                        )
+
     return {
         "command_info": command_info,
         "status": return_code,
@@ -129,6 +155,12 @@ def complete_command(command_info: dict):
         command_info["command"] = (
             f"cd /venvs/{venv} && source bin/activate && {command} && deactivate"
         )
+    # Handle output vars if present
+    if command_info.get("output_variables", None) is not None:
+        for output_variable in command_info["output_variables"]:
+            command_info[
+                "command"
+            ] += f' && echo "output of {output_variable}:${output_variable}"'
     return command_info
 
 

megalinter/tests/test_megalinter/pre_post_test.py

@@ -7,7 +7,7 @@
 import unittest
 import uuid
 
-from megalinter import utilstest
+from megalinter import config, utilstest
 
 
 class PrePostTest(unittest.TestCase):
@@ -25,6 +25,7 @@ def setUp(self):
         )
 
     def test_pre_post_success(self):
+        config.SKIP_DELETE_CONFIG = True
         mega_linter, output = utilstest.call_mega_linter(
             {
                 "MULTI_STATUS": "false",
@@ -42,3 +43,25 @@ def test_pre_post_success(self):
         self.assertIn("descriptor post-command has been run", output)
         self.assertIn("linter pre-command has been run", output)
         self.assertIn("linter post-command has been run", output)
+        self.assertTrue(
+            config.get(self.request_id, "MY_OUTPUT_VARIABLE", "")
+            == "my output variable value",
+            "MY_OUTPUT_VARIABLE should be found",
+        )
+        self.assertTrue(
+            config.get(self.request_id, "MY_OUTPUT_VARIABLE2", "")
+            == "my output variable value2",
+            "MY_OUTPUT_VARIABLE2 should be found",
+        )
+        self.assertTrue(
+            config.get(self.request_id, "MY_OUTPUT_LINTER_VARIABLE", "")
+            == "my output linter variable value",
+            "MY_OUTPUT_LINTER_VARIABLE should be found",
+        )
+        self.assertTrue(
+            config.get(self.request_id, "MY_OUTPUT_LINTER_VARIABLE2", "")
+            == "my output linter variable value2",
+            "MY_OUTPUT_LINTER_VARIABLE2 should be found",
+        )
+        config.SKIP_DELETE_CONFIG = False
+        config.delete(self.request_id)