prevent purge_on_delete due to failed compile

src/inmanta/const.py

@@ -112,3 +112,12 @@ class LogLevel(Enum):
 
 
 INMANTA_URN = "urn:inmanta:"
+
+
+class Compilestate(Enum):
+    success = 1
+    failed = 2
+
+
+EXPORT_META_DATA = "export_metadata"
+META_DATA_COMPILE_STATE = "inmanta:compile:state"

src/inmanta/export.py

@@ -266,6 +266,9 @@ def run(self, types, scopes, metadata={}, no_commit=False, include_status=False,
 
             # call dependency managers
             self._call_dep_manager(types)
+            metadata[const.META_DATA_COMPILE_STATE] = const.Compilestate.success.name
+        else:
+            metadata[const.META_DATA_COMPILE_STATE] = const.Compilestate.failed.name
 
         # validate the dependency graph
         self._validate_graph()
@@ -440,7 +443,7 @@ def call():
                 LOGGER.debug("Uploaded file with hash %s" % hash_id)
 
         # Collecting version information
-        version_info = {"export_metadata": metadata,
+        version_info = {const.EXPORT_META_DATA: metadata,
                         "model": model}
 
         # TODO: start transaction

src/inmanta/server/server.py

@@ -809,36 +809,47 @@ def put_version(self, env, version, resources, resource_state, unknowns, version
             res_obj = rv_dict[t.resource_str()]
             res_obj.provides.append(f.resource_version_id)
 
-        # search for deleted resources
-        resources_to_purge = yield data.Resource.get_deleted_resources(env.id, version, set(rv_dict.keys()))
-        previous_requires = {}
-        for res in resources_to_purge:
-            LOGGER.warning("Purging %s, purged resource based on %s" % (res.resource_id, res.resource_version_id))
-
-            attributes = res.attributes.copy()
-            attributes["purged"] = "true"
-            attributes["requires"] = []
-
-            res_obj = data.Resource.new(env.id, resource_version_id="%s,v=%s" % (res.resource_id, version),
-                                        attributes=attributes)
-            resource_objects.append(res_obj)
-
-            previous_requires[res_obj.resource_id] = res.attributes["requires"]
-            resource_version_ids.append(res_obj.resource_version_id)
-            agents.add(res_obj.agent)
-            rv_dict[res_obj.resource_id] = res_obj
-
-        # invert dependencies on purges
-        for res_id, requires in previous_requires.items():
-            res_obj = rv_dict[res_id]
-            for require in requires:
-                req_id = Id.parse_id(require)
-
-                if req_id.resource_str() in rv_dict:
-                    req_res = rv_dict[req_id.resource_str()]
-
-                    req_res.attributes["requires"].append(res_obj.resource_version_id)
-                    res_obj.provides.append(req_res.resource_version_id)
+        # detect failed compiles
+        def safe_get(input, key, default):
+            if not isinstance(input, dict):
+                return default
+            if key not in input:
+                return default
+            return input[key]
+        metadata = safe_get(version_info, const.EXPORT_META_DATA, {})
+        compile_state = safe_get(metadata, const.META_DATA_COMPILE_STATE, "")
+        failed = compile_state == const.Compilestate.failed.name
+
+        if not failed:
+            # search for deleted resources
+            resources_to_purge = yield data.Resource.get_deleted_resources(env.id, version, set(rv_dict.keys()))
+            previous_requires = {}
+            for res in resources_to_purge:
+                LOGGER.warning("Purging %s, purged resource based on %s" % (res.resource_id, res.resource_version_id))
+
+                attributes = res.attributes.copy()
+                attributes["purged"] = "true"
+                attributes["requires"] = []
+                res_obj = data.Resource.new(env.id, resource_version_id="%s,v=%s" % (res.resource_id, version),
+                                            attributes=attributes)
+                resource_objects.append(res_obj)
+
+                previous_requires[res_obj.resource_id] = res.attributes["requires"]
+                resource_version_ids.append(res_obj.resource_version_id)
+                agents.add(res_obj.agent)
+                rv_dict[res_obj.resource_id] = res_obj
+
+            # invert dependencies on purges
+            for res_id, requires in previous_requires.items():
+                res_obj = rv_dict[res_id]
+                for require in requires:
+                    req_id = Id.parse_id(require)
+
+                    if req_id.resource_str() in rv_dict:
+                        req_res = rv_dict[req_id.resource_str()]
+
+                        req_res.attributes["requires"].append(res_obj.resource_version_id)
+                        res_obj.provides.append(req_res.resource_version_id)
 
         undeployable = [res.resource_id for res in undeployable]
         # get skipped for undeployable
@@ -1798,7 +1809,7 @@ def decomission_environment(self, env, metadata):
                 "message": "Decommission of environment",
                 "type": "api"
             }
-        result = yield self.put_version(env, version, [], {}, [], {"export_metadata": metadata})
+        result = yield self.put_version(env, version, [], {}, [], {const.EXPORT_META_DATA: metadata})
         return result, {"version": version}
 
     @protocol.handle(methods.Decommision.clear_environment, env="id")

tests/conftest.py

@@ -41,6 +41,8 @@
 import re
 from tornado.ioloop import IOLoop
 from inmanta.server.bootloader import InmantaBootloader
+from inmanta.export import cfg_env, unknown_parameters
+import traceback
 
 
 DEFAULT_PORT_ENVVAR = 'MONGOBOX_PORT'
@@ -69,6 +71,7 @@ def reset_all():
     # command.Commander.reset()
     handler.Commander.reset()
     Project._project = None
+    unknown_parameters.clear()
 
 
 @pytest.fixture(scope="function", autouse=True)
@@ -286,6 +289,8 @@ def create_env():
     result = io_loop.run_sync(create_env)
     env_id = result.result["environment"]["id"]
 
+    cfg_env.set(env_id)
+
     yield env_id
 
 
@@ -324,11 +329,15 @@ def tearDownClass(self):
         # reset cwd
         os.chdir(self.cwd)
 
-    def setup_for_snippet(self, snippet, autostd=True):
+    def setup_func(self):
         # init project
         self.project_dir = tempfile.mkdtemp()
         os.symlink(self.env, os.path.join(self.project_dir, ".env"))
 
+    def tear_down_func(self):
+        shutil.rmtree(self.project_dir)
+
+    def setup_for_snippet(self, snippet, autostd=True):
         with open(os.path.join(self.project_dir, "project.yml"), "w") as cfg:
             cfg.write(
                 """
@@ -347,23 +356,34 @@ def setup_for_snippet(self, snippet, autostd=True):
 
         Project.set(Project(self.project_dir, autostd=autostd))
 
-    def do_export(self, deploy=False, include_status=False):
+    def do_export(self, deploy=False, include_status=False, do_raise=True):
         templfile = mktemp("json", "dump", self.project_dir)
 
-        from inmanta.export import Exporter
-
-        (types, scopes) = compiler.do_compile()
-
         class Options(object):
             pass
+
         options = Options()
         options.json = templfile if not deploy else None
         options.depgraph = False
         options.deploy = deploy
         options.ssl = False
 
-        export = Exporter(options=options)
-        return export.run(types, scopes, include_status=include_status)
+        from inmanta.export import Exporter  # noqa: H307
+
+        try:
+            (types, scopes) = compiler.do_compile()
+        except Exception:
+            types, scopes = (None, None)
+            if do_raise:
+                raise
+            else:
+                traceback.print_exc()
+
+        # Even if the compile failed we might have collected additional data such as unknowns. So
+        # continue the export
+
+        export = Exporter(options)
+        return export.run(types, scopes, model_export=False, include_status=include_status)
 
     def setup_for_error(self, snippet, shouldbe):
         self.setup_for_snippet(snippet)
@@ -389,14 +409,20 @@ def setup_for_error_re(self, snippet, shouldbe):
 
 
 @pytest.fixture(scope="session")
-def snippetcompiler():
+def snippetcompiler_global():
     ast = SnippetCompilationTest()
     ast.setUpClass()
     yield ast
-    shutil.rmtree(ast.project_dir)
     ast.tearDownClass()
 
 
+@pytest.fixture(scope="function")
+def snippetcompiler(snippetcompiler_global):
+    snippetcompiler_global.setup_func()
+    yield snippetcompiler_global
+    snippetcompiler_global.tear_down_func()
+
+
 class CLI(object):
 
     def __init__(self, io_loop):

tests/test_export.py

@@ -158,7 +158,6 @@ def test_empty_server_export(snippetcompiler, server, client):
 
 @pytest.mark.gen_test
 def test_server_export(snippetcompiler, server, client, environment):
-    config.Config.set("config", "environment", environment)
     snippetcompiler.setup_for_snippet("""
             h = std::Host(name="test", os=std::linux)
             f = std::ConfigFile(host=h, path="/etc/motd", content="test")