From 09f22f44d9e95486bfeaf176d1ccfd092aec7cf1 Mon Sep 17 00:00:00 2001 From: Aaron Steers <18150651+aaronsteers@users.noreply.github.com> Date: Mon, 23 Mar 2020 17:41:17 -0700 Subject: [PATCH 1/3] infra docs v3 --- slalom/dataops/infra.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/slalom/dataops/infra.py b/slalom/dataops/infra.py index 5a7a395..d9179b2 100644 --- a/slalom/dataops/infra.py +++ b/slalom/dataops/infra.py @@ -149,11 +149,13 @@ def init_and_apply(infra_dir: str = "./infra/", save_output: bool = False): def update_module_docs( tf_dir: str, + *, recursive: bool = True, readme: str = "README.md", footer: bool = True, header: bool = True, special_case_words: List[str] = None, + extra_docs_names: List[str] = ["USAGE.md", "NOTES.md"], git_repo: str = "https://github.com/slalom-ggp/dataops-infra", ): """ @@ -167,6 +169,8 @@ def update_module_docs( readme : Optional (default="README.md"). The filename to create when generating docs. footnote: Optional (default=True). 'True' to include the standard footnote. special_case_words: Optional. A list of words to override special casing rules. + extra_docs_names: (Optional.) A list of filenames which, if found, will be appended + to each module's README.md file. git_repo: Optional. The git repo path to use in rendering 'source' paths. Returns: @@ -175,7 +179,11 @@ def update_module_docs( """ markdown_text = "" if ".git" not in tf_dir and ".terraform" not in tf_dir: - if [x for x in io.list_files(tf_dir) if x.endswith(".tf")]: + tf_files = [x for x in io.list_files(tf_dir) if x.endswith(".tf")] + extra_docs = [ + x for x in io.list_files(tf_dir) if extra_docs_names and x in extra_docs_names + ] + if tf_files: module_title = _proper( os.path.basename(tf_dir), special_case_words=special_case_words ) @@ -195,7 +203,12 @@ def update_module_docs( module_title=module_title, module_path=module_path ) markdown_text += markdown_output - for extra_file in ["NOTES.md"]: + markdown_text += "\n\n##Source Code Files:\n\n* " + markdown_text += "\n* ".join( + ["* [{tf_file}]({tf_file})" for tf_file in tf_files] + ) + markdown_text += "\n\n" + for extra_file in extra_docs: extra_filepath = f"{tf_dir}/{extra_file}" if os.path.isfile(extra_filepath): markdown_text += io.get_text_file_contents(extra_filepath) + "\n" From e3f5400b42f2b779bc17e4e74bde91e935badecd Mon Sep 17 00:00:00 2001 From: Aaron Steers <18150651+aaronsteers@users.noreply.github.com> Date: Mon, 23 Mar 2020 19:05:39 -0700 Subject: [PATCH 2/3] update docs creation code --- slalom/dataops/infra.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/slalom/dataops/infra.py b/slalom/dataops/infra.py index d9179b2..1154598 100644 --- a/slalom/dataops/infra.py +++ b/slalom/dataops/infra.py @@ -141,6 +141,14 @@ def init_and_apply(infra_dir: str = "./infra/", save_output: bool = False): DOCS_FOOTER = """ --------------------- +## Source Files + +_Source code for this module is available using the links below._ + +{src} + +--------------------- + _**NOTE:** This documentation was auto-generated using `terraform-docs` and `s-infra` from `slalom.dataops`. Please do not attempt to manually update this file._ @@ -181,7 +189,9 @@ def update_module_docs( if ".git" not in tf_dir and ".terraform" not in tf_dir: tf_files = [x for x in io.list_files(tf_dir) if x.endswith(".tf")] extra_docs = [ - x for x in io.list_files(tf_dir) if extra_docs_names and x in extra_docs_names + x + for x in io.list_files(tf_dir) + if extra_docs_names and os.path.basename(x) in extra_docs_names ] if tf_files: module_title = _proper( @@ -203,17 +213,17 @@ def update_module_docs( module_title=module_title, module_path=module_path ) markdown_text += markdown_output - markdown_text += "\n\n##Source Code Files:\n\n* " - markdown_text += "\n* ".join( - ["* [{tf_file}]({tf_file})" for tf_file in tf_files] - ) - markdown_text += "\n\n" for extra_file in extra_docs: - extra_filepath = f"{tf_dir}/{extra_file}" - if os.path.isfile(extra_filepath): - markdown_text += io.get_text_file_contents(extra_filepath) + "\n" + markdown_text += io.get_text_file_contents(extra_file) + "\n" if footer: - markdown_text += DOCS_FOOTER + markdown_text += DOCS_FOOTER.format( + src="\n".join( + [ + "* [{f}]({f})".format(f=os.path.basename(tf_file)) + for tf_file in tf_files + ] + ) + ) io.create_text_file(f"{tf_dir}/{readme}", markdown_text) if recursive: for folder in io.list_files(tf_dir): From 15dd9c4f93cf5c5fc1fd96ea89530b2fea365d58 Mon Sep 17 00:00:00 2001 From: Aaron Steers <18150651+aaronsteers@users.noreply.github.com> Date: Fri, 3 Apr 2020 08:53:34 -0700 Subject: [PATCH 3/3] rename and deprecate old name --- slalom/dataops/infra.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/slalom/dataops/infra.py b/slalom/dataops/infra.py index 1154598..c239500 100644 --- a/slalom/dataops/infra.py +++ b/slalom/dataops/infra.py @@ -355,7 +355,7 @@ def _log_issue(module_path, issue_desc, details_list): return result_str -def change_upstream_source( +def update_catalog_source( dir_to_update=".", git_repo="https://github.com/slalom-ggp/dataops-infra", branch="master", @@ -403,6 +403,16 @@ def change_upstream_source( jobs.run_command("terraform fmt -recursive", dir_to_update) +def deprecated(fn, msg): + """Wrapper for deprecated function. Return the function after printing warning.""" + + def _fn(*args, **kwargs): + logging.warning(f"Deprecation warning: {msg}") + _fn(*args, **kwargs) + + return _fn + + def main(): fire.Fire( { @@ -411,7 +421,14 @@ def main(): "apply": apply, "init+apply": init_and_apply, "deploy": init_and_apply, - "change_upstream_source": change_upstream_source, + "update_catalog_source": update_catalog_source, + "change_upstream_source": deprecated( + update_catalog_source, + msg=( + "Command 'update_catalog_source' is deprecated. " + "Please use 'update_catalog_source' instead." + ), + ), "update_module_docs": update_module_docs, "get_tf_metadata": get_tf_metadata, "check_tf_metadata": check_tf_metadata,