Sync settings and CLI arguments

Most CLI arguments can be now saved into the configuration profile JSON.
This also cleans up passing around some, but not all args, into various
classes and methods, instead of just passing all of settings.

Author: DragoonAethis

itch_dl/cli.py

@@ -17,50 +17,52 @@ def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Bulk download stuff from Itch.io.")
     parser.add_argument("url_or_path",
                         help="itch.io URL or path to a game jam entries.json file")
-    parser.add_argument("--api-key", metavar="key", default=None,
-                        help="itch.io API key - https://itch.io/user/settings/api-keys")
     parser.add_argument("--profile", metavar="profile", default=None,
                         help="configuration profile to load")
+
+    # These args must match config.py -> Settings class. Make sure all defaults here
+    # evaluate to False, or apply_args_on_settings will override profile settings.
+    parser.add_argument("--api-key", metavar="key", default=None,
+                        help="itch.io API key - https://itch.io/user/settings/api-keys")
+    parser.add_argument("--user-agent", metavar="agent", default=None,
+                        help="user agent to use when sending HTTP requests")
+    parser.add_argument("--download-to", metavar="path", default=None,
+                        help="directory to save results into (default: current working dir)")
+    parser.add_argument("--mirror-web", action="store_true",
+                        help="try to fetch assets on game sites")
     parser.add_argument("--urls-only", action="store_true",
                         help="print scraped game URLs without downloading them")
-    parser.add_argument("--download-to", metavar="path",
-                        help="directory to save results into (default: current dir)")
-    parser.add_argument("--parallel", metavar="parallel", type=int, default=1,
+    parser.add_argument("--parallel", metavar="parallel", type=int, default=None,
                         help="how many threads to use for downloading games (default: 1)")
-    parser.add_argument("--mirror-web", action="store_true",
-                        help="try to fetch assets on game sites")
     parser.add_argument("--filter-files-glob", metavar="glob", default=None,
                         help="filter downloaded files with a shell-style glob/fnmatch (unmatched files are skipped)")
     parser.add_argument("--filter-files-regex", metavar="regex", default=None,
                         help="filter downloaded files with a Python regex (unmatched files are skipped)")
     parser.add_argument("--verbose", action="store_true",
                         help="print verbose logs")
+
     return parser.parse_args()
     # fmt: on
 
 
-def apply_args_on_settings(args: argparse.Namespace, settings: Settings):
-    """Apply settings overrides from provided command line arguments, if set."""
-    for key in ("api_key", "filter_files_glob", "filter_files_regex"):
-        value = getattr(args, key)
-        if value:
-            setattr(settings, key, value)
-
-
 def run() -> int:
     args = parse_args()
     if args.verbose:
         logging.getLogger().setLevel(logging.DEBUG)
 
-    settings = load_config(profile=args.profile)
-    apply_args_on_settings(args, settings)
+    settings: Settings = load_config(args, profile=args.profile)
+    if settings.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
 
     if not settings.api_key:
         exit(
             "You did not provide an API key which itch-dl requires.\n"
             "See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info."
         )
 
+    url_or_path = args.url_or_path
+    del args  # Do not use `args` beyond this point.
+
     # Check API key validity:
     client = ItchApiClient(settings.api_key, settings.user_agent)
     profile_req = client.get("/profile")
@@ -70,25 +72,24 @@ def run() -> int:
             "See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info."
         )
 
-    jobs = get_jobs_for_url_or_path(args.url_or_path, settings)
+    jobs = get_jobs_for_url_or_path(url_or_path, settings)
     jobs = list(set(jobs))  # Deduplicate, just in case...
     logging.info(f"Found {len(jobs)} URL(s).")
 
     if len(jobs) == 0:
         exit("No URLs to download.")
 
-    if args.urls_only:
+    if settings.urls_only:
         for job in jobs:
             print(job)
 
         return 0
 
-    download_to = os.getcwd()
-    if args.download_to is not None:
-        download_to = os.path.normpath(args.download_to)
-        os.makedirs(download_to, exist_ok=True)
+    # If the download dir is not set, use the current working dir:
+    settings.download_to = os.path.normpath(settings.download_to or os.getcwd())
+    os.makedirs(settings.download_to, exist_ok=True)
 
     # Grab all the download keys (there's no way to fetch them per title...):
     keys = get_download_keys(client)
 
-    return drive_downloads(jobs, download_to, args.mirror_web, settings, keys, parallel=args.parallel)
+    return drive_downloads(jobs, settings, keys)

itch_dl/config.py

@@ -2,13 +2,26 @@
 import json
 import logging
 import platform
+import argparse
 from typing import Optional
 
 import requests
 from pydantic import BaseModel
 
 from . import __version__
 
+OVERRIDABLE_SETTINGS = (
+    "api_key",
+    "user_agent",
+    "download_to",
+    "mirror_web",
+    "urls_only",
+    "parallel",
+    "filter_files_glob",
+    "filter_files_regex",
+    "verbose",
+)
+
 
 class Settings(BaseModel):
     """Available settings for itch-dl. Make sure all of them
@@ -17,9 +30,16 @@ class Settings(BaseModel):
     api_key: Optional[str] = None
     user_agent: str = f"python-requests/{requests.__version__} itch-dl/{__version__}"
 
+    download_to: Optional[str] = None
+    mirror_web: bool = False
+    urls_only: bool = False
+    parallel: int = 1
+
     filter_files_glob: Optional[str] = None
     filter_files_regex: Optional[str] = None
 
+    verbose: bool = False
+
 
 def create_and_get_config_path() -> str:
     """Returns the configuration directory in the appropriate
@@ -37,7 +57,7 @@ def create_and_get_config_path() -> str:
     return os.path.join(base_path, "itch-dl")
 
 
-def load_config(profile: Optional[str] = None) -> Settings:
+def load_config(args: argparse.Namespace, profile: Optional[str] = None) -> Settings:
     """Loads the configuration from the file system if it exists,
     the returns a Settings object."""
     config_path = create_and_get_config_path()
@@ -58,4 +78,13 @@ def load_config(profile: Optional[str] = None) -> Settings:
 
         config_data.update(profile_data)
 
-    return Settings(**config_data)
+    # All settings from the base file:
+    settings = Settings(**config_data)
+
+    # Apply overrides from CLI args:
+    for key in OVERRIDABLE_SETTINGS:
+        value = getattr(args, key)
+        if value:
+            setattr(settings, key, value)
+
+    return settings

itch_dl/downloader.py

@@ -62,10 +62,7 @@ class GameMetadata(TypedDict, total=False):
 
 
 class GameDownloader:
-    def __init__(self, download_to: str, mirror_web: bool, settings: Settings, keys: Dict[int, str]):
-        self.download_to = download_to
-        self.mirror_web = mirror_web
-
+    def __init__(self, settings: Settings, keys: Dict[int, str]):
         self.settings = settings
         self.download_keys = keys
         self.client = ItchApiClient(settings.api_key, settings.user_agent)
@@ -258,7 +255,7 @@ def download(self, url: str, skip_downloaded: bool = True):
 
         author, game = match["author"], match["game"]
 
-        download_path = os.path.join(self.download_to, author, game)
+        download_path = os.path.join(self.settings.download_to, author, game)
         os.makedirs(download_path, exist_ok=True)
 
         paths: Dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()}
@@ -372,7 +369,7 @@ def download(self, url: str, skip_downloaded: bool = True):
             logging.warning(f"Game {title} has external download URLs: {external_urls}")
 
         # TODO: Mirror JS/CSS assets
-        if self.mirror_web:
+        if self.settings.mirror_web:
             os.makedirs(paths["screenshots"], exist_ok=True)
             for screenshot in metadata["screenshots"]:
                 if not screenshot:
@@ -406,20 +403,17 @@ def download(self, url: str, skip_downloaded: bool = True):
 
 def drive_downloads(
     jobs: List[str],
-    download_to: str,
-    mirror_web: bool,
     settings: Settings,
     keys: Dict[int, str],
-    parallel: int = 1,
 ):
-    downloader = GameDownloader(download_to, mirror_web, settings, keys)
+    downloader = GameDownloader(settings, keys)
     tqdm_args = {
         "desc": "Games",
         "unit": "game",
     }
 
-    if parallel > 1:
-        results = thread_map(downloader.download, jobs, max_workers=parallel, **tqdm_args)
+    if settings.parallel > 1:
+        results = thread_map(downloader.download, jobs, max_workers=settings.parallel, **tqdm_args)
     else:
         results = [downloader.download(job) for job in tqdm(jobs, **tqdm_args)]