feat: improve development tooling

.gitignore

@@ -136,4 +136,7 @@ creds/
 
 node_modules/
 fixtures/
-cypress/videos/
+cypress/videos/
+
+.env*
+.secrets/

.tool-versions

@@ -1 +1,2 @@
 python 3.10.10
+poetry 1.8.2

compose.yaml

@@ -4,13 +4,12 @@ services:
     image: postgres:16.1-alpine3.19
     restart: always
     environment:
-      POSTGRES_HOST_AUTH_METHOD: trust
-#      POSTGRES_USER: postgres
-#      POSTGRES_PASSWORD: postgres
-#      POSTGRES_DB: postgres
+      POSTGRES_USER: gary
+      POSTGRES_PASSWORD: w1shl1st
+      POSTGRES_DB: gary_db
     ports:
-      - "5900:5432"
-  redis:
-    image: redis:alpine3.17
-    ports:
-      - "6379:6379"
+      - "5932:5432"
+  # valkey:
+  #   image: valkey/valkey:7-alpine3.19
+  #   ports:
+  #     - "6379:6379"

development.py

@@ -0,0 +1,279 @@
+import argparse
+import gzip
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import boto3
+import botocore
+from dotenv import load_dotenv
+
+
+BASE_DIR = Path(__file__).resolve().parent
+
+load_dotenv()
+
+
+def require_bucket_credentials():
+    envvars = {
+        "CAMINO_ARTIFACT_S3_ACCESS_KEY": os.getenv("CAMINO_ARTIFACT_S3_ACCESS_KEY"),
+        "CAMINO_ARTIFACT_S3_SECRET_KEY": os.getenv("CAMINO_ARTIFACT_S3_SECRET_KEY"),
+        "CAMINO_ARTIFACT_S3_BUCKET_NAME": os.getenv("CAMINO_ARTIFACT_S3_BUCKET_NAME"),
+        "CAMINO_ARTIFACT_S3_ENDPOINT": os.getenv("CAMINO_ARTIFACT_S3_ENDPOINT"),
+    }
+    is_missing = False
+    for envvar, value in envvars.items():
+        if value is None:
+            is_missing = True
+            sys.stderr.write(f"Missing required envvar {envvar}\n")
+    if is_missing:
+        sys.exit(1)
+    return envvars
+
+
+def new_s3_connection():
+    creds = require_bucket_credentials()
+    session = boto3.session.Session()
+    client = session.client(
+        "s3",
+        config=botocore.config.Config(s3={"addressing_style": "virtual"}),
+        region_name="sfo3",
+        endpoint_url="https://sfo3.digitaloceanspaces.com",
+        aws_access_key_id=creds["CAMINO_ARTIFACT_S3_ACCESS_KEY"],
+        aws_secret_access_key=creds["CAMINO_ARTIFACT_S3_SECRET_KEY"],
+    )
+    bucket_name = creds["CAMINO_ARTIFACT_S3_BUCKET_NAME"]
+    return client, bucket_name
+
+def clone_database_from_pgdump_archive(list_sources, from_db, args, database_url):
+    client, bucket_name = new_s3_connection()
+    list_kwargs = {
+        "Bucket": bucket_name,
+    }
+    if list_sources or from_db:
+        list_kwargs["Prefix"] = f"{list_sources or from_db}"
+    items = client.list_objects(**list_kwargs)["Contents"]
+
+    if args._list:
+        for obj in items:
+            print("Object: {}".format(obj["Key"]))
+    if (
+        args._from
+    ):  # To use a specific file, specify the full name of the file instead of just e.g. 'development'
+        item = items[-1]
+        item_key = item["Key"]
+        temp_dir = tempfile.mkdtemp(prefix="db-clone-")
+        destination_path = Path(temp_dir) / item_key
+        client.download_file(
+            Bucket=bucket_name,
+            Key=item_key,
+            Filename=str(destination_path),
+        )
+        if args.verbose:
+            print(f"Downloaded {item_key} to {destination_path}")
+        restore_sql = gzip.decompress(destination_path.read_bytes())
+        result = subprocess.run(
+            ["/usr/bin/env", "bash", "-c", f"psql {database_url}"],
+            input=restore_sql,
+            check=True,
+            capture_output=True,
+        )
+        if args.verbose:
+            print(result.stdout.decode())
+
+def clone_database_from_live_instance(from_db, database_url):
+    try:
+        remote_db_url = json.loads(Path(".secrets/database-credentials.json").read_bytes())["by_key"][from_db]
+        subprocess.run(
+            [
+                "/usr/bin/env",
+                "bash",
+                "-c",
+                f"pg_dump {remote_db_url} | psql {database_url}",
+            ],
+            check=True,
+            stdout=sys.stdout,
+            stderr=sys.stderr,
+            stdin=sys.stdin,
+        )
+    except KeyError:
+        print(f"Database {from_db} not found in database-credentials.json")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error reading database-credentials.json: {e}")
+        sys.exit(1)
+
+secret_files = [
+    ("database-credentials.json", ".secrets/database-credentials.json"),
+    (".env", ".env"),
+]
+
+def main():
+    parser = argparse.ArgumentParser(description="Development tooling.")
+    parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output.")
+    component_sp = parser.add_subparsers(dest="component")
+
+    secrets_parser = component_sp.add_parser("secrets", help="Manage secrets.")
+    secrets_action_parser = secrets_parser.add_subparsers(
+        dest="action", help="Secrets action."
+    )
+    secrets_action_parser.add_parser("get", help="Get secrets from 1Password.")
+    secrets_action_parser.add_parser("upload", help="Upload secrets to 1Password.")
+
+    database_parser = component_sp.add_parser("database", help="Manipulate databases.")
+    database_action_parser = database_parser.add_subparsers(
+        dest="action", help="Database action."
+    )
+    database_cloner = database_action_parser.add_parser(
+        "clone", help="Clone a database locally."
+    )
+    database_cloner.add_argument(
+        "--from", dest="_from", type=str, help="Which database to clone."
+    )
+    database_cloner.add_argument(
+        "--list", dest="_list", type=str, help="List available clone sources."
+    )
+    database_cloner.add_argument(
+        "--using-archive-strategy",
+        dest="_using_pgdump_archive",
+        action="store_true",
+        help="Restore from a pgdump archive instead of a live instance.",
+    )
+
+    args = parser.parse_args()
+
+    match args.component:
+        case "database":
+            database_url = os.getenv("DATABASE_URL")
+            if database_url is None:
+                sys.stderr.write(f"Missing DATABASE_URL envvar\n")
+                sys.exit(1)
+            if args.verbose:
+                print(f"DATABASE_URL: {database_url}")
+            match args.action:
+                case "clone":
+                    if args._using_pgdump_archive:
+                        list_sources = args._list
+                        from_db = args._from
+                        clone_database_from_pgdump_archive(list_sources, from_db, args, database_url)
+                    else:
+                        from_db = args._from
+                        clone_database_from_live_instance(from_db, database_url)
+
+        case "secrets":
+            match args.action:
+                case "upload":
+                    if input(
+                        "Are you sure you want to upload secrets to 1Password? [y/N] "
+                    ).lower() != "y":
+                        print("Aborting.")
+                        sys.exit(1)
+
+                    for secret_file, secret_path in secret_files:
+                        f = Path(secret_path).read_bytes()
+                        try:
+                            subprocess.run(
+                                [
+                                    "/usr/bin/env",
+                                    "bash",
+                                    "-c",
+                                    f"op --account my.1password.com --vault Gary document edit {secret_file}",
+                                ],
+                                check=True,
+                                input=f
+                            )
+                        except subprocess.CalledProcessError:
+                            print(f"Failed to get {secret_file}, creating a new document...")
+                            f.seek(0)
+                            subprocess.run(
+                                [
+                                    "/usr/bin/env",
+                                    "bash",
+                                    "-c",
+                                    f"op --account my.1password.com --vault Gary document create {secret_file}",
+                                ],
+                                check=True,
+                                input=f,
+                            )
+                            print(f"Successfully uploaded {secret_file} to 1Password.")
+
+                case "get":
+                    for secret_file, secret_path in secret_files:
+                        leading_dir = os.path.dirname(secret_path)
+                        if leading_dir and not os.path.exists(leading_dir):
+                            os.makedirs(leading_dir)
+                        try:
+                            subprocess.run(
+                                [
+                                    "/usr/bin/env",
+                                    "bash",
+                                    "-c",
+                                    f"op --account my.1password.com --vault Gary document get {secret_file} > {secret_path}",
+                                ],
+                                check=True,
+                            )
+                        except subprocess.CalledProcessError:
+                            print(f"Failed to get {secret_file}")
+                            sys.exit(1)
+
+                    # secrets_cache_location = os.path.join(BASE_DIR, ".secrets")
+                    # if not os.path.exists(secrets_cache_location):
+                    #     os.makedirs(secrets_cache_location)
+                    # for filename in [
+                    #     "database-credentials.json",
+                    # ]:
+                    #     # calling directly (not in a sub-shell) to retain current login state
+                    #     op_subprocess = subprocess.run(
+                    #         [
+                    #             "op",
+                    #             "--account=my.1password.com",
+                    #             "--vault=Gary",
+                    #             "document",
+                    #             "get",
+                    #             filename,
+                    #         ],
+                    #         check=True,
+                    #         stderr=sys.stderr,
+                    #         stdout=subprocess.PIPE,
+                    #         stdin=sys.stdin,
+                    #     )
+                    #     op_stdout = op_subprocess.stdout
+                    #     if op_stdout.strip() == "":
+                    #         raise Exception(
+                    #             "op unexpectedly returned an empty document"
+                    #         )
+
+                    #     output_fn = Path(f"{secrets_cache_location}/{filename}")
+                    #     output_fn.write_bytes(op_stdout)
+                    # for envfile in [
+                    #     ".env",
+                    # ]:
+                    #     if os.path.exists(envfile):
+                    #         if (
+                    #             input(
+                    #                 f"{envfile} already exists, overwrite? [y/N] "
+                    #             ).lower()
+                    #             != "y"
+                    #         ):
+                    #             print(f"Skipping {envfile}")
+                    #             continue
+                    #     subprocess.run(
+                    #         [
+                    #             "/usr/bin/env",
+                    #             "bash",
+                    #             "-c",
+                    #             f"op --account my.1password.com --vault Gary document get {envfile} > {envfile}",
+                    #         ],
+                    #         check=True,
+                    #     )
+                case _:
+                    secrets_parser.print_help()
+        case _:
+            parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

gary/settings.py

@@ -91,6 +91,7 @@ def get_envvar_list(envvar_name, default=[], separator=",", normalize=True):
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "allauth.account.middleware.AccountMiddleware",
 ]
 
 ROOT_URLCONF = "gary.urls"
@@ -171,6 +172,8 @@ def get_envvar_list(envvar_name, default=[], separator=",", normalize=True):
 SOCIALACCOUNT_AUTO_SIGNUP = True
 SOCIALACCOUNT_LOGIN_ON_GET = True
 
+LOGIN_REDIRECT_URL = "/"
+
 ACCOUNT_EMAIL_REQUIRED = True
 ACCOUNT_AUTHENTICATION_METHOD = 'email'
 ACCOUNT_USER_DISPLAY = "gifter.utils.user_display"

justfile

@@ -2,3 +2,12 @@
 db:
     docker compose down || true
     docker compose up -d
+
+db-clone environment_name="production":
+    poetry run python development.py database clone --from {{environment_name}}
+
+secrets:
+    poetry run python development.py secrets get
+
+upload-secrets:
+    poetry run python development.py secrets upload

manage.py

@@ -3,6 +3,9 @@
 import os
 import sys
 
+from dotenv import load_dotenv
+
+load_dotenv()
 
 def main():
     """Run administrative tasks."""