From 7070cfa8e7861653e6d3111a409386d4ff67961f Mon Sep 17 00:00:00 2001 From: Pete Gadomski Date: Thu, 24 Mar 2022 07:00:44 -0600 Subject: [PATCH 1/3] fix: typing in validation test file --- tests/cli/commands/test_validate.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/cli/commands/test_validate.py b/tests/cli/commands/test_validate.py index aa36c56e..ba7f5327 100644 --- a/tests/cli/commands/test_validate.py +++ b/tests/cli/commands/test_validate.py @@ -1,48 +1,49 @@ from typing import Callable, List +from click import Group, Command from stactools.cli.commands.validate import create_validate_command -from stactools.testing import CliTestCase +from stactools.testing.cli_test import CliTestCase from tests import test_data class ValidatateTest(CliTestCase): - def create_subcommand_functions(self) -> List[Callable]: + def create_subcommand_functions(self) -> List[Callable[[Group], Command]]: return [create_validate_command] - def test_valid_item(self): + def test_valid_item(self) -> None: path = test_data.get_path( "data-files/catalogs/test-case-1/country-1/area-1-1/" "area-1-1-imagery/area-1-1-imagery.json" ) - result = self.run_command(["validate", path, "--no-assets"]) + result = self.run_command(f"validate {path} --no-assets") self.assertEqual(0, result.exit_code) - def test_invalid_item(self): + def test_invalid_item(self) -> None: path = test_data.get_path( "data-files/catalogs/test-case-1/country-1/area-1-1/" "area-1-1-imagery/area-1-1-imagery-invalid.json" ) - result = self.run_command(["validate", path]) + result = self.run_command(f"validate {path}") self.assertEqual(1, result.exit_code) - def test_collection_with_invalid_item(self): + def test_collection_with_invalid_item(self) -> None: path = test_data.get_path( "data-files/catalogs/test-case-1/country-1/area-1-1/collection-invalid.json" ) - result = self.run_command(["validate", path]) + result = self.run_command(f"validate {path}") self.assertEqual(1, result.exit_code) - def test_collection_with_invalid_item_no_validate_all(self): + def test_collection_with_invalid_item_no_validate_all(self) -> None: path = test_data.get_path( "data-files/catalogs/test-case-1/country-1/area-1-1/collection-invalid.json" ) - result = self.run_command(["validate", path, "--no-recurse"]) + result = self.run_command(f"validate {path} --no-recurse") self.assertEqual(0, result.exit_code) - def test_collection_invalid_asset(self): + def test_collection_invalid_asset(self) -> None: path = test_data.get_path( "data-files/catalogs/test-case-1/country-1" "/area-1-1/area-1-1-imagery/area-1-1-imagery.json" ) - result = self.run_command(["validate", path]) + result = self.run_command(f"validate {path}") self.assertEqual(1, result.exit_code) From 4cf011cafe22e64f5932f5e94edac0bb47c13eb3 Mon Sep 17 00:00:00 2001 From: Pete Gadomski Date: Thu, 24 Mar 2022 10:44:11 -0600 Subject: [PATCH 2/3] feat: use stac-validator for validation Co-authored-by: Phil Varner --- requirements-min.txt | 1 + setup.cfg | 1 + src/stactools/cli/commands/validate.py | 119 +++++++++---------------- tests/cli/commands/test_validate.py | 11 ++- 4 files changed, 53 insertions(+), 79 deletions(-) diff --git a/requirements-min.txt b/requirements-min.txt index 2b72324f..b9f7c63e 100644 --- a/requirements-min.txt +++ b/requirements-min.txt @@ -9,3 +9,4 @@ pystac[validation] == 1.2 rasterio == 1.2.9 requests == 2.20 stac-check == 1.2.0 +stac-validator == 3.1.0 diff --git a/setup.cfg b/setup.cfg index e3c10676..f4cec423 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,6 +43,7 @@ install_requires = rasterio >= 1.2.9 requests >= 2.20 stac-check >= 1.2.0 + stac-validator >= 3.1.0 [options.extras_require] all = diff --git a/src/stactools/cli/commands/validate.py b/src/stactools/cli/commands/validate.py index 6e5ca710..bd777eb0 100644 --- a/src/stactools/cli/commands/validate.py +++ b/src/stactools/cli/commands/validate.py @@ -1,97 +1,66 @@ +import json import sys -from typing import List, Optional +from typing import Optional import click -import pystac -from pystac import Catalog, Collection, Item, STACObject, STACValidationError - -from stactools.core.utils import href_exists +from stac_validator.validate import StacValidate def create_validate_command(cli: click.Group) -> click.Command: @cli.command("validate", short_help="Validate a stac object.") @click.argument("href") @click.option( - "--recurse/--no-recurse", + "--recursive/--no-recursive", + help="Recursively validate all STAC objects in this catalog.", default=True, - help=( - "If false, do not validate any children " - "(only useful for Catalogs and Collections" - ), ) @click.option( - "--links/--no-links", - default=True, - help=("If false, do not check any of the objects's links."), + "--validate-links/--no-validate-links", help="Validate links.", default=True ) @click.option( - "--assets/--no-assets", - default=True, - help=("If false, do not check any of the collection's/item's assets."), + "--validate-assets/--no-validate-assets", help="Validate assets.", default=True + ) + @click.option("-v", "--verbose", is_flag=True, help="Enables verbose output.") + @click.option( + "--quiet/--no-quiet", help="Do not print output to console.", default=False ) - def validate_command(href: str, recurse: bool, links: bool, assets: bool) -> None: + @click.option( + "--log-file", + help="Save output to file (local filepath).", + ) + def validate_command( + href: str, + recursive: bool, + validate_links: bool, + validate_assets: bool, + verbose: bool, + quiet: bool, + log_file: Optional[str], + ) -> None: """Validates a STAC object. - Prints any validation errors to stdout. - """ - object = pystac.read_file(href) - - if isinstance(object, Item): - errors = validate(object, None, False, links, assets) - else: - errors = validate(object, object, recurse, links, assets) + This is a thin wrapper around + [stac-validate](https://github.com/stac-utils/stac-validator). Not all + command-line options are exposed. If you want more control over + validation, use `stac-validator` directly. - if not errors: - click.secho("OK", fg="green", nl=False) - click.echo(f" STAC object at {href} is valid!") + If you'd like linting, use `stac lint`. + """ + validate = StacValidate( + href, + recursive=recursive, + links=validate_links, + assets=validate_assets, + verbose=verbose, + no_output=quiet, + log=log_file or "", + ) + is_valid = validate.run() + if not quiet: + click.echo(json.dumps(validate.message, indent=4)) + if is_valid: + sys.exit(0) else: - for error in errors: - click.secho("ERROR", fg="red", nl=False) - click.echo(f" {error}") sys.exit(1) return validate_command - - -def validate( - object: STACObject, - root: Optional[STACObject], - recurse: bool, - links: bool, - assets: bool, -) -> List[str]: - errors: List[str] = [] - - try: - object.validate() - except FileNotFoundError as e: - errors.append(f"File not found: {e}") - except STACValidationError as e: - errors.append(f"{e}\n{e.source}") - - if links: - for link in object.get_links(): - href = link.get_absolute_href() - assert href - if not href_exists(href): - errors.append( - f'Missing link in {object.self_href}: "{link.rel}" -> {link.href}' - ) - - if assets and (isinstance(object, Item) or isinstance(object, Collection)): - for name, asset in object.get_assets().items(): - href = asset.get_absolute_href() - assert href - if not href_exists(href): - errors.append( - f"Asset '{name}' does not exist: {asset.get_absolute_href()}" - ) - - if recurse: - assert isinstance(object, Catalog) or isinstance(object, Collection) - for child in object.get_children(): - errors.extend(validate(child, root, recurse, links, assets)) - for item in object.get_items(): - errors.extend(validate(item, root, False, links, assets)) - - return errors diff --git a/tests/cli/commands/test_validate.py b/tests/cli/commands/test_validate.py index ba7f5327..cdd60247 100644 --- a/tests/cli/commands/test_validate.py +++ b/tests/cli/commands/test_validate.py @@ -1,6 +1,7 @@ from typing import Callable, List -from click import Group, Command +from click import Command, Group + from stactools.cli.commands.validate import create_validate_command from stactools.testing.cli_test import CliTestCase from tests import test_data @@ -15,7 +16,7 @@ def test_valid_item(self) -> None: "data-files/catalogs/test-case-1/country-1/area-1-1/" "area-1-1-imagery/area-1-1-imagery.json" ) - result = self.run_command(f"validate {path} --no-assets") + result = self.run_command(f"validate {path} --no-validate-assets") self.assertEqual(0, result.exit_code) def test_invalid_item(self) -> None: @@ -37,7 +38,7 @@ def test_collection_with_invalid_item_no_validate_all(self) -> None: path = test_data.get_path( "data-files/catalogs/test-case-1/country-1/area-1-1/collection-invalid.json" ) - result = self.run_command(f"validate {path} --no-recurse") + result = self.run_command(f"validate {path} --no-recursive") self.assertEqual(0, result.exit_code) def test_collection_invalid_asset(self) -> None: @@ -46,4 +47,6 @@ def test_collection_invalid_asset(self) -> None: "/area-1-1/area-1-1-imagery/area-1-1-imagery.json" ) result = self.run_command(f"validate {path}") - self.assertEqual(1, result.exit_code) + self.assertEqual( + 0, result.exit_code + ) # unreachable links aren't an error in stac-validator From fc071e0318df5d21f8350bb8d9db6b844cff7786 Mon Sep 17 00:00:00 2001 From: Pete Gadomski Date: Tue, 3 May 2022 06:41:19 -0600 Subject: [PATCH 3/3] chore: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be065155..a9e5cf36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Use black (instead of yapf) for formatting ([#274](https://github.com/stac-utils/stactools/pull/274)) - stac-check version and lint reporting ([#258](https://github.com/stac-utils/stactools/pull/258)) - Sphinx theme ([#284](https://github.com/stac-utils/stactools/pull/284)) +- Use stac-validator for validation ([#289](https://github.com/stac-utils/stactools/pull/289)) ### Fixed