USE-91-handle-empty-crawls (#51)

Why these changes are being introduced:
* Empty crawls were exiting on an error code when this is an expected scenario that should be handled with a clean exit.

How this addresses that need:
* Add NoValidSeedsError exception
* Update harvest CLI command to exit on NoValidSeedsError
* Add command_exit to run after all CLI commands to account for updated harvest CLI command
* Update _handle_subprocess_logging method to raise NoValidSeedsError exception and add a try/except block for JSON decode errors
* Add corresponding CLI and unit tests

Side effects of this change:
* None

Relevant ticket(s):
* https://mitlibraries.atlassian.net/browse/USE-91

Author: ehanson8

harvester/cli.py

@@ -6,12 +6,14 @@
 import os
 from datetime import UTC, datetime, timedelta
 from time import perf_counter
+from typing import Any
 
 import click
 import smart_open  # type: ignore[import]
 
 from harvester.config import configure_logger, configure_sentry
 from harvester.crawl import Crawler
+from harvester.exceptions import NoValidSeedsError
 from harvester.metadata import CrawlMetadataParser
 from harvester.sitemaps import SitemapsParser
 from harvester.utils import require_container
@@ -231,7 +233,7 @@ def generate_metadata_records(
 )
 @click.pass_context
 def harvest(
-    ctx: click.Context,
+    _ctx: click.Context,
     crawl_name: str,
     config_yaml_file: str,
     sitemaps: tuple[str, ...],
@@ -286,7 +288,11 @@ def harvest(
         btrix_args_json=btrix_args_json,
         urls_file=urls_file,
     )
-    crawler.crawl()
+    try:
+        crawler.crawl()
+    except NoValidSeedsError as e:
+        logger.info(e)
+        return
     logger.info("Crawl complete, WACZ archive located at: %s", crawler.wacz_filepath)
 
     # upload WACZ if output file destination provided
@@ -325,6 +331,10 @@ def harvest(
             ) as urls_out:
                 urls_out.write(urls_in.read())
 
+
+@main.result_callback()
+@click.pass_context
+def command_exit(ctx: click.Context, *_args: Any, **_kwargs: Any) -> None:  # noqa: ANN401
     logger.info(
         "Total elapsed: %s",
         str(

harvester/crawl.py

@@ -8,7 +8,11 @@
 
 import smart_open  # type: ignore[import]
 
-from harvester.exceptions import ConfigYamlError, WaczFileDoesNotExist
+from harvester.exceptions import (
+    ConfigYamlError,
+    NoValidSeedsError,
+    WaczFileDoesNotExist,
+)
 from harvester.utils import require_container
 
 logger = logging.getLogger(__name__)
@@ -101,11 +105,21 @@ def _handle_subprocess_logging(self, process: subprocess.Popen[str]) -> None:
                     if '"context":"crawlStatus"' in line:
                         self._log_crawl_count_status(line)
 
-                    # identify fatal logs and raise Runtime exception
+                    # identify fatal logs and handle appropriately
                     if '"logLevel":"fatal"' in line:
-                        raise RuntimeError(
-                            f"Fatal log message detected from crawler, exiting: {line}"
-                        )
+                        try:
+                            log_data = json.loads(line)
+                            if "No valid seeds specified" in log_data.get("message", ""):
+                                raise NoValidSeedsError(log_data.get("message", line))
+
+                            raise RuntimeError(
+                                "Fatal log message detected from crawler, exiting: "
+                                f"{log_data.get('message', line)}"
+                            )
+                        except json.JSONDecodeError as e:
+                            raise RuntimeError(
+                                f"Could not parse fatal log message: {line}"
+                            ) from e
 
         if process.stderr:  # pragma: no cover
             for line in process.stderr:

harvester/exceptions.py

@@ -17,3 +17,7 @@ class ConfigYamlError(Exception):
 
 class ContextManagerRequiredError(Exception):
     pass
+
+
+class NoValidSeedsError(Exception):
+    pass

tests/test_cli.py

@@ -8,6 +8,7 @@
 import smart_open
 
 from harvester.cli import main
+from harvester.exceptions import NoValidSeedsError
 
 
 def test_cli_no_command_options(caplog, runner):
@@ -292,3 +293,27 @@ def test_cli_harvest_with_sitemap_urls_output_file(caplog, runner):
         )
 
         assert mock_write_urls.call_count == 2
+
+
+def test_cli_harvest_handles_no_valid_seeds(caplog, runner):
+    with patch(
+        "harvester.crawl.Crawler.crawl",
+        side_effect=NoValidSeedsError(
+            "No valid seeds specified, aborting crawl. Quitting"
+        ),
+    ):
+        result = runner.invoke(
+            main,
+            [
+                "--verbose",
+                "harvest",
+                "--crawl-name",
+                "homepage",
+                "--config-yaml-file",
+                "tests/fixtures/lib-website-homepage.yaml",
+                "--wacz-output-file",
+                "/tmp/homepage.wacz",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "No valid seeds specified, aborting crawl" in caplog.text

tests/test_crawler.py

@@ -11,6 +11,7 @@
 from harvester.crawl import Crawler
 from harvester.exceptions import (
     ConfigYamlError,
+    NoValidSeedsError,
     RequiresContainerContextError,
     WaczFileDoesNotExist,
 )
@@ -187,3 +188,24 @@ def test_crawl_fatal_log_raises_runtime_error(create_mocked_crawler):
         "subprocess.Popen", return_value=mock_process
     ):
         crawler.crawl()
+
+
+@pytest.mark.usefixtures("_mock_inside_container")
+def test_crawl_no_valid_seeds_raises_exception(create_mocked_crawler):
+    crawler = create_mocked_crawler()
+    stdouts = [
+        (
+            '{"logLevel":"fatal","message":"No valid seeds specified, '
+            'aborting crawl. Quitting","context":"general","details":{}}'
+        )
+    ]
+    mock_process = MagicMock()
+    mock_process.__enter__.return_value = mock_process
+    mock_process.__exit__.return_value = None
+    mock_process.stdout = iter(stdouts)
+    mock_process.stderr = iter([])
+    mock_process.wait.return_value = 0
+    with pytest.raises(NoValidSeedsError), patch(
+        "subprocess.Popen", return_value=mock_process
+    ):
+        crawler.crawl()