From f0d0447e510b8340f6ae5f447840ba276910288f Mon Sep 17 00:00:00 2001 From: JP Maxwell Date: Sat, 17 Oct 2026 01:26:41 -0500 Subject: [PATCH 1/2] Handle Nutch generate status before errexit --- .github/workflows/crawler-tests.yml | 15 ++++++ .gitignore | 1 + conf/nutch/crawler.sh | 11 +++-- readme.md | 9 ++++ tests/test_crawler_control_flow.py | 76 +++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/crawler-tests.yml create mode 100644 tests/test_crawler_control_flow.py diff --git a/.github/workflows/crawler-tests.yml b/.github/workflows/crawler-tests.yml new file mode 100644 index 0000000..b3e1ca9 --- /dev/null +++ b/.github/workflows/crawler-tests.yml @@ -0,0 +1,15 @@ +name: Crawler control flow +on: + push: + pull_request: +permissions: + contents: read +jobs: + offline-tests: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Check shell syntax + run: bash -n conf/nutch/crawler.sh + - name: Exercise crawler without network or search services + run: python3 -m unittest discover -s tests -v diff --git a/.gitignore b/.gitignore index af56f61..e67a40e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store .idea/ +__pycache__/ diff --git a/conf/nutch/crawler.sh b/conf/nutch/crawler.sh index a482c26..494fa96 100755 --- a/conf/nutch/crawler.sh +++ b/conf/nutch/crawler.sh @@ -47,8 +47,13 @@ for i in $(seq ${ITERATIONS}); do batchId=`date +%s`-$RANDOM echo "doing generate" - NUTCH_CONF_DIR=$CONF_DIR $NUTCH_HOME/runtime/local/bin/nutch generate $commonOptions -topN $TOP -noNorm -noFilter -crawlId $CRAWL_ID -batchId $batchId - RETCODE=$? + # Nutch uses 1 for an empty fetch list. Capture it in a conditional so + # errexit does not bypass the status handling below. + if NUTCH_CONF_DIR=$CONF_DIR $NUTCH_HOME/runtime/local/bin/nutch generate $commonOptions -topN $TOP -noNorm -noFilter -crawlId $CRAWL_ID -batchId $batchId; then + RETCODE=0 + else + RETCODE=$? + fi if [ $RETCODE -eq 0 ]; then : # ok: no error elif [ $RETCODE -eq 1 ]; then @@ -84,4 +89,4 @@ curl "http://localhost:$SOLR_PORT/solr/$CORE/suggest?suggest.build=true&wt=json" echo 'Done with all iterations' -exit 0; \ No newline at end of file +exit 0; diff --git a/readme.md b/readme.md index 904e578..259b2f4 100644 --- a/readme.md +++ b/readme.md @@ -1,5 +1,14 @@ # Docker for Openstack Search Service +# Offline crawler control-flow checks + +Run `python3 -m unittest discover -s tests -v` and +`bash -n conf/nutch/crawler.sh`. The tests run the actual crawler shell script +against temporary Nutch/curl stubs. They cover normal batches, Nutch's empty +fetch-list status (1), unexpected generate failures and failures in each later +pipeline stage. They do not fetch pages or modify an index, and do not establish +deployed crawler freshness, robots handling or stale-result deletion behavior. + # build ```` diff --git a/tests/test_crawler_control_flow.py b/tests/test_crawler_control_flow.py new file mode 100644 index 0000000..461f37e --- /dev/null +++ b/tests/test_crawler_control_flow.py @@ -0,0 +1,76 @@ +"""Exercise the real shell script with no network, Nutch, Solr or index writes.""" +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "conf/nutch/crawler.sh" + + +class CrawlerControlFlowTest(unittest.TestCase): + def run_crawler(self, generate_status=0, fail_command=""): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + binaries = root / "bin" + binaries.mkdir() + nutch = root / "runtime/local/bin/nutch" + nutch.parent.mkdir(parents=True) + nutch.write_text('''#!/bin/bash +echo "nutch:$1" >> "$CALL_LOG" +if [ "$1" = generate ]; then exit "$GENERATE_STATUS"; fi +if [ "$1" = "$FAIL_COMMAND" ]; then exit 7; fi +''') + curl = binaries / "curl" + curl.write_text('''#!/bin/bash +echo "curl:$1" >> "$CALL_LOG" +''') + nutch.chmod(0o700) + curl.chmod(0o700) + log = root / "calls" + env = dict(os.environ, PATH=f"{binaries}:{os.environ['PATH']}", + NUTCH_HOME=str(root), SOLR_PORT="8983", CALL_LOG=str(log), + GENERATE_STATUS=str(generate_status), FAIL_COMMAND=fail_command) + result = subprocess.run( + ["bash", str(SCRIPT), "isolated", "1", str(root / "seeds"), + "2", "", "10", str(root / "conf")], + env=env, capture_output=True, text=True, timeout=10) + return result, log.read_text().splitlines() + + def test_success_runs_both_batches_then_finishes_index(self): + result, calls = self.run_crawler() + self.assertEqual(result.returncode, 0, result.stderr) + batch = [f"nutch:{command}" for command in + ("generate", "fetch", "parse", "updatedb", "solrindex", "solrdedup")] + self.assertEqual(calls[:13], ["nutch:inject"] + batch * 2) + self.assertEqual(len(calls), 15) + self.assertIn("/update?", calls[-2]) + self.assertIn("/suggest?", calls[-1]) + + def test_empty_fetch_list_skips_fetch_but_finishes_index(self): + result, calls = self.run_crawler(generate_status=1) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("no more URLs", result.stdout) + self.assertEqual(calls[:2], ["nutch:inject", "nutch:generate"]) + self.assertEqual(len(calls), 4) + self.assertIn("/update?", calls[-2]) + self.assertIn("/suggest?", calls[-1]) + + def test_unexpected_generate_error_preserves_status_and_stops(self): + result, calls = self.run_crawler(generate_status=3) + self.assertEqual(result.returncode, 3) + self.assertIn("Failed with exit value 3", result.stdout) + self.assertEqual(calls, ["nutch:inject", "nutch:generate"]) + + def test_other_pipeline_errors_still_stop_immediately(self): + for command in ("inject", "fetch", "parse", "updatedb", "solrindex", "solrdedup"): + with self.subTest(command=command): + result, calls = self.run_crawler(fail_command=command) + self.assertEqual(result.returncode, 7) + self.assertEqual(calls[-1], f"nutch:{command}") + self.assertFalse(any(call.startswith("curl:") for call in calls)) + + +if __name__ == "__main__": + unittest.main() From 9ff1bd4d841d979bf3caac69dcdd4fadfde46f94 Mon Sep 17 00:00:00 2001 From: JP Maxwell Date: Sat, 17 Oct 2026 01:28:07 -0500 Subject: [PATCH 2/2] Preserve sitemap seeds when retrieval fails --- conf/nutch/crawler.sh | 16 ++++++++++- readme.md | 5 ++++ tests/test_crawler_control_flow.py | 45 ++++++++++++++++++++++++++++-- 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/conf/nutch/crawler.sh b/conf/nutch/crawler.sh index 494fa96..bc54250 100755 --- a/conf/nutch/crawler.sh +++ b/conf/nutch/crawler.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e; +set -o pipefail CORE=$1 CRAWL_ID=$2 @@ -36,7 +37,20 @@ then echo "DOMAIN not set, skipping sitemap retrieval..." else echo "trying to get site map for $DOMAIN..." - curl $DOMAIN/sitemap.xml | grep -e loc | sed 's|\(.*\)<\/loc>$|\1|g' > $SEEDS/sitemap.txt + # Do not replace accepted seeds with an HTTP error or a partial download. + sitemap_tmp=$(mktemp "$SEEDS/.sitemap.XXXXXX") + trap 'rm -f -- "$sitemap_tmp"' EXIT + curl --fail --silent --show-error --connect-timeout 10 --max-time 120 \ + "$DOMAIN/sitemap.xml" | grep -e loc | sed 's|\(.*\)<\/loc>$|\1|g' > "$sitemap_tmp" + test -s "$sitemap_tmp" + # This legacy extractor requires standalone loc lines. Reject residual XML + # rather than injecting malformed seeds when the sitemap format changes. + if grep -Eqv '^https?://[^[:space:]<>]+$' "$sitemap_tmp"; then + echo "Sitemap contains unsupported seed lines" >&2 + exit 1 + fi + mv -- "$sitemap_tmp" "$SEEDS/sitemap.txt" + trap - EXIT fi echo "crawling for core $CORE crawl_id $CRAWL_ID seed $SEEDS"; NUTCH_CONF_DIR=$CONF_DIR $NUTCH_HOME/runtime/local/bin/nutch inject $SEEDS -crawlId $CRAWL_ID diff --git a/readme.md b/readme.md index 259b2f4..6355191 100644 --- a/readme.md +++ b/readme.md @@ -8,6 +8,11 @@ against temporary Nutch/curl stubs. They cover normal batches, Nutch's empty fetch-list status (1), unexpected generate failures and failures in each later pipeline stage. They do not fetch pages or modify an index, and do not establish deployed crawler freshness, robots handling or stale-result deletion behavior. +Sitemap HTTP/download failures, empty/error bodies and unsupported compact XML +stop before injection and preserve previous seeds. Retrieval has a 10-second +connection timeout and 120-second overall limit. Seed replacement is atomic. +The existing extractor still requires standalone loc lines; sitemap indexes and +XML entity decoding require a separate parser change and acceptance. # build diff --git a/tests/test_crawler_control_flow.py b/tests/test_crawler_control_flow.py index 461f37e..2ad6273 100644 --- a/tests/test_crawler_control_flow.py +++ b/tests/test_crawler_control_flow.py @@ -10,7 +10,8 @@ class CrawlerControlFlowTest(unittest.TestCase): - def run_crawler(self, generate_status=0, fail_command=""): + def run_crawler(self, generate_status=0, fail_command="", domain="", + sitemap_status=0, sitemap_body="https://example.test/page\n"): with tempfile.TemporaryDirectory() as directory: root = Path(directory) binaries = root / "bin" @@ -25,17 +26,28 @@ def run_crawler(self, generate_status=0, fail_command=""): curl = binaries / "curl" curl.write_text('''#!/bin/bash echo "curl:$1" >> "$CALL_LOG" +if [ "$1" = --fail ]; then + cat "$SITEMAP_BODY" + exit "$SITEMAP_STATUS" +fi ''') nutch.chmod(0o700) curl.chmod(0o700) log = root / "calls" + seeds = root / "seeds" + seeds.mkdir() + (seeds / "sitemap.txt").write_text("https://example.test/previous\n") + (root / "sitemap-body").write_text(sitemap_body) env = dict(os.environ, PATH=f"{binaries}:{os.environ['PATH']}", NUTCH_HOME=str(root), SOLR_PORT="8983", CALL_LOG=str(log), - GENERATE_STATUS=str(generate_status), FAIL_COMMAND=fail_command) + GENERATE_STATUS=str(generate_status), FAIL_COMMAND=fail_command, + SITEMAP_STATUS=str(sitemap_status), SITEMAP_BODY=str(root / "sitemap-body")) result = subprocess.run( ["bash", str(SCRIPT), "isolated", "1", str(root / "seeds"), - "2", "", "10", str(root / "conf")], + "2", domain, "10", str(root / "conf")], env=env, capture_output=True, text=True, timeout=10) + self.seeds_after = (seeds / "sitemap.txt").read_text() + self.seed_files_after = sorted(p.name for p in seeds.iterdir()) return result, log.read_text().splitlines() def test_success_runs_both_batches_then_finishes_index(self): @@ -71,6 +83,33 @@ def test_other_pipeline_errors_still_stop_immediately(self): self.assertEqual(calls[-1], f"nutch:{command}") self.assertFalse(any(call.startswith("curl:") for call in calls)) + def test_sitemap_replacement_happens_before_injection(self): + result, calls = self.run_crawler(domain="https://example.test") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(calls[:2], ["curl:--fail", "nutch:inject"]) + self.assertEqual(self.seeds_after, "https://example.test/page\n") + self.assertEqual(self.seed_files_after, ["sitemap.txt"]) + + def test_failed_sitemap_preserves_seeds_and_prevents_injection(self): + for status in (22, 28): + with self.subTest(status=status): + result, calls = self.run_crawler( + domain="https://example.test", sitemap_status=status) + self.assertEqual(result.returncode, status) + self.assertEqual(calls, ["curl:--fail"]) + self.assertEqual(self.seeds_after, "https://example.test/previous\n") + self.assertEqual(self.seed_files_after, ["sitemap.txt"]) + + def test_empty_or_non_sitemap_response_preserves_seeds(self): + for body in ("", "Temporarily unavailable\n", + "https://example.test/page\n"): + with self.subTest(body=body): + result, calls = self.run_crawler(domain="https://example.test", sitemap_body=body) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(calls, ["curl:--fail"]) + self.assertEqual(self.seeds_after, "https://example.test/previous\n") + self.assertEqual(self.seed_files_after, ["sitemap.txt"]) + if __name__ == "__main__": unittest.main()