Use passive website probing in S3 test workflows

Author: trewwwsec

README.md

@@ -105,7 +105,7 @@ uv run subrecon -d example.com -o perimeter_report.json
 ```bash
 S3_CANARY="$(uv run python scripts/select_s3_canary.py || true)"
 S3_TARGET="${S3_CANARY:-subrecon-negative-s3-$(date +%s)}"
-uv run subrecon -d "${S3_TARGET}.test" --keywords "$S3_TARGET" --no-ct --no-search --no-subfinder --no-amass --no-gcp --no-azure -o s3_report.json
+uv run subrecon -d "${S3_TARGET}.test" --keywords "$S3_TARGET" --s3-website-probe --no-ct --no-search --no-subfinder --no-amass --no-gcp --no-azure -o s3_report.json
 ```
 
 ### Takeover-focused check

docs/TESTING.md

@@ -85,7 +85,7 @@ uv run subrecon -d example.com --search-providers commoncrawl --no-ct --no-subfi
 # S3 only (auto-select live canary; fallback to deterministic negative control)
 S3_CANARY="$(uv run python scripts/select_s3_canary.py || true)"
 S3_TARGET="${S3_CANARY:-subrecon-negative-s3-$(date +%s)}"
-uv run subrecon -d "${S3_TARGET}.test" --keywords "$S3_TARGET" --no-ct --no-subfinder --no-amass --no-search --no-gcp --no-azure --no-takeover -o /tmp/subrecon_s3_smoke.json
+uv run subrecon -d "${S3_TARGET}.test" --keywords "$S3_TARGET" --s3-website-probe --no-ct --no-subfinder --no-amass --no-search --no-gcp --no-azure --no-takeover -o /tmp/subrecon_s3_smoke.json
 
 # GCS bucket only (existence/listability signal)
 uv run subrecon -d gcp-public-data.test --keywords gcp-public-data-landsat --no-ct --no-subfinder --no-amass --no-search --no-s3 --no-azure --no-takeover -o /tmp/subrecon_gcp_smoke.json
@@ -134,7 +134,7 @@ uv run subrecon -d iana.org --search-providers commoncrawl --no-ct --no-subfinde
 # S3 only (auto-select live canary; fallback to deterministic negative control)
 S3_CANARY="$(uv run python scripts/select_s3_canary.py || true)"
 S3_TARGET="${S3_CANARY:-subrecon-negative-s3-$(date +%s)}"
-uv run subrecon -d "${S3_TARGET}.test" --keywords "$S3_TARGET" --no-ct --no-subfinder --no-amass --no-search --no-gcp --no-azure --no-takeover -o /tmp/subrecon_live3_s3_only.json
+uv run subrecon -d "${S3_TARGET}.test" --keywords "$S3_TARGET" --s3-website-probe --no-ct --no-subfinder --no-amass --no-search --no-gcp --no-azure --no-takeover -o /tmp/subrecon_live3_s3_only.json
 
 # GCS only
 uv run subrecon -d gcp-public-data.test --keywords gcp-public-data-landsat --no-ct --no-subfinder --no-amass --no-search --no-s3 --no-azure --no-takeover -o /tmp/subrecon_live3_gcp_only.json

scripts/run_pilot_test.sh

@@ -152,13 +152,15 @@ PY
     echo "$S3_CANARY_BUCKET" > "$OUT_DIR/s3_canary_bucket.txt"
     run_case s3_only_canary \
       uv run subrecon -d "${S3_CANARY_BUCKET}.test" --keywords "$S3_CANARY_BUCKET" \
+      --s3-website-probe \
       --no-ct --no-subfinder --no-amass --no-search --no-gcp --no-azure --no-takeover
   else
     echo "[!] selector returned empty S3 canary bucket; using negative-control bucket case"
     S3_NEGATIVE_BUCKET="subrecon-negative-s3-$(date +%s)"
     echo "$S3_NEGATIVE_BUCKET" > "$OUT_DIR/s3_negative_bucket.txt"
     run_case s3_only_negative_control \
       uv run subrecon -d "${S3_NEGATIVE_BUCKET}.test" --keywords "$S3_NEGATIVE_BUCKET" \
+      --s3-website-probe \
       --no-ct --no-subfinder --no-amass --no-search --no-gcp --no-azure --no-takeover
   fi
 else
@@ -167,6 +169,7 @@ else
   echo "$S3_NEGATIVE_BUCKET" > "$OUT_DIR/s3_negative_bucket.txt"
   run_case s3_only_negative_control \
     uv run subrecon -d "${S3_NEGATIVE_BUCKET}.test" --keywords "$S3_NEGATIVE_BUCKET" \
+    --s3-website-probe \
     --no-ct --no-subfinder --no-amass --no-search --no-gcp --no-azure --no-takeover
 fi