import pytest from govcrawler.fetcher import chain from govcrawler.fetcher.browser import FetchResult @pytest.fixture(autouse=True) def _env(monkeypatch): monkeypatch.setenv("DB_URL", "postgresql+psycopg://x/x") monkeypatch.setenv("USER_AGENT", "TestBot/1.0") def _ok_result(strategy="httpx", status=200, html=None): html = html if html is not None else "" + "x" * 1000 + "" return FetchResult( url="https://a.com/x", final_url="https://a.com/x", status=status, html=html, fetched_at=0.0, duration_ms=10, is_challenge=False, strategy=strategy, ) def test_tier2_success_no_fallback(monkeypatch): calls = {"http": 0, "browser": 0} def fake_http(url, **kw): calls["http"] += 1 return _ok_result() def fake_browser(url, **kw): calls["browser"] += 1 return _ok_result(strategy="playwright") monkeypatch.setattr(chain, "fetch_html_http", fake_http) monkeypatch.setattr(chain, "fetch_html_browser", fake_browser) fr = chain.fetch_html("https://a.com/x") assert fr.strategy == "httpx" assert calls["browser"] == 0 def test_tier2_412_triggers_fallback(monkeypatch): def fake_http(url, **kw): return FetchResult( url=url, final_url=url, status=412, html="请稍候", fetched_at=0.0, duration_ms=5, is_challenge=True, strategy="httpx", ) def fake_browser(url, **kw): return _ok_result(strategy="playwright", html="" + "y" * 2000 + "") monkeypatch.setattr(chain, "fetch_html_http", fake_http) monkeypatch.setattr(chain, "fetch_html_browser", fake_browser) fr = chain.fetch_html("https://a.com/x") assert fr.strategy == "playwright" assert fr.status == 200 def test_empty_html_triggers_fallback(monkeypatch): def fake_http(url, **kw): return FetchResult( url=url, final_url=url, status=200, html="", fetched_at=0.0, duration_ms=5, is_challenge=False, strategy="httpx", ) monkeypatch.setattr(chain, "fetch_html_http", fake_http) monkeypatch.setattr( chain, "fetch_html_browser", lambda url, **kw: _ok_result(strategy="playwright"), ) fr = chain.fetch_html("https://a.com/x") assert fr.strategy == "playwright" def test_force_browser_skips_tier2(monkeypatch): http_called = {"n": 0} def fake_http(url, **kw): http_called["n"] += 1 return _ok_result() monkeypatch.setattr(chain, "fetch_html_http", fake_http) monkeypatch.setattr( chain, "fetch_html_browser", lambda url, **kw: _ok_result(strategy="playwright"), ) fr = chain.fetch_html("https://a.com/x", force_browser=True) assert fr.strategy == "playwright" assert http_called["n"] == 0 def test_both_tiers_fail_preserves_error(monkeypatch): monkeypatch.setattr( chain, "fetch_html_http", lambda url, **kw: FetchResult( url=url, final_url=url, status=0, html="", fetched_at=0.0, duration_ms=1, is_challenge=False, error="ConnectError: x", strategy="httpx", ), ) monkeypatch.setattr( chain, "fetch_html_browser", lambda url, **kw: FetchResult( url=url, final_url=url, status=0, html="", fetched_at=0.0, duration_ms=1, is_challenge=False, error="TimeoutError: y", strategy="playwright", ), ) fr = chain.fetch_html("https://a.com/x") assert fr.strategy == "playwright" assert "httpx:" in (fr.error or "") and "playwright:" in (fr.error or "")