From 2523897b86813166621951927fc479d3aa16bf8a Mon Sep 17 00:00:00 2001 From: bits-bytes-nn Date: Sun, 11 Oct 2026 20:41:30 +0900 Subject: [PATCH] fix(crawl): report sources whose article pages cannot be read; drop OpenAI OpenAI's feed fetches, but from AWS every article page returns 403, so each post kept only its teaser and died at the content gate in all 29 dev runs on record while crawl health counted the source as ok. Mark a post whose page was needed but unreadable, and alert on an ok source where that holds for every contributed post. Remove the OpenAI feed. --- README.ko.md | 2 +- README.md | 2 +- app/configs/config-template.yaml | 1 - app/main.py | 26 ++++++++++--- app/src/feed_parser.py | 65 ++++++++++++++++++++++++++------ docs/CHANGELOG.md | 7 ++++ docs/design.md | 22 ++++++++--- tests/test_crawl_health.py | 40 ++++++++++++++++++++ tests/test_feed_parser.py | 44 +++++++++++++++++++++ tests/test_main_notifications.py | 31 +++++++++++++++ 10 files changed, 213 insertions(+), 27 deletions(-) diff --git a/README.ko.md b/README.ko.md index 2d585ac..6866315 100644 --- a/README.ko.md +++ b/README.ko.md @@ -24,7 +24,7 @@ Amazon Bedrock 기반(필터 GPT-6.1 Sol, 요약 Claude) · AWS 위에서 CDK로 - **AI 기반 큐레이션**. Amazon Bedrock에서 GPT-6.1 Sol이 글마다 관련성을 점수로 매기고, Claude가 여러 섹션으로 구조화된 요약을 씁니다. - **다중 소스 집계**. RSS와 견고한 HTML 스크레이핑으로 약 20개 기술 블로그(AWS, - Google, Meta, OpenAI, Microsoft, NVIDIA 등)에서 글을 모읍니다. 모든 요청에는 + Google, Meta, Microsoft, NVIDIA 등)에서 글을 모읍니다. 모든 요청에는 SSRF 가드와 소스별 헬스 추적이 따라붙습니다. - **콘텐츠 품질 게이트**. 본문이 너무 빈약한 글은 LLM에 닿기 *전에* 걸러내, 빈 껍데기 글이 다이제스트에 실리지 않게 합니다. diff --git a/README.md b/README.md index 59163a6..05bf8b8 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Powered by Amazon Bedrock (GPT-6.1 Sol filters, Claude summarizes) · orchestrat - **AI-powered curation**. On Amazon Bedrock, GPT-6.1 Sol scores each post for relevance and Claude writes a structured, multi-section summary. - **Multi-source aggregation**. pulls from \~20 tech blogs via RSS and resilient - HTML scraping (AWS, Google, Meta, OpenAI, Microsoft, NVIDIA, and more), with + HTML scraping (AWS, Google, Meta, Microsoft, NVIDIA, and more), with SSRF-guarded requests and per-source health tracking. - **Content quality gate**. drops posts whose visible text is too thin to summarize *before* they reach the LLM, so the digest never ships empty write-ups. diff --git a/app/configs/config-template.yaml b/app/configs/config-template.yaml index 12b8fb4..84ff34c 100644 --- a/app/configs/config-template.yaml +++ b/app/configs/config-template.yaml @@ -24,7 +24,6 @@ scraping: - "https://www.microsoft.com/en-us/research/blog/feed/" # was research.microsoft.com/rss/news.xml (301 redirect) - "https://medium.com/feed/netflix-techblog" # was techblog.netflix.com (301 redirect to Medium) - "https://developer.nvidia.com/blog/feed" - - "https://openai.com/news/rss.xml" - "https://medium.com/feed/@palantir" - "https://medium.com/feed/@Pinterest_Engineering" - "https://qwenlm.github.io/blog/index.xml" diff --git a/app/main.py b/app/main.py index 9d931f1..34ec06a 100644 --- a/app/main.py +++ b/app/main.py @@ -111,7 +111,11 @@ def handler(event: dict[str, Any], context: Any) -> dict[str, int | str]: if ( is_running_in_aws() and topic_arn - and (crawl_report.failed or crawl_report.likely_broken) + and ( + crawl_report.failed + or crawl_report.likely_broken + or crawl_report.pages_unreadable + ) ): _send_crawl_health_alert( default_boto_session, @@ -378,6 +382,7 @@ def _log_run_summary( # point: the empty count is ~12 every week and says nothing. "sources_likely_broken": len(crawl_report.likely_broken), "sources_failed": len(crawl_report.failed), + "sources_pages_unreadable": len(crawl_report.pages_unreadable), "posts_collected": crawl_report.total_posts, "posts_filtered_out": len(filtered_out_posts), "posts_in_digest": len(posts), @@ -997,7 +1002,8 @@ def _send_crawl_health_alert( crawl_report: CrawlReport, expected_flaky: list[str], ) -> None: - """Notify when a crawl source failed, or produced nothing to filter. + """Notify when a crawl source failed, produced nothing to filter, or left + every post without its article page. Two triggers, and the second one is the point. A FAILED source is loud — the fetch itself errored. A source that fetched fine and offered ZERO items is just @@ -1027,8 +1033,13 @@ def unexpected(sources: list[SourceHealth]) -> list[SourceHealth]: failed = unexpected(crawl_report.failed) broken = unexpected(crawl_report.likely_broken) - if not failed and not broken: - suppressed = crawl_report.failed + crawl_report.likely_broken + unreadable = unexpected(crawl_report.pages_unreadable) + if not failed and not broken and not unreadable: + suppressed = ( + crawl_report.failed + + crawl_report.likely_broken + + crawl_report.pages_unreadable + ) if suppressed: logger.info( "All %d unhealthy source(s) are configured as expected-flaky; " @@ -1042,11 +1053,14 @@ def unexpected(sources: list[SourceHealth]) -> list[SourceHealth]: # Named for what it is rather than "empty": the operator's next action is # to check the feed URL or the scraper's selector, not to wait a week. "Sources offering nothing": str(len(broken)), + # The feed is fine but its posts never reach the summarizer: check + # whether the site now blocks the crawler (a 403 or a bot challenge). + "Unreadable article pages": str(len(unreadable)), "Summary": crawl_report.summary_line(), "Detail": crawl_report.format_alert(), } - if broken: - fields["Check these"] = ", ".join(s.url for s in broken) + if broken or unreadable: + fields["Check these"] = ", ".join(s.url for s in broken + unreadable) _publish_alarm( boto_session, topic_arn, diff --git a/app/src/feed_parser.py b/app/src/feed_parser.py index 109f972..9654ea6 100644 --- a/app/src/feed_parser.py +++ b/app/src/feed_parser.py @@ -461,6 +461,10 @@ class Post(BaseModel): tags: list[str] = Field(default_factory=list) urls: list[str] = Field(default_factory=list) score: float = 0.0 + # The feed carried only a teaser and the article page could not be read + # (blocked, failed or unparseable). Crawl health counts these per source; it + # is run state, not part of the saved post. + page_unreadable: bool = Field(default=False, exclude=True) @field_validator("tags", mode="before") @classmethod @@ -482,7 +486,7 @@ def from_entry(cls, entry: feedparser.FeedParserDict) -> "Post": per-scraper source would be a second copy of it that could drift. """ link_str = getattr(entry, "link", "") - content = cls._extract_content_from_entry(entry, link_str) + content, page_unreadable = cls._extract_content_from_entry(entry, link_str) return cls( # Some feeds escape the title twice (Amazon Science sends # "don&#8217;t"), so feedparser hands back "don’t" and the @@ -501,12 +505,15 @@ def from_entry(cls, entry: feedparser.FeedParserDict) -> "Post": # whose summarization failed shipped its raw feed teaser as the # article body while ALSO being reported as filtered-out. images=cls._extract_images(content, link_str), + page_unreadable=page_unreadable, ) @classmethod def _extract_content_from_entry( cls, entry: feedparser.FeedParserDict, link: str - ) -> str: + ) -> tuple[str, bool]: + """The post's content, and whether the article page was needed but could + not be read.""" content_items = getattr(entry, "content", []) content = ( content_items[0].get("value") @@ -519,17 +526,16 @@ def _extract_content_from_entry( # text keeps the scrape decision consistent with the content gate (\u00a711) # so we don't skip scraping and then drop the thin post. feed_text_len = len(visible_text(content)) + if feed_text_len >= ScraperConfig.FULL_SCRAPE_TEXT_THRESHOLD: + return content, False + scraped_content = cls._scrape_full_content(link) # Only adopt the scrape if it actually has MORE readable prose than the # feed content. A JS-rendered/consent-walled page can yield a thin body # that, if it blindly replaced a decent feed teaser, would push the post # below the downstream content gate and silently drop a healthy post. - if ( - feed_text_len < ScraperConfig.FULL_SCRAPE_TEXT_THRESHOLD - and (scraped_content := cls._scrape_full_content(link)) - and len(visible_text(scraped_content)) > feed_text_len - ): + if scraped_content and len(visible_text(scraped_content)) > feed_text_len: content = scraped_content - return content + return content, not scraped_content @staticmethod def _scrape_full_content(url: str) -> str: @@ -994,6 +1000,9 @@ class SourceHealth: #: #: ``None`` means the fetcher did not report a count — unknown, NOT zero. candidates: int | None = None + #: Contributed posts whose article page was needed but could not be read + #: (``Post.page_unreadable``). + unreadable_pages: int = 0 error: str | None = None @@ -1034,6 +1043,20 @@ def likely_broken(self) -> list[SourceHealth]: """ return [s for s in self.empty if s.candidates == 0] + @property + def pages_unreadable(self) -> list[SourceHealth]: + """OK sources whose every contributed post lacked a readable article page. + + The feed fetched, so the source reads as healthy, yet each post carries + only its teaser and dies at the content gate. OpenAI's feed was in that + state for every run on record: the article pages answer with a + Cloudflare challenge. Some unreadable pages among readable ones are + transient errors and are not reported. + """ + return [ + s for s in self.ok if s.post_count and s.unreadable_pages == s.post_count + ] + @property def total_posts(self) -> int: return sum(s.post_count for s in self.sources) @@ -1043,10 +1066,10 @@ def summary_line(self) -> str: empty = f"{len(self.empty)} empty" if broken: empty += f" ({broken} with nothing to filter)" - return ( - f"{len(self.ok)} ok, {empty}, " - f"{len(self.failed)} failed ({self.total_posts} posts)" - ) + ok = f"{len(self.ok)} ok" + if unreadable := len(self.pages_unreadable): + ok += f" ({unreadable} with unreadable article pages)" + return f"{ok}, {empty}, {len(self.failed)} failed ({self.total_posts} posts)" @staticmethod def _empty_reason(source: SourceHealth) -> str: @@ -1067,6 +1090,14 @@ def format_alert(self) -> str: for s in self.failed ) lines.append("") + if self.pages_unreadable: + lines.append("Sources whose article pages could not be read:") + lines.extend( + f" - {s.url} [{s.fetcher}]: {s.unreadable_pages} of " + f"{s.post_count} post(s) left with only the feed teaser" + for s in self.pages_unreadable + ) + lines.append("") if self.empty: lines.append("Empty sources (no posts in window):") lines.extend( @@ -1122,6 +1153,7 @@ def collect_posts(self, start_date: datetime, end_date: datetime) -> list[Post]: try: fetched = fetcher.fetch(start_date, end_date) contributed = 0 + unreadable = 0 for post in fetched: if post.link in seen_links: continue @@ -1139,6 +1171,7 @@ def collect_posts(self, start_date: datetime, end_date: datetime) -> list[Post]: continue all_posts.append(post) contributed += 1 + unreadable += post.page_unreadable seen_links.add(post.link) if title_key: seen_titles.add(title_key) @@ -1161,6 +1194,7 @@ def collect_posts(self, start_date: datetime, end_date: datetime) -> list[Post]: # visible instead of hiding it. post_count=contributed, duplicates=len(fetched) - contributed, + unreadable_pages=unreadable, ) ) if not fetched: @@ -1194,4 +1228,11 @@ def collect_posts(self, start_date: datetime, end_date: datetime) -> list[Post]: len(self.report.likely_broken), [s.url for s in self.report.likely_broken], ) + if self.report.pages_unreadable: + logger.warning( + "%d source(s) left every post with only its feed teaser — the " + "article pages could not be read: %s", + len(self.report.pages_unreadable), + [s.url for s in self.report.pages_unreadable], + ) return sorted(all_posts, key=lambda p: p.published_date, reverse=True) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ad52f4a..0e9d31d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,13 @@ ### 소스 구성 +- **2026-10-11** OpenAI 뉴스 피드를 소스에서 뺐습니다. 피드는 받아지지만 글 페이지가 + AWS Batch에서 매번 403을 돌려줍니다. 같은 브라우저 헤더로 AWS 밖에서 받으면 본문이 오므로 + 송신 IP를 막는 것입니다. + 로그가 남은 2026-08-19 이후 dev 실행 29회 모두 글이 소개 문구만 남아 콘텐츠 게이트에서 + 탈락했고 호에 실린 적이 없습니다. 크롤 헬스는 피드만 보고 이 소스를 `ok`로 셌으므로, 글 + 페이지를 하나도 읽지 못한 소스를 `pages_unreadable`로 따로 알리게 했습니다. 공식 + OpenAI Developers 피드는 블로그가 아니라 문서 목록이라 대체 소스로 쓰지 않았습니다. - **2026-10-11** LinkedIn 엔지니어링 블로그를 dev 소스에 더했습니다. AWS Batch에서 인덱스 6건과 글 본문을 모두 받는 것을 확인했고, 인덱스가 잘라 보여주는 제목은 글 페이지의 `og:title`로 채웁니다. diff --git a/docs/design.md b/docs/design.md index 684929f..cff594c 100644 --- a/docs/design.md +++ b/docs/design.md @@ -529,9 +529,9 @@ HTML 엔티티를 한 번 더 풉니다. 제목을 두 번 이스케이프해 - `SourceStatus`. `OK`(가져와서 글까지 생성), `EMPTY`(가져왔지만 기간 안에 글이 없음), `FAILED`(가져오기 자체가 실패). - `SourceHealth`. 소스별 레코드: `url`, `fetcher`, `status`, `post_count`, - `duplicates`, `candidates`, `error`. + `duplicates`, `candidates`, `unreadable_pages`, `error`. - `CrawlReport`. `SourceHealth`들을 한데 모읍니다. `failed` / `empty` / `ok` / - `likely_broken`으로 나눠 보여 주고, `total_posts`, 한 줄 요약 `summary_line()`, + `likely_broken` / `pages_unreadable`로 나눠 보여 주고, `total_posts`, 한 줄 요약 `summary_line()`, 그리고 사람이 읽기 좋게 정리한 `format_alert()`를 제공합니다. ### `EMPTY`를 가르는 후보 수 @@ -574,16 +574,25 @@ Empty sources (no posts in window): `PostCollector.collect_posts`가 실행하면서 이 리포트를 채우고, `likely_broken` 소스의 URL은 알림과 별개로 매 실행 WARNING 로그에 남깁니다. +### 글 페이지를 읽지 못하는 `OK` 소스 + +피드는 받아지는데 피드에 소개 문구만 있고 글 페이지가 막힌 소스는 `OK`로 집계됩니다. 그러나 +글마다 소개 문구만 남아 콘텐츠 게이트에서 전부 탈락합니다. `Post.page_unreadable`은 피드 +본문이 짧아 글 페이지를 받아야 했는데 받지 못했거나 파싱하지 못한 글에 붙습니다. 저장하는 +글 JSON에는 들어가지 않는 실행 상태입니다. 수집기가 소스별로 이 글 수를 +`SourceHealth.unreadable_pages`에 세고, `CrawlReport.pages_unreadable`은 **기여한 글 전부가** +그런 `OK` 소스만 골라냅니다. 일부만 실패한 소스는 일시적 오류일 수 있어 고르지 않습니다. + ### 크롤 헬스 알림의 트리거 -AWS에서 실행하다 소스가 `FAILED`가 되거나 `likely_broken`이 되면 `main.py`가 크롤 헬스 +AWS에서 실행하다 소스가 `FAILED`, `likely_broken`, `pages_unreadable` 중 하나가 되면 `main.py`가 크롤 헬스 알림을 발행합니다(`_send_crawl_health_alert`, 알림 전체 목록은 [SNS 알림](#sns-알림)). 깨진 소스가 다이제스트에서 조용히 사라지는 대신 바로 손쓸 수 있는 이메일이 됩니다. - **조용한 블로그는 이 알림을 띄우지 않습니다.** 후보가 0이 아니기 때문입니다. 평소에도 비어 있는 소스마다 울리는 알람은 아무도 읽지 않는 알람이고, 그러면 새로 깨진 소스가 그 뒤에 숨습니다. -- **`scraping.expected_flaky_urls`에 적힌 소스는 두 트리거 모두에서 제외합니다.** 매 실행 +- **`scraping.expected_flaky_urls`에 적힌 소스는 모든 트리거에서 제외합니다.** 매 실행 AWS 송신 IP를 거부하는 소스를 위한 장치로, 차단 결과가 `FAILED`로 나오든 후보 0개로 나오든 울리지 않아야 하기 때문입니다. 리포트 본문과 로그에는 그대로 남고, 조용해지는 것은 트리거뿐입니다. 실패가 전부 예상된 것이면 알림 대신 그 사실을 INFO 로그로 남기고, @@ -1754,8 +1763,9 @@ fixture(`no_network`)가 실제 연결과 호스트 이름 조회를 막아, 테 `Retry-After` 준수, `403` 비재시도, 선언된 헤더가 그대로 나가는지를 확인하고, SSRF 가드는 내부 IP와 내부로 해석되는 호스트명, 내부로 향하는 리다이렉트가 모두 막히는지 봅니다. -**크롤 헬스.** 후보 0개만 `likely_broken`이 되는지, `None`을 0으로 취급하지 않는지, -`expected_flaky_urls`가 두 트리거를 모두 억제하면서 리포트에는 남기는지를 확인합니다. +**크롤 헬스.** 후보 0개만 `likely_broken`이 되는지, `None`을 0으로 취급하지 않는지, 글 페이지를 +전부 못 읽은 소스만 `pages_unreadable`이 되는지, `expected_flaky_urls`가 모든 트리거를 +억제하면서 리포트에는 남기는지를 확인합니다. **필터·요약 단계.** 점수 경계값, 범위 밖 점수의 실패 처리, 피드 제목 유지, 응답이 `None`으로 돌아왔을 때의 정렬 유지, 잘못된 출력 처리, 요약 전에 적용되는 `max_posts` 캡을 확인합니다. diff --git a/tests/test_crawl_health.py b/tests/test_crawl_health.py index 513c767..a1ddf7e 100644 --- a/tests/test_crawl_health.py +++ b/tests/test_crawl_health.py @@ -311,3 +311,43 @@ def fetch(self, start, end): collector.collect_posts(*self.WINDOW) assert collector.report.sources[0].candidates == 0 assert len(collector.report.likely_broken) == 1 + + +class TestPagesUnreadable: + """A feed that fetches while every article page is blocked (OpenAI behind a + Cloudflare challenge) used to count as a healthy source.""" + + def _ok(self, posts: int, unreadable: int) -> SourceHealth: + return SourceHealth( + url="https://openai.com/news/rss.xml", + fetcher="RssFetcher", + status=SourceStatus.OK, + post_count=posts, + unreadable_pages=unreadable, + ) + + def test_every_page_unreadable_is_flagged(self): + report = CrawlReport(sources=[self._ok(15, 15)]) + assert report.pages_unreadable == report.sources + assert "1 with unreadable article pages" in report.summary_line() + assert "15 of 15 post(s)" in report.format_alert() + + @pytest.mark.parametrize(("posts", "unreadable"), [(15, 14), (3, 0), (0, 0)]) + def test_partial_or_no_failures_are_not(self, posts, unreadable): + report = CrawlReport(sources=[self._ok(posts, unreadable)]) + assert report.pages_unreadable == [] + assert "unreadable" not in report.summary_line() + + def test_the_collector_counts_contributed_posts_only(self): + blocked = _post("https://openai.com/index/a") + blocked.page_unreadable = True + duplicate = _post("https://openai.com/index/a") + duplicate.page_unreadable = True + collector = PostCollector( + [_OkFetcher("https://openai.com/news/rss.xml", [blocked, duplicate])] + ) + now = datetime.now(UTC) + collector.collect_posts(now, now) + source = collector.report.sources[0] + assert (source.post_count, source.unreadable_pages) == (1, 1) + assert collector.report.pages_unreadable == [source] diff --git a/tests/test_feed_parser.py b/tests/test_feed_parser.py index 82abd1a..772de9f 100644 --- a/tests/test_feed_parser.py +++ b/tests/test_feed_parser.py @@ -328,6 +328,50 @@ def test_richer_scrape_replaces_thin_feed_content(self, monkeypatch): assert post.content == rich +class TestPageUnreadable: + """A teaser-only entry whose article page cannot be read is marked, so crawl + health can name the source instead of reporting it as healthy.""" + + def _entry(self, body: str) -> feedparser.FeedParserDict: + return feedparser.FeedParserDict( + title="T", + link="https://openai.com/index/x", + published="2026-05-30T10:00:00Z", + content=[{"value": body}], + ) + + def test_a_blocked_page_marks_the_post(self, monkeypatch): + from app.src import feed_parser + + monkeypatch.setattr( + feed_parser.Post, "_scrape_full_content", staticmethod(lambda url: "") + ) + post = Post.from_entry(self._entry("

short teaser

")) + assert post.page_unreadable + assert "page_unreadable" not in post.model_dump() + + def test_a_page_that_was_read_does_not(self, monkeypatch): + from app.src import feed_parser + + monkeypatch.setattr( + feed_parser.Post, + "_scrape_full_content", + staticmethod(lambda url: "

tiny

"), + ) + assert not Post.from_entry(self._entry("

short teaser

")).page_unreadable + + def test_a_full_feed_body_needs_no_page(self, monkeypatch): + from app.src import feed_parser + + monkeypatch.setattr( + feed_parser.Post, + "_scrape_full_content", + staticmethod(lambda url: pytest.fail("fetched")), + ) + body = "

" + ("word " * 800) + "

" + assert not Post.from_entry(self._entry(body)).page_unreadable + + class TestRegisteredScrapersResolveToAMappedSource: """Each index-page scraper's own URL must resolve to a real source. diff --git a/tests/test_main_notifications.py b/tests/test_main_notifications.py index 12649a8..37ea498 100644 --- a/tests/test_main_notifications.py +++ b/tests/test_main_notifications.py @@ -3,6 +3,7 @@ from __future__ import annotations +import re import sys from copy import deepcopy from pathlib import Path @@ -161,6 +162,36 @@ def test_an_unknown_count_does_not_alert(self): ) assert session.sns.published == [] + def _unreadable(self) -> CrawlReport: + return CrawlReport( + sources=[ + SourceHealth( + url="https://openai.com/news/rss.xml", + fetcher="RssFetcher", + status=SourceStatus.OK, + post_count=15, + unreadable_pages=15, + ) + ] + ) + + def test_a_source_with_unreadable_pages_alerts(self): + session = _FakeSession() + main._send_crawl_health_alert( + session, "arn:topic", PROJECT, self._unreadable(), [] + ) + assert len(session.sns.published) == 1 + msg = session.sns.published[0]["Message"] + assert re.search(r"Unreadable article pages:\s+1", msg) + assert re.search(r"Check these:\s+https://openai.com/news/rss.xml", msg) + + def test_expected_flaky_suppresses_the_unreadable_pages_trigger(self): + session = _FakeSession() + main._send_crawl_health_alert( + session, "arn:topic", PROJECT, self._unreadable(), ["openai.com"] + ) + assert session.sns.published == [] + def test_expected_flaky_suppresses_the_offering_nothing_trigger_too(self): """x.ai is blocked from AWS egress IPs every week; whether that shows up as FAILED or as zero candidates, it must not page."""