diff --git a/README.ko.md b/README.ko.md index 2d585ac..6866315 100644 --- a/README.ko.md +++ b/README.ko.md @@ -24,7 +24,7 @@ Amazon Bedrock 기반(필터 GPT-6.1 Sol, 요약 Claude) · AWS 위에서 CDK로 - **AI 기반 큐레이션**. Amazon Bedrock에서 GPT-6.1 Sol이 글마다 관련성을 점수로 매기고, Claude가 여러 섹션으로 구조화된 요약을 씁니다. - **다중 소스 집계**. RSS와 견고한 HTML 스크레이핑으로 약 20개 기술 블로그(AWS, - Google, Meta, OpenAI, Microsoft, NVIDIA 등)에서 글을 모읍니다. 모든 요청에는 + Google, Meta, Microsoft, NVIDIA 등)에서 글을 모읍니다. 모든 요청에는 SSRF 가드와 소스별 헬스 추적이 따라붙습니다. - **콘텐츠 품질 게이트**. 본문이 너무 빈약한 글은 LLM에 닿기 *전에* 걸러내, 빈 껍데기 글이 다이제스트에 실리지 않게 합니다. diff --git a/README.md b/README.md index 59163a6..05bf8b8 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Powered by Amazon Bedrock (GPT-6.1 Sol filters, Claude summarizes) · orchestrat - **AI-powered curation**. On Amazon Bedrock, GPT-6.1 Sol scores each post for relevance and Claude writes a structured, multi-section summary. - **Multi-source aggregation**. pulls from \~20 tech blogs via RSS and resilient - HTML scraping (AWS, Google, Meta, OpenAI, Microsoft, NVIDIA, and more), with + HTML scraping (AWS, Google, Meta, Microsoft, NVIDIA, and more), with SSRF-guarded requests and per-source health tracking. - **Content quality gate**. drops posts whose visible text is too thin to summarize *before* they reach the LLM, so the digest never ships empty write-ups. diff --git a/app/configs/config-template.yaml b/app/configs/config-template.yaml index 12b8fb4..84ff34c 100644 --- a/app/configs/config-template.yaml +++ b/app/configs/config-template.yaml @@ -24,7 +24,6 @@ scraping: - "https://www.microsoft.com/en-us/research/blog/feed/" # was research.microsoft.com/rss/news.xml (301 redirect) - "https://medium.com/feed/netflix-techblog" # was techblog.netflix.com (301 redirect to Medium) - "https://developer.nvidia.com/blog/feed" - - "https://openai.com/news/rss.xml" - "https://medium.com/feed/@palantir" - "https://medium.com/feed/@Pinterest_Engineering" - "https://qwenlm.github.io/blog/index.xml" diff --git a/app/main.py b/app/main.py index 9d931f1..34ec06a 100644 --- a/app/main.py +++ b/app/main.py @@ -111,7 +111,11 @@ def handler(event: dict[str, Any], context: Any) -> dict[str, int | str]: if ( is_running_in_aws() and topic_arn - and (crawl_report.failed or crawl_report.likely_broken) + and ( + crawl_report.failed + or crawl_report.likely_broken + or crawl_report.pages_unreadable + ) ): _send_crawl_health_alert( default_boto_session, @@ -378,6 +382,7 @@ def _log_run_summary( # point: the empty count is ~12 every week and says nothing. "sources_likely_broken": len(crawl_report.likely_broken), "sources_failed": len(crawl_report.failed), + "sources_pages_unreadable": len(crawl_report.pages_unreadable), "posts_collected": crawl_report.total_posts, "posts_filtered_out": len(filtered_out_posts), "posts_in_digest": len(posts), @@ -997,7 +1002,8 @@ def _send_crawl_health_alert( crawl_report: CrawlReport, expected_flaky: list[str], ) -> None: - """Notify when a crawl source failed, or produced nothing to filter. + """Notify when a crawl source failed, produced nothing to filter, or left + every post without its article page. Two triggers, and the second one is the point. A FAILED source is loud — the fetch itself errored. A source that fetched fine and offered ZERO items is just @@ -1027,8 +1033,13 @@ def unexpected(sources: list[SourceHealth]) -> list[SourceHealth]: failed = unexpected(crawl_report.failed) broken = unexpected(crawl_report.likely_broken) - if not failed and not broken: - suppressed = crawl_report.failed + crawl_report.likely_broken + unreadable = unexpected(crawl_report.pages_unreadable) + if not failed and not broken and not unreadable: + suppressed = ( + crawl_report.failed + + crawl_report.likely_broken + + crawl_report.pages_unreadable + ) if suppressed: logger.info( "All %d unhealthy source(s) are configured as expected-flaky; " @@ -1042,11 +1053,14 @@ def unexpected(sources: list[SourceHealth]) -> list[SourceHealth]: # Named for what it is rather than "empty": the operator's next action is # to check the feed URL or the scraper's selector, not to wait a week. "Sources offering nothing": str(len(broken)), + # The feed is fine but its posts never reach the summarizer: check + # whether the site now blocks the crawler (a 403 or a bot challenge). + "Unreadable article pages": str(len(unreadable)), "Summary": crawl_report.summary_line(), "Detail": crawl_report.format_alert(), } - if broken: - fields["Check these"] = ", ".join(s.url for s in broken) + if broken or unreadable: + fields["Check these"] = ", ".join(s.url for s in broken + unreadable) _publish_alarm( boto_session, topic_arn, diff --git a/app/src/feed_parser.py b/app/src/feed_parser.py index 109f972..9654ea6 100644 --- a/app/src/feed_parser.py +++ b/app/src/feed_parser.py @@ -461,6 +461,10 @@ class Post(BaseModel): tags: list[str] = Field(default_factory=list) urls: list[str] = Field(default_factory=list) score: float = 0.0 + # The feed carried only a teaser and the article page could not be read + # (blocked, failed or unparseable). Crawl health counts these per source; it + # is run state, not part of the saved post. + page_unreadable: bool = Field(default=False, exclude=True) @field_validator("tags", mode="before") @classmethod @@ -482,7 +486,7 @@ def from_entry(cls, entry: feedparser.FeedParserDict) -> "Post": per-scraper source would be a second copy of it that could drift. """ link_str = getattr(entry, "link", "") - content = cls._extract_content_from_entry(entry, link_str) + content, page_unreadable = cls._extract_content_from_entry(entry, link_str) return cls( # Some feeds escape the title twice (Amazon Science sends # "don&#8217;t"), so feedparser hands back "don’t" and the @@ -501,12 +505,15 @@ def from_entry(cls, entry: feedparser.FeedParserDict) -> "Post": # whose summarization failed shipped its raw feed teaser as the # article body while ALSO being reported as filtered-out. images=cls._extract_images(content, link_str), + page_unreadable=page_unreadable, ) @classmethod def _extract_content_from_entry( cls, entry: feedparser.FeedParserDict, link: str - ) -> str: + ) -> tuple[str, bool]: + """The post's content, and whether the article page was needed but could + not be read.""" content_items = getattr(entry, "content", []) content = ( content_items[0].get("value") @@ -519,17 +526,16 @@ def _extract_content_from_entry( # text keeps the scrape decision consistent with the content gate (\u00a711) # so we don't skip scraping and then drop the thin post. feed_text_len = len(visible_text(content)) + if feed_text_len >= ScraperConfig.FULL_SCRAPE_TEXT_THRESHOLD: + return content, False + scraped_content = cls._scrape_full_content(link) # Only adopt the scrape if it actually has MORE readable prose than the # feed content. A JS-rendered/consent-walled page can yield a thin body # that, if it blindly replaced a decent feed teaser, would push the post # below the downstream content gate and silently drop a healthy post. - if ( - feed_text_len < ScraperConfig.FULL_SCRAPE_TEXT_THRESHOLD - and (scraped_content := cls._scrape_full_content(link)) - and len(visible_text(scraped_content)) > feed_text_len - ): + if scraped_content and len(visible_text(scraped_content)) > feed_text_len: content = scraped_content - return content + return content, not scraped_content @staticmethod def _scrape_full_content(url: str) -> str: @@ -994,6 +1000,9 @@ class SourceHealth: #: #: ``None`` means the fetcher did not report a count — unknown, NOT zero. candidates: int | None = None + #: Contributed posts whose article page was needed but could not be read + #: (``Post.page_unreadable``). + unreadable_pages: int = 0 error: str | None = None @@ -1034,6 +1043,20 @@ def likely_broken(self) -> list[SourceHealth]: """ return [s for s in self.empty if s.candidates == 0] + @property + def pages_unreadable(self) -> list[SourceHealth]: + """OK sources whose every contributed post lacked a readable article page. + + The feed fetched, so the source reads as healthy, yet each post carries + only its teaser and dies at the content gate. OpenAI's feed was in that + state for every run on record: the article pages answer with a + Cloudflare challenge. Some unreadable pages among readable ones are + transient errors and are not reported. + """ + return [ + s for s in self.ok if s.post_count and s.unreadable_pages == s.post_count + ] + @property def total_posts(self) -> int: return sum(s.post_count for s in self.sources) @@ -1043,10 +1066,10 @@ def summary_line(self) -> str: empty = f"{len(self.empty)} empty" if broken: empty += f" ({broken} with nothing to filter)" - return ( - f"{len(self.ok)} ok, {empty}, " - f"{len(self.failed)} failed ({self.total_posts} posts)" - ) + ok = f"{len(self.ok)} ok" + if unreadable := len(self.pages_unreadable): + ok += f" ({unreadable} with unreadable article pages)" + return f"{ok}, {empty}, {len(self.failed)} failed ({self.total_posts} posts)" @staticmethod def _empty_reason(source: SourceHealth) -> str: @@ -1067,6 +1090,14 @@ def format_alert(self) -> str: for s in self.failed ) lines.append("") + if self.pages_unreadable: + lines.append("Sources whose article pages could not be read:") + lines.extend( + f" - {s.url} [{s.fetcher}]: {s.unreadable_pages} of " + f"{s.post_count} post(s) left with only the feed teaser" + for s in self.pages_unreadable + ) + lines.append("") if self.empty: lines.append("Empty sources (no posts in window):") lines.extend( @@ -1122,6 +1153,7 @@ def collect_posts(self, start_date: datetime, end_date: datetime) -> list[Post]: try: fetched = fetcher.fetch(start_date, end_date) contributed = 0 + unreadable = 0 for post in fetched: if post.link in seen_links: continue @@ -1139,6 +1171,7 @@ def collect_posts(self, start_date: datetime, end_date: datetime) -> list[Post]: continue all_posts.append(post) contributed += 1 + unreadable += post.page_unreadable seen_links.add(post.link) if title_key: seen_titles.add(title_key) @@ -1161,6 +1194,7 @@ def collect_posts(self, start_date: datetime, end_date: datetime) -> list[Post]: # visible instead of hiding it. post_count=contributed, duplicates=len(fetched) - contributed, + unreadable_pages=unreadable, ) ) if not fetched: @@ -1194,4 +1228,11 @@ def collect_posts(self, start_date: datetime, end_date: datetime) -> list[Post]: len(self.report.likely_broken), [s.url for s in self.report.likely_broken], ) + if self.report.pages_unreadable: + logger.warning( + "%d source(s) left every post with only its feed teaser — the " + "article pages could not be read: %s", + len(self.report.pages_unreadable), + [s.url for s in self.report.pages_unreadable], + ) return sorted(all_posts, key=lambda p: p.published_date, reverse=True) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ad52f4a..0e9d31d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,13 @@ ### 소스 구성 +- **2026-10-11** OpenAI 뉴스 피드를 소스에서 뺐습니다. 피드는 받아지지만 글 페이지가 + AWS Batch에서 매번 403을 돌려줍니다. 같은 브라우저 헤더로 AWS 밖에서 받으면 본문이 오므로 + 송신 IP를 막는 것입니다. + 로그가 남은 2026-08-19 이후 dev 실행 29회 모두 글이 소개 문구만 남아 콘텐츠 게이트에서 + 탈락했고 호에 실린 적이 없습니다. 크롤 헬스는 피드만 보고 이 소스를 `ok`로 셌으므로, 글 + 페이지를 하나도 읽지 못한 소스를 `pages_unreadable`로 따로 알리게 했습니다. 공식 + OpenAI Developers 피드는 블로그가 아니라 문서 목록이라 대체 소스로 쓰지 않았습니다. - **2026-10-11** LinkedIn 엔지니어링 블로그를 dev 소스에 더했습니다. AWS Batch에서 인덱스 6건과 글 본문을 모두 받는 것을 확인했고, 인덱스가 잘라 보여주는 제목은 글 페이지의 `og:title`로 채웁니다. diff --git a/docs/design.md b/docs/design.md index 684929f..cff594c 100644 --- a/docs/design.md +++ b/docs/design.md @@ -529,9 +529,9 @@ HTML 엔티티를 한 번 더 풉니다. 제목을 두 번 이스케이프해 - `SourceStatus`. `OK`(가져와서 글까지 생성), `EMPTY`(가져왔지만 기간 안에 글이 없음), `FAILED`(가져오기 자체가 실패). - `SourceHealth`. 소스별 레코드: `url`, `fetcher`, `status`, `post_count`, - `duplicates`, `candidates`, `error`. + `duplicates`, `candidates`, `unreadable_pages`, `error`. - `CrawlReport`. `SourceHealth`들을 한데 모읍니다. `failed` / `empty` / `ok` / - `likely_broken`으로 나눠 보여 주고, `total_posts`, 한 줄 요약 `summary_line()`, + `likely_broken` / `pages_unreadable`로 나눠 보여 주고, `total_posts`, 한 줄 요약 `summary_line()`, 그리고 사람이 읽기 좋게 정리한 `format_alert()`를 제공합니다. ### `EMPTY`를 가르는 후보 수 @@ -574,16 +574,25 @@ Empty sources (no posts in window): `PostCollector.collect_posts`가 실행하면서 이 리포트를 채우고, `likely_broken` 소스의 URL은 알림과 별개로 매 실행 WARNING 로그에 남깁니다. +### 글 페이지를 읽지 못하는 `OK` 소스 + +피드는 받아지는데 피드에 소개 문구만 있고 글 페이지가 막힌 소스는 `OK`로 집계됩니다. 그러나 +글마다 소개 문구만 남아 콘텐츠 게이트에서 전부 탈락합니다. `Post.page_unreadable`은 피드 +본문이 짧아 글 페이지를 받아야 했는데 받지 못했거나 파싱하지 못한 글에 붙습니다. 저장하는 +글 JSON에는 들어가지 않는 실행 상태입니다. 수집기가 소스별로 이 글 수를 +`SourceHealth.unreadable_pages`에 세고, `CrawlReport.pages_unreadable`은 **기여한 글 전부가** +그런 `OK` 소스만 골라냅니다. 일부만 실패한 소스는 일시적 오류일 수 있어 고르지 않습니다. + ### 크롤 헬스 알림의 트리거 -AWS에서 실행하다 소스가 `FAILED`가 되거나 `likely_broken`이 되면 `main.py`가 크롤 헬스 +AWS에서 실행하다 소스가 `FAILED`, `likely_broken`, `pages_unreadable` 중 하나가 되면 `main.py`가 크롤 헬스 알림을 발행합니다(`_send_crawl_health_alert`, 알림 전체 목록은 [SNS 알림](#sns-알림)). 깨진 소스가 다이제스트에서 조용히 사라지는 대신 바로 손쓸 수 있는 이메일이 됩니다. - **조용한 블로그는 이 알림을 띄우지 않습니다.** 후보가 0이 아니기 때문입니다. 평소에도 비어 있는 소스마다 울리는 알람은 아무도 읽지 않는 알람이고, 그러면 새로 깨진 소스가 그 뒤에 숨습니다. -- **`scraping.expected_flaky_urls`에 적힌 소스는 두 트리거 모두에서 제외합니다.** 매 실행 +- **`scraping.expected_flaky_urls`에 적힌 소스는 모든 트리거에서 제외합니다.** 매 실행 AWS 송신 IP를 거부하는 소스를 위한 장치로, 차단 결과가 `FAILED`로 나오든 후보 0개로 나오든 울리지 않아야 하기 때문입니다. 리포트 본문과 로그에는 그대로 남고, 조용해지는 것은 트리거뿐입니다. 실패가 전부 예상된 것이면 알림 대신 그 사실을 INFO 로그로 남기고, @@ -1754,8 +1763,9 @@ fixture(`no_network`)가 실제 연결과 호스트 이름 조회를 막아, 테 `Retry-After` 준수, `403` 비재시도, 선언된 헤더가 그대로 나가는지를 확인하고, SSRF 가드는 내부 IP와 내부로 해석되는 호스트명, 내부로 향하는 리다이렉트가 모두 막히는지 봅니다. -**크롤 헬스.** 후보 0개만 `likely_broken`이 되는지, `None`을 0으로 취급하지 않는지, -`expected_flaky_urls`가 두 트리거를 모두 억제하면서 리포트에는 남기는지를 확인합니다. +**크롤 헬스.** 후보 0개만 `likely_broken`이 되는지, `None`을 0으로 취급하지 않는지, 글 페이지를 +전부 못 읽은 소스만 `pages_unreadable`이 되는지, `expected_flaky_urls`가 모든 트리거를 +억제하면서 리포트에는 남기는지를 확인합니다. **필터·요약 단계.** 점수 경계값, 범위 밖 점수의 실패 처리, 피드 제목 유지, 응답이 `None`으로 돌아왔을 때의 정렬 유지, 잘못된 출력 처리, 요약 전에 적용되는 `max_posts` 캡을 확인합니다. diff --git a/tests/test_crawl_health.py b/tests/test_crawl_health.py index 513c767..a1ddf7e 100644 --- a/tests/test_crawl_health.py +++ b/tests/test_crawl_health.py @@ -311,3 +311,43 @@ def fetch(self, start, end): collector.collect_posts(*self.WINDOW) assert collector.report.sources[0].candidates == 0 assert len(collector.report.likely_broken) == 1 + + +class TestPagesUnreadable: + """A feed that fetches while every article page is blocked (OpenAI behind a + Cloudflare challenge) used to count as a healthy source.""" + + def _ok(self, posts: int, unreadable: int) -> SourceHealth: + return SourceHealth( + url="https://openai.com/news/rss.xml", + fetcher="RssFetcher", + status=SourceStatus.OK, + post_count=posts, + unreadable_pages=unreadable, + ) + + def test_every_page_unreadable_is_flagged(self): + report = CrawlReport(sources=[self._ok(15, 15)]) + assert report.pages_unreadable == report.sources + assert "1 with unreadable article pages" in report.summary_line() + assert "15 of 15 post(s)" in report.format_alert() + + @pytest.mark.parametrize(("posts", "unreadable"), [(15, 14), (3, 0), (0, 0)]) + def test_partial_or_no_failures_are_not(self, posts, unreadable): + report = CrawlReport(sources=[self._ok(posts, unreadable)]) + assert report.pages_unreadable == [] + assert "unreadable" not in report.summary_line() + + def test_the_collector_counts_contributed_posts_only(self): + blocked = _post("https://openai.com/index/a") + blocked.page_unreadable = True + duplicate = _post("https://openai.com/index/a") + duplicate.page_unreadable = True + collector = PostCollector( + [_OkFetcher("https://openai.com/news/rss.xml", [blocked, duplicate])] + ) + now = datetime.now(UTC) + collector.collect_posts(now, now) + source = collector.report.sources[0] + assert (source.post_count, source.unreadable_pages) == (1, 1) + assert collector.report.pages_unreadable == [source] diff --git a/tests/test_feed_parser.py b/tests/test_feed_parser.py index 82abd1a..772de9f 100644 --- a/tests/test_feed_parser.py +++ b/tests/test_feed_parser.py @@ -328,6 +328,50 @@ def test_richer_scrape_replaces_thin_feed_content(self, monkeypatch): assert post.content == rich +class TestPageUnreadable: + """A teaser-only entry whose article page cannot be read is marked, so crawl + health can name the source instead of reporting it as healthy.""" + + def _entry(self, body: str) -> feedparser.FeedParserDict: + return feedparser.FeedParserDict( + title="T", + link="https://openai.com/index/x", + published="2026-05-30T10:00:00Z", + content=[{"value": body}], + ) + + def test_a_blocked_page_marks_the_post(self, monkeypatch): + from app.src import feed_parser + + monkeypatch.setattr( + feed_parser.Post, "_scrape_full_content", staticmethod(lambda url: "") + ) + post = Post.from_entry(self._entry("

short teaser

")) + assert post.page_unreadable + assert "page_unreadable" not in post.model_dump() + + def test_a_page_that_was_read_does_not(self, monkeypatch): + from app.src import feed_parser + + monkeypatch.setattr( + feed_parser.Post, + "_scrape_full_content", + staticmethod(lambda url: "

tiny

"), + ) + assert not Post.from_entry(self._entry("

short teaser

")).page_unreadable + + def test_a_full_feed_body_needs_no_page(self, monkeypatch): + from app.src import feed_parser + + monkeypatch.setattr( + feed_parser.Post, + "_scrape_full_content", + staticmethod(lambda url: pytest.fail("fetched")), + ) + body = "

" + ("word " * 800) + "

" + assert not Post.from_entry(self._entry(body)).page_unreadable + + class TestRegisteredScrapersResolveToAMappedSource: """Each index-page scraper's own URL must resolve to a real source. diff --git a/tests/test_main_notifications.py b/tests/test_main_notifications.py index 12649a8..37ea498 100644 --- a/tests/test_main_notifications.py +++ b/tests/test_main_notifications.py @@ -3,6 +3,7 @@ from __future__ import annotations +import re import sys from copy import deepcopy from pathlib import Path @@ -161,6 +162,36 @@ def test_an_unknown_count_does_not_alert(self): ) assert session.sns.published == [] + def _unreadable(self) -> CrawlReport: + return CrawlReport( + sources=[ + SourceHealth( + url="https://openai.com/news/rss.xml", + fetcher="RssFetcher", + status=SourceStatus.OK, + post_count=15, + unreadable_pages=15, + ) + ] + ) + + def test_a_source_with_unreadable_pages_alerts(self): + session = _FakeSession() + main._send_crawl_health_alert( + session, "arn:topic", PROJECT, self._unreadable(), [] + ) + assert len(session.sns.published) == 1 + msg = session.sns.published[0]["Message"] + assert re.search(r"Unreadable article pages:\s+1", msg) + assert re.search(r"Check these:\s+https://openai.com/news/rss.xml", msg) + + def test_expected_flaky_suppresses_the_unreadable_pages_trigger(self): + session = _FakeSession() + main._send_crawl_health_alert( + session, "arn:topic", PROJECT, self._unreadable(), ["openai.com"] + ) + assert session.sns.published == [] + def test_expected_flaky_suppresses_the_offering_nothing_trigger_too(self): """x.ai is blocked from AWS egress IPs every week; whether that shows up as FAILED or as zero candidates, it must not page."""