diff --git a/content-scraper/README.md b/content-scraper/README.md index cc8fd895..803506cd 100644 --- a/content-scraper/README.md +++ b/content-scraper/README.md @@ -27,6 +27,15 @@ The MCP expects a database with the following tables: When installing, configure: - `database.apiUrl`: Database API URL - `database.token`: Authentication token +- `apifyApiToken`: Apify token (required for LinkedIn) +- `linkedinActor` (optional): `harvestapi` (default) or `johnvc` for + [`johnvc/linkedin-posts-api`](https://apify.com/johnvc/linkedin-posts-api). + The default stays HarvestAPI. CLI equivalent: `LINKEDIN_ACTOR=johnvc`. + +When `linkedinActor` is `johnvc`, profile posts are mapped into the existing +`LinkedInRawPost` shape. Share counts are stored as 0, author avatar is null, +and a person profile returns the recent posts LinkedIn shows a signed-out +visitor (usually 3 to 8), which covers the default `maxPosts = 5`. ## Available Tools diff --git a/content-scraper/cli.ts b/content-scraper/cli.ts index edcb0540..235a6bb8 100644 --- a/content-scraper/cli.ts +++ b/content-scraper/cli.ts @@ -23,6 +23,7 @@ import { scrapeAllBlogs } from "./server/lib/blog-scraper.ts"; import { scrapeAllLinkedInSources, scrapeLinkedInProfile, + type LinkedInActorChoice, } from "./server/lib/linkedin-scraper.ts"; import { scrapeAllRedditSources, @@ -84,6 +85,7 @@ ENVIRONMENT: DATABASE_TOKEN Token de autenticação Bearer (obrigatório) OPENROUTER_API_KEY Chave da API OpenRouter (obrigatória para scraping) APIFY_API_TOKEN Token da API Apify (obrigatório para LinkedIn) + LINKEDIN_ACTOR harvestapi (default) or johnvc `; // ============================================================================= @@ -95,14 +97,17 @@ interface CliConfig { databaseToken: string; openrouterApiKey: string; apifyApiToken: string; + linkedinActor: LinkedInActorChoice; } function loadConfig(): CliConfig { + const actorEnv = (process.env.LINKEDIN_ACTOR ?? "harvestapi").trim(); return { databaseApiUrl: process.env.DATABASE_API_URL ?? "", databaseToken: process.env.DATABASE_TOKEN ?? "", openrouterApiKey: process.env.OPENROUTER_API_KEY ?? "", apifyApiToken: process.env.APIFY_API_TOKEN ?? "", + linkedinActor: actorEnv === "johnvc" ? "johnvc" : "harvestapi", }; } @@ -191,6 +196,7 @@ async function cmdScrapeAll( apiKey, config.apifyApiToken, maxPosts, + config.linkedinActor, ); const totalSaved = linkedinResults.reduce( (sum, r) => sum + r.postsSaved, @@ -279,6 +285,7 @@ async function cmdScrapeLinkedIn( profileUrl, 0.7, maxPosts, + config.linkedinActor, ); console.log( `\n✅ Complete! ${result.postsSaved} posts saved, ${result.postsRelevant} relevant\n`, @@ -289,6 +296,7 @@ async function cmdScrapeLinkedIn( apiKey, apifyToken, maxPosts, + config.linkedinActor, ); const totalSaved = results.reduce((sum, r) => sum + r.postsSaved, 0); const totalRelevant = results.reduce( diff --git a/content-scraper/server/lib/linkedin-scraper.test.ts b/content-scraper/server/lib/linkedin-scraper.test.ts new file mode 100644 index 00000000..0b0613c5 --- /dev/null +++ b/content-scraper/server/lib/linkedin-scraper.test.ts @@ -0,0 +1,54 @@ +/** + * Offline fixture tests for the johnvc LinkedIn posts mapper. + * No live Apify calls. + */ +import { describe, expect, test } from "bun:test"; +import { mapJohnvcPostToRaw } from "./linkedin-scraper.ts"; + +const FIXTURE_POST: Record = { + resultType: "post", + postId: "7363908644226125825", + postUrl: + "https://www.linkedin.com/posts/example_activity-7363908644226125825", + postType: "post", + datePosted: "2026-10-01T12:00:00.000Z", + text: "Weekly MCP digest notes from our team.", + authorName: "Example Author", + authorUrl: "https://www.linkedin.com/in/example-author", + authorHeadline: "Builder at Example Co", + numLikes: 12, + numComments: 2, + images: ["https://example.com/img.png"], +}; + +const FIXTURE_ERROR: Record = { + resultType: "error", + error: "UnsupportedUrl", + message: "member-id URL", +}; + +describe("mapJohnvcPostToRaw", () => { + test("maps documented fields onto LinkedInRawPost", () => { + const mapped = mapJohnvcPostToRaw(FIXTURE_POST); + expect(mapped).not.toBeNull(); + expect(mapped!.id).toBe("7363908644226125825"); + expect(mapped!.linkedinUrl).toContain("activity-7363908644226125825"); + expect(mapped!.content).toContain("Weekly MCP digest"); + expect(mapped!.author.name).toBe("Example Author"); + expect(mapped!.author.linkedinUrl).toBe( + "https://www.linkedin.com/in/example-author", + ); + expect(mapped!.author.publicIdentifier).toBe("example-author"); + expect(mapped!.author.info).toBe("Builder at Example Co"); + expect(mapped!.author.avatar).toBeUndefined(); + expect(mapped!.engagement).toEqual({ likes: 12, comments: 2, shares: 0 }); + expect(mapped!.postedAt.date).toBe("2026-10-01T12:00:00.000Z"); + expect(mapped!.postImages).toEqual([ + { url: "https://example.com/img.png" }, + ]); + }); + + test("skips error rows", () => { + expect(mapJohnvcPostToRaw(FIXTURE_ERROR)).toBeNull(); + }); +}); diff --git a/content-scraper/server/lib/linkedin-scraper.ts b/content-scraper/server/lib/linkedin-scraper.ts index 1063a62d..8c8b78be 100644 --- a/content-scraper/server/lib/linkedin-scraper.ts +++ b/content-scraper/server/lib/linkedin-scraper.ts @@ -17,6 +17,10 @@ import { getPublicationWeek, sleep, generateContentHash } from "./utils.ts"; // Apify API configuration const APIFY_API_URL = "https://api.apify.com/v2"; const LINKEDIN_ACTOR_ID = "harvestapi~linkedin-profile-posts"; +const JOHNVC_LINKEDIN_ACTOR_ID = "johnvc~linkedin-posts-api"; + +/** Opt-in Actor choice. Default stays harvestapi. */ +export type LinkedInActorChoice = "harvestapi" | "johnvc"; // Rate limiting const DELAY_BETWEEN_POSTS = 300; @@ -116,25 +120,15 @@ async function createLinkedInContent( } /** - * Fetch LinkedIn posts via Apify + * Start an Apify Actor run, poll until SUCCEEDED, and return dataset items. */ -async function fetchLinkedInPosts( - profileUrl: string, +async function runApifyActorAndFetchItems( + actorId: string, + input: Record, apifyToken: string, - maxPosts = 5, -): Promise { - const input = { - targetUrls: [profileUrl], - maxPosts, - includeReposts: true, - includeQuotePosts: true, - scrapeComments: false, - scrapeReactions: false, - }; - - // Start Actor run +): Promise { const runResponse = await fetch( - `${APIFY_API_URL}/acts/${LINKEDIN_ACTOR_ID}/runs?token=${apifyToken}`, + `${APIFY_API_URL}/acts/${actorId}/runs?token=${apifyToken}`, { method: "POST", headers: { @@ -152,9 +146,8 @@ async function fetchLinkedInPosts( const runData = (await runResponse.json()) as { data: { id: string } }; const runId = runData.data.id; - console.log(`[LinkedIn] Started Apify run: ${runId}`); + console.log(`[LinkedIn] Started Apify run: ${runId} (${actorId})`); - // Poll for completion let status = "RUNNING"; let attempts = 0; const maxAttempts = 60; @@ -185,7 +178,6 @@ async function fetchLinkedInPosts( throw new Error(`Apify run failed with status: ${status}`); } - // Fetch dataset results const datasetResponse = await fetch( `${APIFY_API_URL}/actor-runs/${runId}/dataset/items?token=${apifyToken}`, ); @@ -195,9 +187,136 @@ async function fetchLinkedInPosts( } const posts = await datasetResponse.json(); + return posts as unknown[]; +} + +/** + * Public identifier from a LinkedIn profile URL path segment. + */ +function publicIdentifierFromUrl(url: string | undefined): string { + if (!url || typeof url !== "string") return ""; + try { + const parts = new URL(url).pathname.split("/").filter(Boolean); + const idx = parts.findIndex((p) => p === "in" || p === "company"); + if (idx >= 0 && parts[idx + 1]) return parts[idx + 1]; + return parts[0] ?? ""; + } catch { + return ""; + } +} + +/** + * Map a johnvc/linkedin-posts-api row onto LinkedInRawPost so processLinkedInPost + * stays unchanged. Share counts are 0 (Actor does not return them). Avatar unset. + */ +export function mapJohnvcPostToRaw( + row: Record, +): LinkedInRawPost | null { + const resultType = row.resultType ?? row.result_type; + if (resultType && resultType !== "post") return null; + + const postId = String(row.postId ?? row.id ?? ""); + if (!postId) return null; + + const datePosted = + typeof row.datePosted === "string" + ? row.datePosted + : typeof row.fetchedAt === "string" + ? row.fetchedAt + : new Date().toISOString(); + const timestamp = Date.parse(datePosted); + const authorUrl = + typeof row.authorUrl === "string" ? row.authorUrl : undefined; + const authorName = typeof row.authorName === "string" ? row.authorName : ""; + + const images = Array.isArray(row.images) + ? (row.images as unknown[]) + .map((img) => { + if (typeof img === "string") return { url: img }; + if (img && typeof img === "object" && "url" in img) { + return { url: String((img as { url: unknown }).url) }; + } + return null; + }) + .filter((x): x is { url: string } => x !== null) + : undefined; + + const raw: LinkedInRawPost = { + type: typeof row.postType === "string" ? row.postType : "post", + id: postId, + linkedinUrl: String(row.postUrl ?? row.linkedinUrl ?? ""), + content: String(row.text ?? row.content ?? ""), + author: { + name: authorName, + publicIdentifier: publicIdentifierFromUrl(authorUrl), + linkedinUrl: authorUrl ?? "", + info: + typeof row.authorHeadline === "string" ? row.authorHeadline : undefined, + }, + postedAt: { + timestamp: Number.isFinite(timestamp) ? timestamp : Date.now(), + date: datePosted, + }, + engagement: { + likes: typeof row.numLikes === "number" ? row.numLikes : 0, + comments: typeof row.numComments === "number" ? row.numComments : 0, + shares: 0, + }, + }; + if (images && images.length > 0) raw.postImages = images; + return raw; +} + +/** + * Fetch LinkedIn posts via the default HarvestAPI Actor. + */ +async function fetchLinkedInPosts( + profileUrl: string, + apifyToken: string, + maxPosts = 5, +): Promise { + const input = { + targetUrls: [profileUrl], + maxPosts, + includeReposts: true, + includeQuotePosts: true, + scrapeComments: false, + scrapeReactions: false, + }; + const posts = await runApifyActorAndFetchItems( + LINKEDIN_ACTOR_ID, + input, + apifyToken, + ); return posts as LinkedInRawPost[]; } +/** + * Fetch LinkedIn posts via johnvc/linkedin-posts-api and map to LinkedInRawPost. + */ +async function fetchLinkedInPostsJohnvc( + profileUrl: string, + apifyToken: string, + maxPosts = 5, +): Promise { + const input = { + profileUrls: [profileUrl], + maxPostsPerProfile: maxPosts, + }; + const rows = await runApifyActorAndFetchItems( + JOHNVC_LINKEDIN_ACTOR_ID, + input, + apifyToken, + ); + const out: LinkedInRawPost[] = []; + for (const row of rows) { + if (!row || typeof row !== "object" || Array.isArray(row)) continue; + const mapped = mapJohnvcPostToRaw(row as Record); + if (mapped) out.push(mapped); + } + return out; +} + /** * Get post type */ @@ -336,9 +455,11 @@ export async function scrapeLinkedInProfile( profileUrl: string, authorAuthority = 0.7, maxPosts = 5, + linkedinActor: LinkedInActorChoice = "harvestapi", ): Promise { console.log(`[LinkedIn] Fetching posts from: ${profileUrl}`); console.log(` Authority: ${(authorAuthority * 100).toFixed(0)}%`); + console.log(` Actor: ${linkedinActor}`); const ctx: LinkedInScraperContext = { dbClient, @@ -346,8 +467,10 @@ export async function scrapeLinkedInProfile( apifyApiToken, }; - // Fetch posts via Apify - const posts = await fetchLinkedInPosts(profileUrl, apifyApiToken, maxPosts); + const posts = + linkedinActor === "johnvc" + ? await fetchLinkedInPostsJohnvc(profileUrl, apifyApiToken, maxPosts) + : await fetchLinkedInPosts(profileUrl, apifyApiToken, maxPosts); if (posts.length === 0) { console.log(" ⚠ No posts found"); @@ -406,6 +529,7 @@ export async function scrapeAllLinkedInSources( openrouterApiKey: string, apifyApiToken: string, maxPosts = 5, + linkedinActor: LinkedInActorChoice = "harvestapi", ): Promise { console.log("=".repeat(60)); console.log("LinkedIn Scraper - Scraping All Sources"); @@ -434,6 +558,7 @@ export async function scrapeAllLinkedInSources( source.profile_url, source.authority, maxPosts, + linkedinActor, ); results.push(result); console.log( diff --git a/content-scraper/server/tools/scraping.ts b/content-scraper/server/tools/scraping.ts index f321af0a..76cd877d 100644 --- a/content-scraper/server/tools/scraping.ts +++ b/content-scraper/server/tools/scraping.ts @@ -12,6 +12,7 @@ import { scrapeAllBlogs } from "../lib/blog-scraper.ts"; import { scrapeAllLinkedInSources, scrapeLinkedInProfile, + type LinkedInActorChoice, } from "../lib/linkedin-scraper.ts"; import { scrapeAllRedditSources, @@ -60,6 +61,14 @@ function getApifyApiToken(env: Env): string | null { return env.MESH_REQUEST_CONTEXT?.state?.apifyApiToken ?? null; } +/** + * LinkedIn Actor choice from state. Default harvestapi. + */ +function getLinkedInActor(env: Env): LinkedInActorChoice { + const value = env.MESH_REQUEST_CONTEXT?.state?.linkedinActor; + return value === "johnvc" ? "johnvc" : "harvestapi"; +} + // ============================================================================= // SCRAPE_ALL Tool // ============================================================================= @@ -159,6 +168,7 @@ export const getScrapeAllTool = (env: Env) => openrouterApiKey, apifyApiToken, linkedin_max_posts, + getLinkedInActor(env), ); const totalSaved = linkedinResults.reduce( (sum, r) => sum + r.postsSaved, @@ -332,6 +342,7 @@ export const getScrapeLinkedInTool = (env: Env) => profile_url, 0.7, max_posts, + getLinkedInActor(env), ); return { success: true, @@ -346,6 +357,7 @@ export const getScrapeLinkedInTool = (env: Env) => openrouterApiKey, apifyApiToken, max_posts, + getLinkedInActor(env), ); const totalSaved = results.reduce((sum, r) => sum + r.postsSaved, 0); const totalRelevant = results.reduce( diff --git a/content-scraper/server/types/env.ts b/content-scraper/server/types/env.ts index 87390a2d..e8f5456a 100644 --- a/content-scraper/server/types/env.ts +++ b/content-scraper/server/types/env.ts @@ -64,6 +64,13 @@ export const StateSchema = z.object({ .string() .optional() .describe("Apify API token for LinkedIn scraping (required for LinkedIn)"), + linkedinActor: z + .enum(["harvestapi", "johnvc"]) + .optional() + .default("harvestapi") + .describe( + "LinkedIn posts Actor: harvestapi (default) or johnvc (linkedin-posts-api). Clear/omit to keep harvestapi.", + ), urlFields: z.object({ urls: z.array(UrlEntrySchema).describe("URLs to scrape content from"), }),