feat(content-scraper): optional johnvc/linkedin-posts-api source for LinkedIn profiles

content-scraper/README.md

@@ -27,6 +27,15 @@ The MCP expects a database with the following tables:
 When installing, configure:
 - `database.apiUrl`: Database API URL
 - `database.token`: Authentication token
+- `apifyApiToken`: Apify token (required for LinkedIn)
+- `linkedinActor` (optional): `harvestapi` (default) or `johnvc` for
+  [`johnvc/linkedin-posts-api`](https://apify.com/johnvc/linkedin-posts-api).
+  The default stays HarvestAPI. CLI equivalent: `LINKEDIN_ACTOR=johnvc`.
+
+When `linkedinActor` is `johnvc`, profile posts are mapped into the existing
+`LinkedInRawPost` shape. Share counts are stored as 0, author avatar is null,
+and a person profile returns the recent posts LinkedIn shows a signed-out
+visitor (usually 3 to 8), which covers the default `maxPosts = 5`.
 
 ## Available Tools
 

content-scraper/cli.ts

@@ -23,6 +23,7 @@ import { scrapeAllBlogs } from "./server/lib/blog-scraper.ts";
 import {
   scrapeAllLinkedInSources,
   scrapeLinkedInProfile,
+  type LinkedInActorChoice,
 } from "./server/lib/linkedin-scraper.ts";
 import {
   scrapeAllRedditSources,
@@ -84,6 +85,7 @@ ENVIRONMENT:
   DATABASE_TOKEN        Token de autenticação Bearer (obrigatório)
   OPENROUTER_API_KEY    Chave da API OpenRouter (obrigatória para scraping)
   APIFY_API_TOKEN       Token da API Apify (obrigatório para LinkedIn)
+  LINKEDIN_ACTOR        harvestapi (default) or johnvc
 `;
 
 // =============================================================================
@@ -95,14 +97,17 @@ interface CliConfig {
   databaseToken: string;
   openrouterApiKey: string;
   apifyApiToken: string;
+  linkedinActor: LinkedInActorChoice;
 }
 
 function loadConfig(): CliConfig {
+  const actorEnv = (process.env.LINKEDIN_ACTOR ?? "harvestapi").trim();
   return {
     databaseApiUrl: process.env.DATABASE_API_URL ?? "",
     databaseToken: process.env.DATABASE_TOKEN ?? "",
     openrouterApiKey: process.env.OPENROUTER_API_KEY ?? "",
     apifyApiToken: process.env.APIFY_API_TOKEN ?? "",
+    linkedinActor: actorEnv === "johnvc" ? "johnvc" : "harvestapi",
   };
 }
 
@@ -191,6 +196,7 @@ async function cmdScrapeAll(
           apiKey,
           config.apifyApiToken,
           maxPosts,
+          config.linkedinActor,
         );
         const totalSaved = linkedinResults.reduce(
           (sum, r) => sum + r.postsSaved,
@@ -279,6 +285,7 @@ async function cmdScrapeLinkedIn(
         profileUrl,
         0.7,
         maxPosts,
+        config.linkedinActor,
       );
       console.log(
         `\n✅ Complete! ${result.postsSaved} posts saved, ${result.postsRelevant} relevant\n`,
@@ -289,6 +296,7 @@ async function cmdScrapeLinkedIn(
         apiKey,
         apifyToken,
         maxPosts,
+        config.linkedinActor,
       );
       const totalSaved = results.reduce((sum, r) => sum + r.postsSaved, 0);
       const totalRelevant = results.reduce(

content-scraper/server/lib/linkedin-scraper.test.ts

@@ -0,0 +1,54 @@
+/**
+ * Offline fixture tests for the johnvc LinkedIn posts mapper.
+ * No live Apify calls.
+ */
+import { describe, expect, test } from "bun:test";
+import { mapJohnvcPostToRaw } from "./linkedin-scraper.ts";
+
+const FIXTURE_POST: Record<string, unknown> = {
+  resultType: "post",
+  postId: "7363908644226125825",
+  postUrl:
+    "https://www.linkedin.com/posts/example_activity-7363908644226125825",
+  postType: "post",
+  datePosted: "2026-10-01T12:00:00.000Z",
+  text: "Weekly MCP digest notes from our team.",
+  authorName: "Example Author",
+  authorUrl: "https://www.linkedin.com/in/example-author",
+  authorHeadline: "Builder at Example Co",
+  numLikes: 12,
+  numComments: 2,
+  images: ["https://example.com/img.png"],
+};
+
+const FIXTURE_ERROR: Record<string, unknown> = {
+  resultType: "error",
+  error: "UnsupportedUrl",
+  message: "member-id URL",
+};
+
+describe("mapJohnvcPostToRaw", () => {
+  test("maps documented fields onto LinkedInRawPost", () => {
+    const mapped = mapJohnvcPostToRaw(FIXTURE_POST);
+    expect(mapped).not.toBeNull();
+    expect(mapped!.id).toBe("7363908644226125825");
+    expect(mapped!.linkedinUrl).toContain("activity-7363908644226125825");
+    expect(mapped!.content).toContain("Weekly MCP digest");
+    expect(mapped!.author.name).toBe("Example Author");
+    expect(mapped!.author.linkedinUrl).toBe(
+      "https://www.linkedin.com/in/example-author",
+    );
+    expect(mapped!.author.publicIdentifier).toBe("example-author");
+    expect(mapped!.author.info).toBe("Builder at Example Co");
+    expect(mapped!.author.avatar).toBeUndefined();
+    expect(mapped!.engagement).toEqual({ likes: 12, comments: 2, shares: 0 });
+    expect(mapped!.postedAt.date).toBe("2026-10-01T12:00:00.000Z");
+    expect(mapped!.postImages).toEqual([
+      { url: "https://example.com/img.png" },
+    ]);
+  });
+
+  test("skips error rows", () => {
+    expect(mapJohnvcPostToRaw(FIXTURE_ERROR)).toBeNull();
+  });
+});

content-scraper/server/lib/linkedin-scraper.ts

@@ -17,6 +17,10 @@ import { getPublicationWeek, sleep, generateContentHash } from "./utils.ts";
 // Apify API configuration
 const APIFY_API_URL = "https://api.apify.com/v2";
 const LINKEDIN_ACTOR_ID = "harvestapi~linkedin-profile-posts";
+const JOHNVC_LINKEDIN_ACTOR_ID = "johnvc~linkedin-posts-api";
+
+/** Opt-in Actor choice. Default stays harvestapi. */
+export type LinkedInActorChoice = "harvestapi" | "johnvc";
 
 // Rate limiting
 const DELAY_BETWEEN_POSTS = 300;
@@ -116,25 +120,15 @@ async function createLinkedInContent(
 }
 
 /**
- * Fetch LinkedIn posts via Apify
+ * Start an Apify Actor run, poll until SUCCEEDED, and return dataset items.
  */
-async function fetchLinkedInPosts(
-  profileUrl: string,
+async function runApifyActorAndFetchItems(
+  actorId: string,
+  input: Record<string, unknown>,
   apifyToken: string,
-  maxPosts = 5,
-): Promise<LinkedInRawPost[]> {
-  const input = {
-    targetUrls: [profileUrl],
-    maxPosts,
-    includeReposts: true,
-    includeQuotePosts: true,
-    scrapeComments: false,
-    scrapeReactions: false,
-  };
-
-  // Start Actor run
+): Promise<unknown[]> {
   const runResponse = await fetch(
-    `${APIFY_API_URL}/acts/${LINKEDIN_ACTOR_ID}/runs?token=${apifyToken}`,
+    `${APIFY_API_URL}/acts/${actorId}/runs?token=${apifyToken}`,
     {
       method: "POST",
       headers: {
@@ -152,9 +146,8 @@ async function fetchLinkedInPosts(
   const runData = (await runResponse.json()) as { data: { id: string } };
   const runId = runData.data.id;
 
-  console.log(`[LinkedIn] Started Apify run: ${runId}`);
+  console.log(`[LinkedIn] Started Apify run: ${runId} (${actorId})`);
 
-  // Poll for completion
   let status = "RUNNING";
   let attempts = 0;
   const maxAttempts = 60;
@@ -185,7 +178,6 @@ async function fetchLinkedInPosts(
     throw new Error(`Apify run failed with status: ${status}`);
   }
 
-  // Fetch dataset results
   const datasetResponse = await fetch(
     `${APIFY_API_URL}/actor-runs/${runId}/dataset/items?token=${apifyToken}`,
   );
@@ -195,9 +187,136 @@ async function fetchLinkedInPosts(
   }
 
   const posts = await datasetResponse.json();
+  return posts as unknown[];
+}
+
+/**
+ * Public identifier from a LinkedIn profile URL path segment.
+ */
+function publicIdentifierFromUrl(url: string | undefined): string {
+  if (!url || typeof url !== "string") return "";
+  try {
+    const parts = new URL(url).pathname.split("/").filter(Boolean);
+    const idx = parts.findIndex((p) => p === "in" || p === "company");
+    if (idx >= 0 && parts[idx + 1]) return parts[idx + 1];
+    return parts[0] ?? "";
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * Map a johnvc/linkedin-posts-api row onto LinkedInRawPost so processLinkedInPost
+ * stays unchanged. Share counts are 0 (Actor does not return them). Avatar unset.
+ */
+export function mapJohnvcPostToRaw(
+  row: Record<string, unknown>,
+): LinkedInRawPost | null {
+  const resultType = row.resultType ?? row.result_type;
+  if (resultType && resultType !== "post") return null;
+
+  const postId = String(row.postId ?? row.id ?? "");
+  if (!postId) return null;
+
+  const datePosted =
+    typeof row.datePosted === "string"
+      ? row.datePosted
+      : typeof row.fetchedAt === "string"
+        ? row.fetchedAt
+        : new Date().toISOString();
+  const timestamp = Date.parse(datePosted);
+  const authorUrl =
+    typeof row.authorUrl === "string" ? row.authorUrl : undefined;
+  const authorName = typeof row.authorName === "string" ? row.authorName : "";
+
+  const images = Array.isArray(row.images)
+    ? (row.images as unknown[])
+        .map((img) => {
+          if (typeof img === "string") return { url: img };
+          if (img && typeof img === "object" && "url" in img) {
+            return { url: String((img as { url: unknown }).url) };
+          }
+          return null;
+        })
+        .filter((x): x is { url: string } => x !== null)
+    : undefined;
+
+  const raw: LinkedInRawPost = {
+    type: typeof row.postType === "string" ? row.postType : "post",
+    id: postId,
+    linkedinUrl: String(row.postUrl ?? row.linkedinUrl ?? ""),
+    content: String(row.text ?? row.content ?? ""),
+    author: {
+      name: authorName,
+      publicIdentifier: publicIdentifierFromUrl(authorUrl),
+      linkedinUrl: authorUrl ?? "",
+      info:
+        typeof row.authorHeadline === "string" ? row.authorHeadline : undefined,
+    },
+    postedAt: {
+      timestamp: Number.isFinite(timestamp) ? timestamp : Date.now(),
+      date: datePosted,
+    },
+    engagement: {
+      likes: typeof row.numLikes === "number" ? row.numLikes : 0,
+      comments: typeof row.numComments === "number" ? row.numComments : 0,
+      shares: 0,
+    },
+  };
+  if (images && images.length > 0) raw.postImages = images;
+  return raw;
+}
+
+/**
+ * Fetch LinkedIn posts via the default HarvestAPI Actor.
+ */
+async function fetchLinkedInPosts(
+  profileUrl: string,
+  apifyToken: string,
+  maxPosts = 5,
+): Promise<LinkedInRawPost[]> {
+  const input = {
+    targetUrls: [profileUrl],
+    maxPosts,
+    includeReposts: true,
+    includeQuotePosts: true,
+    scrapeComments: false,
+    scrapeReactions: false,
+  };
+  const posts = await runApifyActorAndFetchItems(
+    LINKEDIN_ACTOR_ID,
+    input,
+    apifyToken,
+  );
   return posts as LinkedInRawPost[];
 }
 
+/**
+ * Fetch LinkedIn posts via johnvc/linkedin-posts-api and map to LinkedInRawPost.
+ */
+async function fetchLinkedInPostsJohnvc(
+  profileUrl: string,
+  apifyToken: string,
+  maxPosts = 5,
+): Promise<LinkedInRawPost[]> {
+  const input = {
+    profileUrls: [profileUrl],
+    maxPostsPerProfile: maxPosts,
+  };
+  const rows = await runApifyActorAndFetchItems(
+    JOHNVC_LINKEDIN_ACTOR_ID,
+    input,
+    apifyToken,
+  );
+  const out: LinkedInRawPost[] = [];
+  for (const row of rows) {
+    if (!row || typeof row !== "object" || Array.isArray(row)) continue;
+    const mapped = mapJohnvcPostToRaw(row as Record<string, unknown>);
+    if (mapped) out.push(mapped);
+  }
+  return out;
+}
+
 /**
  * Get post type
  */
@@ -336,18 +455,22 @@ export async function scrapeLinkedInProfile(
   profileUrl: string,
   authorAuthority = 0.7,
   maxPosts = 5,
+  linkedinActor: LinkedInActorChoice = "harvestapi",
 ): Promise<ScrapeResult> {
   console.log(`[LinkedIn] Fetching posts from: ${profileUrl}`);
   console.log(`    Authority: ${(authorAuthority * 100).toFixed(0)}%`);
+  console.log(`    Actor: ${linkedinActor}`);
 
   const ctx: LinkedInScraperContext = {
     dbClient,
     openrouterApiKey,
     apifyApiToken,
   };
 
-  // Fetch posts via Apify
-  const posts = await fetchLinkedInPosts(profileUrl, apifyApiToken, maxPosts);
+  const posts =
+    linkedinActor === "johnvc"
+      ? await fetchLinkedInPostsJohnvc(profileUrl, apifyApiToken, maxPosts)
+      : await fetchLinkedInPosts(profileUrl, apifyApiToken, maxPosts);
 
   if (posts.length === 0) {
     console.log("    ⚠ No posts found");
@@ -406,6 +529,7 @@ export async function scrapeAllLinkedInSources(
   openrouterApiKey: string,
   apifyApiToken: string,
   maxPosts = 5,
+  linkedinActor: LinkedInActorChoice = "harvestapi",
 ): Promise<ScrapeResult[]> {
   console.log("=".repeat(60));
   console.log("LinkedIn Scraper - Scraping All Sources");
@@ -434,6 +558,7 @@ export async function scrapeAllLinkedInSources(
         source.profile_url,
         source.authority,
         maxPosts,
+        linkedinActor,
       );
       results.push(result);
       console.log(