feat: scrape HN favorites as a Kubernetes CronJob and write to DynamoDB

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

src/favorites.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "vitest";
+import { parseFavorites } from "./favorites.js";
+
+const row = (id: number) => `
+<tr class="athing submission" id="${id}">
+  <td class="title"><span class="titleline"><a href="https://example.com/${id}">Title ${id}</a>
+    <span class="sitebit comhead">(<a href="from?site=example.com"><span class="sitestr">example.com</span></a>)</span></span></td>
+</tr>
+<tr><td class="subtext"><span class="subline">
+  <span class="score">${id} points</span> by <a class="hnuser">alice</a>
+  <span class="age"><a href="item?id=${id}">3 hours ago</a></span>
+  | <a href="item?id=${id}">${id} comments</a></span></td></tr>`;
+
+const page = (n: number) =>
+  `<table>${Array.from({ length: n }, (_, i) => row(i + 1)).join("")}</table>`;
+
+describe("parseFavorites", () => {
+  it("extracts every field of a submission", () => {
+    expect(parseFavorites(page(1))[0]).toEqual({
+      id: "1",
+      title: "Title 1",
+      url: "https://example.com/1",
+      site: "example.com",
+      score: 1,
+      by: "alice",
+      age: "3 hours ago",
+      itemUrl: "https://news.ycombinator.com/item?id=1",
+      comments: 1,
+    });
+  });
+
+  it("caps the result at the limit (10 by default)", () => {
+    expect(parseFavorites(page(15))).toHaveLength(10);
+    expect(parseFavorites(page(15), 3)).toHaveLength(3);
+  });
+
+  it("returns nothing for a page without submissions", () => {
+    expect(parseFavorites("<html><body>rate limited</body></html>")).toEqual(
+      [],
+    );
+  });
+});

src/favorites.ts

@@ -0,0 +1,63 @@
+import { load } from "cheerio";
+
+export interface Favorite {
+  id: string;
+  title: string;
+  url: string;
+  site: string | null;
+  score: number | null;
+  by: string;
+  age: string;
+  itemUrl: string;
+  comments: number;
+}
+
+const HN = "https://news.ycombinator.com";
+
+/** Parse the HTML of an HN /favorites page into at most `limit` favorites. */
+export function parseFavorites(html: string, limit = 10): Favorite[] {
+  const $ = load(html);
+  return $("tr.athing.submission")
+    .map((_, el): Favorite => {
+      const id = $(el).attr("id") || "";
+      const titleAnchor = $(el).find("span.titleline > a").first();
+      const detailsRow = $(el).next();
+
+      const scoreMatch = detailsRow.find(".subtext .score").text().match(/\d+/);
+      const ageLink = detailsRow.find(".subtext .age > a");
+      const itemPath = ageLink.attr("href") || "";
+      const commentsText = detailsRow
+        .find('.subtext a[href^="item?id="]')
+        .last()
+        .text();
+      const commentsMatch = commentsText.match(/\d+/);
+
+      return {
+        id,
+        title: titleAnchor.text(),
+        url: titleAnchor.attr("href") || "",
+        site: $(el).find(".sitebit .sitestr").text() || null,
+        score: scoreMatch ? Number(scoreMatch[0]) : null,
+        by: detailsRow.find(".subtext .hnuser").text() || "",
+        age: ageLink.text() || "",
+        itemUrl: itemPath ? `${HN}/${itemPath}` : "",
+        comments: commentsMatch ? Number(commentsMatch[0]) : 0,
+      };
+    })
+    .get()
+    .slice(0, limit);
+}
+
+export async function fetchFavoritesHtml(user: string): Promise<string> {
+  const resp = await fetch(`${HN}/favorites?id=${encodeURIComponent(user)}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    },
+  });
+  const html = await resp.text();
+  if (!resp.ok) {
+    throw new Error(`HN responded ${resp.status}: ${html.slice(0, 300)}`);
+  }
+  return html;
+}

src/main.ts

@@ -0,0 +1,63 @@
+import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
+import { DynamoDBDocumentClient, PutCommand } from "@aws-sdk/lib-dynamodb";
+import { randomUUID } from "node:crypto";
+import { fetchFavoritesHtml, parseFavorites } from "./favorites.js";
+
+// One-shot job for a Kubernetes CronJob: scrape, write one batch, exit.
+// AWS credentials come from the standard AWS_* env vars (a cluster Secret).
+async function main(): Promise<void> {
+  const tableName = process.env.TABLE_NAME;
+  if (!tableName) throw new Error("TABLE_NAME environment variable is not set");
+  const hnUser = process.env.HN_USER || "noaoh";
+
+  console.log(
+    JSON.stringify({ level: "info", msg: "cron job started", hnUser }),
+  );
+
+  const html = await fetchFavoritesHtml(hnUser);
+  const favorites = parseFavorites(html);
+  if (favorites.length === 0) {
+    throw new Error(`Parsed 0 favorites: ${html.slice(0, 300)}`);
+  }
+
+  const now = new Date();
+  const batchId = randomUUID();
+  const dateKey = now.toISOString().split("T")[0];
+
+  const docClient = DynamoDBDocumentClient.from(new DynamoDBClient({}));
+  await docClient.send(
+    new PutCommand({
+      TableName: tableName,
+      Item: {
+        pk: `date#${dateKey}`,
+        sk: `batch#${batchId}`,
+        fetchedAt: now.toISOString(),
+        batchId,
+        count: favorites.length,
+        favorites,
+      },
+    }),
+  );
+
+  console.log(
+    JSON.stringify({
+      level: "info",
+      msg: "wrote batch",
+      batchId,
+      dateKey,
+      count: favorites.length,
+      table: tableName,
+    }),
+  );
+}
+
+main().catch((err) => {
+  console.error(
+    JSON.stringify({
+      level: "error",
+      msg: "failed to fetch or write favorites",
+      error: err instanceof Error ? err.message : String(err),
+    }),
+  );
+  process.exit(1);
+});