feat: scrape HN favorites as a Kubernetes CronJob and write to DynamoDB

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

.dockerignore

@@ -0,0 +1,4 @@
+node_modules
+dist
+.git
+.forgejo

.forgejo/workflows/ci.yaml

@@ -0,0 +1,29 @@
+# Full pipeline: checks -> build+push image -> bump tag in gitops repo.
+# The heavy lifting lives in nholtware/workflows composite actions; ArgoCD
+# takes over from the gitops commit.
+on:
+  push:
+    branches: [main]
+
+jobs:
+  checks:
+    runs-on: docker
+    steps:
+      - uses: actions/checkout@v4
+      - uses: http://192.168.1.230:3000/nholtware/workflows/node-checks@v1
+      - uses: http://192.168.1.230:3000/nholtware/workflows/lint-checks@v1
+
+  deploy:
+    needs: checks
+    runs-on: host
+    steps:
+      - uses: actions/checkout@v4
+      - uses: http://192.168.1.230:3000/nholtware/workflows/build-push@v1
+        with:
+          app-name: hackernews-favs-cron
+          registry-user: ${{ secrets.REGISTRY_USER }}
+          registry-token: ${{ secrets.REGISTRY_TOKEN }}
+      - uses: http://192.168.1.230:3000/nholtware/workflows/gitops-bump@v1
+        with:
+          app-name: hackernews-favs-cron
+          gitops-token: ${{ secrets.GITOPS_TOKEN }}

.forgejo/workflows/pr.yaml

@@ -0,0 +1,12 @@
+# PR checks: lint + test only. No build/deploy -- that's ci.yaml's job on push to main.
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  checks:
+    runs-on: docker
+    steps:
+      - uses: actions/checkout@v4
+      - uses: http://192.168.1.230:3000/nholtware/workflows/node-checks@v1
+      - uses: http://192.168.1.230:3000/nholtware/workflows/lint-checks@v1

.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+dist/
+.env

.nvmrc

@@ -0,0 +1 @@
+v26

.prettierignore

@@ -0,0 +1,3 @@
+dist
+node_modules
+package-lock.json

.prettierrc.json

@@ -0,0 +1,4 @@
+{
+  "semi": true,
+  "trailingComma": "all"
+}

Dockerfile

@@ -0,0 +1,18 @@
+FROM node:22 AS build
+WORKDIR /app
+COPY package.json package-lock.json ./
+RUN npm ci
+COPY tsconfig.json ./
+COPY src ./src
+RUN npm run build
+
+FROM node:22-slim
+WORKDIR /app
+ENV NODE_ENV=production
+ARG GIT_SHA=dev
+ENV GIT_SHA=${GIT_SHA}
+COPY package.json package-lock.json ./
+RUN npm ci --omit=dev
+COPY --from=build /app/dist ./dist
+USER node
+CMD ["node", "dist/main.js"]

README.md

@@ -0,0 +1,18 @@
+# hackernews-favs-cron
+
+Runs the Hacker News favorites scrape from `nholtware/hackernews-favs` (the AWS
+Lambda version) as a Kubernetes CronJob on the homelab k3s cluster. It still
+writes each batch to the AWS DynamoDB table `hackernews-favs` (same item shape:
+`pk=date#YYYY-MM-DD`, `sk=batch#<uuid>`), so existing readers are unaffected.
+
+- Code: `src/favorites.ts` (fetch + parse), `src/main.ts` (one-shot entrypoint).
+- Deploy: push to `main` -> Forgejo CI builds the image -> `gitops-bump` ->
+  ArgoCD syncs `manifests/hackernews-favs-cron` (CronJob, Sundays 02:00 UTC).
+- Config env: `TABLE_NAME`, `HN_USER` (default `noaoh`), `AWS_REGION`.
+- Credentials: Secret `aws-dynamodb` in namespace `hackernews-favs-cron`, keys
+  `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`. Created by kubectl, never in git.
+  Scope the IAM user to `dynamodb:PutItem` on the table.
+- Run once now: `kubectl create job -n hackernews-favs-cron --from=cronjob/hackernews-favs-cron manual-$(date +%s)`
+- Remove the Lambda schedule once happy: `cdk destroy` in `hackernews-favs`
+  would also delete the table (RemovalPolicy.DESTROY), so disable the EventBridge
+  rule instead.

commitlint.config.js

@@ -0,0 +1,3 @@
+export default {
+  extends: ["@commitlint/config-conventional"],
+};

eslint.config.js

@@ -0,0 +1,8 @@
+import tseslint from "typescript-eslint";
+
+export default tseslint.config(
+  {
+    ignores: ["dist/**"],
+  },
+  ...tseslint.configs.recommended,
+);

package.json

@@ -0,0 +1,34 @@
+{
+  "name": "hackernews-favs-cron",
+  "version": "1.0.0",
+  "description": "Kubernetes CronJob: scrapes Hacker News favorites and writes a batch to AWS DynamoDB",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/main.js",
+    "dev": "tsx src/main.ts",
+    "test": "vitest run",
+    "lint": "eslint . && tsc --noEmit",
+    "format": "prettier --write .",
+    "format:check": "prettier --check ."
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "dependencies": {
+    "@aws-sdk/client-dynamodb": "^3.1141.0",
+    "@aws-sdk/lib-dynamodb": "^3.1141.0",
+    "cheerio": "^1.0.0"
+  },
+  "devDependencies": {
+    "@commitlint/cli": "^19.5.0",
+    "@commitlint/config-conventional": "^19.5.0",
+    "@types/node": "^22.0.0",
+    "eslint": "^9.13.0",
+    "prettier": "^3.3.3",
+    "tsx": "^4.20.0",
+    "typescript": "^5.6.0",
+    "typescript-eslint": "^8.15.0",
+    "vitest": "^4.1.10"
+  }
+}

src/favorites.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "vitest";
+import { parseFavorites } from "./favorites.js";
+
+const row = (id: number) => `
+<tr class="athing submission" id="${id}">
+  <td class="title"><span class="titleline"><a href="https://example.com/${id}">Title ${id}</a>
+    <span class="sitebit comhead">(<a href="from?site=example.com"><span class="sitestr">example.com</span></a>)</span></span></td>
+</tr>
+<tr><td class="subtext"><span class="subline">
+  <span class="score">${id} points</span> by <a class="hnuser">alice</a>
+  <span class="age"><a href="item?id=${id}">3 hours ago</a></span>
+  | <a href="item?id=${id}">${id} comments</a></span></td></tr>`;
+
+const page = (n: number) =>
+  `<table>${Array.from({ length: n }, (_, i) => row(i + 1)).join("")}</table>`;
+
+describe("parseFavorites", () => {
+  it("extracts every field of a submission", () => {
+    expect(parseFavorites(page(1))[0]).toEqual({
+      id: "1",
+      title: "Title 1",
+      url: "https://example.com/1",
+      site: "example.com",
+      score: 1,
+      by: "alice",
+      age: "3 hours ago",
+      itemUrl: "https://news.ycombinator.com/item?id=1",
+      comments: 1,
+    });
+  });
+
+  it("caps the result at the limit (10 by default)", () => {
+    expect(parseFavorites(page(15))).toHaveLength(10);
+    expect(parseFavorites(page(15), 3)).toHaveLength(3);
+  });
+
+  it("returns nothing for a page without submissions", () => {
+    expect(parseFavorites("<html><body>rate limited</body></html>")).toEqual(
+      [],
+    );
+  });
+});

src/favorites.ts

@@ -0,0 +1,63 @@
+import { load } from "cheerio";
+
+export interface Favorite {
+  id: string;
+  title: string;
+  url: string;
+  site: string | null;
+  score: number | null;
+  by: string;
+  age: string;
+  itemUrl: string;
+  comments: number;
+}
+
+const HN = "https://news.ycombinator.com";
+
+/** Parse the HTML of an HN /favorites page into at most `limit` favorites. */
+export function parseFavorites(html: string, limit = 10): Favorite[] {
+  const $ = load(html);
+  return $("tr.athing.submission")
+    .map((_, el): Favorite => {
+      const id = $(el).attr("id") || "";
+      const titleAnchor = $(el).find("span.titleline > a").first();
+      const detailsRow = $(el).next();
+
+      const scoreMatch = detailsRow.find(".subtext .score").text().match(/\d+/);
+      const ageLink = detailsRow.find(".subtext .age > a");
+      const itemPath = ageLink.attr("href") || "";
+      const commentsText = detailsRow
+        .find('.subtext a[href^="item?id="]')
+        .last()
+        .text();
+      const commentsMatch = commentsText.match(/\d+/);
+
+      return {
+        id,
+        title: titleAnchor.text(),
+        url: titleAnchor.attr("href") || "",
+        site: $(el).find(".sitebit .sitestr").text() || null,
+        score: scoreMatch ? Number(scoreMatch[0]) : null,
+        by: detailsRow.find(".subtext .hnuser").text() || "",
+        age: ageLink.text() || "",
+        itemUrl: itemPath ? `${HN}/${itemPath}` : "",
+        comments: commentsMatch ? Number(commentsMatch[0]) : 0,
+      };
+    })
+    .get()
+    .slice(0, limit);
+}
+
+export async function fetchFavoritesHtml(user: string): Promise<string> {
+  const resp = await fetch(`${HN}/favorites?id=${encodeURIComponent(user)}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    },
+  });
+  const html = await resp.text();
+  if (!resp.ok) {
+    throw new Error(`HN responded ${resp.status}: ${html.slice(0, 300)}`);
+  }
+  return html;
+}

src/main.ts

@@ -0,0 +1,63 @@
+import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
+import { DynamoDBDocumentClient, PutCommand } from "@aws-sdk/lib-dynamodb";
+import { randomUUID } from "node:crypto";
+import { fetchFavoritesHtml, parseFavorites } from "./favorites.js";
+
+// One-shot job for a Kubernetes CronJob: scrape, write one batch, exit.
+// AWS credentials come from the standard AWS_* env vars (a cluster Secret).
+async function main(): Promise<void> {
+  const tableName = process.env.TABLE_NAME;
+  if (!tableName) throw new Error("TABLE_NAME environment variable is not set");
+  const hnUser = process.env.HN_USER || "noaoh";
+
+  console.log(
+    JSON.stringify({ level: "info", msg: "cron job started", hnUser }),
+  );
+
+  const html = await fetchFavoritesHtml(hnUser);
+  const favorites = parseFavorites(html);
+  if (favorites.length === 0) {
+    throw new Error(`Parsed 0 favorites: ${html.slice(0, 300)}`);
+  }
+
+  const now = new Date();
+  const batchId = randomUUID();
+  const dateKey = now.toISOString().split("T")[0];
+
+  const docClient = DynamoDBDocumentClient.from(new DynamoDBClient({}));
+  await docClient.send(
+    new PutCommand({
+      TableName: tableName,
+      Item: {
+        pk: `date#${dateKey}`,
+        sk: `batch#${batchId}`,
+        fetchedAt: now.toISOString(),
+        batchId,
+        count: favorites.length,
+        favorites,
+      },
+    }),
+  );
+
+  console.log(
+    JSON.stringify({
+      level: "info",
+      msg: "wrote batch",
+      batchId,
+      dateKey,
+      count: favorites.length,
+      table: tableName,
+    }),
+  );
+}
+
+main().catch((err) => {
+  console.error(
+    JSON.stringify({
+      level: "error",
+      msg: "failed to fetch or write favorites",
+      error: err instanceof Error ? err.message : String(err),
+    }),
+  );
+  process.exit(1);
+});

tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": ["src"]
+}

vitest.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    include: ["src/**/*.test.ts"],
+  },
+});