fix: identify honestly to HN and retry with backoff on 419/429/5xx
The spoofed Chrome User-Agent drew HTTP 419 from HN; an honest UA gets 200. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
35e9c73cf2
commit
bc0b92bd4d
1 changed files with 25 additions and 11 deletions
|
|
@ -48,16 +48,30 @@ export function parseFavorites(html: string, limit = 10): Favorite[] {
|
||||||
.slice(0, limit);
|
.slice(0, limit);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function fetchFavoritesHtml(user: string): Promise<string> {
|
const USER_AGENT =
|
||||||
const resp = await fetch(`${HN}/favorites?id=${encodeURIComponent(user)}`, {
|
"hackernews-favs-cron/1.0 (personal homelab job; one request per week)";
|
||||||
headers: {
|
const RETRYABLE = new Set([419, 429, 500, 502, 503, 504]);
|
||||||
"User-Agent":
|
|
||||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
/**
|
||||||
},
|
* Fetch the favorites page, identifying ourselves honestly. Retries a few
|
||||||
});
|
* times with exponential backoff on rate-limit/transient statuses, then fails
|
||||||
|
* loudly (the CronJob's backoffLimit takes over).
|
||||||
|
*/
|
||||||
|
export async function fetchFavoritesHtml(
|
||||||
|
user: string,
|
||||||
|
attempts = 4,
|
||||||
|
baseDelayMs = 5000,
|
||||||
|
): Promise<string> {
|
||||||
|
const url = `${HN}/favorites?id=${encodeURIComponent(user)}`;
|
||||||
|
for (let attempt = 1; ; attempt++) {
|
||||||
|
const resp = await fetch(url, { headers: { "User-Agent": USER_AGENT } });
|
||||||
const html = await resp.text();
|
const html = await resp.text();
|
||||||
if (!resp.ok) {
|
if (resp.ok) return html;
|
||||||
throw new Error(`HN responded ${resp.status}: ${html.slice(0, 300)}`);
|
if (!RETRYABLE.has(resp.status) || attempt >= attempts) {
|
||||||
|
throw new Error(
|
||||||
|
`HN responded ${resp.status} after ${attempt} attempt(s): ${html.slice(0, 300)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await new Promise((r) => setTimeout(r, baseDelayMs * 2 ** (attempt - 1)));
|
||||||
}
|
}
|
||||||
return html;
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue