refactor: store VidIQ cards in the service worker and drop the API layer
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
IgorVolochay
co-authored by
Claude Sonnet 5.5
Claude Sonnet 5.5
parent
1cf5270704
commit
cdf8c08c7f
-173
@@ -1,173 +0,0 @@
|
||||
export class ApiError extends Error {
|
||||
constructor(kind, message, status = null) {
|
||||
super(message);
|
||||
this.name = 'ApiError';
|
||||
this.kind = kind;
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export function chunk(items, size) {
|
||||
const batches = [];
|
||||
for (let i = 0; i < items.length; i += size) batches.push(items.slice(i, i + size));
|
||||
return batches;
|
||||
}
|
||||
|
||||
export function parseIsoDuration(iso) {
|
||||
const match = /^P(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?$/.exec(iso ?? '');
|
||||
if (!match) return null;
|
||||
const [, days = 0, hours = 0, minutes = 0, seconds = 0] = match;
|
||||
return Number(days) * 86400 + Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds);
|
||||
}
|
||||
|
||||
const THUMBNAIL_ORDER = ['maxres', 'standard', 'high', 'medium', 'default'];
|
||||
|
||||
export function pickThumbnail(thumbnails) {
|
||||
for (const size of THUMBNAIL_ORDER) {
|
||||
const url = thumbnails?.[size]?.url;
|
||||
if (url) return url;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const QUOTA_REASONS = new Set(['quotaExceeded', 'dailyLimitExceeded']);
|
||||
const KEY_REASONS = new Set([
|
||||
'keyInvalid',
|
||||
'API_KEY_INVALID',
|
||||
'accessNotConfigured',
|
||||
'ipRefererBlocked',
|
||||
'API_KEY_HTTP_REFERRER_BLOCKED',
|
||||
'API_KEY_SERVICE_BLOCKED',
|
||||
'SERVICE_DISABLED',
|
||||
]);
|
||||
|
||||
export function errorReasons(body) {
|
||||
const error = body?.error;
|
||||
return [...(error?.errors ?? []), ...(error?.details ?? [])].map((entry) => entry.reason).filter(Boolean);
|
||||
}
|
||||
|
||||
export function classifyError(status, body) {
|
||||
const reasons = errorReasons(body);
|
||||
if (reasons.some((reason) => QUOTA_REASONS.has(reason))) return 'quota';
|
||||
if (reasons.some((reason) => KEY_REASONS.has(reason))) return 'invalid-key';
|
||||
if (status >= 500 || status === 429) return 'server';
|
||||
return 'other';
|
||||
}
|
||||
|
||||
// The Data API quota resets at midnight Pacific Time.
|
||||
export function nextQuotaReset(now = new Date()) {
|
||||
const parts = new Intl.DateTimeFormat('en-US', {
|
||||
timeZone: 'America/Los_Angeles',
|
||||
hourCycle: 'h23',
|
||||
hour: 'numeric',
|
||||
minute: 'numeric',
|
||||
second: 'numeric',
|
||||
}).formatToParts(now);
|
||||
const part = (type) => Number(parts.find((p) => p.type === type).value);
|
||||
const sinceMidnightMs =
|
||||
((part('hour') * 60 + part('minute')) * 60 + part('second')) * 1000 + now.getMilliseconds();
|
||||
return now.getTime() - sinceMidnightMs + 24 * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
const API_BASE = 'https://www.googleapis.com/youtube/v3';
|
||||
const BATCH_SIZE = 50;
|
||||
const RETRIES = 3;
|
||||
const BASE_DELAY_MS = 500;
|
||||
|
||||
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
async function requestOnce(path, params, { apiKey, fetchImpl = fetch }) {
|
||||
const url = `${API_BASE}/${path}?${new URLSearchParams({ ...params, key: apiKey })}`;
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(url);
|
||||
} catch (err) {
|
||||
throw new ApiError('network', err.message);
|
||||
}
|
||||
if (response.ok) {
|
||||
try {
|
||||
return await response.json();
|
||||
} catch {
|
||||
throw new ApiError('server', 'Invalid JSON in API response', response.status);
|
||||
}
|
||||
}
|
||||
let body = null;
|
||||
try {
|
||||
body = await response.json();
|
||||
} catch {
|
||||
// error responses are not always JSON
|
||||
}
|
||||
throw new ApiError(classifyError(response.status, body), body?.error?.message ?? `HTTP ${response.status}`, response.status);
|
||||
}
|
||||
|
||||
async function request(path, params, opts) {
|
||||
const sleep = opts.sleep ?? defaultSleep;
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
try {
|
||||
return await requestOnce(path, params, opts);
|
||||
} catch (err) {
|
||||
const retryable = err instanceof ApiError && (err.kind === 'network' || err.kind === 'server');
|
||||
if (!retryable || attempt >= RETRIES) throw err;
|
||||
await sleep(BASE_DELAY_MS * 2 ** attempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeVideo(item) {
|
||||
const { snippet = {}, statistics = {}, contentDetails = {} } = item;
|
||||
return {
|
||||
videoId: item.id,
|
||||
title: snippet.title ?? '',
|
||||
thumbnailUrl: pickThumbnail(snippet.thumbnails),
|
||||
channelId: snippet.channelId,
|
||||
channelTitle: snippet.channelTitle ?? '',
|
||||
views: Number(statistics.viewCount ?? 0),
|
||||
publishedAt: snippet.publishedAt ?? null,
|
||||
durationSec: parseIsoDuration(contentDetails.duration),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeChannel(item) {
|
||||
const customUrl = item.snippet?.customUrl;
|
||||
const stats = item.statistics ?? {};
|
||||
return {
|
||||
channelId: item.id,
|
||||
title: item.snippet?.title ?? '',
|
||||
url: customUrl?.startsWith('@')
|
||||
? `https://www.youtube.com/${customUrl}`
|
||||
: `https://www.youtube.com/channel/${item.id}`,
|
||||
subscribers: stats.hiddenSubscriberCount || stats.subscriberCount === undefined ? null : Number(stats.subscriberCount),
|
||||
};
|
||||
}
|
||||
|
||||
export async function fetchVideos(ids, opts) {
|
||||
const videos = [];
|
||||
const found = new Set();
|
||||
for (const batch of chunk(ids, BATCH_SIZE)) {
|
||||
const data = await request(
|
||||
'videos',
|
||||
{ part: 'snippet,statistics,contentDetails', id: batch.join(',') },
|
||||
opts,
|
||||
);
|
||||
for (const item of data.items ?? []) {
|
||||
videos.push(normalizeVideo(item));
|
||||
found.add(item.id);
|
||||
}
|
||||
}
|
||||
return { videos, missingIds: ids.filter((id) => !found.has(id)) };
|
||||
}
|
||||
|
||||
// maxResults is sent here only: videos.list rejects it together with `id`,
|
||||
// while channels.list would otherwise default to 5 results.
|
||||
export async function fetchChannels(ids, opts) {
|
||||
const channels = [];
|
||||
for (const batch of chunk(ids, BATCH_SIZE)) {
|
||||
const data = await request(
|
||||
'channels',
|
||||
{ part: 'statistics,snippet', id: batch.join(','), maxResults: String(BATCH_SIZE) },
|
||||
opts,
|
||||
);
|
||||
for (const item of data.items ?? []) channels.push(normalizeChannel(item));
|
||||
}
|
||||
return channels;
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
import { computeScore, shouldSave } from './score.js';
|
||||
|
||||
export const RECHECK_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
const watchUrl = (videoId) => `https://www.youtube.com/watch?v=${videoId}`;
|
||||
|
||||
function isFresh(checkedAt, nowMs) {
|
||||
return checkedAt !== undefined && nowMs - Date.parse(checkedAt) < RECHECK_MS;
|
||||
}
|
||||
|
||||
export async function processBatch(items, { settings, db, api, now = new Date() }) {
|
||||
const nowMs = now.getTime();
|
||||
const nowIso = now.toISOString();
|
||||
const byId = new Map(items.map((item) => [item.videoId, item]));
|
||||
|
||||
const seen = await db.getSeen([...byId.keys()]);
|
||||
const due = [...byId.values()].filter(
|
||||
(item) => item.vidiqScore !== null || !isFresh(seen.get(item.videoId)?.checkedAt, nowMs),
|
||||
);
|
||||
if (due.length === 0) return { checked: 0, saved: 0 };
|
||||
|
||||
const { videos, missingIds } = await api.fetchVideos(due.map((item) => item.videoId));
|
||||
|
||||
const channelIds = [...new Set(videos.map((video) => video.channelId))];
|
||||
const cachedChannels = await db.getChannels(channelIds);
|
||||
const staleIds = channelIds.filter((id) => !isFresh(cachedChannels.get(id)?.fetchedAt, nowMs));
|
||||
const fetchedChannels = staleIds.length > 0 ? await api.fetchChannels(staleIds) : [];
|
||||
const channelRecords = fetchedChannels.map((channel) => ({ ...channel, fetchedAt: nowIso }));
|
||||
const channels = new Map(cachedChannels);
|
||||
for (const record of channelRecords) channels.set(record.channelId, record);
|
||||
|
||||
const existing = await db.getOutliers(videos.map((video) => video.videoId));
|
||||
const outliers = [];
|
||||
for (const video of videos) {
|
||||
const item = byId.get(video.videoId);
|
||||
const previous = existing.get(video.videoId);
|
||||
const channel = channels.get(video.channelId);
|
||||
const subscribers = channel?.subscribers ?? null;
|
||||
const score = computeScore(video.views, subscribers);
|
||||
const vidiqScore = item.vidiqScore ?? previous?.vidiqScore ?? null;
|
||||
|
||||
if (!previous && !shouldSave({ views: video.views, score, vidiqScore }, settings)) continue;
|
||||
|
||||
outliers.push({
|
||||
videoId: video.videoId,
|
||||
title: video.title,
|
||||
url: watchUrl(video.videoId),
|
||||
thumbnailUrl: video.thumbnailUrl,
|
||||
channelId: video.channelId,
|
||||
channelTitle: video.channelTitle,
|
||||
channelUrl: channel?.url ?? `https://www.youtube.com/channel/${video.channelId}`,
|
||||
subscribers,
|
||||
views: video.views,
|
||||
publishedAt: video.publishedAt,
|
||||
durationSec: video.durationSec,
|
||||
isShort: item.viaShortsUrl || previous?.isShort || false,
|
||||
score,
|
||||
vidiqScore,
|
||||
foundOn: previous?.foundOn ?? item.page,
|
||||
firstSeenAt: previous?.firstSeenAt ?? nowIso,
|
||||
updatedAt: nowIso,
|
||||
});
|
||||
}
|
||||
|
||||
// Results first, `seen` last: if anything above throws, the ids are retried later.
|
||||
await db.putChannels(channelRecords);
|
||||
await db.putOutliers(outliers);
|
||||
await db.putSeen([
|
||||
...videos.map((video) => ({ videoId: video.videoId, checkedAt: nowIso, status: 'ok' })),
|
||||
...missingIds.map((videoId) => ({ videoId, checkedAt: nowIso, status: 'unavailable' })),
|
||||
]);
|
||||
return { checked: due.length, saved: outliers.length };
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
export function computeScore(views, subscribers) {
|
||||
if (subscribers === null || subscribers === undefined) return null;
|
||||
return Math.round((views / Math.max(subscribers, 1)) * 100) / 100;
|
||||
}
|
||||
|
||||
export function shouldSave({ views, score, vidiqScore }, { threshold, minViews }) {
|
||||
if (vidiqScore !== null && vidiqScore !== undefined) return true;
|
||||
return score !== null && score !== undefined && views >= minViews && score >= threshold;
|
||||
}
|
||||
Reference in New Issue
Block a user