volochay/outlier-saver-extension
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: store VidIQ cards in the service worker and drop the API layer

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
IgorVolochay
2026-10-25 12:08:02 +03:00
co-authored by Claude Sonnet 5.5
parent 1cf5270704
commit cdf8c08c7f
9 changed files with 18 additions and 934 deletions
Download Patch File Download Diff File Expand all files Collapse all files
manifest.json
+3 -4
View File
@@ -1,10 +1,9 @@
{
"manifest_version": 3,
"name": "Outlier Saver",
"version": "0.1.0",
"description": "Находит на YouTube видео с аномальными просмотрами и сохраняет их локально.",
"version": "0.2.0",
"description": "Сохраняет видео YouTube с высоким outlier score от VidIQ.",
"permissions": ["storage"],
"host_permissions": ["https://www.googleapis.com/youtube/v3/*"],
"background": { "service_worker": "src/background.js", "type": "module" },
"action": { "default_title": "Outlier Saver", "default_popup": "src/popup/popup.html" },
"options_ui": { "page": "src/options/options.html", "open_in_tab": true },
@@ -17,7 +16,7 @@
],
"web_accessible_resources": [
{
"resources": ["src/lib/video-id.js", "src/lib/vidiq.js"],
"resources": ["src/lib/*.js"],
"matches": ["https://www.youtube.com/*"]
}
]
src/background.js
+15 -129
View File
@@ -1,136 +1,22 @@
import { ApiError, fetchChannels, fetchVideos, nextQuotaReset } from './lib/api.js';
import * as db from './lib/db.js';
import { processBatch } from './lib/process.js';
import { getSettings, getStatus, setStatus } from './lib/storage.js';
import { isVideoId } from './lib/video-id.js';
import { getOutliers, putOutliers } from './lib/db.js';
import { buildRecord, sanitizeItem } from './lib/record.js';
import { getSettings } from './lib/storage.js';
const BATCH_SIZE = 50;
const FLUSH_DELAY_MS = 1000;
const MAX_QUEUE = 5000;
const PAGES = new Set(['home', 'watch', 'search', 'channel']);
const MAX_ITEMS_PER_MESSAGE = 200;
// videoId -> queue item. In memory only: if the worker is killed, the content script
// re-sends visible ids after the next navigation, and `seen` is written only after success.
const queue = new Map();
let flushTimer = null;
let flushing = false;
function sanitizeItem(raw) {
if (!raw || !isVideoId(raw.videoId)) return null;
return {
videoId: raw.videoId,
vidiqScore: typeof raw.vidiqScore === 'number' && Number.isFinite(raw.vidiqScore) ? raw.vidiqScore : null,
viaShortsUrl: raw.viaShortsUrl === true,
page: PAGES.has(raw.page) ? raw.page : 'home',
};
async function store(rawItems) {
const items = rawItems.slice(0, MAX_ITEMS_PER_MESSAGE).map(sanitizeItem).filter(Boolean);
if (items.length === 0) return;
const [settings, existing] = await Promise.all([getSettings(), getOutliers(items.map((item) => item.videoId))]);
const now = new Date();
const records = items.map((item) => buildRecord(item, existing.get(item.videoId), settings, now)).filter(Boolean);
await putOutliers(records);
}
function enqueue(rawItems) {
for (const raw of rawItems) {
const item = sanitizeItem(raw);
if (!item) continue;
const queued = queue.get(item.videoId);
if (queued) {
queued.vidiqScore = item.vidiqScore ?? queued.vidiqScore;
queued.viaShortsUrl = queued.viaShortsUrl || item.viaShortsUrl;
} else if (queue.size < MAX_QUEUE) {
queue.set(item.videoId, item);
}
}
scheduleFlush();
}
function requeue(items) {
for (const item of items) if (!queue.has(item.videoId)) queue.set(item.videoId, item);
}
function takeBatch() {
const batch = [];
for (const [videoId, item] of queue) {
if (batch.length >= BATCH_SIZE) break;
batch.push(item);
queue.delete(videoId);
}
return batch;
}
function scheduleFlush() {
if (flushTimer !== null || flushing || queue.size === 0) return;
flushTimer = setTimeout(flush, FLUSH_DELAY_MS);
}
async function updateBadge(status) {
if (status.state === 'ok') {
await chrome.action.setBadgeText({ text: '' });
return;
}
await chrome.action.setBadgeBackgroundColor({ color: '#d93025' });
await chrome.action.setBadgeText({ text: '!' });
}
async function reportStatus(status) {
await setStatus(status);
await updateBadge(status);
}
// Returns true when the flush loop may continue, false when collection must stop.
async function handleError(err, batch) {
if (err instanceof ApiError && err.kind === 'quota') {
requeue(batch);
await reportStatus({ state: 'quota', until: nextQuotaReset(), message: err.message });
return false;
}
if (err instanceof ApiError && err.kind === 'invalid-key') {
requeue(batch);
await reportStatus({ state: 'invalid-key', message: err.message });
return false;
}
console.warn(`Outlier Saver: dropped a batch of ${batch.length} ids`, err);
return true;
}
async function flush() {
flushTimer = null;
if (flushing) return;
flushing = true;
try {
while (queue.size > 0) {
const settings = await getSettings();
if (!settings.apiKey) {
await reportStatus({ state: 'no-key' });
return;
}
const status = await getStatus();
if (status.state === 'invalid-key') return;
if (status.state === 'quota' && Date.now() < status.until) return;
if (status.state !== 'ok') await reportStatus({ state: 'ok' });
const batch = takeBatch();
const apiKey = settings.apiKey;
const api = {
fetchVideos: (ids) => fetchVideos(ids, { apiKey }),
fetchChannels: (ids) => fetchChannels(ids, { apiKey }),
};
try {
await processBatch(batch, { settings, db, api });
} catch (err) {
if (!(await handleError(err, batch))) return;
}
}
} finally {
flushing = false;
}
}
// Messages are handled one after another so two batches never read-modify-write the same video at once.
let queue = Promise.resolve();
chrome.runtime.onMessage.addListener((message) => {
if (message?.type === 'videos-seen' && Array.isArray(message.items)) enqueue(message.items);
if (message?.type !== 'cards' || !Array.isArray(message.items)) return;
queue = queue.then(() => store(message.items)).catch((err) => console.warn('Outlier Saver: failed to store cards', err));
});
// A new API key clears a previous "no key / invalid key / quota" pause.
chrome.storage.onChanged.addListener((changes, area) => {
if (area !== 'local' || !changes.settings) return;
if (changes.settings.oldValue?.apiKey === changes.settings.newValue?.apiKey) return;
reportStatus({ state: 'ok' }).then(scheduleFlush);
});
getStatus().then(updateBadge);
src/lib/api.js
-173
View File
@@ -1,173 +0,0 @@
export class ApiError extends Error {
constructor(kind, message, status = null) {
super(message);
this.name = 'ApiError';
this.kind = kind;
this.status = status;
}
}
export function chunk(items, size) {
const batches = [];
for (let i = 0; i < items.length; i += size) batches.push(items.slice(i, i + size));
return batches;
}
export function parseIsoDuration(iso) {
const match = /^P(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?$/.exec(iso ?? '');
if (!match) return null;
const [, days = 0, hours = 0, minutes = 0, seconds = 0] = match;
return Number(days) * 86400 + Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds);
}
const THUMBNAIL_ORDER = ['maxres', 'standard', 'high', 'medium', 'default'];
export function pickThumbnail(thumbnails) {
for (const size of THUMBNAIL_ORDER) {
const url = thumbnails?.[size]?.url;
if (url) return url;
}
return null;
}
const QUOTA_REASONS = new Set(['quotaExceeded', 'dailyLimitExceeded']);
const KEY_REASONS = new Set([
'keyInvalid',
'API_KEY_INVALID',
'accessNotConfigured',
'ipRefererBlocked',
'API_KEY_HTTP_REFERRER_BLOCKED',
'API_KEY_SERVICE_BLOCKED',
'SERVICE_DISABLED',
]);
export function errorReasons(body) {
const error = body?.error;
return [...(error?.errors ?? []), ...(error?.details ?? [])].map((entry) => entry.reason).filter(Boolean);
}
export function classifyError(status, body) {
const reasons = errorReasons(body);
if (reasons.some((reason) => QUOTA_REASONS.has(reason))) return 'quota';
if (reasons.some((reason) => KEY_REASONS.has(reason))) return 'invalid-key';
if (status >= 500 || status === 429) return 'server';
return 'other';
}
// The Data API quota resets at midnight Pacific Time.
export function nextQuotaReset(now = new Date()) {
const parts = new Intl.DateTimeFormat('en-US', {
timeZone: 'America/Los_Angeles',
hourCycle: 'h23',
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
}).formatToParts(now);
const part = (type) => Number(parts.find((p) => p.type === type).value);
const sinceMidnightMs =
((part('hour') * 60 + part('minute')) * 60 + part('second')) * 1000 + now.getMilliseconds();
return now.getTime() - sinceMidnightMs + 24 * 60 * 60 * 1000;
}
const API_BASE = 'https://www.googleapis.com/youtube/v3';
const BATCH_SIZE = 50;
const RETRIES = 3;
const BASE_DELAY_MS = 500;
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
async function requestOnce(path, params, { apiKey, fetchImpl = fetch }) {
const url = `${API_BASE}/${path}?${new URLSearchParams({ ...params, key: apiKey })}`;
let response;
try {
response = await fetchImpl(url);
} catch (err) {
throw new ApiError('network', err.message);
}
if (response.ok) {
try {
return await response.json();
} catch {
throw new ApiError('server', 'Invalid JSON in API response', response.status);
}
}
let body = null;
try {
body = await response.json();
} catch {
// error responses are not always JSON
}
throw new ApiError(classifyError(response.status, body), body?.error?.message ?? `HTTP ${response.status}`, response.status);
}
async function request(path, params, opts) {
const sleep = opts.sleep ?? defaultSleep;
for (let attempt = 0; ; attempt += 1) {
try {
return await requestOnce(path, params, opts);
} catch (err) {
const retryable = err instanceof ApiError && (err.kind === 'network' || err.kind === 'server');
if (!retryable || attempt >= RETRIES) throw err;
await sleep(BASE_DELAY_MS * 2 ** attempt);
}
}
}
export function normalizeVideo(item) {
const { snippet = {}, statistics = {}, contentDetails = {} } = item;
return {
videoId: item.id,
title: snippet.title ?? '',
thumbnailUrl: pickThumbnail(snippet.thumbnails),
channelId: snippet.channelId,
channelTitle: snippet.channelTitle ?? '',
views: Number(statistics.viewCount ?? 0),
publishedAt: snippet.publishedAt ?? null,
durationSec: parseIsoDuration(contentDetails.duration),
};
}
export function normalizeChannel(item) {
const customUrl = item.snippet?.customUrl;
const stats = item.statistics ?? {};
return {
channelId: item.id,
title: item.snippet?.title ?? '',
url: customUrl?.startsWith('@')
? `https://www.youtube.com/${customUrl}`
: `https://www.youtube.com/channel/${item.id}`,
subscribers: stats.hiddenSubscriberCount || stats.subscriberCount === undefined ? null : Number(stats.subscriberCount),
};
}
export async function fetchVideos(ids, opts) {
const videos = [];
const found = new Set();
for (const batch of chunk(ids, BATCH_SIZE)) {
const data = await request(
'videos',
{ part: 'snippet,statistics,contentDetails', id: batch.join(',') },
opts,
);
for (const item of data.items ?? []) {
videos.push(normalizeVideo(item));
found.add(item.id);
}
}
return { videos, missingIds: ids.filter((id) => !found.has(id)) };
}
// maxResults is sent here only: videos.list rejects it together with `id`,
// while channels.list would otherwise default to 5 results.
export async function fetchChannels(ids, opts) {
const channels = [];
for (const batch of chunk(ids, BATCH_SIZE)) {
const data = await request(
'channels',
{ part: 'statistics,snippet', id: batch.join(','), maxResults: String(BATCH_SIZE) },
opts,
);
for (const item of data.items ?? []) channels.push(normalizeChannel(item));
}
return channels;
}
src/lib/process.js
-73
View File
@@ -1,73 +0,0 @@
import { computeScore, shouldSave } from './score.js';
export const RECHECK_MS = 24 * 60 * 60 * 1000;
const watchUrl = (videoId) => `https://www.youtube.com/watch?v=${videoId}`;
function isFresh(checkedAt, nowMs) {
return checkedAt !== undefined && nowMs - Date.parse(checkedAt) < RECHECK_MS;
}
export async function processBatch(items, { settings, db, api, now = new Date() }) {
const nowMs = now.getTime();
const nowIso = now.toISOString();
const byId = new Map(items.map((item) => [item.videoId, item]));
const seen = await db.getSeen([...byId.keys()]);
const due = [...byId.values()].filter(
(item) => item.vidiqScore !== null || !isFresh(seen.get(item.videoId)?.checkedAt, nowMs),
);
if (due.length === 0) return { checked: 0, saved: 0 };
const { videos, missingIds } = await api.fetchVideos(due.map((item) => item.videoId));
const channelIds = [...new Set(videos.map((video) => video.channelId))];
const cachedChannels = await db.getChannels(channelIds);
const staleIds = channelIds.filter((id) => !isFresh(cachedChannels.get(id)?.fetchedAt, nowMs));
const fetchedChannels = staleIds.length > 0 ? await api.fetchChannels(staleIds) : [];
const channelRecords = fetchedChannels.map((channel) => ({ ...channel, fetchedAt: nowIso }));
const channels = new Map(cachedChannels);
for (const record of channelRecords) channels.set(record.channelId, record);
const existing = await db.getOutliers(videos.map((video) => video.videoId));
const outliers = [];
for (const video of videos) {
const item = byId.get(video.videoId);
const previous = existing.get(video.videoId);
const channel = channels.get(video.channelId);
const subscribers = channel?.subscribers ?? null;
const score = computeScore(video.views, subscribers);
const vidiqScore = item.vidiqScore ?? previous?.vidiqScore ?? null;
if (!previous && !shouldSave({ views: video.views, score, vidiqScore }, settings)) continue;
outliers.push({
videoId: video.videoId,
title: video.title,
url: watchUrl(video.videoId),
thumbnailUrl: video.thumbnailUrl,
channelId: video.channelId,
channelTitle: video.channelTitle,
channelUrl: channel?.url ?? `https://www.youtube.com/channel/${video.channelId}`,
subscribers,
views: video.views,
publishedAt: video.publishedAt,
durationSec: video.durationSec,
isShort: item.viaShortsUrl || previous?.isShort || false,
score,
vidiqScore,
foundOn: previous?.foundOn ?? item.page,
firstSeenAt: previous?.firstSeenAt ?? nowIso,
updatedAt: nowIso,
});
}
// Results first, `seen` last: if anything above throws, the ids are retried later.
await db.putChannels(channelRecords);
await db.putOutliers(outliers);
await db.putSeen([
...videos.map((video) => ({ videoId: video.videoId, checkedAt: nowIso, status: 'ok' })),
...missingIds.map((videoId) => ({ videoId, checkedAt: nowIso, status: 'unavailable' })),
]);
return { checked: due.length, saved: outliers.length };
}
src/lib/score.js
-9
View File
@@ -1,9 +0,0 @@
export function computeScore(views, subscribers) {
if (subscribers === null || subscribers === undefined) return null;
return Math.round((views / Math.max(subscribers, 1)) * 100) / 100;
}
export function shouldSave({ views, score, vidiqScore }, { threshold, minViews }) {
if (vidiqScore !== null && vidiqScore !== undefined) return true;
return score !== null && score !== undefined && views >= minViews && score >= threshold;
}
test/api-helpers.test.js
-93
View File
@@ -1,93 +0,0 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
ApiError,
chunk,
classifyError,
errorReasons,
nextQuotaReset,
parseIsoDuration,
pickThumbnail,
} from '../src/lib/api.js';
test('chunk splits into batches and keeps the remainder', () => {
assert.deepEqual(chunk([1, 2, 3, 4, 5], 2), [[1, 2], [3, 4], [5]]);
assert.deepEqual(chunk([], 50), []);
assert.equal(chunk(Array.from({ length: 120 }, (_, i) => i), 50).map((b) => b.length).join(), '50,50,20');
});
test('parseIsoDuration converts ISO 8601 durations to seconds', () => {
assert.equal(parseIsoDuration('PT1H2M3S'), 3723);
assert.equal(parseIsoDuration('PT45S'), 45);
assert.equal(parseIsoDuration('PT10M'), 600);
assert.equal(parseIsoDuration('P1DT1H'), 90000);
assert.equal(parseIsoDuration('P0D'), 0);
assert.equal(parseIsoDuration('PT0S'), 0);
});
test('parseIsoDuration returns null for garbage', () => {
assert.equal(parseIsoDuration('garbage'), null);
assert.equal(parseIsoDuration(undefined), null);
assert.equal(parseIsoDuration(null), null);
});
test('pickThumbnail prefers the largest available size', () => {
const all = {
default: { url: 'd' },
medium: { url: 'm' },
high: { url: 'h' },
standard: { url: 's' },
maxres: { url: 'x' },
};
assert.equal(pickThumbnail(all), 'x');
delete all.maxres;
assert.equal(pickThumbnail(all), 's');
assert.equal(pickThumbnail({ default: { url: 'd' }, medium: { url: 'm' } }), 'm');
assert.equal(pickThumbnail({}), null);
assert.equal(pickThumbnail(undefined), null);
assert.equal(pickThumbnail(null), null);
});
test('errorReasons collects reasons from errors[] and details[]', () => {
const body = { error: { errors: [{ reason: 'badRequest' }], details: [{ reason: 'API_KEY_INVALID' }] } };
assert.deepEqual(errorReasons(body), ['badRequest', 'API_KEY_INVALID']);
assert.deepEqual(errorReasons(null), []);
assert.deepEqual(errorReasons({}), []);
});
test('classifyError recognises quota, key and server problems', () => {
assert.equal(classifyError(403, { error: { errors: [{ reason: 'quotaExceeded' }] } }), 'quota');
assert.equal(classifyError(403, { error: { errors: [{ reason: 'dailyLimitExceeded' }] } }), 'quota');
assert.equal(
classifyError(400, { error: { errors: [{ reason: 'badRequest' }], details: [{ reason: 'API_KEY_INVALID' }] } }),
'invalid-key',
);
assert.equal(classifyError(403, { error: { errors: [{ reason: 'accessNotConfigured' }] } }), 'invalid-key');
assert.equal(classifyError(500, null), 'server');
assert.equal(classifyError(503, {}), 'server');
assert.equal(classifyError(429, { error: { errors: [{ reason: 'rateLimitExceeded' }] } }), 'server');
assert.equal(classifyError(404, null), 'other');
});
test('ApiError carries its kind and status', () => {
const err = new ApiError('quota', 'out of quota', 403);
assert.ok(err instanceof Error);
assert.equal(err.name, 'ApiError');
assert.equal(err.kind, 'quota');
assert.equal(err.status, 403);
assert.equal(err.message, 'out of quota');
});
test('nextQuotaReset points at the next Pacific midnight (summer time)', () => {
const noon = new Date('2026-10-25T20:00:00Z'); // 13:00 PDT
assert.equal(new Date(nextQuotaReset(noon)).toISOString(), '2026-10-26T07:00:00.000Z');
const justBefore = new Date('2026-10-26T06:59:59Z'); // 23:59:59 PDT
assert.equal(new Date(nextQuotaReset(justBefore)).toISOString(), '2026-10-26T07:00:00.000Z');
const exactly = new Date('2026-10-26T07:00:00Z'); // 00:00:00 PDT
assert.equal(new Date(nextQuotaReset(exactly)).toISOString(), '2026-10-27T07:00:00.000Z');
});
test('nextQuotaReset points at the next Pacific midnight (winter time)', () => {
const noon = new Date('2026-12-15T12:00:00Z'); // 04:00 PST
assert.equal(new Date(nextQuotaReset(noon)).toISOString(), '2026-12-16T08:00:00.000Z');
});
test/api-requests.test.js
-165
View File
@@ -1,165 +0,0 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { ApiError, fetchChannels, fetchVideos } from '../src/lib/api.js';
const jsonResponse = (status, body) => ({ ok: status >= 200 && status < 300, status, json: async () => body });
const noSleep = async () => {};
const videoItem = (id) => ({
id,
snippet: {
title: `Title ${id}`,
channelId: 'UC1',
channelTitle: 'Chan',
publishedAt: '2026-10-01T00:00:00Z',
thumbnails: { high: { url: `https://i.ytimg.com/vi/${id}/hqdefault.jpg` } },
},
statistics: { viewCount: '12345' },
contentDetails: { duration: 'PT10M' },
});
test('fetchVideos splits ids into batches of 50 and normalizes items', async () => {
const ids = Array.from({ length: 120 }, (_, i) => `v${String(i).padStart(10, '0')}`);
const seen = [];
const fetchImpl = async (url) => {
const u = new URL(url);
const batch = u.searchParams.get('id').split(',');
seen.push({ path: u.pathname, size: batch.length, key: u.searchParams.get('key'), part: u.searchParams.get('part') });
return jsonResponse(200, { items: batch.map(videoItem) });
};
const { videos, missingIds } = await fetchVideos(ids, { apiKey: 'KEY', fetchImpl, sleep: noSleep });
assert.deepEqual(seen.map((s) => s.size), [50, 50, 20]);
assert.ok(seen.every((s) => s.path === '/youtube/v3/videos' && s.key === 'KEY'));
assert.ok(seen.every((s) => s.part === 'snippet,statistics,contentDetails'));
assert.equal(videos.length, 120);
assert.deepEqual(missingIds, []);
assert.deepEqual(videos[0], {
videoId: ids[0],
title: `Title ${ids[0]}`,
thumbnailUrl: `https://i.ytimg.com/vi/${ids[0]}/hqdefault.jpg`,
channelId: 'UC1',
channelTitle: 'Chan',
views: 12345,
publishedAt: '2026-10-01T00:00:00Z',
durationSec: 600,
});
});
test('fetchVideos reports ids the API did not return', async () => {
const fetchImpl = async (url) => {
const batch = new URL(url).searchParams.get('id').split(',');
return jsonResponse(200, { items: batch.filter((id) => id !== 'missing0001').map(videoItem) });
};
const { videos, missingIds } = await fetchVideos(['abcdefghijk', 'missing0001'], {
apiKey: 'K',
fetchImpl,
sleep: noSleep,
});
assert.equal(videos.length, 1);
assert.deepEqual(missingIds, ['missing0001']);
});
test('requests are retried on 5xx with exponential backoff', async () => {
const delays = [];
let calls = 0;
const fetchImpl = async () => {
calls += 1;
return calls < 3 ? jsonResponse(503, {}) : jsonResponse(200, { items: [videoItem('abcdefghijk')] });
};
const { videos } = await fetchVideos(['abcdefghijk'], {
apiKey: 'K',
fetchImpl,
sleep: async (ms) => delays.push(ms),
});
assert.equal(videos.length, 1);
assert.equal(calls, 3);
assert.deepEqual(delays, [500, 1000]);
});
test('requests give up after three retries', async () => {
let calls = 0;
const fetchImpl = async () => {
calls += 1;
return jsonResponse(500, {});
};
await assert.rejects(
fetchVideos(['abcdefghijk'], { apiKey: 'K', fetchImpl, sleep: noSleep }),
(err) => err instanceof ApiError && err.kind === 'server' && err.status === 500,
);
assert.equal(calls, 4);
});
test('network failures are retried and reported as kind network', async () => {
let calls = 0;
const fetchImpl = async () => {
calls += 1;
throw new TypeError('Failed to fetch');
};
await assert.rejects(
fetchVideos(['abcdefghijk'], { apiKey: 'K', fetchImpl, sleep: noSleep }),
(err) => err instanceof ApiError && err.kind === 'network',
);
assert.equal(calls, 4);
});
test('quota errors are not retried', async () => {
let calls = 0;
const fetchImpl = async () => {
calls += 1;
return jsonResponse(403, { error: { message: 'quota', errors: [{ reason: 'quotaExceeded' }] } });
};
await assert.rejects(
fetchVideos(['abcdefghijk'], { apiKey: 'K', fetchImpl, sleep: noSleep }),
(err) => err instanceof ApiError && err.kind === 'quota' && err.message === 'quota',
);
assert.equal(calls, 1);
});
test('invalid key errors are not retried', async () => {
let calls = 0;
const fetchImpl = async () => {
calls += 1;
return jsonResponse(400, {
error: { message: 'API key not valid.', errors: [{ reason: 'badRequest' }], details: [{ reason: 'API_KEY_INVALID' }] },
});
};
await assert.rejects(
fetchVideos(['abcdefghijk'], { apiKey: 'bad', fetchImpl, sleep: noSleep }),
(err) => err instanceof ApiError && err.kind === 'invalid-key',
);
assert.equal(calls, 1);
});
test('fetchChannels normalizes subscriber counts and channel urls', async () => {
let params;
const fetchImpl = async (url) => {
params = new URL(url).searchParams;
return jsonResponse(200, {
items: [
{ id: 'UC1', snippet: { title: 'One', customUrl: '@one' }, statistics: { subscriberCount: '1500' } },
{ id: 'UC2', snippet: { title: 'Two' }, statistics: { hiddenSubscriberCount: true } },
{ id: 'UC3', snippet: { title: 'Three', customUrl: 'legacyname' }, statistics: { subscriberCount: '7' } },
],
});
};
const channels = await fetchChannels(['UC1', 'UC2', 'UC3'], { apiKey: 'K', fetchImpl, sleep: noSleep });
assert.equal(params.get('part'), 'statistics,snippet');
assert.equal(params.get('maxResults'), '50');
assert.equal(params.get('id'), 'UC1,UC2,UC3');
assert.deepEqual(channels, [
{ channelId: 'UC1', title: 'One', url: 'https://www.youtube.com/@one', subscribers: 1500 },
{ channelId: 'UC2', title: 'Two', url: 'https://www.youtube.com/channel/UC2', subscribers: null },
{ channelId: 'UC3', title: 'Three', url: 'https://www.youtube.com/channel/UC3', subscribers: 7 },
]);
});
test('fetchChannels of an empty list makes no request', async () => {
const fetchImpl = async () => {
throw new Error('should not be called');
};
assert.deepEqual(await fetchChannels([], { apiKey: 'K', fetchImpl, sleep: noSleep }), []);
});
test/process.test.js
-244
View File
@@ -1,244 +0,0 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { processBatch } from '../src/lib/process.js';
const NOW = new Date('2026-10-25T10:00:00.000Z');
const NOW_ISO = NOW.toISOString();
const HOUR = 3600 * 1000;
const ago = (hours) => new Date(NOW.getTime() - hours * HOUR).toISOString();
const SETTINGS = { threshold: 5, minViews: 10000 };
const ID = 'vid00000001';
function memoryDb() {
const stores = { seen: new Map(), channels: new Map(), outliers: new Map() };
const getter = (name) => async (keys) =>
new Map(keys.filter((key) => stores[name].has(key)).map((key) => [key, stores[name].get(key)]));
const putter = (name, keyField) => async (records) => {
for (const record of records) stores[name].set(record[keyField], record);
};
return {
stores,
getSeen: getter('seen'),
getChannels: getter('channels'),
getOutliers: getter('outliers'),
putSeen: putter('seen', 'videoId'),
putChannels: putter('channels', 'channelId'),
putOutliers: putter('outliers', 'videoId'),
};
}
function fakeApi({ videos = [], channels = [] } = {}) {
const calls = { videos: [], channels: [] };
return {
calls,
fetchVideos: async (ids) => {
calls.videos.push(ids);
const found = videos.filter((v) => ids.includes(v.videoId));
return { videos: found, missingIds: ids.filter((id) => !found.some((v) => v.videoId === id)) };
},
fetchChannels: async (ids) => {
calls.channels.push(ids);
return channels.filter((c) => ids.includes(c.channelId));
},
};
}
const item = (videoId = ID, over = {}) => ({ videoId, vidiqScore: null, viaShortsUrl: false, page: 'home', ...over });
const video = (videoId = ID, over = {}) => ({
videoId,
title: 'Title',
thumbnailUrl: 'https://i.ytimg.com/t.jpg',
channelId: 'UC1',
channelTitle: 'Chan',
views: 60000,
publishedAt: '2026-10-01T00:00:00Z',
durationSec: 600,
...over,
});
const channel = (channelId, subscribers) => ({
channelId,
title: 'Chan',
url: 'https://www.youtube.com/@chan',
subscribers,
});
const run = (items, db, api) => processBatch(items, { settings: SETTINGS, db, api, now: NOW });
test('saves a video whose score reaches the threshold', async () => {
const db = memoryDb();
const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] });
const result = await run([item(ID, { page: 'search' })], db, api);
assert.deepEqual(result, { checked: 1, saved: 1 });
assert.deepEqual(db.stores.outliers.get(ID), {
videoId: ID,
title: 'Title',
url: `https://www.youtube.com/watch?v=${ID}`,
thumbnailUrl: 'https://i.ytimg.com/t.jpg',
channelId: 'UC1',
channelTitle: 'Chan',
channelUrl: 'https://www.youtube.com/@chan',
subscribers: 10000,
views: 60000,
publishedAt: '2026-10-01T00:00:00Z',
durationSec: 600,
isShort: false,
score: 6,
vidiqScore: null,
foundOn: 'search',
firstSeenAt: NOW_ISO,
updatedAt: NOW_ISO,
});
assert.deepEqual(db.stores.seen.get(ID), { videoId: ID, checkedAt: NOW_ISO, status: 'ok' });
assert.deepEqual(db.stores.channels.get('UC1'), { ...channel('UC1', 10000), fetchedAt: NOW_ISO });
});
test('marks a below-threshold video as seen without saving it', async () => {
const db = memoryDb();
const api = fakeApi({ videos: [video()], channels: [channel('UC1', 100000)] });
const result = await run([item()], db, api);
assert.deepEqual(result, { checked: 1, saved: 0 });
assert.equal(db.stores.outliers.size, 0);
assert.equal(db.stores.seen.get(ID).status, 'ok');
});
test('does not save a video below minViews even with a huge score', async () => {
const db = memoryDb();
const api = fakeApi({ videos: [video(ID, { views: 9000 })], channels: [channel('UC1', 100)] });
await run([item()], db, api);
assert.equal(db.stores.outliers.size, 0);
});
test('saves a VidIQ-flagged video even when subscribers are hidden', async () => {
const db = memoryDb();
const api = fakeApi({ videos: [video()], channels: [channel('UC1', null)] });
await run([item(ID, { vidiqScore: 7.5 })], db, api);
const saved = db.stores.outliers.get(ID);
assert.equal(saved.score, null);
assert.equal(saved.subscribers, null);
assert.equal(saved.vidiqScore, 7.5);
});
test('does not save a video with hidden subscribers and no VidIQ score', async () => {
const db = memoryDb();
const api = fakeApi({ videos: [video()], channels: [channel('UC1', null)] });
await run([item()], db, api);
assert.equal(db.stores.outliers.size, 0);
});
test('treats a channel missing from the API response as unknown subscribers', async () => {
const db = memoryDb();
const api = fakeApi({ videos: [video()], channels: [] });
await run([item(ID, { vidiqScore: 4 })], db, api);
assert.equal(db.stores.outliers.get(ID).subscribers, null);
assert.equal(db.stores.outliers.get(ID).channelUrl, 'https://www.youtube.com/channel/UC1');
});
test('skips ids checked less than 24 hours ago', async () => {
const db = memoryDb();
db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(1), status: 'ok' });
const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] });
const result = await run([item()], db, api);
assert.deepEqual(result, { checked: 0, saved: 0 });
assert.equal(api.calls.videos.length, 0);
});
test('rechecks ids older than 24 hours', async () => {
const db = memoryDb();
db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(25), status: 'ok' });
const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] });
const result = await run([item()], db, api);
assert.deepEqual(result, { checked: 1, saved: 1 });
});
test('an item with a VidIQ score bypasses a fresh seen record', async () => {
const db = memoryDb();
db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(1), status: 'ok' });
const api = fakeApi({ videos: [video()], channels: [channel('UC1', 1000000)] });
const result = await run([item(ID, { vidiqScore: 4 })], db, api);
assert.deepEqual(result, { checked: 1, saved: 1 });
assert.equal(db.stores.outliers.get(ID).vidiqScore, 4);
});
test('a repeat sighting refreshes stats and keeps the original metadata', async () => {
const db = memoryDb();
db.stores.outliers.set(ID, {
videoId: ID,
firstSeenAt: '2026-10-20T00:00:00.000Z',
foundOn: 'home',
isShort: true,
vidiqScore: 9,
});
db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(30), status: 'ok' });
const api = fakeApi({ videos: [video(ID, { views: 70000 })], channels: [channel('UC1', 10000)] });
await run([item(ID, { page: 'search', viaShortsUrl: false })], db, api);
const saved = db.stores.outliers.get(ID);
assert.equal(saved.firstSeenAt, '2026-10-20T00:00:00.000Z');
assert.equal(saved.foundOn, 'home');
assert.equal(saved.isShort, true);
assert.equal(saved.vidiqScore, 9);
assert.equal(saved.views, 70000);
assert.equal(saved.score, 7);
assert.equal(saved.updatedAt, NOW_ISO);
});
test('an already saved video is updated even when its score fell below the threshold', async () => {
const db = memoryDb();
db.stores.outliers.set(ID, { videoId: ID, firstSeenAt: ago(100), foundOn: 'home', isShort: false, vidiqScore: null });
db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(30), status: 'ok' });
const api = fakeApi({ videos: [video()], channels: [channel('UC1', 1000000)] });
const result = await run([item()], db, api);
assert.deepEqual(result, { checked: 1, saved: 1 });
assert.equal(db.stores.outliers.get(ID).score, 0.06);
});
test('a short found via a /shorts/ link is flagged', async () => {
const db = memoryDb();
const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] });
await run([item(ID, { viaShortsUrl: true })], db, api);
assert.equal(db.stores.outliers.get(ID).isShort, true);
});
test('records unavailable videos in seen and does not save them', async () => {
const db = memoryDb();
const api = fakeApi({ videos: [], channels: [] });
const result = await run([item()], db, api);
assert.deepEqual(result, { checked: 1, saved: 0 });
assert.deepEqual(db.stores.seen.get(ID), { videoId: ID, checkedAt: NOW_ISO, status: 'unavailable' });
assert.equal(db.stores.outliers.size, 0);
});
test('reuses a cached channel younger than 24 hours', async () => {
const db = memoryDb();
db.stores.channels.set('UC1', { ...channel('UC1', 10000), fetchedAt: ago(1) });
const api = fakeApi({ videos: [video()], channels: [] });
const result = await run([item()], db, api);
assert.equal(api.calls.channels.length, 0);
assert.deepEqual(result, { checked: 1, saved: 1 });
});
test('refetches a cached channel older than 24 hours', async () => {
const db = memoryDb();
db.stores.channels.set('UC1', { ...channel('UC1', 10000), fetchedAt: ago(25) });
const api = fakeApi({ videos: [video()], channels: [channel('UC1', 20000)] });
const result = await run([item()], db, api);
assert.deepEqual(api.calls.channels, [['UC1']]);
assert.equal(db.stores.channels.get('UC1').subscribers, 20000);
assert.deepEqual(result, { checked: 1, saved: 0 }); // score 3 < 5
});
test('an API failure writes nothing and does not mark ids as seen', async () => {
const db = memoryDb();
const api = {
...fakeApi(),
fetchVideos: async () => {
throw new Error('boom');
},
};
await assert.rejects(run([item()], db, api), /boom/);
assert.equal(db.stores.seen.size, 0);
assert.equal(db.stores.outliers.size, 0);
assert.equal(db.stores.channels.size, 0);
});
test/score.test.js
-44
View File
@@ -1,44 +0,0 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { computeScore, shouldSave } from '../src/lib/score.js';
test('computeScore divides views by subscribers', () => {
assert.equal(computeScore(100000, 20000), 5);
assert.equal(computeScore(2500, 1000), 2.5);
});
test('computeScore rounds to two decimals', () => {
assert.equal(computeScore(10000, 3000), 3.33);
});
test('computeScore never divides by zero', () => {
assert.equal(computeScore(5000, 0), 5000);
});
test('computeScore returns null when subscribers are unknown', () => {
assert.equal(computeScore(5000, null), null);
assert.equal(computeScore(5000, undefined), null);
});
const settings = { threshold: 5, minViews: 10000 };
test('shouldSave accepts a video exactly at both limits', () => {
assert.equal(shouldSave({ views: 10000, score: 5, vidiqScore: null }, settings), true);
});
test('shouldSave rejects a score below the threshold', () => {
assert.equal(shouldSave({ views: 50000, score: 4.99, vidiqScore: null }, settings), false);
});
test('shouldSave rejects views below the minimum even with a huge score', () => {
assert.equal(shouldSave({ views: 9999, score: 50, vidiqScore: null }, settings), false);
});
test('shouldSave rejects a null score without a VidIQ score', () => {
assert.equal(shouldSave({ views: 500000, score: null, vidiqScore: null }, settings), false);
});
test('shouldSave accepts any video that has a VidIQ score', () => {
assert.equal(shouldSave({ views: 100, score: null, vidiqScore: 7.5 }, settings), true);
assert.equal(shouldSave({ views: 100, score: 0.01, vidiqScore: 3 }, settings), true);
});