From cdf8c08c7f7ba6c9fd63fbcf916c3de2bd0adfd5 Mon Sep 17 00:00:00 2001 From: IgorVolochay Date: Sun, 25 Oct 2026 12:08:02 +0300 Subject: [PATCH] refactor: store VidIQ cards in the service worker and drop the API layer Co-Authored-By: Claude Sonnet 5.5 --- manifest.json | 7 +- src/background.js | 144 +++------------------- src/lib/api.js | 173 --------------------------- src/lib/process.js | 73 ------------ src/lib/score.js | 9 -- test/api-helpers.test.js | 93 --------------- test/api-requests.test.js | 165 -------------------------- test/process.test.js | 244 -------------------------------------- test/score.test.js | 44 ------- 9 files changed, 18 insertions(+), 934 deletions(-) delete mode 100644 src/lib/api.js delete mode 100644 src/lib/process.js delete mode 100644 src/lib/score.js delete mode 100644 test/api-helpers.test.js delete mode 100644 test/api-requests.test.js delete mode 100644 test/process.test.js delete mode 100644 test/score.test.js diff --git a/manifest.json b/manifest.json index c3416c6..adc56e1 100644 --- a/manifest.json +++ b/manifest.json @@ -1,10 +1,9 @@ { "manifest_version": 3, "name": "Outlier Saver", - "version": "0.1.0", - "description": "Находит на YouTube видео с аномальными просмотрами и сохраняет их локально.", + "version": "0.2.0", + "description": "Сохраняет видео YouTube с высоким outlier score от VidIQ.", "permissions": ["storage"], - "host_permissions": ["https://www.googleapis.com/youtube/v3/*"], "background": { "service_worker": "src/background.js", "type": "module" }, "action": { "default_title": "Outlier Saver", "default_popup": "src/popup/popup.html" }, "options_ui": { "page": "src/options/options.html", "open_in_tab": true }, @@ -17,7 +16,7 @@ ], "web_accessible_resources": [ { - "resources": ["src/lib/video-id.js", "src/lib/vidiq.js"], + "resources": ["src/lib/*.js"], "matches": ["https://www.youtube.com/*"] } ] diff --git a/src/background.js b/src/background.js index 671b872..38caf3c 100644 --- a/src/background.js +++ b/src/background.js @@ -1,136 +1,22 @@ -import { ApiError, fetchChannels, fetchVideos, nextQuotaReset } from './lib/api.js'; -import * as db from './lib/db.js'; -import { processBatch } from './lib/process.js'; -import { getSettings, getStatus, setStatus } from './lib/storage.js'; -import { isVideoId } from './lib/video-id.js'; +import { getOutliers, putOutliers } from './lib/db.js'; +import { buildRecord, sanitizeItem } from './lib/record.js'; +import { getSettings } from './lib/storage.js'; -const BATCH_SIZE = 50; -const FLUSH_DELAY_MS = 1000; -const MAX_QUEUE = 5000; -const PAGES = new Set(['home', 'watch', 'search', 'channel']); +const MAX_ITEMS_PER_MESSAGE = 200; -// videoId -> queue item. In memory only: if the worker is killed, the content script -// re-sends visible ids after the next navigation, and `seen` is written only after success. -const queue = new Map(); -let flushTimer = null; -let flushing = false; - -function sanitizeItem(raw) { - if (!raw || !isVideoId(raw.videoId)) return null; - return { - videoId: raw.videoId, - vidiqScore: typeof raw.vidiqScore === 'number' && Number.isFinite(raw.vidiqScore) ? raw.vidiqScore : null, - viaShortsUrl: raw.viaShortsUrl === true, - page: PAGES.has(raw.page) ? raw.page : 'home', - }; +async function store(rawItems) { + const items = rawItems.slice(0, MAX_ITEMS_PER_MESSAGE).map(sanitizeItem).filter(Boolean); + if (items.length === 0) return; + const [settings, existing] = await Promise.all([getSettings(), getOutliers(items.map((item) => item.videoId))]); + const now = new Date(); + const records = items.map((item) => buildRecord(item, existing.get(item.videoId), settings, now)).filter(Boolean); + await putOutliers(records); } -function enqueue(rawItems) { - for (const raw of rawItems) { - const item = sanitizeItem(raw); - if (!item) continue; - const queued = queue.get(item.videoId); - if (queued) { - queued.vidiqScore = item.vidiqScore ?? queued.vidiqScore; - queued.viaShortsUrl = queued.viaShortsUrl || item.viaShortsUrl; - } else if (queue.size < MAX_QUEUE) { - queue.set(item.videoId, item); - } - } - scheduleFlush(); -} - -function requeue(items) { - for (const item of items) if (!queue.has(item.videoId)) queue.set(item.videoId, item); -} - -function takeBatch() { - const batch = []; - for (const [videoId, item] of queue) { - if (batch.length >= BATCH_SIZE) break; - batch.push(item); - queue.delete(videoId); - } - return batch; -} - -function scheduleFlush() { - if (flushTimer !== null || flushing || queue.size === 0) return; - flushTimer = setTimeout(flush, FLUSH_DELAY_MS); -} - -async function updateBadge(status) { - if (status.state === 'ok') { - await chrome.action.setBadgeText({ text: '' }); - return; - } - await chrome.action.setBadgeBackgroundColor({ color: '#d93025' }); - await chrome.action.setBadgeText({ text: '!' }); -} - -async function reportStatus(status) { - await setStatus(status); - await updateBadge(status); -} - -// Returns true when the flush loop may continue, false when collection must stop. -async function handleError(err, batch) { - if (err instanceof ApiError && err.kind === 'quota') { - requeue(batch); - await reportStatus({ state: 'quota', until: nextQuotaReset(), message: err.message }); - return false; - } - if (err instanceof ApiError && err.kind === 'invalid-key') { - requeue(batch); - await reportStatus({ state: 'invalid-key', message: err.message }); - return false; - } - console.warn(`Outlier Saver: dropped a batch of ${batch.length} ids`, err); - return true; -} - -async function flush() { - flushTimer = null; - if (flushing) return; - flushing = true; - try { - while (queue.size > 0) { - const settings = await getSettings(); - if (!settings.apiKey) { - await reportStatus({ state: 'no-key' }); - return; - } - const status = await getStatus(); - if (status.state === 'invalid-key') return; - if (status.state === 'quota' && Date.now() < status.until) return; - if (status.state !== 'ok') await reportStatus({ state: 'ok' }); - - const batch = takeBatch(); - const apiKey = settings.apiKey; - const api = { - fetchVideos: (ids) => fetchVideos(ids, { apiKey }), - fetchChannels: (ids) => fetchChannels(ids, { apiKey }), - }; - try { - await processBatch(batch, { settings, db, api }); - } catch (err) { - if (!(await handleError(err, batch))) return; - } - } - } finally { - flushing = false; - } -} +// Messages are handled one after another so two batches never read-modify-write the same video at once. +let queue = Promise.resolve(); chrome.runtime.onMessage.addListener((message) => { - if (message?.type === 'videos-seen' && Array.isArray(message.items)) enqueue(message.items); + if (message?.type !== 'cards' || !Array.isArray(message.items)) return; + queue = queue.then(() => store(message.items)).catch((err) => console.warn('Outlier Saver: failed to store cards', err)); }); - -// A new API key clears a previous "no key / invalid key / quota" pause. -chrome.storage.onChanged.addListener((changes, area) => { - if (area !== 'local' || !changes.settings) return; - if (changes.settings.oldValue?.apiKey === changes.settings.newValue?.apiKey) return; - reportStatus({ state: 'ok' }).then(scheduleFlush); -}); - -getStatus().then(updateBadge); diff --git a/src/lib/api.js b/src/lib/api.js deleted file mode 100644 index 37e4afd..0000000 --- a/src/lib/api.js +++ /dev/null @@ -1,173 +0,0 @@ -export class ApiError extends Error { - constructor(kind, message, status = null) { - super(message); - this.name = 'ApiError'; - this.kind = kind; - this.status = status; - } -} - -export function chunk(items, size) { - const batches = []; - for (let i = 0; i < items.length; i += size) batches.push(items.slice(i, i + size)); - return batches; -} - -export function parseIsoDuration(iso) { - const match = /^P(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?$/.exec(iso ?? ''); - if (!match) return null; - const [, days = 0, hours = 0, minutes = 0, seconds = 0] = match; - return Number(days) * 86400 + Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds); -} - -const THUMBNAIL_ORDER = ['maxres', 'standard', 'high', 'medium', 'default']; - -export function pickThumbnail(thumbnails) { - for (const size of THUMBNAIL_ORDER) { - const url = thumbnails?.[size]?.url; - if (url) return url; - } - return null; -} - -const QUOTA_REASONS = new Set(['quotaExceeded', 'dailyLimitExceeded']); -const KEY_REASONS = new Set([ - 'keyInvalid', - 'API_KEY_INVALID', - 'accessNotConfigured', - 'ipRefererBlocked', - 'API_KEY_HTTP_REFERRER_BLOCKED', - 'API_KEY_SERVICE_BLOCKED', - 'SERVICE_DISABLED', -]); - -export function errorReasons(body) { - const error = body?.error; - return [...(error?.errors ?? []), ...(error?.details ?? [])].map((entry) => entry.reason).filter(Boolean); -} - -export function classifyError(status, body) { - const reasons = errorReasons(body); - if (reasons.some((reason) => QUOTA_REASONS.has(reason))) return 'quota'; - if (reasons.some((reason) => KEY_REASONS.has(reason))) return 'invalid-key'; - if (status >= 500 || status === 429) return 'server'; - return 'other'; -} - -// The Data API quota resets at midnight Pacific Time. -export function nextQuotaReset(now = new Date()) { - const parts = new Intl.DateTimeFormat('en-US', { - timeZone: 'America/Los_Angeles', - hourCycle: 'h23', - hour: 'numeric', - minute: 'numeric', - second: 'numeric', - }).formatToParts(now); - const part = (type) => Number(parts.find((p) => p.type === type).value); - const sinceMidnightMs = - ((part('hour') * 60 + part('minute')) * 60 + part('second')) * 1000 + now.getMilliseconds(); - return now.getTime() - sinceMidnightMs + 24 * 60 * 60 * 1000; -} - -const API_BASE = 'https://www.googleapis.com/youtube/v3'; -const BATCH_SIZE = 50; -const RETRIES = 3; -const BASE_DELAY_MS = 500; - -const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); - -async function requestOnce(path, params, { apiKey, fetchImpl = fetch }) { - const url = `${API_BASE}/${path}?${new URLSearchParams({ ...params, key: apiKey })}`; - let response; - try { - response = await fetchImpl(url); - } catch (err) { - throw new ApiError('network', err.message); - } - if (response.ok) { - try { - return await response.json(); - } catch { - throw new ApiError('server', 'Invalid JSON in API response', response.status); - } - } - let body = null; - try { - body = await response.json(); - } catch { - // error responses are not always JSON - } - throw new ApiError(classifyError(response.status, body), body?.error?.message ?? `HTTP ${response.status}`, response.status); -} - -async function request(path, params, opts) { - const sleep = opts.sleep ?? defaultSleep; - for (let attempt = 0; ; attempt += 1) { - try { - return await requestOnce(path, params, opts); - } catch (err) { - const retryable = err instanceof ApiError && (err.kind === 'network' || err.kind === 'server'); - if (!retryable || attempt >= RETRIES) throw err; - await sleep(BASE_DELAY_MS * 2 ** attempt); - } - } -} - -export function normalizeVideo(item) { - const { snippet = {}, statistics = {}, contentDetails = {} } = item; - return { - videoId: item.id, - title: snippet.title ?? '', - thumbnailUrl: pickThumbnail(snippet.thumbnails), - channelId: snippet.channelId, - channelTitle: snippet.channelTitle ?? '', - views: Number(statistics.viewCount ?? 0), - publishedAt: snippet.publishedAt ?? null, - durationSec: parseIsoDuration(contentDetails.duration), - }; -} - -export function normalizeChannel(item) { - const customUrl = item.snippet?.customUrl; - const stats = item.statistics ?? {}; - return { - channelId: item.id, - title: item.snippet?.title ?? '', - url: customUrl?.startsWith('@') - ? `https://www.youtube.com/${customUrl}` - : `https://www.youtube.com/channel/${item.id}`, - subscribers: stats.hiddenSubscriberCount || stats.subscriberCount === undefined ? null : Number(stats.subscriberCount), - }; -} - -export async function fetchVideos(ids, opts) { - const videos = []; - const found = new Set(); - for (const batch of chunk(ids, BATCH_SIZE)) { - const data = await request( - 'videos', - { part: 'snippet,statistics,contentDetails', id: batch.join(',') }, - opts, - ); - for (const item of data.items ?? []) { - videos.push(normalizeVideo(item)); - found.add(item.id); - } - } - return { videos, missingIds: ids.filter((id) => !found.has(id)) }; -} - -// maxResults is sent here only: videos.list rejects it together with `id`, -// while channels.list would otherwise default to 5 results. -export async function fetchChannels(ids, opts) { - const channels = []; - for (const batch of chunk(ids, BATCH_SIZE)) { - const data = await request( - 'channels', - { part: 'statistics,snippet', id: batch.join(','), maxResults: String(BATCH_SIZE) }, - opts, - ); - for (const item of data.items ?? []) channels.push(normalizeChannel(item)); - } - return channels; -} diff --git a/src/lib/process.js b/src/lib/process.js deleted file mode 100644 index f47e68c..0000000 --- a/src/lib/process.js +++ /dev/null @@ -1,73 +0,0 @@ -import { computeScore, shouldSave } from './score.js'; - -export const RECHECK_MS = 24 * 60 * 60 * 1000; - -const watchUrl = (videoId) => `https://www.youtube.com/watch?v=${videoId}`; - -function isFresh(checkedAt, nowMs) { - return checkedAt !== undefined && nowMs - Date.parse(checkedAt) < RECHECK_MS; -} - -export async function processBatch(items, { settings, db, api, now = new Date() }) { - const nowMs = now.getTime(); - const nowIso = now.toISOString(); - const byId = new Map(items.map((item) => [item.videoId, item])); - - const seen = await db.getSeen([...byId.keys()]); - const due = [...byId.values()].filter( - (item) => item.vidiqScore !== null || !isFresh(seen.get(item.videoId)?.checkedAt, nowMs), - ); - if (due.length === 0) return { checked: 0, saved: 0 }; - - const { videos, missingIds } = await api.fetchVideos(due.map((item) => item.videoId)); - - const channelIds = [...new Set(videos.map((video) => video.channelId))]; - const cachedChannels = await db.getChannels(channelIds); - const staleIds = channelIds.filter((id) => !isFresh(cachedChannels.get(id)?.fetchedAt, nowMs)); - const fetchedChannels = staleIds.length > 0 ? await api.fetchChannels(staleIds) : []; - const channelRecords = fetchedChannels.map((channel) => ({ ...channel, fetchedAt: nowIso })); - const channels = new Map(cachedChannels); - for (const record of channelRecords) channels.set(record.channelId, record); - - const existing = await db.getOutliers(videos.map((video) => video.videoId)); - const outliers = []; - for (const video of videos) { - const item = byId.get(video.videoId); - const previous = existing.get(video.videoId); - const channel = channels.get(video.channelId); - const subscribers = channel?.subscribers ?? null; - const score = computeScore(video.views, subscribers); - const vidiqScore = item.vidiqScore ?? previous?.vidiqScore ?? null; - - if (!previous && !shouldSave({ views: video.views, score, vidiqScore }, settings)) continue; - - outliers.push({ - videoId: video.videoId, - title: video.title, - url: watchUrl(video.videoId), - thumbnailUrl: video.thumbnailUrl, - channelId: video.channelId, - channelTitle: video.channelTitle, - channelUrl: channel?.url ?? `https://www.youtube.com/channel/${video.channelId}`, - subscribers, - views: video.views, - publishedAt: video.publishedAt, - durationSec: video.durationSec, - isShort: item.viaShortsUrl || previous?.isShort || false, - score, - vidiqScore, - foundOn: previous?.foundOn ?? item.page, - firstSeenAt: previous?.firstSeenAt ?? nowIso, - updatedAt: nowIso, - }); - } - - // Results first, `seen` last: if anything above throws, the ids are retried later. - await db.putChannels(channelRecords); - await db.putOutliers(outliers); - await db.putSeen([ - ...videos.map((video) => ({ videoId: video.videoId, checkedAt: nowIso, status: 'ok' })), - ...missingIds.map((videoId) => ({ videoId, checkedAt: nowIso, status: 'unavailable' })), - ]); - return { checked: due.length, saved: outliers.length }; -} diff --git a/src/lib/score.js b/src/lib/score.js deleted file mode 100644 index 1c57bf5..0000000 --- a/src/lib/score.js +++ /dev/null @@ -1,9 +0,0 @@ -export function computeScore(views, subscribers) { - if (subscribers === null || subscribers === undefined) return null; - return Math.round((views / Math.max(subscribers, 1)) * 100) / 100; -} - -export function shouldSave({ views, score, vidiqScore }, { threshold, minViews }) { - if (vidiqScore !== null && vidiqScore !== undefined) return true; - return score !== null && score !== undefined && views >= minViews && score >= threshold; -} diff --git a/test/api-helpers.test.js b/test/api-helpers.test.js deleted file mode 100644 index 24971ad..0000000 --- a/test/api-helpers.test.js +++ /dev/null @@ -1,93 +0,0 @@ -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { - ApiError, - chunk, - classifyError, - errorReasons, - nextQuotaReset, - parseIsoDuration, - pickThumbnail, -} from '../src/lib/api.js'; - -test('chunk splits into batches and keeps the remainder', () => { - assert.deepEqual(chunk([1, 2, 3, 4, 5], 2), [[1, 2], [3, 4], [5]]); - assert.deepEqual(chunk([], 50), []); - assert.equal(chunk(Array.from({ length: 120 }, (_, i) => i), 50).map((b) => b.length).join(), '50,50,20'); -}); - -test('parseIsoDuration converts ISO 8601 durations to seconds', () => { - assert.equal(parseIsoDuration('PT1H2M3S'), 3723); - assert.equal(parseIsoDuration('PT45S'), 45); - assert.equal(parseIsoDuration('PT10M'), 600); - assert.equal(parseIsoDuration('P1DT1H'), 90000); - assert.equal(parseIsoDuration('P0D'), 0); - assert.equal(parseIsoDuration('PT0S'), 0); -}); - -test('parseIsoDuration returns null for garbage', () => { - assert.equal(parseIsoDuration('garbage'), null); - assert.equal(parseIsoDuration(undefined), null); - assert.equal(parseIsoDuration(null), null); -}); - -test('pickThumbnail prefers the largest available size', () => { - const all = { - default: { url: 'd' }, - medium: { url: 'm' }, - high: { url: 'h' }, - standard: { url: 's' }, - maxres: { url: 'x' }, - }; - assert.equal(pickThumbnail(all), 'x'); - delete all.maxres; - assert.equal(pickThumbnail(all), 's'); - assert.equal(pickThumbnail({ default: { url: 'd' }, medium: { url: 'm' } }), 'm'); - assert.equal(pickThumbnail({}), null); - assert.equal(pickThumbnail(undefined), null); - assert.equal(pickThumbnail(null), null); -}); - -test('errorReasons collects reasons from errors[] and details[]', () => { - const body = { error: { errors: [{ reason: 'badRequest' }], details: [{ reason: 'API_KEY_INVALID' }] } }; - assert.deepEqual(errorReasons(body), ['badRequest', 'API_KEY_INVALID']); - assert.deepEqual(errorReasons(null), []); - assert.deepEqual(errorReasons({}), []); -}); - -test('classifyError recognises quota, key and server problems', () => { - assert.equal(classifyError(403, { error: { errors: [{ reason: 'quotaExceeded' }] } }), 'quota'); - assert.equal(classifyError(403, { error: { errors: [{ reason: 'dailyLimitExceeded' }] } }), 'quota'); - assert.equal( - classifyError(400, { error: { errors: [{ reason: 'badRequest' }], details: [{ reason: 'API_KEY_INVALID' }] } }), - 'invalid-key', - ); - assert.equal(classifyError(403, { error: { errors: [{ reason: 'accessNotConfigured' }] } }), 'invalid-key'); - assert.equal(classifyError(500, null), 'server'); - assert.equal(classifyError(503, {}), 'server'); - assert.equal(classifyError(429, { error: { errors: [{ reason: 'rateLimitExceeded' }] } }), 'server'); - assert.equal(classifyError(404, null), 'other'); -}); - -test('ApiError carries its kind and status', () => { - const err = new ApiError('quota', 'out of quota', 403); - assert.ok(err instanceof Error); - assert.equal(err.name, 'ApiError'); - assert.equal(err.kind, 'quota'); - assert.equal(err.status, 403); - assert.equal(err.message, 'out of quota'); -}); - -test('nextQuotaReset points at the next Pacific midnight (summer time)', () => { - const noon = new Date('2026-10-25T20:00:00Z'); // 13:00 PDT - assert.equal(new Date(nextQuotaReset(noon)).toISOString(), '2026-10-26T07:00:00.000Z'); - const justBefore = new Date('2026-10-26T06:59:59Z'); // 23:59:59 PDT - assert.equal(new Date(nextQuotaReset(justBefore)).toISOString(), '2026-10-26T07:00:00.000Z'); - const exactly = new Date('2026-10-26T07:00:00Z'); // 00:00:00 PDT - assert.equal(new Date(nextQuotaReset(exactly)).toISOString(), '2026-10-27T07:00:00.000Z'); -}); - -test('nextQuotaReset points at the next Pacific midnight (winter time)', () => { - const noon = new Date('2026-12-15T12:00:00Z'); // 04:00 PST - assert.equal(new Date(nextQuotaReset(noon)).toISOString(), '2026-12-16T08:00:00.000Z'); -}); diff --git a/test/api-requests.test.js b/test/api-requests.test.js deleted file mode 100644 index 9f82c8c..0000000 --- a/test/api-requests.test.js +++ /dev/null @@ -1,165 +0,0 @@ -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { ApiError, fetchChannels, fetchVideos } from '../src/lib/api.js'; - -const jsonResponse = (status, body) => ({ ok: status >= 200 && status < 300, status, json: async () => body }); -const noSleep = async () => {}; - -const videoItem = (id) => ({ - id, - snippet: { - title: `Title ${id}`, - channelId: 'UC1', - channelTitle: 'Chan', - publishedAt: '2026-10-01T00:00:00Z', - thumbnails: { high: { url: `https://i.ytimg.com/vi/${id}/hqdefault.jpg` } }, - }, - statistics: { viewCount: '12345' }, - contentDetails: { duration: 'PT10M' }, -}); - -test('fetchVideos splits ids into batches of 50 and normalizes items', async () => { - const ids = Array.from({ length: 120 }, (_, i) => `v${String(i).padStart(10, '0')}`); - const seen = []; - const fetchImpl = async (url) => { - const u = new URL(url); - const batch = u.searchParams.get('id').split(','); - seen.push({ path: u.pathname, size: batch.length, key: u.searchParams.get('key'), part: u.searchParams.get('part') }); - return jsonResponse(200, { items: batch.map(videoItem) }); - }; - - const { videos, missingIds } = await fetchVideos(ids, { apiKey: 'KEY', fetchImpl, sleep: noSleep }); - - assert.deepEqual(seen.map((s) => s.size), [50, 50, 20]); - assert.ok(seen.every((s) => s.path === '/youtube/v3/videos' && s.key === 'KEY')); - assert.ok(seen.every((s) => s.part === 'snippet,statistics,contentDetails')); - assert.equal(videos.length, 120); - assert.deepEqual(missingIds, []); - assert.deepEqual(videos[0], { - videoId: ids[0], - title: `Title ${ids[0]}`, - thumbnailUrl: `https://i.ytimg.com/vi/${ids[0]}/hqdefault.jpg`, - channelId: 'UC1', - channelTitle: 'Chan', - views: 12345, - publishedAt: '2026-10-01T00:00:00Z', - durationSec: 600, - }); -}); - -test('fetchVideos reports ids the API did not return', async () => { - const fetchImpl = async (url) => { - const batch = new URL(url).searchParams.get('id').split(','); - return jsonResponse(200, { items: batch.filter((id) => id !== 'missing0001').map(videoItem) }); - }; - const { videos, missingIds } = await fetchVideos(['abcdefghijk', 'missing0001'], { - apiKey: 'K', - fetchImpl, - sleep: noSleep, - }); - assert.equal(videos.length, 1); - assert.deepEqual(missingIds, ['missing0001']); -}); - -test('requests are retried on 5xx with exponential backoff', async () => { - const delays = []; - let calls = 0; - const fetchImpl = async () => { - calls += 1; - return calls < 3 ? jsonResponse(503, {}) : jsonResponse(200, { items: [videoItem('abcdefghijk')] }); - }; - const { videos } = await fetchVideos(['abcdefghijk'], { - apiKey: 'K', - fetchImpl, - sleep: async (ms) => delays.push(ms), - }); - assert.equal(videos.length, 1); - assert.equal(calls, 3); - assert.deepEqual(delays, [500, 1000]); -}); - -test('requests give up after three retries', async () => { - let calls = 0; - const fetchImpl = async () => { - calls += 1; - return jsonResponse(500, {}); - }; - await assert.rejects( - fetchVideos(['abcdefghijk'], { apiKey: 'K', fetchImpl, sleep: noSleep }), - (err) => err instanceof ApiError && err.kind === 'server' && err.status === 500, - ); - assert.equal(calls, 4); -}); - -test('network failures are retried and reported as kind network', async () => { - let calls = 0; - const fetchImpl = async () => { - calls += 1; - throw new TypeError('Failed to fetch'); - }; - await assert.rejects( - fetchVideos(['abcdefghijk'], { apiKey: 'K', fetchImpl, sleep: noSleep }), - (err) => err instanceof ApiError && err.kind === 'network', - ); - assert.equal(calls, 4); -}); - -test('quota errors are not retried', async () => { - let calls = 0; - const fetchImpl = async () => { - calls += 1; - return jsonResponse(403, { error: { message: 'quota', errors: [{ reason: 'quotaExceeded' }] } }); - }; - await assert.rejects( - fetchVideos(['abcdefghijk'], { apiKey: 'K', fetchImpl, sleep: noSleep }), - (err) => err instanceof ApiError && err.kind === 'quota' && err.message === 'quota', - ); - assert.equal(calls, 1); -}); - -test('invalid key errors are not retried', async () => { - let calls = 0; - const fetchImpl = async () => { - calls += 1; - return jsonResponse(400, { - error: { message: 'API key not valid.', errors: [{ reason: 'badRequest' }], details: [{ reason: 'API_KEY_INVALID' }] }, - }); - }; - await assert.rejects( - fetchVideos(['abcdefghijk'], { apiKey: 'bad', fetchImpl, sleep: noSleep }), - (err) => err instanceof ApiError && err.kind === 'invalid-key', - ); - assert.equal(calls, 1); -}); - -test('fetchChannels normalizes subscriber counts and channel urls', async () => { - let params; - const fetchImpl = async (url) => { - params = new URL(url).searchParams; - return jsonResponse(200, { - items: [ - { id: 'UC1', snippet: { title: 'One', customUrl: '@one' }, statistics: { subscriberCount: '1500' } }, - { id: 'UC2', snippet: { title: 'Two' }, statistics: { hiddenSubscriberCount: true } }, - { id: 'UC3', snippet: { title: 'Three', customUrl: 'legacyname' }, statistics: { subscriberCount: '7' } }, - ], - }); - }; - - const channels = await fetchChannels(['UC1', 'UC2', 'UC3'], { apiKey: 'K', fetchImpl, sleep: noSleep }); - - assert.equal(params.get('part'), 'statistics,snippet'); - assert.equal(params.get('maxResults'), '50'); - assert.equal(params.get('id'), 'UC1,UC2,UC3'); - assert.deepEqual(channels, [ - { channelId: 'UC1', title: 'One', url: 'https://www.youtube.com/@one', subscribers: 1500 }, - { channelId: 'UC2', title: 'Two', url: 'https://www.youtube.com/channel/UC2', subscribers: null }, - { channelId: 'UC3', title: 'Three', url: 'https://www.youtube.com/channel/UC3', subscribers: 7 }, - ]); -}); - -test('fetchChannels of an empty list makes no request', async () => { - const fetchImpl = async () => { - throw new Error('should not be called'); - }; - assert.deepEqual(await fetchChannels([], { apiKey: 'K', fetchImpl, sleep: noSleep }), []); -}); diff --git a/test/process.test.js b/test/process.test.js deleted file mode 100644 index 6f2e53e..0000000 --- a/test/process.test.js +++ /dev/null @@ -1,244 +0,0 @@ -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { processBatch } from '../src/lib/process.js'; - -const NOW = new Date('2026-10-25T10:00:00.000Z'); -const NOW_ISO = NOW.toISOString(); -const HOUR = 3600 * 1000; -const ago = (hours) => new Date(NOW.getTime() - hours * HOUR).toISOString(); -const SETTINGS = { threshold: 5, minViews: 10000 }; -const ID = 'vid00000001'; - -function memoryDb() { - const stores = { seen: new Map(), channels: new Map(), outliers: new Map() }; - const getter = (name) => async (keys) => - new Map(keys.filter((key) => stores[name].has(key)).map((key) => [key, stores[name].get(key)])); - const putter = (name, keyField) => async (records) => { - for (const record of records) stores[name].set(record[keyField], record); - }; - return { - stores, - getSeen: getter('seen'), - getChannels: getter('channels'), - getOutliers: getter('outliers'), - putSeen: putter('seen', 'videoId'), - putChannels: putter('channels', 'channelId'), - putOutliers: putter('outliers', 'videoId'), - }; -} - -function fakeApi({ videos = [], channels = [] } = {}) { - const calls = { videos: [], channels: [] }; - return { - calls, - fetchVideos: async (ids) => { - calls.videos.push(ids); - const found = videos.filter((v) => ids.includes(v.videoId)); - return { videos: found, missingIds: ids.filter((id) => !found.some((v) => v.videoId === id)) }; - }, - fetchChannels: async (ids) => { - calls.channels.push(ids); - return channels.filter((c) => ids.includes(c.channelId)); - }, - }; -} - -const item = (videoId = ID, over = {}) => ({ videoId, vidiqScore: null, viaShortsUrl: false, page: 'home', ...over }); -const video = (videoId = ID, over = {}) => ({ - videoId, - title: 'Title', - thumbnailUrl: 'https://i.ytimg.com/t.jpg', - channelId: 'UC1', - channelTitle: 'Chan', - views: 60000, - publishedAt: '2026-10-01T00:00:00Z', - durationSec: 600, - ...over, -}); -const channel = (channelId, subscribers) => ({ - channelId, - title: 'Chan', - url: 'https://www.youtube.com/@chan', - subscribers, -}); -const run = (items, db, api) => processBatch(items, { settings: SETTINGS, db, api, now: NOW }); - -test('saves a video whose score reaches the threshold', async () => { - const db = memoryDb(); - const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] }); - - const result = await run([item(ID, { page: 'search' })], db, api); - - assert.deepEqual(result, { checked: 1, saved: 1 }); - assert.deepEqual(db.stores.outliers.get(ID), { - videoId: ID, - title: 'Title', - url: `https://www.youtube.com/watch?v=${ID}`, - thumbnailUrl: 'https://i.ytimg.com/t.jpg', - channelId: 'UC1', - channelTitle: 'Chan', - channelUrl: 'https://www.youtube.com/@chan', - subscribers: 10000, - views: 60000, - publishedAt: '2026-10-01T00:00:00Z', - durationSec: 600, - isShort: false, - score: 6, - vidiqScore: null, - foundOn: 'search', - firstSeenAt: NOW_ISO, - updatedAt: NOW_ISO, - }); - assert.deepEqual(db.stores.seen.get(ID), { videoId: ID, checkedAt: NOW_ISO, status: 'ok' }); - assert.deepEqual(db.stores.channels.get('UC1'), { ...channel('UC1', 10000), fetchedAt: NOW_ISO }); -}); - -test('marks a below-threshold video as seen without saving it', async () => { - const db = memoryDb(); - const api = fakeApi({ videos: [video()], channels: [channel('UC1', 100000)] }); - const result = await run([item()], db, api); - assert.deepEqual(result, { checked: 1, saved: 0 }); - assert.equal(db.stores.outliers.size, 0); - assert.equal(db.stores.seen.get(ID).status, 'ok'); -}); - -test('does not save a video below minViews even with a huge score', async () => { - const db = memoryDb(); - const api = fakeApi({ videos: [video(ID, { views: 9000 })], channels: [channel('UC1', 100)] }); - await run([item()], db, api); - assert.equal(db.stores.outliers.size, 0); -}); - -test('saves a VidIQ-flagged video even when subscribers are hidden', async () => { - const db = memoryDb(); - const api = fakeApi({ videos: [video()], channels: [channel('UC1', null)] }); - await run([item(ID, { vidiqScore: 7.5 })], db, api); - const saved = db.stores.outliers.get(ID); - assert.equal(saved.score, null); - assert.equal(saved.subscribers, null); - assert.equal(saved.vidiqScore, 7.5); -}); - -test('does not save a video with hidden subscribers and no VidIQ score', async () => { - const db = memoryDb(); - const api = fakeApi({ videos: [video()], channels: [channel('UC1', null)] }); - await run([item()], db, api); - assert.equal(db.stores.outliers.size, 0); -}); - -test('treats a channel missing from the API response as unknown subscribers', async () => { - const db = memoryDb(); - const api = fakeApi({ videos: [video()], channels: [] }); - await run([item(ID, { vidiqScore: 4 })], db, api); - assert.equal(db.stores.outliers.get(ID).subscribers, null); - assert.equal(db.stores.outliers.get(ID).channelUrl, 'https://www.youtube.com/channel/UC1'); -}); - -test('skips ids checked less than 24 hours ago', async () => { - const db = memoryDb(); - db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(1), status: 'ok' }); - const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] }); - const result = await run([item()], db, api); - assert.deepEqual(result, { checked: 0, saved: 0 }); - assert.equal(api.calls.videos.length, 0); -}); - -test('rechecks ids older than 24 hours', async () => { - const db = memoryDb(); - db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(25), status: 'ok' }); - const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] }); - const result = await run([item()], db, api); - assert.deepEqual(result, { checked: 1, saved: 1 }); -}); - -test('an item with a VidIQ score bypasses a fresh seen record', async () => { - const db = memoryDb(); - db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(1), status: 'ok' }); - const api = fakeApi({ videos: [video()], channels: [channel('UC1', 1000000)] }); - const result = await run([item(ID, { vidiqScore: 4 })], db, api); - assert.deepEqual(result, { checked: 1, saved: 1 }); - assert.equal(db.stores.outliers.get(ID).vidiqScore, 4); -}); - -test('a repeat sighting refreshes stats and keeps the original metadata', async () => { - const db = memoryDb(); - db.stores.outliers.set(ID, { - videoId: ID, - firstSeenAt: '2026-10-20T00:00:00.000Z', - foundOn: 'home', - isShort: true, - vidiqScore: 9, - }); - db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(30), status: 'ok' }); - const api = fakeApi({ videos: [video(ID, { views: 70000 })], channels: [channel('UC1', 10000)] }); - - await run([item(ID, { page: 'search', viaShortsUrl: false })], db, api); - - const saved = db.stores.outliers.get(ID); - assert.equal(saved.firstSeenAt, '2026-10-20T00:00:00.000Z'); - assert.equal(saved.foundOn, 'home'); - assert.equal(saved.isShort, true); - assert.equal(saved.vidiqScore, 9); - assert.equal(saved.views, 70000); - assert.equal(saved.score, 7); - assert.equal(saved.updatedAt, NOW_ISO); -}); - -test('an already saved video is updated even when its score fell below the threshold', async () => { - const db = memoryDb(); - db.stores.outliers.set(ID, { videoId: ID, firstSeenAt: ago(100), foundOn: 'home', isShort: false, vidiqScore: null }); - db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(30), status: 'ok' }); - const api = fakeApi({ videos: [video()], channels: [channel('UC1', 1000000)] }); - const result = await run([item()], db, api); - assert.deepEqual(result, { checked: 1, saved: 1 }); - assert.equal(db.stores.outliers.get(ID).score, 0.06); -}); - -test('a short found via a /shorts/ link is flagged', async () => { - const db = memoryDb(); - const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] }); - await run([item(ID, { viaShortsUrl: true })], db, api); - assert.equal(db.stores.outliers.get(ID).isShort, true); -}); - -test('records unavailable videos in seen and does not save them', async () => { - const db = memoryDb(); - const api = fakeApi({ videos: [], channels: [] }); - const result = await run([item()], db, api); - assert.deepEqual(result, { checked: 1, saved: 0 }); - assert.deepEqual(db.stores.seen.get(ID), { videoId: ID, checkedAt: NOW_ISO, status: 'unavailable' }); - assert.equal(db.stores.outliers.size, 0); -}); - -test('reuses a cached channel younger than 24 hours', async () => { - const db = memoryDb(); - db.stores.channels.set('UC1', { ...channel('UC1', 10000), fetchedAt: ago(1) }); - const api = fakeApi({ videos: [video()], channels: [] }); - const result = await run([item()], db, api); - assert.equal(api.calls.channels.length, 0); - assert.deepEqual(result, { checked: 1, saved: 1 }); -}); - -test('refetches a cached channel older than 24 hours', async () => { - const db = memoryDb(); - db.stores.channels.set('UC1', { ...channel('UC1', 10000), fetchedAt: ago(25) }); - const api = fakeApi({ videos: [video()], channels: [channel('UC1', 20000)] }); - const result = await run([item()], db, api); - assert.deepEqual(api.calls.channels, [['UC1']]); - assert.equal(db.stores.channels.get('UC1').subscribers, 20000); - assert.deepEqual(result, { checked: 1, saved: 0 }); // score 3 < 5 -}); - -test('an API failure writes nothing and does not mark ids as seen', async () => { - const db = memoryDb(); - const api = { - ...fakeApi(), - fetchVideos: async () => { - throw new Error('boom'); - }, - }; - await assert.rejects(run([item()], db, api), /boom/); - assert.equal(db.stores.seen.size, 0); - assert.equal(db.stores.outliers.size, 0); - assert.equal(db.stores.channels.size, 0); -}); diff --git a/test/score.test.js b/test/score.test.js deleted file mode 100644 index 0a8a226..0000000 --- a/test/score.test.js +++ /dev/null @@ -1,44 +0,0 @@ -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { computeScore, shouldSave } from '../src/lib/score.js'; - -test('computeScore divides views by subscribers', () => { - assert.equal(computeScore(100000, 20000), 5); - assert.equal(computeScore(2500, 1000), 2.5); -}); - -test('computeScore rounds to two decimals', () => { - assert.equal(computeScore(10000, 3000), 3.33); -}); - -test('computeScore never divides by zero', () => { - assert.equal(computeScore(5000, 0), 5000); -}); - -test('computeScore returns null when subscribers are unknown', () => { - assert.equal(computeScore(5000, null), null); - assert.equal(computeScore(5000, undefined), null); -}); - -const settings = { threshold: 5, minViews: 10000 }; - -test('shouldSave accepts a video exactly at both limits', () => { - assert.equal(shouldSave({ views: 10000, score: 5, vidiqScore: null }, settings), true); -}); - -test('shouldSave rejects a score below the threshold', () => { - assert.equal(shouldSave({ views: 50000, score: 4.99, vidiqScore: null }, settings), false); -}); - -test('shouldSave rejects views below the minimum even with a huge score', () => { - assert.equal(shouldSave({ views: 9999, score: 50, vidiqScore: null }, settings), false); -}); - -test('shouldSave rejects a null score without a VidIQ score', () => { - assert.equal(shouldSave({ views: 500000, score: null, vidiqScore: null }, settings), false); -}); - -test('shouldSave accepts any video that has a VidIQ score', () => { - assert.equal(shouldSave({ views: 100, score: null, vidiqScore: 7.5 }, settings), true); - assert.equal(shouldSave({ views: 100, score: 0.01, vidiqScore: 3 }, settings), true); -});