From 97faeedca38a4989cbb65adf2cecb601c9870efa Mon Sep 17 00:00:00 2001 From: IgorVolochay Date: Sun, 25 Oct 2026 10:50:34 +0300 Subject: [PATCH] feat: fetch videos and channels from the YouTube Data API Co-Authored-By: Claude Sonnet 5.5 --- src/lib/api.js | 103 ++++++++++++++++++++++++ test/api-requests.test.js | 165 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 test/api-requests.test.js diff --git a/src/lib/api.js b/src/lib/api.js index 91b7c25..37e4afd 100644 --- a/src/lib/api.js +++ b/src/lib/api.js @@ -68,3 +68,106 @@ export function nextQuotaReset(now = new Date()) { ((part('hour') * 60 + part('minute')) * 60 + part('second')) * 1000 + now.getMilliseconds(); return now.getTime() - sinceMidnightMs + 24 * 60 * 60 * 1000; } + +const API_BASE = 'https://www.googleapis.com/youtube/v3'; +const BATCH_SIZE = 50; +const RETRIES = 3; +const BASE_DELAY_MS = 500; + +const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function requestOnce(path, params, { apiKey, fetchImpl = fetch }) { + const url = `${API_BASE}/${path}?${new URLSearchParams({ ...params, key: apiKey })}`; + let response; + try { + response = await fetchImpl(url); + } catch (err) { + throw new ApiError('network', err.message); + } + if (response.ok) { + try { + return await response.json(); + } catch { + throw new ApiError('server', 'Invalid JSON in API response', response.status); + } + } + let body = null; + try { + body = await response.json(); + } catch { + // error responses are not always JSON + } + throw new ApiError(classifyError(response.status, body), body?.error?.message ?? `HTTP ${response.status}`, response.status); +} + +async function request(path, params, opts) { + const sleep = opts.sleep ?? defaultSleep; + for (let attempt = 0; ; attempt += 1) { + try { + return await requestOnce(path, params, opts); + } catch (err) { + const retryable = err instanceof ApiError && (err.kind === 'network' || err.kind === 'server'); + if (!retryable || attempt >= RETRIES) throw err; + await sleep(BASE_DELAY_MS * 2 ** attempt); + } + } +} + +export function normalizeVideo(item) { + const { snippet = {}, statistics = {}, contentDetails = {} } = item; + return { + videoId: item.id, + title: snippet.title ?? '', + thumbnailUrl: pickThumbnail(snippet.thumbnails), + channelId: snippet.channelId, + channelTitle: snippet.channelTitle ?? '', + views: Number(statistics.viewCount ?? 0), + publishedAt: snippet.publishedAt ?? null, + durationSec: parseIsoDuration(contentDetails.duration), + }; +} + +export function normalizeChannel(item) { + const customUrl = item.snippet?.customUrl; + const stats = item.statistics ?? {}; + return { + channelId: item.id, + title: item.snippet?.title ?? '', + url: customUrl?.startsWith('@') + ? `https://www.youtube.com/${customUrl}` + : `https://www.youtube.com/channel/${item.id}`, + subscribers: stats.hiddenSubscriberCount || stats.subscriberCount === undefined ? null : Number(stats.subscriberCount), + }; +} + +export async function fetchVideos(ids, opts) { + const videos = []; + const found = new Set(); + for (const batch of chunk(ids, BATCH_SIZE)) { + const data = await request( + 'videos', + { part: 'snippet,statistics,contentDetails', id: batch.join(',') }, + opts, + ); + for (const item of data.items ?? []) { + videos.push(normalizeVideo(item)); + found.add(item.id); + } + } + return { videos, missingIds: ids.filter((id) => !found.has(id)) }; +} + +// maxResults is sent here only: videos.list rejects it together with `id`, +// while channels.list would otherwise default to 5 results. +export async function fetchChannels(ids, opts) { + const channels = []; + for (const batch of chunk(ids, BATCH_SIZE)) { + const data = await request( + 'channels', + { part: 'statistics,snippet', id: batch.join(','), maxResults: String(BATCH_SIZE) }, + opts, + ); + for (const item of data.items ?? []) channels.push(normalizeChannel(item)); + } + return channels; +} diff --git a/test/api-requests.test.js b/test/api-requests.test.js new file mode 100644 index 0000000..9f82c8c --- /dev/null +++ b/test/api-requests.test.js @@ -0,0 +1,165 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { ApiError, fetchChannels, fetchVideos } from '../src/lib/api.js'; + +const jsonResponse = (status, body) => ({ ok: status >= 200 && status < 300, status, json: async () => body }); +const noSleep = async () => {}; + +const videoItem = (id) => ({ + id, + snippet: { + title: `Title ${id}`, + channelId: 'UC1', + channelTitle: 'Chan', + publishedAt: '2026-10-01T00:00:00Z', + thumbnails: { high: { url: `https://i.ytimg.com/vi/${id}/hqdefault.jpg` } }, + }, + statistics: { viewCount: '12345' }, + contentDetails: { duration: 'PT10M' }, +}); + +test('fetchVideos splits ids into batches of 50 and normalizes items', async () => { + const ids = Array.from({ length: 120 }, (_, i) => `v${String(i).padStart(10, '0')}`); + const seen = []; + const fetchImpl = async (url) => { + const u = new URL(url); + const batch = u.searchParams.get('id').split(','); + seen.push({ path: u.pathname, size: batch.length, key: u.searchParams.get('key'), part: u.searchParams.get('part') }); + return jsonResponse(200, { items: batch.map(videoItem) }); + }; + + const { videos, missingIds } = await fetchVideos(ids, { apiKey: 'KEY', fetchImpl, sleep: noSleep }); + + assert.deepEqual(seen.map((s) => s.size), [50, 50, 20]); + assert.ok(seen.every((s) => s.path === '/youtube/v3/videos' && s.key === 'KEY')); + assert.ok(seen.every((s) => s.part === 'snippet,statistics,contentDetails')); + assert.equal(videos.length, 120); + assert.deepEqual(missingIds, []); + assert.deepEqual(videos[0], { + videoId: ids[0], + title: `Title ${ids[0]}`, + thumbnailUrl: `https://i.ytimg.com/vi/${ids[0]}/hqdefault.jpg`, + channelId: 'UC1', + channelTitle: 'Chan', + views: 12345, + publishedAt: '2026-10-01T00:00:00Z', + durationSec: 600, + }); +}); + +test('fetchVideos reports ids the API did not return', async () => { + const fetchImpl = async (url) => { + const batch = new URL(url).searchParams.get('id').split(','); + return jsonResponse(200, { items: batch.filter((id) => id !== 'missing0001').map(videoItem) }); + }; + const { videos, missingIds } = await fetchVideos(['abcdefghijk', 'missing0001'], { + apiKey: 'K', + fetchImpl, + sleep: noSleep, + }); + assert.equal(videos.length, 1); + assert.deepEqual(missingIds, ['missing0001']); +}); + +test('requests are retried on 5xx with exponential backoff', async () => { + const delays = []; + let calls = 0; + const fetchImpl = async () => { + calls += 1; + return calls < 3 ? jsonResponse(503, {}) : jsonResponse(200, { items: [videoItem('abcdefghijk')] }); + }; + const { videos } = await fetchVideos(['abcdefghijk'], { + apiKey: 'K', + fetchImpl, + sleep: async (ms) => delays.push(ms), + }); + assert.equal(videos.length, 1); + assert.equal(calls, 3); + assert.deepEqual(delays, [500, 1000]); +}); + +test('requests give up after three retries', async () => { + let calls = 0; + const fetchImpl = async () => { + calls += 1; + return jsonResponse(500, {}); + }; + await assert.rejects( + fetchVideos(['abcdefghijk'], { apiKey: 'K', fetchImpl, sleep: noSleep }), + (err) => err instanceof ApiError && err.kind === 'server' && err.status === 500, + ); + assert.equal(calls, 4); +}); + +test('network failures are retried and reported as kind network', async () => { + let calls = 0; + const fetchImpl = async () => { + calls += 1; + throw new TypeError('Failed to fetch'); + }; + await assert.rejects( + fetchVideos(['abcdefghijk'], { apiKey: 'K', fetchImpl, sleep: noSleep }), + (err) => err instanceof ApiError && err.kind === 'network', + ); + assert.equal(calls, 4); +}); + +test('quota errors are not retried', async () => { + let calls = 0; + const fetchImpl = async () => { + calls += 1; + return jsonResponse(403, { error: { message: 'quota', errors: [{ reason: 'quotaExceeded' }] } }); + }; + await assert.rejects( + fetchVideos(['abcdefghijk'], { apiKey: 'K', fetchImpl, sleep: noSleep }), + (err) => err instanceof ApiError && err.kind === 'quota' && err.message === 'quota', + ); + assert.equal(calls, 1); +}); + +test('invalid key errors are not retried', async () => { + let calls = 0; + const fetchImpl = async () => { + calls += 1; + return jsonResponse(400, { + error: { message: 'API key not valid.', errors: [{ reason: 'badRequest' }], details: [{ reason: 'API_KEY_INVALID' }] }, + }); + }; + await assert.rejects( + fetchVideos(['abcdefghijk'], { apiKey: 'bad', fetchImpl, sleep: noSleep }), + (err) => err instanceof ApiError && err.kind === 'invalid-key', + ); + assert.equal(calls, 1); +}); + +test('fetchChannels normalizes subscriber counts and channel urls', async () => { + let params; + const fetchImpl = async (url) => { + params = new URL(url).searchParams; + return jsonResponse(200, { + items: [ + { id: 'UC1', snippet: { title: 'One', customUrl: '@one' }, statistics: { subscriberCount: '1500' } }, + { id: 'UC2', snippet: { title: 'Two' }, statistics: { hiddenSubscriberCount: true } }, + { id: 'UC3', snippet: { title: 'Three', customUrl: 'legacyname' }, statistics: { subscriberCount: '7' } }, + ], + }); + }; + + const channels = await fetchChannels(['UC1', 'UC2', 'UC3'], { apiKey: 'K', fetchImpl, sleep: noSleep }); + + assert.equal(params.get('part'), 'statistics,snippet'); + assert.equal(params.get('maxResults'), '50'); + assert.equal(params.get('id'), 'UC1,UC2,UC3'); + assert.deepEqual(channels, [ + { channelId: 'UC1', title: 'One', url: 'https://www.youtube.com/@one', subscribers: 1500 }, + { channelId: 'UC2', title: 'Two', url: 'https://www.youtube.com/channel/UC2', subscribers: null }, + { channelId: 'UC3', title: 'Three', url: 'https://www.youtube.com/channel/UC3', subscribers: 7 }, + ]); +}); + +test('fetchChannels of an empty list makes no request', async () => { + const fetchImpl = async () => { + throw new Error('should not be called'); + }; + assert.deepEqual(await fetchChannels([], { apiKey: 'K', fetchImpl, sleep: noSleep }), []); +});