diff --git a/src/lib/process.js b/src/lib/process.js new file mode 100644 index 0000000..f47e68c --- /dev/null +++ b/src/lib/process.js @@ -0,0 +1,73 @@ +import { computeScore, shouldSave } from './score.js'; + +export const RECHECK_MS = 24 * 60 * 60 * 1000; + +const watchUrl = (videoId) => `https://www.youtube.com/watch?v=${videoId}`; + +function isFresh(checkedAt, nowMs) { + return checkedAt !== undefined && nowMs - Date.parse(checkedAt) < RECHECK_MS; +} + +export async function processBatch(items, { settings, db, api, now = new Date() }) { + const nowMs = now.getTime(); + const nowIso = now.toISOString(); + const byId = new Map(items.map((item) => [item.videoId, item])); + + const seen = await db.getSeen([...byId.keys()]); + const due = [...byId.values()].filter( + (item) => item.vidiqScore !== null || !isFresh(seen.get(item.videoId)?.checkedAt, nowMs), + ); + if (due.length === 0) return { checked: 0, saved: 0 }; + + const { videos, missingIds } = await api.fetchVideos(due.map((item) => item.videoId)); + + const channelIds = [...new Set(videos.map((video) => video.channelId))]; + const cachedChannels = await db.getChannels(channelIds); + const staleIds = channelIds.filter((id) => !isFresh(cachedChannels.get(id)?.fetchedAt, nowMs)); + const fetchedChannels = staleIds.length > 0 ? await api.fetchChannels(staleIds) : []; + const channelRecords = fetchedChannels.map((channel) => ({ ...channel, fetchedAt: nowIso })); + const channels = new Map(cachedChannels); + for (const record of channelRecords) channels.set(record.channelId, record); + + const existing = await db.getOutliers(videos.map((video) => video.videoId)); + const outliers = []; + for (const video of videos) { + const item = byId.get(video.videoId); + const previous = existing.get(video.videoId); + const channel = channels.get(video.channelId); + const subscribers = channel?.subscribers ?? null; + const score = computeScore(video.views, subscribers); + const vidiqScore = item.vidiqScore ?? previous?.vidiqScore ?? null; + + if (!previous && !shouldSave({ views: video.views, score, vidiqScore }, settings)) continue; + + outliers.push({ + videoId: video.videoId, + title: video.title, + url: watchUrl(video.videoId), + thumbnailUrl: video.thumbnailUrl, + channelId: video.channelId, + channelTitle: video.channelTitle, + channelUrl: channel?.url ?? `https://www.youtube.com/channel/${video.channelId}`, + subscribers, + views: video.views, + publishedAt: video.publishedAt, + durationSec: video.durationSec, + isShort: item.viaShortsUrl || previous?.isShort || false, + score, + vidiqScore, + foundOn: previous?.foundOn ?? item.page, + firstSeenAt: previous?.firstSeenAt ?? nowIso, + updatedAt: nowIso, + }); + } + + // Results first, `seen` last: if anything above throws, the ids are retried later. + await db.putChannels(channelRecords); + await db.putOutliers(outliers); + await db.putSeen([ + ...videos.map((video) => ({ videoId: video.videoId, checkedAt: nowIso, status: 'ok' })), + ...missingIds.map((videoId) => ({ videoId, checkedAt: nowIso, status: 'unavailable' })), + ]); + return { checked: due.length, saved: outliers.length }; +} diff --git a/test/process.test.js b/test/process.test.js new file mode 100644 index 0000000..6f2e53e --- /dev/null +++ b/test/process.test.js @@ -0,0 +1,244 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { processBatch } from '../src/lib/process.js'; + +const NOW = new Date('2026-10-25T10:00:00.000Z'); +const NOW_ISO = NOW.toISOString(); +const HOUR = 3600 * 1000; +const ago = (hours) => new Date(NOW.getTime() - hours * HOUR).toISOString(); +const SETTINGS = { threshold: 5, minViews: 10000 }; +const ID = 'vid00000001'; + +function memoryDb() { + const stores = { seen: new Map(), channels: new Map(), outliers: new Map() }; + const getter = (name) => async (keys) => + new Map(keys.filter((key) => stores[name].has(key)).map((key) => [key, stores[name].get(key)])); + const putter = (name, keyField) => async (records) => { + for (const record of records) stores[name].set(record[keyField], record); + }; + return { + stores, + getSeen: getter('seen'), + getChannels: getter('channels'), + getOutliers: getter('outliers'), + putSeen: putter('seen', 'videoId'), + putChannels: putter('channels', 'channelId'), + putOutliers: putter('outliers', 'videoId'), + }; +} + +function fakeApi({ videos = [], channels = [] } = {}) { + const calls = { videos: [], channels: [] }; + return { + calls, + fetchVideos: async (ids) => { + calls.videos.push(ids); + const found = videos.filter((v) => ids.includes(v.videoId)); + return { videos: found, missingIds: ids.filter((id) => !found.some((v) => v.videoId === id)) }; + }, + fetchChannels: async (ids) => { + calls.channels.push(ids); + return channels.filter((c) => ids.includes(c.channelId)); + }, + }; +} + +const item = (videoId = ID, over = {}) => ({ videoId, vidiqScore: null, viaShortsUrl: false, page: 'home', ...over }); +const video = (videoId = ID, over = {}) => ({ + videoId, + title: 'Title', + thumbnailUrl: 'https://i.ytimg.com/t.jpg', + channelId: 'UC1', + channelTitle: 'Chan', + views: 60000, + publishedAt: '2026-10-01T00:00:00Z', + durationSec: 600, + ...over, +}); +const channel = (channelId, subscribers) => ({ + channelId, + title: 'Chan', + url: 'https://www.youtube.com/@chan', + subscribers, +}); +const run = (items, db, api) => processBatch(items, { settings: SETTINGS, db, api, now: NOW }); + +test('saves a video whose score reaches the threshold', async () => { + const db = memoryDb(); + const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] }); + + const result = await run([item(ID, { page: 'search' })], db, api); + + assert.deepEqual(result, { checked: 1, saved: 1 }); + assert.deepEqual(db.stores.outliers.get(ID), { + videoId: ID, + title: 'Title', + url: `https://www.youtube.com/watch?v=${ID}`, + thumbnailUrl: 'https://i.ytimg.com/t.jpg', + channelId: 'UC1', + channelTitle: 'Chan', + channelUrl: 'https://www.youtube.com/@chan', + subscribers: 10000, + views: 60000, + publishedAt: '2026-10-01T00:00:00Z', + durationSec: 600, + isShort: false, + score: 6, + vidiqScore: null, + foundOn: 'search', + firstSeenAt: NOW_ISO, + updatedAt: NOW_ISO, + }); + assert.deepEqual(db.stores.seen.get(ID), { videoId: ID, checkedAt: NOW_ISO, status: 'ok' }); + assert.deepEqual(db.stores.channels.get('UC1'), { ...channel('UC1', 10000), fetchedAt: NOW_ISO }); +}); + +test('marks a below-threshold video as seen without saving it', async () => { + const db = memoryDb(); + const api = fakeApi({ videos: [video()], channels: [channel('UC1', 100000)] }); + const result = await run([item()], db, api); + assert.deepEqual(result, { checked: 1, saved: 0 }); + assert.equal(db.stores.outliers.size, 0); + assert.equal(db.stores.seen.get(ID).status, 'ok'); +}); + +test('does not save a video below minViews even with a huge score', async () => { + const db = memoryDb(); + const api = fakeApi({ videos: [video(ID, { views: 9000 })], channels: [channel('UC1', 100)] }); + await run([item()], db, api); + assert.equal(db.stores.outliers.size, 0); +}); + +test('saves a VidIQ-flagged video even when subscribers are hidden', async () => { + const db = memoryDb(); + const api = fakeApi({ videos: [video()], channels: [channel('UC1', null)] }); + await run([item(ID, { vidiqScore: 7.5 })], db, api); + const saved = db.stores.outliers.get(ID); + assert.equal(saved.score, null); + assert.equal(saved.subscribers, null); + assert.equal(saved.vidiqScore, 7.5); +}); + +test('does not save a video with hidden subscribers and no VidIQ score', async () => { + const db = memoryDb(); + const api = fakeApi({ videos: [video()], channels: [channel('UC1', null)] }); + await run([item()], db, api); + assert.equal(db.stores.outliers.size, 0); +}); + +test('treats a channel missing from the API response as unknown subscribers', async () => { + const db = memoryDb(); + const api = fakeApi({ videos: [video()], channels: [] }); + await run([item(ID, { vidiqScore: 4 })], db, api); + assert.equal(db.stores.outliers.get(ID).subscribers, null); + assert.equal(db.stores.outliers.get(ID).channelUrl, 'https://www.youtube.com/channel/UC1'); +}); + +test('skips ids checked less than 24 hours ago', async () => { + const db = memoryDb(); + db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(1), status: 'ok' }); + const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] }); + const result = await run([item()], db, api); + assert.deepEqual(result, { checked: 0, saved: 0 }); + assert.equal(api.calls.videos.length, 0); +}); + +test('rechecks ids older than 24 hours', async () => { + const db = memoryDb(); + db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(25), status: 'ok' }); + const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] }); + const result = await run([item()], db, api); + assert.deepEqual(result, { checked: 1, saved: 1 }); +}); + +test('an item with a VidIQ score bypasses a fresh seen record', async () => { + const db = memoryDb(); + db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(1), status: 'ok' }); + const api = fakeApi({ videos: [video()], channels: [channel('UC1', 1000000)] }); + const result = await run([item(ID, { vidiqScore: 4 })], db, api); + assert.deepEqual(result, { checked: 1, saved: 1 }); + assert.equal(db.stores.outliers.get(ID).vidiqScore, 4); +}); + +test('a repeat sighting refreshes stats and keeps the original metadata', async () => { + const db = memoryDb(); + db.stores.outliers.set(ID, { + videoId: ID, + firstSeenAt: '2026-10-20T00:00:00.000Z', + foundOn: 'home', + isShort: true, + vidiqScore: 9, + }); + db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(30), status: 'ok' }); + const api = fakeApi({ videos: [video(ID, { views: 70000 })], channels: [channel('UC1', 10000)] }); + + await run([item(ID, { page: 'search', viaShortsUrl: false })], db, api); + + const saved = db.stores.outliers.get(ID); + assert.equal(saved.firstSeenAt, '2026-10-20T00:00:00.000Z'); + assert.equal(saved.foundOn, 'home'); + assert.equal(saved.isShort, true); + assert.equal(saved.vidiqScore, 9); + assert.equal(saved.views, 70000); + assert.equal(saved.score, 7); + assert.equal(saved.updatedAt, NOW_ISO); +}); + +test('an already saved video is updated even when its score fell below the threshold', async () => { + const db = memoryDb(); + db.stores.outliers.set(ID, { videoId: ID, firstSeenAt: ago(100), foundOn: 'home', isShort: false, vidiqScore: null }); + db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(30), status: 'ok' }); + const api = fakeApi({ videos: [video()], channels: [channel('UC1', 1000000)] }); + const result = await run([item()], db, api); + assert.deepEqual(result, { checked: 1, saved: 1 }); + assert.equal(db.stores.outliers.get(ID).score, 0.06); +}); + +test('a short found via a /shorts/ link is flagged', async () => { + const db = memoryDb(); + const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] }); + await run([item(ID, { viaShortsUrl: true })], db, api); + assert.equal(db.stores.outliers.get(ID).isShort, true); +}); + +test('records unavailable videos in seen and does not save them', async () => { + const db = memoryDb(); + const api = fakeApi({ videos: [], channels: [] }); + const result = await run([item()], db, api); + assert.deepEqual(result, { checked: 1, saved: 0 }); + assert.deepEqual(db.stores.seen.get(ID), { videoId: ID, checkedAt: NOW_ISO, status: 'unavailable' }); + assert.equal(db.stores.outliers.size, 0); +}); + +test('reuses a cached channel younger than 24 hours', async () => { + const db = memoryDb(); + db.stores.channels.set('UC1', { ...channel('UC1', 10000), fetchedAt: ago(1) }); + const api = fakeApi({ videos: [video()], channels: [] }); + const result = await run([item()], db, api); + assert.equal(api.calls.channels.length, 0); + assert.deepEqual(result, { checked: 1, saved: 1 }); +}); + +test('refetches a cached channel older than 24 hours', async () => { + const db = memoryDb(); + db.stores.channels.set('UC1', { ...channel('UC1', 10000), fetchedAt: ago(25) }); + const api = fakeApi({ videos: [video()], channels: [channel('UC1', 20000)] }); + const result = await run([item()], db, api); + assert.deepEqual(api.calls.channels, [['UC1']]); + assert.equal(db.stores.channels.get('UC1').subscribers, 20000); + assert.deepEqual(result, { checked: 1, saved: 0 }); // score 3 < 5 +}); + +test('an API failure writes nothing and does not mark ids as seen', async () => { + const db = memoryDb(); + const api = { + ...fakeApi(), + fetchVideos: async () => { + throw new Error('boom'); + }, + }; + await assert.rejects(run([item()], db, api), /boom/); + assert.equal(db.stores.seen.size, 0); + assert.equal(db.stores.outliers.size, 0); + assert.equal(db.stores.channels.size, 0); +});