feat: add batch processing with injected db and api

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

src/lib/process.js

@@ -0,0 +1,73 @@
+import { computeScore, shouldSave } from './score.js';
+
+export const RECHECK_MS = 24 * 60 * 60 * 1000;
+
+const watchUrl = (videoId) => `https://www.youtube.com/watch?v=${videoId}`;
+
+function isFresh(checkedAt, nowMs) {
+  return checkedAt !== undefined && nowMs - Date.parse(checkedAt) < RECHECK_MS;
+}
+
+export async function processBatch(items, { settings, db, api, now = new Date() }) {
+  const nowMs = now.getTime();
+  const nowIso = now.toISOString();
+  const byId = new Map(items.map((item) => [item.videoId, item]));
+
+  const seen = await db.getSeen([...byId.keys()]);
+  const due = [...byId.values()].filter(
+    (item) => item.vidiqScore !== null || !isFresh(seen.get(item.videoId)?.checkedAt, nowMs),
+  );
+  if (due.length === 0) return { checked: 0, saved: 0 };
+
+  const { videos, missingIds } = await api.fetchVideos(due.map((item) => item.videoId));
+
+  const channelIds = [...new Set(videos.map((video) => video.channelId))];
+  const cachedChannels = await db.getChannels(channelIds);
+  const staleIds = channelIds.filter((id) => !isFresh(cachedChannels.get(id)?.fetchedAt, nowMs));
+  const fetchedChannels = staleIds.length > 0 ? await api.fetchChannels(staleIds) : [];
+  const channelRecords = fetchedChannels.map((channel) => ({ ...channel, fetchedAt: nowIso }));
+  const channels = new Map(cachedChannels);
+  for (const record of channelRecords) channels.set(record.channelId, record);
+
+  const existing = await db.getOutliers(videos.map((video) => video.videoId));
+  const outliers = [];
+  for (const video of videos) {
+    const item = byId.get(video.videoId);
+    const previous = existing.get(video.videoId);
+    const channel = channels.get(video.channelId);
+    const subscribers = channel?.subscribers ?? null;
+    const score = computeScore(video.views, subscribers);
+    const vidiqScore = item.vidiqScore ?? previous?.vidiqScore ?? null;
+
+    if (!previous && !shouldSave({ views: video.views, score, vidiqScore }, settings)) continue;
+
+    outliers.push({
+      videoId: video.videoId,
+      title: video.title,
+      url: watchUrl(video.videoId),
+      thumbnailUrl: video.thumbnailUrl,
+      channelId: video.channelId,
+      channelTitle: video.channelTitle,
+      channelUrl: channel?.url ?? `https://www.youtube.com/channel/${video.channelId}`,
+      subscribers,
+      views: video.views,
+      publishedAt: video.publishedAt,
+      durationSec: video.durationSec,
+      isShort: item.viaShortsUrl || previous?.isShort || false,
+      score,
+      vidiqScore,
+      foundOn: previous?.foundOn ?? item.page,
+      firstSeenAt: previous?.firstSeenAt ?? nowIso,
+      updatedAt: nowIso,
+    });
+  }
+
+  // Results first, `seen` last: if anything above throws, the ids are retried later.
+  await db.putChannels(channelRecords);
+  await db.putOutliers(outliers);
+  await db.putSeen([
+    ...videos.map((video) => ({ videoId: video.videoId, checkedAt: nowIso, status: 'ok' })),
+    ...missingIds.map((videoId) => ({ videoId, checkedAt: nowIso, status: 'unavailable' })),
+  ]);
+  return { checked: due.length, saved: outliers.length };
+}

test/process.test.js

@@ -0,0 +1,244 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { processBatch } from '../src/lib/process.js';
+
+const NOW = new Date('2026-10-25T10:00:00.000Z');
+const NOW_ISO = NOW.toISOString();
+const HOUR = 3600 * 1000;
+const ago = (hours) => new Date(NOW.getTime() - hours * HOUR).toISOString();
+const SETTINGS = { threshold: 5, minViews: 10000 };
+const ID = 'vid00000001';
+
+function memoryDb() {
+  const stores = { seen: new Map(), channels: new Map(), outliers: new Map() };
+  const getter = (name) => async (keys) =>
+    new Map(keys.filter((key) => stores[name].has(key)).map((key) => [key, stores[name].get(key)]));
+  const putter = (name, keyField) => async (records) => {
+    for (const record of records) stores[name].set(record[keyField], record);
+  };
+  return {
+    stores,
+    getSeen: getter('seen'),
+    getChannels: getter('channels'),
+    getOutliers: getter('outliers'),
+    putSeen: putter('seen', 'videoId'),
+    putChannels: putter('channels', 'channelId'),
+    putOutliers: putter('outliers', 'videoId'),
+  };
+}
+
+function fakeApi({ videos = [], channels = [] } = {}) {
+  const calls = { videos: [], channels: [] };
+  return {
+    calls,
+    fetchVideos: async (ids) => {
+      calls.videos.push(ids);
+      const found = videos.filter((v) => ids.includes(v.videoId));
+      return { videos: found, missingIds: ids.filter((id) => !found.some((v) => v.videoId === id)) };
+    },
+    fetchChannels: async (ids) => {
+      calls.channels.push(ids);
+      return channels.filter((c) => ids.includes(c.channelId));
+    },
+  };
+}
+
+const item = (videoId = ID, over = {}) => ({ videoId, vidiqScore: null, viaShortsUrl: false, page: 'home', ...over });
+const video = (videoId = ID, over = {}) => ({
+  videoId,
+  title: 'Title',
+  thumbnailUrl: 'https://i.ytimg.com/t.jpg',
+  channelId: 'UC1',
+  channelTitle: 'Chan',
+  views: 60000,
+  publishedAt: '2026-10-01T00:00:00Z',
+  durationSec: 600,
+  ...over,
+});
+const channel = (channelId, subscribers) => ({
+  channelId,
+  title: 'Chan',
+  url: 'https://www.youtube.com/@chan',
+  subscribers,
+});
+const run = (items, db, api) => processBatch(items, { settings: SETTINGS, db, api, now: NOW });
+
+test('saves a video whose score reaches the threshold', async () => {
+  const db = memoryDb();
+  const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] });
+
+  const result = await run([item(ID, { page: 'search' })], db, api);
+
+  assert.deepEqual(result, { checked: 1, saved: 1 });
+  assert.deepEqual(db.stores.outliers.get(ID), {
+    videoId: ID,
+    title: 'Title',
+    url: `https://www.youtube.com/watch?v=${ID}`,
+    thumbnailUrl: 'https://i.ytimg.com/t.jpg',
+    channelId: 'UC1',
+    channelTitle: 'Chan',
+    channelUrl: 'https://www.youtube.com/@chan',
+    subscribers: 10000,
+    views: 60000,
+    publishedAt: '2026-10-01T00:00:00Z',
+    durationSec: 600,
+    isShort: false,
+    score: 6,
+    vidiqScore: null,
+    foundOn: 'search',
+    firstSeenAt: NOW_ISO,
+    updatedAt: NOW_ISO,
+  });
+  assert.deepEqual(db.stores.seen.get(ID), { videoId: ID, checkedAt: NOW_ISO, status: 'ok' });
+  assert.deepEqual(db.stores.channels.get('UC1'), { ...channel('UC1', 10000), fetchedAt: NOW_ISO });
+});
+
+test('marks a below-threshold video as seen without saving it', async () => {
+  const db = memoryDb();
+  const api = fakeApi({ videos: [video()], channels: [channel('UC1', 100000)] });
+  const result = await run([item()], db, api);
+  assert.deepEqual(result, { checked: 1, saved: 0 });
+  assert.equal(db.stores.outliers.size, 0);
+  assert.equal(db.stores.seen.get(ID).status, 'ok');
+});
+
+test('does not save a video below minViews even with a huge score', async () => {
+  const db = memoryDb();
+  const api = fakeApi({ videos: [video(ID, { views: 9000 })], channels: [channel('UC1', 100)] });
+  await run([item()], db, api);
+  assert.equal(db.stores.outliers.size, 0);
+});
+
+test('saves a VidIQ-flagged video even when subscribers are hidden', async () => {
+  const db = memoryDb();
+  const api = fakeApi({ videos: [video()], channels: [channel('UC1', null)] });
+  await run([item(ID, { vidiqScore: 7.5 })], db, api);
+  const saved = db.stores.outliers.get(ID);
+  assert.equal(saved.score, null);
+  assert.equal(saved.subscribers, null);
+  assert.equal(saved.vidiqScore, 7.5);
+});
+
+test('does not save a video with hidden subscribers and no VidIQ score', async () => {
+  const db = memoryDb();
+  const api = fakeApi({ videos: [video()], channels: [channel('UC1', null)] });
+  await run([item()], db, api);
+  assert.equal(db.stores.outliers.size, 0);
+});
+
+test('treats a channel missing from the API response as unknown subscribers', async () => {
+  const db = memoryDb();
+  const api = fakeApi({ videos: [video()], channels: [] });
+  await run([item(ID, { vidiqScore: 4 })], db, api);
+  assert.equal(db.stores.outliers.get(ID).subscribers, null);
+  assert.equal(db.stores.outliers.get(ID).channelUrl, 'https://www.youtube.com/channel/UC1');
+});
+
+test('skips ids checked less than 24 hours ago', async () => {
+  const db = memoryDb();
+  db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(1), status: 'ok' });
+  const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] });
+  const result = await run([item()], db, api);
+  assert.deepEqual(result, { checked: 0, saved: 0 });
+  assert.equal(api.calls.videos.length, 0);
+});
+
+test('rechecks ids older than 24 hours', async () => {
+  const db = memoryDb();
+  db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(25), status: 'ok' });
+  const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] });
+  const result = await run([item()], db, api);
+  assert.deepEqual(result, { checked: 1, saved: 1 });
+});
+
+test('an item with a VidIQ score bypasses a fresh seen record', async () => {
+  const db = memoryDb();
+  db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(1), status: 'ok' });
+  const api = fakeApi({ videos: [video()], channels: [channel('UC1', 1000000)] });
+  const result = await run([item(ID, { vidiqScore: 4 })], db, api);
+  assert.deepEqual(result, { checked: 1, saved: 1 });
+  assert.equal(db.stores.outliers.get(ID).vidiqScore, 4);
+});
+
+test('a repeat sighting refreshes stats and keeps the original metadata', async () => {
+  const db = memoryDb();
+  db.stores.outliers.set(ID, {
+    videoId: ID,
+    firstSeenAt: '2026-10-20T00:00:00.000Z',
+    foundOn: 'home',
+    isShort: true,
+    vidiqScore: 9,
+  });
+  db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(30), status: 'ok' });
+  const api = fakeApi({ videos: [video(ID, { views: 70000 })], channels: [channel('UC1', 10000)] });
+
+  await run([item(ID, { page: 'search', viaShortsUrl: false })], db, api);
+
+  const saved = db.stores.outliers.get(ID);
+  assert.equal(saved.firstSeenAt, '2026-10-20T00:00:00.000Z');
+  assert.equal(saved.foundOn, 'home');
+  assert.equal(saved.isShort, true);
+  assert.equal(saved.vidiqScore, 9);
+  assert.equal(saved.views, 70000);
+  assert.equal(saved.score, 7);
+  assert.equal(saved.updatedAt, NOW_ISO);
+});
+
+test('an already saved video is updated even when its score fell below the threshold', async () => {
+  const db = memoryDb();
+  db.stores.outliers.set(ID, { videoId: ID, firstSeenAt: ago(100), foundOn: 'home', isShort: false, vidiqScore: null });
+  db.stores.seen.set(ID, { videoId: ID, checkedAt: ago(30), status: 'ok' });
+  const api = fakeApi({ videos: [video()], channels: [channel('UC1', 1000000)] });
+  const result = await run([item()], db, api);
+  assert.deepEqual(result, { checked: 1, saved: 1 });
+  assert.equal(db.stores.outliers.get(ID).score, 0.06);
+});
+
+test('a short found via a /shorts/ link is flagged', async () => {
+  const db = memoryDb();
+  const api = fakeApi({ videos: [video()], channels: [channel('UC1', 10000)] });
+  await run([item(ID, { viaShortsUrl: true })], db, api);
+  assert.equal(db.stores.outliers.get(ID).isShort, true);
+});
+
+test('records unavailable videos in seen and does not save them', async () => {
+  const db = memoryDb();
+  const api = fakeApi({ videos: [], channels: [] });
+  const result = await run([item()], db, api);
+  assert.deepEqual(result, { checked: 1, saved: 0 });
+  assert.deepEqual(db.stores.seen.get(ID), { videoId: ID, checkedAt: NOW_ISO, status: 'unavailable' });
+  assert.equal(db.stores.outliers.size, 0);
+});
+
+test('reuses a cached channel younger than 24 hours', async () => {
+  const db = memoryDb();
+  db.stores.channels.set('UC1', { ...channel('UC1', 10000), fetchedAt: ago(1) });
+  const api = fakeApi({ videos: [video()], channels: [] });
+  const result = await run([item()], db, api);
+  assert.equal(api.calls.channels.length, 0);
+  assert.deepEqual(result, { checked: 1, saved: 1 });
+});
+
+test('refetches a cached channel older than 24 hours', async () => {
+  const db = memoryDb();
+  db.stores.channels.set('UC1', { ...channel('UC1', 10000), fetchedAt: ago(25) });
+  const api = fakeApi({ videos: [video()], channels: [channel('UC1', 20000)] });
+  const result = await run([item()], db, api);
+  assert.deepEqual(api.calls.channels, [['UC1']]);
+  assert.equal(db.stores.channels.get('UC1').subscribers, 20000);
+  assert.deepEqual(result, { checked: 1, saved: 0 }); // score 3 < 5
+});
+
+test('an API failure writes nothing and does not mark ids as seen', async () => {
+  const db = memoryDb();
+  const api = {
+    ...fakeApi(),
+    fetchVideos: async () => {
+      throw new Error('boom');
+    },
+  };
+  await assert.rejects(run([item()], db, api), /boom/);
+  assert.equal(db.stores.seen.size, 0);
+  assert.equal(db.stores.outliers.size, 0);
+  assert.equal(db.stores.channels.size, 0);
+});