feat: scrape video cards and verify against real markup in headless Chrome

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-25 12:07:08 +03:00
parent 386b88bab5
commit 6de9e9656c
6 changed files with 398 additions and 0 deletions
@@ -0,0 +1,118 @@
+import { after, before, test } from 'node:test';
+import assert from 'node:assert/strict';
+import { fileURLToPath } from 'node:url';
+import { CHROME_PATH, launchBrowser, startStaticServer } from '../tools/browser.js';
+
+const skip = CHROME_PATH ? false : 'Chrome is not installed';
+let server;
+let browser;
+let page;
+
+before(async () => {
+  if (skip) return;
+  server = await startStaticServer(fileURLToPath(new URL('..', import.meta.url)));
+  browser = await launchBrowser();
+  page = await browser.openPage(`${server.origin}/test/fixtures/harness.html`);
+});
+
+after(async () => {
+  page?.close();
+  await browser?.close();
+  await server?.close();
+});
+
+// Puts a fixture into the page (wrapped in `wrapper`, as YouTube nests it), runs `prepare`
+// to tweak the DOM, then scrapes the first video link's card.
+const scrape = (fixture, { wrapper = 'ytd-rich-item-renderer', prepare = '', context = null } = {}) =>
+  page.evaluate(`(async () => {
+    const html = await (await fetch('/test/fixtures/${fixture}')).text();
+    document.body.innerHTML = '<${wrapper}>' + html + '</${wrapper}>';
+    ${prepare}
+    const { scrapeCard, CARD_SELECTOR, LINK_SELECTOR } = await import('/src/lib/card.js');
+    const link = document.querySelector(LINK_SELECTOR);
+    return scrapeCard(link.closest(CARD_SELECTOR), link, ${JSON.stringify(context)});
+  })()`);
+
+test('reads every field from a real home-page card with VidIQ blocks', { skip }, async () => {
+  const item = await scrape('home-card.html');
+  assert.ok(item.thumbnailUrl.startsWith('https://i.ytimg.com/vi/Tts9NHL8D8I/'), item.thumbnailUrl);
+  assert.deepEqual(
+    { ...item, thumbnailUrl: null },
+    {
+      videoId: 'Tts9NHL8D8I',
+      isShort: false,
+      title: 'Agar.io - 88k Score on an empty Experimental Server',
+      thumbnailUrl: null,
+      channelTitle: 'Wun Wun',
+      channelUrl: 'https://www.youtube.com/@wunwun9522',
+      subscribers: 571000,
+      views: 59000000,
+      outlierScore: 146.19,
+    },
+  );
+});
+
+test('home card without VidIQ blocks still has views but no score or subscribers', { skip }, async () => {
+  const item = await scrape('home-card.html', {
+    prepare: "document.querySelectorAll('.vidiq-scope').forEach((el) => el.remove());",
+  });
+  assert.equal(item.outlierScore, null);
+  assert.equal(item.subscribers, null);
+  assert.equal(item.views, 59000000);
+});
+
+test('badge without a link falls back to the capped badge text', { skip }, async () => {
+  const item = await scrape('home-card.html', {
+    prepare: "document.querySelector('[data-hover-popover=\"OutlierPopover\"] a').removeAttribute('href');",
+  });
+  assert.equal(item.outlierScore, 100);
+});
+
+test('a /shorts/ link marks the item as a short', { skip }, async () => {
+  const item = await scrape('home-card.html', {
+    prepare: "document.querySelector('a[href*=\"/watch?v=\"]').setAttribute('href', '/shorts/abcdefghijk');",
+  });
+  assert.equal(item.videoId, 'abcdefghijk');
+  assert.equal(item.isShort, true);
+});
+
+test('search card: old renderer, subscribers inside the metadata line are not mistaken for views', { skip }, async () => {
+  const item = await scrape('search-card.html', { wrapper: 'div' });
+  assert.deepEqual(item, {
+    videoId: 'pyV68KvUwlw',
+    isShort: false,
+    title: 'Minecraft Beaten In 53 Seconds [FWR]',
+    thumbnailUrl: 'https://i.ytimg.com/vi/pyV68KvUwlw/hq720.jpg?sqp=abc&rs=def',
+    channelTitle: 'HanabiYaki',
+    channelUrl: 'https://www.youtube.com/@HanabiYaki',
+    subscribers: 111000,
+    views: 5100000,
+    outlierScore: 7.46,
+  });
+});
+
+test('sidebar card: channel name without a link, views taken from the metadata row', { skip }, async () => {
+  const item = await scrape('sidebar-card.html', { wrapper: 'div' });
+  assert.equal(item.title, 'CБОРНИК - В Гостях У Бабули 21-30 серии');
+  assert.equal(item.channelTitle, 'Уродская Анимация');
+  assert.equal(item.channelUrl, null);
+  assert.equal(item.views, 1800000);
+  assert.equal(item.subscribers, 2820000);
+  assert.equal(item.outlierScore, 12);
+});
+
+test('channel page: the channel comes from the page context when the card names none', { skip }, async () => {
+  const item = await scrape('sidebar-card.html', {
+    wrapper: 'div',
+    prepare: "document.querySelector('.ytContentMetadataViewModelMetadataRow').remove();",
+    context: { channelTitle: 'MrBeast', channelUrl: 'https://www.youtube.com/@MrBeast' },
+  });
+  assert.equal(item.channelTitle, 'MrBeast');
+  assert.equal(item.channelUrl, 'https://www.youtube.com/@MrBeast');
+  assert.equal(item.views, 1800000);
+});
+
+test('thumbnail falls back to a URL built from the video id', { skip }, async () => {
+  const item = await scrape('sidebar-card.html', { wrapper: 'div', prepare: "document.querySelector('img').remove();" });
+  assert.equal(item.thumbnailUrl, 'https://i.ytimg.com/vi/7KXuP7Ws8W8/mqdefault.jpg');
+});