feat: scrape video cards and verify against real markup in headless Chrome

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-25 12:07:08 +03:00
parent 386b88bab5
commit 6de9e9656c
6 changed files with 398 additions and 0 deletions
@@ -0,0 +1,118 @@
+import { after, before, test } from 'node:test';
+import assert from 'node:assert/strict';
+import { fileURLToPath } from 'node:url';
+import { CHROME_PATH, launchBrowser, startStaticServer } from '../tools/browser.js';
+
+const skip = CHROME_PATH ? false : 'Chrome is not installed';
+let server;
+let browser;
+let page;
+
+before(async () => {
+  if (skip) return;
+  server = await startStaticServer(fileURLToPath(new URL('..', import.meta.url)));
+  browser = await launchBrowser();
+  page = await browser.openPage(`${server.origin}/test/fixtures/harness.html`);
+});
+
+after(async () => {
+  page?.close();
+  await browser?.close();
+  await server?.close();
+});
+
+// Puts a fixture into the page (wrapped in `wrapper`, as YouTube nests it), runs `prepare`
+// to tweak the DOM, then scrapes the first video link's card.
+const scrape = (fixture, { wrapper = 'ytd-rich-item-renderer', prepare = '', context = null } = {}) =>
+  page.evaluate(`(async () => {
+    const html = await (await fetch('/test/fixtures/${fixture}')).text();
+    document.body.innerHTML = '<${wrapper}>' + html + '</${wrapper}>';
+    ${prepare}
+    const { scrapeCard, CARD_SELECTOR, LINK_SELECTOR } = await import('/src/lib/card.js');
+    const link = document.querySelector(LINK_SELECTOR);
+    return scrapeCard(link.closest(CARD_SELECTOR), link, ${JSON.stringify(context)});
+  })()`);
+
+test('reads every field from a real home-page card with VidIQ blocks', { skip }, async () => {
+  const item = await scrape('home-card.html');
+  assert.ok(item.thumbnailUrl.startsWith('https://i.ytimg.com/vi/Tts9NHL8D8I/'), item.thumbnailUrl);
+  assert.deepEqual(
+    { ...item, thumbnailUrl: null },
+    {
+      videoId: 'Tts9NHL8D8I',
+      isShort: false,
+      title: 'Agar.io - 88k Score on an empty Experimental Server',
+      thumbnailUrl: null,
+      channelTitle: 'Wun Wun',
+      channelUrl: 'https://www.youtube.com/@wunwun9522',
+      subscribers: 571000,
+      views: 59000000,
+      outlierScore: 146.19,
+    },
+  );
+});
+
+test('home card without VidIQ blocks still has views but no score or subscribers', { skip }, async () => {
+  const item = await scrape('home-card.html', {
+    prepare: "document.querySelectorAll('.vidiq-scope').forEach((el) => el.remove());",
+  });
+  assert.equal(item.outlierScore, null);
+  assert.equal(item.subscribers, null);
+  assert.equal(item.views, 59000000);
+});
+
+test('badge without a link falls back to the capped badge text', { skip }, async () => {
+  const item = await scrape('home-card.html', {
+    prepare: "document.querySelector('[data-hover-popover=\"OutlierPopover\"] a').removeAttribute('href');",
+  });
+  assert.equal(item.outlierScore, 100);
+});
+
+test('a /shorts/ link marks the item as a short', { skip }, async () => {
+  const item = await scrape('home-card.html', {
+    prepare: "document.querySelector('a[href*=\"/watch?v=\"]').setAttribute('href', '/shorts/abcdefghijk');",
+  });
+  assert.equal(item.videoId, 'abcdefghijk');
+  assert.equal(item.isShort, true);
+});
+
+test('search card: old renderer, subscribers inside the metadata line are not mistaken for views', { skip }, async () => {
+  const item = await scrape('search-card.html', { wrapper: 'div' });
+  assert.deepEqual(item, {
+    videoId: 'pyV68KvUwlw',
+    isShort: false,
+    title: 'Minecraft Beaten In 53 Seconds [FWR]',
+    thumbnailUrl: 'https://i.ytimg.com/vi/pyV68KvUwlw/hq720.jpg?sqp=abc&rs=def',
+    channelTitle: 'HanabiYaki',
+    channelUrl: 'https://www.youtube.com/@HanabiYaki',
+    subscribers: 111000,
+    views: 5100000,
+    outlierScore: 7.46,
+  });
+});
+
+test('sidebar card: channel name without a link, views taken from the metadata row', { skip }, async () => {
+  const item = await scrape('sidebar-card.html', { wrapper: 'div' });
+  assert.equal(item.title, 'CБОРНИК - В Гостях У Бабули 21-30 серии');
+  assert.equal(item.channelTitle, 'Уродская Анимация');
+  assert.equal(item.channelUrl, null);
+  assert.equal(item.views, 1800000);
+  assert.equal(item.subscribers, 2820000);
+  assert.equal(item.outlierScore, 12);
+});
+
+test('channel page: the channel comes from the page context when the card names none', { skip }, async () => {
+  const item = await scrape('sidebar-card.html', {
+    wrapper: 'div',
+    prepare: "document.querySelector('.ytContentMetadataViewModelMetadataRow').remove();",
+    context: { channelTitle: 'MrBeast', channelUrl: 'https://www.youtube.com/@MrBeast' },
+  });
+  assert.equal(item.channelTitle, 'MrBeast');
+  assert.equal(item.channelUrl, 'https://www.youtube.com/@MrBeast');
+  assert.equal(item.views, 1800000);
+});
+
+test('thumbnail falls back to a URL built from the video id', { skip }, async () => {
+  const item = await scrape('sidebar-card.html', { wrapper: 'div', prepare: "document.querySelector('img').remove();" });
+  assert.equal(item.thumbnailUrl, 'https://i.ytimg.com/vi/7KXuP7Ws8W8/mqdefault.jpg');
+});
@@ -0,0 +1,4 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>harness</title>
+<body></body>
@@ -0,0 +1,46 @@
+<!-- Synthetic: modelled on the search-result card structure observed on youtube.com on 2026-10-25;
+     the VidIQ blocks copy the structure of home-card.html. -->
+<ytd-video-renderer>
+  <div id="dismissible">
+    <ytd-thumbnail>
+      <a id="thumbnail" href="/watch?v=pyV68KvUwlw&amp;pp=ygUSbWluZWNyYWZ0IHNwZWVkcnVu">
+        <img src="https://i.ytimg.com/vi/pyV68KvUwlw/hq720.jpg?sqp=abc&amp;rs=def">
+        <div id="overlays"><span id="text" aria-label="1 минута 29 секунд">1:29</span></div>
+      </a>
+    </ytd-thumbnail>
+    <div class="text-wrapper">
+      <div id="meta">
+        <div id="title-wrapper">
+          <h3 class="title-and-badge">
+            <a id="video-title" href="/watch?v=pyV68KvUwlw&amp;pp=ygUSbWluZWNyYWZ0IHNwZWVkcnVu"
+               aria-label="Minecraft Beaten In 53 Seconds [FWR] 1 минута 29 секунд" title="Minecraft Beaten In 53 Seconds [FWR]">
+              <yt-formatted-string>Minecraft Beaten In 53 Seconds [FWR]</yt-formatted-string>
+            </a>
+          </h3>
+        </div>
+        <ytd-video-meta-block>
+          <div id="metadata">
+            <div id="byline-container">
+              <ytd-channel-name id="channel-name"><div id="container"><div id="text-container">
+                <yt-formatted-string id="text" title="HanabiYaki"><a href="/@HanabiYaki">HanabiYaki</a></yt-formatted-string>
+              </div></div></ytd-channel-name>
+            </div>
+            <div id="metadata-line">
+              <span class="inline-metadata-item">5,1&nbsp;млн</span>
+              <span class="inline-metadata-item">2 г. назад</span>
+              <div class="vidiq-scope">
+                <span data-vidiq-subscriber-count="">111&nbsp;тыс.</span><span>подписчики</span>
+              </div>
+            </div>
+          </div>
+          <span class="inline-video-stats vidiq-scope">
+            <div data-hover-popover="VphPopover"><span>29 VPH</span></div>
+            <div data-hover-popover="OutlierPopover">
+              <a href="https://app.vidiq.com/outliers?utm_source=chrome&amp;outlier_score=7.456&amp;video_id=pyV68KvUwlw"><span>7.5x</span></a>
+            </div>
+          </span>
+        </ytd-video-meta-block>
+      </div>
+    </div>
+  </div>
+</ytd-video-renderer>
@@ -0,0 +1,36 @@
+<!-- Synthetic: modelled on the watch-page sidebar lockup observed on youtube.com on 2026-10-25;
+     the VidIQ blocks copy the structure of home-card.html. -->
+<yt-lockup-view-model class="lockup">
+  <div class="ytLockupViewModelHost content-id-7KXuP7Ws8W8">
+    <a class="ytLockupViewModelContentImage" href="/watch?v=7KXuP7Ws8W8">
+      <yt-thumbnail-view-model><div class="ytThumbnailViewModelImage"><img src="https://i.ytimg.com/vi/7KXuP7Ws8W8/hqdefault.jpg"></div></yt-thumbnail-view-model>
+    </a>
+    <div class="ytLockupViewModelMetadata">
+      <yt-lockup-metadata-view-model><div class="ytLockupMetadataViewModelTextContainer">
+        <h3 class="ytLockupMetadataViewModelHeadingReset" title="CБОРНИК - В Гостях У Бабули 21-30 серии">
+          <a class="ytLockupMetadataViewModelTitle" href="/watch?v=7KXuP7Ws8W8"><span>CБОРНИК - В Гостях У Бабули 21-30 серии</span></a>
+        </h3>
+        <div class="ytLockupMetadataViewModelMetadata">
+          <yt-content-metadata-view-model>
+            <div class="ytContentMetadataViewModelMetadataRow">
+              <span class="ytAttributedStringHost ytContentMetadataViewModelMetadataText">Уродская Анимация</span>
+            </div>
+            <div class="ytContentMetadataViewModelMetadataRow">
+              <span class="ytAttributedStringHost ytContentMetadataViewModelMetadataText" aria-label="1,8 миллиона просмотров">1,8&nbsp;млн</span>
+              <span class="ytContentMetadataViewModelDelimiter"></span>
+              <span class="ytAttributedStringHost ytContentMetadataViewModelMetadataText" aria-label="3 недели назад">3 нед. назад</span>
+            </div>
+            <div class="vidiq-scope">
+              <span data-vidiq-subscriber-count="">2,82&nbsp;млн</span><span>подписчики</span>
+            </div>
+            <span class="inline-video-stats vidiq-scope">
+              <div data-hover-popover="OutlierPopover">
+                <a href="https://app.vidiq.com/outliers?outlier_score=12&amp;video_id=7KXuP7Ws8W8"><span>12x</span></a>
+              </div>
+            </span>
+          </yt-content-metadata-view-model>
+        </div>
+      </div></yt-lockup-metadata-view-model>
+    </div>
+  </div>
+</yt-lockup-view-model>