feat: scrape video cards and verify against real markup in headless Chrome

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-25 12:07:08 +03:00
parent 386b88bab5
commit 6de9e9656c
6 changed files with 398 additions and 0 deletions
@@ -0,0 +1,87 @@
+import { parseViews } from './numbers.js';
+import { parseVideoLink } from './video-id.js';
+import { readVidiqData } from './vidiq.js';
+
+export const LINK_SELECTOR = 'a[href*="/watch?v="], a[href*="/shorts/"]';
+export const CARD_SELECTOR = [
+  'ytd-rich-item-renderer',
+  'ytd-video-renderer',
+  'ytd-compact-video-renderer',
+  'ytd-grid-video-renderer',
+  'yt-lockup-view-model',
+  'ytm-shorts-lockup-view-model',
+  'ytd-reel-item-renderer',
+].join(',');
+
+// Spans that hold the channel name, the view count and the age in the new and old card layouts.
+const METADATA_TEXT = '.ytContentMetadataViewModelMetadataText, .inline-metadata-item, #metadata-line span';
+const CHANNEL_LINK = 'a[href^="/@"], a[href^="/channel/"], a[href^="/c/"], a[href^="/user/"]';
+const RELATIVE_TIME = /назад|(^|\s)ago(\s|$)/i;
+
+// VidIQ puts its own text (subscribers, VPH, score) inside `.vidiq-scope`; it must not be read as YouTube's.
+const outsideVidiq = (element) => !element.closest('.vidiq-scope');
+
+function readTitle(card) {
+  const heading = card.querySelector('h3[title]')?.getAttribute('title')?.trim();
+  if (heading) return heading;
+  const oldTitle = card.querySelector('#video-title');
+  const oldText = (oldTitle?.getAttribute('title') || oldTitle?.textContent || '').trim();
+  if (oldText) return oldText;
+  return card.querySelector('.ytLockupMetadataViewModelTitle')?.textContent.trim() || null;
+}
+
+function readThumbnail(card, videoId) {
+  const image = card.querySelector('img[src^="https://i.ytimg.com/vi/"], img[src^="https://i.ytimg.com/vi_webp/"]');
+  return image ? image.src : `https://i.ytimg.com/vi/${videoId}/mqdefault.jpg`;
+}
+
+function readViews(card) {
+  for (const element of card.querySelectorAll(METADATA_TEXT)) {
+    if (!outsideVidiq(element)) continue;
+    const views = parseViews(element.textContent) ?? parseViews(element.getAttribute('aria-label'));
+    if (views !== null) return views;
+  }
+  return null;
+}
+
+function absoluteChannelUrl(href) {
+  try {
+    const url = new URL(href, 'https://www.youtube.com');
+    return url.origin + url.pathname;
+  } catch {
+    return null;
+  }
+}
+
+function readChannel(card, context) {
+  const link = [...card.querySelectorAll(CHANNEL_LINK)].find((a) => outsideVidiq(a) && a.textContent.trim());
+  if (link) return { channelTitle: link.textContent.trim(), channelUrl: absoluteChannelUrl(link.getAttribute('href')) };
+
+  // New layouts show the channel name as plain text: the first metadata text that is neither a count nor an age.
+  for (const element of card.querySelectorAll(METADATA_TEXT)) {
+    const text = element.textContent.trim();
+    if (outsideVidiq(element) && text && parseViews(text) === null && !RELATIVE_TIME.test(text)) {
+      return { channelTitle: text, channelUrl: context?.channelUrl ?? null };
+    }
+  }
+  return { channelTitle: context?.channelTitle ?? null, channelUrl: context?.channelUrl ?? null };
+}
+
+// Reads one video card. `context` supplies the channel on channel pages, where cards do not name it.
+export function scrapeCard(card, link, context = null) {
+  const parsed = parseVideoLink(link.getAttribute('href'));
+  if (!parsed) return null;
+  const { outlierScore, subscribers } = readVidiqData(card);
+  const { channelTitle, channelUrl } = readChannel(card, context);
+  return {
+    videoId: parsed.videoId,
+    isShort: parsed.isShort,
+    title: readTitle(card),
+    thumbnailUrl: readThumbnail(card, parsed.videoId),
+    channelTitle,
+    channelUrl,
+    subscribers,
+    views: readViews(card),
+    outlierScore,
+  };
+}
@@ -0,0 +1,118 @@
+import { after, before, test } from 'node:test';
+import assert from 'node:assert/strict';
+import { fileURLToPath } from 'node:url';
+import { CHROME_PATH, launchBrowser, startStaticServer } from '../tools/browser.js';
+
+const skip = CHROME_PATH ? false : 'Chrome is not installed';
+let server;
+let browser;
+let page;
+
+before(async () => {
+  if (skip) return;
+  server = await startStaticServer(fileURLToPath(new URL('..', import.meta.url)));
+  browser = await launchBrowser();
+  page = await browser.openPage(`${server.origin}/test/fixtures/harness.html`);
+});
+
+after(async () => {
+  page?.close();
+  await browser?.close();
+  await server?.close();
+});
+
+// Puts a fixture into the page (wrapped in `wrapper`, as YouTube nests it), runs `prepare`
+// to tweak the DOM, then scrapes the first video link's card.
+const scrape = (fixture, { wrapper = 'ytd-rich-item-renderer', prepare = '', context = null } = {}) =>
+  page.evaluate(`(async () => {
+    const html = await (await fetch('/test/fixtures/${fixture}')).text();
+    document.body.innerHTML = '<${wrapper}>' + html + '</${wrapper}>';
+    ${prepare}
+    const { scrapeCard, CARD_SELECTOR, LINK_SELECTOR } = await import('/src/lib/card.js');
+    const link = document.querySelector(LINK_SELECTOR);
+    return scrapeCard(link.closest(CARD_SELECTOR), link, ${JSON.stringify(context)});
+  })()`);
+
+test('reads every field from a real home-page card with VidIQ blocks', { skip }, async () => {
+  const item = await scrape('home-card.html');
+  assert.ok(item.thumbnailUrl.startsWith('https://i.ytimg.com/vi/Tts9NHL8D8I/'), item.thumbnailUrl);
+  assert.deepEqual(
+    { ...item, thumbnailUrl: null },
+    {
+      videoId: 'Tts9NHL8D8I',
+      isShort: false,
+      title: 'Agar.io - 88k Score on an empty Experimental Server',
+      thumbnailUrl: null,
+      channelTitle: 'Wun Wun',
+      channelUrl: 'https://www.youtube.com/@wunwun9522',
+      subscribers: 571000,
+      views: 59000000,
+      outlierScore: 146.19,
+    },
+  );
+});
+
+test('home card without VidIQ blocks still has views but no score or subscribers', { skip }, async () => {
+  const item = await scrape('home-card.html', {
+    prepare: "document.querySelectorAll('.vidiq-scope').forEach((el) => el.remove());",
+  });
+  assert.equal(item.outlierScore, null);
+  assert.equal(item.subscribers, null);
+  assert.equal(item.views, 59000000);
+});
+
+test('badge without a link falls back to the capped badge text', { skip }, async () => {
+  const item = await scrape('home-card.html', {
+    prepare: "document.querySelector('[data-hover-popover=\"OutlierPopover\"] a').removeAttribute('href');",
+  });
+  assert.equal(item.outlierScore, 100);
+});
+
+test('a /shorts/ link marks the item as a short', { skip }, async () => {
+  const item = await scrape('home-card.html', {
+    prepare: "document.querySelector('a[href*=\"/watch?v=\"]').setAttribute('href', '/shorts/abcdefghijk');",
+  });
+  assert.equal(item.videoId, 'abcdefghijk');
+  assert.equal(item.isShort, true);
+});
+
+test('search card: old renderer, subscribers inside the metadata line are not mistaken for views', { skip }, async () => {
+  const item = await scrape('search-card.html', { wrapper: 'div' });
+  assert.deepEqual(item, {
+    videoId: 'pyV68KvUwlw',
+    isShort: false,
+    title: 'Minecraft Beaten In 53 Seconds [FWR]',
+    thumbnailUrl: 'https://i.ytimg.com/vi/pyV68KvUwlw/hq720.jpg?sqp=abc&rs=def',
+    channelTitle: 'HanabiYaki',
+    channelUrl: 'https://www.youtube.com/@HanabiYaki',
+    subscribers: 111000,
+    views: 5100000,
+    outlierScore: 7.46,
+  });
+});
+
+test('sidebar card: channel name without a link, views taken from the metadata row', { skip }, async () => {
+  const item = await scrape('sidebar-card.html', { wrapper: 'div' });
+  assert.equal(item.title, 'CБОРНИК - В Гостях У Бабули 21-30 серии');
+  assert.equal(item.channelTitle, 'Уродская Анимация');
+  assert.equal(item.channelUrl, null);
+  assert.equal(item.views, 1800000);
+  assert.equal(item.subscribers, 2820000);
+  assert.equal(item.outlierScore, 12);
+});
+
+test('channel page: the channel comes from the page context when the card names none', { skip }, async () => {
+  const item = await scrape('sidebar-card.html', {
+    wrapper: 'div',
+    prepare: "document.querySelector('.ytContentMetadataViewModelMetadataRow').remove();",
+    context: { channelTitle: 'MrBeast', channelUrl: 'https://www.youtube.com/@MrBeast' },
+  });
+  assert.equal(item.channelTitle, 'MrBeast');
+  assert.equal(item.channelUrl, 'https://www.youtube.com/@MrBeast');
+  assert.equal(item.views, 1800000);
+});
+
+test('thumbnail falls back to a URL built from the video id', { skip }, async () => {
+  const item = await scrape('sidebar-card.html', { wrapper: 'div', prepare: "document.querySelector('img').remove();" });
+  assert.equal(item.thumbnailUrl, 'https://i.ytimg.com/vi/7KXuP7Ws8W8/mqdefault.jpg');
+});
@@ -0,0 +1,4 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>harness</title>
+<body></body>
@@ -0,0 +1,46 @@
+<!-- Synthetic: modelled on the search-result card structure observed on youtube.com on 2026-10-25;
+     the VidIQ blocks copy the structure of home-card.html. -->
+<ytd-video-renderer>
+  <div id="dismissible">
+    <ytd-thumbnail>
+      <a id="thumbnail" href="/watch?v=pyV68KvUwlw&amp;pp=ygUSbWluZWNyYWZ0IHNwZWVkcnVu">
+        <img src="https://i.ytimg.com/vi/pyV68KvUwlw/hq720.jpg?sqp=abc&amp;rs=def">
+        <div id="overlays"><span id="text" aria-label="1 минута 29 секунд">1:29</span></div>
+      </a>
+    </ytd-thumbnail>
+    <div class="text-wrapper">
+      <div id="meta">
+        <div id="title-wrapper">
+          <h3 class="title-and-badge">
+            <a id="video-title" href="/watch?v=pyV68KvUwlw&amp;pp=ygUSbWluZWNyYWZ0IHNwZWVkcnVu"
+               aria-label="Minecraft Beaten In 53 Seconds [FWR] 1 минута 29 секунд" title="Minecraft Beaten In 53 Seconds [FWR]">
+              <yt-formatted-string>Minecraft Beaten In 53 Seconds [FWR]</yt-formatted-string>
+            </a>
+          </h3>
+        </div>
+        <ytd-video-meta-block>
+          <div id="metadata">
+            <div id="byline-container">
+              <ytd-channel-name id="channel-name"><div id="container"><div id="text-container">
+                <yt-formatted-string id="text" title="HanabiYaki"><a href="/@HanabiYaki">HanabiYaki</a></yt-formatted-string>
+              </div></div></ytd-channel-name>
+            </div>
+            <div id="metadata-line">
+              <span class="inline-metadata-item">5,1&nbsp;млн</span>
+              <span class="inline-metadata-item">2 г. назад</span>
+              <div class="vidiq-scope">
+                <span data-vidiq-subscriber-count="">111&nbsp;тыс.</span><span>подписчики</span>
+              </div>
+            </div>
+          </div>
+          <span class="inline-video-stats vidiq-scope">
+            <div data-hover-popover="VphPopover"><span>29 VPH</span></div>
+            <div data-hover-popover="OutlierPopover">
+              <a href="https://app.vidiq.com/outliers?utm_source=chrome&amp;outlier_score=7.456&amp;video_id=pyV68KvUwlw"><span>7.5x</span></a>
+            </div>
+          </span>
+        </ytd-video-meta-block>
+      </div>
+    </div>
+  </div>
+</ytd-video-renderer>
@@ -0,0 +1,36 @@
+<!-- Synthetic: modelled on the watch-page sidebar lockup observed on youtube.com on 2026-10-25;
+     the VidIQ blocks copy the structure of home-card.html. -->
+<yt-lockup-view-model class="lockup">
+  <div class="ytLockupViewModelHost content-id-7KXuP7Ws8W8">
+    <a class="ytLockupViewModelContentImage" href="/watch?v=7KXuP7Ws8W8">
+      <yt-thumbnail-view-model><div class="ytThumbnailViewModelImage"><img src="https://i.ytimg.com/vi/7KXuP7Ws8W8/hqdefault.jpg"></div></yt-thumbnail-view-model>
+    </a>
+    <div class="ytLockupViewModelMetadata">
+      <yt-lockup-metadata-view-model><div class="ytLockupMetadataViewModelTextContainer">
+        <h3 class="ytLockupMetadataViewModelHeadingReset" title="CБОРНИК - В Гостях У Бабули 21-30 серии">
+          <a class="ytLockupMetadataViewModelTitle" href="/watch?v=7KXuP7Ws8W8"><span>CБОРНИК - В Гостях У Бабули 21-30 серии</span></a>
+        </h3>
+        <div class="ytLockupMetadataViewModelMetadata">
+          <yt-content-metadata-view-model>
+            <div class="ytContentMetadataViewModelMetadataRow">
+              <span class="ytAttributedStringHost ytContentMetadataViewModelMetadataText">Уродская Анимация</span>
+            </div>
+            <div class="ytContentMetadataViewModelMetadataRow">
+              <span class="ytAttributedStringHost ytContentMetadataViewModelMetadataText" aria-label="1,8 миллиона просмотров">1,8&nbsp;млн</span>
+              <span class="ytContentMetadataViewModelDelimiter"></span>
+              <span class="ytAttributedStringHost ytContentMetadataViewModelMetadataText" aria-label="3 недели назад">3 нед. назад</span>
+            </div>
+            <div class="vidiq-scope">
+              <span data-vidiq-subscriber-count="">2,82&nbsp;млн</span><span>подписчики</span>
+            </div>
+            <span class="inline-video-stats vidiq-scope">
+              <div data-hover-popover="OutlierPopover">
+                <a href="https://app.vidiq.com/outliers?outlier_score=12&amp;video_id=7KXuP7Ws8W8"><span>12x</span></a>
+              </div>
+            </span>
+          </yt-content-metadata-view-model>
+        </div>
+      </div></yt-lockup-metadata-view-model>
+    </div>
+  </div>
+</yt-lockup-view-model>
@@ -0,0 +1,107 @@
+import { spawn, spawnSync } from 'node:child_process';
+import { createServer } from 'node:http';
+import { existsSync, mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { extname, join, resolve, sep } from 'node:path';
+
+const sleep = (ms) => new Promise((done) => setTimeout(done, ms));
+
+export const CHROME_PATH =
+  ['google-chrome', 'google-chrome-stable', 'chromium', 'chromium-browser'].find(
+    (name) => spawnSync('which', [name], { stdio: 'ignore' }).status === 0,
+  ) ?? null;
+
+const TYPES = { '.html': 'text/html; charset=utf-8', '.js': 'text/javascript; charset=utf-8', '.css': 'text/css' };
+
+// Serves a directory over http://127.0.0.1 so a page can import ES modules from it.
+export function startStaticServer(root) {
+  const base = resolve(root);
+  const server = createServer((req, res) => {
+    const path = resolve(join(base, decodeURIComponent(new URL(req.url, 'http://localhost').pathname)));
+    let isFile = false;
+    try {
+      isFile = path.startsWith(base + sep) && statSync(path).isFile();
+    } catch {
+      // not found
+    }
+    if (!isFile) {
+      res.writeHead(404).end('not found');
+      return;
+    }
+    res.writeHead(200, { 'content-type': TYPES[extname(path)] ?? 'application/octet-stream' });
+    res.end(readFileSync(path));
+  });
+  return new Promise((done) => {
+    server.listen(0, '127.0.0.1', () => {
+      done({ origin: `http://127.0.0.1:${server.address().port}`, close: () => new Promise((closed) => server.close(closed)) });
+    });
+  });
+}
+
+export async function launchBrowser() {
+  const profile = mkdtempSync(join(tmpdir(), 'outlier-saver-chrome-'));
+  // Chrome puts a unix socket in TMPDIR and dies with "Socket path too long" if that path is long.
+  const { TMPDIR: _ignored, ...env } = process.env;
+  const proc = spawn(
+    CHROME_PATH,
+    ['--headless=new', '--remote-debugging-port=0', `--user-data-dir=${profile}`, '--no-first-run', '--no-default-browser-check', 'about:blank'],
+    { stdio: 'ignore', env },
+  );
+  const portFile = join(profile, 'DevToolsActivePort');
+  let port = null;
+  for (let attempt = 0; attempt < 80 && port === null; attempt += 1) {
+    if (existsSync(portFile)) port = Number(readFileSync(portFile, 'utf8').split('\n')[0]);
+    else await sleep(250);
+  }
+  if (port === null) {
+    proc.kill();
+    throw new Error('Chrome did not start');
+  }
+
+  return {
+    async openPage(url) {
+      const target = await (await fetch(`http://127.0.0.1:${port}/json/new?${url}`, { method: 'PUT' })).json();
+      const ws = new WebSocket(target.webSocketDebuggerUrl);
+      await new Promise((opened, failed) => {
+        ws.onopen = opened;
+        ws.onerror = failed;
+      });
+      let nextId = 0;
+      const pending = new Map();
+      ws.onmessage = ({ data }) => {
+        const message = JSON.parse(data);
+        const waiter = pending.get(message.id);
+        if (!waiter) return;
+        pending.delete(message.id);
+        if (message.error) waiter.fail(new Error(message.error.message));
+        else waiter.done(message.result);
+      };
+      const send = (method, params = {}) =>
+        new Promise((done, fail) => {
+          nextId += 1;
+          pending.set(nextId, { done, fail });
+          ws.send(JSON.stringify({ id: nextId, method, params }));
+        });
+      const evaluate = async (expression) => {
+        const result = await send('Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true });
+        if (result.exceptionDetails) {
+          throw new Error(result.exceptionDetails.exception?.description ?? result.exceptionDetails.text);
+        }
+        return result.result.value;
+      };
+      // A fresh tab is "complete" on about:blank before it navigates, so wait for the real page.
+      for (let attempt = 0; attempt < 80; attempt += 1) {
+        const state = await evaluate('JSON.stringify([location.href, document.readyState])');
+        const [href, readyState] = JSON.parse(state);
+        if (href !== 'about:blank' && readyState === 'complete') break;
+        await sleep(100);
+      }
+      return { evaluate, close: () => ws.close() };
+    },
+    async close() {
+      proc.kill('SIGTERM');
+      await sleep(300);
+      rmSync(profile, { recursive: true, force: true, maxRetries: 3 });
+    },
+  };
+}