From 6de9e9656c3827680f5e83b79fddcd8c025d5c01 Mon Sep 17 00:00:00 2001 From: IgorVolochay Date: Sun, 25 Oct 2026 12:07:08 +0300 Subject: [PATCH] feat: scrape video cards and verify against real markup in headless Chrome Co-Authored-By: Claude Sonnet 5.5 --- src/lib/card.js | 87 +++++++++++++++++++++++ test/card.dom.test.js | 118 ++++++++++++++++++++++++++++++++ test/fixtures/harness.html | 4 ++ test/fixtures/search-card.html | 46 +++++++++++++ test/fixtures/sidebar-card.html | 36 ++++++++++ tools/browser.js | 107 +++++++++++++++++++++++++++++ 6 files changed, 398 insertions(+) create mode 100644 src/lib/card.js create mode 100644 test/card.dom.test.js create mode 100644 test/fixtures/harness.html create mode 100644 test/fixtures/search-card.html create mode 100644 test/fixtures/sidebar-card.html create mode 100644 tools/browser.js diff --git a/src/lib/card.js b/src/lib/card.js new file mode 100644 index 0000000..6183d0b --- /dev/null +++ b/src/lib/card.js @@ -0,0 +1,87 @@ +import { parseViews } from './numbers.js'; +import { parseVideoLink } from './video-id.js'; +import { readVidiqData } from './vidiq.js'; + +export const LINK_SELECTOR = 'a[href*="/watch?v="], a[href*="/shorts/"]'; +export const CARD_SELECTOR = [ + 'ytd-rich-item-renderer', + 'ytd-video-renderer', + 'ytd-compact-video-renderer', + 'ytd-grid-video-renderer', + 'yt-lockup-view-model', + 'ytm-shorts-lockup-view-model', + 'ytd-reel-item-renderer', +].join(','); + +// Spans that hold the channel name, the view count and the age in the new and old card layouts. +const METADATA_TEXT = '.ytContentMetadataViewModelMetadataText, .inline-metadata-item, #metadata-line span'; +const CHANNEL_LINK = 'a[href^="/@"], a[href^="/channel/"], a[href^="/c/"], a[href^="/user/"]'; +const RELATIVE_TIME = /назад|(^|\s)ago(\s|$)/i; + +// VidIQ puts its own text (subscribers, VPH, score) inside `.vidiq-scope`; it must not be read as YouTube's. +const outsideVidiq = (element) => !element.closest('.vidiq-scope'); + +function readTitle(card) { + const heading = card.querySelector('h3[title]')?.getAttribute('title')?.trim(); + if (heading) return heading; + const oldTitle = card.querySelector('#video-title'); + const oldText = (oldTitle?.getAttribute('title') || oldTitle?.textContent || '').trim(); + if (oldText) return oldText; + return card.querySelector('.ytLockupMetadataViewModelTitle')?.textContent.trim() || null; +} + +function readThumbnail(card, videoId) { + const image = card.querySelector('img[src^="https://i.ytimg.com/vi/"], img[src^="https://i.ytimg.com/vi_webp/"]'); + return image ? image.src : `https://i.ytimg.com/vi/${videoId}/mqdefault.jpg`; +} + +function readViews(card) { + for (const element of card.querySelectorAll(METADATA_TEXT)) { + if (!outsideVidiq(element)) continue; + const views = parseViews(element.textContent) ?? parseViews(element.getAttribute('aria-label')); + if (views !== null) return views; + } + return null; +} + +function absoluteChannelUrl(href) { + try { + const url = new URL(href, 'https://www.youtube.com'); + return url.origin + url.pathname; + } catch { + return null; + } +} + +function readChannel(card, context) { + const link = [...card.querySelectorAll(CHANNEL_LINK)].find((a) => outsideVidiq(a) && a.textContent.trim()); + if (link) return { channelTitle: link.textContent.trim(), channelUrl: absoluteChannelUrl(link.getAttribute('href')) }; + + // New layouts show the channel name as plain text: the first metadata text that is neither a count nor an age. + for (const element of card.querySelectorAll(METADATA_TEXT)) { + const text = element.textContent.trim(); + if (outsideVidiq(element) && text && parseViews(text) === null && !RELATIVE_TIME.test(text)) { + return { channelTitle: text, channelUrl: context?.channelUrl ?? null }; + } + } + return { channelTitle: context?.channelTitle ?? null, channelUrl: context?.channelUrl ?? null }; +} + +// Reads one video card. `context` supplies the channel on channel pages, where cards do not name it. +export function scrapeCard(card, link, context = null) { + const parsed = parseVideoLink(link.getAttribute('href')); + if (!parsed) return null; + const { outlierScore, subscribers } = readVidiqData(card); + const { channelTitle, channelUrl } = readChannel(card, context); + return { + videoId: parsed.videoId, + isShort: parsed.isShort, + title: readTitle(card), + thumbnailUrl: readThumbnail(card, parsed.videoId), + channelTitle, + channelUrl, + subscribers, + views: readViews(card), + outlierScore, + }; +} diff --git a/test/card.dom.test.js b/test/card.dom.test.js new file mode 100644 index 0000000..b890a9b --- /dev/null +++ b/test/card.dom.test.js @@ -0,0 +1,118 @@ +import { after, before, test } from 'node:test'; +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import { CHROME_PATH, launchBrowser, startStaticServer } from '../tools/browser.js'; + +const skip = CHROME_PATH ? false : 'Chrome is not installed'; +let server; +let browser; +let page; + +before(async () => { + if (skip) return; + server = await startStaticServer(fileURLToPath(new URL('..', import.meta.url))); + browser = await launchBrowser(); + page = await browser.openPage(`${server.origin}/test/fixtures/harness.html`); +}); + +after(async () => { + page?.close(); + await browser?.close(); + await server?.close(); +}); + +// Puts a fixture into the page (wrapped in `wrapper`, as YouTube nests it), runs `prepare` +// to tweak the DOM, then scrapes the first video link's card. +const scrape = (fixture, { wrapper = 'ytd-rich-item-renderer', prepare = '', context = null } = {}) => + page.evaluate(`(async () => { + const html = await (await fetch('/test/fixtures/${fixture}')).text(); + document.body.innerHTML = '<${wrapper}>' + html + ''; + ${prepare} + const { scrapeCard, CARD_SELECTOR, LINK_SELECTOR } = await import('/src/lib/card.js'); + const link = document.querySelector(LINK_SELECTOR); + return scrapeCard(link.closest(CARD_SELECTOR), link, ${JSON.stringify(context)}); + })()`); + +test('reads every field from a real home-page card with VidIQ blocks', { skip }, async () => { + const item = await scrape('home-card.html'); + assert.ok(item.thumbnailUrl.startsWith('https://i.ytimg.com/vi/Tts9NHL8D8I/'), item.thumbnailUrl); + assert.deepEqual( + { ...item, thumbnailUrl: null }, + { + videoId: 'Tts9NHL8D8I', + isShort: false, + title: 'Agar.io - 88k Score on an empty Experimental Server', + thumbnailUrl: null, + channelTitle: 'Wun Wun', + channelUrl: 'https://www.youtube.com/@wunwun9522', + subscribers: 571000, + views: 59000000, + outlierScore: 146.19, + }, + ); +}); + +test('home card without VidIQ blocks still has views but no score or subscribers', { skip }, async () => { + const item = await scrape('home-card.html', { + prepare: "document.querySelectorAll('.vidiq-scope').forEach((el) => el.remove());", + }); + assert.equal(item.outlierScore, null); + assert.equal(item.subscribers, null); + assert.equal(item.views, 59000000); +}); + +test('badge without a link falls back to the capped badge text', { skip }, async () => { + const item = await scrape('home-card.html', { + prepare: "document.querySelector('[data-hover-popover=\"OutlierPopover\"] a').removeAttribute('href');", + }); + assert.equal(item.outlierScore, 100); +}); + +test('a /shorts/ link marks the item as a short', { skip }, async () => { + const item = await scrape('home-card.html', { + prepare: "document.querySelector('a[href*=\"/watch?v=\"]').setAttribute('href', '/shorts/abcdefghijk');", + }); + assert.equal(item.videoId, 'abcdefghijk'); + assert.equal(item.isShort, true); +}); + +test('search card: old renderer, subscribers inside the metadata line are not mistaken for views', { skip }, async () => { + const item = await scrape('search-card.html', { wrapper: 'div' }); + assert.deepEqual(item, { + videoId: 'pyV68KvUwlw', + isShort: false, + title: 'Minecraft Beaten In 53 Seconds [FWR]', + thumbnailUrl: 'https://i.ytimg.com/vi/pyV68KvUwlw/hq720.jpg?sqp=abc&rs=def', + channelTitle: 'HanabiYaki', + channelUrl: 'https://www.youtube.com/@HanabiYaki', + subscribers: 111000, + views: 5100000, + outlierScore: 7.46, + }); +}); + +test('sidebar card: channel name without a link, views taken from the metadata row', { skip }, async () => { + const item = await scrape('sidebar-card.html', { wrapper: 'div' }); + assert.equal(item.title, 'CБОРНИК - В Гостях У Бабули 21-30 серии'); + assert.equal(item.channelTitle, 'Уродская Анимация'); + assert.equal(item.channelUrl, null); + assert.equal(item.views, 1800000); + assert.equal(item.subscribers, 2820000); + assert.equal(item.outlierScore, 12); +}); + +test('channel page: the channel comes from the page context when the card names none', { skip }, async () => { + const item = await scrape('sidebar-card.html', { + wrapper: 'div', + prepare: "document.querySelector('.ytContentMetadataViewModelMetadataRow').remove();", + context: { channelTitle: 'MrBeast', channelUrl: 'https://www.youtube.com/@MrBeast' }, + }); + assert.equal(item.channelTitle, 'MrBeast'); + assert.equal(item.channelUrl, 'https://www.youtube.com/@MrBeast'); + assert.equal(item.views, 1800000); +}); + +test('thumbnail falls back to a URL built from the video id', { skip }, async () => { + const item = await scrape('sidebar-card.html', { wrapper: 'div', prepare: "document.querySelector('img').remove();" }); + assert.equal(item.thumbnailUrl, 'https://i.ytimg.com/vi/7KXuP7Ws8W8/mqdefault.jpg'); +}); diff --git a/test/fixtures/harness.html b/test/fixtures/harness.html new file mode 100644 index 0000000..733cab6 --- /dev/null +++ b/test/fixtures/harness.html @@ -0,0 +1,4 @@ + + +harness + diff --git a/test/fixtures/search-card.html b/test/fixtures/search-card.html new file mode 100644 index 0000000..32746b5 --- /dev/null +++ b/test/fixtures/search-card.html @@ -0,0 +1,46 @@ + + +
+ + + +
1:29
+ + +
+
+ + +
+
+ +
+
+ 5,1 млн + 2 г. назад +
+ 111 тыс.подписчики +
+
+
+ +
29 VPH
+
+ 7.5x +
+ + +
+
+
+ diff --git a/test/fixtures/sidebar-card.html b/test/fixtures/sidebar-card.html new file mode 100644 index 0000000..7ff8cae --- /dev/null +++ b/test/fixtures/sidebar-card.html @@ -0,0 +1,36 @@ + + +
+ +
+ +
+
+

+ CБОРНИК - В Гостях У Бабули 21-30 серии +

+
+ +
+ Уродская Анимация +
+
+ 1,8 млн + + 3 нед. назад +
+
+ 2,82 млнподписчики +
+ +
+ 12x +
+ + +
+
+
+
+ diff --git a/tools/browser.js b/tools/browser.js new file mode 100644 index 0000000..e3d5907 --- /dev/null +++ b/tools/browser.js @@ -0,0 +1,107 @@ +import { spawn, spawnSync } from 'node:child_process'; +import { createServer } from 'node:http'; +import { existsSync, mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { extname, join, resolve, sep } from 'node:path'; + +const sleep = (ms) => new Promise((done) => setTimeout(done, ms)); + +export const CHROME_PATH = + ['google-chrome', 'google-chrome-stable', 'chromium', 'chromium-browser'].find( + (name) => spawnSync('which', [name], { stdio: 'ignore' }).status === 0, + ) ?? null; + +const TYPES = { '.html': 'text/html; charset=utf-8', '.js': 'text/javascript; charset=utf-8', '.css': 'text/css' }; + +// Serves a directory over http://127.0.0.1 so a page can import ES modules from it. +export function startStaticServer(root) { + const base = resolve(root); + const server = createServer((req, res) => { + const path = resolve(join(base, decodeURIComponent(new URL(req.url, 'http://localhost').pathname))); + let isFile = false; + try { + isFile = path.startsWith(base + sep) && statSync(path).isFile(); + } catch { + // not found + } + if (!isFile) { + res.writeHead(404).end('not found'); + return; + } + res.writeHead(200, { 'content-type': TYPES[extname(path)] ?? 'application/octet-stream' }); + res.end(readFileSync(path)); + }); + return new Promise((done) => { + server.listen(0, '127.0.0.1', () => { + done({ origin: `http://127.0.0.1:${server.address().port}`, close: () => new Promise((closed) => server.close(closed)) }); + }); + }); +} + +export async function launchBrowser() { + const profile = mkdtempSync(join(tmpdir(), 'outlier-saver-chrome-')); + // Chrome puts a unix socket in TMPDIR and dies with "Socket path too long" if that path is long. + const { TMPDIR: _ignored, ...env } = process.env; + const proc = spawn( + CHROME_PATH, + ['--headless=new', '--remote-debugging-port=0', `--user-data-dir=${profile}`, '--no-first-run', '--no-default-browser-check', 'about:blank'], + { stdio: 'ignore', env }, + ); + const portFile = join(profile, 'DevToolsActivePort'); + let port = null; + for (let attempt = 0; attempt < 80 && port === null; attempt += 1) { + if (existsSync(portFile)) port = Number(readFileSync(portFile, 'utf8').split('\n')[0]); + else await sleep(250); + } + if (port === null) { + proc.kill(); + throw new Error('Chrome did not start'); + } + + return { + async openPage(url) { + const target = await (await fetch(`http://127.0.0.1:${port}/json/new?${url}`, { method: 'PUT' })).json(); + const ws = new WebSocket(target.webSocketDebuggerUrl); + await new Promise((opened, failed) => { + ws.onopen = opened; + ws.onerror = failed; + }); + let nextId = 0; + const pending = new Map(); + ws.onmessage = ({ data }) => { + const message = JSON.parse(data); + const waiter = pending.get(message.id); + if (!waiter) return; + pending.delete(message.id); + if (message.error) waiter.fail(new Error(message.error.message)); + else waiter.done(message.result); + }; + const send = (method, params = {}) => + new Promise((done, fail) => { + nextId += 1; + pending.set(nextId, { done, fail }); + ws.send(JSON.stringify({ id: nextId, method, params })); + }); + const evaluate = async (expression) => { + const result = await send('Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true }); + if (result.exceptionDetails) { + throw new Error(result.exceptionDetails.exception?.description ?? result.exceptionDetails.text); + } + return result.result.value; + }; + // A fresh tab is "complete" on about:blank before it navigates, so wait for the real page. + for (let attempt = 0; attempt < 80; attempt += 1) { + const state = await evaluate('JSON.stringify([location.href, document.readyState])'); + const [href, readyState] = JSON.parse(state); + if (href !== 'about:blank' && readyState === 'complete') break; + await sleep(100); + } + return { evaluate, close: () => ws.close() }; + }, + async close() { + proc.kill('SIGTERM'); + await sleep(300); + rmSync(profile, { recursive: true, force: true, maxRetries: 3 }); + }, + }; +}