feat: scrape video cards and verify against real markup in headless Chrome

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-25 12:07:08 +03:00
parent 386b88bab5
commit 6de9e9656c
6 changed files with 398 additions and 0 deletions
@@ -0,0 +1,87 @@
+import { parseViews } from './numbers.js';
+import { parseVideoLink } from './video-id.js';
+import { readVidiqData } from './vidiq.js';
+
+export const LINK_SELECTOR = 'a[href*="/watch?v="], a[href*="/shorts/"]';
+export const CARD_SELECTOR = [
+  'ytd-rich-item-renderer',
+  'ytd-video-renderer',
+  'ytd-compact-video-renderer',
+  'ytd-grid-video-renderer',
+  'yt-lockup-view-model',
+  'ytm-shorts-lockup-view-model',
+  'ytd-reel-item-renderer',
+].join(',');
+
+// Spans that hold the channel name, the view count and the age in the new and old card layouts.
+const METADATA_TEXT = '.ytContentMetadataViewModelMetadataText, .inline-metadata-item, #metadata-line span';
+const CHANNEL_LINK = 'a[href^="/@"], a[href^="/channel/"], a[href^="/c/"], a[href^="/user/"]';
+const RELATIVE_TIME = /назад|(^|\s)ago(\s|$)/i;
+
+// VidIQ puts its own text (subscribers, VPH, score) inside `.vidiq-scope`; it must not be read as YouTube's.
+const outsideVidiq = (element) => !element.closest('.vidiq-scope');
+
+function readTitle(card) {
+  const heading = card.querySelector('h3[title]')?.getAttribute('title')?.trim();
+  if (heading) return heading;
+  const oldTitle = card.querySelector('#video-title');
+  const oldText = (oldTitle?.getAttribute('title') || oldTitle?.textContent || '').trim();
+  if (oldText) return oldText;
+  return card.querySelector('.ytLockupMetadataViewModelTitle')?.textContent.trim() || null;
+}
+
+function readThumbnail(card, videoId) {
+  const image = card.querySelector('img[src^="https://i.ytimg.com/vi/"], img[src^="https://i.ytimg.com/vi_webp/"]');
+  return image ? image.src : `https://i.ytimg.com/vi/${videoId}/mqdefault.jpg`;
+}
+
+function readViews(card) {
+  for (const element of card.querySelectorAll(METADATA_TEXT)) {
+    if (!outsideVidiq(element)) continue;
+    const views = parseViews(element.textContent) ?? parseViews(element.getAttribute('aria-label'));
+    if (views !== null) return views;
+  }
+  return null;
+}
+
+function absoluteChannelUrl(href) {
+  try {
+    const url = new URL(href, 'https://www.youtube.com');
+    return url.origin + url.pathname;
+  } catch {
+    return null;
+  }
+}
+
+function readChannel(card, context) {
+  const link = [...card.querySelectorAll(CHANNEL_LINK)].find((a) => outsideVidiq(a) && a.textContent.trim());
+  if (link) return { channelTitle: link.textContent.trim(), channelUrl: absoluteChannelUrl(link.getAttribute('href')) };
+
+  // New layouts show the channel name as plain text: the first metadata text that is neither a count nor an age.
+  for (const element of card.querySelectorAll(METADATA_TEXT)) {
+    const text = element.textContent.trim();
+    if (outsideVidiq(element) && text && parseViews(text) === null && !RELATIVE_TIME.test(text)) {
+      return { channelTitle: text, channelUrl: context?.channelUrl ?? null };
+    }
+  }
+  return { channelTitle: context?.channelTitle ?? null, channelUrl: context?.channelUrl ?? null };
+}
+
+// Reads one video card. `context` supplies the channel on channel pages, where cards do not name it.
+export function scrapeCard(card, link, context = null) {
+  const parsed = parseVideoLink(link.getAttribute('href'));
+  if (!parsed) return null;
+  const { outlierScore, subscribers } = readVidiqData(card);
+  const { channelTitle, channelUrl } = readChannel(card, context);
+  return {
+    videoId: parsed.videoId,
+    isShort: parsed.isShort,
+    title: readTitle(card),
+    thumbnailUrl: readThumbnail(card, parsed.videoId),
+    channelTitle,
+    channelUrl,
+    subscribers,
+    views: readViews(card),
+    outlierScore,
+  };
+}