Ozon-parser/Ozon-parser.py

import time
import json

from selenium import webdriver
from selenium_stealth import stealth
from bs4 import BeautifulSoup

from curl_cffi import requests

def init_webdriver():
    driver = webdriver.Chrome()
    stealth(driver,
            languages=["en-US", "en"],
            vendor="Google Inc.",
            platform="Win32",
            webgl_vendor="Intel Inc.",
            renderer="Intel Iris OpenGL Engine",
            fix_hairline=True)
    driver.maximize_window()
    return driver

def scrolldown(driver, deep):
    for _ in range(deep):
        driver.execute_script('window.scrollBy(0, 500)')
        time.sleep(0.1)

def get_product_info(product_url):
    session = requests.Session()

    raw_data = session.get("https://www.ozon.ru/api/composer-api.bx/page/json/v2?url=" + product_url)
    json_data = json.loads(raw_data.content.decode())

    full_name = json_data["seo"]["title"]

    if json_data["layout"][0]["component"] == "userAdultModal":
        product_id = str(full_name.split()[-1])[1:-1]
        print(product_id, full_name)
        return (product_id, full_name, "Товар для лиц старше 18 лет", None, None)
    else:
        description = json.loads(json_data["seo"]["script"][0]["innerHTML"])["description"]
        image_url = json.loads(json_data["seo"]["script"][0]["innerHTML"])["image"]
        price = json.loads(json_data["seo"]["script"][0]["innerHTML"])["offers"]["price"] + " " +\
                json.loads(json_data["seo"]["script"][0]["innerHTML"])["offers"]["priceCurrency"]
        product_id = json.loads(json_data["seo"]["script"][0]["innerHTML"])["sku"]

        return (product_id, full_name, description, price, image_url)

def get_mainpage_cards(driver, url):
    driver.get(url)
    scrolldown(driver, 50)
    main_page_html = BeautifulSoup(driver.page_source, "html.parser")

    content = main_page_html.find("div", {"class": "container"})
    content = content.findChildren(recursive=False)[-1].find("div")
    content = content.findChildren(recursive=False)
    content = [item for item in content if "freshIsland" in str(item)][-1]
    content = content.find("div").find("div").find("div")
    content = content.findChildren(recursive=False)

    all_cards = list()
    for layer in content:
        layer = layer.find("div")
        cards = layer.findChildren(recursive=False)

        cards_in_layer = list()
        for card in cards:
            card = card.findChildren(recursive=False)

            card_name = card[2].find("span", {"class": "tsBody500Medium"}).contents[0]
            card_url = card[2].find("a", href=True)["href"]
            product_url = "https://ozon.ru/" + card_url

            product_id, full_name, description, price, image_url = get_product_info(card_url)
            card_info = {product_id: {"short_name": card_name,
                                      "full_name": full_name,
                                      "description": description,
                                      "url": product_url,
                                      "price": price,
                                      "image_url": image_url
                                      }
                         }
            cards_in_layer.append(card_info)
            print(product_id, "- DONE")

        all_cards.extend(cards_in_layer)
    return all_cards

def get_searchpage_cards(driver, url, all_cards = []):
    driver.get(url)
    scrolldown(driver, 20)
    search_page_html = BeautifulSoup(driver.page_source, "html.parser")

    content = search_page_html.find("div", {"id": "layoutPage"})
    content = content.find("div")

    content_with_cards = content.find("div", {"class": "widget-search-result-container"})
    content_with_cards = content_with_cards.find("div").findChildren(recursive=False)

    cards_in_page = list()
    for card in content_with_cards:
        card_url = card.find("a", href=True)["href"]
        card_name = card.find("span", {"class": "tsBody500Medium"}).contents[0]

        product_url = "https://ozon.ru/" + card_url

        product_id, full_name, description, price, image_url = get_product_info(card_url)
        card_info = {product_id: {"short_name": card_name,
                                  "full_name": full_name,
                                  "description": description,
                                  "url": product_url,
                                  "price": price,
                                  "image_url": image_url
                                  }
                     }
        cards_in_page.append(card_info)
        print(product_id, "- DONE")

    content_with_next = [div for div in content.find_all("a", href=True) if "Дальше" in str(div)]
    if not content_with_next:
        return cards_in_page
    else:
        next_page_url = "https://www.ozon.ru" + content_with_next[0]["href"]
        all_cards.extend(get_searchpage_cards(driver, next_page_url, cards_in_page))
        return all_cards


if __name__ == "__main__":
    url_ozon = "https://www.ozon.ru"

    driver = init_webdriver()

    search_list = ["пиво+светлое", "бандана+мужская+с+черепами", "сухарики+кириешки", "RTX+4090"]
    end_list = list()

    try:
        main_cards = get_mainpage_cards(driver, url_ozon)
        print("Я успешно нашёл", len(main_cards), "на главной странице")
        end_list.append("MAIN")
    except:
        print("Я упал на парсинге главной страницы")

    for search_tag in search_list:
        url_search = f"https://www.ozon.ru/search/?text={search_tag}&from_global=true"

        try:
            search_cards = get_searchpage_cards(driver, url_search)
            print("Я успешно нашёл", len(search_cards), "по поиску", search_tag)
            end_list.append(search_tag)
        except:
            print("Я упал на", search_tag)
    print(end_list)

    driver.quit()