From ae9aba80e63d133130dd90037263483efbd2f589 Mon Sep 17 00:00:00 2001 From: Igor Volochay Date: Mon, 8 Jul 2024 12:55:47 +0300 Subject: [PATCH] add parser --- Ozon-parser.py | 153 ++++++++++++++++++++++++++++++++++++++++++++ get-product-info.py | 20 ++++++ 2 files changed, 173 insertions(+) create mode 100644 Ozon-parser.py create mode 100644 get-product-info.py diff --git a/Ozon-parser.py b/Ozon-parser.py new file mode 100644 index 0000000..d70916b --- /dev/null +++ b/Ozon-parser.py @@ -0,0 +1,153 @@ +import time +import json + +from selenium import webdriver +from selenium_stealth import stealth +from bs4 import BeautifulSoup + +from curl_cffi import requests + +def init_webdriver(): + driver = webdriver.Chrome() + stealth(driver, + languages=["en-US", "en"], + vendor="Google Inc.", + platform="Win32", + webgl_vendor="Intel Inc.", + renderer="Intel Iris OpenGL Engine", + fix_hairline=True) + driver.maximize_window() + return driver + +def scrolldown(driver, deep): + for _ in range(deep): + driver.execute_script('window.scrollBy(0, 500)') + time.sleep(0.1) + +def get_product_info(product_url): + session = requests.Session() + + raw_data = session.get("https://www.ozon.ru/api/composer-api.bx/page/json/v2?url=" + product_url) + json_data = json.loads(raw_data.content.decode()) + + full_name = json_data["seo"]["title"] + + if json_data["layout"][0]["component"] == "userAdultModal": + product_id = str(full_name.split()[-1])[1:-1] + print(product_id, full_name) + return (product_id, full_name, "Товар для лиц старше 18 лет", None, None) + else: + description = json.loads(json_data["seo"]["script"][0]["innerHTML"])["description"] + image_url = json.loads(json_data["seo"]["script"][0]["innerHTML"])["image"] + price = json.loads(json_data["seo"]["script"][0]["innerHTML"])["offers"]["price"] + " " +\ + json.loads(json_data["seo"]["script"][0]["innerHTML"])["offers"]["priceCurrency"] + product_id = json.loads(json_data["seo"]["script"][0]["innerHTML"])["sku"] + + return (product_id, full_name, description, price, image_url) + +def get_mainpage_cards(driver, url): + driver.get(url) + scrolldown(driver, 50) + main_page_html = BeautifulSoup(driver.page_source, "html.parser") + + content = main_page_html.find("div", {"class": "container"}) + content = content.findChildren(recursive=False)[-1].find("div") + content = content.findChildren(recursive=False) + content = [item for item in content if "freshIsland" in str(item)][-1] + content = content.find("div").find("div").find("div") + content = content.findChildren(recursive=False) + + all_cards = list() + for layer in content: + layer = layer.find("div") + cards = layer.findChildren(recursive=False) + + cards_in_layer = list() + for card in cards: + card = card.findChildren(recursive=False) + + card_name = card[2].find("span", {"class": "tsBody500Medium"}).contents[0] + card_url = card[2].find("a", href=True)["href"] + product_url = "https://ozon.ru/" + card_url + + product_id, full_name, description, price, image_url = get_product_info(card_url) + card_info = {product_id: {"short_name": card_name, + "full_name": full_name, + "description": description, + "url": product_url, + "price": price, + "image_url": image_url + } + } + cards_in_layer.append(card_info) + print(product_id, "- DONE") + + all_cards.extend(cards_in_layer) + return all_cards + +def get_searchpage_cards(driver, url, all_cards = []): + driver.get(url) + scrolldown(driver, 20) + search_page_html = BeautifulSoup(driver.page_source, "html.parser") + + content = search_page_html.find("div", {"id": "layoutPage"}) + content = content.find("div") + + content_with_cards = content.find("div", {"class": "widget-search-result-container"}) + content_with_cards = content_with_cards.find("div").findChildren(recursive=False) + + cards_in_page = list() + for card in content_with_cards: + card_url = card.find("a", href=True)["href"] + card_name = card.find("span", {"class": "tsBody500Medium"}).contents[0] + + product_url = "https://ozon.ru/" + card_url + + product_id, full_name, description, price, image_url = get_product_info(card_url) + card_info = {product_id: {"short_name": card_name, + "full_name": full_name, + "description": description, + "url": product_url, + "price": price, + "image_url": image_url + } + } + cards_in_page.append(card_info) + print(product_id, "- DONE") + + content_with_next = [div for div in content.find_all("a", href=True) if "Дальше" in str(div)] + if not content_with_next: + return cards_in_page + else: + next_page_url = "https://www.ozon.ru" + content_with_next[0]["href"] + all_cards.extend(get_searchpage_cards(driver, next_page_url, cards_in_page)) + return all_cards + + +if __name__ == "__main__": + url_ozon = "https://www.ozon.ru" + + driver = init_webdriver() + + search_list = ["пиво+светлое", "бандана+мужская+с+черепами", "сухарики+кириешки", "RTX+4090"] + end_list = list() + + try: + main_cards = get_mainpage_cards(driver, url_ozon) + print("Я успешно нашёл", len(main_cards), "на главной странице") + end_list.append("MAIN") + except: + print("Я упал на парсинге главной страницы") + + for search_tag in search_list: + url_search = f"https://www.ozon.ru/search/?text={search_tag}&from_global=true" + + try: + search_cards = get_searchpage_cards(driver, url_search) + print("Я успешно нашёл", len(search_cards), "по поиску", search_tag) + end_list.append(search_tag) + except: + print("Я упал на", search_tag) + print(end_list) + + driver.quit() diff --git a/get-product-info.py b/get-product-info.py new file mode 100644 index 0000000..46df158 --- /dev/null +++ b/get-product-info.py @@ -0,0 +1,20 @@ +from curl_cffi import requests +import json + +# формируем сессию и получаем куки +s = requests.Session() +s.get("https://www.ozon.ru") +print(*str(s.cookies).split(", "), sep="\n") + +url_prod = "https://www.ozon.ru/api/composer-api.bx/page/json/v2?url=/product/artbuk-h-r-a-m-monohromnyy-uzhas-h-r-a-m-1173897628/?avtc=1&avte=2&avts=1719315501" + +r = s.get("https://www.ozon.ru/api/composer-api.bx/page/json/v2?url=/product/futbolka-rustactic-politsiya-991026398") + +#print(r) +json_data = json.loads(r.content.decode()) +print(json_data["seo"]["title"]) +print(json.loads(json_data["seo"]["script"][0]["innerHTML"])["description"]) +print(json.loads(json_data["seo"]["script"][0]["innerHTML"])["image"]) +print(json.loads(json_data["seo"]["script"][0]["innerHTML"])["offers"]["price"] + " " + + json.loads(json_data["seo"]["script"][0]["innerHTML"])["offers"]["priceCurrency"]) +print(json.loads(json_data["seo"]["script"][0]["innerHTML"])["sku"]) \ No newline at end of file