add parser

Ozon-parser.py

@@ -0,0 +1,153 @@
+import time
+import json
+
+from selenium import webdriver
+from selenium_stealth import stealth
+from bs4 import BeautifulSoup
+
+from curl_cffi import requests
+
+def init_webdriver():
+    driver = webdriver.Chrome()
+    stealth(driver,
+            languages=["en-US", "en"],
+            vendor="Google Inc.",
+            platform="Win32",
+            webgl_vendor="Intel Inc.",
+            renderer="Intel Iris OpenGL Engine",
+            fix_hairline=True)
+    driver.maximize_window()
+    return driver
+
+def scrolldown(driver, deep):
+    for _ in range(deep):
+        driver.execute_script('window.scrollBy(0, 500)')
+        time.sleep(0.1)
+
+def get_product_info(product_url):
+    session = requests.Session()
+
+    raw_data = session.get("https://www.ozon.ru/api/composer-api.bx/page/json/v2?url=" + product_url)
+    json_data = json.loads(raw_data.content.decode())
+
+    full_name = json_data["seo"]["title"]
+
+    if json_data["layout"][0]["component"] == "userAdultModal":
+        product_id = str(full_name.split()[-1])[1:-1]
+        print(product_id, full_name)
+        return (product_id, full_name, "Товар для лиц старше 18 лет", None, None)
+    else:
+        description = json.loads(json_data["seo"]["script"][0]["innerHTML"])["description"]
+        image_url = json.loads(json_data["seo"]["script"][0]["innerHTML"])["image"]
+        price = json.loads(json_data["seo"]["script"][0]["innerHTML"])["offers"]["price"] + " " +\
+                json.loads(json_data["seo"]["script"][0]["innerHTML"])["offers"]["priceCurrency"]
+        product_id = json.loads(json_data["seo"]["script"][0]["innerHTML"])["sku"]
+
+        return (product_id, full_name, description, price, image_url)
+
+def get_mainpage_cards(driver, url):
+    driver.get(url)
+    scrolldown(driver, 50)
+    main_page_html = BeautifulSoup(driver.page_source, "html.parser")
+
+    content = main_page_html.find("div", {"class": "container"})
+    content = content.findChildren(recursive=False)[-1].find("div")
+    content = content.findChildren(recursive=False)
+    content = [item for item in content if "freshIsland" in str(item)][-1]
+    content = content.find("div").find("div").find("div")
+    content = content.findChildren(recursive=False)
+
+    all_cards = list()
+    for layer in content:
+        layer = layer.find("div")
+        cards = layer.findChildren(recursive=False)
+
+        cards_in_layer = list()
+        for card in cards:
+            card = card.findChildren(recursive=False)
+
+            card_name = card[2].find("span", {"class": "tsBody500Medium"}).contents[0]
+            card_url = card[2].find("a", href=True)["href"]
+            product_url = "https://ozon.ru/" + card_url
+
+            product_id, full_name, description, price, image_url = get_product_info(card_url)
+            card_info = {product_id: {"short_name": card_name,
+                                      "full_name": full_name,
+                                      "description": description,
+                                      "url": product_url,
+                                      "price": price,
+                                      "image_url": image_url
+                                      }
+                         }
+            cards_in_layer.append(card_info)
+            print(product_id, "- DONE")
+
+        all_cards.extend(cards_in_layer)
+    return all_cards
+
+def get_searchpage_cards(driver, url, all_cards = []):
+    driver.get(url)
+    scrolldown(driver, 20)
+    search_page_html = BeautifulSoup(driver.page_source, "html.parser")
+
+    content = search_page_html.find("div", {"id": "layoutPage"})
+    content = content.find("div")
+
+    content_with_cards = content.find("div", {"class": "widget-search-result-container"})
+    content_with_cards = content_with_cards.find("div").findChildren(recursive=False)
+
+    cards_in_page = list()
+    for card in content_with_cards:
+        card_url = card.find("a", href=True)["href"]
+        card_name = card.find("span", {"class": "tsBody500Medium"}).contents[0]
+
+        product_url = "https://ozon.ru/" + card_url
+
+        product_id, full_name, description, price, image_url = get_product_info(card_url)
+        card_info = {product_id: {"short_name": card_name,
+                                  "full_name": full_name,
+                                  "description": description,
+                                  "url": product_url,
+                                  "price": price,
+                                  "image_url": image_url
+                                  }
+                     }
+        cards_in_page.append(card_info)
+        print(product_id, "- DONE")
+
+    content_with_next = [div for div in content.find_all("a", href=True) if "Дальше" in str(div)]
+    if not content_with_next:
+        return cards_in_page
+    else:
+        next_page_url = "https://www.ozon.ru" + content_with_next[0]["href"]
+        all_cards.extend(get_searchpage_cards(driver, next_page_url, cards_in_page))
+        return all_cards
+
+
+if __name__ == "__main__":
+    url_ozon = "https://www.ozon.ru"
+
+    driver = init_webdriver()
+
+    search_list = ["пиво+светлое", "бандана+мужская+с+черепами", "сухарики+кириешки", "RTX+4090"]
+    end_list = list()
+
+    try:
+        main_cards = get_mainpage_cards(driver, url_ozon)
+        print("Я успешно нашёл", len(main_cards), "на главной странице")
+        end_list.append("MAIN")
+    except:
+        print("Я упал на парсинге главной страницы")
+
+    for search_tag in search_list:
+        url_search = f"https://www.ozon.ru/search/?text={search_tag}&from_global=true"
+
+        try:
+            search_cards = get_searchpage_cards(driver, url_search)
+            print("Я успешно нашёл", len(search_cards), "по поиску", search_tag)
+            end_list.append(search_tag)
+        except:
+            print("Я упал на", search_tag)
+    print(end_list)
+
+    driver.quit()

get-product-info.py

@@ -0,0 +1,20 @@
+from curl_cffi import requests
+import json
+
+# формируем сессию и получаем куки
+s = requests.Session()
+s.get("https://www.ozon.ru")
+print(*str(s.cookies).split(", "), sep="\n")
+
+url_prod = "https://www.ozon.ru/api/composer-api.bx/page/json/v2?url=/product/artbuk-h-r-a-m-monohromnyy-uzhas-h-r-a-m-1173897628/?avtc=1&avte=2&avts=1719315501"
+
+r = s.get("https://www.ozon.ru/api/composer-api.bx/page/json/v2?url=/product/futbolka-rustactic-politsiya-991026398")
+
+#print(r)
+json_data = json.loads(r.content.decode())
+print(json_data["seo"]["title"])
+print(json.loads(json_data["seo"]["script"][0]["innerHTML"])["description"])
+print(json.loads(json_data["seo"]["script"][0]["innerHTML"])["image"])
+print(json.loads(json_data["seo"]["script"][0]["innerHTML"])["offers"]["price"] + " " +
+      json.loads(json_data["seo"]["script"][0]["innerHTML"])["offers"]["priceCurrency"])
+print(json.loads(json_data["seo"]["script"][0]["innerHTML"])["sku"])