Create crawler + downloader

2023-12-22 18:06:42 +03:00
parent 3c71f14fb3
commit ae15ef9ce3
1 changed files with 64 additions and 0 deletions
@@ -0,0 +1,64 @@
+import time
+
+from requests import get
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse, unquote
+from pywebcopy import save_webpage
+
+def get_all_links(url, deph, deph_counter=0):
+    all_links = {url}
+
+    if deph_counter < deph:
+        try:
+            req = get(url)
+            soup = BeautifulSoup(req.text, 'html.parser')
+        except:
+            print(f"С сайтом ({url}) какие-то проблемы. Ухожу в сон на 5 секунд...")
+            time.sleep(5)
+            try:
+                req = get(url)
+                soup = BeautifulSoup(req.text, 'html.parser')
+            except:
+                return
+
+        for link in soup.find_all('a'):
+            link = link.get('href')
+            try:
+                if link:
+                    if "http" in link:
+                        all_links = all_links.union(get_all_links(link, deph, deph_counter+1))
+                    elif link == "/":
+                        continue
+                    else:
+                        url = urlparse(url).scheme + "://" + urlparse(url).netloc
+                        all_links = all_links.union(get_all_links(url+link, deph, deph_counter+1))
+            except:
+                print(link)
+
+    return all_links
+
+def save_page(url):
+    dir_name = unquote(urlparse(url).netloc + urlparse(url).path)
+    for char in ["\\", "/", ":", "*", "?", "\"", "<", ">", "|"]:
+        dir_name = dir_name.replace(char, "!")
+
+    try:
+        save_webpage(
+            url=url,
+            project_folder="sites/",
+            project_name=str(dir_name),
+            open_in_browser=False,
+            threaded=True
+        )
+    except:
+        print(" ! Не удалось скачать:", dir_name, url)
+        return
+
+if __name__ == '__main__':
+    start_url = "https://habr.com/ru/articles/669766/"
+
+    links = get_all_links(start_url, 0)
+
+    print(len(links))
+    for link in links:
+        save_page(link)