65 lines
2.0 KiB
Python
65 lines
2.0 KiB
Python
import time
|
||
|
||
from requests import get
|
||
from bs4 import BeautifulSoup
|
||
from urllib.parse import urlparse, unquote
|
||
from pywebcopy import save_webpage
|
||
|
||
def get_all_links(url, deph, deph_counter=0):
|
||
all_links = {url}
|
||
|
||
if deph_counter < deph:
|
||
try:
|
||
req = get(url)
|
||
soup = BeautifulSoup(req.text, 'html.parser')
|
||
except:
|
||
print(f"С сайтом ({url}) какие-то проблемы. Ухожу в сон на 5 секунд...")
|
||
time.sleep(5)
|
||
try:
|
||
req = get(url)
|
||
soup = BeautifulSoup(req.text, 'html.parser')
|
||
except:
|
||
return
|
||
|
||
for link in soup.find_all('a'):
|
||
link = link.get('href')
|
||
try:
|
||
if link:
|
||
if "http" in link:
|
||
all_links = all_links.union(get_all_links(link, deph, deph_counter+1))
|
||
elif link == "/":
|
||
continue
|
||
else:
|
||
url = urlparse(url).scheme + "://" + urlparse(url).netloc
|
||
all_links = all_links.union(get_all_links(url+link, deph, deph_counter+1))
|
||
except:
|
||
print(link)
|
||
|
||
return all_links
|
||
|
||
def save_page(url):
|
||
dir_name = unquote(urlparse(url).netloc + urlparse(url).path)
|
||
for char in ["\\", "/", ":", "*", "?", "\"", "<", ">", "|"]:
|
||
dir_name = dir_name.replace(char, "!")
|
||
|
||
try:
|
||
save_webpage(
|
||
url=url,
|
||
project_folder="sites/",
|
||
project_name=str(dir_name),
|
||
open_in_browser=False,
|
||
threaded=True
|
||
)
|
||
except:
|
||
print(" ! Не удалось скачать:", dir_name, url)
|
||
return
|
||
|
||
if __name__ == '__main__':
|
||
start_url = "https://habr.com/ru/articles/669766/"
|
||
|
||
links = get_all_links(start_url, 0)
|
||
|
||
print(len(links))
|
||
for link in links:
|
||
save_page(link)
|