Тренируюсь на сайте спарсить авторов цитат и не могу понять из-за чего именно ошибка. Судя по коду ошибки, походу что-то не то с передаваемыми url, но что именно не знаю.
Это первый файл, который парсит:
import requests
from bs4 import BeautifulSoup
headers = {'User=Agent':
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 '
'(.NET CLR 3.5.30729)'}
def get_url():
links_list = []
for count_link in range(1, 100):
url = f'https://quotes.toscrape.com/page/{count_link}/'
resp = requests.get(url, headers=headers)
soup = BeautifulSoup(resp.text, 'lxml')
data = soup.find_all('div', class_='quote')
result = soup.find_all('span', class_='text')
if len(result) != 0:
for item in data:
link_author = 'https://quotes.toscrape.com' + item.find('a', text='(about)').get('href')
if link_author not in links_list:
links_list.append(link_author)
else:
break
yield links_list
def get_author():
for url in get_url():
resp = requests.get(url, headers=headers)
soup = BeautifulSoup(resp.text, 'lxml')
data = soup.find('div', class_='author-details')
name = data.find('h3', class_='author-title').text.strip()
date_of_birth = data.find('span', class_='author-born-date').text.strip()
place_of_birth = data.find('span', class_='author-born-location').text.strip()
description_author = data.find('div', class_='author-description').text.strip()
yield name, date_of_birth, place_of_birth, description_author
Это второй файл, который записывает данные в файл:
import xlsxwriter
from test_scrap import get_author
def writer(param):
book = xlsxwriter.Workbook(r'C:\code\test_scrap\data.xlsx')
page = book.add_worksheet('Author')
row = 0
column = 0
page.set_column('A:A', 30)
page.set_column('B:B', 30)
page.set_column('C:C', 30)
page.set_column('D:E', 100)
for item in param:
page.write(row, column, item[0])
page.write(row, column + 1, item[1])
page.write(row, column + 2, item[2])
page.write(row, column + 3, item[3])
row += 1
book.close()
writer(get_author())
А вот сам текст ошибки:
Traceback (most recent call last):
File "C:\code\test_scrap\writer_scrap.py", line 27, in <module>
writer(get_author())
File "C:\code\test_scrap\writer_scrap.py", line 17, in writer
for item in param:
File "C:\code\test_scrap\test_scrap.py", line 29, in get_author
resp = requests.get(url, headers=headers)
File "C:\code\venv\lib\site-packages\requests\api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "C:\code\venv\lib\site-packages\requests\api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "C:\code\venv\lib\site-packages\requests\sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
File "C:\code\venv\lib\site-packages\requests\sessions.py", line 695, in send
adapter = self.get_adapter(url=request.url)
File "C:\code\venv\lib\site-packages\requests\sessions.py", line 792, in get_adapter
raise InvalidSchema(f"No connection adapters were found for {url!r}")
requests.exceptions.InvalidSchema: No connection adapters were found for "['https://quotes.toscrape.com/author/Albert-Einstein', 'https://quotes.toscrape.com/author/J-K-Rowling', 'https://quotes.toscrape.com/author/Jane-Austen', 'https://quotes.toscrape.com/author/Marilyn-Monroe', 'https://quotes.toscrape.com/author/Andre-Gide', 'https://quotes.toscrape.com/author/Thomas-A-Edison', 'https://quotes.toscrape.com/author/Eleanor-Roosevelt', 'https://quotes.toscrape.com/author/Steve-Martin', 'https://quotes.toscrape.com/author/Bob-Marley', 'https://quotes.toscrape.com/author/Dr-Seuss', 'https://quotes.toscrape.com/author/Douglas-Adams', 'https://quotes.toscrape.com/author/Elie-Wiesel', 'https://quotes.toscrape.com/author/Friedrich-Nietzsche', 'https://quotes.toscrape.com/author/Mark-Twain', 'https://quotes.toscrape.com/author/Allen-Saunders', 'https://quotes.toscrape.com/author/Pablo-Neruda', 'https://quotes.toscrape.com/author/Ralph-Waldo-Emerson', 'https://quotes.toscrape.com/author/Mother-Teresa', 'https://quotes.toscrape.com/author/Garrison-Keillor', 'https://quotes.toscrape.com/author/Jim-Henson', 'https://quotes.toscrape.com/author/Charles-M-Schulz', 'https://quotes.toscrape.com/author/William-Nicholson', 'https://quotes.toscrape.com/author/Jorge-Luis-Borges', 'https://quotes.toscrape.com/author/George-Eliot', 'https://quotes.toscrape.com/author/George-R-R-Martin', 'https://quotes.toscrape.com/author/C-S-Lewis', 'https://quotes.toscrape.com/author/Martin-Luther-King-Jr', 'https://quotes.toscrape.com/author/James-Baldwin', 'https://quotes.toscrape.com/author/Haruki-Murakami', 'https://quotes.toscrape.com/author/Alexandre-Dumas-fils', 'https://quotes.toscrape.com/author/Stephenie-Meyer', 'https://quotes.toscrape.com/author/Ernest-Hemingway', 'https://quotes.toscrape.com/author/Helen-Keller', 'https://quotes.toscrape.com/author/George-Bernard-Shaw', 'https://quotes.toscrape.com/author/Charles-Bukowski', 'https://quotes.toscrape.com/author/Suzanne-Collins', 'https://quotes.toscrape.com/author/J-R-R-Tolkien', 'https://quotes.toscrape.com/author/Alfred-Tennyson', 'https://quotes.toscrape.com/author/Terry-Pratchett', 'https://quotes.toscrape.com/author/J-D-Salinger', 'https://quotes.toscrape.com/author/George-Carlin', 'https://quotes.toscrape.com/author/John-Lennon', 'https://quotes.toscrape.com/author/W-C-Fields', 'https://quotes.toscrape.com/author/Ayn-Rand', 'https://quotes.toscrape.com/author/Jimi-Hendrix', 'https://quotes.toscrape.com/author/J-M-Barrie', 'https://quotes.toscrape.com/author/E-E-Cummings', 'https://quotes.toscrape.com/author/Khaled-Hosseini', 'https://quotes.toscrape.com/author/Harper-Lee', 'https://quotes.toscrape.com/author/Madeleine-LEngle']"
Process finished with exit code 1