Ссылка на код: https://repl.it/@VitalikBerezous/TiredTurboPolygon#main.py
При написании парсера на Python с помощью requests и BeautifulSoup4 появляется ошибка несовместимости типов данных.
Код:
import requests
from bs4 import BeautifulSoup
URL = 'https://auto.ria.com/uk/newauto/marka-jeep/'
HEADERS = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0', 'accept': '*/*'}
HOST = 'https://auto.ria.com'
def get_html(url, params=None):
r = requests.get(url, headers = HEADERS, params= params)
return r
def get_pages_count(html):
soup = BeautifulSoup(html, 'html.parser')
pagination = soup.find_all('span', class_='mhide')
if pagination:
return int(pagination[-1].get_text())
else:
return 1
print(pagination)
def get_content(html):
soup = BeautifulSoup(html, 'html.parser')
items = soup.find_all('div', class_='proposition')
cars = []
for item in items:
uah_price = item.find('span', class_='grey size13')
if uah_price:
uah_price = uah_price.get_text(strip=True)
else:
uah_price = 'Цену уточняйте'
cars.append({
'title': item.find('h3', class_='proposition_name').get_text(strip=True),
'link': HOST + item.find('h3', class_='proposition_name').a.get('href'),
'usd_price': item.find('span', class_='size18').get_text(strip=True),
'uah_price': uah_price,
'city': item.find('div', class_='proposition_region size13').strong.get_text(strip=True),
})
print(cars)
def parse():
html = get_html(URL)
if html.status_code == 200:
cars = []
pages_count = get_pages_count(html.text)
for page in range(1, pages_count + 1):
print(f'Парсинг страницы {page} из {pages_count}...')
html = get_html(URL, params={'page': page})
cars.extend(get_content(html.text)) #<=======
#cars = get_content(html.text)
print(cars)
else:
print('Error')
parse()
Ошибка:
Парсинг страницы 1 из 3...
Traceback (most recent call last):
File "C:\Vit\Python\webparser\parser.py", line 58, in <module>
parse()
File "C:\Vit\Python\webparser\parser.py", line 52, in parse
cars.extend(get_content(html.text))
TypeError: 'NoneType' object is not iterable
[{'title': 'Jeep Compass 2019', 'link': 'https://auto.ria.com/newauto/auto-jeep-compass-1838277.html', 'usd_price': '29 900 $', 'uah_price': '840 390 грн', 'city': 'Київ'}, {'title': 'Jeep Grand Cherokee 2018', 'link': 'https://auto.ria.com/newauto/auto-jeep-grand-cherokee-1838297.html', 'usd_price': '55 000 $', 'uah_price': '1 545 869 грн', 'city': 'Київ'}, {'title': 'Jeep Grand Cherokee 2018', 'link': 'https://auto.ria.com/newauto/auto-jeep-grand-cherokee-1810691.html', 'usd_price': '55 000 $', 'uah_price': '1 546 050 грн', 'city': 'Одеса'}, {'title': 'Jeep Grand Cherokee 2020', 'link': 'https://auto.ria.com/newauto/auto-jeep-grand-cherokee-1811781.html', 'usd_price': '59 500 $', 'uah_price': '1 672 545 грн', 'city': 'Одеса'}, {'title': 'Jeep Wrangler 2018', 'link': 'https://auto.ria.com/newauto/auto-jeep-wrangler-1810694.html', 'usd_price': '53 700 $', 'uah_price': '1 509 507 грн', 'city': 'Одеса'}, {'title': 'Jeep Grand Cherokee 2021', 'link': 'https://auto.ria.com/newauto/auto-jeep-grand-cherokee-1836192.html', 'usd_price': '60 000 $', 'uah_price': '1 686 402 грн', 'city': 'Київ'}, {'title': 'Jeep Compass 2019', 'link': 'https://auto.ria.com/newauto/auto-jeep-compass-1838186.html', 'usd_price': '35 900 $', 'uah_price': '1 009 031 грн', 'city': 'Київ'}, {'title': 'Jeep Renegade 2020', 'link': 'https://auto.ria.com/newauto/auto-jeep-renegade-1838198.html', 'usd_price': '30 000 $', 'uah_price': '843 201 грн', 'city': 'Київ'}, {'title': 'Jeep Wrangler 2020', 'link': 'https://auto.ria.com/newauto/auto-jeep-wrangler-1838190.html', 'usd_price': '65 000 $', 'uah_price': '1 826 936 грн', 'city': 'Київ'}, {'title': 'Jeep Compass 2019', 'link': 'https://auto.ria.com/newauto/auto-jeep-compass-1810690.html', 'usd_price': '32 200 $', 'uah_price': '905 142 грн', 'city': 'Одеса'}]
Process finished with exit code 1