При запуске скрипта парсит только 6 объявлений, хотя selenium с кнопкой взаимодействует. В чём может быть проблема?
# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
import csv
from selenium import webdriver
def get_html(url):
r = requests.get(url)
return r.text
def write_csv(data):
with open('avito-realty.csv', 'a', encoding = 'utf-8') as file:
writer = csv.writer(file)
writer.writerow((data['title'], data['price'], data['metro'], data['url']))
def get_data(html):
soup = BeautifulSoup(html, 'lxml')
ads = soup.find('div', class_='recommendations-list js-recommendations-list').find_all('div', class_ = 'recommendations-item')
driver = webdriver.Edge()
driver.get('https://www.avito.ru/moskva/nedvizhimost')
button = driver.find_element_by_xpath('//button[@class="button button-origin button-origin_large button-origin_full-width recommendations-show-more js-recommendations-show-more"]').click()
# title, price, metro, url
for ad in ads:
try:
title = ad.find('a', class_ = 'description-title-link js-item-link').text
except:
title = ''
try:
price = ad.find('div', class_ = 'option price').text.strip()
except:
price = ''
try:
metro = ad.find('div', class_ = 'fader item_gallery-ellipsis').text
except:
metro = ''
try:
url = 'https://www.avito.ru' + ad.find('a', class_ = 'description-title-link js-item-link').get('href')
except:
url = ''
data = {'title' : title,
'price' : price,
'metro' : metro,
'url' : url}
write_csv(data)
def main():
url = 'https://www.avito.ru/moskva/nedvizhimost'
html = get_html(url)
get_data(html)
if __name__ == '__main__':
main()