Пытаюсь парсить Avito без использования Selenium.
После нескольких удачно спарсеных страниц, Avito при GET запросе отдаёт мне страницу с загрузкой (проверкa браузера).
Kак можно обойти, не используя вебдрайвер?
Прокси и asyncio.sleep
пробовал, не помогает.
Вот код:
import aiohttp
import asyncio
from bs4 import BeautifulSoup as bs
from concurrent.futures import ThreadPoolExecutor
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0'}
def set_proxies():
file = open('Proxy.txt', 'r')
proxies_file = file.read().splitlines()
proxies_list = []
for each in proxies_file:
proxies_list.append('http://' + str(each))
return proxies_list
def get_item(html):
products = []
soup = bs(html, 'lxml')
items = soup.find_all('div', {'itemtype': 'http://schema.org/Product'})
for item in items:
title = item.find('a', class_='snippet-link')['title']
price = item.find('span', {'data-marker': 'item-price'}).text
link = 'https://www.avito.ru' + item.find('a', class_='snippet-link')['href']
products.append((title, price.strip(), link))
return products
async def gen_urls(number):
root_url = 'https://www.avito.ru/chelyabinsk/tovary_dlya_detey_i_igrushki/kupit-velosipedy_i_samokaty-ASgBAgICAUT~AaCGAQ?q=%D1%8D%D0%BB%D0%B5%D0%BA%D1%82%D1%80%D0%BE%D1%81%D0%B0%D0%BC%D0%BE%D0%BA%D0%B0%D1%82&p=2'
for i in range(number):
num_page = root_url[:-1] + str(i)
yield num_page
async def request(session, url, proxy):
async with session.get(url, proxy=proxy, headers=headers) as response:
return await response.text()
async def crawl(proxy, client, pool):
loop = asyncio.get_event_loop()
proxies = await proxy
for prox in proxies:
try:
for request_future in asyncio.as_completed([request(client, url, prox) async for url in gen_urls(10)]):
result = await loop.run_in_executor(pool, get_item, await request_future)
print(result)
except:
continue
async def main(proxies):
loop = asyncio.get_event_loop()
with ThreadPoolExecutor() as pool:
async with aiohttp.ClientSession() as client:
proxy_future = loop.create_future()
proxy_future.set_result(proxies)
await crawl(proxy_future, client, pool)
if __name__ == '__main__':
asyncio.run(main(set_proxies()))