Задача состоит в парсинге данных с сайта tripadvisor. Последующего картографирования полученных данных. Проблема возникла на этапе получения данных о месте(street-address, locality) - последние строчки кода. ОН НЕ ПОЛУЧАЕТ ДАННЫХ С САЙТА
import urllib3
from bs4 import BeautifulSoup
import time
import json
url = "https://www.tripadvisor.ru/Attractions-g60763-Activities-New_York_City_New_York.html"
def fetch(url):
http = urllib3.PoolManager()
r = http.request('GET', 'https://www.tripadvisor.ru/Attractions-g60763-Activities-New_York_City_New_York.html')
data = r.data
return data
def extract_data(data):
#locate link to attraction page
baseUrl = "https://www.tripadvisor.ru"
website = baseUrl + data.findChildren('a')[0]['href']
coords = extract_address(website)
print("waiting 1 sec...")
time.sleep(1)
name = data.findChildren('a')[0].string.strip()
numReviews = data.findChildren('span', 'more')[0].findChildren('a')[0].string.strip()
print ("site name: " + name)
print ("number of reviews: " + numReviews)
print ('-------')
return name + " | " + numReviews + "|" + str(coords['lat']) + "|" + str(coords['lng']) + "\n"
#function to open attraction page and extract adress
def extract_address(url):
print("opening attraction page: " + url)
page = fetch (url)
soup = BeautifulSoup(page)
#parse HTML for address components
street = soup.find('span',{"class":"street-address"})
print("street=")
print(street)
locality = soup.find('span',{"class":"locality"})
print("locality=")
print(locality)
try:
region = soup.findAll('span', 'region')[0].string.stirp()
except Exception as e:
region = ""
КОГДА ДЕЛАЮ ПРИНТ - ПОЛУЧАЮ: street= None locality= None waiting 1 sec... site name: Центральный парк number of reviews: 124 546 отзывов
НЕ ПОНИМАЮ ПОЧЕМУ NONE