У меня есть парсер, который я сделал насколько мне хватило знаний. Он в принципе меня устраивает, если бы не было повторов и пропусков. Часто он может пропустить или записать страницу несколько раз подрят, вообщем работает со сбоями. Как можно было бы избавиться от этого? Использую модуль time для того чтобы успел прогрузиться сайт.
import lxml.html
from selenium import webdriver
import time
executable_path = "A:\Programs\chromedriver.exe"
webdriver = webdriver.Chrome(executable_path=executable_path)
webdriver.get ('https://site.com/page1')
time.sleep(3)
q=webdriver.find_element_by_xpath("xpath1")
w=webdriver.find_element_by_xpath("xpath2")
e=webdriver.find_element_by_xpath("xpath3")
r=webdriver.find_element_by_xpath("xpath4")
t=webdriver.find_element_by_xpath("xpath5")
y=webdriver.find_element_by_xpath("xpath6")
u=webdriver.find_element_by_xpath("xpath7")
i=webdriver.find_element_by_xpath("xpath8")
o=webdriver.find_element_by_xpath("xpath9")
p=webdriver.find_element_by_xpath("xpath10")
a=webdriver.find_element_by_xpath("xpath11")
s=webdriver.find_element_by_xpath("xpath12")
c=webdriver.find_element_by_xpath("xpath13")
g=webdriver.find_element_by_xpath("xpath14")
h=webdriver.find_element_by_xpath("xpath15")
j=webdriver.find_element_by_xpath("xpath16")
k=webdriver.find_element_by_xpath("xpath17")
l=webdriver.find_element_by_xpath("xpath18")
z=webdriver.find_element_by_xpath("xpath19")
x=webdriver.find_element_by_xpath("xpath20")
f=open("A:\\Programs\\text.txt","w")
f.write(q.text + str(w.text)+"\n")
f.write(e.text + str(r.text)+"\n")
f.write(t.text + str(y.text)+"\n")
f.write(u.text + str(i.text)+"\n")
f.write(o.text + str(p.text)+"\n")
f.write(a.text + str(s.text)+"\n")
f.write(c.text + str(g.text)+"\n")
f.write(h.text + str(j.text)+"\n")
f.write(k.text + str(l.text)+"\n")
f.write(z.text + str(x.text)+"\n")
webdriver.get ('https://site.com/page2')
time.sleep(3)
q=webdriver.find_element_by_xpath("xpath1")
w=webdriver.find_element_by_xpath("xpath2")
e=webdriver.find_element_by_xpath("xpath3")
r=webdriver.find_element_by_xpath("xpath4")
t=webdriver.find_element_by_xpath("xpath5")
y=webdriver.find_element_by_xpath("xpath6")
u=webdriver.find_element_by_xpath("xpath7")
i=webdriver.find_element_by_xpath("xpath8")
o=webdriver.find_element_by_xpath("xpath9")
p=webdriver.find_element_by_xpath("xpath10")
a=webdriver.find_element_by_xpath("xpath11")
s=webdriver.find_element_by_xpath("xpath12")
c=webdriver.find_element_by_xpath("xpath13")
g=webdriver.find_element_by_xpath("xpath14")
h=webdriver.find_element_by_xpath("xpath15")
j=webdriver.find_element_by_xpath("xpath16")
k=webdriver.find_element_by_xpath("xpath17")
l=webdriver.find_element_by_xpath("xpath18")
z=webdriver.find_element_by_xpath("xpath19")
x=webdriver.find_element_by_xpath("xpath20")
f.write(q.text + str(w.text)+"\n")
f.write(e.text + str(r.text)+"\n")
f.write(t.text + str(y.text)+"\n")
f.write(u.text + str(i.text)+"\n")
f.write(o.text + str(p.text)+"\n")
f.write(a.text + str(s.text)+"\n")
f.write(c.text + str(g.text)+"\n")
f.write(h.text + str(j.text)+"\n")
f.write(k.text + str(l.text)+"\n")
f.write(z.text + str(x.text)+"\n")
#и тд... около 300 страниц
f.close()
20 разных xpath
:
w=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[1]/div/div[1]/div[2]/span[1]/span[1]")
e=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[2]/div/div[2]/span[1]")
r=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[2]/div/div[1]/div[2]/span[1]/span[1]")
t=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[3]/div/div[2]/span[1]")
y=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[3]/div/div[1]/div[2]/span[1]/span[1]")
u=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[4]/div/div[2]/span[1]")
i=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[4]/div/div[1]/div[2]/span[1]/span[1]")
o=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[5]/div/div[2]/span[1]")
p=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[5]/div/div[1]/div[2]/span[1]/span[1]")
a=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[6]/div/div[2]/span[1]")
s=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[6]/div/div[1]/div[2]/span[1]/span[1]")
c=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[7]/div/div[2]/span[1]")
g=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[7]/div/div[1]/div[2]/span[1]/span[1]")
h=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[8]/div/div[2]/span[1]")
j=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[8]/div/div[1]/div[2]/span[1]/span[1]")
k=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[9]/div/div[2]/span[1]")
l=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[9]/div/div[1]/div[2]/span[1]/span[1]")
z=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[10]/div/div[2]/span[1]")
x=webdriver.find_element_by_xpath("/html/body/div[1]/div[7]/div[2]/div[1]/div[4]/div[2]/div[2]/div/div[1]/a[10]/div/div[1]/div[2]/span[1]/span[1]")