我如何从该网站获取以下信息，检查gif在下一页中还有更多评论？我想使用硒和网络驱动程序
<span class="a-profile-name">NAME</span>
<i data-hook="review-star-rating" class="a-icon a-icon-star a-star-2 review-rating"><span class="a-icon-alt">2.0 out of 5 stars</span></I>
<span data-hook="review-date" class="a-size-base a-color-secondary review-date">Reviewed in the United States on January 23, 2019</span> 评论主体：
穿着约4个月后，鞋底完全脱胶 在办公环境中。我无法想象一对合法的 匡威运动鞋的品质如此低劣。我不是专家 但我认为它们是假的。
这两种鞋都不值钱。
我更喜欢使用selenium，因为我可以轻松地移至下一页并存储收集的数据。
对于这些字段中的每个字段，我应该有单独的列表，这些列表收集：author, dates, stars, review's title and review's body。
示例如下：
具有2226个评分评论。
您认为硒有作用吗？
代码（代码中包含缺少的信息，可能搜索的一部分也不正确）：
from bs4 import BeautifulSoup
import time
from selenium import webdriver
import re
def spider():
driver = webdriver.Chrome('path/chromedriver'))
driver.get('https://www.amazon.com/Converse-Chuck-Taylor-Star-Core/dp/B07KLM7JRL/ref=sr_1_1?dchild=1&keywords=converse&qid=1596469913&sr=8-1&th=1') #in th I should add page number info
time.sleep(1)
search = driver.find_element_by_name('q')
time.sleep(2)
search.submit()
author = []
dates = []
score = []
review_min = []
review = []
while True:
soup = BeautifulSoup(driver.page_source,'lxml')
result_div = soup.find_all('div', attrs={'class': 'g'})
time.sleep(2)
for r in result_div:
# here there should be the part to get info about author, dates, scores, ...
time.sleep(1)
# part where I append results scraped
next_page_btn =driver.find_elements_by_xpath("//a[@id='pnnext']")
if len(next_page_btn) <1:
print("no more pages left")
break
element =WebDriverWait(driver,100).until(expected_conditions.element_to_be_clickable((By.ID,'pnnext')))
driver.execute_script("return arguments[0].scrollIntoView();", element)
element.click()
time.sleep(2)
driver.quit()
答案 0 :(得分：1)
您的解决方案需要由几层组成。每个层负责不同的动作和行为。
负责导航和页面迭代-将在每个页面重复。
负责商品-将提取单个商品评论信息，并将重复页面中的每个商品。
这是最棘手的部分，因为它必须在不同的页面中打开每个项目（如果使用“后退”，它将刷新并丢失数据），导航到新页面，切换，提取，关闭并切换回-所以我们回到点0，准备下一个项目。
负责评论-将提取单个项目的所有评论，并将重复每个项目页面的所有评论
For Each Page Extract
> Item, For Each Item Extract
> Reviews
结果将是具有以下格式的评论项目的数组
{
"product": "My Product",
"link": "https://products/my_product",
"reviews": [
{ "author": "foo", "date": "0000-000"... },
{ "author": "bar", "date": "0000-000"... },
...
]
}
这将是您的起点，您可以实施缺少的部分。这将在单个页面中提取所有项目的评论。
按原样运行示例，只需更改驱动程序路径即可。
import re
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait
def spider(page_number: int):
# setup: web driver > wait object > url format > page number
driver = webdriver.Chrome('D:\\automation-env\\web-drivers\\chromedriver.exe')
wait = WebDriverWait(driver, 15)
url_format =\
"https://www.amazon.com/Converse-Chuck-Taylor-Star-Core/dp/B07KLM7JRL/" \
"ref=sr_1_1?" \
"dchild=1&" \
"keywords=converse&" \
"qid=1596469913&" \
"sr=8-1&" \
"th={page_number}"
try:
# navigate
driver.get(url_format.format(page_number=page_number))
driver.maximize_window()
# search your product
__search(driver_wait=wait, search_for='converse')
# cache item
rate_locator = (By.XPATH, "//i[contains(@class,'a-star-small-')]")
items = wait.until(expected_conditions.visibility_of_all_elements_located(rate_locator))
# product cycle
reviews = []
for i in range(len(items)):
reviews.append(__product_cycle(on_driver=driver, on_element=items[i], on_element_index=i + 1))
# output
print(reviews)
except Exception as e:
print(e)
finally:
if driver is not None:
driver.quit()
# execute search product
def __search(driver_wait: WebDriverWait, search_for: str):
# search
search = driver_wait.until(expected_conditions.element_to_be_clickable((By.ID, 'twotabsearchtextbox')))
search.clear()
search.send_keys(search_for)
search.submit()
# execute an extraction on single item in the products list
# you can add more logic to extract the rest of the review
def __product_cycle(on_driver, on_element, on_element_index):
# hover the review element
ActionChains(driver=on_driver).move_to_element(on_element).perform()
# open reviews in new page (the index is here to handle amazon keeping in the DOM all reviews already inspected)
wait = WebDriverWait(on_driver, 15)
link_element_locator = (By.XPATH, "(//a[.='See all customer reviews'])[" + f'{on_element_index}' + "]")
link_element =\
wait.until(expected_conditions.element_to_be_clickable(link_element_locator))
link = link_element.get_attribute(name='href')
on_driver.execute_script(script="window.open('about:blank', '_blank');")
on_driver.switch_to_window(on_driver.window_handles[1])
on_driver.get(link)
# cache review elements
review_locator = (By.XPATH, "//div[contains(@id,'customer_review-')]")
review_elements = wait.until(expected_conditions.visibility_of_all_elements_located(review_locator))
# extract reviews for page
# if you want to iterate pages put this inside page iteration loop
reviews = {
"product": on_driver.title,
"link": on_driver.current_url,
"data": []
}
reviews_data = []
for e in review_elements:
reviews["data"].append(__get_item_review(on_driver, e))
# return to point 0
on_driver.close()
on_driver.switch_to_window(on_driver.window_handles[0])
# results
return reviews
# extracts a single item reviews collection
def __get_item_review(on_driver, on_element) -> dict:
# locators
author_locator = ".//span[@class='a-profile-name']"
date_locator = ".//span[@data-hook='review-date']"
score_locator = ".//a[.//i[@data-hook='review-star-rating']]"
review_locator = ".//div[@data-hook='review-collapsed']/span"
# data
review_data = {
'author': on_element.find_element_by_xpath(author_locator).text.strip(),
'date': re.findall('(?<=on ).*', on_element.find_element_by_xpath(date_locator).text.strip())[0],
'score': re.findall('\\d+.\\d+', on_element.find_element_by_xpath(score_locator).get_attribute("title"))[0],
'review': on_element.find_element_by_xpath(review_locator).text.strip(),
}
# TODO: add more logic to get also the hidden reviews for this item.
# results data
return review_data
spider(page_number=1)