from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service as ChromeService
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
url = "https://demo.vuestorefront.io/"
with webdriver.Chrome(service=ChromeService(ChromeDriverManager().install())) as driver:
driver.get(url)
input = driver.find_element(By.CSS_SELECTOR, "input[type='search']")
input.send_keys("laundry" + Keys.ENTER)
el = WebDriverWait(driver, timeout=3).until(
lambda d: d.find_element(By.CLASS_NAME, "sf-product-card__title"))
items = driver.find_elements(By.CLASS_NAME, "sf-product-card__title")
for item in items:
print(item.text)
from bs4 import BeautifulSoup
soup = BeautifulSoup(r.content, 'html.parser')
for item in soup.find_all('span', {'class': 'sf-product-card__title'}):
print(item.text)
import requests
from bs4 import BeautifulSoup
r = requests.get('https://demo.vuestorefront.io/c/kitchen')
soup = BeautifulSoup(r.content, 'html.parser')
for item in soup.find_all('span', {'class': 'sf-product-card__title'}):
print(item.text)
恭喜!你做到了; 您已成功使用 Request Python 库进行网络抓取。您的输出应如下所示:
[Sample] Tiered Wire Basket
[Sample] Oak Cheese Grater
[Sample] 1 L Le Parfait Jar
[Sample] Chemex Coffeemaker 3 Cup
[Sample] Able Brewing System
4. BeautifulSoup
Beautiful Soup是一个强大的 Python 网络抓取库,特别适用于解析 XML 和 HTML 文档。它的便利性是其最受欢迎的福利之一。Beautiful Soup 建立在著名的 Python 解析包之上,允许您尝试不同的技术。
for item in response.css('.sf-product-card__title'):
print(item.xpath('string(.)').get())
使用以下代码创建一个名为“scrapyTest.py”的新文件:
import scrapy
class kitchenSpider(scrapy.Spider):
name='mySpider'
start_urls = ['https://demo.vuestorefront.io/c/kitchen',]
def parse(self, response):
for item in response.css('.sf-product-card__title'):
print(item.xpath('string(.)').get())
通过在终端中执行以下脚本来运行蜘蛛,您应该会在屏幕上看到打印的项目列表:
scrapy runspider scrapyTest.py
[Sample] Tiered Wire Basket
[Sample] Oak Cheese Grater
[Sample] 1 L Le Parfait Jar
[Sample] Chemex Coffeemaker 3 Cup
[Sample] Able Brewing System