收录快的网站,wordpress 系统安装,网站主页设计要点,建筑工程网络计划以下是使用Python编写的针对淘宝、天猫、京东详情页的爬虫实例。请注意#xff0c;这些实例仅供参考#xff0c;实际使用时可能需要根据网站结构的变化进行调整#xff0c;并且需要遵守各平台的爬虫协议和法律法规。
淘宝详情页爬虫实例 环境准备#xff1a; Python 3.xSe…以下是使用Python编写的针对淘宝、天猫、京东详情页的爬虫实例。请注意这些实例仅供参考实际使用时可能需要根据网站结构的变化进行调整并且需要遵守各平台的爬虫协议和法律法规。
淘宝详情页爬虫实例 环境准备 Python 3.xSelenium库ChromeDriver或对应浏览器的WebDriver 代码实现
from selenium import webdriverfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.common.keys import Keysimport timeimport csv# 初始化WebDriverdriver webdriver.Chrome()# 打开淘宝并搜索商品driver.get(https://www.taobao.com/)driver.maximize_window() # 最大化浏览器窗口driver.implicitly_wait(10) # 设置隐式等待时间# 搜索商品这里以“手机”为例search_keyword 手机driver.find_element(By.XPATH, //*[idq]).send_keys(search_keyword)driver.find_element(By.XPATH, //*[idJ_TSearchForm]/div[1]/button).click()# 等待搜索结果加载完成time.sleep(5)# 解析搜索结果页面并提取商品详情页链接product_links []for item in driver.find_elements(By.XPATH, //div[classgrid g-clearfix]/div/div):detail_url item.find_element(By.XPATH, .//div[classpic]/a).get_attribute(href)product_links.append(detail_url)# 遍历商品详情页链接并提取所需信息with open(taobao_products.csv, w, newline, encodingutf-8) as csvfile:fieldnames [title, price, seller, location, detail_url]writer csv.DictWriter(csvfile, fieldnamesfieldnames)writer.writeheader()for link in product_links:driver.get(link)time.sleep(3) # 等待详情页加载完成title driver.find_element(By.XPATH, //*[idJ_DetailHeader]/div[1]/h1).textprice driver.find_element(By.XPATH, //*[idJ_StrPrice]/em).textseller driver.find_element(By.XPATH, //*[idJ_OtherOptions]/div[1]/p[1]/a).textlocation driver.find_element(By.XPATH, //*[idJ_OtherOptions]/div[1]/p[2]/span).textwriter.writerow({title: title,price: price,seller: seller,location: location,detail_url: link})# 关闭WebDriverdriver.quit()
天猫详情页爬虫实例
环境准备与淘宝相同。代码实现以搜索“羽毛球”为例
from selenium import webdriverfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECimport timeimport csv# 初始化WebDriverdriver webdriver.Chrome()# 打开天猫并搜索商品driver.get(https://list.tmall.com/)driver.maximize_window()driver.implicitly_wait(10)# 搜索商品这里以“羽毛球”为例search_keyword 羽毛球driver.get(fhttps://list.tmall.com/search_product.htm?q{search_keyword})# 等待搜索结果加载完成wait WebDriverWait(driver, 10)page_total_element wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, .tm-pagination .ui-page-item.ui-page-item-last em)))page_total page_total_element.text# 解析搜索结果页面并提取商品信息product_info []for page in range(1, int(page_total) 1):try:# 如果是非第一页则进行翻页操作if page 1:input_element wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, .ui-page div.ui-page-wrap b.ui-page-skip form input.ui-page-skipTo)))input_element.clear()input_element.send_keys(page)submit_button wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, .ui-page div.ui-page-wrap b.ui-page-skip form button.ui-btn-s)))submit_button.click()time.sleep(2) # 等待页面加载# 提取当前页的商品信息goods driver.find_elements(By.CSS_SELECTOR, #J_ItemList .product)for good in goods:title good.find_element(By.CSS_SELECTOR, .productTitle).textprice good.find_element(By.CSS_SELECTOR, .productPrice).text.replace(¥, )detail_url good.find_element(By.CSS_SELECTOR, .productImg).get_attribute(href)product_info.append({title: title,price: price,detail_url: detail_url})except Exception as e:print(fError on page {page}: {e})# 将商品信息写入CSV文件with open(tmall_products.csv, w, newline, encodingutf-8) as csvfile:fieldnames [title, price, detail_url]writer csv.DictWriter(csvfile, fieldnamesfieldnames)writer.writeheader()for product in product_info:writer.writerow(product)# 关闭WebDriverdriver.quit()
京东详情页爬虫实例
环境准备与淘宝相同。代码实现以搜索“手机”为例并提取详情页图片
from selenium import webdriverfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECimport timeimport osimport requests# 初始化WebDriverdriver webdriver.Chrome()# 打开京东并搜索商品driver.get(https://search.jd.com/)driver.maximize_window()driver.implicitly_wait(10)# 搜索商品这里以“手机”为例search_keyword 手机driver.find_element(By.XPATH, //*[idkey]).send_keys(search_keyword)driver.find_element(By.XPATH, //*[idsearch]/div/button).click()# 等待搜索结果加载完成wait WebDriverWait(driver, 10)# 提取商品详情页链接并进入详情页提取图片product_links []for item in driver.find_elements(By.CSS_SELECTOR, .gl-item):detail_url item.find_element(By.CSS_SELECTOR, .p-name em a).get_attribute(href)product_links.append(detail_url)headers {User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36}for link in product_links:driver.get(link)time.sleep(3) # 等待详情页加载完成# 提取图片链接并下载image_urls []try:images driver.find_elements(By.CSS_SELECTOR, .sku-gallery img)for img in images:image_urls.append(img.get_attribute(src))except Exception as e:print(fError extracting images from {link}: {e})continueimage_dir f./jd_images/{link.split(/)[-1]}if not os.path.exists(image_dir):os.makedirs(image_dir)
