Selenium爬取文章实例
这一篇主要是解决一个问题：浏览器操作下拉才能显示更多的内容
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
import time
thepaper_dict = {}
web = webdriver.Chrome(service=Service('chromedriver.exe'))
web.implicitly_wait(10)
web.get('https://www.thepaper.cn')
time.sleep(5)
web.maximize_window()
time.sleep(5)
web.find_element(By.XPATH,'//*[@id="navMenu"]/ul/li[5]/a').click()
time.sleep(10)
'''重复下拉，尽可能多的获取内容'''
for i in range(5):
web.execute_script('window.scrollTo(0,3500)')
time.sleep(5)
tes_list = web.find_elements(By.CLASS_NAME,"small_toplink__GmZhY")
for i in tes_list:
title = i.find_element(By.TAG_NAME,'h2').text
url =i.find_element(By.TAG_NAME,'a').get_attribute('href')
thepaper_dict.update({title:url})
web.close()