selenium爬虫根据域名后缀查询网站数量

laolao's Blogs / 2023-05-29 / 原文

本质是对‘site’功能的自动化查询

import re
from selenium import webdriver

a = {'org.do', ...., 'org.ua'}

driver = webdriver.Chrome()
driver.get("https://www.baidu.com/")
p_input = driver.find_element_by_id('kw')
dir_ = {}
d_ = []
import time

for aa in a:
    p_input.send_keys('site:{}'.format(aa))
    p_btn = driver.find_element_by_id('su')
    p_btn.click()
    time.sleep(2)
    try:
        text = driver.find_element_by_xpath('//*[@id="content_left"]/div[1]/div/p[1]/b').text
        number = re.findall("\d+", text)  # 输出结果为列表
        if int(''.join(number)) < 100:
            d_.append(aa)
        dir_[aa] = text
    except:
        d_.append(aa)
        dir_[aa] = 0
    p_input.clear()

print(d_)
print(dir_)
import pandas as pd

pd.DataFrame(dir_, index=[0]).to_excel('2.xlsx', index=False)
driver.close()

　小修一下bug

import pandas as pd                                                
da_={}                                                             
for j,i in a.items():                                              
   print(type(i))                                                  
   if type(i)==type('str'):                                        
       n=re.findall("\d+", i)                                      
       da_[j]=int(''.join(n))                                      
   else:                                                           
       da_[j]=0                                                    
pd.DataFrame.from_dict(da_,orient='index').to_excel('2.xlsx')      
print(da_)