selenium爬取多个网站及通过GUI界面点击爬取

发布时间：2026/7/29 13:39:45

selenium爬取多个网站及通过GUI界面点击爬取

selenium爬取代码

webcrawl.py

import re
import time
import json
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import TimeoutException, StaleElementReferenceException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ECclass AgriInfoSpider:def __init__(self):chrome_options = Options()chrome_options.add_argument("--headless")  # 无界面模式chrome_options.add_argument("--start-maximized")chrome_options.add_argument("--disable-extensions")chrome_options.add_argument("--disable-gpu")chrome_options.add_argument("--disable-dev-shm-usage")chrome_options.add_argument("--no-sandbox")self.driver = webdriver.Chrome(options=chrome_options, executable_path='C:\\Users\\Administrator\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe')def save_to_json(self, item, filename):with open(filename, 'a', encoding='utf-8') as f:json.dump(item, f, ensure_ascii=False)f.write('\n')def close(self):self.driver.quit()# TODO# 农业网-农业科技def agronet(self, stop_event):self.driver.get('http://www.agronet.com.cn/Tech/List.html')self.driver.implicitly_wait(60)# 获取行业industrys = self.driver.find_elements(By.XPATH, '//dl[@class="product_classification_nav"]/dd/ul/li/a')item = {}order = 0# 点击各个行业for m, industry in enumerate(industrys):if stop_event.is_set():breakitem["industry"] = industry.textindustry.click()# 确保页面正确到达# WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//dl[@class="arrow_700"]/dt/span/em[2]'), item["industry"]))articles = self.driver.find_elements(By.XPATH, '//dl[@class="arrow_700"]/dd/ul/li')while True:if stop_event.is_set():breakfor i, article in enumerate(articles):if stop_event.is_set():breakitem["order"] = ordertry:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//dl[@class="arrow_700"]/dt/em'), "农业技术文章列表"))except TimeoutException:continue# 文章标题article = self.driver.find_elements(By.XPATH, '//dl[@class="arrow_700"]/dd/ul/li/span/a')[i]item["title"] = article.textitem["date"] = re.search(r'\d{4}-\d{1,2}-\d{1,2}', self.driver.find_element(By.XPATH, '//dl[@class="arrow_700"]/dd/ul/li/div').text).group()item["source"] = self.driver.find_element(By.XPATH, '//dl[@class="arrow_700"]/dd/ul/li/em').text# 点击文章article.click()# 获取所有打开的窗口句柄window_handles = self.driver.window_handles# 切换标签页self.driver.switch_to.window(window_handles[-1])try:# 获取内容content = self.driver.find_elements(By.XPATH, '//div[@class="font_bottom"]/p')content_lists = [c.text.strip() for c in content]item["content"] = [''.join(content_lists)]except:item["content"] = []# 写入文件self.save_to_json(item, './results/agronet.json')# 关闭新标签页self.driver.close()# 切换回原始的标签页self.driver.switch_to.window(self.driver.window_handles[0])order += 1# 点击下一页try:if stop_event.is_set():breaknext_page = self.driver.find_element(By.XPATH, '//a[contains(text(), "下一页")]')next_page.click()if self.driver.current_url == 'http://www.agronet.com.cn/Message/Error?aspxerrorpath=/Tech/List':breakexcept:break# 中国农网-三农头条def farmer(self, stop_event):self.driver.get('https://www.farmer.com.cn/farmer/xw/sntt/list.shtml')# 获取所有文章articles = self.driver.find_elements(By.XPATH, '//div[contains(@class, "index-font")]')item = {}order = 0# 点击文章while True:if stop_event.is_set():breakfor article in articles:if stop_event.is_set():breakitem["order"] = orderitem["title"] = article.textarticle.click()# 获取所有打开的窗口句柄window_handles = self.driver.window_handles# 切换标签页self.driver.switch_to.window(window_handles[-1])# 确保到达文章详情标签页try:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="index-title"]/span[3]'), "详情"))except TimeoutException:continueitem["author"] = self.driver.find_element(By.XPATH, '//div[@class="index-introduce"]/ul/li[2]/span').textitem["date"] = self.driver.find_element(By.XPATH, '//div[@class="index-introduce"]/ul/div/span').textitem["source"] = self.driver.find_element(By.XPATH, '//div[@class="index-introduce"]/ul/li[1]/span').textcontent = self.driver.find_elements(By.XPATH, '//div[@class="textList"]/p')content_lists = [c.text.strip() for c in content]item["content"] = [''.join(content_lists)]# 写入文件self.save_to_json(item, './results/farmer.json')# 关闭新标签页self.driver.close()# 切换回原始的标签页self.driver.switch_to.window(self.driver.window_handles[0])order += 1# 点击下一页try:if stop_event.is_set():breaknext_page = self.driver.find_element(By.XPATH, '//a[contains(text(), "下一页")]')next_page.click()except:break# 中国农业农村信息网-数据-市场动态def agri(self,stop_event):self.driver.get('http://www.agri.cn/sj/scdt/')item = {}order = 0while True:if stop_event.is_set():break# 获取所有文章articles = self.driver.find_elements(By.XPATH, '//ul[@class="nxw_list_ul"]/li/div/div/p[1]/a')for i, article in enumerate(articles):if stop_event.is_set():breakitem["order"] = ordertry:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[contains(@class, "title_common") and contains(@class, "title_common_w")]'), "市场动态"))except TimeoutException:continue# 点击文章article = self.driver.find_elements(By.XPATH, '//ul[@class="nxw_list_ul"]/li/div/div/p[1]/a')[i]article.click()# 获取所有打开的窗口句柄window_handles = self.driver.window_handles# 切换标签页self.driver.switch_to.window(window_handles[-1])# 确保到达文章详情标签页try:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="bread_nav"]/a[3]'), "市场动态"))except TimeoutException:continueitem["title"] = self.driver.find_element(By.XPATH, '//div[@class="detailCon_info_tit"]').textitem["date"] = re.search(r'\d{4}-\d{1,2}-\d{1,2}', self.driver.find_element(By.XPATH, '//div[@class="updateInfo_mess"]/span[1]').text).group()item["source"] = re.search(r'来源：(.+)', self.driver.find_element(By.XPATH, '//div[@class="updateInfo_mess"]/span[3]').text).group()try:content = self.driver.find_elements(By.XPATH, '//div[contains(@class, "content_body_box") and contains(@class, "ArticleDetails")]/p')content_lists = [c.text.strip() for c in content]item["content"] = [''.join(content_lists)]except:item["content"] = []# 写入文件self.save_to_json(item, './results/agri.json')# 关闭新标签页self.driver.close()# 切换回原始的标签页self.driver.switch_to.window(self.driver.window_handles[0])order += 1# 点击下一页pages_sum = re.search(r'\d+', self.driver.find_element(By.XPATH, '//font[contains(@class, "clear") and contains(@class, "jump_con")]/span[2]').text).group()pages_cur = re.search(r'\d+', self.driver.find_element(By.XPATH, '//a[@class=" cur"]').text).group()if pages_cur != pages_sum:next_page = self.driver.find_element(By.XPATH, '//a[contains(text(), "下一页")]')next_page.click()else:break# 新农网-农业新闻-行业资讯def xinnong(self, stop_event):self.driver.get('http://www.xinnong.net/news/hangye/list_14_1.html')self.driver.implicitly_wait(60)item = {}order = 0while True:if stop_event.is_set():break# 获取所有文章articles = self.driver.find_elements(By.XPATH, '//div[@class="newslist"]/ul/li/a')for i, article in enumerate(articles):if stop_event.is_set():breakitem["order"] = ordertry:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="lsttit"]/h1'), "行业资讯"))except TimeoutException:continue# 点击文章article = self.driver.find_elements(By.XPATH, '//div[@class="newslist"]/ul/li/a')[i]article.click()# 获取所有打开的窗口句柄window_handles = self.driver.window_handles# 切换标签页self.driver.switch_to.window(window_handles[-1])# 确保到达文章详情标签页try:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="spos"]/a[3]'), "行业资讯"))except TimeoutException:continueitem["title"] = self.driver.find_element(By.XPATH, '//div[@class="arctit"]/h1').textitem["date"] = re.search(r'\d{4}-\d{1,2}-\d{1,2}', self.driver.find_element(By.XPATH, '//div[@class="arcinfo"]').text).group()item["source"] = re.search(r'来源：(.+)', self.driver.find_element(By.XPATH, '//div[@class="arcinfo"]').text).group()content = self.driver.find_elements(By.XPATH, '//div[@class="arcont"]/p')content_lists = [c.text.strip() for c in content]item["content"] = [''.join(content_lists)]# 写入文件self.save_to_json(item, './results/xinnong.json')# 关闭新标签页self.driver.close()# 切换回原始的标签页self.driver.switch_to.window(self.driver.window_handles[0])order += 1# 点击下一页try:if stop_event.is_set():breaknext_page = self.driver.find_element(By.XPATH, '//a[contains(text(), "下一页")]')next_page.click()except:break# 富农网-行业资讯def richagri(self, stop_event):self.driver.get('http://www.richagri.com/news')item = {}order = 0while True:if stop_event.is_set():break# 获取所有文章articles = self.driver.find_elements(By.XPATH, '//div[@class="head"]/ul/li/a')for i, article in enumerate(articles):if stop_event.is_set():breakitem["order"] = ordertry:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="head"]/div[2]/a[2]'), "行业资讯"))except TimeoutException:continue# 点击文章article = self.driver.find_elements(By.XPATH, '//div[@class="head"]/ul/li/a')[i]article.click()# 确保到达文章详情标签页try:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="head"]/div[2]/a[2]'), "行业资讯"))except TimeoutException:continueitem["title"] = self.driver.find_element(By.XPATH, '//div[@class="head"]/b').textitem["date"] = re.search(r'\d{4}-\d{1,2}-\d{1,2}', self.driver.find_element(By.XPATH, '//div[@class="head"]/font').text).group()content = self.driver.find_elements(By.XPATH, '//div[@class="head"]')content_lists = [c.text.strip('\n') for c in content]content_lists = re.search(r'时间:(\d{4}-\d{1,2}-\d{1,2})\n(.*?)(?=\n回顶部)', content_lists[0], re.S)item["content"] = [''.join(content_lists.group(2))]# 写入文件self.save_to_json(item, './results/richagri.json')# 返回原始的标签页self.driver.back()order += 1# 点击下一页try:if stop_event.is_set():breaknext_page = self.driver.find_element(By.XPATH, '//a[contains(text(), "下页")]')next_page.click()except:break# 金农网-分析def jinnong(self, stop_event):self.driver.get('https://www.jinnong.cn/1002/')# 存储已经爬取的文章链接crawled_links = set()item = {}order = 0while True:if stop_event.is_set():break# 获取所有文章articles = self.driver.find_elements(By.XPATH, '//ul[@class="left-side-items"]/li/a')for i, article in enumerate(articles):if stop_event.is_set():breakarticle_link = article.get_attribute('href')if article_link not in crawled_links:item["order"] = ordertry:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//span[@class="current-path-2"]'), "农业市场分析"))except TimeoutException:continue# 点击文章article = self.driver.find_elements(By.XPATH, '//ul[@class="left-side-items"]/li/a')[i]article.click()# 获取所有打开的窗口句柄window_handles = self.driver.window_handles# 切换标签页self.driver.switch_to.window(window_handles[-1])# 确保到达文章详情标签页try:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="current-path"]/span[3]'), "正文"))except TimeoutException:continueitem["title"] = self.driver.find_element(By.XPATH, '//div[@class="article-title"]/h1').textitem["author"] = self.driver.find_element(By.XPATH, '//div[@class="article-title"]/div/div[1]/span[3]').textitem["date"] = self.driver.find_element(By.XPATH, '//div[@class="article-title"]/div/div[1]/span[1]').textitem["source"] = re.search(r'来源：(.+)', self.driver.find_element(By.XPATH, '//div[@class="article-title"]/div/div[1]/span[2]').text).group()content = self.driver.find_elements(By.XPATH, '//div[@class="article-conte-infor"]')content_lists = [c.text.strip() for c in content]item["content"] = [''.join(content_lists)]# 写入文件self.save_to_json(item, './results/jinnong.json')# 记录已爬取的链接crawled_links.add(article_link)# 关闭新标签页self.driver.close()# 切换回原始的标签页self.driver.switch_to.window(self.driver.window_handles[0])order += 1# 点击加载更多button = self.driver.find_element(By.CSS_SELECTOR, '.click_more a span').textif button == "点击加载更多":next_page = self.driver.find_element(By.CSS_SELECTOR, '.click_more a')self.driver.execute_script("arguments[0].click();", next_page)# 结束elif button == "已加载全部":break# 中国乡村振兴服务网-新闻动态-行业资讯def xczxfw(self, stop_event):self.driver.get('http://www.xczxfw.org.cn/news/12/list')item = {}order = 0while True:if stop_event.is_set():break# 获取所有文章articles = self.driver.find_elements(By.XPATH, '//div[@class="zdhd"]/dl/dd/p/a')# articles = self.driver.find_elements(By.XPATH, '//dl[@class="lb_dt"]/dd/p/a')for i, article in enumerate(articles):if stop_event.is_set():breakitem["order"] = order# 点击文章article = self.driver.find_elements(By.XPATH, '//div[@class="zdhd"]/dl/dd/p/a')[i]# article = self.driver.find_elements(By.XPATH, '//dl[@class="lb_dt"]/dd/p/a')[i]article.click()# 获取所有打开的窗口句柄window_handles = self.driver.window_handles# 切换标签页self.driver.switch_to.window(window_handles[-1])# 确保到达文章详情标签页try:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="gy_1"]/span/a[2]'), "行业资讯"))except TimeoutException:continueitem["title"] = self.driver.find_element(By.XPATH, '//div[@class="zdhd"]/h1').textitem["date"] = re.search(r'\d{4}-\d{1,2}-\d{1,2} \d{2}:\d{2}', self.driver.find_element(By.XPATH, '//div[@class="zdhd"]/h2').text).group()item["source"] = re.search(r'来源：(.+)', self.driver.find_element(By.XPATH, '//div[@class="zdhd"]/h2').text).group()content = self.driver.find_elements(By.XPATH, '//div[@class="com_de"]/p')content_lists = [c.text.strip() for c in content]item["content"] = [''.join(content_lists)]# 写入文件self.save_to_json(item, './results/xczxfw.json')# 关闭新标签页self.driver.close()# 切换回原始的标签页self.driver.switch_to.window(self.driver.window_handles[0])order += 1# 点击下一页try:if stop_event.is_set():breaknext_page = self.driver.find_element(By.XPATH, '//a[contains(text(), "下一页")]')next_page.click()except:break# 农博网-农博数据中心-实用技术def shujuaweb(self, stop_event):self.driver.get('http://shuju.aweb.com.cn/technology/technology-0-1.shtml')item = {}order = 0while True:if stop_event.is_set():break# 获取所有文章articles = self.driver.find_elements(By.XPATH, '//ul[@class="newList2"]/li/a[2]')for i, article in enumerate(articles):if stop_event.is_set():breakitem["order"] = ordertry:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//h2[@class="h2s"]'), "实用技术："))except TimeoutException:continue# 点击文章article = self.driver.find_elements(By.XPATH, '//ul[@class="newList2"]/li/a[2]')[i]article.click()# 获取所有打开的窗口句柄window_handles = self.driver.window_handles# 切换标签页self.driver.switch_to.window(window_handles[-1])try:# 确保到达文章详情标签页WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//ul[@class="sub"]/li[7]/a/span'), "实用技术"))except TimeoutException:continueitem["title"] = self.driver.find_element(By.XPATH, '//div[@id="content"]/div[1]/div[1]/div/p[1]').textitem["date"] = re.search(r'\d{4}年\d{1,2}月\d{1,2}日 \d{2}:\d{2}', self.driver.find_element(By.XPATH, '//div[@id="content"]/div[1]/div[1]/div/p[2]/span').text).group()# 避免来源为空报错source_element = self.driver.find_element(By.XPATH, '//div[@id="content"]/div[1]/div[1]/div/p[2]/span')source_match = re.search(r'来源：(.+)', source_element.text)item["source"] = source_match.group() if source_match else ""content = self.driver.find_elements(By.XPATH, '//ul[@class="name"]/following-sibling::p')content_lists = [c.text.strip() for c in content]item["content"] = [''.join(content_lists)]# 写入文件self.save_to_json(item, './results/shujuaweb.json')# 关闭新标签页self.driver.close()# 切换回原始的标签页self.driver.switch_to.window(self.driver.window_handles[0])order += 1# 点击下一页try:if stop_event.is_set():breaknext_page = self.driver.find_element(By.XPATH, '//a[contains(text(), "下一页")]')next_page.click()except:break# 三农综合信息服务平台-12316头条-动态def agri_12316(self, stop_event):self.driver.get('http://12316.agri.cn/news/A12316dt/index.html')self.driver.implicitly_wait(60)item = {}order = 0while True:if stop_event.is_set():break# 获取所有文章articles = self.driver.find_elements(By.XPATH, '//ul[@class="dongtai_list"]/table[last()]/tbody/tr/td/li/a/p')for i, article in enumerate(articles):if stop_event.is_set():breakitem["order"] = ordertry:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//p[@class="weizhi"]'), "12316头条-动态"))except TimeoutException:continue# 点击文章article = self.driver.find_elements(By.XPATH, '//ul[@class="dongtai_list"]/table[last()]/tbody/tr/td/li/a/p')[i]article.click()# 确保到达文章详情标签页try:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="content"]/p[1]'), "正文"))except TimeoutException:continueitem["title"] = self.driver.find_element(By.XPATH, '//div[@class="detail_box"]/h3').textitem["date"] = self.driver.find_element(By.XPATH, '//p[@class="zuozhe"]/span[1]/i').textitem["source"] = self.driver.find_element(By.XPATH, '//p[@class="zuozhe"]/span[2]/i').textcontent = self.driver.find_elements(By.XPATH, '//div[@class="news_box"]')content_lists = [c.text.strip() for c in content]item["content"] = [''.join(content_lists)]# 写入文件self.save_to_json(item, './results/agri_12316.json')# 返回原始的标签页self.driver.back()order += 1# 点击下一页try:if stop_event.is_set():breaknext_page = self.driver.find_element(By.XPATH, '//a[contains(text(), "下一页")]')next_page.click()except:break# 吾谷网-农技通def wugu(self, stop_event):self.driver.get('http://www.wugu.com.cn/?cat=6')self.driver.implicitly_wait(60)item = {}order = 0# 获取总页数try:pages = self.driver.find_elements(By.XPATH, '//div[@class="nav-links"]/a')total_pages = int(pages[1].text)except (ValueError, IndexError):total_pages = 1for current_page in range(1, total_pages + 1):if stop_event.is_set():break# 打开每一页page_url = f'http://www.wugu.com.cn/?paged={current_page}&cat=6'self.driver.get(page_url)# 获取所有文章articles = self.driver.find_elements(By.XPATH, '//div[@class="mg-posts-sec-inner"]/article/div/h4/a')for i, article in enumerate(articles):if stop_event.is_set():breakitem["order"] = ordertry:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="mg-breadcrumb-title"]/h1'), "分类：农技通"))# 点击文章article = self.driver.find_elements(By.XPATH, '//div[@class="mg-posts-sec-inner"]/article/div/h4/a')[i]self.driver.execute_script("arguments[0].click();", article)except TimeoutException:continue# 确保到达文章详情标签页try:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="mg-header"]/div[1]/a'), "农技通"))except TimeoutException:continueitem["title"] = self.driver.find_element(By.XPATH, '//div[@class="mg-header"]/h1/a').textitem["date"] = self.driver.find_element(By.XPATH, '//div[@class="mg-header"]/div[2]/div/span[1]').textcontent = self.driver.find_elements(By.XPATH, '//div[@class="mg-blog-post-box"]/article/p')content_lists = [c.text.strip() for c in content]item["content"] = [''.join(content_lists)]# 写入文件self.save_to_json(item, './results/wugu.json')# 返回原始的标签页self.driver.back()order += 1# 新农村商网-农业资讯-政策法规def mofcom(self, stop_event):self.driver.get('http://nc.mofcom.gov.cn/nyzx/zcfg')self.driver.implicitly_wait(60)item = {}order = 0while True:if stop_event.is_set():break# 获取所有文章articles = self.driver.find_elements(By.XPATH, '//div[@id="showList"]/div//ul/li/h5/a')for i, article in enumerate(articles):if stop_event.is_set():breakitem["order"] = ordertry:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[contains(@class, "w") and contains(@class, "u-nav-wrap")]/span'), "政策法规"))except TimeoutException:continue# 重新获取文章列表articles = self.driver.find_elements(By.XPATH, '//div[@id="showList"]/div//ul/li/h5/a')  # 重新获取文章列表article = articles[i]  # 重新获取对应的文章元素article_link = self.driver.find_element(By.XPATH, '//div[@id="showList"]/div//ul/li/h5/a')try:# 点击文章链接self.driver.execute_script("arguments[0].click();", article_link)except StaleElementReferenceException:continue# 获取所有打开的窗口句柄window_handles = self.driver.window_handles# 切换标签页self.driver.switch_to.window(window_handles[-1])# 确保到达文章详情标签页try:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[contains(@class, "w") and contains(@class, "u-nav-wrap")]'), "正文"))except TimeoutException:continueitem["title"] = self.driver.find_element(By.XPATH, '//h3[@class="u-tt"]').textitem["date"] = self.driver.find_element(By.XPATH, '//span[@class="u-time"]').text# 避免来源为空报错source_element = self.driver.find_element(By.XPATH, '//span[@class="u-source"]')source_match = re.search(r'信息来源：(.+)', source_element.text)item["source"] = source_match.group() if source_match else ""content = self.driver.find_elements(By.XPATH, '//div[@class="u-txt"]/p')content_lists = [c.text.strip() for c in content]item["content"] = [''.join(content_lists)]# 写入文件self.save_to_json(item, './results/mofcom.json')# 关闭新标签页self.driver.close()# 切换回原始的标签页self.driver.switch_to.window(self.driver.window_handles[0])order += 1# 点击下一页try:if stop_event.is_set():breaknext_page = WebDriverWait(self.driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//a[contains(text(), "下一页")]')))self.driver.execute_script("arguments[0].click();", next_page)except:break# 惠农网-行业资讯-行情资讯def cnhnb(self, stop_event):self.driver.get('https://news.cnhnb.com/hqjd/?pi=1')item = {}order = 0while True:if stop_event.is_set():break# 获取所有文章articles = self.driver.find_elements(By.XPATH, '//div[@class="latest-list"]/div/div[2]/div[1]/a')for i, article in enumerate(articles):if stop_event.is_set():breakitem["order"] = ordertry:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="ci_crumbs"]/dl/dd[2]'), "行情解读"))except TimeoutException:continue# 分类item["classify"] = self.driver.find_element(By.XPATH, '//span[@class="ct-s"]').text# 点击文章article = self.driver.find_elements(By.XPATH, '//div[@class="latest-list"]/div/div[2]/div[1]/a')[i]article.click()# 确保到达文章详情标签页try:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//div[@class="ci_crumbs"]/dl/dd[2]/a'), "行情解读"))except TimeoutException:continueitem["title"] = self.driver.find_element(By.XPATH, '//div[@class="title"]/h1').textitem["date"] = re.search(r'\d{4}-\d{1,2}-\d{1,2} \d{2}:\d{2}', self.driver.find_element(By.XPATH, '//div[@class="d-tips"]').text).group()# 避免来源为空报错source_element = self.driver.find_element(By.XPATH, '//div[@class="d-tips"]')source_match = re.search(r'来源：([^采编]+)', source_element.text)item["source"] = source_match.group(1) if source_match else ""content = self.driver.find_elements(By.XPATH, '//div[@class="content"]/p')content_lists = [c.text.strip() for c in content]item["content"] = [''.join(content_lists)]# 写入文件self.save_to_json(item, './results/cnhnb.json')# 返回原始的标签页self.driver.back()order += 1# 点击下一页try:if stop_event.is_set():breaknext_page = self.driver.find_element(By.XPATH, '//button[@class="btn-next"]')WebDriverWait(self.driver, 20).until(EC.element_to_be_clickable(next_page))next_page.click()except:break# # 农一网 商城(舍弃)# def agri_16899(self):#     self.driver.get('https://www.16899.com/Product/ProductList.html')#     self.driver.implicitly_wait(60)##     item = {}#     order = 0#     while True:#         # 获取所有文章#         articles = self.driver.find_elements(By.XPATH, '//div[@id="postData"]/div[1]/dl/dd[1]/a')##         for i, article in enumerate(articles):#             item["order"] = order#             try:#                 WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//li[@class="selected"]/a'), "农资商城"))#             except TimeoutException:#                 continue##             # 重新获取文章列表#             article = self.driver.find_elements(By.XPATH, '//div[@id="postData"]/div[1]/dl/dd[1]/a')[i]#             # 点击文章#             article.click()##             # 获取所有打开的窗口句柄#             window_handles = self.driver.window_handles#             # 切换标签页#             self.driver.switch_to.window(window_handles[-1])##             # 确保到达文章详情标签页#             try:#                 WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//li[@class="active"]/a'), "产品介绍"))#             except TimeoutException:#                 continue##             # 分类#             item["classify"] = self.driver.find_element(By.XPATH, '//h1[@class="pro_name"]/span[1]').text##             item["title"] = self.driver.find_element(By.XPATH, '//h1[@class="pro_name"]/span[2]').text#             item["price"] = self.driver.find_element(By.XPATH, '//span[@id="strongprice"]').text##             # 写入文件#             self.save_to_json(item, 'agri_16899.json')##             # 关闭新标签页#             self.driver.close()##             # 切换回原始的标签页#             self.driver.switch_to.window(self.driver.window_handles[0])##             order += 1##         # 点击下一页#         try:#             next_page = self.driver.find_element(By.XPATH, '//a[contains(text(), "下页")]')#             if next_page.get_attribute("class") == "disabled":#                 break#             else:#                 next_page.click()#         except:#             break# 191农资人-精华帖-植保技术def agri_191(self, stop_event):self.driver.get('https://www.191.cn/searcher.php?digest=1&starttime=&endtime=&fid=3')self.driver.implicitly_wait(60)item = {}order = 0# 获取总页数try:pages = self.driver.find_elements(By.XPATH, '//div[@class="pages"]/a')total_pages = int(pages[-1].text)except (ValueError, IndexError):total_pages = 1for current_page in range(1, total_pages + 1):if stop_event.is_set():break# 打开每一页page_url = f'https://www.191.cn/searcher.php?type=special&condition=digest&authorid=&fid=3&starttime=&endtime=&page={current_page}'self.driver.get(page_url)# 获取所有文章articles = self.driver.find_elements(By.XPATH, '//div[@class="dlA"]/dl/dt/a')for i, article in enumerate(articles):if stop_event.is_set():breakitem["order"] = ordertry:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//li[@id="fid_3"]/a'), "植保技术"))except TimeoutException:continue# 重新获取文章列表article = self.driver.find_elements(By.XPATH, '//div[@class="dlA"]/dl/dt/a')[i]# 点击文章article.click()# 获取所有打开的窗口句柄window_handles = self.driver.window_handles# 切换标签页self.driver.switch_to.window(window_handles[-1])# 确保到达文章详情标签页try:WebDriverWait(self.driver, 20).until(EC.text_to_be_present_in_element((By.XPATH, '//td[@id="td_tpc"]/div[1]/a'), "楼主"))except TimeoutException:continueitem["title"] = self.driver.find_element(By.XPATH, '//h1[@id="subject_tpc"]').textitem["date"] = self.driver.find_element(By.XPATH, '//td[@id="td_tpc"]/div[1]/span[2]').textcontent = self.driver.find_elements(By.XPATH, '//div[@id="read_tpc"]')content_lists = [c.text.strip() for c in content]item["content"] = [''.join(content_lists)]# 写入文件self.save_to_json(item, './results/agri_191.json')# 关闭新标签页self.driver.close()# 切换回原始的标签页self.driver.switch_to.window(self.driver.window_handles[0])order += 1

GUI界面代码

main.py

# 同时爬取所有网站
# import traceback
# import concurrent.futures
# from webcrawl import AgriInfoSpider
#
#
# def run_spider(spider_method):
#     spider = AgriInfoSpider()
#
#     try:
#         spider_method()
#     except Exception as e:
#         print(f"{spider_method.__name__} 爬虫发生错误: {str(e)}")
#         traceback.print_exc()
#
#     spider.close()
#
#
# if __name__ == "__main__":
#     spider_methods = [
#         AgriInfoSpider().agronet,
#         AgriInfoSpider().farmer,
#         AgriInfoSpider().agri,
#         AgriInfoSpider().xinnong,
#         AgriInfoSpider().richagri,
#         AgriInfoSpider().jinnong,
#         AgriInfoSpider().xczxfw,
#         AgriInfoSpider().shujuaweb,
#         AgriInfoSpider().agri_12316,
#         AgriInfoSpider().wugu,
#         AgriInfoSpider().mofcom,
#         AgriInfoSpider().cnhnb,
#         AgriInfoSpider().agri_191,
#     ]
#
#     with concurrent.futures.ThreadPoolExecutor() as executor:
#         executor.map(run_spider, spider_methods)import tkinter as tk
from tkinter import messagebox
from threading import Thread, Event
from webcrawl import AgriInfoSpiderclass App:def __init__(self, root):self.running_spiders = {}  # 跟踪已经运行的爬虫self.root = rootself.root.title("农业信息爬虫")self.root.geometry("900x400")self.create_listbox()self.create_stop_button()self.create_run_button()def create_listbox(self):self.listbox = tk.Listbox(self.root, bd=0, highlightthickness=0, bg=self.root.cget('bg'), selectmode=tk.SINGLE, font=('楷体', 12))self.listbox.pack(side=tk.LEFT, fill=tk.BOTH, expand=True, padx=100, pady=70)self.spider_functions = {"农业网-农业科技": "agronet","中国农网-三农头条": "farmer","中国农业农村信息网-数据-市场动态": "agri","新农网-农业新闻-行业资讯": "xinnong","富农网-行业资讯": "richagri","金农网-分析": "jinnong","中国乡村振兴服务网-新闻动态-行业资讯": "xczxfw","农博网-农博数据中心-实用技术": "shujuaweb","三农综合信息服务平台-12316头条-动态": "agri_12316","吾谷网-农技通": "wugu","新农村商网-农业资讯-政策法规": "mofcom","惠农网-行业资讯-行情资讯": "cnhnb","191农资人-精华帖-植保技术": "agri_191"}for spider_name in self.spider_functions:self.listbox.insert(tk.END, spider_name)scrollbar = tk.Scrollbar(self.root, command=self.listbox.yview)scrollbar.pack(side=tk.RIGHT, fill=tk.Y)self.listbox.config(yscrollcommand=scrollbar.set)def create_run_button(self):self.run_button = tk.Button(self.root, text="运行", command=self.run_spider, font=('黑体', 12))self.run_button.pack(side=tk.RIGHT, padx=(0, 20), pady=50)def create_stop_button(self):self.stop_button = tk.Button(self.root, text="停止", command=self.stop_spider, state=tk.DISABLED, font=('黑体', 12))self.stop_button.pack(side=tk.RIGHT, padx=(20, 150), pady=50)def run_spider(self):selected_index = self.listbox.curselection()if not selected_index:messagebox.showwarning("警告", "请先选择一个爬虫")returnselected_item = self.listbox.get(selected_index)spider_name = self.spider_functions.get(selected_item)if spider_name:if spider_name in self.running_spiders:messagebox.showinfo("提示", f"{selected_item} 爬虫已经在运行")else:stop_event = Event()  # 创建一个Event对象spider = AgriInfoSpider()thread = Thread(target=self.run_spider_function, args=(spider, spider_name, stop_event))thread.start()self.running_spiders[spider_name] = {"thread": thread, "stop_event": stop_event}  # 将新运行的爬虫加入跟踪字典self.stop_button.config(state=tk.NORMAL)  # 启用停止按钮else:messagebox.showwarning("警告", "选择的爬虫不存在")def run_spider_function(self, spider, spider_name, stop_event):try:getattr(spider, spider_name)(stop_event)except Exception as e:messagebox.showerror("错误", f"爬虫运行出错: {e}")finally:self.root.after(0, self.update_stop_button, spider_name)def stop_spider(self):selected_index = self.listbox.curselection()if not selected_index:messagebox.showwarning("警告", "请先选择一个爬虫")returnselected_item = self.listbox.get(selected_index)spider_name = self.spider_functions.get(selected_item)if spider_name and spider_name in self.running_spiders:Thread(target=self.stop_spider_thread, args=(spider_name, selected_item)).start()else:messagebox.showwarning("警告", "选择的爬虫不存在或未运行")def stop_spider_thread(self, spider_name, selected_item):spider_info = self.running_spiders[spider_name]spider_info["stop_event"].set()spider_info["thread"].join()self.root.after(0, self.update_stop_button, spider_name)messagebox.showinfo("提示", f"{selected_item} 爬虫已停止")def update_stop_button(self, spider_name):del self.running_spiders[spider_name]if not self.running_spiders:self.stop_button.config(state=tk.DISABLED)if __name__ == "__main__":root = tk.Tk()app = App(root)root.mainloop()

运行的GUI界面及结果如下

最开始的界面：
GUI界面
点击运行：
点击运行
运行第二个爬虫：
运行第二个爬虫
停止运行：
停止运行

相关新闻

雍禾植发袁宣心中有“术”，雍禾医疗帮用户重启人生

雍禾植发袁宣心中有“术”，雍禾医疗帮用户重启人生

从公立医院烧伤外科来到雍禾，可以说是袁宣职业生涯里最重要的一个决定。据了解，袁宣医生所在的雍禾植发，是国内最大的毛发医疗机构。截至2022年12月31日，雍禾医疗已组建1341人的专业医疗团队，其中毛发医生294人&#x…

2026/7/29 13:39:45
熟悉HDFS常用操作

熟悉HDFS常用操作

1. 利用Hadoop提供的Shell命令完成下列任务 （1）向HDFS中上传任意文本文件，如果指定的文件在HDFS中已经存在，由用户指定是追加到原有文件末尾还是覆盖原有的文件。 #检查文件是否存在./bin/hdfs dfs -test -e text.txt echo $? #结果是1 代表已存在 #根据结果判断出文件已存…

2026/7/26 18:50:26
Selenium-java 定位元素时切换iFrame时的方法

Selenium-java 定位元素时切换iFrame时的方法

具体方法如下图所示，如果iFrame中嵌套多层iFrame需要逐层定位到需要的那一层iFrame,完成操作后，执行该代码：driver.switchTo() .defaultContent() ; 是返回最顶部的frame

2026/7/27 14:29:23

最新新闻

AI合同模板生成落地难题全破解（法律风控×NLP模型×私有化部署大揭秘）

AI合同模板生成落地难题全破解（法律风控×NLP模型×私有化部署大揭秘）

更多请点击： https://intelliparadigm.com 第一章：AI合同模板生成落地难题全破解（法律风控NLP模型私有化部署大揭秘） AI驱动的合同模板生成在实际企业落地中，常面临三重断层：法律条款的强合规性与模型泛化…

2026/7/29 13:39:02
Python客户端高效访问Tiled科学数据服务指南

Python客户端高效访问Tiled科学数据服务指南

1. 项目概述"使用Python客户端导航Tiled"这个项目标题看似简单，却蕴含着一个数据科学家日常工作中非常实用的技能点。作为一名长期与海量数据集打交道的从业者，我深刻理解高效访问和浏览结构化数据的重要性。Tiled作为一种新兴的数据服务框架&…

2026/7/29 13:39:02
机器人视觉控制实战：从DFRobot杯赛题到宇树G1跑道居中项目全解析

机器人视觉控制实战：从DFRobot杯赛题到宇树G1跑道居中项目全解析

1. 项目缘起：从“DFRobot杯”看一个赛事的五年沉淀五年前，当“DFRobot杯”这个名字第一次出现在国内创客与机器人教育的视野中时，它可能只是众多科创赛事中的一个新面孔。五年，对于一项赛事而言，足以经历从蹒跚学步到稳…

2026/7/29 13:39:02
Spring Boot构建个性化旅游推荐系统实战

Spring Boot构建个性化旅游推荐系统实战

1. 项目概述：当Spring Boot遇上个性化旅游推荐 去年帮朋友优化过一个旅游平台的推荐模块，发现市面上大多数推荐系统要么过于简单（仅按热度排序），要么复杂到需要专业算法团队维护。而Spring Boot的轻量级特性恰好能在两…

2026/7/29 13:39:17
小分子亲和探针：生物素标记双氢青蒿素Biotin-DHA靶点垂钓工具

小分子亲和探针：生物素标记双氢青蒿素Biotin-DHA靶点垂钓工具

生物素-双氢青蒿素 Biotin-DHA 双氢青蒿素（DHA，Dihydroartemisinin）是青蒿素最重要活性衍生物，不仅具备强效抗疟活性，近年大量研究证实其拥有抗肿瘤、抗纤维化、调控铁死亡、抗炎、免疫调节等广泛药理作用。 生物素 …

2026/7/29 13:39:17
如何3小时完成STM32 VL53L1X激光测距项目开发：终极实战指南

如何3小时完成STM32 VL53L1X激光测距项目开发：终极实战指南

如何3小时完成STM32 VL53L1X激光测距项目开发：终极实战指南 【免费下载链接】VL53L1X_STM32_module A simple VL53L1x module for STM32.Using software IIC 项目地址: https://gitcode.com/gh_mirrors/vl/VL53L1X_STM32_module 想要为你的STM32项目添加厘米…

2026/7/29 13:39:17

日新闻

自律同行，突破无界！NANK南卡正式官宣曾舜晞成为品牌代言人

自律同行，突破无界！NANK南卡正式官宣曾舜晞成为品牌代言人

近日，国际专注开放式技术研发的声学品牌Nank南卡，正式官宣实力艺人曾舜晞担任品牌代言人。消息一经发出便轰动全网。为什么耳机品牌不选择流量明星、老牌歌手？而且是选择曾舜晞？让我们一起来探索一下！比起短期的流量&a…

2026/7/29 0:00:42
【RT-DETR多模态创新改进】CVPR 2025 | 独家特征融合创新改进篇 | 引入RLAB残差线性注意力模块，有效融合并强调多尺度特征，多种改进点，适合红外与可见光融合目标检测任务，有效涨点

【RT-DETR多模态创新改进】CVPR 2025 | 独家特征融合创新改进篇 | 引入RLAB残差线性注意力模块，有效融合并强调多尺度特征，多种改进点，适合红外与可见光融合目标检测任务，有效涨点

一、本文介绍 🔥本文在RT-DETR多模态融合目标检测中引入RLAB残差线性注意力模块，可在不同模态特征交互阶段进行多次残差细化，使可见光、红外等特征在尺度、语义和空间位置上更好对齐；随后将细化特征与解码器输出拼接并生成Q、K、V，通过线性注意力自适应强化关键通道、目…

2026/7/29 0:00:42
AI编程系列02：合并知识功能，给 AI 问数和 RAG 场景打基础

AI编程系列02：合并知识功能，给 AI 问数和 RAG 场景打基础

AI编程系列02：合并知识功能，给 AI 问数和 RAG 场景打基础 在上一期「AI编程系列」中，我们学习了如何构建一个基础的 AI 问答系统，通过简单的输入输出让模型回应问题。但现实世界中的 AI 应用往往需要处理更复杂的场景：…

2026/7/29 0:00:42

周新闻

Apex英雄目标检测数据集 深度学习框架YOLO如何训练APEX数据集

Apex英雄目标检测数据集 深度学习框架YOLO如何训练APEX数据集

Apex检测数据集数据集详情检测类别： allies enemy tag图片总量：7247张训练集：5139张验证集：1425张测试集：683张标注状态：全部已标注，即拿即用数据格式：支持YOLO格式及其他格式&#…

2026/7/29 11:47:07
智慧飞行 大疆无人机一站式智能管控平台/支持大疆机场/私有化部署 从航线规划、自动飞行、AI识别到数据管理，一个平台全搞定

智慧飞行 大疆无人机一站式智能管控平台/支持大疆机场/私有化部署 从航线规划、自动飞行、AI识别到数据管理，一个平台全搞定

智慧飞行-大疆无人机一站式智能管控平台/支持大疆机场/私有化部署 企业级全域智能无人机一体化管控平台源码！ 专为电力巡检、安防监控、应急救援、测绘勘察等行业打造，让您的无人机舰队实现 “无人化、自动化、智能化” 管理！ 🎯 …

2026/7/29 11:47:11
揭秘ChatGPT+Mathematica协同教学：为什么92%的初学者在72小时内建立函数直觉？

揭秘ChatGPT+Mathematica协同教学：为什么92%的初学者在72小时内建立函数直觉？

更多请点击： https://codechina.net 第一章：AI帮助理解数学概念 人工智能正以前所未有的方式重塑数学学习的路径。通过自然语言处理与符号计算的深度融合，AI不仅能解析抽象定义，还能将定理、证明和几何直觉转化为可交互、可验证的…

2026/7/29 11:47:11

月新闻

[C++]内存管理：串顺序存储的内存回收

[C++]内存管理：串顺序存储的内存回收

在串（字符串）的顺序存储中，内存回收的方式取决于字符串的存储方式以及所使用的编程语言和相关库。以下以 C 为例进行说明，因为 C 对内存管理有较为直接的控制。 1. 基于 char 数组的串顺序存储 如果使用普通的 char 数组来存储字…

2026/7/29 2:28:15
移动端游戏功耗测试实战：电流、功率、亮度和场景对比

移动端游戏功耗测试实战：电流、功率、亮度和场景对比

移动端游戏功耗测试：先控制变量，再比较优化是否真的省电 摘要：功耗测试最容易犯的错误，是拿两次不同温度、不同亮度、不同场景的平均功率直接比较。本文给出一套可复现的游戏功耗测试方法，覆盖引擎特性验证、版本回归和黑盒体验测试，并说明如何把功耗与帧率、温控、CPU/G…

2026/7/29 2:28:16
足球口袋教练 HarmonyOS 离线应用实战（03/20）：ArkUI 首页仪表盘搭建

足球口袋教练 HarmonyOS 离线应用实战（03/20）：ArkUI 首页仪表盘搭建

本文是“足球口袋教练 HarmonyOS 离线应用实战”系列第 3 篇。示例项目是一个 HarmonyOS / ArkTS / ArkUI 编写的离线足球训练助手，围绕真实页面、真实截图和可复现操作展开。 本篇要解决的问题 训练 App 的首页不能只展示欢迎语，它要解决“我现在该点哪…

2026/7/29 12:06:51