浙江高职高校网站爬虫——浙江工贸职业技术学院

import requests from lxml import etree import os

def write_page(title,author,source,time,test):#写入文件 print(“正在保存:”+title) with open(title+’.txt’,‘w’,encoding=‘UTF-8’) as file: file.write(title+’\n’) file.write(author+’\n’) file.write(source+’\n’) file.write(time+’\n’) file.write(test) file.close() def spider_data(url):#爬取每条通知的数据 response = requests.get(url) response.encoding = ‘utf-8’ html = etree.HTML(response.text) title=’’.join(html.xpath(’//h2/text()’)) zz=’’.join(html.xpath("//div[@class=‘zz’]/text()")) test=’’.join(html.xpath(’//*[@id=“vsb_content”]/p/text()’)) author=zz.split()[0] source=zz.split()[1] time=zz.split()[2]+" "+zz.split()[3]

write_page(title,author,source,time,test) #print(type(title))

def spider_url(url):#爬取每条通知的url

response = requests.get(url) response.encoding = 'utf-8' result = response.text html = etree.HTML(result) all_url = html.xpath("//div[@class='right-1']/ul/li/a/@href") url = [] for i in all_url: if i.startswith("../"): url.append('http://www.zjitc.net' + i.lstrip('..')) return url

def spider_list_url():#爬取每页的url url = ‘http://www.zjitc.net/xwzx/tztg.htm’ num = int(input(“请输入要爬取得页数:”)) url_lsit = [] url_lsit.append(url) for i in range(num - 1): response = requests.get(url) response.encoding = ‘utf-8’ result = response.text html = etree.HTML(result) url_next = html.xpath("//a[contains(string(), ‘下页’)]/@href")[0] if url_next.startswith(‘tztg’): url = ‘http://www.zjitc.net/xwzx/’ + url_next else: url = ‘http://www.zjitc.net/xwzx/tztg/’ + url_next url_lsit.append(url) return url_lsit if name==“main”: for i in spider_list_url(): for j in spider_url(i): spider_data(j)