微博关键词搜索并爬取前40页内容与图片

微博关键词搜索并爬取前40页内容与图片

# -*- coding: utf-8 -*- """ @author: tanderick """ import requests import re import os import urllib.parse import time #header文件 headers ={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0)'} #搜索名词 keyword = '简历' #创建同名文件夹 filepath = str(r'C:/weibo/'+keyword) if not os.path.exists(filepath): os.mkdir(filepath) #搜索名词下40页源码并保存为txt文件 kw=urllib.parse.quote(keyword) s_url ='https://s.weibo.com/weibo?q='+kw+'&wvr=6&b=1&Refer=SWeibo_box' f = requests.get(s_url,headers = headers) for i in range(40): html = requests.get(s_url+'&page='+str(i),headers = headers) html = html.text html =urllib.parse.unquote(html) print(i) with open(filepath+'/'+keyword+'.txt','a',encoding ="utf-8") as f: f.write(html) time.sleep(0.5) #打开该文件 with open(filepath+'/'+keyword+'.txt','r',encoding ="utf-8") as h: html = h.read() #解析内容并下载 uids = re.findall('<a href="//weibo.com/(.*?)?refer_flag=1001030103_" class=".*?" target=".*?" nick-name="(.*?)" suda-data=".*?">.*?</a>',html) contents = re.findall(' <p class="txt" node-type="feed_list_content" nick-name=".*?">(.*?)</p>',html,re.S) pic_id = re.findall('<!--card-wrap-->(.*?)<!--/card-wrap-->',html,re.S) for i in range(len(uids)): uid,nickname = uids[i] out_filepath =filepath+'/'+nickname if not os.path.exists(out_filepath): os.mkdir(out_filepath) with open(out_filepath+'/微博内容.txt','a',encoding ="utf-8") as f: f.write(str(uids[i])+'\r\n'+re.sub('<.*?>','',contents[i],re.S)) #获取用户名与微博内容 pic_urls1 = re.findall('img src="(.*?)jpg".*?',pic_id[i]) pic_urls2 = re.findall('cover_img=(.*?)jpg.*?',pic_id[i]) for url1 in pic_urls1: url1 = re.sub(r'https:','',str(url1)) filename = url1.split('/')[-1] response = requests.get(r'http:'+url1+'jpg',headers=headers) with open(out_filepath+'/'+filename+'jpg','wb') as f: f.write(response.content) print(r'http:'+url1+'jpg'+'下载完成') for url2 in pic_urls2: url2 = re.sub(r'https:','',str(url2)) filename = url1.split('/')[-1] response = requests.get(r'http:'+url2+'jpg',headers=headers) with open(out_filepath+'/'+filename+'jpg','wb') as f: f.write(response.content) print(r'http:'+url2+'jpg'+'下载完成') #下载图片