用pycharm，别用啥vscode，pycharm最好用 先创建一个scrapy项目，首先创建一个python项目，然后去终端输入指令 “scrapy startproject module”，其中module为模块名
这是我的项目结构 ，因为我爬取的是晋江，所以爬来的小说名字非常的沙雕
然后如果你没用命令行，是自己建的项目，那一定要加入这个.cfg文件，不然是跑不起来的。
然后middlewares.py是不用动的，自己只需要写pipeline,spider,items这三个，data和cloud是我用来分析的，和爬虫无关。
先看最简单的item把
可以写多个类对应不同的爬取结果 里面的每个属性都是随便设的，你需要爬取什么就加入什么
import scrapy class MyscrapyItem(scrapy.Item): # 小说名称 novelName = scrapy.Field() # 作者名称 authorName = scrapy.Field() # 小说内容页链接 novelContent = scrapy.Field() class NovelItem(scrapy.Item): description = scrapy.Field() novelName = scrapy.Field() Day1 = scrapy.Field() Day2 = scrapy.Field() Day3 = scrapy.Field() Day4 = scrapy.Field() Day5 = scrapy.Field()然后我第一步是去网站里把vip金榜（？或许是这个）里的小说名称+作者名称+链接爬取过来 Myspider.py
import scrapy from myscrapy.items import MyscrapyItem,NovelItem class Spider(scrapy.Spider): # 爬虫名 name = "myspider" # 起始链接 start_urls = ['http://www.jjwxc.net/fenzhan/yq/'] name_list=[] def parse(self, response): # 开始爬取 print("spider start") # 取出所有<li>标签中style=_width:183px;的标签 node_list = response.xpath("//li[@style='_width:183px;']") next_url=[] # 遍历list for x in range(len(node_list)): node=node_list[x] # 创建item字段对象用来存储信息 item = MyscrapyItem() # extract() : 将xpath对象转换为Unicode字符串 novelName = node.xpath("./a/@alt").extract() self.name_list.append(novelName) authorName = node.xpath("./a/label/text()").extract() novelContent = node.xpath("./a/@href").extract() # 对的到的信息进行一点加工,并放入item中 item['novelName'] = novelName[0].split(" ")[0] item['authorName'] = authorName[0] item['novelContent'] = "http://www.jjwxc.net/" + novelContent[0] urls=item['novelContent'] next_url.append(item['novelContent'][0]) yield item然后在pipeline中进行存储 存储为csv格式，Novel.csv就是结果
import csv from myscrapy.items import MyscrapyItem,NovelItem class MyscrapyPipeline: # 初始化 def __init__(self): self.f = open("Novel.csv", "w") self.writer = csv.writer(self.f) self.writer.writerow(['novelName', 'authorName', 'novelContent']) self.test=None def process_item(self, item, spider): #根据不同的item进行分别处理 if isinstance(item, MyscrapyItem): novel_list = [item['novelName'], item['authorName'], item['novelContent']] self.writer.writerow(novel_list) self.writer.close() return item elif isinstance(item, NovelItem): name=item['novelName'] self.test=test(name) novel_list = [item['description'],item['Day1'],item['Day2'],item['Day3'],item['Day4'],item['Day5']] self.test.writer.writerow(novel_list) return item #写入csv def close_spider(self, spider): # 关闭 self.test.close() self.writer.close() self.f.close() #上面的是生成所有小说的汇总 #这个是专门处理每本小说的类 class test(): # 初始化 def __init__(self,name): self.f = open(name+".csv", "w") self.writer = csv.writer(self.f) self.writer.writerow(['description','Day1','Day2','Day3','Day4','Day5']) def close(self): self.writer.close() self.f.close()效果
然后进一步对每本小说继续爬取
import scrapy from myscrapy.items import MyscrapyItem,NovelItem class Spider(scrapy.Spider): # 爬虫名 name = "myspider" # 起始链接 start_urls = ['http://www.jjwxc.net/fenzhan/yq/'] name_list=[] def parse(self, response): # 开始爬取 print("spider start") # 取出所有<li>标签中style=_width:183px;的标签 node_list = response.xpath("//li[@style='_width:183px;']") next_url=[] # 遍历list for x in range(len(node_list)): node=node_list[x] # 创建item字段对象用来存储信息 item = MyscrapyItem() # extract() : 将xpath对象转换为Unicode字符串 novelName = node.xpath("./a/@alt").extract() self.name_list.append(novelName) authorName = node.xpath("./a/label/text()").extract() novelContent = node.xpath("./a/@href").extract() # 对的到的信息进行一点加工,并放入item中 item['novelName'] = novelName[0].split(" ")[0] item['authorName'] = authorName[0] item['novelContent'] = "http://www.jjwxc.net/" + novelContent[0] urls=item['novelContent'] next_url.append(item['novelContent'][0]) yield item #对爬取到的新链接进行再次爬取 yield scrapy.Request(url=urls, callback=self.parse_novel) #对每本小说继续爬取 def parse_novel(self, response): print("开始爬取小说") #新的Item Novelitem = NovelItem() #取出小说名称 Name = response.xpath("//span[@ itemprop='articleSection']") for node in Name: novelName=node.xpath("./text()").extract() Novelitem['novelName']=novelName[0] #取出包含字数的标签 node_list = response.xpath("//td[@itemprop='wordCount']") #取出前五章的字数并装入item中 Day1 = node_list[0].xpath("./text()").extract() Novelitem['Day1'] = Day1[0] Day2 = node_list[1].xpath("./text()").extract() Novelitem['Day2'] = Day2[0] Day3 = node_list[2].xpath("./text()").extract() Novelitem['Day3'] = Day3[0] Day4 = node_list[3].xpath("./text()").extract() Novelitem['Day4'] = Day4[0] Day5 = node_list[4].xpath("./text()").extract() Novelitem['Day5'] = Day5[0] #取出小说的描述用于词云 descriptions=response.xpath("//div[@itemprop='description']").extract()[0] #拼接并过滤标签<br> description="" for x in descriptions: description=(description+x).replace("<br>"," ") #加入item Novelitem['description'] = description yield Novelitem先把对应的包装上 pip install -i https://pypi.tuna.tsinghua.edu.cn/simple pandas pip install -i https://pypi.tuna.tsinghua.edu.cn/simple jieba pip install -i https://pypi.tuna.tsinghua.edu.cn/simple wordcloud pip install -i https://pypi.tuna.tsinghua.edu.cn/simple matplotlib
然后爬虫生成了每本小说的csv文件，再在data.py中进行分析 simhei.ttf是字体，不然里面的中文全是方格
import pandas as pd import matplotlib.pyplot as plt import matplotlib.font_manager as fm import os class data(): #文件目录 #urls=os.getcwd().replace("/myscrapy/spiders","") urls="/Users/polarischen/PycharmProjects/Testscrapy/myscrapy/" print(urls) #加入中文字体 myfont = fm.FontProperties(fname=urls+"/simhei.ttf",size=7.0) #导入数据 data1 = pd.read_csv(urls+"/天才女友.csv") data2= pd.read_csv(urls+"/不学习就要继承亿万家产.csv") data3 = pd.read_csv(urls+"/人参精穿成年代文中小闺女.csv") data4= pd.read_csv(urls+"/我成了反派的挂件[穿书].csv") #x轴 x=['Day1','Day2','Day3','Day4','Day5'] #y轴数据 num1=data1.iloc[:,[1,2,3,4,5]] num2=data2.iloc[:,[1,2,3,4,5]] num3=data3.iloc[:,[1,2,3,4,5]] num4=data4.iloc[:,[1,2,3,4,5]] print(type(num1.loc[0])) #绘图 plt.plot(x,num1.loc[0], label='天才女友') plt.plot(x, num2.loc[0],label='不学习就要继承亿万家产') plt.plot(x, num3.loc[0], label='人参精穿成年代文中小闺女') plt.plot(x, num4.loc[0], label='我成了反派的挂件[穿书]') # 显示图例, 图例中内容由 label 定义 plt.legend(prop=myfont) # 横坐标轴的标题 plt.ylabel('字数',fontproperties=myfont) # 纵坐标轴的标题 plt.xlabel('日期',fontproperties=myfont) plt.title("小说前五章的字数",fontproperties=myfont) plt.show()效果图
然后再创建cloud.py进行词云图的创建,自己随便找一个csv文件里小说的description搞一个txt文件就行。我的就叫description.txt
# 第三方库 from wordcloud import WordCloud import jieba import os import matplotlib.pyplot as plt # 数据地址(此处filename为.txt文件所在地址) filename1 = os.getcwd().replace("/myscrapy/spiders", "")+"/description.txt" # 数据加载 with open(filename1, encoding='UTF-8') as f1: data1 = f1.read(); # 用jieba分别进行分词分析 wordList_jieba1 = jieba.cut(data1, cut_all=False); # 链接成新文档 data1 = ','.join(wordList_jieba1); # 字体 font = os.getcwd().replace("/myscrapy/spiders", "")+"/simhei.ttf" # 词云分析 wc1 = WordCloud(font_path=font).generate(data1); # 词云展示 plt.imshow(wc1, interpolation='bilinear'); plt.axis('off'); plt.show();效果图： 结束
