爬虫大作业

难点是后端 首先是实现注册登录模块 对应在mysql中建立user和user_action两张表用以储存用户信息 对应在代码端 访问数据库的mysqlmodule.exports = { mysql: { host : 'localhost'， user : 'root', password: 'root', database: 'crawl', connectionLimit: 10 } }; 登录/注册端

app.use(session({ secret: 'sessiontest',//与cookieParser中的一致 resave: true, saveUninitialized: false, // 是否保存未初始化的会话 cookie : { maxAge : 1000 * 60 * 60, // 设置 session 的有效时间，单位毫秒 }, })); $scope.check_pwd = function () { var data = JSON.stringify({ username: $scope.username, password: $scope.password }); $http.post("/users/login", data) .then( function (res) { if(res.data.msg=='ok') { window.location.href='/news.html'; }else{ $scope.msg=res.data.msg; } }, function (err) { $scope.msg = err.data; }); }; $scope.doAdd = function () { if($scope.add_password!==$scope.confirm_password){ // $timeout(function () { // $scope.msg = '两次密码不一致！'; // },100); $scope.msg = '两次密码不一致！'; } else { var data = JSON.stringify({ username: $scope.add_username, password: $scope.add_password }); $http.post("/users/register", data) .then(function (res) { if(res.data.msg=='成功注册！请登录') { $scope.msg=res.data.msg; $timeout(function () { window.location.href='index.html'; },2000); } else { $scope.msg = res.data.msg; } }, function (err) { $scope.msg = err.data; }); } }； router.post('/login', function(req, res) { var username = req.body.username; var password = req.body.password; // var sess = req.session; userDAO.getByUsername(username, function (user) { if(user.length==0){ res.json({msg:'用户不存在！请检查后输入'}); }else { if(password===user[0].password){ req.session['username'] = username; res.cookie('username', username); res.json({msg: 'ok'}); // res.json({msg:'ok'}); }else{ res.json({msg:'用户名或密码错误！请检查后输入'}); } } }); }); /* add users */ router.post('/register', function (req, res) { var add_user = req.body; // 先检查用户是否存在 userDAO.getByUsername(add_user.username, function (user) { if (user.length != 0) { // res.render('index', {msg:'用户不存在！'}); res.json({msg: '用户已存在！'}); }else { userDAO.add(add_user, function (success) { res.json({msg: '成功注册！请登录'}); }) } }); }); // 退出登录 router.get('/logout', function(req, res, next){ req.session.destroy(function(err) { if(err){ res.json('退出登录失败'); return; } // req.session.loginUser = null; res.clearCookie('username'); res.json({result:'/index.html'}); }); });

查询端

$scope.search = function () { var title1 = $scope.title1; var title2 = $scope.title2; var selectTitle = $scope.selectTitle; var content1 = $scope.content1; var content2 = $scope.content2; var selectContent = $scope.selectContent; var sorttime = $scope.sorttime; // 检查用户传的参数是否有问题 //用户有可能这样输入：___ and/or 新冠（直接把查询词输在了第二个位置） if(typeof title1=="undefined" && typeof title2!="undefined" && title2.length>0){ title1 = title2; } if(typeof content1=="undefined" && typeof content2!="undefined" && content2.length>0){ content1 = content2; } // 用户可能一个查询词都不输入，默认就是查找全部数据 var myurl = `/news/search?t1=${title1}&ts=${selectTitle}&t2=${title2}&c1=${content1}&cs=${selectContent}&c2=${content2}&stime=${sorttime}`; $http.get(myurl).then( function (res) { if(res.data.message=='data'){ $scope.isisshowresult = true; //显示表格查询结果 // $scope.searchdata = res.data; $scope.initPageSort(res.data.result) }else { window.location.href=res.data.result; } },function (err) { $scope.msg = err.data; }); };cript $scope.showSearch = function () { $scope.isShow = true; $scope.isisshowresult = false; // 再次回到查询页面时，表单里要保证都空的 $scope.title1=undefined; $scope.title2=undefined; $scope.selectTitle='AND'; $scope.content1=undefined; $scope.content2=undefined; $scope.selectContent='AND'; $scope.sorttime=undefined; }; $scope.logout = function () { // $http.get().then(); $http.get("/users/logout").then( function (res) { window.location.href=res.data.result; },function (err) { $scope.msg = err.data; } ); };

分页功能

$scope.first = function () { $scope.selectPage(1);}; $scope.last = function () { $scope.selectPage($scope.pages);};

之后是我们的图表 1.柱状图

$scope.histogram = function () { $scope.isShow = false; $http.get("/news/histogram") .then( function (res) { if(res.data.message=='url'){ window.location.href=res.data.result; }else { // var newdata = washdata(data); let xdata = [], ydata = [], newdata; var pattern = /\d{4}-(\d{2}-\d{2})/; res.data.result.forEach(function (element) { // "x":"2020-04-28T16:00:00.000Z" ,对x进行处理,只取 月日 xdata.push(pattern.exec(element["x"])[1]); ydata.push(element["y"]); }); newdata = {"xdata": xdata, "ydata": ydata}; var myChart = echarts.init(document.getElementById('main1')); // 指定图表的配置项和数据 var option = { title: { text: '新闻发布数 随时间变化' }, tooltip: {}, legend: { data: ['新闻发布数'] }, xAxis: { data: newdata["xdata"] }, yAxis: {}, series: [{ name: '新闻数目', type: 'bar', data: newdata["ydata"] }] }; // 使用刚指定的配置项和数据显示图表。 myChart.setOption(option); } }, function (err) { $scope.msg = err.data; }); };

2.云词

$scope.wordcloud = function () { $scope.isShow = false; $http.get("/news/wordcloud").then( function (res) { if(res.data.message=='url'){ window.location.href=res.data.result; }else { var mainContainer = document.getElementById('main1'); var chart = echarts.init(mainContainer); var data = []; for (var name in res.data.result) { data.push({ name: name, value: Math.sqrt(res.data.result[name]) }) } var maskImage = new Image(); maskImage.src = './images/logo.png'; var option = { title: { text: '所有新闻内容 jieba分词 的词云展示' }, series: [{ type: 'wordCloud', sizeRange: [12, 60], rotationRange: [-90, 90], rotationStep: 45, gridSize: 2, shape: 'circle', maskImage: maskImage, drawOutOfBound: false, textStyle: { normal: { fontFamily: 'sans-serif', fontWeight: 'bold', // Color can be a callback function or a color string color: function () { // Random color return 'rgb(' + [ Math.round(Math.random() * 160), Math.round(Math.random() * 160), Math.round(Math.random() * 160) ].join(',') + ')'; } }, emphasis: { shadowBlur: 10, shadowColor: '#333' } }, data: data }] }; maskImage.onload = function () { // option.series[0].data = data; chart.clear(); chart.setOption(option); }; window.onresize = function () { chart.resize(); }; } }); } });

3.折线图

$scope.line = function () { $scope.isShow = false; $http.get("/news/line").then( function (res) { if(res.data.message=='url'){ window.location.href=res.data.result; }else { var myChart = echarts.init(document.getElementById("main1")); option = { title: { text: '"疫情"该词在新闻中的出现次数随时间变化图' }, xAxis: { type: 'category', data: Object.keys(res.data.result) }, yAxis: { type: 'value' }, series: [{ data: Object.values(res.data.result), type: 'line', itemStyle: {normal: {label: {show: true}}} }], }; if (option && typeof option === "object") { myChart.setOption(option, true); } } }); };

新冠肺炎 一词 出现频率

饼图

$scope.pie = function () { $scope.isShow = false; $http.get("/news/pie").then( function (res) { if(res.data.message=='url'){ window.location.href=res.data.result; }else { let newdata = []; var pattern = /./;//匹配名字 res.data.result.forEach(function (element) { // "x": 责任编辑：李夏君 ,对x进行处理,只取 名字 newdata.push({name: pattern.exec(element["x"])[1], value: element["y"]}); }); var myChart = echarts.init(document.getElementById('main1')); var app = {}; option = null; // 指定图表的配置项和数据 var option = { title: { text: '作者发布新闻数量', x: 'center' }, tooltip: { trigger: 'item', formatter: "{a} <br/>{b} : {c} ({d}%)" }, legend: { orient: 'vertical', left: 'left', // data: ['直接访问', '邮件营销', '联盟广告', '视频广告', '搜索引擎'] }, series: [ { name: '访问来源', type: 'pie', radius: '55%', center: ['50%', '60%'], data: newdata, itemStyle: { emphasis: { shadowBlur: 10, shadowOffsetX: 0, shadowColor: 'rgba(0, 0, 0, 0.5)' } } } ] }; // myChart.setOption(option); app.currentIndex = -1; setInterval(function () { var dataLen = option.series[0].data.length; // 取消之前高亮的图形 myChart.dispatchAction({ type: 'downplay', seriesIndex: 0, dataIndex: app.currentIndex }); app.currentIndex = (app.currentIndex + 1) % dataLen; // 高亮当前图形 myChart.dispatchAction({ type: 'highlight', seriesIndex: 0, dataIndex: app.currentIndex }); // 显示 tooltip myChart.dispatchAction({ type: 'showTip', seriesIndex: 0, dataIndex: app.currentIndex }); }, 1000); if (option && typeof option === "object") { myChart.setOption(option, true); } ; } }); };

不知道是浏览器问题还是什么，饼图是一直加载不出来的。我怀疑是我电脑修改过位色导致的。