影评分析

import pandas as pd import jieba from wordcloud import WordCloud import matplotlib.pyplot as plt import numpy as np data = pd.read_csv('comment.csv',lineterminator="\n") #查看每一列的缺失值 data.isnull().sum()

数据预处理

#填充缺失值 data = data.fillna('Unknow') print(data.isnull().sum()) #删除重复记录 data.drop_duplicates(inplace=True) #文本内容清洗，清楚特殊符号，用正则表达式 import re pattern = r"[!\"#$%&'()*+,-./:;<=>?@[\\\]^_^{|}~—！，。？、￥…（）：【】《》‘’“”\s]+" re_obj = re.compile(pattern) def clear(text): return re_obj.sub("",text) data['comment'] = data['comment'].apply(clear) #分词 def cut_word(text): return jieba.cut(text) data["comment"] = data["comment"].apply(cut_word) #停用词处理 def get_stopword(): s = set() with open("hit_stopwords.txt", encoding="UTF-8") as f: for line in f: s.add(line.strip()) return s def remove_stopword(words): return [word for word in words if word not in stopword] stopword = get_stopword() data["comment"] = data["comment"].apply(remove_stopword)

探索性分析

#城市分析 crank = data.groupby("city")["name"].count().reset_index().sort_values(by='name',ascending=False)[:10] fig, ax = plt.subplots(1,2) #创建画布 一行 两列 fig.set_size_inches(15,5) #设置画布大小 count = data["city"].value_counts() top = count.iloc[:10] bottom = count.iloc[-10:] for index, d, title in zip(range(2), [top, bottom], ["前10城市","后10城市"]): a = sns.barplot(d.index, d.values, ax=ax[index]) #旋转45度 a.set_xticklabels(a.get_xticklabels(),rotation=45) a.set_title(title) #评分分析 score = data.groupby("score")["name"].count().reset_index() score.columns=["评分","数量"] sns.countplot(x="score",data=data,log=True) #统计出现频率最多的15个词汇 from itertools import chain from collections import Counter li_2d = data['comment'].tolist() #将二维列表转换为一维 li_1d = list(chain.from_iterable(li_2d)) print(f'总词汇量：{len(li_1d)}') c = Counter(li_1d) print(f'不重复词汇量：{len(c)}') common = c.most_common(15) import seaborn as sns plt.rcParams['font.family'] = ['sans-serif'] plt.rcParams['font.sans-serif'] = ['SimHei'] d = dict(common) plt.figure(figsize=(15, 5)) sns.barplot(list(d.keys()), list(d.values()))

建模分析

#建立模型 def join(text_list): return " ".join(text_list) data['comment'] = data['comment'].apply(join) #构造目标列 data['target'] = np.where(data['score'] >=4.5, 2,np.where(data['score'] >=3,1,0)) data['target'].value_counts() #下采样 p = data[data['target'] == 2] m = data[data['target'] == 1] n = data[data['target'] == 0] p = p.sample(len(m)) m = m.sample(len(m)) data2 = pd.concat([p,m,n],axis=0) data2['target'].value_counts() # 构建训练集和测试集。 from sklearn.model_selection import train_test_split X = data2['comment'] y = data2['target'] X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.25) print('训练集样本数：',X_train.shape[0],'测试集样本数',X_test.shape[0]) #特征选择 vec = TfidfVectorizer(ngram_range=(1,2),max_df=0.5,min_df=1) X_train_trans = vec.fit_transform(X_train) X_test_trans = vec.fit_transform(X_test) display(X_train_trans,X_test_trans)

逻辑回归

from sklearn.linear_model import LogisticRegression from sklearn.metrics import classification_report lr = LogisticRegression(class_weight='balanced',multi_class='ovr',solver='sag') lr.fit(X_train_trans,y_train) y_hat = lr.predict(X_test_trans) print(classification_report(y_test,y_hat))

朴素贝叶斯

from sklearn.naive_bayes import ComplementNB gnb = ComplementNB() gnb.fit(X_train_trans,y_train) y_hat = gnb.predict(X_test_trans) print(classification_report(y_test,y_hat))