从movie中取相同电影的映射关系，然后从details中取这些id对应的信息，id重新设置从0开始
import pandas as pd import pymongo from setting import setting db = pymongo.MongoClient(host=setting['host'])['movie'] //连接数据库 db.authenticate(setting['username'], setting['password'])//用户验证 //excludes = ['insertStamp', 'timestamp', 'rating', 'rateNum', 'cover'] //excludes = {v: False for v in excludes} //data = db['details'].find({}, excludes) //df = pd.DataFrame.from_records(data) //df.to_csv('details.csv', index=False) doubanIds = [] table = {'maoyan': {}, 'mtime': {}} //otherSrcId -> doubanId //只需要source字段的数据 for doc in db['movie'].find({}, {'source': True}, no_cursor_timeout=True): doubanId = doc['source']['douban']['sourceId'] doubanIds.append(doubanId) for src in ['maoyan', 'mtime']: if src in doc['source']: table[src][doc['source'][src]['sourceId']] = doubanId print('douban num: ', len(doubanIds)) details = pd.read_csv('details.csv', index=False) douban = details[(details['source'] == 'douban') & details['sourceId'].isin(doubanIds)] maoyan = details[(details['source'] == 'maoyan') & details['sourceId'].isin(table['maoyan'].keys())] mtime = details[(details['source'] == 'mtime') & details['sourceId'].isin(table['mtime'].keys())] df = douban.append(maoyan).append(mtime).reset_index() //拼接起来，重新设置index，从0开始，作为新的id trans = {'douban': {}, 'maoyan': {}, 'mtime': {}} for id, row in df.iterrows(): trans[row['source']][str(row['sourceId'])] = id # srcId -> newId with open('link.csv', 'w') as f: for src in ['maoyan', 'mtime']: for k, v in table[src].items(): f.write("{}\t{}\n".format(trans[src][k], trans['douban'][v])) # otherNewId, doubanNewId df['id'] = df.index df.drop(columns=['imdb', 'source', 'sourceId']).to_csv('data.csv', index=False) //drop去掉不需要的列输出link.csv每行表示其他来源和douban的这个id是同一个电影 data.csv是link中电影对应地信息
