一.数据集

train.dat共393366项，第一维是用户的代号，第二维是商品代号，第三项是用户对该商品的评分，第四项是评论数，第五项是评论内容。

test.dat第一维是用户代号，第二维是商品代号。

二.简单思路及实现过程

这里简单先贴出代码实现，后期再慢慢补上具体过程说明~

1．数据预处理

去除部分无关常用词，这里调用nltk包，将评论词词词根化等，进行评论数据的预处理，存为new dat.dat

import nltk

from nltk.corpus import stopwords

from nltk.stem.porter import PorterStemmer

from sklearn.model_selection import train_test_split

def textPrecessing(text):

wordLst = nltk.word_tokenize(text)

filtered = [w for w in wordLst if w not in stopwords.words('english')]

refiltered =nltk.pos_tag(filtered)

filtered = [w for w, pos in refiltered if pos.startswith('NN')]

ps = PorterStemmer()

filtered = [ps.stem(w) for w in filtered]

return " ".join(filtered)

def split_word():

x = []

y = []

with open("‪E:/project/o/comdata/train.dat", encoding='utf-8') as f:

for data in f.readlines():

data = data.strip("\n").split(" ")

temp = []

temp.append(' '.join(data[0:4]))

text = data[4:]

temp.append(textPrecessing(" ".join(text)))

x.append(' '.join(temp))

with open("‪E:/project/o/comdata/newdat.dat", "w", encoding='utf-8') as w:

for i in x:

w.write(i)

w.write('\n')

f.close()

w.close()

with open('train.dat',encoding='UTF-8') as file_object:     #获取训练集

for j in range(303366):

line = file_object.readline()

if(line):

traindata.append(line.strip('\n').split(' ')[0:4]) #

word=line.strip('\n').split(' ')[4:]

comm=[]

for w in word:

if w not in stoplist:

comm.append(w)

commentdata.append(comm)   # get comment information

2.获取对应的用户名-id，物品-id字典，构建用户id-物品id 评分表

useridlist={} #用户名对应序号

itemidlist={} #物品对应序号

num=0

mun=0

uu=0

for item in traindata:

if item[0] in useridlist:

uu+=1

if item[0] not in useridlist:

useridlist[item[0]]=num

num+=1

for item in traindata:

if item[1] not in itemidlist:

itemidlist[item[1]]=mun

mun+=1

scorematr=np.zeros((len(useridlist),len(itemidlist))) #评分表ui

for item in traindata:

i=int(useridlist[item[0]])

j=int(itemidlist[item[1]])

scorematr[i][j]=int(item[2])

3.尝试LFM法

def Lfm(T,k):

alpha = np.float32(0.05)

lambda_ = np.float32(0.08)

m, n = T.shape

u = np.float32(np.random.rand(m,k))

v = np.float32(np.random.ran77dn(k,n))

du=np.float32(0)

dv=np.float32(0)

for t in range(500):

for i in range(m):

for j in range(n):

if math.fabs(T[i][j]) > 1e-4:

err = T[i][j] - np.dot(u[i],v[:,j])

for t in range(k):

du = err * v[t][j] - lambda_ * u[i][t]

dv = err * u[i][t] - lambda_ * v[t][j]

u[i][t] += alpha * du; v[t][j] += alpha * dv

return u,v

#u,v=Lfm(scorearr) print(np.dot(u,v))

四．尝试LDA法/word2vec法基于评论预测

① LDA 模型

dictionary = corpora.Dictionary(commentdata)  #建立字典

dictionary.filter_tokens(bad_ids=['i','the','is','and','very','a','an','had','about','for','it','if','of','to'])

dictionary.filter_extremes(no_above=40/100)

#dictionary.filter_n_most_frequent(8)

print(dictionary.token2id)   #打印字典表

corpus = [dictionary.doc2bow(text) for text in commentdata]

print(corpus[3:5])

print("corpus----")

lda=LdaModel(corpus=corpus,id2word=dictionary,num_topics=20)

# 训练LDA模型

lda.save("./lda.model")

# lda=LdaModel.load("./lda.model")

② word2vec 模型

model="./pinglun.model"

if os.path.exists(model):

model=gensim.models.Word2Vec.load("./pinglun.model")

else :

model = gensim.models.word2vec.Word2Vec(commentdata, size=200,

workers=7,min_count=2)  #训练 word2vec模型

model.save("./pinglun.model")

(1)求θuij

① LDA方法的θ取法

def getHuik(doc_topic,n,bo):   #获得Huik

Huikarr = np.zeros((u0, i0, n))

for ui in traindata:

u=int(useridlist[ui[0]])

i=int(itemidlist[ui[1]])

comid=Bui[u][i]  #u,i对应的评论的序号

item = doc_topic[int(comid)]  #遍历评论的词汇，与主题特征词

for topid in range(5):

Huikarr[u][i][topid]=float(item[topid][1]) #评论ui对应topid的概率

bo[int(comid)][topid]=float(item[topid][1])

return Huikarr

② word2vec的θ取法

def getHuik(doc_topic):   #获得Huik

for top in topic:

topid=topictokenlist[top]

temp=topic[topid]

model = gensim.models.Word2Vec.load("./pinglun.model")

simlar=model.most_similar(temp,topn=50) #主题的特征词集,取了10个

for ui in traindata:

u=int(useridlist[ui[0]])

i=int(itemidlist[ui[1]])

comid=Bui[u][i]  #u,i对应的评论的序号

for item in commentdata[int(comid)]:  #遍历评论的词汇，与主题特征词

tempcom=0  # 统计一条评论中主题特征词的相似和

if item in simlar:

tempcom+=model.similarity(item,temp)

print("zhaodoa!!!")

Huikarr[u][i][topid]=tempcom

return Huikarr

(1)求puij ‘

def getpuj0(u,j): #求单个的puj’

Huij=getHuik()  #得θuij数组

tempujs = 0

for item in traindata:

i=int(itemidlist[item[1]]) # 物品号

tempujs+=Huij[u][i][j]

return tempujs/Cu[u] #得到puj ‘

Puj0=np.zeros((len(useridlist),len(topictokenlist)))

def getPuj0():  #求全体puj ’数组

for u in range(len(useridlist)):

for  j in range((len(topictokenlist))):

Puj0[u][j]=getpuj0(u,j)

return Puj0

(3)求 puj

def getPuj1():

bottom=np.zeros((len(useridlist)))

for u in range(len(useridlist)):

temp=0

for j in range(len(topictokenlist)):

temp+=Puj0[u][j]

bottom[u]=temp

for u in range(len(useridlist)):

for j in range((len(topictokenlist))):

Puj1[u][j]=Puj0[u][j]/bottom[u]

return Puj1

(2)求qij’

def getqij0(i,j):

Huij=getHuik() #uij数组

tempu = 0

for u in range(len(useridlist)):

tempu+=Huij[u][i][j]

return tempu/Ci[i] #得到p'u

# j

Qij0=np.zeros((len(itemidlist),len(topictokenlist)))

def getQij0():

for i in range(len(itemidlist)):

for j in range(len(topictokenlist)):

Qij0[i][j]=getqij0(i,j)

return Qij0

(4)求qij

def getQij1():

bottom = np.zeros((len(itemidlist)))

for i in range(len(itemidlist)):

temp = 0

for j in range(len(topictokenlist)):

temp += Qij0[i][j]

bottom[i] = temp

for i in range(len((itemidlist))):

for j in range((len(topictokenlist))):

Qij1[i][j] = Qij0[i][j] / bottom[i]

return Qij1

(5)产生用户u未评论物品i的主题分布

def getTuij0(Huij):

Tuij0 = np.zeros((393366, 5))

Qij1=getQij1(Huij)

Puj1=getPuj1(Huij)

Tuij0 = np.zeros((len(useridlist), len(itemidlist),5))

for u in range(len(useridlist)):

for i in range(len(itemidlist)):

mm=int(Bui[u][i])

for j in range(5):

Tuij0[m][j]=Puj1[u][j]*Qij1[i][j]

return Tuij0

def getTuij_input(Huij):

Tuij_input = np.zeros((393366,5))

bottom=np.zeros((393366,1))

Tuij0=getTuij0(Huij)

for u in range(len(useridlist)):

for i in range(len(itemidlist)):

temp=0

hh=int(Bui[u][i])

for j in range(5):

temp+=Tuij0[hh][j]

jj=Bui[u][i]

bottom[jj]=temp

for u in range(len(useridlist)):

for i in range(len(itemidlist)):

for j in range(5):

tok=Bui[u][i]

Tuij_input[tok][j]=Tuij0[tok][j]/bottom[tok]

return Tuij_input

(6)线性回归与预测

bo=np.zeros((393366,5))

m=getHuik(doc_topic,5,bo)

clf= sklearn.linear_model.LinearRegression()

clf.fit(bo,mark)

Test_input=getTuij_input(m)

print("结果：")

for item in testdata:

#  item=testdata[1]

print(item)

u=int(useridlist[item[0]])

i=int(itemidlist[item[1]])

print(clf.predict(Test_input[u][i]))     #输出预测评分