Python3.7版本，转载自：https://blog.csdn.net/juanjuan1314/article/details/75124046
wangyi_title.txt文件下载地址：链接:https://pan.baidu.com/s/1uL75P13t98YHMqgv3Kx7TQ 密码:oqxt
对原文有修改，原文代码是Python2，有很多问题。
# coding:utf-8
import sys
import gensim
import sklearn
import numpy as np
from gensim.models.doc2vec import Doc2Vec, LabeledSentence
TaggededDocument = gensim.models.doc2vec.TaggedDocument
def get_datasest():
with open("wangyi_title.txt", 'r') as cf:
docs = cf.readlines()
print(len(docs))
x_train = []
#y = np.concatenate(np.ones(len(docs)))
for i, text in enumerate(docs):
word_list = text.split(' ')
l = len(word_list)
word_list[l-1] = word_list[l-1].strip()
document = TaggededDocument(word_list, tags=[i])
x_train.append(document)
return x_train
def getVecs(model, corpus, vector_size):
vecs = [np.array(model.docvecs[z.tags[0]].reshape(1, vector_size)) for z in corpus]
return np.concatenate(vecs)
def train(x_train, vector_size=200, epoch_num=1):
model_dm = Doc2Vec(x_train,min_count=1, window = 3, vector_size = vector_size, sample=1e-3, negative=5, workers=4)
model_dm.train(x_train, total_examples=model_dm.corpus_count, epochs=70)
model_dm.save('model_dm_wangyi')
return model_dm
def test():
model_dm = Doc2Vec.load("model_dm_wangyi")
test_text = ['《', '舞林', '争霸' '》', '十强' '出炉', '复活', '舞者', '澳门', '踢馆']
inferred_vector_dm = model_dm.infer_vector(test_text)
print(inferred_vector_dm)
sims = model_dm.docvecs.most_similar([inferred_vector_dm], topn=10)
return sims
if __name__ == '__main__':
x_train = get_datasest()
model_dm = train(x_train)
sims = test()
for count, sim in sims:
sentence = x_train[count]
words = ''
for word in sentence[0]:
words = words + word + ' '
print (words, sim, len(sentence[0]))用了网页的热门娱乐新闻标题作为训练语料，输出结果如下，很相似的句子确实少。
42754
[-2.1229391e-05 -5.3489220e-04 -1.4628534e-03 1.0101878e-03
1.9613570e-03 1.2337929e-04 -1.5623088e-03 1.4899696e-03
1.7250431e-04 1.7861715e-03 5.4765341e-04 1.7854273e-03
1.4752866e-04 -4.7224312e-04 -2.0143031e-03 -1.3678997e-03
2.1347464e-03 -4.2291704e-04 -2.2612642e-03 1.9719985e-03
-1.7474928e-03 6.7744928e-04 -1.1667489e-03 1.4224678e-03
-4.9147848e-04 1.9250986e-03 1.5286671e-04 -1.0706087e-03
-1.2940766e-03 -1.1336872e-03 -4.8530920e-04 1.4789804e-03
1.7939236e-03 -1.2773223e-03 -2.4406663e-03 1.9606731e-03
2.4594443e-03 1.5459055e-03 -9.8075520e-04 1.6827125e-03
1.4778823e-03 2.0646905e-03 -3.4740806e-05 -1.5140681e-03
-7.6300337e-04 -2.1761435e-03 -1.9383265e-04 6.5391039e-04
-9.3230215e-04 3.8053558e-04 1.6529204e-05 1.5503957e-03
5.2016345e-04 1.2898637e-03 1.7284699e-03 2.2767365e-03
-9.5764997e-05 -8.4209896e-05 -1.5726103e-03 2.2212588e-03
-5.9885468e-04 -2.1759607e-03 -1.9564391e-03 1.2035059e-03
-3.9055874e-04 1.1362566e-03 1.0841021e-03 -9.0546644e-04
2.3774474e-03 1.3961376e-03 -1.8707723e-03 1.5263865e-03
-1.1634092e-03 -2.2435680e-03 1.8672579e-03 5.6013430e-04
2.3103815e-03 1.2101847e-03 -2.4156671e-03 -5.1514624e-04
2.1143679e-03 2.3558659e-03 -1.0352633e-03 -8.4526307e-04
2.2150134e-03 5.3238236e-05 -2.3913602e-03 -1.5362124e-04
1.5323326e-03 2.4526857e-03 -1.6107119e-04 -3.4444834e-04
1.6401864e-03 1.0141496e-03 3.7656463e-04 -1.2738963e-04
-1.1323770e-03 -2.0433934e-03 3.7525350e-04 -1.6017296e-04
-3.3818476e-04 2.2791843e-03 -1.4202974e-03 -2.7641861e-04
1.1009629e-04 -4.2639120e-04 1.8214980e-03 -1.7151656e-03
-1.5390049e-03 -1.3191046e-03 1.7080955e-03 1.1002786e-03
1.6142949e-03 1.8982554e-03 -7.0945674e-04 -4.6570468e-04
9.8265568e-04 -7.4710487e-04 2.4075075e-03 -2.1547875e-03
-2.1082300e-03 -1.8821321e-03 9.6265052e-04 -1.1552537e-03
-1.6849015e-03 -1.2968426e-03 -1.5383511e-04 -7.5135130e-04
-1.8727558e-03 5.2730407e-04 -2.3783895e-03 2.4225495e-03
2.3140633e-03 -1.0093495e-03 1.5953591e-03 -1.6097585e-03
-5.1834644e-04 5.6184967e-05 2.8760443e-04 2.0393797e-03
1.4612459e-03 2.1953927e-03 -2.1270583e-03 -9.9687604e-04
1.2225753e-03 2.0009447e-03 4.6715033e-04 2.1180776e-03
2.8774102e-04 -8.8365687e-06 -1.7047256e-03 -9.7245700e-04
-4.0429382e-04 1.9775415e-03 -2.2045472e-03 1.5636642e-03
-1.9885909e-03 2.0202452e-03 2.1154643e-03 1.7958126e-03
1.0514902e-03 1.9323002e-03 -1.5818867e-03 1.3666560e-03
-9.1630412e-04 3.2067264e-04 1.7956816e-04 -2.3987342e-03
9.4504084e-04 -2.9586093e-04 -1.6545136e-03 -9.1628381e-04
-1.2085686e-04 8.3511556e-04 9.2640345e-04 -1.0981049e-03
-2.6373079e-04 -1.1188543e-04 -1.0378383e-03 3.7422587e-04
-2.0860252e-03 8.9370640e-04 1.1446123e-03 -1.3295287e-03
1.2766315e-03 1.3684760e-03 2.1959674e-03 6.3199044e-04
-2.7432822e-04 5.7462428e-04 2.3212784e-03 9.1525499e-04
1.9918189e-03 1.1947503e-03 -1.1286519e-03 -6.5884611e-04
-6.7673821e-04 -3.2887704e-04 -1.9954341e-03 -1.1857023e-04]
/usr/local/lib/python3.7/site-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.
if np.issubdtype(vec.dtype, np.int):
《全民目击》错失上影节 导演称是为品质让路 0.26860833168029785 2
[星态度]蔡依林：在感情里我不是强势者 0.2664705514907837 1
《大魔术师》发剧照 梁朝伟刘青云等人搞笑耍宝 0.23988798260688782 2
三池崇史戛纳回应媒体反馈：恶评在意料之中 0.23537132143974304 1
《单身男女》试映爆满获好评 本周四正式上映 0.22329775989055634 2
金鸡百花节门票创“史上天价”最贵11560元 0.2214442789554596 1
《霍比特人》登杂志封面 曝彼得杰克逊工作照 0.2193169891834259 2
章子怡回应与汪峰婚期：应该快了 我也不小了 0.21707454323768616 2
《画壁》发MV 邓超:这是属于我和孙俪的爱情 0.21643859148025513 2
《生化危机5》曝战神自白预告 重现屠魔之旅 0.21323256194591522 2
[Finished in 114.1s]1
1