文章目录
一、DeepFM模型
DeepFM模型是2017年哈尔滨工业大学和华为公司联合提出的深度推荐模型，它是由FM模型和Wide&Deep模型进行了整合。其主要特点是DeepFM两个部分共享底层输出特征，无需进行特征工程，通过FM代替Wide部分，加强了浅层网络部分特征组合能力。其模型结构图如下：
DeepFM模型的paper地址如下： https://arxiv.org/pdf/1804.04950.pdf
DeepFM模型主要特点：
- 端到端模型，无需特征工程；
- 加强低阶特征组合和高阶特征组合之间的联系；
- 将FM模型和DNN模型进行组合，并结合Wide&Deep的思想，使用FM代替Wide部分，并共享隐藏输入部分。
二、算法原理
2.1 FM部分
可以看下FM基本原理，该部分与FM模型完全等价。
2.2 DNN部分
这个部分是简单的全连接网络，但是其中使用的输入：Dense Embeddings，与上述FM部分的是同一层。DNN与FM的Dense Embedding共享（Dense Embeddings在FM部分为了计算二阶特征交叉，而在DNN部分为了当做DNN的输入数据），这也是DeepFM与Wide&Deep模型最大的不同之处。
上图只展示了DeepFM-D部分的结构，DeepFM模型的Deep部分也可以参考PNN模型中product部分。
2.3 模型小结
最后将FM部分的输出结果和DNN部分的输出结果输入到下图公式：
在DeepFM模型中，为了防止过拟合，在FM部分采取L2_norm正则，在DNN部分采取dropout。
DeepFM模型实验细节看文献部分。
三、代码实现
采取的数据是movielens 100.为了操作的方便，只为了展示FM实现的过程，只选取了uid、itemId作为输入特征，rating作为lable。
3.1 数据集
u.item： 电影信息数据
movie id | movie title | release date | video release date |IMDb URL |unknown | Action | Adventure | Animation |Children's | Comedy | Crime |Documentary | Drama | Fantasy |Film-Noir | Horror | Musical | Mystery |Romance | Sci-Fi |Thriller | War | Western
1|Toy Story (1995)|01-Jan-1995||http://us.imdb.com/M/title-exact?Toy%20Story%20(1995)|0|0|0|1|1|1|0|0|0|0|0|0|0|0|0|0|0|0|0
2|GoldenEye (1995)|01-Jan-1995||http://us.imdb.com/M/title-exact?GoldenEye%20(1995)|0|1|1|0|0|0|0|0|0|0|0|0|0|0|0|0|1|0|0
3|Four Rooms (1995)|01-Jan-1995||http://us.imdb.com/M/title-exact?Four%20Rooms%20(1995)|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|1|0|0
u.user： 用户信息数据
user id | age | gender | occupation | zip code
1|24|M|technician|85711
2|53|F|other|94043
3|23|M|writer|32067
ua.base： 训练数据集
ua.test: 测试数据集
user id | item id | rating | timestamp
1 1 5 874965758
1 2 3 876893171
1 3 4 878542960
3.2 数据处理
将uid和itemId使用one-hot编码，将rating作为输出标签，其评分等级为[0-5]，大于3为1（表示用户感兴趣）小于3为0（表示用户不感兴趣）。
# 数据加载
def loadData():
# user信息(只取uid)
userInfo = pd.read_csv('../data/u.user', sep='\|', names=['uid', 'age', 'gender', 'occupation','zip code'])
uid_ = userInfo['uid']
userId_dum = pd.get_dummies(userInfo['uid'], columns=['uid'], prefix='uid_')
userId_dum['uid']=uid_
# item信息（只取itemId）
header = ['item_id', 'title', 'release_date', 'video_release_date', 'IMDb_URL', 'unknown', 'Action', 'Adventure', 'Animation', 'Children',
'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy', 'Film-Noir', 'Horror', 'Musical', 'Mystery', 'Romance', 'Sci-Fi',
'Thriller', 'War', 'Western']
ItemInfo = pd.read_csv('../data/u.item', sep='|', names=header, encoding = "ISO-8859-1")
ItemInfo = ItemInfo.drop(columns=['title', 'release_date', 'video_release_date', 'IMDb_URL', 'unknown'])
item_id_ = ItemInfo['item_id']
item_Id_dum = pd.get_dummies(ItemInfo['item_id'], columns=['item_id'], prefix='item_id_')
item_Id_dum['item_id']=item_id_
# 训练数据
trainData = pd.read_csv('../data/ua.base', sep='\t', names=['uid', 'item_id', 'rating', 'time'])
trainData = trainData.drop(columns=['time'])
trainData['rating']=trainData.rating.apply(lambda x:1 if int(x)>3 else 0)
Y_train=pd.get_dummies(trainData['rating'],columns=['rating'],prefix='y_')
X_train = pd.merge(trainData, userId_dum, how='left')
X_train = pd.merge(X_train, item_Id_dum, how='left')
X_train=X_train.drop(columns=['uid','item_id','rating'])
# 测试数据
testData = pd.read_csv('../data/ua.test', sep='\t', names=['uid', 'item_id', 'rating', 'time'])
testData = testData.drop(columns=['time'])
testData['rating']=testData.rating.apply(lambda x:1 if int(x)>3 else 0)
Y_test=pd.get_dummies(testData['rating'],columns=['rating'],prefix='y_')
X_test = pd.merge(testData, userId_dum, how='left')
X_test = pd.merge(X_test, item_Id_dum, how='left')
X_test=X_test.drop(columns=['uid','item_id','rating'])
# 对应域 uid itemid
# user信息 uid
# item信息 itemid
field_index={
}
userField=['uid']
itemField=['itemId']
field=userField+itemField
# 每个域的长度
userFieldLen=[len(uid_)]
itemFieldLen=[len(item_id_)]
field_len = userFieldLen + itemFieldLen
j=0
field_arange=[0]
for field_n in range(len(field)):
field_arange.append(field_arange[field_n]+field_len[field_n])
return X_train.values,Y_train.values,X_test.values,Y_test.values,field_arange,len(field)
3.2 DeepFM模型
class DeepFM:
def __init__(self,vec_dim,learning_rate,feature_length,field_arange,field_len,dnn_layers,dropout_rate,lamda):
self.vec_dim=vec_dim
self.learning_rate=learning_rate
self.feature_length=feature_length
self.field_arange=field_arange
self.field_len=field_len
self.dnn_layers=dnn_layers
self.dropout_rate=dropout_rate
self.l2_reg = tf.contrib.layers.l2_regularizer(float(lamda))
def add_input(self):
self.X = tf.placeholder(dtype=tf.float32, name='input_x',shape=[None,self.feature_length])
self.Y = tf.placeholder(dtype=tf.float32, name='input_y', shape=[None, 2])
# 创建计算规则
def inference(self):
with tf.variable_scope('line_part'):
linear_w = tf.get_variable(shape=[self.feature_length, 2],dtype=tf.float32,name='linear_w')
first_out = tf.matmul(self.X, linear_w)
with tf.variable_scope('emb_part'):
Emb = [tf.get_variable(name='Embedding_%d'%i,
shape=[self.field_arange[i+1]-self.field_arange[i], self.vec_dim],
dtype=tf.float32,regularizer=self.l2_reg) for i in range(self.field_len)]
Emb_part = tf.concat([tf.matmul(tf.slice(self.X,[0,self.field_arange[i]],
[-1,self.field_arange[i+1]-self.field_arange[i]]), Emb[i])
for i in range(self.field_len)], axis=1)
x=Emb_part
# x = tf.layers.dropout(Emb_part, rate=self.dropout_rate)
with tf.variable_scope('interaction_part'):
sum_emb_square = tf.square(tf.reduce_sum(Emb_part, axis=1, keep_dims=True))
square_emb_sum = tf.reduce_sum(tf.square(Emb_part), axis=1, keep_dims=True)
second_out = 0.5 * (sum_emb_square - square_emb_sum)
self.fm=tf.add(first_out,second_out)
in_num = self.field_len * self.vec_dim
with tf.variable_scope('dnn_part'):
for i in range(len(self.dnn_layers)):
out_num = self.dnn_layers[i]
w = tf.get_variable(name='w_%d'%i, shape=[in_num, out_num], dtype=tf.float32)
b = tf.get_variable(name='b_%d'%i, shape=[out_num], dtype=tf.float32)
x = tf.matmul(x, w) + b
if out_num == 2:
self.dnn_part = x
else:
# 为了防止dnn部分过拟合 对其加dropout
x = tf.layers.dropout(tf.nn.relu(x), rate=self.dropout_rate)
in_num = out_num
with tf.variable_scope('output_part'):
output = tf.add(self.fm,self.dnn_part)
out_w = tf.get_variable(name='out_w', shape=[2, 2], dtype=tf.float32)
out_b = tf.get_variable(name='out_b', shape=[2], dtype=tf.float32)
self.y_out = tf.matmul(output, out_w) + out_b
def add_loss(self):
self.loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=self.Y, logits=self.y_out))
#计算accuracy
def add_accuracy(self):
# accuracy
self.correct_prediction = tf.equal(tf.cast(tf.argmax(self.y_out,1), tf.float32), tf.cast(tf.argmax(self.Y,1), tf.float32))
self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32))
# self.auc_value = tf.metrics.auc(tf.argmax(self.y_out,1),tf.argmax(self.Y,1), curve='ROC')
#训练
def train(self):
optimizer = tf.train.AdagradOptimizer(self.learning_rate)
self.train_op = optimizer.minimize(self.loss)
#构建图
def build_graph(self):
self.add_input()
self.inference()
self.add_loss()
self.add_accuracy()
self.train()
3.4 训练和测试
def train_model(sess, model, X_train,Y_train,batch_size, epochs=100):
num = len(X_train) // batch_size+1
for step in range(epochs):
print("epochs{0}:".format(step+1))
for i in range(num):
index = np.random.choice(len(X_train), batch_size)
batch_x = X_train[index]
batch_y = Y_train[index]
feed_dict = {
model.X: batch_x,
model.Y: batch_y}
sess.run(model.train_op, feed_dict=feed_dict)
# print("Iteration {0}: with minibatch training loss = {1}"
# .format(step+1, loss))
if (i+1)%100==0:
loss ,accuracy,y_out= sess.run([model.loss,model.accuracy,model.y_out], feed_dict=feed_dict)
auc = metrics.roc_auc_score(batch_y, y_out)
print("Iteration {0}: with minibatch training loss = {1} accuracy = {2} auc={3}"
.format(step+1, loss,accuracy,auc))
def test_model(sess,model,X_test,Y_test):
loss,y_out, accuracy= sess.run([model.loss, model.y_out,model.accuracy], feed_dict={
model.X: X_test, model.Y: Y_test})
print("loss={0} accuracy={1} auc={2}".format(loss,accuracy,metrics.roc_auc_score(Y_test, y_out)))
3.5 完整代码
import numpy as np
import pandas as pd
import tensorflow as tf
import sklearn.metrics as metrics
# 数据加载
def loadData():
# user信息(只取uid)
userInfo = pd.read_csv('../data/u.user', sep='\|', names=['uid', 'age', 'gender', 'occupation','zip code'])
uid_ = userInfo['uid']
userId_dum = pd.get_dummies(userInfo['uid'], columns=['uid'], prefix='uid_')
userId_dum['uid']=uid_
# item信息（只取itemId）
header = ['item_id', 'title', 'release_date', 'video_release_date', 'IMDb_URL', 'unknown', 'Action', 'Adventure', 'Animation', 'Children',
'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy', 'Film-Noir', 'Horror', 'Musical', 'Mystery', 'Romance', 'Sci-Fi',
'Thriller', 'War', 'Western']
ItemInfo = pd.read_csv('../data/u.item', sep='|', names=header, encoding = "ISO-8859-1")
ItemInfo = ItemInfo.drop(columns=['title', 'release_date', 'video_release_date', 'IMDb_URL', 'unknown'])
item_id_ = ItemInfo['item_id']
item_Id_dum = pd.get_dummies(ItemInfo['item_id'], columns=['item_id'], prefix='item_id_')
item_Id_dum['item_id']=item_id_
# 训练数据
trainData = pd.read_csv('../data/ua.base', sep='\t', names=['uid', 'item_id', 'rating', 'time'])
trainData = trainData.drop(columns=['time'])
trainData['rating']=trainData.rating.apply(lambda x:1 if int(x)>3 else 0)
Y_train=pd.get_dummies(trainData['rating'],columns=['rating'],prefix='y_')
X_train = pd.merge(trainData, userId_dum, how='left')
X_train = pd.merge(X_train, item_Id_dum, how='left')
X_train=X_train.drop(columns=['uid','item_id','rating'])
# 测试数据
testData = pd.read_csv('../data/ua.test', sep='\t', names=['uid', 'item_id', 'rating', 'time'])
testData = testData.drop(columns=['time'])
testData['rating']=testData.rating.apply(lambda x:1 if int(x)>3 else 0)
Y_test=pd.get_dummies(testData['rating'],columns=['rating'],prefix='y_')
X_test = pd.merge(testData, userId_dum, how='left')
X_test = pd.merge(X_test, item_Id_dum, how='left')
X_test=X_test.drop(columns=['uid','item_id','rating'])
# 对应域 uid itemid
# user信息 uid
# item信息 itemid
field_index={
}
userField=['uid']
itemField=['itemId']
field=userField+itemField
# 每个域的长度
userFieldLen=[len(uid_)]
itemFieldLen=[len(item_id_)]
field_len = userFieldLen + itemFieldLen
j=0
field_arange=[0]
for field_n in range(len(field)):
field_arange.append(field_arange[field_n]+field_len[field_n])
return X_train.values,Y_train.values,X_test.values,Y_test.values,field_arange,len(field)
class DeepFM:
def __init__(self,vec_dim,learning_rate,feature_length,field_arange,field_len,dnn_layers,dropout_rate,lamda):
self.vec_dim=vec_dim
self.learning_rate=learning_rate
self.feature_length=feature_length
self.field_arange=field_arange
self.field_len=field_len
self.dnn_layers=dnn_layers
self.dropout_rate=dropout_rate
self.l2_reg = tf.contrib.layers.l2_regularizer(float(lamda))
def add_input(self):
self.X = tf.placeholder(dtype=tf.float32, name='input_x',shape=[None,self.feature_length])
self.Y = tf.placeholder(dtype=tf.float32, name='input_y', shape=[None, 2])
# 创建计算规则
def inference(self):
with tf.variable_scope('line_part'):
linear_w = tf.get_variable(shape=[self.feature_length, 2],dtype=tf.float32,name='linear_w')
first_out = tf.matmul(self.X, linear_w)
with tf.variable_scope('emb_part'):
Emb = [tf.get_variable(name='Embedding_%d'%i,
shape=[self.field_arange[i+1]-self.field_arange[i], self.vec_dim],
dtype=tf.float32,regularizer=self.l2_reg) for i in range(self.field_len)]
Emb_part = tf.concat([tf.matmul(tf.slice(self.X,[0,self.field_arange[i]],
[-1,self.field_arange[i+1]-self.field_arange[i]]), Emb[i])
for i in range(self.field_len)], axis=1)
x=Emb_part
# x = tf.layers.dropout(Emb_part, rate=self.dropout_rate)
with tf.variable_scope('interaction_part'):
sum_emb_square = tf.square(tf.reduce_sum(Emb_part, axis=1, keep_dims=True))
square_emb_sum = tf.reduce_sum(tf.square(Emb_part), axis=1, keep_dims=True)
second_out = 0.5 * (sum_emb_square - square_emb_sum)
self.fm=tf.add(first_out,second_out)
in_num = self.field_len * self.vec_dim
with tf.variable_scope('dnn_part'):
for i in range(len(self.dnn_layers)):
out_num = self.dnn_layers[i]
w = tf.get_variable(name='w_%d'%i, shape=[in_num, out_num], dtype=tf.float32)
b = tf.get_variable(name='b_%d'%i, shape=[out_num], dtype=tf.float32)
x = tf.matmul(x, w) + b
if out_num == 2:
self.dnn_part = x
else:
# 为了防止dnn部分过拟合 对其加dropout
x = tf.layers.dropout(tf.nn.relu(x), rate=self.dropout_rate)
in_num = out_num
with tf.variable_scope('output_part'):
output = tf.add(self.fm,self.dnn_part)
out_w = tf.get_variable(name='out_w', shape=[2, 2], dtype=tf.float32)
out_b = tf.get_variable(name='out_b', shape=[2], dtype=tf.float32)
self.y_out = tf.matmul(output, out_w) + out_b
def add_loss(self):
self.loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=self.Y, logits=self.y_out))
#计算accuracy
def add_accuracy(self):
# accuracy
self.correct_prediction = tf.equal(tf.cast(tf.argmax(self.y_out,1), tf.float32), tf.cast(tf.argmax(self.Y,1), tf.float32))
self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32))
# self.auc_value = tf.metrics.auc(tf.argmax(self.y_out,1),tf.argmax(self.Y,1), curve='ROC')
#训练
def train(self):
optimizer = tf.train.AdagradOptimizer(self.learning_rate)
self.train_op = optimizer.minimize(self.loss)
#构建图
def build_graph(self):
self.add_input()
self.inference()
self.add_loss()
self.add_accuracy()
self.train()
def train_model(sess, model, X_train,Y_train,batch_size, epochs=100):
num = len(X_train) // batch_size+1
for step in range(epochs):
print("epochs{0}:".format(step+1))
for i in range(num):
index = np.random.choice(len(X_train), batch_size)
batch_x = X_train[index]
batch_y = Y_train[index]
feed_dict = {
model.X: batch_x,
model.Y: batch_y}
sess.run(model.train_op, feed_dict=feed_dict)
# print("Iteration {0}: with minibatch training loss = {1}"
# .format(step+1, loss))
if (i+1)%100==0:
loss ,accuracy,y_out= sess.run([model.loss,model.accuracy,model.y_out], feed_dict=feed_dict)
auc = metrics.roc_auc_score(batch_y, y_out)
print("Iteration {0}: with minibatch training loss = {1} accuracy = {2} auc={3}"
.format(step+1, loss,accuracy,auc))
def test_model(sess,model,X_test,Y_test):
loss,y_out, accuracy= sess.run([model.loss, model.y_out,model.accuracy], feed_dict={
model.X: X_test, model.Y: Y_test})
print("loss={0} accuracy={1} auc={2}".format(loss,accuracy,metrics.roc_auc_score(Y_test, y_out)))
if __name__ == '__main__':
X_train,Y_train,X_test,Y_test,field_arange,field_len=loadData()
learning_rate = 0.001
batch_size = 128
vec_dim = 800
feature_length = X_train.shape[1]
dnn_layers=[300,200,100,2]
dropout_rate=0.9
lamda=0.5
model = DeepFM(vec_dim ,learning_rate ,feature_length,field_arange,field_len,dnn_layers,dropout_rate,lamda)
model.build_graph()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print('start training...')
train_model(sess,model,X_train,Y_train,batch_size,epochs=10)
print('start testing...')
test_model(sess,model,X_test,Y_test)