启动训练前，复用前面章节的数据处理和神经网络模型代码，已阅读可直接跳过。

    1. import random
    2. import numpy as np
    3. from PIL import Image
    4. import paddle.fluid as fluid
    5. import paddle.fluid.dygraph as dygraph
    6. from paddle.fluid.dygraph import Linear, Embedding, Conv2D, Pool2D
    8. class MovieLen(object):
    9.     def __init__(self, use_poster):
    10.         self.use_poster = use_poster
    11.         # 声明每个数据文件的路径
    12.         usr_info_path = "./work/ml-1m/users.dat"
    13.         if use_poster:
    14.             rating_path = "./work/ml-1m/new_rating.txt"
    15.         else:
    16.             rating_path = "./work/ml-1m/ratings.dat"
    18.         movie_info_path = "./work/ml-1m/movies.dat"
    19.         self.poster_path = "./work/ml-1m/posters/"
    20.         # 得到电影数据
    21.         self.movie_info, self.movie_cat, self.movie_title = self.get_movie_info(movie_info_path)
    22.         # 记录电影的最大ID
    23.         self.max_mov_cat = np.max([self.movie_cat[k] for k in self.movie_cat])
    24.         self.max_mov_tit = np.max([self.movie_title[k] for k in self.movie_title])
    25.         self.max_mov_id = np.max(list(map(int, self.movie_info.keys())))
    26.         # 记录用户数据的最大ID
    27.         self.max_usr_id = 0
    28.         self.max_usr_age = 0
    29.         self.max_usr_job = 0
    30.         # 得到用户数据
    31.         self.usr_info = self.get_usr_info(usr_info_path)
    32.         # 得到评分数据
    33.         self.rating_info = self.get_rating_info(rating_path)
    34.         # 构建数据集 
    35.         self.dataset = self.get_dataset(usr_info=self.usr_info,
    36.                                         rating_info=self.rating_info,
    37.                                         movie_info=self.movie_info)
    38.         # 划分数据集，获得数据加载器
    39.         self.train_dataset = self.dataset[:int(len(self.dataset)*0.9)]
    40.         self.valid_dataset = self.dataset[int(len(self.dataset)*0.9):]
    41.         print("##Total dataset instances: ", len(self.dataset))
    42.         print("##MovieLens dataset information: \nusr num: {}\n"
    43.               "movies num: {}".format(len(self.usr_info),len(self.movie_info)))
    44.     # 得到电影数据
    45.     def get_movie_info(self, path):
    46.         # 打开文件，编码方式选择ISO-8859-1，读取所有数据到data中 
    47.         with open(path, 'r', encoding="ISO-8859-1") as f:
    48.             data = f.readlines()
    49.         # 建立三个字典，分别用户存放电影所有信息，电影的名字信息、类别信息
    50.         movie_info, movie_titles, movie_cat = {}, {}, {}
    51.         # 对电影名字、类别中不同的单词计数
    52.         t_count, c_count = 1, 1
    54.         count_tit = {}
    55.         # 按行读取数据并处理
    56.         for item in data:
    57.             item = item.strip().split("::")
    58.             v_id = item[0]
    59.             v_title = item[1][:-7]
    60.             cats = item[2].split('|')
    61.             v_year = item[1][-5:-1]
    63.             titles = v_title.split()
    64.             # 统计电影名字的单词，并给每个单词一个序号，放在movie_titles中
    65.             for t in titles:
    66.                 if t not in movie_titles:
    67.                     movie_titles[t] = t_count
    68.                     t_count += 1
    69.             # 统计电影类别单词，并给每个单词一个序号，放在movie_cat中
    70.             for cat in cats:
    71.                 if cat not in movie_cat:
    72.                     movie_cat[cat] = c_count
    73.                     c_count += 1
    74.             # 补0使电影名称对应的列表长度为15
    75.             v_tit = [movie_titles[k] for k in titles]
    76.             while len(v_tit)<15:
    77.                 v_tit.append(0)
    78.             # 补0使电影种类对应的列表长度为6
    79.             v_cat = [movie_cat[k] for k in cats]
    80.             while len(v_cat)<6:
    81.                 v_cat.append(0)
    82.             # 保存电影数据到movie_info中
    83.             movie_info[v_id] = {'mov_id': int(v_id),
    84.                                 'title': v_tit,
    85.                                 'category': v_cat,
    86.                                 'years': int(v_year)}
    87.         return movie_info, movie_cat, movie_titles
    89.     def get_usr_info(self, path):
    90.         # 性别转换函数，M-0， F-1
    91.         def gender2num(gender):
    92.             return 1 if gender == 'F' else 0
    94.         # 打开文件，读取所有行到data中
    95.         with open(path, 'r') as f:
    96.             data = f.readlines()
    97.         # 建立用户信息的字典
    98.         use_info = {}
    100.         max_usr_id = 0
    101.         #按行索引数据
    102.         for item in data:
    103.             # 去除每一行中和数据无关的部分
    104.             item = item.strip().split("::")
    105.             usr_id = item[0]
    106.             # 将字符数据转成数字并保存在字典中
    107.             use_info[usr_id] = {'usr_id': int(usr_id),
    108.                                 'gender': gender2num(item[1]),
    109.                                 'age': int(item[2]),
    110.                                 'job': int(item[3])}
    111.             self.max_usr_id = max(self.max_usr_id, int(usr_id))
    112.             self.max_usr_age = max(self.max_usr_age, int(item[2]))
    113.             self.max_usr_job = max(self.max_usr_job, int(item[3]))
    114.         return use_info
    115.     # 得到评分数据
    116.     def get_rating_info(self, path):
    117.         # 读取文件里的数据
    118.         with open(path, 'r') as f:
    119.             data = f.readlines()
    120.         # 将数据保存在字典中并返回
    121.         rating_info = {}
    122.         for item in data:
    123.             item = item.strip().split("::")
    124.             usr_id,movie_id,score = item[0],item[1],item[2]
    125.             if usr_id not in rating_info.keys():
    126.                 rating_info[usr_id] = {movie_id:float(score)}
    127.             else:
    128.                 rating_info[usr_id][movie_id] = float(score)
    129.         return rating_info
    130.     # 构建数据集
    131.     def get_dataset(self, usr_info, rating_info, movie_info):
    132.         trainset = []
    133.         for usr_id in rating_info.keys():
    134.             usr_ratings = rating_info[usr_id]
    135.             for movie_id in usr_ratings:
    136.                 trainset.append({'usr_info': usr_info[usr_id],
    137.                                  'mov_info': movie_info[movie_id],
    138.                                  'scores': usr_ratings[movie_id]})
    139.         return trainset
    141.     def load_data(self, dataset=None, mode='train'):
    142.         use_poster = False
    144.         # 定义数据迭代Batch大小
    145.         BATCHSIZE = 256
    147.         data_length = len(dataset)
    148.         index_list = list(range(data_length))
    149.         # 定义数据迭代加载器
    150.         def data_generator():
    151.             # 训练模式下，打乱训练数据
    152.             if mode == 'train':
    153.                 random.shuffle(index_list)
    154.             # 声明每个特征的列表
    155.             usr_id_list,usr_gender_list,usr_age_list,usr_job_list = [], [], [], []
    156.             mov_id_list,mov_tit_list,mov_cat_list,mov_poster_list = [], [], [], []
    157.             score_list = []
    158.             # 索引遍历输入数据集
    159.             for idx, i in enumerate(index_list):
    160.                 # 获得特征数据保存到对应特征列表中
    161.                 usr_id_list.append(dataset[i]['usr_info']['usr_id'])
    162.                 usr_gender_list.append(dataset[i]['usr_info']['gender'])
    163.                 usr_age_list.append(dataset[i]['usr_info']['age'])
    164.                 usr_job_list.append(dataset[i]['usr_info']['job'])
    166.                 mov_id_list.append(dataset[i]['mov_info']['mov_id'])
    167.                 mov_tit_list.append(dataset[i]['mov_info']['title'])
    168.                 mov_cat_list.append(dataset[i]['mov_info']['category'])
    169.                 mov_id = dataset[i]['mov_info']['mov_id']
    171.                 if use_poster:
    172.                     # 不使用图像特征时，不读取图像数据，加快数据读取速度
    173.                     poster = Image.open(self.poster_path+'mov_id{}.jpg'.format(str(mov_id[0])))
    174.                     poster = poster.resize([64, 64])
    175.                     if len(poster.size) <= 2:
    176.                         poster = poster.convert("RGB")
    178.                     mov_poster_list.append(np.array(poster))
    180.                 score_list.append(int(dataset[i]['scores']))
    181.                 # 如果读取的数据量达到当前的batch大小，就返回当前批次
    182.                 if len(usr_id_list)==BATCHSIZE:
    183.                     # 转换列表数据为数组形式，reshape到固定形状
    184.                     usr_id_arr = np.array(usr_id_list)
    185.                     usr_gender_arr = np.array(usr_gender_list)
    186.                     usr_age_arr = np.array(usr_age_list)
    187.                     usr_job_arr = np.array(usr_job_list)
    189.                     mov_id_arr = np.array(mov_id_list)
    190.                     mov_cat_arr = np.reshape(np.array(mov_cat_list), [BATCHSIZE, 6]).astype(np.int64)
    191.                     mov_tit_arr = np.reshape(np.array(mov_tit_list), [BATCHSIZE, 1, 15]).astype(np.int64)
    193.                     if use_poster:
    194.                         mov_poster_arr = np.reshape(np.array(mov_poster_list)/127.5 - 1, [BATCHSIZE, 3, 64, 64]).astype(np.float32)
    195.                     else:
    196.                         mov_poster_arr = np.array([0.])
    198.                     scores_arr = np.reshape(np.array(score_list), [-1, 1]).astype(np.float32)
    200.                     # 放回当前批次数据
    201.                     yield [usr_id_arr, usr_gender_arr, usr_age_arr, usr_job_arr], \
    202.                            [mov_id_arr, mov_cat_arr, mov_tit_arr, mov_poster_arr], scores_arr
    204.                     # 清空数据
    205.                     usr_id_list, usr_gender_list, usr_age_list, usr_job_list = [], [], [], []
    206.                     mov_id_list, mov_tit_list, mov_cat_list, score_list = [], [], [], []
    207.                     mov_poster_list = []
    208.         return data_generator
    210. class Model(dygraph.layers.Layer):
    211.     def __init__(self, use_poster, use_mov_title, use_mov_cat, use_age_job):
    212.         super(Model, self).__init__()
    214.         # 将传入的name信息和bool型参数添加到模型类中
    215.         self.use_mov_poster = use_poster
    216.         self.use_mov_title = use_mov_title
    217.         self.use_usr_age_job = use_age_job
    218.         self.use_mov_cat = use_mov_cat
    220.         # 获取数据集的信息，并构建训练和验证集的数据迭代器
    221.         Dataset = MovieLen(self.use_mov_poster)
    222.         self.Dataset = Dataset
    223.         self.trainset = self.Dataset.train_dataset
    224.         self.valset = self.Dataset.valid_dataset
    225.         self.train_loader = self.Dataset.load_data(dataset=self.trainset, mode='train')
    226.         self.valid_loader = self.Dataset.load_data(dataset=self.valset, mode='valid')
    228.         """ define network layer for embedding usr info """
    229.         USR_ID_NUM = Dataset.max_usr_id + 1
    230.         # 对用户ID做映射，并紧接着一个Linear层
    231.         self.usr_emb = Embedding([USR_ID_NUM, 32], is_sparse=False)
    232.         self.usr_fc = Linear(32, 32)
    234.         # 对用户性别信息做映射，并紧接着一个Linear层
    235.         USR_GENDER_DICT_SIZE = 2
    236.         self.usr_gender_emb = Embedding([USR_GENDER_DICT_SIZE, 16])
    237.         self.usr_gender_fc = Linear(16, 16)
    239.         # 对用户年龄信息做映射，并紧接着一个Linear层
    240.         USR_AGE_DICT_SIZE = Dataset.max_usr_age + 1
    241.         self.usr_age_emb = Embedding([USR_AGE_DICT_SIZE, 16])
    242.         self.usr_age_fc = Linear(16, 16)
    244.         # 对用户职业信息做映射，并紧接着一个Linear层
    245.         USR_JOB_DICT_SIZE = Dataset.max_usr_job + 1
    246.         self.usr_job_emb = Embedding([USR_JOB_DICT_SIZE, 16])
    247.         self.usr_job_fc = Linear(16, 16)
    249.         # 新建一个Linear层，用于整合用户数据信息
    250.         self.usr_combined = Linear(80, 200, act='tanh')
    252.         """ define network layer for embedding usr info """
    253.         # 对电影ID信息做映射，并紧接着一个Linear层
    254.         MOV_DICT_SIZE = Dataset.max_mov_id + 1
    255.         self.mov_emb = Embedding([MOV_DICT_SIZE, 32])
    256.         self.mov_fc = Linear(32, 32)
    258.         # 对电影类别做映射
    259.         CATEGORY_DICT_SIZE = len(Dataset.movie_cat) + 1
    260.         self.mov_cat_emb = Embedding([CATEGORY_DICT_SIZE, 32], is_sparse=False)
    261.         self.mov_cat_fc = Linear(32, 32)
    263.         # 对电影名称做映射
    264.         MOV_TITLE_DICT_SIZE = len(Dataset.movie_title) + 1
    265.         self.mov_title_emb = Embedding([MOV_TITLE_DICT_SIZE, 32], is_sparse=False)
    266.         self.mov_title_conv = Conv2D(1, 1, filter_size=(3, 1), stride=(2,1), padding=0, act='relu')
    267.         self.mov_title_conv2 = Conv2D(1, 1, filter_size=(3, 1), stride=1, padding=0, act='relu')
    269.         # 新建一个Linear层，用于整合电影特征
    270.         self.mov_concat_embed = Linear(96, 200, act='tanh')
    272.     # 定义计算用户特征的前向运算过程
    273.     def get_usr_feat(self, usr_var):
    274.         """ get usr features"""
    275.         # 获取到用户数据
    276.         usr_id, usr_gender, usr_age, usr_job = usr_var
    277.         # 将用户的ID数据经过embedding和Linear计算，得到的特征保存在feats_collect中
    278.         feats_collect = []
    279.         usr_id = self.usr_emb(usr_id)
    280.         usr_id = self.usr_fc(usr_id)
    281.         usr_id = fluid.layers.relu(usr_id)
    282.         feats_collect.append(usr_id)
    284.         # 计算用户的性别特征，并保存在feats_collect中
    285.         usr_gender = self.usr_gender_emb(usr_gender)
    286.         usr_gender = self.usr_gender_fc(usr_gender)
    287.         usr_gender = fluid.layers.relu(usr_gender)
    288.         feats_collect.append(usr_gender)
    289.         # 选择是否使用用户的年龄-职业特征
    290.         if self.use_usr_age_job:
    291.             # 计算用户的年龄特征，并保存在feats_collect中
    292.             usr_age = self.usr_age_emb(usr_age)
    293.             usr_age = self.usr_age_fc(usr_age)
    294.             usr_age = fluid.layers.relu(usr_age)
    295.             feats_collect.append(usr_age)
    296.             # 计算用户的职业特征，并保存在feats_collect中
    297.             usr_job = self.usr_job_emb(usr_job)
    298.             usr_job = self.usr_job_fc(usr_job)
    299.             usr_job = fluid.layers.relu(usr_job)
    300.             feats_collect.append(usr_job)
    302.         # 将用户的特征级联，并通过Linear层得到最终的用户特征
    303.         usr_feat = fluid.layers.concat(feats_collect, axis=1)
    304.         usr_feat = self.usr_combined(usr_feat)
    305.         return usr_feat
    307.         # 定义电影特征的前向计算过程
    308.     def get_mov_feat(self, mov_var):
    309.         """ get movie features"""
    310.         # 获得电影数据
    311.         mov_id, mov_cat, mov_title, mov_poster = mov_var
    312.         feats_collect = []
    313.         # 获得batchsize的大小
    314.         batch_size = mov_id.shape[0]
    315.         # 计算电影ID的特征，并存在feats_collect中
    316.         mov_id = self.mov_emb(mov_id)
    317.         mov_id = self.mov_fc(mov_id)
    318.         mov_id = fluid.layers.relu(mov_id)
    319.         feats_collect.append(mov_id)
    321.         # 如果使用电影的种类数据，计算电影种类特征的映射
    322.         if self.use_mov_cat:
    323.             # 计算电影种类的特征映射，对多个种类的特征求和得到最终特征
    324.             mov_cat = self.mov_cat_emb(mov_cat)
    325.             mov_cat = fluid.layers.reduce_sum(mov_cat, dim=1, keep_dim=False)
    327.             mov_cat = self.mov_cat_fc(mov_cat)
    328.             feats_collect.append(mov_cat)
    330.         if self.use_mov_title:
    331.             # 计算电影名字的特征映射，对特征映射使用卷积计算最终的特征
    332.             mov_title = self.mov_title_emb(mov_title)
    333.             mov_title = self.mov_title_conv2(self.mov_title_conv(mov_title))
    334.             mov_title = fluid.layers.reduce_sum(mov_title, dim=2, keep_dim=False)
    335.             mov_title = fluid.layers.relu(mov_title)
    336.             mov_title = fluid.layers.reshape(mov_title, [batch_size, -1])
    338.             feats_collect.append(mov_title)
    340.         # 使用一个全连接层，整合所有电影特征，映射为一个200维的特征向量
    341.         mov_feat = fluid.layers.concat(feats_collect, axis=1)
    342.         mov_feat = self.mov_concat_embed(mov_feat)
    343.         return mov_feat
    345.     # 定义个性化推荐算法的前向计算
    346.     def forward(self, usr_var, mov_var):
    347.         # 计算用户特征和电影特征
    348.         usr_feat = self.get_usr_feat(usr_var)
    349.         mov_feat = self.get_mov_feat(mov_var)
    350.         # 根据计算的特征计算相似度
    351.         res = fluid.layers.cos_sim(usr_feat, mov_feat)
    352.         # 将相似度扩大范围到和电影评分相同数据范围
    353.         res = fluid.layers.scale(res, scale=5)
    354.         return usr_feat, mov_feat, res
    355.    
    1. # 解压数据集
    2. !cd work && unzip -o -q ml-1m.zip