CCF&滴滴出行——路况状态时空预测AI算法竞赛baseline方案三（LSTM模型）

楼主最近在参加一个比赛：比赛链接如下：
https://www.datafountain.cn/competitions/466
现在分享自己的代码，该方案目前成绩在65/2865

运行环境: python 3.7.9，tensorflow 2.0.0，keras 2.3.1
Name:Model_LSTM
一：数据预处理+特征工程

# Version: python 3.7.9  tensorflow 2.0.0 keras 2.3.1
# Name:Model LSTM
import numpy as np
import pandas as pd
import os
import datetime
from tqdm import tqdm
from collections import Counter
import warnings
warnings.filterwarnings("ignore")




def get_info(x):
    return [i.split(":")[-1] for i in x.split(" ")]
def get_speed(x):
    return np.array([i.split(",")[0] for i in x],dtype='float16')
def get_eta(x):
    return np.array([i.split(",")[1] for i in x],dtype="float16")
def get_state(x):
    return np.array([i.split(",")[2] for i in x])
def get_cnt(x):
    return np.array([i.split(",")[3] for i in x],dtype="int16")


def get_feature(input_file_path_his, input_file_path_attr,input_file_path_topo, mode):
    # his
    df = pd.read_csv(input_file_path_his, sep=";", header=None)
    df["link"] = df[0].apply(lambda x: x.split(" ")[0]).astype(int)
    df["label"] = df[0].apply(lambda x: x.split(" ") [1]).astype(int)
    df["current_slice_id"] = df[0].apply(lambda x: x.split(" ")[2]).astype(int)
    df["future_slice_id"] = df[0].apply(lambda x: x.split(" ")[3]).astype(int)
    df["time_diff"] = df["future_slice_id"] - df["current_slice_id"]
    df = df.drop([0], axis=1)

    if mode == "is_train":
        df["label"] = df["label"].map(lambda x: 3 if x >= 3 else x)
        df['label'] -= 1
    else:
        df = df.drop(["label"], axis=1)

    df["current_state_last"] = df[1].apply(lambda x: x.split(" ")[-1].split(":")[-1])
    # 路况速度,eta速度,路况状态,参与路况计算的车辆数
    df["current_speed"] = df["current_state_last"].apply(lambda x: x.split(",")[0])
    df["current_eat_speed"] = df["current_state_last"].apply(lambda x: x.split(",")[1])
    df["current_state"] = df["current_state_last"].apply(lambda x: x.split(",")[2])
    df["current_count"] = df["current_state_last"].apply(lambda x: x.split(",")[3])
    df = df.drop(["current_state_last"], axis=1)
    for i in tqdm(range(1, 6, 1)):
        flag = f"his_{(6-i)*7}"
        df["history_info"] = df[i].apply(get_info)

        # speed
        df["his_speed"] = df["history_info"].apply(get_speed)
        df[f'{flag}_speed_mean'] = df["his_speed"].apply(lambda x: x.mean())

        # eta
        df["his_eta"] = df["history_info"].apply(get_eta)
        df[f"{flag}_eta_mean"] = df["his_eta"].apply(lambda x: x.mean())

        # state
        df["his_state"] = df["history_info"].apply(get_state)
        df[f"{flag}_state_max"] = df["his_state"].apply(lambda x: Counter(x).most_common()[0][0])
        df[f"{flag}_state_min"] = df["his_state"].apply(lambda x: Counter(x).most_common()[-1][0])

        # cnt
        df["his_cnt"] = df["history_info"].apply(get_cnt)
        df[f"{flag}_cnt_mean"] = df["his_cnt"].apply(lambda x: x.mean())
        df = df.drop([i, "history_info", "his_speed", "his_eta", "his_state", "his_cnt"], axis=1)
        # break

    df2 = pd.read_csv(input_file_path_attr, sep='\t',
                       names=['link', 'length', 'direction', 'path_class', 'speed_class',
                              'LaneNum', 'speed_limit', 'level', 'width'], header=None)
    df = df.merge(df2, on='link', how='left')

    if mode =="is_train":
        output_file_path =f"./data/{mode}_{input_file_path_his.split('/')[-1].split('.')[0]}" +".csv"
        df.to_csv(output_file_path,index =False,mode='w', header=True)

    else:
        output_file_path=f"./data/{input_file_path_his.split('/')[-1].split('.')[0]}" +".csv"
        df.to_csv(output_file_path,index = False,mode='w', header=True)
    # print(df.dtypes)

if __name__ =="__main__":
    print(datetime.datetime.now())
    #训练集
    get_feature(input_file_path_his="D:/traffic-fix/input_data/traffic/20190701.txt",\
                input_file_path_attr="D:/traffic-fix/input_data/road_attribute/attr.txt",\
                input_file_path_topo="D:/traffic-fix/input_data/road_topo/topo.txt",mode="is_train")
    #测试集
    get_feature(input_file_path_his="D:/traffic-fix/test_data/20190801_testdata.txt",\
                input_file_path_attr="D:/traffic-fix/input_data/road_attribute/attr.txt",\
                input_file_path_topo="D:/traffic-fix/input_data/road_topo/topo.txt",mode="is_test")
    print(datetime.datetime.now())

二：建立模型+模型评估

# Version: python 3.7.9  tensorflow 2.0.0 keras 2.3.1
# Name:Model LSTM
# =======================================================================

import tensorflow as tf
import keras as K
import pandas as pd
import numpy as np
from keras.utils import np_utils
import datetime
from keras.layers import Dense, LSTM
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical
from keras.models import Sequential


def LSTM_Model(train=None,label=None, test=None, use_features=None, n_class=3):
    # lb = LabelBinarizer()
    # Y = lb.fit_transform(train[label].values)
    # 1. 处理数据
    Y = np_utils.to_categorical(train[label], num_classes=n_class)
    train_x, test_x, train_y, test_y = train_test_split(train[use_features], Y,
                                                        train_size=0.7, test_size=0.3, random_state=0)

    # 输入shape(samples,timesteps,input_dim)
    # samples表示数据的长度,timesteps表示步长,input_dim表示特征维度
    timesteps = 1
    train_X = train_x.values.reshape((train_x.shape[0], timesteps, train_x.shape[1]))
    test_X = test_x.values.reshape((test_x.shape[0], timesteps, test_x.shape[1]))
    test_x_pre = test[use_features].values.reshape((test[use_features].shape[0], timesteps, test[use_features].shape[1]))

    # 2. 建立模型
    nb_lstm_outputs = 30  #神经元个数
    model = Sequential()
    model.add(LSTM(units=nb_lstm_outputs, input_shape=(train_X.shape[1],train_X.shape[2])))
    model.add(Dense(n_class, activation='softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    # 3.训练模型
    model.fit(train_X, train_y, epochs=10, batch_size=128, verbose=1)
    model.summary()
    # 4. 评估模型
    score = model.evaluate(test_X, test_y,batch_size=128, verbose=1)
    print(score)
    # 5. 预测结果
    predictions = model.predict(test_x_pre)
    test["label"] = np.argmax(predictions, axis=1) + 1

    return test[["link", 'current_slice_id', 'future_slice_id', "label"]]

if __name__ =="__main__":

    train = pd.read_csv('./data/is_train_20190701.csv')
    test = pd.read_csv("./data/20190801_testdata.csv")

    # print(train.dtypes)
    # print(train["link_id_length"].unique())

    del_feature = ['link','label']
    use_features = [i for i in train.columns if i not in del_feature]

    # train['label'] = train['label'].apply(lambda x: 1 if x == 32 else 0)
    category = ["direction","pathclass","speedclass","LaneNum","level"]
    print(datetime.datetime.now())

    submit =LSTM_Model(train=train,label="label", test=test, use_features=use_features,n_class=3)
    submit.to_csv('submit.csv', index=False, encoding='utf8')

    print(datetime.datetime.now())