import pandas as pd
import numpy as np
# 使用GBM的算法
import lightgbm as lgb
#import xgboost as xgb
# 引入科学计算的函数库，从中引入vstack：增加维数 csr_matrix：稀疏矩阵 save_npz：保存文件，load_npz：加载文件
from scipy.sparse import vstack, csr_matrix, save_npz, load_npz
# 引入机器学习函数库之数据预处理的库，在数据预处理的库中引入独热编码方式以及标签编码方式
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
# 引入机器学习函数库值模型选择的库，从中导入k折交叉验证的方法
from sklearn.model_selection import StratifiedKFold
#from sklearn.metrics import roc_auc_score
# 引入垃圾回收的库
import gc
# 初始化垃圾回收机制
gc.enable()
# 设置数据的类型
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
print('Download Train and Test Data.\n')
#读取训练集的文件，读取的方式把定义的dtypes的字典赋给dtype参数
train = pd.read_csv('../input/train.csv', dtype=dtypes, low_memory=True)
train['MachineIdentifier'] = train.index.astype('uint32')
#读取测试集的文件，读取的方式把定义的dtypes的字典赋给dtype参数
test = pd.read_csv('../input/test.csv', dtype=dtypes, low_memory=True)
test['MachineIdentifier'] = test.index.astype('uint32')
# 垃圾回收机制中回收函数，在变量生命周期结束之时，回收所对的空间，防止内存泄漏
gc.collect()
print('Transform all features to category.\n')
# 第1步：如何提高分类准确性的关键--即:将数据提纯
for usecol in train.columns.tolist()[1:-1]:# 首尾两列不是特征值，且末尾列是所分的类别
train[usecol] = train[usecol].astype('str')# 转换函数，将每一列的数据类型都转换为字符串
test[usecol] = test[usecol].astype('str') # 转换函数，将每一列的数据类型都转换为字符串
# 这样的做的目的是将所有变量的特征值的可能取值能够使用独热编码
# Fit LabelEncoder：一种直接按照序号进行编码的编码方式
le = LabelEncoder().fit(np.unique(train[usecol].unique().tolist() + test[usecol].unique().tolist()))
# 在后面由于要把缺失值用0来填充，因此在编码时，需要+1
train[usecol] = le.transform(train[usecol])+1
test[usecol] = le.transform(test[usecol])+1
# 注意：agg和aggrigate函数是功能相同，agg_tr和agg_te的变量是存储训练集中
agg_tr = (train
.groupby([usecol])# 在训练集的每一列特征的特征取值，统计同一个类别在这总样本中出现的次数，并将统计结果聚合在新的一列
.aggregate({'MachineIdentifier': 'count'})# count是非NA值的数量
.reset_index()# 重新设置索引
.rename({'MachineIdentifier': 'Train'}, axis=1))
agg_te = (test
.groupby([usecol])
.aggregate({'MachineIdentifier': 'count'})# （记住：聚合时选择不同的标题）将相同的变量取值进行分组，并统计每一个取值相同变量的个数
.reset_index()
.rename({'MachineIdentifier': 'Test'}, axis=1))# rename替换标题
# 将测试集与训练集合并（agg即为aggrigate的简称），填充缺失值
agg = pd.merge(agg_tr, agg_te, on=usecol, how='outer').replace(np.nan, 0)
'''
筛选出每一个特征值中可能取值占比重适中的可能取值
'''
# 选择在训练集中出现次数超过1000的特征取值
agg = agg[(agg['Train'] > 1000)].reset_index(drop=True)# 将符合条件的特征取值重新编排索引
agg['Total'] = agg['Train'] + agg['Test']
# 这一预处理目的是找出训练集的非空取值所占总数的合适比例，便于使用分类器是分类准确
agg = agg[(agg['Train'] / agg['Total'] > 0.2) & (agg['Train'] / agg['Total'] < 0.8)]
agg[usecol+'Copy'] = agg[usecol]
# 这是筛选后的训练集
train[usecol] = (pd.merge(train[[usecol]],
agg[[usecol, usecol+'Copy']],
on=usecol, how='left')[usecol+'Copy']
.replace(np.nan, 0).astype('int').astype('category'))
# 这是筛选后的测试集
test[usecol] = (pd.merge(test[[usecol]],
agg[[usecol, usecol+'Copy']],
on=usecol, how='left')[usecol+'Copy']
.replace(np.nan, 0).astype('int').astype('category'))
# 不用的变量可以delete掉，这是一个好的编程习惯防止内存泄漏
del le, agg_tr, agg_te, agg, usecol
gc.collect()
# 建立训练集目标值的数组
y_train = np.array(train['HasDetections'])
train_ids = train.index
test_ids = test.index
# 不用的变量可以delete掉，这是一个好的编程习惯防止内存泄漏
del train['HasDetections'], train['MachineIdentifier'], test['MachineIdentifier']
gc.collect()
print("If you don't want use Sparse Matrix choose Kernel Version 2 to get simple solution.\n")
print('--------------------------------------------------------------------------------------------------------')
print('Transform Data to Sparse Matrix.')
print('Sparse Matrix can be used to fit a lot of models, eg. XGBoost, LightGBM, Random Forest, K-Means and etc.')
print('To concatenate Sparse Matrices by column use hstack()')
print('Read more about Sparse Matrix https://docs.scipy.org/doc/scipy/reference/sparse.html')
print('Good Luck!')
print('--------------------------------------------------------------------------------------------------------')
# 初始化独立编码的对象，用训练集进行拟合
ohe = OneHotEncoder(categories='auto', sparse=True, dtype='uint8').fit(train)
# 现在进行训练集和测试集的采集，以下是采集方法，这种方法是分段划分
m = 100000
# 采取等距抽样的方式选取训练集和测试集部分的数据，作为要用机器学习算法的训练集和测试集，这一些数据的都以稀疏矩阵的方式进行存储
train = vstack([ohe.transform(train[i*m:(i+1)*m]) for i in range(train.shape[0] // m + 1)])
test = vstack([ohe.transform(test[i*m:(i+1)*m]) for i in range(test.shape[0] // m + 1)])
# 随后将分段标记索引的数据集分别按照训练集和测试集的类别存储在两个文件
save_npz('train.npz', train, compressed=True)
save_npz('test.npz', test, compressed=True)
# 不用的变量可以delete掉，这是一个好的编程习惯防止内存泄漏
del ohe, train, test
gc.collect()
# 第2步：对划分后的训练集和测试集进行验证
#创建交叉验证的对象，n_splits：划分的组数；shuffle：是否重新洗牌；random_state：随机种子数
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
# 将训练集和测试集进行分开
skf.get_n_splits(train_ids, y_train)
lgb_test_result = np.zeros(test_ids.shape[0])
#lgb_train_result = np.zeros(train_ids.shape[0])
#xgb_test_result = np.zeros(test_ids.shape[0])
#xgb_train_result = np.zeros(train_ids.shape[0])
counter = 0
print('\nLightGBM\n')
# 构建循环进行交叉验证
for train_index, test_index in skf.split(train_ids, y_train):
print('Fold {}\n'.format(counter + 1))
train = load_npz('train.npz')
X_fit = vstack([train[train_index[i*m:(i+1)*m]] for i in range(train_index.shape[0] // m + 1)])
X_val = vstack([train[test_index[i*m:(i+1)*m]] for i in range(test_index.shape[0] // m + 1)])
X_fit, X_val = csr_matrix(X_fit, dtype='float32'), csr_matrix(X_val, dtype='float32')
y_fit, y_val = y_train[train_index], y_train[test_index]
# 不用的变量可以delete掉，这是一个好的编程习惯防止内存泄漏
del train
gc.collect()
lgb_model = lgb.LGBMClassifier(max_depth=-1,
n_estimators=30000,
learning_rate=0.05,
num_leaves=2**12-1,
colsample_bytree=0.28,
objective='binary',
n_jobs=-1)
#xgb_model = xgb.XGBClassifier(max_depth=6,
# n_estimators=30000,
# colsample_bytree=0.2,
# learning_rate=0.1,
# objective='binary:logistic',
# n_jobs=-1)
lgb_model.fit(X_fit,y_fit,
eval_metric='auc', eval_set=[(X_val, y_val)],
verbose=100, early_stopping_rounds=100)
#xgb_model.fit(X_fit, y_fit, eval_metric='auc',
# eval_set=[(X_val, y_val)],
# verbose=1000, early_stopping_rounds=300)
#lgb_train_result[test_index] += lgb_model.predict_proba(X_val)[:,1]
#xgb_train_result[test_index] += xgb_model.predict_proba(X_val)[:,1]
# 不用的变量可以delete掉，这是一个好的编程习惯防止内存泄漏
del X_fit, X_val, y_fit, y_val, train_index, test_index
gc.collect()
test = load_npz('test.npz')
test = csr_matrix(test, dtype='float32')
lgb_test_result += lgb_model.predict_proba(test)[:,1]
#xgb_test_result += xgb_model.predict_proba(test)[:,1]
counter += 1
# 不用的变量可以delete掉，这是一个好的编程习惯防止内存泄漏
del test
gc.collect()
#Stop fitting to prevent time limit error
#if counter == 3 : break
#print('\nLigthGBM VAL AUC Score: {}'.format(roc_auc_score(y_train, lgb_train_result)))
#print('\nXGBoost VAL AUC Score: {}'.format(roc_auc_score(y_train, xgb_train_result)))
# 第3步：用来计算该分类器进行分类的准确率
submission = pd.read_csv('../input/sample_submission.csv')
submission['HasDetections'] = lgb_test_result / counter
submission.to_csv('lgb_submission.csv', index=False)
#submission['HasDetections'] = xgb_test_result / counter
#submission.to_csv('xgb_submission.csv', index=False)
#submission['HasDetections'] = 0.5 * lgb_test_result / counter + 0.5 * xgb_test_result / counter
#submission.to_csv('lgb_xgb_submission.csv', index=False)
#完成！并结束程序
print('\nDone.')
