#!/usr/bin/env python  
# coding: utf-8  
  
import warnings  
warnings.filterwarnings("ignore")  
  
import pandas as pd  
import numpy as np  
import pickle  
import matplotlib.pyplot as plt  
plt.style.use('seaborn')  
import tensorflow as tf  
import seaborn as sns  
from sklearn.model_selection import train_test_split  
from keras.models import Model, load_model  
from keras.layers import Input, Dense  
from keras.callbacks import ModelCheckpoint  
from keras import regularizers  
from sklearn.preprocessing import StandardScaler  
from sklearn.metrics import roc_curve, auc, precision_recall_curve  
# 安利一個(gè)異常檢測Python庫 https://github.com/yzhao062/Pyod  
  
# 讀取數(shù)據(jù) ：信用卡欺詐數(shù)據(jù)集地址https://www.kaggle.com/mlg-ulb/creditcardfraud  
d = pd.read_csv('creditcard.csv')  
  
# 查看樣本比例  
num_nonfraud = np.sum(d['Class'] == 0)  
num_fraud = np.sum(d['Class'] == 1)  
plt.bar(['Fraud', 'non-fraud'], [num_fraud, num_nonfraud], color='dodgerblue')  
plt.show()  
  
# 刪除時(shí)間列，對Amount進(jìn)行標(biāo)準(zhǔn)化  
data = d.drop(['Time'], axis=1)  
data['Amount'] = StandardScaler().fit_transform(data[['Amount']])  
  
# 為無監(jiān)督新穎點(diǎn)檢測方法，只提取負(fù)樣本，并且按照8:2切成訓(xùn)練集和測試集  
mask = (data['Class'] == 0)  
X_train, X_test = train_test_split(data[mask], test_size=0.2, random_state=0)  
X_train = X_train.drop(['Class'], axis=1).values  
X_test = X_test.drop(['Class'], axis=1).values  
  
# 提取所有正樣本，作為測試集的一部分  
X_fraud = data[~mask].drop(['Class'], axis=1).values  
  
# 構(gòu)建Autoencoder網(wǎng)絡(luò)模型  
# 隱藏層節(jié)點(diǎn)數(shù)分別為16，8，8，16  
# epoch為5，batch size為32  
input_dim = X_train.shape[1]  
encoding_dim = 16  
num_epoch = 5  
batch_size = 32  
  
input_layer = Input(shape=(input_dim, ))  
encoder = Dense(encoding_dim, activation="tanh",   
                activity_regularizer=regularizers.l1(10e-5))(input_layer)  
encoder = Dense(int(encoding_dim / 2), activation="relu")(encoder)  
decoder = Dense(int(encoding_dim / 2), activation='tanh')(encoder)  
decoder = Dense(input_dim, activation='relu')(decoder)  
autoencoder = Model(inputs=input_layer, outputs=decoder)  
autoencoder.compile(optimizer='adam',   
                    loss='mean_squared_error',   
                    metrics=['mae'])  
  
# 模型保存為model.h5，并開始訓(xùn)練模型  
checkpointer = ModelCheckpoint(filepath="model.h5",  
                               verbose=0,  
                               save_best_only=True)  
history = autoencoder.fit(X_train, X_train,  
                          epochs=num_epoch,  
                          batch_size=batch_size,  
                          shuffle=True,  
                          validation_data=(X_test, X_test),  
                          verbose=1,   
                          callbacks=[checkpointer]).history  
  
  
# 畫出損失函數(shù)曲線  
plt.figure(figsize=(14, 5))  
plt.subplot(121)  
plt.plot(history['loss'], c='dodgerblue', lw=3)  
plt.plot(history['val_loss'], c='coral', lw=3)  
plt.title('model loss')  
plt.ylabel('mse'); plt.xlabel('epoch')  
plt.legend(['train', 'test'], loc='upper right')  
  
plt.subplot(122)  
plt.plot(history['mae'], c='dodgerblue', lw=3)  
plt.plot(history['val_mae'], c='coral', lw=3)  
plt.title('model mae')  
plt.ylabel('mae'); plt.xlabel('epoch')  
plt.legend(['train', 'test'], loc='upper right')  
  
  
# 讀取模型  
autoencoder = load_model('model.h5')  
  
# 利用autoencoder重建測試集  
pred_test = autoencoder.predict(X_test)  
# 重建欺詐樣本  
pred_fraud = autoencoder.predict(X_fraud)    
  
# 計(jì)算重構(gòu)MSE和MAE誤差  
mse_test = np.mean(np.power(X_test - pred_test, 2), axis=1)  
mse_fraud = np.mean(np.power(X_fraud - pred_fraud, 2), axis=1)  
mae_test = np.mean(np.abs(X_test - pred_test), axis=1)  
mae_fraud = np.mean(np.abs(X_fraud - pred_fraud), axis=1)  
mse_df = pd.DataFrame()  
mse_df['Class'] = [0] * len(mse_test) + [1] * len(mse_fraud)  
mse_df['MSE'] = np.hstack([mse_test, mse_fraud])  
mse_df['MAE'] = np.hstack([mae_test, mae_fraud])  
mse_df = mse_df.sample(frac=1).reset_index(drop=True)  
  
# 分別畫出測試集中正樣本和負(fù)樣本的還原誤差MAE和MSE  
markers = ['o', '^']  
markers = ['o', '^']  
colors = ['dodgerblue', 'coral']  
labels = ['Non-fraud', 'Fraud']  
  
plt.figure(figsize=(14, 5))  
plt.subplot(121)  
for flag in [1, 0]:  
    temp = mse_df[mse_df['Class'] == flag]  
    plt.scatter(temp.index,   
                temp['MAE'],    
                alpha=0.7,   
                marker=markers[flag],   
                c=colors[flag],   
                label=labels[flag])  
plt.title('Reconstruction MAE')  
plt.ylabel('Reconstruction MAE'); plt.xlabel('Index')  
plt.subplot(122)  
for flag in [1, 0]:  
    temp = mse_df[mse_df['Class'] == flag]  
    plt.scatter(temp.index,   
                temp['MSE'],    
                alpha=0.7,   
                marker=markers[flag],   
                c=colors[flag],   
                label=labels[flag])  
plt.legend(loc=[1, 0], fontsize=12); plt.title('Reconstruction MSE')  
plt.ylabel('Reconstruction MSE'); plt.xlabel('Index')  
plt.show()  
# 下圖分別是MAE和MSE重構(gòu)誤差，其中橘黃色的點(diǎn)是信用欺詐，也就是異常點(diǎn)；藍(lán)色是正常點(diǎn)。我們可以看出異常點(diǎn)的重構(gòu)誤差整體很高。  
  
# 畫出Precision-Recall曲線  
plt.figure(figsize=(14, 6))  
for i, metric in enumerate(['MAE', 'MSE']):  
    plt.subplot(1, 2, i+1)  
    precision, recall, _ = precision_recall_curve(mse_df['Class'], mse_df[metric])  
    pr_auc = auc(recall, precision)  
    plt.title('Precision-Recall curve based on %s\nAUC = %0.2f'%(metric, pr_auc))  
    plt.plot(recall[:-2], precision[:-2], c='coral', lw=4)  
    plt.xlabel('Recall'); plt.ylabel('Precision')  
plt.show()  
  
# 畫出ROC曲線  
plt.figure(figsize=(14, 6))  
for i, metric in enumerate(['MAE', 'MSE']):  
    plt.subplot(1, 2, i+1)  
    fpr, tpr, _ = roc_curve(mse_df['Class'], mse_df[metric])  
    roc_auc = auc(fpr, tpr)  
    plt.title('Receiver Operating Characteristic based on %s\nAUC = %0.2f'%(metric, roc_auc))  
    plt.plot(fpr, tpr, c='coral', lw=4)  
    plt.plot([0,1],[0,1], c='dodgerblue', ls='--')  
    plt.ylabel('TPR'); plt.xlabel('FPR')  
plt.show()  
# 不管是用MAE還是MSE作為劃分標(biāo)準(zhǔn)，模型的表現(xiàn)都算是很好的。PR AUC分別是0.51和0.44，而ROC AUC都達(dá)到了0.95。  
  
# 畫出MSE、MAE散點(diǎn)圖  
markers = ['o', '^']  
colors = ['dodgerblue', 'coral']  
labels = ['Non-fraud', 'Fraud']  
  
plt.figure(figsize=(10, 5))  
for flag in [1, 0]:  
    temp = mse_df[mse_df['Class'] == flag]  
    plt.scatter(temp['MAE'],   
                temp['MSE'],    
                alpha=0.7,   
                marker=markers[flag],   
                c=colors[flag],   
                label=labels[flag])  
plt.legend(loc=[1, 0])  
plt.ylabel('Reconstruction RMSE'); plt.xlabel('Reconstruction MAE')  
plt.show()