NN_save_model.py

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
from sklearn.preprocessing import LabelEncoder
import tensorflow as tf

import f_score_metrics

# Read "train.csv" file
df = pd.read_csv("DataSet/train.csv")

# Splitting the 'is_fraud?' column
labels = df["is_fraud?"].copy().to_numpy()
labels = labels.astype(int)
df = df.drop("is_fraud?", axis=1)
df = df.set_index(df.columns[0])

# Delete $ symbol from amount column
df['amount'] = df['amount'].str.replace('$', '').astype(float)

# Split "zip" by units
df["zip_1"] = df["zip"] // 10000
df["zip_2"] = (df["zip"] - df["zip_1"]) // 100
df["zip_4"] = df["zip"] % 100

# Drop "zip"
df = df.drop("zip", axis=1)

# Replace NaN 
df["merchant_state"] = df["merchant_state"].fillna("Online")
df = df.fillna(-1)

# Change float64 with int64
df["amount"] = round(df["amount"] * 10)
df["amount"] = df["amount"].astype("int64")
df["zip_2"] = df["zip_2"].astype("int64")
df["zip_4"] = df["zip_4"].astype("int64")

df["merchant_id"] = df["merchant_id"].astype("int64")
df["mcc"] = df["mcc"].astype("int64")
df["merchant_city"] = df["merchant_city"].astype("category")
df["merchant_state"] = df["merchant_state"].astype("category")
df["errors?"] = df["errors?"].astype("category")
df["use_chip"] = df["use_chip"].astype("category")
df["user_id"] = df["user_id"].astype("int64")
df["card_id"] = df["card_id"].astype("int64")
df["zip_1"] = df["zip_1"].astype("int64")

print(df.head(5))

# Create additional data
# User average amount
user_avg_amount = df.groupby("user_id")["amount"].mean().reset_index()
user_avg_amount['amount'] = np.round(user_avg_amount['amount'])
user_avg_amount.columns = ['user_id', 'user_avg_amount']

# Merchant average amount
merchant_avg_amount = df.groupby("merchant_id")["amount"].mean().reset_index()
merchant_avg_amount['amount'] = np.round(merchant_avg_amount['amount'])
merchant_avg_amount.columns = ["merchant_id", "merchant_avg_amount"]

# Add to Original Dataset
df = pd.merge(df, user_avg_amount, on="user_id", how="left")
df = pd.merge(df, merchant_avg_amount, on="merchant_id", how="left")


# Labeling to categorical datas
le_city = LabelEncoder()
le_city.fit(df["merchant_city"])
df["merchant_city"] = le_city.transform(df["merchant_city"])

le_state = LabelEncoder()
le_state.fit(df["merchant_state"])
df["merchant_state"] = le_state.transform(df["merchant_state"])

le_errors = LabelEncoder()
le_errors.fit(df["errors?"])
df["errors?"] = le_errors.transform(df["errors?"])

le_chip = LabelEncoder()
le_chip.fit(df["use_chip"])
df["use_chip"] = le_chip.transform(df["use_chip"])

# errors? -> errors
df.rename(columns={"errors?" : "errors"}, inplace=True)

# Print Data sample
print(f"\n{df.head(5)}\n")
print(labels[:5])
print()
print(le_errors.classes_)
print()

# Validate
print(df.dtypes)

# Split Datas for train & test
X_train, X_test, y_train, y_test = train_test_split(df, labels, test_size=0.1, random_state=1225)

# Shift to tf.data.Dataset
train_dataset = tf.data.Dataset.from_tensor_slices((dict(X_train.to_dict('list')), y_train))
test_dataset = tf.data.Dataset.from_tensor_slices((dict(X_test.to_dict("list")), y_test))
df_dataset = tf.data.Dataset.from_tensor_slices((dict(df.to_dict("list")), labels))

# One-hot encoding to "card_id", "zip_1"
def one_hot_encode(features):
    features["card_id"] = tf.one_hot(features["card_id"], depth=10)
    features["zip_1"] = tf.one_hot(features["zip_1"], depth=10)
    features["use_chip"] = tf.one_hot(features["use_chip"], depth=3)
    return features

train_dataset = train_dataset.map(lambda x, y: (one_hot_encode(x), y))
test_dataset = test_dataset.map(lambda x, y: (one_hot_encode(x), y))
df_dataset = df_dataset.map(lambda x, y: (one_hot_encode(x), y))

# Change to vector
def reshape_scalars(x, y):
    reshaped_x = {}
    for key, value in x.items():
        if len(value.shape) == 0:  # 스칼라 값인 경우
            reshaped_x[key] = tf.cast(tf.reshape(value, (1,)), dtype=tf.float32)
        else:
            reshaped_x[key] = tf.cast(value, dtype=tf.float32)
    return reshaped_x, y

# Dataset 객체에 map 함수 적용
train_dataset = train_dataset.map(reshape_scalars)
test_dataset = test_dataset.map(reshape_scalars)
df_dataset = df_dataset.map(reshape_scalars)

# print for validate
for item, label in df_dataset.take(1):
    for key, value in item.items():
        print(f"{key}: {value.numpy()}")
    print(label)

# Count fraud or not
total_samples = len(y_train)
num_not_fraud = np.count_nonzero(y_train == 0)
num_fraud = np.count_nonzero(y_train == 1)

class_weight = {
    0: total_samples / (2 * num_not_fraud),
    1: total_samples / (2 * num_fraud)
}


# Build Neural Network
class Logistic_Model(tf.keras.Model):
    def __init__(self, units: int, output_dim:int, output_dim_small:int, output_dim_large:int, kernel_l2_lambda: float, 
                 activity_l2_lambda: float, activity_l2_small: float, activity_l2_big: float,
                 dropout_rate: float , kernel_initializer: str, dropout_small: float, dropout_big: float):
        super(Logistic_Model, self).__init__()

        self.units = units
        self.output_dim = output_dim
        self.output_dim_small = output_dim_small
        self.output_dim_large = output_dim_large
        self.kernel_l2_lambda = kernel_l2_lambda
        self.activity_l2_lambda = activity_l2_lambda
        self.dropout_rate = dropout_rate
        self.kernel_initializer = kernel_initializer
        self.activity_l2_small = activity_l2_small
        self.activity_l2_big = activity_l2_big
        self.dropout_small = dropout_small
        self.dropout_big = dropout_big

        self.input_user_id = tf.keras.layers.Embedding(
            input_dim=2000, output_dim=self.output_dim, input_length=1, activity_regularizer=tf.keras.regularizers.l2(self.activity_l2_small), mask_zero=False)
        self.input_amount = tf.keras.layers.Embedding(
            input_dim=20000, output_dim=self.output_dim_large, input_length=1, activity_regularizer=tf.keras.regularizers.l2(self.activity_l2_big), mask_zero=False)
        self.input_mer_id = tf.keras.layers.Embedding(
            input_dim=25076, output_dim=self.output_dim_large, input_length=1, activity_regularizer=tf.keras.regularizers.l2(self.activity_l2_big), mask_zero=False)
        self.input_mer_ct = tf.keras.layers.Embedding(
            input_dim=4400, output_dim=self.output_dim, input_length=1, activity_regularizer=tf.keras.regularizers.l2(self.activity_l2_small), mask_zero=False)
        self.input_mer_st = tf.keras.layers.Embedding(
            input_dim=130, output_dim=self.output_dim, input_length=1, activity_regularizer=tf.keras.regularizers.l2(self.activity_l2_small), mask_zero=False)
        self.input_mcc = tf.keras.layers.Embedding(
            input_dim=110, output_dim=self.output_dim, input_length=1, activity_regularizer=tf.keras.regularizers.l2(self.activity_l2_small), mask_zero=False)
        self.input_zip2 = tf.keras.layers.Embedding(
            input_dim=1000, output_dim=self.output_dim, input_length=1, activity_regularizer=tf.keras.regularizers.l2(self.activity_l2_small), mask_zero=False)
        self.input_zip4 = tf.keras.layers.Embedding(
            input_dim=100, output_dim=self.output_dim, input_length=1, activity_regularizer=tf.keras.regularizers.l2(self.activity_l2_small), mask_zero=False)
        self.input_user_avg = tf.keras.layers.Embedding(
            input_dim=2000, output_dim=self.output_dim, input_length=1, activity_regularizer=tf.keras.regularizers.l2(self.activity_l2_small), mask_zero=False)
        self.input_mer_avg = tf.keras.layers.Embedding(
            input_dim=2000, output_dim=self.output_dim, input_length=1, activity_regularizer=tf.keras.regularizers.l2(self.activity_l2_small), mask_zero=False)

        self.input_card_id = tf.keras.layers.Dense(units=output_dim_small, activation="relu", kernel_initializer="he_normal")
        self.input_use_chip = tf.keras.layers.Dense(units=output_dim_small, activation="relu", kernel_initializer="he_normal")
        self.input_zip1 = tf.keras.layers.Dense(units=output_dim_small, activation="relu", kernel_initializer="he_normal")

        self.hidden = tf.keras.layers.Dense(
            units=self.units,
            kernel_regularizer=tf.keras.regularizers.L2(self.kernel_l2_lambda),
            activity_regularizer=tf.keras.regularizers.L2(self.activity_l2_lambda),
            activation="relu",
            kernel_initializer=self.kernel_initializer, # he_normal or he_uniform
            name="hidden"
        )

        self.dropout = tf.keras.layers.Dropout(self.dropout_rate)
        self.dropout_small_layer = tf.keras.layers.Dropout(self.dropout_small)
        self.dropout_big_layer = tf.keras.layers.Dropout(self.dropout_big)

        self.output_layer = tf.keras.layers.Dense(1, activation="sigmoid")

    def call(self, inputs: tf.data.Dataset):

        user_id_out = self.input_user_id(inputs["user_id"])
        user_id_out = tf.squeeze(user_id_out, axis=1)
        amount_out = self.input_amount(inputs["amount"])
        amount_out = tf.squeeze(amount_out, axis=1)
        mer_id_out = self.input_mer_id(inputs["merchant_id"])
        mer_id_out = tf.squeeze(mer_id_out, axis=1)
        mer_ct_out = self.input_mer_ct(inputs["merchant_city"])
        mer_ct_out = tf.squeeze(mer_ct_out, axis=1)
        mer_st_out = self.input_mer_st(inputs["merchant_state"])
        mer_st_out = tf.squeeze(mer_st_out, axis=1)
        mcc_out = self.input_mcc(inputs["mcc"])
        mcc_out = tf.squeeze(mcc_out, axis=1)
        zip2_out = self.input_zip2(inputs["zip_2"])
        zip2_out = tf.squeeze(zip2_out, axis=1)
        zip4_out = self.input_zip4(inputs["zip_4"])
        zip4_out = tf.squeeze(zip4_out, axis=1)
        user_avg_out = self.input_user_avg(inputs["user_avg_amount"])
        user_avg_out = tf.squeeze(user_avg_out, axis=1)
        mer_avg_out = self.input_mer_avg(inputs["merchant_avg_amount"])
        mer_avg_out = tf.squeeze(mer_avg_out, axis=1)

        card_id_out = self.input_card_id(inputs["card_id"])
        use_chip_out = self.input_use_chip(inputs["use_chip"])
        zip1_out = self.input_zip1(inputs["zip_1"])

        user_id_out = self.dropout_small_layer(user_id_out)
        amount_out = self.dropout_big_layer(amount_out)
        mer_id_out = self.dropout_big_layer(mer_id_out)
        mer_ct_out = self.dropout_small_layer(mer_ct_out)
        mer_st_out = self.dropout_small_layer(mer_st_out)
        mcc_out = self.dropout_small_layer(mcc_out)
        zip2_out = self.dropout_small_layer(zip2_out)
        zip4_out = self.dropout_small_layer(zip4_out)
        user_avg_out = self.dropout_small_layer(user_avg_out)
        mer_avg_out = self.dropout_small_layer(mer_avg_out)
        
        x = tf.concat([user_id_out, card_id_out, amount_out, inputs["errors"], mer_id_out, mer_ct_out, mer_st_out, 
                       mcc_out, mcc_out, use_chip_out, zip1_out, zip2_out, zip4_out, user_avg_out, mer_avg_out], axis=1)

        x = self.hidden(x)
        x = self.dropout(x)
        output = self.output_layer(x)
        
        return output
    
    def get_config(self):
        config = super().get_config()
        config.update({
            'units': self.units,
            'output_dim': self.output_dim,
            'output_dim_small': self.output_dim_small,
            'output_dim_large': self.output_dim_large,
            'kernel_l2_lambda': self.kernel_l2_lambda,
            'activity_l2_lambda': self.activity_l2_lambda,
            "activity_l2_small": self.activity_l2_small,
            "activity_l2_big": self.activity_l2_big,
            'dropout_rate': self.dropout_rate,
            "dropout_small": self.dropout_small,
            "dropout_big": self.dropout_big,
            'kernel_initializer': self.kernel_initializer
        })
        return config

    @classmethod
    def from_config(cls, config):
        return cls(**config)
    
# Model Set & Learn
early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)
model = Logistic_Model(units=972, output_dim=257, output_dim_small=4, output_dim_large=363, 
                       kernel_l2_lambda=1.135931893834352e-06, activity_l2_lambda=0.0007376979639327542, 
                       dropout_rate=0.05,  activity_l2_small=0.00015649062805372037, activity_l2_big=0.010755911495755333, 
                       dropout_small=0.0, dropout_big=0.45, kernel_initializer="he_normal")
batch_size = 107
lr = 0.02132
model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=lr), loss='binary_crossentropy', metrics=[f_score_metrics.F1Score()])
model.fit(df_dataset.batch(batch_size),
            epochs=100,
            class_weight=class_weight,
            validation_data=test_dataset.batch(batch_size),
            callbacks=[early_stopping])

# Save model
model.save("ensembledb/nn_full_model_1")

# Predict & Validate
y_pred = model.predict(test_dataset.batch(batch_size))
y_pred = (y_pred > 0.5).astype(int).flatten()  # Convert probabilities to binary labels and flatten to 1D array
model_metric = f1_score(y_test, y_pred)