import pandas as pd
import numpy as np
from pathlib import Path

data = Path('../data/data-en-hi-de-fr.csv')
df = pd.read_csv(data)
df.head()

df.labels.value_counts()

df = df[['labels', 'text_fr']]
df['labels'] = df['labels'].map({'ham': 0, 'spam': 1})
df = df.rename(columns={'text_fr': 'text', 'labels': 'is_spam'})

df.head().style.set_properties(subset=['text'], **{'width': '1000px'}).hide(axis='index')

import html
import unicodedata
import re

def preprocess_text(text):
    # Unescape HTML
    text = html.unescape(text)
    # Lowercase text
    text = text.lower()
    # Remove accents
    text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8')
    # Remove URLs and replace with 'URL'
    text = re.sub(r'http\S+', 'URL', text)
    # Remove emails and replace with 'EMAIL'
    text = re.sub(r'\b\S+@\S+\.\S+\b', 'EMAIL', text)
    # Remove potential phone numbers and replace with 'PHONE'
    text = re.sub(r'\b\d{5,}(-\d{5,})*\b', 'PHONE', text)
    # Remove digits and replace with 'DIGIT'
    text = re.sub(r'\d+', ' DIGIT ', text)
    # Remove single quotes
    text = text.replace("'", '')
    # Remove any non-alphanumeric characters
    text = re.sub(r'[^a-zA-Z\s]', ' ', text)
    return text

df['text'] = df['text'].apply(preprocess_text)

import nltk
nltk.download('stopwords')
nltk.download('punkt_tab')

# Tokenize text
from nltk.tokenize import word_tokenize
df['tokens'] = df['text'].apply(lambda x: word_tokenize(x, language='french'))

# Remove stopwords
from nltk.corpus import stopwords
stop_words = set(stopwords.words('french'))
df['tokens'] = df['tokens'].apply(lambda x: [word for word in x if word not in stop_words])

# Stem tokens
from nltk.stem import SnowballStemmer
df['tokens'] = df['tokens'].apply(lambda x: [SnowballStemmer('french').stem(word) for word in x])

# Join tokens back together
df['text'] = df['tokens'].apply(lambda x: ' '.join(x))

from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer

preprocessor = Pipeline([
    ('vectorizer', CountVectorizer(min_df=5, max_df=0.7)),
    ('tfidf_transformer', TfidfTransformer(use_idf=True)),
])

preprocessor_param = {
    'preprocessor__vectorizer__ngram_range': [(1, 1), (1, 2)],
}

from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC

models = [
    {
        'model_name' : 'Naive Bayes',
        'model_filepath' : Path('naive_bayes_model.pkl'),
        'model_instance' : MultinomialNB(),
        'model_param' : 
        {
            'model__alpha' : np.linspace(0.1, 1, 5)
        },
        'best_model' : None,
    },
    {
        'model_name' : 'Logistic Regression',
        'model_filepath' : Path('logistic_regression_model.pkl'),
        'model_instance' : LogisticRegression(penalty='l2', max_iter=500),
        'model_param' : 
        {
            'model__C' : np.linspace(0.1, 1, 5)
        },
        'best_model' : None,
    },
    {
        'model_name' : 'SVC',
        'model_filepath' : Path('svc_model.pkl'),
        'model_instance' : SVC(kernel = 'rbf', degree = 3, gamma = 'scale', probability=True),
        'model_param' : 
        { 
            'model__C' : np.linspace(0.1, 1, 5) 
        },
        'best_model': None,
    }
]

from sklearn.model_selection import train_test_split, GridSearchCV

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(df['text'], df['is_spam'], test_size=0.3, random_state=42)

for model_infos in models:
    
    pipeline = Pipeline([
        ('preprocessor', preprocessor),
        ('model', model_infos['model_instance'])
    ])
    
    param_grid = {**preprocessor_param, **model_infos['model_param']}
    
    # Find best hyperparameters
    grid_search = GridSearchCV(pipeline, param_grid, cv=5, n_jobs=-1)
    grid_search.fit(X_train, y_train)
    print(f"Best parameters for {model_infos['model_name']}: {grid_search.best_params_}")
    
    # Save model with best hyperparameters
    model_infos['best_model'] = grid_search.best_estimator_

from utils import roc_plot, precision_recall_plot, table_report

model_infos = models[0]
best_model = model_infos['best_model']
y_pred = best_model.predict(X_test)
y_pred_proba = best_model.predict_proba(X_test)

table_report(y_test, y_pred)

precision_recall_plot(y_test, y_pred_proba[:, 1])

roc_plot(y_test, y_pred_proba[:, 1])

model_infos = models[1]
best_model = model_infos['best_model']
y_pred = best_model.predict(X_test)
y_pred_proba = best_model.predict_proba(X_test)

table_report(y_test, y_pred)

precision_recall_plot(y_test, y_pred_proba[:, 1])

roc_plot(y_test, y_pred_proba[:, 1])

model_infos = models[2]
best_model = model_infos['best_model']
y_pred = best_model.predict(X_test)
y_pred_proba = best_model.predict_proba(X_test)

table_report(y_test, y_pred)

precision_recall_plot(y_test, y_pred_proba[:, 1])

roc_plot(y_test, y_pred_proba[:, 1])

import pickle

DATA_FOLDER = Path('../data')
SAVE = False

if SAVE:
    # Refit on all data and save model
    for model_infos in models:
        best_model = model_infos['best_model']
        best_model.fit(df['text'], df['is_spam'])
        
        with open(DATA_FOLDER / model_infos['model_filepath'], 'wb') as f:
            pickle.dump(best_model, f)

import ipywidgets as widgets
from IPython.display import display

# Load the saved model
model_filepath = Path('naive_bayes_model.pkl')  # Change this to the appropriate model file path
with open(DATA_FOLDER / model_filepath, 'rb') as f:
    loaded_model = pickle.load(f)

# Create input/output widgets
input_text = widgets.Textarea(
    value='',
    placeholder='Type something',
    description='Input:',
    disabled=False
)

output = widgets.Output()

# Define a function to perform inference
def on_text_submit(change):
    with output:
        output.clear_output()
        text = change['new']
        text = preprocess_text(text)
        
        tokens = word_tokenize(text, language='french')
        tokens = [word for word in tokens if word not in stop_words]
        tokens = [SnowballStemmer('french').stem(word) for word in tokens]
        
        text = ' '.join(tokens)

        prediction_proba = loaded_model.predict_proba([text])
        print(f"Spam probability (between 0 and 1): {prediction_proba[0][1]*100:.2f}%")

# Bind the function to the input widget
input_text.observe(on_text_submit, names='value')

display(input_text, output)