
# Install the 'contractions' library using pip
!pip install contractions

# Import necessary libraries
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
import contractions
import re
import nltk
import numpy as np
import tensorflow

# Download NLTK resources
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')

# Import additional libraries
from wordcloud import WordCloud, STOPWORDS
from bs4 import BeautifulSoup
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score
from keras.models import Sequential
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Dense, Embedding, LSTM, SpatialDropout1D, Dropout, Bidirectional
from sklearn.preprocessing import LabelBinarizer
from nltk.tokenize import RegexpTokenizer

# Mount Google Drive in Google Colab
from google.colab import drive
drive.mount('/content/drive')

Collecting contractions
  Downloading contractions-0.1.73-py2.py3-none-any.whl (8.7 kB)
Collecting textsearch>=0.0.21 (from contractions)
  Downloading textsearch-0.0.24-py2.py3-none-any.whl (7.6 kB)
Collecting anyascii (from textsearch>=0.0.21->contractions)
  Downloading anyascii-0.3.2-py3-none-any.whl (289 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 289.9/289.9 kB 6.1 MB/s eta 0:00:00
Collecting pyahocorasick (from textsearch>=0.0.21->contractions)
  Downloading pyahocorasick-2.0.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (110 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 110.8/110.8 kB 11.4 MB/s eta 0:00:00
Installing collected packages: pyahocorasick, anyascii, textsearch, contractions
Successfully installed anyascii-0.3.2 contractions-0.1.73 pyahocorasick-2.0.0 textsearch-0.0.24

[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.
[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Unzipping tokenizers/punkt.zip.
[nltk_data] Downloading package wordnet to /root/nltk_data...

Mounted at /content/drive


# Mount Google drive to access the dataset only if you are using Google Colab
drive.mount('/content/drive/')

Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount("/content/drive/", force_remount=True).


# Read the data
tweets = pd.read_csv('/content/drive/MyDrive/7-NLP/project/Tweets.csv')
data = tweets.copy()


# Display the head of the DataFrame
data.head()


# Display the tail of the DataFrame
data.tail()


# Display information about the DataFrame structure
data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 14640 entries, 0 to 14639
Data columns (total 15 columns):
 #   Column                        Non-Null Count  Dtype  
---  ------                        --------------  -----  
 0   tweet_id                      14640 non-null  int64  
 1   airline_sentiment             14640 non-null  object 
 2   airline_sentiment_confidence  14640 non-null  float64
 3   negativereason                9178 non-null   object 
 4   negativereason_confidence     10522 non-null  float64
 5   airline                       14640 non-null  object 
 6   airline_sentiment_gold        40 non-null     object 
 7   name                          14640 non-null  object 
 8   negativereason_gold           32 non-null     object 
 9   retweet_count                 14640 non-null  int64  
 10  text                          14640 non-null  object 
 11  tweet_coord                   1019 non-null   object 
 12  tweet_created                 14640 non-null  object 
 13  tweet_location                9907 non-null   object 
 14  user_timezone                 9820 non-null   object 
dtypes: float64(2), int64(2), object(11)
memory usage: 1.7+ MB


# Display the dimensions (number of rows and columns) of the DataFrame
data.shape

(14640, 15)


# Check for missing values in each column and sum them up
data.isna().sum()

tweet_id                            0
airline_sentiment                   0
airline_sentiment_confidence        0
negativereason                   5462
negativereason_confidence        4118
airline                             0
airline_sentiment_gold          14600
name                                0
negativereason_gold             14608
retweet_count                       0
text                                0
tweet_coord                     13621
tweet_created                       0
tweet_location                   4733
user_timezone                    4820
dtype: int64


# Check for duplicated rows in the DataFrame and sum them up
data.duplicated().sum()

36


# Remove duplicated rows, keeping the first occurrence of each duplicated row
data = data.drop_duplicates(keep='first')

# Reset the index after removing duplicates and dropping the original index
data = data.reset_index(drop=True)


# Check for duplicated rows in the DataFrame and sum them up
data.duplicated().sum()

0


# Display the unique values in the 'airline_sentiment' column
data['airline_sentiment'].unique()

array(['neutral', 'positive', 'negative'], dtype=object)


# function to create labeled barplots


def labeled_barplot(data, feature, perc=False, n=None):
    """
    Barplot with percentage at the top

    data: dataframe
    feature: dataframe column
    perc: whether to display percentages instead of count (default is False)
    n: displays the top n category levels (default is None, i.e., display all levels)
    """

    total = len(data[feature])  # length of the column
    count = data[feature].nunique()
    if n is None:
        plt.figure(figsize=(count + 1, 5))
    else:
        plt.figure(figsize=(n + 1, 5))

    plt.xticks(rotation=90, fontsize=15)
    ax = sns.countplot(
        data=data,
        x=feature,
        palette="Paired",
        order=data[feature].value_counts().index[:n].sort_values(),
    )

    for p in ax.patches:
        if perc == True:
            label = "{:.1f}%".format(
                100 * p.get_height() / total
            )  # percentage of each class of the category
        else:
            label = p.get_height()  # count of each level of the category

        x = p.get_x() + p.get_width() / 2  # width of the plot
        y = p.get_height()  # height of the plot

        ax.annotate(
            label,
            (x, y),
            ha="center",
            va="center",
            size=12,
            xytext=(0, 5),
            textcoords="offset points",
        )  # annotate the percentage

    plt.show()  # show the plot


labeled_barplot(data, "airline", perc=True)


labeled_barplot(data, "airline_sentiment", perc=True)


# Group the data by 'airline_sentiment' and 'airline', then count the occurrences
airline_sentiment = data.groupby(['airline_sentiment', 'airline']).airline_sentiment.count().unstack()

# Plot a bar chart based on the grouped data
airline_sentiment.plot(kind='bar')

<Axes: xlabel='airline_sentiment'>


labeled_barplot(data, "negativereason", perc=True)


def plotBoW(cleaned_words):
    """
    Plot a word cloud based on Bag of Words.

    Parameters:
    - cleaned_words (str): Text data containing cleaned words.

    Returns:
    None
    """

    # Create a WordCloud object
    wordcloud = WordCloud(
        stopwords=STOPWORDS,       # Set stopwords for the word cloud
        colormap='RdBu',           # Set the color map
        background_color='white',  # Set the background color
        width=3000,                 # Set the width of the word cloud
        height=2500                 # Set the height of the word cloud
    ).generate(cleaned_words)

    # Create a figure and display the word cloud
    plt.figure(1, figsize=(12, 12))
    plt.imshow(wordcloud)
    plt.axis('off')  # Turn off the axis
    plt.show()


# Filter the DataFrame to include only negative sentiment tweets
airline_tweets = data[data['airline_sentiment'] == 'negative']

# Combine all the text from the 'text' column into a single string
words = ' '.join(data['text'])

# Clean the text by removing URLs, mentions, and 'RT' (Retweet) tags
cleaned_word_negative = " ".join([word for word in words.split()
                                   if 'http' not in word
                                   and not word.startswith('@')
                                   and word != 'RT'])


# Plot a word cloud
plotBoW(cleaned_word_negative)


# Filter the DataFrame to include only positive sentiment tweets
airline_tweets = data[data['airline_sentiment'] == 'positive']

# Combine all the text from the 'text' column into a single string
words = ' '.join(data['text'])

# Clean the text by removing URLs, mentions, and 'RT' (Retweet) tags
cleaned_word_positive = " ".join([word for word in words.split()
                                   if 'http' not in word
                                   and not word.startswith('@')
                                   and word != 'RT'])


# Plot a word cloud
plotBoW(cleaned_word_positive)


# Select and keep only the 'text' and 'airline_sentiment' columns in the DataFrame
data = data[['text', 'airline_sentiment']]


# Define a function to strip HTML tags from text
def strip_html(text):
    """
    Strip HTML tags from the given text.

    Parameters:
    - text (str): Input text containing HTML tags.

    Returns:
    - str: Text with HTML tags removed.
    """
    soup = BeautifulSoup(text, "html.parser")
    return soup.get_text()

# Apply the strip_html function to the 'text' column of the DataFrame
data['text'] = data['text'].apply(strip_html)

# Display the head of the DataFrame after stripping HTML tags
data.head()

<ipython-input-25-d10b79162125>:12: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.
  soup = BeautifulSoup(text, "html.parser")


# Define a function to replace contractions in the text
def replace_contractions(text):
    """
    Replace contractions in the given text.

    Parameters:
    - text (str): Input text containing contractions.

    Returns:
    - str: Text with contractions replaced.
    """
    return contractions.fix(text)

# Apply the replace_contractions function to the 'text' column of the DataFrame
data['text'] = data['text'].apply(lambda x: replace_contractions(x))

# Display the head of the DataFrame after replacing contractions
data.head()


# Define a function to remove numbers from the text
def remove_numbers(text):
    """
    Remove numerical digits from the given text.

    Parameters:
    - text (str): Input text containing numerical digits.

    Returns:
    - str: Text with numerical digits removed.
    """
    text = re.sub(r'\d+', '', text)
    return text

# Apply the remove_numbers function to the 'text' column of the DataFrame
data['text'] = data['text'].apply(remove_numbers)

# Display the head of the DataFrame after removing numbers from the 'text' column
data.head()


# Define a function to remove mentions (words starting with '@') from the text
def remove_mentions(text):
    """
    Remove mentions (words starting with '@') from the given text.

    Parameters:
    - text (str): Input text containing mentions.

    Returns:
    - str: Text with mentions removed.
    """
    text = re.sub(r'@\w+', '', text)
    return text

# Apply the remove_mentions function to the 'text' column of the DataFrame
data['text'] = data['text'].apply(remove_mentions)

# Display the head of the DataFrame after removing mentions from the 'text' column
data.head()


# Apply nltk.word_tokenize to tokenize the 'text' column of the DataFrame
data['text'] = data.apply(lambda row: nltk.word_tokenize(row['text']), axis=1)

# Display the head of the DataFrame after tokenizing the 'text' column
data.head()


# Get the list of English stopwords from nltk
stopwords = stopwords.words('english')

# Define a function to remove stopwords from a list of tokens
def remove_stopwords(tokens):
    """
    Remove stopwords from the given list of tokens.

    Parameters:
    - tokens (list): List of tokens.

    Returns:
    - list: List of tokens with stopwords removed.
    """
    return [token for token in tokens if token not in stopwords]

# Apply the remove_stopwords function to the 'text' column of the DataFrame
data['text'] = data['text'].apply(remove_stopwords)

# Display the head of the DataFrame after removing stopwords from the 'text' column
data.head()


# Define a function to remove emojis or special characters from a list of tokens
def remove_special_characters(tokens):
    # Define a regular expression pattern to match emojis or special characters
    pattern = r'[^\w\s]|\d'

    # Create a RegexpTokenizer with the defined pattern
    tokenizer = RegexpTokenizer(pattern)

    # Tokenize the list of tokens and filter out tokens that match the pattern
    filtered_tokens = [token for token in tokens if not tokenizer.tokenize(token)]

    # Return the filtered tokens
    return filtered_tokens


# Apply the remove_special_characters function to the 'text' column of the DataFrame
data['text'] = data['text'].apply(remove_special_characters)

# Display the head of the DataFrame after removing emojis or special characters from the 'text' column
data.head()


# Define a function to remove emojis or special characters from a list of tokens
def to_lowercase(tokens):
  return [token.lower() for token in tokens]

# Apply the remove_special_characters function to the 'text' column of the DataFrame
data['text'] = data['text'].apply(to_lowercase)

# Display the head of the DataFrame after removing emojis or special characters from the 'text' column
data.head()


# Create a WordNet Lemmatizer
lemmatizer = WordNetLemmatizer()

# Define a function to lemmatize a list of words
def lemmatize_list(words):
    """
    Lemmatize a list of words using WordNet Lemmatizer.

    Parameters:
    - words (list): List of words.

    Returns:
    - str: Lemmatized words joined into a string.
    """
    new_words = [lemmatizer.lemmatize(word, pos='v') for word in words]
    return ' '.join(new_words)

# Apply the lemmatize_list function to the 'text' column of the DataFrame
data['text'] = data['text'].apply(lemmatize_list)

# Display the head of the DataFrame after lemmatizing the 'text' column
data.head()


def metrics_score(actual, predicted):
    """
    Display a confusion matrix as a heatmap and provide metrics scores.

    Parameters:
    - actual (array-like): True labels.
    - predicted (array-like): Predicted labels.

    Returns:
    None
    """
    # Calculate confusion matrix
    cm = confusion_matrix(actual, predicted)

    # Plot the confusion matrix as a heatmap
    plt.figure(figsize=(8, 5))
    sns.heatmap(cm, annot=True, fmt='.2f', xticklabels=['negative', 'positive', 'neutral'], yticklabels=['negative', 'positive', 'neutral'])

    # Set labels and display the plot
    plt.ylabel('Actual')
    plt.xlabel('Predicted')
    plt.show()


# Create a CountVectorizer with a maximum of 1000 features
Count_vec = CountVectorizer(max_features=1000)

# Transform the 'text' column using CountVectorizer
data_features = Count_vec.fit_transform(data['text'])

# Convert the sparse matrix to a dense array
data_features = data_features.toarray()

# Display the shape of the resulting array
data_features.shape

(14604, 1000)


# Features (X) are the transformed data from CountVectorizer
X = data_features

# Target variable (y) is the 'airline_sentiment' column
y = data['airline_sentiment']


# Split the data into training and testing sets
# - X_train and y_train are the features and labels for training
# - X_test and y_test are the features and labels for testing
# - test_size=0.25 specifies that 25% of the data will be used for testing
# - shuffle=False ensures that the data is not shuffled before splitting
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, shuffle=False)


# Create a RandomForestClassifier with 100 trees and 4 jobs (parallel processing)
cv_forest = RandomForestClassifier(n_estimators=100, n_jobs=4)

# Fit the RandomForestClassifier to the training data
cv_forest.fit(X_train, y_train)

# Display the details of the trained RandomForestClassifier
print(cv_forest)

# Perform cross-validation on the RandomForestClassifier and print the mean accuracy
cross_val_accuracy = np.mean(cross_val_score(cv_forest, X_train, y_train, cv=5, scoring='accuracy'))
print(f"Cross-Validation Mean Accuracy: {cross_val_accuracy}")

RandomForestClassifier(n_jobs=4)
Cross-Validation Mean Accuracy: 0.735417450800181


# Initialize an empty list to store cross-validation scores
cv_scores = []

# Define a range of values for the number of estimators (trees)
n_estimators = list(range(100, 401, 50))

# Loop through each value of the number of estimators
for estimator in n_estimators:
    # Create a RandomForestClassifier with the current number of estimators
    cv_forest = RandomForestClassifier(n_estimators=estimator, n_jobs=4)

    # Perform cross-validation and append the mean accuracy to cv_scores
    scores = cross_val_score(cv_forest, X_train, y_train, cv=5, scoring='accuracy')
    cv_scores.append(scores.mean())

    # Print the number of estimators and the corresponding mean accuracy
    print(f"Estimator:{estimator}, Mean score: {scores.mean()}")

Estimator:100, Mean score: 0.7365125909438571
Estimator:150, Mean score: 0.7357823724701925
Estimator:200, Mean score: 0.7356000158389759
Estimator:250, Mean score: 0.736330025905062
Estimator:300, Mean score: 0.7365128827144671
Estimator:350, Mean score: 0.7347780980307569
Estimator:400, Mean score: 0.7368779294290257


# Calculate the error (misclassification error) for each number of estimators
error = [1 - x for x in cv_scores]

# Find the optimal number of estimators with the minimum error
optimal_learners = n_estimators[error.index(min(error))]

# Plot the misclassification error over the range of number of estimators
plt.plot(n_estimators, error)

# Annotate the plot with the optimal number of estimators and the corresponding error
xy = (optimal_learners, min(error))
plt.annotate('(%s, %s)' % xy, xy=xy, textcoords='data')

# Set labels for the axes
plt.xlabel("Number of base learners")
plt.ylabel("Misclassification Error")

# Display the plot
plt.show()


# Create a RandomForestClassifier with the optimal number of estimators
classifier_cv = RandomForestClassifier(n_estimators=400, n_jobs=4)

# Fit the RandomForestClassifier to the training data
classifier_cv.fit(X_train, y_train)

# Predict the labels for the test set
y_pred_cv = classifier_cv.predict(X_test)

# Display the predicted labels
y_pred_cv

array(['negative', 'negative', 'negative', ..., 'negative', 'negative',
       'negative'], dtype=object)


metrics_score(y_test, y_pred_cv)


# Generate a classification report for the predicted labels
cr1 = {
    'name':'countvectorizer',
    'report':classification_report(y_test, y_pred_cv)
  }
# Print the classification report
print(cr1['report'])

              precision    recall  f1-score   support

    negative       0.86      0.88      0.87      2627
     neutral       0.50      0.47      0.48       574
    positive       0.64      0.63      0.63       450

    accuracy                           0.78      3651
   macro avg       0.67      0.66      0.66      3651
weighted avg       0.78      0.78      0.78      3651


# Get the names of all features from the CountVectorizer
all_features = Count_vec.get_feature_names_out()

# Initialize a string to store the top features
top_features = ''

# Get the feature importances from the trained classifier
feat = classifier_cv.feature_importances_

# Sort the features based on their importance in descending order
features = np.argsort(feat)[::-1]

# Concatenate the names of the top 20 features
for i in features[0:20]:
    top_features += all_features[i]
    top_features += ' '

# Plot a Word Cloud of the top features
plotBoW(top_features)


# Create a TfidfVectorizer with a maximum of 2000 features
vectorizer = TfidfVectorizer(max_features=2000)

# Transform the 'text' column using TfidfVectorizer
data_features = vectorizer.fit_transform(data['text'])

# Convert the sparse matrix to a dense array
data_features = data_features.toarray()

# Display the shape of the resulting array
data_features.shape

(14604, 2000)


# Features (X) are the transformed data from TfidfVectorizer
X = data_features

# Target variable (y) is the 'airline_sentiment' column
y = data['airline_sentiment']


# Split the data into training and testing sets
# - X_train and y_train are the features and labels for training
# - X_test and y_test are the features and labels for testing
# - test_size=0.25 specifies that 25% of the data will be used for testing
# - random_state=42 ensures reproducibility by fixing the random seed
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)


# Create a RandomForestClassifier with 100 trees and 4 jobs (parallel processing)
tfidf_forest = RandomForestClassifier(n_estimators=100, n_jobs=4)

# Fit the RandomForestClassifier to the training data
tfidf_forest.fit(X_train, y_train)

# Display the details of the trained RandomForestClassifier
print(tfidf_forest)

# Perform cross-validation on the RandomForestClassifier and print the mean accuracy
cross_val_accuracy_tfidf = np.mean(cross_val_score(tfidf_forest, X_train, y_train, cv=5, scoring='accuracy'))
print(f"Cross-Validation Mean Accuracy: {cross_val_accuracy_tfidf}")

RandomForestClassifier(n_jobs=4)
Cross-Validation Mean Accuracy: 0.7528537041320971


# Initialize an empty list to store cross-validation scores for TF-IDF features
tfidf_scores = []

# Define a range of values for the number of estimators (trees)
n_estimators = list(range(20, 101, 20))

# Loop through each value of the number of estimators
for estimator in n_estimators:
    # Create a RandomForestClassifier with the current number of estimators
    tfidf_forest = RandomForestClassifier(n_estimators=estimator, n_jobs=4)

    # Perform cross-validation and append the mean accuracy to tfidf_scores
    scores = cross_val_score(tfidf_forest, X_train, y_train, cv=5, scoring='accuracy')
    tfidf_scores.append(scores.mean())

    # Print the number of estimators and the corresponding mean accuracy
    print(f"Estimator: {estimator}, Mean score: {scores.mean()}")

Estimator: 20, Mean score: 0.7452756711245048
Estimator: 40, Mean score: 0.7502059275283487
Estimator: 60, Mean score: 0.7498411725843999
Estimator: 80, Mean score: 0.75084511357171
Estimator: 100, Mean score: 0.7528542043102855


# Calculate the error (misclassification error) for each number of estimators for TF-IDF features
error = [1 - x for x in tfidf_scores]

# Find the optimal number of estimators with the minimum error
optimal_learners_tfidf = n_estimators[error.index(min(error))]

# Plot the misclassification error over the range of number of estimators
plt.plot(n_estimators, error)

# Annotate the plot with the optimal number of estimators and the corresponding error
xy = (optimal_learners_tfidf, min(error))
plt.annotate('(%s, %s)' % xy, xy=xy, textcoords='data')

# Set labels for the axes
plt.xlabel("Number of base learners")
plt.ylabel("Misclassification Error")

# Display the plot
plt.show()


# Create a RandomForestClassifier with the optimal number of estimators for TF-IDF features
classifier_tfidf = RandomForestClassifier(n_estimators=100, n_jobs=4)

# Fit the RandomForestClassifier to the training data for TF-IDF features
classifier_tfidf.fit(X_train, y_train)

# Predict the labels for the test set using TF-IDF features
y_pred_tfidf = classifier_tfidf.predict(X_test)

# Display the predicted labels for TF-IDF features
y_pred_tfidf

array(['neutral', 'negative', 'negative', ..., 'negative', 'negative',
       'negative'], dtype=object)


metrics_score(y_test, y_pred_tfidf)


# Generate a classification report for the predicted labels using TF-IDF features
cr2 = {
    'name': 'TF-IDF',
    'report':classification_report(y_test, y_pred_tfidf)
  }
# Print the classification report for TF-IDF features
print(cr2['report'])

              precision    recall  f1-score   support

    negative       0.81      0.92      0.86      2330
     neutral       0.62      0.46      0.53       741
    positive       0.73      0.59      0.65       580

    accuracy                           0.77      3651
   macro avg       0.72      0.65      0.68      3651
weighted avg       0.76      0.77      0.76      3651


# Get the names of all features from the TfidfVectorizer
all_features = vectorizer.get_feature_names_out()

# Initialize a string to store the top features for TF-IDF
top_features = ''

# Get the feature importances from the trained classifier for TF-IDF
feat = classifier_tfidf.feature_importances_

# Sort the features based on their importance in descending order for TF-IDF
features = np.argsort(feat)[::-1]

# Concatenate the names of the top 20 features for TF-IDF
for i in features[0:20]:
    top_features += all_features[i]
    top_features += ' '

# Plot a Word Cloud of the top features for TF-IDF
plotBoW(top_features)


def graphs_and_report(history, predicted):
    # Plot training & validation loss values
    plt.plot(history.history['loss'])
    plt.plot(history.history['val_loss'])
    plt.title('Model loss')
    plt.ylabel('Loss')
    plt.xlabel('Epoch')
    plt.legend(['Train', 'Test'], loc='upper left')
    plt.show()

    # Plot training & validation accuracy values
    plt.plot(history.history['accuracy'])
    plt.plot(history.history['val_accuracy'])
    plt.title('Model accuracy')
    plt.ylabel('Accuracy')
    plt.xlabel('Epoch')
    plt.legend(['Train', 'Test'], loc='upper left')
    plt.show()


# Create a Tokenizer with a maximum vocabulary size of 800 words
tokenizer = Tokenizer(num_words=800, split=' ')

# Fit the Tokenizer on the 'text' column of the data
tokenizer.fit_on_texts(data['text'])


# Convert the text data to sequences using the trained Tokenizer
X = tokenizer.texts_to_sequences(data['text'])

# Pad the sequences to ensure uniform length
X = pad_sequences(X)

print(X)

[[  0   0   0 ...   0  95  34]
 [  0   0   0 ... 420 211 107]
 [  0   0   0 ...  33  88 111]
 ...
 [  0   0   0 ... 365 354 167]
 [  0   0   0 ... 232   1  20]
 [  0   0   0 ...  68 109   2]]


# Create a LabelBinarizer to encode the categorical target variable
enc = LabelBinarizer()

# Fit the LabelBinarizer on the 'airline_sentiment' column and transform it to binary labels
y_encoded = enc.fit_transform(data['airline_sentiment'])

# Split the data into training and testing sets
# - X_train and y_train are the features and labels for training
# - X_test and y_test are the features and labels for testing
# - test_size=0.30 specifies that 30% of the data will be used for testing
# - random_state=42 ensures reproducibility by fixing the random seed
X_train, X_test, y_train, y_test = train_test_split(X, y_encoded, test_size=0.30, random_state=42)


# Create a Sequential model
model = Sequential()

# Add an Embedding layer with vocabulary size 800, embedding dimension 120, and input length determined by X.shape[1]
model.add(Embedding(800, 120, input_length=X.shape[1]))

# Add a SpatialDropout1D layer with dropout rate 0.2
model.add(SpatialDropout1D(0.2))

# Add an LSTM layer with 256 units, dropout rate 0.2, and recurrent dropout rate 0.2
model.add(LSTM(256, dropout=0.2, recurrent_dropout=0.2))

# Add a Dense layer with 124 units and ReLU activation function
model.add(Dense(124, activation='relu'))

# Add a Dropout layer with dropout rate 0.2
model.add(Dropout(0.2))

# Add another Dense layer with 64 units and ReLU activation function
model.add(Dense(64, activation='relu'))

# Add the output Dense layer with 3 units and softmax activation function for multi-class classification
model.add(Dense(3, activation='softmax'))

# Compile the model using categorical crossentropy loss, Adam optimizer, and accuracy as the metric
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

# Display a summary of the model architecture
model.summary()

Model: "sequential"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding (Embedding)       (None, 24, 120)           96000     
                                                                 
 spatial_dropout1d (Spatial  (None, 24, 120)           0         
 Dropout1D)                                                      
                                                                 
 lstm (LSTM)                 (None, 256)               386048    
                                                                 
 dense (Dense)               (None, 124)               31868     
                                                                 
 dropout (Dropout)           (None, 124)               0         
                                                                 
 dense_1 (Dense)             (None, 64)                8000      
                                                                 
 dense_2 (Dense)             (None, 3)                 195       
                                                                 
=================================================================
Total params: 522111 (1.99 MB)
Trainable params: 522111 (1.99 MB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________


# Train the model using the training data
history = model.fit(X_train, y_train, epochs=10, batch_size=32, verbose='auto')

Epoch 1/10
320/320 [==============================] - 82s 242ms/step - loss: 0.7061 - accuracy: 0.7014
Epoch 2/10
320/320 [==============================] - 80s 250ms/step - loss: 0.5572 - accuracy: 0.7777
Epoch 3/10
320/320 [==============================] - 77s 239ms/step - loss: 0.5107 - accuracy: 0.7952
Epoch 4/10
320/320 [==============================] - 76s 237ms/step - loss: 0.4898 - accuracy: 0.8026
Epoch 5/10
320/320 [==============================] - 89s 280ms/step - loss: 0.4639 - accuracy: 0.8173
Epoch 6/10
320/320 [==============================] - 75s 236ms/step - loss: 0.4374 - accuracy: 0.8277
Epoch 7/10
320/320 [==============================] - 75s 236ms/step - loss: 0.4126 - accuracy: 0.8324
Epoch 8/10
320/320 [==============================] - 77s 241ms/step - loss: 0.3957 - accuracy: 0.8367
Epoch 9/10
320/320 [==============================] - 77s 240ms/step - loss: 0.3738 - accuracy: 0.8465
Epoch 10/10
320/320 [==============================] - 76s 237ms/step - loss: 0.3545 - accuracy: 0.8530


# Make predictions on the test set using the trained LSTM model
y_pred_lstm = model.predict(X_test)

# Extract the class indices with the highest probability for each sample
y_pred_arg = np.argmax(y_pred_lstm, axis=1)

# Extract the class indices from the one-hot encoded true labels for each sample
y_test_arg = np.argmax(y_test, axis=1)

# Print the predicted class indices
print("Predicted Class Indices:")
print(y_pred_arg)

# Print the true class indices
print("\nTrue Class Indices:")
print(y_test_arg)

137/137 [==============================] - 7s 48ms/step
Predicted Class Indices:
[1 0 0 ... 0 0 0]

True Class Indices:
[1 0 0 ... 0 0 1]


# Generate a classification report for the predicted class indices and true class indices
cr3 = {
    'name':'LSTM',
    'report':classification_report(y_test_arg, y_pred_arg)
  }
# Print the classification report
print(cr3['report'])

              precision    recall  f1-score   support

           0       0.85      0.86      0.85      2796
           1       0.58      0.54      0.56       885
           2       0.68      0.69      0.68       701

    accuracy                           0.77      4382
   macro avg       0.70      0.70      0.70      4382
weighted avg       0.77      0.77      0.77      4382


# Loop over the models (assumes you have cr1, cr2, cr3 defined)
for i in range(1, 4):

    # Use eval to retrieve the classification report for the current model
    cr = eval(f'cr{i}')

    print(f"Model {i} - {cr['name']}")

    # Print the classification report for the current model
    print(cr['report'])

Model 1 - countvectorizer
              precision    recall  f1-score   support

    negative       0.86      0.88      0.87      2627
     neutral       0.50      0.47      0.48       574
    positive       0.64      0.63      0.63       450

    accuracy                           0.78      3651
   macro avg       0.67      0.66      0.66      3651
weighted avg       0.78      0.78      0.78      3651

Model 2 - TF-IDF
              precision    recall  f1-score   support

    negative       0.81      0.92      0.86      2330
     neutral       0.62      0.46      0.53       741
    positive       0.73      0.59      0.65       580

    accuracy                           0.77      3651
   macro avg       0.72      0.65      0.68      3651
weighted avg       0.76      0.77      0.76      3651

Model 3 - LSTM
              precision    recall  f1-score   support

           0       0.85      0.86      0.85      2796
           1       0.58      0.54      0.56       885
           2       0.68      0.69      0.68       701

    accuracy                           0.77      4382
   macro avg       0.70      0.70      0.70      4382
weighted avg       0.77      0.77      0.77      4382

	tweet_id	airline_sentiment	airline_sentiment_confidence	negativereason	negativereason_confidence	airline	airline_sentiment_gold	name	negativereason_gold	text	tweet_coord	tweet_created	tweet_location	user_timezone
0	570306133677760513	neutral	1.0000	NaN	NaN	Virgin America	NaN	cairdin	NaN	@VirginAmerica What @dhepburn said.	NaN	2015-02-24 11:35:52 -0800	NaN	Eastern Time (US & Canada)
1	570301130888122368	positive	0.3486	NaN	0.0000	Virgin America	NaN	jnardino	NaN	@VirginAmerica plus you've added commercials t...	NaN	2015-02-24 11:15:59 -0800	NaN	Pacific Time (US & Canada)
2	570301083672813571	neutral	0.6837	NaN	NaN	Virgin America	NaN	yvonnalynn	NaN	@VirginAmerica I didn't today... Must mean I n...	NaN	2015-02-24 11:15:48 -0800	Lets Play	Central Time (US & Canada)
3	570301031407624196	negative	1.0000	Bad Flight	0.7033	Virgin America	NaN	jnardino	NaN	@VirginAmerica it's really aggressive to blast...	NaN	2015-02-24 11:15:36 -0800	NaN	Pacific Time (US & Canada)
4	570300817074462722	negative	1.0000	Can't Tell	1.0000	Virgin America	NaN	jnardino	NaN	@VirginAmerica and it's a really big bad thing...	NaN	2015-02-24 11:14:45 -0800	NaN	Pacific Time (US & Canada)

	tweet_id	airline_sentiment	airline_sentiment_confidence	negativereason	negativereason_confidence	airline	airline_sentiment_gold	name	negativereason_gold	text	tweet_coord	tweet_created	tweet_location	user_timezone
14635	569587686496825344	positive	0.3487	NaN	0.0000	American	NaN	KristenReenders	NaN	@AmericanAir thank you we got on a different f...	NaN	2015-02-22 12:01:01 -0800	NaN	NaN
14636	569587371693355008	negative	1.0000	Customer Service Issue	1.0000	American	NaN	itsropes	NaN	@AmericanAir leaving over 20 minutes Late Flig...	NaN	2015-02-22 11:59:46 -0800	Texas	NaN
14637	569587242672398336	neutral	1.0000	NaN	NaN	American	NaN	sanyabun	NaN	@AmericanAir Please bring American Airlines to...	NaN	2015-02-22 11:59:15 -0800	Nigeria,lagos	NaN
14638	569587188687634433	negative	1.0000	Customer Service Issue	0.6659	American	NaN	SraJackson	NaN	@AmericanAir you have my money, you change my ...	NaN	2015-02-22 11:59:02 -0800	New Jersey	Eastern Time (US & Canada)
14639	569587140490866689	neutral	0.6771	NaN	0.0000	American	NaN	daviddtwu	NaN	@AmericanAir we have 8 ppl so we need 2 know h...	NaN	2015-02-22 11:58:51 -0800	dallas, TX	NaN

🤖 Proyectos de IA

📍 Navegación

Introduction to Computer Vision: Twitter US Airline Sentiment¶

Problem Statement¶

Context¶

Objective¶

Data Dictionary¶

Importing necessary libraries¶

Loading the dataset¶

Data Overview¶

Observations¶

Observations¶

Exploratory Data Analysis¶

Observations¶

Observations¶

Data Preparation for Modeling¶

Model Building¶

Using countvectorizer¶

Observations¶

Observations¶

Using TF-IDF (Term Frequency- Inverse Document Frequency)¶

Observations¶

Observations¶

Using LSTM¶

Observations¶

Summary¶

Summary¶

Decision¶

Actionable Insights and Recommendations¶

Happy Learning!¶

	text	airline_sentiment
0	@VirginAmerica What @dhepburn said.	neutral
1	@VirginAmerica plus you have added commercials...	positive
2	@VirginAmerica I did not today... Must mean I ...	neutral
3	@VirginAmerica it is really aggressive to blas...	negative
4	@VirginAmerica and it is a really big bad thin...	negative

	text	airline_sentiment
0	What said.	neutral
1	plus you have added commercials to the experi...	positive
2	I did not today... Must mean I need to take a...	neutral
3	it is really aggressive to blast obnoxious "e...	negative
4	and it is a really big bad thing about it	negative

	text	airline_sentiment
0	[What, said, .]	neutral
1	[plus, you, have, added, commercials, to, the,...	positive
2	[I, did, not, today, ..., Must, mean, I, need,...	neutral
3	[it, is, really, aggressive, to, blast, obnoxi...	negative
4	[and, it, is, a, really, big, bad, thing, abou...	negative

	text	airline_sentiment
0	[What, said, .]	neutral
1	[plus, added, commercials, experience, ..., ta...	positive
2	[I, today, ..., Must, mean, I, need, take, ano...	neutral
3	[really, aggressive, blast, obnoxious, ``, ent...	negative
4	[really, big, bad, thing]	negative

	text	airline_sentiment
0	[What, said]	neutral
1	[plus, added, commercials, experience, tacky]	positive
2	[I, today, Must, mean, I, need, take, another,...	neutral
3	[really, aggressive, blast, obnoxious, enterta...	negative
4	[really, big, bad, thing]	negative

	text	airline_sentiment
0	what say	neutral
1	plus add commercials experience tacky	positive
2	i today must mean i need take another trip	neutral
3	really aggressive blast obnoxious entertainmen...	negative
4	really big bad thing	negative