29  Imbalanced Data (Titanic Dataset)

When working with imbalanced data, we have a couple of options available to us.

For example, we can - create additional synthetic data - use features built in to models to affect how much weight it gives samples from the minority class

Let’s import our titanic dataset as before.

from sklearn.model_selection import train_test_split
import pandas as pd
import numpy as np
from sklearn.metrics import roc_auc_score, precision_score, recall_score, f1_score, \
                            confusion_matrix, ConfusionMatrixDisplay
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import NearestNeighbors
from imblearn.over_sampling import SMOTE
from sklearn.preprocessing import StandardScaler
from xgboost import XGBClassifier
try:
    data = pd.read_csv("data/processed_data.csv")

except FileNotFoundError:
    # Download processed data:
    address = 'https://raw.githubusercontent.com/MichaelAllen1966/' + \
                '1804_python_healthcare/master/titanic/data/processed_data.csv'

    data = pd.read_csv(address)

    # Create a data subfolder if one does not already exist
    import os
    data_directory ='./data/'
    if not os.path.exists(data_directory):
        os.makedirs(data_directory)

    # Save data
    data.to_csv(data_directory + 'processed_data.csv', index=False)

data = data.astype(float)

# Drop Passengerid (axis=1 indicates we are removing a column rather than a row)
# We drop passenger ID as it is not original data

data.drop('PassengerId', inplace=True, axis=1)

X = data.drop('Survived',axis=1) # X = all 'data' except the 'survived' column
y = data['Survived'] # y = 'survived' column from 'data'

feature_names = X.columns.tolist()

X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
X_train, X_validate, y_train, y_validate = train_test_split(X_train_val, y_train_val,
                                                            test_size=0.2, random_state=42)

print(f"Training Dataset Samples: {len(X_train)}")
print(f"Validation Dataset Samples: {len(X_validate)}")
print(f"Testing Dataset Samples: {len(X_test)}")
Training Dataset Samples: 569
Validation Dataset Samples: 143
Testing Dataset Samples: 179
# Let's also check that the class splits are as expected
number_positive_class = np.sum(data['Survived'] == 1)
number_negative_class = np.sum(data['Survived'] == 0)

print (f"Positives : {number_positive_class}")
print (f"Negatives : {number_negative_class}")
Positives : 342
Negatives : 549

We’ll also create a function for comparing our outputs.

def fit_train(name="Logistic Regression",
              X_train=X_train, X_validate=X_validate,
              y_train=y_train, y_validate=y_validate,
              model=LogisticRegression()
              ):

     model.fit(X_train, y_train)

     y_pred_train = model.predict(X_train)
     y_pred_val = model.predict(X_validate)

     tn, fp, fn, tp = confusion_matrix(y_validate, y_pred_val, labels=[0, 1]).ravel()

     return pd.DataFrame({
            'Accuracy (training)': np.mean(y_pred_train == y_train),
            'Accuracy (validation)': np.mean(y_pred_val == y_validate),
            'Precision (validation)': precision_score(y_validate, y_pred_val, average='macro'),
            'Recall (validation)': recall_score(y_validate, y_pred_val, average='macro'),
            "AUC": roc_auc_score(y_validate, y_pred_val),
            "f1": f1_score(y_validate, y_pred_val, average='macro'),
            "FP": fp,
            "FN": fn
          }, index=[name]
).round(3)

29.1 In-model options

Certain models have options to allow us to account for imbalanced data automatically.

results_df = fit_train("Standard", model=XGBClassifier())
results_df
Accuracy (training) Accuracy (validation) Precision (validation) Recall (validation) AUC f1 FP FN
Standard 0.979 0.797 0.788 0.786 0.786 0.787 14 15 
model = XGBClassifier(
    random_state=42,
    scale_pos_weight=number_negative_class/number_positive_class
    )
results_df = pd.concat([results_df,
    fit_train("With Imbalanced Data Parameter", model=model)]
)
results_df
Accuracy (training) Accuracy (validation) Precision (validation) Recall (validation) AUC f1 FP FN
Standard 0.979 0.797 0.788 0.786 0.786 0.787 14 15
With Imbalanced Data Parameter 0.988 0.811 0.802 0.800 0.800 0.801 13 14

29.1.1 Logistic Regression

def standardise_data(X_train, X_test):

    # Initialise a new scaling object for normalising input data
    sc = StandardScaler()

    # Set up the scaler just on the training set
    sc.fit(X_train)

    # Apply the scaler to the training and test sets
    train_std=sc.transform(X_train)
    test_std=sc.transform(X_test)

    return train_std, test_std
X_train_std, X_test_std = standardise_data(X_train, X_test)
results_df = fit_train("Logistic Regression", model=LogisticRegression(),
                       X_train=X_train_std,
                       X_validate=X_test_std,
                       y_train=y_train,
                       y_validate=y_test)
results_df
Accuracy (training) Accuracy (validation) Precision (validation) Recall (validation) AUC f1 FP FN
Logistic Regression 0.8 0.832 0.827 0.827 0.827 0.827 15 15

According to the documentation, “The “balanced” mode uses the values of y to automatically adjust weights inversely proportional to class frequencies in the input data as n_samples / (n_classes * np.bincount(y)).”

What will this be in our case?

len(X_train_std) / (2 * np.bincount(y_train))
array([0.79691877, 1.34198113])
model_lr = LogisticRegression(
    class_weight="balanced"
)

results_df = pd.concat([results_df,
    fit_train("With Imbalanced Data Parameter", model=model_lr,
                       X_train=X_train_std,
                       X_validate=X_test_std,
                       y_train=y_train,
                       y_validate=y_test)

    ]
)
results_df
Accuracy (training) Accuracy (validation) Precision (validation) Recall (validation) AUC f1 FP FN
Logistic Regression 0.800 0.832 0.827 0.827 0.827 0.827 15 15
With Imbalanced Data Parameter 0.803 0.804 0.800 0.807 0.807 0.801 22 13

In our case, this doesn’t seem to have helped - but it’s good to know and interesting to see the strong impact on false positives and false negatives. If we were interested in maximising some aspect of our model, the slight loss in overall performance may feel worthwhile to us.

29.1.2 Upsampling using synthetic data generators like SMOTE

def make_synthetic_data_smote(X, y, number_of_samples=[1000,1000]):
    """
    Synthetic data generation for two classes.

    Inputs
    ------
    original_data: X, y numpy arrays (y should have label 0 and 1)
    number_of_samples: number of samples to generate (list for y=0, y=1)
    (Note - number_of_samples has default of 1000 samples for each class
    if no numbers are specified at the point of calling the function)

    Returns
    -------
    X_synthetic: NumPy array
    y_synthetic: NumPy array

    """

    # Count instances in each class
    count_label_0 = np.sum(y==0)
    count_label_1 = np.sum(y==1)

    # SMOTE requires final class counts; add current counts to required counts
    # (which are passed into the function)
    n_class_0 = number_of_samples[0] + count_label_0
    n_class_1 = number_of_samples[1] + count_label_1

    # Use SMOTE to sample data points.  The number of points that we pass over
    # to SMOTE is calculated above (the number of synthetic data samples we
    # want, which we passed into the function + the counts from the original
    # data).  This tells SMOTE how many TOTAL data points are needed (original
    # + synthetic) for each class.  It then uses the original data to generate
    # new synthetic data points.
    # For example, imagine our original data has 100 samples for class 0 and 50
    # for class 1, and we tell SMOTE we want 100 synthetic data points for
    # class 0 and 150 synthetic data points for class 1.  We tell SMOTE that we
    # need a total of 200 data points for class 0 (100 original + 100 synthetic)
    # and 200 data points for class 1 (50 original + 150 synthetic).  It will
    # then fill those data points by taking the original data (which will fill
    # up the first 100 "slots" for class 0, and the first 50 "slots" for class 1)
    # and then use these original data points to sample new synthetic data points
    # to fill the remaining "slots" in each class.
    X_resampled, y_resampled = SMOTE(
        sampling_strategy = {0:n_class_0, 1:n_class_1}).fit_resample(X, y)

    # Get just the additional (synthetic) data points.  By using len(X) for the
    # X (input feature) data, and len(y) for the y (output label) data, we skip
    # the original data, and just start from the newly created synthetic data,
    # generated by SMOTE (above)
    X_synthetic = X_resampled[len(X):]
    y_synthetic = y_resampled[len(y):]

    return X_synthetic, y_synthetic
X.head()
Pclass Age SibSp Parch Fare AgeImputed EmbarkedImputed CabinLetterImputed CabinNumber CabinNumberImputed ... Embarked_missing CabinLetter_A CabinLetter_B CabinLetter_C CabinLetter_D CabinLetter_E CabinLetter_F CabinLetter_G CabinLetter_T CabinLetter_missing
0 3.0 22.0 1.0 0.0 7.2500 0.0 0.0 1.0 0.0 1.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0
1 1.0 38.0 1.0 0.0 71.2833 0.0 0.0 0.0 85.0 0.0 ... 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0
2 3.0 26.0 0.0 0.0 7.9250 0.0 0.0 1.0 0.0 1.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0
3 1.0 35.0 1.0 0.0 53.1000 0.0 0.0 0.0 123.0 0.0 ... 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0
4 3.0 35.0 0.0 0.0 8.0500 0.0 0.0 1.0 0.0 1.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0

5 rows × 24 columns

# Get full list of column names (the names of our features)
X_col_names = X.columns.tolist()

# Set categorical one-hots cols using common prefix
categorical = ['CabinLetter_', 'Embarked_']

one_hot_cols = []
for col in categorical:
    one_hot_cols.append([x for x in X_col_names if x[0:len(col)] == col])

# Set integer columns
integer_cols = ['Age', 'Pclass']

# Don't need to explicitly set float cols

# Set binary columns
binary_cols = ['SibSp', 'Parch', 'AgeImputed', 'EmbarkedImputed']
# Generate synthetic data again, but this time with 250 extra synthetic data
# points for the positive class (double what we need), and 0 for the negative
# class
X_synthetic, y_synthetic = make_synthetic_data_smote(
    X_train, y_train, number_of_samples=[0, 350]
)

# Set y_label
y_label = "Survived"

# Create a data frame with id to store the synthetic data
synth_df = pd.DataFrame()

# Transfer X values to the new DataFrame
synth_df=pd.concat([synth_df,
                    pd.DataFrame(X_synthetic, columns=X.columns.to_list())],
                    axis=1)
synth_df
Pclass Age SibSp Parch Fare AgeImputed EmbarkedImputed CabinLetterImputed CabinNumber CabinNumberImputed ... Embarked_missing CabinLetter_A CabinLetter_B CabinLetter_C CabinLetter_D CabinLetter_E CabinLetter_F CabinLetter_G CabinLetter_T CabinLetter_missing
569 1.000000 34.674002 1.000000 0.000000 90.000000 0.0 0.0 0.0 90.555015 0.0 ... 0.0 0.0 0.000000 1.000000 0.0 0.000000 0.0 0.0 0.0 0.0
570 1.000000 35.938509 0.000000 0.000000 223.817179 0.0 0.0 0.0 35.837817 0.0 ... 0.0 0.0 0.229055 0.770945 0.0 0.000000 0.0 0.0 0.0 0.0
571 1.000000 55.300072 0.000000 0.850004 153.462500 0.0 0.0 0.0 125.000000 0.0 ... 0.0 0.0 0.000000 1.000000 0.0 0.000000 0.0 0.0 0.0 0.0
572 1.000000 22.943377 0.000000 0.528311 61.745149 0.0 0.0 0.0 33.943377 0.0 ... 0.0 0.0 0.471689 0.000000 0.0 0.528311 0.0 0.0 0.0 0.0
573 1.025236 27.050471 0.025236 0.000000 30.386439 0.0 0.0 1.0 0.000000 1.0 ... 0.0 0.0 0.000000 0.000000 0.0 0.000000 0.0 0.0 0.0 1.0
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
914 2.000000 25.995775 1.000000 0.751056 29.004225 0.0 0.0 1.0 0.000000 1.0 ... 0.0 0.0 0.000000 0.000000 0.0 0.000000 0.0 0.0 0.0 1.0
915 2.945161 62.945161 0.000000 0.000000 9.637540 0.0 0.0 1.0 0.000000 1.0 ... 0.0 0.0 0.000000 0.000000 0.0 0.000000 0.0 0.0 0.0 1.0
916 3.000000 28.000000 0.000000 0.000000 7.750000 1.0 0.0 1.0 0.000000 1.0 ... 0.0 0.0 0.000000 0.000000 0.0 0.000000 0.0 0.0 0.0 1.0
917 1.773067 33.067333 0.226933 0.000000 22.100026 0.0 0.0 0.0 105.992533 0.0 ... 0.0 0.0 0.000000 0.226933 0.0 0.773067 0.0 0.0 0.0 0.0
918 3.000000 15.809182 0.000000 0.000000 7.892575 0.0 0.0 1.0 0.000000 1.0 ... 0.0 0.0 0.000000 0.000000 0.0 0.000000 0.0 0.0 0.0 1.0

350 rows × 24 columns

# Make integer as necessary by rounding the raw synthetic data
for col in integer_cols:
    synth_df[col] = synth_df[col].round(0)

# Round binary cols and clip so values under 0 or above 1
# are set to 0 and 1 respectively (this won't happen with
# SMOTE, as it will only sample between the two points (so
# points sampled between binary points will always be
# between 0 and 1) but it can happen with other methods)
for col in binary_cols:
    synth_df[col] = np.clip(synth_df[col],0,1).round(0)

# Add y data with a label
y_list = list(y_synthetic)
synth_df[y_label] = y_list

# Shuffle data
synth_df = synth_df.sample(frac=1.0)
# Standardise synthetic data (based on real training data)
X_train_std, X_synth_std = standardise_data(X_train, X_synthetic)

# Get ALL real X data (combine standardised training + test data)
# We do this because we need to check for duplicates / very close
# values in all of the real data we've got
X_real_std = np.concatenate([X_train_std, X_test_std], axis=0)

# Use SciKitLearn neighbors.NearestNeighbors to find nearest neighbour
# to each data point. First, we fit to the real standardised data
# (all of it, train + test set).  Then we can give it the synthetic data
# and ask it to give us the cartesian distance and ID of its nearest
# real world data point neighbour for each synthetic data point.
nn = NearestNeighbors(n_neighbors=1, algorithm='auto').fit(X_real_std)
dists, idxs = nn.kneighbors(X_synth_std)

# Store the index and ids (indices) in the synthetic data DataFrame
# Flatten just reduces something in more than 1 dimension down to
# 1 dimension (eg a list of lists becomes a single list)
synth_df['distance_to_closest_real'] = list(dists.flatten())
synth_df['closest_X_real_row_index'] = list(idxs.flatten())
# Get points with zero distance to real (use distance of <0.001 as effectively identical)
identical = synth_df['distance_to_closest_real'] < 0.001

print (f'Proportion of data points identical to real data points = {identical.mean():0.3f}')
# Remove points with zero (or effectively zero) distance to a real data point.  We
# do this by setting up a mask that says we only want to see data points where the "identical"
# criterion we specified above is false (ie they're not identical).  Then we apply that
# mask and overwrite our existing synthetic data DataFrame so we've now only got data points
# that are not identical to real world data points.
mask = identical == False
synth_df = synth_df[mask]
Proportion of data points identical to real data points = 0.074
# Proportion of points to remove
proportion_to_remove = 0.1

# Sort by distance, with highest distances (those we want to keep) at
# the top
synth_by_distance = synth_df.sort_values(
    'distance_to_closest_real', ascending=False)

# Limit data.  Calculate the number of entries to keep as being the
# total number of synthetic data points we've now got (after having
# removed ones identical to real world data points) multiplied by
# the proportion we want to keep (the inverse of the proportion to remove).
# As we've sorted in descending order by distance, we can then just
# use .head to identify how much of the top of list we want to keep
# (90% in this case, where we're removing the 10% that are closest - at
# the bottom)
number_to_keep = int(len(synth_by_distance) * (1 - proportion_to_remove))
synth_by_distance = synth_by_distance.head(number_to_keep)

# Shuffle and store back in synth_df (frac=1 gives us a sample size of 100%
# (ie - all of the ones we said above we wanted to keep))
synth_df = synth_by_distance.sample(frac=1)
# Keep only a random sample of 150 of the remaining synthetic datapoints
# We don't need a mask here as ALL our synthetic datapoints are for class 1
# (positive).
synth_df = synth_df.sample(150)
synth_df
Pclass Age SibSp Parch Fare AgeImputed EmbarkedImputed CabinLetterImputed CabinNumber CabinNumberImputed ... CabinLetter_C CabinLetter_D CabinLetter_E CabinLetter_F CabinLetter_G CabinLetter_T CabinLetter_missing Survived distance_to_closest_real closest_X_real_row_index
812 1.0 40.0 0.0 0.0 79.770103 0.0 1.0 0.000000 26.938916 0.000000 ... 0.000000 0.132635 0.0 0.0 0.0 0.0 0.000000 1.0 3.069914 214
634 2.0 32.0 1.0 1.0 26.175431 0.0 0.0 1.000000 0.000000 1.000000 ... 0.000000 0.000000 0.0 0.0 0.0 0.0 1.000000 1.0 0.328739 550
720 3.0 2.0 0.0 1.0 12.870792 0.0 0.0 1.000000 0.000000 1.000000 ... 0.000000 0.000000 0.0 0.0 0.0 0.0 1.000000 1.0 0.156506 266
886 1.0 41.0 0.0 0.0 79.085888 0.0 1.0 0.000000 29.397383 0.000000 ... 0.000000 0.279477 0.0 0.0 0.0 0.0 0.000000 1.0 0.645736 101
881 1.0 11.0 1.0 1.0 147.110861 0.0 0.0 0.000000 24.788982 0.000000 ... 1.000000 0.000000 0.0 0.0 0.0 0.0 0.000000 1.0 3.136116 372
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
635 2.0 1.0 1.0 1.0 18.613431 0.0 0.0 1.000000 0.000000 1.000000 ... 0.000000 0.000000 0.0 0.0 0.0 0.0 1.000000 1.0 0.311295 639
789 1.0 13.0 0.0 1.0 79.786677 0.0 0.0 0.000000 20.720365 0.000000 ... 0.000000 0.531185 0.0 0.0 0.0 0.0 0.000000 1.0 0.378668 429
702 2.0 24.0 1.0 1.0 65.049929 0.0 0.0 0.968794 0.062412 0.968794 ... 0.031206 0.000000 0.0 0.0 0.0 0.0 0.968794 1.0 0.565451 282
755 2.0 31.0 0.0 0.0 13.000000 0.0 0.0 1.000000 0.000000 1.000000 ... 0.000000 0.000000 0.0 0.0 0.0 0.0 1.000000 1.0 0.314695 301
890 3.0 4.0 0.0 1.0 22.490998 0.0 0.0 1.000000 0.000000 1.000000 ... 0.000000 0.000000 0.0 0.0 0.0 0.0 1.000000 1.0 0.677551 160

150 rows × 27 columns

# Add synthetic data for positive class (class 1) to real data
# We'll make a separate copy of the original dataframe with the new synthetic
# data points added, keeping our original data intact.
augmented_data = pd.concat([data, synth_df])

# We'll also get rid of the two columns we added -
# distance_to_closest_real and closest_X_real_row_index as we do not want these
# to be used in a Logistic Regression model.
augmented_data.drop('distance_to_closest_real', axis=1, inplace=True)
augmented_data.drop('closest_X_real_row_index', axis=1, inplace=True)
# Let's have a look at our new dataframe
augmented_data
Survived Pclass Age SibSp Parch Fare AgeImputed EmbarkedImputed CabinLetterImputed CabinNumber ... Embarked_missing CabinLetter_A CabinLetter_B CabinLetter_C CabinLetter_D CabinLetter_E CabinLetter_F CabinLetter_G CabinLetter_T CabinLetter_missing
0 0.0 3.0 22.0 1.0 0.0 7.250000 0.0 0.0 1.000000 0.000000 ... 0.0 0.000000 0.0 0.000000 0.000000 0.0 0.0 0.0 0.0 1.000000
1 1.0 1.0 38.0 1.0 0.0 71.283300 0.0 0.0 0.000000 85.000000 ... 0.0 0.000000 0.0 1.000000 0.000000 0.0 0.0 0.0 0.0 0.000000
2 1.0 3.0 26.0 0.0 0.0 7.925000 0.0 0.0 1.000000 0.000000 ... 0.0 0.000000 0.0 0.000000 0.000000 0.0 0.0 0.0 0.0 1.000000
3 1.0 1.0 35.0 1.0 0.0 53.100000 0.0 0.0 0.000000 123.000000 ... 0.0 0.000000 0.0 1.000000 0.000000 0.0 0.0 0.0 0.0 0.000000
4 0.0 3.0 35.0 0.0 0.0 8.050000 0.0 0.0 1.000000 0.000000 ... 0.0 0.000000 0.0 0.000000 0.000000 0.0 0.0 0.0 0.0 1.000000
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
635 1.0 2.0 1.0 1.0 1.0 18.613431 0.0 0.0 1.000000 0.000000 ... 0.0 0.000000 0.0 0.000000 0.000000 0.0 0.0 0.0 0.0 1.000000
789 1.0 1.0 13.0 0.0 1.0 79.786677 0.0 0.0 0.000000 20.720365 ... 0.0 0.468815 0.0 0.000000 0.531185 0.0 0.0 0.0 0.0 0.000000
702 1.0 2.0 24.0 1.0 1.0 65.049929 0.0 0.0 0.968794 0.062412 ... 0.0 0.000000 0.0 0.031206 0.000000 0.0 0.0 0.0 0.0 0.968794
755 1.0 2.0 31.0 0.0 0.0 13.000000 0.0 0.0 1.000000 0.000000 ... 0.0 0.000000 0.0 0.000000 0.000000 0.0 0.0 0.0 0.0 1.000000
890 1.0 3.0 4.0 0.0 1.0 22.490998 0.0 0.0 1.000000 0.000000 ... 0.0 0.000000 0.0 0.000000 0.000000 0.0 0.0 0.0 0.0 1.000000

1041 rows × 25 columns

# Let's also check that the class splits are as expected
number_positive_class = np.sum(augmented_data['Survived'] == 1)
number_negative_class = np.sum(augmented_data['Survived'] == 0)

print (f"Positives : {number_positive_class}")
print (f"Negatives : {number_negative_class}")
Positives : 492
Negatives : 549
X_aug = augmented_data.drop('Survived',axis=1) # X = all 'data' except the 'stroke' column
y_aug = augmented_data['Survived'] # y = 'stroke' column from 'data'
X_train_aug, X_test_aug, y_train_aug, y_test_aug = train_test_split(X_aug, y_aug, test_size = 0.25, random_state=42)
X_train_std_aug, X_test_std_aug = standardise_data(X_train_aug, X_test_aug)

30 Compare performance

results_df = fit_train(X_train=X_train_std_aug, X_validate=X_test_std_aug,
            y_train=y_train_aug, y_validate=y_test_aug,
            name="With Synthetic Data", )
pd.concat([
    results_df,
    fit_train(X_train=X_train_std, X_validate=X_test_std,
            y_train=y_train, y_validate=y_test,
            name="Without Synthetic Data")])
Accuracy (training) Accuracy (validation) Precision (validation) Recall (validation) AUC f1 FP FN
With Synthetic Data 0.8 0.793 0.795 0.791 0.791 0.792 22 32
Without Synthetic Data 0.8 0.832 0.827 0.827 0.827 0.827 15 15