In [1]:
import pandas as pd
import numpy as np
import gzip
from collections import defaultdict
from keras.utils.np_utils import to_categorical
import matplotlib.pyplot as plt
from keras.layers import Dense, LSTM, Embedding
from keras.models import Sequential
from keras.preprocessing.text import Tokenizer,text_to_word_sequence

from keras.utils.np_utils import to_categorical
from keras.utils import pad_sequences
In [2]:
vocab_size = 400000
embedding_size = 50
maxlen = 10
embeddings_path = 'glove.6B.50d.txt.gz'
In [3]:
import pandas as pd
from sklearn.model_selection import train_test_split

df = pd.read_csv('Consumer_Complaints.csv.zip', encoding='latin-1')
df = df[pd.notnull(df['Consumer Complaint'])]
train, test  = train_test_split( df, test_size=0.20, random_state=10)
In [4]:
train.shape
Out[4]:
(222251, 19)
In [5]:
test.shape
Out[5]:
(55563, 19)
In [6]:
train.head()
Out[6]:
Date received Product Sub-product Issue Sub-issue Consumer Complaint Company Public Response Company State ZIP code Tags Consumer consent provided? Submitted via Date Sent to Company Company Response to Consumer Timely response? Consumer disputed? Complaint ID Unnamed: 18
549929 10/13/2015 Prepaid card Payroll card Fraud or scam NaN RushCard had notify all customers that it will... NaN Empowerment Ventures, LLC MD 210XX NaN Consent provided Web 11-02-2015 Closed with monetary relief Yes No 1604724 NaN
191026 09/29/2016 Credit card NaN Other NaN You have failed to respond to a certified lett... NaN JPMORGAN CHASE & CO. FL 334XX Servicemember Consent provided Web 09/29/2016 Closed with explanation Yes Yes 2139057 NaN
401077 10-04-2016 Mortgage Conventional fixed mortgage Loan modification,collection,foreclosure NaN I have a mortgage on a house XXXX. When I move... NaN OLD NATIONAL BANK WA 983XX Older American Consent provided Web 10-06-2016 Closed with explanation Yes Yes 2143603 NaN
358052 07-06-2015 Debt collection Mortgage False statements or representation Attempted to collect wrong amount XXXX XXXX, XXXX bac XXXX XXXX XXXX XXXX, XXXX ... Company chooses not to provide a public response BANK OF AMERICA, NATIONAL ASSOCIATION GA 303XX NaN Consent provided Web 07/21/2015 Closed with explanation Yes No 1453268 NaN
466771 01-05-2016 Credit reporting NaN Credit reporting company's investigation Inadequate help over the phone XXXX XXXX XXXX XXXX. Box XXXX XXXX, XXXX XXXX ... Company chooses not to provide a public response Experian Information Solutions Inc. MD 208XX NaN Consent provided Web 01-11-2016 Closed with explanation Yes No 1727584 NaN
In [7]:
# Common methods to read data and get embeddings
def read_data():
    df_train = train
    df_val = test
    categories = list(set(df_train.Product.values))
    return df_train,df_val,categories

def load_embeddings():
    word_index = {}
    embeddings = np.zeros((vocab_size,embedding_size))
    with gzip.open(embeddings_path) as file:
        for i,line in enumerate(file):
            line_tokens = line.split()
            word = line_tokens[0].decode('utf8')
            embeddings[i] = np.asarray(line_tokens[1:],dtype='float32')
            word_index[word] = i
    return embeddings,word_index
In [8]:
def get_embedding(word,word_index,embeddings):
    if word in word_index:
        return embeddings[word_index[word]].reshape(((embedding_size,1)))
    else:
        return np.zeros((embedding_size,1))
In [9]:
# Methods for Neural Network Model
def prepare_data(df_train,df_val,categories):
    train_text = df_train['Consumer Complaint'].tolist()
    val_text = df_val['Consumer Complaint'].tolist()
    tk = Tokenizer(num_words = vocab_size, lower = True)
    tk.fit_on_texts(train_text + val_text)
    x_train = pad_sequences(tk.texts_to_sequences(train_text),maxlen=maxlen)
    x_val = pad_sequences(tk.texts_to_sequences(val_text),maxlen=maxlen)
    y_train = category_to_one_hot(df_train['Product'].values,categories)
    y_val = category_to_one_hot(df_val['Product'].values,categories) 
    return tk.word_index,x_train,y_train,x_val,y_val

def prepare_data_from_full_word_index(df_train,df_val,categories,word_index):
    train_text = df_train['Consumer Complaint'].tolist()
    val_text = df_val['Consumer Complaint'].tolist()
    x_train = get_pad_sequences(train_text,word_index)
    x_val = get_pad_sequences(val_text,word_index)
    y_train = category_to_one_hot(df_train['Product'].values,categories)
    y_val = category_to_one_hot(df_val['Product'].values,categories) 
    return word_index,x_train,y_train,x_val,y_val

def get_pad_sequences(text_list,word_index):
    seqs = []
    for text in text_list:
        word_seq = text_to_word_sequence(text.lower())
        seq = []
        for word in word_seq:
          if word in word_index:
            seq.append(word_index[word])
        seqs.append(seq)
    return pad_sequences(seqs,maxlen)


# Convert the list of categories to one_hot vector
def category_to_one_hot(cat_list,cat_master):
    cat_dict = {}
    for i,cat in enumerate(cat_master):
        cat_dict[cat] = i
    cat_integers = [cat_dict[cat] for cat in cat_list]
    return to_categorical(cat_integers,num_classes=len(cat_master))

# Convert one_hot to category
def one_hot_to_category(cat_one_hot_list,cat_master):
    return [cat_master[cat_one_hot.argmax()] for cat_one_hot in cat_one_hot_list]

# Get the embedding weights for the model
def get_embedding_matrix_for_model(embeddings,word_index):
    train_val_words = min(vocab_size, len(word_index)) +1
    embedding_matrix = np.zeros((train_val_words, embedding_size))
    for word, i in word_index.items():
        embedding_vector = get_embedding(word,word_index,embeddings).flatten()
        if embedding_vector is not None: 
            embedding_matrix[i] = embedding_vector
    return embedding_matrix

# Build the keras model
def build_model(embedding_matrix,categories):
    model = Sequential()
    model.add(Embedding(embedding_matrix.shape[0], embedding_size, weights=[embedding_matrix],input_length=maxlen,trainable=False))
    model.add(LSTM(32))
#   We don't lose much by replacing LSTM with this flatten layer (as we have short sequences)
#   model.add(Flatten())
    model.add(Dense(32, activation='relu'))
    model.add(Dense(len(categories), activation='sigmoid'))
    model.compile(optimizer='rmsprop',
              loss='categorical_crossentropy',
              metrics=['acc'])
    return model

def get_val(numerator,divisor):
    return float('nan') if divisor == 0 else np.round(numerator/divisor,3)

def analyze_predictions(categories,y_true,y_pred):
    tp = defaultdict(int)
    tn = defaultdict(int)
    fp = defaultdict(int)
    fn = defaultdict(int)
    precisions = []
    recalls = []
    f1s = []
    cat_counts = defaultdict(int)
    for cat in y_true:
        cat_counts[cat]+=1 
    correct = 0
    conf_mat = defaultdict(dict)
    for cat1 in categories:
        for cat2 in categories:
            conf_mat[cat1][cat2] = 0
    for y,y_hat in zip(y_true,y_pred):
        conf_mat[y][y_hat]+=1
        if y == y_hat:
            correct+=1
            tp[y]+=1
        else:
            fp[y_hat]+=1
            fn[y]+=1
    print('Overall Accuracy:',round(correct/len(y_pred),3))
    for cat in categories:
        precision = get_val(tp[cat],tp[cat]+fp[cat])
        recall = get_val(tp[cat],(tp[cat]+fn[cat]))
        f1 = get_val(2*precision*recall,precision + recall)
        precisions.append(precision)
        recalls.append(recall)  
        f1s.append(f1)
        print('{} --> Precision:{},Recall:{},F1:{}'.format(cat,precision,recall,f1))
    print ('\nAverages---> Precision:{}, Recall:{}, F1:{}'.format(np.round(np.nanmean(precisions),3),                                                                                                np.round(np.nanmean(recalls),3),
                                                               np.round(np.nanmean(f1s),3))
          )
          
    print('\nConfusion Matrix')
    for cat1 in categories:
        print('\n'  +cat1+'({}) --> '.format(cat_counts[cat1]),end='')
        for cat2 in categories:
            print('{}({})'.format(cat2,conf_mat[cat1][cat2]),end=' , ')
    print('')
    


# From Deep Learning with Python book
def make_history_plot(history):
    acc = history.history['acc']
    val_acc = history.history['val_acc']
    loss = history.history['loss']
    val_loss = history.history['val_loss']

    epochs = range(1, len(acc) + 1)

    plt.plot(epochs, acc, 'bo', label='Training acc')
    plt.plot(epochs, val_acc, 'b', color='green',label='Validation acc')
    plt.title('Training and validation accuracy')
    plt.legend()

    plt.figure()

    plt.plot(epochs, loss, 'bo', label='Training loss')
    plt.plot(epochs, val_loss, 'b', color='green',label='Validation loss')
    plt.title('Training and validation loss')
    plt.legend()
    plt.show()
In [10]:
import numpy as np
np.random.seed(42) # for reproducibility
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot
import matplotlib.pyplot as plt
%matplotlib inline
In [11]:
df_train,df_val,categories = read_data()
In [12]:
#Load Glove 50-d embeddings
embeddings,word_index = load_embeddings()
In [13]:
#Prepare the data for the model
tk_word_index,x_train,y_train,x_val,y_val = prepare_data_from_full_word_index(df_train,df_val,categories,word_index)
In [14]:
# Get the embedding matrix for the model, build model, display model summary
embedding_matrix = get_embedding_matrix_for_model(embeddings,word_index)
model = build_model(embedding_matrix,categories)
model.summary()

Model: "sequential"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding (Embedding)       (None, 10, 50)            20000050  
                                                                 
 lstm (LSTM)                 (None, 32)                10624     
                                                                 
 dense (Dense)               (None, 32)                1056      
                                                                 
 dense_1 (Dense)             (None, 18)                594       
                                                                 
=================================================================
Total params: 20,012,324
Trainable params: 12,274
Non-trainable params: 20,000,050
_________________________________________________________________
In [15]:
# Train the model, record history
history = model.fit(x_train, y_train,
                    epochs=5,
                    batch_size=24,
                    shuffle=False,
                    validation_data=(x_val, y_val))

Epoch 1/5

2023-01-12 00:57:03.587094: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz

9261/9261 [==============================] - 18s 2ms/step - loss: 2.0021 - acc: 0.3386 - val_loss: 1.9310 - val_acc: 0.3609
Epoch 2/5
9261/9261 [==============================] - 17s 2ms/step - loss: 1.8987 - acc: 0.3755 - val_loss: 1.8980 - val_acc: 0.3740
Epoch 3/5
9261/9261 [==============================] - 17s 2ms/step - loss: 1.8717 - acc: 0.3857 - val_loss: 1.8833 - val_acc: 0.3826
Epoch 4/5
9261/9261 [==============================] - 17s 2ms/step - loss: 1.8580 - acc: 0.3909 - val_loss: 1.8773 - val_acc: 0.3847
Epoch 5/5
9261/9261 [==============================] - 17s 2ms/step - loss: 1.8492 - acc: 0.3948 - val_loss: 1.8731 - val_acc: 0.3881
In [16]:
make_history_plot(history)

/var/folders/16/3468kndx5l1_zj5r84tsybgc0000gn/T/ipykernel_37214/4293647875.py:129: UserWarning: color is redundantly defined by the 'color' keyword argument and the fmt string "b" (-> color=(0.0, 0.0, 1.0, 1)). The keyword argument will take precedence.
  plt.plot(epochs, val_acc, 'b', color='green',label='Validation acc')
/var/folders/16/3468kndx5l1_zj5r84tsybgc0000gn/T/ipykernel_37214/4293647875.py:136: UserWarning: color is redundantly defined by the 'color' keyword argument and the fmt string "b" (-> color=(0.0, 0.0, 1.0, 1)). The keyword argument will take precedence.
  plt.plot(epochs, val_loss, 'b', color='green',label='Validation loss')
In [17]:
# Make and analyze training predictions
train_predictions = one_hot_to_category(model.predict(x_train),categories)
analyze_predictions(categories,df_train['Product'].values,train_predictions)

6946/6946 [==============================] - 4s 596us/step
Overall Accuracy: 0.398
Payday loan, title loan, or personal loan --> Precision:nan,Recall:0.0,F1:nan
Debt collection --> Precision:0.444,Recall:0.65,F1:0.528
Other financial service --> Precision:nan,Recall:0.0,F1:nan
Vehicle loan or lease --> Precision:nan,Recall:0.0,F1:nan
Credit card or prepaid card --> Precision:nan,Recall:0.0,F1:nan
Mortgage --> Precision:0.362,Recall:0.626,F1:0.459
Bank account or service --> Precision:0.318,Recall:0.275,F1:0.295
Credit reporting --> Precision:0.583,Recall:0.083,F1:0.145
Prepaid card --> Precision:nan,Recall:0.0,F1:nan
Credit reporting, credit repair services, or other personal consumer reports --> Precision:0.377,Recall:0.582,F1:0.458
Consumer Loan --> Precision:0.446,Recall:0.023,F1:0.044
Student loan --> Precision:0.554,Recall:0.201,F1:0.295
Checking or savings account --> Precision:nan,Recall:0.0,F1:nan
Payday loan --> Precision:nan,Recall:0.0,F1:nan
Money transfer, virtual currency, or money service --> Precision:nan,Recall:0.0,F1:nan
Money transfers --> Precision:nan,Recall:0.0,F1:nan
Virtual currency --> Precision:nan,Recall:0.0,F1:nan
Credit card --> Precision:0.335,Recall:0.176,F1:0.231

Averages---> Precision:0.427, Recall:0.145, F1:0.307

Confusion Matrix

Payday loan, title loan, or personal loan(1747) --> Payday loan, title loan, or personal loan(0) , Debt collection(648) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(660) , Bank account or service(76) , Credit reporting(3) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(244) , Consumer Loan(11) , Student loan(58) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(47) , 
Debt collection(50617) --> Payday loan, title loan, or personal loan(0) , Debt collection(32881) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(7254) , Bank account or service(581) , Credit reporting(255) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(8716) , Consumer Loan(37) , Student loan(350) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(543) , 
Other financial service(232) --> Payday loan, title loan, or personal loan(0) , Debt collection(85) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(71) , Bank account or service(34) , Credit reporting(0) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(34) , Consumer Loan(0) , Student loan(5) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(3) , 
Vehicle loan or lease(2224) --> Payday loan, title loan, or personal loan(0) , Debt collection(731) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(790) , Bank account or service(53) , Credit reporting(2) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(455) , Consumer Loan(84) , Student loan(45) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(64) , 
Credit card or prepaid card(8563) --> Payday loan, title loan, or personal loan(0) , Debt collection(1951) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(2522) , Bank account or service(682) , Credit reporting(13) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(1770) , Consumer Loan(5) , Student loan(91) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(1529) , 
Mortgage(35020) --> Payday loan, title loan, or personal loan(0) , Debt collection(6608) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(21916) , Bank account or service(1006) , Credit reporting(58) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(4228) , Consumer Loan(6) , Student loan(670) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(528) , 
Bank account or service(11944) --> Payday loan, title loan, or personal loan(0) , Debt collection(2495) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(3622) , Bank account or service(3281) , Credit reporting(21) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(1680) , Consumer Loan(10) , Student loan(52) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(783) , 
Credit reporting(25412) --> Payday loan, title loan, or personal loan(0) , Debt collection(6572) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(3253) , Bank account or service(140) , Credit reporting(2119) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(12885) , Consumer Loan(16) , Student loan(149) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(278) , 
Prepaid card(1162) --> Payday loan, title loan, or personal loan(0) , Debt collection(269) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(346) , Bank account or service(250) , Credit reporting(0) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(133) , Consumer Loan(1) , Student loan(6) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(157) , 
Credit reporting, credit repair services, or other personal consumer reports(39224) --> Payday loan, title loan, or personal loan(0) , Debt collection(9175) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(5058) , Bank account or service(296) , Credit reporting(1068) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(22828) , Consumer Loan(31) , Student loan(322) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(446) , 
Consumer Loan(7555) --> Payday loan, title loan, or personal loan(0) , Debt collection(2712) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(2651) , Bank account or service(188) , Credit reporting(22) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(1431) , Consumer Loan(172) , Student loan(166) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(213) , 
Student loan(13289) --> Payday loan, title loan, or personal loan(0) , Debt collection(3492) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(5132) , Bank account or service(187) , Credit reporting(22) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(1583) , Consumer Loan(4) , Student loan(2675) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(194) , 
Checking or savings account(5173) --> Payday loan, title loan, or personal loan(0) , Debt collection(1049) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(1494) , Bank account or service(1482) , Credit reporting(6) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(831) , Consumer Loan(1) , Student loan(18) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(292) , 
Payday loan(1379) --> Payday loan, title loan, or personal loan(0) , Debt collection(629) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(453) , Bank account or service(76) , Credit reporting(2) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(132) , Consumer Loan(0) , Student loan(55) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(32) , 
Money transfer, virtual currency, or money service(2533) --> Payday loan, title loan, or personal loan(0) , Debt collection(607) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(830) , Bank account or service(633) , Credit reporting(0) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(382) , Consumer Loan(2) , Student loan(3) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(76) , 
Money transfers(1183) --> Payday loan, title loan, or personal loan(0) , Debt collection(299) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(373) , Bank account or service(305) , Credit reporting(2) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(171) , Consumer Loan(0) , Student loan(4) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(29) , 
Virtual currency(12) --> Payday loan, title loan, or personal loan(0) , Debt collection(2) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(3) , Bank account or service(4) , Credit reporting(0) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(3) , Consumer Loan(0) , Student loan(0) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(0) , 
Credit card(14982) --> Payday loan, title loan, or personal loan(0) , Debt collection(3845) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(4175) , Bank account or service(1038) , Credit reporting(40) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(3090) , Consumer Loan(6) , Student loan(157) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(2631) ,
In [18]:
# Make and analyze validation predictions
val_predictions = one_hot_to_category(model.predict(x_val),categories)
analyze_predictions(categories,df_val['Product'].values,val_predictions)

1737/1737 [==============================] - 1s 602us/step
Overall Accuracy: 0.388
Payday loan, title loan, or personal loan --> Precision:nan,Recall:0.0,F1:nan
Debt collection --> Precision:0.432,Recall:0.638,F1:0.515
Other financial service --> Precision:nan,Recall:0.0,F1:nan
Vehicle loan or lease --> Precision:nan,Recall:0.0,F1:nan
Credit card or prepaid card --> Precision:nan,Recall:0.0,F1:nan
Mortgage --> Precision:0.359,Recall:0.616,F1:0.454
Bank account or service --> Precision:0.308,Recall:0.265,F1:0.285
Credit reporting --> Precision:0.543,Recall:0.074,F1:0.13
Prepaid card --> Precision:nan,Recall:0.0,F1:nan
Credit reporting, credit repair services, or other personal consumer reports --> Precision:0.364,Recall:0.559,F1:0.441
Consumer Loan --> Precision:0.381,Recall:0.023,F1:0.043
Student loan --> Precision:0.548,Recall:0.206,F1:0.299
Checking or savings account --> Precision:nan,Recall:0.0,F1:nan
Payday loan --> Precision:nan,Recall:0.0,F1:nan
Money transfer, virtual currency, or money service --> Precision:nan,Recall:0.0,F1:nan
Money transfers --> Precision:nan,Recall:0.0,F1:nan
Virtual currency --> Precision:nan,Recall:0.0,F1:nan
Credit card --> Precision:0.313,Recall:0.159,F1:0.211

Averages---> Precision:0.406, Recall:0.141, F1:0.297

Confusion Matrix

Payday loan, title loan, or personal loan(439) --> Payday loan, title loan, or personal loan(0) , Debt collection(167) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(165) , Bank account or service(17) , Credit reporting(0) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(59) , Consumer Loan(2) , Student loan(18) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(11) , 
Debt collection(12651) --> Payday loan, title loan, or personal loan(0) , Debt collection(8067) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(1805) , Bank account or service(141) , Credit reporting(69) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(2302) , Consumer Loan(11) , Student loan(110) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(146) , 
Other financial service(61) --> Payday loan, title loan, or personal loan(0) , Debt collection(16) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(22) , Bank account or service(9) , Credit reporting(1) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(11) , Consumer Loan(0) , Student loan(1) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(1) , 
Vehicle loan or lease(567) --> Payday loan, title loan, or personal loan(0) , Debt collection(159) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(204) , Bank account or service(22) , Credit reporting(0) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(113) , Consumer Loan(31) , Student loan(14) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(24) , 
Credit card or prepaid card(2096) --> Payday loan, title loan, or personal loan(0) , Debt collection(497) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(615) , Bank account or service(195) , Credit reporting(6) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(402) , Consumer Loan(0) , Student loan(21) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(360) , 
Mortgage(8817) --> Payday loan, title loan, or personal loan(0) , Debt collection(1725) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(5427) , Bank account or service(241) , Credit reporting(9) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(1119) , Consumer Loan(2) , Student loan(173) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(121) , 
Bank account or service(2943) --> Payday loan, title loan, or personal loan(0) , Debt collection(638) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(857) , Bank account or service(781) , Credit reporting(4) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(434) , Consumer Loan(4) , Student loan(13) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(212) , 
Credit reporting(6181) --> Payday loan, title loan, or personal loan(0) , Debt collection(1705) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(810) , Bank account or service(32) , Credit reporting(457) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(3059) , Consumer Loan(4) , Student loan(34) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(80) , 
Prepaid card(288) --> Payday loan, title loan, or personal loan(0) , Debt collection(74) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(80) , Bank account or service(64) , Credit reporting(0) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(26) , Consumer Loan(0) , Student loan(1) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(43) , 
Credit reporting, credit repair services, or other personal consumer reports(9782) --> Payday loan, title loan, or personal loan(0) , Debt collection(2434) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(1322) , Bank account or service(80) , Credit reporting(267) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(5471) , Consumer Loan(12) , Student loan(80) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(116) , 
Consumer Loan(1919) --> Payday loan, title loan, or personal loan(0) , Debt collection(672) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(646) , Bank account or service(75) , Credit reporting(4) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(372) , Consumer Loan(45) , Student loan(42) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(63) , 
Student loan(3400) --> Payday loan, title loan, or personal loan(0) , Debt collection(892) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(1294) , Bank account or service(35) , Credit reporting(6) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(428) , Consumer Loan(0) , Student loan(702) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(43) , 
Checking or savings account(1316) --> Payday loan, title loan, or personal loan(0) , Debt collection(246) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(413) , Bank account or service(340) , Credit reporting(3) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(226) , Consumer Loan(0) , Student loan(6) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(82) , 
Payday loan(369) --> Payday loan, title loan, or personal loan(0) , Debt collection(157) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(127) , Bank account or service(28) , Credit reporting(0) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(32) , Consumer Loan(1) , Student loan(16) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(8) , 
Money transfer, virtual currency, or money service(556) --> Payday loan, title loan, or personal loan(0) , Debt collection(131) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(160) , Bank account or service(145) , Credit reporting(0) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(94) , Consumer Loan(0) , Student loan(3) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(23) , 
Money transfers(314) --> Payday loan, title loan, or personal loan(0) , Debt collection(74) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(113) , Bank account or service(67) , Credit reporting(1) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(45) , Consumer Loan(0) , Student loan(3) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(11) , 
Virtual currency(4) --> Payday loan, title loan, or personal loan(0) , Debt collection(0) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(1) , Bank account or service(1) , Credit reporting(0) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(2) , Consumer Loan(0) , Student loan(0) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(0) , 
Credit card(3860) --> Payday loan, title loan, or personal loan(0) , Debt collection(1024) , Other financial service(0) , Vehicle loan or lease(0) , Credit card or prepaid card(0) , Mortgage(1075) , Bank account or service(259) , Credit reporting(15) , Prepaid card(0) , Credit reporting, credit repair services, or other personal consumer reports(825) , Consumer Loan(6) , Student loan(44) , Checking or savings account(0) , Payday loan(0) , Money transfer, virtual currency, or money service(0) , Money transfers(0) , Virtual currency(0) , Credit card(612) ,
In [ ]: