Monday, March 23, 2020

Beginning Deep Learning, Working with the Boston Housing Dataset

This code is all part of my deep learning journey and as always, is being placed here so I can always revisit it as I continue to expand on my learning of this topic.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
#!/usr/bin/env python3


'''
 Beginning my deep learning journey -
 This part of the journey focus on the binary or two class classification problem.
 Learning to classify the Boston Housing dataset into positive and negative reviews based on 
 text content


 File: dlBoston.py
 Author: Nik Alleyne
 Author Blog: www.securitynik.com
 Date: 2020-02-04
'''

import numpy as np
from keras.datasets import boston_housing
from keras import (models, layers)
from matplotlib import pyplot as plt


def build_model(X_train):
    model = models.Sequential()
    model.add(layers.Dense(64, activation='relu', input_shape=(X_train.shape[1],)))
    model.add(layers.Dense(64, activation='relu'))
    
    '''
    Final layer does not have an activation function defined
    By not specifying an activation function, the model is 
    free to learn and predict the linear values
    '''

    model.add(layers.Dense(1))

    '''
    Mean Squared Error (mse) is widely used in regression problems
    Mean Absolute Error (mae) is used to monitor the absolute value
    between the predictions and the targets
    '''
    model.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])
    return model




def main():
    # Devide the data into training and testing sets
    (X_train, y_train), (X_test, y_test) = boston_housing.load_data()
    

    # Get the shape of both the training and testing set
    print('\n[*] X_train shape: {}'.format(X_train.shape))
    print('[*] y_train shape: {}'.format(y_train.shape))
    print('[*] X_test shape: {}'.format(X_test.shape))
    print('[*] y_test shape: {}'.format(y_test.shape))

    print('\n[*] Sample records from X_train\n {}'.format(X_train))
    print('\n[*] Sample record from y_train\n {}'.format(y_train))

    mean = X_train.mean(axis=0)
    X_train -= mean
    print('\n[*] X_train after finding the mean \n{}'.format(X_train))

    std_deviation = X_train.std(axis=0)
    X_train /= std_deviation
    print('\n[*] X_train after finding the standard deviation \n{}'.format(X_train))

    X_test -= mean
    X_test /= std_deviation
    print('\n[*] X_test after finding the mean \n{}'.format(X_test))
    

    #Setting up cross validation
    k = 4
    num_validation_samples = len(X_train) // k
    print('[*] Num Validation samples {}'.format(num_validation_samples))
    
    num_epochs = 2
    all_scores = []
    mae_histories = []

    for i in range(k):
        print('[*] Proessing fold: {}'.format(i))
        X_train_val = X_train[i * num_validation_samples: (i + 1) * num_validation_samples]
        y_train_val = y_train[i * num_validation_samples: (i + 1) * num_validation_samples]

        X_train_patial = np.concatenate([X_train[:i * num_validation_samples], X_train[(i+1) * num_validation_samples:]], axis=0)
        y_train_patial = np.concatenate([y_train[:i * num_validation_samples], y_train[(i+1) * num_validation_samples:]], axis=0)
        model = build_model(X_train)
        nn_history = model.fit(X_train_patial, y_train_patial, epochs=num_epochs, batch_size=1, verbose=1)
        val_mse, val_mae = model.evaluate(X_train_val, y_train_val, verbose=1)
        all_scores.append(val_mae)
        
    print('[*] History information from nn_history.history \n{}'.format(nn_history.history))
    mae_histories = nn_history.history['mae']

    print('\n[*] X_train Validation samples\n{}'.format(X_train_val))
    print('\n[*] y_train validation samples\n{}'.format(y_train_val))

    print('[*] All scores \n{}'.format(all_scores))
    print('[*] Here is the mean of all scores {}'.format(np.mean(all_scores)))
    print('[*] Here is the mae scores {}'.format(mae_histories))
    print('[*] Here is the mae mean scores {}'.format(np.mean(mae_histories)))


if __name__ == '__main__':
    main()


'''
References:
https://www.manning.com/books/deep-learning-with-python

'''

Posted by at No comments:

Learning about KMeans and the Elbow Curve

This code is all part of my deep learning journey and as always, is being placed here so I can always revisit it as I continue to expand on my learning of this topic.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
#!/usr/bin/env python3

'''
Continuting my journey learning about machine learning
This code is focused on learning about Clustering and KMeans with a specific focus on the Elbow Method
Author: Nik Alleyne
Author Blog: www.securitynik.com
Filename: KMeans-pkts-elbow.py
'''




import pandas as pd
import numpy as np
from sklearn.cluster import KMeans
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot as plt



def main():
    pkt_df = pd.read_csv('/tmp/FeaturePktEnginFinal.csv', verbose=True)
    print(pkt_df.columns)

    # Looking for null data
    print('[*] Looking for NULL data \n{}'.format(pkt_df.isnull()))

    # Filling NULL data
    pkt_df.fillna(0, inplace=True)


    # Leveraging MinMax Scaler to attempt to give the data better representation
    min_max_scaler = MinMaxScaler()
    print('[*] Here is your MinMax Scaler information \n{}'.format(min_max_scaler))
    
    # Scaling the destination ports and tcp length
    pkt_df['tcpdport'] = min_max_scaler.fit_transform(pkt_df[['tcpdport']])
    pkt_df['tcplen'] = min_max_scaler.fit_transform(pkt_df[['tcplen']])
    print('\n[*] Here the Min Max Scaled TCP Length \n{}'.format(pkt_df['tcplen']))
    print('\n[*] Here the Min Max Scaled TCP Destination Ports \n{}'.format(pkt_df['tcpdport']))


    # Setup a new KMeans classifier to work on the scaled data
    km_scaled = KMeans(n_clusters=3)
    km_scaled_predict = km_scaled.fit_predict(pkt_df[['tcpdport', 'tcplen']])
    print('[*] Here are your new clusters \n{}'.format(km_scaled_predict))
    print('\n[*] Once again, here are your cluster centers \n{}'.format(km_scaled.cluster_centers_))

    # Add km_scaled_predict as a new column
    pkt_df['km_scaled_predict'] = km_scaled_predict
    print('[*] Here is the new data \n{}'.format(pkt_df))

    # Plot a new scatter plot
    # Create 3 new data frames to plot the graphs
    pkt_df0 = pkt_df[pkt_df.km_scaled_predict == 0]
    pkt_df1 = pkt_df[pkt_df.km_scaled_predict == 1]
    pkt_df2 = pkt_df[pkt_df.km_scaled_predict == 2]

    # Scatter plot the packet size
    plt.scatter(pkt_df0.tcpdport, pkt_df0['tcplen'], color='green')
    plt.scatter(pkt_df1.tcpdport, pkt_df1['tcplen'], color='red')
    plt.scatter(pkt_df2.tcpdport, pkt_df2['tcplen'], color='blue')

    '''
    Add the centroids to the scatter plots
    All rows and first column [:, 0]
    All rows and second column [:, 1]
    '''
    plt.scatter(km_scaled.cluster_centers_[:,0], km_scaled.cluster_centers_[:,1], color='black', marker='*', label='centroid')

    plt.xlabel('TCP Destination Port')
    plt.ylabel('TCP Packet Length')
    plt.legend()
    plt.show()

    # Finding the optimal K value using Elbow Method
    sum_of_sqr_err = []
    for k in range(1,10):
        km_clf = KMeans(n_clusters=k)
        km_clf.fit(pkt_df[['tcplen', 'tcpdport']])
        
        # Inertia is the Sum Of Squares Error (SSE)
        sum_of_sqr_err.append(km_clf.inertia_)
    
    # Print the SSE values
    print('\n[*] Here are your SSE Values \n{}'.format(sum_of_sqr_err))

    # plot the elbow graph
    plt.xlabel('K')
    plt.ylabel('Sum of Squared')
    plt.plot(range(1,10), sum_of_sqr_err)
    plt.show()
    

    # Ploting new clusters with K=2
    km_scaled = KMeans(n_clusters=2)
    km_scaled_predict = km_scaled.fit_predict(pkt_df[['tcpdport', 'tcplen']])
    print('[*] Here are your new clusters \n{}'.format(km_scaled_predict))
    print('\n[*] Once again, here are your cluster centers \n{}'.format(km_scaled.cluster_centers_))

    # Add km_scaled_predict as a new column
    pkt_df['km_scaled_predict'] = km_scaled_predict
    print('[*] Here is the new data \n{}'.format(pkt_df))

    # Plot a new scatter plot
    # Create 3 new data frames to plot the graphs
    pkt_df0 = pkt_df[pkt_df.km_scaled_predict == 0]
    pkt_df1 = pkt_df[pkt_df.km_scaled_predict == 1]
    pkt_df2 = pkt_df[pkt_df.km_scaled_predict == 2]

    # Scatter plot the packet size
    plt.scatter(pkt_df0.tcpdport, pkt_df0['tcplen'], color='green')
    plt.scatter(pkt_df1.tcpdport, pkt_df1['tcplen'], color='red')
    plt.scatter(pkt_df2.tcpdport, pkt_df2['tcplen'], color='blue')

    '''
    Add the centroids to the scatter plots
    All rows and first column [:, 0]
    All rows and second column [:, 1]
    '''
    plt.scatter(km_scaled.cluster_centers_[:,0], km_scaled.cluster_centers_[:,1], color='black', marker='*', label='centroid')

    plt.xlabel('TCP Destination Port')
    plt.ylabel('TCP Packet Length')
    plt.legend()
    plt.show()


    plt.close('all')


    

if __name__ == '__main__':
    main()


'''
References:
https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.drop.html
https://dev.to/nexttech/k-means-clustering-with-scikit-learn-14kk
https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.fillna.html
https://www.youtube.com/watch?v=ZueoXMgCd1c
https://www.youtube.com/watch?v=EItlUEPCIzM
https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html
'''



dad
Posted by at No comments:

Beginning Deep Learning with Dogs and Cats Classification

This code is all part of my deep learning journey and as always, is being placed here so I can always revisit it as I continue to expand on my learning of this topic.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
#!/usr/bin/env python3


'''
 Beginning my deep learning journey
 File: dl_catsVdogs_v3.py

 This is a continuation from dl_cats_Vdogs.py.
 This version 3 addresses the over fitting issues via usage of a pre-trained network
 A pre-trained network is a saved version of a network which was trained on a 
 larger dataset. 
 The two ways to use pre-trained networks are feature extraction and fine-tuning

 Feature extraction uses representation learned from previous models to extract interesting features.
 These features are then run against a newly trained classifier


 Author: Nik Alleyne
 Author Blog: www.securitynik.com
 Date: 2020-03-06
'''

import os, shutil
from keras import (models, optimizers)
from keras.applications import VGG16
from keras.layers import (Conv2D, MaxPooling2D, Flatten, Dense, Dropout)
from keras.preprocessing.image import (ImageDataGenerator, image)
from matplotlib import pyplot as plt

def main():
    cats_dogs_dataset = './PetImages'
    cats_dogs = '/tmp/cats_dogs'
    
    print('[*] Checking to the if "{}" directory exists ...'.format(cats_dogs))
    if (os.path.exists(cats_dogs)):
        print('[-] Deleting directory {}'.format(cats_dogs))
        shutil.rmtree(cats_dogs)
    
    print('[+] Making temporary image directory ...')
    os.mkdir(cats_dogs)

    print(' [+] Creating training set directory ...')
    train_dir = os.path.join(cats_dogs, 'train_set')
    cats_train = os.path.join(train_dir, 'cats')
    dogs_train = os.path.join(train_dir, 'dogs')
    os.mkdir(train_dir)
    os.mkdir(cats_train)
    os.mkdir(dogs_train)

    print(' [+] Creating validation set directory  ...')
    val_dir = os.path.join(cats_dogs, 'val_set')
    cats_val = os.path.join(val_dir, 'cats')
    dogs_val = os.path.join(val_dir, 'dogs')
    os.mkdir(val_dir)
    os.mkdir(cats_val)
    os.mkdir(dogs_val)

    print(' [+] Creating testing set directory  ...')
    test_dir = os.path.join(cats_dogs, 'test_set')
    cats_test = os.path.join(test_dir, 'cats')
    dogs_test = os.path.join(test_dir, 'dogs')
    os.mkdir(test_dir)
    os.mkdir(cats_test)
    os.mkdir(dogs_test)

    # copy the first 1000 cat images from for training data
    cat_train_images = ['{}.jpg'.format(i) for i in range(1000)]
    for cat in cat_train_images:
        src = os.path.join(cats_dogs_dataset +'/Cat', cat)
        dst = os.path.join(cats_train, cat)
        shutil.copyfile(src, dst)


    # copy the first 1000 dog images for training data
    dog_train_images = ['{}.jpg'.format(i) for i in range(1000)]
    for dog in dog_train_images:
        src = os.path.join(cats_dogs_dataset +'/Dog', dog)
        dst = os.path.join(dogs_train, dog)
        shutil.copyfile(src, dst)


     # copy the next 500 cat images and used as validation data
    cat_val_images = ['{}.jpg'.format(i) for i in range(1000, 1500)]
    for cat in cat_val_images:
        src = os.path.join(cats_dogs_dataset +'/Cat', cat)
        dst = os.path.join(cats_val, cat)
        shutil.copyfile(src, dst)


    # copy the next 500 dog images and used as validation data
    dog_val_images = ['{}.jpg'.format(i) for i in range(1000, 1500)]
    for dog in dog_val_images:
        src = os.path.join(cats_dogs_dataset +'/Dog', dog)
        dst = os.path.join(dogs_val, dog)
        shutil.copyfile(src, dst)
   

     # copy the final 500 cat images and used as test data
    cat_test_images = ['{}.jpg'.format(i) for i in range(1500, 2000)]
    for cat in cat_test_images:
        src = os.path.join(cats_dogs_dataset +'/Cat', cat)
        dst = os.path.join(cats_test, cat)
        shutil.copyfile(src, dst)

    # copy the final 500 dog images and used as test data
    dog_test_images = ['{}.jpg'.format(i) for i in range(1500, 2000)]
    for dog in dog_test_images:
        src = os.path.join(cats_dogs_dataset +'/Dog', dog)
        dst = os.path.join(dogs_test, dog)
        shutil.copyfile(src, dst)


    # Verifying the number of cat records in each directory
    print('[*] Total cat training images: {}:{}'.format(cats_train, len(os.listdir(cats_train))))
    print('[*] Total cat validation images: {}:{}'.format(cats_val, len(os.listdir(cats_val))))
    print('[*] Total cat test images: {}:{}'.format(cats_test, len(os.listdir(cats_test))))

    # Verifying the number of dog records in each directory
    print('[*] Total cat training images: {}:{}'.format(dogs_train, len(os.listdir(dogs_train))))
    print('[*] Total cat validation images: {}:{}'.format(dogs_val, len(os.listdir(dogs_val))))
    print('[*] Total cat test images: {}:{}'.format(dogs_test, len(os.listdir(dogs_test))))

    # Initiate the VGG16 convulational base
    pretrained_model_base = VGG16(weights='imagenet', include_top=False, input_shape=(150, 150, 3))
    
    #rint the summary of the network
    print('[*] Here is the convulational network base \n{}'.format(pretrained_model_base.summary()))


    '''
    Create the colvolution neural network from the pretrained model.
    This is meant to help to address the overfitting related to 
    small datasets. This is being used in conjunction with data augmentation
    '''

    convnet = models.Sequential()
    convnet.add(pretrained_model_base)

    # Flatten the layers
    convnet.add(Flatten())

    # Add dense layers
    convnet.add(Dense(512, activation='relu'))
    convnet.add(Dense(1, activation='sigmoid'))
    
    # Compile the model
    convnet.compile(loss='binary_crossentropy', optimizer=optimizers.RMSprop(lr=0.01), metrics=['accuracy'])

    

    print('[*] Here is the model summary \n{}'.format(convnet.summary()))

    # Train the network using data augmentation
    training_data_generator = ImageDataGenerator(rescale=1.0/255, rotation_range=40, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, horizontal_flip=True,fill_mode='nearest')

    # Testing data should not be augmented
    testing_data_generator = ImageDataGenerator(rescale=1.0/255)

    training_data_generated = training_data_generator.flow_from_directory(train_dir, target_size=(150, 150), batch_size=32, class_mode='binary')
    validation_data_generated = testing_data_generator.flow_from_directory(val_dir, target_size=(150, 150), batch_size=32, class_mode='binary')
    convnet.compile(loss='binary_crossentropy', optimizer=optimizers.RMSprop(lr=0.02), metrics=['accuracy'])

    '''
    Try to fit on the training data to see what occurrs. 
    Set the epoch and steps as 10 just for my learning. 
    These numbers should be larger
    '''

    convnet_history = convnet.fit_generator(training_data_generated, steps_per_epoch=5, epochs=10, validation_data=validation_data_generated, validation_steps=8)
    print('[*] Here is what the network history looks like:\n {}'.format(convnet_history.history))
    
    # Save the model
    convnet.save('/tmp/convnet_v3.h5')

    '''
    Plot the loss and accuracy of the model
    This is being done over the training and validation data
    '''
    train_accuracy = convnet_history.history['accuracy']
    train_loss = convnet_history.history['loss']

    validation_accuracy = convnet_history.history['val_accuracy']
    validation_loss = convnet_history.history['val_loss']

    convnet_epochs = range(1, len(train_accuracy) + 1)

    # Plot the graph to compare the training and validation loss and accuracy
    plt.plot(convnet_epochs, train_accuracy, 'b', label='Training Accuracy')
    plt.plot(convnet_epochs, validation_accuracy, 'bo', label='Validation Accuracy')
    plt.title('Training vs Validation Accuracy')
    plt.legend()
    plt.figure()
    
    plt.plot(convnet_epochs, train_loss, 'b', label='Training Loss')
    plt.plot(convnet_epochs, validation_loss, 'bo', label='Validation Loss')
    plt.title('Training vs Validation Accuracy')
    plt.legend()
    
    plt.show()




if __name__ == '__main__':
    main()



'''
References:
    https://www.microsoft.com/en-us/download/details.aspx?id=54765
    https://keras.io/preprocessing/image/
    https://keras.io/models/sequential/
    https://keras.io/applications/#extract-features-with-vgg16

'''





Posted by at No comments:

Beginning Deep Learning IMDB Dataset

This code is all part of my deep learning journey and as always, is being placed here so I can always revisit it as I continue to expand on my learning of this topic.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
#!/usr/bin/env python3


'''
 Beginning my deep learning journey -
 This part of the journey focus on the binary or two class classification problem.
 Learning to classify the IMDB dataset into positive and negative reviews based on 
 text content


 File: dlIMDB.py
 Author: Nik Alleyne
 Author Blog: www.securitynik.com
 Date: 2020-01-31
'''

import numpy as np
from keras.datasets import imdb
from keras import (optimizers, layers, models)
from matplotlib import pyplot as plt
from keras.utils.vis_utils import plot_model


# Function used to vectorize data, making into a set of 1s nd 0s
def vectorize_data(sequences, dimension=10000):
    results = np.zeros((len(sequences), dimension))
    for i, sequence in enumerate(sequences):
        results[i, sequence] = 1
    return results



def main():
    ''' 
    split the data into training and testing
    Use the top 10,000 most frequently seen words
    Discard words least seen
    '''
    (X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=10000, maxlen=None)
    
    # Looking for the total number of words in the entire set
    total_words = len(np.unique(np.hstack(X_train))) + len(np.unique(np.hstack(X_test))) 
    print('[*] Total Words in the dataset: {}'.format(total_words))
    
    # Taking a look at the data
    print('[*] X_train sample data:\n{}'.format(X_train[5]))
    print('\n[*] y_train sample data: {}'.format(y_train[5]))
    print('\n[*] X_test sample data:\n{}'.format(X_test[5]))
    print('\n[*] y_test sample data: {}'.format(y_test[5]))

    # Get the shape of both the training and testing set
    print('\n[*] X_train shape: {}'.format(X_train.shape))
    print('[*] y_train shape: {}'.format(y_train.shape))
    print('[*] X_test shape: {}'.format(X_test.shape))
    print('[*] y_test shape: {}'.format(y_test.shape))

    '''
    Encode both the training and testing data
    so that it can be fed to the neural network
    First vectorize the training and testing data
    '''
    X_train = vectorize_data(X_train).astype('float32')
    X_test = vectorize_data(X_test).astype('float32')
    print('\n[*] Encoded X_train data: \n {}'.format(X_train))

    '''
    Convert the testing data to np array
    '''
    y_train = np.asarray(y_train).astype('float32')
    y_test = np.asarray(y_test).astype('float32')


    # Create the validation set of 10000 records from the training set
    X_train_val = X_train[:10000]
    y_train_val = y_train[:10000]

    '''
    Create a new training and testing set from the remainder of the 
    X_train after the validation records have been extracted
    '''
    X_train_partial = X_train[10000:]
    y_train_partial = y_train[10000:]

    print('\n [*] Here are your unique classes {}'.format(np.unique(y_train)))

    # Build the neural network
    model = models.Sequential()
    model.add(layers.Dense(64, activation='relu', input_shape=(10000,)))
    model.add(layers.Dense(64, activation='relu'))
    model.add(layers.Dense(1, activation='sigmoid'))

    # Compile the model
    model.compile(optimizer=optimizers.RMSprop(lr=0.001), loss='binary_crossentropy', metrics=['accuracy'])
    
    '''
    Fit the model and test it using the validation data
    The returned results have a history object which has a member history
    Take the results from the history object and store it in the nn_history variable
    this variable tracks the accuracy & loss and validation accuracy & validation loss
    '''
    nn_history = model.fit(X_train_partial, y_train_partial, epochs=3, batch_size=512, validation_data=(X_train_val, y_train_val))
    print('\n[*] Here is the results from history_nn\n{}'.format(nn_history.history))
    print('\n[*] Here are the keys for nn_history.history: {}'.format(nn_history.history.keys()))
    
    '''
    Perform an evaulation of the model
    based oh the test data
    '''
    results = model.evaluate(X_test, y_test, verbose=1)
    print('[*] Here is the loss results for the evaluated model {}'.format(results[0]))
    print('[*] Here is the accuracy results for the evaluated model {}'.format(results[1]))


    #Ploting the training and validation loss
    nn_history_loss = nn_history.history['loss']
    nn_history_val_loss = nn_history.history['val_loss']
    epochs = range(1, len(nn_history_loss) + 1)

    plt.plot(epochs, nn_history_loss, 'bo', label='Training Loss')
    plt.plot(epochs, nn_history_val_loss, 'b', label='Validation Loss')
    plt.title(' Training vs Validation Loss ')
    plt.xlabel('epochs')
    plt.ylabel('loss')
    plt.legend()
    plt.show()

    '''
    Quite interesting for me, as the training loss decreased,
    the validation loss increased
    '''

    #Plotting the training and validation accuracy
    nn_history_accuracy = nn_history.history['accuracy']
    nn_history_val_accuracy = nn_history.history['val_accuracy']

    plt.clf()
    plt.plot(epochs, nn_history_accuracy, 'bo', label='Training Accuracy')
    plt.plot(epochs, nn_history_val_accuracy, 'b', label='Validation Accuracy')
    plt.title(' Training vs Validation Accuracy ')
    plt.xlabel('epochs')
    plt.ylabel('loss')
    plt.legend()
    plt.show()
    plt.close('all')

    ''' 
    Another finding was that as the training accuracy increased,
    the validation acurracy basically flatlined. 
    These findings apparently ties back into overfitting
    The model is doing well on the training data but horrible
    on the validation data
    The fact that a model performs well on training data does not mean
    it will perform well on data it has never seen before

    I adjusted the epoch a few times and it sees 3 might be the sweet spot 
    for my example

    '''

    # Time to make a prediction on the trained model
    predict_sentiment = model.predict(X_test)
    print('[*] Here are your sentiments for the testing data \n{}'.format(predict_sentiment))

    print('\n[*] Model Summary Information \n{}'.format(model.summary()))

    # Create a visual plot of the model
    plot_model(model, to_file='/tmp/model.png', show_shapes=True, show_layer_names=True)


if __name__ == '__main__':
    main()


'''
Referenes:
https://www.manning.com/books/deep-learning-with-python
https://keras.io/getting-started/sequential-model-guide/
https://keras.io/optimizers/
https://keras.io/datasets/#imdb-movie-reviews-sentiment-classification
https://machinelearningmastery.com/predict-sentiment-movie-reviews-using-deep-learning/
https://towardsdatascience.com/machine-learning-word-embedding-sentiment-classification-using-keras-b83c28087456
https://www.stackabuse.com/python-for-nlp-movie-sentiment-analysis-using-deep-learning-in-keras/
https://machinelearningmastery.com/visualize-deep-learning-neural-network-model-keras/
https://machinelearningmastery.com/tutorial-first-neural-network-python-keras/
'''


Posted by at No comments:
Subscribe to: Posts (Atom)