Learning by practicing

Monday, March 23, 2020

Beginning Deep Learning with Dogs and Cats Classification

This code is all part of my deep learning journey and as always, is being placed here so I can always revisit it as I continue to expand on my learning of this topic.

#!/usr/bin/env python3


'''
 Beginning my deep learning journey
 File: dl_catsVdogs_v3.py

 This is a continuation from dl_cats_Vdogs.py.
 This version 3 addresses the over fitting issues via usage of a pre-trained network
 A pre-trained network is a saved version of a network which was trained on a 
 larger dataset. 
 The two ways to use pre-trained networks are feature extraction and fine-tuning

 Feature extraction uses representation learned from previous models to extract interesting features.
 These features are then run against a newly trained classifier


 Author: Nik Alleyne
 Author Blog: www.securitynik.com
 Date: 2020-03-06
'''

import os, shutil
from keras import (models, optimizers)
from keras.applications import VGG16
from keras.layers import (Conv2D, MaxPooling2D, Flatten, Dense, Dropout)
from keras.preprocessing.image import (ImageDataGenerator, image)
from matplotlib import pyplot as plt

def main():
    cats_dogs_dataset = './PetImages'
    cats_dogs = '/tmp/cats_dogs'
    
    print('[*] Checking to the if "{}" directory exists ...'.format(cats_dogs))
    if (os.path.exists(cats_dogs)):
        print('[-] Deleting directory {}'.format(cats_dogs))
        shutil.rmtree(cats_dogs)
    
    print('[+] Making temporary image directory ...')
    os.mkdir(cats_dogs)

    print(' [+] Creating training set directory ...')
    train_dir = os.path.join(cats_dogs, 'train_set')
    cats_train = os.path.join(train_dir, 'cats')
    dogs_train = os.path.join(train_dir, 'dogs')
    os.mkdir(train_dir)
    os.mkdir(cats_train)
    os.mkdir(dogs_train)

    print(' [+] Creating validation set directory  ...')
    val_dir = os.path.join(cats_dogs, 'val_set')
    cats_val = os.path.join(val_dir, 'cats')
    dogs_val = os.path.join(val_dir, 'dogs')
    os.mkdir(val_dir)
    os.mkdir(cats_val)
    os.mkdir(dogs_val)

    print(' [+] Creating testing set directory  ...')
    test_dir = os.path.join(cats_dogs, 'test_set')
    cats_test = os.path.join(test_dir, 'cats')
    dogs_test = os.path.join(test_dir, 'dogs')
    os.mkdir(test_dir)
    os.mkdir(cats_test)
    os.mkdir(dogs_test)

    # copy the first 1000 cat images from for training data
    cat_train_images = ['{}.jpg'.format(i) for i in range(1000)]
    for cat in cat_train_images:
        src = os.path.join(cats_dogs_dataset +'/Cat', cat)
        dst = os.path.join(cats_train, cat)
        shutil.copyfile(src, dst)


    # copy the first 1000 dog images for training data
    dog_train_images = ['{}.jpg'.format(i) for i in range(1000)]
    for dog in dog_train_images:
        src = os.path.join(cats_dogs_dataset +'/Dog', dog)
        dst = os.path.join(dogs_train, dog)
        shutil.copyfile(src, dst)


     # copy the next 500 cat images and used as validation data
    cat_val_images = ['{}.jpg'.format(i) for i in range(1000, 1500)]
    for cat in cat_val_images:
        src = os.path.join(cats_dogs_dataset +'/Cat', cat)
        dst = os.path.join(cats_val, cat)
        shutil.copyfile(src, dst)


    # copy the next 500 dog images and used as validation data
    dog_val_images = ['{}.jpg'.format(i) for i in range(1000, 1500)]
    for dog in dog_val_images:
        src = os.path.join(cats_dogs_dataset +'/Dog', dog)
        dst = os.path.join(dogs_val, dog)
        shutil.copyfile(src, dst)
   

     # copy the final 500 cat images and used as test data
    cat_test_images = ['{}.jpg'.format(i) for i in range(1500, 2000)]
    for cat in cat_test_images:
        src = os.path.join(cats_dogs_dataset +'/Cat', cat)
        dst = os.path.join(cats_test, cat)
        shutil.copyfile(src, dst)

    # copy the final 500 dog images and used as test data
    dog_test_images = ['{}.jpg'.format(i) for i in range(1500, 2000)]
    for dog in dog_test_images:
        src = os.path.join(cats_dogs_dataset +'/Dog', dog)
        dst = os.path.join(dogs_test, dog)
        shutil.copyfile(src, dst)


    # Verifying the number of cat records in each directory
    print('[*] Total cat training images: {}:{}'.format(cats_train, len(os.listdir(cats_train))))
    print('[*] Total cat validation images: {}:{}'.format(cats_val, len(os.listdir(cats_val))))
    print('[*] Total cat test images: {}:{}'.format(cats_test, len(os.listdir(cats_test))))

    # Verifying the number of dog records in each directory
    print('[*] Total cat training images: {}:{}'.format(dogs_train, len(os.listdir(dogs_train))))
    print('[*] Total cat validation images: {}:{}'.format(dogs_val, len(os.listdir(dogs_val))))
    print('[*] Total cat test images: {}:{}'.format(dogs_test, len(os.listdir(dogs_test))))

    # Initiate the VGG16 convulational base
    pretrained_model_base = VGG16(weights='imagenet', include_top=False, input_shape=(150, 150, 3))
    
    #rint the summary of the network
    print('[*] Here is the convulational network base \n{}'.format(pretrained_model_base.summary()))


    '''
    Create the colvolution neural network from the pretrained model.
    This is meant to help to address the overfitting related to 
    small datasets. This is being used in conjunction with data augmentation
    '''

    convnet = models.Sequential()
    convnet.add(pretrained_model_base)

    # Flatten the layers
    convnet.add(Flatten())

    # Add dense layers
    convnet.add(Dense(512, activation='relu'))
    convnet.add(Dense(1, activation='sigmoid'))
    
    # Compile the model
    convnet.compile(loss='binary_crossentropy', optimizer=optimizers.RMSprop(lr=0.01), metrics=['accuracy'])

    

    print('[*] Here is the model summary \n{}'.format(convnet.summary()))

    # Train the network using data augmentation
    training_data_generator = ImageDataGenerator(rescale=1.0/255, rotation_range=40, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, horizontal_flip=True,fill_mode='nearest')

    # Testing data should not be augmented
    testing_data_generator = ImageDataGenerator(rescale=1.0/255)

    training_data_generated = training_data_generator.flow_from_directory(train_dir, target_size=(150, 150), batch_size=32, class_mode='binary')
    validation_data_generated = testing_data_generator.flow_from_directory(val_dir, target_size=(150, 150), batch_size=32, class_mode='binary')
    convnet.compile(loss='binary_crossentropy', optimizer=optimizers.RMSprop(lr=0.02), metrics=['accuracy'])

    '''
    Try to fit on the training data to see what occurrs. 
    Set the epoch and steps as 10 just for my learning. 
    These numbers should be larger
    '''

    convnet_history = convnet.fit_generator(training_data_generated, steps_per_epoch=5, epochs=10, validation_data=validation_data_generated, validation_steps=8)
    print('[*] Here is what the network history looks like:\n {}'.format(convnet_history.history))
    
    # Save the model
    convnet.save('/tmp/convnet_v3.h5')

    '''
    Plot the loss and accuracy of the model
    This is being done over the training and validation data
    '''
    train_accuracy = convnet_history.history['accuracy']
    train_loss = convnet_history.history['loss']

    validation_accuracy = convnet_history.history['val_accuracy']
    validation_loss = convnet_history.history['val_loss']

    convnet_epochs = range(1, len(train_accuracy) + 1)

    # Plot the graph to compare the training and validation loss and accuracy
    plt.plot(convnet_epochs, train_accuracy, 'b', label='Training Accuracy')
    plt.plot(convnet_epochs, validation_accuracy, 'bo', label='Validation Accuracy')
    plt.title('Training vs Validation Accuracy')
    plt.legend()
    plt.figure()
    
    plt.plot(convnet_epochs, train_loss, 'b', label='Training Loss')
    plt.plot(convnet_epochs, validation_loss, 'bo', label='Validation Loss')
    plt.title('Training vs Validation Accuracy')
    plt.legend()
    
    plt.show()




if __name__ == '__main__':
    main()



'''
References:
    https://www.microsoft.com/en-us/download/details.aspx?id=54765
    https://keras.io/preprocessing/image/
    https://keras.io/models/sequential/
    https://keras.io/applications/#extract-features-with-vgg16

'''

Beginning Deep Learning IMDB Dataset

This code is all part of my deep learning journey and as always, is being placed here so I can always revisit it as I continue to expand on my learning of this topic.

#!/usr/bin/env python3


'''
 Beginning my deep learning journey -
 This part of the journey focus on the binary or two class classification problem.
 Learning to classify the IMDB dataset into positive and negative reviews based on 
 text content


 File: dlIMDB.py
 Author: Nik Alleyne
 Author Blog: www.securitynik.com
 Date: 2020-01-31
'''

import numpy as np
from keras.datasets import imdb
from keras import (optimizers, layers, models)
from matplotlib import pyplot as plt
from keras.utils.vis_utils import plot_model


# Function used to vectorize data, making into a set of 1s nd 0s
def vectorize_data(sequences, dimension=10000):
    results = np.zeros((len(sequences), dimension))
    for i, sequence in enumerate(sequences):
        results[i, sequence] = 1
    return results



def main():
    ''' 
    split the data into training and testing
    Use the top 10,000 most frequently seen words
    Discard words least seen
    '''
    (X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=10000, maxlen=None)
    
    # Looking for the total number of words in the entire set
    total_words = len(np.unique(np.hstack(X_train))) + len(np.unique(np.hstack(X_test))) 
    print('[*] Total Words in the dataset: {}'.format(total_words))
    
    # Taking a look at the data
    print('[*] X_train sample data:\n{}'.format(X_train[5]))
    print('\n[*] y_train sample data: {}'.format(y_train[5]))
    print('\n[*] X_test sample data:\n{}'.format(X_test[5]))
    print('\n[*] y_test sample data: {}'.format(y_test[5]))

    # Get the shape of both the training and testing set
    print('\n[*] X_train shape: {}'.format(X_train.shape))
    print('[*] y_train shape: {}'.format(y_train.shape))
    print('[*] X_test shape: {}'.format(X_test.shape))
    print('[*] y_test shape: {}'.format(y_test.shape))

    '''
    Encode both the training and testing data
    so that it can be fed to the neural network
    First vectorize the training and testing data
    '''
    X_train = vectorize_data(X_train).astype('float32')
    X_test = vectorize_data(X_test).astype('float32')
    print('\n[*] Encoded X_train data: \n {}'.format(X_train))

    '''
    Convert the testing data to np array
    '''
    y_train = np.asarray(y_train).astype('float32')
    y_test = np.asarray(y_test).astype('float32')


    # Create the validation set of 10000 records from the training set
    X_train_val = X_train[:10000]
    y_train_val = y_train[:10000]

    '''
    Create a new training and testing set from the remainder of the 
    X_train after the validation records have been extracted
    '''
    X_train_partial = X_train[10000:]
    y_train_partial = y_train[10000:]

    print('\n [*] Here are your unique classes {}'.format(np.unique(y_train)))

    # Build the neural network
    model = models.Sequential()
    model.add(layers.Dense(64, activation='relu', input_shape=(10000,)))
    model.add(layers.Dense(64, activation='relu'))
    model.add(layers.Dense(1, activation='sigmoid'))

    # Compile the model
    model.compile(optimizer=optimizers.RMSprop(lr=0.001), loss='binary_crossentropy', metrics=['accuracy'])
    
    '''
    Fit the model and test it using the validation data
    The returned results have a history object which has a member history
    Take the results from the history object and store it in the nn_history variable
    this variable tracks the accuracy & loss and validation accuracy & validation loss
    '''
    nn_history = model.fit(X_train_partial, y_train_partial, epochs=3, batch_size=512, validation_data=(X_train_val, y_train_val))
    print('\n[*] Here is the results from history_nn\n{}'.format(nn_history.history))
    print('\n[*] Here are the keys for nn_history.history: {}'.format(nn_history.history.keys()))
    
    '''
    Perform an evaulation of the model
    based oh the test data
    '''
    results = model.evaluate(X_test, y_test, verbose=1)
    print('[*] Here is the loss results for the evaluated model {}'.format(results[0]))
    print('[*] Here is the accuracy results for the evaluated model {}'.format(results[1]))


    #Ploting the training and validation loss
    nn_history_loss = nn_history.history['loss']
    nn_history_val_loss = nn_history.history['val_loss']
    epochs = range(1, len(nn_history_loss) + 1)

    plt.plot(epochs, nn_history_loss, 'bo', label='Training Loss')
    plt.plot(epochs, nn_history_val_loss, 'b', label='Validation Loss')
    plt.title(' Training vs Validation Loss ')
    plt.xlabel('epochs')
    plt.ylabel('loss')
    plt.legend()
    plt.show()

    '''
    Quite interesting for me, as the training loss decreased,
    the validation loss increased
    '''

    #Plotting the training and validation accuracy
    nn_history_accuracy = nn_history.history['accuracy']
    nn_history_val_accuracy = nn_history.history['val_accuracy']

    plt.clf()
    plt.plot(epochs, nn_history_accuracy, 'bo', label='Training Accuracy')
    plt.plot(epochs, nn_history_val_accuracy, 'b', label='Validation Accuracy')
    plt.title(' Training vs Validation Accuracy ')
    plt.xlabel('epochs')
    plt.ylabel('loss')
    plt.legend()
    plt.show()
    plt.close('all')

    ''' 
    Another finding was that as the training accuracy increased,
    the validation acurracy basically flatlined. 
    These findings apparently ties back into overfitting
    The model is doing well on the training data but horrible
    on the validation data
    The fact that a model performs well on training data does not mean
    it will perform well on data it has never seen before

    I adjusted the epoch a few times and it sees 3 might be the sweet spot 
    for my example

    '''

    # Time to make a prediction on the trained model
    predict_sentiment = model.predict(X_test)
    print('[*] Here are your sentiments for the testing data \n{}'.format(predict_sentiment))

    print('\n[*] Model Summary Information \n{}'.format(model.summary()))

    # Create a visual plot of the model
    plot_model(model, to_file='/tmp/model.png', show_shapes=True, show_layer_names=True)


if __name__ == '__main__':
    main()


'''
Referenes:
https://www.manning.com/books/deep-learning-with-python
https://keras.io/getting-started/sequential-model-guide/
https://keras.io/optimizers/
https://keras.io/datasets/#imdb-movie-reviews-sentiment-classification
https://machinelearningmastery.com/predict-sentiment-movie-reviews-using-deep-learning/
https://towardsdatascience.com/machine-learning-word-embedding-sentiment-classification-using-keras-b83c28087456
https://www.stackabuse.com/python-for-nlp-movie-sentiment-analysis-using-deep-learning-in-keras/
https://machinelearningmastery.com/visualize-deep-learning-neural-network-model-keras/
https://machinelearningmastery.com/tutorial-first-neural-network-python-keras/
'''

Beginning Machine Learning - Detecting Fraudulent Transactions

This code is all part of my deep learning journey and as always, is being placed here so I can always revisit it as I continue to expand on my learning of this topic.

#!/usr/bin/env python3

'''
Continuing my machine learning journey

Author: Nik Alleyne
Author Blog: www.securitynik.com
file: payment_fraud.csv

'''

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (accuracy_score, classification_report, confusion_matrix)


def main():
    train_test_size = 0.33
    # import dataset
    fraud_df = pd.read_csv('./payment_fraud.csv', verbose=True)
    print('[*] First 10 records \n{}'.format(fraud_df.head(10)))
    print('\n[*] Here is another 10 sample records \n{}'.format(fraud_df.sample(10)))

    print('\n[*] Converting the payment method from categorical variable to numeric ...')
    fraud_df.paymentMethod.replace(['paypal', 'creditcard', 'storecredit'], [0,1,2], inplace=True)
    print('[*] Here is another 10 sample records after conversion \n{}'.format(fraud_df.sample(10)))

    # Create the X axis
    X_axis = fraud_df.drop('label', axis=1)
    print('[*] Sample 10 records from the X_axis \n{}'.format(X_axis.sample(10)))

    # Create the Y axis
    y_axis = fraud_df['label']
    print('\n[*] Sample 10 records from the X_axis \n{}'.format(y_axis.sample(10)))
    print('[*] Shape of the X_axis is: {}'.format(X_axis.shape))
    print('[*] Shape of the y_axis is: {}'.format(y_axis.shape))

    # Split the data into training and testing set
    print('[*] Splitting the data. Testing size is: {}%'.format(train_test_size*100))
    X_train, X_test, y_train, y_test = train_test_split(X_axis, y_axis, test_size=train_test_size, shuffle=True)
    print('\n[*] Shape of the X_train is: {}'.format(X_train.shape))
    print('[*] Shape of the X_test is: {}'.format(X_test.shape))
    print('\n[*] Shape of the y_train is: {}'.format(y_train.shape))
    print('[*] Shape of the y_test is: {}'.format(y_test.shape))

    '''
    First use logistic regression classifier
    The solver was set to 'lbfgs' because of a warning which was 
    produced when the classifier was created without it 
    '''
    lr_clf = LogisticRegression(solver='lbfgs')
    lr_clf.fit(X_train, y_train)
    print('\n[*] Here is the lr_clf aftering being fitted \n{}'.format(lr_clf))

    # Making a prediction on the test data
    predict_fraud = lr_clf.predict(X_test)
    print('\n[*] Here are your prediction on possible fraudlent transactions \n{}'.format([i for i in predict_fraud]))

    # Test the accuracy
    print('\n[*] Accuracy Score: {}'.format(accuracy_score(predict_fraud, y_test)))
    print('[*] Confusion Matrix: \n{}'.format(confusion_matrix(y_test, predict_fraud)))
    print('[*] Classification report on your prediction \n{}'.format(classification_report(y_test,predict_fraud)))


    # Let's make a prediction on a user's input
    a = [[1,2,3,4,5]]
    print('[*] Enter your 5 feature values as command separated')
    user_input = input('[*] Example: 1,2,3,4,5: ')
    user_input = user_input.split(',')
    user_input = [int(i) for i in user_input]
    print('[*] You Entered: {}'.format(user_input))
    print('[*] Here is your label:{} \n 0-Not Fraud \n 1-Fraud'.format(lr_clf.predict([user_input])))
    print('[*] Here is the probability score: {}'.format(lr_clf.predict_proba([user_input])))



if __name__ == '__main__':
    main()


'''
Reference:
https://raw.githubusercontent.com/oreilly-mlsec/book-resources/master/chapter2/datasets/payment_fraud.csv
https://www.amazon.com/Machine-Learning-Security-Protecting-Algorithms-dp-1491979909/dp/1491979909/
https://stackoverflow.com/questions/23307301/replacing-column-values-in-a-pandas-dataframe
https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.get_dummies.htddml

'''

Beginning Deep Learning Working With the Reuters Dataset

This code is all part of my deep learning journey and as always, is being placed here so I can always revisit it as I continue to expand on my learning of this topic.

#!/usr/bin/env python3


'''
 Beginning my deep learning journey -
 This part of the journey focus on the binary or two class classification problem.
 Learning to classify the Reuters Newswire dataset into positive and negative reviews based on 
 text content


 File: dlReuters.py
 Author: Nik Alleyne
 Author Blog: www.securitynik.com
 Date: 2020-02-04
'''

import numpy as np
from keras import (models, layers)
from keras.datasets import reuters
from keras.utils.np_utils import to_categorical
from matplotlib import pyplot as plt
from keras.utils.vis_utils import plot_model


# Function used to vectorize data, making into a set of 1s nd 0s
def vectorize_data(sequences, dimension=10000):
    results = np.zeros((len(sequences), dimension))
    for i, sequence in enumerate(sequences):
        results[i, sequence] = 1
    return results


def main():
    '''
    Split the dataset into training and testing
    using the 10,000 most frequently occurring word
    '''
    (X_train, y_train), (X_test, y_test) = reuters.load_data(num_words=10000)

    # Get the shape of the training and testing data
    print('\n[*] X_train shape: {}'.format(X_train.shape))
    print('[*] y_train shape: {}'.format(y_train.shape))
    print('[*] X_test shape: {}'.format(X_test.shape))
    print('[*] y_test shape: {}'.format(y_test.shape))

    # Each sample record is a list of integers
    print('[*] Sample record from X_train: \n{}'.format(X_train[0]))

    #Vectorize the training data
    X_train = vectorize_data(X_train).astype('float32')
    X_test = vectorize_data(X_test).astype('float32')

    # One Hot Encode the labels
    y_train = to_categorical(y_train)
    y_test = to_categorical(y_test)
    print('[*] Sample y_train data after one hot encoded \n{}'.format(y_train))

    # Create a validation set
    X_train_val = X_train[:1000]
    y_train_val = y_train[:1000]

    X_train_partial = X_train[1000:]
    y_train_partial = y_train[1000:]


    # Build the 3 node neural network
    model = models.Sequential()
    model.add(layers.Dense(64, activation='relu', input_shape=(10000,)))
    model.add(layers.Dense(64, activation='relu'))
    model.add(layers.Dense(46, activation='softmax'))

    # Compile the model
    model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['acc'])
    
    # Fit the model
    nn_history = model.fit(X_train, y_train, epochs=20, batch_size=512, validation_data=(X_train_val, y_train_val))
    print('\n[*] Here is the content of nn_history.history \n{}'.format(nn_history.history))

    # Plotting the loss for for the training and validation set
    val_loss = nn_history.history['val_loss']
    train_loss = nn_history.history['loss']
    
    print('[*] Length of validation Loss is {}'.format(len(val_loss)))
    print('[*] Length of training Loss is {}'.format(len(train_loss)))

    val_acc = nn_history.history['val_acc']
    train_acc = nn_history.history['acc']
        
    epochs = range(1, len(val_loss) + 1)
    plt.figure(figsize=(8,8))
    plt.plot(epochs, val_loss, color='green', marker='+', label='Validation Loss', linestyle='dashed', linewidth=2, markersize=15)
    plt.plot(epochs, train_loss, color='red', marker='.', label='Training Loss', linestyle='dashed', linewidth=2, markersize=15)
    plt.plot(epochs, val_acc, color='blue', marker='*', label='Validation Accuracy', linestyle='dashed', linewidth=2, markersize=15)
    plt.plot(epochs, train_acc, color='orange', marker='d', label='Training Accuracy', linestyle='dashed', linewidth=2, markersize=15)
    plt.xlabel('Epoch')
    plt.ylabel('Loss/Accuracy')
    plt.title('Training/Validation vs Accuracy/Loss')
    plt.legend()
    plt.show()

    plt.close('all')

    # Evaluate the test data
    results = model.evaluate(X_test, y_test)
    print('[*] Loss on the test data is: {}'.format(results[0]))
    print('[*] Accuracy on the test data is: {}'.format(results[1]))

    #Perform a prediction on the test set
    reuters_predict = model.predict(X_test)
    print('[*] Here are your predictions \n{}'.format(reuters_predict))
    print('[*] Here is the model summary \n{}'.format(model.summary()))

    print('[*] Here is the shape for predictions: {}'.format(reuters_predict[0].shape))
    
    #Visualize the model via a graph
    plot_model(model, to_file='/tmp/reuters-model.png', show_shapes=True, show_layer_names=True)

    print('[*] Largest prediction entry {}'.format(np.argmax(reuters_predict[0])))



if __name__ == '__main__':
    main()


'''
References:
https://www.manning.com/books/deep-learning-with-python
https://matplotlib.org/3.1.0/api/_as_gen/matplotlib.pyplot.plot.html#matplotlib.pyplot.plot
https://matplotlib.org/3.1.0/api/markers_api.html#module-matplotlib.markers
'''

Beginning Deep Learning - Working MNIST and Convolutional Neural Networks

This code is all part of my deep learning journey and as always, is being placed here so I can always revisit it as I continue to expand on my learning of this topic.

#!/usr/bin/env python3


'''
 Continuing my deep learning journey
 File: dlMNIST-convnet.py
 Author: Nik Alleyne
 Author Blog: www.securitynik.com
 Date: 2020-02-26
'''

from time import sleep
from keras.datasets import mnist
from keras import models
from keras.layers import (Dense, Activation, Conv2D, MaxPooling2D, Flatten)
from keras.utils import to_categorical
from matplotlib import pyplot as plt

def main():
    (X_train, y_train), (X_test, y_test) = mnist.load_data()
    print('[*] Your tensor has "{}" dimensions'.format(X_train.ndim))
    print('[*] The shape of X_train is:{}'.format(X_train.shape))
    print('[*] The shape of y_train is:{}'.format(y_train.shape))
    print('[*] The shape of X_test is:{}'.format(X_test.shape))
    print('[*] The shape of y_test is:{}'.format(y_test.shape))
    
    # Reshape the training and testing data
    X_train = X_train.reshape((60000,28, 28, 1))
    X_test = X_test.reshape((10000,28, 28, 1))

    # Converting to training and testing data to type float
    X_train = X_train.astype('float32')/255
    X_test = X_test.astype('float32')/255

    # Convert the labels to categrical type
    y_train = to_categorical(y_train)
    y_test = to_categorical(y_test)

    '''
    Build the convolution network
    The first layer has a feature map of 28, 28, 1
    starts with dept of 32 and ends with depth 64 (output depth)
    Size of the patches are (3x3) - (window height, window width)
    The 2*2 represents the max pooling window

    '''
    conv_nn = models.Sequential()
    conv_nn.add(Conv2D(32, (3,3), activation='relu', input_shape=(28, 28, 1)))
    conv_nn.add(MaxPooling2D(2,2))

    conv_nn.add(Conv2D(64, (3,3), activation='relu'))
    conv_nn.add(MaxPooling2D(2,2))

    conv_nn.add(Conv2D(64, (3,3), activation='relu'))

    # Flatten the network
    conv_nn.add(Flatten())
    conv_nn.add(Dense(64, activation='relu'))

    # Output layer
    conv_nn.add(Dense(10, activation='softmax'))
    

    # Compile the modell
    conv_nn.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
    print('[*] Here is your model summary \n{}'.format(conv_nn.summary()))

    # Fit on the training set
    conv_nn.fit(X_train, y_train, epochs=10, batch_size=64 )

    # Evaluate the model on the test data set
    testing_loss, testing_accuracy = conv_nn.evaluate(X_test, y_test)

    print('\n[*] Your testing accuracy is: {}'.format(testing_loss))
    print('[*] Your testing loss is: {}'.format(testing_accuracy))

    

if __name__ == '__main__':
    main()


'''
Referenes:
https://www.manning.com/books/deep-learning-with-python
https://keras.io/getting-started/sequential-model-guide/
https://keras.io/optimizers/

'''

Friday, March 20, 2020

Working with Time - Automating Epoch To Local

Recently, I encountered a situation where I had to have a large number of epoch times converted to local time. On most days, for a one off conversion, you can leverage the Linux command prompt with

1 2	root@securitynik:~# date --date '@1582902198' Fri 28 Feb 2020 10:03:18 AM EST

However, in this case, with a large number of times, we need to automate that conversion. The script below addressed this need.

Here is a sample of the file:

root@securitynik:~# cat Possible-Password-Spraying.csv | grep --perl-regexp "TimeGenerated=[0-9]*" --color=always --only-matching | awk --field-separator='=' '{ print $2 }' | more

1582902198
1582902292
1582902223
1582902225
1582902200
1582902160
1582902158
1582902156
1582902162
1582902155
1582902154
....

Script to solve the problem.

#!/usr/bin/env python3

'''
This script is meant to take a file containing epoch times
and converting the times to local times

In Unix, epoch time is the number of seconds since January 1, 1970
In Windows epoch time is the number of 100ns intervals since January 1, 1601
https://devblogs.microsoft.com/oldnewthing/20090306-00/?p=18913
https://www.computerhope.com/jargon/e/epoch.htm

'''

__version__ = '0.1'
__author__ = 'Nik Alleyne'
__contact__ = 'nalleyne@forsythe.com'
__maintainer__ = 'Nik Alleyne'
__status__ = 'Development'
__date__ = '2020-03-20'



import time

if __name__ == '__main__':
    '''
    create a file point, to point to the file we wish to read
    Note this file must already exist. Here is the command I used to 
    extract the time from a window log using IBM WinCollect Agent.
    
    root@securitynik:~# cat Possible-Password-Spraying.csv | grep --perl-regexp "TimeGenerated=[0-9]*" --color=always --only-matching | awk --field-separator='=' '{ print $2 }' > /tmp/epoch.txt
    '''
    epoch_fp = open('/tmp/epoch.txt', 'r')

    # Read all lines in the file one by one
    for epoch_line in epoch_fp.readlines():
        #print('{}'.format(epoch_line))
        # the "strip('\n')" is used to remove the extra spaces between two lines
        epoch_line = epoch_line.strip('\n')

        # The file also still contains some special characters
        epoch_line = ''.join(filter(str.isalnum, epoch_line )).replace('mK','')
        
        #Remove special characters from the line
        #print(float(epoch_line))

        # Convert the epoch to local time
        print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(float(epoch_line))))

Monday, March 16, 2020

Looking to get your hands on some great free SANS Content

SANS instructors produce thousands of free content-rich resources for the information security community annually. These
resources are aimed to provide the latest in research and technology available to help support awareness and growth across a wide range of IT and OT security considerations.

Currently available content relates to:
* Cybersecurity News, Updates & Content
* Tools & Workstations
* Scholarship & Community Programs
* Free Training
* Curricula Specific Resources

To learn more more visit: https://www.sans.org/free