Hi I am training an autoencoder model for anomaly detection.Here is my training code
def model_save(i,start,end):
df_normal = pd.read_csv("/home/ram/Downloads/19_channel_2.csv")
df_normal = df_normal.drop(['Unnamed: 0'],axis=1)
len_values = len((df_normal))
#print(len_values)
point = ((len_values))%18
if point==0:
df_normal = pd.DataFrame(df_normal.iloc[:,start:end])
else:
df_normal = pd.DataFrame(df_normal.iloc[:-point,start:end])
#df_normal = pd.DataFrame(df_normal.iloc[:-point,:])
df_normal = df_normal.T
df_normal = (df_normal.values).reshape(-1,18)
df_normal = pd.DataFrame(df_normal)
RANDOM_SEED = 101
X_train = df_normal.values
print('Training data size :', X_train.shape)
scaler = MinMaxScaler()
X_train_scaled = scaler.fit_transform(X_train)
input_dim = X_train.shape[1]
encoding_dim = 6
input_layer = Input(shape=(input_dim, ))
encoder = Dense(encoding_dim, activation="tanh",activity_regularizer=regularizers.l1(10e-5))(input_layer)
encoder = Dense(int(encoding_dim / 2), activation="tanh")(encoder)
encoder = Dense(int(2), activation="tanh")(encoder)
decoder = Dense(int(encoding_dim/ 2), activation='tanh')(encoder)
decoder = Dense(int(encoding_dim), activation='tanh')(decoder)
decoder = Dense(input_dim, activation='tanh')(decoder)
autoencoder = Model(inputs=input_layer, outputs=decoder)
#autoencoder.summary()
nb_epoch = 100
batch_size = 50
autoencoder.compile(optimizer='adam', loss='mse' )
t_ini = datetime.datetime.now()
autoencoder.fit(X_train_scaled, X_train_scaled,
epochs=nb_epoch,
batch_size=batch_size,
shuffle=True,
validation_split=0.1,
verbose=0
)
t_fin = datetime.datetime.now()
print('Time to run the model: {} Sec.'.format((t_fin - t_ini).total_seconds()))
autoencoder.save("/home/ram/Downloads/AutoEncoderModels/Keras/auto_encoder_model_19H_C2_zone_pos_" + "S" + str(start) + '_' + 'E' + str(end) + '.h5' )
print("Saved model " + str(i) + " to disk")
with open("/home/ram/Downloads/AutoEncoderScalars/auto_encoder_scalar_19H_C2_zone_pos_" + "S" + str(start) + '_' + 'E' + str(end) + ".pkl" , "wb") as outfile:
pkl.dump(scaler, outfile)
print("Saved scalar " + str(i) + " to disk")
K.clear_session()
This is my code for testing the model
from keras.models import load_model
import pickle as pkl
import numpy as np
import json
import pandas as pd
def reconstruction_error(i,X_test_scaled,autoencoder):
predictions = autoencoder.predict(X_test_scaled)
print(predictions)
mse = np.mean(np.power(X_test_scaled - predictions, 2), axis=1)
df_error = pd.DataFrame({'reconstruction_error': mse})
#print(df_error)
outliers = df_error.index[df_error.reconstruction_error > 0.1].tolist()
#print(outliers)
print("The no of anomalies for zone" + str(i) + "is" + str(len(outliers)))
global sum_outliers
sum_outliers += len(outliers)
print(sum_outliers)
def model_run(i,start,end):
df_test_2 = pd.DataFrame(df_temp.iloc[:,start:end])
df_test_T = df_test_2.T
df_test_reshaped = df_test_T.values.reshape(-1,5)
autoencoder = load_model('/home/ram/Downloads/AutoEncoderModels/Keras/auto_encoder_model_19H_C2_zone_pos_S' + str(start) + '_' + 'E' + str(end) + '.h5')
with open("/home/ram/Downloads/AutoEncoderScalars/zone" + str(i) + ".pkl", "rb") as infile:
scaler = pkl.load(infile)
X_test_scaled =scaler.transform(df_test_reshaped)
reconstruction_error(i,X_test_scaled,autoencoder)
When I run the above test data code in different computers,The reconstruction error I receive each time is different.I have set the random seed constant each time I trained the model.But how does it give a different result on test data? All the answers I viewed so far are requesting me to change the code in the training code.But here I am getting different results when I run it on test data.I am using jupyter notebook
I tried various answers in the web like shuffle= False, manual_variable_initialization(True)
But none of them seems to work