Commit 5f0faef9 authored by Eva Lina Fesefeldt's avatar Eva Lina Fesefeldt
Browse files

Abbildungen L-BFGS MNIST

parent a4dbd00f
# mit gegebenem Optimierer trainieren, Gewichte im Minimum abspeichern
import numpy as np
from numpy.linalg import norm
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.python.keras import optimizers
from tensorflow.python.keras.backend import learning_phase
from scipy.sparse import diags
from scipy.linalg import norm, eig, eigh, eigvalsh, eigvals
from helper import vector_flat_shape_to_trainable_shape
from hessian_lanczos import lanczos_sym_knn
from generate_dataset import generate_tictactoe
from helper import matrix_trainable_shape_to_flat_shape
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("opt", type=str) # "sgd_with_decay", "sgd", "adam", "adagrad"
parser.add_argument("--batch_size", type=int, dest="batch_size")
parser.add_argument("--LR", type=float, dest="LR")
parser.add_argument("--epochs", type=int, dest="epochs")
args = parser.parse_args()
optimizer_str = args.opt
if args.batch_size is not None:
batch_size = args.batch_size
else:
batch_size = 32
print("Setting batch size to default (32)")
if args.LR is not None:
LR = args.LR
else:
LR = 0.01
print("Setting learning rate to default (0.01)")
if args.epochs is not None:
epochs = args.epochs
else:
epochs = 10000
print("Setting batch sizeepochs to default (10000)")
print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')))
loss_array = np.zeros((100,))
lambdamax_array = np.zeros((100,))
filename_loss = "results/MSE_loss_" + optimizer_str + "_batch" + str(batch_size) + "_LR" + str(LR) + "_epochs" + str(epochs)
filename_lambdamax= "results/lambdamax_" + optimizer_str + "_batch" + str(batch_size) + "_LR" + str(LR) + "_epochs" + str(epochs)
# MNIST laden
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
# normalisieren
x_train, x_test = x_train / 255.0, x_test / 255.0
# Reshape
input_dim = 28*28
x_train = tf.reshape(x_train, (60000, 28*28))
y_train = tf.one_hot(y_train, depth=10)
for size_hidden_layer in range(50,251):
number_of_parameters = input_dim*size_hidden_layer + size_hidden_layer + 10 * size_hidden_layer + 10
model = Sequential()
model.add(Dense(size_hidden_layer, input_dim = input_dim, activation='sigmoid'))
model.add(Dense(10, input_dim=size_hidden_layer, activation='sigmoid'))
# Gewichte und Biase initialisieren, um nachher Vergleichbarkeit zu haben
filename_W1 = "initializers/W_1_n" + str(size_hidden_layer) + ".npy"
filename_b1 = "initializers/b_1_n" + str(size_hidden_layer) + ".npy"
filename_W2 = "initializers/W_2_n" + str(size_hidden_layer) + ".npy"
filename_b2 = "initializers/b_2_n" + str(size_hidden_layer) + ".npy"
W_1 = np.load(filename_W1)
b_1 = np.load(filename_b1)
W_2 = np.load(filename_W2)
b_2 = np.load(filename_b2)
list_of_weights_and_biases = [W_1, b_1, W_2, b_2]
model.set_weights(list_of_weights_and_biases)
dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
# Klassifizierer trainieren
loss_fn = tf.keras.losses.MeanSquaredError()
if optimizer_str == "sgd":
opt = tf.keras.optimizers.SGD(learning_rate=LR)
if optimizer_str == "adam":
opt = tf.keras.optimizers.Adam(learning_rate=LR)
if optimizer_str == "sgd_with_decay":
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=LR, decay_steps=10000, decay_rate=0.9)
opt = tf.keras.optimizers.SGD(learning_rate=lr_schedule)
if optimizer_str == "adagrad":
opt = tf.keras.optimizers.Adagrad(learning_rate=LR)
model.compile(optimizer = opt, loss=loss_fn)
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=0)
# Loss speichern
weights_and_bias = model.get_weights()
predictions = model.predict(x_train)
loss_array[size_hidden_layer-1] = loss_fn(y_train, predictions).numpy()
m = 30
# Lanczos-Verfahren für KNN
r_numpy = np.reshape(np.random.rand(number_of_parameters), (number_of_parameters,))
r_flat_shape = tf.constant(r_numpy, dtype='float32')
r = vector_flat_shape_to_trainable_shape(r_flat_shape)
S = np.eye(m+1,m+1)
a1, b1, Q = lanczos_sym_knn(model, x_train, y_train, r, loss_fn, S, m)
T = diags([b1, a1, b1], [-1, 0, 1], shape=(m+1, m), format='csr').toarray()
eig_T = eigvalsh(T[:m,:m])
lambda_max = max(abs(eig_T))
lambdamax_array[size_hidden_layer-1] = lambda_max
np.save(filename_loss, loss_array)
np.save(filename_lambdamax, lambdamax_array)
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment