Commit 746fb9f5 authored by Eva Lina Fesefeldt's avatar Eva Lina Fesefeldt
Browse files

Plots angepasst

parent fbd35d46
import numpy as np
from scipy.special import comb
import matplotlib.pyplot as plt
import math
# Finde alle Vektoren x \in {0,1}^n mit genau 5 Einträgen = 1 (weiß)
# Gibt eine Matrix der Größe (N,n) zurück, deren Zeilen die gesuchten Vektoren sind
def find_combinations(n,k):
# Rekursionsanfang
if k==0:
return np.zeros(n)
if n==1 & k==1:
return np.array([1])
if n==1 & k==0:
return np.array([0])
# Anzahl der möglichen Kombinationen: k aus n auswählen, Matrix anlegen
N = int(comb(n,k))
X = np.zeros((N,n))
# Setze den ersten Eintrag auf 1 (weiß) und rufe das Subproblem auf
number_of_combinations_problem_1 = int(comb(n-1,k-1))
X[0:number_of_combinations_problem_1,0] = 1
X[0:number_of_combinations_problem_1,1:n] = find_combinations(n-1,k-1)
if number_of_combinations_problem_1 == N:
return X
# Belasse den ersten Eintrag bei 0 (schwarz) und rufe das Subproblem auf
X[number_of_combinations_problem_1:,1:n] = find_combinations(n-1,k)
return X
# (weiß gewinnt, schwarz gewinnt, niemand gewinnt)
def winner_one_line(x1,x2,x3):
if x1 != x2 or x2 != x3:
return np.array([0,0,1]).T
if x1 == 1:
return np.array([1,0,0]).T
return np.array([0,1,0]).T
def one_tictactoe_label(x):
strikes = np.zeros((3, 8))
# Alle Möglichkeiten zu gewinnen
strikes[:,0] = winner_one_line(x[0], x[4], x[8]) # Diagonale
strikes[:,1] = winner_one_line(x[2], x[4], x[6]) # Antidiagonale
strikes[:,2] = winner_one_line(x[0], x[1], x[2]) # Horizontal 1
strikes[:,3] = winner_one_line(x[3], x[4], x[5]) # Horizontal 2
strikes[:,4] = winner_one_line(x[6], x[7], x[8]) # Horizontal 3
strikes[:,5] = winner_one_line(x[0], x[3], x[6]) # Vertikal 1
strikes[:,6] = winner_one_line(x[1], x[4], x[7]) # Vertikal 2
strikes[:,7] = winner_one_line(x[2], x[5], x[8]) # Vertikal 3
# Eine Farbe gewinnt, falls sie mindestens einen Strike hat und die andere Farbe keine Strikes hat
strikes_white = np.sum(strikes[0,:])
strikes_black = np.sum(strikes[1,:])
# Weiß gewinnt
if strikes_black == 0 and strikes_white > 0:
return np.array([1,0,0])
# Schwarz gewinnt
if strikes_white == 0 and strikes_black > 0:
return np.array([0,1,0])
return np.array([0,0,1])
def tictactoe_labels(X):
N,n = X.shape
labels = np.zeros((N,3))
for i in range(N):
labels[i,:] = one_tictactoe_label(X[i,:])
return labels.astype(float)
def generate_tictactoe():
n = 9
k = 5
N = int(comb(n,k))
X = np.zeros((N,n))
X = find_combinations(n,k).astype(float)
labels = tictactoe_labels(X)
return X, labels
from generate_dataset import *
from plots import show_fields
set, labels = generate_tictactoe()
index_schwarz = labels[:,1] == 1
schwarz_gewinnt = set[index_schwarz,:]
label_schwarz = labels[index_schwarz,:]
index_weiß = labels[:,0] == 1
weiß_gewinnt = set[index_weiß,:]
label_weiß = labels[index_weiß,:]
index_unentschieden = labels[:,2] == 1
unentschieden = set[index_unentschieden,:]
label_unentschieden = labels[index_unentschieden,:]
schwarz_gewinnt_auswahl = schwarz_gewinnt[:3,:]
label_schwarz_auswahl = label_schwarz[:3,:]
weiß_gewinnt_auswahl = weiß_gewinnt[:3,:]
label_weiß_auswahl = label_weiß[:3,:]
unentschieden_auswahl = unentschieden[:3,:]
label_unenschieden_auswahl = label_unentschieden[:3,:]
auswahl = np.concatenate((schwarz_gewinnt_auswahl, weiß_gewinnt_auswahl, unentschieden_auswahl))
label_auswahl = np.concatenate((label_schwarz_auswahl, label_weiß_auswahl, label_unenschieden_auswahl))
pi = np.random.permutation(9)
auswahl = auswahl[pi]
label_auswahl = label_auswahl[pi]
show_fields(auswahl, label_auswahl, 9)
print()
\ No newline at end of file
from numpy.lib.function_base import delete
import tensorflow as tf
import matplotlib as mpl
import matplotlib.pyplot as plt
import math
from generate_dataset import *
# Alle Spiegelungen und Drehungen entfernen, um den ganzen Datensatz darstellen zu können
def imshow_zero_center(image, n):
lim = tf.reduce_max(abs(image))
plt.imshow(image, vmin=-lim, vmax=lim, cmap='seismic')
plt.title("Hesse-Matrix für n = " + str(n))
plt.colorbar()
plt.show()
def vecshow_zero_center(image, title):
image = image.T
lim = tf.reduce_max(abs(image))
plt.imshow(image, vmin=-lim, vmax=lim, cmap='seismic')
plt.title(title)
plt.colorbar()
plt.show()
def show_eigenvalues_semilogy(A, n):
plt.semilogy(tf.math.abs(A), '.b')
plt.xlabel("Eigenwertindex")
plt.ylabel("Betrag des Eigenwertes")
plt.title("Eigenwerte der Hesse-Matrix für n = " + str(n))
plt.show()
def show_eigenvalues_semilogx_complex_plane(A, n):
plt.semilogx(tf.math.real(A), tf.math.imag(A), '.r', alpha=0.3)
plt.xlabel("Realteil")
plt.ylabel("Imaginärteil")
plt.title("Eigenwerte der Hesse-Matrix für n = " + str(n))
# Visualisieren des Feldes
def show_fields(X, labels, n):
#plt.rcParams.update({"text.usetex": True, "font.family": "sans-serif", "font.sans-serif": ["Helvetica"]})
mpl.rcParams.update({'font.size': 8})
plt.figure(figsize=(7, 7))
ncols = 3
nrows = math.ceil(n / ncols)
for i in range(n):
ax = plt.subplot(nrows, ncols, i + 1)
plt.imshow(X[i].reshape(3, 3))
plt.gray()
if labels[i,0] == 1:
title = 'Weiß gewinnt'
if labels[i,1] == 1:
title = 'Schwarz gewinnt'
if labels[i,2] == 1:
title = 'Unentschieden'
ax.set_title(title)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
def find_reflection_y(field, set_of_fields):
m,n = set_of_fields.shape
field = np.reshape(field, (3,3))
are_equal = False
field_reflection = np.copy(field)
field_reflection[:,0] = field[:,2]
field_reflection[:,2] = field[:,0]
for i in range(m):
test = np.reshape(set_of_fields[i,:], (3,3))
#fig, (ax1, ax2) = plt.subplots(1,2)
#ax1.imshow(field)
#ax1.set_title("Original")
#ax2.imshow(field_reflection)
#ax2.set_title("Gespiegelt y-Richtung")
#plt.show()
are_equal = np.array_equal(test, field_reflection)
if (are_equal == True):
break
return are_equal
def find_reflection_x(field, set_of_fields):
m,n = set_of_fields.shape
field = np.reshape(field, (3,3))
are_equal = False
field_reflection = np.copy(field)
field_reflection[0,:] = field[2,:]
field_reflection[2,:] = field[0,:]
#fig, (ax1, ax2) = plt.subplots(1,2)
#ax1.imshow(field)
#ax1.set_title("Original")
#ax2.imshow(field_reflection)
#ax2.set_title("Gespiegelt x-Richtung")
#plt.show()
for i in range(m):
test = np.reshape(set_of_fields[i,:], (3,3))
are_equal = np.array_equal(test, field_reflection)
if (are_equal == True):
break
return are_equal
def rot_90(test):
test_rot = np.copy(test)
test_rot[0,0] = test[0,2]
test_rot[0,1] = test[1,2]
test_rot[0,2] = test[2,2]
test_rot[1,0] = test[0,1]
test_rot[1,2] = test[2,1]
test_rot[2,0] = test[0,0]
test_rot[2,1] = test[1,0]
test_rot[2,2] = test[2,0]
return test_rot
def find_rotation(field, set_of_fields):
m,n = set_of_fields.shape
field = np.reshape(field, (3,3))
are_equal = False
for i in range(m):
test = np.reshape(set_of_fields[i,:], (3,3))
#fig, (ax1, ax2, ax3, ax4) = plt.subplots(1,4)
#ax1.imshow(test)
#ax1.set_title("Original")
# 90 Grad
field_rotation = rot_90(field)
are_equal = np.array_equal(test, field_rotation)
if (are_equal == True):
break
#ax2.imshow(field_rotation)
#.set_title("90 Grad")
# 180 Grad
field_rotation = rot_90(field_rotation)
are_equal = np.array_equal(test, field_rotation)
if (are_equal == True):
break
#ax3.imshow(field_rotation)
#ax3.set_title("180 Grad")
# 270 Grad
field_rotation = rot_90(field_rotation)
are_equal = np.array_equal(test, field_rotation)
if (are_equal == True):
break
#ax4.imshow(field_rotation)
#ax4.set_title("270 Grad")
#plt.show()
return are_equal
train_set, labels = generate_tictactoe()
i = 1
for k in range(1, 126):
# Ist die aktuelle Zeile eine Spiegelung in x-Richtung
if (find_reflection_x(train_set[i,:], train_set[:i-1,:]) == True):
train_set = np.delete(train_set, obj=i, axis=0)
labels = np.delete(labels, obj=i, axis=0)
# Ist die aktuelle Zeile eine Spiegelung in y-Richtung
elif (find_reflection_y(train_set[i,:], train_set[:i-1,:]) == True):
train_set = np.delete(train_set, obj=i, axis=0)
labels = np.delete(labels, obj=i, axis=0)
# Ist die aktuelle Zeile eine Drehung
elif (find_rotation(train_set[i,:], train_set[:i-1,:]) == True):
train_set = np.delete(train_set, obj=i, axis=0)
labels = np.delete(labels, obj=i, axis=0)
else:
i += 1
n,m = train_set.shape
mpl.rcParams.update({'font.size': 8})
plt.figure(figsize=(6, 8), constrained_layout=True)
ncols = 5
nrows = math.ceil(n / ncols)
for i in range(n):
ax = plt.subplot(nrows, ncols, i + 1)
plt.imshow(train_set[i].reshape(3, 3))
plt.gray()
if labels[i,0] == 1:
title = 'Weiß gewinnt'
if labels[i,1] == 1:
title = 'Schwarz gewinnt'
if labels[i,2] == 1:
title = 'Unentschieden'
ax.set_title(title)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
#pi = np.random.permutation(9)
#auswahl = auswahl[pi]
#label_auswahl = label_auswahl[pi]
#show_fields(auswahl, label_auswahl, 9)
# Jacobi-Vector-Products
# Forward Mode AD
import tensorflow as tf
from tensorflow.python.eager import forwardprop
from tensorflow.python.ops.gen_array_ops import reshape
def _back_over_forward_hvp(model, images, labels, vector, loss_fn):
layer1 = model.layers[0]
layer2 = model.layers[1]
x = tf.constant(images)
with tf.GradientTape() as grad_tape:
grad_tape.watch(model.trainable_variables)
with forwardprop.ForwardAccumulator(model.trainable_variables, vector) as acc:
x = layer1(x)
x = layer2(x)
loss = loss_fn(x, labels)
return grad_tape.gradient(acc.jvp(loss), model.trainable_variables)
def _tf_gradients_forward_over_back_hvp(model, images, labels, vector, loss_fn):
with tf.GradientTape() as grad_tape:
y = model(images)
loss = loss_fn(y, labels)
variables = model.trainable_variables
grads = grad_tape.gradient(loss, variables)
helpers = tf.nest.map_structure(tf.ones_like, grads)
transposing = tf.gradients(grads, variables, helpers)
return tf.gradients(transposing, helpers, vector)
def _back_over_back_hvp(model, images, labels, vector, loss_fn):
with tf.GradientTape() as outer_tape:
with tf.GradientTape() as inner_tape:
y = model(images)
loss = loss_fn(y, labels)
grads = inner_tape.gradient(loss, model.trainable_variables)
return outer_tape.gradient(
grads, model.trainable_variables, output_gradients=vector)
if __name__ == "__main__":
x = tf.constant([[2.0, 3.0], [1.0, 4.0]])
targets = tf.constant([[1.], [-1.]])
dense = tf.keras.layers.Dense(1)
dense.build([None, 2])
# dense.kernel (2 Parameter) und dense.bias (1 Parameter)
# primals: Parameter, tangents: vector in Jacobi-Vector-Produkt
#with tf.autodiff.ForwardAccumulator(primals=dense.kernel, tangents=tf.constant([[1.], [0.]])) as acc:
# loss = tf.reduce_sum((dense(x) - targets) ** 2.)
#print(acc.jvp(loss))
#HV = _back_over_forward_hvp()
\ No newline at end of file
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
mpl.rcParams.update({'font.size': 8})
Hv = np.load("results/Hv_analytisch.npy").reshape((16,1))
Hv_BOF = np.load("results/Hv_BOF_n1.npy").reshape((16,1))
Hv_nested_tapes = np.load("results/Hv_nested_tapes_n1.npy").reshape((16,1))
Hv = np.load("results/Hv_analytisch.npy").reshape((1,16))
Hv_BOF = np.load("results/Hv_BOF_n1.npy").reshape((1,16))
Hv_nested_tapes = np.load("results/Hv_nested_tapes_n1.npy").reshape((1,16))
fig, (ax1, ax2, ax3) = plt.subplots(1,3, sharey=True, figsize= (5,5))
fig, (ax1, ax2, ax3) = plt.subplots(3,1, sharey=True, figsize= (6,2), constrained_layout = False)
fig.suptitle("")
ax1.imshow(Hv)
ax2.imshow(Hv_BOF)
ax3.imshow(Hv_nested_tapes)
lim = max(np.max(np.abs(Hv)), np.max(np.abs(Hv_BOF)), np.max(np.abs(Hv_nested_tapes)))
ax1.imshow(Hv, vmin=-lim, vmax=lim, cmap="seismic")
ax1.set_title("Analytisches HVP")
ax2.imshow(Hv_nested_tapes, vmin=-lim, vmax=lim, cmap="seismic")
ax2.set_title("HVP mit expliziter AD-Hesse-Matrix")
im = ax3.imshow(Hv_BOF, vmin=-lim, vmax=lim, cmap="seismic")
ax3.set_title("Back-over-Forward HVP")
ax1.axis("off")
ax2.axis('off')
ax3.axis('off')
fig.colorbar(im, location="bottom")
#fig.colorbar()
plt.show()
\ No newline at end of file
......@@ -3,7 +3,7 @@ from generate_dataset import generate_tictactoe
from helper import matrix_trainable_shape_to_flat_shape
import numpy as np
size_hidden_layer = 5
size_hidden_layer = 2
# Erstelle Daten
train_set, train_labels = generate_tictactoe()
......
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
from numpy.core.fromnumeric import size
size_hidden_layer = 3
filename = "results/hesse_nestedgrad_n" + str(size_hidden_layer) + ".npy"
hessian = np.load(filename)
filename_n1 = "results/hesse_nestedgrad_n1.npy"
hessian_n1 = np.load(filename_n1)
lim = np.max(np.abs(hessian))
filename_n2 = "results/hesse_nestedgrad_n2.npy"
hessian_n2 = np.load(filename_n2)
plt.imshow(hessian, vmin=-lim, vmax=lim, cmap='seismic')
plt.title("Hesse-Matrix n = " +str(size_hidden_layer))
plt.colorbar()
filename_n3 = "results/hesse_nestedgrad_n3.npy"
hessian_n3 = np.load(filename_n3)
lim = max(np.max(np.abs(hessian_n1)), np.max(np.abs(hessian_n1)), np.max(np.abs(hessian_n1)))
mpl.rcParams.update({'font.size': 8})
fig, (ax1, ax2, ax3) = plt.subplots(1,3, figsize= (6,2), constrained_layout = True)
ax1.imshow(hessian_n1, vmin=-lim, vmax=lim, cmap='seismic')
ax1.set_title("n = 1")
ax2.imshow(hessian_n2, vmin=-lim, vmax=lim, cmap='seismic')
ax2.set_title("n = 2")
im = ax3.imshow(hessian_n3, vmin=-lim, vmax=lim, cmap='seismic')
ax3.set_title("n = 3")
hessian_n1_grenze1 = 9 - 0.5
hessian_n1_grenze2 = 10 - 0.5
hessian_n1_grenze3 = 13 - 0.5
hessian_n2_grenze1 = 9*2 - 0.5
hessian_n2_grenze2 = 9*2 + 2 - 0.5
hessian_n2_grenze3 = 9*2 + 2 + 3*2 - 0.5
hessian_n3_grenze1 = 9*3 - 0.5
hessian_n3_grenze2 = 9*3 + 3 - 0.5
hessian_n3_grenze3 = 9*3 + 3 + 3*3 - 0.5
ax1.vlines([hessian_n1_grenze1, hessian_n1_grenze2, hessian_n1_grenze3], -0.5, 15.5)
ax1.hlines([hessian_n1_grenze1, hessian_n1_grenze2, hessian_n1_grenze3], -0.5, 15.5)
ax2.vlines([hessian_n2_grenze1, hessian_n2_grenze2, hessian_n2_grenze3], -0.5, 28.5)
ax2.hlines([hessian_n2_grenze1, hessian_n2_grenze2, hessian_n2_grenze3], -0.5, 28.5)
ax3.vlines([hessian_n3_grenze1, hessian_n3_grenze2, hessian_n3_grenze3], -0.5, 41.5)
ax3.hlines([hessian_n3_grenze1, hessian_n3_grenze2, hessian_n3_grenze3], -0.5, 41.5)
ax1.axis('off')
ax2.axis('off')
ax3.axis('off')
fig.colorbar(im)
plt.show()
......@@ -4,13 +4,9 @@ import tensorflow as tf
import matplotlib.pyplot as plt
from generate_dataset import generate_tictactoe
from analytical_derivative_n1 import grad_and_hesse_matrix
from helper import matrix_trainable_shape_to_flat_shape, vector_flat_shape_to_trainable_shape, vector_trainable_shape_to_flat_shape
from jvp import _back_over_forward_hvp, _tf_gradients_forward_over_back_hvp, _back_over_back_hvp
from scipy.sparse import diags, triu, random
from scipy.linalg import norm, eig, eigh, eigvalsh, eigvals
from helper import matrix_trainable_shape_to_flat_shape
def eigenvalues_hesse(size_hidden_layer):
for size_hidden_layer in range(1,100):
# Erstelle Daten
train_set, train_labels = generate_tictactoe()
norms = norm(train_labels, ord=1, axis=0)
......@@ -24,16 +20,27 @@ def eigenvalues_hesse(size_hidden_layer):
model = Sequential()
model.add(Dense(size_hidden_layer, input_dim = 9,activation='sigmoid'))
model.add(Dense(3, input_dim=size_hidden_layer, activation='sigmoid'))
model.summary()
#model.summary()
# Gewichte und Biase initialisieren, um nachher Vergleichbarkeit zu haben
filename_W1 = "initializers/W_1_n" + str(size_hidden_layer) + ".npy"
filename_b1 = "initializers/b_1_n" + str(size_hidden_layer) + ".npy"
filename_W2 = "initializers/W_2_n" + str(size_hidden_layer) + ".npy"
filename_b2 = "initializers/b_2_n" + str(size_hidden_layer) + ".npy"
W_1 = np.load(filename_W1)
b_1 = np.load(filename_b1)
W_2 = np.load(filename_W2)
b_2 = np.load(filename_b2)
list_of_weights_and_biases = [W_1, b_1, W_2, b_2]
model.set_weights(list_of_weights_and_biases)
dataset = tf.data.Dataset.from_tensor_slices((train_set, train_labels))
# Klassifizierer trainieren
loss_fn = tf.keras.losses.MeanSquaredError()
model.compile(optimizer='adam', loss=loss_fn)
#model.fit(train_set, train_labels, batch_size=32, epochs=10000, verbose=0)
weights_and_bias = model.get_weights()
predictions = model.predict(train_set)
print("Loss: ", loss_fn(train_labels, predictions).numpy())
......@@ -57,29 +64,10 @@ def eigenvalues_hesse(size_hidden_layer):
hessian_nested_tapes = matrix_trainable_shape_to_flat_shape(model, h)
# Eigenwerte komplett berechnen und plotten
# Eigenwerte komplett berechnen
eig = tf.linalg.eigh(hessian_nested_tapes)
eig_axis = np.linspace(0,1,number_of_parameters)
#eig_axis = np.linspace(0,1,number_of_parameters)
return eig[0].numpy(), eig_axis
filename = "results/hesse_eigenwerte_n" + str(size_hidden_layer) + ".npy"
np.save(filename, eig[0].numpy())