Commit 6c7ce97d authored by Saad Jbabdi's avatar Saad Jbabdi
Browse files

new deeplearning pracs

parent 45b4f532
This diff is collapsed.
This diff is collapsed.
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Generative Adversarial Networks\n",
"\n",
"Let's try to understand these fuckers\n",
"\n",
"\n",
"Network1:\n",
"\n",
"z -> G -> fake -> D -> out1\n",
"\n",
"Network2:\n",
"\n",
"x -> D -> out2\n",
"\n",
"D tries to make out1 near 0 and out2 near 1\n",
"G tries to make out1 near 1\n",
"\n",
"Alternate training D and G by training networks 1 and 2.\n",
"\n",
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from keras.models import Sequential\n",
"from keras.layers import Dense, LeakyReLU\n",
"from keras.optimizers import Adam\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"\n",
"# Change these as you see fit\n",
"noise_dim = 1\n",
"batch_size = 128\n",
"steps_per_epoch = 10\n",
"epochs = 100\n",
"data_dim = 1\n",
"optimizer = Adam(0.0002, 0.5)\n",
"\n",
"\n",
"# Generator network\n",
"# You can play with the number of layers and the depth of each layer\n",
"# To me, LeakyReLu worked better than ReLU. Seems to need to have\n",
"# non flat areas to get gradients to work well\n",
"def create_generator():\n",
" G = Sequential()\n",
" \n",
" G.add(Dense(20, input_dim=noise_dim))\n",
" G.add(LeakyReLU(0.3))\n",
" G.add(Dense(20))\n",
" G.add(LeakyReLU(0.3))\n",
" G.add(Dense(data_dim, activation='linear'))\n",
" \n",
" # Binary crossentropy does the log business\n",
" G.compile(loss='binary_crossentropy', optimizer=optimizer)\n",
" return G\n",
"\n",
"# Apparently this needs to be clever so deeper?\n",
"def create_descriminator():\n",
" D = Sequential()\n",
" \n",
" D.add(Dense(10, input_dim=data_dim))\n",
" D.add(LeakyReLU(0.2))\n",
" D.add(Dense(10))\n",
" D.add(LeakyReLU(0.2))\n",
" D.add(Dense(10))\n",
" D.add(LeakyReLU(0.2))\n",
" D.add(Dense(10))\n",
" D.add(LeakyReLU(0.2))\n",
" D.add(Dense(1, activation='sigmoid'))\n",
" \n",
" D.compile(loss='binary_crossentropy', optimizer=optimizer)\n",
" return D\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Here we actually create D and G\n",
" There is just one trick with D.trainable=False\n",
" This tells keras to not touch D when training the combined model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from keras.layers import Input\n",
"from keras.models import Model\n",
"\n",
"D = create_descriminator()\n",
"G = create_generator()\n",
"\n",
"D.trainable = False\n",
"\n",
"gan_input = Input(shape=(noise_dim,))\n",
"Fake = G(gan_input)\n",
"\n",
"gan_output = D(Fake)\n",
"\n",
"gan = Model(gan_input, gan_output)\n",
"gan.compile(loss='binary_crossentropy', optimizer=optimizer)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"# Create some data with three bumps \n",
"\n",
"n = 1000\n",
"A = np.random.random((data_dim,data_dim))\n",
"cov = A.T@A\n",
"mu1 = np.random.randn(data_dim)\n",
"mu2 = np.random.randn(data_dim)+5\n",
"mu3 = np.random.randn(data_dim)-5\n",
"data1 = np.random.multivariate_normal(mu1,cov,int(n/3))\n",
"data2 = np.random.multivariate_normal(mu2,cov,int(n/3)) \n",
"data3 = np.random.multivariate_normal(mu3,cov,int(n/3)) \n",
"data = np.concatenate((data1,data2,data3),axis=0)\n",
"\n",
"_ = plt.hist(data,100)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"# Training\n",
"\n",
"Dx = np.zeros(epochs)\n",
"DGz = np.zeros(epochs)\n",
"for epoch in range(epochs):\n",
" for batch in range(steps_per_epoch):\n",
" # random noise realisation\n",
" z = np.random.normal(0, 1, size=(batch_size, noise_dim))\n",
" # fake data = G(z)\n",
" fake_x = G.predict(z)\n",
" # real data = some batch of data\n",
" real_x = data[np.random.randint(0, data.shape[0], size=batch_size)]\n",
" # concatenate real and fake data\n",
" x = np.concatenate((real_x, fake_x))\n",
"\n",
" # target for D\n",
" y_D = np.zeros(2*batch_size)\n",
" y_D[:batch_size] = 1 # D=1 for real\n",
"\n",
" # train D\n",
" D_loss = D.train_on_batch(x, y_D)\n",
"\n",
" # target for G is all 1's for noise \n",
" y_G = np.ones(batch_size)\n",
" \n",
" # train G\n",
"# G_loss = gan.train_on_batch(z, y_G)\n",
" \n",
" \n",
"\n",
" print(f'Epoch: {epoch} \\t Discriminator Loss: {D_loss} \\t\\t Generator Loss: {G_loss}')\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Compare data distrib to G distribution\n",
"# Also overlay D(x)\n",
"# Also overlay pdata/(pdata+pfake)\n",
"\n",
"bins = np.linspace(np.quantile(data,.05),np.quantile(data,.95),60)\n",
"pdata,_,_ = plt.hist(data,bins=bins,density=True,alpha=.5)\n",
"pdata = pdata/np.sum(pdata)\n",
"z = np.random.randn(1000)\n",
"Gz = G.predict(z)\n",
"\n",
"pfake,_,_ = plt.hist(Gz,bins=bins,density=True,alpha=.5)\n",
"pfake = pfake/np.sum(pfake)\n",
"\n",
"plt.plot(bins,D.predict(bins))\n",
"plt.plot(bins[1:],pdata/(pdata+pfake))\n",
"\n",
"plt.legend(['D(x)','pdata/(pdata+pfake)','data dist','fake dist'])\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"\n",
"\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.11"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
%% Cell type:markdown id: tags:
# Generative Adversarial Networks
Let's try to understand these fuckers
Network1:
z -> G -> fake -> D -> out1
Network2:
x -> D -> out2
D tries to make out1 near 0 and out2 near 1
G tries to make out1 near 1
Alternate training D and G by training networks 1 and 2.
%% Cell type:markdown id: tags:
%% Cell type:code id: tags:
``` python
from keras.models import Sequential
from keras.layers import Dense, LeakyReLU
from keras.optimizers import Adam
import numpy as np
import matplotlib.pyplot as plt
# Change these as you see fit
noise_dim = 1
batch_size = 128
steps_per_epoch = 10
epochs = 100
data_dim = 1
optimizer = Adam(0.0002, 0.5)
# Generator network
# You can play with the number of layers and the depth of each layer
# To me, LeakyReLu worked better than ReLU. Seems to need to have
# non flat areas to get gradients to work well
def create_generator():
G = Sequential()
G.add(Dense(20, input_dim=noise_dim))
G.add(LeakyReLU(0.3))
G.add(Dense(20))
G.add(LeakyReLU(0.3))
G.add(Dense(data_dim, activation='linear'))
# Binary crossentropy does the log business
G.compile(loss='binary_crossentropy', optimizer=optimizer)
return G
# Apparently this needs to be clever so deeper?
def create_descriminator():
D = Sequential()
D.add(Dense(10, input_dim=data_dim))
D.add(LeakyReLU(0.2))
D.add(Dense(10))
D.add(LeakyReLU(0.2))
D.add(Dense(10))
D.add(LeakyReLU(0.2))
D.add(Dense(10))
D.add(LeakyReLU(0.2))
D.add(Dense(1, activation='sigmoid'))
D.compile(loss='binary_crossentropy', optimizer=optimizer)
return D
```
%% Cell type:markdown id: tags:
### Here we actually create D and G
There is just one trick with D.trainable=False
This tells keras to not touch D when training the combined model
%% Cell type:code id: tags:
``` python
from keras.layers import Input
from keras.models import Model
D = create_descriminator()
G = create_generator()
D.trainable = False
gan_input = Input(shape=(noise_dim,))
Fake = G(gan_input)
gan_output = D(Fake)
gan = Model(gan_input, gan_output)
gan.compile(loss='binary_crossentropy', optimizer=optimizer)
```
%% Cell type:code id: tags:
``` python
# Create some data with three bumps
n = 1000
A = np.random.random((data_dim,data_dim))
cov = A.T@A
mu1 = np.random.randn(data_dim)
mu2 = np.random.randn(data_dim)+5
mu3 = np.random.randn(data_dim)-5
data1 = np.random.multivariate_normal(mu1,cov,int(n/3))
data2 = np.random.multivariate_normal(mu2,cov,int(n/3))
data3 = np.random.multivariate_normal(mu3,cov,int(n/3))
data = np.concatenate((data1,data2,data3),axis=0)
_ = plt.hist(data,100)
```
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
# Training
Dx = np.zeros(epochs)
DGz = np.zeros(epochs)
for epoch in range(epochs):
for batch in range(steps_per_epoch):
# random noise realisation
z = np.random.normal(0, 1, size=(batch_size, noise_dim))
# fake data = G(z)
fake_x = G.predict(z)
# real data = some batch of data
real_x = data[np.random.randint(0, data.shape[0], size=batch_size)]
# concatenate real and fake data
x = np.concatenate((real_x, fake_x))
# target for D
y_D = np.zeros(2*batch_size)
y_D[:batch_size] = 1 # D=1 for real
# train D
D_loss = D.train_on_batch(x, y_D)
# target for G is all 1's for noise
y_G = np.ones(batch_size)
# train G
# G_loss = gan.train_on_batch(z, y_G)
print(f'Epoch: {epoch} \t Discriminator Loss: {D_loss} \t\t Generator Loss: {G_loss}')
```
%% Cell type:code id: tags:
``` python
# Compare data distrib to G distribution
# Also overlay D(x)
# Also overlay pdata/(pdata+pfake)
bins = np.linspace(np.quantile(data,.05),np.quantile(data,.95),60)
pdata,_,_ = plt.hist(data,bins=bins,density=True,alpha=.5)
pdata = pdata/np.sum(pdata)
z = np.random.randn(1000)
Gz = G.predict(z)
pfake,_,_ = plt.hist(Gz,bins=bins,density=True,alpha=.5)
pfake = pfake/np.sum(pfake)
plt.plot(bins,D.predict(bins))
plt.plot(bins[1:],pdata/(pdata+pfake))
plt.legend(['D(x)','pdata/(pdata+pfake)','data dist','fake dist'])
```
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
```
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment