first commit

timsainb · May 13, 2019 · 2931eb8 · 2931eb8
commit 2931eb8
Show file tree

Hide file tree

Showing 21 changed files with 9,054 additions and 0 deletions.
diff --git a/0.0-Autoencoder-fashion-mnist.ipynb b/0.0-Autoencoder-fashion-mnist.ipynb
diff --git a/1.0-Variational-Autoencoder-fashion-mnist.ipynb b/1.0-Variational-Autoencoder-fashion-mnist.ipynb
diff --git a/2.0-GAN-fashion-mnist.ipynb b/2.0-GAN-fashion-mnist.ipynb
diff --git a/3.0-WGAN-GP-fashion-mnist.ipynb b/3.0-WGAN-GP-fashion-mnist.ipynb
diff --git a/4.0-seq2seq-fashion-mnist.ipynb b/4.0-seq2seq-fashion-mnist.ipynb
diff --git a/5.0-GAIA-fashion-mnist.ipynb b/5.0-GAIA-fashion-mnist.ipynb
diff --git a/6.0-VAE-GAN-fashion-mnist.ipynb b/6.0-VAE-GAN-fashion-mnist.ipynb
diff --git a/7-Copy1.0-Tensorflow-spectrograms-and-inversion.ipynb b/7-Copy1.0-Tensorflow-spectrograms-and-inversion.ipynb
diff --git a/7-Copy2.0-Tensorflow-spectrograms-and-inversion.ipynb b/7-Copy2.0-Tensorflow-spectrograms-and-inversion.ipynb
diff --git a/7.0-NSYNTH-iterator.ipynb b/7.0-NSYNTH-iterator.ipynb
@@ -0,0 +1,162 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2019-05-12T22:52:56.853920Z",
+     "start_time": "2019-05-12T22:52:52.330852Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import tensorflow as tf\n",
+    "import tensorflow_datasets as tfds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2019-05-12T22:52:56.867372Z",
+     "start_time": "2019-05-12T22:52:56.859781Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "env: CUDA_VISIBLE_DEVICES=1\n"
+     ]
+    }
+   ],
+   "source": [
+    "# make visible the only one GPU\n",
+    "%env CUDA_VISIBLE_DEVICES=1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2019-05-12T22:53:13.902647Z",
+     "start_time": "2019-05-12T22:53:13.897719Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2.0.0-dev20190510\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(tf.__version__)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Download/load dataset\n",
+    "Tensorflow datasets will automatically download or load the dataset for you at this location"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2019-05-12T23:07:00.512745Z",
+     "start_time": "2019-05-12T23:07:00.505247Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "DATASET_LOC = '/mnt/cube/tsainbur/Projects/github_repos/AVGN_419/AVGN/data/tensorflow_datasets/'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2019-05-12T23:07:01.825833Z",
+     "start_time": "2019-05-12T23:07:00.667541Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "ds_train, ds_test = tfds.load(\n",
+    "    name=\"nsynth\", split=[\"train\", \"test\"], data_dir=DATASET_LOC\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2019-05-12T23:08:08.939470Z",
+     "start_time": "2019-05-12T23:08:08.916653Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<_OptionsDataset shapes: {id: (), audio: (64000,), pitch: (), velocity: (), instrument: {label: (), family: (), source: ()}, qualities: {bright: (), dark: (), distortion: (), fast_decay: (), long_release: (), multiphonic: (), nonlinear_env: (), percussive: (), reverb: (), tempo-synced: ()}}, types: {id: tf.string, audio: tf.float32, pitch: tf.int64, velocity: tf.int64, instrument: {label: tf.int64, family: tf.int64, source: tf.int64}, qualities: {bright: tf.bool, dark: tf.bool, distortion: tf.bool, fast_decay: tf.bool, long_release: tf.bool, multiphonic: tf.bool, nonlinear_env: tf.bool, percussive: tf.bool, reverb: tf.bool, tempo-synced: tf.bool}}>"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ds_train"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/7.0-Tensorflow-spectrograms-and-inversion.ipynb b/7.0-Tensorflow-spectrograms-and-inversion.ipynb
diff --git a/8.0-seq2seq-NSYNTH.ipynb b/8.0-seq2seq-NSYNTH.ipynb
@@ -0,0 +1,32 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/imgs/ae.png b/imgs/ae.png
diff --git a/imgs/gaia.png b/imgs/gaia.png
diff --git a/imgs/gan.png b/imgs/gan.png
diff --git a/imgs/nsynth-dataset.png b/imgs/nsynth-dataset.png
diff --git a/imgs/seq2seq.png b/imgs/seq2seq.png
diff --git a/imgs/spectrogram-inversion.png b/imgs/spectrogram-inversion.png
diff --git a/imgs/vae.png b/imgs/vae.png
diff --git a/imgs/vaegan.png b/imgs/vaegan.png
diff --git a/readme.md b/readme.md
@@ -0,0 +1,46 @@
+Generative models in Tensorflow 2.X
+==============================
+
+[Tim Sainburg](https://timsainburg.com/) (PhD Candidate, UCSD, Gentner Laboratory)
+
+This is a small project to implement a number of generative models in Tensorflow 2. Layers and optimizers use Keras. The models are implemented for two datasets: [fashion MNIST](https://github.com/zalandoresearch/fashion-mnist), and [NSYNTH](https://magenta.tensorflow.org/datasets/nsynth). Networks were written with the goal of being as simple and consistent as possible while still being readable. 
+
+## Included models:
+### Autoencoder (AE) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/timsainb/tensorflow2-generative-models/)
+A simple autoencoder network.
+![an autoencoder](imgs/ae.png)
+
+### Variational Autoencoder (VAE) ([article](https://arxiv.org/abs/1312.6114)) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/timsainb/tensorflow2-generative-models/)
+The original variational autoencoder network,  using [tensorflow_probability](https://github.com/tensorflow/probability)
+![variational autoencoder](imgs/vae.png)
+
+### Generative Adversarial Network (GAN) ([article](https://arxiv.org/abs/1406.2661)) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/timsainb/tensorflow2-generative-models/)
+GANs are a form of neural network in which two sub-networks (the encoder and decoder) are trained on opposing loss functions: an encoder that is trained to produce data which is indiscernable from the true data, and a decoder that is trained to discern between the data and generated data.
+![gan](imgs/gan.png)
+
+### Wasserstein GAN with Gradient Penalty (WGAN-GP) ([article](https://arxiv.org/abs/1701.07875)) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/timsainb/tensorflow2-generative-models/)
+WGAN-GP is a GAN that improves over the original loss function to improve training stability. 
+![wgan gp](imgs/gan.png)
+
+### VAE-GAN ([article](https://arxiv.org/abs/1512.09300)) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/timsainb/tensorflow2-generative-models/)
+VAE-GAN combines the VAE and GAN to autoencode over a latent representation of data in the generator to improve over the pixelwise error function used in autoencoders. 
+![vae gan](imgs/vaegan.png)
+
+### Generative adversarial interpolative autoencoder (GAIA) ([article](https://arxiv.org/abs/1807.06650)) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/timsainb/tensorflow2-generative-models/)
+GAIA is an autoencoder trained to learn convex latent representations by adversarially training on interpolations in latent space projections of real data. 
+![generative adversarial interpolative autoencoding network](imgs/gaia.png)
+
+## Other Notebooks:
+
+### Seq2Seq Autoencoder (without attention) (Fasion MNIST: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/timsainb/tensorflow2-generative-models/) | NSYNTH: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/timsainb/tensorflow2-generative-models/))
+Seq2Seq models use recurrent neural network cells (like LSTMs) to better capture sequential organization in data. This implementation uses Convolutional Layers as input to the LSTM cells, and a single Bidirectional LSTM layer. 
+![a seq2seq bidirectional lstm in tensorflow 2.0](imgs/seq2seq.png)
+
+### Spectrogramming, Mel Scaling, and Inversion in Tensorflow [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/timsainb/tensorflow2-generative-models/)
+Tensorflow as a signal processing package that allows us to generate spectrograms from waveforms in numpy. This notebook can serve as background for the Iterator for Nsynth notebook, or for any other spectrogram inversion in Tensorflow project. Spectrogram inversion is done using the Griffin-Lim algorithm. 
+![spectrogram inversion in tensorflow 2.0](imgs/spectrogram-inversion.png)
+
+
+### Iterator for NSynth [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/timsainb/tensorflow2-generative-models/)
+The NSYNTH dataset is a set of thousands of musical notes saved as waveforms. To input these into a Seq2Seq model as spectrograms, I wrote a small dataset class that converts to spectrogram in tensorflow (using the code from the spectrogramming notebook). 
+![a dataset iterator for tensorflow 2.0](imgs/nsynth-dataset.png)