Extending TF Support for RNN & Upsample Layers

ide/static/js/content.js

@@ -286,7 +286,7 @@ class Content extends React.Component {
     var bias_params = 0;
 
     var filter_layers = ["Convolution", "Deconvolution"];
-    var fc_layers = ["InnerProduct", "Embed", "Recurrent", "LSTM"];
+    var fc_layers = ["InnerProduct", "Embed", "Recurrent", "LSTM", "GRU", "RNN"];
 
     if(filter_layers.includes(layer.info.type)) {
       // if layer is Conv or DeConv calculating total parameter of the layer using:

ide/static/js/data.js

@@ -1531,6 +1531,19 @@ export default {
         type: 'number',
         required: false
       },
+      return_sequences: { // Only Keras
+        name: 'Return Sequences',
+        value: false,
+        type: 'checkbox',
+        required: false
+      },
+      activation: { // Only Keras
+        name: 'Activation',
+        value: 'None',
+        type: 'select',
+        options: ['softmax', 'elu', 'selu', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'],
+        required: false
+      },
       caffe: {
         name: 'Available Caffe',
         value: true,
@@ -1677,6 +1690,19 @@ export default {
         type: 'number',
         required: false
       },
+      return_sequences: { // Only Keras
+        name: 'Return Sequences',
+        value: false,
+        type: 'checkbox',
+        required: false
+      },
+      activation: { // Only Keras
+        name: 'Activation',
+        value: 'None',
+        type: 'select',
+        options: ['softmax', 'elu', 'selu', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'],
+        required: false
+      },
       caffe: {
         name: 'Available Caffe',
         value: false,
@@ -1835,6 +1861,13 @@ export default {
         type: 'checkbox',
         required: false
       },
+      activation: { // Only Keras
+        name: 'Activation',
+        value: 'None',
+        type: 'select',
+        options: ['softmax', 'elu', 'selu', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'],
+        required: false
+      },
       caffe: {
         name: 'Available Caffe',
         value: true,

ide/utils/shapes.py

@@ -161,7 +161,7 @@ def get_layer_shape(layer):
     elif(layer['info']['type'] in ['Convolution', 'Pooling', 'Deconvolution', 'DepthwiseConv']):
         return filter(layer)
 
-    elif(layer['info']['type'] in ['InnerProduct', 'Recurrent', 'RNN', 'LSTM', 'Embed']):
+    elif(layer['info']['type'] in ['InnerProduct', 'Recurrent', 'RNN', 'LSTM', 'Embed', 'GRU']):
         return output(layer)
 
     elif(layer['info']['type'] == 'Flatten'):

keras_app/views/layers_export.py

@@ -45,6 +45,20 @@
     'None': None
 }
 
+activationMap = {
+    'relu': 'relu',
+    'tanh': 'tanh',
+    'softmax': 'softmax',
+    'elu': 'elu',
+    'selu': 'selu',
+    'softplus': 'softplus',
+    'softsign': 'softsign',
+    'sigmoid': 'sigmoid',
+    'hard_sigmoid': 'hard_sigmoid',
+    'linear': 'linear',
+    'None': None
+}
+
 
 # ********** Data Layers **********
 def data(layer, layer_in, layerId):
@@ -485,6 +499,9 @@ def recurrent(layer, layer_in, layerId, tensor=True):
         return_sequences = layer['params']['return_sequences']
     else:
         return_sequences = False
+    activation = None
+    if ('activation' in layer['params']):
+        activation = activationMap[layer['params']['activation']]
     if (layer['info']['type'] == 'GRU'):
         recurrent_activation = layer['params']['recurrent_activation']
         out[layerId] = GRU(units, kernel_initializer=kernel_initializer,
@@ -496,7 +513,8 @@ def recurrent(layer, layer_in, layerId, tensor=True):
                            bias_regularizer=bias_regularizer, activity_regularizer=activity_regularizer,
                            kernel_constraint=kernel_constraint, recurrent_constraint=recurrent_constraint,
                            bias_constraint=bias_constraint, use_bias=use_bias, dropout=dropout,
-                           recurrent_dropout=recurrent_dropout)
+                           recurrent_dropout=recurrent_dropout, return_sequences=return_sequences,
+                           activation=activation)
     elif (layer['info']['type'] == 'LSTM'):
         recurrent_activation = layer['params']['recurrent_activation']
         unit_forget_bias = layer['params']['unit_forget_bias']
@@ -509,7 +527,8 @@ def recurrent(layer, layer_in, layerId, tensor=True):
                             bias_regularizer=bias_regularizer, activity_regularizer=activity_regularizer,
                             kernel_constraint=kernel_constraint, recurrent_constraint=recurrent_constraint,
                             bias_constraint=bias_constraint, use_bias=use_bias, dropout=dropout,
-                            recurrent_dropout=recurrent_dropout, return_sequences=return_sequences)
+                            recurrent_dropout=recurrent_dropout, return_sequences=return_sequences,
+                            activation=activation)
     else:
         out[layerId] = SimpleRNN(units, kernel_initializer=kernel_initializer,
                                  bias_initializer=bias_initializer,
@@ -522,7 +541,8 @@ def recurrent(layer, layer_in, layerId, tensor=True):
                                  recurrent_constraint=recurrent_constraint,
                                  bias_constraint=bias_constraint,
                                  use_bias=use_bias, dropout=dropout,
-                                 recurrent_dropout=recurrent_dropout)
+                                 recurrent_dropout=recurrent_dropout, return_sequences=return_sequences,
+                                 activation=activation)
     if tensor:
         out[layerId] = out[layerId](*layer_in)
     return out

keras_app/views/layers_import.py

@@ -348,6 +348,8 @@ def Recurrent(layer):
         params['recurrent_constraint'] = layer.recurrent_constraint.__class__.__name__
     if (layer.bias_constraint):
         params['bias_constraint'] = layer.bias_constraint.__class__.__name__
+    if (layer.activation):
+        params['activation'] = layer.activation.func_name
     params['use_bias'] = layer.use_bias
     params['dropout'] = layer.dropout
     params['recurrent_dropout'] = layer.recurrent_dropout

tensorflow_app/views/import_graphdef.py

@@ -12,22 +12,36 @@
 # map from operation name(tensorflow) to layer name(caffe)
 op_layer_map = {'Placeholder': 'Input', 'Conv2D': 'Convolution', 'Conv3D': 'Convolution',
                 'MaxPool': 'Pooling', 'MaxPool3D': 'Pooling', 'AvgPool3D': 'Pooling',
-                'DepthwiseConv2dNative': 'DepthwiseConv', 'MatMul': 'InnerProduct',
-                'Prod': 'InnerProduct', 'LRN': 'LRN', 'Concat': 'Concat',
-                'AvgPool': 'Pooling', 'Reshape': 'Flatten', 'FusedBatchNorm': 'BatchNorm',
-                'Conv2DBackpropInput': 'Deconvolution'}
-
-activation_map = {'Sigmoid': 'Sigmoid', 'Softplus': 'Softplus', 'Softsign': 'Softsign',
-                  'Elu': 'ELU', 'LeakyRelu': 'ReLU', 'Softmax': 'Softmax',
-                  'Relu': 'ReLU', 'Tanh': 'TanH', 'SELU': 'SELU'}
-
-name_map = {'flatten': 'Flatten', 'dropout': 'Dropout', 'lrn': 'LRN', 'concatenate': 'Concat',
-            'batch': 'BatchNorm', 'add': 'Eltwise', 'mul': 'Eltwise'}
-
+                'AvgPool': 'Pooling', 'DepthwiseConv2dNative': 'DepthwiseConv', 'Reshape': 'Flatten',
+                'MatMul': 'InnerProduct', 'Prod': 'InnerProduct', 'FusedBatchNorm': 'BatchNorm',
+                'LRN': 'LRN', 'Concat': 'Concat', 'Conv2DBackpropInput': 'Deconvolution',
+                'ResizeNearestNeighbor': 'Upsample', 'Elu': 'ELU', 'Relu': 'ReLU', 'SELU': 'SELU',
+                'LeakyRelu': 'ReLU', 'Softsign': 'Softsign', 'Tanh': 'TanH', 'Sigmoid': 'Sigmoid',
+                'Softmax': 'Softmax', 'Softplus': 'Softplus'}
+
+name_map = {'flatten': 'Flatten', 'dropout': 'Dropout', 'batch': 'BatchNorm',
+            'add': 'Eltwise', 'mul': 'Eltwise', 'dense': 'InnerProduct',
+            'BasicLSTMCell': 'LSTM', 'LSTMCell': 'LSTM', 'BasicRNNCell': 'RNN',
+            'RNNCell': 'RNN', 'GRUCell': 'GRU', 'lstm': 'LSTM', 'simple': 'RNN',
+            'gru': 'GRU', 'concatenate': 'Concat', 'lrn': 'LRN'}
+
+# weights and bias intializer map more initializer need to be added
 initializer_map = {'random_uniform': 'RandomUniform', 'random_normal': 'RandomNormal',
                    'Const': 'Constant', 'zeros': 'Zeros', 'ones': 'Ones',
                    'eye': 'Identity', 'truncated_normal': 'TruncatedNormal'}
 
+# separate activation map for rnn layers
+activation_map = {'Relu': 'relu', 'Elu': 'elu', 'Softsign': 'softsign',
+                  'Softplus': 'softplus', 'Sigmoid': 'sigmoid', 'Tanh': 'tanh',
+                  'Softmax': 'softmax'}
+
+
+def check_rnn(node_name):
+    # check if an op is of rnn layer
+    if str(node_name).split('/')[0] == 'rnn':
+        return True
+    return False
+
 
 def get_layer_name(node_name):
     i = node_name.find('/')
@@ -42,7 +56,9 @@ def get_layer_name(node_name):
 
 def get_layer_type(node_name):
     i = node_name.find('_')
-    if i == -1:
+    if check_rnn(node_name):
+        name = 'rnn'
+    elif i == -1:
         name = str(node_name)
     else:
         name = str(node_name[:i])
@@ -166,24 +182,34 @@ def import_graph_def(request):
         graph = tf.get_default_graph()
         session = tf.Session(graph=graph)
 
+        # to hold name of most recent rnn layer
+        rnn_input_flag = False
+
         for node in graph.get_operations():
             name = get_layer_name(node.name)
+            layer_type = get_layer_type(node.name)
             if node.type == 'NoOp':
                 # if init ops is found initialize graph_def
                 init_op = session.graph.get_operation_by_name(node.name)
                 session.run(init_op)
                 continue
+            # separate case for a rnn layer
             if name not in d:
                 d[name] = {'type': [], 'input': [], 'output': [], 'params': {}}
                 order.append(name)
             if node.type in op_layer_map:
-                if (node.type == "FusedBatchNorm"):
+                if (node.type in ["FusedBatchNorm", "Softmax"]):
                     d[name]['type'] = []
-                d[name]['type'].append(op_layer_map[node.type])
-            elif node.type in activation_map:
-                d[name]['type'].append(activation_map[node.type])
+                # not allowing redundant ops in layer type
+                if (op_layer_map[node.type] not in d[name]['type']):
+                    d[name]['type'].append(op_layer_map[node.type])
             else:  # For cases where the ops are composed of only basic ops
-                layer_type = get_layer_type(node.name)
+                if layer_type == 'rnn':
+                    node_name = str(node.name).split('/')
+                    if re.match('.*CellZeroState.*', node_name[1]):
+                        layer_type = node_name[1].split('ZeroState')[0]
+                        # reset layer type of rnn layer to remove basic ops
+                        d[name]['type'] = []
                 if layer_type in name_map:
                     if name_map[layer_type] not in d[name]['type']:
                         d[name]['type'].append(name_map[layer_type])
@@ -218,7 +244,7 @@ def import_graph_def(request):
                 continue
             name = get_layer_name(node.name)
             layer = d[name]
-
+            print(node.name)
             if layer['type'][0] == 'Input':
                 input_dim = [int(dim.size) for dim in node.get_attr('shape').dim]
                 # Swapping channel value to convert NCHW/NCDHW format
@@ -227,10 +253,13 @@ def import_graph_def(request):
                 # preserving input shape to calculate deconv output shape
                 input_layer_name = node.name
                 input_layer_dim = input_dim[:]
-                temp = input_dim[1]
-                input_dim[1] = input_dim[len(input_dim) - 1]
-                input_dim[len(input_dim) - 1] = temp
-
+                for outputId in layer['output']:
+                    if (len(d[outputId]['type']) > 0 and d[outputId]['type'][0] in ['LSTM', 'RNN', 'GRU']):
+                        rnn_input_flag = True
+                if (not rnn_input_flag):
+                    temp = input_dim[1]
+                    input_dim[1] = input_dim[len(input_dim) - 1]
+                    input_dim[len(input_dim) - 1] = temp
                 layer['params']['dim'] = str(input_dim)[1:-1]
 
             elif layer['type'][0] == 'Convolution':
@@ -423,6 +452,16 @@ def import_graph_def(request):
                         return JsonResponse({'result': 'error', 'error':
                                              'Missing shape info in GraphDef'})
 
+            elif layer['type'][0] == 'Upsample':
+                layer['params']['layer_type'] = '2D'
+                # parameter identification can be improvised further
+                if str(node.name) == name + '/Const':
+                    layer['params']['size_h'] = int(
+                        node.get_attr('value').tensor_shape.dim[0].size)
+                    layer['params']['size_w'] = int(
+                        node.get_attr('value').tensor_shape.dim[0].size)
+                pass
+
             elif layer['type'][0] == 'InnerProduct':
                 # searching for weights and kernel ops to extract num_outputs
                 # of IneerProduct layer also considering repeat & stack layer
@@ -507,13 +546,90 @@ def import_graph_def(request):
             elif layer['type'][0] == 'Flatten':
                 pass
 
+            elif layer['type'][0] == 'Concat':
+                pass
+
             elif layer['type'][0] == 'Dropout':
                 if ('rate' in node.node_def.attr):
                     layer['params']['rate'] = node.get_attr('rate')
                 if ('seed' in node.node_def.attr):
                     layer['params']['seed'] = node.get_attr('seed')
                 if ('training' in node.node_def.attr):
                     layer['params']['trainable'] = node.get_attr('training')
+
+            elif layer['type'][0] == 'LSTM':
+                if re.match('.*'+name+'/Const', str(node.name)):
+                    try:
+                        layer['params']['num_output'] = node.get_attr(
+                            'value').int_val[0]
+                    except:
+                        pass
+                if re.match('.*/kernel/Initializer.*', str(node.name)):
+                    w_filler = str(node.name).split('/')[4]
+                    layer['params']['weight_filler'] = initializer_map[w_filler]
+                if re.match('.*/bias/Initializer.*', str(node.name)):
+                    b_filler = str(node.name).split('/')[4]
+                    layer['params']['bias_filler'] = initializer_map[b_filler]
+                if re.match('.*/while/.*/add/y', str(node.name)):
+                    layer['params']['unit_forget_bias'] = node.get_attr(
+                        'value').float_val[0]
+                if re.match('.*/while/.*/', str(node.name)):
+                    activation = str(node.name).split('/')
+                    if len(activation) == 4 and activation[3] in activation_map:
+                        layer['params']['recurrent_activation'] = activation_map[activation[3]]
+                if str(node.type) == "TensorArrayGatherV3":
+                    try:
+                        layer['params']['num_output'] = int(node.get_attr(
+                            'element_shape').dim[1].size)
+                    except:
+                        pass
+                layer['params']['return_sequences'] = True
+
+            elif layer['type'][0] == 'RNN':
+                if re.match('.*'+name+'/Const', str(node.name)):
+                    try:
+                        layer['params']['num_output'] = node.get_attr(
+                            'value').int_val[0]
+                    except:
+                        pass
+                if re.match('.*/kernel/Initializer.*', str(node.name)):
+                    w_filler = str(node.name).split('/')[4]
+                    layer['params']['weight_filler'] = initializer_map[w_filler]
+                if re.match('.*/bias/Initializer.*', str(node.name)):
+                    b_filler = str(node.name).split('/')[4]
+                    layer['params']['bias_filler'] = initializer_map[b_filler]
+                if str(node.type) == "TensorArrayGatherV3":
+                    try:
+                        layer['params']['num_output'] = int(node.get_attr(
+                            'element_shape').dim[1].size)
+                    except:
+                        pass
+                layer['params']['return_sequences'] = True
+
+            elif layer['type'][0] == 'GRU':
+                if re.match('.*'+name+'/Const', str(node.name)):
+                    try:
+                        layer['params']['num_output'] = node.get_attr(
+                            'value').int_val[0]
+                    except:
+                        pass
+                if re.match('.*/gates/kernel/Initializer.*', str(node.name)):
+                    w_filler = str(node.name).split('/')[5]
+                    layer['params']['weight_filler'] = initializer_map[w_filler]
+                if re.match('.*/gates/bias/Initializer.*', str(node.name)):
+                    b_filler = str(node.name).split('/')[5]
+                    layer['params']['bias_filler'] = initializer_map[b_filler]
+                if re.match('.*/while/.*/', str(node.name)):
+                    activation = str(node.name).split('/')
+                    if len(activation) == 4 and activation[3] in activation_map:
+                        layer['params']['recurrent_activation'] = activation_map[activation[3]]
+                if str(node.type) == "TensorArrayGatherV3":
+                    try:
+                        layer['params']['num_output'] = int(node.get_attr(
+                            'element_shape').dim[1].size)
+                    except:
+                        pass
+                layer['params']['return_sequences'] = True
         net = {}
         batch_norms = []
         for key in d.keys():