def testVariableCollectionsWithArgScopeNested(self):
   with self.test_session():
     with arg_scope([variables_lib2.variable], collections='A'):
       a = variables_lib2.variable('a', [])
       with arg_scope([variables_lib2.variable], collections='B'):
         b = variables_lib2.variable('b', [])
     self.assertEquals(a, ops.get_collection('A')[0])
     self.assertEquals(b, ops.get_collection('B')[0])

 def testVariableCollectionsWithArgScopeNonNested(self):
   with self.test_session():
     with arg_scope([variables_lib2.variable], collections='A'):
       a = variables_lib2.variable('a', [])
     with arg_scope([variables_lib2.variable], collections='B'):
       b = variables_lib2.variable('b', [])
     variables_lib2.variable('c', [])
     self.assertListEqual([a], ops.get_collection('A'))
     self.assertListEqual([b], ops.get_collection('B'))

def alexnet_v2_arg_scope(weight_decay=0.0005):
  with arg_scope(
      [layers.conv2d, layers_lib.fully_connected],
      activation_fn=nn_ops.relu,
      biases_initializer=init_ops.constant_initializer(0.1),
      weights_regularizer=regularizers.l2_regularizer(weight_decay)):
    with arg_scope([layers.conv2d], padding='SAME'):
      with arg_scope([layers_lib.max_pool2d], padding='VALID') as arg_sc:
        return arg_sc

 def testReuseArgScope(self):
   func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
   key_op = _key_op(func1)
   current_scope = {key_op: func1_kwargs.copy()}
   with self.test_session():
     with arg_scope([func1], a=1, b=None, c=[1]) as scope1:
       pass
     with arg_scope(scope1) as scope:
       self.assertDictEqual(scope, current_scope)

  def testArgScopeObjectCreatedWithinScopeInheritsArgScope(self):
    def get_scope_object():
      with arg_scope([func1], a=1, b=None, c=[1]) as sc:
        return sc

    with arg_scope([func1], b=2, d=10):
      with arg_scope(get_scope_object()):
        args, kwargs = func1(0)
        self.assertTupleEqual(args, (0,))
        self.assertDictEqual(kwargs, {'a': 1, 'b': None, 'c': [1], 'd': 10})

 def testClearArgScope(self):
   func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
   key_op = _key_op(func1)
   func1_scope = {key_op: func1_kwargs.copy()}
   with self.test_session():
     with arg_scope([func1], a=1, b=None, c=[1]) as sc1:
       self.assertEqual(sc1, func1_scope)
       with arg_scope({}) as sc2:
         self.assertEqual(sc2, {})
       with arg_scope([]) as current_arg_scope:
         self.assertEqual(current_arg_scope, func1_scope)

 def testNestedArgScope(self):
   func1_args = (0,)
   func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
   with arg_scope([func1], a=1, b=None, c=[1]):
     args, kwargs = func1(0)
     self.assertTupleEqual(args, func1_args)
     self.assertDictEqual(kwargs, func1_kwargs)
     func1_kwargs['b'] = 2
     with arg_scope([func1], b=2):
       args, kwargs = func1(0)
       self.assertTupleEqual(args, func1_args)
       self.assertDictEqual(kwargs, func1_kwargs)

 def testCurrentArgScopeNested(self):
   func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
   func2_kwargs = {'b': 2, 'd': [2]}
   key = _key_op
   current_scope = {
       key(func1): func1_kwargs.copy(),
       key(func2): func2_kwargs.copy()
   }
   with self.test_session():
     with arg_scope([func1], a=1, b=None, c=[1]):
       with arg_scope([func2], b=2, d=[2]) as scope:
         self.assertDictEqual(scope, current_scope)

  def testNestedArgScopeObjectCreatedOutsideScopeOverridesArgScope(self):

    def get_scope_object():
      with arg_scope([func1], a=1, b=None, c=[1]) as sc:
        return sc

    scope_object = get_scope_object()
    with arg_scope([func1], b=2, d=10):
      with arg_scope(scope_object):
        args, kwargs = func1(0)
        self.assertTupleEqual(args, (0,))
        self.assertDictEqual(kwargs, {'a': 1, 'b': None, 'c': [1]})

def resnet_arg_scope(is_training=True,
                     weight_decay=0.0001,
                     batch_norm_decay=0.997,
                     batch_norm_epsilon=1e-5,
                     batch_norm_scale=True):
  """Defines the default ResNet arg scope.

  TODO(gpapan): The batch-normalization related default values above are
    appropriate for use in conjunction with the reference ResNet models
    released at https://github.com/KaimingHe/deep-residual-networks. When
    training ResNets from scratch, they might need to be tuned.

  Args:
    is_training: Whether or not we are training the parameters in the batch
      normalization layers of the model.
    weight_decay: The weight decay to use for regularizing the model.
    batch_norm_decay: The moving average decay when estimating layer activation
      statistics in batch normalization.
    batch_norm_epsilon: Small constant to prevent division by zero when
      normalizing activations by their variance in batch normalization.
    batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the
      activations in the batch normalization layer.

  Returns:
    An `arg_scope` to use for the resnet models.
  """
  batch_norm_params = {
      'is_training': is_training,
      'decay': batch_norm_decay,
      'epsilon': batch_norm_epsilon,
      'scale': batch_norm_scale,
      'updates_collections': ops.GraphKeys.UPDATE_OPS,
  }

  with arg_scope(
      [layers_lib.conv2d],
      weights_regularizer=regularizers.l2_regularizer(weight_decay),
      weights_initializer=initializers.variance_scaling_initializer(),
      activation_fn=nn_ops.relu,
      normalizer_fn=layers.batch_norm,
      normalizer_params=batch_norm_params):
    with arg_scope([layers.batch_norm], **batch_norm_params):
      # The following implies padding='SAME' for pool1, which makes feature
      # alignment easier for dense prediction tasks. This is also used in
      # https://github.com/facebook/fb.resnet.torch. However the accompanying
      # code of 'Deep Residual Learning for Image Recognition' uses
      # padding='VALID' for pool1. You can switch to that choice by setting
      # tf.contrib.framework.arg_scope([tf.contrib.layers.max_pool2d], padding='VALID').
      with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc:
        return arg_sc

 def testPartiallySharedArgScope(self):
   func1_args = (0,)
   func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
   func2_args = (1,)
   func2_kwargs = {'a': 1, 'b': None, 'd': [2]}
   with arg_scope([func1, func2], a=1, b=None):
     with arg_scope([func1], c=[1]):
       with arg_scope([func2], d=[2]):
         args, kwargs = func1(0)
         self.assertTupleEqual(args, func1_args)
         self.assertDictEqual(kwargs, func1_kwargs)
         args, kwargs = func2(1)
         self.assertTupleEqual(args, func2_args)
         self.assertDictEqual(kwargs, func2_kwargs)

def inception_v3_arg_scope(weight_decay=0.00004,
                           batch_norm_var_collection='moving_vars',
                           batch_norm_decay=0.9997,
                           batch_norm_epsilon=0.001,
                           updates_collections=ops.GraphKeys.UPDATE_OPS,
                           use_fused_batchnorm=True):
  """Defines the default InceptionV3 arg scope.

  Args:
    weight_decay: The weight decay to use for regularizing the model.
    batch_norm_var_collection: The name of the collection for the batch norm
      variables.
    batch_norm_decay: Decay for batch norm moving average
    batch_norm_epsilon: Small float added to variance to avoid division by zero
    updates_collections: Collections for the update ops of the layer
    use_fused_batchnorm: Enable fused batchnorm.

  Returns:
    An `arg_scope` to use for the inception v3 model.
  """
  batch_norm_params = {
      # Decay for the moving averages.
      'decay': batch_norm_decay,
      # epsilon to prevent 0s in variance.
      'epsilon': batch_norm_epsilon,
      # collection containing update_ops.
      'updates_collections': updates_collections,
      # Use fused batch norm if possible.
      'fused': use_fused_batchnorm,
      # collection containing the moving mean and moving variance.
      'variables_collections': {
          'beta': None,
          'gamma': None,
          'moving_mean': [batch_norm_var_collection],
          'moving_variance': [batch_norm_var_collection],
      }
  }

  # Set weight_decay for weights in Conv and FC layers.
  with arg_scope(
      [layers.conv2d, layers_lib.fully_connected],
      weights_regularizer=regularizers.l2_regularizer(weight_decay)):
    with arg_scope(
        [layers.conv2d],
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=nn_ops.relu,
        normalizer_fn=layers_lib.batch_norm,
        normalizer_params=batch_norm_params) as sc:
      return sc

def inception_v1_arg_scope(weight_decay=0.00004,
                           use_batch_norm=True,
                           batch_norm_var_collection='moving_vars'):
  """Defines the default InceptionV1 arg scope.

  Note: Althougth the original paper didn't use batch_norm we found it useful.

  Args:
    weight_decay: The weight decay to use for regularizing the model.
    use_batch_norm: "If `True`, batch_norm is applied after each convolution.
    batch_norm_var_collection: The name of the collection for the batch norm
      variables.

  Returns:
    An `arg_scope` to use for the inception v3 model.
  """
  batch_norm_params = {
      # Decay for the moving averages.
      'decay': 0.9997,
      # epsilon to prevent 0s in variance.
      'epsilon': 0.001,
      # collection containing update_ops.
      'updates_collections': ops.GraphKeys.UPDATE_OPS,
      # collection containing the moving mean and moving variance.
      'variables_collections': {
          'beta': None,
          'gamma': None,
          'moving_mean': [batch_norm_var_collection],
          'moving_variance': [batch_norm_var_collection],
      }
  }
  if use_batch_norm:
    normalizer_fn = layers_lib.batch_norm
    normalizer_params = batch_norm_params
  else:
    normalizer_fn = None
    normalizer_params = {}
  # Set weight_decay for weights in Conv and FC layers.
  with arg_scope(
      [layers.conv2d, layers_lib.fully_connected],
      weights_regularizer=regularizers.l2_regularizer(weight_decay)):
    with arg_scope(
        [layers.conv2d],
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=nn_ops.relu,
        normalizer_fn=normalizer_fn,
        normalizer_params=normalizer_params) as sc:
      return sc

 def testEndPointsV2(self):
   """Test the end points of a tiny v2 bottleneck network."""
   blocks = [
       resnet_v2.resnet_v2_block(
           'block1', base_depth=1, num_units=2, stride=2),
       resnet_v2.resnet_v2_block(
           'block2', base_depth=2, num_units=2, stride=1),
   ]
   inputs = create_test_input(2, 32, 16, 3)
   with arg_scope(resnet_utils.resnet_arg_scope()):
     _, end_points = self._resnet_plain(inputs, blocks, scope='tiny')
   expected = [
       'tiny/block1/unit_1/bottleneck_v2/shortcut',
       'tiny/block1/unit_1/bottleneck_v2/conv1',
       'tiny/block1/unit_1/bottleneck_v2/conv2',
       'tiny/block1/unit_1/bottleneck_v2/conv3',
       'tiny/block1/unit_2/bottleneck_v2/conv1',
       'tiny/block1/unit_2/bottleneck_v2/conv2',
       'tiny/block1/unit_2/bottleneck_v2/conv3',
       'tiny/block2/unit_1/bottleneck_v2/shortcut',
       'tiny/block2/unit_1/bottleneck_v2/conv1',
       'tiny/block2/unit_1/bottleneck_v2/conv2',
       'tiny/block2/unit_1/bottleneck_v2/conv3',
       'tiny/block2/unit_2/bottleneck_v2/conv1',
       'tiny/block2/unit_2/bottleneck_v2/conv2',
       'tiny/block2/unit_2/bottleneck_v2/conv3'
   ]
   self.assertItemsEqual(expected, end_points)

  def grad_fn(inputs, variables, outputs, output_grads):
    """Recompute outputs for gradient computation."""
    del outputs
    # Recompute outputs
    with framework_ops.control_dependencies(output_grads):
      if use_data_dep_:
        inputs = _force_data_dependency(output_grads, inputs)
      with contrib_framework_ops.arg_scope(cached_arg_scope[0]):
        with variable_scope.variable_scope(cached_vs[0], reuse=True):
          outputs = fn(*inputs)

    if not (isinstance(outputs, list) or isinstance(outputs, tuple)):
      outputs = [outputs]
    outputs = list(outputs)
    grads = gradients_impl.gradients(outputs, inputs + variables, output_grads)

    if tupleize_grads:
      if use_data_dep_:
        grads = _tuple_with_data_dep(grads)
      else:
        grads = control_flow_ops.tuple(grads)

    grad_inputs = grads[:len(inputs)]
    grad_vars = grads[len(inputs):]
    return grad_inputs, grad_vars

  def testVariableWithDeviceFunction(self):

    class DevFn(object):

      def __init__(self):
        self.counter = -1

      def __call__(self, op):
        self.counter += 1
        return 'cpu:%d' % self.counter

    with self.test_session():
      with arg_scope([variables_lib2.variable], device=DevFn()):
        a = variables_lib2.variable('a', [])
        b = variables_lib2.variable('b', [])
        c = variables_lib2.variable('c', [], device='cpu:12')
        d = variables_lib2.variable('d', [])
        with ops.device('cpu:99'):
          e_init = constant_op.constant(12)
        e = variables_lib2.variable('e', initializer=e_init)
      self.assertDeviceEqual(a.device, 'cpu:0')
      self.assertEqual(a.initial_value.op.colocation_groups(),
                       a.op.colocation_groups())
      self.assertDeviceEqual(b.device, 'cpu:1')
      self.assertEqual(b.initial_value.op.colocation_groups(),
                       b.op.colocation_groups())
      self.assertDeviceEqual(c.device, 'cpu:12')
      self.assertEqual(c.initial_value.op.colocation_groups(),
                       c.op.colocation_groups())
      self.assertDeviceEqual(d.device, 'cpu:2')
      self.assertEqual(d.initial_value.op.colocation_groups(),
                       d.op.colocation_groups())
      self.assertDeviceEqual(e.device, 'cpu:3')
      self.assertDeviceEqual(e.initial_value.device, 'cpu:99')

def slim_net_original(image, keep_prob):
    with arg_scope([layers.conv2d, layers.fully_connected], biases_initializer=tf.random_normal_initializer(stddev=0.1)):

        # conv2d(inputs, num_outputs, kernel_size, stride=1, padding='SAME',
        # activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None,
        # weights_initializer=initializers.xavier_initializer(), weights_regularizer=None,
        # biases_initializer=init_ops.zeros_initializer, biases_regularizer=None, scope=None):
        net = layers.conv2d(image, 32, [5, 5], scope='conv1', weights_regularizer=regularizers.l1_regularizer(0.5))

        # max_pool(inputs, kernel_size, stride=2, padding='VALID', scope=None)
        net = layers.max_pool2d(net, 2, scope='pool1')

        net = layers.conv2d(net, 64, [5, 5], scope='conv2', weights_regularizer=regularizers.l2_regularizer(0.5))
        summaries.summarize_tensor(net, tag='conv2')

        net = layers.max_pool2d(net, 2, scope='pool2')

        net = layers.flatten(net, scope='flatten1')

        # fully_connected(inputs, num_outputs, activation_fn=nn.relu, normalizer_fn=None,
        # normalizer_params=None, weights_initializer=initializers.xavier_initializer(),
        # weights_regularizer=None, biases_initializer=init_ops.zeros_initializer,
        # biases_regularizer=None, scope=None):
        net = layers.fully_connected(net, 1024, scope='fc1')

        # dropout(inputs, keep_prob=0.5, is_training=True, scope=None)
        net = layers.dropout(net, keep_prob=keep_prob, scope='dropout1')

        net = layers.fully_connected(net, 10, scope='fc2')
    return net

  def testVariableGPUPlacement(self):

    with ops.Graph().as_default():
      device_fn = variables_lib2.VariableDeviceChooser(device_type='GPU')
      with arg_scope([variables_lib2.variable], device=device_fn):
        a = variables_lib2.variable('a', [])
        b = variables_lib2.variable('b', [])
        c = variables_lib2.variable('c', [], device='cpu:12')
        d = variables_lib2.variable('d', [])
        with ops.device('cpu:99'):
          e_init = constant_op.constant(12)
        e = variables_lib2.variable('e', initializer=e_init)
      # The values below highlight how the VariableDeviceChooser puts initial
      # values on the same device as the variable job.
      self.assertDeviceEqual(a.device, '/gpu:0')
      self.assertEqual(a.initial_value.op.colocation_groups(),
                       a.op.colocation_groups())
      self.assertDeviceEqual(b.device, '/gpu:0')
      self.assertEqual(b.initial_value.op.colocation_groups(),
                       b.op.colocation_groups())
      self.assertDeviceEqual(c.device, '/cpu:12')
      self.assertEqual(c.initial_value.op.colocation_groups(),
                       c.op.colocation_groups())
      self.assertDeviceEqual(d.device, '/gpu:0')
      self.assertEqual(d.initial_value.op.colocation_groups(),
                       d.op.colocation_groups())
      self.assertDeviceEqual(e.device, '/gpu:0')
      self.assertDeviceEqual(e.initial_value.device, '/cpu:99')

 def _resnet_plain(self, inputs, blocks, output_stride=None, scope=None):
   """A plain ResNet without extra layers before or after the ResNet blocks."""
   with variable_scope.variable_scope(scope, values=[inputs]):
     with arg_scope([layers.conv2d], outputs_collections='end_points'):
       net = resnet_utils.stack_blocks_dense(inputs, blocks, output_stride)
       end_points = utils.convert_collection_to_dict('end_points')
       return net, end_points

def vgg_arg_scope(weight_decay=0.0005):
  """Defines the VGG arg scope.

  Args:
    weight_decay: The l2 regularization coefficient.

  Returns:
    An arg_scope.
  """
  with arg_scope(
      [layers.conv2d, layers_lib.fully_connected],
      activation_fn=nn_ops.relu,
      weights_regularizer=regularizers.l2_regularizer(weight_decay),
      biases_initializer=init_ops.zeros_initializer()):
    with arg_scope([layers.conv2d], padding='SAME') as arg_sc:
      return arg_sc