Source code for ivy.neural_net_stateful.initializers

# local
import ivy


# Constant #
# ---------#

[docs]class Constant:
[docs] def __init__(self, constant): self._constant = constant
[docs] def create_variables(self, var_shape, dev_str, fan_out=None, fan_in=None): """ Create internal variables for the layer """ return ivy.variable(ivy.ones(var_shape, dev_str=dev_str) * self._constant)
[docs]class Zeros(Constant):
[docs] def __init__(self): super().__init__(0.)
[docs]class Ones(Constant):
[docs] def __init__(self): super().__init__(1.)
# Uniform # # --------#
[docs]class Uniform:
[docs] def __init__(self, numerator, fan_mode, power, gain): if fan_mode not in ['fan_in', 'fan_out', 'fan_sum', 'fan_avg']: raise Exception('Invalid fan mode, must be one of [ fan_in | fan_out | fan_sum | fan_avg ]') self._numerator = numerator self._fan_mode = fan_mode self._power = power self._gain = gain
[docs] def create_variables(self, var_shape, dev_str, fan_out=None, fan_in=None): """ Create internal variables for the layer """ if self._fan_mode == 'fan_in': if fan_in is None: raise Exception('input_channels must be specified for fan_in denominator mode.') fan = fan_in elif self._fan_mode == 'fan_out': if fan_in is None: raise Exception('output_channels must be specified for fan_out denominator mode.') fan = fan_out elif self._fan_mode == 'fan_sum': if fan_in is None or fan_out is None: raise Exception('input_channels and output_channels must both be specified for' 'fan_sum denominator mode.') fan = fan_in + fan_out elif self._fan_mode == 'fan_avg': if fan_in is None or fan_out is None: raise Exception('input_channels and output_channels must both be specified for' 'fan_avg denominator mode.') fan = (fan_in + fan_out) / 2 else: raise Exception('Invalid denominator mode, must be one of [ fan_in | fan_out | fan_sum | fan_avg ]') wlim = ((self._numerator / fan) ** self._power) * self._gain return ivy.variable(ivy.random_uniform(-wlim, wlim, var_shape, dev_str=dev_str))
[docs]class GlorotUniform(Uniform):
[docs] def __init__(self): super().__init__(6, 'fan_sum', 0.5, 1)
[docs]class FirstLayerSiren(Uniform):
[docs] def __init__(self): super().__init__(1, 'fan_in', 1, 1)
[docs]class Siren(Uniform):
[docs] def __init__(self, w0=30): super().__init__(6, 'fan_in', 0.5, 1/w0)
# Gaussian # # ---------#
[docs]class KaimingNormal:
[docs] def __init__(self, mean=0, fan_mode='fan_in'): if fan_mode not in ['fan_in', 'fan_out', 'fan_sum', 'fan_avg']: raise Exception('Invalid fan mode, must be one of [ fan_in | fan_out | fan_sum | fan_avg ]') self._mean = mean self._fan_mode = fan_mode
[docs] def create_variables(self, var_shape, dev_str, fan_out=None, fan_in=None, negative_slope=0.): """ Create internal variables for the layer """ if self._fan_mode == 'fan_in': if fan_in is None: raise Exception('input_channels must be specified for fan_in denominator mode.') fan = fan_in elif self._fan_mode == 'fan_out': if fan_in is None: raise Exception('output_channels must be specified for fan_out denominator mode.') fan = fan_out elif self._fan_mode == 'fan_sum': if fan_in is None or fan_out is None: raise Exception('input_channels and output_channels must both be specified for' 'fan_sum denominator mode.') fan = fan_in + fan_out elif self._fan_mode == 'fan_avg': if fan_in is None or fan_out is None: raise Exception('input_channels and output_channels must both be specified for' 'fan_avg denominator mode.') fan = (fan_in + fan_out) / 2 else: raise Exception('Invalid denominator mode, must be one of [ fan_in | fan_out | fan_sum | fan_avg ]') std = (2/((1+negative_slope**2)*fan)) ** 0.5 return ivy.variable(ivy.random_normal(self._mean, std, var_shape, dev_str=dev_str))