# noqa

"""Common ML and ML-adjacent algorithms implemented in NumPy"""

from . import utils

from . import preprocessing

from . import gmm

from . import hmm

from . import lda

from . import linear_models

from . import neural_nets

from . import ngram

from . import nonparametric

from . import rl_models

from . import trees

from . import bandits

from . import factorization

from . import tests


"""A Gaussian mixture model class"""

import numpy as np

from numpy_ml.utils.misc import logsumexp, log_gaussian_pdf

class GMM(object):

def __init__(self, C=3, seed=None):

A Gaussian mixture model trained via the expectation maximization

algorithm.

Parameters

C : int

The number of clusters / mixture components in the GMM. Default is

seed : int

Seed for the random number generator. Default is None.

Attributes

N : int

The number of examples in the training dataset.

d : int

The dimension of each example in the training dataset.

pi : :py:class:`ndarray <numpy.ndarray>` of shape `(C,)`

The cluster priors.

Q : :py:class:`ndarray <numpy.ndarray>` of shape `(N, C)`

The variational distribution `q(T)`.

mu : :py:class:`ndarray <numpy.ndarray>` of shape `(C, d)`

The cluster means.

sigma : :py:class:`ndarray <numpy.ndarray>` of shape `(C, d, d)`

The cluster covariance matrices.

self.elbo = None

self.parameters = {}

self.hyperparameters = {

"C": C,

"seed": seed,

self.is_fit = False

if seed:

np.random.seed(seed)

def _initialize_params(self, X):

"""Randomly initialize the starting GMM parameters."""

N, d = X.shape

C = self.hyperparameters["C"]

rr = np.random.rand(C)

self.parameters = {

"pi": rr / rr.sum(),  # cluster priors

"Q": np.zeros((N, C)),  # variational distribution q(T)

"mu": np.random.uniform(-5, 10, C * d).reshape(C, d),  # cluster means

"sigma": np.array([np.eye(d) for _ in range(C)]),  # cluster covariances

self.elbo = None

self.is_fit = False

def likelihood_lower_bound(self, X):

"""Compute the LLB under the current GMM parameters."""

N = X.shape[0]

P = self.parameters

C = self.hyperparameters["C"]

pi, Q, mu, sigma = P["pi"], P["Q"], P["mu"], P["sigma"]

eps = np.finfo(float).eps

expec1, expec2 = 0.0, 0.0

for i in range(N):

x_i = X[i]

for c in range(C):

pi_k = pi[c]

z_nk = Q[i, c]

mu_k = mu[c, :]

sigma_k = sigma[c, :, :]

log_pi_k = np.log(pi_k + eps)

log_p_x_i = log_gaussian_pdf(x_i, mu_k, sigma_k)

prob = z_nk * (log_p_x_i + log_pi_k)

expec1 += prob

expec2 += z_nk * np.log(z_nk + eps)

loss = expec1 - expec2

return loss

def fit(self, X, max_iter=100, tol=1e-3, verbose=False):

Fit the parameters of the GMM on some training data.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, d)`

A collection of `N` training data points, each with dimension `d`.

max_iter : int

The maximum number of EM updates to perform before terminating

training. Default is 100.

tol : float

The convergence tolerance. Training is terminated if the difference

in VLB between the current and previous iteration is less than

`tol`. Default is 1e-3.

verbose : bool

Whether to print the VLB at each training iteration. Default is

False.

Returns

success : {0, -1}

Whether training terminated without incident (0) or one of the

mixture components collapsed and training was halted prematurely

prev_vlb = -np.inf

for _iter in range(max_iter):

try:

vlb = self.likelihood_lower_bound(X)

if verbose:

print(f"{_iter + 1}. Lower bound: {vlb}")

converged = _iter > 0 and np.abs(vlb - prev_vlb) <= tol

if np.isnan(vlb) or converged:

break

prev_vlb = vlb

except np.linalg.LinAlgError:

print("Singular matrix: components collapsed")

return -1

self.elbo = vlb

self.is_fit = True

return 0

def predict(self, X, soft_labels=True):

Return the log probability of each data point in `X` under each

mixture components.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(M, d)`

A collection of `M` data points, each with dimension `d`.

soft_labels : bool

If True, return the log probabilities of the M data points in X

under each mixture component. If False, return only the ID of the

most probable mixture. Default is True.

Returns

y : :py:class:`ndarray <numpy.ndarray>` of shape `(M, C)` or `(M,)`

If `soft_labels` is True, `y` is a 2D array where index (i,j) gives

the log probability of the `i` th data point under the `j` th

mixture component. If `soft_labels` is False, `y` is a 1D array

where the `i` th index contains the ID of the most probable mixture

component.

assert self.is_fit, "Must call the `.fit` method before making predictions"

P = self.parameters

C = self.hyperparameters["C"]

mu, sigma = P["mu"], P["sigma"]

y = []

for x_i in X:

cprobs = [log_gaussian_pdf(x_i, mu[c, :], sigma[c, :, :]) for c in range(C)]

if not soft_labels:

y.append(np.argmax(cprobs))

else:

y.append(cprobs)

return np.array(y)

def _E_step(self, X):

P = self.parameters

C = self.hyperparameters["C"]

for i, x_i in enumerate(X):

denom_vals = []

for c in range(C):

pi_c = pi[c]

mu_c = mu[c, :]

sigma_c = sigma[c, :, :]

log_pi_c = np.log(pi_c)

log_p_x_i = log_gaussian_pdf(x_i, mu_c, sigma_c)

# log N(X_i | mu_c, Sigma_c) + log pi_c

denom_vals.append(log_p_x_i + log_pi_c)

# log \sum_c exp{ log N(X_i | mu_c, Sigma_c) + log pi_c } ]

log_denom = logsumexp(denom_vals)

q_i = np.exp([num - log_denom for num in denom_vals])

np.testing.assert_allclose(np.sum(q_i), 1, err_msg="{}".format(np.sum(q_i)))

Q[i, :] = q_i

def _M_step(self, X):

N, d = X.shape

P = self.parameters

C = self.hyperparameters["C"]

denoms = np.sum(Q, axis=0)

# update cluster priors

pi = denoms / N

# update cluster means

nums_mu = [np.dot(Q[:, c], X) for c in range(C)]

for ix, (num, den) in enumerate(zip(nums_mu, denoms)):

mu[ix, :] = num / den if den > 0 else np.zeros_like(num)

# update cluster covariances

for c in range(C):

mu_c = mu[c, :]

n_c = denoms[c]

outer = np.zeros((d, d))

for i in range(N):

wic = Q[i, c]

xi = X[i, :]

outer += wic * np.outer(xi - mu_c, xi - mu_c)

outer = outer / n_c if n_c > 0 else outer

sigma[c, :, :] = outer

np.testing.assert_allclose(np.sum(pi), 1, err_msg="{}".format(np.sum(pi)))


from .gmm import *


from ..utils.kernels import KernelInitializer

class KernelRegression:

def __init__(self, kernel=None):

A Nadaraya-Watson kernel regression model.

Notes

The Nadaraya-Watson regression model is

.. math::

f(x) = \sum_i w_i(x) y_i

where the sample weighting functions, :math:`w_i`, are simply

.. math::

w_i(x) = \\frac{k(x, x_i)}{\sum_j k(x, x_j)}

with `k` being the kernel function.

Observe that `k`-nearest neighbors

(:class:`~numpy_ml.nonparametric.KNN`) regression is a special case of

kernel regression where the `k` closest observations have a weight

`1/k`, and all others have weight 0.

Parameters

kernel : str, :doc:`Kernel <numpy_ml.utils.kernels>` object, or dict

The kernel to use. If None, default to

:class:`~numpy_ml.utils.kernels.LinearKernel`. Default is None.

self.parameters = {"X": None, "y": None}

self.hyperparameters = {"kernel": str(kernel)}

self.kernel = KernelInitializer(kernel)()

def fit(self, X, y):

Fit the regression model to the data and targets in `X` and `y`.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, M)`

An array of N examples to generate predictions on

y : :py:class:`ndarray <numpy.ndarray>` of shape `(N, ...)`

Predicted targets for the `N` rows in `X`

self.parameters = {"X": X, "y": y}

def predict(self, X):

Generate predictions for the targets associated with the rows in `X`.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N', M')`

An array of `N'` examples to generate predictions on

Returns

y : :py:class:`ndarray <numpy.ndarray>` of shape `(N', ...)`

Predicted targets for the `N'` rows in `X`

K = self.kernel

P = self.parameters

sim = K(P["X"], X)

return (sim * P["y"][:, None]).sum(axis=0) / sim.sum(axis=0)


import warnings

import numpy as np

from numpy.linalg import slogdet, inv

try:

_SCIPY = True

from scipy.stats import norm

except:

_SCIPY = False

warnings.warn(

"Could not import scipy.stats. Confidence scores "

"for GPRegression are restricted to 95% bounds"

from ..utils.kernels import KernelInitializer

class GPRegression:

def __init__(self, kernel="RBFKernel", alpha=1e-10):

A Gaussian Process (GP) regression model.

.. math::

y \mid X, f  &\sim  \mathcal{N}( [f(x_1), \ldots, f(x_n)], \\alpha I ) \\\\

f \mid X     &\sim  \\text{GP}(0, K)

for data :math:`D = \{(x_1, y_1), \ldots, (x_n, y_n) \}` and a covariance matrix :math:`K_{ij}

= \\text{kernel}(x_i, x_j)` for all :math:`i, j \in \{1, \ldots, n \}`.

Parameters

kernel : str

The kernel to use in fitting the GP prior. Default is 'RBFKernel'.

alpha : float

An isotropic noise term for the diagonal in the GP covariance, `K`.

Larger values correspond to the expectation of greater noise in the

observed data points. Default is 1e-10.

self.kernel = KernelInitializer(kernel)()

self.parameters = {"GP_mean": None, "GP_cov": None, "X": None}

self.hyperparameters = {"kernel": str(self.kernel), "alpha": alpha}

def fit(self, X, y):

Fit the GP prior to the training data.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, M)`

A training dataset of `N` examples, each with dimensionality `M`.

y : :py:class:`ndarray <numpy.ndarray>` of shape `(N, O)`

A collection of real-valued training targets for the

examples in `X`, each with dimension `O`.

mu = np.zeros(X.shape[0])

K = self.kernel(X, X)

self.parameters["X"] = X

self.parameters["y"] = y

self.parameters["GP_cov"] = K

self.parameters["GP_mean"] = mu

def predict(self, X, conf_interval=0.95, return_cov=False):

Return the MAP estimate for :math:`y^*`, corresponding the mean/mode of

the posterior predictive distribution, :math:`p(y^* \mid x^*, X, y)`.

Notes

Under the GP regression model, the posterior predictive distribution is

.. math::

y^* \mid x^*, X, y \sim \mathcal{N}(\mu^*, \\text{cov}^*)

where

.. math::

\mu^*  &=  K^* (K + \\alpha I)^{-1} y \\\\

\\text{cov}^*  &=  K^{**} - K^{*'} (K + \\alpha I)^{-1} K^*

and

.. math::

K  &=  \\text{kernel}(X, X) \\\\

K^*  &=  \\text{kernel}(X, X^*) \\\\

K^{**}  &=  \\text{kernel}(X^*, X^*)

NB. This implementation uses the inefficient but general purpose

`np.linalg.inv` routine to invert :math:`(K + \\alpha I)`. A more

efficient way is to rely on the fact that `K` (and hence also :math:`K

+ \\alpha I`) is symmetric positive (semi-)definite and take the inner

product of the inverse of its (lower) Cholesky decompositions:

.. math::

Q^{-1} = \\text{cholesky}(Q)^{-1 \\top} \\text{cholesky}(Q)^{-1}

For more details on a production-grade implementation, see Algorithm

2.1 in Rasmussen & Williams (2006).

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape (N, M)

conf_interval : float in (0, 1)

The percentage confidence bound to return for each prediction. If

the scipy package is not available, this value is always set to

0.95. Default is 0.95.

return_cov : bool

If True, also return the covariance (`cov*`) of the posterior

predictive distribution for the points in `X`. Default is False.

Returns

y_pred : :py:class:`ndarray <numpy.ndarray>` of shape `(N, O)`

The predicted values for each point in `X`, each with

dimensionality `O`.

conf : :py:class:`ndarray <numpy.ndarray>` of shape `(N, O)`

The % conf_interval confidence bound for each `y_pred`. The conf %

confidence interval for the `i`'th prediction is ``[y[i] - conf[i],

y[i] + conf[i]]``.

cov : :py:class:`ndarray <numpy.ndarray>` of shape `(N, N)`

The covariance (`cov*`) of the posterior predictive distribution for

`X`. Only returned if `return_cov` is True.

if conf_interval != 0.95 and not _SCIPY:

fstr = "Cannot compute {}% confidence score without scipy.stats"

warnings.warn(fstr.format(conf_interval))

X_star = X

X = self.parameters["X"]

y = self.parameters["y"]

K = self.parameters["GP_cov"]

alpha = self.hyperparameters["alpha"]

K_star = self.kernel(X_star, X)

K_star_star = self.kernel(X_star, X_star)

sig = np.eye(K.shape[0]) * alpha

K_y_inv = inv(K + sig)

pp_mean = K_star @ K_y_inv @ y

pp_cov = K_star_star - K_star @ K_y_inv @ K_star.T

# if we can't use scipy, ignore the passed value for `conf_interval`

# and return the 95% confidence bound.

# (norm.ppf == inverse CDF for standard normal)

percentile = 1.96 if not _SCIPY else norm.ppf(conf_interval)

conf = percentile * np.sqrt(np.diag(pp_cov))

return (pp_mean, conf) if not return_cov else (pp_mean, conf, pp_cov)

def marginal_log_likelihood(self, kernel_params=None):

Compute the log of the marginal likelihood (i.e., the log model

evidence), :math:`p(y \mid X, \\text{kernel_params})`.

Notes

Under the GP regression model, the marginal likelihood is normally

distributed:

.. math::

y | X, \\theta  \sim  \mathcal{N}(0, K + \\alpha I)

Hence,

.. math::

\log p(y \mid X, \\theta) =

-0.5 \log \det(K + \\alpha I) -

0.5 y^\\top (K + \\alpha I)^{-1} y + \\frac{n}{2} \log 2 \pi

where :math:`K = \\text{kernel}(X, X)`, :math:`\\theta` is the set of

kernel parameters, and `n` is the number of dimensions in `K`.

Parameters

kernel_params : dict

Parameters for the kernel function. If None, calculate the

marginal likelihood under the kernel parameters defined at model

initialization. Default is None.

Returns

marginal_log_likelihood : float

The log likelihood of the training targets given the kernel

parameterized by `kernel_params` and the training inputs,

marginalized over all functions `f`.

X = self.parameters["X"]

y = self.parameters["y"]

alpha = self.hyperparameters["alpha"]

K = self.parameters["GP_cov"]

if kernel_params is not None:

# create a new kernel with parameters `kernel_params` and recalc

# the GP covariance matrix

summary_dict = self.kernel.summary_dict()

summary_dict["parameters"].update(kernel_params)

kernel = KernelInitializer(summary_dict)()

K = kernel(X, X)

# add isotropic noise to kernel diagonal

K += np.eye(K.shape[0]) * alpha

Kinv = inv(K)

Klogdet = -0.5 * slogdet(K)[1]

const = K.shape[0] / 2 * np.log(2 * np.pi)

# handle both uni- and multidimensional target values

if y.ndim == 1:

y = y[:, np.newaxis]

# sum over each dimension of y

marginal_ll = np.sum([Klogdet - 0.5 * _y.T @ Kinv @ _y - const for _y in y.T])

return marginal_ll

def sample(self, X, n_samples=1, dist="posterior_predictive"):

Sample functions from the GP prior or posterior predictive

distribution.

Parameters

The collection of datapoints to generate predictions on. Only used if

`dist` = 'posterior_predictive'.

n_samples: int

The number of samples to generate. Default is 1.

dist : {"posterior_predictive", "prior"}

The distribution to draw samples from. Default is

Returns

samples : :py:class:`ndarray <numpy.ndarray>` of shape `(n_samples, O, N)`

The generated samples for the points in `X`.

mvnorm = np.random.multivariate_normal

if dist == "prior":

mu = np.zeros((X.shape[0], 1))

cov = self.kernel(X, X)

elif dist == "posterior_predictive":

mu, _, cov = self.predict(X, return_cov=True)

else:

raise ValueError("Unrecognized dist: '{}'".format(dist))

if mu.ndim == 1:

mu = mu[:, np.newaxis]

samples = np.array([mvnorm(_mu, cov, size=n_samples) for _mu in mu.T])

return samples.swapaxes(0, 1)


Popular nonparameteric regression and classification models.

The nonparametric module contains an assortment of nonparametric models that

don't fit elsewhere in the package. For other nonparametric models, see the

``numpy_ml.trees`` module.

from .gp import *

from .knn import *

from .kernel_regression import *


"""A k-Nearest Neighbors (KNN) model for both classiciation and regression."""

from collections import Counter

import numpy as np

from ..utils.data_structures import BallTree

class KNN:

def __init__(

self, k=5, leaf_size=40, classifier=True, metric=None, weights="uniform",

A `k`-nearest neighbors (kNN) model relying on a ball tree for efficient

computation.

Parameters

k : int

The number of neighbors to use during prediction. Default is 5.

leaf_size : int

The maximum number of datapoints at each leaf in the ball tree.

Default is 40.

classifier : bool

Whether to treat the values in Y as class labels (classifier =

True) or real-valued targets (classifier = False). Default is True.

metric : :doc:`Distance metric <numpy_ml.utils.distance_metrics>` or None

The distance metric to use for computing nearest neighbors. If

None, use the :func:`~numpy_ml.utils.distance_metrics.euclidean`

metric by default. Default is None.

weights : {'uniform', 'distance'}

How to weight the predictions from each neighbors. 'uniform'

assigns uniform weights to each neighbor, while 'distance' assigns

weights proportional to the inverse of the distance from the query

point. Default is 'uniform'.

self._ball_tree = BallTree(leaf_size=leaf_size, metric=metric)

self.hyperparameters = {

"id": "KNN",

"k": k,

"leaf_size": leaf_size,

"classifier": classifier,

"metric": str(metric),

"weights": weights,

def fit(self, X, y):

r"""

Fit the model to the data and targets in `X` and `y`

Parameters

X : numpy array of shape `(N, M)`

An array of `N` examples to generate predictions on.

y : numpy array of shape `(N, *)`

Targets for the `N` rows in `X`.

if X.ndim != 2:

raise Exception("X must be two-dimensional")

self._ball_tree.fit(X, y)

def predict(self, X):

r"""

Generate predictions for the targets associated with the rows in `X`.

Parameters

X : numpy array of shape `(N', M')`

An array of `N'` examples to generate predictions on.

Returns

y : numpy array of shape `(N', *)`

Predicted targets for the `N'` rows in `X`.

predictions = []

H = self.hyperparameters

for x in X:

pred = None

nearest = self._ball_tree.nearest_neighbors(H["k"], x)

targets = [n.val for n in nearest]

if H["classifier"]:

if H["weights"] == "uniform":

# for consistency with sklearn / scipy.stats.mode, return

# the smallest class ID in the event of a tie

counts = Counter(targets).most_common()

pred, _ = sorted(counts, key=lambda x: (-x[1], x[0]))[0]

elif H["weights"] == "distance":

best_score = -np.inf

for label in set(targets):

scores = [1 / n.distance for n in nearest if n.val == label]

pred = label if np.sum(scores) > best_score else pred

else:

if H["weights"] == "uniform":

pred = np.mean(targets)

elif H["weights"] == "distance":

weights = [1 / n.distance for n in nearest]

pred = np.average(targets, weights=weights)

predictions.append(pred)

return np.array(predictions)


"""A module of basic building blcoks for constructing neural networks"""

from . import utils

from . import losses

from . import activations

from . import schedulers

from . import optimizers

from . import wrappers

from . import layers

from . import initializers

from . import modules

from . import models


"""A collection of activation function objects for building neural networks"""

from math import erf

from abc import ABC, abstractmethod

import numpy as np

class ActivationBase(ABC):

def __init__(self, **kwargs):

"""Initialize the ActivationBase object"""

def __call__(self, z):

"""Apply the activation function to an input"""

if z.ndim == 1:

z = z.reshape(1, -1)

return self.fn(z)

@abstractmethod

def fn(self, z):

raise NotImplementedError

@abstractmethod

def grad(self, x, **kwargs):

"""Compute the gradient of the activation function wrt the input"""

raise NotImplementedError

class Sigmoid(ActivationBase):

def __init__(self):

"""A logistic sigmoid activation function."""

def __str__(self):

"""Return a string representation of the activation function"""

return "Sigmoid"

def fn(self, z):

r"""

Evaluate the logistic sigmoid, :math:`\sigma`, on the elements of input `z`.

.. math::

\sigma(x_i) = \frac{1}{1 + e^{-x_i}}

return 1 / (1 + np.exp(-z))

def grad(self, x):

r"""

Evaluate the first derivative of the logistic sigmoid on the elements of `x`.

.. math::

\frac{\partial \sigma}{\partial x_i} = \sigma(x_i) (1 - \sigma(x_i))

fn_x = self.fn(x)

return fn_x * (1 - fn_x)

def grad2(self, x):

r"""

Evaluate the second derivative of the logistic sigmoid on the elements of `x`.

.. math::

\frac{\partial^2 \sigma}{\partial x_i^2} =

\frac{\partial \sigma}{\partial x_i} (1 - 2 \sigma(x_i))

fn_x = self.fn(x)

return fn_x * (1 - fn_x) * (1 - 2 * fn_x)

class ReLU(ActivationBase):

A rectified linear activation function.

Notes

"ReLU units can be fragile during training and can "die". For example, a

large gradient flowing through a ReLU neuron could cause the weights to

update in such a way that the neuron will never activate on any datapoint

again. If this happens, then the gradient flowing through the unit will

forever be zero from that point on. That is, the ReLU units can

irreversibly die during training since they can get knocked off the data

manifold.

For example, you may find that as much as 40% of your network can be "dead"

(i.e. neurons that never activate across the entire training dataset) if

the learning rate is set too high. With a proper setting of the learning

rate this is less frequently an issue." [*]_

References

.. [*] Karpathy, A. "CS231n: Convolutional neural networks for visual recognition."

def __init__(self):

def __str__(self):

"""Return a string representation of the activation function"""

return "ReLU"

def fn(self, z):

r"""

Evaulate the ReLU function on the elements of input `z`.

.. math::

&=  z_i \ \ \ \ &&\text{if }z_i > 0 \\

&=  0 \ \ \ \ &&\text{otherwise}

return np.clip(z, 0, np.inf)

def grad(self, x):

r"""

Evaulate the first derivative of the ReLU function on the elements of input `x`.

.. math::

\frac{\partial \text{ReLU}}{\partial x_i}

&=  1 \ \ \ \ &&\text{if }x_i > 0 \\

&=  0   \ \ \ \ &&\text{otherwise}

return (x > 0).astype(int)

def grad2(self, x):

r"""

Evaulate the second derivative of the ReLU function on the elements of

input `x`.

.. math::

\frac{\partial^2 \text{ReLU}}{\partial x_i^2}  =  0

return np.zeros_like(x)

class LeakyReLU(ActivationBase):

'Leaky' version of a rectified linear unit (ReLU).

Notes

Leaky ReLUs [*]_ are designed to address the vanishing gradient problem in

ReLUs by allowing a small non-zero gradient when `x` is negative.

Parameters

alpha: float

Activation slope when x < 0. Default is 0.3.

References

.. [*] Mass, L. M., Hannun, A. Y, & Ng, A. Y. (2013). "Rectifier

nonlinearities improve neural network acoustic models." *Proceedings of

the 30th International Conference of Machine Learning, 30*.

def __init__(self, alpha=0.3):

self.alpha = alpha

def __str__(self):

return "Leaky ReLU(alpha={})".format(self.alpha)

def fn(self, z):

r"""

Evaluate the leaky ReLU function on the elements of input `z`.

.. math::

&=  z_i \ \ \ \ &&\text{if } z_i > 0 \\

&=  \alpha z_i \ \ \ \ &&\text{otherwise}

_z = z.copy()

_z[z < 0] = _z[z < 0] * self.alpha

return _z

def grad(self, x):

r"""

Evaluate the first derivative of the leaky ReLU function on the elements

of input `x`.

.. math::

\frac{\partial \text{LeakyReLU}}{\partial x_i}

&=  1 \ \ \ \ &&\text{if }x_i > 0 \\

&=  \alpha \ \ \ \ &&\text{otherwise}

out = np.ones_like(x)

out[x < 0] *= self.alpha

return out

def grad2(self, x):

r"""

Evaluate the second derivative of the leaky ReLU function on the

elements of input `x`.

.. math::

\frac{\partial^2 \text{LeakyReLU}}{\partial x_i^2}  =  0

return np.zeros_like(x)

class GELU(ActivationBase):

def __init__(self, approximate=True):

r"""

A Gaussian error linear unit (GELU). [*]_

Notes

A ReLU alternative. GELU weights inputs by their value, rather than

gates inputs by their sign, as in vanilla ReLUs.

References

.. [*] Hendrycks, D., & Gimpel, K. (2016). "Bridging nonlinearities and

stochastic regularizers with Gaussian error linear units." *CoRR*.

Parameters

approximate : bool

Whether to use a faster but less precise approximation to the Gauss

error function when calculating the unit activation and gradient.

Default is True.

self.approximate = True

def __str__(self):

"""Return a string representation of the activation function"""

return f"GELU(approximate={self.approximate})"

def fn(self, z):

r"""

Compute the GELU function on the elements of input `z`.

.. math::

\text{GELU}(z_i) = z_i P(Z \leq z_i) = z_i \Phi(z_i)

= z_i \cdot \frac{1}{2}(1 + \text{erf}(x/\sqrt{2}))

pi, sqrt, tanh = np.pi, np.sqrt, np.tanh

if self.approximate:

return 0.5 * z * (1 + tanh(sqrt(2 / pi) * (z + 0.044715 * z ** 3)))

return 0.5 * z * (1 + erf(z / sqrt(2)))

def grad(self, x):

r"""

Evaluate the first derivative of the GELU function on the elements

of input `x`.

.. math::

\frac{\partial \text{GELU}}{\partial x_i}  =

\frac{1}{2} + \frac{1}{2}\left(\text{erf}(\frac{x}{\sqrt{2}}) +

\frac{x + \text{erf}'(\frac{x}{\sqrt{2}})}{\sqrt{2}}\right)

where :math:`\text{erf}'(x) = \frac{2}{\sqrt{\pi}} \cdot \exp\{-x^2\}`.

pi, exp, sqrt, tanh = np.pi, np.exp, np.sqrt, np.tanh

s = x / sqrt(2)

erf_prime = lambda x: (2 / sqrt(pi)) * exp(-(x ** 2))  # noqa: E731

if self.approximate:

approx = tanh(sqrt(2 / pi) * (x + 0.044715 * x ** 3))

dx = 0.5 + 0.5 * approx + ((0.5 * x * erf_prime(s)) / sqrt(2))

else:

dx = 0.5 + 0.5 * erf(s) + ((0.5 * x * erf_prime(s)) / sqrt(2))

return dx

def grad2(self, x):

r"""

Evaluate the second derivative of the GELU function on the elements

of input `x`.

.. math::

\frac{\partial^2 \text{GELU}}{\partial x_i^2} =

\frac{1}{2\sqrt{2}} \left\[

\text{erf}'(\frac{x}{\sqrt{2}}) +

\frac{1}{\sqrt{2}} \text{erf}''(\frac{x}{\sqrt{2}})

\right]

where :math:`\text{erf}'(x) = \frac{2}{\sqrt{\pi}} \cdot \exp\{-x^2\}` and

:math:`\text{erf}''(x) = \frac{-4x}{\sqrt{\pi}} \cdot \exp\{-x^2\}`.

pi, exp, sqrt = np.pi, np.exp, np.sqrt

s = x / sqrt(2)

erf_prime2 = lambda x: -4 * x * exp(-(x ** 2)) / sqrt(pi)  # noqa: E731

ddx = (1 / 2 * sqrt(2)) * (1 + erf_prime(s) + (erf_prime2(s) / sqrt(2)))

return ddx

class Tanh(ActivationBase):

def __init__(self):

"""A hyperbolic tangent activation function."""

def __str__(self):

"""Return a string representation of the activation function"""

return "Tanh"

def fn(self, z):

"""Compute the tanh function on the elements of input `z`."""

return np.tanh(z)

def grad(self, x):

r"""

Evaluate the first derivative of the tanh function on the elements

of input `x`.

.. math::

\frac{\partial \tanh}{\partial x_i}  =  1 - \tanh(x)^2

return 1 - np.tanh(x) ** 2

def grad2(self, x):

r"""

Evaluate the second derivative of the tanh function on the elements

of input `x`.

.. math::

\frac{\partial^2 \tanh}{\partial x_i^2} =

-2 \tanh(x) \left(\frac{\partial \tanh}{\partial x_i}\right)

tanh_x = np.tanh(x)

return -2 * tanh_x * (1 - tanh_x ** 2)

class Affine(ActivationBase):

def __init__(self, slope=1, intercept=0):

An affine activation function.

Parameters

slope: float

Activation slope. Default is 1.

intercept: float

Intercept/offset term. Default is 0.

self.slope = slope

self.intercept = intercept

def __str__(self):

return "Affine(slope={}, intercept={})".format(self.slope, self.intercept)

def fn(self, z):

r"""

Evaluate the Affine activation on the elements of input `z`.

.. math::

\text{Affine}(z_i)  =  \text{slope} \times z_i + \text{intercept}

return self.slope * z + self.intercept

def grad(self, x):

r"""

Evaluate the first derivative of the Affine activation on the elements

of input `x`.

.. math::

\frac{\partial \text{Affine}}{\partial x_i}  =  \text{slope}

return self.slope * np.ones_like(x)

def grad2(self, x):

r"""

Evaluate the second derivative of the Affine activation on the elements

of input `x`.

.. math::

\frac{\partial^2 \text{Affine}}{\partial x_i^2}  =  0

return np.zeros_like(x)

class Identity(Affine):

def __init__(self):

Identity activation function.

Notes

:class:`Identity` is syntactic sugar for :class:`Affine` with

slope = 1 and intercept = 0.

super().__init__(slope=1, intercept=0)

def __str__(self):

return "Identity"

class ELU(ActivationBase):

def __init__(self, alpha=1.0):

r"""

An exponential linear unit (ELU).

Notes

ELUs are intended to address the fact that ReLUs are strictly nonnegative

and thus have an average activation > 0, increasing the chances of internal

covariate shift and slowing down learning. ELU units address this by (1)

allowing negative values when :math:`x < 0`, which (2) are bounded by a value

:math:`-\alpha`. Similar to :class:`LeakyReLU`, the negative activation

values help to push the average unit activation towards 0. Unlike

:class:`LeakyReLU`, however, the boundedness of the negative activation

allows for greater robustness in the face of large negative values,

allowing the function to avoid conveying the *degree* of "absence"

(negative activation) in the input. [*]_

Parameters

alpha : float

Slope of negative segment. Default is 1.

References

.. [*] Clevert, D. A., Unterthiner, T., Hochreiter, S. (2016). "Fast

and accurate deep network learning by exponential linear units

(ELUs)". *4th International Conference on Learning

Representations*.

self.alpha = alpha

def __str__(self):

"""Return a string representation of the activation function"""

return "ELU(alpha={})".format(self.alpha)

def fn(self, z):

r"""

Evaluate the ELU activation on the elements of input `z`.

.. math::

&=  z_i \ \ \ \ &&\text{if }z_i > 0 \\

&=  \alpha (e^{z_i} - 1) \ \ \ \ &&\text{otherwise}

# z if z > 0  else alpha * (e^z - 1)

return np.where(z > 0, z, self.alpha * (np.exp(z) - 1))

def grad(self, x):

r"""

Evaluate the first derivative of the ELU activation on the elements

of input `x`.

.. math::

\frac{\partial \text{ELU}}{\partial x_i}

&=  1 \ \ \ \ &&\text{if } x_i > 0 \\

&=  \alpha e^{x_i} \ \ \ \ &&\text{otherwise}

# 1 if x > 0 else alpha * e^(z)

return np.where(x > 0, np.ones_like(x), self.alpha * np.exp(x))

def grad2(self, x):

r"""

Evaluate the second derivative of the ELU activation on the elements

of input `x`.

.. math::

\frac{\partial^2 \text{ELU}}{\partial x_i^2}

&=  0 \ \ \ \ &&\text{if } x_i > 0 \\

# 0 if x > 0 else alpha * e^(z)

return np.where(x >= 0, np.zeros_like(x), self.alpha * np.exp(x))

class Exponential(ActivationBase):

def __init__(self):

"""An exponential (base e) activation function"""

def __str__(self):

return "Exponential"

def fn(self, z):

r"""

Evaluate the activation function

.. math::

\text{Exponential}(z_i) = e^{z_i}

return np.exp(z)

def grad(self, x):

r"""

Evaluate the first derivative of the exponential activation on the elements

of input `x`.

.. math::

\frac{\partial \text{Exponential}}{\partial x_i}  =  e^{x_i}

return np.exp(x)

def grad2(self, x):

r"""

Evaluate the second derivative of the exponential activation on the elements

of input `x`.

.. math::

\frac{\partial^2 \text{Exponential}}{\partial x_i^2}  =  e^{x_i}

return np.exp(x)

class SELU(ActivationBase):

r"""

A scaled exponential linear unit (SELU).

Notes

SELU units, when used in conjunction with proper weight initialization and

regularization techniques, encourage neuron activations to converge to

zero-mean and unit variance without explicit use of e.g., batchnorm.

For SELU units, the :math:`\alpha` and :math:`\text{scale}` values are

constants chosen so that the mean and variance of the inputs are preserved

between consecutive layers. As such the authors propose weights be

initialized using Lecun-Normal initialization: :math:`w_{ij} \sim

\mathcal{N}(0, 1 / \text{fan_in})`, and to use the dropout variant

:math:`\alpha`-dropout during regularization. [*]_

See the reference for more information (especially the appendix ;-) ).

References

.. [*] Klambauer, G., Unterthiner, T., & Hochreiter, S. (2017).

"Self-normalizing neural networks." *Advances in Neural Information

Processing Systems, 30.*

def __init__(self):

self.alpha = 1.6732632423543772848170429916717

self.scale = 1.0507009873554804934193349852946

self.elu = ELU(alpha=self.alpha)

def __str__(self):

"""Return a string representation of the activation function"""

return "SELU"

def fn(self, z):

r"""

Evaluate the SELU activation on the elements of input `z`.

.. math::

\text{SELU}(z_i)  =  \text{scale} \times \text{ELU}(z_i, \alpha)

which is simply

.. math::

&= \text{scale} \times z_i \ \ \ \ &&\text{if }z_i > 0 \\

&= \text{scale} \times \alpha (e^{z_i} - 1) \ \ \ \ &&\text{otherwise}

return self.scale * self.elu.fn(z)

def grad(self, x):

r"""

Evaluate the first derivative of the SELU activation on the elements

of input `x`.

.. math::

\frac{\partial \text{SELU}}{\partial x_i}

&=  \text{scale} \ \ \ \ &&\text{if } x_i > 0 \\

&=  \text{scale} \times \alpha e^{x_i} \ \ \ \ &&\text{otherwise}

return np.where(

x >= 0, np.ones_like(x) * self.scale, np.exp(x) * self.alpha * self.scale,

def grad2(self, x):

r"""

Evaluate the second derivative of the SELU activation on the elements

of input `x`.

.. math::

\frac{\partial^2 \text{SELU}}{\partial x_i^2}

&=  0 \ \ \ \ &&\text{if } x_i > 0 \\

return np.where(x > 0, np.zeros_like(x), np.exp(x) * self.alpha * self.scale)

class HardSigmoid(ActivationBase):

def __init__(self):

A "hard" sigmoid activation function.

Notes

The hard sigmoid is a piecewise linear approximation of the logistic

sigmoid that is computationally more efficient to compute.

def __str__(self):

return "Hard Sigmoid"

def fn(self, z):

r"""

Evaluate the hard sigmoid activation on the elements of input `z`.

.. math::

&= 0 \ \ \ \ &&\text{if }z_i < -2.5 \\

&= 0.2 z_i + 0.5 \ \ \ \ &&\text{if }-2.5 \leq z_i \leq 2.5 \\

&= 1 \ \ \ \ &&\text{if }z_i > 2.5

return np.clip((0.2 * z) + 0.5, 0.0, 1.0)

def grad(self, x):

r"""

Evaluate the first derivative of the hard sigmoid activation on the elements

of input `x`.

.. math::

\frac{\partial \text{HardSigmoid}}{\partial x_i}

&=  0.2 \ \ \ \ &&\text{if } -2.5 \leq x_i \leq 2.5\\

&=  0 \ \ \ \ &&\text{otherwise}

return np.where((x >= -2.5) & (x <= 2.5), 0.2, 0)

def grad2(self, x):

r"""

Evaluate the second derivative of the hard sigmoid activation on the elements

of input `x`.

.. math::

\frac{\partial^2 \text{HardSigmoid}}{\partial x_i^2} =  0

return np.zeros_like(x)

class SoftPlus(ActivationBase):

def __init__(self):

A softplus activation function.

Notes

In contrast to :class:`ReLU`, the softplus activation is differentiable

everywhere (including 0). It is, however, less computationally efficient to

compute.

The derivative of the softplus activation is the logistic sigmoid.

def __str__(self):

"""Return a string representation of the activation function"""

return "SoftPlus"

def fn(self, z):

r"""

Evaluate the softplus activation on the elements of input `z`.

.. math::

\text{SoftPlus}(z_i) = \log(1 + e^{z_i})

return np.log(np.exp(z) + 1)

def grad(self, x):

r"""

Evaluate the first derivative of the softplus activation on the elements

of input `x`.

.. math::

\frac{\partial \text{SoftPlus}}{\partial x_i} = \frac{e^{x_i}}{1 + e^{x_i}}

exp_x = np.exp(x)

return exp_x / (exp_x + 1)

def grad2(self, x):

r"""

Evaluate the second derivative of the softplus activation on the elements

of input `x`.

.. math::

\frac{\partial^2 \text{SoftPlus}}{\partial x_i^2} =

\frac{e^{x_i}}{(1 + e^{x_i})^2}

exp_x = np.exp(x)

return exp_x / ((exp_x + 1) ** 2)


from .activations import *


Common neural network loss functions.

This module implements loss objects that can be used during neural network

training.

from .losses import *


from abc import ABC, abstractmethod

import numpy as np

from ...utils.testing import is_binary, is_stochastic

from ..initializers import (

WeightInitializer,

ActivationInitializer,

OptimizerInitializer,

class ObjectiveBase(ABC):

def __init__(self):

@abstractmethod

def loss(self, y_true, y_pred):

pass

@abstractmethod

def grad(self, y_true, y_pred, **kwargs):

pass

class SquaredError(ObjectiveBase):

def __init__(self):

A squared-error / `L2` loss.

Notes

For real-valued target **y** and predictions :math:`\hat{\mathbf{y}}`, the

squared error is

.. math::

\mathcal{L}(\mathbf{y}, \hat{\mathbf{y}})

= 0.5 ||\hat{\mathbf{y}} - \mathbf{y}||_2^2

def __call__(self, y, y_pred):

return self.loss(y, y_pred)

def __str__(self):

return "SquaredError"

@staticmethod

def loss(y, y_pred):

Compute the squared error between `y` and `y_pred`.

Parameters

y : :py:class:`ndarray <numpy.ndarray>` of shape (n, m)

Ground truth values for each of `n` examples

y_pred : :py:class:`ndarray <numpy.ndarray>` of shape (n, m)

Predictions for the `n` examples in the batch.

Returns

loss : float

The sum of the squared error across dimensions and examples.

return 0.5 * np.linalg.norm(y_pred - y) ** 2

@staticmethod

def grad(y, y_pred, z, act_fn):

Gradient of the squared error loss with respect to the pre-nonlinearity

input, `z`.

Notes

The current method computes the gradient :math:`\\frac{\partial

\mathcal{L}}{\partial \mathbf{z}}`, where

.. math::

\mathcal{L}(\mathbf{z})

&=  \\text{squared_error}(\mathbf{y}, g(\mathbf{z})) \\\\

g(\mathbf{z})

&=  \\text{act_fn}(\mathbf{z})

The gradient with respect to :math:`\mathbf{z}` is then

.. math::

\\frac{\partial \mathcal{L}}{\partial \mathbf{z}}

= (g(\mathbf{z}) - \mathbf{y}) \left(

\\frac{\partial g}{\partial \mathbf{z}} \\right)

Parameters

Ground truth values for each of `n` examples.

act_fn : :doc:`Activation <numpy_ml.neural_nets.activations>` object

The activation function for the output layer of the network.

Returns

grad : :py:class:`ndarray <numpy.ndarray>` of shape (n, m)

The gradient of the squared error loss with respect to `z`.

return (y_pred - y) * act_fn.grad(z)

class CrossEntropy(ObjectiveBase):

def __init__(self):

A cross-entropy loss.

Notes

For a one-hot target **y** and predicted class probabilities

:math:`\hat{\mathbf{y}}`, the cross entropy is

.. math::

\mathcal{L}(\mathbf{y}, \hat{\mathbf{y}})

= \sum_i y_i \log \hat{y}_i

def __call__(self, y, y_pred):

return self.loss(y, y_pred)

def __str__(self):

return "CrossEntropy"

@staticmethod

def loss(y, y_pred):

Compute the cross-entropy (log) loss.

Notes

This method returns the sum (not the average!) of the losses for each

sample.

Parameters

y : :py:class:`ndarray <numpy.ndarray>` of shape (n, m)

Class labels (one-hot with `m` possible classes) for each of `n`

examples.

y_pred : :py:class:`ndarray <numpy.ndarray>` of shape (n, m)

Probabilities of each of `m` classes for the `n` examples in the

batch.

Returns

loss : float

The sum of the cross-entropy across classes and examples.

# prevent taking the log of 0

eps = np.finfo(float).eps

# each example is associated with a single class; sum the negative log

# probability of the correct label over all samples in the batch.

# observe that we are taking advantage of the fact that y is one-hot

# encoded

cross_entropy = -np.sum(y * np.log(y_pred + eps))

return cross_entropy

@staticmethod

def grad(y, y_pred):

Compute the gradient of the cross entropy loss with regard to the

softmax input, `z`.

Notes

The gradient for this method goes through both the cross-entropy loss

AND the softmax non-linearity to return :math:`\\frac{\partial

\mathcal{L}}{\partial \mathbf{z}}` (rather than :math:`\\frac{\partial

\mathcal{L}}{\partial \\text{softmax}(\mathbf{z})}`).

In particular, let:

.. math::

\mathcal{L}(\mathbf{z})

= \\text{cross_entropy}(\\text{softmax}(\mathbf{z})).

The current method computes:

.. math::

\\frac{\partial \mathcal{L}}{\partial \mathbf{z}}

&= \\text{softmax}(\mathbf{z}) - \mathbf{y} \\\\

&=  \hat{\mathbf{y}} - \mathbf{y}

Parameters

y : :py:class:`ndarray <numpy.ndarray>` of shape `(n, m)`

A one-hot encoding of the true class labels. Each row constitues a

training example, and each column is a different class.

y_pred: :py:class:`ndarray <numpy.ndarray>` of shape `(n, m)`

The network predictions for the probability of each of `m` class

labels on each of `n` examples in a batch.

Returns

grad : :py:class:`ndarray <numpy.ndarray>` of shape (n, m)

The gradient of the cross-entropy loss with respect to the *input*

to the softmax function.

# derivative of xe wrt z is y_pred - y_true, hence we can just

# subtract 1 from the probability of the correct class labels

grad = y_pred - y

# [optional] scale the gradients by the number of examples in the batch

# n, m = y.shape

# grad /= n

return grad

class VAELoss(ObjectiveBase):

def __init__(self):

The variational lower bound for a variational autoencoder with Bernoulli

units.

Notes

The VLB to the sum of the binary cross entropy between the true input and

the predicted output (the "reconstruction loss") and the KL divergence

between the learned variational distribution :math:`q` and the prior,

:math:`p`, assumed to be a unit Gaussian.

.. math::

\\text{VAELoss} =

\\text{cross_entropy}(\mathbf{y}, \hat{\mathbf{y}})

+ \\mathbb{KL}[q \ || \ p]

where :math:`\mathbb{KL}[q \ || \ p]` is the Kullback-Leibler

divergence between the distributions :math:`q` and :math:`p`.

References

.. [1] Kingma, D. P. & Welling, M. (2014). "Auto-encoding variational Bayes".

*arXiv preprint arXiv:1312.6114.* https://arxiv.org/pdf/1312.6114.pdf

def __call__(self, y, y_pred, t_mean, t_log_var):

return self.loss(y, y_pred, t_mean, t_log_var)

def __str__(self):

return "VAELoss"

@staticmethod

def loss(y, y_pred, t_mean, t_log_var):

Variational lower bound for a Bernoulli VAE.

Parameters

y : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, N)`

The original images.

y_pred : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, N)`

The VAE reconstruction of the images.

t_mean: :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, T)`

Mean of the variational distribution :math:`q(t \mid x)`.

t_log_var: :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, T)`

Log of the variance vector of the variational distribution

:math:`q(t \mid x)`.

Returns

loss : float

The VLB, averaged across the batch.

# prevent nan on log(0)

eps = np.finfo(float).eps

y_pred = np.clip(y_pred, eps, 1 - eps)

# reconstruction loss: binary cross-entropy

rec_loss = -np.sum(y * np.log(y_pred) + (1 - y) * np.log(1 - y_pred), axis=1)

# KL divergence between the variational distribution q and the prior p,

# a unit gaussian

kl_loss = -0.5 * np.sum(1 + t_log_var - t_mean ** 2 - np.exp(t_log_var), axis=1)

loss = np.mean(kl_loss + rec_loss)

return loss

@staticmethod

def grad(y, y_pred, t_mean, t_log_var):

Compute the gradient of the VLB with regard to the network parameters.

Parameters

The original images.

The VAE reconstruction of the images.

Mean of the variational distribution :math:`q(t | x)`.

:math:`q(t | x)`.

Returns

dY_pred : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, N)`

The gradient of the VLB with regard to `y_pred`.

dLogVar : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, T)`

The gradient of the VLB with regard to `t_log_var`.

dMean : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, T)`

The gradient of the VLB with regard to `t_mean`.

N = y.shape[0]

eps = np.finfo(float).eps

y_pred = np.clip(y_pred, eps, 1 - eps)

dY_pred = -y / (N * y_pred) - (y - 1) / (N - N * y_pred)

dLogVar = (np.exp(t_log_var) - 1) / (2 * N)

dMean = t_mean / N

return dY_pred, dLogVar, dMean

class WGAN_GPLoss(ObjectiveBase):

def __init__(self, lambda_=10):

The loss function for a Wasserstein GAN [*]_ [*]_ with gradient penalty.

Notes

Assuming an optimal critic, minimizing this quantity wrt. the generator

parameters corresponds to minimizing the Wasserstein-1 (earth-mover)

distance between the fake and real data distributions.

The formula for the WGAN-GP critic loss is

.. math::

&=  \sum_{x \in X_{real}} p(x) D(x)

- \sum_{x' \in X_{fake}} p(x') D(x') \\\\

&=  \\text{WGANLoss} + \lambda

(||\\nabla_{X_{interp}} D(X_{interp})||_2 - 1)^2

where

.. math::

X_{fake}  &=   \\text{Generator}(\mathbf{z}) \\\\

X_{interp}   &=   \\alpha X_{real} + (1 - \\alpha) X_{fake} \\\\

and

.. math::

\mathbf{z}  &\sim  \mathcal{N}(0, \mathbb{1}) \\\\

\\alpha  &\sim  \\text{Uniform}(0, 1)

References

.. [*] Gulrajani, I., Ahmed, F., Arjovsky, M., Dumoulin, V., &

Courville, A. (2017) "Improved training of Wasserstein GANs"

*Advances in Neural Information Processing Systems, 31*: 5769-5779.

.. [*] Goodfellow, I. J, Abadie, P. A., Mirza, M., Xu, B., Farley, D.

W., Ozair, S., Courville, A., & Bengio, Y. (2014) "Generative

adversarial nets" *Advances in Neural Information Processing

Systems, 27*: 2672-2680.

Parameters

lambda_ : float

The gradient penalty coefficient. Default is 10.

self.lambda_ = lambda_

def __call__(self, Y_fake, module, Y_real=None, gradInterp=None):

Computes the generator and critic loss using the WGAN-GP value

function.

Parameters

Y_fake : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex,)`

The output of the critic for `X_fake`.

module : {'C', 'G'}

Whether to calculate the loss for the critic ('C') or the generator

('G'). If calculating loss for the critic, `Y_real` and

`gradInterp` must not be None.

Y_real : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex,)`, or None

The output of the critic for `X_real`. Default is None.

gradInterp : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, n_feats)`, or None

The gradient of the critic output for `X_interp` wrt. `X_interp`.

Default is None.

Returns

loss : float

Depending on the setting for `module`, either the critic or

generator loss, averaged over examples in the minibatch.

return self.loss(Y_fake, module, Y_real=Y_real, gradInterp=gradInterp)

def __str__(self):

return "WGANLossGP(lambda_={})".format(self.lambda_)

def loss(self, Y_fake, module, Y_real=None, gradInterp=None):

function.

Parameters

Y_fake : :py:class:`ndarray <numpy.ndarray>` of shape (n_ex,)

The output of the critic for `X_fake`.

module : {'C', 'G'}

`gradInterp` must not be None.

Y_real : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex,)` or None

gradInterp : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, n_feats)` or None

Default is None.

Returns

loss : float

# calc critic loss including gradient penalty

if module == "C":

X_interp_norm = np.linalg.norm(gradInterp, axis=1, keepdims=True)

gradient_penalty = (X_interp_norm - 1) ** 2

loss = (

Y_fake.mean() - Y_real.mean() + self.lambda_ * gradient_penalty.mean()

# calc generator loss

elif module == "G":

loss = -Y_fake.mean()

else:

raise ValueError("Unrecognized module: {}".format(module))

return loss

def grad(self, Y_fake, module, Y_real=None, gradInterp=None):

Computes the gradient of the generator or critic loss with regard to

its inputs.

Parameters

Y_fake : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex,)`

The output of the critic for `X_fake`.

module : {'C', 'G'}

Whether to calculate the gradient for the critic loss ('C') or the

generator loss ('G'). If calculating grads for the critic, `Y_real`

and `gradInterp` must not be None.

The output of the critic for `X_real`. Default is None.

The gradient of the critic output on `X_interp` wrt. `X_interp`.

Default is None.

Returns

grads : tuple

If `module` == 'C', returns a 3-tuple containing the gradient of

the critic loss with regard to (`Y_fake`, `Y_real`, `gradInterp`).

If `module` == 'G', returns the gradient of the generator with

regard to `Y_fake`.

eps = np.finfo(float).eps

n_ex_fake = Y_fake.shape[0]

# calc gradient of the critic loss

if module == "C":

n_ex_real = Y_real.shape[0]

dY_fake = -1 / n_ex_fake * np.ones_like(Y_fake)

dY_real = 1 / n_ex_real * np.ones_like(Y_real)

# differentiate through gradient penalty

X_interp_norm = np.linalg.norm(gradInterp, axis=1, keepdims=True) + eps

dGradInterp = (

(2 / n_ex_fake)

* self.lambda_

* (X_interp_norm - 1)

* (gradInterp / X_interp_norm)

grad = (dY_fake, dY_real, dGradInterp)

# calc gradient of the generator loss

elif module == "G":

grad = -1 / n_ex_fake * np.ones_like(Y_fake)

else:

return grad

class NCELoss(ObjectiveBase):

def __init__(

self,

optimizer=None,

A noise contrastive estimation (NCE) loss function.

Notes

Noise contrastive estimation is a candidate sampling method often

used to reduce the computational challenge of training a softmax

layer on problems with a large number of output classes. It proceeds by

training a logistic regression model to discriminate between samples

from the true data distribution and samples from an artificial noise

distribution.

It can be shown that as the ratio of negative samples to data samples

goes to infinity, the gradient of the NCE loss converges to the

original softmax gradient.

For input data **X**, target labels `targets`, loss parameters **W** and

**b**, and noise samples `noise` sampled from the noise distribution `Q`,

the NCE loss is

.. math::

\\text{NCE}(X, targets) =

\\text{cross_entropy}(\mathbf{y}_{targets}, \hat{\mathbf{y}}_{targets}) +

\\text{cross_entropy}(\mathbf{y}_{noise}, \hat{\mathbf{y}}_{noise})

where

.. math::

&=  \sigma(\mathbf{W}[targets] \mathbf{X} + \mathbf{b}[targets] - \log Q(targets)) \\\\

&=  \sigma(\mathbf{W}[noise] \mathbf{X} + \mathbf{b}[noise] - \log Q(noise))

In the above equations, :math:`\sigma` is the logistic sigmoid

function, and :math:`Q(x)` corresponds to the probability of the values

in `x` under `Q`.

References

.. [1] Gutmann, M. & Hyvarinen, A. (2010). Noise-contrastive

estimation: A new estimation principle for unnormalized statistical

models. *AISTATS, 13*: 297-304.

.. [2] Minh, A. & Teh, Y. W. (2012). A fast and simple algorithm for

training neural probabilistic language models. *ICML, 29*: 1751-1758.

Parameters

n_classes : int

The total number of output classes in the model.

noise_sampler : :class:`~numpy_ml.utils.data_structures.DiscreteSampler` instance

The negative sampler. Defines a distribution over all classes in

the dataset.

num_negative_samples : int

The number of negative samples to draw for each target / batch of

targets.

init : {'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform'}

The weight initialization strategy. Default is 'glorot_uniform'.

optimizer : str, :doc:`Optimizer <numpy_ml.neural_nets.optimizers>` object, or None

The optimization strategy to use when performing gradient updates

within the :meth:`update` method.  If None, use the :class:`SGD

<numpy_ml.neural_nets.optimizers.SGD>` optimizer with

default parameters. Default is None.

subtract_log_label_prob : bool

Whether to subtract the log of the probability of each label under

the noise distribution from its respective logit. Set to False for

negative sampling, True for NCE. Default is True.

Attributes

gradients : dict

The accumulated parameter gradients.

parameters: dict

The loss parameter values.

hyperparameters: dict

The loss hyperparameter values.

derived_variables: dict

Useful intermediate values computed during the loss computation.

self.init = init

self.n_in = None

self.trainable = True

self.n_classes = n_classes

self.noise_sampler = noise_sampler

self.num_negative_samples = num_negative_samples

self.act_fn = ActivationInitializer("Sigmoid")()

self.optimizer = OptimizerInitializer(optimizer)()

self.subtract_log_label_prob = subtract_log_label_prob

self.is_initialized = False

def _init_params(self):

init_weights = WeightInitializer(str(self.act_fn), mode=self.init)

self.X = []

b = np.zeros((1, self.n_classes))

W = init_weights((self.n_classes, self.n_in))

self.parameters = {"W": W, "b": b}

self.gradients = {"W": np.zeros_like(W), "b": np.zeros_like(b)}

self.derived_variables = {

"y_pred": [],

"target": [],

"true_w": [],

"true_b": [],

"sampled_b": [],

"sampled_w": [],

"out_labels": [],

"target_logits": [],

"noise_samples": [],

"noise_logits": [],

self.is_initialized = True

@property

def hyperparameters(self):

return {

"id": "NCELoss",

"n_in": self.n_in,

"init": self.init,

"n_classes": self.n_classes,

"noise_sampler": self.noise_sampler,

"num_negative_samples": self.num_negative_samples,

"subtract_log_label_prob": self.subtract_log_label_prob,

"optimizer": {

"cache": self.optimizer.cache,

"hyperparameters": self.optimizer.hyperparameters,

def __call__(self, X, target, neg_samples=None, retain_derived=True):

return self.loss(X, target, neg_samples, retain_derived)

def __str__(self):

keys = [

"{}={}".format(k, v)

for k, v in self.hyperparameters.items()

if k not in ["id", "optimizer"]

] + ["optimizer={}".format(self.optimizer)]

return "NCELoss({})".format(", ".join(keys))

def freeze(self):

Freeze the loss parameters at their current values so they can no

longer be updated.

self.trainable = False

def unfreeze(self):

"""Unfreeze the layer parameters so they can be updated."""

self.trainable = True

def flush_gradients(self):

"""Erase all the layer's derived variables and gradients."""

assert self.trainable, "NCELoss is frozen"

self.X = []

for k, v in self.derived_variables.items():

self.derived_variables[k] = []

for k, v in self.gradients.items():

self.gradients[k] = np.zeros_like(v)

def update(self, cur_loss=None):

Update the loss parameters using the accrued gradients and optimizer.

Flush all gradients once the update is complete.

self.optimizer.step()

for k, v in self.gradients.items():

if k in self.parameters:

self.parameters[k] = self.optimizer(self.parameters[k], v, k, cur_loss)

def loss(self, X, target, neg_samples=None, retain_derived=True):

Compute the NCE loss for a collection of inputs and associated targets.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, n_c, n_in)`

Layer input. A minibatch of `n_ex` examples, where each example is

an `n_c` by `n_in` matrix (e.g., the matrix of `n_c` context

embeddings, each of dimensionality `n_in`, for a CBOW model).

target : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex,)`

Integer indices of the target class(es) for each example in the

minibatch (e.g., the target word id for an example in a CBOW model).

neg_samples : :py:class:`ndarray <numpy.ndarray>` of shape (`num_negative_samples`,) or None

An optional array of negative samples to use during the loss

calculation. These will be used instead of samples draw from

``self.noise_sampler``. Default is None.

retain_derived : bool

Whether to retain the variables calculated during the forward pass

for use later during backprop. If False, this suggests the layer

will not be expected to backprop through with regard to this input.

Default is True.

Returns

loss : float

The NCE loss summed over the minibatch and samples.

y_pred : :py:class:`ndarray <numpy.ndarray>` of shape (`n_ex`, `n_c`)

The network predictions for the conditional probability of each

target given each context: entry (`i`, `j`) gives the predicted

probability of target `i` under context vector `j`.

if not self.is_initialized:

self.n_in = X.shape[-1]

loss, Z_target, Z_neg, y_pred, y_true, noise_samples = self._loss(

X, target, neg_samples

# cache derived variables for gradient calculation

if retain_derived:

self.X.append(X)

self.derived_variables["y_pred"].append(y_pred)

self.derived_variables["target"].append(target)

self.derived_variables["out_labels"].append(y_true)

self.derived_variables["target_logits"].append(Z_target)

self.derived_variables["noise_samples"].append(noise_samples)

self.derived_variables["noise_logits"].append(Z_neg)

return loss, np.squeeze(y_pred[..., :1], -1)

def _loss(self, X, target, neg_samples):

"""Actual computation of NCE loss"""

fstr = "X must have shape (n_ex, n_c, n_in), but got {} dims instead"

assert X.ndim == 3, fstr.format(X.ndim)

W = self.parameters["W"]

b = self.parameters["b"]

# sample negative samples from the noise distribution

if neg_samples is None:

neg_samples = self.noise_sampler(self.num_negative_samples)

assert len(neg_samples) == self.num_negative_samples

# get the probability of the negative sample class and the target

# class under the noise distribution

p_neg_samples = self.noise_sampler.probs[neg_samples]

p_target = np.atleast_2d(self.noise_sampler.probs[target])

# save the noise samples for debugging

noise_samples = (neg_samples, p_target, p_neg_samples)

# compute the logit for the negative samples and target

Z_target = X @ W[target].T + b[0, target]

Z_neg = X @ W[neg_samples].T + b[0, neg_samples]

# subtract the log probability of each label under the noise dist

if self.subtract_log_label_prob:

n, m = Z_target.shape[0], Z_neg.shape[0]

Z_target[range(n), ...] -= np.log(p_target)

Z_neg[range(m), ...] -= np.log(p_neg_samples)

# only retain the probability of the target under its associated

# minibatch example

aa, _, cc = Z_target.shape

Z_target = Z_target[range(aa), :, range(cc)][..., None]

# p_target = (n_ex, n_c, 1)

# p_neg = (n_ex, n_c, n_samples)

pred_p_target = self.act_fn(Z_target)

pred_p_neg = self.act_fn(Z_neg)

# if we're in evaluation mode, ignore the negative samples - just

# return the binary cross entropy on the targets

y_pred = pred_p_target

if self.trainable:

# (n_ex, n_c, 1 + n_samples) (target is first column)

y_pred = np.concatenate((y_pred, pred_p_neg), axis=-1)

n_targets = 1

y_true = np.zeros_like(y_pred)

y_true[..., :n_targets] = 1

# binary cross entropy

eps = np.finfo(float).eps

np.clip(y_pred, eps, 1 - eps, y_pred)

loss = -np.sum(y_true * np.log(y_pred) + (1 - y_true) * np.log(1 - y_pred))

return loss, Z_target, Z_neg, y_pred, y_true, noise_samples

def grad(self, retain_grads=True, update_params=True):

Compute the gradient of the NCE loss with regard to the inputs,

weights, and biases.

Parameters

retain_grads : bool

Whether to include the intermediate parameter gradients computed

during the backward pass in the final parameter update. Default is

True.

update_params : bool

Whether to perform a single step of gradient descent on the layer

weights and bias using the calculated gradients. If `retain_grads`

is False, this option is ignored and the parameter gradients are

not updated. Default is True.

Returns

dLdX : :py:class:`ndarray <numpy.ndarray>` of shape (`n_ex`, `n_in`) or list of arrays

The gradient of the loss with regard to the layer input(s) `X`.

assert self.trainable, "NCE loss is frozen"

dX = []

for input_idx, x in enumerate(self.X):

dx, dw, db = self._grad(x, input_idx)

dX.append(dx)

if retain_grads:

self.gradients["W"] += dw

self.gradients["b"] += db

dX = dX[0] if len(self.X) == 1 else dX

if retain_grads and update_params:

self.update()

return dX

def _grad(self, X, input_idx):

"""Actual computation of gradient wrt. loss weights + input"""

W, b = self.parameters["W"], self.parameters["b"]

y_pred = self.derived_variables["y_pred"][input_idx]

target = self.derived_variables["target"][input_idx]

y_true = self.derived_variables["out_labels"][input_idx]

Z_neg = self.derived_variables["noise_logits"][input_idx]

Z_target = self.derived_variables["target_logits"][input_idx]

neg_samples = self.derived_variables["noise_samples"][input_idx][0]

# the number of target classes per minibatch example

n_targets = 1

# calculate the grad of the binary cross entropy wrt. the network

# predictions

preds, classes = y_pred.flatten(), y_true.flatten()

dLdp_real = ((1 - classes) / (1 - preds)) - (classes / preds)

dLdp_real = dLdp_real.reshape(*y_pred.shape)

# partition the gradients into target and negative sample portions

dLdy_pred_target = dLdp_real[..., :n_targets]

dLdy_pred_neg = dLdp_real[..., n_targets:]

# compute gradients of the loss wrt the data and noise logits

dLdZ_target = dLdy_pred_target * self.act_fn.grad(Z_target)

dLdZ_neg = dLdy_pred_neg * self.act_fn.grad(Z_neg)

# compute param gradients on target + negative samples

dB_neg = dLdZ_neg.sum(axis=(0, 1))

dB_target = dLdZ_target.sum(axis=(1, 2))

dW_neg = (dLdZ_neg.transpose(0, 2, 1) @ X).sum(axis=0)

dW_target = (dLdZ_target.transpose(0, 2, 1) @ X).sum(axis=1)

# TODO: can this be done with np.einsum instead?

dX_target = np.vstack(

[dLdZ_target[[ix]] @ W[[t]] for ix, t in enumerate(target)]

dX_neg = dLdZ_neg @ W[neg_samples]

hits = list(set(target).intersection(set(neg_samples)))

hit_ixs = [np.where(target == h)[0] for h in hits]

# adjust param gradients if there's an accidental hit

if len(hits) != 0:

hit_ixs = np.concatenate(hit_ixs)

target = np.delete(target, hit_ixs)

dB_target = np.delete(dB_target, hit_ixs)

dW_target = np.delete(dW_target, hit_ixs, 0)

dX = dX_target + dX_neg

# use np.add.at to ensure that repeated indices in the target (or

# possibly in neg_samples if sampling is done with replacement) are

# properly accounted for

dB = np.zeros_like(b).flatten()

np.add.at(dB, target, dB_target)

np.add.at(dB, neg_samples, dB_neg)

dB = dB.reshape(*b.shape)

dW = np.zeros_like(W)

np.add.at(dW, target, dW_target)

np.add.at(dW, neg_samples, dW_neg)

return dX, dW, dB


from .wrappers import *


A collection of objects thats can wrap / otherwise modify arbitrary neural

network layers.

from abc import ABC, abstractmethod

import numpy as np

class WrapperBase(ABC):

def __init__(self, wrapped_layer):

"""An abstract base class for all Wrapper instances"""

self._base_layer = wrapped_layer

if hasattr(wrapped_layer, "_base_layer"):

self._base_layer = wrapped_layer._base_layer

@abstractmethod

def _init_wrapper_params(self):

raise NotImplementedError

@abstractmethod

def forward(self, z, **kwargs):

"""Overwritten by inherited class"""

raise NotImplementedError

@abstractmethod

def backward(self, out, **kwargs):

"""Overwritten by inherited class"""

raise NotImplementedError

@property

def trainable(self):

"""Whether the base layer is frozen"""

return self._base_layer.trainable

@property

def parameters(self):

"""A dictionary of the base layer parameters"""

return self._base_layer.parameters

@property

def hyperparameters(self):

"""A dictionary of the base layer's hyperparameters"""

hp = self._base_layer.hyperparameters

hpw = self._wrapper_hyperparameters

if "wrappers" in hp:

hp["wrappers"].append(hpw)

else:

hp["wrappers"] = [hpw]

return hp

@property

def derived_variables(self):

A dictionary of the intermediate values computed during layer

training.

dv = self._base_layer.derived_variables.copy()

if "wrappers" in dv:

dv["wrappers"].append(self._wrapper_derived_variables)

else:

dv["wrappers"] = [self._wrapper_derived_variables]

return dv

@property

def gradients(self):

"""A dictionary of the current layer parameter gradients."""

return self._base_layer.gradients

@property

def act_fn(self):

"""The activation function for the base layer."""

return self._base_layer.act_fn

@property

def X(self):

"""The collection of layer inputs."""

return self._base_layer.X

def _init_params(self):

hp = self._wrapper_hyperparameters

if "wrappers" in self._base_layer.hyperparameters:

self._base_layer.hyperparameters["wrappers"].append(hp)

else:

self._base_layer.hyperparameters["wrappers"] = [hp]

def freeze(self):

Freeze the base layer's parameters at their current values so they can

no longer be updated.

def unfreeze(self):

"""Unfreeze the base layer's parameters so they can be updated."""

def flush_gradients(self):

"""Erase all the wrapper and base layer's derived variables and gradients."""

assert self.trainable, "Layer is frozen"

for k, v in self._wrapper_derived_variables.items():

self._wrapper_derived_variables[k] = []

def update(self, lr):

Update the base layer's parameters using the accrued gradients and

layer optimizer. Flush all gradients once the update is complete.

def _set_wrapper_params(self, pdict):

for k, v in pdict.items():

if k in self._wrapper_hyperparameters:

self._wrapper_hyperparameters[k] = v

return self

def set_params(self, summary_dict):

Set the base layer parameters from a dictionary of values.

Parameters

summary_dict : dict

A dictionary of layer parameters and hyperparameters. If a required

parameter or hyperparameter is not included within `summary_dict`,

this method will use the value in the current layer's

:meth:`summary` method.

Returns

layer : :doc:`Layer <numpy_ml.neural_nets.layers>` object

The newly-initialized layer.

return self._base_layer.set_params(summary_dict)

def summary(self):

"""Return a dict of the layer parameters, hyperparameters, and ID."""

return {

"layer": self.hyperparameters["layer"],

"layer_wrappers": [i["wrapper"] for i in self.hyperparameters["wrappers"]],

"parameters": self.parameters,

"hyperparameters": self.hyperparameters,

class Dropout(WrapperBase):

def __init__(self, wrapped_layer, p):

A dropout regularization wrapper.

Notes

During training, a dropout layer zeroes each element of the layer input

with probability `p` and scales the activation by `1 / (1 - p)` (to reflect

the fact that on average only `(1 - p) * N` units are active on any

training pass). At test time, does not adjust elements of the input at

all (ie., simply computes the identity function).

Parameters

wrapped_layer : :doc:`Layer <numpy_ml.neural_nets.layers>` instance

The layer to apply dropout to.

p : float in [0, 1)

The dropout propbability during training

self.p = p

def _init_wrapper_params(self):

self._wrapper_derived_variables = {"dropout_mask": []}

self._wrapper_hyperparameters = {"wrapper": "Dropout", "p": self.p}

def forward(self, X, retain_derived=True):

Compute the layer output with dropout for a single minibatch.

Parameters

Layer input, representing the `n_in`-dimensional features for a

minibatch of `n_ex` examples.

retain_derived : bool

Whether to retain the variables calculated during the forward pass

for use later during backprop. If False, this suggests the layer

will not be expected to backprop through wrt. this input. Default

is True.

Returns

Y : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, n_out)`

Layer output for each of the `n_ex` examples.

scaler, mask = 1.0, np.ones(X.shape).astype(bool)

if self.trainable:

scaler = 1.0 / (1.0 - self.p)

mask = np.random.rand(*X.shape) >= self.p

X = mask * X

if retain_derived:

self._wrapper_derived_variables["dropout_mask"].append(mask)

return scaler * self._base_layer.forward(X, retain_derived)

def backward(self, dLdy, retain_grads=True):

Backprop from the base layer's outputs to inputs.

Parameters

dLdy : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, n_out)` or list of arrays

The gradient(s) of the loss wrt. the layer output(s).

retain_grads : bool

Whether to include the intermediate parameter gradients computed

during the backward pass in the final parameter update. Default is

True.

Returns

dLdX : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, n_in)` or list of arrays

The gradient of the loss wrt. the layer input(s) `X`.

"""  # noqa: E501

assert self.trainable, "Layer is frozen"

dLdy *= 1.0 / (1.0 - self.p)

return self._base_layer.backward(dLdy, retain_grads)

def init_wrappers(layer, wrappers_list):

Initialize the layer wrappers in `wrapper_list` and return a wrapped

`layer` object.

Parameters

The base layer object to apply the wrappers to.

wrappers : list of dicts

A list of parameter dictionaries for a the wrapper objects. The

wrappers are initialized and applied to the the layer sequentially.

Returns

wrapped_layer : :class:`WrapperBase` instance

The wrapped layer object

for wr in wrappers_list:

if wr["wrapper"] == "Dropout":

layer = Dropout(layer, 1)._set_wrapper_params(wr)

else:

raise NotImplementedError("{}".format(wr["wrapper"]))

return layer


from .layers import *