"""A module containing exploration policies for various multi-armed bandit problems."""

from abc import ABC, abstractmethod

from collections import defaultdict

import numpy as np

from ..utils.testing import is_number

class BanditPolicyBase(ABC):

def __init__(self):

"""A simple base class for multi-armed bandit policies"""

self.step = 0

self.ev_estimates = {}

self.is_initialized = False

def __repr__(self):

"""Return a string representation of the policy"""

HP = self.hyperparameters

params = ", ".join(["{}={}".format(k, v) for (k, v) in HP.items() if k != "id"])

return "{}({})".format(HP["id"], params)

@property

def hyperparameters(self):

"""A dictionary containing the policy hyperparameters"""

pass

@property

def parameters(self):

"""A dictionary containing the current policy parameters"""

pass

def act(self, bandit, context=None):

Select an arm and sample from its payoff distribution.

Parameters

bandit : :class:`Bandit <numpy_ml.bandits.bandits.Bandit>` instance

The multi-armed bandit to act upon

context : :py:class:`ndarray <numpy.ndarray>` of shape `(D,)` or None

The context vector for the current timestep if interacting with a

contextual bandit. Otherwise, this argument is unused. Default is

None.

Returns

rwd : float

The reward received after pulling ``arm_id``.

arm_id : int

The arm that was pulled to generate ``rwd``.

if not self.is_initialized:

arm_id = self._select_arm(bandit, context)

rwd = self._pull_arm(bandit, arm_id, context)

self._update_params(arm_id, rwd, context)

return rwd, arm_id

def reset(self):

"""Reset the policy parameters and counters to their initial states."""

self.step = 0

self.is_initialized = False

def _pull_arm(self, bandit, arm_id, context):

"""Execute a bandit action and return the received reward."""

self.step += 1

return bandit.pull(arm_id, context)

@abstractmethod

def _select_arm(self, bandit, context):

"""Select an arm based on the current context"""

pass

@abstractmethod

def _update_params(self, bandit, context):

"""Update the policy parameters after an interaction"""

pass

@abstractmethod

def _initialize_params(self, bandit):

Initialize any policy-specific parameters that depend on information

from the bandit environment.

pass

@abstractmethod

def _reset_params(self):

Reset any model-specific parameters. This gets called within the

public `self.reset()` method.

pass

class EpsilonGreedy(BanditPolicyBase):

def __init__(self, epsilon=0.05, ev_prior=0.5):

r"""

An epsilon-greedy policy for multi-armed bandit problems.

Notes

Epsilon-greedy policies greedily select the arm with the highest

expected payoff with probability :math:`1-\epsilon`, and selects an arm

uniformly at random with probability :math:`\epsilon`:

.. math::

P(a) = \left\{

\begin{array}{lr}

\epsilon / N + (1 - \epsilon) &\text{if }

a = \arg \max_{a' \in \mathcal{A}}

\mathbb{E}_{q_{\hat{\theta}}}[r \mid a']\\

\epsilon / N &\text{otherwise}

\end{array}

\right.

where :math:`N = |\mathcal{A}|` is the number of arms,

:math:`q_{\hat{\theta}}` is the estimate of the arm payoff

distribution under current model parameters :math:`\hat{\theta}`, and

:math:`\mathbb{E}_{q_{\hat{\theta}}}[r \mid a']` is the expected

reward under :math:`q_{\hat{\theta}}` of receiving reward `r` after

taking action :math:`a'`.

Parameters

epsilon : float in [0, 1]

The probability of taking a random action. Default is 0.05.

ev_prior : float

The starting expected payoff for each arm before any data has been

observed. Default is 0.5.

self.epsilon = epsilon

self.ev_prior = ev_prior

self.pull_counts = defaultdict(lambda: 0)

@property

def parameters(self):

"""A dictionary containing the current policy parameters"""

return {"ev_estimates": self.ev_estimates}

@property

def hyperparameters(self):

"""A dictionary containing the policy hyperparameters"""

return {

"id": "EpsilonGreedy",

"epsilon": self.epsilon,

"ev_prior": self.ev_prior,

def _initialize_params(self, bandit):

from the bandit environment.

self.ev_estimates = {i: self.ev_prior for i in range(bandit.n_arms)}

self.is_initialized = True

def _select_arm(self, bandit, context=None):

if np.random.rand() < self.epsilon:

arm_id = np.random.choice(bandit.n_arms)

else:

ests = self.ev_estimates

(arm_id, _) = max(ests.items(), key=lambda x: x[1])

return arm_id

def _update_params(self, arm_id, reward, context=None):

E, C = self.ev_estimates, self.pull_counts

C[arm_id] += 1

E[arm_id] += (reward - E[arm_id]) / (C[arm_id])

def _reset_params(self):

public `self.reset()` method.

self.ev_estimates = {}

class UCB1(BanditPolicyBase):

def __init__(self, C=1, ev_prior=0.5):

r"""

A UCB1 policy for multi-armed bandit problems.

Notes

The UCB1 algorithm [*]_ guarantees the cumulative regret is bounded by log

`t`, where `t` is the current timestep. To make this guarantee UCB1

assumes all arm payoffs are between 0 and 1.

Under UCB1, the upper confidence bound on the expected value for

pulling arm `a` at timestep `t` is:

.. math::

\text{UCB}(a, t) = \text{EV}_t(a) + C \sqrt{\frac{2 \log t}{N_t(a)}}

where :math:`\text{EV}_t(a)` is the average of the rewards recieved so

far from pulling arm `a`, `C` is a free parameter controlling the

"optimism" of the confidence upper bound for :math:`\text{UCB}(a, t)`

(for logarithmic regret bounds, `C` must equal 1), and :math:`N_t(a)`

is the number of times arm `a` has been pulled during the previous `t -

1` timesteps.

References

.. [*] Auer, P., Cesa-Bianchi, N., & Fischer, P. (2002). Finite-time

analysis of the multiarmed bandit problem. *Machine Learning,

Parameters

C : float in (0, +infinity)

A confidence/optimisim parameter affecting the degree of

exploration, where larger values encourage greater exploration. The

UCB1 algorithm assumes `C=1`. Default is 1.

ev_prior : float

The starting expected value for each arm before any data has been

observed. Default is 0.5.

self.C = C

self.ev_prior = ev_prior

@property

def parameters(self):

"""A dictionary containing the current policy parameters"""

return {"ev_estimates": self.ev_estimates}

@property

def hyperparameters(self):

"""A dictionary containing the policy hyperparameters"""

return {

"C": self.C,

"id": "UCB1",

"ev_prior": self.ev_prior,

def _initialize_params(self, bandit):

from the bandit environment.

self.is_initialized = True

def _select_arm(self, bandit, context=None):

# add eps to avoid divide-by-zero errors on the first pull of each arm

eps = np.finfo(float).eps

N, T = bandit.n_arms, self.step + 1

E, C = self.ev_estimates, self.pull_counts

scores = [E[a] + self.C * np.sqrt(np.log(T) / (C[a] + eps)) for a in range(N)]

return np.argmax(scores)

def _update_params(self, arm_id, reward, context=None):

C[arm_id] += 1

E[arm_id] += (reward - E[arm_id]) / (C[arm_id])

def _reset_params(self):

public :method:`reset` method.

self.ev_estimates = {}

self.pull_counts = defaultdict(lambda: 0)

class ThompsonSamplingBetaBinomial(BanditPolicyBase):

def __init__(self, alpha=1, beta=1):

r"""

A conjugate Thompson sampling [1]_ [2]_ policy for multi-armed bandits with

Bernoulli likelihoods.

Notes

The policy assumes independent Beta priors on the Bernoulli arm payoff

probabilities, :math:`\theta`:

.. math::

\theta_k \sim \text{Beta}(\alpha_k, \beta_k) \\

r \mid \theta_k \sim \text{Bernoulli}(\theta_k)

where :math:`k \in \{1,\ldots,K \}` indexes arms in the MAB and

:math:`\theta_k` is the parameter of the Bernoulli likelihood for arm

`k`. The sampler begins by selecting an arm with probability

proportional to its payoff probability under the initial Beta prior.

After pulling the sampled arm and receiving a reward, `r`, the sampler

computes the posterior over the model parameters (arm payoffs) via

Bayes' rule, and then samples a new action in proportion to its payoff

probability under this posterior. This process (i.e., sample action

from posterior, take action and receive reward, compute updated

posterior) is repeated until the number of trials is exhausted.

Note that due to the conjugacy between the Beta prior and Bernoulli

likelihood the posterior for each arm will also be Beta-distributed and

can computed and sampled from efficiently:

.. math::

\theta_k \mid r \sim \text{Beta}(\alpha_k + r, \beta_k + 1 - r)

References

.. [1] Thompson, W. (1933). On the likelihood that one unknown

probability exceeds another in view of the evidence of two samples.

*Biometrika, 25(3/4)*, 285-294.

.. [2] Chapelle, O., & Li, L. (2011). An empirical evaluation of

Thompson sampling. *Advances in Neural Information Processing

Systems, 24*, 2249-2257.

Parameters

alpha : float or list of length `K`

Parameter for the Beta prior on arm payouts. If a float, this value

will be used in the prior for all of the `K` arms.

beta : float or list of length `K`

self.alphas, self.betas = [], []

self.alpha, self.beta = alpha, beta

self.is_initialized = False

@property

def parameters(self):

"""A dictionary containing the current policy parameters"""

return {

"ev_estimates": self.ev_estimates,

"alphas": self.alphas,

"betas": self.betas,

@property

def hyperparameters(self):

"""A dictionary containing the policy hyperparameters"""

return {

"id": "ThompsonSamplingBetaBinomial",

"alpha": self.alpha,

"beta": self.beta,

def _initialize_params(self, bandit):

bhp = bandit.hyperparameters

fstr = "ThompsonSamplingBetaBinomial only defined for BernoulliBandit, got: {}"

assert bhp["id"] == "BernoulliBandit", fstr.format(bhp["id"])

# initialize the model prior

if is_number(self.alpha):

self.alphas = [self.alpha] * bandit.n_arms

if is_number(self.beta):

self.betas = [self.beta] * bandit.n_arms

assert len(self.alphas) == len(self.betas) == bandit.n_arms

self.ev_estimates = {i: self._map_estimate(i, 1) for i in range(bandit.n_arms)}

self.is_initialized = True

def _select_arm(self, bandit, context):

if not self.is_initialized:

# draw a sample from the current model posterior

posterior_sample = np.random.beta(self.alphas, self.betas)

# greedily select an action based on this sample

return np.argmax(posterior_sample)

def _update_params(self, arm_id, rwd, context):

Compute the parameters of the Beta posterior, P(payoff prob | rwd),

for arm `arm_id`.

self.alphas[arm_id] += rwd

self.betas[arm_id] += 1 - rwd

self.ev_estimates[arm_id] = self._map_estimate(arm_id, rwd)

def _map_estimate(self, arm_id, rwd):

"""Compute the current MAP estimate for an arm's payoff probability"""

A, B = self.alphas, self.betas

if A[arm_id] > 1 and B[arm_id] > 1:

map_payoff_prob = (A[arm_id] - 1) / (A[arm_id] + B[arm_id] - 2)

elif A[arm_id] < 1 and B[arm_id] < 1:

map_payoff_prob = rwd  # 0 or 1 equally likely, make a guess

elif A[arm_id] <= 1 and B[arm_id] > 1:

map_payoff_prob = 0

elif A[arm_id] > 1 and B[arm_id] <= 1:

map_payoff_prob = 1

else:

map_payoff_prob = 0.5

return map_payoff_prob

def _reset_params(self):

public `self.reset()` method.

self.alphas, self.betas = [], []

self.ev_estimates = {}

class LinUCB(BanditPolicyBase):

def __init__(self, alpha=1):

A disjoint linear UCB policy [*]_ for contextual linear bandits.

Notes

LinUCB is only defined for :class:`ContextualLinearBandit <numpy_ml.bandits.ContextualLinearBandit>` environments.

References

.. [*] Li, L., Chu, W., Langford, J., & Schapire, R. (2010). A

contextual-bandit approach to personalized news article

recommendation. In *Proceedings of the 19th International Conference

on World Wide Web*, 661-670.

Parameters

alpha : float

A confidence/optimisim parameter affecting the amount of

exploration. Default is 1.

"""  # noqa

self.alpha = alpha

self.A, self.b = [], []

self.is_initialized = False

@property

def parameters(self):

"""A dictionary containing the current policy parameters"""

return {"ev_estimates": self.ev_estimates, "A": self.A, "b": self.b}

@property

def hyperparameters(self):

"""A dictionary containing the policy hyperparameters"""

return {

"id": "LinUCB",

"alpha": self.alpha,

def _initialize_params(self, bandit):

from the bandit environment.

bhp = bandit.hyperparameters

fstr = "LinUCB only defined for contextual linear bandits, got: {}"

assert bhp["id"] == "ContextualLinearBandit", fstr.format(bhp["id"])

self.A, self.b = [], []

for _ in range(bandit.n_arms):

self.A.append(np.eye(bandit.D))

self.b.append(np.zeros(bandit.D))

self.is_initialized = True

def _select_arm(self, bandit, context):

probs = []

for a in range(bandit.n_arms):

C, A, b = context[:, a], self.A[a], self.b[a]

A_inv = np.linalg.inv(A)

theta_hat = A_inv @ b

p = theta_hat @ C + self.alpha * np.sqrt(C.T @ A_inv @ C)

probs.append(p)

return np.argmax(probs)

def _update_params(self, arm_id, rwd, context):

"""Compute the parameters for A and b."""

self.A[arm_id] += context[:, arm_id] @ context[:, arm_id].T

self.b[arm_id] += rwd * context[:, arm_id]

def _reset_params(self):

public `self.reset()` method.

self.A, self.b = [], []

self.ev_estimates = {}


"""Algorithms for approximate matrix factorization"""

from copy import deepcopy

import numpy as np

class VanillaALS:

def __init__(self, K, alpha=1, max_iter=200, tol=1e-4):

r"""

Approximately factor a real-valued matrix using regularized alternating

least-squares (ALS).

Notes

The regularized ALS minimization problem is

.. math::

\min_{\mathbf{W}, \mathbf{H}} ||\mathbf{X} - \mathbf{WH}||^2 -

\alpha \left(

||\mathbf{W}||^2 + ||\mathbf{H}||^2

\right)

where :math:`||\cdot||` denotes the Frobenius norm, **X** is the

:math:`N \times M` data matrix, :math:`\mathbf{W}` and

:math:`\mathbf{H}` are learned factor matrices with dimensions :math:`N

\times K` and :math:`K \times M`, respectively, and :math:`\alpha` is a

user-defined regularization weight.

ALS proceeds by alternating between fixing **W** and optimizing for

**H** and fixing **H** and optimizing for **W**. Vanilla ALS has no

convergance guarantees and the objective function is prone to

oscillation across updates, particularly for dense input matrices [1]_.

References

.. [1] Gillis, N. (2014). The why and how of nonnegative matrix

factorization.  *Regularization, optimization, kernels, and support

vector machines, 12(257)*, 257-291.

Parameters

K : int

The number of latent factors to include in the factor matrices W

and H.

alpha : float

The L2 regularization weight on the factor matrices. Larger

values result in more aggressive regularization. Default is 1.

max_iter : int

The maximum number of iterations to run before stopping. Default is

tol : float

The tolerance for the stopping condition. Default is 1e-4.

self.K = K

self.W = None

self.H = None

self.tol = tol

self.alpha = alpha

self.max_iter = max_iter

@property

def parameters(self):

"""Return a dictionary of the current model parameters"""

return {"W": self.W, "H": self.H}

@property

def hyperparameters(self):

"""Return a dictionary of the model hyperparameters"""

return {

"id": "ALSFactor",

"K": self.K,

"tol": self.tol,

"alpha": self.alpha,

"max_iter": self.max_iter,

def _init_factor_matrices(self, X, W=None, H=None):

"""Randomly initialize the factor matrices"""

N, M = X.shape

scale = np.sqrt(X.mean() / self.K)

self.W = np.random.rand(N, self.K) * scale if W is None else W

self.H = np.random.rand(self.K, M) * scale if H is None else H

assert self.W.shape == (N, self.K)

assert self.H.shape == (self.K, M)

def _loss(self, X, Xhat):

"""Regularized Frobenius loss"""

alpha, W, H = self.alpha, self.W, self.H

sq_fnorm = lambda x: np.sum(x ** 2)  # noqa: E731

return sq_fnorm(X - Xhat) + alpha * (sq_fnorm(W) + sq_fnorm(H))

def _update_factor(self, X, A):

"""Perform the ALS update"""

T1 = np.linalg.inv(A.T @ A + self.alpha * np.eye(self.K))

return X @ A @ T1

def fit(self, X, W=None, H=None, n_initializations=10, verbose=False):

Factor a data matrix into two low rank factors via ALS.

Parameters

X : numpy array of shape `(N, M)`

The data matrix to factor.

W : numpy array of shape `(N, K)` or None

An initial value for the `W` factor matrix. If None, initialize `W`

randomly. Default is None.

H : numpy array of shape `(K, M)` or None

An initial value for the `H` factor matrix. If None, initialize `H`

randomly. Default is None.

n_initializations : int

Number of re-initializations of the algorithm to perform before

taking the answer with the lowest reconstruction error. This value

is ignored and set to 1 if both `W` and `H` are not None. Default

is 10.

verbose : bool

Whether to print the loss at each iteration. Default is False.

if W is not None and H is not None:

n_initializations = 1

best_loss = np.inf

for f in range(n_initializations):

if verbose:

print("\nINITIALIZATION {}".format(f + 1))

new_W, new_H, loss = self._fit(X, W, H, verbose)

if loss <= best_loss:

best_loss = loss

best_W, best_H = deepcopy(new_W), deepcopy(new_H)

self.W, self.H = best_W, best_H

if verbose:

print("\nFINAL LOSS: {}".format(best_loss))

def _fit(self, X, W, H, verbose):

self._init_factor_matrices(X, W, H)

W, H = self.W, self.H

for i in range(self.max_iter):

W = self._update_factor(X, H.T)

H = self._update_factor(X.T, W).T

loss = self._loss(X, W @ H)

if verbose:

print("[Iter {}] Loss: {:.8f}".format(i + 1, loss))

if loss <= self.tol:

break

return W, H, loss

class NMF:

def __init__(self, K, max_iter=200, tol=1e-4):

r"""

Nonnegative matrix factorization (NMF) performed using fast

hierarchical alternating least squares (HALS) [*]_.

Notes

The NMF minimization problem is

.. math::

\ \ \ \ \text{subject to } \mathbf{W}, \mathbf{H} \geq 0

where :math:`||\cdot||` denotes the Frobenius norm, and the notation

:math:`\mathbf{A} \geq 0` indicates that each element of **A** is

greater than or equal to 0. In the above equation, **X** is the

:math:`N \times M` data matrix, :math:`\mathbf{W}` and

\times K` and :math:`K \times M`, respectively.

As with other ALS-based approaches, there is no guarantee that NMF will

converge to a stationary point, let alone a global minimum. As a result

it is generally good practice to run the algorithm multiple times with

different initializations, taking the outcome that achieves the lowest

reconstruction error.

References

.. [*] Cichocki, A., & Phan, A. (2009). Fast local algorithms for

large scale nonnegative matrix and tensor factorizations. *IEICE

Transactions on Fundamentals of Electronics, Communications and

Computer Sciences, 92(3)*, 708-721.

Parameters

K : int

The number of latent factors to include in the factor matrices **W**

and **H**.

max_iter : int

tol : float

The tolerance for the stopping condition. Default is 1e-4.

self.K = K

self.W = None

self.H = None

self.tol = tol

self.max_iter = max_iter

@property

def parameters(self):

"""Return a dictionary of the current model parameters"""

return {"W": self.W, "H": self.H}

@property

def hyperparameters(self):

"""Return a dictionary of the model hyperparameters"""

return {

"id": "NMF",

"K": self.K,

"tol": self.tol,

"max_iter": self.max_iter,

def _init_factor_matrices(self, X, W, H):

"""Initialize the factor matrices using vanilla ALS"""

ALS = None

N, M = X.shape

# initialize factors using ALS if not already defined

if W is None:

ALS = VanillaALS(self.K, alpha=0, max_iter=200)

ALS.fit(X, verbose=False)

W = ALS.W / np.linalg.norm(ALS.W, axis=0)

if H is None:

H = np.abs(np.random.rand(self.K, M)) if ALS is None else ALS.H

assert W.shape == (N, self.K)

assert H.shape == (self.K, M)

self.H = H

self.W = W

def _loss(self, X, Xhat):

"""Return the least-squares reconstruction loss between X and Xhat"""

return np.sum((X - Xhat) ** 2)

def _update_H(self, X, W, H):

"""Perform the fast HALS update for H"""

eps = np.finfo(float).eps

XtW = X.T @ W  # dim: (M, K)

WtW = W.T @ W  # dim: (K, K)

for k in range(self.K):

H[k, :] += XtW[:, k] - H.T @ WtW[:, k]

H[k, :] = np.clip(H[k, :], eps, np.inf)  # enforce nonnegativity

return H

def _update_W(self, X, W, H):

"""Perform the fast HALS update for W"""

eps = np.finfo(float).eps

XHt = X @ H.T  # dim: (N, K)

HHt = H @ H.T  # dim: (K, K)

for k in range(self.K):

W[:, k] = W[:, k] * HHt[k, k] + XHt[:, k] - W @ HHt[:, k]

W[:, k] = np.clip(W[:, k], eps, np.inf)  # enforce nonnegativity

# renormalize the new column

n = np.linalg.norm(W[:, k])

W[:, k] /= n if n > 0 else 1.0

return W

r"""

Factor a data matrix into two nonnegative low rank factor matrices via

fast HALS.

Notes

This method implements Algorithm 2 from [*]_. In contrast to vanilla

ALS, HALS proceeds by minimizing a *set* of local cost functions with

the same global minima. Each cost function is defined on a "residue" of

the factor matrices **W** and **H**:

.. math::

\mathbf{X}^{(j)} :=

\mathbf{X} - \mathbf{WH}^\top + \mathbf{w}_j \mathbf{h}_j^\top

where :math:`\mathbf{X}^{(j)}` is the :math:`j^{th}` residue, **X** is

the input data matrix, and :math:`\mathbf{w}_j` and

:math:`\mathbf{h}_j` are the :math:`j^{th}` columns of the current

factor matrices **W** and **H**. HALS proceeds by minimizing the cost

for each residue, first with respect to :math:`\mathbf{w}_j`, and then

with respect to :math:`\mathbf{h}_j`. In either case, the cost for

residue `j`, :math:`\mathcal{L}^{(j)}` is simply:

.. math::

\mathcal{L}^{(j)} :=

|| \mathbf{X}^{(j)} - \mathbf{w}_j \mathbf{h}_j^\top ||

where :math:`||\cdot||` denotes the Frobenius norm. For NMF,

minimization is performed under the constraint that all elements of

both **W** and **H** are nonnegative.

References

Computer Sciences, 92(3)*, 708-721.

Parameters

X : numpy array of shape `(N, M)`

The data matrix to factor.

W : numpy array of shape `(N, K)` or None

**W** using vanilla ALS. Default is None.

H : numpy array of shape `(K, M)` or None

**H** using vanilla ALS. Default is None.

n_initializations : int

is 10.

verbose : bool

if W is not None and H is not None:

n_initializations = 1

best_loss = np.inf

for f in range(n_initializations):

if verbose:

print("\nINITIALIZATION {}".format(f + 1))

new_W, new_H, loss = self._fit(X, W, H, verbose)

if loss <= best_loss:

best_loss = loss

best_W, best_H = deepcopy(new_W), deepcopy(new_H)

self.W, self.H = best_W, best_H

if verbose:

print("\nFINAL LOSS: {}".format(best_loss))

def _fit(self, X, W, H, verbose):

self._init_factor_matrices(X, W, H)

W, H = self.W, self.H

for i in range(self.max_iter):

H = self._update_H(X, W, H)

W = self._update_W(X, W, H)

loss = self._loss(X, W @ H)

if verbose:

print("[Iter {}] Loss: {:.8f}".format(i + 1, loss))

if loss <= self.tol:

break

return W, H, loss


"""Algorithms for approximate matrix factorization"""

from .factors import *


import numpy as np

from scipy.special import digamma, polygamma, gammaln

class LDA(object):

def __init__(self, T=10):

Vanilla (non-smoothed) LDA model trained using variational EM.

Generates maximum-likelihood estimates for model paramters

`alpha` and `beta`.

Parameters

T : int

Number of topics

Attributes

D : int

Number of documents

N : list of length `D`

Number of words in each document

V : int

Number of unique word tokens across all documents

phi : :py:class:`ndarray <numpy.ndarray>` of shape `(D, N[d], T)`

Variational approximation to word-topic distribution

gamma : :py:class:`ndarray <numpy.ndarray>` of shape `(D, T)`

Variational approximation to document-topic distribution

alpha : :py:class:`ndarray <numpy.ndarray>` of shape `(1, T)`

Parameter for the Dirichlet prior on the document-topic distribution

beta  : :py:class:`ndarray <numpy.ndarray>` of shape `(V, T)`

Word-topic distribution

self.T = T

def _maximize_phi(self):

Optimize variational parameter phi

ϕ_{t, n} ∝ β_{t, w_n}  e^( Ψ(γ_t) )

D = self.D

N = self.N

T = self.T

phi = self.phi

beta = self.beta

gamma = self.gamma

corpus = self.corpus

for d in range(D):

for n in range(N[d]):

for t in range(T):

w_n = int(corpus[d][n])

phi[d][n, t] = beta[w_n, t] * np.exp(dg(gamma, d, t))

# Normalize over topics

phi[d][n, :] = phi[d][n, :] / np.sum(phi[d][n, :])

return phi

def _maximize_gamma(self):

Optimize variational parameter gamma

γ_t = α_t + \sum_{n=1}^{N_d} ϕ_{t, n}

D = self.D

phi = self.phi

alpha = self.alpha

gamma = np.tile(alpha, (D, 1)) + np.array(

list(map(lambda x: np.sum(x, axis=0), phi))

return gamma

def _maximize_beta(self):

Optimize model parameter beta

β_{t, n} ∝ \sum_{d=1}^D \sum_{i=1}^{N_d} ϕ_{d, t, n} [ i = n]

T = self.T

V = self.V

phi = self.phi

beta = self.beta

corpus = self.corpus

for n in range(V):

# Construct binary mask [i == n] to be the same shape as phi

mask = [np.tile((doc == n), (T, 1)).T for doc in corpus]

beta[n, :] = np.sum(

np.array(list(map(lambda x: np.sum(x, axis=0), phi * mask))), axis=0

# Normalize over words

for t in range(T):

beta[:, t] = beta[:, t] / np.sum(beta[:, t])

return beta

def _maximize_alpha(self, max_iters=1000, tol=0.1):

Optimize alpha using Blei's O(n) Newton-Raphson modification

for a Hessian with special structure

D = self.D

T = self.T

alpha = self.alpha

gamma = self.gamma

for _ in range(max_iters):

alpha_old = alpha

#  Calculate gradient

g = D * (digamma(np.sum(alpha)) - digamma(alpha)) + np.sum(

digamma(gamma) - np.tile(digamma(np.sum(gamma, axis=1)), (T, 1)).T,

#  Calculate Hessian diagonal component

h = -D * polygamma(1, alpha)

#  Calculate Hessian constant component

z = D * polygamma(1, np.sum(alpha))

#  Calculate constant

c = np.sum(g / h) / (z ** (-1.0) + np.sum(h ** (-1.0)))

#  Update alpha

alpha = alpha - (g - c) / h

#  Check convergence

if np.sqrt(np.mean(np.square(alpha - alpha_old))) < tol:

break

return alpha

def _E_step(self):

Maximize the VLB with respect to the variational parameters, γ and ϕ

self.phi = self._maximize_phi()

self.gamma = self._maximize_gamma()

def _M_step(self):

Maximize the VLB with respect to the model parameters, α and β

self.beta = self._maximize_beta()

self.alpha = self._maximize_alpha()

def VLB(self):

Return the variational lower bound associated with the current model

parameters.

phi = self.phi

alpha = self.alpha

beta = self.beta

gamma = self.gamma

corpus = self.corpus

D = self.D

T = self.T

N = self.N

a, b, c, _d = 0, 0, 0, 0

for d in range(D):

a += (

gammaln(np.sum(alpha))

- np.sum(gammaln(alpha))

+ np.sum([(alpha[t] - 1) * dg(gamma, d, t) for t in range(T)])

_d += (

gammaln(np.sum(gamma[d, :]))

- np.sum(gammaln(gamma[d, :]))

+ np.sum([(gamma[d, t] - 1) * dg(gamma, d, t) for t in range(T)])

for n in range(N[d]):

w_n = int(corpus[d][n])

b += np.sum([phi[d][n, t] * dg(gamma, d, t) for t in range(T)])

c += np.sum([phi[d][n, t] * np.log(beta[w_n, t]) for t in range(T)])

_d += np.sum([phi[d][n, t] * np.log(phi[d][n, t]) for t in range(T)])

return a + b + c - _d

def initialize_parameters(self):

Provide reasonable initializations for model and variational parameters.

T = self.T

V = self.V

N = self.N

D = self.D

# initialize model parameters

self.alpha = 100 * np.random.dirichlet(10 * np.ones(T), 1)[0]

self.beta = np.random.dirichlet(np.ones(V), T).T

# initialize variational parameters

self.phi = np.array([1 / T * np.ones([N[d], T]) for d in range(D)])

self.gamma = np.tile(self.alpha, (D, 1)) + np.tile(N / T, (T, 1)).T

def train(self, corpus, verbose=False, max_iter=1000, tol=5):

Train the LDA model on a corpus of documents (bags of words).

Parameters

corpus : list of length `D`

A list of lists, with each sublist containing the tokenized text of

a single document.

verbose : bool

Whether to print the VLB at each training iteration. Default is

True.

max_iter : int

The maximum number of training iterations to perform before

breaking. Default is 1000.

tol : int

Break the training loop if the difference betwen the VLB on the

current iteration and the previous iteration is less than `tol`.

Default is 5.

self.D = len(corpus)

self.V = len(set(np.concatenate(corpus)))

self.N = np.array([len(d) for d in corpus])

self.corpus = corpus

vlb = -np.inf

for i in range(max_iter):

old_vlb = vlb

vlb = self.VLB()

delta = vlb - old_vlb

if verbose:

print("Iteration {}: {:.3f} (delta: {:.2f})".format(i + 1, vlb, delta))

if delta < tol:

break

#                                Utils                                #

def dg(gamma, d, t):

E[log X_t] where X_t ~ Dir

return digamma(gamma[d, t]) - digamma(np.sum(gamma[d, :]))


from .lda import *
from .lda_smoothed import *


import numpy as np

class SmoothedLDA(object):

def __init__(self, T, **kwargs):

A smoothed LDA model trained using collapsed Gibbs sampling. Generates

posterior mean estimates for model parameters `phi` and `theta`.

Parameters

T : int

Number of topics

Attributes

D : int

Number of documents

N : int

Total number of words across all documents

V : int

Number of unique word tokens across all documents

phi : :py:class:`ndarray <numpy.ndarray>` of shape `(N[d], T)`

The word-topic distribution

theta : :py:class:`ndarray <numpy.ndarray>` of shape `(D, T)`

The document-topic distribution

alpha : :py:class:`ndarray <numpy.ndarray>` of shape `(1, T)`

Parameter for the Dirichlet prior on the document-topic distribution

beta  : :py:class:`ndarray <numpy.ndarray>` of shape `(V, T)`

Parameter for the Dirichlet prior on the topic-word distribution

self.T = T

self.alpha = (50.0 / self.T) * np.ones(self.T)

if "alpha" in kwargs.keys():

self.alpha = (kwargs["alpha"]) * np.ones(self.T)

self.beta = 0.01

if "beta" in kwargs.keys():

self.beta = kwargs["beta"]

def _init_params(self, texts, tokens):

self.tokens = tokens

self.D = len(texts)

self.V = len(np.unique(self.tokens))

self.N = np.sum(np.array([len(doc) for doc in texts]))

self.word_document = np.zeros(self.N)

# now that we know the number of tokens in our corpus, we can set beta

self.beta = self.beta * np.ones(self.V)

count = 0

for doc_idx, doc in enumerate(texts):

for word_idx, word in enumerate(doc):

word_idx = word_idx + count

self.word_document[word_idx] = doc_idx

count = count + len(doc)

def train(self, texts, tokens, n_gibbs=2000):

Trains a topic model on the documents in texts.

Parameters

texts : array of length `(D,)`

The training corpus represented as an array of subarrays, where

each subarray corresponds to the tokenized words of a single

document.

tokens : array of length `(V,)`

The set of unique tokens in the documents in `texts`.

n_gibbs : int

The number of steps to run the collapsed Gibbs sampler during

training. Default is 2000.

Returns

C_wt : :py:class:`ndarray <numpy.ndarray>` of shape (V, T)

The word-topic count matrix

C_dt : :py:class:`ndarray <numpy.ndarray>` of shape (D, T)

The document-topic count matrix

assignments : :py:class:`ndarray <numpy.ndarray>` of shape (N, n_gibbs)

The topic assignments for each word in the corpus on each Gibbs

step.

self._init_params(texts, tokens)

C_wt, C_dt, assignments = self._gibbs_sampler(n_gibbs, texts)

self.fit_params(C_wt, C_dt)

return C_wt, C_dt, assignments

def what_did_you_learn(self, top_n=10):

Print the `top_n` most probable words under each topic

for tt in range(self.T):

top_idx = np.argsort(self.phi[:, tt])[::-1][:top_n]

top_tokens = self.tokens[top_idx]

print("\nTop Words for Topic %s:\n" % (str(tt)))

for token in top_tokens:

print("\t%s\n" % (str(token)))

def fit_params(self, C_wt, C_dt):

Estimate `phi`, the word-topic distribution, and `theta`, the

topic-document distribution.

Parameters

The word-topic count matrix

The document-topic count matrix

Returns

phi : :py:class:`ndarray <numpy.ndarray>` of shape `(V, T)`

The word-topic distribution

The document-topic distribution

self.phi = np.zeros([self.V, self.T])

self.theta = np.zeros([self.D, self.T])

b, a = self.beta[0], self.alpha[0]

for ii in range(self.V):

for jj in range(self.T):

self.phi[ii, jj] = (C_wt[ii, jj] + b) / (

np.sum(C_wt[:, jj]) + self.V * b

for dd in range(self.D):

for jj in range(self.T):

self.theta[dd, jj] = (C_dt[dd, jj] + a) / (

np.sum(C_dt[dd, :]) + self.T * a

return self.phi, self.theta

def _estimate_topic_prob(self, ii, d, C_wt, C_dt):

Compute an approximation of the conditional probability that token ii

is assigned to topic jj given all previous topic assignments and the

current document d: p(t_i = j | t_{-i}, w_i, d_i)

p_vec = np.zeros(self.T)

b, a = self.beta[0], self.alpha[0]

for jj in range(self.T):

# prob of word ii under topic jj

frac1 = (C_wt[ii, jj] + b) / (np.sum(C_wt[:, jj]) + self.V * b)

# prob of topic jj under document d

frac2 = (C_dt[d, jj] + a) / (np.sum(C_dt[d, :]) + self.T * a)

p_vec[jj] = frac1 * frac2

return p_vec / np.sum(p_vec)

def _gibbs_sampler(self, n_gibbs, texts):

Collapsed Gibbs sampler for estimating the posterior distribution over

topic assignments.

# Initialize count matrices

C_wt = np.zeros([self.V, self.T])

C_dt = np.zeros([self.D, self.T])

assignments = np.zeros([self.N, n_gibbs + 1])

# Randomly initialize topic assignments for words

for ii in range(self.N):

token_idx = np.concatenate(texts)[ii]

assignments[ii, 0] = np.random.randint(0, self.T)

doc = self.word_document[ii]

C_dt[doc, assignments[ii, 0]] += 1

C_wt[token_idx, assignments[ii, 0]] += 1

# run collapsed Gibbs sampler

for gg in range(n_gibbs):

print("Gibbs iteration {} of {}".format(gg + 1, n_gibbs))

for jj in range(self.N):

token_idx = np.concatenate(texts)[jj]

# Decrement count matrices by 1

doc = self.word_document[jj]

C_wt[token_idx, assignments[jj, gg]] -= 1

C_dt[doc, assignments[jj, gg]] -= 1

# Draw new topic from our approximation of the conditional dist.

p_topics = self._estimate_topic_prob(token_idx, doc, C_wt, C_dt)

sampled_topic = np.nonzero(np.random.multinomial(1, p_topics))[0][0]

# Update count matrices

C_wt[token_idx, sampled_topic] += 1

C_dt[doc, sampled_topic] += 1

assignments[jj, gg + 1] = sampled_topic

return C_wt, C_dt, assignments


# flake8: noqa

from collections import Counter

# gold-standard imports

import huffman

import numpy as np

from scipy.fftpack import dct

from sklearn.preprocessing import StandardScaler

from sklearn.feature_extraction.text import TfidfVectorizer

# try:

#     from librosa.core.time_frequency import fft_frequencies

# except ImportError:

#     # for librosa >= 0.8.0

#     from librosa import fft_frequencies

# from librosa.feature import mfcc as lr_mfcc

# from librosa.util import frame

# from librosa.filters import mel

# numpy-ml implementations

from numpy_ml.preprocessing.general import Standardizer

from numpy_ml.preprocessing.nlp import HuffmanEncoder, TFIDFEncoder

from numpy_ml.preprocessing.dsp import (

DCT,

DFT,

mfcc,

from numpy_ml.utils.testing import random_paragraph

def test_tfidf(N=15):

i = 0

while i < N:

docs = []

n_docs = np.random.randint(1, 10)

for d in range(n_docs):

n_lines = np.random.randint(1, 1000)

lines = [random_paragraph(np.random.randint(1, 10)) for _ in range(n_lines)]

docs.append("\n".join([" ".join(l) for l in lines]))

smooth = bool(np.random.randint(2))

tfidf = TFIDFEncoder(

lowercase=True,

gold = TfidfVectorizer(

input="content",

norm=None,

lowercase=True,

tfidf.fit(docs)

mine = tfidf.transform(ignore_special_chars=True)

theirs = gold.fit_transform(docs).toarray()

np.testing.assert_almost_equal(mine, theirs)

print("PASSED")

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_LSTMCell(N=15):

from numpy_ml.neural_nets.layers import LSTMCell

N = np.inf if N is None else N

i = 1

while i < N + 1:

n_ex = np.random.randint(1, 10)

n_in = np.random.randint(1, 10)

n_out = np.random.randint(1, 10)

n_t = np.random.randint(1, 10)

X = random_tensor((n_ex, n_in, n_t), standardize=True)

# initialize LSTM layer

L1 = LSTMCell(n_out=n_out)

# forward prop

Cs = []

y_preds = []

for t in range(n_t):

y_pred, Ct = L1.forward(X[:, :, t])

Cs.append(Ct)

# backprop

dLdX = []

dLdAt = np.ones_like(y_preds[t])

for t in reversed(range(n_t)):

dLdXt = L1.backward(dLdAt)

dLdX.insert(0, dLdXt)

dLdX = np.dstack(dLdX)

y_preds = np.dstack(y_preds)

Cs = np.array(Cs)

# get gold standard gradients

gold_mod = TorchLSTMCell(n_in, n_out, L1.parameters)

golds = gold_mod.extract_grads(X)

params = [

(X, "X"),

(np.array(Cs), "C"),

(y_preds, "y"),

(L1.parameters["bo"].T, "bo"),

(L1.parameters["bu"].T, "bu"),

(L1.parameters["bf"].T, "bf"),

(L1.parameters["bc"].T, "bc"),

(L1.parameters["Wo"], "Wo"),

(L1.parameters["Wu"], "Wu"),

(L1.parameters["Wf"], "Wf"),

(L1.parameters["Wc"], "Wc"),

(L1.gradients["bo"].T, "dLdBo"),

(L1.gradients["bu"].T, "dLdBu"),

(L1.gradients["bf"].T, "dLdBf"),

(L1.gradients["bc"].T, "dLdBc"),

(L1.gradients["Wo"], "dLdWo"),

(L1.gradients["Wu"], "dLdWu"),

(L1.gradients["Wf"], "dLdWf"),

(L1.gradients["Wc"], "dLdWc"),

(dLdX, "dLdX"),

print("Case {}".format(i))

for ix, (mine, label) in enumerate(params):

mine,

golds[label],

err_msg=err_fmt(params, golds, ix),

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

mine, label = params[ix]

label, mine, label, golds[label]

err_msg += warn_str

return err_msg

def test_MultiplyLayer(N=15):

from numpy_ml.neural_nets.layers import Multiply

from numpy_ml.neural_nets.activations import Tanh, ReLU, Sigmoid, Affine

N = np.inf if N is None else N

acts = [

(Tanh(), nn.Tanh(), "Tanh"),

(Sigmoid(), nn.Sigmoid(), "Sigmoid"),

(ReLU(), nn.ReLU(), "ReLU"),

(Affine(), TorchLinearActivation(), "Affine"),

i = 1

while i < N + 1:

Xs = []

n_ex = np.random.randint(1, 100)

n_in = np.random.randint(1, 100)

n_entries = np.random.randint(2, 5)

for _ in range(n_entries):

Xs.append(random_tensor((n_ex, n_in), standardize=True))

act_fn, torch_fn, act_fn_name = acts[np.random.randint(0, len(acts))]

# initialize Add layer

L1 = Multiply(act_fn)

# forward prop

y_pred = L1.forward(Xs)

# backprop

dLdy = np.ones_like(y_pred)

dLdXs = L1.backward(dLdy)

# get gold standard gradients

gold_mod = TorchMultiplyLayer(torch_fn)

golds = gold_mod.extract_grads(Xs)

params = [(Xs, "Xs"), (y_pred, "Y")]

params.extend(

[(dldxi, "dLdX{}".format(i + 1)) for i, dldxi in enumerate(dLdXs)]

print("\nTrial {}".format(i))

print("n_ex={}, n_in={}".format(n_ex, n_in))

print("n_entries={}, act_fn={}".format(n_entries, str(act_fn)))

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=1

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import numpy as np

import scipy

import networkx as nx

from sklearn.neighbors import BallTree as sk_BallTree

from sklearn.metrics.pairwise import rbf_kernel as sk_rbf

from sklearn.metrics.pairwise import linear_kernel as sk_linear

from sklearn.metrics.pairwise import polynomial_kernel as sk_poly

from numpy_ml.utils.distance_metrics import (

hamming,

euclidean,

chebyshev,

manhattan,

minkowski,

from numpy_ml.utils.kernels import LinearKernel, PolynomialKernel, RBFKernel

from numpy_ml.utils.data_structures import BallTree

from numpy_ml.utils.graphs import (

Edge,

DiGraph,

UndirectedGraph,

def test_chebyshev(N=1):

i = 0

while i < N:

N = np.random.randint(1, 100)

x = np.random.rand(N)

y = np.random.rand(N)

mine = chebyshev(x, y)

theirs = scipy.spatial.distance.chebyshev(x, y)

np.testing.assert_almost_equal(mine, theirs)

print("PASSED")

i += 1


# flake8: noqa

import numpy as np

from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor

from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor

from sklearn.metrics import accuracy_score, mean_squared_error

from sklearn.datasets import make_regression, make_blobs

from sklearn.model_selection import train_test_split

from numpy_ml.trees.gbdt import GradientBoostedDecisionTree

from numpy_ml.trees.dt import DecisionTree, Node, Leaf

from numpy_ml.trees.rf import RandomForest

from numpy_ml.utils.testing import random_tensor

def clone_tree(dtree):

children_left = dtree.tree_.children_left

children_right = dtree.tree_.children_right

feature = dtree.tree_.feature

threshold = dtree.tree_.threshold

values = dtree.tree_.value

def grow(node_id):

l, r = children_left[node_id], children_right[node_id]

if l == r:

return Leaf(values[node_id].argmax())

n = Node(None, None, (feature[node_id], threshold[node_id]))

n.left = grow(l)

n.right = grow(r)

return n

node_id = 0

root = Node(None, None, (feature[node_id], threshold[node_id]))

root.left = grow(children_left[node_id])

root.right = grow(children_right[node_id])

return root

def compare_trees(mine, gold):

clone = clone_tree(gold)

mine = mine.root

def test(mine, clone):

if isinstance(clone, Node) and isinstance(mine, Node):

assert mine.feature == clone.feature, "Node {} not equal".format(depth)

np.testing.assert_allclose(mine.threshold, clone.threshold)

test(mine.left, clone.left, depth + 1)

test(mine.right, clone.right, depth + 1)

elif isinstance(clone, Leaf) and isinstance(mine, Leaf):

np.testing.assert_allclose(mine.value, clone.value)

return

else:

raise ValueError("Nodes at depth {} are not equal".format(depth))

depth = 0

ok = True

while ok:

assert mine.feature == clone.feature

test(mine.left, clone.left, depth + 1)

return

else:

def test_DecisionTree(N=1):

i = 1

while i <= N:

n_ex = np.random.randint(2, 100)

n_feats = np.random.randint(2, 100)

max_depth = np.random.randint(1, 5)

classifier = np.random.choice([True, False])

if classifier:

# create classification problem

n_classes = np.random.randint(2, 10)

X, Y = make_blobs(

n_samples=n_ex, centers=n_classes, n_features=n_feats, random_state=i

X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.3, random_state=i)

# initialize model

def loss(yp, y):

return 1 - accuracy_score(yp, y)

criterion = np.random.choice(["entropy", "gini"])

mine = DecisionTree(

classifier=classifier, max_depth=max_depth, criterion=criterion

gold = DecisionTreeClassifier(

criterion=criterion,

splitter="best",

else:

# create regeression problem

X, Y = make_regression(n_samples=n_ex, n_features=n_feats, random_state=i)

# initialize model

criterion = "mse"

loss = mean_squared_error

mine = DecisionTree(

criterion=criterion, max_depth=max_depth, classifier=classifier

gold = DecisionTreeRegressor(

criterion=criterion, max_depth=max_depth, splitter="best"

print("Trial {}".format(i))

print("\tClassifier={}, criterion={}".format(classifier, criterion))

print("\tmax_depth={}, n_feats={}, n_ex={}".format(max_depth, n_feats, n_ex))

if classifier:

print("\tn_classes: {}".format(n_classes))

# fit 'em

mine.fit(X, Y)

gold.fit(X, Y)

# get preds on training set

y_pred_mine = mine.predict(X)

y_pred_gold = gold.predict(X)

loss_mine = loss(y_pred_mine, Y)

loss_gold = loss(y_pred_gold, Y)

# get preds on test set

y_pred_mine_test = mine.predict(X_test)

y_pred_gold_test = gold.predict(X_test)

loss_mine_test = loss(y_pred_mine_test, Y_test)

loss_gold_test = loss(y_pred_gold_test, Y_test)

try:

np.testing.assert_almost_equal(loss_mine, loss_gold)

print("\tLoss on training: {}".format(loss_mine))

except AssertionError as e:

print("\tTraining losses not equal:\n{}".format(e))

try:

np.testing.assert_almost_equal(loss_mine_test, loss_gold_test)

print("\tLoss on test: {}".format(loss_mine_test))

except AssertionError as e:

print("\tTest losses not equal:\n{}".format(e))

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_Conv1D(N=15):

from numpy_ml.neural_nets.layers import Conv1D

from numpy_ml.neural_nets.activations import Tanh, ReLU, Sigmoid, Affine

N = np.inf if N is None else N

acts = [

(Tanh(), nn.Tanh(), "Tanh"),

(Sigmoid(), nn.Sigmoid(), "Sigmoid"),

(ReLU(), nn.ReLU(), "ReLU"),

(Affine(), TorchLinearActivation(), "Affine"),

i = 1

while i < N + 1:

n_ex = np.random.randint(1, 10)

l_in = np.random.randint(1, 10)

n_in, n_out = np.random.randint(1, 3), np.random.randint(1, 3)

f_width = min(l_in, np.random.randint(1, 5))

p, s = np.random.randint(0, 5), np.random.randint(1, 3)

d = np.random.randint(0, 5)

fc = f_width * (d + 1) - d

l_out = int(1 + (l_in + 2 * p - fc) / s)

if l_out <= 0:

continue

X = random_tensor((n_ex, l_in, n_in), standardize=True)

# randomly select an activation function

act_fn, torch_fn, act_fn_name = acts[np.random.randint(0, len(acts))]

# initialize Conv2D layer

L1 = Conv1D(

pad=p,

stride=s,

dilation=d,

# forward prop

y_pred = L1.forward(X)

# backprop

dLdy = np.ones_like(y_pred)

dLdX = L1.backward(dLdy)

# get gold standard gradients

gold_mod = TorchConv1DLayer(

n_in, n_out, torch_fn, L1.parameters, L1.hyperparameters

golds = gold_mod.extract_grads(X)

params = [

(L1.X[0], "X"),

(y_pred, "y"),

(L1.parameters["W"], "W"),

(L1.parameters["b"], "b"),

(L1.gradients["W"], "dLdW"),

(L1.gradients["b"], "dLdB"),

(dLdX, "dLdX"),

print("\nTrial {}".format(i))

print("pad={}, stride={}, f_width={}, n_ex={}".format(p, s, f_width, n_ex))

print("l_in={}, n_in={}".format(l_in, n_in))

print("l_out={}, n_out={}".format(l_out, n_out))

print("dilation={}".format(d))

for ix, (mine, label) in enumerate(params):

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=4

print("\tPASSED {}".format(label))

i += 1

def test_pad1D(N=15):

from .nn_torch_models import TorchCausalConv1d, torchify

N = np.inf if N is None else N

i = 1

while i < N + 1:

p = np.random.choice(["same", "causal"])

n_ex = np.random.randint(1, 10)

l_in = np.random.randint(1, 10)

s = np.random.randint(1, 3)

d = np.random.randint(0, 5)

X_pad, _ = pad1D(X, p, kernel_width=f_width, stride=s, dilation=d)

# initialize Conv2D layer

L1 = Conv1D(out_ch=n_out, kernel_width=f_width, pad=0, stride=s, dilation=d)

# forward prop

try:

y_pred = L1.forward(X_pad)

except ValueError:

continue

# ignore n. output channels

print("Trial {}".format(i))

print("p={} d={} s={} l_in={} f_width={}".format(p, d, s, l_in, f_width))

print("n_ex={} n_in={} n_out={}".format(n_ex, n_in, n_out))

assert y_pred.shape[:2] == X.shape[:2], print(

"y_pred.shape={} X.shape={}".format(y_pred.shape, X.shape)

if p == "causal":

gold = TorchCausalConv1d(

stride=s,

dilation=d + 1,

bias=True,

if s != 1:

print(

"TorchCausalConv1D does not do `same` padding for stride > 1. Skipping"

continue

XT = torchify(np.moveaxis(X, [0, 1, 2], [0, -1, -2]))

else:

gold = nn.Conv1d(

stride=s,

dilation=d + 1,

bias=True,

XT = torchify(np.moveaxis(X_pad, [0, 1, 2], [0, -1, -2]))

# import weights and biases

# (f[0], n_in, n_out) -> (n_out, n_in, f[0])

b = L1.parameters["b"]

W = np.moveaxis(L1.parameters["W"], [0, 1, 2], [-1, -2, -3])

assert gold.weight.shape == W.shape

assert gold.bias.shape == b.flatten().shape

gold.weight = nn.Parameter(torch.FloatTensor(W))

gold.bias = nn.Parameter(torch.FloatTensor(b.flatten()))

outT = gold(XT)

if outT.ndimension() == 2:

import ipdb

gold_out = np.moveaxis(outT.detach().numpy(), [0, 1, 2], [0, -1, -2])

assert gold_out.shape[:2] == X.shape[:2]

[(y_pred.shape, "out.shape"), (y_pred, "out")],

{"out.shape": gold_out.shape, "out": gold_out},

print("PASSED\n")

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_Pool2D(N=15):

from numpy_ml.neural_nets.layers import Pool2D

N = np.inf if N is None else N

i = 1

while i < N + 1:

n_ex = np.random.randint(1, 10)

in_rows = np.random.randint(1, 10)

in_cols = np.random.randint(1, 10)

n_in = np.random.randint(1, 3)

f_shape = (

min(in_rows, np.random.randint(1, 5)),

min(in_cols, np.random.randint(1, 5)),

p, s = np.random.randint(0, max(1, min(f_shape) // 2)), np.random.randint(1, 3)

#  mode = ["max", "average"][np.random.randint(0, 2)]

mode = "average"

out_rows = int(1 + (in_rows + 2 * p - f_shape[0]) / s)

out_cols = int(1 + (in_cols + 2 * p - f_shape[1]) / s)

X = random_tensor((n_ex, in_rows, in_cols, n_in), standardize=True)

print("\nmode: {}".format(mode))

print("pad={}, stride={}, f_shape={}, n_ex={}".format(p, s, f_shape, n_ex))

print("in_rows={}, in_cols={}, n_in={}".format(in_rows, in_cols, n_in))

print("out_rows={}, out_cols={}, n_out={}".format(out_rows, out_cols, n_in))

# initialize Pool2D layer

L1 = Pool2D(kernel_shape=f_shape, pad=p, stride=s, mode=mode)

# forward prop

y_pred = L1.forward(X)

# backprop

dLdy = np.ones_like(y_pred)

dLdX = L1.backward(dLdy)

# get gold standard gradients

gold_mod = TorchPool2DLayer(n_in, L1.hyperparameters)

golds = gold_mod.extract_grads(X)

params = [(L1.X[0], "X"), (y_pred, "y"), (dLdX, "dLdX")]

for ix, (mine, label) in enumerate(params):

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=4

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import numpy as np

from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier

from sklearn.gaussian_process import GaussianProcessRegressor

from numpy_ml.nonparametric.knn import KNN

from numpy_ml.nonparametric.gp import GPRegression

from numpy_ml.utils.distance_metrics import euclidean

def test_gp_regression(N=15):

i = 0

while i < N:

alpha = np.random.rand()

N = np.random.randint(2, 100)

M = np.random.randint(2, 100)

K = np.random.randint(1, N)

J = np.random.randint(1, 3)

X = np.random.rand(N, M)

y = np.random.rand(N, J)

X_test = np.random.rand(K, M)

gp = GPRegression(kernel="RBFKernel(sigma=1)", alpha=alpha)

gold = GaussianProcessRegressor(

kernel=None, alpha=alpha, optimizer=None, normalize_y=False

gp.fit(X, y)

gold.fit(X, y)

preds, _ = gp.predict(X_test)

gold_preds = gold.predict(X_test)

np.testing.assert_almost_equal(preds.reshape(-1), gold_preds.reshape(-1))

mll = gp.marginal_log_likelihood().reshape(-1)

gold_mll = gold.log_marginal_likelihood().reshape(-1)

np.testing.assert_almost_equal(mll, gold_mll)

print("PASSED")

i += 1


# flake8: noqa

import time

import numpy as np

from numpy.testing import assert_almost_equal

from scipy.special import expit

import torch

import torch.nn.functional as F

from numpy_ml.utils.testing import random_stochastic_matrix, random_tensor

def torch_gradient_generator(fn, **kwargs):

def get_grad(z):

z1 = torch.autograd.Variable(torch.from_numpy(z), requires_grad=True)

z2 = fn(z1, **kwargs).sum()

grad = z1.grad.numpy()

return grad

return get_grad

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

#                            Test Suite                               #

#  def test_activations(N=50):

#      print("Testing Sigmoid activation")

#      time.sleep(1)

#      test_sigmoid_activation(N)

#      test_sigmoid_grad(N)

#      #  print("Testing Softmax activation")

#      #  time.sleep(1)

#      #  test_softmax_activation(N)

#      #  test_softmax_grad(N)

#      print("Testing Tanh activation")

#      time.sleep(1)

#      test_tanh_grad(N)

#      print("Testing ReLU activation")

#      time.sleep(1)

#      test_relu_activation(N)

#      test_relu_grad(N)

#      print("Testing ELU activation")

#      time.sleep(1)

#      test_elu_activation(N)

#      test_elu_grad(N)

#      print("Testing SELU activation")

#      time.sleep(1)

#      test_selu_activation(N)

#      test_selu_grad(N)

#      print("Testing LeakyRelu activation")

#      time.sleep(1)

#      test_leakyrelu_activation(N)

#      test_leakyrelu_grad(N)

#      print("Testing SoftPlus activation")

#      time.sleep(1)

#      test_softplus_activation(N)

#      test_softplus_grad(N)

#                          Activations                                #

def test_tanh_grad(N=50):

from numpy_ml.neural_nets.activations import Tanh

N = np.inf if N is None else N

mine = Tanh()

gold = torch_gradient_generator(torch.tanh)

i = 0

while i < N:

n_ex = np.random.randint(1, 100)

n_dims = np.random.randint(1, 100)

z = random_tensor((n_ex, n_dims))

assert_almost_equal(mine.grad(z), gold(z))

print("PASSED")

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

#                         Loss Functions                              #

def test_WGAN_GP_loss(N=5):

from numpy_ml.neural_nets.losses import WGAN_GPLoss

N = np.inf if N is None else N

i = 1

while i < N:

lambda_ = np.random.randint(0, 10)

n_ex = np.random.randint(1, 10)

n_feats = np.random.randint(2, 10)

Y_real = random_tensor([n_ex], standardize=True)

Y_fake = random_tensor([n_ex], standardize=True)

gradInterp = random_tensor([n_ex, n_feats], standardize=True)

mine = WGAN_GPLoss(lambda_=lambda_)

C_loss = mine(Y_fake, "C", Y_real, gradInterp)

G_loss = mine(Y_fake, "G")

C_dY_fake, dY_real, dGradInterp = mine.grad(Y_fake, "C", Y_real, gradInterp)

G_dY_fake = mine.grad(Y_fake, "G")

golds = TorchWGANGPLoss(lambda_).extract_grads(Y_real, Y_fake, gradInterp)

if np.isnan(golds["C_dGradInterp"]).any():

continue

params = [

(Y_real, "Y_real"),

(Y_fake, "Y_fake"),

(gradInterp, "gradInterp"),

(C_loss, "C_loss"),

(G_loss, "G_loss"),

(-dY_real, "C_dY_real"),

(-C_dY_fake, "C_dY_fake"),

(dGradInterp, "C_dGradInterp"),

(G_dY_fake, "G_dY_fake"),

print("\nTrial {}".format(i))

for ix, (mine, label) in enumerate(params):

mine,

golds[label],

err_msg=err_fmt(params, golds, ix),

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def grad_check_RNN(model, loss_func, param_name, n_t, X, epsilon=1e-7):

Manual gradient calc for vanilla RNN parameters

if param_name in ["Ba", "Bx"]:

param_name = param_name.lower()

elif param_name in ["X", "y"]:

return None

param_orig = model.parameters[param_name]

grads = np.zeros_like(param_orig)

for flat_ix, val in enumerate(param_orig.flat):

param = deepcopy(param_orig)

md_ix = np.unravel_index(flat_ix, param.shape)

# plus

y_preds_plus = []

param[md_ix] = val + epsilon

model.parameters[param_name] = param

for t in range(n_t):

y_pred_plus = model.forward(X[:, :, t])

y_preds_plus += [y_pred_plus]

loss_plus = loss_func(y_preds_plus)

# minus

y_preds_minus = []

param[md_ix] = val - epsilon

model.parameters[param_name] = param

for t in range(n_t):

y_pred_minus = model.forward(X[:, :, t])

y_preds_minus += [y_pred_minus]

loss_minus = loss_func(y_preds_minus)

grad = (loss_plus - loss_minus) / (2 * epsilon)

grads[md_ix] = grad

return grads.T

#                               Modules                               #

def test_MultiHeadedAttentionModule(N=15):

from numpy_ml.neural_nets.modules import MultiHeadedAttentionModule

N = np.inf if N is None else N

i = 1

while i < N + 1:

n_ex = np.random.randint(1, 10)

latent_dim = np.random.randint(1, 20)

n_heads = np.random.randint(2, 10)

d_k = d_v = n_heads * latent_dim

Q = random_tensor((n_ex, d_k), standardize=True)

K = random_tensor((n_ex, d_k), standardize=True)

V = random_tensor((n_ex, d_v), standardize=True)

mine = MultiHeadedAttentionModule(n_heads=n_heads, dropout_p=0)

# forward prop

y_pred = mine.forward(Q, K, V)

# backprop

dLdy = np.ones_like(y_pred)

dLdQ, dLdK, dLdV = mine.backward(dLdy)

# get gold standard gradients

params = mine.parameters

hparams = mine.hyperparameters

gold_mod = TorchMultiHeadedAttentionModule(params, hparams)

golds = gold_mod.extract_grads(Q, K, V)

dv = mine.derived_variables

params = mine.parameters["components"]

grads = mine.gradients["components"]

params = [

(Q, "Q"),

(K, "K"),

(V, "V"),

(mine.n_heads, "n_heads"),

(mine.latent_dim, "latent_dim"),

(params["O"]["W"], "O_W"),

(params["K"]["W"], "K_W"),

(params["V"]["W"], "V_W"),

(params["Q"]["W"], "Q_W"),

(params["O"]["b"], "O_b"),

(params["K"]["b"], "K_b"),

(params["V"]["b"], "V_b"),

(params["Q"]["b"], "Q_b"),

(dv["Q_proj"], "Q_proj"),

(dv["K_proj"], "K_proj"),

(dv["V_proj"], "V_proj"),

(dv["attention_weights"][0], "weights"),

(dv["attention_out"], "attn_out"),

(y_pred, "Y"),

(dLdy, "dLdy"),

(dv["dQ_proj"], "dQ_proj"),

(dv["dK_proj"], "dK_proj"),

(dv["dV_proj"], "dV_proj"),

(grads["O"]["W"], "dO_W"),

(grads["V"]["W"], "dV_W"),

(grads["K"]["W"], "dK_W"),

(grads["Q"]["W"], "dQ_W"),

(grads["O"]["b"], "dO_b"),

(grads["V"]["b"], "dV_b"),

(grads["K"]["b"], "dK_b"),

(grads["Q"]["b"], "dQ_b"),

(dLdQ, "dQ"),

(dLdK, "dK"),

(dLdV, "dV"),

print("\nTrial {}".format(i))

print(

"n_ex={} d_k=d_v={} latent_dim={} n_heads={}".format(

n_ex, d_k, latent_dim, n_heads

for ix, (mine, label) in enumerate(params):

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=4

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_Conv2D(N=15):

from numpy_ml.neural_nets.layers import Conv2D

from numpy_ml.neural_nets.activations import Tanh, ReLU, Sigmoid, Affine

N = np.inf if N is None else N

acts = [

(Tanh(), nn.Tanh(), "Tanh"),

(Sigmoid(), nn.Sigmoid(), "Sigmoid"),

(ReLU(), nn.ReLU(), "ReLU"),

(Affine(), TorchLinearActivation(), "Affine"),

i = 1

while i < N + 1:

n_ex = np.random.randint(1, 10)

in_rows = np.random.randint(1, 10)

in_cols = np.random.randint(1, 10)

n_in, n_out = np.random.randint(1, 3), np.random.randint(1, 3)

f_shape = (

min(in_rows, np.random.randint(1, 5)),

min(in_cols, np.random.randint(1, 5)),

p, s = np.random.randint(0, 5), np.random.randint(1, 3)

d = np.random.randint(0, 5)

fr, fc = f_shape[0] * (d + 1) - d, f_shape[1] * (d + 1) - d

out_rows = int(1 + (in_rows + 2 * p - fr) / s)

out_cols = int(1 + (in_cols + 2 * p - fc) / s)

if out_rows <= 0 or out_cols <= 0:

continue

X = random_tensor((n_ex, in_rows, in_cols, n_in), standardize=True)

# randomly select an activation function

act_fn, torch_fn, act_fn_name = acts[np.random.randint(0, len(acts))]

# initialize Conv2D layer

L1 = Conv2D(

pad=p,

stride=s,

dilation=d,

# forward prop

y_pred = L1.forward(X)

# backprop

dLdy = np.ones_like(y_pred)

dLdX = L1.backward(dLdy)

# get gold standard gradients

gold_mod = TorchConv2DLayer(

n_in, n_out, torch_fn, L1.parameters, L1.hyperparameters

golds = gold_mod.extract_grads(X)

params = [

(L1.X[0], "X"),

(y_pred, "y"),

(L1.parameters["W"], "W"),

(L1.parameters["b"], "b"),

(L1.gradients["W"], "dLdW"),

(L1.gradients["b"], "dLdB"),

(dLdX, "dLdX"),

print("\nTrial {}".format(i))

print("pad={}, stride={}, f_shape={}, n_ex={}".format(p, s, f_shape, n_ex))

print("in_rows={}, in_cols={}, n_in={}".format(in_rows, in_cols, n_in))

print("out_rows={}, out_cols={}, n_out={}".format(out_rows, out_cols, n_out))

print("dilation={}".format(d))

for ix, (mine, label) in enumerate(params):

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=4

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_BatchNorm2D(N=15):

from numpy_ml.neural_nets.layers import BatchNorm2D

N = np.inf if N is None else N

i = 1

while i < N + 1:

n_ex = np.random.randint(2, 10)

in_rows = np.random.randint(1, 10)

in_cols = np.random.randint(1, 10)

n_in = np.random.randint(1, 3)

# initialize BatchNorm2D layer

X = random_tensor((n_ex, in_rows, in_cols, n_in), standardize=True)

L1 = BatchNorm2D()

# forward prop

y_pred = L1.forward(X)

# standard sum loss

dLdy = np.ones_like(X)

dLdX = L1.backward(dLdy)

# get gold standard gradients

gold_mod = TorchBatchNormLayer(

n_in, L1.parameters, mode="2D", epsilon=L1.epsilon, momentum=L1.momentum

golds = gold_mod.extract_grads(X, Y_true=None)

params = [

(L1.X[0], "X"),

(L1.hyperparameters["momentum"], "momentum"),

(L1.hyperparameters["epsilon"], "epsilon"),

(L1.parameters["scaler"].T, "scaler"),

(L1.parameters["intercept"], "intercept"),

(L1.parameters["running_mean"], "running_mean"),

#  (L1.parameters["running_var"], "running_var"),

(y_pred, "y"),

(L1.gradients["scaler"], "dLdScaler"),

(L1.gradients["intercept"], "dLdIntercept"),

(dLdX, "dLdX"),

print("Trial {}".format(i))

for ix, (mine, label) in enumerate(params):

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=3

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import tempfile

import nltk

import numpy as np

from ..preprocessing.nlp import tokenize_words

from ..ngram import AdditiveNGram, MLENGram

from ..utils.testing import random_paragraph

class MLEGold:

def __init__(

self, N, K=1, unk=True, filter_stopwords=True, filter_punctuation=True

self.N = N

self.K = K

self.unk = unk

self.filter_stopwords = filter_stopwords

self.filter_punctuation = filter_punctuation

self.hyperparameters = {

"N": N,

"K": K,

"unk": unk,

"filter_stopwords": filter_stopwords,

"filter_punctuation": filter_punctuation,

def train(self, corpus_fp, vocab=None, encoding=None):

N = self.N

H = self.hyperparameters

models, counts = {}, {}

grams = {n: [] for n in range(1, N + 1)}

gg = {n: [] for n in range(1, N + 1)}

filter_punc, filter_stop = H["filter_punctuation"], H["filter_stopwords"]

n_words = 0

tokens = set([])

with open(corpus_fp, "r", encoding=encoding) as text:

for line in text:

words = tokenize_words(line, filter_punc, filter_stop)

if vocab is not None:

words = vocab.filter(words, H["unk"])

if len(words) == 0:

continue

n_words += len(words)

tokens.update(words)

# calculate n, n-1, ... 1-grams

for n in range(1, N + 1):

grams[n].append(

nltk.ngrams(

words,

n,

gg[n].extend(

list(

nltk.ngrams(

words,

n,

for n in range(1, N + 1):

counts[n] = nltk.FreqDist(gg[n])

models[n] = nltk.lm.MLE(order=n)

models[n].fit(grams[n], tokens)

self.counts = counts

self.n_words = n_words

self._models = models

self.n_tokens = len(vocab) if vocab is not None else len(tokens)

def log_prob(self, words, N):

assert N in self.counts, "You do not have counts for {}-grams".format(N)

if N > len(words):

err = "Not enough words for a gram-size of {}: {}".format(N, len(words))

raise ValueError(err)

total_prob = 0

for ngram in nltk.ngrams(words, N):

total_prob += self._log_ngram_prob(ngram)

return total_prob

def _log_ngram_prob(self, ngram):

N = len(ngram)

return self._models[N].logscore(ngram[-1], ngram[:-1])

class AdditiveGold:

def __init__(

self.N = N

self.K = K

self.unk = unk

self.hyperparameters = {

"N": N,

"K": K,

"unk": unk,

"filter_stopwords": filter_stopwords,

N = self.N

H = self.hyperparameters

models, counts = {}, {}

gg = {n: [] for n in range(1, N + 1)}

n_words = 0

tokens = set()

for line in text:

if vocab is not None:

words = vocab.filter(words, H["unk"])

if len(words) == 0:

continue

n_words += len(words)

tokens.update(words)

# calculate n, n-1, ... 1-grams

for n in range(1, N + 1):

grams[n].append(

nltk.ngrams(

words,

n,

gg[n].extend(

list(

nltk.ngrams(

words,

n,

for n in range(1, N + 1):

counts[n] = nltk.FreqDist(gg[n])

models[n] = nltk.lm.Lidstone(order=n, gamma=self.K)

models[n].fit(grams[n], tokens)

self.counts = counts

self._models = models

self.n_words = n_words

def log_prob(self, words, N):

if N > len(words):

raise ValueError(err)

total_prob = 0

for ngram in nltk.ngrams(words, N):

return total_prob

def _log_ngram_prob(self, ngram):

N = len(ngram)

def test_mle():

N = np.random.randint(2, 5)

gold = MLEGold(N, unk=True, filter_stopwords=False, filter_punctuation=False)

mine = MLENGram(N, unk=True, filter_stopwords=False, filter_punctuation=False)

with tempfile.NamedTemporaryFile() as temp:

temp.write(bytes(" ".join(random_paragraph(1000)), encoding="utf-8-sig"))

gold.train(temp.name, encoding="utf-8-sig")

mine.train(temp.name, encoding="utf-8-sig")

for k in mine.counts[N].keys():

if k[0] == k[1] and k[0] in ("<bol>", "<eol>"):

continue

err_str = "{}, mine: {}, gold: {}"

assert mine.counts[N][k] == gold.counts[N][k], err_str.format(

k, mine.counts[N][k], gold.counts[N][k]

M = mine.log_prob(k, N)

G = gold.log_prob(k, N) / np.log2(np.e)  # convert to log base e

np.testing.assert_allclose(M, G)

print("PASSED")


# flake8: noqa

import numpy as np

from sklearn import datasets

from sklearn.model_selection import train_test_split

from sklearn import naive_bayes

from numpy_ml.linear_models import GaussianNBClassifier

from numpy_ml.utils.testing import random_tensor

def test_GaussianNB(N=10):

N = np.inf if N is None else N

i = 1

eps = np.finfo(float).eps

while i < N + 1:

n_ex = np.random.randint(1, 300)

n_feats = np.random.randint(1, 100)

n_classes = np.random.randint(2, 10)

X = random_tensor((n_ex, n_feats), standardize=True)

y = np.random.randint(0, n_classes, size=n_ex)

X_test = random_tensor((n_ex, n_feats), standardize=True)

NB = GaussianNBClassifier(eps=1e-09)

NB.fit(X, y)

preds = NB.predict(X_test)

sklearn_NB = naive_bayes.GaussianNB()

sklearn_NB.fit(X, y)

sk_preds = sklearn_NB.predict(X_test)

for j in range(len(NB.labels)):

P = NB.parameters

jointi = np.log(sklearn_NB.class_prior_[j])

jointi_mine = np.log(P["prior"][j])

np.testing.assert_almost_equal(jointi, jointi_mine)

n_jk = -0.5 * np.sum(np.log(2.0 * np.pi * sklearn_NB.sigma_[j, :] + eps))

n_jk_mine = -0.5 * np.sum(np.log(2.0 * np.pi * P["sigma"][j] + eps))

np.testing.assert_almost_equal(n_jk_mine, n_jk)

n_jk2 = n_jk - 0.5 * np.sum(

((X_test - sklearn_NB.theta_[j, :]) ** 2) / (sklearn_NB.sigma_[j, :]), 1

n_jk2_mine = n_jk_mine - 0.5 * np.sum(

((X_test - P["mean"][j]) ** 2) / (P["sigma"][j]), 1

np.testing.assert_almost_equal(n_jk2_mine, n_jk2, decimal=4)

llh = jointi + n_jk2

llh_mine = jointi_mine + n_jk2_mine

np.testing.assert_almost_equal(llh_mine, llh, decimal=4)

np.testing.assert_almost_equal(P["prior"], sklearn_NB.class_prior_)

np.testing.assert_almost_equal(P["mean"], sklearn_NB.theta_)

np.testing.assert_almost_equal(P["sigma"], sklearn_NB.sigma_)

sklearn_NB._joint_log_likelihood(X_test),

np.testing.assert_almost_equal(preds, sk_preds)

print("PASSED")

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_BatchNorm1D(N=15):

from numpy_ml.neural_nets.layers import BatchNorm1D

N = np.inf if N is None else N

i = 1

while i < N + 1:

n_ex = np.random.randint(2, 1000)

n_in = np.random.randint(1, 1000)

X = random_tensor((n_ex, n_in), standardize=True)

# initialize BatchNorm1D layer

L1 = BatchNorm1D()

# forward prop

y_pred = L1.forward(X)

# backprop

dLdy = np.ones_like(y_pred)

dLdX = L1.backward(dLdy)

# get gold standard gradients

gold_mod = TorchBatchNormLayer(

n_in, L1.parameters, "1D", epsilon=L1.epsilon, momentum=L1.momentum

golds = gold_mod.extract_grads(X)

params = [

(L1.X[0], "X"),

(y_pred, "y"),

(L1.parameters["scaler"].T, "scaler"),

(L1.parameters["intercept"], "intercept"),

(L1.parameters["running_mean"], "running_mean"),

#  (L1.parameters["running_var"], "running_var"),

(L1.gradients["scaler"], "dLdScaler"),

(L1.gradients["intercept"], "dLdIntercept"),

(dLdX, "dLdX"),

print("Trial {}".format(i))

for ix, (mine, label) in enumerate(params):

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=1

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import numpy as np

import scipy

import networkx as nx

from sklearn.neighbors import BallTree as sk_BallTree

from sklearn.metrics.pairwise import rbf_kernel as sk_rbf

from sklearn.metrics.pairwise import linear_kernel as sk_linear

from sklearn.metrics.pairwise import polynomial_kernel as sk_poly

from numpy_ml.utils.distance_metrics import (

hamming,

euclidean,

chebyshev,

manhattan,

minkowski,

from numpy_ml.utils.kernels import LinearKernel, PolynomialKernel, RBFKernel

from numpy_ml.utils.data_structures import BallTree

from numpy_ml.utils.graphs import (

Edge,

DiGraph,

UndirectedGraph,

#                               Kernels                               #

def test_radial_basis_kernel(N=1):

i = 0

while i < N:

N = np.random.randint(1, 100)

M = np.random.randint(1, 100)

C = np.random.randint(1, 1000)

gamma = np.random.rand()

X = np.random.rand(N, C)

Y = np.random.rand(M, C)

# sklearn (gamma) <-> mine (sigma) conversion:

# gamma = 1 / (2 * sigma^2)

# sigma = np.sqrt(1 / 2 * gamma)

mine = RBFKernel(sigma=np.sqrt(1 / (2 * gamma)))(X, Y)

gold = sk_rbf(X, Y, gamma=gamma)

np.testing.assert_almost_equal(mine, gold)

print("PASSED")

i += 1


# flake8: noqa

import numpy as np

from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor

from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor

from sklearn.metrics import accuracy_score, mean_squared_error

from sklearn.datasets import make_regression, make_blobs

from sklearn.model_selection import train_test_split

from numpy_ml.trees.gbdt import GradientBoostedDecisionTree

from numpy_ml.trees.dt import DecisionTree, Node, Leaf

from numpy_ml.trees.rf import RandomForest

from numpy_ml.utils.testing import random_tensor

def clone_tree(dtree):

children_left = dtree.tree_.children_left

children_right = dtree.tree_.children_right

feature = dtree.tree_.feature

threshold = dtree.tree_.threshold

values = dtree.tree_.value

def grow(node_id):

l, r = children_left[node_id], children_right[node_id]

if l == r:

return Leaf(values[node_id].argmax())

n = Node(None, None, (feature[node_id], threshold[node_id]))

n.left = grow(l)

n.right = grow(r)

return n

node_id = 0

root = Node(None, None, (feature[node_id], threshold[node_id]))

root.left = grow(children_left[node_id])

root.right = grow(children_right[node_id])

return root

def compare_trees(mine, gold):

clone = clone_tree(gold)

mine = mine.root

def test(mine, clone):

if isinstance(clone, Node) and isinstance(mine, Node):

assert mine.feature == clone.feature, "Node {} not equal".format(depth)

np.testing.assert_allclose(mine.threshold, clone.threshold)

test(mine.left, clone.left, depth + 1)

test(mine.right, clone.right, depth + 1)

elif isinstance(clone, Leaf) and isinstance(mine, Leaf):

np.testing.assert_allclose(mine.value, clone.value)

return

else:

raise ValueError("Nodes at depth {} are not equal".format(depth))

depth = 0

ok = True

while ok:

assert mine.feature == clone.feature

test(mine.left, clone.left, depth + 1)

return

else:

def test_gbdt(N=1):

i = 1

while i <= N:

n_ex = np.random.randint(2, 100)

n_feats = np.random.randint(2, 100)

n_trees = np.random.randint(2, 100)

max_depth = np.random.randint(1, 5)

classifier = np.random.choice([True, False])

if classifier:

# create classification problem

n_classes = np.random.randint(2, 10)

X, Y = make_blobs(

n_samples=n_ex, centers=n_classes, n_features=n_feats, random_state=i

X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.3, random_state=i)

# initialize model

def loss(yp, y):

return 1 - accuracy_score(yp, y)

# initialize model

criterion = np.random.choice(["entropy", "gini"])

mine = GradientBoostedDecisionTree(

classifier=classifier,

loss="crossentropy",

gold = RandomForestClassifier(

criterion=criterion,

bootstrap=True,

else:

# create regeression problem

X, Y = make_regression(n_samples=n_ex, n_features=n_feats, random_state=i)

# initialize model

criterion = "mse"

loss = mean_squared_error

mine = GradientBoostedDecisionTree(

classifier=classifier,

loss="mse",

gold = RandomForestRegressor(

criterion=criterion,

bootstrap=True,

print("Trial {}".format(i))

print("\tClassifier={}, criterion={}".format(classifier, criterion))

print("\tmax_depth={}, n_feats={}, n_ex={}".format(max_depth, n_feats, n_ex))

if classifier:

print("\tn_classes: {}".format(n_classes))

# fit 'em

mine.fit(X, Y)

gold.fit(X, Y)

# get preds

y_pred_mine = mine.predict(X)

y_pred_gold = gold.predict(X)

loss_mine = loss(y_pred_mine, Y)

loss_gold = loss(y_pred_gold, Y)

# get preds on test set

y_pred_mine_test = mine.predict(X_test)

y_pred_gold_test = gold.predict(X_test)

loss_mine_test = loss(y_pred_mine_test, Y_test)

loss_gold_test = loss(y_pred_gold_test, Y_test)

try:

np.testing.assert_almost_equal(loss_mine, loss_gold)

print("\tLoss on training: {}".format(loss_mine))

except AssertionError as e:

print("\tTraining losses not equal:\n{}".format(e))

try:

np.testing.assert_almost_equal(loss_mine_test, loss_gold_test)

print("\tLoss on test: {}".format(loss_mine_test))

except AssertionError as e:

print("\tTest losses not equal:\n{}".format(e))

print("PASSED")

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_LayerNorm2D(N=15):

from numpy_ml.neural_nets.layers import LayerNorm2D

N = np.inf if N is None else N

i = 1

while i < N + 1:

n_ex = np.random.randint(2, 10)

in_rows = np.random.randint(1, 10)

in_cols = np.random.randint(1, 10)

n_in = np.random.randint(1, 3)

# initialize LayerNorm2D layer

X = random_tensor((n_ex, in_rows, in_cols, n_in), standardize=True)

L1 = LayerNorm2D()

# forward prop

y_pred = L1.forward(X)

# standard sum loss

dLdy = np.ones_like(X)

dLdX = L1.backward(dLdy)

# get gold standard gradients

gold_mod = TorchLayerNormLayer(

[n_in, in_rows, in_cols], L1.parameters, mode="2D", epsilon=L1.epsilon

golds = gold_mod.extract_grads(X, Y_true=None)

params = [

(L1.X[0], "X"),

(L1.hyperparameters["epsilon"], "epsilon"),

(L1.parameters["scaler"], "scaler"),

(L1.parameters["intercept"], "intercept"),

(y_pred, "y"),

(L1.gradients["scaler"], "dLdScaler"),

(L1.gradients["intercept"], "dLdIntercept"),

(dLdX, "dLdX"),

print("Trial {}".format(i))

for ix, (mine, label) in enumerate(params):

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=3

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def grad_check_RNN(model, loss_func, param_name, n_t, X, epsilon=1e-7):

Manual gradient calc for vanilla RNN parameters

if param_name in ["Ba", "Bx"]:

param_name = param_name.lower()

elif param_name in ["X", "y"]:

return None

param_orig = model.parameters[param_name]

grads = np.zeros_like(param_orig)

for flat_ix, val in enumerate(param_orig.flat):

param = deepcopy(param_orig)

md_ix = np.unravel_index(flat_ix, param.shape)

# plus

y_preds_plus = []

param[md_ix] = val + epsilon

model.parameters[param_name] = param

for t in range(n_t):

y_pred_plus = model.forward(X[:, :, t])

y_preds_plus += [y_pred_plus]

loss_plus = loss_func(y_preds_plus)

# minus

y_preds_minus = []

param[md_ix] = val - epsilon

model.parameters[param_name] = param

for t in range(n_t):

y_pred_minus = model.forward(X[:, :, t])

y_preds_minus += [y_pred_minus]

loss_minus = loss_func(y_preds_minus)

grad = (loss_plus - loss_minus) / (2 * epsilon)

grads[md_ix] = grad

return grads.T

#                               Modules                               #

def test_WaveNetModule(N=10):

from numpy_ml.neural_nets.modules import WavenetResidualModule

N = np.inf if N is None else N

i = 1

while i < N + 1:

n_ex = np.random.randint(1, 10)

l_in = np.random.randint(1, 10)

ch_residual, ch_dilation = np.random.randint(1, 5), np.random.randint(1, 5)

f_width = min(l_in, np.random.randint(1, 5))

d = np.random.randint(0, 5)

X_main = np.zeros_like(

random_tensor((n_ex, l_in, ch_residual), standardize=True)

X_main[0][0][0] = 1.0

X_skip = np.zeros_like(

# initialize Conv2D layer

L1 = WavenetResidualModule(

dilation=d,

# forward prop

Y_main, Y_skip = L1.forward(X_main, X_skip)

# backprop

dLdY_skip = np.ones_like(Y_skip)

dLdY_main = np.ones_like(Y_main)

dLdX_main, dLdX_skip = L1.backward(dLdY_skip, dLdY_main)

_, conv_1x1_pad = pad1D(

L1._dv["multiply_gate_out"], "same", kernel_width=1, stride=1, dilation=0

if conv_1x1_pad[0] != conv_1x1_pad[1]:

print("Skipping")

continue

conv_1x1_pad = conv_1x1_pad[0]

# get gold standard gradients

gold_mod = TorchWavenetModule(L1.parameters, L1.hyperparameters, conv_1x1_pad)

golds = gold_mod.extract_grads(X_main, X_skip)

dv = L1.derived_variables

pc = L1.parameters["components"]

gr = L1.gradients["components"]

params = [

(L1.X_main, "X_main"),

(L1.X_skip, "X_skip"),

(pc["conv_dilation"]["W"], "conv_dilation_W"),

(pc["conv_dilation"]["b"], "conv_dilation_b"),

(pc["conv_1x1"]["W"], "conv_1x1_W"),

(pc["conv_1x1"]["b"], "conv_1x1_b"),

(dv["conv_dilation_out"], "conv_dilation_out"),

(dv["tanh_out"], "tanh_out"),

(dv["sigm_out"], "sigm_out"),

(dv["multiply_gate_out"], "multiply_gate_out"),

(dv["conv_1x1_out"], "conv_1x1_out"),

(Y_main, "Y_main"),

(Y_skip, "Y_skip"),

(dLdY_skip, "dLdY_skip"),

(dLdY_main, "dLdY_main"),

(dv["dLdConv_1x1"], "dLdConv_1x1_out"),

(gr["conv_1x1"]["W"], "dLdConv_1x1_W"),

(gr["conv_1x1"]["b"], "dLdConv_1x1_b"),

(dv["dLdMultiply"], "dLdMultiply_out"),

(dv["dLdTanh"], "dLdTanh_out"),

(dv["dLdSigmoid"], "dLdSigm_out"),

(dv["dLdConv_dilation"], "dLdConv_dilation_out"),

(gr["conv_dilation"]["W"], "dLdConv_dilation_W"),

(gr["conv_dilation"]["b"], "dLdConv_dilation_b"),

(dLdX_main, "dLdX_main"),

(dLdX_skip, "dLdX_skip"),

print("\nTrial {}".format(i))

print("f_width={}, n_ex={}".format(f_width, n_ex))

print("l_in={}, ch_residual={}".format(l_in, ch_residual))

print("ch_dilation={} dilation={}".format(ch_dilation, d))

for ix, (mine, label) in enumerate(params):

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=4

print("\tPASSED {}".format(label))

i += 1

#                                Utils                                #


# flake8: noqa

from collections import Counter

# gold-standard imports

import huffman

import numpy as np

from scipy.fftpack import dct

from sklearn.preprocessing import StandardScaler

from sklearn.feature_extraction.text import TfidfVectorizer

# try:

#     from librosa.core.time_frequency import fft_frequencies

# except ImportError:

#     # for librosa >= 0.8.0

#     from librosa import fft_frequencies

# from librosa.feature import mfcc as lr_mfcc

# from librosa.util import frame

# from librosa.filters import mel

# numpy-ml implementations

from numpy_ml.preprocessing.general import Standardizer

from numpy_ml.preprocessing.nlp import HuffmanEncoder, TFIDFEncoder

from numpy_ml.preprocessing.dsp import (

DCT,

DFT,

mfcc,

from numpy_ml.utils.testing import random_paragraph

def test_standardizer(N=15):

i = 0

while i < N:

mean = bool(np.random.randint(2))

std = bool(np.random.randint(2))

N = np.random.randint(2, 100)

M = np.random.randint(2, 100)

X = np.random.rand(N, M)

S = Standardizer(with_mean=mean, with_std=std)

S.fit(X)

mine = S.transform(X)

theirs = StandardScaler(with_mean=mean, with_std=std)

gold = theirs.fit_transform(X)

np.testing.assert_almost_equal(mine, gold)

print("PASSED")

i += 1


# flake8: noqa

from collections import Counter

# gold-standard imports

import huffman

import numpy as np

from scipy.fftpack import dct

# try:

# except ImportError:

#     # for librosa >= 0.8.0

# from librosa.util import frame

# from librosa.filters import mel

# numpy-ml implementations

DCT,

DFT,

mfcc,

def test_dft(N=15):

i = 0

while i < N:

N = np.random.randint(2, 100)

signal = np.random.rand(N)

mine = DFT(signal)

theirs = np.fft.rfft(signal)

np.testing.assert_almost_equal(mine.real, theirs.real)

print("PASSED")

i += 1


# flake8: noqa

import numpy as np

import scipy

import networkx as nx

from sklearn.neighbors import BallTree as sk_BallTree

from sklearn.metrics.pairwise import rbf_kernel as sk_rbf

from sklearn.metrics.pairwise import linear_kernel as sk_linear

from sklearn.metrics.pairwise import polynomial_kernel as sk_poly

from numpy_ml.utils.distance_metrics import (

hamming,

euclidean,

chebyshev,

manhattan,

minkowski,

from numpy_ml.utils.kernels import LinearKernel, PolynomialKernel, RBFKernel

from numpy_ml.utils.data_structures import BallTree

from numpy_ml.utils.graphs import (

Edge,

DiGraph,

UndirectedGraph,

def test_euclidean(N=1):

i = 0

while i < N:

N = np.random.randint(1, 100)

x = np.random.rand(N)

y = np.random.rand(N)

mine = euclidean(x, y)

theirs = scipy.spatial.distance.euclidean(x, y)

np.testing.assert_almost_equal(mine, theirs)

print("PASSED")

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_LayerNorm1D(N=15):

from numpy_ml.neural_nets.layers import LayerNorm1D

N = np.inf if N is None else N

i = 1

while i < N + 1:

n_ex = np.random.randint(2, 1000)

n_in = np.random.randint(1, 1000)

X = random_tensor((n_ex, n_in), standardize=True)

# initialize BatchNorm1D layer

L1 = LayerNorm1D()

# forward prop

y_pred = L1.forward(X)

# backprop

dLdy = np.ones_like(y_pred)

dLdX = L1.backward(dLdy)

# get gold standard gradients

gold_mod = TorchLayerNormLayer(n_in, L1.parameters, "1D", epsilon=L1.epsilon)

golds = gold_mod.extract_grads(X)

params = [

(L1.X[0], "X"),

(y_pred, "y"),

(L1.parameters["scaler"].T, "scaler"),

(L1.parameters["intercept"], "intercept"),

(L1.gradients["scaler"], "dLdScaler"),

(L1.gradients["intercept"], "dLdIntercept"),

(dLdX, "dLdX"),

print("Trial {}".format(i))

for ix, (mine, label) in enumerate(params):

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=3

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_FullyConnected(N=15):

from numpy_ml.neural_nets.layers import FullyConnected

from numpy_ml.neural_nets.activations import Tanh, ReLU, Sigmoid, Affine

N = np.inf if N is None else N

acts = [

(Tanh(), nn.Tanh(), "Tanh"),

(Sigmoid(), nn.Sigmoid(), "Sigmoid"),

(ReLU(), nn.ReLU(), "ReLU"),

(Affine(), TorchLinearActivation(), "Affine"),

i = 1

while i < N + 1:

n_ex = np.random.randint(1, 100)

n_in = np.random.randint(1, 100)

n_out = np.random.randint(1, 100)

X = random_tensor((n_ex, n_in), standardize=True)

# randomly select an activation function

act_fn, torch_fn, act_fn_name = acts[np.random.randint(0, len(acts))]

# initialize FC layer

L1 = FullyConnected(n_out=n_out, act_fn=act_fn)

# forward prop

y_pred = L1.forward(X)

# backprop

dLdy = np.ones_like(y_pred)

dLdX = L1.backward(dLdy)

# get gold standard gradients

gold_mod = TorchFCLayer(n_in, n_out, torch_fn, L1.parameters)

golds = gold_mod.extract_grads(X)

params = [

(L1.X[0], "X"),

(y_pred, "y"),

(L1.parameters["W"].T, "W"),

(L1.parameters["b"], "b"),

(dLdy, "dLdy"),

(L1.gradients["W"].T, "dLdW"),

(L1.gradients["b"], "dLdB"),

(dLdX, "dLdX"),

print("\nTrial {}\nact_fn={}".format(i, act_fn_name))

for ix, (mine, label) in enumerate(params):

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=3

print("\tPASSED {}".format(label))

i += 1