# flake8: noqa

import numpy as np

from sklearn.linear_model import LinearRegression as LinearRegressionGold

from numpy_ml.linear_models import LinearRegression

from numpy_ml.utils.testing import random_tensor

def test_linear_regression(N=10):

N = np.inf if N is None else N

i = 1

while i < N + 1:

train_samples = np.random.randint(2, 30)

update_samples = np.random.randint(1, 30)

n_samples = train_samples + update_samples

# ensure n_feats < train_samples, otherwise multiple solutions are

# possible

n_feats = np.random.randint(1, train_samples)

target_dim = np.random.randint(1, 10)

fit_intercept = np.random.choice([True, False])

X = random_tensor((n_samples, n_feats), standardize=True)

y = random_tensor((n_samples, target_dim), standardize=True)

weighted = np.random.choice([True, False])

weights = np.random.rand(n_samples) if weighted else np.ones(n_samples)

X_train, X_update = X[:train_samples], X[train_samples:]

y_train, y_update = y[:train_samples], y[train_samples:]

w_train, w_update = weights[:train_samples], weights[train_samples:]

print(f"Weights: {weighted}")

print(f"Fit intercept: {fit_intercept}")

# Fit gold standard model on the entire dataset

lr_gold = LinearRegressionGold(fit_intercept=fit_intercept)

lr_gold.fit(X, y, sample_weight=weights)

lr_mine = LinearRegression(fit_intercept=fit_intercept)

lr_mine.fit(X, y, weights=weights)

# check that model predictions match

lr_mine.predict(X), lr_gold.predict(X), decimal=5

print("\t1. Overall model predictions match")

# check that model coefficients match

beta = lr_mine.beta.T[:, 1:] if fit_intercept else lr_mine.beta.T

np.testing.assert_almost_equal(beta, lr_gold.coef_, decimal=6)

print("\t2. Overall model coefficients match")

# Fit our model on just (X_train, y_train)...

lr = LinearRegression(fit_intercept=fit_intercept)

lr.fit(X_train, y_train, weights=w_train)

do_single_sample_update = np.random.choice([True, False])

# ...then update our model on the examples (X_update, y_update)

if do_single_sample_update:

for x_new, y_new, w_new in zip(X_update, y_update, w_update):

lr.update(x_new, y_new, w_new)

else:

lr.update(X_update, y_update, w_update)

# check that model predictions match

np.testing.assert_almost_equal(lr.predict(X), lr_gold.predict(X), decimal=5)

print("\t3. Iterative model predictions match")

# check that model coefficients match

beta = lr.beta.T[:, 1:] if fit_intercept else lr.beta.T

print("\t4. Iterative model coefficients match")

print("\tPASSED\n")

i += 1


"""Unit tests for various numpy-ml modules"""


# flake8: noqa

import numpy as np

import scipy

import networkx as nx

from sklearn.neighbors import BallTree as sk_BallTree

from sklearn.metrics.pairwise import rbf_kernel as sk_rbf

from sklearn.metrics.pairwise import linear_kernel as sk_linear

from sklearn.metrics.pairwise import polynomial_kernel as sk_poly

from numpy_ml.utils.distance_metrics import (

hamming,

euclidean,

chebyshev,

manhattan,

minkowski,

from numpy_ml.utils.kernels import LinearKernel, PolynomialKernel, RBFKernel

from numpy_ml.utils.data_structures import BallTree

from numpy_ml.utils.graphs import (

Edge,

DiGraph,

UndirectedGraph,

#                               Kernels                               #

def test_linear_kernel(N=1):

i = 0

while i < N:

N = np.random.randint(1, 100)

M = np.random.randint(1, 100)

C = np.random.randint(1, 1000)

X = np.random.rand(N, C)

Y = np.random.rand(M, C)

mine = LinearKernel()(X, Y)

gold = sk_linear(X, Y)

np.testing.assert_almost_equal(mine, gold)

print("PASSED")

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_AddLayer(N=15):

from numpy_ml.neural_nets.layers import Add

from numpy_ml.neural_nets.activations import Tanh, ReLU, Sigmoid, Affine

N = np.inf if N is None else N

acts = [

(Tanh(), nn.Tanh(), "Tanh"),

(Sigmoid(), nn.Sigmoid(), "Sigmoid"),

(ReLU(), nn.ReLU(), "ReLU"),

(Affine(), TorchLinearActivation(), "Affine"),

i = 1

while i < N + 1:

Xs = []

n_ex = np.random.randint(1, 100)

n_in = np.random.randint(1, 100)

n_entries = np.random.randint(2, 5)

for _ in range(n_entries):

Xs.append(random_tensor((n_ex, n_in), standardize=True))

act_fn, torch_fn, act_fn_name = acts[np.random.randint(0, len(acts))]

# initialize Add layer

L1 = Add(act_fn)

# forward prop

y_pred = L1.forward(Xs)

# backprop

dLdy = np.ones_like(y_pred)

dLdXs = L1.backward(dLdy)

# get gold standard gradients

gold_mod = TorchAddLayer(torch_fn)

golds = gold_mod.extract_grads(Xs)

params = [(Xs, "Xs"), (y_pred, "Y")]

params.extend(

[(dldxi, "dLdX{}".format(i + 1)) for i, dldxi in enumerate(dLdXs)]

print("\nTrial {}".format(i))

print("n_ex={}, n_in={}".format(n_ex, n_in))

print("n_entries={}, act_fn={}".format(n_entries, str(act_fn)))

for ix, (mine, label) in enumerate(params):

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=1

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import time

import numpy as np

from numpy.testing import assert_almost_equal

from scipy.special import expit

import torch

import torch.nn.functional as F

from numpy_ml.utils.testing import random_stochastic_matrix, random_tensor

def torch_gradient_generator(fn, **kwargs):

def get_grad(z):

z1 = torch.autograd.Variable(torch.from_numpy(z), requires_grad=True)

z2 = fn(z1, **kwargs).sum()

grad = z1.grad.numpy()

return grad

return get_grad

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

#                            Test Suite                               #

#  def test_activations(N=50):

#      print("Testing Sigmoid activation")

#      time.sleep(1)

#      test_sigmoid_activation(N)

#      test_sigmoid_grad(N)

#      #  print("Testing Softmax activation")

#      #  time.sleep(1)

#      #  test_softmax_activation(N)

#      #  test_softmax_grad(N)

#      print("Testing Tanh activation")

#      time.sleep(1)

#      test_tanh_grad(N)

#      print("Testing ReLU activation")

#      time.sleep(1)

#      test_relu_activation(N)

#      test_relu_grad(N)

#      print("Testing ELU activation")

#      time.sleep(1)

#      test_elu_activation(N)

#      test_elu_grad(N)

#      print("Testing SELU activation")

#      time.sleep(1)

#      test_selu_activation(N)

#      test_selu_grad(N)

#      print("Testing LeakyRelu activation")

#      time.sleep(1)

#      test_leakyrelu_activation(N)

#      test_leakyrelu_grad(N)

#      print("Testing SoftPlus activation")

#      time.sleep(1)

#      test_softplus_activation(N)

#      test_softplus_grad(N)

#                          Activations                                #

def test_gelu_activation(N=50):

from numpy_ml.neural_nets.activations import GELU

N = np.inf if N is None else N

i = 0

while i < N:

n_dims = np.random.randint(1, 100)

z = random_stochastic_matrix(1, n_dims)

approx = np.random.choice([True, False])

mine = GELU(approximate=False)

mine_approx = GELU(approximate=True)

gold = lambda z: F.gelu(torch.FloatTensor(z)).numpy()

np.testing.assert_allclose(mine.fn(z), gold(z), rtol=1e-3)

assert_almost_equal(mine.fn(z), mine_approx.fn(z))

print("PASSED")

i += 1

def test_gelu_grad(N=50):

N = np.inf if N is None else N

mine = GELU(approximate=False)

mine_approx = GELU(approximate=True)

gold = torch_gradient_generator(F.gelu)

i = 0

while i < N:

n_ex = np.random.randint(1, 100)

n_dims = np.random.randint(1, 100)

z = random_tensor((n_ex, n_dims))

assert_almost_equal(mine.grad(z), gold(z), decimal=3)

assert_almost_equal(mine.grad(z), mine_approx.grad(z))

print("PASSED")

i += 1


# flake8: noqa

from collections import Counter

# gold-standard imports

import huffman

import numpy as np

from scipy.fftpack import dct

from sklearn.preprocessing import StandardScaler

from sklearn.feature_extraction.text import TfidfVectorizer

# try:

#     from librosa.core.time_frequency import fft_frequencies

# except ImportError:

#     # for librosa >= 0.8.0

#     from librosa import fft_frequencies

# from librosa.feature import mfcc as lr_mfcc

# from librosa.util import frame

# from librosa.filters import mel

# numpy-ml implementations

from numpy_ml.preprocessing.general import Standardizer

from numpy_ml.preprocessing.nlp import HuffmanEncoder, TFIDFEncoder

from numpy_ml.preprocessing.dsp import (

DCT,

DFT,

mfcc,

from numpy_ml.utils.testing import random_paragraph

def test_dct(N=15):

i = 0

while i < N:

N = np.random.randint(2, 100)

signal = np.random.rand(N)

ortho = bool(np.random.randint(2))

mine = DCT(signal, orthonormal=ortho)

theirs = dct(signal, norm="ortho" if ortho else None)

np.testing.assert_almost_equal(mine, theirs)

print("PASSED")

i += 1


# flake8: noqa

import numpy as np

import scipy

import networkx as nx

from sklearn.neighbors import BallTree as sk_BallTree

from sklearn.metrics.pairwise import rbf_kernel as sk_rbf

from sklearn.metrics.pairwise import linear_kernel as sk_linear

from sklearn.metrics.pairwise import polynomial_kernel as sk_poly

from numpy_ml.utils.distance_metrics import (

hamming,

euclidean,

chebyshev,

manhattan,

minkowski,

from numpy_ml.utils.kernels import LinearKernel, PolynomialKernel, RBFKernel

from numpy_ml.utils.data_structures import BallTree

from numpy_ml.utils.graphs import (

Edge,

DiGraph,

UndirectedGraph,

def test_minkowski(N=1):

i = 0

while i < N:

N = np.random.randint(1, 100)

p = 1 + np.random.rand() * 10

x = np.random.rand(N)

y = np.random.rand(N)

mine = minkowski(x, y, p)

theirs = scipy.spatial.distance.minkowski(x, y, p)

print("PASSED")

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def grad_check_RNN(model, loss_func, param_name, n_t, X, epsilon=1e-7):

Manual gradient calc for vanilla RNN parameters

if param_name in ["Ba", "Bx"]:

param_name = param_name.lower()

elif param_name in ["X", "y"]:

return None

param_orig = model.parameters[param_name]

grads = np.zeros_like(param_orig)

for flat_ix, val in enumerate(param_orig.flat):

param = deepcopy(param_orig)

md_ix = np.unravel_index(flat_ix, param.shape)

# plus

y_preds_plus = []

param[md_ix] = val + epsilon

model.parameters[param_name] = param

for t in range(n_t):

y_pred_plus = model.forward(X[:, :, t])

y_preds_plus += [y_pred_plus]

loss_plus = loss_func(y_preds_plus)

# minus

y_preds_minus = []

param[md_ix] = val - epsilon

model.parameters[param_name] = param

for t in range(n_t):

y_pred_minus = model.forward(X[:, :, t])

y_preds_minus += [y_pred_minus]

loss_minus = loss_func(y_preds_minus)

grad = (loss_plus - loss_minus) / (2 * epsilon)

grads[md_ix] = grad

return grads.T

#                               Modules                               #

def test_BidirectionalLSTM(N=15):

from numpy_ml.neural_nets.modules import BidirectionalLSTM

N = np.inf if N is None else N

i = 1

while i < N + 1:

n_ex = np.random.randint(1, 10)

n_in = np.random.randint(1, 10)

n_out = np.random.randint(1, 10)

n_t = np.random.randint(1, 10)

X = random_tensor((n_ex, n_in, n_t), standardize=True)

# initialize LSTM layer

L1 = BidirectionalLSTM(n_out=n_out)

# forward prop

y_pred = L1.forward(X)

# backprop

dLdA = np.ones_like(y_pred)

dLdX = L1.backward(dLdA)

# get gold standard gradients

gold_mod = TorchBidirectionalLSTM(n_in, n_out, L1.parameters)

golds = gold_mod.extract_grads(X)

pms, grads = L1.parameters["components"], L1.gradients["components"]

params = [

(X, "X"),

(y_pred, "y"),

(pms["cell_fwd"]["bo"].T, "bo_f"),

(pms["cell_fwd"]["bu"].T, "bu_f"),

(pms["cell_fwd"]["bf"].T, "bf_f"),

(pms["cell_fwd"]["bc"].T, "bc_f"),

(pms["cell_fwd"]["Wo"], "Wo_f"),

(pms["cell_fwd"]["Wu"], "Wu_f"),

(pms["cell_fwd"]["Wf"], "Wf_f"),

(pms["cell_fwd"]["Wc"], "Wc_f"),

(pms["cell_bwd"]["bo"].T, "bo_b"),

(pms["cell_bwd"]["bu"].T, "bu_b"),

(pms["cell_bwd"]["bf"].T, "bf_b"),

(pms["cell_bwd"]["bc"].T, "bc_b"),

(pms["cell_bwd"]["Wo"], "Wo_b"),

(pms["cell_bwd"]["Wu"], "Wu_b"),

(pms["cell_bwd"]["Wf"], "Wf_b"),

(pms["cell_bwd"]["Wc"], "Wc_b"),

(grads["cell_fwd"]["bo"].T, "dLdBo_f"),

(grads["cell_fwd"]["bu"].T, "dLdBu_f"),

(grads["cell_fwd"]["bf"].T, "dLdBf_f"),

(grads["cell_fwd"]["bc"].T, "dLdBc_f"),

(grads["cell_fwd"]["Wo"], "dLdWo_f"),

(grads["cell_fwd"]["Wu"], "dLdWu_f"),

(grads["cell_fwd"]["Wf"], "dLdWf_f"),

(grads["cell_fwd"]["Wc"], "dLdWc_f"),

(grads["cell_bwd"]["bo"].T, "dLdBo_b"),

(grads["cell_bwd"]["bu"].T, "dLdBu_b"),

(grads["cell_bwd"]["bf"].T, "dLdBf_b"),

(grads["cell_bwd"]["bc"].T, "dLdBc_b"),

(grads["cell_bwd"]["Wo"], "dLdWo_b"),

(grads["cell_bwd"]["Wu"], "dLdWu_b"),

(grads["cell_bwd"]["Wf"], "dLdWf_b"),

(grads["cell_bwd"]["Wc"], "dLdWc_b"),

(dLdX, "dLdX"),

print("Case {}".format(i))

for ix, (mine, label) in enumerate(params):

mine,

golds[label],

err_msg=err_fmt(params, golds, ix),

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_Embedding(N=15):

from numpy_ml.neural_nets.layers import Embedding

N = np.inf if N is None else N

i = 1

while i < N + 1:

vocab_size = np.random.randint(1, 2000)

n_ex = np.random.randint(1, 100)

n_in = np.random.randint(1, 100)

emb_dim = np.random.randint(1, 100)

X = np.random.randint(0, vocab_size, (n_ex, n_in))

# initialize Embedding layer

L1 = Embedding(n_out=emb_dim, vocab_size=vocab_size)

# forward prop

y_pred = L1.forward(X)

# backprop

dLdy = np.ones_like(y_pred)

#  dLdX = L1.backward(dLdy)

# get gold standard gradients

gold_mod = TorchEmbeddingLayer(vocab_size, emb_dim, L1.parameters)

golds = gold_mod.extract_grads(X)

params = [

(L1.X[0], "X"),

(y_pred, "y"),

(L1.parameters["W"], "W"),

(dLdy, "dLdy"),

(L1.gradients["W"], "dLdW"),

#  (dLdX, "dLdX"),

print("\nTrial {}".format(i))

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=3

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import numpy as np

import scipy

import networkx as nx

from sklearn.neighbors import BallTree as sk_BallTree

from sklearn.metrics.pairwise import rbf_kernel as sk_rbf

from sklearn.metrics.pairwise import linear_kernel as sk_linear

from sklearn.metrics.pairwise import polynomial_kernel as sk_poly

from numpy_ml.utils.distance_metrics import (

hamming,

euclidean,

chebyshev,

manhattan,

minkowski,

from numpy_ml.utils.kernels import LinearKernel, PolynomialKernel, RBFKernel

from numpy_ml.utils.data_structures import BallTree

from numpy_ml.utils.graphs import (

Edge,

DiGraph,

UndirectedGraph,

def test_ball_tree(N=1):

i = 0

while i < N:

N = np.random.randint(2, 100)

M = np.random.randint(2, 100)

k = np.random.randint(1, N)

ls = np.min([np.random.randint(1, 10), N - 1])

X = np.random.rand(N, M)

BT = BallTree(leaf_size=ls, metric=euclidean)

BT.fit(X)

x = np.random.rand(M)

mine = BT.nearest_neighbors(k, x)

assert len(mine) == k

mine_neighb = np.array([n.key for n in mine])

mine_dist = np.array([n.distance for n in mine])

sort_ix = np.argsort(mine_dist)

mine_dist = mine_dist[sort_ix]

mine_neighb = mine_neighb[sort_ix]

sk = sk_BallTree(X, leaf_size=ls)

theirs_dist, ind = sk.query(x.reshape(1, -1), k=k)

sort_ix = np.argsort(theirs_dist.flatten())

theirs_dist = theirs_dist.flatten()[sort_ix]

theirs_neighb = X[ind.flatten()[sort_ix]]

for j in range(len(theirs_dist)):

np.testing.assert_almost_equal(mine_neighb[j], theirs_neighb[j])

np.testing.assert_almost_equal(mine_dist[j], theirs_dist[j])

print("PASSED")

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_RNNCell(N=15):

from numpy_ml.neural_nets.layers import RNNCell

N = np.inf if N is None else N

i = 1

while i < N + 1:

n_ex = np.random.randint(1, 10)

n_in = np.random.randint(1, 10)

n_out = np.random.randint(1, 10)

n_t = np.random.randint(1, 10)

X = random_tensor((n_ex, n_in, n_t), standardize=True)

# initialize RNN layer

L1 = RNNCell(n_out=n_out)

# forward prop

y_preds = []

for t in range(n_t):

y_pred = L1.forward(X[:, :, t])

y_preds += [y_pred]

# backprop

dLdX = []

dLdAt = np.ones_like(y_preds[t])

for t in reversed(range(n_t)):

dLdXt = L1.backward(dLdAt)

dLdX.insert(0, dLdXt)

dLdX = np.dstack(dLdX)

# get gold standard gradients

gold_mod = TorchRNNCell(n_in, n_out, L1.parameters)

golds = gold_mod.extract_grads(X)

params = [

(X, "X"),

(np.array(y_preds), "y"),

(L1.parameters["ba"].T, "ba"),

(L1.parameters["bx"].T, "bx"),

(L1.parameters["Wax"].T, "Wax"),

(L1.parameters["Waa"].T, "Waa"),

(L1.gradients["ba"].T, "dLdBa"),

(L1.gradients["bx"].T, "dLdBx"),

(L1.gradients["Wax"].T, "dLdWax"),

(L1.gradients["Waa"].T, "dLdWaa"),

(dLdX, "dLdX"),

print("Trial {}".format(i))

for ix, (mine, label) in enumerate(params):

mine,

golds[label],

err_msg=err_fmt(params, golds, ix),

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import time

import numpy as np

from scipy.special import expit

import torch

import torch.nn.functional as F

from numpy_ml.utils.testing import random_stochastic_matrix, random_tensor

def torch_gradient_generator(fn, **kwargs):

def get_grad(z):

z1 = torch.autograd.Variable(torch.from_numpy(z), requires_grad=True)

z2 = fn(z1, **kwargs).sum()

grad = z1.grad.numpy()

return grad

return get_grad

mine, label = params[ix]

label, mine, label, golds[label]

err_msg += warn_str

return err_msg

#                            Test Suite                               #

#  def test_activations(N=50):

#      print("Testing Sigmoid activation")

#      time.sleep(1)

#      test_sigmoid_activation(N)

#      test_sigmoid_grad(N)

#      #  print("Testing Softmax activation")

#      #  time.sleep(1)

#      #  test_softmax_activation(N)

#      #  test_softmax_grad(N)

#      print("Testing Tanh activation")

#      time.sleep(1)

#      test_tanh_grad(N)

#      print("Testing ReLU activation")

#      time.sleep(1)

#      test_relu_activation(N)

#      test_relu_grad(N)

#      print("Testing ELU activation")

#      time.sleep(1)

#      test_elu_activation(N)

#      test_elu_grad(N)

#      print("Testing SELU activation")

#      time.sleep(1)

#      test_selu_activation(N)

#      test_selu_grad(N)

#      print("Testing LeakyRelu activation")

#      time.sleep(1)

#      test_leakyrelu_activation(N)

#      test_leakyrelu_grad(N)

#      print("Testing SoftPlus activation")

#      time.sleep(1)

#      test_softplus_activation(N)

#      test_softplus_grad(N)

#                          Activations                                #

def test_elu_activation(N=50):

from numpy_ml.neural_nets.activations import ELU

N = np.inf if N is None else N

i = 0

while i < N:

n_dims = np.random.randint(1, 10)

z = random_tensor((1, n_dims))

alpha = np.random.uniform(0, 10)

mine = ELU(alpha)

gold = lambda z, a: F.elu(torch.from_numpy(z), alpha).numpy()

assert_almost_equal(mine.fn(z), gold(z, alpha))

print("PASSED")

i += 1

def test_elu_grad(N=50):

N = np.inf if N is None else N

i = 0

while i < N:

n_ex = np.random.randint(1, 10)

n_dims = np.random.randint(1, 10)

alpha = np.random.uniform(0, 10)

z = random_tensor((n_ex, n_dims))

mine = ELU(alpha)

gold = torch_gradient_generator(F.elu, alpha=alpha)

assert_almost_equal(mine.grad(z), gold(z), decimal=6)

print("PASSED")

i += 1


# flake8: noqa

import time

import numpy as np

from numpy.testing import assert_almost_equal

from scipy.special import expit

import torch

import torch.nn.functional as F

def get_grad(z):

z2 = fn(z1, **kwargs).sum()

grad = z1.grad.numpy()

return grad

return get_grad

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

#                            Test Suite                               #

#  def test_activations(N=50):

#      print("Testing Sigmoid activation")

#      time.sleep(1)

#      test_sigmoid_activation(N)

#      test_sigmoid_grad(N)

#      #  print("Testing Softmax activation")

#      #  time.sleep(1)

#      #  test_softmax_activation(N)

#      #  test_softmax_grad(N)

#      print("Testing Tanh activation")

#      time.sleep(1)

#      test_tanh_grad(N)

#      print("Testing ReLU activation")

#      time.sleep(1)

#      test_relu_activation(N)

#      test_relu_grad(N)

#      print("Testing ELU activation")

#      time.sleep(1)

#      test_elu_activation(N)

#      test_elu_grad(N)

#      print("Testing SELU activation")

#      time.sleep(1)

#      test_selu_activation(N)

#      test_selu_grad(N)

#      print("Testing LeakyRelu activation")

#      time.sleep(1)

#      test_leakyrelu_activation(N)

#      test_leakyrelu_grad(N)

#      print("Testing SoftPlus activation")

#      time.sleep(1)

#      test_softplus_activation(N)

#      test_softplus_grad(N)

#                          Activations                                #

def test_softplus_activation(N=50):

from numpy_ml.neural_nets.activations import SoftPlus

N = np.inf if N is None else N

mine = SoftPlus()

gold = lambda z: F.softplus(torch.FloatTensor(z)).numpy()

i = 0

while i < N:

n_dims = np.random.randint(1, 100)

z = random_stochastic_matrix(1, n_dims)

assert_almost_equal(mine.fn(z), gold(z))

print("PASSED")

i += 1

def test_softplus_grad(N=50):

N = np.inf if N is None else N

mine = SoftPlus()

gold = torch_gradient_generator(F.softplus)

i = 0

while i < N:

n_ex = np.random.randint(1, 100)

n_dims = np.random.randint(1, 100)

z = random_tensor((n_ex, n_dims), standardize=True)

assert_almost_equal(mine.grad(z), gold(z))

print("PASSED")

i += 1


# flake8: noqa

import time

import numpy as np

from numpy.testing import assert_almost_equal

from scipy.special import expit

import torch

import torch.nn.functional as F

from numpy_ml.utils.testing import random_stochastic_matrix, random_tensor

def torch_gradient_generator(fn, **kwargs):

def get_grad(z):

z1 = torch.autograd.Variable(torch.from_numpy(z), requires_grad=True)

z2 = fn(z1, **kwargs).sum()

grad = z1.grad.numpy()

return grad

return get_grad

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

#                            Test Suite                               #

#  def test_activations(N=50):

#      print("Testing Sigmoid activation")

#      time.sleep(1)

#      test_sigmoid_activation(N)

#      test_sigmoid_grad(N)

#      #  print("Testing Softmax activation")

#      #  time.sleep(1)

#      #  test_softmax_activation(N)

#      #  test_softmax_grad(N)

#      print("Testing Tanh activation")

#      time.sleep(1)

#      test_tanh_grad(N)

#      print("Testing ReLU activation")

#      time.sleep(1)

#      test_relu_activation(N)

#      test_relu_grad(N)

#      print("Testing ELU activation")

#      time.sleep(1)

#      test_elu_activation(N)

#      test_elu_grad(N)

#      print("Testing SELU activation")

#      time.sleep(1)

#      test_selu_activation(N)

#      test_selu_grad(N)

#      print("Testing LeakyRelu activation")

#      time.sleep(1)

#      test_leakyrelu_activation(N)

#      test_leakyrelu_grad(N)

#      print("Testing SoftPlus activation")

#      time.sleep(1)

#      test_softplus_activation(N)

#      test_softplus_grad(N)

#                          Activations                                #

def test_relu_activation(N=50):

from numpy_ml.neural_nets.activations import ReLU

N = np.inf if N is None else N

mine = ReLU()

gold = lambda z: F.relu(torch.FloatTensor(z)).numpy()

i = 0

while i < N:

n_dims = np.random.randint(1, 100)

z = random_stochastic_matrix(1, n_dims)

assert_almost_equal(mine.fn(z), gold(z))

print("PASSED")

i += 1

def test_relu_grad(N=50):

N = np.inf if N is None else N

mine = ReLU()

gold = torch_gradient_generator(F.relu)

i = 0

while i < N:

n_ex = np.random.randint(1, 100)

n_dims = np.random.randint(1, 100)

z = random_tensor((n_ex, n_dims))

assert_almost_equal(mine.grad(z), gold(z))

print("PASSED")

i += 1


# flake8: noqa

import tempfile

import nltk

import numpy as np

from ..preprocessing.nlp import tokenize_words

from ..ngram import AdditiveNGram, MLENGram

from ..utils.testing import random_paragraph

class MLEGold:

def __init__(

self, N, K=1, unk=True, filter_stopwords=True, filter_punctuation=True

self.N = N

self.K = K

self.unk = unk

self.filter_stopwords = filter_stopwords

self.filter_punctuation = filter_punctuation

self.hyperparameters = {

"N": N,

"K": K,

"unk": unk,

"filter_stopwords": filter_stopwords,

"filter_punctuation": filter_punctuation,

def train(self, corpus_fp, vocab=None, encoding=None):

N = self.N

H = self.hyperparameters

models, counts = {}, {}

grams = {n: [] for n in range(1, N + 1)}

gg = {n: [] for n in range(1, N + 1)}

filter_punc, filter_stop = H["filter_punctuation"], H["filter_stopwords"]

n_words = 0

tokens = set([])

with open(corpus_fp, "r", encoding=encoding) as text:

for line in text:

words = tokenize_words(line, filter_punc, filter_stop)

if vocab is not None:

words = vocab.filter(words, H["unk"])

if len(words) == 0:

continue

n_words += len(words)

tokens.update(words)

# calculate n, n-1, ... 1-grams

for n in range(1, N + 1):

grams[n].append(

nltk.ngrams(

words,

n,

gg[n].extend(

list(

nltk.ngrams(

words,

n,

for n in range(1, N + 1):

counts[n] = nltk.FreqDist(gg[n])

models[n] = nltk.lm.MLE(order=n)

models[n].fit(grams[n], tokens)

self.counts = counts

self.n_words = n_words

self._models = models

self.n_tokens = len(vocab) if vocab is not None else len(tokens)

def log_prob(self, words, N):

assert N in self.counts, "You do not have counts for {}-grams".format(N)

if N > len(words):

err = "Not enough words for a gram-size of {}: {}".format(N, len(words))

raise ValueError(err)

total_prob = 0

for ngram in nltk.ngrams(words, N):

total_prob += self._log_ngram_prob(ngram)

return total_prob

def _log_ngram_prob(self, ngram):

N = len(ngram)

return self._models[N].logscore(ngram[-1], ngram[:-1])

class AdditiveGold:

def __init__(

self.N = N

self.K = K

self.unk = unk

self.hyperparameters = {

"N": N,

"K": K,

"unk": unk,

"filter_stopwords": filter_stopwords,

N = self.N

H = self.hyperparameters

models, counts = {}, {}

gg = {n: [] for n in range(1, N + 1)}

n_words = 0

tokens = set()

for line in text:

if vocab is not None:

words = vocab.filter(words, H["unk"])

if len(words) == 0:

continue

n_words += len(words)

tokens.update(words)

# calculate n, n-1, ... 1-grams

for n in range(1, N + 1):

grams[n].append(

nltk.ngrams(

words,

n,

gg[n].extend(

list(

nltk.ngrams(

words,

n,

for n in range(1, N + 1):

counts[n] = nltk.FreqDist(gg[n])

models[n] = nltk.lm.Lidstone(order=n, gamma=self.K)

models[n].fit(grams[n], tokens)

self.counts = counts

self._models = models

self.n_words = n_words

def log_prob(self, words, N):

if N > len(words):

raise ValueError(err)

total_prob = 0

for ngram in nltk.ngrams(words, N):

return total_prob

def _log_ngram_prob(self, ngram):

N = len(ngram)

def test_additive():

K = np.random.rand()

N = np.random.randint(2, 5)

gold = AdditiveGold(

N, K, unk=True, filter_stopwords=False, filter_punctuation=False

mine = AdditiveNGram(

with tempfile.NamedTemporaryFile() as temp:

temp.write(bytes(" ".join(random_paragraph(1000)), encoding="utf-8-sig"))

gold.train(temp.name, encoding="utf-8-sig")

mine.train(temp.name, encoding="utf-8-sig")

for k in mine.counts[N].keys():

if k[0] == k[1] and k[0] in ("<bol>", "<eol>"):

continue

err_str = "{}, mine: {}, gold: {}"

assert mine.counts[N][k] == gold.counts[N][k], err_str.format(

k, mine.counts[N][k], gold.counts[N][k]

M = mine.log_prob(k, N)

G = gold.log_prob(k, N) / np.log2(np.e)  # convert to log base e

np.testing.assert_allclose(M, G)

print("PASSED")


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_DPAttention(N=15):

from numpy_ml.neural_nets.layers import DotProductAttention

N = np.inf if N is None else N

i = 1

while i < N + 1:

n_ex = np.random.randint(1, 10)

d_k = np.random.randint(1, 100)

d_v = np.random.randint(1, 100)

Q = random_tensor((n_ex, d_k), standardize=True)

K = random_tensor((n_ex, d_k), standardize=True)

V = random_tensor((n_ex, d_v), standardize=True)

# initialize DotProductAttention layer

mine = DotProductAttention(scale=True, dropout_p=0)

# forward prop

y_pred = mine.forward(Q, K, V)

# backprop

dLdy = np.ones_like(y_pred)

dLdQ, dLdK, dLdV = mine.backward(dLdy)

# get gold standard gradients

gold_mod = TorchSDPAttentionLayer()

golds = gold_mod.extract_grads(Q, K, V)

params = [

(mine.X[0][0], "Q"),

(mine.X[0][1], "K"),

(mine.X[0][2], "V"),

(y_pred, "Y"),

(dLdV, "dLdV"),

(dLdK, "dLdK"),

(dLdQ, "dLdQ"),

print("\nTrial {}".format(i))

print("n_ex={} d_k={} d_v={}".format(n_ex, d_k, d_v))

for ix, (mine, label) in enumerate(params):

mine, golds[label], err_msg=err_fmt(params, golds, ix), decimal=4

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

#                         Loss Functions                              #

def test_squared_error(N=15):

from numpy_ml.neural_nets.losses import SquaredError

N = np.inf if N is None else N

mine = SquaredError()

gold = (

lambda y, y_pred: mean_squared_error(y, y_pred)

* y_pred.shape[0]

* y_pred.shape[1]

* 0.5

# ensure we get 0 when the two arrays are equal

n_dims = np.random.randint(2, 100)

n_examples = np.random.randint(1, 1000)

y = y_pred = random_tensor((n_examples, n_dims))

assert_almost_equal(mine.loss(y, y_pred), gold(y, y_pred))

print("PASSED")

i = 1

while i < N:

n_dims = np.random.randint(2, 100)

n_examples = np.random.randint(1, 1000)

y = random_tensor((n_examples, n_dims))

assert_almost_equal(mine.loss(y, y_pred), gold(y, y_pred), decimal=5)

print("PASSED")

i += 1

def test_squared_error_grad(N=15):

from numpy_ml.neural_nets.activations import Tanh

N = np.inf if N is None else N

mine = SquaredError()

gold = torch_mse_grad

act = Tanh()

i = 1

while i < N:

n_dims = np.random.randint(2, 100)

n_examples = np.random.randint(1, 1000)

y = random_tensor((n_examples, n_dims))

# raw inputs

z = random_tensor((n_examples, n_dims))

y_pred = act.fn(z)

mine.grad(y, y_pred, z, act), 0.5 * gold(y, z, torch.tanh), decimal=4

print("PASSED")

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def grad_check_RNN(model, loss_func, param_name, n_t, X, epsilon=1e-7):

Manual gradient calc for vanilla RNN parameters

if param_name in ["Ba", "Bx"]:

param_name = param_name.lower()

elif param_name in ["X", "y"]:

return None

param_orig = model.parameters[param_name]

grads = np.zeros_like(param_orig)

for flat_ix, val in enumerate(param_orig.flat):

param = deepcopy(param_orig)

md_ix = np.unravel_index(flat_ix, param.shape)

# plus

y_preds_plus = []

param[md_ix] = val + epsilon

model.parameters[param_name] = param

for t in range(n_t):

y_pred_plus = model.forward(X[:, :, t])

y_preds_plus += [y_pred_plus]

loss_plus = loss_func(y_preds_plus)

# minus

y_preds_minus = []

param[md_ix] = val - epsilon

model.parameters[param_name] = param

for t in range(n_t):

y_pred_minus = model.forward(X[:, :, t])

y_preds_minus += [y_pred_minus]

loss_minus = loss_func(y_preds_minus)

grad = (loss_plus - loss_minus) / (2 * epsilon)

grads[md_ix] = grad

return grads.T

#                               Modules                               #

def test_SkipConnectionConvModule(N=15):

from numpy_ml.neural_nets.modules import SkipConnectionConvModule

from numpy_ml.neural_nets.activations import Tanh, ReLU, Sigmoid, Affine

N = np.inf if N is None else N

acts = [

(Tanh(), nn.Tanh(), "Tanh"),

(Sigmoid(), nn.Sigmoid(), "Sigmoid"),

(ReLU(), nn.ReLU(), "ReLU"),

(Affine(), TorchLinearActivation(), "Affine"),

i = 1

while i < N + 1:

n_ex = np.random.randint(2, 10)

in_rows = np.random.randint(2, 10)

in_cols = np.random.randint(2, 10)

n_in = np.random.randint(2, 5)

n_out1 = np.random.randint(2, 5)

n_out2 = np.random.randint(2, 5)

f_shape1 = (

min(in_rows, np.random.randint(1, 5)),

min(in_cols, np.random.randint(1, 5)),

f_shape2 = (

min(in_rows, np.random.randint(1, 5)),

min(in_cols, np.random.randint(1, 5)),

f_shape_skip = (

min(in_rows, np.random.randint(1, 5)),

min(in_cols, np.random.randint(1, 5)),

s1 = np.random.randint(1, 5)

s2 = np.random.randint(1, 5)

s_skip = np.random.randint(1, 5)

# randomly select an activation function

act_fn, torch_fn, act_fn_name = acts[np.random.randint(0, len(acts))]

X = random_tensor((n_ex, in_rows, in_cols, n_in), standardize=True)

p1 = (np.random.randint(1, 5), np.random.randint(1, 5))

p2 = (np.random.randint(1, 5), np.random.randint(1, 5))

# initialize SkipConnectionConv module

L1 = SkipConnectionConvModule(

# forward prop

try:

y_pred = L1.forward(X)

except (ValueError, AssertionError):

print("Invalid padding; Retrying")

continue

ps = L1.hyperparameters["pad_skip"]

if ps[0] != ps[1] or ps[2] != ps[3]:

continue

pad_skip = (ps[0], ps[2])

# backprop

dLdy = np.ones_like(y_pred)

dLdX = L1.backward(dLdy)

# get gold standard gradients

gold_mod = TorchSkipConnectionConv(

golds = gold_mod.extract_grads(X)

params = L1.parameters["components"]

grads = L1.gradients["components"]

params = [

(X, "X"),

(params["conv1"]["W"], "conv1_W"),

(params["conv1"]["b"], "conv1_b"),

(params["batchnorm1"]["scaler"].T, "bn1_scaler"),

(params["batchnorm1"]["intercept"], "bn1_intercept"),

(params["batchnorm1"]["running_mean"], "bn1_running_mean"),

#  (params["batchnorm1"]["running_var"], "bn1_running_var"),

(params["conv2"]["W"], "conv2_W"),

(params["conv2"]["b"], "conv2_b"),

(params["batchnorm2"]["scaler"].T, "bn2_scaler"),

(params["batchnorm2"]["intercept"], "bn2_intercept"),

(params["batchnorm2"]["running_mean"], "bn2_running_mean"),

#  (params["batchnorm2"]["running_var"], "bn2_running_var"),

(params["conv_skip"]["W"], "conv_skip_W"),

(params["conv_skip"]["b"], "conv_skip_b"),

(params["batchnorm_skip"]["scaler"].T, "bn_skip_scaler"),

(params["batchnorm_skip"]["intercept"], "bn_skip_intercept"),

(params["batchnorm_skip"]["running_mean"], "bn_skip_running_mean"),

#  (params["batchnorm_skip"]["running_var"], "bn_skip_running_var"),

(L1._dv["conv1_out"], "act1_out"),

(L1._dv["batchnorm1_out"], "bn1_out"),

(L1._dv["conv2_out"], "conv2_out"),

(L1._dv["batchnorm2_out"], "bn2_out"),

(L1._dv["conv_skip_out"], "conv_skip_out"),

(L1._dv["batchnorm_skip_out"], "bn_skip_out"),

(y_pred, "Y"),

(dLdy, "dLdY"),

(L1.derived_variables["dLdBn2"], "dLdBn2_out"),

(L1.derived_variables["dLdConv2"], "dLdConv2_out"),

(L1.derived_variables["dLdBnSkip"], "dLdBnSkip_out"),

(L1.derived_variables["dLdConvSkip"], "dLdConvSkip_out"),

(L1.derived_variables["dLdBn1"], "dLdBn1_out"),

(L1.derived_variables["dLdConv1"], "dLdActFn1_out"),

(dLdX, "dLdX"),

(grads["batchnorm_skip"]["scaler"].T, "dLdBnSkip_scaler"),

(grads["batchnorm_skip"]["intercept"], "dLdBnSkip_intercept"),

(grads["conv_skip"]["W"], "dLdConvSkip_W"),

(grads["conv_skip"]["b"], "dLdConvSkip_b"),

(grads["batchnorm2"]["scaler"].T, "dLdBn2_scaler"),

(grads["batchnorm2"]["intercept"], "dLdBn2_intercept"),

(grads["conv2"]["W"], "dLdConv2_W"),

(grads["conv2"]["b"], "dLdConv2_b"),

(grads["batchnorm1"]["scaler"].T, "dLdBn1_scaler"),

(grads["batchnorm1"]["intercept"], "dLdBn1_intercept"),

(grads["conv1"]["W"], "dLdConv1_W"),

(grads["conv1"]["b"], "dLdConv1_b"),

print("\nTrial {}".format(i))

print("act_fn={}, n_ex={}".format(act_fn, n_ex))

print("in_rows={}, in_cols={}, n_in={}".format(in_rows, in_cols, n_in))

print("pad1={}, stride1={}, f_shape1={}".format(p1, s1, f_shape1))

print("pad2={}, stride2={}, f_shape2={}".format(p2, s2, f_shape2))

print("stride_skip={}, f_shape_skip={}".format(s_skip, f_shape_skip))

warn_str = (

"\n[NOTE] The tests in this module can fail sometimes during "

"backprop due to the ReLU issue: while the difference in the forward pass "

"between z=-1e-9 and z=1e-9 is miniscule, the difference during the backward "

"pass is significant due to ReLU's kink about 0."

for ix, (mine, label) in enumerate(params):

mine,

golds[label],

err_msg=err_fmt(params, golds, ix, warn_str),

print("\tPASSED {}".format(label))

i += 1


# flake8: noqa

from collections import Counter

# gold-standard imports

import huffman

import numpy as np

from scipy.fftpack import dct

from sklearn.preprocessing import StandardScaler

from sklearn.feature_extraction.text import TfidfVectorizer

# try:

#     from librosa.core.time_frequency import fft_frequencies

# except ImportError:

#     # for librosa >= 0.8.0

#     from librosa import fft_frequencies

# from librosa.feature import mfcc as lr_mfcc

# from librosa.util import frame

# from librosa.filters import mel

# numpy-ml implementations

from numpy_ml.preprocessing.general import Standardizer

from numpy_ml.preprocessing.nlp import HuffmanEncoder, TFIDFEncoder

from numpy_ml.preprocessing.dsp import (

DCT,

DFT,

mfcc,

from numpy_ml.utils.testing import random_paragraph

def test_huffman(N=15):

i = 0

while i < N:

n_words = np.random.randint(1, 100)

para = random_paragraph(n_words)

HT = HuffmanEncoder()

HT.fit(para)

my_dict = HT._item2code

their_dict = huffman.codebook(Counter(para).items())

for k, v in their_dict.items():

fstr = "their_dict['{}'] = {}, but my_dict['{}'] = {}"

assert k in my_dict, "key `{}` not in my_dict".format(k)

assert my_dict[k] == v, fstr.format(k, v, k, my_dict[k])

print("PASSED")

i += 1


# flake8: noqa

import numpy as np

from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier

from sklearn.gaussian_process import GaussianProcessRegressor

from numpy_ml.nonparametric.knn import KNN

from numpy_ml.nonparametric.gp import GPRegression

from numpy_ml.utils.distance_metrics import euclidean

def test_knn_regression(N=15):

i = 0

while i < N:

N = np.random.randint(2, 100)

M = np.random.randint(2, 100)

k = np.random.randint(1, N)

ls = np.min([np.random.randint(1, 10), N - 1])

weights = np.random.choice(["uniform", "distance"])

X = np.random.rand(N, M)

X_test = np.random.rand(N, M)

y = np.random.rand(N)

knn = KNN(

k=k, leaf_size=ls, metric=euclidean, classifier=False, weights=weights

knn.fit(X, y)

preds = knn.predict(X_test)

gold = KNeighborsRegressor(

weights=weights,

metric="minkowski",

gold.fit(X, y)

gold_preds = gold.predict(X_test)

for mine, theirs in zip(preds, gold_preds):

np.testing.assert_almost_equal(mine, theirs)

print("PASSED")

i += 1

def test_knn_clf(N=15):

i = 0

while i < N:

N = np.random.randint(2, 100)

M = np.random.randint(2, 100)

k = np.random.randint(1, N)

n_classes = np.random.randint(2, 10)

weights = "uniform"

X = np.random.rand(N, M)

X_test = np.random.rand(N, M)

y = np.random.randint(0, n_classes, size=N)

knn = KNN(k=k, leaf_size=ls, metric=euclidean, classifier=True, weights=weights)

knn.fit(X, y)

preds = knn.predict(X_test)

gold = KNeighborsClassifier(

metric="minkowski",

weights=weights,

gold.fit(X, y)

gold_preds = gold.predict(X_test)

print("PASSED")

i += 1


# flake8: noqa

import numpy as np

from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor

from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor

from sklearn.metrics import accuracy_score, mean_squared_error

from sklearn.datasets import make_regression, make_blobs

from sklearn.model_selection import train_test_split

from numpy_ml.trees.gbdt import GradientBoostedDecisionTree

from numpy_ml.trees.dt import DecisionTree, Node, Leaf

from numpy_ml.trees.rf import RandomForest

from numpy_ml.utils.testing import random_tensor

def clone_tree(dtree):

children_left = dtree.tree_.children_left

children_right = dtree.tree_.children_right

feature = dtree.tree_.feature

threshold = dtree.tree_.threshold

values = dtree.tree_.value

def grow(node_id):

l, r = children_left[node_id], children_right[node_id]

if l == r:

return Leaf(values[node_id].argmax())

n = Node(None, None, (feature[node_id], threshold[node_id]))

n.left = grow(l)

n.right = grow(r)

return n

node_id = 0

root = Node(None, None, (feature[node_id], threshold[node_id]))

root.left = grow(children_left[node_id])

root.right = grow(children_right[node_id])

return root

def compare_trees(mine, gold):

clone = clone_tree(gold)

mine = mine.root

def test(mine, clone):

if isinstance(clone, Node) and isinstance(mine, Node):

assert mine.feature == clone.feature, "Node {} not equal".format(depth)

np.testing.assert_allclose(mine.threshold, clone.threshold)

test(mine.left, clone.left, depth + 1)

test(mine.right, clone.right, depth + 1)

elif isinstance(clone, Leaf) and isinstance(mine, Leaf):

np.testing.assert_allclose(mine.value, clone.value)

return

else:

raise ValueError("Nodes at depth {} are not equal".format(depth))

depth = 0

ok = True

while ok:

assert mine.feature == clone.feature

test(mine.left, clone.left, depth + 1)

return

else:

def test_RandomForest(N=1):

i = 1

while i <= N:

n_ex = np.random.randint(2, 100)

n_feats = np.random.randint(2, 100)

n_trees = np.random.randint(2, 100)

max_depth = np.random.randint(1, 5)

classifier = np.random.choice([True, False])

if classifier:

# create classification problem

n_classes = np.random.randint(2, 10)

X, Y = make_blobs(

n_samples=n_ex, centers=n_classes, n_features=n_feats, random_state=i

X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.3, random_state=i)

# initialize model

def loss(yp, y):

return 1 - accuracy_score(yp, y)

# initialize model

criterion = np.random.choice(["entropy", "gini"])

mine = RandomForest(

classifier=classifier,

criterion=criterion,

gold = RandomForestClassifier(

criterion=criterion,

bootstrap=True,

else:

# create regeression problem

X, Y = make_regression(n_samples=n_ex, n_features=n_feats, random_state=i)

# initialize model

criterion = "mse"

loss = mean_squared_error

mine = RandomForest(

criterion=criterion,

classifier=classifier,

gold = RandomForestRegressor(

criterion=criterion,

bootstrap=True,

print("Trial {}".format(i))

print("\tClassifier={}, criterion={}".format(classifier, criterion))

print("\tmax_depth={}, n_feats={}, n_ex={}".format(max_depth, n_feats, n_ex))

if classifier:

print("\tn_classes: {}".format(n_classes))

# fit 'em

mine.fit(X, Y)

gold.fit(X, Y)

# get preds

y_pred_mine = mine.predict(X)

y_pred_gold = gold.predict(X)

loss_mine = loss(y_pred_mine, Y)

loss_gold = loss(y_pred_gold, Y)

# get preds on test set

y_pred_mine_test = mine.predict(X_test)

y_pred_gold_test = gold.predict(X_test)

loss_mine_test = loss(y_pred_mine_test, Y_test)

loss_gold_test = loss(y_pred_gold_test, Y_test)

try:

np.testing.assert_almost_equal(loss_mine, loss_gold)

print("\tLoss on training: {}".format(loss_mine))

except AssertionError as e:

print("\tTraining losses not equal:\n{}".format(e))

try:

np.testing.assert_almost_equal(loss_mine_test, loss_gold_test)

print("\tLoss on test: {}".format(loss_mine_test))

except AssertionError as e:

print("\tTest losses not equal:\n{}".format(e))

print("PASSED")

i += 1


# flake8: noqa

import numpy as np

import scipy

import networkx as nx

from sklearn.neighbors import BallTree as sk_BallTree

from sklearn.metrics.pairwise import rbf_kernel as sk_rbf

from sklearn.metrics.pairwise import linear_kernel as sk_linear

from sklearn.metrics.pairwise import polynomial_kernel as sk_poly

from numpy_ml.utils.distance_metrics import (

hamming,

euclidean,

chebyshev,

manhattan,

minkowski,

from numpy_ml.utils.kernels import LinearKernel, PolynomialKernel, RBFKernel

from numpy_ml.utils.data_structures import BallTree

from numpy_ml.utils.graphs import (

Edge,

DiGraph,

UndirectedGraph,

#                               Kernels                               #

def test_polynomial_kernel(N=1):

i = 0

while i < N:

N = np.random.randint(1, 100)

M = np.random.randint(1, 100)

C = np.random.randint(1, 1000)

gamma = np.random.rand()

d = np.random.randint(1, 5)

c0 = np.random.rand()

X = np.random.rand(N, C)

Y = np.random.rand(M, C)

mine = PolynomialKernel(gamma=gamma, d=d, c0=c0)(X, Y)

gold = sk_poly(X, Y, gamma=gamma, degree=d, coef0=c0)

np.testing.assert_almost_equal(mine, gold)

print("PASSED")

i += 1


# flake8: noqa

import time

import numpy as np

from numpy.testing import assert_almost_equal

from scipy.special import expit

import torch

import torch.nn.functional as F

from numpy_ml.utils.testing import random_stochastic_matrix, random_tensor

def torch_gradient_generator(fn, **kwargs):

def get_grad(z):

z1 = torch.autograd.Variable(torch.from_numpy(z), requires_grad=True)

z2 = fn(z1, **kwargs).sum()

grad = z1.grad.numpy()

return grad

return get_grad

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

#                            Test Suite                               #

#  def test_activations(N=50):

#      print("Testing Sigmoid activation")

#      time.sleep(1)

#      test_sigmoid_activation(N)

#      test_sigmoid_grad(N)

#      #  print("Testing Softmax activation")

#      #  time.sleep(1)

#      #  test_softmax_activation(N)

#      #  test_softmax_grad(N)

#      print("Testing Tanh activation")

#      time.sleep(1)

#      test_tanh_grad(N)

#      print("Testing ReLU activation")

#      time.sleep(1)

#      test_relu_activation(N)

#      test_relu_grad(N)

#      print("Testing ELU activation")

#      time.sleep(1)

#      test_elu_activation(N)

#      test_elu_grad(N)

#      print("Testing SELU activation")

#      time.sleep(1)

#      test_selu_activation(N)

#      test_selu_grad(N)

#      print("Testing LeakyRelu activation")

#      time.sleep(1)

#      test_leakyrelu_activation(N)

#      test_leakyrelu_grad(N)

#      print("Testing SoftPlus activation")

#      time.sleep(1)

#      test_softplus_activation(N)

#      test_softplus_grad(N)

#                          Activations                                #

def test_selu_activation(N=50):

from numpy_ml.neural_nets.activations import SELU

N = np.inf if N is None else N

mine = SELU()

gold = lambda z: F.selu(torch.FloatTensor(z)).numpy()

i = 0

while i < N:

n_dims = np.random.randint(1, 100)

z = random_stochastic_matrix(1, n_dims)

assert_almost_equal(mine.fn(z), gold(z))

print("PASSED")

i += 1

def test_selu_grad(N=50):

N = np.inf if N is None else N

mine = SELU()

gold = torch_gradient_generator(F.selu)

i = 0

while i < N:

n_ex = np.random.randint(1, 100)

n_dims = np.random.randint(1, 100)

z = random_tensor((n_ex, n_dims))

assert_almost_equal(mine.grad(z), gold(z), decimal=6)

print("PASSED")

i += 1


# flake8: noqa

import time

from copy import deepcopy

import numpy as np

from numpy.testing import assert_almost_equal

from sklearn.metrics import log_loss, mean_squared_error

# for testing sigmoid

from scipy.special import expit

import torch

import torch.nn as nn

import torch.nn.functional as F

from numpy_ml.neural_nets.utils import (

from numpy_ml.utils.testing import (

from .nn_torch_models import (

# TFNCELoss,

# WGAN_GP_tf,

TorchVAELoss,

TorchFCLayer,

TorchRNNCell,

TorchLSTMCell,

TorchAddLayer,

TorchWGANGPLoss,

TorchWavenetModule,

TorchMultiplyLayer,

TorchLayerNormLayer,

TorchBatchNormLayer,

TorchEmbeddingLayer,

TorchLinearActivation,

TorchSDPAttentionLayer,

TorchBidirectionalLSTM,

TorchSkipConnectionConv,

TorchSkipConnectionIdentity,

TorchMultiHeadedAttentionModule,

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

def test_softmax_activation(N=15):

from numpy_ml.neural_nets.layers import Softmax

N = np.inf if N is None else N

mine = Softmax()

gold = lambda z: F.softmax(torch.FloatTensor(z), dim=1).numpy()

i = 0

while i < N:

n_dims = np.random.randint(1, 100)

z = random_stochastic_matrix(1, n_dims)

assert_almost_equal(mine.forward(z), gold(z))

print("PASSED")

i += 1

def test_softmax_grad(N=15):

from functools import partial

N = np.inf if N is None else N

p_soft = partial(F.softmax, dim=1)

gold = torch_gradient_generator(p_soft)

i = 0

while i < N:

mine = Softmax()

n_ex = np.random.randint(1, 3)

n_dims = np.random.randint(1, 50)

z = random_tensor((n_ex, n_dims), standardize=True)

out = mine.forward(z)

gold(z),

err_msg="Theirs:\n{}\n\nMine:\n{}\n".format(

gold(z), mine.backward(np.ones_like(out))

print("PASSED")

i += 1


# flake8: noqa

import time

import numpy as np

from numpy.testing import assert_almost_equal

from scipy.special import expit

import torch

import torch.nn.functional as F

from numpy_ml.utils.testing import random_stochastic_matrix, random_tensor

def torch_gradient_generator(fn, **kwargs):

def get_grad(z):

z1 = torch.autograd.Variable(torch.from_numpy(z), requires_grad=True)

z2 = fn(z1, **kwargs).sum()

grad = z1.grad.numpy()

return grad

return get_grad

#                           Debug Formatter                           #

def err_fmt(params, golds, ix, warn_str=""):

mine, label = params[ix]

err_msg = "-" * 25 + " DEBUG " + "-" * 25 + "\n"

prev_mine, prev_label = params[max(ix - 1, 0)]

err_msg += "Mine (prev) [{}]:\n{}\n\nTheirs (prev) [{}]:\n{}".format(

prev_label, prev_mine, prev_label, golds[prev_label]

err_msg += "\n\nMine [{}]:\n{}\n\nTheirs [{}]:\n{}".format(

label, mine, label, golds[label]

err_msg += warn_str

err_msg += "\n" + "-" * 23 + " END DEBUG " + "-" * 23

return err_msg

#                            Test Suite                               #

#  def test_activations(N=50):

#      print("Testing Sigmoid activation")

#      time.sleep(1)

#      test_sigmoid_activation(N)

#      test_sigmoid_grad(N)

#      #  print("Testing Softmax activation")

#      #  time.sleep(1)

#      #  test_softmax_activation(N)

#      #  test_softmax_grad(N)

#      print("Testing Tanh activation")

#      time.sleep(1)

#      test_tanh_grad(N)

#      print("Testing ReLU activation")

#      time.sleep(1)

#      test_relu_activation(N)

#      test_relu_grad(N)

#      print("Testing ELU activation")

#      time.sleep(1)

#      test_elu_activation(N)

#      test_elu_grad(N)

#      print("Testing SELU activation")

#      time.sleep(1)

#      test_selu_activation(N)

#      test_selu_grad(N)

#      print("Testing LeakyRelu activation")

#      time.sleep(1)

#      test_leakyrelu_activation(N)

#      test_leakyrelu_grad(N)

#      print("Testing SoftPlus activation")

#      time.sleep(1)

#      test_softplus_activation(N)

#      test_softplus_grad(N)

#                          Activations                                #

def test_leakyrelu_activation(N=50):

from numpy_ml.neural_nets.activations import LeakyReLU

N = np.inf if N is None else N

i = 0

while i < N:

n_dims = np.random.randint(1, 100)

z = random_stochastic_matrix(1, n_dims)

alpha = np.random.uniform(0, 10)

mine = LeakyReLU(alpha=alpha)

gold = lambda z: F.leaky_relu(torch.FloatTensor(z), alpha).numpy()

assert_almost_equal(mine.fn(z), gold(z))

print("PASSED")

i += 1

def test_leakyrelu_grad(N=50):

N = np.inf if N is None else N

i = 0

while i < N:

n_ex = np.random.randint(1, 10)

n_dims = np.random.randint(1, 10)

alpha = np.random.uniform(0, 10)

z = random_tensor((n_ex, n_dims))

mine = LeakyReLU(alpha)

gold = torch_gradient_generator(F.leaky_relu, negative_slope=alpha)

assert_almost_equal(mine.grad(z), gold(z), decimal=6)

print("PASSED")

i += 1