"""A module for naive Bayes classifiers"""

import numpy as np

class GaussianNBClassifier:

def __init__(self, eps=1e-6):

r"""

A naive Bayes classifier for real-valued data.

Notes

The naive Bayes model assumes the features of each training example

:math:`\mathbf{x}` are mutually independent given the example label

*y*:

.. math::

P(\mathbf{x}_i \mid y_i) = \prod_{j=1}^M P(x_{i,j} \mid y_i)

where :math:`M` is the rank of the :math:`i^{th}` example

:math:`\mathbf{x}_i` and :math:`y_i` is the label associated with the

:math:`i^{th}` example.

Combining the conditional independence assumption with a simple

application of Bayes' theorem gives the naive Bayes classification

rule:

.. math::

\hat{y} &= \arg \max_y P(y \mid \mathbf{x}) \\

&= \arg \max_y  P(y) P(\mathbf{x} \mid y) \\

&= \arg \max_y  P(y) \prod_{j=1}^M P(x_j \mid y)

In the final expression, the prior class probability :math:`P(y)` can

be specified in advance or estimated empirically from the training

data.

In the Gaussian version of the naive Bayes model, the feature

likelihood is assumed to be normally distributed for each class:

.. math::

\mathbf{x}_i \mid y_i = c, \theta \sim \mathcal{N}(\mu_c, \Sigma_c)

where :math:`\theta` is the set of model parameters: :math:`\{\mu_1,

\Sigma_1, \ldots, \mu_K, \Sigma_K\}`, :math:`K` is the total number of

unique classes present in the data, and the parameters for the Gaussian

associated with class :math:`c`, :math:`\mu_c` and :math:`\Sigma_c`

(where :math:`1 \leq c \leq K`), are estimated via MLE from the set of

training examples with label :math:`c`.

Parameters

eps : float

A value added to the variance to prevent numerical error. Default

is 1e-6.

Attributes

parameters : dict

Dictionary of model parameters: "mean", the `(K, M)` array of

feature means under each class, "sigma", the `(K, M)` array of

feature variances under each class, and "prior", the `(K,)` array of

empirical prior probabilities for each class label.

hyperparameters : dict

Dictionary of model hyperparameters

labels : :py:class:`ndarray <numpy.ndarray>` of shape `(K,)`

An array containing the unique class labels for the training

examples.

self.labels = None

self.hyperparameters = {"eps": eps}

self.parameters = {

"mean": None,  # shape: (K, M)

"sigma": None,  # shape: (K, M)

"prior": None,  # shape: (K,)

def fit(self, X, y):

Fit the model parameters via maximum likelihood.

Notes

The model parameters are stored in the :py:attr:`parameters

<numpy_ml.linear_models.GaussianNBClassifier.parameters>` attribute.

The following keys are present:

"mean": :py:class:`ndarray <numpy.ndarray>` of shape `(K, M)`

Feature means for each of the `K` label classes

"sigma": :py:class:`ndarray <numpy.ndarray>` of shape `(K, M)`

Feature variances for each of the `K` label classes

"prior": :py:class:`ndarray <numpy.ndarray>` of shape `(K,)`

Prior probability of each of the `K` label classes, estimated

empirically from the training data

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, M)`

A dataset consisting of `N` examples, each of dimension `M`

y: :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

The class label for each of the `N` examples in `X`

Returns

self : :class:`GaussianNBClassifier <numpy_ml.linear_models.GaussianNBClassifier>` instance

"""  # noqa: E501

P = self.parameters

H = self.hyperparameters

self.labels = np.unique(y)

K = len(self.labels)

N, M = X.shape

P["mean"] = np.zeros((K, M))

P["sigma"] = np.zeros((K, M))

P["prior"] = np.zeros((K,))

for i, c in enumerate(self.labels):

X_c = X[y == c, :]

P["mean"][i, :] = np.mean(X_c, axis=0)

P["sigma"][i, :] = np.var(X_c, axis=0) + H["eps"]

P["prior"][i] = X_c.shape[0] / N

return self

def predict(self, X):

Use the trained classifier to predict the class label for each example

in **X**.

Parameters

X: :py:class:`ndarray <numpy.ndarray>` of shape `(N, M)`

A dataset of `N` examples, each of dimension `M`

Returns

labels : :py:class:`ndarray <numpy.ndarray>` of shape `(N)`

The predicted class labels for each example in `X`

return self.labels[self._log_posterior(X).argmax(axis=1)]

def _log_posterior(self, X):

r"""

Compute the (unnormalized) log posterior for each class.

Parameters

Returns

log_posterior : :py:class:`ndarray <numpy.ndarray>` of shape `(N, K)`

Unnormalized log posterior probability of each class for each

example in `X`

K = len(self.labels)

log_posterior = np.zeros((X.shape[0], K))

for i in range(K):

log_posterior[:, i] = self._log_class_posterior(X, i)

return log_posterior

def _log_class_posterior(self, X, class_idx):

r"""

Compute the (unnormalized) log posterior for the label at index

`class_idx` in :py:attr:`labels <numpy_ml.linear_models.GaussianNBClassifier.labels>`.

Notes

Unnormalized log posterior for example :math:`\mathbf{x}_i` and class

:math:`c` is::

.. math::

\log P(y_i = c \mid \mathbf{x}_i, \theta)

&\propto \log P(y=c \mid \theta) +

\log P(\mathbf{x}_i \mid y_i = c, \theta) \\

&\propto \log P(y=c \mid \theta)

\sum{j=1}^M \log P(x_j \mid y_i = c, \theta)

In the Gaussian naive Bayes model, the feature likelihood for class

:math:`c`, :math:`P(\mathbf{x}_i \mid y_i = c, \theta)` is assumed to

be normally distributed

.. math::

Parameters

class_idx : int

The index of the current class in :py:attr:`labels`

Returns

log_class_posterior : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

Unnormalized log probability of the label at index `class_idx`

for each example in `X`

"""  # noqa: E501

P = self.parameters

mu = P["mean"][class_idx]

prior = P["prior"][class_idx]

sigsq = P["sigma"][class_idx]

# log likelihood = log X | N(mu, sigsq)

log_likelihood = -0.5 * np.sum(np.log(2 * np.pi * sigsq))

log_likelihood -= 0.5 * np.sum(((X - mu) ** 2) / sigsq, axis=1)

return log_likelihood + np.log(prior)


"""A module of Bayesian linear regression models."""

import numpy as np

import scipy.stats as stats

from numpy_ml.utils.testing import is_number, is_symmetric_positive_definite

class BayesianLinearRegressionUnknownVariance:

def __init__(self, alpha=1, beta=2, mu=0, V=None, fit_intercept=True):

r"""

Bayesian linear regression model with unknown variance. Assumes a

conjugate normal-inverse-gamma joint prior on the model parameters and

error variance.

Notes

The current model uses a conjugate normal-inverse-gamma joint prior on

model parameters **b** and error variance :math:`\sigma^2`. The joint

and marginal posteriors over each are:

.. math::

\mathbf{b}, \sigma^2 &\sim

\text{N-\Gamma^{-1}}(\mu, \mathbf{V}^{-1}, \alpha, \beta) \\

\sigma^2 &\sim \text{InverseGamma}(\alpha, \beta) \\

\mathbf{b} \mid \sigma^2 &\sim \mathcal{N}(\mu, \sigma^2 \mathbf{V})

Parameters

alpha : float

The shape parameter for the Inverse-Gamma prior on

:math:`\sigma^2`. Must be strictly greater than 0. Default is 1.

beta : float

The scale parameter for the Inverse-Gamma prior on

mu : :py:class:`ndarray <numpy.ndarray>` of shape `(M,)` or float

is ``np.ones(M) * mu``. Default is 0.

V : :py:class:`ndarray <numpy.ndarray>` of shape `(N, N)` or `(N,)` or None

A symmetric positive definite matrix that when multiplied

element-wise by :math:`\sigma^2` gives the covariance matrix for

the Gaussian prior on `b`. If a list, assume ``V = diag(V)``. If

None, assume `V` is the identity matrix.  Default is None.

fit_intercept : bool

Whether to fit an intercept term in addition to the coefficients in

b. If True, the estimates for b will have `M + 1` dimensions, where

the first dimension corresponds to the intercept. Default is True.

Attributes

posterior : dict or None

Frozen random variables for the posterior distributions

:math:`P(\sigma^2 \mid X)` and :math:`P(b \mid X, \sigma^2)`.

posterior_predictive : dict or None

Frozen random variable for the posterior predictive distribution,

:math:`P(y \mid X)`. This value is only set following a call to

:meth:`predict <numpy_ml.linear_models.BayesianLinearRegressionUnknownVariance.predict>`.

"""  # noqa: E501

# this is a placeholder until we know the dimensions of X

V = 1.0 if V is None else V

if isinstance(V, list):

V = np.array(V)

if isinstance(V, np.ndarray):

if V.ndim == 1:

V = np.diag(V)

elif V.ndim == 2:

fstr = "V must be symmetric positive definite"

assert is_symmetric_positive_definite(V), fstr

self.V = V

self.mu = mu

self.beta = beta

self.alpha = alpha

self.fit_intercept = fit_intercept

self.posterior = None

self.posterior_predictive = None

def fit(self, X, y):

Compute the posterior over model parameters using the data in `X` and

`y`.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, M)`

A dataset consisting of `N` examples, each of dimension `M`.

y : :py:class:`ndarray <numpy.ndarray>` of shape `(N, K)`

The targets for each of the `N` examples in `X`, where each target

has dimension `K`.

Returns

self : :class:`BayesianLinearRegressionUnknownVariance<numpy_ml.linear_models.BayesianLinearRegressionUnknownVariance>` instance

"""  # noqa: E501

# convert X to a design matrix if we're fitting an intercept

if self.fit_intercept:

X = np.c_[np.ones(X.shape[0]), X]

N, M = X.shape

alpha, beta, V, mu = self.alpha, self.beta, self.V, self.mu

if is_number(V):

V *= np.eye(M)

if is_number(mu):

mu *= np.ones(M)

# sigma

I = np.eye(N)  # noqa: E741

a = y - (X @ mu)

b = np.linalg.inv(X @ V @ X.T + I)

c = y - (X @ mu)

shape = N + alpha

sigma = (1 / shape) * (alpha * beta ** 2 + a @ b @ c)

scale = sigma ** 2

# sigma is the mode of the inverse gamma prior on sigma^2

sigma = scale / (shape - 1)

# mean

V_inv = np.linalg.inv(V)

L = np.linalg.inv(V_inv + X.T @ X)

R = V_inv @ mu + X.T @ y

mu = L @ R

cov = L * sigma

# posterior distribution for sigma^2 and b

self.posterior = {

"sigma**2": stats.distributions.invgamma(a=shape, scale=scale),

"b | sigma**2": stats.multivariate_normal(mean=mu, cov=cov),

return self

def predict(self, X):

Return the MAP prediction for the targets associated with `X`.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(Z, M)`

A dataset consisting of `Z` new examples, each of dimension `M`.

Returns

y_pred : :py:class:`ndarray <numpy.ndarray>` of shape `(Z, K)`

The model predictions for the items in `X`.

if self.fit_intercept:

X = np.c_[np.ones(X.shape[0]), X]

I = np.eye(X.shape[0])  # noqa: E741

mu = X @ self.posterior["b | sigma**2"].mean

cov = X @ self.posterior["b | sigma**2"].cov @ X.T + I

# MAP estimate for y corresponds to the mean of the posterior

# predictive

self.posterior_predictive = stats.multivariate_normal(mu, cov)

return mu

class BayesianLinearRegressionKnownVariance:

def __init__(self, mu=0, sigma=1, V=None, fit_intercept=True):

r"""

Bayesian linear regression model with known error variance and

conjugate Gaussian prior on model parameters.

Notes

Uses a conjugate Gaussian prior on the model coefficients **b**. The

posterior over model coefficients is then

.. math::

\mathbf{b} \mid \mu, \sigma^2, \mathbf{V}

\sim \mathcal{N}(\mu, \sigma^2 \mathbf{V})

Ridge regression is a special case of this model where :math:`\mu =

\mathbf{0}`, :math:`\sigma = 1` and :math:`\mathbf{V} = \mathbf{I}`

(ie., the prior on the model coefficients **b** is a zero-mean, unit

covariance Gaussian).

Parameters

The mean of the Gaussian prior on `b`. If a float, assume `mu` is

``np.ones(M) * mu``. Default is 0.

sigma : float

The square root of the scaling term for covariance of the Gaussian

prior on `b`. Default is 1.

V : :py:class:`ndarray <numpy.ndarray>` of shape `(N,N)` or `(N,)` or None

A symmetric positive definite matrix that when multiplied

element-wise by ``sigma ** 2`` gives the covariance matrix for the

Gaussian prior on `b`. If a list, assume ``V = diag(V)``. If None,

assume `V` is the identity matrix. Default is None.

fit_intercept : bool

`b`. If True, the estimates for `b` will have `M + 1` dimensions, where

Attributes

posterior : dict or None

Frozen random variable for the posterior distribution :math:`P(b

\mid X, \sigma^2)`.

posterior_predictive : dict or None

:meth:`predict <numpy_ml.linear_models.BayesianLinearRegressionKnownVariance.predict>`.

"""  # noqa: E501

# this is a placeholder until we know the dimensions of X

V = 1.0 if V is None else V

if isinstance(V, list):

V = np.array(V)

if isinstance(V, np.ndarray):

if V.ndim == 1:

V = np.diag(V)

elif V.ndim == 2:

fstr = "V must be symmetric positive definite"

assert is_symmetric_positive_definite(V), fstr

self.posterior = {}

self.posterior_predictive = {}

self.V = V

self.mu = mu

self.sigma = sigma

self.fit_intercept = fit_intercept

def fit(self, X, y):

`y`.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, M)`

y : :py:class:`ndarray <numpy.ndarray>` of shape `(N, K)`

has dimension `K`.

if self.fit_intercept:

X = np.c_[np.ones(X.shape[0]), X]

N, M = X.shape

if is_number(self.V):

self.V *= np.eye(M)

if is_number(self.mu):

self.mu *= np.ones(M)

V = self.V

mu = self.mu

sigma = self.sigma

V_inv = np.linalg.inv(V)

L = np.linalg.inv(V_inv + X.T @ X)

R = V_inv @ mu + X.T @ y

mu = L @ R

cov = L * sigma ** 2

# posterior distribution over b conditioned on sigma

self.posterior["b"] = stats.multivariate_normal(mu, cov)

def predict(self, X):

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(Z, M)`

Returns

The MAP predictions for the targets associated with the items in

`X`.

if self.fit_intercept:

X = np.c_[np.ones(X.shape[0]), X]

I = np.eye(X.shape[0])  # noqa: E741

mu = X @ self.posterior["b"].mean

cov = X @ self.posterior["b"].cov @ X.T + I

# MAP estimate for y corresponds to the mean/mode of the gaussian

# posterior predictive distribution

return mu


"""Linear regression module."""

import numpy as np

class LinearRegression:

def __init__(self, fit_intercept=True):

r"""

A weighted linear least-squares regression model.

Notes

In weighted linear least-squares regression [1]_, a real-valued target

vector, **y**, is modeled as a linear combination of covariates, **X**,

and model coefficients, :math:`\beta`:

.. math::

y_i = \beta^\top \mathbf{x}_i + \epsilon_i

In this equation :math:`\epsilon_i \sim \mathcal{N}(0, \sigma^2_i)` is

the error term associated with example :math:`i`, and

:math:`\sigma^2_i` is the variance of the corresponding example.

Under this model, the maximum-likelihood estimate for the regression

coefficients, :math:`\beta`, is:

.. math::

\hat{\beta} = \Sigma^{-1} \mathbf{X}^\top \mathbf{Wy}

where :math:`\Sigma^{-1} = (\mathbf{X}^\top \mathbf{WX})^{-1}` and

**W** is a diagonal matrix of weights, with each entry inversely

proportional to the variance of the corresponding measurement. When

**W** is the identity matrix the examples are weighted equally and the

model reduces to standard linear least squares [2]_.

References

.. [1] https://en.wikipedia.org/wiki/Weighted_least_squares

.. [2] https://en.wikipedia.org/wiki/General_linear_model

Parameters

fit_intercept : bool

Whether to fit an intercept term in addition to the model

coefficients. Default is True.

Attributes

beta : :py:class:`ndarray <numpy.ndarray>` of shape `(M, K)` or None

Fitted model coefficients.

sigma_inv : :py:class:`ndarray <numpy.ndarray>` of shape `(N, N)` or None

Inverse of the data covariance matrix.

self.beta = None

self.sigma_inv = None

self.fit_intercept = fit_intercept

self._is_fit = False

def update(self, X, y, weights=None):

r"""

Incrementally update the linear least-squares coefficients for a set of

new examples.

Notes

The recursive least-squares algorithm [3]_ [4]_ is used to efficiently

update the regression parameters as new examples become available. For

a single new example :math:`(\mathbf{x}_{t+1}, \mathbf{y}_{t+1})`, the

parameter updates are

.. math::

\beta_{t+1} = \left(

\mathbf{X}_{1:t}^\top \mathbf{X}_{1:t} +

\mathbf{x}_{t+1}\mathbf{x}_{t+1}^\top \right)^{-1}

\mathbf{X}_{1:t}^\top \mathbf{Y}_{1:t} +

\mathbf{x}_{t+1}^\top \mathbf{y}_{t+1}

where :math:`\beta_{t+1}` are the updated regression coefficients,

:math:`\mathbf{X}_{1:t}` and :math:`\mathbf{Y}_{1:t}` are the set of

examples observed from timestep 1 to *t*.

In the single-example case, the RLS algorithm uses the Sherman-Morrison

formula [5]_ to avoid re-inverting the covariance matrix on each new

update. In the multi-example case (i.e., where :math:`\mathbf{X}_{t+1}`

and :math:`\mathbf{y}_{t+1}` are matrices of `N` examples each), we use

the generalized Woodbury matrix identity [6]_ to update the inverse

covariance. This comes at a performance cost, but is still more

performant than doing multiple single-example updates if *N* is large.

References

.. [3] Gauss, C. F. (1821) *Theoria combinationis observationum

erroribus minimis obnoxiae*, Werke, 4. Gottinge

.. [4] https://en.wikipedia.org/wiki/Recursive_least_squares_filter

.. [5] https://en.wikipedia.org/wiki/Sherman%E2%80%93Morrison_formula

.. [6] https://en.wikipedia.org/wiki/Woodbury_matrix_identity

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, M)`

A dataset consisting of `N` examples, each of dimension `M`

y : :py:class:`ndarray <numpy.ndarray>` of shape `(N, K)`

The targets for each of the `N` examples in `X`, where each target

has dimension `K`

weights : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)` or None

Weights associated with the examples in `X`. Examples

with larger weights exert greater influence on model fit.  When

`y` is a vector (i.e., `K = 1`), weights should be set to the

reciporical of the variance for each measurement (i.e., :math:`w_i

= 1/\sigma^2_i`). When `K > 1`, it is assumed that all columns of

`y` share the same weight :math:`w_i`. If None, examples are

weighted equally, resulting in the standard linear least squares

update.  Default is None.

Returns

self : :class:`LinearRegression <numpy_ml.linear_models.LinearRegression>` instance

"""  # noqa: E501

if not self._is_fit:

raise RuntimeError("You must call the `fit` method before calling `update`")

X, y = np.atleast_2d(X), np.atleast_2d(y)

X1, Y1 = X.shape[0], y.shape[0]

weights = np.ones(X1) if weights is None else np.atleast_1d(weights)

weights = np.squeeze(weights) if weights.size > 1 else weights

err_str = f"weights must have shape ({X1},) but got {weights.shape}"

assert weights.shape == (X1,), err_str

# scale X and y by the weight associated with each example

W = np.diag(np.sqrt(weights))

X, y = W @ X, W @ y

self._update1D(X, y, W) if X1 == Y1 == 1 else self._update2D(X, y, W)

return self

def _update1D(self, x, y, w):

"""Sherman-Morrison update for a single example"""

beta, S_inv = self.beta, self.sigma_inv

# convert x to a design vector if we're fitting an intercept

if self.fit_intercept:

x = np.c_[np.diag(w), x]

# update the inverse of the covariance matrix via Sherman-Morrison

S_inv -= (S_inv @ x.T @ x @ S_inv) / (1 + x @ S_inv @ x.T)

# update the model coefficients

beta += S_inv @ x.T @ (y - x @ beta)

def _update2D(self, X, y, W):

"""Woodbury update for multiple examples"""

beta, S_inv = self.beta, self.sigma_inv

# convert X to a design matrix if we're fitting an intercept

if self.fit_intercept:

X = np.c_[np.diag(W), X]

I = np.eye(X.shape[0])  # noqa: E741

# update the inverse of the covariance matrix via Woodbury identity

S_inv -= S_inv @ X.T @ np.linalg.pinv(I + X @ S_inv @ X.T) @ X @ S_inv

# update the model coefficients

beta += S_inv @ X.T @ (y - X @ beta)

def fit(self, X, y, weights=None):

r"""

Fit regression coefficients via maximum likelihood.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, M)`

A dataset consisting of `N` examples, each of dimension `M`.

y : :py:class:`ndarray <numpy.ndarray>` of shape `(N, K)`

has dimension `K`.

Weights associated with the examples in `X`. Examples

update.  Default is None.

Returns

"""  # noqa: E501

N = X.shape[0]

weights = np.ones(N) if weights is None else np.atleast_1d(weights)

err_str = f"weights must have shape ({N},) but got {weights.shape}"

assert weights.shape == (N,), err_str

# scale X and y by the weight associated with each example

W = np.diag(np.sqrt(weights))

X, y = W @ X, W @ y

if self.fit_intercept:

X = np.c_[np.sqrt(weights), X]

self.sigma_inv = np.linalg.pinv(X.T @ X)

self.beta = np.atleast_2d(self.sigma_inv @ X.T @ y)

self._is_fit = True

return self

def predict(self, X):

Use the trained model to generate predictions on a new collection of

data points.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(Z, M)`

A dataset consisting of `Z` new examples, each of dimension `M`.

Returns

y_pred : :py:class:`ndarray <numpy.ndarray>` of shape `(Z, K)`

The model predictions for the items in `X`.

if self.fit_intercept:

X = np.c_[np.ones(X.shape[0]), X]

return X @ self.beta


"""Ridge regression module"""

import numpy as np

class RidgeRegression:

def __init__(self, alpha=1, fit_intercept=True):

r"""

A ridge regression model with maximum likelihood fit via the normal

equations.

Notes

Ridge regression is a biased estimator for linear models which adds an

additional penalty proportional to the L2-norm of the model

coefficients to the standard mean-squared-error loss:

.. math::

\mathcal{L}_{Ridge} = (\mathbf{y} - \mathbf{X} \beta)^\top

(\mathbf{y} - \mathbf{X} \beta) + \alpha ||\beta||_2^2

where :math:`\alpha` is a weight controlling the severity of the

penalty.

Given data matrix **X** and target vector **y**, the maximum-likelihood

estimate for ridge coefficients, :math:`\beta`, is:

.. math::

\hat{\beta} =

\left(\mathbf{X}^\top \mathbf{X} + \alpha \mathbf{I} \right)^{-1}

\mathbf{X}^\top \mathbf{y}

It turns out that this estimate for :math:`\beta` also corresponds to

the MAP estimate if we assume a multivariate Gaussian prior on the

model coefficients, assuming that the data matrix **X** has been

standardized and the target values **y** centered at 0:

.. math::

\beta \sim \mathcal{N}\left(\mathbf{0}, \frac{1}{2M} \mathbf{I}\right)

Parameters

alpha : float

L2 regularization coefficient. Larger values correspond to larger

penalty on the L2 norm of the model coefficients. Default is 1.

fit_intercept : bool

Whether to fit an additional intercept term. Default is True.

Attributes

beta : :py:class:`ndarray <numpy.ndarray>` of shape `(M, K)` or None

Fitted model coefficients.

self.beta = None

self.alpha = alpha

self.fit_intercept = fit_intercept

def fit(self, X, y):

Fit the regression coefficients via maximum likelihood.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, M)`

A dataset consisting of `N` examples, each of dimension `M`.

y : :py:class:`ndarray <numpy.ndarray>` of shape `(N, K)`

The targets for each of the `N` examples in `X`, where each target

has dimension `K`.

Returns

self : :class:`RidgeRegression <numpy_ml.linear_models.RidgeRegression>` instance

"""  # noqa: E501

# convert X to a design matrix if we're fitting an intercept

if self.fit_intercept:

X = np.c_[np.ones(X.shape[0]), X]

A = self.alpha * np.eye(X.shape[1])

pseudo_inverse = np.linalg.inv(X.T @ X + A) @ X.T

self.beta = pseudo_inverse @ y

return self

def predict(self, X):

Use the trained model to generate predictions on a new collection of

data points.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(Z, M)`

A dataset consisting of `Z` new examples, each of dimension `M`.

Returns

y_pred : :py:class:`ndarray <numpy.ndarray>` of shape `(Z, K)`

The model predictions for the items in `X`.

if self.fit_intercept:

X = np.c_[np.ones(X.shape[0]), X]

return np.dot(X, self.beta)


# flake8: noqa

import numpy as np

import matplotlib.pyplot as plt

import seaborn as sns

# https://seaborn.pydata.org/generated/seaborn.set_context.html

# https://seaborn.pydata.org/generated/seaborn.set_style.html

sns.set_context("paper", font_scale=1)

from numpy_ml.lda import LDA

def generate_corpus():

# Generate some fake data

D = 300

T = 10

V = 30

N = np.random.randint(150, 200, size=D)

# Create a document-topic distribution for 3 different types of documents

alpha1 = np.array((20, 15, 10, 1, 1, 1, 1, 1, 1, 1))

alpha2 = np.array((1, 1, 1, 10, 15, 20, 1, 1, 1, 1))

alpha3 = np.array((1, 1, 1, 1, 1, 1, 10, 12, 15, 18))

# Arbitrarily choose each topic to have 3 very common, diagnostic words

# These words are barely shared with any other topic

beta_probs = (

np.ones((V, T)) + np.array([np.arange(V) % T == t for t in range(T)]).T * 19

beta_gen = np.array(list(map(lambda x: np.random.dirichlet(x), beta_probs.T))).T

corpus = []

theta = np.empty((D, T))

# Generate each document from the LDA model

for d in range(D):

# Draw topic distribution for the document

if d < (D / 3):

theta[d, :] = np.random.dirichlet(alpha1, 1)[0]

elif d < 2 * (D / 3):

theta[d, :] = np.random.dirichlet(alpha2, 1)[0]

else:

theta[d, :] = np.random.dirichlet(alpha3, 1)[0]

doc = np.array([])

for n in range(N[d]):

# Draw a topic according to the document's topic distribution

z_n = np.random.choice(np.arange(T), p=theta[d, :])

# Draw a word according to the topic-word distribution

w_n = np.random.choice(np.arange(V), p=beta_gen[:, z_n])

doc = np.append(doc, w_n)

corpus.append(doc)

return corpus, T

def plot_unsmoothed():

corpus, T = generate_corpus()

L = LDA(T)

L.train(corpus, verbose=False)

fig, axes = plt.subplots(1, 2)

ax1 = sns.heatmap(L.beta, xticklabels=[], yticklabels=[], ax=axes[0])

ax1.set_title("Recovered topic-word distribution")

ax2 = sns.heatmap(L.gamma, xticklabels=[], yticklabels=[], ax=axes[1])

ax2.set_title("Recovered document-topic distribution")

plt.savefig("img/plot_unsmoothed.png", dpi=300)

plt.close("all")


# flake8: noqa

import numpy as np

from sklearn.datasets.samples_generator import make_blobs

from scipy.stats import multivariate_normal

import matplotlib.pyplot as plt

import seaborn as sns

# https://seaborn.pydata.org/generated/seaborn.set_context.html

# https://seaborn.pydata.org/generated/seaborn.set_style.html

sns.set_context("paper", font_scale=1)

from numpy_ml.gmm import GMM

from matplotlib.colors import ListedColormap

def plot_countour(X, x, y, z, ax, xlim, ylim):

def fixed_aspect_ratio(ratio, ax):

Set a fixed aspect ratio on matplotlib plots

regardless of axis units

xvals, yvals = ax.get_xlim(), ax.get_ylim()

xrange = xvals[1] - xvals[0]

yrange = yvals[1] - yvals[0]

ax.set_aspect(ratio * (xrange / yrange), adjustable="box")

# contour the gridded data, plotting dots at the randomly spaced data points.

ax.contour(x, y, z, 6, linewidths=0.5, colors="k")

fixed_aspect_ratio(1, ax)

return ax

def plot_clusters(model, X, ax):

C = model.C

xmin = min(X[:, 0]) - 0.1 * (max(X[:, 0]) - min(X[:, 0]))

xmax = max(X[:, 0]) + 0.1 * (max(X[:, 0]) - min(X[:, 0]))

ymin = min(X[:, 1]) - 0.1 * (max(X[:, 1]) - min(X[:, 1]))

ymax = max(X[:, 1]) + 0.1 * (max(X[:, 1]) - min(X[:, 1]))

for c in range(C):

rv = multivariate_normal(model.mu[c], model.sigma[c], allow_singular=True)

x = np.linspace(xmin, xmax, 500)

y = np.linspace(ymin, ymax, 500)

X1, Y1 = np.meshgrid(x, y)

xy = np.column_stack([X1.flat, Y1.flat])

# density values at the grid points

Z = rv.pdf(xy).reshape(X1.shape)

ax = plot_countour(X, X1, Y1, Z, ax=ax, xlim=(xmin, xmax), ylim=(ymin, ymax))

ax.plot(model.mu[c, 0], model.mu[c, 1], "ro")

# plot data points

cm = ListedColormap(sns.color_palette().as_hex())

labels = model.Q.argmax(1)

uniq = set(labels)

for i in uniq:

ax.scatter(X[labels == i, 0], X[labels == i, 1], c=cm.colors[i - 1], s=30)

return ax

def plot():

fig, axes = plt.subplots(4, 4)

fig.set_size_inches(10, 10)

for i, ax in enumerate(axes.flatten()):

n_ex = 150

n_in = 2

n_classes = np.random.randint(2, 4)

X, y = make_blobs(

n_samples=n_ex, centers=n_classes, n_features=n_in, random_state=i

X -= X.mean(axis=0)

# take best fit over 10 runs

best_elbo = -np.inf

for k in range(10):

_G = GMM(C=n_classes, seed=k * 3)

ret = _G.fit(X, max_iter=100, verbose=False)

while ret != 0:

print("Components collapsed; Refitting")

if _G.best_elbo > best_elbo:

best_elbo = _G.best_elbo

G = _G

ax = plot_clusters(G, X, ax)

ax.set_title("# Classes: {}; Final VLB: {:.2f}".format(n_classes, G.best_elbo))

plt.savefig("img/plot.png", dpi=300)

plt.close("all")


# flake8: noqa

import numpy as np

import matplotlib.pyplot as plt

import seaborn as sns

# https://seaborn.pydata.org/generated/seaborn.set_context.html

# https://seaborn.pydata.org/generated/seaborn.set_style.html

sns.set_context("notebook", font_scale=0.7)

from numpy_ml.neural_nets.activations import (

Affine,

ReLU,

LeakyReLU,

Tanh,

Sigmoid,

ELU,

Exponential,

SELU,

HardSigmoid,

SoftPlus,

def plot_activations():

fig, axes = plt.subplots(2, 5, sharex=True, sharey=True)

fns = [

Affine(),

Tanh(),

Sigmoid(),

ReLU(),

LeakyReLU(),

ELU(),

Exponential(),

SELU(),

HardSigmoid(),

SoftPlus(),

for ax, fn in zip(axes.flatten(), fns):

X = np.linspace(-3, 3, 100).astype(float).reshape(100, 1)

ax.plot(X, fn(X), label=r"$y$", alpha=1.0)

ax.plot(X, fn.grad(X), label=r"$\frac{dy}{dx}$", alpha=1.0)

ax.plot(X, fn.grad2(X), label=r"$\frac{d^2 y}{dx^2}$", alpha=1.0)

ax.hlines(0, -3, 3, lw=1, linestyles="dashed", color="k")

ax.vlines(0, -1.2, 1.2, lw=1, linestyles="dashed", color="k")

ax.set_ylim(-1.1, 1.1)

ax.set_xlim(-3, 3)

ax.set_yticks([-1, 0, 1])

#  ax.yaxis.set_visible(False)

ax.legend(frameon=False)

sns.despine(left=True, bottom=True)

fig.set_size_inches(10, 5)

plt.savefig("img/plot.png", dpi=300)

plt.close("all")

if __name__ == "__main__":


# flake8: noqa

import time

import numpy as np

import matplotlib.pyplot as plt

import seaborn as sns

from numpy_ml.neural_nets.schedulers import (

ConstantScheduler,

ExponentialScheduler,

NoamScheduler,

KingScheduler,

def king_loss_fn(x):

if x <= 250:

return -0.25 * x + 82.50372665317208

elif 250 < x <= 600:

return 20.00372665317208

elif 600 < x <= 700:

return -0.2 * x + 140.00372665317207

else:

return 0.003726653172066108

def plot_schedulers():

fig, axes = plt.subplots(2, 2)

schedulers = [

[ConstantScheduler(lr=0.01), "lr=1e-2"],

[ConstantScheduler(lr=0.008), "lr=8e-3"],

[ConstantScheduler(lr=0.006), "lr=6e-3"],

[ConstantScheduler(lr=0.004), "lr=4e-3"],

[ConstantScheduler(lr=0.002), "lr=2e-3"],

ExponentialScheduler(

lr=0.01, stage_length=250, staircase=False, decay=0.4

"lr=0.01, stage=250, stair=False, decay=0.4",

ExponentialScheduler(

lr=0.01, stage_length=250, staircase=True, decay=0.4

"lr=0.01, stage=250, stair=True, decay=0.4",

ExponentialScheduler(

lr=0.01, stage_length=125, staircase=True, decay=0.1

"lr=0.01, stage=125, stair=True, decay=0.1",

ExponentialScheduler(

lr=0.001, stage_length=250, staircase=False, decay=0.1

"lr=0.001, stage=250, stair=False, decay=0.1",

ExponentialScheduler(

lr=0.001, stage_length=125, staircase=False, decay=0.8

"lr=0.001, stage=125, stair=False, decay=0.8",

ExponentialScheduler(

lr=0.01, stage_length=250, staircase=False, decay=0.01

"lr=0.01, stage=250, stair=False, decay=0.01",

NoamScheduler(model_dim=512, scale_factor=1, warmup_steps=250),

"dim=512, scale=1, warmup=250",

NoamScheduler(model_dim=256, scale_factor=1, warmup_steps=250),

"dim=256, scale=1, warmup=250",

NoamScheduler(model_dim=512, scale_factor=1, warmup_steps=500),

"dim=512, scale=1, warmup=500",

NoamScheduler(model_dim=256, scale_factor=1, warmup_steps=500),

"dim=512, scale=1, warmup=500",

NoamScheduler(model_dim=512, scale_factor=2, warmup_steps=500),

"dim=512, scale=2, warmup=500",

NoamScheduler(model_dim=512, scale_factor=0.5, warmup_steps=500),

"dim=512, scale=0.5, warmup=500",

#      KingScheduler(initial_lr=0.01, patience=100, decay=0.1),

#      "lr=0.01, patience=100, decay=0.8",

#      KingScheduler(initial_lr=0.01, patience=300, decay=0.999),

#      "lr=0.01, patience=300, decay=0.999",

KingScheduler(initial_lr=0.009, patience=150, decay=0.995),

"lr=0.009, patience=150, decay=0.9999",

KingScheduler(initial_lr=0.008, patience=100, decay=0.995),

"lr=0.008, patience=100, decay=0.995",

KingScheduler(initial_lr=0.007, patience=50, decay=0.995),

"lr=0.007, patience=50, decay=0.995",

KingScheduler(initial_lr=0.005, patience=25, decay=0.9),

"lr=0.005, patience=25, decay=0.99",

for ax, schs, title in zip(

axes.flatten(), schedulers, ["Constant", "Exponential", "Noam", "King"]

t0 = time.time()

print("Running {} scheduler".format(title))

X = np.arange(1, 1000)

loss = np.array([king_loss_fn(x) for x in X])

# scale loss to fit on same axis as lr

scale = 0.01 / loss[0]

loss *= scale

if title == "King":

ax.plot(X, loss, ls=":", label="Loss")

for sc, lg in schs:

Y = np.array([sc(x, ll) for x, ll in zip(X, loss)])

ax.plot(X, Y, label=lg, alpha=0.6)

ax.set_ylabel("Learning rate")

ax.set_title("{} scheduler".format(title))

print(

"Finished plotting {} runs of {} in {:.2f}s".format(

len(schs), title, time.time() - t0

plt.savefig("plot.png", dpi=300)

plt.close("all")

if __name__ == "__main__":


# flake8: noqa

import numpy as np

from matplotlib import pyplot as plt

import seaborn as sns

# https://seaborn.pydata.org/generated/seaborn.set_context.html

# https://seaborn.pydata.org/generated/seaborn.set_style.html

sns.set_context("notebook", font_scale=0.8)

from hmmlearn.hmm import MultinomialHMM as MHMM

from numpy_ml.hmm import MultinomialHMM

def generate_training_data(params, n_steps=500, n_examples=15):

hmm = MultinomialHMM(A=params["A"], B=params["B"], pi=params["pi"])

# generate a new sequence

observations = []

for i in range(n_examples):

latent, obs = hmm.generate(

n_steps, params["latent_states"], params["obs_types"]

assert len(latent) == len(obs) == n_steps

observations.append(obs)

observations = np.array(observations)

return observations

def default_hmm():

obs_types = [0, 1, 2, 3]

latent_states = ["H", "C"]

# derived variables

V = len(obs_types)

N = len(latent_states)

# define a very simple HMM with T=3 observations

O = np.array([1, 3, 1]).reshape(1, -1)

A = np.array([[0.9, 0.1], [0.5, 0.5]])

B = np.array([[0.2, 0.7, 0.09, 0.01], [0.1, 0.0, 0.8, 0.1]])

pi = np.array([0.75, 0.25])

return {

"latent_states": latent_states,

"obs_types": obs_types,

"V": V,

"N": N,

"O": O,

"A": A,

"B": B,

"pi": pi,

def plot_matrices(params, best, best_theirs):

cmap = "copper"

ll_mine, best = best

ll_theirs, best_theirs = best_theirs

fig, axes = plt.subplots(3, 3)

axes = {

"A": [axes[0, 0], axes[0, 1], axes[0, 2]],

"B": [axes[1, 0], axes[1, 1], axes[1, 2]],

"pi": [axes[2, 0], axes[2, 1], axes[2, 2]],

for k, tt in [("A", "Transition"), ("B", "Emission"), ("pi", "Prior")]:

true_ax, est_ax, est_theirs_ax = axes[k]

true, est, est_theirs = params[k], best[k], best_theirs[k]

if k == "pi":

true = true.reshape(-1, 1)

est = est.reshape(-1, 1)

est_theirs = est_theirs.reshape(-1, 1)

true_ax = sns.heatmap(

true,

cmap=cmap,

cbar=False,

annot=True,

xticklabels=[],

yticklabels=[],

est_ax = sns.heatmap(

est,

cmap=cmap,

annot=True,

cbar=False,

xticklabels=[],

yticklabels=[],

est_theirs_ax = sns.heatmap(

cmap=cmap,

annot=True,

cbar=False,

xticklabels=[],

yticklabels=[],

true_ax.set_title("{} (True)".format(tt))

est_ax.set_title("{} (Mine)".format(tt))

est_theirs_ax.set_title("{} (hmmlearn)".format(tt))

fig.suptitle("LL (mine): {:.2f}, LL (hmmlearn): {:.2f}".format(ll_mine, ll_theirs))

plt.tight_layout(rect=[0, 0.03, 1, 0.95])

plt.savefig("img/plot.png", dpi=300)

plt.close()

def test_HMM():

np.set_printoptions(precision=5, suppress=True)

P = default_hmm()

ls, obs = P["latent_states"], P["obs_types"]

# generate a new sequence

O = generate_training_data(P, n_steps=30, n_examples=25)

tol = 1e-5

n_runs = 5

best, best_theirs = (-np.inf, []), (-np.inf, [])

for _ in range(n_runs):

hmm = MultinomialHMM()

A_, B_, pi_ = hmm.fit(O, ls, obs, tol=tol, verbose=True)

theirs = MHMM(

tol=tol,

verbose=True,

algorithm="viterbi",

O_flat = O.reshape(1, -1).flatten().reshape(-1, 1)

theirs = theirs.fit(O_flat, lengths=[O.shape[1]] * O.shape[0])

hmm2 = MultinomialHMM(A=A_, B=B_, pi=pi_)

like = np.sum([hmm2.log_likelihood(obs) for obs in O])

like_theirs = theirs.score(O_flat, lengths=[O.shape[1]] * O.shape[0])

if like > best[0]:

best = (like, {"A": A_, "B": B_, "pi": pi_})

if like_theirs > best_theirs[0]:

best_theirs = (

"A": theirs.transmat_,

"B": theirs.emissionprob_,

"pi": theirs.startprob_,

print("Final log likelihood of sequence: {:.5f}".format(best[0]))

print("Final log likelihood of sequence (theirs): {:.5f}".format(best_theirs[0]))

plot_matrices(P, best, best_theirs)


# flake8: noqa

import numpy as np

from sklearn.metrics import accuracy_score, mean_squared_error

from sklearn.datasets import make_blobs, make_regression

from sklearn.model_selection import train_test_split

import matplotlib.pyplot as plt

# https://seaborn.pydata.org/generated/seaborn.set_context.html

# https://seaborn.pydata.org/generated/seaborn.set_style.html

import seaborn as sns

sns.set_context("paper", font_scale=0.9)

from numpy_ml.trees import GradientBoostedDecisionTree, DecisionTree, RandomForest

def plot():

fig, axes = plt.subplots(4, 4)

fig.set_size_inches(10, 10)

for ax in axes.flatten():

n_ex = 100

n_trees = 50

n_feats = np.random.randint(2, 100)

max_depth_d = np.random.randint(1, 100)

max_depth_r = np.random.randint(1, 10)

classifier = np.random.choice([True, False])

if classifier:

# create classification problem

n_classes = np.random.randint(2, 10)

X, Y = make_blobs(n_samples=n_ex, centers=n_classes, n_features=2)

X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.3)

n_feats = min(n_feats, X.shape[1])

# initialize model

def loss(yp, y):

return accuracy_score(yp, y)

# initialize model

criterion = np.random.choice(["entropy", "gini"])

mine = RandomForest(

classifier=classifier,

criterion=criterion,

mine_d = DecisionTree(

criterion=criterion, max_depth=max_depth_d, classifier=classifier

mine_g = GradientBoostedDecisionTree(

classifier=classifier,

loss="crossentropy",

else:

# create regeression problem

X, Y = make_regression(n_samples=n_ex, n_features=1)

n_feats = min(n_feats, X.shape[1])

# initialize model

criterion = "mse"

loss = mean_squared_error

mine = RandomForest(

criterion=criterion,

classifier=classifier,

mine_d = DecisionTree(

mine_g = GradientBoostedDecisionTree(

classifier=classifier,

loss="mse",

# fit 'em

mine.fit(X, Y)

mine_d.fit(X, Y)

mine_g.fit(X, Y)

# get preds on test set

y_pred_mine_test = mine.predict(X_test)

y_pred_mine_test_d = mine_d.predict(X_test)

y_pred_mine_test_g = mine_g.predict(X_test)

loss_mine_test = loss(y_pred_mine_test, Y_test)

loss_mine_test_d = loss(y_pred_mine_test_d, Y_test)

loss_mine_test_g = loss(y_pred_mine_test_g, Y_test)

if classifier:

entries = [

("RF", loss_mine_test, y_pred_mine_test),

("DT", loss_mine_test_d, y_pred_mine_test_d),

("GB", loss_mine_test_g, y_pred_mine_test_g),

(lbl, test_loss, preds) = entries[np.random.randint(3)]

ax.set_title("{} Accuracy: {:.2f}%".format(lbl, test_loss * 100))

for i in np.unique(Y_test):

ax.scatter(

X_test[preds == i, 0].flatten(),

X_test[preds == i, 1].flatten(),

#  s=0.5,

else:

X_ax = np.linspace(

np.min(X_test.flatten()) - 1, np.max(X_test.flatten()) + 1, 100

).reshape(-1, 1)

y_pred_mine_test = mine.predict(X_ax)

y_pred_mine_test_d = mine_d.predict(X_ax)

y_pred_mine_test_g = mine_g.predict(X_ax)

ax.scatter(X_test.flatten(), Y_test.flatten(), c="b", alpha=0.5)

#  s=0.5)

ax.plot(

#  linewidth=0.5,

label="GB".format(n_trees, n_feats, max_depth_d),

color="red",

ax.plot(

#  linewidth=0.5,

label="RF".format(n_trees, n_feats, max_depth_r),

color="cornflowerblue",

ax.plot(

#  linewidth=0.5,

color="yellowgreen",

"GB: {:.1f} / RF: {:.1f} / DT: {:.1f} ".format(

loss_mine_test_g, loss_mine_test, loss_mine_test_d

ax.legend()

plt.savefig("plot.png", dpi=300)

plt.close("all")


# flake8: noqa

import numpy as np

import matplotlib.pyplot as plt

import seaborn as sns

# https://seaborn.pydata.org/generated/seaborn.set_context.html

# https://seaborn.pydata.org/generated/seaborn.set_style.html

sns.set_context("paper", font_scale=0.5)

from numpy_ml.nonparametric import GPRegression, KNN, KernelRegression

from numpy_ml.linear_models.lm import LinearRegression

from sklearn.model_selection import train_test_split

def random_regression_problem(n_ex, n_in, n_out, d=3, intercept=0, std=1, seed=0):

coef = np.random.uniform(0, 50, size=d)

coef[-1] = intercept

y = []

X = np.random.uniform(-100, 100, size=(n_ex, n_in))

for x in X:

val = np.polyval(coef, x) + np.random.normal(0, std)

y.append(val)

y = np.array(y)

X_train, X_test, y_train, y_test = train_test_split(

X, y, test_size=0.3, random_state=seed

return X_train, y_train, X_test, y_test, coef

def plot_regression():

fig, axes = plt.subplots(4, 4)

for i, ax in enumerate(axes.flatten()):

n_in = 1

n_out = 1

d = np.random.randint(1, 5)

n_ex = np.random.randint(5, 500)

std = np.random.randint(0, 1000)

intercept = np.random.rand() * np.random.randint(-300, 300)

X_train, y_train, X_test, y_test, coefs = random_regression_problem(

n_ex, n_in, n_out, d=d, intercept=intercept, std=std, seed=i

LR = LinearRegression(fit_intercept=True)

LR.fit(X_train, y_train)

y_pred = LR.predict(X_test)

loss = np.mean((y_test.flatten() - y_pred.flatten()) ** 2)

d = 3

best_loss = np.inf

for gamma in np.linspace(1e-10, 1, 100):

for c0 in np.linspace(-1, 1000, 100):

kernel = "PolynomialKernel(d={}, gamma={}, c0={})".format(d, gamma, c0)

KR_poly = KernelRegression(kernel=kernel)

KR_poly.fit(X_train, y_train)

y_pred_poly = KR_poly.predict(X_test)

loss_poly = np.mean((y_test.flatten() - y_pred_poly.flatten()) ** 2)

if loss_poly <= best_loss:

KR_poly_best = kernel

best_loss = loss_poly

print("Best kernel: {} || loss: {:.4f}".format(KR_poly_best, best_loss))

KR_poly = KernelRegression(kernel=KR_poly_best)

KR_poly.fit(X_train, y_train)

KR_rbf = KernelRegression(kernel="RBFKernel(sigma=1)")

KR_rbf.fit(X_train, y_train)

y_pred_rbf = KR_rbf.predict(X_test)

loss_rbf = np.mean((y_test.flatten() - y_pred_rbf.flatten()) ** 2)

xmin = min(X_test) - 0.1 * (max(X_test) - min(X_test))

xmax = max(X_test) + 0.1 * (max(X_test) - min(X_test))

X_plot = np.linspace(xmin, xmax, 100)

y_plot = LR.predict(X_plot)

y_plot_poly = KR_poly.predict(X_plot)

y_plot_rbf = KR_rbf.predict(X_plot)

ax.scatter(X_test, y_test, alpha=0.5)

ax.plot(X_plot, y_plot, label="OLS", alpha=0.5)

ax.plot(

X_plot, y_plot_poly, label="KR (poly kernel, d={})".format(d), alpha=0.5

ax.plot(X_plot, y_plot_rbf, label="KR (rbf kernel)", alpha=0.5)

ax.legend()

#  ax.set_title(

#      "MSE\nLR: {:.2f} KR (poly): {:.2f}\nKR (rbf): {:.2f}".format(

#          loss, loss_poly, loss_rbf

plt.savefig("img/kr_plots.png", dpi=300)

plt.close("all")

def plot_knn():

fig, axes = plt.subplots(4, 4)

for i, ax in enumerate(axes.flatten()):

n_in = 1

n_out = 1

d = np.random.randint(1, 5)

n_ex = np.random.randint(5, 500)

std = np.random.randint(0, 1000)

LR.fit(X_train, y_train)

y_pred = LR.predict(X_test)

knn_1 = KNN(k=1, classifier=False, leaf_size=10, weights="uniform")

knn_1.fit(X_train, y_train)

y_pred_1 = knn_1.predict(X_test)

loss_1 = np.mean((y_test.flatten() - y_pred_1.flatten()) ** 2)

knn_5 = KNN(k=5, classifier=False, leaf_size=10, weights="uniform")

knn_5.fit(X_train, y_train)

y_pred_5 = knn_5.predict(X_test)

loss_5 = np.mean((y_test.flatten() - y_pred_5.flatten()) ** 2)

knn_10 = KNN(k=10, classifier=False, leaf_size=10, weights="uniform")

knn_10.fit(X_train, y_train)

y_pred_10 = knn_10.predict(X_test)

loss_10 = np.mean((y_test.flatten() - y_pred_10.flatten()) ** 2)

X_plot = np.linspace(xmin, xmax, 100)

y_plot = LR.predict(X_plot)

y_plot_1 = knn_1.predict(X_plot)

y_plot_5 = knn_5.predict(X_plot)

y_plot_10 = knn_10.predict(X_plot)

ax.scatter(X_test, y_test, alpha=0.5)

ax.plot(X_plot, y_plot_1, label="KNN (k=1)", alpha=0.5)

ax.plot(X_plot, y_plot_5, label="KNN (k=5)", alpha=0.5)

ax.plot(X_plot, y_plot_10, label="KNN (k=10)", alpha=0.5)

ax.legend()

#  ax.set_title(

#          loss, loss_poly, loss_rbf

plt.savefig("img/knn_plots.png", dpi=300)

plt.close("all")

def plot_gp():

sns.set_context("paper", font_scale=0.65)

X_test = np.linspace(-10, 10, 100)

X_train = np.array([-3, 0, 7, 1, -9])

y_train = np.sin(X_train)

fig, axes = plt.subplots(2, 2)

alphas = [0, 1e-10, 1e-5, 1]

for ix, (ax, alpha) in enumerate(zip(axes.flatten(), alphas)):

G = GPRegression(kernel="RBFKernel", alpha=alpha)

G.fit(X_train, y_train)

y_pred, conf = G.predict(X_test)

ax.plot(X_train, y_train, "rx", label="observed")

ax.plot(X_test, np.sin(X_test), label="true fn")

ax.plot(X_test, y_pred, "--", label="MAP (alpha={})".format(alpha))

ax.fill_between(X_test, y_pred + conf, y_pred - conf, alpha=0.1)

sns.despine()

ax.legend()

plt.savefig("img/gp_alpha.png", dpi=300)

plt.close("all")

def plot_gp_dist():

sns.set_context("paper", font_scale=0.95)

X_test = np.linspace(-10, 10, 100)

X_train = np.array([-3, 0, 7, 1, -9])

y_train = np.sin(X_train)

fig, axes = plt.subplots(1, 3)

G = GPRegression(kernel="RBFKernel", alpha=0)

G.fit(X_train, y_train)

y_pred_prior = G.sample(X_test, 3, "prior")

y_pred_posterior = G.sample(X_test, 3, "posterior_predictive")

for prior_sample in y_pred_prior:

axes[0].plot(X_test, prior_sample.ravel(), lw=1)

axes[0].set_title("Prior samples")

for post_sample in y_pred_posterior:

axes[1].plot(X_test, post_sample.ravel(), lw=1)

axes[1].plot(X_train, y_train, "ko", ms=1.2)

axes[1].set_title("Posterior samples")

y_pred, conf = G.predict(X_test)

axes[2].plot(X_test, np.sin(X_test), lw=1, label="true function")

axes[2].plot(X_test, y_pred, lw=1, label="MAP estimate")

axes[2].fill_between(X_test, y_pred + conf, y_pred - conf, alpha=0.1)

axes[2].plot(X_train, y_train, "ko", ms=1.2, label="observed")

axes[2].set_title("Posterior mean")

fig.set_size_inches(6, 2)

plt.savefig("img/gp_dist.png", dpi=300)

plt.close("all")


"""Miscellaneous plots for multi-arm bandit validation"""

from collections import namedtuple

import numpy as np

from numpy_ml.bandits import (

MultinomialBandit,

BernoulliBandit,

ShortestPathBandit,

ContextualLinearBandit,

from numpy_ml.bandits.trainer import BanditTrainer

from numpy_ml.bandits.policies import (

EpsilonGreedy,

ThompsonSamplingBetaBinomial,

LinUCB,

from numpy_ml.utils.graphs import random_DAG, DiGraph, Edge

def random_multinomial_mab(n_arms=10, n_choices_per_arm=5, reward_range=[0, 1]):

"""Generate a random multinomial multi-armed bandit environemt"""

payoffs = []

payoff_probs = []

lo, hi = reward_range

for a in range(n_arms):

p = np.random.uniform(size=n_choices_per_arm)

p = p / p.sum()

r = np.random.uniform(low=lo, high=hi, size=n_choices_per_arm)

payoffs.append(list(r))

return MultinomialBandit(payoffs, payoff_probs)

def random_bernoulli_mab(n_arms=10):

"""Generate a random Bernoulli multi-armed bandit environemt"""

p = np.random.uniform(size=n_arms)

payoff_probs = p / p.sum()

return BernoulliBandit(payoff_probs)

def plot_epsilon_greedy_multinomial_payoff():

Evaluate an epsilon-greedy policy on a random multinomial bandit

problem

N = np.random.randint(2, 30)  # n arms

K = np.random.randint(2, 10)  # n payoffs / arm

ep_length = 1

rrange = [0, 1]

n_duplicates = 5

n_episodes = 5000

mab = random_multinomial_mab(N, K, rrange)

policy = EpsilonGreedy(epsilon=0.05, ev_prior=rrange[1] / 2)

policy = BanditTrainer().train(policy, mab, ep_length, n_episodes, n_duplicates)

def plot_ucb1_multinomial_payoff():

"""Evaluate the UCB1 policy on a multinomial bandit environment"""

N = np.random.randint(2, 30)  # n arms

ep_length = 1

C = 1

rrange = [0, 1]

n_duplicates = 5

n_episodes = 5000

policy = UCB1(C=C, ev_prior=rrange[1] / 2)

def plot_thompson_sampling_beta_binomial_payoff():

Evaluate the ThompsonSamplingBetaBinomial policy on a random Bernoulli

multi-armed bandit.

N = np.random.randint(2, 30)  # n arms

ep_length = 1

n_duplicates = 5

n_episodes = 5000

mab = random_bernoulli_mab(N)

policy = ThompsonSamplingBetaBinomial(alpha=1, beta=1)

def plot_lin_ucb():

"""Plot the linUCB policy on a contextual linear bandit problem"""

ep_length = 1

K = np.random.randint(2, 25)

D = np.random.randint(2, 10)

n_duplicates = 5

n_episodes = 5000

cmab = ContextualLinearBandit(K, D, 1)

policy = LinUCB(alpha=1)

policy = BanditTrainer().train(policy, cmab, ep_length, n_episodes, n_duplicates)

def plot_ucb1_gaussian_shortest_path():

Plot the UCB1 policy on a graph shortest path problem each edge weight

drawn from an independent univariate Gaussian

ep_length = 1

n_duplicates = 5

n_episodes = 5000

p = np.random.rand()

n_vertices = np.random.randint(5, 15)

Gaussian = namedtuple("Gaussian", ["mean", "variance", "EV", "sample"])

# create randomly-weighted edges

print("Building graph")

E = []

G = random_DAG(n_vertices, p)

V = G.vertices

for e in G.edges:

mean, var = np.random.uniform(0, 1), np.random.uniform(0, 1)

w = lambda: np.random.normal(mean, var)  # noqa: E731

rv = Gaussian(mean, var, mean, w)

E.append(Edge(e.fr, e.to, rv))

G = DiGraph(V, E)

while not G.path_exists(V[0], V[-1]):

print("Skipping")

idx = np.random.randint(0, len(V))

V[idx], V[-1] = V[-1], V[idx]

mab = ShortestPathBandit(G, V[0], V[-1])

policy = UCB1(C=1, ev_prior=0.5)

def plot_comparison():

Use the BanditTrainer to compare several policies on the same bandit

problem

ep_length = 1

K = 10

n_duplicates = 5

n_episodes = 5000

cmab = random_bernoulli_mab(n_arms=K)

policy1 = EpsilonGreedy(epsilon=0.05, ev_prior=0.5)

policy2 = UCB1(C=1, ev_prior=0.5)

policy3 = ThompsonSamplingBetaBinomial(alpha=1, beta=1)

policies = [policy1, policy2, policy3]

BanditTrainer().compare(

policies, cmab, ep_length, n_episodes, n_duplicates,


# flake8: noqa

import gym

from numpy_ml.rl_models.trainer import Trainer

from numpy_ml.rl_models.agents import (

CrossEntropyAgent,

MonteCarloAgent,

TemporalDifferenceAgent,

DynaAgent,

def test_cross_entropy_agent():

seed = 12345

max_steps = 300

n_episodes = 50

retain_prcnt = 0.2

n_samples_per_episode = 500

env = gym.make("LunarLander-v2")

agent = CrossEntropyAgent(env, n_samples_per_episode, retain_prcnt)

trainer = Trainer(agent, env)

trainer.train(

n_episodes, max_steps, seed=seed, plot=True, verbose=True, render_every=None

def test_monte_carlo_agent():

seed = 12345

max_steps = 300

n_episodes = 10000

epsilon = 0.05

off_policy = True

smooth_factor = 0.001

temporal_discount = 0.95

env = gym.make("Copy-v0")

agent = MonteCarloAgent(env, off_policy, temporal_discount, epsilon)

trainer = Trainer(agent, env)

trainer.train(

seed=seed,

plot=True,

verbose=True,

def test_temporal_difference_agent():

seed = 12345

max_steps = 200

n_episodes = 5000

lr = 0.4

n_tilings = 10

epsilon = 0.10

off_policy = True

grid_dims = [100, 100]

smooth_factor = 0.005

temporal_discount = 0.999

env = gym.make("LunarLander-v2")

obs_max = 1

obs_min = -1

agent = TemporalDifferenceAgent(

env,

lr=lr,

epsilon=epsilon,

trainer = Trainer(agent, env)

trainer.train(

seed=seed,

plot=True,

verbose=True,

def test_dyna_agent():

seed = 12345

max_steps = 200

n_episodes = 150

lr = 0.4

q_plus = False

n_tilings = 10

epsilon = 0.10

grid_dims = [10, 10]

smooth_factor = 0.01

temporal_discount = 0.99

explore_weight = 0.05

n_simulated_actions = 25

obs_max, obs_min = 1, -1

env = gym.make("Taxi-v2")

agent = DynaAgent(

env,

lr=lr,

epsilon=epsilon,

n_simulated_actions=n_simulated_actions,

trainer = Trainer(agent, env)

trainer.train(

seed=seed,

plot=True,

verbose=True,


# flake8: noqa

import numpy as np

import matplotlib.pyplot as plt

import seaborn as sns

# https://seaborn.pydata.org/generated/seaborn.set_context.html

# https://seaborn.pydata.org/generated/seaborn.set_style.html

sns.set_context("notebook", font_scale=1)

from numpy_ml.ngram import MLENGram, AdditiveNGram, GoodTuringNGram

def plot_count_models(GT, N):

NC = GT._num_grams_with_count

mod = GT._count_models[N]

max_n = max(GT.counts[N].values())

emp = [NC(n + 1, N) for n in range(max_n)]

prd = [np.exp(mod.predict(np.array([n + 1]))) for n in range(max_n + 10)]

plt.scatter(range(max_n), emp, c="r", label="actual")

plt.plot(range(max_n + 10), prd, "-", label="model")

plt.ylim([-1, 100])

plt.xlabel("Count ($r$)")

plt.ylabel("Count-of-counts ($N_r$)")

plt.legend()

plt.savefig("test.png")

plt.close()

def compare_probs(fp, N):

MLE = MLENGram(N, unk=False, filter_punctuation=False, filter_stopwords=False)

MLE.train(fp, encoding="utf-8-sig")

add_y, mle_y, gtt_y = [], [], []

addu_y, mleu_y, gttu_y = [], [], []

seen = ("<bol>", "the")

unseen = ("<bol>", "asdf")

GTT = GoodTuringNGram(

N, conf=1.96, unk=False, filter_stopwords=False, filter_punctuation=False

GTT.train(fp, encoding="utf-8-sig")

gtt_prob = GTT.log_prob(seen, N)

gtt_prob_u = GTT.log_prob(unseen, N)

for K in np.linspace(0, 10, 20):

ADD = AdditiveNGram(

N, K, unk=False, filter_punctuation=False, filter_stopwords=False

ADD.train(fp, encoding="utf-8-sig")

add_prob = ADD.log_prob(seen, N)

mle_prob = MLE.log_prob(seen, N)

mle_prob_u = MLE.log_prob(unseen, N)

add_prob_u = ADD.log_prob(unseen, N)

plt.plot(np.linspace(0, 10, 20), add_y, label="Additive (seen ngram)")

plt.plot(np.linspace(0, 10, 20), addu_y, label="Additive (unseen ngram)")

#  plt.plot(np.linspace(0, 10, 20), gtt_y, label="Good-Turing (seen ngram)")

#  plt.plot(np.linspace(0, 10, 20), gttu_y, label="Good-Turing (unseen ngram)")

plt.plot(np.linspace(0, 10, 20), mle_y, "--", label="MLE (seen ngram)")

plt.xlabel("K")

plt.ylabel("log P(sequence)")

plt.legend()

plt.close("all")

def plot_gt_freqs(fp):

Draws a scatterplot of the empirical frequencies of the counted species

versus their Simple Good Turing smoothed values, in rank order. Depends on

pylab and matplotlib.

MLE = MLENGram(1, filter_punctuation=False, filter_stopwords=False)

MLE.train(fp, encoding="utf-8-sig")

counts = dict(MLE.counts[1])

GT = GoodTuringNGram(1, filter_stopwords=False, filter_punctuation=False)

GT.train(fp, encoding="utf-8-sig")

ADD = AdditiveNGram(1, 1, filter_punctuation=False, filter_stopwords=False)

ADD.train(fp, encoding="utf-8-sig")

tot = float(sum(counts.values()))

freqs = dict([(token, cnt / tot) for token, cnt in counts.items()])

sgt_probs = dict([(tok, np.exp(GT.log_prob(tok, 1))) for tok in counts.keys()])

as_probs = dict([(tok, np.exp(ADD.log_prob(tok, 1))) for tok in counts.keys()])

X, Y = np.arange(len(freqs)), sorted(freqs.values(), reverse=True)

plt.loglog(X, Y, "k+", alpha=0.25, label="MLE")

X, Y = np.arange(len(sgt_probs)), sorted(sgt_probs.values(), reverse=True)

plt.loglog(X, Y, "r+", alpha=0.25, label="simple Good-Turing")

X, Y = np.arange(len(as_probs)), sorted(as_probs.values(), reverse=True)

plt.loglog(X, Y, "b+", alpha=0.25, label="Laplace smoothing")

plt.xlabel("Rank")

plt.ylabel("Probability")

plt.legend()

plt.close("all")


# flake8: noqa

import numpy as np

from sklearn.model_selection import train_test_split

from sklearn.datasets.samples_generator import make_blobs

from sklearn.linear_model import LogisticRegression as LogisticRegression_sk

from sklearn.datasets import make_regression

from sklearn.metrics import zero_one_loss, r2_score

import matplotlib.pyplot as plt

import seaborn as sns

# https://seaborn.pydata.org/generated/seaborn.set_context.html

# https://seaborn.pydata.org/generated/seaborn.set_style.html

sns.set_context("paper", font_scale=0.5)

from numpy_ml.linear_models import (

RidgeRegression,

LinearRegression,

BayesianLinearRegressionKnownVariance,

BayesianLinearRegressionUnknownVariance,

LogisticRegression,

#                           Data Generators                           #

def random_binary_tensor(shape, sparsity=0.5):

X = (np.random.rand(*shape) >= (1 - sparsity)).astype(float)

return X

def random_regression_problem(n_ex, n_in, n_out, intercept=0, std=1, seed=0):

X, y, coef = make_regression(

bias=intercept,

noise=std,

coef=True,

X_train, X_test, y_train, y_test = train_test_split(

X, y, test_size=0.3, random_state=seed

return X_train, y_train, X_test, y_test, coef

def random_classification_problem(n_ex, n_classes, n_in, seed=0):

X, y = make_blobs(

n_samples=n_ex, centers=n_classes, n_features=n_in, random_state=seed

X, y, test_size=0.3, random_state=seed

return X_train, y_train, X_test, y_test

#                                Plots                                #

def plot_logistic():

fig, axes = plt.subplots(4, 4)

for i, ax in enumerate(axes.flatten()):

n_in = 1

n_ex = 150

X_train, y_train, X_test, y_test = random_classification_problem(

n_ex, n_classes=2, n_in=n_in, seed=i

LR = LogisticRegression(penalty="l2", gamma=0.2, fit_intercept=True)

LR.fit(X_train, y_train, lr=0.1, tol=1e-7, max_iter=1e7)

y_pred = (LR.predict(X_test) >= 0.5) * 1.0

loss = zero_one_loss(y_test, y_pred) * 100.0

LR_sk = LogisticRegression_sk(

penalty="l2", tol=0.0001, C=0.8, fit_intercept=True, random_state=i

LR_sk.fit(X_train, y_train)

y_pred_sk = (LR_sk.predict(X_test) >= 0.5) * 1.0

loss_sk = zero_one_loss(y_test, y_pred_sk) * 100.0

xmin = min(X_test) - 0.1 * (max(X_test) - min(X_test))

xmax = max(X_test) + 0.1 * (max(X_test) - min(X_test))

X_plot = np.linspace(xmin, xmax, 100)

y_plot = LR.predict(X_plot)

y_plot_sk = LR_sk.predict_proba(X_plot.reshape(-1, 1))[:, 1]

ax.scatter(X_test[y_pred == 0], y_test[y_pred == 0], alpha=0.5)

ax.scatter(X_test[y_pred == 1], y_test[y_pred == 1], alpha=0.5)

ax.plot(X_plot, y_plot, label="mine", alpha=0.75)

ax.plot(X_plot, y_plot_sk, label="sklearn", alpha=0.75)

ax.legend()

ax.set_title("Loss mine: {:.2f} Loss sklearn: {:.2f}".format(loss, loss_sk))

plt.savefig("plot_logistic.png", dpi=300)

plt.close("all")

def plot_bayes():

n_in = 1

n_out = 1

n_ex = 20

std = 15

intercept = 10

X_train, y_train, X_test, y_test, coefs = random_regression_problem(

n_ex, n_in, n_out, intercept=intercept, std=std, seed=0

# add some outliers

x1, x2 = X_train[0] + 0.5, X_train[6] - 0.3

y1 = np.dot(x1, coefs) + intercept + 25

y2 = np.dot(x2, coefs) + intercept - 31

X_train = np.vstack([X_train, np.array([x1, x2])])

y_train = np.hstack([y_train, [y1[0], y2[0]]])

LR = LinearRegression(fit_intercept=True)

LR.fit(X_train, y_train)

y_pred = LR.predict(X_test)

loss = np.mean((y_test - y_pred) ** 2)

ridge = RidgeRegression(alpha=1, fit_intercept=True)

ridge.fit(X_train, y_train)

y_pred = ridge.predict(X_test)

loss_ridge = np.mean((y_test - y_pred) ** 2)

LR_var = BayesianLinearRegressionKnownVariance(

mu=np.c_[intercept, coefs][0], sigma=np.sqrt(std), V=None, fit_intercept=True,

LR_var.fit(X_train, y_train)

y_pred_var = LR_var.predict(X_test)

loss_var = np.mean((y_test - y_pred_var) ** 2)

LR_novar = BayesianLinearRegressionUnknownVariance(

LR_novar.fit(X_train, y_train)

y_pred_novar = LR_novar.predict(X_test)

loss_novar = np.mean((y_test - y_pred_novar) ** 2)

X_plot = np.linspace(xmin, xmax, 100)

y_plot = LR.predict(X_plot)

y_plot_ridge = ridge.predict(X_plot)

y_plot_var = LR_var.predict(X_plot)

y_plot_novar = LR_novar.predict(X_plot)

y_true = [np.dot(x, coefs) + intercept for x in X_plot]

fig, axes = plt.subplots(1, 4)

axes = axes.flatten()

axes[0].scatter(X_test, y_test)

axes[0].plot(X_plot, y_plot, label="MLE")

axes[0].plot(X_plot, y_true, label="True fn")

axes[0].set_title("Linear Regression\nMLE Test MSE: {:.2f}".format(loss))

#  axes[0].fill_between(X_plot, y_plot - error, y_plot + error)

axes[1].scatter(X_test, y_test)

axes[1].plot(X_plot, y_plot_ridge, label="MLE")

axes[1].plot(X_plot, y_true, label="True fn")

"Ridge Regression (alpha=1)\nMLE Test MSE: {:.2f}".format(loss_ridge)

axes[2].plot(X_plot, y_plot_var, label="MAP")

mu, cov = LR_var.posterior["b"].mean, LR_var.posterior["b"].cov

for k in range(200):

b_samp = np.random.multivariate_normal(mu, cov)

y_samp = [np.dot(x, b_samp[1]) + b_samp[0] for x in X_plot]

axes[2].plot(X_plot, y_samp, alpha=0.05)

axes[2].scatter(X_test, y_test)

axes[2].plot(X_plot, y_true, label="True fn")

"Bayesian Regression (known variance)\nMAP Test MSE: {:.2f}".format(loss_var)

axes[3].plot(X_plot, y_plot_novar, label="MAP")

mu = LR_novar.posterior["b | sigma**2"].mean

cov = LR_novar.posterior["b | sigma**2"].cov

for k in range(200):

axes[3].plot(X_plot, y_samp, alpha=0.05)

axes[3].scatter(X_test, y_test)

axes[3].plot(X_plot, y_true, label="True fn")

"Bayesian Regression (unknown variance)\nMAP Test MSE: {:.2f}".format(

for ax in axes:

fig.set_size_inches(7.5, 1.875)

plt.savefig("plot_bayes.png", dpi=300)

plt.close("all")

def plot_regression():

fig, axes = plt.subplots(4, 4)

for i, ax in enumerate(axes.flatten()):

n_in = 1

n_out = 1

n_ex = 50

std = np.random.randint(0, 100)

intercept = np.random.rand() * np.random.randint(-300, 300)

n_ex, n_in, n_out, intercept=intercept, std=std, seed=i

LR = LinearRegression(fit_intercept=True)

LR.fit(X_train, y_train)

y_pred = LR.predict(X_test)

loss = np.mean((y_test - y_pred) ** 2)

r2 = r2_score(y_test, y_pred)

mu=np.c_[intercept, coefs][0],

sigma=np.sqrt(std),

V=None,

LR_var.fit(X_train, y_train)

y_pred_var = LR_var.predict(X_test)

loss_var = np.mean((y_test - y_pred_var) ** 2)

r2_var = r2_score(y_test, y_pred_var)

alpha=1, beta=2, mu=np.c_[intercept, coefs][0], V=None, fit_intercept=True,

LR_novar.fit(X_train, y_train)

y_pred_novar = LR_novar.predict(X_test)

r2_novar = r2_score(y_test, y_pred_novar)

X_plot = np.linspace(xmin, xmax, 100)

y_plot = LR.predict(X_plot)

y_plot_var = LR_var.predict(X_plot)

y_plot_novar = LR_novar.predict(X_plot)

ax.scatter(X_test, y_test, marker="x", alpha=0.5)

ax.plot(X_plot, y_plot, label="linear regression", alpha=0.5)

ax.plot(X_plot, y_plot_var, label="Bayes (w var)", alpha=0.5)

ax.plot(X_plot, y_plot_novar, label="Bayes (no var)", alpha=0.5)

ax.legend()

"MSE\nLR: {:.2f} Bayes (w var): {:.2f}\nBayes (no var): {:.2f}".format(

loss, loss_var, loss_novar

plt.savefig("plot_regression.png", dpi=300)

plt.close("all")