import numpy as np

from numpy.lib.stride_tricks import as_strided

from ..utils.windows import WindowInitializer

#                          Signal Resampling                          #

def batch_resample(X, new_dim, mode="bilinear"):

Resample each image (or similar grid-based 2D signal) in a batch to

`new_dim` using the specified resampling strategy.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, in_rows, in_cols, in_channels)`

An input image volume

new_dim : 2-tuple of `(out_rows, out_cols)`

The dimension to resample each image to

mode : {'bilinear', 'neighbor'}

The resampling strategy to employ. Default is 'bilinear'.

Returns

resampled : :py:class:`ndarray <numpy.ndarray>` of shape `(n_ex, out_rows, out_cols, in_channels)`

The resampled image volume.

if mode == "bilinear":

interpolate = bilinear_interpolate

elif mode == "neighbor":

interpolate = nn_interpolate_2D

else:

raise NotImplementedError("Unrecognized resampling mode: {}".format(mode))

out_rows, out_cols = new_dim

n_ex, in_rows, in_cols, n_in = X.shape

# compute coordinates to resample

x = np.tile(np.linspace(0, in_cols - 2, out_cols), out_rows)

y = np.repeat(np.linspace(0, in_rows - 2, out_rows), out_cols)

# resample each image

resampled = []

for i in range(n_ex):

r = interpolate(X[i, ...], x, y)

r = r.reshape(out_rows, out_cols, n_in)

resampled.append(r)

return np.dstack(resampled)

def nn_interpolate_2D(X, x, y):

Estimates of the pixel values at the coordinates (x, y) in `X` using a

nearest neighbor interpolation strategy.

Notes

Assumes the current entries in `X` reflect equally-spaced samples from a 2D

integer grid.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(in_rows, in_cols, in_channels)`

An input image sampled along a grid of `in_rows` by `in_cols`.

x : list of length `k`

A list of x-coordinates for the samples we wish to generate

y : list of length `k`

A list of y-coordinates for the samples we wish to generate

Returns

samples : :py:class:`ndarray <numpy.ndarray>` of shape `(k, in_channels)`

The samples for each (x,y) coordinate computed via nearest neighbor

interpolation

nx, ny = np.around(x), np.around(y)

nx = np.clip(nx, 0, X.shape[1] - 1).astype(int)

ny = np.clip(ny, 0, X.shape[0] - 1).astype(int)

return X[ny, nx, :]

def nn_interpolate_1D(X, t):

Estimates of the signal values at `X[t]` using a nearest neighbor

interpolation strategy.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(in_length, in_channels)`

An input image sampled along an integer `in_length`

t : list of length `k`

A list of coordinates for the samples we wish to generate

Returns

interpolation

nt = np.clip(np.around(t), 0, X.shape[0] - 1).astype(int)

return X[nt, :]

def bilinear_interpolate(X, x, y):

Estimates of the pixel values at the coordinates (x, y) in `X` via bilinear

interpolation.

Notes

Assumes the current entries in X reflect equally-spaced

samples from a 2D integer grid.

Modified from https://bit.ly/2NMb1Dr

Parameters

x : list of length `k`

y : list of length `k`

Returns

samples : list of length `(k, in_channels)`

The samples for each (x,y) coordinate computed via bilinear

interpolation

x0 = np.floor(x).astype(int)

y0 = np.floor(y).astype(int)

x1 = x0 + 1

y1 = y0 + 1

x0 = np.clip(x0, 0, X.shape[1] - 1)

y0 = np.clip(y0, 0, X.shape[0] - 1)

x1 = np.clip(x1, 0, X.shape[1] - 1)

y1 = np.clip(y1, 0, X.shape[0] - 1)

Ia = X[y0, x0, :].T

Ib = X[y1, x0, :].T

Ic = X[y0, x1, :].T

Id = X[y1, x1, :].T

wa = (x1 - x) * (y1 - y)

wb = (x1 - x) * (y - y0)

wc = (x - x0) * (y1 - y)

wd = (x - x0) * (y - y0)

return (Ia * wa).T + (Ib * wb).T + (Ic * wc).T + (Id * wd).T

#                        Fourier Decomposition                        #

def DCT(frame, orthonormal=True):

A naive :math:`O(N^2)` implementation of the 1D discrete cosine transform-II

(DCT-II).

Notes

For a signal :math:`\mathbf{x} = [x_1, \ldots, x_N]` consisting of `N`

samples, the `k` th DCT coefficient, :math:`c_k`, is

.. math::

c_k = 2 \sum_{n=0}^{N-1} x_n \cos(\pi k (2 n + 1) / (2 N))

where `k` ranges from :math:`0, \ldots, N-1`.

The DCT is highly similar to the DFT -- whereas in a DFT the basis

functions are sinusoids, in a DCT they are restricted solely to cosines. A

signal's DCT representation tends to have more of its energy concentrated

in a smaller number of coefficients when compared to the DFT, and is thus

commonly used for signal compression. [1]

.. [1] Smoother signals can be accurately approximated using fewer DFT / DCT

coefficients, resulting in a higher compression ratio. The DCT naturally

yields a continuous extension at the signal boundaries due its use of

even basis functions (cosine). This in turn produces a smoother

extension in comparison to DFT or DCT approximations, resulting in a

higher compression.

Parameters

frame : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

A signal frame consisting of N samples

orthonormal : bool

Scale to ensure the coefficient vector is orthonormal. Default is True.

Returns

dct : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

The discrete cosine transform of the samples in `frame`.

N = len(frame)

out = np.zeros_like(frame)

for k in range(N):

for (n, xn) in enumerate(frame):

out[k] += xn * np.cos(np.pi * k * (2 * n + 1) / (2 * N))

scale = np.sqrt(1 / (4 * N)) if k == 0 else np.sqrt(1 / (2 * N))

out[k] *= 2 * scale if orthonormal else 2

return out

def __DCT2(frame):

"""Currently broken"""

N = len(frame)  # window length

k = np.arange(N, dtype=float)

F = k.reshape(1, -1) * k.reshape(-1, 1)

K = np.divide(F, k, out=np.zeros_like(F), where=F != 0)

FC = np.cos(F * np.pi / N + K * np.pi / 2 * N)

return 2 * (FC @ frame)

def DFT(frame, positive_only=True):

A naive :math:`O(N^2)` implementation of the 1D discrete Fourier transform (DFT).

Notes

The Fourier transform decomposes a signal into a linear combination of

sinusoids (ie., basis elements in the space of continuous periodic

functions).  For a sequence :math:`\mathbf{x} = [x_1, \ldots, x_N]` of N

evenly spaced samples, the `k` th DFT coefficient is given by:

.. math::

c_k = \sum_{n=0}^{N-1} x_n \exp(-2 \pi i k n / N)

where `i` is the imaginary unit, `k` is an index ranging from `0, ..., N-1`,

and :math:`X_k` is the complex coefficient representing the phase

(imaginary part) and amplitude (real part) of the `k` th sinusoid in the

DFT spectrum. The frequency of the `k` th sinusoid is :math:`(k 2 \pi / N)`

radians per sample.

When applied to a real-valued input, the negative frequency terms are the

complex conjugates of the positive-frequency terms and the overall spectrum

is symmetric (excluding the first index, which contains the zero-frequency

/ intercept term).

Parameters

frame : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

A signal frame consisting of N samples

positive_only : bool

Whether to only return the coefficients for the positive frequency

terms. Default is True.

Returns

spectrum : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)` or `(N // 2 + 1,)` if `real_only`

The coefficients of the frequency spectrum for `frame`, including

imaginary components.

N = len(frame)  # window length

# F[i,j] = coefficient for basis vector i, timestep j (i.e., k * n)

F = np.arange(N).reshape(1, -1) * np.arange(N).reshape(-1, 1)

F = np.exp(F * (-1j * 2 * np.pi / N))

# vdot only operates on vectors (rather than ndarrays), so we have to

# loop over each basis vector in F explicitly

spectrum = np.array([np.vdot(f, frame) for f in F])

return spectrum[: (N // 2) + 1] if positive_only else spectrum

def dft_bins(N, fs=44000, positive_only=True):

Calc the frequency bin centers for a DFT with `N` coefficients.

Parameters

N : int

The number of frequency bins in the DFT

fs : int

The sample rate/frequency of the signal (in Hz). Default is 44000.

positive_only : bool

Whether to only return the bins for the positive frequency

terms. Default is True.

Returns

bins : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)` or `(N // 2 + 1,)` if `positive_only`

The frequency bin centers associated with each coefficient in the

DFT spectrum

if positive_only:

freq_bins = np.linspace(0, fs / 2, 1 + N // 2, endpoint=True)

else:

l, r = (1 + (N - 1) / 2, (1 - N) / 2) if N % 2 else (N / 2, -N / 2)

freq_bins = np.r_[np.arange(l), np.arange(r, 0)] * fs / N

return freq_bins

def magnitude_spectrum(frames):

Compute the magnitude spectrum (i.e., absolute value of the DFT spectrum)

for each frame in `frames`. Assumes each frame is real-valued only.

Parameters

frames : :py:class:`ndarray <numpy.ndarray>` of shape `(M, N)`

A sequence of `M` frames each consisting of `N` samples

Returns

magnitude_spec : :py:class:`ndarray <numpy.ndarray>` of shape `(M, N // 2 + 1)`

The magnitude spectrum for each frame in `frames`. Only includes the

coefficients for the positive spectrum frequencies.

return np.vstack([np.abs(DFT(frame, positive_only=True)) for frame in frames])

def power_spectrum(frames, scale=False):

Compute the power spectrum for a signal represented as a collection of

frames. Assumes each frame is real-valued only.

The power spectrum is simply the square of the magnitude spectrum, possibly

scaled by the number of FFT bins. It measures how the energy of the signal

is distributed over the frequency domain.

Parameters

scale : bool

Whether the scale by the number of DFT bins. Default is False.

Returns

power_spec : :py:class:`ndarray <numpy.ndarray>` of shape `(M, N // 2 + 1)`

The power spectrum for each frame in `frames`. Only includes the

scaler = frames.shape[1] // 2 + 1 if scale else 1

return (1 / scaler) * magnitude_spectrum(frames) ** 2

#                       Preprocessing Utils                           #

def to_frames(x, frame_width, stride, writeable=False):

Convert a 1D signal x into overlapping windows of width `frame_width` using

a hop length of `stride`.

Notes

If ``(len(x) - frame_width) % stride != 0`` then some number of the samples

in x will be dropped. Specifically::

n_dropped_frames = len(x) - frame_width - stride * (n_frames - 1)

where::

n_frames = (len(x) - frame_width) // stride + 1

This method uses low-level stride manipulation to avoid creating an

additional copy of `x`. The downside is that if ``writeable`=True``,

modifying the `frame` output can result in unexpected behavior:

>>> out = to_frames(np.arange(6), 5, 1)

>>> out

array([[0, 1, 2, 3, 4],

[1, 2, 3, 4, 5]])

>>> out[0, 1] = 99

>>> out

array([[ 0, 99,  2,  3,  4],

[99,  2,  3,  4,  5]])

Parameters

x : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

A 1D signal consisting of N samples

frame_width : int

The width of a single frame window in samples

stride : int

The hop size / number of samples advanced between consecutive frames

writeable : bool

If set to False, the returned array will be readonly. Otherwise it will

be writable if `x` was. It is advisable to set this to False whenever

possible to avoid unexpected behavior (see NB 2 above). Default is False.

Returns

frame: :py:class:`ndarray <numpy.ndarray>` of shape `(n_frames, frame_width)`

The collection of overlapping frames stacked into a matrix

assert x.ndim == 1

assert stride >= 1

assert len(x) >= frame_width

# get the size for an element in x in bits

byte = x.itemsize

return as_strided(

x,

shape=(n_frames, frame_width),

strides=(byte * stride, byte),

writeable=writeable,

def autocorrelate1D(x):

Autocorrelate a 1D signal `x` with itself.

Notes

The `k` th term in the 1 dimensional autocorrelation is

.. math::

a_k = \sum_n x_{n + k} x_n

NB. This is a naive :math:`O(N^2)` implementation.  For a faster :math:`O(N

\log N)` approach using the FFT, see [1].

References

.. [1] https://en.wikipedia.org/wiki/Autocorrelation#Efficient%computation

Parameters

A 1D signal consisting of N samples

Returns

auto : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

The autocorrelation of `x` with itself

N = len(x)

auto = np.zeros(N)

for k in range(N):

for n in range(N - k):

auto[k] += x[n + k] * x[n]

return auto

#                               Filters                               #

def preemphasis(x, alpha):

Increase the amplitude of high frequency bands + decrease the amplitude of

lower bands.

Notes

Preemphasis filtering is (was?) a common transform in speech processing,

where higher frequencies tend to be more useful during signal

disambiguation.

.. math::

\\text{preemphasis}( x_t ) = x_t - \\alpha x_{t-1}

Parameters

x : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

A 1D signal consisting of `N` samples

alpha : float in [0, 1)

The preemphasis coefficient. A value of 0 corresponds to no

filtering

Returns

out : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

The filtered signal

return np.concatenate([x[:1], x[1:] - alpha * x[:-1]])

def cepstral_lifter(mfccs, D):

A simple sinusoidal filter applied in the Mel-frequency domain.

Notes

Cepstral lifting helps to smooth the spectral envelope and dampen the

magnitude of the higher MFCC coefficients while keeping the other

coefficients unchanged. The filter function is:

.. math::

\\text{lifter}( x_n ) = x_n \left(1 + \\frac{D \sin(\pi n / D)}{2}\\right)

Parameters

mfccs : :py:class:`ndarray <numpy.ndarray>` of shape `(G, C)`

Matrix of Mel cepstral coefficients. Rows correspond to frames, columns

to cepstral coefficients

D : int in :math:`[0, +\infty]`

The filter coefficient. 0 corresponds to no filtering, larger values

correspond to greater amounts of smoothing

Returns

out : :py:class:`ndarray <numpy.ndarray>` of shape `(G, C)`

The lifter'd MFCC coefficients

if D == 0:

return mfccs

n = np.arange(mfccs.shape[1])

return mfccs * (1 + (D / 2) * np.sin(np.pi * n / D))

def mel_spectrogram(

x,

window="hamming",

center=True,

Apply the Mel-filterbank to the power spectrum for a signal `x`.

Notes

The Mel spectrogram is the projection of the power spectrum of the framed

and windowed signal onto the basis set provided by the Mel filterbank.

Parameters

x : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

A 1D signal consisting of N samples

window_duration : float

The duration of each frame / window (in seconds). Default is 0.025.

stride_duration : float

The duration of the hop between consecutive windows (in seconds).

Default is 0.01.

mean_normalize : bool

Whether to subtract the coefficient means from the final filter values

to improve the signal-to-noise ratio. Default is True.

window : {'hamming', 'hann', 'blackman_harris'}

The windowing function to apply to the signal before FFT. Default is

'hamming'.

n_filters : int

The number of mel filters to include in the filterbank. Default is 20.

center : bool

Whether to the `k` th frame of the signal should *begin* at index ``x[k *

stride_len]`` (center = False) or be *centered* at ``x[k * stride_len]``

(center = True). Default is False.

alpha : float in [0, 1)

The coefficient for the preemphasis filter. A value of 0 corresponds to

no filtering. Default is 0.95.

fs : int

The sample rate/frequency for the signal. Default is 44000.

Returns

filter_energies : :py:class:`ndarray <numpy.ndarray>` of shape `(G, n_filters)`

The (possibly mean_normalized) power for each filter in the Mel

filterbank (i.e., the Mel spectrogram). Rows correspond to frames,

columns to filters

energy_per_frame : :py:class:`ndarray <numpy.ndarray>` of shape `(G,)`

The total energy in each frame of the signal

eps = np.finfo(float).eps

window_fn = WindowInitializer()(window)

stride = round(stride_duration * fs)

frame_width = round(window_duration * fs)

N = frame_width

# add a preemphasis filter to the raw signal

x = preemphasis(x, alpha)

# convert signal to overlapping frames and apply a window function

x = np.pad(x, N // 2, "reflect") if center else x

frames = to_frames(x, frame_width, stride, fs)

window = np.tile(window_fn(frame_width), (frames.shape[0], 1))

frames = frames * window

# compute the power spectrum

power_spec = power_spectrum(frames)

energy_per_frame = np.sum(power_spec, axis=1)

energy_per_frame[energy_per_frame == 0] = eps

# compute the power at each filter in the Mel filterbank

fbank = mel_filterbank(N, n_filters=n_filters, fs=fs)

filter_energies = power_spec @ fbank.T

filter_energies -= np.mean(filter_energies, axis=0) if mean_normalize else 0

filter_energies[filter_energies == 0] = eps

return filter_energies, energy_per_frame

#                       Mel-Frequency Features                        #

def mfcc(

x,

center=True,

window="hann",

normalize=True,

Compute the Mel-frequency cepstral coefficients (MFCC) for a signal.

Notes

Computing MFCC features proceeds in the following stages:

1. Convert the signal into overlapping frames and apply a window fn

2. Compute the power spectrum at each frame

3. Apply the mel filterbank to the power spectra to get mel filterbank powers

4. Take the logarithm of the mel filterbank powers at each frame

5. Take the discrete cosine transform (DCT) of the log filterbank

energies and retain only the first k coefficients to further reduce

the dimensionality

MFCCs were developed in the context of HMM-GMM automatic speech recognition

(ASR) systems and can be used to provide a somewhat speaker/pitch

invariant representation of phonemes.

Parameters

x : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

A 1D signal consisting of N samples

fs : int

The sample rate/frequency for the signal. Default is 44000.

n_mfccs : int

The number of cepstral coefficients to return (including the intercept

coefficient). Default is 13.

alpha : float in [0, 1)

The preemphasis coefficient. A value of 0 corresponds to no

filtering. Default is 0.95.

center : bool

Whether to the kth frame of the signal should *begin* at index ``x[k *

(center = True). Default is True.

n_filters : int

The number of filters to include in the Mel filterbank. Default is 20.

normalize : bool

Whether to mean-normalize the MFCC values. Default is True.

lifter_coef : int in :math:[0, + \infty]`

The cepstral filter coefficient. 0 corresponds to no filtering, larger

values correspond to greater amounts of smoothing. Default is 22.

window : {'hamming', 'hann', 'blackman_harris'}

The windowing function to apply to the signal before taking the DFT.

Default is 'hann'.

stride_duration : float

The duration of the hop between consecutive windows (in seconds).

Default is 0.01.

window_duration : float

The duration of each frame / window (in seconds). Default is 0.025.

replace_intercept : bool

Replace the first MFCC coefficient (the intercept term) with the

log of the total frame energy instead. Default is True.

Returns

mfccs : :py:class:`ndarray <numpy.ndarray>` of shape `(G, C)`

Matrix of Mel-frequency cepstral coefficients. Rows correspond to

frames, columns to cepstral coefficients

# map the power spectrum for the (framed + windowed representation of) `x`

# onto the mel scale

filter_energies, frame_energies = mel_spectrogram(

x=x,

fs=fs,

alpha=alpha,

center=center,

window=window,

log_energies = 10 * np.log10(filter_energies)

# perform a DCT on the log-mel coefficients to further reduce the data

# dimensionality -- the early DCT coefficients will capture the majority of

# the data, allowing us to discard coefficients > n_mfccs

mfccs = np.array([DCT(frame) for frame in log_energies])[:, :n_mfccs]

mfccs = cepstral_lifter(mfccs, D=lifter_coef)

mfccs -= np.mean(mfccs, axis=0) if normalize else 0

if replace_intercept:

# the 0th MFCC coefficient doesn't tell us anything about the spectrum;

# replace it with the log of the frame energy for something more

# informative

mfccs[:, 0] = np.log(frame_energies)

return mfccs

def mel2hz(mel, formula="htk"):

Convert the mel-scale representation of a signal into Hz

Parameters

mel : :py:class:`ndarray <numpy.ndarray>` of shape `(N, \*)`

An array of mel frequencies to convert

formula : {"htk", "slaney"}

The Mel formula to use. "htk" uses the formula used by the Hidden

Markov Model Toolkit, and described in O'Shaughnessy (1987). "slaney"

uses the formula used in the MATLAB auditory toolbox (Slaney, 1998).

Default is 'htk'

Returns

hz : :py:class:`ndarray <numpy.ndarray>` of shape `(N, \*)`

The frequencies of the items in `mel`, in Hz

fstr = "formula must be either 'htk' or 'slaney' but got '{}'"

assert formula in ["htk", "slaney"], fstr.format(formula)

if formula == "htk":

return 700 * (10 ** (mel / 2595) - 1)

raise NotImplementedError("slaney")

def hz2mel(hz, formula="htk"):

Convert the frequency representaiton of a signal in Hz into the mel scale.

Parameters

formula : {"htk", "slaney"}

Default is 'htk'.

Returns

An array of mel frequencies to convert.

if formula == "htk":

return 2595 * np.log10(1 + hz / 700)

raise NotImplementedError("slaney")

def mel_filterbank(

N, n_filters=20, fs=44000, min_freq=0, max_freq=None, normalize=True

Compute the filters in a Mel filterbank and return the corresponding

transformation matrix

Notes

The Mel scale is a perceptual scale designed to simulate the way the human

ear works. Pitches judged by listeners to be equal in perceptual /

psychological distance have equal distance on the Mel scale.  Practically,

this corresponds to a scale with higher resolution at low frequencies and

lower resolution at higher (> 500 Hz) frequencies.

Each filter in the Mel filterbank is triangular with a response of 1 at its

center and a linear decay on both sides until it reaches the center

frequency of the next adjacent filter.

This implementation is based on code in the (superb) LibROSA package [1].

References

.. [1] McFee et al. (2015). "librosa: Audio and music signal analysis in

Python", *Proceedings of the 14th Python in Science Conference*

Parameters

N : int

The number of DFT bins

n_filters : int

The number of mel filters to include in the filterbank. Default is 20.

min_freq : int

Minimum filter frequency (in Hz). Default is 0.

max_freq : int

Maximum filter frequency (in Hz). Default is 0.

fs : int

The sample rate/frequency for the signal. Default is 44000.

normalize : bool

If True, scale the Mel filter weights by their area in Mel space.

Default is True.

Returns

fbank : :py:class:`ndarray <numpy.ndarray>` of shape `(n_filters, N // 2 + 1)`

The mel-filterbank transformation matrix. Rows correspond to filters,

columns to DFT bins.

max_freq = fs / 2 if max_freq is None else max_freq

min_mel, max_mel = hz2mel(min_freq), hz2mel(max_freq)

fbank = np.zeros((n_filters, N // 2 + 1))

# uniformly spaced values on the mel scale, translated back into Hz

mel_bins = mel2hz(np.linspace(min_mel, max_mel, n_filters + 2))

# the centers of the frequency bins for the DFT

hz_bins = dft_bins(N, fs)

mel_spacing = np.diff(mel_bins)

# ramps[i] = mel_bins[i] - hz_bins

ramps = mel_bins.reshape(-1, 1) - hz_bins.reshape(1, -1)

for i in range(n_filters):

# calc the filter values on the left and right across the bins ...

left = -ramps[i] / mel_spacing[i]

right = ramps[i + 2] / mel_spacing[i + 1]

# .. and set them zero when they cross the x-axis

fbank[i] = np.maximum(0, np.minimum(left, right))

if normalize:

energy_norm = 2.0 / (mel_bins[2 : n_filters + 2] - mel_bins[:n_filters])

fbank *= energy_norm[:, np.newaxis]

return fbank


from . import general
from . import nlp
from . import dsp


"""Common preprocessing utilities for working with text data"""

import re

import heapq

import os.path as op

from collections import Counter, OrderedDict, defaultdict

import numpy as np

# This list of English stop words is taken from the "Glasgow Information

# Retrieval Group". The original list can be found at

# http://ir.dcs.gla.ac.uk/resources/linguistic_utils/stop_words

_STOP_WORDS = set(

"a about above across after afterwards again against all almost alone "

"along already also although always am among amongst amoungst amount an "

"and another any anyhow anyone anything anyway anywhere are around as at "

"back be became because become becomes becoming been before beforehand "

"behind being below beside besides between beyond bill both bottom but by "

"call can cannot cant co con could couldnt cry de describe detail do done "

"down due during each eg eight either eleven else elsewhere empty enough "

"etc even ever every everyone everything everywhere except few fifteen "

"fifty fill find fire first five for former formerly forty found four from "

"front full further get give go had has hasnt have he hence her here "

"hereafter hereby herein hereupon hers herself him himself his how however "

"hundred i ie if in inc indeed interest into is it its itself keep last "

"latter latterly least less ltd made many may me meanwhile might mill mine "

"more moreover most mostly move much must my myself name namely neither "

"never nevertheless next nine no nobody none noone nor not nothing now "

"nowhere of off often on once one only onto or other others otherwise our "

"ours ourselves out over own part per perhaps please put rather re same see "

"seem seemed seeming seems serious several she should show side since "

"sincere six sixty so some somehow someone something sometime sometimes "

"somewhere still such system take ten than that the their them themselves "

"then thence there thereafter thereby therefore therein thereupon these "

"they thick thin third this those though three through throughout thru thus "

"to together too top toward towards twelve twenty two un under until up "

"upon us very via was we well were what whatever when whence whenever where "

"whereafter whereas whereby wherein whereupon wherever whether which while "

"whither who whoever whole whom whose why will with within without would "

"yet you your yours yourself yourselves"

).split(" "),

_WORD_REGEX = re.compile(r"(?u)\b\w\w+\b")  # sklearn default

_WORD_REGEX_W_PUNC = re.compile(r"(?u)\w+|[^a-zA-Z0-9\s]")

_WORD_REGEX_W_PUNC_AND_WHITESPACE = re.compile(r"(?u)s?\w+\s?|\s?[^a-zA-Z0-9\s]\s?")

_PUNC_BYTE_REGEX = re.compile(

r"(33|34|35|36|37|38|39|40|41|42|43|44|45|"

r"46|47|58|59|60|61|62|63|64|91|92|93|94|"

_PUNCTUATION = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"

_PUNC_TABLE = str.maketrans("", "", _PUNCTUATION)

def ngrams(sequence, N):

"""Return all `N`-grams of the elements in `sequence`"""

assert N >= 1

return list(zip(*[sequence[i:] for i in range(N)]))

def tokenize_whitespace(

line, lowercase=True, filter_stopwords=True, filter_punctuation=True, **kwargs,

Split a string at any whitespace characters, optionally removing

punctuation and stop-words in the process.

line = line.lower() if lowercase else line

words = line.split()

line = [strip_punctuation(w) for w in words] if filter_punctuation else line

return remove_stop_words(words) if filter_stopwords else words

def tokenize_words(

Split a string into individual words, optionally removing punctuation and

stop-words in the process.

REGEX = _WORD_REGEX if filter_punctuation else _WORD_REGEX_W_PUNC

words = REGEX.findall(line.lower() if lowercase else line)

def tokenize_words_bytes(

line,

lowercase=True,

**kwargs,

stop-words in the process. Translate each word into a list of bytes.

words = tokenize_words(

line,

lowercase=lowercase,

**kwargs,

words = [" ".join([str(i) for i in w.encode(encoding)]) for w in words]

return words

def tokenize_bytes_raw(line, encoding="utf-8", splitter=None, **kwargs):

Convert the characters in `line` to a collection of bytes. Each byte is

represented in decimal as an integer between 0 and 255.

Parameters

line : str

The string to tokenize.

encoding : str

The encoding scheme for the characters in `line`. Default is `'utf-8'`.

splitter : {'punctuation', None}

If `'punctuation'`, split the string at any punctuation character

before encoding into bytes. If None, do not split `line` at all.

Default is None.

Returns

bytes : list

A list of the byte-encoded characters in `line`. Each item in the list

is a string of space-separated integers between 0 and 255 representing

the bytes encoding the characters in `line`.

byte_str = [" ".join([str(i) for i in line.encode(encoding)])]

if splitter == "punctuation":

byte_str = _PUNC_BYTE_REGEX.sub(r"-\1-", byte_str[0]).split("-")

return byte_str

def bytes_to_chars(byte_list, encoding="utf-8"):

Decode bytes (represented as an integer between 0 and 255) to characters in

the specified encoding.

hex_array = [hex(a).replace("0x", "") for a in byte_list]

hex_array = " ".join([h if len(h) > 1 else f"0{h}" for h in hex_array])

return bytearray.fromhex(hex_array).decode(encoding)

def tokenize_chars(line, lowercase=True, filter_punctuation=True, **kwargs):

Split a string into individual characters, optionally removing punctuation

and stop-words in the process.

line = line.lower() if lowercase else line

line = strip_punctuation(line) if filter_punctuation else line

chars = list(re.sub(" {2,}", " ", line).strip())

return chars

def remove_stop_words(words):

"""Remove stop words from a list of word strings"""

return [w for w in words if w.lower() not in _STOP_WORDS]

def strip_punctuation(line):

"""Remove punctuation from a string"""

return line.translate(_PUNC_TABLE).strip()

#                          Byte-Pair Encoder                          #

class BytePairEncoder(object):

def __init__(self, max_merges=3000, encoding="utf-8"):

A byte-pair encoder for sub-word embeddings.

Notes

Byte-pair encoding [1][2] is a compression algorithm that iteratively

replaces the most frequently ocurring byte pairs in a set of documents

with a new, single token. It has gained popularity as a preprocessing

step for many NLP tasks due to its simplicity and expressiveness: using

a base coebook of just 256 unique tokens (bytes), any string can be

encoded.

References

.. [1] Gage, P. (1994). A new algorithm for data compression. *C

Users Journal, 12(2)*, 23–38.

.. [2] Sennrich, R., Haddow, B., & Birch, A. (2015). Neural machine

translation of rare words with subword units, *Proceedings of the

54th Annual Meeting of the Association for Computational

Linguistics,* 1715-1725.

Parameters

max_merges : int

The maximum number of byte pair merges to perform during the

:meth:`fit` operation. Default is 3000.

encoding : str

The encoding scheme for the documents used to train the encoder.

Default is `'utf-8'`.

self.parameters = {

"max_merges": max_merges,

"encoding": encoding,

# initialize the byte <-> token and token <-> byte dictionaries. bytes

# are represented in decimal as integers between 0 and 255. there is a

# 1:1 correspondence between token and byte representations up to 255.

self.byte2token = OrderedDict({i: i for i in range(256)})

self.token2byte = OrderedDict({v: k for k, v in self.byte2token.items()})

def fit(self, corpus_fps, encoding="utf-8"):

Train a byte pair codebook on a set of documents.

Parameters

corpus_fps : str or list of strs

The filepath / list of filepaths for the document(s) to be used to

learn the byte pair codebook.

encoding : str

The text encoding for documents. Common entries are either 'utf-8'

(no header byte), or 'utf-8-sig' (header byte). Default is

vocab = (

Vocabulary(

lowercase=False,

tokenizer="bytes",

.fit(corpus_fps, encoding=encoding)

.counts

# iteratively merge the most common byte bigram across the documents

for _ in range(self.parameters["max_merges"]):

pair_counts = self._get_counts(vocab)

most_common_bigram = max(pair_counts, key=pair_counts.get)

vocab = self._merge(most_common_bigram, vocab)

token_bytes = set()

for k in vocab.keys():

token_bytes = token_bytes.union([w for w in k.split(" ") if "-" in w])

for i, t in enumerate(token_bytes):

byte_tuple = tuple(int(j) for j in t.split("-"))

self.token2byte[256 + i] = byte_tuple

self.byte2token[byte_tuple] = 256 + i

return self

def _get_counts(self, vocab):

"""Collect bigram counts for the tokens in vocab"""

pair_counts = defaultdict(int)

for word, count in vocab.items():

pairs = ngrams(word.split(" "), 2)

for p in pairs:

pair_counts[p] += count

return pair_counts

def _merge(self, bigram, vocab):

"""Replace `bigram` with a single token and update vocab accordingly"""

v_out = {}

bg = re.escape(" ".join(bigram))

bigram_regex = re.compile(r"(?<!\S)" + bg + r"(?!\S)")

for word in vocab.keys():

# bigram "a b" becomes "a-b"

w_out = bigram_regex.sub("-".join(bigram), word)

v_out[w_out] = vocab[word]

return v_out

def transform(self, text):

Transform the words in `text` into their byte pair encoded token IDs.

Parameters

text: str or list of `N` strings

The list of strings to encode

Returns

codes : list of `N` lists

A list of byte pair token IDs for each of the `N` strings in

`text`.

Examples

>>> B = BytePairEncoder(max_merges=100).fit("./example.txt")

>>> encoded_tokens = B.transform("Hello! How are you 😁 ?")

>>> encoded_tokens

[[72, 879, 474, ...]]

if isinstance(text, str):

text = [text]

return [self._transform(string) for string in text]

def _transform(self, text):

"""Transform a single text string to a list of byte-pair IDs"""

P = self.parameters

_bytes = tokenize_bytes_raw(text, encoding=P["encoding"])

encoded = []

for w in _bytes:

l, r = 0, len(w)

w = [int(i) for i in w.split(" ")]

while l < len(w):

candidate = tuple(w[l:r])

if len(candidate) > 1 and candidate in self.byte2token:

# candidate is a collection of several bytes and is in our

# vocab

encoded.append(self.byte2token[candidate])

l, r = r, len(w)

elif len(candidate) == 1:

# candidate is a single byte and should always be in our

# vocab

l, r = r, len(w)

else:

# candidate is not in vocab, so we decrease our context

# window by 1 and try again

r -= 1

return encoded

def inverse_transform(self, codes):

Transform an encoded sequence of byte pair codeword IDs back into

human-readable text.

Parameters

codes : list of `N` lists

A list of `N` lists. Each sublist is a collection of integer

byte-pair token IDs representing a particular text string.

Returns

text: list of `N` strings

The decoded strings corresponding to the `N` sublists in `codes`.

Examples

>>> encoded_tokens

[[72, 879, 474, ...]]

>>> B.inverse_transform(encoded_tokens)

Hello! How are you 😁 ?

if isinstance(codes[0], int):

codes = [codes]

decoded = []

P = self.parameters

for code in codes:

_bytes = [self.token2byte[t] if t > 255 else [t] for t in code]

_bytes = [b for blist in _bytes for b in blist]

decoded.append(bytes_to_chars(_bytes, encoding=P["encoding"]))

return decoded

@property

def codebook(self):

A list of the learned byte pair codewords, decoded into human-readable

format

return [

for t in self.byte2token.keys()

if isinstance(t, tuple)

@property

def tokens(self):

"""A list of the byte pair codeword IDs"""

return list(self.token2byte.keys())

#                            Huffman Tree                             #

class Node(object):

def __init__(self, key, val):

self.key = key

self.val = val

self.left = None

self.right = None

def __gt__(self, other):

"""Greater than"""

if not isinstance(other, Node):

return -1

return self.val > other.val

def __ge__(self, other):

"""Greater than or equal to"""

if not isinstance(other, Node):

return -1

return self.val >= other.val

def __lt__(self, other):

"""Less than"""

if not isinstance(other, Node):

return -1

return self.val < other.val

def __le__(self, other):

"""Less than or equal to"""

if not isinstance(other, Node):

return -1

return self.val <= other.val

class HuffmanEncoder(object):

def fit(self, text):

Build a Huffman tree for the tokens in `text` and compute each token's

binary encoding.

Notes

In a Huffman code, tokens that occur more frequently are (generally)

represented using fewer bits. Huffman codes produce the minimum expected

codeword length among all methods for encoding tokens individually.

Huffman codes correspond to paths through a binary tree, with 1

corresponding to "move right" and 0 corresponding to "move left". In

contrast to standard binary trees, the Huffman tree is constructed from the

bottom up. Construction begins by initializing a min-heap priority queue

consisting of each token in the corpus, with priority corresponding to the

token frequency. At each step, the two most infrequent tokens in the corpus

are removed and become the children of a parent pseudotoken whose

"frequency" is the sum of the frequencies of its children. This new parent

pseudotoken is added to the priority queue and the process is repeated

recursively until no tokens remain.

Parameters

text: list of strs or :class:`Vocabulary` instance

The tokenized text or a pretrained :class:`Vocabulary` object to use for

building the Huffman code.

def transform(self, text):

Transform the words in `text` into their Huffman-code representations.

Parameters

text: list of `N` strings

The list of words to encode

Returns

codes : list of `N` binary strings

The encoded words in `text`

if isinstance(text, str):

text = [text]

for token in set(text):

if token not in self._item2code:

raise Warning("Token '{}' not in Huffman tree. Skipping".format(token))

return [self._item2code.get(t, None) for t in text]

def inverse_transform(self, codes):

Transform an encoded sequence of bit-strings back into words.

Parameters

codes : list of `N` binary strings

A list of encoded bit-strings, represented as strings.

Returns

text: list of `N` strings

The decoded text.

if isinstance(codes, str):

codes = [codes]

for code in set(codes):

if code not in self._code2item:

raise Warning("Code '{}' not in Huffman tree. Skipping".format(code))

return [self._code2item.get(c, None) for c in codes]

@property

def tokens(self):

"""A list the unique tokens in `text`"""

return list(self._item2code.keys())

@property

def codes(self):

"""A list with the Huffman code for each unique token in `text`"""

return list(self._code2item.keys())

def _counter(self, text):

counts = {}

for item in text:

counts[item] = counts.get(item, 0) + 1

return counts

def _build_tree(self, text):

"""Construct Huffman Tree"""

PQ = []

if isinstance(text, Vocabulary):

counts = text.counts

else:

counts = self._counter(text)

for (k, c) in counts.items():

PQ.append(Node(k, c))

# create a priority queue with priority = item frequency

heapq.heapify(PQ)

while len(PQ) > 1:

node1 = heapq.heappop(PQ)  # item with smallest frequency

node2 = heapq.heappop(PQ)  # item with second smallest frequency

parent = Node(None, node1.val + node2.val)

parent.left = node1

parent.right = node2

heapq.heappush(PQ, parent)

self._root = heapq.heappop(PQ)

def _generate_codes(self):

current_code = ""

self._item2code = {}

self._code2item = {}

self._build_code(self._root, current_code)

def _build_code(self, root, current_code):

if root is None:

return

if root.key is not None:

self._item2code[root.key] = current_code

self._code2item[current_code] = root.key

return

# 0 = move left, 1 = move right

self._build_code(root.left, current_code + "0")

self._build_code(root.right, current_code + "1")

#                             Containers                              #

class Token:

def __init__(self, word):

self.count = 0

self.word = word

def __repr__(self):

"""A string representation of the token"""

return "Token(word='{}', count={})".format(self.word, self.count)

class TFIDFEncoder:

def __init__(

self,

vocab=None,

lowercase=True,

tokenizer="words",

r"""

An object for compiling and encoding the term-frequency

inverse-document-frequency (TF-IDF) representation of the tokens in a

text corpus.

Notes

TF-IDF is intended to reflect how important a word is to a document in

a collection or corpus. For a word token `w` in a document `d`, and a

corpus, :math:`D = \{d_1, \ldots, d_N\}`, we have:

.. math::

\text{TF}(w, d)  &=  \text{num. occurences of }w \text{ in document }d \\

\text{IDF}(w, D)  &=  \log \frac{|D|}{|\{ d \in D: t \in d \}|}

Parameters

vocab : :class:`Vocabulary` object or list-like

An existing vocabulary to filter the tokens in the corpus against.

Default is None.

lowercase : bool

Whether to convert each string to lowercase before tokenization.

Default is True.

min_count : int

Minimum number of times a token must occur in order to be included

in vocab. Default is 0.

smooth_idf : bool

Whether to add 1 to the denominator of the IDF calculation to avoid

divide-by-zero errors. Default is True.

max_tokens : int

Only add the `max_tokens` most frequent tokens that occur more

than `min_count` to the vocabulary.  If None, add all tokens

greater that occur more than than `min_count`. Default is None.

input_type : {'files', 'strings'}

If 'files', the sequence input to `fit` is expected to be a list

of filepaths. If 'strings', the input is expected to be a list of

lists, each sublist containing the raw strings for a single

document in the corpus. Default is 'filename'.

filter_stopwords : bool

Whether to remove stopwords before encoding the words in the

corpus. Default is True.

filter_punctuation : bool

Whether to remove punctuation before encoding the words in the

corpus. Default is True.

tokenizer : {'whitespace', 'words', 'characters', 'bytes'}

Strategy to follow when mapping strings to tokens. The

`'whitespace'` tokenizer splits strings at whitespace characters.

The `'words'` tokenizer splits strings using a "word" regex. The

`'characters'` tokenizer splits strings into individual characters.

The `'bytes'` tokenizer splits strings into a collection of

individual bytes.

# create a function to filter against words in the vocab

self._filter_vocab = lambda words: words

if isinstance(vocab, Vocabulary):

self._filter_vocab = vocab.filter

elif isinstance(vocab, (list, np.ndarray, set)):

vocab = set(vocab)

self._filter_vocab = lambda words: [

w if w in vocab else "<unk>" for w in words

if input_type not in ["files", "strings"]:

fstr = "`input_type` must be either 'files' or 'strings', but got {}"

raise ValueError(fstr.format(input_type))

self._tokens = None

self._idx2doc = None

self.term_freq = None

self.idx2token = None

self.token2idx = None

self.inv_doc_freq = None

self.hyperparameters = {

"id": "TFIDFEncoder",

"encoding": None,

"vocab": vocab

if not isinstance(vocab, Vocabulary)

else vocab.hyperparameters,

"lowercase": lowercase,

"min_count": min_count,

"input_type": input_type,

"max_tokens": max_tokens,

"smooth_idf": smooth_idf,

"tokenizer": tokenizer

if not isinstance(vocab, Vocabulary)

else vocab.hyperparameters["tokenizer"],

"filter_stopwords": filter_stopwords

if not isinstance(vocab, Vocabulary)

else vocab.hyperparameters["filter_stopwords"],

"filter_punctuation": filter_punctuation

if not isinstance(vocab, Vocabulary)

else vocab.hyperparameters["filter_punctuation"],

def fit(self, corpus_seq, encoding="utf-8-sig"):

Compute term-frequencies and inverse document frequencies on a

collection of documents.

Parameters

corpus_seq : str or list of strs

The filepath / list of filepaths / raw string contents of the

document(s) to be encoded, in accordance with the `input_type`

parameter passed to the :meth:`__init__` method. Each document is

expected to be a string of tokens separated by whitespace.

encoding : str

Specifies the text encoding for corpus if `input_type` is `files`.

Common entries are either 'utf-8' (no header byte), or 'utf-8-sig'

(header byte). Default is 'utf-8-sig'.

Returns

self

H = self.hyperparameters

if isinstance(corpus_seq, str):

corpus_seq = [corpus_seq]

if H["input_type"] == "files":

for corpus_fp in corpus_seq:

assert op.isfile(corpus_fp), "{} does not exist".format(corpus_fp)

tokens = []

idx2token, token2idx = {}, {}

# encode special tokens

for tt in ["<bol>", "<eol>", "<unk>"]:

token2idx[tt] = len(tokens)

idx2token[len(tokens)] = tt

tokens.append(Token(tt))

min_count = H["min_count"]

max_tokens = H["max_tokens"]

H["encoding"] = encoding

bol_ix = token2idx["<bol>"]

eol_ix = token2idx["<eol>"]

idx2doc, term_freq = {}, {}

# encode the text in `corpus_fps` without any filtering ...

for d_ix, doc in enumerate(corpus_seq):

doc_count = {}

idx2doc[d_ix] = doc if H["input_type"] == "files" else None

token2idx, idx2token, tokens, doc_count = self._encode_document(

doc, token2idx, idx2token, tokens, doc_count, bol_ix, eol_ix,

term_freq[d_ix] = doc_count

self._tokens = tokens

self._idx2doc = idx2doc

self.token2idx = token2idx

self.idx2token = idx2token

self.term_freq = term_freq

# ... retain only the top `max_tokens` most frequent tokens, coding

# everything else as <unk> ...

if max_tokens is not None and len(tokens) > max_tokens:

# ... replace all words occurring less than `min_count` by <unk> ...

if min(self._tokens, key=lambda t: t.count).count < min_count:

# ... sort tokens alphabetically and reindex ...

# ... finally, calculate inverse document frequency

return self

def _encode_document(

self, doc, word2idx, idx2word, tokens, doc_count, bol_ix, eol_ix,

"""Perform tokenization and compute token counts for a single document"""

H = self.hyperparameters

lowercase = H["lowercase"]

filter_stop = H["filter_stopwords"]

filter_punc = H["filter_punctuation"]

if H["input_type"] == "files":

with open(doc, "r", encoding=H["encoding"]) as handle:

doc = handle.read()

tokenizer_dict = {

"words": tokenize_words,

"characters": tokenize_chars,

"whitespace": tokenize_whitespace,

"bytes": tokenize_bytes_raw,

tokenizer = tokenizer_dict[H["tokenizer"]]

n_words = 0

lines = doc.split("\n")

for line in lines:

words = tokenizer(

line,

lowercase=lowercase,

encoding=H["encoding"],

words = self._filter_vocab(words)

n_words += len(words)

for ww in words:

if ww not in word2idx:

word2idx[ww] = len(tokens)

idx2word[len(tokens)] = ww

tokens.append(Token(ww))

t_idx = word2idx[ww]

tokens[t_idx].count += 1

doc_count[t_idx] = doc_count.get(t_idx, 0) + 1

# wrap line in <bol> and <eol> tags

tokens[bol_ix].count += 1

tokens[eol_ix].count += 1

doc_count[bol_ix] = doc_count.get(bol_ix, 0) + 1

doc_count[eol_ix] = doc_count.get(eol_ix, 0) + 1

return word2idx, idx2word, tokens, doc_count

def _keep_top_n_tokens(self):

N = self.hyperparameters["max_tokens"]

doc_counts, word2idx, idx2word = {}, {}, {}

tokens = sorted(self._tokens, key=lambda x: x.count, reverse=True)

# reindex the top-N tokens...

unk_ix = None

for idx, tt in enumerate(tokens[:N]):

word2idx[tt.word] = idx

idx2word[idx] = tt.word

if tt.word == "<unk>":

unk_ix = idx

# ... if <unk> isn't in the top-N, add it, replacing the Nth

# most-frequent word and adjust the <unk> count accordingly ...

if unk_ix is None:

unk_ix = self.token2idx["<unk>"]

old_count = tokens[N - 1].count

tokens[N - 1] = self._tokens[unk_ix]

tokens[N - 1].count += old_count

word2idx["<unk>"] = N - 1

idx2word[N - 1] = "<unk>"

# ... and recode all dropped tokens as "<unk>"

for tt in tokens[N:]:

tokens[unk_ix].count += tt.count

# ... finally, reindex the word counts for each document

doc_counts = {}

for d_ix in self.term_freq.keys():

doc_counts[d_ix] = {}

for old_ix, d_count in self.term_freq[d_ix].items():

word = self.idx2token[old_ix]

new_ix = word2idx.get(word, unk_ix)

doc_counts[d_ix][new_ix] = doc_counts[d_ix].get(new_ix, 0) + d_count

self._tokens = tokens[:N]

self.token2idx = word2idx

self.idx2token = idx2word

self.term_freq = doc_counts

assert len(self._tokens) <= N

def _drop_low_freq_tokens(self):

Replace all tokens that occur less than `min_count` with the `<unk>`

token.

H = self.hyperparameters

unk_token = self._tokens[self.token2idx["<unk>"]]

eol_token = self._tokens[self.token2idx["<eol>"]]

bol_token = self._tokens[self.token2idx["<bol>"]]

tokens = [unk_token, eol_token, bol_token]

unk_idx = 0

word2idx = {"<unk>": 0, "<eol>": 1, "<bol>": 2}

idx2word = {0: "<unk>", 1: "<eol>", 2: "<bol>"}

special = {"<eol>", "<bol>", "<unk>"}

for tt in self._tokens:

if tt.word not in special:

if tt.count < H["min_count"]:

tokens[unk_idx].count += tt.count

else:

word2idx[tt.word] = len(tokens)

idx2word[len(tokens)] = tt.word

tokens.append(tt)

# reindex document counts

doc_counts = {}

for d_idx in self.term_freq.keys():

doc_counts[d_idx] = {}

for old_idx, d_count in self.term_freq[d_idx].items():

word = self.idx2token[old_idx]

new_idx = word2idx.get(word, unk_idx)

doc_counts[d_idx][new_idx] = doc_counts[d_idx].get(new_idx, 0) + d_count

self._tokens = tokens

self.token2idx = word2idx

self.idx2token = idx2word

self.term_freq = doc_counts

def _sort_tokens(self):

# sort tokens alphabetically and recode

ix = 0

token2idx, idx2token, = (

special = ["<eol>", "<bol>", "<unk>"]

words = sorted(self.token2idx.keys())

term_freq = {d: {} for d in self.term_freq.keys()}

for w in words:

if w not in special:

old_ix = self.token2idx[w]

token2idx[w], idx2token[ix] = ix, w

for d in self.term_freq.keys():

if old_ix in self.term_freq[d]:

count = self.term_freq[d][old_ix]

term_freq[d][ix] = count

ix += 1

for w in special:

token2idx[w] = len(token2idx)

idx2token[len(idx2token)] = w

self.token2idx = token2idx

self.idx2token = idx2token

self.term_freq = term_freq

self.vocab_counts = Counter({t.word: t.count for t in self._tokens})

def _calc_idf(self):

Compute the (smoothed-) inverse-document frequency for each token in

the corpus.

For a word token `w`, the IDF is simply

IDF(w) = log ( |D| / |{ d in D: w in d }| ) + 1

where D is the set of all documents in the corpus,

D = {d1, d2, ..., dD}

If `smooth_idf` is True, we perform additive smoothing on the number of

documents containing a given word, equivalent to pretending that there

exists a final D+1st document that contains every word in the corpus:

SmoothedIDF(w) = log ( |D| + 1 / [1 + |{ d in D: w in d }|] ) + 1

inv_doc_freq = {}

smooth_idf = self.hyperparameters["smooth_idf"]

tf, doc_idxs = self.term_freq, self._idx2doc.keys()

D = len(self._idx2doc) + int(smooth_idf)

for word, w_ix in self.token2idx.items():

d_count = int(smooth_idf)

d_count += np.sum([1 if w_ix in tf[d_ix] else 0 for d_ix in doc_idxs])

inv_doc_freq[w_ix] = 1 if d_count == 0 else np.log(D / d_count) + 1

self.inv_doc_freq = inv_doc_freq

def transform(self, ignore_special_chars=True):

Generate the term-frequency inverse-document-frequency encoding of a

text corpus.

Parameters

ignore_special_chars : bool

Whether to drop columns corresponding to "<eol>", "<bol>", and

"<unk>" tokens from the final tfidf encoding. Default is True.

Returns

tfidf : numpy array of shape `(D, M [- 3])`

The encoded corpus, with each row corresponding to a single

document, and each column corresponding to a token id. The mapping

between column numbers and tokens is stored in the `idx2token`

attribute IFF `ignore_special_chars` is False. Otherwise, the

mappings are not accurate.

D, N = len(self._idx2doc), len(self._tokens)

tf = np.zeros((D, N))

idf = np.zeros((D, N))

for d_ix in self._idx2doc.keys():

words, counts = zip(*self.term_freq[d_ix].items())

docs = np.ones(len(words), dtype=int) * d_ix

tf[docs, words] = counts

words = sorted(self.idx2token.keys())

idf = np.tile(np.array([self.inv_doc_freq[w] for w in words]), (D, 1))

tfidf = tf * idf

if ignore_special_chars:

idxs = [

tfidf = np.delete(tfidf, idxs, 1)

return tfidf

class Vocabulary:

def __init__(

self,

lowercase=True,

tokenizer="words",

An object for compiling and encoding the unique tokens in a text corpus.

Parameters

lowercase : bool

Whether to convert each string to lowercase before tokenization.

Default is True.

min_count : int

Minimum number of times a token must occur in order to be included

in vocab. If `None`, include all tokens from `corpus_fp` in vocab.

Default is None.

max_tokens : int

Only add the `max_tokens` most frequent tokens that occur more

than `min_count` to the vocabulary.  If None, add all tokens

that occur more than than `min_count`. Default is None.

filter_stopwords : bool

Whether to remove stopwords before encoding the words in the

corpus. Default is True.

filter_punctuation : bool

Whether to remove punctuation before encoding the words in the

corpus. Default is True.

tokenizer : {'whitespace', 'words', 'characters', 'bytes'}

Strategy to follow when mapping strings to tokens. The

`'whitespace'` tokenizer splits strings at whitespace characters.

The `'words'` tokenizer splits strings using a "word" regex. The

`'characters'` tokenizer splits strings into individual characters.

The `'bytes'` tokenizer splits strings into a collection of

individual bytes.

self.hyperparameters = {

"id": "Vocabulary",

"encoding": None,

"corpus_fps": None,

"lowercase": lowercase,

"min_count": min_count,

"max_tokens": max_tokens,

"filter_stopwords": filter_stopwords,

"filter_punctuation": filter_punctuation,

"tokenizer": tokenizer,

def __len__(self):

"""Return the number of tokens in the vocabulary"""

return len(self._tokens)

def __iter__(self):

"""Return an iterator over the tokens in the vocabulary"""

return iter(self._tokens)

def __contains__(self, word):

"""Assert whether `word` is a token in the vocabulary"""

return word in self.token2idx

def __getitem__(self, key):

Return the token (if key is an integer) or the index (if key is a string)

for the key in the vocabulary, if it exists.

if isinstance(key, str):

return self._tokens[self.token2idx[key]]

if isinstance(key, int):

return self._tokens[key]

@property

def n_tokens(self):

"""The number of unique word tokens in the vocabulary"""

return len(self.token2idx)

@property

def n_words(self):

"""The total number of words in the corpus"""

return sum(self.counts.values())

@property

def shape(self):

return self._tokens.shape

def most_common(self, n=5):

"""Return the top `n` most common tokens in the corpus"""

return self.counts.most_common()[:n]

def words_with_count(self, k):

"""Return all tokens that occur `k` times in the corpus"""

return [w for w, c in self.counts.items() if c == k]

def filter(self, words, unk=True):  # noqa: A003

Filter (or replace) any word in `words` that is not present in

`Vocabulary`.

Parameters

words : list of strs

A list of words to filter

unk : bool

Whether to replace any out of vocabulary words in `words` with the

``<unk>`` token (True) or skip them entirely (False).  Default is

True.

Returns

filtered : list of strs

The list of words filtered against the words in Vocabulary.

if unk:

return [w if w in self else "<unk>" for w in words]

return [w for w in words if w in self]

def words_to_indices(self, words):

Convert the words in `words` to their token indices. If a word is not

in the vocabulary, return the index for the ``<unk>`` token

Parameters

words : list of strs

A list of words to filter

Returns

indices : list of ints

The token indices for each word in `words`

unk_ix = self.token2idx["<unk>"]

lowercase = self.hyperparameters["lowercase"]

words = [w.lower() for w in words] if lowercase else words

return [self.token2idx[w] if w in self else unk_ix for w in words]

def indices_to_words(self, indices):

Convert the indices in `indices` to their word values. If an index is

not in the vocabulary, return the ``<unk>`` token.

Parameters

indices : list of ints

Returns

words : list of strs

The word strings corresponding to each token index in `indices`

unk = "<unk>"

return [self.idx2token[i] if i in self.idx2token else unk for i in indices]

def fit(self, corpus_fps, encoding="utf-8-sig"):

Compute the vocabulary across a collection of documents.

Parameters

corpus_fps : str or list of strs

The filepath / list of filepaths for the document(s) to be encoded.

Each document is expected to be encoded as newline-separated

string of text, with adjacent tokens separated by a whitespace

character.

encoding : str

Specifies the text encoding for corpus. Common entries are either

'utf-8' (no header byte), or 'utf-8-sig' (header byte). Default is

Returns

self

if isinstance(corpus_fps, str):

corpus_fps = [corpus_fps]

for corpus_fp in corpus_fps:

assert op.isfile(corpus_fp), "{} does not exist".format(corpus_fp)

tokens = []

H = self.hyperparameters

idx2word, word2idx = {}, {}

tokenizer_dict = {

"words": tokenize_words,

"characters": tokenize_chars,

"whitespace": tokenize_whitespace,

"bytes": tokenize_bytes_raw,

min_count = H["min_count"]

lowercase = H["lowercase"]

max_tokens = H["max_tokens"]

filter_stop = H["filter_stopwords"]

filter_punc = H["filter_punctuation"]

tokenizer = tokenizer_dict[H["tokenizer"]]

H["encoding"] = encoding

H["corpus_fps"] = corpus_fps

# encode special tokens

for tt in ["<bol>", "<eol>", "<unk>"]:

word2idx[tt] = len(tokens)

idx2word[len(tokens)] = tt

tokens.append(Token(tt))

bol_ix = word2idx["<bol>"]

eol_ix = word2idx["<eol>"]

for d_ix, doc_fp in enumerate(corpus_fps):

with open(doc_fp, "r", encoding=H["encoding"]) as doc:

for line in doc:

words = tokenizer(

line,

lowercase=lowercase,

encoding=H["encoding"],

for ww in words:

if ww not in word2idx:

word2idx[ww] = len(tokens)

idx2word[len(tokens)] = ww

tokens.append(Token(ww))

t_idx = word2idx[ww]

tokens[t_idx].count += 1

# wrap line in <bol> and <eol> tags

tokens[bol_ix].count += 1

tokens[eol_ix].count += 1

self._tokens = tokens

self.token2idx = word2idx

self.idx2token = idx2word

# replace all words occurring less than `min_count` by <unk>

if min_count is not None:

# retain only the top `max_tokens` most frequent tokens, coding

# everything else as <unk>

if max_tokens is not None and len(tokens) > max_tokens:

counts = {w: self._tokens[ix].count for w, ix in self.token2idx.items()}

self.counts = Counter(counts)

self._tokens = np.array(self._tokens)

return self

def _keep_top_n_tokens(self):

word2idx, idx2word = {}, {}

N = self.hyperparameters["max_tokens"]

tokens = sorted(self._tokens, key=lambda x: x.count, reverse=True)

# reindex the top-N tokens...

unk_ix = None

for idx, tt in enumerate(tokens[:N]):

word2idx[tt.word] = idx

idx2word[idx] = tt.word

if tt.word == "<unk>":

unk_ix = idx

# ... if <unk> isn't in the top-N, add it, replacing the Nth

# most-frequent word and adjusting the <unk> count accordingly ...

if unk_ix is None:

unk_ix = self.token2idx["<unk>"]

old_count = tokens[N - 1].count

tokens[N - 1] = self._tokens[unk_ix]

tokens[N - 1].count += old_count

word2idx["<unk>"] = N - 1

idx2word[N - 1] = "<unk>"

# ... and recode all dropped tokens as "<unk>"

for tt in tokens[N:]:

tokens[unk_ix].count += tt.count

self._tokens = tokens[:N]

self.token2idx = word2idx

self.idx2token = idx2word

assert len(self._tokens) <= N

def _drop_low_freq_tokens(self):

Replace all tokens that occur less than `min_count` with the `<unk>`

token.

unk_idx = 0

unk_token = self._tokens[self.token2idx["<unk>"]]

eol_token = self._tokens[self.token2idx["<eol>"]]

bol_token = self._tokens[self.token2idx["<bol>"]]

H = self.hyperparameters

tokens = [unk_token, eol_token, bol_token]

word2idx = {"<unk>": 0, "<eol>": 1, "<bol>": 2}

idx2word = {0: "<unk>", 1: "<eol>", 2: "<bol>"}

special = {"<eol>", "<bol>", "<unk>"}

for tt in self._tokens:

if tt.word not in special:

if tt.count < H["min_count"]:

tokens[unk_idx].count += tt.count

else:

word2idx[tt.word] = len(tokens)

idx2word[len(tokens)] = tt.word

tokens.append(tt)

self._tokens = tokens

self.token2idx = word2idx

self.idx2token = idx2word


import json

import hashlib

import warnings

import numpy as np

try:

from scipy.sparse import csr_matrix

_SCIPY = True

except ImportError:

warnings.warn("Scipy not installed. FeatureHasher can only create dense matrices")

_SCIPY = False

def minibatch(X, batchsize=256, shuffle=True):

Compute the minibatch indices for a training dataset.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, \*)`

The dataset to divide into minibatches. Assumes the first dimension

represents the number of training examples.

batchsize : int

The desired size of each minibatch. Note, however, that if ``X.shape[0] %

batchsize > 0`` then the final batch will contain fewer than batchsize

entries. Default is 256.

shuffle : bool

Whether to shuffle the entries in the dataset before dividing into

minibatches. Default is True.

Returns

mb_generator : generator

A generator which yields the indices into `X` for each batch.

n_batches: int

The number of batches.

N = X.shape[0]

ix = np.arange(N)

n_batches = int(np.ceil(N / batchsize))

if shuffle:

np.random.shuffle(ix)

def mb_generator():

for i in range(n_batches):

yield ix[i * batchsize : (i + 1) * batchsize]

return mb_generator(), n_batches

class OneHotEncoder:

def __init__(self):

Convert between category labels and their one-hot vector

representations.

Parameters

categories : list of length `C`

self._is_fit = False

self.hyperparameters = {}

self.parameters = {"categories": None}

def __call__(self, labels):

return self.transform(labels)

def fit(self, categories):

Create mappings between columns and category labels.

Parameters

categories : list of length `C`

self.parameters["categories"] = categories

self.cat2idx = {c: i for i, c in enumerate(categories)}

self.idx2cat = {i: c for i, c in enumerate(categories)}

self._is_fit = True

def transform(self, labels, categories=None):

Convert a list of labels into a one-hot encoding.

Parameters

labels : list of length `N`

A list of category labels.

categories : list of length `C`

List of the unique category labels for the items to encode. Default

is None.

Returns

Y : :py:class:`ndarray <numpy.ndarray>` of shape `(N, C)`

The one-hot encoded labels. Each row corresponds to an example,

with a single 1 in the column corresponding to the respective

label.

if not self._is_fit:

categories = set(labels) if categories is None else categories

self.fit(categories)

unknown = list(set(labels) - set(self.cat2idx.keys()))

assert len(unknown) == 0, "Unrecognized label(s): {}".format(unknown)

N, C = len(labels), len(self.cat2idx)

cols = np.array([self.cat2idx[c] for c in labels])

Y = np.zeros((N, C))

Y[np.arange(N), cols] = 1

return Y

def inverse_transform(self, Y):

Convert a one-hot encoding back into the corresponding labels

Parameters

One-hot encoded labels. Each row corresponds to an example, with a

single 1 in the column associated with the label for that example

Returns

labels : list of length `N`

The list of category labels corresponding to the nonzero columns in

`Y`

C = len(self.cat2idx)

assert Y.ndim == 2, "Y must be 2D, but has shape {}".format(Y.shape)

assert Y.shape[1] == C, "Y must have {} columns, got {}".format(C, Y.shape[1])

return [self.idx2cat[ix] for ix in Y.nonzero()[1]]

class Standardizer:

def __init__(self, with_mean=True, with_std=True):

Feature-wise standardization for vector inputs.

Notes

Due to the sensitivity of empirical mean and standard deviation

calculations to extreme values, `Standardizer` cannot guarantee

balanced feature scales in the presence of outliers. In particular,

note that because outliers for each feature can have different

magnitudes, the spread of the transformed data on each feature can be

very different.

Similar to sklearn, `Standardizer` uses a biased estimator for the

standard deviation: ``numpy.std(x, ddof=0)``.

Parameters

with_mean : bool

Whether to scale samples to have 0 mean during transformation.

Default is True.

with_std : bool

Whether to scale samples to have unit variance during

transformation. Default is True.

self.with_mean = with_mean

self.with_std = with_std

self._is_fit = False

@property

def hyperparameters(self):

H = {"with_mean": self.with_mean, "with_std": self.with_std}

return H

@property

def parameters(self):

params = {

"mean": self._mean if hasattr(self, "mean") else None,

"std": self._std if hasattr(self, "std") else None,

return params

def __call__(self, X):

return self.transform(X)

def fit(self, X):

Store the feature-wise mean and standard deviation across the samples

in `X` for future scaling.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, C)`

An array of N samples, each with dimensionality `C`

if not isinstance(X, np.ndarray):

X = np.array(X)

if X.shape[0] < 2:

raise ValueError("`X` must contain at least 2 samples")

std = np.ones(X.shape[1])

mean = np.zeros(X.shape[1])

if self.with_mean:

mean = np.mean(X, axis=0)

if self.with_std:

std = np.std(X, axis=0, ddof=0)

self._mean = mean

self._std = std

self._is_fit = True

def transform(self, X):

Standardize features by removing the mean and scaling to unit variance.

For a sample `x`, the standardized score is calculated as:

.. math::

z = (x - u) / s

where `u` is the mean of the training samples or zero if `with_mean` is

False, and `s` is the standard deviation of the training samples or 1

if `with_std` is False.

Parameters

An array of N samples, each with dimensionality `C`.

Returns

Z : :py:class:`ndarray <numpy.ndarray>` of shape `(N, C)`

The feature-wise standardized version of `X`.

if not self._is_fit:

raise Exception("Must call `fit` before using the `transform` method")

return (X - self._mean) / self._std

def inverse_transform(self, Z):

Convert a collection of standardized features back into the original

feature space.

For a standardized sample `z`, the unstandardized score is calculated as:

.. math::

x = z s + u

if `with_std` is False.

Parameters

An array of `N` standardized samples, each with dimensionality `C`.

Returns

The unstandardixed samples from `Z`.

assert self._is_fit, "Must fit `Standardizer` before calling inverse_transform"

P = self.parameters

mean, std = P["mean"], P["std"]

return Z * std + mean

class FeatureHasher:

def __init__(self, n_dim=256, sparse=True):

Convert a collection of features to a fixed-dimensional matrix using

the hashing trick.

Notes

Uses the md5 hash.

Parameters

n_dim : int

The dimensionality of each example in the output feature matrix.

Small numbers of features are likely to cause hash collisions, but

large numbers will cause larger overall parameter dimensions for

any (linear) learning agent. Default is 256.

sparse : bool

Whether the resulting feature matrix should be a sparse

:py:class:`csr_matrix <scipy.sparse.csr_matrix>` or dense

:py:class:`ndarray <numpy.ndarray>`. Default is True.

self.n_dim = n_dim

self.hash = hashlib.md5

self.sparse = sparse and _SCIPY

def encode(self, examples):

Encode a collection of multi-featured examples into a

`n_dim`-dimensional feature matrix via feature hashing.

Notes

Feature hashing works by applying a hash function to the features of an

example and using the hash values as column indices in the resulting

feature matrix. The entries at each hashed feature column correspond to

the values for that example and feature. For example, given the

following two input examples:

>>> examples = [

{"furry": 1, "quadruped": 1, "domesticated": 1},

{"nocturnal": 1, "quadruped": 1},

and a hypothetical hash function `H` mapping strings to [0, 127], we have:

>>> feature_mat = zeros(2, 128)

>>> ex1_cols = [H("furry"), H("quadruped"), H("domesticated")]

>>> ex2_cols = [H("nocturnal"), H("quadruped")]

>>> feat_mat[0, ex1_cols] = 1

>>> feat_mat[1, ex2_cols] = 1

To better handle hash collisions, it is common to multiply the feature

value by the sign of the digest for the corresponding feature name.

Parameters

examples : dict or list of dicts

A collection of `N` examples, each represented as a dict where keys

correspond to the feature name and values correspond to the feature

value.

Returns

table : :py:class:`ndarray <numpy.ndarray>` or :py:class:`csr_matrix <scipy.sparse.csr_matrix>` of shape `(N, n_dim)`

The encoded feature matrix

if isinstance(examples, dict):

examples = [examples]

sparse = self.sparse

return self._encode_sparse(examples) if sparse else self._encode_dense(examples)

def _encode_dense(self, examples):

N = len(examples)

table = np.zeros(N, self.n_dim)  # dense

for row, feat_dict in enumerate(examples):

for f_id, val in feat_dict.items():

if isinstance(f_id, str):

f_id = f_id.encode("utf-8")

# use json module to convert the feature id into a unique

# string compatible with the buffer API (required by hashlib)

if isinstance(f_id, (tuple, dict, list)):

f_id = json.dumps(f_id, sort_keys=True).encode("utf-8")

h = int(self.hash(f_id).hexdigest(), base=16)

col = h % self.n_dim

table[row, col] += np.sign(h) * val

return table

def _encode_sparse(self, examples):

N = len(examples)

idxs, data = [], []

for f_id, val in feat_dict.items():

if isinstance(f_id, str):

f_id = f_id.encode("utf-8")

col = h % self.n_dim

idxs.append((row, col))

data.append(np.sign(h) * val)

table = csr_matrix((data, zip(*idxs)), shape=(N, self.n_dim))

return table


from .hmm import *


"""Hidden Markov model module"""

import numpy as np

from numpy_ml.utils.misc import logsumexp

class MultinomialHMM:

def __init__(self, A=None, B=None, pi=None, eps=None):

r"""

A simple hidden Markov model with multinomial emission distribution.

Parameters

A : :py:class:`ndarray <numpy.ndarray>` of shape `(N, N)` or None

The transition matrix between latent states in the HMM. Index `i`,

`j` gives the probability of transitioning from latent state `i` to

latent state `j`. Default is None.

B : :py:class:`ndarray <numpy.ndarray>` of shape `(N, V)` or None

The emission matrix. Entry `i`, `j` gives the probability of latent

state i emitting an observation of type `j`. Default is None.

pi : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)` or None

The prior probability of each latent state. If None, use a uniform

prior over states. Default is None.

eps : float or None

Epsilon value to avoid :math:`\log(0)` errors. If None, defaults to

the machine epsilon. Default is None.

Attributes

A : :py:class:`ndarray <numpy.ndarray>` of shape `(N, N)`

latent state `j`.

B : :py:class:`ndarray <numpy.ndarray>` of shape `(N, V)`

state `i` emitting an observation of type `j`.

N : int

The number of unique latent states

V : int

The number of unique observation types

O : :py:class:`ndarray <numpy.ndarray>` of shape `(I, T)`

The collection of observed training sequences.

I : int

The number of sequences in `O`.

T : int

The number of observations in each sequence in `O`.

eps = np.finfo(float).eps if eps is None else eps

# prior probability of each latent state

if pi is not None:

pi[pi == 0] = eps

# number of latent state types

N = None

if A is not None:

N = A.shape[0]

A[A == 0] = eps

# number of observation types

V = None

if B is not None:

V = B.shape[1]

B[B == 0] = eps

self.parameters = {

"A": A,  # transition matrix

"B": B,  # emission matrix

"pi": pi,  # prior probability of each latent state

self.hyperparameters = {

"eps": eps,  # epsilon

self.derived_variables = {

"N": N,  # number of latent state types

"V": V,  # number of observation types

def generate(self, n_steps, latent_state_types, obs_types):

Sample a sequence from the HMM.

Parameters

n_steps : int

The length of the generated sequence

latent_state_types : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

A collection of labels for the latent states

obs_types : :py:class:`ndarray <numpy.ndarray>` of shape `(V,)`

A collection of labels for the observations

Returns

states : :py:class:`ndarray <numpy.ndarray>` of shape `(n_steps,)`

The sampled latent states.

emissions : :py:class:`ndarray <numpy.ndarray>` of shape `(n_steps,)`

The sampled emissions.

P = self.parameters

A, B, pi = P["A"], P["B"], P["pi"]

# sample the initial latent state

s = np.random.multinomial(1, pi).argmax()

states = [latent_state_types[s]]

# generate an emission given latent state

v = np.random.multinomial(1, B[s, :]).argmax()

emissions = [obs_types[v]]

# sample a latent transition, rinse, and repeat

for i in range(n_steps - 1):

s = np.random.multinomial(1, A[s, :]).argmax()

return np.array(states), np.array(emissions)

def log_likelihood(self, O):

r"""

Given the HMM parameterized by :math:`(A`, B, \pi)` and an observation

sequence `O`, compute the marginal likelihood of `O`,

:math:`P(O \mid A,B,\pi)`, by marginalizing over latent states.

Notes

The log likelihood is computed efficiently via DP using the forward

algorithm, which produces a 2D trellis, ``forward`` (sometimes referred

to as `alpha` in the literature), where entry `i`, `j` represents the

probability under the HMM of being in latent state `i` after seeing the

first `j` observations:

.. math::

\mathtt{forward[i,j]} = P(o_1, \ldots, o_j, q_j=i \mid A, B, \pi)

type `i`.

The DP step is:

.. math::

\mathtt{forward[i,j]}

&= \sum_{s'=1}^N \mathtt{forward[s',j-1]} \cdot

\mathtt{A[s',i]} \cdot \mathtt{B[i,o_j]} \\

&= \sum_{s'=1}^N P(o_1, \ldots, o_{j-1}, q_{j-1}=s' \mid A, B, \pi)

P(q_j=i \mid q_{j-1}=s') P(o_j \mid q_j=i)

In words, ``forward[i,j]`` is the weighted sum of the values computed on

the previous timestep. The weight on each previous state value is the

product of the probability of transitioning from that state to state `i`

and the probability of emitting observation `j` in state `i`.

Parameters

O : :py:class:`ndarray <numpy.ndarray>` of shape `(1, T)`

A single set of observations.

Returns

likelihood : float

The likelihood of the observations `O` under the HMM.

if O.ndim == 1:

O = O.reshape(1, -1)  # noqa: E741

I, T = O.shape  # noqa: E741

if I != 1:  # noqa: E741

raise ValueError("Likelihood only accepts a single sequence")

forward = self._forward(O[0])

log_likelihood = logsumexp(forward[:, T - 1])

return log_likelihood

def decode(self, O):

r"""

Given the HMM parameterized by :math:`(A, B, \pi)` and an observation

sequence :math:`O = o_1, \ldots, o_T`, compute the most probable

sequence of latent states, :math:`Q = q_1, \ldots, q_T`.

Notes

HMM decoding is done efficiently via DP using the Viterbi algorithm,

which produces a 2D trellis, ``viterbi``, where entry `i`, `j` represents the

probability under the HMM of being in state `i` at time `j` after having

passed through the *most probable* state sequence :math:`q_1,\ldots,q_{j-1}`:

.. math::

\mathtt{viterbi[i,j]} =

\max_{q_1, \ldots, q_{j-1}}

P(o_1, \ldots, o_j, q_1, \ldots, q_{j-1}, q_j=i \mid A, B, \pi)

type `i`, and :math:`\max_{q_1,\ldots,q_{j-1}}` represents the maximum over

all possible latent state sequences for the first `j-1` observations.

The DP step is:

.. math::

\mathtt{viterbi[i,j]} &=

\max_{s'=1}^N \mathtt{viterbi[s',j-1]} \cdot

\mathtt{A[s',i]} \cdot \mathtt{B[i,o_j]} \\

&=  \max_{s'=1}^N

P(o_1,\ldots, o_j, q_1, \ldots, q_{j-1}, q_j=i \mid A, B, \pi)

P(q_j=i \mid q_{j-1}=s') P(o_j \mid q_j=i)

In words, ``viterbi[i,j]`` is the weighted sum of the values computed

on the previous timestep. The weight on each value is the product of

the probability of transitioning from that state to state `i` and the

probability of emitting observation `j` in state `i`.

To compute the most probable state sequence we maintain a second

trellis, ``back_pointer``, whose `i`, `j` entry contains the value of the

latent state at timestep `j-1` that is most likely to lead to latent

state `i` at timestep `j`.

When we have completed the ``viterbi`` and ``back_pointer`` trellises for

all `T` timseteps/observations, we greedily move backwards through the

``back_pointer`` trellis to construct the best path for the full

sequence of observations.

Parameters

O : :py:class:`ndarray <numpy.ndarray>` of shape `(T,)`

An observation sequence of length `T`.

Returns

best_path : list of length `T`

The most probable sequence of latent states for observations `O`.

best_path_prob : float

The probability of the latent state sequence in `best_path` under

the HMM.

P = self.parameters

N = self.derived_variables["N"]

eps = self.hyperparameters["eps"]

A, B, pi = P["A"], P["B"], P["pi"]

if O.ndim == 1:

O = O.reshape(1, -1)  # noqa: E741

# number of observations in each sequence

T = O.shape[1]

# number of training sequences

I = O.shape[0]  # noqa: E741

if I != 1:  # noqa: E741

raise ValueError("Can only decode a single sequence (O.shape[0] must be 1)")

# initialize the viterbi and back_pointer matrices

viterbi = np.zeros((N, T))

back_pointer = np.zeros((N, T)).astype(int)

ot = O[0, 0]

for s in range(N):

back_pointer[s, 0] = 0

viterbi[s, 0] = np.log(pi[s] + eps) + np.log(B[s, ot] + eps)

for t in range(1, T):

ot = O[0, t]

for s in range(N):

seq_probs = [

viterbi[s_, t - 1] + np.log(A[s_, s] + eps) + np.log(B[s, ot] + eps)

for s_ in range(N)

viterbi[s, t] = np.max(seq_probs)

back_pointer[s, t] = np.argmax(seq_probs)

best_path_log_prob = viterbi[:, T - 1].max()

# backtrack through the trellis to get the most likely sequence of

# latent states

pointer = viterbi[:, T - 1].argmax()

best_path = [pointer]

for t in reversed(range(1, T)):

pointer = back_pointer[pointer, t]

best_path = best_path[::-1]

return best_path, best_path_log_prob

def _forward(self, Obs):

r"""

Computes the forward probability trellis for an HMM parameterized by

:math:`(A, B, \pi)`.

Notes

The forward trellis (sometimes referred to as `alpha` in the HMM

literature), is a 2D array where entry `i`, `j` represents the probability

under the HMM of being in latent state `i` after seeing the first `j`

observations:

.. math::

\mathtt{forward[i,j]} =

P(o_1, \ldots, o_j, q_j=i \mid A, B, \pi)

type `i`.

The DP step is::

.. math::

forward[i,j] &=

\sum_{s'=1}^N forward[s',j-1] \times A[s',i] \times B[i,o_j] \\

\times P(q_j=i \mid q_{j-1}=s') \times P(o_j \mid q_j=i)

on the previous timestep. The weight on each previous state value is

the product of the probability of transitioning from that state to

state `i` and the probability of emitting observation `j` in state `i`.

Parameters

Obs : :py:class:`ndarray <numpy.ndarray>` of shape `(T,)`

An observation sequence of length `T`.

Returns

forward : :py:class:`ndarray <numpy.ndarray>` of shape `(N, T)`

The forward trellis.

P = self.parameters

N = self.derived_variables["N"]

eps = self.hyperparameters["eps"]

A, B, pi = P["A"], P["B"], P["pi"]

T = Obs.shape[0]

# initialize the forward probability matrix

forward = np.zeros((N, T))

ot = Obs[0]

for s in range(N):

forward[s, 0] = np.log(pi[s] + eps) + np.log(B[s, ot] + eps)

for t in range(1, T):

ot = Obs[t]

for s in range(N):

forward[s, t] = logsumexp(

forward[s_, t - 1]

+ np.log(A[s_, s] + eps)

+ np.log(B[s, ot] + eps)

for s_ in range(N)

]  # noqa: C812

return forward

def _backward(self, Obs):

r"""

Compute the backward probability trellis for an HMM parameterized by

:math:`(A, B, \pi)`.

Notes

The backward trellis (sometimes referred to as `beta` in the HMM

literature), is a 2D array where entry `i`,`j` represents the probability

of seeing the observations from time `j+1` onward given that the HMM is

in state `i` at time `j`

.. math::

\mathtt{backward[i,j]} = P(o_{j+1},o_{j+2},\ldots,o_T \mid q_j=i,A,B,\pi)

Here :math:`q_j = i` indicates that the hidden state at time `j` is of type `i`.

The DP step is::

backward[i,j] &=

\sum_{s'=1}^N backward[s',j+1] \times A[i, s'] \times B[s',o_{j+1}] \\

&= \sum_{s'=1}^N P(o_{j+1}, o_{j+2}, \ldots, o_T \mid q_j=i, A, B, pi)

\times P(q_{j+1}=s' \mid q_{j}=i) \times P(o_{j+1} \mid q_{j+1}=s')

In words, ``backward[i,j]`` is the weighted sum of the values computed

on the following timestep. The weight on each state value from the

`j+1`'th timestep is the product of the probability of transitioning from

state i to that state and the probability of emitting observation `j+1`

from that state.

Parameters

Obs : :py:class:`ndarray <numpy.ndarray>` of shape `(T,)`

A single observation sequence of length `T`.

Returns

backward : :py:class:`ndarray <numpy.ndarray>` of shape `(N, T)`

The backward trellis.

P = self.parameters

A, B = P["A"], P["B"]

N = self.derived_variables["N"]

eps = self.hyperparameters["eps"]

T = Obs.shape[0]

# initialize the backward trellis

backward = np.zeros((N, T))

for s in range(N):

backward[s, T - 1] = 0

for t in reversed(range(T - 1)):

ot1 = Obs[t + 1]

for s in range(N):

backward[s, t] = logsumexp(

np.log(A[s, s_] + eps)

+ np.log(B[s_, ot1] + eps)

+ backward[s_, t + 1]

for s_ in range(N)

]  # noqa: C812

return backward

def _initialize_parameters(self):

P = self.parameters

A, B, pi = P["A"], P["B"], P["pi"]

N, V = self.derived_variables["N"], self.derived_variables["V"]

# Uniform initialization of prior over latent states

if pi is None:

pi = np.ones(N)

pi = pi / pi.sum()

# Uniform initialization of A

if A is None:

A = np.ones((N, N))

A = A / A.sum(axis=1)[:, None]

# Random initialization of B

if B is None:

B = np.random.rand(N, V)

B = B / B.sum(axis=1)[:, None]

P["A"], P["B"], P["pi"] = A, B, pi

def fit(

self,

O,

pi=None,

verbose=False,

Given an observation sequence `O` and the set of possible latent states,

learn the MLE HMM parameters `A` and `B`.

Notes

Model fitting is done iterativly using the Baum-Welch/Forward-Backward

algorithm, a special case of the EM algorithm.

We begin with an intial estimate for the transition (`A`) and emission

(`B`) matrices and then use these to derive better and better estimates

by computing the forward probability for an observation and then

dividing that probability mass among all the paths that contributed to

it.

Parameters

O : :py:class:`ndarray <numpy.ndarray>` of shape `(I, T)`

The set of `I` training observations, each of length `T`.

latent_state_types : list of length `N`

The collection of valid latent states.

observation_types : list of length `V`

The collection of valid observation states.

pi : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

The prior probability of each latent state. If None, assume each

latent state is equally likely a priori. Default is None.

tol : float

The tolerance value. If the difference in log likelihood between

two epochs is less than this value, terminate training. Default is

verbose : bool

Print training stats after each epoch. Default is True.

Returns

A : :py:class:`ndarray <numpy.ndarray>` of shape `(N, N)`

The estimated transition matrix.

B : :py:class:`ndarray <numpy.ndarray>` of shape `(N, V)`

The estimated emission matrix.

The estimated prior probabilities of each latent state.

# observations

if O.ndim == 1:

O = O.reshape(1, -1)  # noqa: E741

# number of training examples (I) and their lengths (T)

I, T = O.shape

# number of types of observation

self.derived_variables["V"] = len(observation_types)

# number of latent state types

self.derived_variables["N"] = len(latent_state_types)

P = self.parameters

# iterate E and M steps until convergence criteria is met

step, delta = 0, np.inf

ll_prev = np.sum([self.log_likelihood(o) for o in O])

while delta > tol:

gamma, xi, phi = self._E_step(O)

P["A"], P["B"], P["pi"] = self._M_step(O, gamma, xi, phi)

ll = np.sum([self.log_likelihood(o) for o in O])

delta = ll - ll_prev

ll_prev = ll

step += 1

if verbose:

fstr = "[Epoch {}] LL: {:.3f} Delta: {:.5f}"

print(fstr.format(step, ll_prev, delta))

#  return A, B, pi

def _E_step(self, O):

r"""

Run a single E-step update for the Baum-Welch/Forward-Backward

algorithm. This step estimates ``xi`` and ``gamma``, the excepted

state-state transition counts and the expected state-occupancy counts,

respectively.

``xi[i,j,k]`` gives the probability of being in state `i` at time `k`

and state `j` at time `k+1` given the observed sequence `O` and the

current estimates for transition (`A`) and emission (`B`) matrices::

.. math::

xi[i,j,k] &= P(q_k=i,q_{k+1}=j \mid O,A,B,pi) \\

&= \frac{

P(q_k=i,q_{k+1}=j,O \mid A,B,pi)

}{P(O \mid A,B,pi)} \\

&= \frac{

P(o_1,o_2,\ldots,o_k,q_k=i \mid A,B,pi) \times

P(q_{k+1}=j \mid q_k=i) \times

P(o_{k+1} \mid q_{k+1}=j) \times

P(o_{k+2},o_{k+3},\ldots,o_T \mid q_{k+1}=j,A,B,pi)

}{P(O \mid A,B,pi)} \\

&= \frac{

\mathtt{fwd[j, k] * self.A[j, i] *

self.B[i, o_{k+1}] * bwd[i, k + 1]}

}{\mathtt{fwd[:, T].sum()}}

The expected number of transitions from state `i` to state `j` across the

entire sequence is then the sum over all timesteps: ``xi[i,j,:].sum()``.

``gamma[i,j]`` gives the probability of being in state `i` at time `j`

.. math:: \mathtt{gamma[i,j]} = P(q_j = i \mid O, A, B, \pi)

Parameters

O : :py:class:`ndarray <numpy.ndarray>` of shape `(I, T)`

The set of `I` training observations, each of length `T`.

Returns

gamma : :py:class:`ndarray <numpy.ndarray>` of shape `(I, N, T)`

The estimated state-occupancy count matrix.

xi : :py:class:`ndarray <numpy.ndarray>` of shape `(I, N, N, T)`

The estimated state-state transition count matrix.

phi : :py:class:`ndarray <numpy.ndarray>` of shape `(I, N)`

The estimated prior counts for each latent state.

I, T = O.shape

P = self.parameters

A, B = P["A"], P["B"]

N = self.derived_variables["N"]

eps = self.hyperparameters["eps"]

phi = np.zeros((I, N))

gamma = np.zeros((I, N, T))

xi = np.zeros((I, N, N, T))

for i in range(I):

Obs = O[i, :]

fwd = self._forward(Obs)

bwd = self._backward(Obs)

log_likelihood = logsumexp(fwd[:, T - 1])

t = T - 1

for si in range(N):

gamma[i, si, t] = fwd[si, t] + bwd[si, t] - log_likelihood

phi[i, si] = fwd[si, 0] + bwd[si, 0] - log_likelihood

for t in range(T - 1):

ot1 = Obs[t + 1]

for si in range(N):

for sj in range(N):

xi[i, si, sj, t] = (

fwd[si, t]

+ np.log(A[si, sj] + eps)

+ np.log(B[sj, ot1] + eps)

+ bwd[sj, t + 1]

- log_likelihood

return gamma, xi, phi

def _M_step(self, O, gamma, xi, phi):

Run a single M-step update for the Baum-Welch/Forward-Backward

algorithm.

Parameters

The estimated starting count matrix for each latent state.

Returns

A : :py:class:`ndarray <numpy.ndarray>` of shape `(N, N)`

The estimated transition matrix.

B : :py:class:`ndarray <numpy.ndarray>` of shape `(N, V)`

The estimated emission matrix.

pi : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

The estimated prior probabilities for each latent state.

I, T = O.shape

P = self.parameters

DV = self.derived_variables

eps = self.hyperparameters["eps"]

N, V = DV["N"], DV["V"]

A, B, pi = P["A"], P["B"], P["pi"]

# initialize the estimated transition (A) and emission (B) matrices

A = np.zeros((N, N))

B = np.zeros((N, V))

pi = np.zeros(N)

count_gamma = np.zeros((I, N, V))

count_xi = np.zeros((I, N, N))

for i in range(I):

Obs = O[i, :]

for si in range(N):

for vk in range(V):

if not (Obs == vk).any():

count_gamma[i, si, vk] = np.log(eps)

else:

count_gamma[i, si, vk] = logsumexp(gamma[i, si, Obs == vk])

for sj in range(N):

count_xi[i, si, sj] = logsumexp(xi[i, si, sj, :])

pi = logsumexp(phi, axis=0) - np.log(I + eps)

np.testing.assert_almost_equal(np.exp(pi).sum(), 1)

for si in range(N):

for vk in range(V):

B[si, vk] = logsumexp(count_gamma[:, si, vk]) - logsumexp(

count_gamma[:, si, :]  # noqa: C812

for sj in range(N):

A[si, sj] = logsumexp(count_xi[:, si, sj]) - logsumexp(

count_xi[:, si, :]  # noqa: C812

np.testing.assert_almost_equal(np.exp(A[si, :]).sum(), 1)

np.testing.assert_almost_equal(np.exp(B[si, :]).sum(), 1)

return np.exp(A), np.exp(B), np.exp(pi)


"""A module for the generalized linear model."""

import numpy as np

from numpy_ml.linear_models.linear_regression import LinearRegression

eps = np.finfo(float).eps

_GLM_LINKS = {

"logit": {

"link": lambda mu: np.log((mu + eps) / (1 - mu + eps)),

"inv_link": lambda eta: 1.0 / (1.0 + np.exp(-eta)),

"link_prime": lambda x: (1 / (x + eps)) + (1 / (1 - x + eps)),

"theta": lambda mu: np.log((mu + eps) / (1 - mu + eps)),

"phi": lambda x: np.ones(x.shape[0]),

"a": lambda phi: phi,

"b": lambda theta: np.log(1 + np.exp(theta)),

"p": 1,

"b_prime": lambda theta: np.exp(theta) / (1 + np.exp(theta)),

"b_prime2": lambda theta: np.exp(theta) / ((1 + np.exp(theta)) ** 2),

"identity": {

"link": lambda mu: mu,

"inv_link": lambda eta: eta,

"link_prime": lambda x: np.ones_like(x),

"theta": lambda mu: mu,

"phi": lambda x: np.var(x, axis=0),

"a": lambda phi: phi,

"b": lambda theta: 0.5 * theta ** 2,

"p": 1,

"b_prime": lambda theta: theta,

"b_prime2": lambda theta: np.ones_like(theta),

"log": {

"link": lambda mu: np.log(mu + eps),

"inv_link": lambda eta: np.exp(eta),

"link_prime": lambda x: 1 / (x + eps),

"theta": lambda mu: np.log(mu + eps),

"phi": lambda x: np.ones(x.shape[0]),

"a": lambda phi: phi,

"p": 1,

"b": lambda theta: np.exp(theta),

"b_prime": lambda theta: np.exp(theta),

"b_prime2": lambda theta: np.exp(theta),

class GeneralizedLinearModel:

def __init__(self, link, fit_intercept=True, tol=1e-5, max_iter=100):

r"""

A generalized linear model with maximum likelihood fit via

iteratively reweighted least squares (IRLS).

Notes

The generalized linear model (GLM) [7]_ [8]_ assumes that each target/dependent

variable :math:`y_i` in target vector :math:`\mathbf{y} = (y_1, \ldots,

y_n)`, has been drawn independently from a pre-specified distribution

in the exponential family [11]_ with unknown mean :math:`\mu_i`. The GLM

models a (one-to-one, continuous, differentiable) function, *g*, of

this mean value as a linear combination of the model parameters

:math:`\mathbf{b}` and observed covariates, :math:`\mathbf{x}_i`:

.. math::

g(\mathbb{E}[y_i \mid \mathbf{x}_i]) =

g(\mu_i) = \mathbf{b}^\top \mathbf{x}_i

where *g* is known as the "link function" associated with the GLM.  The

choice of link function is informed by the instance of the exponential

family the target is drawn from. Common examples:

.. csv-table::

:header: "Distribution", "Link", "Formula"

:widths: 25, 20, 30

"Normal", "Identity", ":math:`g(x) = x`"

"Bernoulli", "Logit", ":math:`g(x) = \log(x) - \log(1 - x)`"

"Binomial", "Logit", ":math:`g(x) = \log(x) - \log(n - x)`"

"Poisson", "Log", ":math:`g(x) = \log(x)`"

An iteratively re-weighted least squares (IRLS) algorithm [9]_ can be

employed to find the maximum likelihood estimate for the model

parameters :math:`\beta` in any instance of the generalized linear

model. IRLS is equivalent to Fisher scoring [10]_, which itself is

a slight modification of classic Newton-Raphson for finding the zeros

of the first derivative of the model log-likelihood.

References

.. [7] Nelder, J., & Wedderburn, R. (1972). Generalized linear

models. *Journal of the Royal Statistical Society, Series A

(General), 135(3)*: 370–384.

.. [8] https://en.wikipedia.org/wiki/Generalized_linear_model

.. [9] https://en.wikipedia.org/wiki/Iteratively_reweighted_least_squares

.. [10] https://en.wikipedia.org/wiki/Scoring_algorithm

.. [11] https://en.wikipedia.org/wiki/Exponential_family

Parameters

link: {'identity', 'logit', 'log'}

The link function to use during modeling.

fit_intercept: bool

Whether to fit an intercept term in addition to the model

coefficients. Default is True.

tol : float

The minimum difference between successive iterations of IRLS

Default is 1e-5.

max_iter: int

The maximum number of iteratively reweighted least squares

iterations to run during fitting. Default is 100.

Attributes

beta : :py:class:`ndarray <numpy.ndarray>` of shape `(M, 1)` or None

Fitted model coefficients.

err_str = f"Valid link functions are {list(_GLM_LINKS.keys())} but got {link}"

assert link in _GLM_LINKS, err_str

self._is_fit = False

self.tol = tol

self.link = link

self.beta = None

self.max_iter = max_iter

self.fit_intercept = fit_intercept

def fit(self, X, y):

Find the maximum likelihood GLM coefficients via IRLS.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, M)`

A dataset consisting of `N` examples, each of dimension `M`.

y : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

The targets for each of the `N` examples in `X`.

Returns

self : :class:`GeneralizedLinearModel <numpy_ml.linear_models.GeneralizedLinearModel>` instance

"""  # noqa: E501

y = np.squeeze(y)

assert y.ndim == 1

N, M = X.shape

L = _GLM_LINKS[self.link]

# starting values for parameters

mu = np.ones_like(y) * np.mean(y)

eta = L["link"](mu)

theta = L["theta"](mu)

# convert X to a design matrix if we're fitting an intercept

if self.fit_intercept:

X = np.c_[np.ones(N), X]

# IRLS for GLM

i = 0

diff, beta = np.inf, np.inf

while diff > (self.tol * M):

if i > self.max_iter:

print("Warning: Model did not converge")

break

# compute first-order Taylor approx.

z = eta + (y - mu) * L["link_prime"](mu)

w = L["p"] / (L["b_prime2"](theta) * L["link_prime"](mu) ** 2)

# perform weighted least-squares on z

wlr = LinearRegression(fit_intercept=False)

beta_new = wlr.fit(X, z, weights=w).beta.ravel()

eta = X @ beta_new

mu = L["inv_link"](eta)

theta = L["theta"](mu)

diff = np.linalg.norm(beta - beta_new, ord=1)

beta = beta_new

i += 1

self.beta = beta

self._is_fit = True

return self

def predict(self, X):

r"""

Use the trained model to generate predictions for the distribution

means, :math:`\mu`, associated with the collection of data points in

**X**.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(Z, M)`

A dataset consisting of `Z` new examples, each of dimension `M`.

Returns

mu_pred : :py:class:`ndarray <numpy.ndarray>` of shape `(Z,)`

The model predictions for the expected value of the target

associated with each item in `X`.

assert self._is_fit, "Must call `fit` before generating predictions"

L = _GLM_LINKS[self.link]

# convert X to a design matrix if we're using an intercept

if self.fit_intercept:

X = np.c_[np.ones(X.shape[0]), X]

mu_pred = L["inv_link"](X @ self.beta)

return mu_pred.ravel()


"""Logistic regression module"""

import numpy as np

class LogisticRegression:

def __init__(self, penalty="l2", gamma=0, fit_intercept=True):

r"""

A simple binary logistic regression model fit via gradient descent on

the penalized negative log likelihood.

Notes

In simple binary logistic regression, the entries in a binary target

vector :math:`\mathbf{y} = (y_1, \ldots, y_N)` are assumed to have been

drawn from a series of independent Bernoulli random variables with

expected values :math:`p_1, \ldots, p_N`. The binary logistic regession

model models the logit of these unknown mean parameters as a linear

function of the model coefficients, :math:`\mathbf{b}`, and the

covariates for the corresponding example, :math:`\mathbf{x}_i`:

.. math::

\text{Logit}(p_i) =

\log \left( \frac{p_i}{1 - p_i} \right) = \mathbf{b}^\top\mathbf{x}_i

The model predictions :math:`\hat{\mathbf{y}}` are the expected values

of the Bernoulli parameters for each example:

.. math::

\hat{y}_i =

\mathbb{E}[y_i \mid \mathbf{x}_i] = \sigma(\mathbf{b}^\top \mathbf{x}_i)

where :math:`\sigma` is the logistic sigmoid function :math:`\sigma(x)

= \frac{1}{1 + e^{-x}}`. Under this model, the (penalized) negative log

likelihood of the targets **y** is

.. math::

- \log \mathcal{L}(\mathbf{b}, \mathbf{y}) = -\frac{1}{N} \left[

\left(

\sum_{i=0}^N y_i \log(\hat{y}_i) +

(1-y_i) \log(1-\hat{y}_i)

\right) - R(\mathbf{b}, \gamma)

\right]

where

.. math::

R(\mathbf{b}, \gamma) = \left\{

\begin{array}{lr}

\frac{\gamma}{2} ||\mathbf{b}||_2^2 & :\texttt{ penalty = 'l2'}\\

\gamma ||\mathbf{b}||_1 & :\texttt{ penalty = 'l1'}

\end{array}

\right.

is a regularization penalty, :math:`\gamma` is a regularization weight,

`N` is the number of examples in **y**, :math:`\hat{y}_i` is the model

prediction on example *i*, and **b** is the vector of model

coefficients.

Parameters

penalty : {'l1', 'l2'}

The type of regularization penalty to apply on the coefficients

`beta`. Default is 'l2'.

gamma : float

The regularization weight. Larger values correspond to larger

regularization penalties, and a value of 0 indicates no penalty.

Default is 0.

fit_intercept : bool

Whether to fit an intercept term in addition to the coefficients in

b. If True, the estimates for `beta` will have `M + 1` dimensions,

where the first dimension corresponds to the intercept. Default is

True.

Attributes

beta : :py:class:`ndarray <numpy.ndarray>` of shape `(M, 1)` or None

Fitted model coefficients.

err_msg = "penalty must be 'l1' or 'l2', but got: {}".format(penalty)

assert penalty in ["l2", "l1"], err_msg

self.beta = None

self.gamma = gamma

self.penalty = penalty

self.fit_intercept = fit_intercept

def fit(self, X, y, lr=0.01, tol=1e-7, max_iter=1e7):

Fit the regression coefficients via gradient descent on the negative

log likelihood.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(N, M)`

A dataset consisting of `N` examples, each of dimension `M`.

y : :py:class:`ndarray <numpy.ndarray>` of shape `(N,)`

The binary targets for each of the `N` examples in `X`.

lr : float

The gradient descent learning rate. Default is 1e-7.

max_iter : float

The maximum number of iterations to run the gradient descent

solver. Default is 1e7.

# convert X to a design matrix if we're fitting an intercept

if self.fit_intercept:

X = np.c_[np.ones(X.shape[0]), X]

l_prev = np.inf

self.beta = np.random.rand(X.shape[1])

for _ in range(int(max_iter)):

y_pred = _sigmoid(X @ self.beta)

loss = self._NLL(X, y, y_pred)

if l_prev - loss < tol:

return

l_prev = loss

self.beta -= lr * self._NLL_grad(X, y, y_pred)

def _NLL(self, X, y, y_pred):

r"""

Penalized negative log likelihood of the targets under the current

model.

.. math::

\text{NLL} = -\frac{1}{N} \left[

\left(

\sum_{i=0}^N y_i \log(\hat{y}_i) + (1-y_i) \log(1-\hat{y}_i)

\right) - R(\mathbf{b}, \gamma)

\right]

N, M = X.shape

beta, gamma = self.beta, self.gamma

order = 2 if self.penalty == "l2" else 1

norm_beta = np.linalg.norm(beta, ord=order)

nll = -np.log(y_pred[y == 1]).sum() - np.log(1 - y_pred[y == 0]).sum()

penalty = (gamma / 2) * norm_beta ** 2 if order == 2 else gamma * norm_beta

return (penalty + nll) / N

def _NLL_grad(self, X, y, y_pred):

"""Gradient of the penalized negative log likelihood wrt beta"""

N, M = X.shape

p, beta, gamma = self.penalty, self.beta, self.gamma

d_penalty = gamma * beta if p == "l2" else gamma * np.sign(beta)

return -((y - y_pred) @ X + d_penalty) / N

def predict(self, X):

Use the trained model to generate prediction probabilities on a new

collection of data points.

Parameters

X : :py:class:`ndarray <numpy.ndarray>` of shape `(Z, M)`

A dataset consisting of `Z` new examples, each of dimension `M`.

Returns

y_pred : :py:class:`ndarray <numpy.ndarray>` of shape `(Z,)`

The model prediction probabilities for the items in `X`.

if self.fit_intercept:

X = np.c_[np.ones(X.shape[0]), X]

return _sigmoid(X @ self.beta)

def _sigmoid(x):

"""The logistic sigmoid function"""

return 1 / (1 + np.exp(-x))


"""A module containing assorted linear models."""

from .ridge import RidgeRegression

from .glm import GeneralizedLinearModel

from .logistic import LogisticRegression

from .bayesian_regression import (

BayesianLinearRegressionKnownVariance,

BayesianLinearRegressionUnknownVariance,

from .naive_bayes import GaussianNBClassifier

from .linear_regression import LinearRegression