from abc import ABC, abstractmethod

from collections import defaultdict

from itertools import combinations, permutations

import numpy as np

#                          Graph Components                           #

class Edge(object):

def __init__(self, fr, to, w=None):

A generic directed edge object.

Parameters

fr: int

The id of the vertex the edge goes from

to: int

The id of the vertex the edge goes to

w: float, :class:`Object` instance, or None

The edge weight, if applicable. If weight is an arbitrary Object it

must have a method called 'sample' which takes no arguments and

returns a random sample from the weight distribution. If `w` is

None, no weight is assumed. Default is None.

self.fr = fr

self.to = to

self._w = w

def __repr__(self):

return "{} -> {}, weight: {}".format(self.fr, self.to, self._w)

@property

def weight(self):

return self._w.sample() if hasattr(self._w, "sample") else self._w

def reverse(self):

"""Reverse the edge direction"""

return Edge(self.t, self.f, self.w)

#                             Graph Types                             #

class Graph(ABC):

def __init__(self, V, E):

self._I2V = {i: v for i, v in zip(range(len(V)), V)}

self._V2I = {v: i for i, v in zip(range(len(V)), V)}

self._G = {i: set() for i in range(len(V))}

self._V = V

self._E = E

def __getitem__(self, v_i):

return self.get_neighbors(v_i)

def get_index(self, v):

"""Get the internal index for a given vetex"""

return self._V2I[v]

def get_vertex(self, v_i):

"""Get the original vertex from a given internal index"""

return self._I2V[v_i]

@property

def vertices(self):

return self._V

@property

def indices(self):

return list(range(len(self.vertices)))

@property

def edges(self):

return self._E

def get_neighbors(self, v_i):

Return the internal indices of the vertices reachable from the vertex

with index `v_i`.

return [self._V2I[e.to] for e in self._G[v_i]]

def to_matrix(self):

"""Return an adjacency matrix representation of the graph"""

adj_mat = np.zeros((len(self._V), len(self._V)))

for e in self.edges:

fr, to = self._V2I[e.fr], self._V2I[e.to]

adj_mat[fr, to] = 1 if e.weight is None else e.weight

return adj_mat

def to_adj_dict(self):

"""Return an adjacency dictionary representation of the graph"""

adj_dict = defaultdict(lambda: list())

for e in self.edges:

return adj_dict

def path_exists(self, s_i, e_i):

Check whether a path exists from vertex index `s_i` to `e_i`.

Parameters

s_i: Int

The interal index of the start vertex

e_i: Int

The internal index of the end vertex

Returns

path_exists : Boolean

Whether or not a valid path exists between `s_i` and `e_i`.

queue = [(s_i, [s_i])]

while len(queue):

c_i, path = queue.pop(0)

nbrs_not_on_path = set(self.get_neighbors(c_i)) - set(path)

for n_i in nbrs_not_on_path:

queue.append((n_i, path + [n_i]))

if n_i == e_i:

return True

return False

def all_paths(self, s_i, e_i):

Find all simple paths between `s_i` and `e_i` in the graph.

Notes

Uses breadth-first search. Ignores all paths with repeated vertices.

Parameters

s_i: Int

The interal index of the start vertex

e_i: Int

The internal index of the end vertex

Returns

complete_paths : list of lists

A list of all paths from `s_i` to `e_i`. Each path is represented

as a list of interal vertex indices.

complete_paths = []

queue = [(s_i, [s_i])]

while len(queue):

c_i, path = queue.pop(0)

for n_i in nbrs_not_on_path:

if n_i == e_i:

complete_paths.append(path + [n_i])

else:

queue.append((n_i, path + [n_i]))

return complete_paths

@abstractmethod

def _build_adjacency_list(self):

pass

class DiGraph(Graph):

def __init__(self, V, E):

A generic directed graph object.

Parameters

V : list

A list of vertex IDs.

E : list of :class:`Edge <numpy_ml.utils.graphs.Edge>` objects

A list of directed edges connecting pairs of vertices in ``V``.

super().__init__(V, E)

self.is_directed = True

self._topological_ordering = []

def _build_adjacency_list(self):

"""Encode directed graph as an adjancency list"""

# assumes no parallel edges

for e in self.edges:

fr_i = self._V2I[e.fr]

def reverse(self):

"""Reverse the direction of all edges in the graph"""

return DiGraph(self.vertices, [e.reverse() for e in self.edges])

def topological_ordering(self):

Returns a (non-unique) topological sort / linearization of the nodes

IFF the graph is acyclic, otherwise returns None.

Notes

A topological sort is an ordering on the nodes in `G` such that for every

directed edge :math:`u \\rightarrow v` in the graph, `u` appears before

`v` in the ordering.  The topological ordering is produced by ordering

the nodes in `G` by their DFS "last visit time," from greatest to

smallest.

This implementation follows a recursive, DFS-based approach [1]_ which

may break if the graph is very large. For an iterative version, see

Khan's algorithm [2]_ .

References

.. [1] Tarjan, R. (1976), Edge-disjoint spanning trees and depth-first

search, *Acta Informatica, 6 (2)*: 171–185.

.. [2] Kahn, A. (1962), Topological sorting of large networks,

*Communications of the ACM, 5 (11)*: 558–562.

Returns

ordering : list or None

A topoligical ordering of the vertex indices if the graph is a DAG,

otherwise None.

ordering = []

visited = set()

def dfs(v_i, path=None):

"""A simple DFS helper routine"""

path = set([v_i]) if path is None else path

for nbr_i in self.get_neighbors(v_i):

if nbr_i in path:

return True  # cycle detected!

elif nbr_i not in visited:

is_cyclic = dfs(nbr_i, path)

if is_cyclic:

return True

# insert to the beginning of the ordering

ordering.insert(0, v_i)

path -= set([v_i])

return False

for s_i in self.indices:

if s_i not in visited:

is_cyclic = dfs(s_i)

if is_cyclic:

return None

return ordering

def is_acyclic(self):

"""Check whether the graph contains cycles"""

return self.topological_ordering() is not None

class UndirectedGraph(Graph):

def __init__(self, V, E):

A generic undirected graph object.

Parameters

V : list

A list of vertex IDs.

A list of edges connecting pairs of vertices in ``V``. For any edge

connecting vertex `u` to vertex `v`, :class:`UndirectedGraph

<numpy_ml.utils.graphs.UndirectedGraph>` will assume that there

exists a corresponding edge connecting `v` to `u`, even if this is

not present in `E`.

super().__init__(V, E)

self.is_directed = False

def _build_adjacency_list(self):

"""Encode undirected, unweighted graph as an adjancency list"""

# assumes no parallel edges

# each edge appears twice as (u,v) and (v,u)

for e in self.edges:

fr_i = self._V2I[e.fr]

to_i = self._V2I[e.to]

#                          Graph Generators                           #

def random_unweighted_graph(n_vertices, edge_prob=0.5, directed=False):

Generate an unweighted Erdős-Rényi random graph [*]_.

References

.. [*] Erdős, P. and Rényi, A. (1959). On Random Graphs, *Publ. Math. 6*, 290.

Parameters

n_vertices : int

The number of vertices in the graph.

edge_prob : float in [0, 1]

The probability of forming an edge between two vertices. Default is

directed : bool

Whether the edges in the graph should be directed. Default is False.

Returns

G : :class:`Graph` instance

The resulting random graph.

vertices = list(range(n_vertices))

candidates = permutations(vertices, 2) if directed else combinations(vertices, 2)

edges = []

for (fr, to) in candidates:

if np.random.rand() <= edge_prob:

edges.append(Edge(fr, to))

return DiGraph(vertices, edges) if directed else UndirectedGraph(vertices, edges)

def random_DAG(n_vertices, edge_prob=0.5):

Create a 'random' unweighted directed acyclic graph by pruning all the

backward connections from a random graph.

Parameters

n_vertices : int

The number of vertices in the graph.

edge_prob : float in [0, 1]

The probability of forming an edge between two vertices in the

underlying random graph, before edge pruning. Default is 0.5.

Returns

G : :class:`Graph` instance

The resulting DAG.

G = random_unweighted_graph(n_vertices, edge_prob, directed=True)

# prune edges to remove backwards connections between vertices

G = DiGraph(G.vertices, [e for e in G.edges if e.fr < e.to])

# if we pruned away all the edges, generate a new graph

while not len(G.edges):

return G


"""Utilities for writing unit tests"""

import numbers

import numpy as np

#                             Assertions                              #

def is_symmetric(X):

"""Check that an array `X` is symmetric along its main diagonal"""

return np.allclose(X, X.T)

def is_symmetric_positive_definite(X):

"""Check that a matrix `X` is a symmetric and positive-definite."""

if is_symmetric(X):

try:

# if matrix is symmetric, check whether the Cholesky decomposition

# (defined only for symmetric/Hermitian positive definite matrices)

# exists

np.linalg.cholesky(X)

return True

except np.linalg.LinAlgError:

return False

return False

def is_stochastic(X):

"""True if `X` contains probabilities that sum to 1 along the columns"""

msg = "Array should be stochastic along the columns"

assert len(X[X < 0]) == len(X[X > 1]) == 0, msg

assert np.allclose(np.sum(X, axis=1), np.ones(X.shape[0])), msg

return True

def is_number(a):

"""Check that a value `a` is numeric"""

return isinstance(a, numbers.Number)

def is_one_hot(x):

"""Return True if array `x` is a binary array with a single 1"""

msg = "Matrix should be one-hot binary"

assert np.array_equal(x, x.astype(bool)), msg

assert np.allclose(np.sum(x, axis=1), np.ones(x.shape[0])), msg

return True

def is_binary(x):

"""Return True if array `x` consists only of binary values"""

msg = "Matrix must be binary"

return True

#                           Data Generators                           #

def random_one_hot_matrix(n_examples, n_classes):

"""Create a random one-hot matrix of shape (`n_examples`, `n_classes`)"""

X = np.eye(n_classes)

X = X[np.random.choice(n_classes, n_examples)]

return X

def random_stochastic_matrix(n_examples, n_classes):

"""Create a random stochastic matrix of shape (`n_examples`, `n_classes`)"""

X = np.random.rand(n_examples, n_classes)

X /= X.sum(axis=1, keepdims=True)

return X

def random_tensor(shape, standardize=False):

Create a random real-valued tensor of shape `shape`. If `standardize` is

True, ensure each column has mean 0 and std 1.

offset = np.random.randint(-300, 300, shape)

X = np.random.rand(*shape) + offset

if standardize:

eps = np.finfo(float).eps

X = (X - X.mean(axis=0)) / (X.std(axis=0) + eps)

return X

def random_binary_tensor(shape, sparsity=0.5):

Create a random binary tensor of shape `shape`. `sparsity` is a value

between 0 and 1 controlling the ratio of 0s to 1s in the output tensor.

return (np.random.rand(*shape) >= (1 - sparsity)).astype(float)

def random_paragraph(n_words, vocab=None):

Generate a random paragraph consisting of `n_words` words. If `vocab` is

not None, words will be drawn at random from this list. Otherwise, words

will be sampled uniformly from a collection of 26 Latin words.

if vocab is None:

vocab = [

"at",

"stet",

"accusam",

"aliquyam",

"clita",

"lorem",

"ipsum",

"dolor",

"dolore",

"dolores",

"sit",

"amet",

"consetetur",

"sadipscing",

"elitr",

"sed",

"diam",

"nonumy",

"eirmod",

"duo",

"ea",

"eos",

"erat",

"est",

"et",

"gubergren",

return [np.random.choice(vocab) for _ in range(n_words)]

#                           Custom Warnings                           #

class DependencyWarning(RuntimeWarning):

pass


"""A module for different N-gram smoothing models"""

import textwrap

from abc import ABC, abstractmethod

from collections import Counter

import numpy as np

from numpy_ml.linear_models import LinearRegression

from numpy_ml.preprocessing.nlp import tokenize_words, ngrams, strip_punctuation

class NGramBase(ABC):

def __init__(self, N, unk=True, filter_stopwords=True, filter_punctuation=True):

A simple word-level N-gram language model.

Notes

This is not optimized code and will be slow for large corpora. To see

how industry-scale NGram models are handled, see the SRLIM-format:

self.N = N

self.unk = unk

self.filter_stopwords = filter_stopwords

self.filter_punctuation = filter_punctuation

self.hyperparameters = {

"N": N,

"unk": unk,

"filter_stopwords": filter_stopwords,

"filter_punctuation": filter_punctuation,

def train(self, corpus_fp, vocab=None, encoding=None):

Compile the n-gram counts for the text(s) in `corpus_fp`.

Notes

After running `train`, the ``self.counts`` attribute will store

dictionaries of the `N`, `N-1`, ..., 1-gram counts.

Parameters

corpus_fp : str

The path to a newline-separated text corpus file.

If not None, only the words in `vocab` will be used to construct

the language model; all out-of-vocabulary words will either be

mappend to ``<unk>`` (if ``self.unk = True``) or removed (if

``self.unk = False``). Default is None.

encoding : str or None

Specifies the text encoding for corpus. Common entries are 'utf-8',

'utf-8-sig', 'utf-16'. Default is None.

return self._train(corpus_fp, vocab=vocab, encoding=encoding)

def _train(self, corpus_fp, vocab=None, encoding=None):

"""Actual N-gram training logic"""

H = self.hyperparameters

grams = {N: [] for N in range(1, self.N + 1)}

counts = {N: Counter() for N in range(1, self.N + 1)}

filter_stop, filter_punc = H["filter_stopwords"], H["filter_punctuation"]

_n_words = 0

tokens = {"<unk>"}

bol, eol = ["<bol>"], ["<eol>"]

with open(corpus_fp, "r", encoding=encoding) as text:

for line in text:

line = strip_punctuation(line) if filter_punc else line

words = tokenize_words(line, filter_stopwords=filter_stop)

if vocab is not None:

words = vocab.filter(words, H["unk"])

if len(words) == 0:

continue

_n_words += len(words)

tokens.update(words)

# calculate n, n-1, ... 1-grams

for N in range(1, self.N + 1):

words_padded = bol * max(1, N - 1) + words + eol * max(1, N - 1)

grams[N].extend(ngrams(words_padded, N))

for N in counts.keys():

counts[N].update(grams[N])

n_words = {N: np.sum(list(counts[N].values())) for N in range(1, self.N + 1)}

n_words[1] = _n_words

n_tokens = {N: len(counts[N]) for N in range(2, self.N + 1)}

n_tokens[1] = len(vocab) if vocab is not None else len(tokens)

self.counts = counts

self.n_words = n_words

self.n_tokens = n_tokens

def completions(self, words, N):

Return the distribution over proposed next words under the `N`-gram

language model.

Parameters

words : list or tuple of strings

The initial sequence of words

N : int

The gram-size of the language model to use to generate completions

Returns

probs : list of (word, log_prob) tuples

The list of possible next words and their log probabilities under

the `N`-gram language model (unsorted)

N = min(N, len(words) + 1)

assert N in self.counts, "You do not have counts for {}-grams".format(N)

assert len(words) >= N - 1, "`words` must have at least {} words".format(N - 1)

probs = []

base = tuple(w.lower() for w in words[-N + 1 :])

for k in self.counts[N].keys():

if k[:-1] == base:

c_prob = self._log_ngram_prob(base + k[-1:])

probs.append((k[-1], c_prob))

return probs

def generate(self, N, seed_words=["<bol>"], n_sentences=5):

Use the `N`-gram language model to generate sentences.

Parameters

N : int

The gram-size of the model to generate from

seed_words : list of strs

A list of seed words to use to condition the initial sentence

generation. Default is ``["<bol>"]``.

sentences : int

The number of sentences to generate from the `N`-gram model.

Default is 50.

Returns

sentences : str

Samples from the `N`-gram model, joined by white spaces, with

individual sentences separated by newlines.

counter = 0

sentences = []

words = seed_words.copy()

while counter < n_sentences:

nextw, probs = zip(*self.completions(words, N))

probs = np.exp(probs) / np.exp(probs).sum()  # renormalize probs if smoothed

next_word = np.random.choice(nextw, p=probs)

# if we reach the end of a sentence, save it and start a new one

if next_word == "<eol>":

S = " ".join([w for w in words if w != "<bol>"])

S = textwrap.fill(S, 90, initial_indent="", subsequent_indent="   ")

print(S)

sentences.append(words)

words = seed_words.copy()

counter += 1

continue

return sentences

def perplexity(self, words, N):

r"""

Calculate the model perplexity on a sequence of words.

Notes

Perplexity, `PP`, is defined as

.. math::

PP(W)  =  \left( \frac{1}{p(W)} \right)^{1 / n}

or simply

.. math::

PP(W)  &=  \exp(-\log p(W) / n) \\

&=  \exp(H(W))

where :math:`W = [w_1, \ldots, w_k]` is a sequence of words, `H(w)` is

the cross-entropy of `W` under the current model, and `n` is the number

of `N`-grams in `W`.

Minimizing perplexity is equivalent to maximizing the probability of

`words` under the `N`-gram model. It may also be interpreted as the

average branching factor when predicting the next word under the

language model.

Parameters

N : int

The gram-size of the model to calculate perplexity with.

words : list or tuple of strings

The sequence of words to compute perplexity on.

Returns

perplexity : float

The model perlexity for the words in `words`.

return np.exp(self.cross_entropy(words, N))

def cross_entropy(self, words, N):

r"""

Calculate the model cross-entropy on a sequence of words against the

empirical distribution of words in a sample.

Notes

Model cross-entropy, `H`, is defined as

.. math::

H(W) = -\frac{\log p(W)}{n}

where :math:`W = [w_1, \ldots, w_k]` is a sequence of words, and `n` is

the number of `N`-grams in `W`.

The model cross-entropy is proportional (not equal, since we use base

`e`) to the average number of bits necessary to encode `W` under the

model distribution.

Parameters

N : int

The gram-size of the model to calculate cross-entropy on.

words : list or tuple of strings

The sequence of words to compute cross-entropy on.

Returns

H : float

The model cross-entropy for the words in `words`.

n_ngrams = len(ngrams(words, N))

return -(1 / n_ngrams) * self.log_prob(words, N)

def _log_prob(self, words, N):

Calculate the log probability of a sequence of words under the

`N`-gram model

if N > len(words):

err = "Not enough words for a gram-size of {}: {}".format(N, len(words))

raise ValueError(err)

total_prob = 0

for ngram in ngrams(words, N):

total_prob += self._log_ngram_prob(ngram)

return total_prob

def _n_completions(self, words, N):

Return the number of unique word tokens that could follow the sequence

`words` under the *unsmoothed* `N`-gram language model.

assert len(words) <= N - 1, "Need > {} words to use {}-grams".format(N - 2, N)

if isinstance(words, list):

words = tuple(words)

base = words[-N + 1 :]

return len([k[-1] for k in self.counts[N].keys() if k[:-1] == base])

def _num_grams_with_count(self, C, N):

Return the number of unique `N`-gram tokens that occur exactly `C`

times

assert C > 0

# cache count values for future calls

if not hasattr(self, "_NC"):

self._NC = {N: {} for N in range(1, self.N + 1)}

if C not in self._NC[N]:

self._NC[N][C] = len([k for k, v in self.counts[N].items() if v == C])

return self._NC[N][C]

@abstractmethod

def log_prob(self, words, N):

Compute the log probability of a sequence of words under the

unsmoothed, maximum-likelihood `N`-gram language model.

raise NotImplementedError

@abstractmethod

def _log_ngram_prob(self, ngram):

"""Return the unsmoothed log probability of the ngram"""

raise NotImplementedError

class MLENGram(NGramBase):

A simple, unsmoothed N-gram model.

Parameters

N : int

The maximum length (in words) of the context-window to use in the

langauge model. Model will compute all n-grams from 1, ..., N.

unk : bool

Whether to include the ``<unk>`` (unknown) token in the LM. Default

is True.

filter_stopwords : bool

Whether to remove stopwords before training. Default is True.

filter_punctuation : bool

Whether to remove punctuation before training. Default is True.

super().__init__(N, unk, filter_stopwords, filter_punctuation)

self.hyperparameters["id"] = "MLENGram"

def log_prob(self, words, N):

Parameters

words : list of strings

A sequence of words

N : int

The gram-size of the language model to use when calculating the log

probabilities of the sequence

Returns

total_prob : float

The total log-probability of the sequence `words` under the

`N`-gram language model

return self._log_prob(words, N)

def _log_ngram_prob(self, ngram):

N = len(ngram)

num = self.counts[N][ngram]

den = self.counts[N - 1][ngram[:-1]] if N > 1 else self.n_words[1]

return np.log(num) - np.log(den) if (den > 0 and num > 0) else -np.inf

class AdditiveNGram(NGramBase):

def __init__(

self, N, K=1, unk=True, filter_stopwords=True, filter_punctuation=True,

An N-Gram model with smoothed probabilities calculated via additive /

Lidstone smoothing.

Notes

The resulting estimates correspond to the expected value of the

posterior, `p(ngram_prob | counts)`, when using a symmetric Dirichlet

prior on counts with parameter `K`.

Parameters

N : int

langauge model. Model will compute all n-grams from 1, ..., N

K : float

The pseudocount to add to each observation. Larger values allocate

more probability toward unseen events. When `K` = 1, the model is

known as Laplace smoothing.  When `K` = 0.5, the model is known as

expected likelihood estimation (ELE) or the Jeffreys-Perks law.

Default is 1.

unk : bool

is True.

filter_stopwords : bool

Whether to remove stopwords before training. Default is True.

filter_punctuation : bool

Whether to remove punctuation before training. Default is True.

super().__init__(N, unk, filter_stopwords, filter_punctuation)

self.hyperparameters["id"] = "AdditiveNGram"

self.hyperparameters["K"] = K

def log_prob(self, words, N):

r"""

Compute the smoothed log probability of a sequence of words under the

`N`-gram language model with additive smoothing.

Notes

For a bigram, additive smoothing amounts to:

.. math::

P(w_i \mid w_{i-1}) = \frac{A + K}{B + KV}

where

.. math::

A  &=  \text{Count}(w_{i-1}, w_i) \\

B  &=  \sum_j \text{Count}(w_{i-1}, w_j) \\

V  &= |\{ w_j \ : \ \text{Count}(w_{i-1}, w_j) > 0 \}|

This is equivalent to pretending we've seen every possible `N`-gram

sequence at least `K` times.

Additive smoothing can be problematic, as it:

- Treats each predicted word in the same way

- Can assign too much probability mass to unseen `N`-grams

Parameters

words : list of strings

A sequence of words.

N : int

probabilities of the sequence.

Returns

total_prob : float

The total log-probability of the sequence `words` under the

`N`-gram language model.

return self._log_prob(words, N)

def _log_ngram_prob(self, ngram):

"""Return the smoothed log probability of the ngram"""

N = len(ngram)

K = self.hyperparameters["K"]

counts, n_words, n_tokens = self.counts, self.n_words[1], self.n_tokens[1]

ctx = ngram[:-1]

num = counts[N][ngram] + K

ctx_count = counts[N - 1][ctx] if N > 1 else n_words

den = ctx_count + K * n_tokens

return np.log(num / den) if den != 0 else -np.inf

class GoodTuringNGram(NGramBase):

def __init__(

self, N, conf=1.96, unk=True, filter_stopwords=True, filter_punctuation=True,

An N-Gram model with smoothed probabilities calculated with the simple

Good-Turing estimator from Gale (2001).

Parameters

N : int

The maximum length (in words) of the context-window to use in the

langauge model. Model will compute all n-grams from 1, ..., N.

conf: float

The multiplier of the standard deviation of the empirical smoothed

count (the default, 1.96, corresponds to a 95% confidence

interval). Controls how many datapoints are smoothed using the

log-linear model.

unk : bool

is True.

filter_stopwords : bool

Whether to remove stopwords before training. Default is True.

filter_punctuation : bool

Whether to remove punctuation before training. Default is True.

super().__init__(N, unk, filter_stopwords, filter_punctuation)

self.hyperparameters["id"] = "GoodTuringNGram"

self.hyperparameters["conf"] = conf

def train(self, corpus_fp, vocab=None, encoding=None):

Compile the n-gram counts for the text(s) in `corpus_fp`. Upon

completion the `self.counts` attribute will store dictionaries of the

`N`, `N-1`, ..., 1-gram counts.

Parameters

corpus_fp : str

The path to a newline-separated text corpus file

vocab : :class:`~numpy_ml.preprocessing.nlp.Vocabulary` instance or None.

If not None, only the words in `vocab` will be used to construct

the language model; all out-of-vocabulary words will either be

mappend to ``<unk>`` (if ``self.unk = True``) or removed (if

``self.unk = False``). Default is None.

encoding : str  or None

'utf-8-sig', 'utf-16'. Default is None.

self._train(corpus_fp, vocab=vocab, encoding=encoding)

def log_prob(self, words, N):

r"""

`N`-gram language model with Good-Turing smoothing.

Notes

For a bigram, Good-Turing smoothing amounts to:

.. math::

P(w_i \mid w_{i-1}) = \frac{C^*}{\text{Count}(w_{i-1})}

where :math:`C^*` is the Good-Turing smoothed estimate of the bigram

count:

.. math::

C^* = \frac{(c + 1) \text{NumCounts}(c + 1, 2)}{\text{NumCounts}(c, 2)}

where

.. math::

c  &=  \text{Count}(w_{i-1}, w_i) \\

\text{NumCounts}(r, k)  &=

|\{ k\text{-gram} : \text{Count}(k\text{-gram}) = r \}|

In words, the probability of an `N`-gram that occurs `r` times in the

corpus is estimated by dividing up the probability mass occupied by

N-grams that occur `r+1` times.

For large values of `r`, NumCounts becomes unreliable. In this case, we

compute a smoothed version of NumCounts using a power law function:

.. math::

\log \text{NumCounts}(r) = b + a \log r

Under the Good-Turing estimator, the total probability assigned to

unseen `N`-grams is equal to the relative occurrence of `N`-grams that

appear only once.

Parameters

words : list of strings

A sequence of words.

N : int

probabilities of the sequence.

Returns

total_prob : float

The total log-probability of the sequence `words` under the

`N`-gram language model.

return self._log_prob(words, N)

def _calc_smoothed_counts(self):

use_interp = False

counts = self.counts

NC = self._num_grams_with_count

conf = self.hyperparameters["conf"]

totals = {N: 0 for N in range(1, self.N + 1)}

smooth_counts = {N: {} for N in range(1, self.N + 1)}

# calculate the probability of all <unk> (i.e., unseen) n-grams

self._p0 = {n: NC(1, n) / sum(counts[n].values()) for n in range(1, self.N + 1)}

# fit log-linear models for predicting smoothed counts in absence of

# real data

LM = self._count_models

for N in range(1, self.N + 1):

for C in sorted(set(counts[N].values())):

# estimate the interpolated count using the log-linear model

c1_lm = np.exp(LM[N].predict(np.c_[np.log(C + 1)])).item()

c0_lm = np.exp(LM[N].predict(np.c_[np.log(C)])).item()

count_interp = ((C + 1) * c1_lm) / c0_lm

# if we have previously been using the interpolated count, or

# if the number of ocurrences of C+1 is 0, use the interpolated

# count as the smoothed count value C*

c1, c0 = NC(C + 1, N), NC(C, N)

if use_interp or c1 == 0:

use_interp = True

smooth_counts[N][C] = count_interp

totals[N] += c0 * smooth_counts[N][C]

continue

# estimate the smoothed count C* empirically if the number of

# terms with count C + 1 > 0

count_emp = ((C + 1) * c1) / c0

# compute the approximate variance of the empirical smoothed

# count C* given C

t = conf * np.sqrt((C + 1) ** 2 * (c1 / c0 ** 2) * (1 + c1 / c0))

# if the difference between the empirical and interpolated

# smoothed counts is greater than t, the empirical estimate

# tends to be more accurate. otherwise, use interpolated

if np.abs(count_interp - count_emp) > t:

smooth_counts[N][C] = count_emp

totals[N] += c0 * smooth_counts[N][C]

continue

use_interp = True

smooth_counts[N][C] = count_interp

totals[N] += c0 * smooth_counts[N][C]

self._smooth_totals = totals

self._smooth_counts = smooth_counts

def _log_ngram_prob(self, ngram):

"""Return the smoothed log probability of the ngram"""

N = len(ngram)

sc, T = self._smooth_counts[N], self._smooth_totals[N]

n_tokens, n_seen = self.n_tokens[N], len(self.counts[N])

# approx. prob of an out-of-vocab ngram (i.e., a fraction of p0)

n_unseen = max((n_tokens ** N) - n_seen, 1)

prob = np.log(self._p0[N] / n_unseen)

if ngram in self.counts[N]:

C = self.counts[N][ngram]

prob = np.log(1 - self._p0[N]) + np.log(sc[C]) - np.log(T)

return prob

def _fit_count_models(self):

Perform the averaging transform proposed by Church and Gale (1991):

estimate the expected count-of-counts by the *density* of

count-of-count values.

self._count_models = {}

NC = self._num_grams_with_count

for N in range(1, self.N + 1):

X, Y = [], []

sorted_counts = sorted(set(self.counts[N].values()))  # r

for ix, j in enumerate(sorted_counts):

i = 0 if ix == 0 else sorted_counts[ix - 1]

k = 2 * j - i if ix == len(sorted_counts) - 1 else sorted_counts[ix + 1]

y = 2 * NC(j, N) / (k - i)

X.append(j)

Y.append(y)

# fit log-linear model: log(counts) ~ log(average_transform(counts))

self._count_models[N] = LinearRegression(fit_intercept=True)

self._count_models[N].fit(np.log(X), np.log(Y))

b, a = self._count_models[N].beta

if a > -1:

fstr = "[Warning] Log-log averaging transform has slope > -1 for N={}"

print(fstr.format(N))


from .ngram import *


from . import rl_utils
from . import agents
from . import trainer
from . import tiles


"""Reinforcement learning agents that can be run on OpenAI gym environs"""

from abc import ABC, abstractmethod

from collections import defaultdict

import numpy as np

from .rl_utils import EnvModel, env_stats, tile_state_space

from ..utils.data_structures import Dict

class AgentBase(ABC):

def __init__(self, env):

self.env = env

self.parameters = {}

self.hyperparameters = {}

self.derived_variables = {}

self.env_info = env_stats(env)

def _create_2num_dicts(self, obs_encoder=None, act_encoder=None):

E = self.env_info

n_states = np.prod(E["n_obs_per_dim"])

n_actions = np.prod(E["n_actions_per_dim"])

# create action -> scalar dictionaries

self._num2action = Dict()

self._action2num = Dict(act_encoder)

if n_actions != np.inf:

self._action2num = {act: i for i, act in enumerate(E["action_ids"])}

self._num2action = {i: act for act, i in self._action2num.items()}

# create obs -> scalar dictionaries

self._num2obs = Dict()

self._obs2num = Dict(obs_encoder)

if n_states != np.inf:

self._obs2num = {act: i for i, act in enumerate(E["obs_ids"])}

self._num2obs = {i: act for act, i in self._obs2num.items()}

def flush_history(self):

"""Clear the episode history"""

for k, v in self.episode_history.items():

self.episode_history[k] = []

@abstractmethod

def act(self, obs):

"""Generate an action given the current observation"""

raise NotImplementedError

@abstractmethod

def greedy_policy(self, **kwargs):

Take a greedy action.

Returns

total_reward : float

The total reward on the episode.

n_steps : float

The total number of steps taken on the episode.

raise NotImplementedError

@abstractmethod

def run_episode(self, max_steps, render=False):

Run the agent on a single episode.

Parameters

max_steps : int

The maximum number of steps to run an episode

render : bool

Whether to render the episode during training

Returns

reward : float

The total reward on the episode, averaged over the theta samples.

steps : float

The total number of steps taken on the episode, averaged over the

theta samples.

raise NotImplementedError

@abstractmethod

def update(self):

r"""

Update the agent parameters according to the rewards accrued on the

current episode.

Returns

avg_reward : float

The average reward earned by the best `retain_prcnt` theta samples

on the current episode.

raise NotImplementedError

class CrossEntropyAgent(AgentBase):

def __init__(self, env, n_samples_per_episode=500, retain_prcnt=0.2):

r"""

A cross-entropy method agent.

Notes

The cross-entropy method [1]_ [2]_ agent only operates on ``envs`` with

discrete action spaces.

On each episode the agent generates `n_theta_samples` of the parameters

(:math:`\theta`) for its behavior policy. The `i`'th sample at

timestep `t` is:

.. math::

\theta_i  &=  \{\mathbf{W}_i^{(t)}, \mathbf{b}_i^{(t)} \} \\

\theta_i  &\sim  \mathcal{N}(\mu^{(t)}, \Sigma^{(t)})

Weights (:math:`\mathbf{W}_i`) and bias (:math:`\mathbf{b}_i`) are the

parameters of the softmax policy:

.. math::

\mathbf{z}_i  &=  \text{obs} \cdot \mathbf{W}_i + \mathbf{b}_i \\

p(a_i^{(t + 1)})  &=  \frac{e^{\mathbf{z}_i}}{\sum_j e^{z_{ij}}} \\

a^{(t + 1)}  &=  \arg \max_j p(a_j^{(t+1)})

At the end of each episode, the agent takes the top `retain_prcnt`

highest scoring :math:`\theta` samples and combines them to generate

the mean and variance of the distribution of :math:`\theta` for the

next episode:

.. math::

\mu^{(t+1)}  &=  \text{avg}(\texttt{best_thetas}^{(t)}) \\

\Sigma^{(t+1)}  &=  \text{var}(\texttt{best_thetas}^{(t)})

References

.. [1] Mannor, S., Rubinstein, R., & Gat, Y. (2003). The cross entropy

method for fast policy search. In *Proceedings of the 20th Annual

ICML, 20*.

.. [2] Rubinstein, R. (1997). optimization of computer simulation

models with rare events, *European Journal of Operational Research,

99*, 89–112.

Parameters

env : :meth:`gym.wrappers` or :meth:`gym.envs` instance

The environment to run the agent on.

n_samples_per_episode : int

The number of theta samples to evaluate on each episode. Default is 500.

retain_prcnt: float

The percentage of `n_samples_per_episode` to use when calculating

the parameter update at the end of the episode. Default is 0.2.

self.retain_prcnt = retain_prcnt

self.n_samples_per_episode = n_samples_per_episode

def _init_params(self):

E = self.env_info

assert not E["continuous_actions"], "Action space must be discrete"

b_len = np.prod(E["n_actions_per_dim"])

W_len = b_len * np.prod(E["obs_dim"])

theta_dim = b_len + W_len

# init mean and variance for mv gaussian with dimensions theta_dim

theta_mean = np.random.rand(theta_dim)

theta_var = np.ones(theta_dim)

self.parameters = {"theta_mean": theta_mean, "theta_var": theta_var}

self.derived_variables = {

"b_len": b_len,

"W_len": W_len,

"W_samples": [],

"b_samples": [],

"episode_num": 0,

"cumulative_rewards": [],

self.hyperparameters = {

"agent": "CrossEntropyAgent",

"retain_prcnt": self.retain_prcnt,

"n_samples_per_episode": self.n_samples_per_episode,

self.episode_history = {"rewards": [], "state_actions": []}

def act(self, obs):

r"""

Generate actions according to a softmax policy.

Notes

The softmax policy assumes that the pmf over actions in state :math:`x_t` is

given by:

.. math::

\pi(a | x^{(t)}) = \text{softmax}(

\text{obs}^{(t)} \cdot \mathbf{W}_i^{(t)} + \mathbf{b}_i^{(t)} )

where :math:`\mathbf{W}` is a learned weight matrix, `obs` is the observation

at timestep `t`, and **b** is a learned bias vector.

Parameters

obs : int or :py:class:`ndarray <numpy.ndarray>`

An observation from the environment.

Returns

action : int, float, or :py:class:`ndarray <numpy.ndarray>`

An action sampled from the distribution over actions defined by the

softmax policy.

E, P = self.env_info, self.parameters

W, b = P["W"], P["b"]

s = self._obs2num[obs]

s = np.array([s]) if E["obs_dim"] == 1 else s

# compute softmax

Z = s.T @ W + b

e_Z = np.exp(Z - np.max(Z, axis=-1, keepdims=True))

action_probs = e_Z / e_Z.sum(axis=-1, keepdims=True)

# sample action

a = np.random.multinomial(1, action_probs).argmax()

return self._num2action[a]

def run_episode(self, max_steps, render=False):

Run the agent on a single episode.

Parameters

max_steps : int

The maximum number of steps to run an episode

render : bool

Whether to render the episode during training

Returns

reward : float

The total reward on the episode, averaged over the theta samples.

steps : float

The total number of steps taken on the episode, averaged over the

theta samples.

E, D = self.env_info, self.derived_variables

n_actions = np.prod(E["n_actions_per_dim"])

W_len, obs_dim = D["W_len"], E["obs_dim"]

steps, rewards = [], []

for theta in D["theta_samples"]:

W = theta[:W_len].reshape(obs_dim, n_actions)

b = theta[W_len:]

total_rwd, n_steps = self._episode(W, b, max_steps, render)

# return the average reward and average number of steps across all

# samples on the current episode

D["episode_num"] += 1

D["cumulative_rewards"] = rewards

return np.mean(D["cumulative_rewards"]), np.mean(steps)

def _episode(self, W, b, max_steps, render):

Run the agent for an episode.

Parameters

W : :py:class:`ndarray <numpy.ndarray>` of shape `(obs_dim, n_actions)`

The weights for the softmax policy.

b : :py:class:`ndarray <numpy.ndarray>` of shape `(bias_len, )`

The bias for the softmax policy.

max_steps : int

The maximum number of steps to run the episode.

render : bool

Whether to render the episode during training.

Returns

reward : float

The total reward on the episode.

steps : float

The total number of steps taken on the episode.

rwds, sa = [], []

H = self.episode_history

total_reward, n_steps = 0.0, 1

obs = self.env.reset()

self.parameters["W"] = W

self.parameters["b"] = b

for i in range(max_steps):

if render:

self.env.render()

n_steps += 1

action = self.act(obs)

s, a = self._obs2num[obs], self._action2num[action]

sa.append((s, a))

obs, reward, done, _ = self.env.step(action)

rwds.append(reward)

total_reward += reward

if done:

break

H["rewards"].append(rwds)

return total_reward, n_steps

def update(self):

r"""

Update :math:`\mu` and :math:`\Sigma` according to the rewards accrued on

the current episode.

Returns

avg_reward : float

The average reward earned by the best `retain_prcnt` theta samples

on the current episode.

D, P = self.derived_variables, self.parameters

n_retain = int(self.retain_prcnt * self.n_samples_per_episode)

# sort the cumulative rewards for each theta sample from greatest to least

sorted_y_val_idxs = np.argsort(D["cumulative_rewards"])[::-1]

top_idxs = sorted_y_val_idxs[:n_retain]

# update theta_mean and theta_var with the best theta value

P["theta_mean"] = np.mean(D["theta_samples"][top_idxs], axis=0)

P["theta_var"] = np.var(D["theta_samples"][top_idxs], axis=0)

def _sample_thetas(self):

Sample `n_samples_per_episode` thetas from a multivariate Gaussian with

mean `theta_mean` and covariance `diag(theta_var)`

P, N = self.parameters, self.n_samples_per_episode

Mu, Sigma = P["theta_mean"], np.diag(P["theta_var"])

samples = np.random.multivariate_normal(Mu, Sigma, N)

self.derived_variables["theta_samples"] = samples

def greedy_policy(self, max_steps, render=True):

Execute a greedy policy using the current agent parameters.

Parameters

max_steps : int

The maximum number of steps to run the episode.

render : bool

Whether to render the episode during execution.

Returns

total_reward : float

The total reward on the episode.

n_steps : float

The total number of steps taken on the episode.

E, D, P = self.env_info, self.derived_variables, self.parameters

sample = np.random.multivariate_normal(Mu, Sigma, 1)

W_len, obs_dim = D["W_len"], E["obs_dim"]

n_actions = np.prod(E["n_actions_per_dim"])

W = sample[0, :W_len].reshape(obs_dim, n_actions)

b = sample[0, W_len:]

total_reward, n_steps = self._episode(W, b, max_steps, render)

return total_reward, n_steps

class MonteCarloAgent(AgentBase):

def __init__(self, env, off_policy=False, temporal_discount=0.9, epsilon=0.1):

A Monte-Carlo learning agent trained using either first-visit Monte

Carlo updates (on-policy) or incremental weighted importance sampling

(off-policy).

Parameters

env : :class:`gym.wrappers` or :class:`gym.envs` instance

The environment to run the agent on.

off_policy : bool

Whether to use a behavior policy separate from the target policy

during training. If False, use the same epsilon-soft policy for

both behavior and target policies. Default is False.

temporal_discount : float between [0, 1]

The discount factor used for downweighting future rewards. Smaller

values result in greater discounting of future rewards. Default is

epsilon : float between [0, 1]

The epsilon value in the epsilon-soft policy. Larger values

encourage greater exploration during training. Default is 0.1.

self.epsilon = epsilon

self.off_policy = off_policy

self.temporal_discount = temporal_discount

def _init_params(self):

E = self.env_info

assert not E["continuous_actions"], "Action space must be discrete"

assert not E["continuous_observations"], "Observation space must be discrete"

n_states = np.prod(E["n_obs_per_dim"])

n_actions = np.prod(E["n_actions_per_dim"])

# behavior policy is stochastic, epsilon-soft policy

self.behavior_policy = self.target_policy = self._epsilon_soft_policy

if self.off_policy:

self.parameters["C"] = np.zeros((n_states, n_actions))

# target policy is deterministic, greedy policy

self.target_policy = self._greedy

# initialize Q function

self.parameters["Q"] = np.random.rand(n_states, n_actions)

# initialize returns object for each state-action pair

self.derived_variables = {

"returns": {(s, a): [] for s in range(n_states) for a in range(n_actions)},

"episode_num": 0,

self.hyperparameters = {

"agent": "MonteCarloAgent",

"epsilon": self.epsilon,

"off_policy": self.off_policy,

"temporal_discount": self.temporal_discount,

self.episode_history = {"state_actions": [], "rewards": []}

def _epsilon_soft_policy(self, s, a=None):

r"""

Epsilon-soft exploration policy.

Notes

Soft policies are necessary for first-visit Monte Carlo methods, as

they require continual exploration (i.e., each state-action pair must

have nonzero probability of occurring).

In epsilon-soft policies, :math:`\pi(a \mid s) > 0` for all :math:`s

\in S` and all :math:`a \in A(s)` at the start of training. As learning

progresses, :math:`pi` gradually shifts closer and closer to a

deterministic optimal policy.

In particular, we have:

.. math::

\pi(a \mid s)  &=

1 - \epsilon + \frac{\epsilon}{|A(s)|}  &&\text{if} a = a^*

\pi(a \mid s)  &=

\frac{\epsilon}{|A(s)|}                 &&\text{if} a \neq a^*

where :math:`|A(s)|` is the number of actions available in state `s`

and :math:`a^* \in A(s)` is the greedy action in state `s` (i.e.,

:math:`a^* = \arg \max_a Q(s, a)`).

Note that epsilon-greedy policies are instances of epsilon-soft

policies, defined as policies for which :math:`\pi(a|s) \geq \epsilon / |A(s)|`

for all states and actions.

Parameters

s : int, float, or tuple

The state number for the current observation, as returned by

a : int, float, tuple, or None

The action number in the current state, as returned by

``self._action2num[obs]``. If None, sample an action from the

action probabilities in state `s`, otherwise, return the

probability of action `a` under the epsilon-soft policy. Default is

None.

Returns

action : int, float, or :py:class:`ndarray <numpy.ndarray>`

If `a` is None, this is an action sampled from the distribution

over actions defined by the epsilon-soft policy. If `a` is not

None, this is the probability of `a` under the epsilon-soft policy.

E, P = self.env_info, self.parameters

# TODO: this assumes all actions are available in every state

n_actions = np.prod(E["n_actions_per_dim"])

a_star = P["Q"][s, :].argmax()

p_a_star = 1.0 - self.epsilon + (self.epsilon / n_actions)

p_a = self.epsilon / n_actions

action_probs = np.ones(n_actions) * p_a

action_probs[a_star] = p_a_star

np.testing.assert_allclose(np.sum(action_probs), 1)

if a is not None:

return action_probs[a]

# sample action

a = np.random.multinomial(1, action_probs).argmax()

return self._num2action[a]

def _greedy(self, s, a=None):

A greedy behavior policy.

Notes

Only used when off-policy is True.

Parameters

s : int, float, or tuple

a : int, float, or tuple

``self._action2num[obs]``. If None, sample an action from the action

probabilities in state `s`, otherwise, return the probability of

action `a` under the greedy policy. Default is None.

Returns

over actions defined by the greedy policy. If `a` is not

None, this is the probability of `a` under the greedy policy.

a_star = self.parameters["Q"][s, :].argmax()

if a is None:

out = self._num2action[a_star]

else:

out = 1 if a == a_star else 0

return out

def _on_policy_update(self):

r"""

Update the `Q` function using an on-policy first-visit Monte Carlo

update.

Notes

The on-policy first-visit Monte Carlo update is

.. math::

Q'(s, a) \leftarrow

\text{avg}(\text{reward following first visit to } (s, a)

\text{ across all episodes})

RL agents seek to learn action values conditional on subsequent optimal

behavior, but they need to behave non-optimally in order to explore all

actions (to find the optimal actions).

The on-policy approach is a compromise -- it learns action values not

for the optimal policy, but for a *near*-optimal policy that still

explores (the epsilon-soft policy).

D, P, HS = self.derived_variables, self.parameters, self.episode_history

ep_rewards = HS["rewards"]

sa_tuples = set(HS["state_actions"])

locs = [HS["state_actions"].index(sa) for sa in sa_tuples]

cumulative_returns = [np.sum(ep_rewards[i:]) for i in locs]

# update Q value with the average of the first-visit return across

# episodes

for (s, a), cr in zip(sa_tuples, cumulative_returns):

D["returns"][(s, a)].append(cr)

P["Q"][s, a] = np.mean(D["returns"][(s, a)])

def _off_policy_update(self):

Update `Q` using weighted importance sampling.

Notes

In importance sampling updates, we account for the fact that we are

updating a different policy from the one we used to generate behavior

by weighting the accumulated rewards by the ratio of the probability of

the trajectory under the target policy versus its probability under

the behavior policies. This is known as the importance sampling weight.

In weighted importance sampling, we scale the accumulated rewards for a

trajectory by their importance sampling weight, then take the

*weighted* average using the importance sampling weight. This weighted

average then becomes the value for the trajectory.

W   = importance sampling weight

G_t = total discounted reward from time t until episode end

C_n = sum of importance weights for the first n rewards

This algorithm converges to Q* in the limit.

P = self.parameters

HS = self.episode_history

ep_rewards = HS["rewards"]

T = len(ep_rewards)

G, W = 0.0, 1.0

for t in reversed(range(T)):

s, a = HS["state_actions"][t]

G = self.temporal_discount * G + ep_rewards[t]

P["C"][s, a] += W

# update Q(s, a) using weighted importance sampling

P["Q"][s, a] += (W / P["C"][s, a]) * (G - P["Q"][s, a])

# multiply the importance sampling ratio by the current weight

W *= self.target_policy(s, a) / self.behavior_policy(s, a)

if W == 0.0:

break

def act(self, obs):

r"""

Execute the behavior policy--an :math:`\epsilon`-soft policy used to

generate actions during training.

Parameters

obs : int, float, or :py:class:`ndarray <numpy.ndarray>` as returned by ``env.step(action)``

An observation from the environment.

Returns

action : int, float, or :py:class:`ndarray <numpy.ndarray>`

An action sampled from the distribution over actions defined by the

epsilon-soft policy.

"""  # noqa: E501

s = self._obs2num[obs]

return self.behavior_policy(s)

def run_episode(self, max_steps, render=False):

Run the agent on a single episode.

Parameters

max_steps : int

The maximum number of steps to run an episode.

render : bool

Whether to render the episode during training.

Returns

reward : float

The total reward on the episode.

steps : float

The number of steps taken on the episode.

D = self.derived_variables

total_rwd, n_steps = self._episode(max_steps, render)

D["episode_num"] += 1

return total_rwd, n_steps

def _episode(self, max_steps, render):

Execute agent on an episode.

Parameters

max_steps : int

The maximum number of steps to run the episode.

render : bool

Returns

reward : float

The total reward on the episode.

steps : float

obs = self.env.reset()

HS = self.episode_history

total_reward, n_steps = 0.0, 0

for i in range(max_steps):

if render:

self.env.render()

n_steps += 1

action = self.act(obs)

s = self._obs2num[obs]

a = self._action2num[action]

# store (state, action) tuple

HS["state_actions"].append((s, a))

# take action

obs, reward, done, info = self.env.step(action)

# record rewards

HS["rewards"].append(reward)

total_reward += reward

if done:

break

return total_reward, n_steps

def update(self):

Update the parameters of the model following the completion of an

episode. Flush the episode history after the update is complete.

H = self.hyperparameters

if H["off_policy"]:

else:

def greedy_policy(self, max_steps, render=True):

Execute a greedy policy using the current agent parameters.

Parameters

max_steps : int

render : bool

Whether to render the episode during execution.

Returns

total_reward : float

The total reward on the episode.

n_steps : float

The total number of steps taken on the episode.

H = self.episode_history

obs = self.env.reset()

total_reward, n_steps = 0.0, 0

for i in range(max_steps):

if render:

self.env.render()

n_steps += 1

action = self._greedy(obs)

s = self._obs2num[obs]

a = self._action2num[action]

# store (state, action) tuple

H["state_actions"].append((s, a))

# take action

# record rewards

H["rewards"].append(reward)

total_reward += reward

if done:

break

return total_reward, n_steps

class TemporalDifferenceAgent(AgentBase):

def __init__(

self,

env,

grid_dims=[8, 8],

r"""

A temporal difference learning agent with expected SARSA (on-policy) [3]_ or

TD(0) `Q`-learning (off-policy) [4]_ updates.

Notes

The expected SARSA on-policy TD(0) update is:

.. math::

Q(s, a) \leftarrow Q(s, a) + \eta \left(

r + \gamma \mathbb{E}_\pi[Q(s', a') \mid s'] - Q(s, a)

\right)

and the TD(0) off-policy Q-learning upate is:

.. math::

Q(s, a) \leftarrow Q(s, a) + \eta (

r + \gamma \max_a \left\{ Q(s', a) \right\} - Q(s, a)

where in each case we have taken action `a` in state `s`, received

reward `r`, and transitioned into state :math:`s'`. In the above

equations, :math:`\eta` is a learning rate parameter, :math:`\gamma` is

a temporal discount factor, and :math:`\mathbb{E}_\pi[ Q[s', a'] \mid

s']` is the expected value under the current policy :math:`\pi` of the

Q function conditioned that we are in state :math:`s'`.

Observe that the expected SARSA update can be used for both on- and

off-policy methods. In an off-policy context, if the target policy is

greedy and the expectation is taken wrt. the target policy then the

expected SARSA update is exactly Q-learning.

NB. For this implementation the agent requires a discrete action

space, but will try to discretize the observation space via tiling if

it is continuous.

References

.. [3] Rummery, G. & Niranjan, M. (1994). *On-Line Q-learning Using

Connectionist Systems*. Tech Report 166. Cambridge University

Department of Engineering.

.. [4] Watkins, C. (1989). Learning from delayed rewards. *PhD thesis,

Cambridge University*.

Parameters

env : gym.wrappers or gym.envs instance

The environment to run the agent on.

lr : float

Learning rate for the Q function updates. Default is 0.05.

epsilon : float between [0, 1]

The epsilon value in the epsilon-soft policy. Larger values

encourage greater exploration during training. Default is 0.1.

n_tilings : int

The number of overlapping tilings to use if the ``env`` observation

space is continuous. Unused if observation space is discrete.

Default is 8.

obs_max : float or :py:class:`ndarray <numpy.ndarray>`

The value to treat as the max value of the observation space when

calculating the grid widths if the observation space is continuous.

If None, use ``env.observation_space.high``. Unused if observation

space is discrete. Default is None.

obs_min : float or :py:class:`ndarray <numpy.ndarray>`

The value to treat as the min value of the observation space when

calculating grid widths if the observation space is continuous. If

None, use ``env.observation_space.low``. Unused if observation

space is discrete. Default is None.

grid_dims : list

The number of rows and columns in each tiling grid if the env

observation space is continuous. Unused if observation space is

discrete. Default is [8, 8].

off_policy : bool

Whether to use a behavior policy separate from the target policy

during training. If False, use the same epsilon-soft policy for

both behavior and target policies. Default is False.

temporal_discount : float between [0, 1]

The discount factor used for downweighting future rewards. Smaller

values result in greater discounting of future rewards. Default is

self.lr = lr

self.obs_max = obs_max

self.obs_min = obs_min

self.epsilon = epsilon

self.n_tilings = n_tilings

self.grid_dims = grid_dims

self.off_policy = off_policy

self.temporal_discount = temporal_discount

def _init_params(self):

E = self.env_info

assert not E["continuous_actions"], "Action space must be discrete"

obs_encoder = None

if E["continuous_observations"]:

obs_encoder, _ = tile_state_space(

self.env,

self._create_2num_dicts(obs_encoder=obs_encoder)

# behavior policy is stochastic, epsilon-soft policy

self.behavior_policy = self.target_policy = self._epsilon_soft_policy

if self.off_policy:

# target policy is deterministic, greedy policy

self.target_policy = self._greedy

# initialize Q function

self.parameters["Q"] = defaultdict(np.random.rand)

# initialize returns object for each state-action pair

self.derived_variables = {"episode_num": 0}

self.hyperparameters = {

"agent": "TemporalDifferenceAgent",

"lr": self.lr,

"obs_max": self.obs_max,

"obs_min": self.obs_min,

"epsilon": self.epsilon,

"n_tilings": self.n_tilings,

"grid_dims": self.grid_dims,

"off_policy": self.off_policy,

"temporal_discount": self.temporal_discount,

self.episode_history = {"state_actions": [], "rewards": []}

def run_episode(self, max_steps, render=False):

Run the agent on a single episode without updating the priority queue

or performing backups.

Parameters

max_steps : int

The maximum number of steps to run an episode

render : bool

Whether to render the episode during training

Returns

reward : float

The total reward on the episode, averaged over the theta samples.

steps : float

The total number of steps taken on the episode, averaged over the

theta samples.

return self._episode(max_steps, render, update=False)

def train_episode(self, max_steps, render=False):

Train the agent on a single episode.

Parameters

max_steps : int

The maximum number of steps to run an episode.

render : bool

Whether to render the episode during training.

Returns

reward : float

The total reward on the episode.

steps : float

The number of steps taken on the episode.

D = self.derived_variables

total_rwd, n_steps = self._episode(max_steps, render, update=True)

D["episode_num"] += 1

return total_rwd, n_steps

def _episode(self, max_steps, render, update=True):

Run or train the agent on an episode.

Parameters

max_steps : int

The maximum number of steps to run the episode.

render : bool

update : bool

Whether to perform the Q function backups after each step. Default

is True.

Returns

reward : float

The total reward on the episode.

steps : float

obs = self.env.reset()

HS = self.episode_history

action = self.act(obs)

s = self._obs2num[obs]

a = self._action2num[action]

# store initial (state, action) tuple

HS["state_actions"].append((s, a))

total_reward, n_steps = 0.0, 0

for i in range(max_steps):

if render:

self.env.render()

# take action

obs, reward, done, info = self.env.step(action)

n_steps += 1

# record rewards

HS["rewards"].append(reward)

total_reward += reward

# generate next state and action

action = self.act(obs)

s_ = self._obs2num[obs] if not done else None

a_ = self._action2num[action]

# store next (state, action) tuple

HS["state_actions"].append((s_, a_))

if update:

self.update()

if done:

break

return total_reward, n_steps

def _epsilon_soft_policy(self, s, a=None):

r"""

Epsilon-soft exploration policy.

In epsilon-soft policies, :math:`\pi(a|s) > 0` for all s ∈ S and all a

∈ A(s) at the start of training. As learning progresses, :math:`\pi`

gradually shifts closer and closer to a deterministic optimal policy.

In particular, we have:

pi(a|s) = 1 - epsilon + (epsilon / |A(s)|) IFF a == a*

pi(a|s) = epsilon / |A(s)|                 IFF a != a*

where

|A(s)| is the number of actions available in state s

a* ∈ A(s) is the greedy action in state s (i.e., a* = argmax_a Q(s, a))

Note that epsilon-greedy policies are instances of epsilon-soft

policies, defined as policies for which pi(a|s) >= epsilon / |A(s)| for

all states and actions.

Parameters

s : int, float, or tuple

The state number for the current observation, as returned by

a : int, float, or tuple

The action number in the current state, as returned by

self._action2num[obs]. If None, sample an action from the action

probabilities in state s, otherwise, return the probability of

action `a` under the epsilon-soft policy. Default is None.

Returns

If `a` is None:

action : int, float, or :py:class:`ndarray <numpy.ndarray>` as returned by `self._num2action`

If `a` is None, returns an action sampled from the distribution

over actions defined by the epsilon-soft policy.

If `a` is not None:

action_prob : float in range [0, 1]

If `a` is not None, returns the probability of `a` under the

epsilon-soft policy.

"""  # noqa: E501

E, P = self.env_info, self.parameters

# TODO: this assumes all actions are available in every state

n_actions = np.prod(E["n_actions_per_dim"])

a_star = np.argmax([P["Q"][(s, aa)] for aa in range(n_actions)])

p_a_star = 1.0 - self.epsilon + (self.epsilon / n_actions)

p_a = self.epsilon / n_actions

action_probs = np.ones(n_actions) * p_a

action_probs[a_star] = p_a_star

np.testing.assert_allclose(np.sum(action_probs), 1)

if a is not None:

return action_probs[a]

# sample action

a = np.random.multinomial(1, action_probs).argmax()

return self._num2action[a]

def _greedy(self, s, a=None):

A greedy behavior policy. Only used when off-policy is true.

Parameters

s : int, float, or tuple

a : int, float, or tuple

``self._action2num[obs]``. If None, sample an action from the

action probabilities in state `s`, otherwise, return the

probability of action `a` under the greedy policy. Default is None.

Returns

If `a` is None:

action : int, float, or :py:class:`ndarray <numpy.ndarray>` as returned by ``self._num2action``

over actions defined by the greedy policy.

If `a` is not None:

action_prob : float in range [0, 1]

greedy policy.

"""  # noqa: E501

P, E = self.parameters, self.env_info

if a is None:

out = self._num2action[a_star]

else:

out = 1 if a == a_star else 0

return out

def _on_policy_update(self, s, a, r, s_, a_):

Update the Q function using the expected SARSA on-policy TD(0) update:

Q[s, a] <- Q[s, a] + lr * [

r + temporal_discount * E[Q[s', a'] | s'] - Q[s, a]

where

E[ Q[s', a'] | s'] is the expected value of the Q function over all

a_ given that we're in state s' under the current policy

NB. the expected SARSA update can be used for both on- and off-policy

methods. In an off-policy context, if the target policy is greedy and

the expectation is taken wrt. the target policy then the expected SARSA

update is exactly Q-learning.

Parameters

s : int as returned by `self._obs2num`

The id for the state/observation at timestep t-1

a : int as returned by `self._action2num`

The id for the action taken at timestep t-1

r : float

The reward after taking action `a` in state `s` at timestep t-1

s_ : int as returned by `self._obs2num`

The id for the state/observation at timestep t

a_ : int as returned by `self._action2num`

The id for the action taken at timestep t

Q, E, pi = self.parameters["Q"], self.env_info, self.behavior_policy

# TODO: this assumes that all actions are available in each state

n_actions = np.prod(E["n_actions_per_dim"])

# compute the expected value of Q(s', a') given that we are in state s'

E_Q = np.sum([pi(s_, aa) * Q[(s_, aa)] for aa in range(n_actions)]) if s_ else 0

# perform the expected SARSA TD(0) update

qsa = Q[(s, a)]

Q[(s, a)] = qsa + self.lr * (r + self.temporal_discount * E_Q - qsa)

def _off_policy_update(self, s, a, r, s_):

Update the `Q` function using the TD(0) Q-learning update:

Q[s, a] <- Q[s, a] + lr * (

r + temporal_discount * max_a { Q[s', a] } - Q[s, a]

Parameters

s : int as returned by `self._obs2num`

The id for the state/observation at timestep `t-1`

a : int as returned by `self._action2num`

The id for the action taken at timestep `t-1`

r : float

The reward after taking action `a` in state `s` at timestep `t-1`

s_ : int as returned by `self._obs2num`

The id for the state/observation at timestep `t`

Q, E = self.parameters["Q"], self.env_info

qsa = Q[(s, a)]

Qs_ = [Q[(s_, aa)] for aa in range(n_actions)] if s_ else [0]

Q[(s, a)] = qsa + self.lr * (r + self.temporal_discount * np.max(Qs_) - qsa)

def update(self):

"""Update the parameters of the model online after each new state-action."""

H, HS = self.hyperparameters, self.episode_history

(s, a), r = HS["state_actions"][-2], HS["rewards"][-1]

s_, a_ = HS["state_actions"][-1]

if H["off_policy"]:

self._off_policy_update(s, a, r, s_)

else:

self._on_policy_update(s, a, r, s_, a_)

def act(self, obs):

r"""

Execute the behavior policy--an :math:`\epsilon`-soft policy used to

generate actions during training.

Parameters

An observation from the environment.

Returns

action : int, float, or :py:class:`ndarray <numpy.ndarray>`

An action sampled from the distribution over actions defined by the

epsilon-soft policy.

"""  # noqa: E501

s = self._obs2num[obs]

return self.behavior_policy(s)

def greedy_policy(self, max_steps, render=True):

Execute a deterministic greedy policy using the current agent

parameters.

Parameters

max_steps : int

The maximum number of steps to run the episode.

render : bool

Whether to render the episode during execution.

Returns

total_reward : float

The total reward on the episode.

n_steps : float

The total number of steps taken on the episode.

H = self.episode_history

obs = self.env.reset()

total_reward, n_steps = 0.0, 0

for i in range(max_steps):

if render:

self.env.render()

s = self._obs2num[obs]

action = self._greedy(s)

a = self._action2num[action]

# store (state, action) tuple

H["state_actions"].append((s, a))

# take action

obs, reward, done, info = self.env.step(action)

n_steps += 1

# record rewards

H["rewards"].append(reward)

total_reward += reward

if done:

break

return total_reward, n_steps

class DynaAgent(AgentBase):

def __init__(

self,

env,

grid_dims=[8, 8],

r"""

A Dyna-`Q` / Dyna-`Q+` agent [5]_ with full TD(0) `Q`-learning updates via

prioritized-sweeping [6]_ .

Notes

This approach consists of three components: a planning method involving

simulated actions, a direct RL method where the agent directly interacts

with the environment, and a model-learning method where the agent

learns to better represent the environment during planning.

During planning, the agent performs random-sample one-step tabular

Q-planning with prioritized sweeping. This entails using a priority

queue to retrieve the state-action pairs from the agent's history which

would stand to have the largest change to their Q-values if backed up.

Specifically, for state action pair `(s, a)` the priority value is:

.. math::

P = \sum_{s'} p(s') | r + \gamma \max_a \{Q(s', a) \} - Q(s, a) |

which corresponds to the absolute magnitude of the TD(0) Q-learning

backup for the pair.

When the first pair in the queue is backed up, the effect on each of

its predecessor pairs is computed. If the predecessor's priority is

greater than a small threshold the pair is added to the queue and the

process is repeated until either the queue is empty or we have exceeded

`n_simulated_actions` updates. These backups occur without the agent

taking any action in the environment and thus constitute simulations

based on the agent's current model of the environment (i.e., its

tabular state-action history).

During the direct RL phase, the agent takes an action based on its

current behavior policy and Q function and receives a reward from the

environment. The agent logs this state-action-reward-new state tuple in

its interaction table (i.e., environment model) and updates its Q

function using a full-backup version of the Q-learning update:

.. math::

Q(s, a) \leftarrow Q(s, a) + \eta \sum_{r, s'} p(r, s' \mid s, a)

\left(r + \gamma \max_a \{ Q(s', a) \} - Q(s, a) \right)

References

.. [5] Sutton, R. (1990). Integrated architectures for learning,

planning, and reacting based on approximating dynamic programming.

In *Proceedings of the 7th Annual ICML*, 216-224.

.. [6] Moore, A. & Atkeson, C. (1993). Prioritized sweeping:

Reinforcement learning with less data and less time. *Machine

Learning, 13(1)*, 103-130.

Parameters

env : :class:`gym.wrappers` or :class:`gym.envs` instance

The environment to run the agent on

lr : float

Learning rate for the `Q` function updates. Default is 0.05.

epsilon : float between [0, 1]

The epsilon value in the epsilon-soft policy. Larger values

encourage greater exploration during training. Default is 0.1.

n_tilings : int

The number of overlapping tilings to use if the env observation

space is continuous. Unused if observation space is discrete.

Default is 8.

obs_max : float or :py:class:`ndarray <numpy.ndarray>` or None

The value to treat as the max value of the observation space when

calculating the grid widths if the observation space is continuous.

If None, use :meth:`env.observation_space.high`. Unused if observation

space is discrete. Default is None.

obs_min : float or :py:class:`ndarray <numpy.ndarray>` or None

The value to treat as the min value of the observation space when

calculating grid widths if the observation space is continuous. If

None, use :meth:`env.observation_space.low`. Unused if observation

space is discrete. Default is None.

grid_dims : list

The number of rows and columns in each tiling grid if the env

observation space is continuous. Unused if observation space is

discrete. Default is `[8, 8]`.

q_plus : bool

Whether to add incentives for visiting states that the agent hasn't

encountered recently. Default is False.

explore_weight : float

Amount to incentivize exploring states that the agent hasn't

recently visited. Only used if `q_plus` is True. Default is 0.05.

temporal_discount : float between [0, 1]

The discount factor used for downweighting future rewards. Smaller

values result in greater discounting of future rewards. Default is

n_simulated_actions : int

THe number of simulated actions to perform for each "real" action.

Default is 50.

self.lr = lr

self.q_plus = q_plus

self.obs_max = obs_max

self.obs_min = obs_min

self.epsilon = epsilon

self.n_tilings = n_tilings

self.grid_dims = grid_dims

self.explore_weight = explore_weight

self.temporal_discount = temporal_discount

self.n_simulated_actions = n_simulated_actions

def _init_params(self):

E = self.env_info

assert not E["continuous_actions"], "Action space must be discrete"

obs_encoder = None

if E["continuous_observations"]:

obs_encoder, _ = tile_state_space(

self.env,

self._create_2num_dicts(obs_encoder=obs_encoder)

self.behavior_policy = self.target_policy = self._epsilon_soft_policy

# initialize Q function and model

self.parameters["Q"] = defaultdict(np.random.rand)

self.parameters["model"] = EnvModel()

# initialize returns object for each state-action pair

self.derived_variables = {

"episode_num": 0,

"sweep_queue": {},

"visited": set(),

"steps_since_last_visit": defaultdict(lambda: 0),

if self.q_plus:

self.derived_variables["steps_since_last_visit"] = defaultdict(

np.random.rand,

self.hyperparameters = {

"agent": "DynaAgent",

"lr": self.lr,

"q_plus": self.q_plus,

"obs_max": self.obs_max,

"obs_min": self.obs_min,

"epsilon": self.epsilon,

"n_tilings": self.n_tilings,

"grid_dims": self.grid_dims,

"explore_weight": self.explore_weight,

"temporal_discount": self.temporal_discount,

"n_simulated_actions": self.n_simulated_actions,

self.episode_history = {"state_actions": [], "rewards": []}

def act(self, obs):

r"""

Execute the behavior policy--an :math:`\epsilon`-soft policy used to

generate actions during training.

Parameters

An observation from the environment.

Returns

action : int, float, or :py:class:`ndarray <numpy.ndarray>`

An action sampled from the distribution over actions defined by the

epsilon-soft policy.

"""  # noqa: E501

s = self._obs2num[obs]

return self.behavior_policy(s)

def _epsilon_soft_policy(self, s, a=None):

Epsilon-soft exploration policy.

In epsilon-soft policies, pi(a|s) > 0 for all s ∈ S and all a ∈ A(s) at

the start of training. As learning progresses, pi gradually shifts

closer and closer to a deterministic optimal policy.

In particular, we have:

pi(a|s) = 1 - epsilon + (epsilon / |A(s)|) IFF a == a*

pi(a|s) = epsilon / |A(s)|                 IFF a != a*

where

|A(s)| is the number of actions available in state s

Note that epsilon-greedy policies are instances of epsilon-soft

all states and actions.

Parameters

s : int, float, or tuple

The state number for the current observation, as returned by

a : int, float, or tuple

The action number in the current state, as returned by

self._action2num[obs]. If None, sample an action from the action

probabilities in state s, otherwise, return the probability of

action `a` under the epsilon-soft policy. Default is None.

Returns

If `a` is None:

action : int, float, or :py:class:`ndarray <numpy.ndarray>` as returned by :meth:`_num2action`

If `a` is None, returns an action sampled from the distribution

over actions defined by the epsilon-soft policy.

If `a` is not None:

action_prob : float in range [0, 1]

If `a` is not None, returns the probability of `a` under the

epsilon-soft policy.

"""  # noqa: E501

E, P = self.env_info, self.parameters

# TODO: this assumes all actions are available in every state

n_actions = np.prod(E["n_actions_per_dim"])

a_star = np.argmax([P["Q"][(s, aa)] for aa in range(n_actions)])

p_a_star = 1.0 - self.epsilon + (self.epsilon / n_actions)

p_a = self.epsilon / n_actions

action_probs = np.ones(n_actions) * p_a

action_probs[a_star] = p_a_star

np.testing.assert_allclose(np.sum(action_probs), 1)

if a is not None:

return action_probs[a]

# sample action

a = np.random.multinomial(1, action_probs).argmax()

return self._num2action[a]

def _greedy(self, s, a=None):

A greedy behavior policy.

Parameters

s : int, float, or tuple

a : int, float, or tuple

action `a` under the greedy policy. Default is None.

Returns

If `a` is None:

over actions defined by the greedy policy.

If `a` is not None:

action_prob : float in range [0, 1]

greedy policy.

"""  # noqa: E501

E, Q = self.env_info, self.parameters["Q"]

a_star = np.argmax([Q[(s, aa)] for aa in range(n_actions)])

if a is None:

out = self._num2action[a_star]

else:

out = 1 if a == a_star else 0

return out

def update(self):

Update the priority queue with the most recent (state, action) pair and

perform random-sample one-step tabular Q-planning.

Notes

The planning algorithm uses a priority queue to retrieve the

state-action pairs from the agent's history which will result in the

largest change to its `Q`-value if backed up. When the first pair in

the queue is backed up, the effect on each of its predecessor pairs is

computed. If the predecessor's priority is greater than a small

threshold the pair is added to the queue and the process is repeated

until either the queue is empty or we exceed `n_simulated_actions`

updates.

s, a = self.episode_history["state_actions"][-1]

self._update_queue(s, a)

def _update_queue(self, s, a):

Update the priority queue by calculating the priority for (s, a) and

inserting it into the queue if it exceeds a fixed (small) threshold.

Parameters

s : int as returned by `self._obs2num`

The id for the state/observation

a : int as returned by `self._action2num`

The id for the action taken from state `s`

sweep_queue = self.derived_variables["sweep_queue"]

# TODO: what's a good threshold here?

priority = self._calc_priority(s, a)

if priority >= 0.001:

if (s, a) in sweep_queue:

sweep_queue[(s, a)] = max(priority, sweep_queue[(s, a)])

else:

sweep_queue[(s, a)] = priority

def _calc_priority(self, s, a):

Compute the "priority" for state-action pair (s, a). The priority P is

defined as:

P = sum_{s_} p(s_) * abs(r + temporal_discount * max_a {Q[s_, a]} - Q[s, a])

which corresponds to the absolute magnitude of the TD(0) Q-learning

backup for (s, a).

Parameters

s : int as returned by `self._obs2num`

The id for the state/observation

Returns

priority : float

The absolute magnitude of the full-backup TD(0) Q-learning update

for (s, a)

priority = 0.0

E = self.env_info

Q = self.parameters["Q"]

env_model = self.parameters["model"]

n_actions = np.prod(E["n_actions_per_dim"])

outcome_probs = env_model.outcome_probs(s, a)

for (r, s_), p_rs_ in outcome_probs:

max_q = np.max([Q[(s_, aa)] for aa in range(n_actions)])

P = p_rs_ * (r + self.temporal_discount * max_q - Q[(s, a)])

priority += np.abs(P)

return priority

def _simulate_behavior(self):

Perform random-sample one-step tabular Q-planning with prioritized

sweeping.

Notes

This approach uses a priority queue to retrieve the state-action pairs

from the agent's history with largest change to their Q-values if

backed up. When the first pair in the queue is backed up, the effect on

each of its predecessor pairs is computed. If the predecessor's

priority is greater than a small threshold the pair is added to the

queue and the process is repeated until either the queue is empty or we

have exceeded a `n_simulated_actions` updates.

env_model = self.parameters["model"]

sweep_queue = self.derived_variables["sweep_queue"]

for _ in range(self.n_simulated_actions):

if len(sweep_queue) == 0:

break

# select (s, a) pair with the largest update (priority)

sq_items = list(sweep_queue.items())

(s_sim, a_sim), _ = sorted(sq_items, key=lambda x: x[1], reverse=True)[0]

# remove entry from queue

del sweep_queue[(s_sim, a_sim)]

# update Q function for (s_sim, a_sim) using the full-backup

# version of the TD(0) Q-learning update

self._update(s_sim, a_sim)

# get all (_s, _a) pairs that lead to s_sim (ie., s_sim's predecessors)

pairs = env_model.state_action_pairs_leading_to_outcome(s_sim)

# add predecessors to queue if their priority exceeds thresh

for (_s, _a) in pairs:

self._update_queue(_s, _a)

def _update(self, s, a):

Update Q using a full-backup version of the TD(0) Q-learning update:

Q(s, a) = Q(s, a) + lr *

sum_{r, s'} [

p(r, s' | s, a) * (r + gamma * max_a { Q(s', a) } - Q(s, a))

Parameters

s : int as returned by ``self._obs2num``

The id for the state/observation

a : int as returned by ``self._action2num``

The id for the action taken from state `s`

update = 0.0

env_model = self.parameters["model"]

E, D, Q = self.env_info, self.derived_variables, self.parameters["Q"]

n_actions = np.prod(E["n_actions_per_dim"])

# sample rewards from the model

outcome_probs = env_model.outcome_probs(s, a)

for (r, s_), p_rs_ in outcome_probs:

# encourage visiting long-untried actions by adding a "bonus"

# reward proportional to the sqrt of the time since last visit

if self.q_plus:

r += self.explore_weight * np.sqrt(D["steps_since_last_visit"][(s, a)])

max_q = np.max([Q[(s_, a_)] for a_ in range(n_actions)])

update += p_rs_ * (r + self.temporal_discount * max_q - Q[(s, a)])

# update Q value for (s, a) pair

Q[(s, a)] += self.lr * update

def run_episode(self, max_steps, render=False):

Run the agent on a single episode without performing `Q`-function

backups.

Parameters

max_steps : int

The maximum number of steps to run an episode.

render : bool

Whether to render the episode during training.

Returns

reward : float

The total reward on the episode.

steps : float

The number of steps taken on the episode.

return self._episode(max_steps, render, update=False)

def train_episode(self, max_steps, render=False):

Train the agent on a single episode.

Parameters

max_steps : int

render : bool

Returns

reward : float

The total reward on the episode.

steps : float

D = self.derived_variables

total_rwd, n_steps = self._episode(max_steps, render, update=True)

D["episode_num"] += 1

return total_rwd, n_steps

def _episode(self, max_steps, render, update=True):

Run or train the agent on an episode.

Parameters

max_steps : int

The maximum number of steps to run the episode.

render : bool

update : bool

Whether to perform the `Q` function backups after each step.

Default is True.

Returns

reward : float

The total reward on the episode.

steps : float

obs = self.env.reset()

env_model = self.parameters["model"]

HS, D = self.episode_history, self.derived_variables

action = self.act(obs)

s = self._obs2num[obs]

a = self._action2num[action]

# store initial (state, action) tuple

HS["state_actions"].append((s, a))

total_reward, n_steps = 0.0, 0

for i in range(max_steps):

if render:

self.env.render()

# take action

obs, reward, done, info = self.env.step(action)

n_steps += 1

# record rewards

HS["rewards"].append(reward)

total_reward += reward

# generate next state and action

action = self.act(obs)

s_ = self._obs2num[obs] if not done else None

a_ = self._action2num[action]

# update model

env_model[(s, a, reward, s_)] += 1

# update history counter

for k in D["steps_since_last_visit"].keys():

D["steps_since_last_visit"][k] += 1

D["steps_since_last_visit"][(s, a)] = 0

if update:

self.update()

# store next (state, action) tuple

HS["state_actions"].append((s_, a_))

s, a = s_, a_

if done:

break

return total_reward, n_steps

def greedy_policy(self, max_steps, render=True):

Execute a deterministic greedy policy using the current agent

parameters.

Parameters

max_steps : int

render : bool

Whether to render the episode during execution.

Returns

total_reward : float

The total reward on the episode.

n_steps : float

The total number of steps taken on the episode.

H = self.episode_history

obs = self.env.reset()

total_reward, n_steps = 0.0, 0

for i in range(max_steps):

if render:

self.env.render()

s = self._obs2num[obs]

action = self._greedy(s)

a = self._action2num[action]

# store (state, action) tuple

H["state_actions"].append((s, a))

# take action

n_steps += 1

# record rewards

H["rewards"].append(reward)

total_reward += reward

if done:

break

return total_reward, n_steps


from time import time

import numpy as np

class Trainer(object):

def __init__(self, agent, env):

An object to facilitate agent training and evaluation.

Parameters

agent : :class:`AgentBase` instance

The agent to train.

env : ``gym.wrappers`` or ``gym.envs`` instance

The environment to run the agent on.

self.env = env

self.agent = agent

self.rewards = {"total": [], "smooth_total": [], "n_steps": [], "duration": []}

def _train_episode(self, max_steps, render_every=None):

t0 = time()

if "train_episode" in dir(self.agent):

# online training updates over the course of the episode

reward, n_steps = self.agent.train_episode(max_steps)

else:

# offline training updates upon completion of the episode

reward, n_steps = self.agent.run_episode(max_steps)

self.agent.update()

duration = time() - t0

return reward, duration, n_steps

def train(

self,

seed=None,

plot=True,

verbose=True,

Train an agent on an OpenAI gym environment, logging training

statistics along the way.

Parameters

n_episodes : int

The number of episodes to train the agent across.

max_steps : int

The maximum number of steps the agent can take on each episode.

seed : int or None

A seed for the random number generator. Default is None.

plot : bool

Whether to generate a plot of the cumulative reward as a function

of training episode. Default is True.

verbose : bool

Whether to print intermediate run statistics to stdout during

training. Default is True.

smooth_factor : float in [0, 1]

The amount to smooth the cumulative reward across episodes. Larger

values correspond to less smoothing.

if seed:

np.random.seed(seed)

self.env.seed(seed=seed)

t0 = time()

render_every = np.inf if render_every is None else render_every

sf = smooth_factor

for ep in range(n_episodes):

tot_rwd, duration, n_steps = self._train_episode(max_steps)

smooth_tot = tot_rwd if ep == 0 else (1 - sf) * smooth_tot + sf * tot_rwd

if verbose:

fstr = "[Ep. {:2}] {:<6.2f} Steps | Total Reward: {:<7.2f}"

fstr += " | Smoothed Total: {:<7.2f} | Duration: {:<6.2f}s"

print(fstr.format(ep + 1, n_steps, tot_rwd, smooth_tot, duration))

if (ep + 1) % render_every == 0:

fstr = "\tGreedy policy total reward: {:.2f}, n_steps: {:.2f}"

total, n_steps = self.agent.greedy_policy(max_steps)

print(fstr.format(total, n_steps))

self.rewards["duration"].append(duration)

self.rewards["smooth_total"].append(smooth_tot)

train_time = (time() - t0) / 60

fstr = "Training took {:.2f} mins [{:.2f}s/episode]"

print(fstr.format(train_time, np.mean(self.rewards["duration"])))

rwd_greedy, n_steps = self.agent.greedy_policy(max_steps, render=False)

fstr = "Final greedy reward: {:.2f} | n_steps: {:.2f}"

print(fstr.format(rwd_greedy, n_steps))

if plot:

def plot_rewards(self, rwd_greedy):

Plot the cumulative reward per episode as a function of episode number.

Notes

Saves plot to the file ``./img/<agent>-<env>.png``

Parameters

rwd_greedy : float

The cumulative reward earned with a final execution of a greedy

target policy.

try:

import matplotlib.pyplot as plt

import seaborn as sns

# https://seaborn.pydata.org/generated/seaborn.set_context.html

# https://seaborn.pydata.org/generated/seaborn.set_style.html

sns.set_context("notebook", font_scale=1)

except:

fstr = "Error importing `matplotlib` and `seaborn` -- plotting functionality is disabled"

raise ImportError(fstr)

R = self.rewards

fig, ax = plt.subplots()

x = np.arange(len(R["total"]))

y = R["smooth_total"]

y_raw = R["total"]

ax.plot(x, y, label="smoothed")

ax.plot(x, y_raw, alpha=0.5, label="raw")

ax.axhline(y=rwd_greedy, xmin=min(x), xmax=max(x), ls=":", label="final greedy")

ax.legend()

sns.despine()

env = self.agent.env_info["id"]

agent = self.agent.hyperparameters["agent"]

ax.set_ylabel("Cumulative reward")

ax.set_title("{} on '{}'".format(agent, env))

plt.savefig("img/{}-{}.png".format(agent, env))

plt.close("all")


"""Utilities for training and evaluating RL models on OpenAI gym environments"""

import warnings

from itertools import product

from collections import defaultdict

import numpy as np

from numpy_ml.utils.testing import DependencyWarning

from numpy_ml.rl_models.tiles.tiles3 import tiles, IHT

NO_PD = False

try:

import pandas as pd

except ModuleNotFoundError:

NO_PD = True

try:

import gym

except ModuleNotFoundError:

fstr = (

"Agents in `numpy_ml.rl_models` use the OpenAI gym for training. "

"To install the gym environments, run `pip install gym`. For more"

" information, see https://github.com/openai/gym."

warnings.warn(fstr, DependencyWarning)

class EnvModel(object):

A simple tabular environment model that maintains the counts of each

reward-outcome pair given the state and action that preceded them. The

model can be queried with

>>> M = EnvModel()

>>> M[(state, action, reward, next_state)] += 1

>>> M[(state, action, reward, next_state)]

>>> M.state_action_pairs()

[(state, action)]

>>> M.outcome_probs(state, action)

[(next_state, 1)]

def __init__(self):

super(EnvModel, self).__init__()

self._model = defaultdict(lambda: defaultdict(lambda: 0))

def __setitem__(self, key, value):

"""Set self[key] to value"""

s, a, r, s_ = key

self._model[(s, a)][(r, s_)] = value

def __getitem__(self, key):

"""Return the value associated with key"""

s, a, r, s_ = key

return self._model[(s, a)][(r, s_)]

def __contains__(self, key):

"""True if EnvModel contains `key`, else False"""

s, a, r, s_ = key

p1 = (s, a) in self.state_action_pairs()

p2 = (r, s_) in self.reward_outcome_pairs()

return p1 and p2

def state_action_pairs(self):

"""Return all (state, action) pairs in the environment model"""

return list(self._model.keys())

def reward_outcome_pairs(self, s, a):

Return all (reward, next_state) pairs associated with taking action `a`

in state `s`.

return list(self._model[(s, a)].keys())

def outcome_probs(self, s, a):

Return the probability under the environment model of each outcome

state after taking action `a` in state `s`.

Parameters

s : int as returned by ``self._obs2num``

The id for the state/observation.

a : int as returned by ``self._action2num``

The id for the action taken from state `s`.

Returns

outcome_probs : list of (state, prob) tuples

A list of each possible outcome and its associated probability

under the model.

items = list(self._model[(s, a)].items())

total_count = np.sum([c for (_, c) in items])

outcome_probs = [c / total_count for (_, c) in items]

outcomes = [p for (p, _) in items]

return list(zip(outcomes, outcome_probs))

def state_action_pairs_leading_to_outcome(self, outcome):

Return all (state, action) pairs that have a nonzero probability of

producing `outcome` under the current model.

Parameters

outcome : int

The outcome state.

Returns

pairs : list of (state, action) tuples

A list of all (state, action) pairs with a nonzero probability of

producing `outcome` under the model.

pairs = []

for sa in self.state_action_pairs():

outcomes = [o for (r, o) in self.reward_outcome_pairs(*sa)]

if outcome in outcomes:

pairs.append(sa)

return pairs

def tile_state_space(

env,

grid_size=(4, 4),

Return a function to encode the continous observations generated by `env`

in terms of a collection of `n_tilings` overlapping tilings (each with

dimension `grid_size`) of the state space.

Arguments

env : ``gym.wrappers.time_limit.TimeLimit`` instance

An openAI environment.

n_tilings : int

The number of overlapping tilings to use. Should be a power of 2. This

determines the dimension of the discretized tile-encoded state vector.

obs_max : float or np.ndarray

The value to treat as the max value of the observation space when

calculating the grid widths. If None, use

``env.observation_space.high``. Default is None.

obs_min : float or np.ndarray

The value to treat as the min value of the observation space when

``env.observation_space.low``. Default is None.

state_action : bool

Whether to use tile coding to encode state-action values (True) or just

state values (False). Default is False.

grid_size : list of length 2

A list of ints representing the coarseness of the tilings. E.g., a

`grid_size` of [4, 4] would mean each tiling consisted of a 4x4 tile

grid. Default is [4, 4].

Returns

encode_obs_as_tile : function

A function which takes as input continous observation vector and

returns a set of the indices of the active tiles in the tile coded

observation space.

n_states : int

An integer reflecting the total number of unique states possible under

this tile coding regimen.

obs_max = np.nan_to_num(env.observation_space.high) if obs_max is None else obs_max

obs_min = np.nan_to_num(env.observation_space.low) if obs_min is None else obs_min

if state_action:

if env_stats["tuple_action"]:

n = [space.n - 1.0 for space in env.action_spaces.spaces]

else:

n = [env.action_space.n]

obs_max = np.concatenate([obs_max, n])

obs_min = np.concatenate([obs_min, np.zeros_like(n)])

obs_range = obs_max - obs_min

scale = 1.0 / obs_range

# scale (state-)observation vector

scale_obs = lambda obs: obs * scale  # noqa: E731

n_tiles = np.prod(grid_size) * n_tilings

n_states = np.prod([n_tiles - i for i in range(n_tilings)])

iht = IHT(16384)

def encode_obs_as_tile(obs):

obs = scale_obs(obs)

return tuple(tiles(iht, n_tilings, obs))

return encode_obs_as_tile, n_states

def get_gym_environs():

"""List all valid OpenAI ``gym`` environment ids"""

return [e.id for e in gym.envs.registry.all()]

def get_gym_stats():

"""Return a pandas DataFrame of the environment IDs."""

df = []

for e in gym.envs.registry.all():

print(e.id)

cols = [

"id",

#  "action_ids",

"deterministic",

#  "obs_ids",

"seed",

return df if NO_PD else pd.DataFrame(df)[cols]

def is_tuple(env):

Check if the action and observation spaces for `env` are instances of

``gym.spaces.Tuple`` or ``gym.spaces.Dict``.

Notes

A tuple space is a tuple of *several* (possibly multidimensional)

action/observation spaces. For our purposes, a tuple space is necessarily

multidimensional.

Returns

tuple_action : bool

Whether the `env`'s action space is an instance of ``gym.spaces.Tuple``

or ``gym.spaces.Dict``.

tuple_obs : bool

Whether the `env`'s observation space is an instance of

tuple_space, dict_space = gym.spaces.Tuple, gym.spaces.dict.Dict

tuple_action = isinstance(env.action_space, (tuple_space, dict_space))

tuple_obs = isinstance(env.observation_space, (tuple_space, dict_space))

return tuple_action, tuple_obs

def is_multidimensional(env):

Check if the action and observation spaces for `env` are multidimensional

or ``Tuple`` spaces.

Notes

A multidimensional space is any space whose actions / observations have

more than one element in them. This includes ``Tuple`` spaces, but also

includes single action/observation spaces with several dimensions.

Parameters

env : ``gym.wrappers`` or ``gym.envs`` instance

The environment to evaluate.

Returns

md_action : bool

Whether the `env`'s action space is multidimensional.

md_obs : bool

Whether the `env`'s observation space is multidimensional.

tuple_action : bool

Whether the `env`'s action space is a ``Tuple`` instance.

tuple_obs : bool

Whether the `env`'s observation space is a ``Tuple`` instance.

md_action, md_obs = True, True

tuple_action, tuple_obs = is_tuple(env)

if not tuple_action:

act = env.action_space.sample()

md_action = isinstance(act, (list, tuple, np.ndarray)) and len(act) > 1

if not tuple_obs:

OS = env.observation_space

obs = OS.low if "low" in dir(OS) else OS.sample()  # sample causes problems

md_obs = isinstance(obs, (list, tuple, np.ndarray)) and len(obs) > 1

return md_action, md_obs, tuple_action, tuple_obs

def is_continuous(env, tuple_action, tuple_obs):

Check if an `env`'s observation and action spaces are continuous.

Parameters

The environment to evaluate.

tuple_action : bool

Whether the `env`'s action space is an instance of `gym.spaces.Tuple`

or `gym.spaces.Dict`.

tuple_obs : bool

Whether the `env`'s observation space is an instance of `gym.spaces.Tuple`

or `gym.spaces.Dict`.

Returns

cont_action : bool

Whether the `env`'s action space is continuous.

cont_obs : bool

Whether the `env`'s observation space is continuous.

Continuous = gym.spaces.box.Box

if tuple_obs:

spaces = env.observation_space.spaces

cont_obs = all(isinstance(s, Continuous) for s in spaces)

else:

cont_obs = isinstance(env.observation_space, Continuous)

if tuple_action:

spaces = env.action_space.spaces

cont_action = all(isinstance(s, Continuous) for s in spaces)

else:

cont_action = isinstance(env.action_space, Continuous)

return cont_action, cont_obs

def action_stats(env, md_action, cont_action):

Get information on `env`'s action space.

Parameters

md_action : bool

Whether the `env`'s action space is multidimensional.

cont_action : bool

Returns

n_actions_per_dim : list of length (action_dim,)

The number of possible actions for each dimension of the action space.

action_ids : list or None

A list of all valid actions within the space. If `cont_action` is

True, this value will be None.

action_dim : int or None

The number of dimensions in a single action.

if cont_action:

action_dim = 1

action_ids = None

n_actions_per_dim = [np.inf]

if md_action:

action_dim = env.action_space.shape[0]

n_actions_per_dim = [np.inf for _ in range(action_dim)]

else:

if md_action:

n_actions_per_dim = [

space.n if hasattr(space, "n") else np.inf

for space in env.action_space.spaces

action_ids = (

if np.inf in n_actions_per_dim

else list(product(*[range(i) for i in n_actions_per_dim]))

action_dim = len(n_actions_per_dim)

else:

action_dim = 1

n_actions_per_dim = [env.action_space.n]

action_ids = list(range(n_actions_per_dim[0]))

return n_actions_per_dim, action_ids, action_dim

def obs_stats(env, md_obs, cont_obs):

Get information on the observation space for `env`.

Parameters

env : ``gym.wrappers`` or ``gym.envs`` instance

The environment to evaluate.

md_obs : bool

cont_obs : bool

Returns

n_obs_per_dim : list of length (obs_dim,)

The number of possible observation classes for each dimension of the

observation space.

obs_ids : list or None

A list of all valid observations within the space. If `cont_obs` is

True, this value will be None.

obs_dim : int or None

The number of dimensions in a single observation.

if cont_obs:

obs_ids = None

obs_dim = env.observation_space.shape[0]

n_obs_per_dim = [np.inf for _ in range(obs_dim)]

else:

if md_obs:

n_obs_per_dim = [

for space in env.observation_space.spaces

obs_ids = (

if np.inf in n_obs_per_dim

else list(product(*[range(i) for i in n_obs_per_dim]))

obs_dim = len(n_obs_per_dim)

else:

obs_dim = 1

n_obs_per_dim = [env.observation_space.n]

obs_ids = list(range(n_obs_per_dim[0]))

return n_obs_per_dim, obs_ids, obs_dim

def env_stats(env):

Compute statistics for the current environment.

Parameters

The environment to evaluate.

Returns

env_info : dict

A dictionary containing information about the action and observation

spaces of `env`.

md_action, md_obs, tuple_action, tuple_obs = is_multidimensional(env)

cont_action, cont_obs = is_continuous(env, tuple_action, tuple_obs)

n_actions_per_dim, action_ids, action_dim = action_stats(

env, md_action, cont_action,

n_obs_per_dim, obs_ids, obs_dim = obs_stats(env, md_obs, cont_obs)

env_info = {

"id": env.spec.id,

"seed": env.spec.seed if "seed" in dir(env.spec) else None,

"deterministic": bool(~env.spec.nondeterministic),

"tuple_actions": tuple_action,

"tuple_observations": tuple_obs,

"multidim_actions": md_action,

"multidim_observations": md_obs,

"continuous_actions": cont_action,

"continuous_observations": cont_obs,

"n_actions_per_dim": n_actions_per_dim,

"action_dim": action_dim,

"n_obs_per_dim": n_obs_per_dim,

"obs_dim": obs_dim,

"action_ids": action_ids,

"obs_ids": obs_ids,

return env_info


from . import tiles3


Tile Coding Software version 3.0beta

by Rich Sutton

based on a program created by Steph Schaeffer and others

External documentation and recommendations on the use of this code is available in the

reinforcement learning textbook by Sutton and Barto, and on the web.

These need to be understood before this code is.

This software is for Python 3 or more.

This is an implementation of grid-style tile codings, based originally on

the UNH CMAC code (see http://www.ece.unh.edu/robots/cmac.htm), but by now highly changed.

Here we provide a function, "tiles", that maps floating and integer

variables to a list of tiles, and a second function "tiles-wrap" that does the same while

wrapping some floats to provided widths (the lower wrap value is always 0).

The float variables will be gridded at unit intervals, so generalization

will be by approximately 1 in each direction, and any scaling will have

to be done externally before calling tiles.

Num-tilings should be a power of 2, e.g., 16. To make the offsetting work properly, it should

also be greater than or equal to four times the number of floats.

The first argument is either an index hash table of a given size (created by (make-iht size)),

an integer "size" (range of the indices from 0), or nil (for testing, indicating that the tile

coordinates are to be returned without being converted to indices).

from math import floor

from itertools import zip_longest

basehash = hash

class IHT:

"Structure to handle collisions"

def __init__(self, sizeval):

self.size = sizeval

self.overfullCount = 0

self.dictionary = {}

def __str__(self):

"Prepares a string for printing whenever this object is printed"

return (

"Collision table:"

+ " size:"

+ str(self.size)

+ " overfullCount:"

+ str(self.overfullCount)

+ " dictionary:"

+ str(len(self.dictionary))

+ " items"

def count(self):

return len(self.dictionary)

def fullp(self):

return len(self.dictionary) >= self.size

def getindex(self, obj, readonly=False):

d = self.dictionary

if obj in d:

return d[obj]

elif readonly:

return None

size = self.size

count = self.count()

if count >= size:

if self.overfullCount == 0:

print("IHT full, starting to allow collisions")

self.overfullCount += 1

return basehash(obj) % self.size

else:

d[obj] = count

return count

def hashcoords(coordinates, m, readonly=False):

if type(m) == IHT:

return m.getindex(tuple(coordinates), readonly)

if type(m) == int:

return basehash(tuple(coordinates)) % m

if m == None:

return coordinates

def tiles(ihtORsize, numtilings, floats, ints=[], readonly=False):

"""returns num-tilings tile indices corresponding to the floats and ints"""

qfloats = [floor(f * numtilings) for f in floats]

Tiles = []

for tiling in range(numtilings):

tilingX2 = tiling * 2

coords = [tiling]

b = tiling

for q in qfloats:

coords.append((q + b) // numtilings)

b += tilingX2

coords.extend(ints)

Tiles.append(hashcoords(coords, ihtORsize, readonly))

return Tiles

def tileswrap(ihtORsize, numtilings, floats, wrapwidths, ints=[], readonly=False):

"""returns num-tilings tile indices corresponding to the floats and ints,

wrapping some floats"""

Tiles = []

for tiling in range(numtilings):

tilingX2 = tiling * 2

coords = [tiling]

b = tiling

for q, width in zip_longest(qfloats, wrapwidths):

c = (q + b % numtilings) // numtilings

coords.append(c % width if width else c)

b += tilingX2

coords.extend(ints)

return Tiles