Prompt:
Problem:
I have a set of data and I want to compare which line describes it best (polynomials of different orders, exponential or logarithmic).
I use Python and Numpy and for polynomial fitting there is a function polyfit(). But I found no such functions for exponential and logarithmic fitting.
How do I fit y = A*exp(Bx) + C ? The result should be an np.array of [A, B, C]. I know that polyfit performs bad for this function, so I would like to use curve_fit to solve the problem, and it should start from initial guess p0.
A:
<code>
import numpy as np
import scipy.optimize
y = np.array([1, 7, 20, 50, 79])
x = np.array([10, 19, 30, 35, 51])
p0 = (4, 0.1, 1)
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

result = scipy.optimize.curve_fit(lambda t,a,b, c: a*np.exp(b*t) + c,  x,  y,  p0=p0)[0]

import numpy as np
import copy
import scipy.optimize

def generate_test_case(test_case_id):

    def define_test_input(test_case_id):
        if test_case_id == 1:
            y = np.array([1, 7, 20, 50, 79])
            x = np.array([10, 19, 30, 35, 51])
            p0 = (4, 0.1, 1)
        return x, y, p0

    def generate_ans(data):
        _a = data
        x, y, p0 = _a
        result = scipy.optimize.curve_fit(
            lambda t, a, b, c: a * np.exp(b * t) + c, x, y, p0=p0
        )[0]
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert np.allclose(result, ans)
    return 1

exec_context = r"""
import numpy as np
import scipy.optimize
x, y, p0 = test_input
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I can't figure out how to do a Two-sample KS test in Scipy.
After reading the documentation scipy kstest
I can see how to test where a distribution is identical to standard normal distribution
from scipy.stats import kstest
import numpy as np
x = np.random.normal(0,1,1000)
test_stat = kstest(x, 'norm')
#>>> test_stat
#(0.021080234718821145, 0.76584491300591395)
Which means that at p-value of 0.76 we can not reject the null hypothesis that the two distributions are identical.
However, I want to compare two distributions and see if I can reject the null hypothesis that they are identical, something like:
from scipy.stats import kstest
import numpy as np
x = np.random.normal(0,1,1000)
z = np.random.normal(1.1,0.9, 1000)
and test whether x and z are identical
I tried the naive:
test_stat = kstest(x, z)
and got the following error:
TypeError: 'numpy.ndarray' object is not callable
Is there a way to do a two-sample KS test in Python? If so, how should I do it?
Thank You in Advance
A:
<code>
from scipy import stats
import numpy as np
np.random.seed(42)
x = np.random.normal(0, 1, 1000)
y = np.random.normal(0, 1, 1000)
</code>
statistic, p_value = ... # put solution in these variables
BEGIN SOLUTION
<code>

import numpy as np
import copy
from scipy import stats

def generate_test_case(test_case_id):

    def define_test_input(test_case_id):
        if test_case_id == 1:
            np.random.seed(42)
            x = np.random.normal(0, 1, 1000)
            y = np.random.normal(0, 1, 1000)
        elif test_case_id == 2:
            np.random.seed(42)
            x = np.random.normal(0, 1, 1000)
            y = np.random.normal(1.1, 0.9, 1000)
        return x, y

    def generate_ans(data):
        _a = data
        x, y = _a
        statistic, p_value = stats.ks_2samp(x, y)
        return [statistic, p_value]

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_allclose(result, ans)
    return 1

exec_context = r"""
from scipy import stats
import numpy as np
np.random.seed(42)
x, y = test_input
[insert]
result = [statistic, p_value]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(2):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I can't figure out how to do a Two-sample KS test in Scipy.
After reading the documentation scipy kstest
I can see how to test where a distribution is identical to standard normal distribution
from scipy.stats import kstest
import numpy as np
x = np.random.normal(0,1,1000)
test_stat = kstest(x, 'norm')
#>>> test_stat
#(0.021080234718821145, 0.76584491300591395)
Which means that at p-value of 0.76 we can not reject the null hypothesis that the two distributions are identical.
However, I want to compare two distributions and see if I can reject the null hypothesis that they are identical, something like:
from scipy.stats import kstest
import numpy as np
x = np.random.normal(0,1,1000)
z = np.random.normal(1.1,0.9, 1000)
and test whether x and z are identical
I tried the naive:
test_stat = kstest(x, z)
and got the following error:
TypeError: 'numpy.ndarray' object is not callable
Is there a way to do a two-sample KS test in Python, then test whether I can reject the null hypothesis that the two distributions are identical(result=True means able to reject, and the vice versa) based on alpha? If so, how should I do it?
Thank You in Advance
A:
<code>
from scipy import stats
import numpy as np
np.random.seed(42)
x = np.random.normal(0, 1, 1000)
y = np.random.normal(0, 1, 1000)
alpha = 0.01
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

s, p = stats.ks_2samp(x, y)
result = (p <= alpha)

import numpy as np
import copy
from scipy import stats

def generate_test_case(test_case_id):

    def define_test_input(test_case_id):
        if test_case_id == 1:
            np.random.seed(42)
            x = np.random.normal(0, 1, 1000)
            y = np.random.normal(0, 1, 1000)
        elif test_case_id == 2:
            np.random.seed(42)
            x = np.random.normal(0, 1, 1000)
            y = np.random.normal(1.1, 0.9, 1000)
        alpha = 0.01
        return x, y, alpha

    def generate_ans(data):
        _a = data
        x, y, alpha = _a
        s, p = stats.ks_2samp(x, y)
        result = p <= alpha
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_array_equal(result, ans)
    return 1

exec_context = r"""
from scipy import stats
import numpy as np
x, y, alpha = test_input
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(2):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
According to the SciPy documentation it is possible to minimize functions with multiple variables, yet it doesn't tell how to optimize on such functions.
from scipy.optimize import minimize
from math import sqrt, sin, pi, cos
def f(c):
  return sqrt((sin(pi/2) + sin(0) + sin(c) - 2)**2 + (cos(pi/2) + cos(0) + cos(c) - 1)**2)
print minimize(f, 3.14/2 + 3.14/7)

The above code does try to minimize the function f, but for my task I need to minimize with respect to three variables, starting from `initial_guess`.
Simply introducing a second argument and adjusting minimize accordingly yields an error (TypeError: f() takes exactly 2 arguments (1 given)).
How does minimize work when minimizing with multiple variables.
I need to minimize f(a,b,c)=((a+b-c)-2)**2 + ((3*a-b-c))**2 + sin(b) + cos(b) + 4.
Result should be a list=[a,b,c], the parameters of minimized function.

A:
<code>
import scipy.optimize as optimize
from math import sqrt, sin, pi, cos

initial_guess = [-1, 0, -3]
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

def g(params):
    import numpy as np
    a, b, c = params
    return ((a+b-c)-2)**2 + ((3*a-b-c))**2 + np.sin(b) + np.cos(b) + 4

res = optimize.minimize(g, initial_guess)
result = res.x

import numpy as np
import copy
from scipy import optimize

def generate_test_case(test_case_id):

    def define_test_input(test_case_id):
        if test_case_id == 1:
            a = [-1, 0, -3]
        return a

    def generate_ans(data):
        _a = data
        initial_guess = _a

        def g(params):
            a, b, c = params
            return (
                ((a + b - c) - 2) ** 2
                + ((3 * a - b - c)) ** 2
                + np.sin(b)
                + np.cos(b)
                + 4
            )

        res = optimize.minimize(g, initial_guess)
        result = res.x
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    def g(params):
        a, b, c = params
        return (
            ((a + b - c) - 2) ** 2 + ((3 * a - b - c)) ** 2 + np.sin(b) + np.cos(b) + 4
        )

    assert abs(g(result) - g(ans)) < 1e-2
    return 1

exec_context = r"""
import scipy.optimize as optimize
from math import sqrt, sin, pi, cos
initial_guess = test_input
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
How does one convert a list of Z-scores from the Z-distribution (standard normal distribution, Gaussian distribution) to left-tailed p-values? I have yet to find the magical function in Scipy's stats module to do this, but one must be there.
A:
<code>
import numpy as np
import scipy.stats
z_scores = np.array([-3, -2, 0, 2, 2.5])
</code>
p_values = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import copy
import scipy.stats

def generate_test_case(test_case_id):

    def define_test_input(test_case_id):
        if test_case_id == 1:
            a = np.array([-3, -2, 0, 2, 2.5])
        return a

    def generate_ans(data):
        _a = data
        z_scores = _a
        temp = np.array(z_scores)
        p_values = scipy.stats.norm.cdf(temp)
        return p_values

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_allclose(result, ans)
    return 1

exec_context = r"""
import numpy as np
import scipy.stats
z_scores = test_input
[insert]
result = p_values
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
How does one convert a list of Z-scores from the Z-distribution (standard normal distribution, Gaussian distribution) to left-tailed p-values? Original data is sampled from X ~ N(mu, sigma). I have yet to find the magical function in Scipy's stats module to do this, but one must be there.
A:
<code>
import scipy.stats
import numpy as np
z_scores = [-3, -2, 0, 2, 2.5]
mu = 3
sigma = 4
</code>
p_values = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import pandas as pd
import scipy
from scipy import sparse
import scipy.stats
import copy
import io
from scipy import integrate

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            z_scores = [-3, -2, 0, 2, 2.5]
            mu = 3
            sigma = 4
        return z_scores, mu, sigma

    def generate_ans(data):
        _a = data
        z_scores, mu, sigma = _a
        temp = np.array(z_scores)
        p_values = scipy.stats.norm.cdf(temp)
        return p_values

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_allclose(result, ans)
    return 1

exec_context = r"""
import numpy as np
import scipy.stats
z_scores, mu, sigma = test_input
[insert]
result = p_values
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
How does one convert a left-tailed p-value to a z_score from the Z-distribution (standard normal distribution, Gaussian distribution)? I have yet to find the magical function in Scipy's stats module to do this, but one must be there.
A:
<code>
import numpy as np
import scipy.stats
p_values = [0.1, 0.225, 0.5, 0.75, 0.925, 0.95]
</code>
z_scores = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import copy
import scipy.stats

def generate_test_case(test_case_id):

    def define_test_input(test_case_id):
        if test_case_id == 1:
            a = [0.1, 0.225, 0.5, 0.75, 0.925, 0.95]
        return a

    def generate_ans(data):
        _a = data
        p_values = _a
        z_scores = scipy.stats.norm.ppf(p_values)
        return z_scores

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_allclose(result, ans)
    return 1

exec_context = r"""
import numpy as np
import scipy.stats
p_values = test_input
[insert]
result = z_scores
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I have been trying to get the result of a lognormal distribution using Scipy. I already have the Mu and Sigma, so I don't need to do any other prep work. If I need to be more specific (and I am trying to be with my limited knowledge of stats), I would say that I am looking for the cumulative function (cdf under Scipy). The problem is that I can't figure out how to do this with just the mean and standard deviation on a scale of 0-1 (ie the answer returned should be something from 0-1). I'm also not sure which method from dist, I should be using to get the answer. I've tried reading the documentation and looking through SO, but the relevant questions (like this and this) didn't seem to provide the answers I was looking for.
Here is a code sample of what I am working with. Thanks. Here mu and stddev stands for mu and sigma in probability density function of lognorm.
from scipy.stats import lognorm
stddev = 0.859455801705594
mu = 0.418749176686875
total = 37
dist = lognorm.cdf(total,mu,stddev)
UPDATE:
So after a bit of work and a little research, I got a little further. But I still am getting the wrong answer. The new code is below. According to R and Excel, the result should be .7434, but that's clearly not what is happening. Is there a logic flaw I am missing?
stddev = 2.0785
mu = 1.744
x = 25
dist = lognorm([mu],loc=stddev)
dist.cdf(x)  # yields=0.96374596, expected=0.7434
A:
<code>
import numpy as np
from scipy import stats
stddev = 2.0785
mu = 1.744
x = 25
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import copy
from scipy import stats

def generate_test_case(test_case_id):

    def define_test_input(test_case_id):
        if test_case_id == 1:
            stddev = 2.0785
            mu = 1.744
            x = 25
        elif test_case_id == 2:
            stddev = 2
            mu = 1
            x = 20
        return x, mu, stddev

    def generate_ans(data):
        _a = data
        x, mu, stddev = _a
        result = stats.lognorm(s=stddev, scale=np.exp(mu)).cdf(x)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert np.allclose(result, ans)
    return 1

exec_context = r"""
import numpy as np
from scipy import stats
x, mu, stddev = test_input
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(2):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I have been trying to get the arithmetic result of a lognormal distribution using Scipy. I already have the Mu and Sigma, so I don't need to do any other prep work. If I need to be more specific (and I am trying to be with my limited knowledge of stats), I would say that I am looking for the expected value and median of the distribution. The problem is that I can't figure out how to do this with just the mean and standard deviation. I'm also not sure which method from dist, I should be using to get the answer. I've tried reading the documentation and looking through SO, but the relevant questions (like this and this) didn't seem to provide the answers I was looking for.
Here is a code sample of what I am working with. Thanks. Here mu and stddev stands for mu and sigma in probability density function of lognorm.
from scipy.stats import lognorm
stddev = 0.859455801705594
mu = 0.418749176686875
total = 37
dist = lognorm(total,mu,stddev)
What should I do next?
A:
<code>
import numpy as np
from scipy import stats
stddev = 2.0785
mu = 1.744
</code>
expected_value, median = ... # put solution in these variables
BEGIN SOLUTION
<code>

expected_value = np.exp(mu + stddev ** 2 / 2)
median = np.exp(mu)

import numpy as np
import copy

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            stddev = 2.0785
            mu = 1.744
        return mu, stddev

    def generate_ans(data):
        _a = data
        mu, stddev = _a
        expected_value = np.exp(mu + stddev**2 / 2)
        median = np.exp(mu)
        return [expected_value, median]

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert np.allclose(result, ans)
    return 1

exec_context = r"""
import numpy as np
from scipy import stats
mu, stddev = test_input
[insert]
result = [expected_value, median]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I have this example of matrix by matrix multiplication using numpy arrays:
import numpy as np
m = np.array([[1,2,3],[4,5,6],[7,8,9]])
c = np.array([0,1,2])
m * c
array([[ 0,  2,  6],
       [ 0,  5, 12],
       [ 0,  8, 18]])
How can i do the same thing if m is scipy sparse CSR matrix? The result should be csr_matrix as well.
This gives dimension mismatch:
sp.sparse.csr_matrix(m)*sp.sparse.csr_matrix(c)

A:
<code>
from scipy import sparse
import numpy as np
sa = sparse.csr_matrix(np.array([[1,2,3],[4,5,6],[7,8,9]]))
sb = sparse.csr_matrix(np.array([0,1,2]))
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import copy
from scipy import sparse

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            sa = sparse.csr_matrix(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]))
            sb = sparse.csr_matrix(np.array([0, 1, 2]))
        elif test_case_id == 2:
            sa = sparse.random(10, 10, density=0.2, format="csr", random_state=42)
            sb = sparse.random(10, 1, density=0.4, format="csr", random_state=45)
        return sa, sb

    def generate_ans(data):
        _a = data
        sa, sb = _a
        result = sa.multiply(sb)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert type(result) == sparse.csr.csr_matrix
    assert len(sparse.find(result != ans)[0]) == 0
    return 1

exec_context = r"""
from scipy import sparse
import numpy as np
sa, sb = test_input
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(2):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I have this example of matrix by matrix multiplication using numpy arrays:
import numpy as np
m = np.array([[1,2,3],[4,5,6],[7,8,9]])
c = np.array([0,1,2])
m * c
array([[ 0,  2,  6],
       [ 0,  5, 12],
       [ 0,  8, 18]])
How can i do the same thing if m is scipy sparse CSR matrix? The result should be csr_matrix as well.
This gives dimension mismatch:
sp.sparse.csr_matrix(m)*sp.sparse.csr_matrix(c)

A:
<code>
from scipy import sparse
import numpy as np
example_sA = sparse.csr_matrix(np.array([[1,2,3],[4,5,6],[7,8,9]]))
example_sB = sparse.csr_matrix(np.array([0,1,2]))
def f(sA = example_sA, sB = example_sB):
    # return the solution in this function
    # result = f(sA, sB)
    ### BEGIN SOLUTION

    result = sA.multiply(sB)

    return result

import numpy as np
import copy
from scipy import sparse

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            sa = sparse.csr_matrix(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]))
            sb = sparse.csr_matrix(np.array([0, 1, 2]))
        elif test_case_id == 2:
            sa = sparse.random(10, 10, density=0.2, format="csr", random_state=42)
            sb = sparse.random(10, 1, density=0.4, format="csr", random_state=45)
        return sa, sb

    def generate_ans(data):
        _a = data
        sA, sB = _a
        result = sA.multiply(sB)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert type(result) == sparse.csr.csr_matrix
    assert len(sparse.find(result != ans)[0]) == 0
    return 1

exec_context = r"""
from scipy import sparse
import numpy as np
sA, sB = test_input
def f(sA, sB):
[insert]
result = f(sA, sB)
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(2):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I have some data that comes in the form (x, y, z, V) where x,y,z are distances, and V is the moisture. I read a lot on StackOverflow about interpolation by python like this and this valuable posts, but all of them were about regular grids of x, y, z. i.e. every value of x contributes equally with every point of y, and every point of z. On the other hand, my points came from 3D finite element grid (as below), where the grid is not regular.
The two mentioned posts 1 and 2, defined each of x, y, z as a separate numpy array then they used something like cartcoord = zip(x, y) then scipy.interpolate.LinearNDInterpolator(cartcoord, z) (in a 3D example). I can not do the same as my 3D grid is not regular, thus not each point has a contribution to other points, so if when I repeated these approaches I found many null values, and I got many errors.
Here are 10 sample points in the form of [x, y, z, V]
data = [[27.827, 18.530, -30.417, 0.205] , [24.002, 17.759, -24.782, 0.197] ,
[22.145, 13.687, -33.282, 0.204] , [17.627, 18.224, -25.197, 0.197] ,
[29.018, 18.841, -38.761, 0.212] , [24.834, 20.538, -33.012, 0.208] ,
[26.232, 22.327, -27.735, 0.204] , [23.017, 23.037, -29.230, 0.205] ,
[28.761, 21.565, -31.586, 0.211] , [26.263, 23.686, -32.766, 0.215]]

I want to get the interpolated value V of the point (25, 20, -30).
How can I get it?

A:
<code>
import numpy as np
import scipy.interpolate

points = np.array([
        [ 27.827,  18.53 , -30.417], [ 24.002,  17.759, -24.782],
        [ 22.145,  13.687, -33.282], [ 17.627,  18.224, -25.197],
        [ 29.018,  18.841, -38.761], [ 24.834,  20.538, -33.012],
        [ 26.232,  22.327, -27.735], [ 23.017,  23.037, -29.23 ],
        [ 28.761,  21.565, -31.586], [ 26.263,  23.686, -32.766]])
V = np.array([0.205,  0.197,  0.204,  0.197,  0.212,
                   0.208,  0.204,  0.205, 0.211,  0.215])
request = np.array([[25, 20, -30]])
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import copy
import scipy.interpolate

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            points = np.array(
                [
                    [27.827, 18.53, -30.417],
                    [24.002, 17.759, -24.782],
                    [22.145, 13.687, -33.282],
                    [17.627, 18.224, -25.197],
                    [29.018, 18.841, -38.761],
                    [24.834, 20.538, -33.012],
                    [26.232, 22.327, -27.735],
                    [23.017, 23.037, -29.23],
                    [28.761, 21.565, -31.586],
                    [26.263, 23.686, -32.766],
                ]
            )
            values = np.array(
                [0.205, 0.197, 0.204, 0.197, 0.212, 0.208, 0.204, 0.205, 0.211, 0.215]
            )
            request = np.array([[25, 20, -30]])
        return points, values, request

    def generate_ans(data):
        _a = data
        points, V, request = _a
        result = scipy.interpolate.griddata(points, V, request)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_allclose(result, ans, atol=1e-3)
    return 1

exec_context = r"""
import numpy as np
import scipy.interpolate
points, V, request = test_input
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I have some data that comes in the form (x, y, z, V) where x,y,z are distances, and V is the moisture. I read a lot on StackOverflow about interpolation by python like this and this valuable posts, but all of them were about regular grids of x, y, z. i.e. every value of x contributes equally with every point of y, and every point of z. On the other hand, my points came from 3D finite element grid (as below), where the grid is not regular.
The two mentioned posts 1 and 2, defined each of x, y, z as a separate numpy array then they used something like cartcoord = zip(x, y) then scipy.interpolate.LinearNDInterpolator(cartcoord, z) (in a 3D example). I can not do the same as my 3D grid is not regular, thus not each point has a contribution to other points, so if when I repeated these approaches I found many null values, and I got many errors.
Here are 10 sample points in the form of [x, y, z, V]
data = [[27.827, 18.530, -30.417, 0.205] , [24.002, 17.759, -24.782, 0.197] ,
[22.145, 13.687, -33.282, 0.204] , [17.627, 18.224, -25.197, 0.197] ,
[29.018, 18.841, -38.761, 0.212] , [24.834, 20.538, -33.012, 0.208] ,
[26.232, 22.327, -27.735, 0.204] , [23.017, 23.037, -29.230, 0.205] ,
[28.761, 21.565, -31.586, 0.211] , [26.263, 23.686, -32.766, 0.215]]

I want to get the interpolated value V of the point (25, 20, -30) and (27, 20, -32) as a list.
How can I get it?

A:
<code>
import numpy as np
import scipy.interpolate

points = np.array([
        [ 27.827,  18.53 , -30.417], [ 24.002,  17.759, -24.782],
        [ 22.145,  13.687, -33.282], [ 17.627,  18.224, -25.197],
        [ 29.018,  18.841, -38.761], [ 24.834,  20.538, -33.012],
        [ 26.232,  22.327, -27.735], [ 23.017,  23.037, -29.23 ],
        [ 28.761,  21.565, -31.586], [ 26.263,  23.686, -32.766]])
V = np.array([0.205,  0.197,  0.204,  0.197,  0.212,
                   0.208,  0.204,  0.205, 0.211,  0.215])
request = np.array([[25, 20, -30], [27, 20, -32]])
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import copy
import scipy.interpolate

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            points = np.array(
                [
                    [27.827, 18.53, -30.417],
                    [24.002, 17.759, -24.782],
                    [22.145, 13.687, -33.282],
                    [17.627, 18.224, -25.197],
                    [29.018, 18.841, -38.761],
                    [24.834, 20.538, -33.012],
                    [26.232, 22.327, -27.735],
                    [23.017, 23.037, -29.23],
                    [28.761, 21.565, -31.586],
                    [26.263, 23.686, -32.766],
                ]
            )
            values = np.array(
                [0.205, 0.197, 0.204, 0.197, 0.212, 0.208, 0.204, 0.205, 0.211, 0.215]
            )
            request = np.array([[25, 20, -30], [27, 20, -32]])
        return points, values, request

    def generate_ans(data):
        _a = data
        points, V, request = _a
        result = scipy.interpolate.griddata(points, V, request).tolist()
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_allclose(result, ans, atol=1e-3)
    return 1

exec_context = r"""
import numpy as np
import scipy.interpolate
points, V, request = test_input
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I have a numpy array for an image that I read in from a FITS file. I rotated it by N degrees using scipy.ndimage.interpolation.rotate. Then I want to figure out where some point (x,y) in the original non-rotated frame ends up in the rotated image -- i.e., what are the rotated frame coordinates (x',y')?
This should be a very simple rotation matrix problem but if I do the usual mathematical or programming based rotation equations, the new (x',y') do not end up where they originally were. I suspect this has something to do with needing a translation matrix as well because the scipy rotate function is based on the origin (0,0) rather than the actual center of the image array.
Can someone please tell me how to get the rotated frame (x',y')? As an example, you could use
from scipy import misc
from scipy.ndimage import rotate
data_orig = misc.face()
data_rot = rotate(data_orig,66) # data array
x0,y0 = 580,300 # left eye; (xrot,yrot) should point there
A:
<code>
from scipy import misc
from scipy.ndimage import rotate
import numpy as np
data_orig = misc.face()
x0,y0 = 580,300 # left eye; (xrot,yrot) should point there
angle = np.random.randint(1, 360)
</code>
data_rot, xrot, yrot = ... # put solution in these variables
BEGIN SOLUTION
<code>

def rot_ans(image, xy, angle):
    im_rot = rotate(image,angle)
    org_center = (np.array(image.shape[:2][::-1])-1)/2.
    rot_center = (np.array(im_rot.shape[:2][::-1])-1)/2.
    org = xy-org_center
    a = np.deg2rad(angle)
    new = np.array([org[0]*np.cos(a) + org[1]*np.sin(a),
            -org[0]*np.sin(a) + org[1]*np.cos(a) ])
    return im_rot, new+rot_center
data_rot, (xrot, yrot) =rot_ans(data_orig, np.array([x0, y0]), angle)

import numpy as np
import copy
from scipy import misc
from scipy.ndimage import rotate

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            data_orig = misc.face()
            x0, y0 = 580, 300  # left eye; (xrot,yrot) should point there
            np.random.seed(42)
            angle = np.random.randint(1, 360)
        return data_orig, x0, y0, angle

    def generate_ans(data):
        _a = data
        data_orig, x0, y0, angle = _a

        def rot_ans(image, xy, angle):
            im_rot = rotate(image, angle)
            org_center = (np.array(image.shape[:2][::-1]) - 1) / 2.0
            rot_center = (np.array(im_rot.shape[:2][::-1]) - 1) / 2.0
            org = xy - org_center
            a = np.deg2rad(angle)
            new = np.array(
                [
                    org[0] * np.cos(a) + org[1] * np.sin(a),
                    -org[0] * np.sin(a) + org[1] * np.cos(a),
                ]
            )
            return im_rot, new + rot_center

        data_rot, (xrot, yrot) = rot_ans(data_orig, np.array([x0, y0]), angle)
        return [data_rot, (xrot, yrot)]

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    res, (x1, y1) = result
    answer, (x_ans, y_ans) = ans
    assert np.allclose((x1, y1), (x_ans, y_ans))
    assert np.allclose(res, answer)
    return 1

exec_context = r"""
from scipy import misc
from scipy.ndimage import rotate
import numpy as np
data_orig, x0, y0, angle = test_input
[insert]
result = [data_rot, (xrot, yrot)]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
How can I extract the main diagonal(1-d array) of a sparse matrix? The matrix is created in scipy.sparse. I want equivalent of np.diagonal(), but for sparse matrix.

A:
<code>
import numpy as np
from scipy.sparse import csr_matrix

arr = np.random.rand(4, 4)
M = csr_matrix(arr)
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import copy
from scipy.sparse import csr_matrix

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            arr = np.array(
                [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]
            )
        elif test_case_id == 2:
            np.random.seed(42)
            arr = np.random.rand(6, 6)
        return arr

    def generate_ans(data):
        _a = data
        arr = _a
        M = csr_matrix(arr)
        result = M.A.diagonal(0)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_array_equal(result, ans)
    return 1

exec_context = r"""
import numpy as np
from scipy.sparse import csr_matrix
arr = test_input
M = csr_matrix(arr)
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(2):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I simulate times in the range 0 to T according to a Poisson process. The inter-event times are exponential and we know that the distribution of the times should be uniform in the range 0 to T.
def poisson_simul(rate, T):
    time = random.expovariate(rate)
    times = [0]
    while (times[-1] < T):
        times.append(time+times[-1])
        time = random.expovariate(rate)
    return times[1:]
I would simply like to run one of the tests for uniformity, for example the Kolmogorov-Smirnov test. I can't work out how to do this in scipy however. If I do
import random
from scipy.stats import kstest
times = poisson_simul(1, 100)
print kstest(times, "uniform")
it is not right . It gives me
(1.0, 0.0)
I just want to test the hypothesis that the points are uniformly chosen from the range 0 to T. How do you do this in scipy? The result should be KStest result.
A:
<code>
from scipy import stats
import random
import numpy as np
def poisson_simul(rate, T):
    time = random.expovariate(rate)
    times = [0]
    while (times[-1] < T):
        times.append(time+times[-1])
        time = random.expovariate(rate)
    return times[1:]
rate = 1.0
T = 100.0
times = poisson_simul(rate, T)
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import random
import copy
from scipy import stats

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:

            def poisson_simul(rate, T):
                time = random.expovariate(rate)
                times = [0]
                while times[-1] < T:
                    times.append(time + times[-1])
                    time = random.expovariate(rate)
                return times[1:]

            random.seed(42)
            rate = 1.0
            T = 100.0
            times = poisson_simul(rate, T)
        return rate, T, times

    def generate_ans(data):
        _a = data
        rate, T, times = _a
        result = stats.kstest(times, stats.uniform(loc=0, scale=T).cdf)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_allclose(result, ans)
    return 1

exec_context = r"""
from scipy import stats
import random
import numpy as np
def poisson_simul(rate, T):
    time = random.expovariate(rate)
    times = [0]
    while (times[-1] < T):
        times.append(time+times[-1])
        time = random.expovariate(rate)
    return times[1:]
rate, T, times = test_input
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I simulate times in the range 0 to T according to a Poisson process. The inter-event times are exponential and we know that the distribution of the times should be uniform in the range 0 to T.
def poisson_simul(rate, T):
    time = random.expovariate(rate)
    times = [0]
    while (times[-1] < T):
        times.append(time+times[-1])
        time = random.expovariate(rate)
    return times[1:]
I would simply like to run one of the tests for uniformity, for example the Kolmogorov-Smirnov test. I can't work out how to do this in scipy however. If I do
import random
from scipy.stats import kstest
times = poisson_simul(1, 100)
print kstest(times, "uniform")
it is not right . It gives me
(1.0, 0.0)
I just want to test the hypothesis that the points are uniformly chosen from the range 0 to T. How do you do this in scipy? The result should be KStest result.
A:
<code>
from scipy import stats
import random
import numpy as np
def poisson_simul(rate, T):
    time = random.expovariate(rate)
    times = [0]
    while (times[-1] < T):
        times.append(time+times[-1])
        time = random.expovariate(rate)
    return times[1:]
example_rate = 1.0
example_T = 100.0
example_times = poisson_simul(example_rate, example_T)
def f(times = example_times, rate = example_rate, T = example_T):
    # return the solution in this function
    # result = f(times, rate, T)
    ### BEGIN SOLUTION

import numpy as np
import random
import copy
from scipy import stats

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:

            def poisson_simul(rate, T):
                time = random.expovariate(rate)
                times = [0]
                while times[-1] < T:
                    times.append(time + times[-1])
                    time = random.expovariate(rate)
                return times[1:]

            random.seed(42)
            rate = 1.0
            T = 100.0
            times = poisson_simul(rate, T)
        return rate, T, times

    def generate_ans(data):
        _a = data
        rate, T, times = _a
        result = stats.kstest(times, stats.uniform(loc=0, scale=T).cdf)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_allclose(result, ans)
    return 1

exec_context = r"""
from scipy import stats
import random
import numpy as np
def poisson_simul(rate, T):
    time = random.expovariate(rate)
    times = [0]
    while (times[-1] < T):
        times.append(time+times[-1])
        time = random.expovariate(rate)
    return times[1:]
rate, T, times = test_input
def f(times, rate, T):
[insert]
result = f(times, rate, T)
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I simulate times in the range 0 to T according to a Poisson process. The inter-event times are exponential and we know that the distribution of the times should be uniform in the range 0 to T.
def poisson_simul(rate, T):
    time = random.expovariate(rate)
    times = [0]
    while (times[-1] < T):
        times.append(time+times[-1])
        time = random.expovariate(rate)
    return times[1:]
I would simply like to run one of the tests for uniformity, for example the Kolmogorov-Smirnov test. I can't work out how to do this in scipy however. If I do
import random
from scipy.stats import kstest
times = poisson_simul(1, 100)
print kstest(times, "uniform")
it is not right . It gives me
(1.0, 0.0)
I just want to test the hypothesis that the points are uniformly chosen from the range 0 to T. How do you do this in scipy? Another question is how to interpret the result? What I want is just `True` for unifomity or `False` vice versa. Suppose I want a confidence level of 95%.
A:
<code>
from scipy import stats
import random
import numpy as np
def poisson_simul(rate, T):
    time = random.expovariate(rate)
    times = [0]
    while (times[-1] < T):
        times.append(time+times[-1])
        time = random.expovariate(rate)
	return times[1:]
rate = 1.0
T = 100.0
times = poisson_simul(rate, T)
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

res= stats.kstest(times, stats.uniform(loc=0, scale=T).cdf)

if res[1] < 0.05:
    result = False
else:
    result = True

import numpy as np
import random
import copy
from scipy import stats

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        def poisson_simul(rate, T):
            time = random.expovariate(rate)
            times = [0]
            while times[-1] < T:
                times.append(time + times[-1])
                time = random.expovariate(rate)
            return times[1:]

        if test_case_id == 1:
            random.seed(42)
            rate = 1
            T = 100
            times = poisson_simul(rate, T)
        elif test_case_id == 2:
            random.seed(45)
            rate = 1
            T = 500
            times = poisson_simul(rate, T)
        return rate, T, times

    def generate_ans(data):
        _a = data
        rate, T, times = _a
        res = stats.kstest(times, stats.uniform(loc=0, scale=T).cdf)
        if res[1] < 0.05:
            result = False
        else:
            result = True
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_array_equal(result, ans)
    return 1

exec_context = r"""
from scipy import stats
import random
import numpy as np
def poisson_simul(rate, T):
    time = random.expovariate(rate)
    times = [0]
    while (times[-1] < T):
        times.append(time+times[-1])
        time = random.expovariate(rate)
    return times[1:]
rate, T, times = test_input
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(2):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I have two csr_matrix, c1, c2.

I want a new matrix Feature = [c1, c2]. But if I directly concatenate them horizontally this way, there's an error that says the matrix Feature is a list. How can I achieve the matrix concatenation and still get the same type of matrix, i.e. a csr_matrix?

And it doesn't work if I do this after the concatenation: Feature = csr_matrix(Feature) It gives the error:

Traceback (most recent call last):
  File "yelpfilter.py", line 91, in <module>
    Feature = csr_matrix(Feature)
  File "c:\python27\lib\site-packages\scipy\sparse\compressed.py", line 66, in __init__
    self._set_self( self.__class__(coo_matrix(arg1, dtype=dtype)) )
  File "c:\python27\lib\site-packages\scipy\sparse\coo.py", line 185, in __init__
    self.row, self.col = M.nonzero()
TypeError: __nonzero__ should return bool or int, returned numpy.bool_

A:
<code>
from scipy import sparse
c1 = sparse.csr_matrix([[0, 0, 1, 0], [2, 0, 0, 0], [0, 0, 0, 0]])
c2 = sparse.csr_matrix([[0, 3, 4, 0], [0, 0, 0, 5], [6, 7, 0, 8]])
</code>
Feature = ... # put solution in this variable
BEGIN SOLUTION
<code>

import copy
from scipy import sparse

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            c1 = sparse.csr_matrix([[0, 0, 1, 0], [2, 0, 0, 0], [0, 0, 0, 0]])
            c2 = sparse.csr_matrix([[0, 3, 4, 0], [0, 0, 0, 5], [6, 7, 0, 8]])
        return c1, c2

    def generate_ans(data):
        _a = data
        c1, c2 = _a
        Feature = sparse.hstack((c1, c2)).tocsr()
        return Feature

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert type(result) == sparse.csr_matrix
    assert len(sparse.find(ans != result)[0]) == 0
    return 1

exec_context = r"""
from scipy import sparse
c1, c2 = test_input
[insert]
result = Feature
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I have two csr_matrix, c1 and c2.

I want a new sparse matrix Feature = [c1, c2], that is, to stack c1 and c2 horizontally to get a new sparse matrix.

To make use of sparse matrix's memory efficiency, I don't want results as dense arrays.

But if I directly concatenate them this way, there's an error that says the matrix Feature is a list.

And if I try this: Feature = csr_matrix(Feature) It gives the error:

Traceback (most recent call last):
  File "yelpfilter.py", line 91, in <module>
    Feature = csr_matrix(Feature)
  File "c:\python27\lib\site-packages\scipy\sparse\compressed.py", line 66, in __init__
    self._set_self( self.__class__(coo_matrix(arg1, dtype=dtype)) )
  File "c:\python27\lib\site-packages\scipy\sparse\coo.py", line 185, in __init__
    self.row, self.col = M.nonzero()
TypeError: __nonzero__ should return bool or int, returned numpy.bool_

Any help would be appreciated!

A:
<code>
from scipy import sparse
c1 = sparse.csr_matrix([[0, 0, 1, 0], [2, 0, 0, 0], [0, 0, 0, 0]])
c2 = sparse.csr_matrix([[0, 3, 4, 0], [0, 0, 0, 5], [6, 7, 0, 8]])
</code>
Feature = ... # put solution in this variable
BEGIN SOLUTION
<code>

import copy
from scipy import sparse

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            c1 = sparse.csr_matrix([[0, 0, 1, 0], [2, 0, 0, 0], [0, 0, 0, 0]])
            c2 = sparse.csr_matrix([[0, 3, 4, 0], [0, 0, 0, 5], [6, 7, 0, 8]])
        return c1, c2

    def generate_ans(data):
        _a = data
        c1, c2 = _a
        Feature = sparse.hstack((c1, c2)).tocsr()
        return Feature

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert type(result) == sparse.csr_matrix
    assert len(sparse.find(ans != result)[0]) == 0
    return 1

exec_context = r"""
from scipy import sparse
c1, c2 = test_input
[insert]
result = Feature
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I have two csr_matrix, c1 and c2.

I want a new matrix
Feature = [c1
           c2].

That is, I want to concatenate c1 and c2 in vertical direction.

But I don't know how to represent the concatenation or how to form the format.

How can I achieve the matrix concatenation and still get the same type of matrix, i.e. a csr_matrix?

Any help would be appreciated.

A:
<code>
from scipy import sparse
c1 = sparse.csr_matrix([[0, 0, 1, 0], [2, 0, 0, 0], [0, 0, 0, 0]])
c2 = sparse.csr_matrix([[0, 3, 4, 0], [0, 0, 0, 5], [6, 7, 0, 8]])
</code>
Feature = ... # put solution in this variable
BEGIN SOLUTION
<code>

import copy
from scipy import sparse

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            c1 = sparse.csr_matrix([[0, 0, 1, 0], [2, 0, 0, 0], [0, 0, 0, 0]])
            c2 = sparse.csr_matrix([[0, 3, 4, 0], [0, 0, 0, 5], [6, 7, 0, 8]])
        return c1, c2

    def generate_ans(data):
        _a = data
        c1, c2 = _a
        Feature = sparse.vstack((c1, c2))
        return Feature

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert type(result) == sparse.csr_matrix
    assert len(sparse.find(ans != result)[0]) == 0
    return 1

exec_context = r"""
from scipy import sparse
c1, c2 = test_input
[insert]
result = Feature
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
Given two sets of points in n-dimensional space, how can one map points from one set to the other, such that each point is only used once and the total euclidean distance between the pairs of points is minimized?
For example,
import matplotlib.pyplot as plt
import numpy as np
# create six points in 2d space; the first three belong to set "A" and the
# second three belong to set "B"
x = [1, 2, 3, 1.8, 1.9, 3.4]
y = [2, 3, 1, 2.6, 3.4, 0.4]
colors = ['red'] * 3 + ['blue'] * 3
plt.scatter(x, y, c=colors)
plt.show()
So in the example above, the goal would be to map each red point to a blue point such that each blue point is only used once and the sum of the distances between points is minimized.
The application I have in mind involves a fairly small number of datapoints in 3-dimensional space, so the brute force approach might be fine, but I thought I would check to see if anyone knows of a more efficient or elegant solution first.
The result should be an assignment of points in second set to corresponding elements in the first set.
For example, a matching solution is
Points1 <-> Points2
    0   ---     2
    1   ---     0
    2   ---     1
and the result is [2, 0, 1]

A:
<code>
import numpy as np
import scipy.spatial
import scipy.optimize
points1 = np.array([(x, y) for x in np.linspace(-1,1,7) for y in np.linspace(-1,1,7)])
N = points1.shape[0]
points2 = 2*np.random.rand(N,2)-1
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import copy
import scipy.spatial
import scipy.optimize

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            np.random.seed(100)
            points1 = np.array(
                [(x, y) for x in np.linspace(-1, 1, 7) for y in np.linspace(-1, 1, 7)]
            )
            N = points1.shape[0]
            points2 = 2 * np.random.rand(N, 2) - 1
        return points1, N, points2

    def generate_ans(data):
        _a = data
        points1, N, points2 = _a
        C = scipy.spatial.distance.cdist(points1, points2)
        _, result = scipy.optimize.linear_sum_assignment(C)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_array_equal(result, ans)
    return 1

exec_context = r"""
import numpy as np
import scipy.spatial
import scipy.optimize
np.random.seed(100)
points1, N, points2 = test_input
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
Given two sets of points in n-dimensional space, how can one map points from one set to the other, such that each point is only used once and the total Manhattan distance between the pairs of points is minimized?
For example,
import matplotlib.pyplot as plt
import numpy as np
# create six points in 2d space; the first three belong to set "A" and the
# second three belong to set "B"
x = [1, 2, 3, 1.8, 1.9, 3.4]
y = [2, 3, 1, 2.6, 3.4, 0.4]
colors = ['red'] * 3 + ['blue'] * 3
plt.scatter(x, y, c=colors)
plt.show()
So in the example above, the goal would be to map each red point to a blue point such that each blue point is only used once and the sum of the distances between points is minimized.
The application I have in mind involves a fairly small number of datapoints in 3-dimensional space, so the brute force approach might be fine, but I thought I would check to see if anyone knows of a more efficient or elegant solution first.
The result should be an assignment of points in second set to corresponding elements in the first set.
For example, a matching solution is
Points1 <-> Points2
    0   ---     2
    1   ---     0
    2   ---     1
and the result is [2, 0, 1]

A:
<code>
import numpy as np
import scipy.spatial
import scipy.optimize
points1 = np.array([(x, y) for x in np.linspace(-1,1,7) for y in np.linspace(-1,1,7)])
N = points1.shape[0]
points2 = 2*np.random.rand(N,2)-1
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

C = scipy.spatial.distance.cdist(points1, points2, metric='minkowski', p=1)
_, result = scipy.optimize.linear_sum_assignment(C)

import numpy as np
import copy
import scipy.spatial
import scipy.optimize

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            np.random.seed(100)
            points1 = np.array(
                [(x, y) for x in np.linspace(-1, 1, 7) for y in np.linspace(-1, 1, 7)]
            )
            N = points1.shape[0]
            points2 = 2 * np.random.rand(N, 2) - 1
        return points1, N, points2

    def generate_ans(data):
        _a = data
        points1, N, points2 = _a
        C = scipy.spatial.distance.cdist(points1, points2, metric="minkowski", p=1)
        _, result = scipy.optimize.linear_sum_assignment(C)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_array_equal(result, ans)
    return 1

exec_context = r"""
import numpy as np
import scipy.spatial
import scipy.optimize
points1, N, points2 = test_input
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I want to remove diagonal elements from a sparse matrix. Since the matrix is sparse, these elements shouldn't be stored once removed.
Scipy provides a method to set diagonal elements values: setdiag
If I try it using lil_matrix, it works:
>>> a = np.ones((2,2))
>>> c = lil_matrix(a)
>>> c.setdiag(0)
>>> c
<2x2 sparse matrix of type '<type 'numpy.float64'>'
    with 2 stored elements in LInked List format>
However with csr_matrix, it seems diagonal elements are not removed from storage:
>>> b = csr_matrix(a)
>>> b
<2x2 sparse matrix of type '<type 'numpy.float64'>'
    with 4 stored elements in Compressed Sparse Row format>

>>> b.setdiag(0)
>>> b
<2x2 sparse matrix of type '<type 'numpy.float64'>'
    with 4 stored elements in Compressed Sparse Row format>

>>> b.toarray()
array([[ 0.,  1.],
       [ 1.,  0.]])
Through a dense array, we have of course:
>>> csr_matrix(b.toarray())
<2x2 sparse matrix of type '<type 'numpy.float64'>'
    with 2 stored elements in Compressed Sparse Row format>
Is that intended? If so, is it due to the compressed format of csr matrices? Is there any workaround else than going from sparse to dense to sparse again?
A:
<code>
from scipy import sparse
import numpy as np
a = np.ones((2, 2))
b = sparse.csr_matrix(a)
</code>
b = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import copy
import tokenize, io
from scipy import sparse

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            a = np.ones((2, 2))
            b = sparse.csr_matrix(a)
        elif test_case_id == 2:
            a = []
            b = sparse.csr_matrix([])
        return a, b

    def generate_ans(data):
        _a = data
        a, b = _a
        b = sparse.csr_matrix(a)
        b.setdiag(0)
        b.eliminate_zeros()
        return b

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert type(result) == type(ans)
    assert len(sparse.find(result != ans)[0]) == 0
    assert result.nnz == ans.nnz
    return 1

exec_context = r"""
from scipy import sparse
import numpy as np
a, b = test_input
[insert]
result = b
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(2):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)

def test_string(solution: str):
    tokens = []
    for token in tokenize.tokenize(io.BytesIO(solution.encode("utf-8")).readline):
        tokens.append(token.string)
    assert (
        "toarray" not in tokens
        and "array" not in tokens
        and "todense" not in tokens
        and "A" not in tokens
    )


Prompt:
Problem:
I am working with a 2D numpy array made of 512x512=262144 values. Such values are of float type and range from 0.0 to 1.0. The array has an X,Y coordinate system which originates in the top left corner: thus, position (0,0) is in the top left corner, while position (512,512) is in the bottom right corner.
This is how the 2D array looks like (just an excerpt):
X,Y,Value
0,0,0.482
0,1,0.49
0,2,0.496
0,3,0.495
0,4,0.49
0,5,0.489
0,6,0.5
0,7,0.504
0,8,0.494
0,9,0.485

I would like to be able to:
Count the number of regions of cells which value exceeds a given threshold, i.e. 0.75;

Note: If two elements touch horizontally, vertically or diagnoally, they belong to one region.

A:
<code>
import numpy as np
from scipy import ndimage

np.random.seed(10)
gen = np.random.RandomState(0)
img = gen.poisson(2, size=(512, 512))
img = ndimage.gaussian_filter(img.astype(np.double), (30, 30))
img -= img.min()
img /= img.max()
threshold = 0.75
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import copy
from scipy import ndimage

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            np.random.seed(10)
            gen = np.random.RandomState(0)
            img = gen.poisson(2, size=(512, 512))
            img = ndimage.gaussian_filter(img.astype(np.double), (30, 30))
            img -= img.min()
            img /= img.max()
        return img

    def generate_ans(data):
        _a = data
        img = _a
        threshold = 0.75
        blobs = img > threshold
        labels, result = ndimage.label(blobs)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_array_equal(result, ans)
    return 1

exec_context = r"""
import numpy as np
from scipy import ndimage
img = test_input
threshold = 0.75
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I am working with a 2D numpy array made of 512x512=262144 values. Such values are of float type and range from 0.0 to 1.0. The array has an X,Y coordinate system which originates in the top left corner: thus, position (0,0) is in the top left corner, while position (512,512) is in the bottom right corner.
This is how the 2D array looks like (just an excerpt):
X,Y,Value
0,0,0.482
0,1,0.49
0,2,0.496
0,3,0.495
0,4,0.49
0,5,0.489
0,6,0.5
0,7,0.504
0,8,0.494
0,9,0.485

I would like to be able to:
Count the number of regions of cells which value below a given threshold, i.e. 0.75;

Note: If two elements touch horizontally, vertically or diagnoally, they belong to one region.

A:
<code>
import numpy as np
from scipy import ndimage

np.random.seed(10)
gen = np.random.RandomState(0)
img = gen.poisson(2, size=(512, 512))
img = ndimage.gaussian_filter(img.astype(np.double), (30, 30))
img -= img.min()
img /= img.max()
threshold = 0.75
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

import numpy as np
import copy
from scipy import ndimage

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            np.random.seed(10)
            gen = np.random.RandomState(0)
            img = gen.poisson(2, size=(512, 512))
            img = ndimage.gaussian_filter(img.astype(np.double), (30, 30))
            img -= img.min()
            img /= img.max()
        return img

    def generate_ans(data):
        _a = data
        img = _a
        threshold = 0.75
        blobs = img < threshold
        labels, result = ndimage.label(blobs)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_array_equal(result, ans)
    return 1

exec_context = r"""
import numpy as np
from scipy import ndimage
img = test_input
threshold = 0.75
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I am working with a 2D numpy array made of 512x512=262144 values. Such values are of float type and range from 0.0 to 1.0. The array has an X,Y coordinate system which originates in the top left corner: thus, position (0,0) is in the top left corner, while position (512,512) is in the bottom right corner.
This is how the 2D array looks like (just an excerpt):
X,Y,Value
0,0,0.482
0,1,0.49
0,2,0.496
0,3,0.495
0,4,0.49
0,5,0.489
0,6,0.5
0,7,0.504
0,8,0.494
0,9,0.485

I would like to be able to:
Count the number of regions of cells which value exceeds a given threshold, i.e. 0.75;

Note: If two elements touch horizontally, vertically or diagnoally, they belong to one region.

A:
<code>
import numpy as np
from scipy import ndimage
np.random.seed(10)
gen = np.random.RandomState(0)
img = gen.poisson(2, size=(512, 512))
img = ndimage.gaussian_filter(img.astype(np.double), (30, 30))
img -= img.min()
example_img /= img.max()
def f(img = example_img):
    threshold = 0.75
    # return the solution in this function
    # result = f(img)
    ### BEGIN SOLUTION

import numpy as np
import copy
from scipy import ndimage

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            np.random.seed(10)
            gen = np.random.RandomState(0)
            img = gen.poisson(2, size=(512, 512))
            img = ndimage.gaussian_filter(img.astype(np.double), (30, 30))
            img -= img.min()
            img /= img.max()
        return img

    def generate_ans(data):
        _a = data
        img = _a
        threshold = 0.75
        blobs = img > threshold
        labels, result = ndimage.label(blobs)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_array_equal(result, ans)
    return 1

exec_context = r"""
import numpy as np
from scipy import ndimage
def f(img):
    threshold = 0.75
[insert]
img = test_input
result = f(img)
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
I am working with a 2D numpy array made of 512x512=262144 values. Such values are of float type and range from 0.0 to 1.0. The array has an X,Y coordinate system which originates in the top left corner: thus, position (0,0) is in the top left corner, while position (512,512) is in the bottom right corner.
This is how the 2D array looks like (just an excerpt):
X,Y,Value
0,0,0.482
0,1,0.49
0,2,0.496
0,3,0.495
0,4,0.49
0,5,0.489
0,6,0.5
0,7,0.504
0,8,0.494
0,9,0.485

I would like to be able to:
Find the regions of cells which value exceeds a given threshold, say 0.75;

Note: If two elements touch horizontally, vertically or diagnoally, they belong to one region.

Determine the distance between the center of mass of such regions and the top left corner, which has coordinates (0,0).
Please output the distances as a list.

A:
<code>
import numpy as np
from scipy import ndimage

np.random.seed(10)
gen = np.random.RandomState(0)
img = gen.poisson(2, size=(512, 512))
img = ndimage.gaussian_filter(img.astype(np.double), (30, 30))
img -= img.min()
img /= img.max()
threshold = 0.75
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

blobs = img > threshold
labels, nlabels = ndimage.label(blobs)
r, c = np.vstack(ndimage.center_of_mass(img, labels, np.arange(nlabels) + 1)).T
# find their distances from the top-left corner
d = np.sqrt(r * r + c * c)
result = sorted(d)

import numpy as np
import copy
from scipy import ndimage

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            np.random.seed(10)
            gen = np.random.RandomState(0)
            img = gen.poisson(2, size=(512, 512))
            img = ndimage.gaussian_filter(img.astype(np.double), (30, 30))
            img -= img.min()
            img /= img.max()
        return img

    def generate_ans(data):
        _a = data
        img = _a
        threshold = 0.75
        blobs = img > threshold
        labels, nlabels = ndimage.label(blobs)
        r, c = np.vstack(ndimage.center_of_mass(img, labels, np.arange(nlabels) + 1)).T
        d = np.sqrt(r * r + c * c)
        result = sorted(d)
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_allclose(sorted(result), ans)
    return 1

exec_context = r"""
import numpy as np
from scipy import ndimage
img = test_input
threshold = 0.75
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)


Prompt:
Problem:
Is there a simple and efficient way to make a sparse scipy matrix (e.g. lil_matrix, or csr_matrix) symmetric?
Currently I have a lil sparse matrix, and not both of sA[i,j] and sA[j,i] have element for any i,j.
When populating a large sparse co-occurrence matrix it would be highly inefficient to fill in [row, col] and [col, row] at the same time. What I'd like to be doing is:
for i in data:
    for j in data:
        if have_element(i, j):
            lil_sparse_matrix[i, j] = some_value
            # want to avoid this:
            # lil_sparse_matrix[j, i] = some_value
# this is what I'm looking for:
lil_sparse.make_symmetric()
and it let sA[i,j] = sA[j,i] for any i, j.

This is similar to stackoverflow's numpy-smart-symmetric-matrix question, but is particularly for scipy sparse matrices.

A:

import numpy as np
from scipy.sparse import lil_matrix
from scipy import sparse

M= sparse.random(10, 10, density=0.1, format='lil')

M = ... # put solution in this variable
BEGIN SOLUTION

import copy
import tokenize, io
from scipy import sparse

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            sA = sparse.random(10, 10, density=0.1, format="lil", random_state=42)
        return sA

    def generate_ans(data):
        _a = data
        M = _a
        rows, cols = M.nonzero()
        M[cols, rows] = M[rows, cols]
        return M

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert len(sparse.find(result != ans)[0]) == 0
    return 1

exec_context = r"""
import numpy as np
from scipy.sparse import lil_matrix
from scipy import sparse
M = test_input
[insert]
result = M
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)

def test_string(solution: str):
    tokens = []
    for token in tokenize.tokenize(io.BytesIO(solution.encode("utf-8")).readline):
        tokens.append(token.string)
    assert "while" not in tokens and "for" not in tokens


Prompt:
Problem:
Is there a simple and efficient way to make a sparse scipy matrix (e.g. lil_matrix, or csr_matrix) symmetric?
Currently I have a lil sparse matrix, and not both of sA[i,j] and sA[j,i] have element for any i,j.
When populating a large sparse co-occurrence matrix it would be highly inefficient to fill in [row, col] and [col, row] at the same time. What I'd like to be doing is:
for i in data:
    for j in data:
        if have_element(i, j):
            lil_sparse_matrix[i, j] = some_value
            # want to avoid this:
            # lil_sparse_matrix[j, i] = some_value
# this is what I'm looking for:
lil_sparse.make_symmetric()
and it let sA[i,j] = sA[j,i] for any i, j.

This is similar to stackoverflow's numpy-smart-symmetric-matrix question, but is particularly for scipy sparse matrices.

A:

import numpy as np
from scipy.sparse import lil_matrix
example_sA = sparse.random(10, 10, density=0.1, format='lil')
def f(sA = example_sA):
    # return the solution in this function
    # sA = f(sA)
    ### BEGIN SOLUTION

import copy
import tokenize, io
from scipy import sparse

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            sA = sparse.random(10, 10, density=0.1, format="lil", random_state=42)
        return sA

    def generate_ans(data):
        _a = data
        sA = _a
        rows, cols = sA.nonzero()
        sA[cols, rows] = sA[rows, cols]
        return sA

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert len(sparse.find(result != ans)[0]) == 0
    return 1

exec_context = r"""
import numpy as np
from scipy.sparse import lil_matrix
def f(sA):
[insert]
sA = test_input
result = f(sA)
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)

def test_string(solution: str):
    tokens = []
    for token in tokenize.tokenize(io.BytesIO(solution.encode("utf-8")).readline):
        tokens.append(token.string)
    assert "while" not in tokens and "for" not in tokens


Prompt:
Problem:

I'm trying to reduce noise in a binary python array by removing all completely isolated single cells, i.e. setting "1" value cells to 0 if they are completely surrounded by other "0"s like this:
0 0 0
0 1 0
0 0 0
 I have been able to get a working solution by removing blobs with sizes equal to 1 using a loop, but this seems like a very inefficient solution for large arrays.
In this case, eroding and dilating my array won't work as it will also remove features with a width of 1. I feel the solution lies somewhere within the scipy.ndimage package, but so far I haven't been able to crack it. Any help would be greatly appreciated!

A:
<code>
import numpy as np
import scipy.ndimage
square = np.zeros((32, 32))
square[10:-10, 10:-10] = 1
np.random.seed(12)
x, y = (32*np.random.random((2, 20))).astype(int)
square[x, y] = 1
</code>
square = ... # put solution in this variable
BEGIN SOLUTION
<code>

def filter_isolated_cells(array, struct):
    filtered_array = np.copy(array)
    id_regions, num_ids = scipy.ndimage.label(filtered_array, structure=struct)
    id_sizes = np.array(scipy.ndimage.sum(array, id_regions, range(num_ids + 1)))
    area_mask = (id_sizes == 1)
    filtered_array[area_mask[id_regions]] = 0
    return filtered_array
square = filter_isolated_cells(square, struct=np.ones((3,3)))

import numpy as np
import copy
import tokenize, io
import scipy.ndimage

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            square = np.zeros((32, 32))
            square[10:-10, 10:-10] = 1
            np.random.seed(12)
            x, y = (32 * np.random.random((2, 20))).astype(int)
            square[x, y] = 1
        return square

    def generate_ans(data):
        _a = data
        square = _a

        def filter_isolated_cells(array, struct):
            filtered_array = np.copy(array)
            id_regions, num_ids = scipy.ndimage.label(filtered_array, structure=struct)
            id_sizes = np.array(
                scipy.ndimage.sum(array, id_regions, range(num_ids + 1))
            )
            area_mask = id_sizes == 1
            filtered_array[area_mask[id_regions]] = 0
            return filtered_array

        square = filter_isolated_cells(square, struct=np.ones((3, 3)))
        return square

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_array_equal(result, ans)
    return 1

exec_context = r"""
import numpy as np
import scipy.ndimage
square = test_input
[insert]
result = square
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)

def test_string(solution: str):
    tokens = []
    for token in tokenize.tokenize(io.BytesIO(solution.encode("utf-8")).readline):
        tokens.append(token.string)
    assert "while" not in tokens and "for" not in tokens


Prompt:
Problem:

I'm trying to reduce noise in a python image array by removing all completely isolated single cells, i.e. setting nonzero value cells to 0 if they are completely surrounded by other "0"s like this:
0 0 0
0 8 0
0 0 0
 I have been able to get a working solution by removing blobs with sizes equal to 1 using a loop, but this seems like a very inefficient solution for large arrays.
In this case, eroding and dilating my array won't work as it will also remove features with a width of 1. I feel the solution lies somewhere within the scipy.ndimage package, but so far I haven't been able to crack it. Any help would be greatly appreciated!

A:
<code>
import numpy as np
import scipy.ndimage
square = np.zeros((32, 32))
square[10:-10, 10:-10] = np.random.randint(1, 255, size = (12, 12))
np.random.seed(12)
x, y = (32*np.random.random((2, 20))).astype(int)
square[x, y] = np.random.randint(1, 255, size = (20,))

</code>
square = ... # put solution in this variable
BEGIN SOLUTION
<code>

def filter_isolated_cells(array, struct):
    filtered_array = np.copy(array)
    id_regions, num_ids = scipy.ndimage.label(filtered_array, structure=struct)
    id_sizes = np.array(scipy.ndimage.sum(array, id_regions, range(num_ids + 1)))
    area_mask = (id_sizes == 1)
    filtered_array[area_mask[id_regions]] = 0
    return filtered_array
arr = np.sign(square)
filtered_array = filter_isolated_cells(arr, struct=np.ones((3,3)))
square = np.where(filtered_array==1, square, 0)

import numpy as np
import copy
import tokenize, io
import scipy.ndimage

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            np.random.seed(42)
            square = np.zeros((32, 32))
            square[10:-10, 10:-10] = np.random.randint(1, 255, size=(12, 12))
            np.random.seed(12)
            x, y = (32 * np.random.random((2, 20))).astype(int)
            square[x, y] = np.random.randint(1, 255, size=(20,))
        return square

    def generate_ans(data):
        _a = data
        square = _a

        def filter_isolated_cells(array, struct):
            filtered_array = np.copy(array)
            id_regions, num_ids = scipy.ndimage.label(filtered_array, structure=struct)
            id_sizes = np.array(
                scipy.ndimage.sum(array, id_regions, range(num_ids + 1))
            )
            area_mask = id_sizes == 1
            filtered_array[area_mask[id_regions]] = 0
            return filtered_array

        arr = np.sign(square)
        filtered_array = filter_isolated_cells(arr, struct=np.ones((3, 3)))
        square = np.where(filtered_array == 1, square, 0)
        return square

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    np.testing.assert_array_equal(result, ans)
    return 1

exec_context = r"""
import numpy as np
import scipy.ndimage
square = test_input
[insert]
result = square
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)

def test_string(solution: str):
    tokens = []
    for token in tokenize.tokenize(io.BytesIO(solution.encode("utf-8")).readline):
        tokens.append(token.string)
    assert "while" not in tokens and "for" not in tokens


Prompt:
Problem:
I have a sparse 988x1 vector (stored in col, a column in a csr_matrix) created through scipy.sparse. Is there a way to gets its mean and standard deviation without having to convert the sparse matrix to a dense one?
numpy.mean seems to only work for dense vectors.

A:
<code>
import numpy as np
from scipy.sparse import csr_matrix

np.random.seed(10)
arr = np.random.randint(4,size=(988,988))
sA = csr_matrix(arr)
col = sA.getcol(0)
</code>
mean, standard_deviation = ... # put solution in these variables
BEGIN SOLUTION
<code>

import numpy as np
import copy
import tokenize, io
from scipy.sparse import csr_matrix

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            np.random.seed(10)
            arr = np.random.randint(4, size=(988, 988))
            A = csr_matrix(arr)
            col = A.getcol(0)
        elif test_case_id == 2:
            np.random.seed(42)
            arr = np.random.randint(4, size=(988, 988))
            A = csr_matrix(arr)
            col = A.getcol(0)
        elif test_case_id == 3:
            np.random.seed(80)
            arr = np.random.randint(4, size=(988, 988))
            A = csr_matrix(arr)
            col = A.getcol(0)
        elif test_case_id == 4:
            np.random.seed(100)
            arr = np.random.randint(4, size=(988, 988))
            A = csr_matrix(arr)
            col = A.getcol(0)
        return col

    def generate_ans(data):
        _a = data
        col = _a
        mean = col.mean()
        N = col.shape[0]
        sqr = col.copy()  # take a copy of the col
        sqr.data **= 2  # square the data, i.e. just the non-zero data
        standard_deviation = np.sqrt(sqr.sum() / N - col.mean() ** 2)
        return [mean, standard_deviation]

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert np.allclose(result, ans)
    return 1

exec_context = r"""
import numpy as np
from scipy.sparse import csr_matrix
col = test_input
[insert]
result = [mean, standard_deviation]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(4):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)

def test_string(solution: str):
    tokens = []
    for token in tokenize.tokenize(io.BytesIO(solution.encode("utf-8")).readline):
        tokens.append(token.string)
    assert (
        "toarray" not in tokens
        and "array" not in tokens
        and "todense" not in tokens
        and "A" not in tokens
    )


Prompt:
Problem:
I have a sparse 988x1 vector (stored in col, a column in a csr_matrix) created through scipy.sparse. Is there a way to gets its max and min value without having to convert the sparse matrix to a dense one?
numpy.max seems to only work for dense vectors.

A:
<code>
import numpy as np
from scipy.sparse import csr_matrix

np.random.seed(10)
arr = np.random.randint(4,size=(988,988))
sA = csr_matrix(arr)
col = sA.getcol(0)
</code>
Max, Min = ... # put solution in these variables
BEGIN SOLUTION
<code>

import numpy as np
import copy
import tokenize, io
from scipy.sparse import csr_matrix

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            np.random.seed(10)
            arr = np.random.randint(4, size=(988, 988))
            A = csr_matrix(arr)
            col = A.getcol(0)
        elif test_case_id == 2:
            np.random.seed(42)
            arr = np.random.randint(4, size=(988, 988))
            A = csr_matrix(arr)
            col = A.getcol(0)
        elif test_case_id == 3:
            np.random.seed(80)
            arr = np.random.randint(4, size=(988, 988))
            A = csr_matrix(arr)
            col = A.getcol(0)
        elif test_case_id == 4:
            np.random.seed(100)
            arr = np.random.randint(4, size=(988, 988))
            A = csr_matrix(arr)
            col = A.getcol(0)
        return col

    def generate_ans(data):
        _a = data
        col = _a
        Max, Min = col.max(), col.min()
        return [Max, Min]

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert np.allclose(result, ans)
    return 1

exec_context = r"""
import numpy as np
from scipy.sparse import csr_matrix
col = test_input
[insert]
result = [Max, Min]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(4):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)

def test_string(solution: str):
    tokens = []
    for token in tokenize.tokenize(io.BytesIO(solution.encode("utf-8")).readline):
        tokens.append(token.string)
    assert (
        "toarray" not in tokens
        and "array" not in tokens
        and "todense" not in tokens
        and "A" not in tokens
    )


Prompt:
Problem:
I have a sparse 988x1 vector (stored in col, a column in a csr_matrix) created through scipy.sparse. Is there a way to gets its median and mode value without having to convert the sparse matrix to a dense one?
numpy.median seems to only work for dense vectors.

A:
<code>
import numpy as np
from scipy.sparse import csr_matrix

np.random.seed(10)
arr = np.random.randint(4,size=(988,988))
sA = csr_matrix(arr)
col = sA.getcol(0)
</code>
Median, Mode = ... # put solution in these variables
BEGIN SOLUTION
<code>

n = col.shape[0]
val = col.data
for i in range(n-len(val)):
    val = np.append(val,0)
Median, Mode = np.median(val), np.argmax(np.bincount(val))

import numpy as np
import copy
import tokenize, io
from scipy.sparse import csr_matrix

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            np.random.seed(10)
            arr = np.random.randint(4, size=(988, 988))
            A = csr_matrix(arr)
            col = A.getcol(0)
        elif test_case_id == 2:
            np.random.seed(42)
            arr = np.random.randint(4, size=(988, 988))
            A = csr_matrix(arr)
            col = A.getcol(0)
        elif test_case_id == 3:
            np.random.seed(80)
            arr = np.random.randint(4, size=(988, 988))
            A = csr_matrix(arr)
            col = A.getcol(0)
        elif test_case_id == 4:
            np.random.seed(100)
            arr = np.random.randint(4, size=(988, 988))
            A = csr_matrix(arr)
            col = A.getcol(0)
        return col

    def generate_ans(data):
        _a = data
        col = _a
        n = col.shape[0]
        val = col.data
        for i in range(n - len(val)):
            val = np.append(val, 0)
        Median, Mode = np.median(val), np.argmax(np.bincount(val))
        return [Median, Mode]

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert np.allclose(result, ans)
    return 1

exec_context = r"""
import numpy as np
from scipy.sparse import csr_matrix
col = test_input
[insert]
result = [Median, Mode]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(4):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)

def test_string(solution: str):
    tokens = []
    for token in tokenize.tokenize(io.BytesIO(solution.encode("utf-8")).readline):
        tokens.append(token.string)
    assert (
        "toarray" not in tokens
        and "array" not in tokens
        and "todense" not in tokens
        and "A" not in tokens
    )


Prompt:
Problem:
I'd like to achieve a fourier series development for a x-y-dataset using numpy and scipy.
At first I want to fit my data with the first 8 cosines and plot additionally only the first harmonic. So I wrote the following two function defintions:
# fourier series defintions
tau = 0.045
def fourier8(x, a1, a2, a3, a4, a5, a6, a7, a8):
    return a1 * np.cos(1 * np.pi / tau * x) + \
           a2 * np.cos(2 * np.pi / tau * x) + \
           a3 * np.cos(3 * np.pi / tau * x) + \
           a4 * np.cos(4 * np.pi / tau * x) + \
           a5 * np.cos(5 * np.pi / tau * x) + \
           a6 * np.cos(6 * np.pi / tau * x) + \
           a7 * np.cos(7 * np.pi / tau * x) + \
           a8 * np.cos(8 * np.pi / tau * x)
def fourier1(x, a1):
    return a1 * np.cos(1 * np.pi / tau * x)
Then I use them to fit my data:
# import and filename
filename = 'data.txt'
import numpy as np
from scipy.optimize import curve_fit
z, Ua = np.loadtxt(filename,delimiter=',', unpack=True)
tau = 0.045
popt, pcov = curve_fit(fourier8, z, Ua)
which works as desired
But know I got stuck making it generic for arbitary orders of harmonics, e.g. I want to fit my data with the first fifteen harmonics.
How could I achieve that without defining fourier1, fourier2, fourier3 ... , fourier15?
By the way, initial guess of a1,a2,… should be set to default value.

A:
<code>
from scipy.optimize import curve_fit
import numpy as np
s = '''1.000000000000000021e-03,2.794682735905079767e+02
4.000000000000000083e-03,2.757183469104809888e+02
1.400000000000000029e-02,2.791403179603880176e+02
2.099999999999999784e-02,1.781413355804160119e+02
3.300000000000000155e-02,-2.798375517344049968e+02
4.199999999999999567e-02,-2.770513900380149721e+02
5.100000000000000366e-02,-2.713769422793179729e+02
6.900000000000000577e-02,1.280740698304900036e+02
7.799999999999999989e-02,2.800801708984579932e+02
8.999999999999999667e-02,2.790400329037249776e+02'''.replace('\n', ';')
arr = np.matrix(s)
z = np.array(arr[:, 0]).squeeze()
Ua = np.array(arr[:, 1]).squeeze()
tau = 0.045
degree = 15
</code>
popt, pcov = ... # put solution in these variables
BEGIN SOLUTION
<code>

def fourier(x, *a):
    ret = a[0] * np.cos(np.pi / tau * x)
    for deg in range(1, len(a)):
        ret += a[deg] * np.cos((deg+1) * np.pi / tau * x)
    return ret

popt, pcov = curve_fit(fourier, z, Ua, [1.0] * degree)


Prompt:
Problem:
I have a raster with a set of unique ID patches/regions which I've converted into a two-dimensional Python numpy array. I would like to calculate pairwise Euclidean distances between all regions to obtain the minimum distance separating the nearest edges of each raster patch. As the array was originally a raster, a solution needs to account for diagonal distances across cells (I can always convert any distances measured in cells back to metres by multiplying by the raster resolution).
I've experimented with the cdist function from scipy.spatial.distance as suggested in this answer to a related question, but so far I've been unable to solve my problem using the available documentation. As an end result I would ideally have a N*N array in the form of "from ID, to ID, distance", including distances between all possible combinations of regions.
Here's a sample dataset resembling my input data:
import numpy as np
import matplotlib.pyplot as plt
# Sample study area array
example_array = np.array([[0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0],
                          [0, 0, 2, 0, 2, 2, 0, 6, 0, 3, 3, 3],
                          [0, 0, 0, 0, 2, 2, 0, 0, 0, 3, 3, 3],
                          [0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0],
                          [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3],
                          [1, 1, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3],
                          [1, 1, 1, 0, 0, 0, 3, 3, 3, 0, 0, 3],
                          [1, 1, 1, 0, 0, 0, 3, 3, 3, 0, 0, 0],
                          [1, 1, 1, 0, 0, 0, 3, 3, 3, 0, 0, 0],
                          [1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                          [1, 0, 1, 0, 0, 0, 0, 5, 5, 0, 0, 0],
                          [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4]])
# Plot array
plt.imshow(example_array, cmap="spectral", interpolation='nearest')
A:
<code>
import numpy as np
import scipy.spatial.distance
example_array = np.array([[0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0],
                          [0, 0, 2, 0, 2, 2, 0, 6, 0, 3, 3, 3],
                          [0, 0, 0, 0, 2, 2, 0, 0, 0, 3, 3, 3],
                          [0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0],
                          [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3],
                          [1, 1, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3],
                          [1, 1, 1, 0, 0, 0, 3, 3, 3, 0, 0, 3],
                          [1, 1, 1, 0, 0, 0, 3, 3, 3, 0, 0, 0],
                          [1, 1, 1, 0, 0, 0, 3, 3, 3, 0, 0, 0],
                          [1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                          [1, 0, 1, 0, 0, 0, 0, 5, 5, 0, 0, 0],
                          [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4]])
</code>
result = ... # put solution in this variable
BEGIN SOLUTION
<code>

import itertools
n = example_array.max()+1
indexes = []
for k in range(1, n):
    tmp = np.nonzero(example_array == k)
    tmp = np.asarray(tmp).T
    indexes.append(tmp)
result = np.zeros((n-1, n-1))
for i, j in itertools.combinations(range(n-1), 2):
    d2 = scipy.spatial.distance.cdist(indexes[i], indexes[j], metric='sqeuclidean')
    result[i, j] = result[j, i] = d2.min()**0.5

import numpy as np
import itertools
import copy
import scipy.spatial.distance

def generate_test_case(test_case_id):
    def define_test_input(test_case_id):
        if test_case_id == 1:
            example_array = np.array(
                [
                    [0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0],
                    [0, 0, 2, 0, 2, 2, 0, 6, 0, 3, 3, 3],
                    [0, 0, 0, 0, 2, 2, 0, 0, 0, 3, 3, 3],
                    [0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0],
                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3],
                    [1, 1, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3],
                    [1, 1, 1, 0, 0, 0, 3, 3, 3, 0, 0, 3],
                    [1, 1, 1, 0, 0, 0, 3, 3, 3, 0, 0, 0],
                    [1, 1, 1, 0, 0, 0, 3, 3, 3, 0, 0, 0],
                    [1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    [1, 0, 1, 0, 0, 0, 0, 5, 5, 0, 0, 0],
                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4],
                ]
            )
        return example_array

    def generate_ans(data):
        _a = data
        example_array = _a
        n = example_array.max() + 1
        indexes = []
        for k in range(1, n):
            tmp = np.nonzero(example_array == k)
            tmp = np.asarray(tmp).T
            indexes.append(tmp)
        result = np.zeros((n - 1, n - 1))
        for i, j in itertools.combinations(range(n - 1), 2):
            d2 = scipy.spatial.distance.cdist(
                indexes[i], indexes[j], metric="sqeuclidean"
            )
            result[i, j] = result[j, i] = d2.min() ** 0.5
        return result

    test_input = define_test_input(test_case_id)
    expected_result = generate_ans(copy.deepcopy(test_input))
    return test_input, expected_result

def exec_test(result, ans):
    assert np.allclose(result, ans)
    return 1

exec_context = r"""
import numpy as np
import scipy.spatial.distance
example_array = test_input
[insert]
"""

def test_execution(solution: str):
    code = exec_context.replace("[insert]", solution)
    for i in range(1):
        test_input, expected_result = generate_test_case(i + 1)
        test_env = {"test_input": test_input}
        exec(code, test_env)
        assert exec_test(test_env["result"], expected_result)