许多 numpy 函数提供了使用轴 = 参数在特定轴上操作的选项。我的问题是
import pandas as pd
import numpy as np
def nanmoving_mean(data,window,axis=0):
kw = {'center':True,'window':window,'min_periods':1}
if len(data.shape)==1:
return pd.Series(data).rolling(**kw).mean().as_matrix()
elif len(data.shape)>=2:
tmp = np.swapaxes(data,0,axis)
tmpshp = tmp.shape
tmp = np.reshape( tmp, (tmpshp[0],-1), order='C' )
tmp = pd.DataFrame(tmp).rolling(**kw).mean().as_matrix()
tmp = np.reshape( tmp, tmpshp, order='C' )
return np.swapaxes(tmp,0,axis)
else:
print('Invalid dimension!')
return None
data = np.random.randint(10,size=(2,3,6))
print(data)
nanmoving_mean(data,window=3,axis=2)
最佳答案
我们可以有一个使用 2D
convolution 的方法.
基本步骤是:
NaNs
与 0s
因为我们需要对输入数据进行加窗求和。 Scipy's convolve2d
获取窗口求和对于数据值NaNs
的面具.我们将使用边界元素作为零。 NaNs
的窗口计数从窗口大小获取负责求和的有效元素的计数。 intervaled-summations
也可以通过 Scipy's
1D uniform-filter
获得那相对来说效率更高。另一个好处是我们可以指定要执行这些求和/平均的轴。2D convolution
和 1D uniform filter
,下面列出的方法很少。from scipy.signal import convolve2d as conv2
from scipy.ndimage.filters import uniform_filter1d as uniff
def nanmoving_mean_numpy(data, W): # data: input array, W: Window size
N = data.shape[-1]
hW = (W-1)//2
nan_mask = np.isnan(data)
data1 = np.where(nan_mask,0,data)
value_sums = conv2(data1.reshape(-1,N),np.ones((1,W)),'same', boundary='fill')
nan_sums = conv2(nan_mask.reshape(-1,N),np.ones((1,W)),'same', boundary='fill')
value_sums.shape = data.shape
nan_sums.shape = data.shape
b_sizes = hW+1+np.arange(hW) # Boundary sizes
count = np.hstack(( b_sizes , W*np.ones(N-2*hW), b_sizes[::-1] ))
return value_sums/(count - nan_sums)
def nanmoving_mean_numpy_v2(data, W): # data: input array, W: Window size
N = data.shape[-1]
hW = (W-1)//2
nan_mask = np.isnan(data)
data1 = np.where(nan_mask,0,data)
value_sums = uniff(data1,size=W, axis=-1, mode='constant')*W
nan_sums = conv2(nan_mask.reshape(-1,N),np.ones((1,W)),'same', boundary='fill')
nan_sums.shape = data.shape
b_sizes = hW+1+np.arange(hW) # Boundary sizes
count = np.hstack(( b_sizes , W*np.ones(N-2*hW,dtype=int), b_sizes[::-1] ))
out = value_sums/(count - nan_sums)
out = np.where(np.isclose( count, nan_sums), np.nan, out)
return out
def nanmoving_mean_numpy_v3(data, W): # data: input array, W: Window size
N = data.shape[-1]
hW = (W-1)//2
nan_mask = np.isnan(data)
data1 = np.where(nan_mask,0,data)
nan_avgs = uniff(nan_mask.astype(float),size=W, axis=-1, mode='constant')
b_sizes = hW+1+np.arange(hW) # Boundary sizes
count = np.hstack(( b_sizes , W*np.ones(N-2*hW), b_sizes[::-1] ))
scale = ((count/float(W)) - nan_avgs)
out = uniff(data1,size=W, axis=-1, mode='constant')/scale
out = np.where(np.isclose( scale, 0), np.nan, out)
return out
In [807]: # Create random input array and insert NaNs
...: data = np.random.randint(10,size=(20,30,60)).astype(float)
...:
...: # Add 10% NaNs across the data randomly
...: idx = np.random.choice(data.size,size=int(data.size*0.1),replace=0)
...: data.ravel()[idx] = np.nan
...:
...: W = 5 # Window size
...:
In [808]: %timeit nanmoving_mean(data,window=W,axis=2)
...: %timeit nanmoving_mean_numpy(data, W)
...: %timeit nanmoving_mean_numpy_v2(data, W)
...: %timeit nanmoving_mean_numpy_v3(data, W)
...:
10 loops, best of 3: 22.3 ms per loop
100 loops, best of 3: 3.31 ms per loop
100 loops, best of 3: 2.99 ms per loop
1000 loops, best of 3: 1.76 ms per loop
In [811]: # Create random input array and insert NaNs
...: data = np.random.randint(10,size=(120,130,160)).astype(float)
...:
...: # Add 10% NaNs across the data randomly
...: idx = np.random.choice(data.size,size=int(data.size*0.1),replace=0)
...: data.ravel()[idx] = np.nan
...:
In [812]: %timeit nanmoving_mean(data,window=W,axis=2)
...: %timeit nanmoving_mean_numpy(data, W)
...: %timeit nanmoving_mean_numpy_v2(data, W)
...: %timeit nanmoving_mean_numpy_v3(data, W)
...:
1 loops, best of 3: 796 ms per loop
1 loops, best of 3: 486 ms per loop
1 loops, best of 3: 275 ms per loop
10 loops, best of 3: 161 ms per loop
https://stackoverflow.com/questions/41311268/
相关文章:
.net - Android Xamarin - 带有 ObservableCollection 的
apache-spark - Spark 溢出独立于分配的执行程序内存
android - 安装失败 INSTALL_PARSE_FAILED_MANIFEST_MALFO
angularjs - 在 Controller 函数之外的脚本中读取 AngularJS 表达式
elixir - 编译 riak_core 项目时,Mix 在 deps.compile 上挂起
r - 如何在 R 中获取面板数据固定效应回归的 corr(u_i, Xb)
angularjs - 在带有过滤的ng-repeat中反向计数