Loading commonml/stats/__init__.py 0 → 100644 +5 −0 Original line number Diff line number Diff line from .agg import ( rolling_mean, mean_std ) from . import timeseries No newline at end of file commonml/stats/agg.py 0 → 100644 +54 −0 Original line number Diff line number Diff line """ Aggregation functions over numpy arrays. """ from typing import Tuple import numpy as np def rolling_mean(arr: np.ndarray, window: int=None) -> np.ndarray: """ Calculate rolling mean of a sequence. Resulting array is of length len(arr) - window + 1. Parameters ---------- arr : np.ndarray Array to take average of. window : int, optional Size of averaging window, by default square root of array length Returns ------- np.ndarray An array of means """ arr = np.asarray(arr) window = int(np.sqrt(len(arr))) if window is None else window return np.convolve(arr, np.ones(window), 'valid') / window def mean_std(arr: np.ndarray, axis: int=0) -> Tuple[np.ndarray, np.ndarray]: """ Calculate means and standard deviations of arrays, ignoring NaNs. Parameters ---------- arr : np.ndarray Array to calculate mean and standard deviation for. axis : int, optional Axis along which to reduce, by default 0 (i.e. mean of each element across rows of a 2D array for example) Returns ------- Tuple[np.ndarray, np.ndarray] Two arrays of same shape as `arr` for mean and std """ arr = np.asarray(arr) means = np.nanmean(arr, axis=axis) stds = np.nanstd(arr, axis=axis) return means, stds commonml/stats/timeseries.py 0 → 100644 +46 −0 Original line number Diff line number Diff line """ Timeseries-specific processing. """ from typing import List import numpy as np import pandas as pd from sklearn.preprocessing import MinMaxScaler from tslearn.metrics import dtw from tslearn.utils import to_time_series def similarity_matrix(timeseries: List[pd.DataFrame], columns: List[str]=None, beta: float=1) -> np.ndarray: """ Calculate a similarity measure (not metric) between time series. The series do not have be of the same length. Used Dynamic Time Warping. Parameters ---------- timeseries : List[pd.DataFrame] List of data frames, where each dataframe is a multivariate time series. columns : List[str], optional Columns to use to measure similarity, by default all columns are used beta : float, optional The factor in the exponential when normalizing the measures, by default 1 Returns ------- np.ndarray A square matrix of dimensions `len(timeseries), len(timeseries)` """ columns = timeseries[0].columns if columns is None else columns scaler = MinMaxScaler(feature_range=(-1, 1)) scaler.fit(pd.concat(timeseries, axis=0)[columns]) dissimilarities = np.zeros((len(timeseries), len(timeseries))) + np.inf # initially all tickers are dissimilar for i, arr1 in enumerate(timeseries): for j, arr2 in enumerate(timeseries): dissimilarities[i, j] = dtw( to_time_series(scaler.transform(arr1[columns])), to_time_series(scaler.transform(arr2[columns]))) return np.exp(-beta * dissimilarities / dissimilarities.std()) Loading
commonml/stats/__init__.py 0 → 100644 +5 −0 Original line number Diff line number Diff line from .agg import ( rolling_mean, mean_std ) from . import timeseries No newline at end of file
commonml/stats/agg.py 0 → 100644 +54 −0 Original line number Diff line number Diff line """ Aggregation functions over numpy arrays. """ from typing import Tuple import numpy as np def rolling_mean(arr: np.ndarray, window: int=None) -> np.ndarray: """ Calculate rolling mean of a sequence. Resulting array is of length len(arr) - window + 1. Parameters ---------- arr : np.ndarray Array to take average of. window : int, optional Size of averaging window, by default square root of array length Returns ------- np.ndarray An array of means """ arr = np.asarray(arr) window = int(np.sqrt(len(arr))) if window is None else window return np.convolve(arr, np.ones(window), 'valid') / window def mean_std(arr: np.ndarray, axis: int=0) -> Tuple[np.ndarray, np.ndarray]: """ Calculate means and standard deviations of arrays, ignoring NaNs. Parameters ---------- arr : np.ndarray Array to calculate mean and standard deviation for. axis : int, optional Axis along which to reduce, by default 0 (i.e. mean of each element across rows of a 2D array for example) Returns ------- Tuple[np.ndarray, np.ndarray] Two arrays of same shape as `arr` for mean and std """ arr = np.asarray(arr) means = np.nanmean(arr, axis=axis) stds = np.nanstd(arr, axis=axis) return means, stds
commonml/stats/timeseries.py 0 → 100644 +46 −0 Original line number Diff line number Diff line """ Timeseries-specific processing. """ from typing import List import numpy as np import pandas as pd from sklearn.preprocessing import MinMaxScaler from tslearn.metrics import dtw from tslearn.utils import to_time_series def similarity_matrix(timeseries: List[pd.DataFrame], columns: List[str]=None, beta: float=1) -> np.ndarray: """ Calculate a similarity measure (not metric) between time series. The series do not have be of the same length. Used Dynamic Time Warping. Parameters ---------- timeseries : List[pd.DataFrame] List of data frames, where each dataframe is a multivariate time series. columns : List[str], optional Columns to use to measure similarity, by default all columns are used beta : float, optional The factor in the exponential when normalizing the measures, by default 1 Returns ------- np.ndarray A square matrix of dimensions `len(timeseries), len(timeseries)` """ columns = timeseries[0].columns if columns is None else columns scaler = MinMaxScaler(feature_range=(-1, 1)) scaler.fit(pd.concat(timeseries, axis=0)[columns]) dissimilarities = np.zeros((len(timeseries), len(timeseries))) + np.inf # initially all tickers are dissimilar for i, arr1 in enumerate(timeseries): for j, arr2 in enumerate(timeseries): dissimilarities[i, j] = dtw( to_time_series(scaler.transform(arr1[columns])), to_time_series(scaler.transform(arr2[columns]))) return np.exp(-beta * dissimilarities / dissimilarities.std())
