Substantial optimisation

Ludicrously large change for one single commit. Moved univariate fitters to univariate folder. Substantial change to mle fitting logic with large clean up of code and removal of older functions. Changed some moments to use scipy stats moments. Moved GumbelLEV into tests and highest level import. Implemented use of expm1 and log1p where it was logical to do so.

Author: derrynknife

pyproject.toml

@@ -5,6 +5,7 @@ line-length = 79
 # So isort is compatible with black
 [tool.isort]
 profile = "black"
+skip = ["__init__.py"]
 
 # mypy
 [tool.mypy]

surpyval/__init__.py

@@ -3,14 +3,14 @@
 from autograd import numpy as np
 
 from surpyval.distribution import Distribution
-from surpyval.nonparametric import (
+from surpyval.univariate.nonparametric import (
     FlemingHarrington,
     KaplanMeier,
     NelsonAalen,
     NonParametric,
     Turnbull,
 )
-from surpyval.parametric import (
+from surpyval.univariate.parametric import (
     Bernoulli,
     Beta,
     CustomDistribution,
@@ -22,6 +22,7 @@
     Gamma,
     Gauss,
     Gumbel,
+    GumbelLEV,
     InstantlyOccurs,
     Logistic,
     LogLogistic,
@@ -34,13 +35,6 @@
     Uniform,
     Weibull,
 )
-from surpyval.regression import (
-    CoxPH,
-    ExponentialPH,
-    RandomSurvivalForest,
-    SurvivalTree,
-    WeibullPH,
-)
 from surpyval.utils import (
     fs_to_xcn,
     fs_to_xrd,
@@ -58,6 +52,14 @@
 
 from .fit_best import fit_best
 
+from surpyval.regression import (  # isort: skip
+    CoxPH,
+    ExponentialPH,
+    RandomSurvivalForest,
+    SurvivalTree,
+    WeibullPH,
+)
+
 NUM = np.float64
 TINIEST = np.finfo(np.float64).tiny
 EPS = np.sqrt(np.finfo(NUM).eps)

surpyval/fit_best.py

@@ -3,7 +3,7 @@
 
 import numpy as np
 
-from surpyval.parametric import (
+from surpyval.univariate.parametric import (
     Beta,
     Exponential,
     ExpoWeibull,

surpyval/parametric/fitters/__init__.py

@@ -4,10 +4,16 @@
 import autograd.numpy as np
 
 import surpyval as surv
-from surpyval.parametric.parametric_fitter import ParametricFitter
 from surpyval.regression.lifemodels.lifemodel import LifeModel
+from surpyval.univariate.parametric import (
+    Gumbel,
+    Logistic,
+    LogNormal,
+    Normal,
+    Weibull,
+)
+from surpyval.univariate.parametric.parametric_fitter import ParametricFitter
 
-from ..parametric import Gumbel, Logistic, LogNormal, Normal, Weibull
 from .accelerated_failure_time import AcceleratedFailureTimeFitter
 from .cox_ph import CoxPH
 from .forest.forest import RandomSurvivalForest

surpyval/regression/__init__.py

@@ -2,8 +2,8 @@
 from scipy.optimize import minimize
 
 import surpyval
+from surpyval.univariate.parametric.fitters import bounds_convert
 
-from ..parametric.fitters import bounds_convert, fix_idx_and_function
 from .regression import Regression
 
 
@@ -154,9 +154,8 @@ def fit(self, Z, x, c=None, n=None, t=None, init=[], fixed={}):
             **{k: v + len(self.param_map) for k, v in phi_param_map.items()},
         }
 
-        transform, inv_trans, funcs, inv_f = bounds_convert(x, bounds)
-        const, fixed_idx, not_fixed = fix_idx_and_function(
-            fixed, param_map, funcs
+        transform, inv_trans, const, fixed_idx, not_fixed = bounds_convert(
+            x, bounds, fixed, param_map
         )
 
         init = transform(init)[not_fixed]

surpyval/regression/accelerated_failure_time.py

@@ -17,12 +17,13 @@
 from scipy.optimize import root
 from scipy.stats import norm
 
-from ..nonparametric import (
+from surpyval.univariate.nonparametric import (
     FlemingHarrington,
     KaplanMeier,
     NelsonAalen,
     Turnbull,
 )
+
 from ..utils import validate_coxph, validate_coxph_df_inputs
 from .proportional_hazards import ProportionalHazardsModel
 from .regression import Regression

surpyval/regression/cox_ph.py

@@ -6,8 +6,8 @@
 from numpy.typing import ArrayLike, NDArray
 
 from surpyval import Exponential, NelsonAalen, Weibull
-from surpyval.parametric import NeverOccurs
 from surpyval.regression.forest.log_rank_split import log_rank_split
+from surpyval.univariate.parametric import NeverOccurs
 from surpyval.utils.surpyval_data import SurpyvalData
 
 

surpyval/regression/forest/node.py

@@ -5,8 +5,8 @@
 from scipy.optimize import minimize
 
 import surpyval
+from surpyval.univariate.parametric.fitters import bounds_convert
 
-from ..parametric.fitters import bounds_convert, fix_idx_and_function
 from .regression import Regression
 
 
@@ -288,9 +288,8 @@ def fit(
         }
         self.param_map = param_map
 
-        transform, inv_trans, funcs, inv_f = bounds_convert(x, bounds)
-        const, fixed_idx, not_fixed = fix_idx_and_function(
-            fixed, param_map, funcs
+        transform, inv_trans, const, fixed_idx, not_fixed = bounds_convert(
+            x, bounds, fixed, param_map
         )
 
         init = transform(init)[not_fixed]

surpyval/regression/parameter_substitution.py

@@ -4,8 +4,8 @@
 from scipy.optimize import minimize
 
 import surpyval
+from surpyval.univariate.parametric.fitters import bounds_convert
 
-from ..parametric.fitters import bounds_convert, fix_idx_and_function
 from ..utils import _get_idx
 from .regression import Regression
 
@@ -233,9 +233,8 @@ def fit(self, Z, x, c=None, n=None, t=None, init=[], fixed={}):
         model.phi_param_map = self.phi_param_map
         param_map = {**self.param_map, **phi_param_map}
 
-        transform, inv_trans, funcs, inv_f = bounds_convert(x, bounds)
-        const, fixed_idx, not_fixed = fix_idx_and_function(
-            fixed, param_map, funcs
+        transform, inv_trans, const, fixed_idx, not_fixed = bounds_convert(
+            x, bounds, fixed, param_map
         )
 
         init = transform(init)[not_fixed]

surpyval/regression/proportional_hazards.py

@@ -0,0 +1,66 @@
+import numpy as np
+
+from surpyval import (
+    Beta,
+    Gamma,
+    Gumbel,
+    GumbelLEV,
+    Logistic,
+    LogLogistic,
+    LogNormal,
+    Normal,
+    Weibull,
+)
+
+
+def test_zi():
+    for dist in [Beta, Weibull, LogNormal, LogLogistic, Gamma]:
+        for zeros in [1, 10, 100, 1000, 10000]:
+            x = dist.random(100, 10, 2)
+            x = np.concatenate((x, np.zeros(zeros)))
+            model = dist.fit(x, zi=True)
+            assert model.res.success
+
+
+def test_lfp():
+    for dist in [
+        Beta,
+        Gamma,
+        Gumbel,
+        GumbelLEV,
+        Logistic,
+        LogLogistic,
+        LogNormal,
+        Normal,
+        Weibull,
+    ]:
+        for censored in [1, 10, 100, 1000, 10000]:
+            x = dist.random(100, 10, 2)
+            c = np.concatenate((np.zeros_like(x), np.ones(censored)))
+            x = np.concatenate((x, x.max() * np.ones(censored)))
+            model = dist.fit(x, c=c, lfp=True)
+            assert model.res.success
+
+
+def test_lfp_zi():
+    for dist in [Gamma, Weibull, LogNormal, LogLogistic]:
+        for zi_lfp_values in [1, 10, 100]:
+            for num_samples in [100, 1000, 10000]:
+                x = dist.random(num_samples, 10, 2)
+                c = np.concatenate(
+                    (
+                        np.zeros_like(x),
+                        np.zeros(zi_lfp_values),
+                        np.ones(zi_lfp_values),
+                    )
+                )
+                x = np.concatenate(
+                    (
+                        x,
+                        np.zeros(zi_lfp_values),
+                        x.max() * np.ones(zi_lfp_values) + 1,
+                    )
+                )
+                model = dist.fit(x, c=c, zi=True, lfp=True)
+                if not model.res.success:
+                    raise ValueError(model, model.res)

surpyval/tests/test_lfp_zi.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from surpyval import nonparametric as nonp
+from surpyval.univariate.nonparametric import rank_adjust
 from surpyval.utils import xcnt_handler
 
 
@@ -26,7 +26,7 @@ def filliben(x, c, n, t):
     c = c[idx2]
     N = len(x)
 
-    ranks = nonp.rank_adjust(x, c)
+    ranks = rank_adjust(x, c)
     d = 1 - c
     r = np.linspace(N, 1, num=N)
 

surpyval/univariate/__init__.py

@@ -1,6 +1,8 @@
 import numpy as np
 
-from surpyval.nonparametric.nonparametric_fitter import NonParametricFitter
+from surpyval.univariate.nonparametric.nonparametric_fitter import (
+    NonParametricFitter,
+)
 
 
 def fh_h(r_i, d_i):

surpyval/nonparametric/__init__.py surpyval/univariate/nonparametric/__init__.py

@@ -1,6 +1,8 @@
 import numpy as np
 
-from surpyval.nonparametric.nonparametric_fitter import NonParametricFitter
+from surpyval.univariate.nonparametric.nonparametric_fitter import (
+    NonParametricFitter,
+)
 
 
 def kaplan_meier(r, d):

surpyval/nonparametric/filliben.py surpyval/univariate/nonparametric/filliben.py

@@ -1,6 +1,8 @@
 import numpy as np
 
-from surpyval.nonparametric.nonparametric_fitter import NonParametricFitter
+from surpyval.univariate.nonparametric.nonparametric_fitter import (
+    NonParametricFitter,
+)
 
 
 def nelson_aalen(r, d):