diff --git a/src/nns/arma.py b/src/nns/arma.py index 3a1c7ef4..d08d53e5 100644 --- a/src/nns/arma.py +++ b/src/nns/arma.py @@ -248,6 +248,7 @@ def nns_arma_optim( weight_score = math.inf if objective_l == "min" else -math.inf if _improves_or_ties_bias(weight_score, nns_score, objective_l): nns_weights = equal_weights + nns_score = weight_score predicted = equal_predicted errors = predicted - actual bias = _finite_gravity(errors) @@ -280,12 +281,14 @@ def nns_arma_optim( ) shrink_score = float(objective_fn(shrink_predicted, actual)) nns_shrink = _improves(shrink_score, nns_score, objective_l) + best_score = shrink_score if nns_shrink else nns_score if nns_shrink: final_predicted = shrink_predicted - regressed_values = _smooth_regressed_variable(values) + # Smooth the training window only, so the validation score never sees the validation data. + regressed_training = _smooth_regressed_variable(values[:train_n]) smooth_predicted = _arma_optim_prediction( - regressed_values, + regressed_training, train_n, h_eval, nns_periods, @@ -295,14 +298,13 @@ def nns_arma_optim( shrink=True, ) smooth_score = float(objective_fn(smooth_predicted, actual)) - nns_regress = _improves(smooth_score, nns_score, objective_l) + nns_regress = _improves(smooth_score, best_score, objective_l) if nns_regress: final_predicted = smooth_predicted del final_predicted # Apply the nns.regress decision: it was tested on the smoothed # series with shrink=True, so the returned forecast uses the same inputs. - final_values = regressed_values if nns_regress else original_values if nns_regress: nns_shrink = True @@ -314,6 +316,7 @@ def nns_arma_optim( if h_oos is None: result_h = h_eval + final_values = regressed_training if nns_regress else original_values model_results = _arma_optim_prediction( final_values, train_n, @@ -326,6 +329,9 @@ def nns_arma_optim( ) else: result_h = h_oos + final_values = ( + _smooth_regressed_variable(original_values) if nns_regress else original_values + ) model_results = _arma_optim_prediction( final_values, None, @@ -413,6 +419,12 @@ def _finish( 'Nothing dynamic about static seasonal factors! Please set "dynamic = FALSE" ' 'or "seasonal.factor = FALSE"' ) + if weights is not None and not isinstance(weights, str) and dynamic: + raise ValueError( + 'Numeric "weights" cannot be used with "dynamic = TRUE", which re-selects the ' + 'seasonal lags at every step. Please set "weights" to NULL or "equal", or ' + '"dynamic = FALSE"' + ) method_l = method.lower() if method_l not in {"lin", "nonlin", "both", "means"}: @@ -486,7 +498,7 @@ def _finish( lags, lag_weights = _resolve_lags_and_weights( current, seasonal_factor=seasonal_factor, - weights=None, + weights=weights, best_periods=best_periods, modulo=modulo, mod_only=mod_only, diff --git a/tests/_r_cache/NNS.ARMA.optim.custom.json b/tests/_r_cache/NNS.ARMA.optim.custom.json index 62c6eb78..bd19992b 100644 --- a/tests/_r_cache/NNS.ARMA.optim.custom.json +++ b/tests/_r_cache/NNS.ARMA.optim.custom.json @@ -141,90 +141,187 @@ "weights": null }, "4b0479ad3acb12c284d99ef15964b3aeca7ac354b9d704e1d010be36e12c39b6": { - "bias.shift": 0.738730098896512, + "bias.shift": 0.0, "errors": [ - -0.870820659735998, - 4.42997139978953, - 3.4004934316573, - 1.20635395391275, - -5.89623696390112, - -3.15842333726781, - -3.56541406532329, - -2.05399141077677, - -1.06933766403026, - 0.936214975501464, - 3.77536506956469, - -0.0302256913052688, - -4.00218758221989, - -0.242021724864784, - -0.307813644922234, - 1.11822607535573, - 0.969095497077795, - -0.808604815455446, - -2.12394953274366, - -3.50806090269815, - -3.32857951814596, - -2.20315072827786, - 4.78810209080733, - 6.25073057598107, - 5.42399821168026, - 1.09975378940172, - -0.845915498490157, - -4.3950622963491, - -3.64730842446673, - -0.44858150106873, - -1.84783562366948, - -1.27161675871394 + -0.0972245584577962, + 5.20141772521018, + 2.97085946490023, + 1.21642185131353, + -6.01720558328814, + -2.90671974024676, + -3.30066362552529, + -1.92461671541887, + -1.27968755251956, + 0.742468513537979, + 3.69142212174162, + 0.210146045811769, + -2.92000010016601, + 0.657394221927852, + -0.224875200039506, + 1.4888113751915, + 1.10282688640545, + -0.565373396576319, + -2.2385070499262, + -2.90174033328285, + -3.23443849752157, + -2.41963496814687, + 4.31912511257162, + 5.98980926408563, + 5.80931020859563, + 1.78400543155194, + -0.328941893440131, + -3.56476326073753, + -2.57072479778962, + 0.352891501223993, + -1.73758716274106, + -0.87416334212498 ], "lower.pred.int": [ - 46.9193746824952, - 45.9217414462411, - 45.256195175462, - 43.2367804096736, - 41.0755792156105, - 40.0180654465558, - 41.3672980923222, - 44.4575750821314, - 46.5640655519224, - 47.5635866486534 + 46.6794651800623, + 45.6818319438082, + 45.0162856730291, + 42.9968709072407, + 40.8356697131776, + 39.7781559441229, + 41.1273885898893, + 44.2176655796984, + 46.3241560494894, + 47.3236771462205 ], "method": "lin", "nns.regress": 0.0, - "obj.fn": 1.05473085257718, + "obj.fn": 1.02118564406747, "periods": [ 12.0, 14.0 ], "results": [ - 53.2681177750397, - 52.2704845387856, - 51.6049382680065, - 49.5855235022181, - 47.424322308155, - 46.3668085391003, - 47.7160411848667, - 50.8063181746758, - 52.9128086444668, - 53.9123297411979 + 52.5293876761432, + 51.531754439889, + 50.86620816911, + 48.8467934033215, + 46.6855922092585, + 45.6280784402038, + 46.9773110859702, + 50.0675880757793, + 52.1740785455703, + 53.1735996423014 ], "shrink": 1.0, "upper.pred.int": [ - 59.6168608675841, - 58.61922763133, - 57.9536813605509, - 55.9342665947625, - 53.7730654006995, - 52.7155516316447, - 54.0647842774111, - 57.1550612672203, - 59.2615517370113, - 60.2610728337423 + 56.6459504594546, + 55.6483172232005, + 54.9827709524214, + 52.963356186633, + 50.8021549925699, + 49.7446412235152, + 51.0938738692816, + 54.1841508590907, + 56.2906413288817, + 57.2901624256128 ], "weights": [ 0.5, 0.5 ] }, + "620db4b9c7d7c77bcaeec563048cc55a1475062327576c563c8f4471b5d1ac49": { + "bias.shift": 0.565523334603564, + "errors": [ + -0.201881367286262, + 0.182803790203536, + 1.75047586910001, + 0.691271737636647, + -0.0320850925105276, + 0.156428362474244, + 1.15801064432018, + -0.0060097329962332, + -0.465347856008115, + -1.34713892429127, + -0.761067652318275, + -2.97561753181145, + -2.81138189986887, + -4.57750807674585, + -2.38434418815688, + -2.50483172185046, + -2.42682439755257, + 0.281586655641576, + 0.932413287910293, + 2.35267046222258 + ], + "lower.pred.int": [ + 38.0221406022244, + 38.1971897597142, + 38.3722248386106, + 38.5471987071473, + 38.7145028770001, + 38.8896673319849, + 38.9432326138308, + 39.1250802365144, + 39.3067341135025, + 39.4881420452194, + 39.6556253171924, + 39.8368734376992, + 39.8622590696418, + 40.0508878927648, + 40.2391377813538, + 40.4269512476602, + 40.5875005719581, + 40.7741966251522, + 40.7799102574209, + 40.9748024317332 + ], + "method": "nonlin", + "nns.regress": 1.0, + "obj.fn": 2.64171880518269, + "periods": 6.0, + "results": [ + 40.6542919673173, + 40.8293411248071, + 41.0043762037036, + 41.1793500722402, + 41.346654242093, + 41.5218186970778, + 41.5753839789237, + 41.7572316016073, + 41.9388854785954, + 42.1202934103123, + 42.2877766822853, + 42.4690248027921, + 42.4944104347347, + 42.6830392578577, + 42.8712891464467, + 43.0591026127531, + 43.219651937051, + 43.4063479902451, + 43.4120616225139, + 43.6069537968261 + ], + "shrink": 1.0, + "upper.pred.int": [ + 43.9053787006157, + 44.0804278581056, + 44.255462937002, + 44.4304368055387, + 44.5977409753915, + 44.7729054303763, + 44.8264707122222, + 45.0083183349058, + 45.1899722118939, + 45.3713801436108, + 45.5388634155837, + 45.7201115360906, + 45.7454971680331, + 45.9341259911562, + 46.1223758797451, + 46.3101893460515, + 46.4707386703494, + 46.6574347235436, + 46.6631483558123, + 46.8580405301246 + ], + "weights": null + }, "6a7f2183afc4e4098c97a5e6d2a661d9380807678813ae632c339bca5cac822c": { "bias.shift": 2.39977647950691, "errors": [ @@ -292,18 +389,115 @@ ], "shrink": 0.0, "upper.pred.int": [ - 487.738525907524, - 459.019317312136, - 483.619684188446, - 537.277441001863, - 549.290281672723, - 615.263814167482, - 712.949872867692, - 692.68624603331, - 585.204327794316, - 536.899034293268, - 457.336403266014, - 501.688342469368 + 506.652632986148, + 477.933424390761, + 502.533791267071, + 556.191548080488, + 568.204388751348, + 634.177921246107, + 731.863979946316, + 711.600353111935, + 604.118434872941, + 555.813141371893, + 476.250510344639, + 520.602449547993 + ], + "weights": null + }, + "af6f78073645b5eadb91a42b6891097c5924a1501bca2c15f4b5bb00e6571bf0": { + "bias.shift": -3.44356792523116, + "errors": [ + 2.87240370967742, + 3.40413939193549, + 3.29490157903226, + 1.51091361290322, + 3.17790714296407, + 3.28332717365269, + 3.3047801511976, + 4.36086634655689, + 5.72456372080838, + 6.33210890119761, + 3.5836547005988, + 3.4762363982036, + 6.3403435997682, + 4.82005077161966, + 2.12328258953061, + -0.209609463202632, + 4.15275729832894, + 3.8840364584165, + 2.86455854743489, + 3.61527949122886 + ], + "lower.pred.int": [ + 37.9677445917303, + 36.5881732739883, + 34.8687114610851, + 34.1692834949561, + 36.6792820250169, + 36.2780450557055, + 35.2792940332504, + 36.1498342286097, + 36.3198786028612, + 36.5873267832504, + 35.4899735826516, + 35.5146272802564, + 39.545022481821, + 37.8021096536725, + 35.4747564715834, + 34.5276954188502, + 37.5541241803818, + 36.9379253404693, + 35.7615494294877, + 36.7942513732817 + ], + "method": "nonlin", + "nns.regress": 0.0, + "obj.fn": 26.358743389983, + "periods": 12.0, + "results": [ + 40.8606087844463, + 39.4810374667043, + 37.7615756538011, + 37.0621476876721, + 39.5721462177329, + 39.1709092484215, + 38.1721582259664, + 39.0426984213257, + 39.2127427955772, + 39.4801909759664, + 38.3828377753676, + 38.4074914729724, + 42.437886674537, + 40.6949738463885, + 38.3676206642994, + 37.4205596115662, + 40.4469883730978, + 39.8307895331853, + 38.6544136222037, + 39.6871155659977 + ], + "shrink": 0.0, + "upper.pred.int": [ + 43.6965377117298, + 42.3169663939878, + 40.5975045810846, + 39.8980766149556, + 42.4080751450164, + 42.006838175705, + 41.00808715325, + 41.8786273486092, + 42.0486717228607, + 42.31611990325, + 41.2187667026511, + 41.2434204002559, + 45.2738156018206, + 43.530902773672, + 41.203549591583, + 40.2564885388497, + 43.2829173003813, + 42.6667184604689, + 41.4903425494872, + 42.5230444932812 ], "weights": null }, @@ -409,65 +603,65 @@ "ee57704e6be7852b86a9becdd1802960f083367665c2f6496e662e67048651ac": { "bias.shift": 0.0, "errors": [ - 79.9368421052632, - 87.6368421052632, - 4.56315789473683, - -18.2947368421053, - -62.7578947368421, - -62.0157894736842, - -90.7834586466165, - -112.897744360902, - -37.5022556390977, - -15.6037593984963, - 8.25563909774428, - 20.0398496240602 + -45.3908786460726, + -25.3224615030013, + -87.3365957976206, + -75.4228511668198, + -97.5772399468264, + -147.806188127279, + -226.039580534683, + -234.973383132998, + -136.995005290042, + -79.0935456156952, + -32.2648369968624, + -73.5156300394074 ], "lower.pred.int": [ - 232.915724635944, - 235.632437021281, - 238.31071744145, - 241.004539574701, - 243.743494835578, - 246.528729809762, - 240.385470475255, - 243.140785433079, - 245.855084301631, - 248.584939790848, - 251.360936143881, - 254.183984861509 + 341.840736117741, + 343.909153260812, + 345.895018966192, + 347.808763596993, + 349.654374816987, + 351.425426636534, + 349.192034229131, + 351.258231630815, + 353.236609473771, + 355.138069148118, + 356.966777766951, + 358.715984724406 ], "method": "lin", "nns.regress": 1.0, "obj.fn": 2.98551970897416, "periods": 6.0, "results": [ - 318.435066741207, - 321.151779126544, - 323.830059546713, - 326.523881679964, - 329.262836940841, - 332.048071915025, - 325.904812580518, - 328.660127538342, - 331.374426406894, - 334.104281896112, - 336.880278249144, - 339.703326966772 + 314.609121353927, + 316.677538496999, + 318.663404202379, + 320.57714883318, + 322.422760053174, + 324.193811872721, + 321.960419465317, + 324.026616867002, + 326.004994709958, + 327.906454384305, + 329.735163003138, + 331.484369960593 ], "shrink": 1.0, "upper.pred.int": [ - 403.95440884647, - 406.671121231807, - 409.349401651976, - 412.043223785227, - 414.782179046104, - 417.567414020288, - 411.424154685781, - 414.179469643605, - 416.893768512157, - 419.623624001375, - 422.399620354407, - 425.222669072036 + 547.125708772389, + 549.19412591546, + 551.179991620841, + 553.093736251642, + 554.939347471635, + 556.710399291183, + 554.477006883779, + 556.543204285463, + 558.52158212842, + 560.423041802766, + 562.251750421599, + 564.000957379054 ], "weights": null } diff --git a/tests/invariants/test_arma.py b/tests/invariants/test_arma.py index 01d77f11..4b29cd92 100644 --- a/tests/invariants/test_arma.py +++ b/tests/invariants/test_arma.py @@ -1,6 +1,7 @@ from __future__ import annotations import warnings +from typing import Any from unittest import mock import numpy as np @@ -42,6 +43,29 @@ def test_nns_arma_numeric_seasonal_dynamic_raises() -> None: nns_arma(variable, h=3, seasonal_factor=5, dynamic=True) +def test_nns_arma_numeric_weights_dynamic_raises() -> None: + variable = np.sin(np.arange(1, 40, dtype=np.float64)) + 2.0 + + with pytest.raises(ValueError, match="dynamic"): + nns_arma( + variable, h=3, seasonal_factor=False, weights=np.array([1.0, 0.0]), dynamic=True + ) + + +def test_nns_arma_dynamic_equal_weights_match_repeated_one_step_forecasts() -> None: + # As in patched R, weights="equal" applies to the lags each dynamic step selects + # (they used to be replaced by the NNS.seas weights). + variable = 100.0 + np.arange(60.0) + 15.0 * np.sin(np.arange(60.0) * np.pi / 6.0) + options: dict[str, Any] = {"seasonal_factor": False, "best_periods": 2, "weights": "equal"} + + dynamic = nns_arma(variable, h=3, dynamic=True, **options) + path = variable.copy() + for _ in range(3): + path = np.append(path, nns_arma(path, h=1, **options)) + + np.testing.assert_allclose(dynamic, path[-3:], rtol=1e-12) + + def test_numeric_seasonal_weights_constant_subsample_is_finite() -> None: # Regression: a lag-subsample with zero variance gives CV 0, so the seasonal # weighting 1 / (CV / baseline) = 1 / 0 = Inf and the normalisation collapsed to diff --git a/tests/parity/test_arma.py b/tests/parity/test_arma.py index d3cdc28f..ffac973e 100644 --- a/tests/parity/test_arma.py +++ b/tests/parity/test_arma.py @@ -487,20 +487,60 @@ def _two_cycle_series(seed: int) -> np.ndarray: ) +def _random_walk(seed: int) -> np.ndarray: + return np.round(30.0 + np.cumsum(np.random.default_rng(seed).normal(0.0, 1.0, 100)), 6) + + +_OPTIM_VALUE_KEYS = ("errors", "results", "lower.pred.int", "bias.shift") + + @pytest.mark.parity @pytest.mark.parametrize( - ("name", "variable", "h", "training_set", "seasonal_factor", "objective", "approximation"), + ( + "name", + "variable", + "h", + "training_set", + "seasonal_factor", + "objective", + "approximation", + "value_keys", + ), [ # "both" is scored with NNS.ARMA(method="both"). - ("both-scored", _AIR_PASSENGERS_132, None, 120, [6, 12, 18, 24], "min", True), + ( + "both-scored", + _AIR_PASSENGERS_132, + None, + 120, + [6, 12, 18, 24], + "min", + True, + _OPTIM_VALUE_KEYS, + ), # Here the separately averaged lin/nonlin paths would pick "both" instead. - ("both-scored-full", _AIR_PASSENGERS, 12, None, [1, 2, 12], "min", True), - # The equal-weights candidate is evaluated (and chosen here). - ("equal-weights", _two_cycle_series(33), 10, None, [4, 6, 7, 12, 14, 24], "min", True), - # The nns.regress decision is applied to the results (shrink TRUE). - ("regress-max", _AIR_PASSENGERS_132, None, 120, [6, 12, 24], "max", True), + ("both-scored-full", _AIR_PASSENGERS, 12, None, [1, 2, 12], "min", True, _OPTIM_VALUE_KEYS), + # The equal-weights candidate is evaluated (and chosen here); obj.fn is its + # score. The R values also carry the R optim-pi-from-returned-model change, + # so only the search outcome is compared here. + ("equal-weights", _two_cycle_series(33), 10, None, [4, 6, 7, 12, 14, 24], "min", True, ()), + # The nns.regress decision on a training-window smooth, applied to the results. + ("regress-max", _AIR_PASSENGERS_132, None, 120, [6, 12, 24], "max", True, ()), + # The smoothed candidate no longer sees the validation window: nns.regress + # TRUE -> FALSE (walk 13); stays TRUE on a training-only smooth (walk 16). + ("walk-13", _random_walk(13), None, 80, [1, 2, 4, 6, 12], "min", True, ()), + ("walk-16", _random_walk(16), None, 80, [1, 2, 4, 6, 12], "min", True, ()), # R now evaluates nonlin/both without the linear approximation. - ("exhaustive", _AIR_PASSENGERS_132, None, 120, [6, 12, 18, 24], "min", False), + ( + "exhaustive", + _AIR_PASSENGERS_132, + None, + 120, + [6, 12, 18, 24], + "min", + False, + _OPTIM_VALUE_KEYS, + ), ], ) def test_nns_arma_optim_selection_matches_patched_r( @@ -511,6 +551,7 @@ def test_nns_arma_optim_selection_matches_patched_r( seasonal_factor: list[int], objective: str, approximation: bool, + value_keys: tuple[str, ...], ) -> None: del name expected = _dict_any( @@ -540,12 +581,39 @@ def test_nns_arma_optim_selection_matches_patched_r( assert actual["weights"] is None else: np.testing.assert_allclose(actual["weights"], _array_any(expected["weights"])) - for key in ("periods", "obj.fn", "bias.shift", "errors", "results", "lower.pred.int"): + for key in ("periods", "obj.fn", *value_keys): np.testing.assert_allclose( _array_any(actual[key]), _array_any(expected[key]), atol=COMPOUND, equal_nan=True ) +@pytest.mark.parity +def test_nns_arma_optim_forecasts_from_the_training_only_smooth_like_r() -> None: + # The R values carry the optim-pi bias change, so compare the forecast before + # the bias shift; a smooth that sees the validation window moves it by ~0.14. + variable = _random_walk(16) + expected = _dict_any( + nns_arma_optim_custom( + variable.tolist(), + h=None, + training_set=80, + seasonal_factor=[1, 2, 4, 6, 12], + objective="min", + linear_approximation=True, + ) + ) + actual = nns_arma_optim( + variable, training_set=80, seasonal_factor=[1, 2, 4, 6, 12], print_trace=False + ) + + assert bool(actual["nns.regress"]) + np.testing.assert_allclose( + np.asarray(actual["results"]) - actual["bias.shift"], + _array_any(expected["results"]) - _array_any(expected["bias.shift"]), + atol=COMPOUND, + ) + + @pytest.mark.parity def test_nns_arma_optim_without_pred_int_matches_r() -> None: # R 13.3 returns numeric(0) bounds for pred.int = NULL; Python used to raise TypeError.