From fa008f646ca99a38da6bdd5bd04ab749a67eaece Mon Sep 17 00:00:00 2001 From: Yunchu Lee Date: Wed, 29 Jun 2022 14:52:34 +0900 Subject: [PATCH 1/2] resolve issues detected by pre-commit tests Signed-off-by: Yunchu Lee --- .coveragerc | 13 + .flake8 | 2 +- .gitignore | 3 + .pre-commit-config.yaml | 4 +- hpo/hpopt_status.json | 2805 ++++++++++++++++++++++++++++++ hpopt/__init__.py | 11 +- hpopt/{__asha.py => asha.py.bak} | 486 ++++-- hpopt/base.py | 331 +++- hpopt/{dummy.py => dummy.py.bak} | 0 hpopt/hpopt.py | 530 +----- hpopt/hyperband.py | 630 ++++--- hpopt/logger.py | 29 +- hpopt/smbo.py | 257 ++- hpopt/utils.py | 145 ++ pytest.ini | 3 + tests/test_asha.py | 76 - tests/test_base.py | 153 ++ tests/test_bayes_opt.py | 292 ++-- tests/test_hpopt.py | 146 +- tests/test_hyperband.py | 226 +++ tests/test_utils.py | 121 ++ 21 files changed, 4789 insertions(+), 1474 deletions(-) create mode 100644 .coveragerc create mode 100644 hpo/hpopt_status.json rename hpopt/{__asha.py => asha.py.bak} (55%) rename hpopt/{dummy.py => dummy.py.bak} (100%) create mode 100644 hpopt/utils.py create mode 100644 pytest.ini delete mode 100644 tests/test_asha.py create mode 100644 tests/test_base.py create mode 100644 tests/test_hyperband.py create mode 100644 tests/test_utils.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..6b3147e --- /dev/null +++ b/.coveragerc @@ -0,0 +1,13 @@ +[run] +data_file = .reports/.coverage +source = + hpopt/ +branch = true +omit = + */__init__.py + */tests/* + setup.py + +[report] +omit = + __init__.py \ No newline at end of file diff --git a/.flake8 b/.flake8 index ba57eea..5abdcf0 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] extend-ignore = E203 -exclude = .venv +exclude = .venv, tests/* max-complexity = 6 max-line-length = 120 diff --git a/.gitignore b/.gitignore index 4fffac2..7f88907 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ .vscode +.trunk venv __pycache__ *.egg-info +.reports +tmp/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b877aa8..168f0a7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: rev: "5.9.2" hooks: - id: isort - alias: isort_ote_sdk + alias: isort_hpopt name: "isort (hpopt)" args: ["--settings", ".isort.cfg"] files: '^hpopt/.*\.py' @@ -16,7 +16,7 @@ repos: hooks: - id: black name: "black (hpopt)" - args: [--line-length, "88"] + args: [--line-length, "120"] files: '^(hpopt)/.*\.py' - repo: https://github.com/PyCQA/flake8 diff --git a/hpo/hpopt_status.json b/hpo/hpopt_status.json new file mode 100644 index 0000000..4ad5e4c --- /dev/null +++ b/hpo/hpopt_status.json @@ -0,0 +1,2805 @@ +{ + "search_algorithm": "asha", + "search_space": { + "num": { + "_type": "uniform", + "_range": [ + 0, + 10, + 2 + ] + } + }, + "metric": "mAP", + "subset_ratio": 1.0, + "image_resize": [ + 0, + 0 + ], + "full_dataset_size": 100, + "reduction_factor": 2, + "config_list": [ + { + "trial_id": 0, + "config": { + "num": 9.190785461339159 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 1, + "config": { + "num": 3.097413973751765 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 2, + "config": { + "num": 4.626790903257594 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 3, + "config": { + "num": 7.14584684482963 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 4, + "config": { + "num": 7.691122718278853 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 5, + "config": { + "num": 2.5231666074388412 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 6, + "config": { + "num": 2.4271403322583263 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 7, + "config": { + "num": 7.645726088162319 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 8, + "config": { + "num": 7.659156231300316 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 9, + "config": { + "num": 6.350931543561577 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 10, + "config": { + "num": 8.905786366166112 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 11, + "config": { + "num": 3.3000455889385694 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 12, + "config": { + "num": 6.394145010940132 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 13, + "config": { + "num": 2.540735335798492 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 14, + "config": { + "num": 9.507639669909935 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 15, + "config": { + "num": 0.7550296737496631 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 16, + "config": { + "num": 1.2407402178660754 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 17, + "config": { + "num": 5.883844470217729 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 18, + "config": { + "num": 0.4731854077362718 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 19, + "config": { + "num": 5.020015669312075 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 20, + "config": { + "num": 7.954425232050779 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 21, + "config": { + "num": 8.849398535436928 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 22, + "config": { + "num": 3.127458664689693 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 23, + "config": { + "num": 1.3036740600600405 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 24, + "config": { + "num": 3.000398445914463 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 25, + "config": { + "num": 0.2420639350784637 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 26, + "config": { + "num": 8.586667187576104 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 27, + "config": { + "num": 7.0314151583301925 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 28, + "config": { + "num": 9.65618692841225 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 29, + "config": { + "num": 5.1328172620348 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 30, + "config": { + "num": 4.823033757341769 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 31, + "config": { + "num": 2.685297032149588 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 32, + "config": { + "num": 8.067637852460514 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 33, + "config": { + "num": 2.937816878589834 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 34, + "config": { + "num": 0.8231603741911775 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 35, + "config": { + "num": 4.013031403681442 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 36, + "config": { + "num": 3.3974378027885486 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 37, + "config": { + "num": 8.915129121617177 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 38, + "config": { + "num": 2.5643946172618537 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 39, + "config": { + "num": 5.491146153691347 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 40, + "config": { + "num": 1.6926320392285954 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 41, + "config": { + "num": 9.523515834154061 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 42, + "config": { + "num": 1.521537778666836 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 43, + "config": { + "num": 4.2613421187933636 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 44, + "config": { + "num": 2.563109861227919 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 45, + "config": { + "num": 0.8010871374986228 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 46, + "config": { + "num": 2.081515884793429 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 47, + "config": { + "num": 9.528538795780523 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 48, + "config": { + "num": 0.7678490511474345 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 49, + "config": { + "num": 0.8955261022940542 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 50, + "config": { + "num": 3.374798309692444 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 51, + "config": { + "num": 0.7771328949545819 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 52, + "config": { + "num": 8.888982717054512 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 53, + "config": { + "num": 7.099119872025392 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 54, + "config": { + "num": 3.5282897790124714 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 55, + "config": { + "num": 0.14777408015121396 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 56, + "config": { + "num": 2.367017113486342 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 57, + "config": { + "num": 2.3082050199945616 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 58, + "config": { + "num": 4.452871793915758 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 59, + "config": { + "num": 7.174515358594414 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 60, + "config": { + "num": 7.504109983360671 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 61, + "config": { + "num": 8.321565598402138 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 62, + "config": { + "num": 1.8373814035623992 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 63, + "config": { + "num": 8.14363312276332 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 64, + "config": { + "num": 0.6566735442210925 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 65, + "config": { + "num": 7.111536853610464 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 66, + "config": { + "num": 5.501582341699751 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 67, + "config": { + "num": 1.0832683561356515 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 68, + "config": { + "num": 5.4512960974240965 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 69, + "config": { + "num": 6.187343032504048 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 70, + "config": { + "num": 8.09869882272887 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 71, + "config": { + "num": 2.7885893589296087 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 72, + "config": { + "num": 2.8714575075045143 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 73, + "config": { + "num": 5.7147289368461776 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 74, + "config": { + "num": 1.284185559710177 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 75, + "config": { + "num": 2.5773053050310346 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 76, + "config": { + "num": 9.556120464708469 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 77, + "config": { + "num": 1.2141001086712855 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 78, + "config": { + "num": 0.8038481365796979 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 79, + "config": { + "num": 8.475325810316622 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 80, + "config": { + "num": 8.208978129689083 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 81, + "config": { + "num": 5.133228843760133 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 82, + "config": { + "num": 5.9505547594634844 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 83, + "config": { + "num": 4.615936469101927 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 84, + "config": { + "num": 9.38837893635139 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 85, + "config": { + "num": 7.8299490095057065 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 86, + "config": { + "num": 9.280824719193346 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 87, + "config": { + "num": 6.0711586827173925 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 88, + "config": { + "num": 1.317315785421248 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 89, + "config": { + "num": 4.353744417685904 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 90, + "config": { + "num": 8.031866718094513 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 91, + "config": { + "num": 3.752003579475658 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 92, + "config": { + "num": 7.2700466502546615 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 93, + "config": { + "num": 6.861328404556323 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 94, + "config": { + "num": 9.56468287726371 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 95, + "config": { + "num": 8.707273858029929 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 96, + "config": { + "num": 2.7674173209274224 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 97, + "config": { + "num": 0.053823131136835345 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 98, + "config": { + "num": 9.753424468390657 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 99, + "config": { + "num": 3.3399605474177596 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 100, + "config": { + "num": 0.0029079084877980943 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 101, + "config": { + "num": 6.497373170406503 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 102, + "config": { + "num": 0.8991829228584458 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 103, + "config": { + "num": 4.726862945417775 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 104, + "config": { + "num": 4.644130174455235 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 105, + "config": { + "num": 4.762549771501221 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 106, + "config": { + "num": 4.354391014449166 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 107, + "config": { + "num": 1.6440700560293098 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 108, + "config": { + "num": 0.6193636570337602 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 109, + "config": { + "num": 0.1318746267154025 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 110, + "config": { + "num": 2.0863467910045532 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 111, + "config": { + "num": 0.4952512903123829 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 112, + "config": { + "num": 6.8257935357958495 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 113, + "config": { + "num": 8.587922574841434 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 114, + "config": { + "num": 7.2113453714521825 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 115, + "config": { + "num": 8.433496699383468 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 116, + "config": { + "num": 3.5548731177762836 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 117, + "config": { + "num": 8.949918440984739 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 118, + "config": { + "num": 7.449254757522867 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 119, + "config": { + "num": 9.853591253401854 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 120, + "config": { + "num": 8.597569759328902 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 121, + "config": { + "num": 6.302132703374234 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 122, + "config": { + "num": 1.6730163593180691 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 123, + "config": { + "num": 8.060809877403917 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 124, + "config": { + "num": 5.66312122704362 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 125, + "config": { + "num": 2.990172365481877 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 126, + "config": { + "num": 7.344274329927388 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 127, + "config": { + "num": 6.1655788408835 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 128, + "config": { + "num": 7.115786377483245 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 129, + "config": { + "num": 6.377813387650217 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 130, + "config": { + "num": 9.969897161611627 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 131, + "config": { + "num": 0.11820991704570516 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 132, + "config": { + "num": 9.40250393879324 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 133, + "config": { + "num": 4.1022636893584945 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 134, + "config": { + "num": 5.110686139464054 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 135, + "config": { + "num": 9.895756478876056 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 136, + "config": { + "num": 7.383243148907463 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 137, + "config": { + "num": 7.968780549103198 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 138, + "config": { + "num": 9.378018619003933 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 139, + "config": { + "num": 6.549412178507513 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 140, + "config": { + "num": 0.5991866544898194 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 141, + "config": { + "num": 6.2285202807125 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 142, + "config": { + "num": 7.7713690444458186 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 143, + "config": { + "num": 1.0848333801871113 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 144, + "config": { + "num": 1.4994710302760594 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 145, + "config": { + "num": 9.854240642265209 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 146, + "config": { + "num": 0.25683919617691897 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 147, + "config": { + "num": 9.924279382515932 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 148, + "config": { + "num": 9.47293078232542 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 149, + "config": { + "num": 5.212121418638024 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 150, + "config": { + "num": 2.00183345818398 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 151, + "config": { + "num": 7.677932782169259 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 152, + "config": { + "num": 5.116544251114111 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 153, + "config": { + "num": 5.782763350569699 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 154, + "config": { + "num": 5.747338318222328 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 155, + "config": { + "num": 5.92448224367312 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 156, + "config": { + "num": 7.604118434552617 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 157, + "config": { + "num": 9.084709463391166 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 158, + "config": { + "num": 0.3338335884692889 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 159, + "config": { + "num": 3.8151189662472174 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 160, + "config": { + "num": 4.016871972954591 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 161, + "config": { + "num": 2.4026576997333438 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 162, + "config": { + "num": 0.649757881936428 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 163, + "config": { + "num": 5.6417853799838635 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 164, + "config": { + "num": 4.713376810403927 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 165, + "config": { + "num": 0.32036909478711406 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 166, + "config": { + "num": 1.4919793679590432 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 167, + "config": { + "num": 2.841092897070354 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 168, + "config": { + "num": 2.256573332056556 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 169, + "config": { + "num": 5.858779253292971 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 170, + "config": { + "num": 2.9481031853677053 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 171, + "config": { + "num": 5.600780720599121 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 172, + "config": { + "num": 6.280269557236756 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 173, + "config": { + "num": 1.0379014922249474 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 174, + "config": { + "num": 7.086620844245659 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 175, + "config": { + "num": 5.683051983075958 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 176, + "config": { + "num": 4.187483829106893 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 177, + "config": { + "num": 6.2455588903724655 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 178, + "config": { + "num": 4.564488068402602 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 179, + "config": { + "num": 8.971180846222165 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 180, + "config": { + "num": 3.633654147869909 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 181, + "config": { + "num": 2.7986836803072377 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 182, + "config": { + "num": 0.8203453550488415 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 183, + "config": { + "num": 4.380776007931217 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 184, + "config": { + "num": 4.735446413784708 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 185, + "config": { + "num": 5.530099559131739 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 186, + "config": { + "num": 3.361632916401296 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 187, + "config": { + "num": 1.8691787147391081 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 188, + "config": { + "num": 7.445004505158117 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 189, + "config": { + "num": 0.02953545354371978 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 190, + "config": { + "num": 3.207809001962998 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 191, + "config": { + "num": 3.2150956063048097 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 192, + "config": { + "num": 0.19690495919416184 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 193, + "config": { + "num": 0.8264962378372209 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 194, + "config": { + "num": 7.545000432077893 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 195, + "config": { + "num": 7.40877827278973 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 196, + "config": { + "num": 2.840332616620276 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 197, + "config": { + "num": 6.219161021698637 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 198, + "config": { + "num": 2.574655602448015 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 199, + "config": { + "num": 4.81733002658639 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 200, + "config": { + "num": 7.34472388990503 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 201, + "config": { + "num": 4.639614276025074 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 202, + "config": { + "num": 1.7941634506381554 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 203, + "config": { + "num": 9.082379640208988 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 204, + "config": { + "num": 7.104122316369578 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 205, + "config": { + "num": 2.4050001175777336 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 206, + "config": { + "num": 3.3375671609409685 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 207, + "config": { + "num": 5.085024332109943 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 208, + "config": { + "num": 9.266879675284006 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 209, + "config": { + "num": 4.671897253922229 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 210, + "config": { + "num": 2.817089525341927 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 211, + "config": { + "num": 8.40542412176976 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 212, + "config": { + "num": 9.802448851975038 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 213, + "config": { + "num": 8.404339975011055 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 214, + "config": { + "num": 7.387068486878721 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 215, + "config": { + "num": 7.516261187700223 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 216, + "config": { + "num": 6.736841254689033 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 217, + "config": { + "num": 2.680902780136869 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 218, + "config": { + "num": 6.033551399459606 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 219, + "config": { + "num": 7.076938071480596 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 220, + "config": { + "num": 0.2541303602901335 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 221, + "config": { + "num": 4.65867923030035 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 222, + "config": { + "num": 2.633437704600541 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 223, + "config": { + "num": 7.083565669132578 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 224, + "config": { + "num": 1.794647289420207 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 225, + "config": { + "num": 2.172948161036877 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 226, + "config": { + "num": 0.04621479870294043 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 227, + "config": { + "num": 9.202304653415162 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 228, + "config": { + "num": 7.596150118836761 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 229, + "config": { + "num": 0.9518544869732726 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 230, + "config": { + "num": 7.204690525690955 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 231, + "config": { + "num": 1.1011053337379073 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 232, + "config": { + "num": 2.8223864148571023 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 233, + "config": { + "num": 5.398665978819745 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 234, + "config": { + "num": 3.224085932908096 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 235, + "config": { + "num": 2.891615130460777 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 236, + "config": { + "num": 2.821013554889842 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 237, + "config": { + "num": 4.919828223677216 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 238, + "config": { + "num": 2.3038471975803834 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 239, + "config": { + "num": 8.228249340879067 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 240, + "config": { + "num": 8.710903812853198 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 241, + "config": { + "num": 6.773557494468769 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 242, + "config": { + "num": 8.918376850571361 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 243, + "config": { + "num": 9.055242244531291 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 244, + "config": { + "num": 9.63097922303654 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 245, + "config": { + "num": 2.8043886440925023 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 246, + "config": { + "num": 4.670153553213389 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 247, + "config": { + "num": 9.073928848644591 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 248, + "config": { + "num": 5.211495198871836 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 249, + "config": { + "num": 8.216134472466125 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 250, + "config": { + "num": 1.190460678957782 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 251, + "config": { + "num": 6.790129835218906 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 252, + "config": { + "num": 0.6008609833615286 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 253, + "config": { + "num": 4.196437399427799 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 254, + "config": { + "num": 1.2067942387980524 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 255, + "config": { + "num": 1.12054657174122 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 256, + "config": { + "num": 7.802818941177437 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 257, + "config": { + "num": 1.986512134964007 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 258, + "config": { + "num": 3.1436396897499463 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 259, + "config": { + "num": 2.2585748326464463 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 260, + "config": { + "num": 8.041293081899305 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 261, + "config": { + "num": 8.082387956900238 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 262, + "config": { + "num": 5.319074902235118 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 263, + "config": { + "num": 3.915648081083183 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 264, + "config": { + "num": 1.724508540246329 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 265, + "config": { + "num": 2.8594203159065934 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 266, + "config": { + "num": 1.1403023823068292 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 267, + "config": { + "num": 3.909251367472917 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 268, + "config": { + "num": 3.6453388289228075 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 269, + "config": { + "num": 8.940559595435415 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 270, + "config": { + "num": 7.861217744803311 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 271, + "config": { + "num": 7.929317241660692 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 272, + "config": { + "num": 4.651250499405957 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 273, + "config": { + "num": 9.058633464630338 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 274, + "config": { + "num": 2.2989152992616892 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 275, + "config": { + "num": 2.219957703904023 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 276, + "config": { + "num": 4.074585468061028 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 277, + "config": { + "num": 8.3584834166828 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 278, + "config": { + "num": 0.39594842665798824 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 279, + "config": { + "num": 5.974048841008725 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 280, + "config": { + "num": 5.107846955884289 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 281, + "config": { + "num": 9.325828431139508 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 282, + "config": { + "num": 4.747095746526035 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 283, + "config": { + "num": 5.763336377474086 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 284, + "config": { + "num": 7.531978865066535 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 285, + "config": { + "num": 8.080236190514578 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 286, + "config": { + "num": 5.829393254752268 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 287, + "config": { + "num": 7.302462842372158 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 288, + "config": { + "num": 9.342724451704491 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 289, + "config": { + "num": 4.396726563385789 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 290, + "config": { + "num": 7.368614273526446 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 291, + "config": { + "num": 6.383367872319824 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 292, + "config": { + "num": 0.3791697879801659 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 293, + "config": { + "num": 9.20267555273517 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 294, + "config": { + "num": 2.2596645194020804 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 295, + "config": { + "num": 8.789283038590984 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 296, + "config": { + "num": 3.256972063692436 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 297, + "config": { + "num": 7.046950037634665 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 298, + "config": { + "num": 4.300386623414655 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 299, + "config": { + "num": 0.25760908154325857 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 300, + "config": { + "num": 4.133755704568225 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 301, + "config": { + "num": 3.826850841348459 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 302, + "config": { + "num": 3.7616874595019723 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 303, + "config": { + "num": 3.2370144199313664 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 304, + "config": { + "num": 7.074466519397707 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 305, + "config": { + "num": 0.09357756209379398 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 306, + "config": { + "num": 5.031863397177632 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 307, + "config": { + "num": 9.509956105868255 + }, + "status": 0, + "score": null, + "bracket": 0 + }, + { + "trial_id": 308, + "config": { + "num": 6.282183274998365 + }, + "status": 0, + "score": null, + "bracket": 0 + } + ], + "num_gen_config": 0 +} \ No newline at end of file diff --git a/hpopt/__init__.py b/hpopt/__init__.py index 1492d38..66cc7fa 100644 --- a/hpopt/__init__.py +++ b/hpopt/__init__.py @@ -1,13 +1,12 @@ # Copyright (C) 2018-2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # +""" exporting hpopt APIs +""" +from .base import SearchSpace, Status from .hpopt import ( - SearchSpace, - Status, create, - createDummyOpt, - createHpoDataset, finalize_trial, get_best_score, get_best_score_with_num_imgs, @@ -15,6 +14,7 @@ get_previous_status, get_status_path, get_trial_path, + get_trial_results, load_json, report, reportOOM, @@ -26,12 +26,11 @@ "get_current_status", "get_previous_status", "create", - "createHpoDataset", - "createDummyOpt", "SearchSpace", "Status", "get_status_path", "get_trial_path", + "get_trial_results", "get_best_score", "get_best_score_with_num_imgs", "finalize_trial", diff --git a/hpopt/__asha.py b/hpopt/asha.py.bak similarity index 55% rename from hpopt/__asha.py rename to hpopt/asha.py.bak index 63b2a1a..677834d 100644 --- a/hpopt/__asha.py +++ b/hpopt/asha.py.bak @@ -3,15 +3,15 @@ # import json +import logging import os from math import ceil, log -import logging -from typing import Optional, Union, List, Dict +from typing import Dict, List, Optional, Union from bayes_opt import BayesianOptimization, UtilityFunction -from hpopt.hpopt_abstract import HpOpt import hpopt +from hpopt.hpopt_abstract import HpOpt logger = logging.getLogger(__name__) @@ -35,31 +35,40 @@ class AsyncHyperBand(HpOpt): num_brackets (int): Number of brackets. Each bracket has a different halving rate, specified by the reduction factor. """ - def __init__(self, - num_brackets: Optional[int] = None, - min_iterations: Optional[int] = None, - reduction_factor: int = 2, - default_hyper_parameters: Optional[Union[Dict, List[Dict]]] = None, - **kwargs): + + def __init__( + self, + num_brackets: Optional[int] = None, + min_iterations: Optional[int] = None, + reduction_factor: int = 2, + default_hyper_parameters: Optional[Union[Dict, List[Dict]]] = None, + **kwargs, + ): super(AsyncHyperBand, self).__init__(**kwargs) if min_iterations is not None: if type(min_iterations) != int: - raise TypeError('min_iterations should be int type') + raise TypeError("min_iterations should be int type") elif min_iterations < 1: - raise ValueError(f'min_iterations should be bigger than 1. Your value is {min_iterations}') + raise ValueError( + f"min_iterations should be bigger than 1. Your value is {min_iterations}" + ) if type(reduction_factor) != int: - raise TypeError('reduction_factor should be int type') + raise TypeError("reduction_factor should be int type") elif reduction_factor < 1: - raise ValueError('reduction_factor should be bigger than 1.' - f' Your value is {reduction_factor}') + raise ValueError( + "reduction_factor should be bigger than 1." + f" Your value is {reduction_factor}" + ) if num_brackets is not None: if type(num_brackets) != int: - raise TypeError('num_brackets should be int type') + raise TypeError("num_brackets should be int type") elif num_brackets < 1: - raise ValueError(f'num_brackets should be bigger than 1. Your value is {num_brackets}') + raise ValueError( + f"num_brackets should be bigger than 1. Your value is {num_brackets}" + ) self.min_iterations = min_iterations self.reduction_factor = reduction_factor @@ -69,26 +78,41 @@ def __init__(self, self.hpo_status = {} # HPO auto configurator - if self.num_trials is None or self.max_iterations is None or self.subset_ratio is None: + if ( + self.num_trials is None + or self.max_iterations is None + or self.subset_ratio is None + ): self.updatable_schedule = True - self.num_trials, self.max_iterations, self.subset_ratio, \ - self.expected_total_epochs, self.num_brackets, self.min_iterations \ - = self.auto_config(expected_time_ratio=self.expected_time_ratio, - num_full_epochs=self.num_full_iterations, - full_dataset_size=self.full_dataset_size, - subset_ratio=self.subset_ratio, - non_pure_train_ratio=self.non_pure_train_ratio, - num_hyperparams=len(self.search_space), - reduction_factor=self.reduction_factor, - parallelism=self.num_workers, - min_epochs=self.min_iterations, - max_epochs=self.max_iterations) - logger.debug(f"auto-config: num_trials : {self.num_trials} min_iterations : {self.min_iterations} " - f"max_iterations : {self.max_iterations} subset_ratio {self.subset_ratio} " - f"expected_total_epochs : {self.expected_total_epochs} num_brackets: {self.num_brackets}") + ( + self.num_trials, + self.max_iterations, + self.subset_ratio, + self.expected_total_epochs, + self.num_brackets, + self.min_iterations, + ) = self.auto_config( + expected_time_ratio=self.expected_time_ratio, + num_full_epochs=self.num_full_iterations, + full_dataset_size=self.full_dataset_size, + subset_ratio=self.subset_ratio, + non_pure_train_ratio=self.non_pure_train_ratio, + num_hyperparams=len(self.search_space), + reduction_factor=self.reduction_factor, + parallelism=self.num_workers, + min_epochs=self.min_iterations, + max_epochs=self.max_iterations, + ) + logger.debug( + f"auto-config: num_trials : {self.num_trials} min_iterations : {self.min_iterations} " + f"max_iterations : {self.max_iterations} subset_ratio {self.subset_ratio} " + f"expected_total_epochs : {self.expected_total_epochs} num_brackets: {self.num_brackets}" + ) else: if self.num_brackets is None: - self.num_brackets = int(log(self.max_iterations) / log(self.reduction_factor)) + 1 + self.num_brackets = ( + int(log(self.max_iterations) / log(self.reduction_factor)) + 1 + ) if self.min_iterations is None: self.min_iterations = 1 @@ -97,17 +121,25 @@ def __init__(self, # It will be used for generating trials. self.bayesopt_space = {} for param in self.search_space: - self.bayesopt_space[param] = (self.search_space[param].lower_space(), - self.search_space[param].upper_space()) - - self.optimizer = BayesianOptimization(f=self.obj, - pbounds=self.bayesopt_space, - verbose=self.verbose, - random_state=None) - - self.uf = UtilityFunction(kind="ucb", kappa=2.5, xi=0.0, - kappa_decay=1, - kappa_decay_delay=self.num_init_trials) + self.bayesopt_space[param] = ( + self.search_space[param].lower_space(), + self.search_space[param].upper_space(), + ) + + self.optimizer = BayesianOptimization( + f=self.obj, + pbounds=self.bayesopt_space, + verbose=self.verbose, + random_state=None, + ) + + self.uf = UtilityFunction( + kind="ucb", + kappa=2.5, + xi=0.0, + kappa_decay=1, + kappa_decay_delay=self.num_init_trials, + ) if self.hasCategoricalParam(self.search_space): self.optimizer.set_gp_params(alpha=1e-3) @@ -116,59 +148,63 @@ def __init__(self, hpo_file_path = hpopt.get_status_path(self.save_path) if self.resume is True and os.path.exists(hpo_file_path): - with open(hpo_file_path, 'rt') as json_file: + with open(hpo_file_path, "rt") as json_file: self.hpo_status = json.load(json_file) json_file.close() - if self.hpo_status['search_algorithm'] != 'asha': + if self.hpo_status["search_algorithm"] != "asha": logger.error("Search algorithm is changed. Stop resuming.") raise ValueError("Search algorithm is changed.") - if self.hpo_status['search_space'] != {ss: self.search_space[ss].__dict__ for ss in self.search_space}: + if self.hpo_status["search_space"] != { + ss: self.search_space[ss].__dict__ for ss in self.search_space + }: logger.error("Search space is changed. Stop resuming.") raise ValueError("Search space is changed.") - if self.hpo_status['metric'] != self.metric: + if self.hpo_status["metric"] != self.metric: logger.error("Metric is changed. Stop resuming.") raise ValueError("Metric is changed.") - if self.hpo_status['subset_ratio'] != self.subset_ratio: + if self.hpo_status["subset_ratio"] != self.subset_ratio: logger.error("subset_ratio is changed. Stop resuming.") raise ValueError("subset_ratio is changed.") - if self.hpo_status['image_resize'] != self.image_resize: + if self.hpo_status["image_resize"] != self.image_resize: logger.error("image_resize is changed. Stop resuming.") raise ValueError("image_resize is changed.") - if self.hpo_status['full_dataset_size'] != self.full_dataset_size: + if self.hpo_status["full_dataset_size"] != self.full_dataset_size: logger.error("The size of dataset is changed. Stop resuming.") raise ValueError("Dataset is changed.") - if self.hpo_status['reduction_factor'] != self.reduction_factor: + if self.hpo_status["reduction_factor"] != self.reduction_factor: logger.error("reduction_factor is changed. Stop resuming.") raise ValueError("reduction_factor is changed.") else: - self.hpo_status['search_algorithm'] = 'asha' - self.hpo_status['search_space'] = {ss: self.search_space[ss].__dict__ for ss in self.search_space} - self.hpo_status['metric'] = self.metric - self.hpo_status['subset_ratio'] = self.subset_ratio - self.hpo_status['image_resize'] = self.image_resize - self.hpo_status['full_dataset_size'] = self.full_dataset_size - self.hpo_status['reduction_factor'] = self.reduction_factor - self.hpo_status['config_list'] = [] + self.hpo_status["search_algorithm"] = "asha" + self.hpo_status["search_space"] = { + ss: self.search_space[ss].__dict__ for ss in self.search_space + } + self.hpo_status["metric"] = self.metric + self.hpo_status["subset_ratio"] = self.subset_ratio + self.hpo_status["image_resize"] = self.image_resize + self.hpo_status["full_dataset_size"] = self.full_dataset_size + self.hpo_status["reduction_factor"] = self.reduction_factor + self.hpo_status["config_list"] = [] # num_gen_config represents the number of assigned configs. # If 1 configs are done and 2 configs are running, num_gen_config is 3. - self.hpo_status['num_gen_config'] = 0 + self.hpo_status["num_gen_config"] = 0 # Resumes from the previous results. - for idx, config in enumerate(self.hpo_status['config_list']): - if config['status'] == hpopt.Status.STOP: + for idx, config in enumerate(self.hpo_status["config_list"]): + if config["status"] == hpopt.Status.STOP: # This case means that config is done. - self.hpo_status['num_gen_config'] += 1 + self.hpo_status["num_gen_config"] += 1 else: # This case means that config should be in the ready state. - config['status'] = hpopt.Status.READY + config["status"] = hpopt.Status.READY trial_file_path = hpopt.get_trial_path(self.save_path, idx) if os.path.exists(trial_file_path): # Incomplete informations should be removed before starting. @@ -177,9 +213,11 @@ def __init__(self, # Initialize the brackets self.rungs_in_brackets = [] for s in range(self.num_brackets): - self.rungs_in_brackets.append(self.get_rungs(self.min_iterations, - self.max_iterations, - self.reduction_factor, s)) + self.rungs_in_brackets.append( + self.get_rungs( + self.min_iterations, self.max_iterations, self.reduction_factor, s + ) + ) # Get the max rung iterations # self.max_rung = 1 # for rungs in self.rungs_in_brackets: @@ -192,7 +230,9 @@ def __init__(self, # n0 is come from the notation in the paper, which means # the initial number of trials in each bracket. - n0_in_brackets = self.get_num_trials_in_brackets(self.reduction_factor, self.num_brackets) + n0_in_brackets = self.get_num_trials_in_brackets( + self.reduction_factor, self.num_brackets + ) # Assign each trial to a bracket # according to the ratio of the relative number of trials in each bracket @@ -204,23 +244,25 @@ def __init__(self, default_hyper_parameters = [default_hyper_parameters] for idx, default_hyper_parameter in enumerate(default_hyper_parameters): - self.hpo_status['config_list'].append({ - "trial_id" : idx, - "config" : default_hyper_parameter, - "status" : hpopt.Status.READY, - "score" : None, - 'bracket': 0, - }) + self.hpo_status["config_list"].append( + { + "trial_id": idx, + "config": default_hyper_parameter, + "status": hpopt.Status.READY, + "score": None, + "bracket": 0, + } + ) - num_ready_configs = len(self.hpo_status['config_list']) + num_ready_configs = len(self.hpo_status["config_list"]) for i in range(len(n0_in_brackets)): n0_in_brackets[i] = int(brackets_ratio[i] * self.num_trials) - n0_in_brackets[0] += (self.num_trials - sum(n0_in_brackets)) + n0_in_brackets[0] += self.num_trials - sum(n0_in_brackets) - for trial in self.hpo_status['config_list']: - n0_in_brackets[trial['bracket']] -= 1 + for trial in self.hpo_status["config_list"]: + n0_in_brackets[trial["bracket"]] -= 1 # Generate trial configs up to self.num_trials for i in range(num_ready_configs, self.num_trials): @@ -245,13 +287,15 @@ def __init__(self, if retry_count > 100: return None - self.hpo_status['config_list'].append( - {'trial_id': i, - 'config': config, - 'status': hpopt.Status.READY, - 'score': None, - 'bracket': bracket_id} - ) + self.hpo_status["config_list"].append( + { + "trial_id": i, + "config": config, + "status": hpopt.Status.READY, + "score": None, + "bracket": bracket_id, + } + ) # Sync the current status to the file system. self.save_results() @@ -259,7 +303,7 @@ def __init__(self, # Get the list of iteration numbers for rungs in the bracket def get_rungs(self, min_t: int, max_t: int, rf: int, s: int): MAX_RUNGS = int(log(max_t - min_t + 1) / log(rf) - s + 1) - rungs = [(min_t - 1) + rf**(k + s) for k in reversed(range(MAX_RUNGS))] + rungs = [(min_t - 1) + rf ** (k + s) for k in reversed(range(MAX_RUNGS))] return rungs @@ -267,7 +311,7 @@ def get_num_trials_in_brackets(self, reduction_factor: int, num_brackets: int): brackets = [] for s in reversed(range(num_brackets)): - n = int(ceil(int(num_brackets/(s+1))*reduction_factor**s)) + n = int(ceil(int(num_brackets / (s + 1)) * reduction_factor**s)) brackets.append(n) return brackets @@ -275,7 +319,7 @@ def get_num_trials_in_brackets(self, reduction_factor: int, num_brackets: int): def save_results(self): hpo_file_path = hpopt.get_status_path(self.save_path) oldmask = os.umask(0o077) - with open(hpo_file_path, 'wt') as json_file: + with open(hpo_file_path, "wt") as json_file: json.dump(self.hpo_status, json_file, indent=4) json_file.close() os.umask(oldmask) @@ -285,23 +329,25 @@ def get_next_sample(self): self.update_scores() # Check if unassigned configs - if self.hpo_status['num_gen_config'] >= self.num_trials: + if self.hpo_status["num_gen_config"] >= self.num_trials: return None # Check total number of executed epochs if self.expected_total_epochs > 0: num_executed_epochs = self.get_num_executed_epochs() - logger.debug(f"self.expected_total_epochs {self.expected_total_epochs} " - f"num_executed_epochs {num_executed_epochs}") + logger.debug( + f"self.expected_total_epochs {self.expected_total_epochs} " + f"num_executed_epochs {num_executed_epochs}" + ) if num_executed_epochs >= self.expected_total_epochs: return None # Choose a config next_config = None trial_id = None - for idx, config in enumerate(self.hpo_status['config_list']): - if config['status'] is hpopt.Status.READY: - config['status'] = hpopt.Status.RUNNING + for idx, config in enumerate(self.hpo_status["config_list"]): + if config["status"] is hpopt.Status.READY: + config["status"] = hpopt.Status.RUNNING next_config = config trial_id = idx break @@ -309,39 +355,41 @@ def get_next_sample(self): if next_config is None: return None - self.hpo_status['num_gen_config'] += 1 + self.hpo_status["num_gen_config"] += 1 self.save_results() new_config = {} - new_config['params'] = next_config['config'] - new_config['iterations'] = self.max_iterations - new_config['trial_id'] = trial_id - new_config['bracket'] = next_config['bracket'] - new_config['rungs'] = self.rungs_in_brackets[next_config['bracket']] - new_config['file_path'] = hpopt.get_trial_path(self.save_path, new_config['trial_id']) - new_config['subset_ratio'] = self.subset_ratio - new_config['resize_width'] = self.image_resize[0] - new_config['resize_height'] = self.image_resize[1] - new_config['mode'] = self.mode + new_config["params"] = next_config["config"] + new_config["iterations"] = self.max_iterations + new_config["trial_id"] = trial_id + new_config["bracket"] = next_config["bracket"] + new_config["rungs"] = self.rungs_in_brackets[next_config["bracket"]] + new_config["file_path"] = hpopt.get_trial_path( + self.save_path, new_config["trial_id"] + ) + new_config["subset_ratio"] = self.subset_ratio + new_config["resize_width"] = self.image_resize[0] + new_config["resize_height"] = self.image_resize[1] + new_config["mode"] = self.mode return new_config def update_scores(self): - for trial_id, config_item in enumerate(self.hpo_status['config_list']): - if config_item['status'] == hpopt.Status.RUNNING: + for trial_id, config_item in enumerate(self.hpo_status["config_list"]): + if config_item["status"] == hpopt.Status.RUNNING: current_status = hpopt.get_current_status(self.save_path, trial_id) if current_status == hpopt.Status.STOP: score = hpopt.get_best_score(self.save_path, trial_id, self.mode) if score is not None: - config_item['score'] = score - config_item['status'] = hpopt.Status.STOP - real_config = config_item['config'] - logger.info(f'#{trial_id} | {real_config} | {score}') + config_item["score"] = score + config_item["status"] = hpopt.Status.STOP + real_config = config_item["config"] + logger.info(f"#{trial_id} | {real_config} | {score}") elif current_status == hpopt.Status.CUDAOOM: - config_item['status'] = hpopt.Status.READY - self.hpo_status['num_gen_config'] -= 1 + config_item["status"] = hpopt.Status.READY + self.hpo_status["num_gen_config"] -= 1 trial_file_path = hpopt.get_trial_path(self.save_path, trial_id) if os.path.exists(trial_file_path): @@ -357,21 +405,30 @@ def update_scores(self): if self.updatable_schedule is True: logger.debug("asha update-config start") - new_num_trials, new_max_iterations, new_subset_ratio, \ - new_expected_total_epochs, new_num_brackets, new_min_iterations \ - = self.auto_config(expected_time_ratio=self.expected_time_ratio, - num_full_epochs=self.num_full_iterations, - full_dataset_size=self.full_dataset_size, - subset_ratio=self.subset_ratio, - non_pure_train_ratio=self.non_pure_train_ratio, - num_hyperparams=len(self.search_space), - reduction_factor=self.reduction_factor, - parallelism=self.num_workers, - min_epochs=self.min_iterations, - max_epochs=self.max_iterations) - logger.debug(f"update-config: num_trials : {new_num_trials} min_iterations : {new_min_iterations} " - f"max_iterations : {new_max_iterations} subset_ratio : {new_subset_ratio} " - f"expected_total_epochs : {new_expected_total_epochs} num_brackets : {new_num_brackets}") + ( + new_num_trials, + new_max_iterations, + new_subset_ratio, + new_expected_total_epochs, + new_num_brackets, + new_min_iterations, + ) = self.auto_config( + expected_time_ratio=self.expected_time_ratio, + num_full_epochs=self.num_full_iterations, + full_dataset_size=self.full_dataset_size, + subset_ratio=self.subset_ratio, + non_pure_train_ratio=self.non_pure_train_ratio, + num_hyperparams=len(self.search_space), + reduction_factor=self.reduction_factor, + parallelism=self.num_workers, + min_epochs=self.min_iterations, + max_epochs=self.max_iterations, + ) + logger.debug( + f"update-config: num_trials : {new_num_trials} min_iterations : {new_min_iterations} " + f"max_iterations : {new_max_iterations} subset_ratio : {new_subset_ratio} " + f"expected_total_epochs : {new_expected_total_epochs} num_brackets : {new_num_brackets}" + ) self.expected_total_epochs = new_expected_total_epochs @@ -386,70 +443,91 @@ def shrink_bs_search_space(self, not_allowed_config): return # Check if the search space type for batch size if qloguniform or quniform. - if self.search_space[self.batch_size_name].type not in ['qloguniform', 'quniform']: + if self.search_space[self.batch_size_name].type not in [ + "qloguniform", + "quniform", + ]: return - not_allowed_bs = not_allowed_config['config'][self.batch_size_name] - new_upper_bound = not_allowed_bs - self.search_space[self.batch_size_name].range[2] + not_allowed_bs = not_allowed_config["config"][self.batch_size_name] + new_upper_bound = ( + not_allowed_bs - self.search_space[self.batch_size_name].range[2] + ) # if new_upper_bound is greater than the current upper bound, update only this trial. if self.search_space[self.batch_size_name].range[1] <= new_upper_bound: - not_allowed_config['config'] = self.get_real_config(self.optimizer.suggest(self.uf)) + not_allowed_config["config"] = self.get_real_config( + self.optimizer.suggest(self.uf) + ) return self.search_space[self.batch_size_name].range[1] = new_upper_bound # if the new upper bound is less than the current lower bound, # update the lower bound to be half of the new upper bound. - if self.search_space[self.batch_size_name].range[0] > self.search_space[self.batch_size_name].range[1]: + if ( + self.search_space[self.batch_size_name].range[0] + > self.search_space[self.batch_size_name].range[1] + ): new_lower_bound = self.search_space[self.batch_size_name].range[1] // 2 self.search_space[self.batch_size_name].range[0] = max(new_lower_bound, 2) - if self.search_space[self.batch_size_name].range[0] > self.search_space[self.batch_size_name].range[1]: - raise ValueError('This model cannot be trained even with batch size of 2.') + if ( + self.search_space[self.batch_size_name].range[0] + > self.search_space[self.batch_size_name].range[1] + ): + raise ValueError( + "This model cannot be trained even with batch size of 2." + ) # Reset search space self.bayesopt_space = {} for param in self.search_space: - self.bayesopt_space[param] = (self.search_space[param].lower_space(), - self.search_space[param].upper_space()) - - self.optimizer = BayesianOptimization(f=self.obj, - pbounds=self.bayesopt_space, - verbose=self.verbose, - random_state=None) + self.bayesopt_space[param] = ( + self.search_space[param].lower_space(), + self.search_space[param].upper_space(), + ) + + self.optimizer = BayesianOptimization( + f=self.obj, + pbounds=self.bayesopt_space, + verbose=self.verbose, + random_state=None, + ) if self.hasCategoricalParam(self.search_space): self.optimizer.set_gp_params(alpha=1e-3) - for idx, config in enumerate(self.hpo_status['config_list']): - if config['status'] is hpopt.Status.READY: - config['config'] = self.get_real_config(self.optimizer.suggest(self.uf)) + for idx, config in enumerate(self.hpo_status["config_list"]): + if config["status"] is hpopt.Status.READY: + config["config"] = self.get_real_config(self.optimizer.suggest(self.uf)) def get_num_executed_epochs(self): num_executed_epochs = 0 - for trial_id, config_item in enumerate(self.hpo_status['config_list']): + for trial_id, config_item in enumerate(self.hpo_status["config_list"]): trial_file_path = hpopt.get_trial_path(self.save_path, trial_id) trial_results = hpopt.load_json(trial_file_path) if trial_results is not None: - scores = trial_results.get('scores', None) + scores = trial_results.get("scores", None) if scores is not None: num_executed_epochs += len(scores) return num_executed_epochs - def auto_config(self, - expected_time_ratio: Union[int, float], - num_full_epochs: int, - full_dataset_size: Optional[int], - subset_ratio: Optional[Union[float, int]], - non_pure_train_ratio: float, - num_hyperparams: int, - reduction_factor: int, - parallelism: int, - min_epochs: Optional[int], - max_epochs: Optional[int]): + def auto_config( + self, + expected_time_ratio: Union[int, float], + num_full_epochs: int, + full_dataset_size: Optional[int], + subset_ratio: Optional[Union[float, int]], + non_pure_train_ratio: float, + num_hyperparams: int, + reduction_factor: int, + parallelism: int, + min_epochs: Optional[int], + max_epochs: Optional[int], + ): # All arguments should be specified. if expected_time_ratio is None: raise ValueError("expected_time_ratio should be specified.") @@ -476,10 +554,10 @@ def auto_config(self, # 1. Decide the max epochs # Check if there are early stopped trials # if True: - # set the max epochs to the min. epochs of them + # set the max epochs to the min. epochs of them # else - # Check the max. epochs in the results - # set the max epochs to the next rung schedule larger than the number + # Check the max. epochs in the results + # set the max epochs to the next rung schedule larger than the number # 1. Config target parameters as large as possible if max_epochs is None: @@ -496,7 +574,9 @@ def auto_config(self, subset_ratio = 0.2 if full_dataset_size == 0: - logger.warning("Sub-dataset isn't used because full_dataset_size value is 0.") + logger.warning( + "Sub-dataset isn't used because full_dataset_size value is 0." + ) subset_ratio = 1.0 elif (full_dataset_size * subset_ratio) < self.min_subset_size: if full_dataset_size > self.min_subset_size: @@ -513,12 +593,12 @@ def auto_config(self, # Update num_full_epochs from previous trials max_epochs_in_trials = 0 - if self.hpo_status.get('config_list', None) is not None: - for trial_id, config_item in enumerate(self.hpo_status['config_list']): + if self.hpo_status.get("config_list", None) is not None: + for trial_id, config_item in enumerate(self.hpo_status["config_list"]): trial_file_path = hpopt.get_trial_path(self.save_path, trial_id) trial_results = hpopt.load_json(trial_file_path) if trial_results is not None: - scores = trial_results.get('scores', None) + scores = trial_results.get("scores", None) if scores is not None: if len(scores) > max_epochs_in_trials: max_epochs_in_trials = len(scores) @@ -547,27 +627,47 @@ def auto_config(self, while current_time_ratio < expected_time_ratio: num_trials = num_trials + 1 - num_total_epochs = self.get_total_epochs(num_trials, reduction_factor, - num_brackets, new_min_epochs, new_max_epochs) - - current_time_ratio = num_total_epochs / num_full_epochs / parallelism * \ - ((1 - non_pure_train_ratio) * subset_ratio + non_pure_train_ratio) + num_total_epochs = self.get_total_epochs( + num_trials, + reduction_factor, + num_brackets, + new_min_epochs, + new_max_epochs, + ) + + current_time_ratio = ( + num_total_epochs + / num_full_epochs + / parallelism + * ((1 - non_pure_train_ratio) * subset_ratio + non_pure_train_ratio) + ) # If the remained number of epochs is less than the max_epochs_in_trials, # HPO is terminated. # num_total_epochs = num_total_epochs - max_epochs_in_trials - return num_trials, max_epochs, subset_ratio, num_total_epochs, num_brackets, min_epochs - - def get_total_epochs(self, - num_trials: int, - reduction_factor: int, - num_brackets: int, - min_epochs: int, - max_epochs: int): + return ( + num_trials, + max_epochs, + subset_ratio, + num_total_epochs, + num_brackets, + min_epochs, + ) + + def get_total_epochs( + self, + num_trials: int, + reduction_factor: int, + num_brackets: int, + min_epochs: int, + max_epochs: int, + ): num_total_epochs = 0 - num_trials_in_brackets = self.get_num_trials_in_brackets(reduction_factor, num_brackets) + num_trials_in_brackets = self.get_num_trials_in_brackets( + reduction_factor, num_brackets + ) brackets_total = sum(num_trials_in_brackets) brackets_ratio = [float(b / brackets_total) for b in num_trials_in_brackets] @@ -575,27 +675,33 @@ def get_total_epochs(self, for i in range(len(num_trials_in_brackets)): num_trials_in_brackets[i] = int(brackets_ratio[i] * num_trials) - num_trials_in_brackets[0] += (num_trials - sum(num_trials_in_brackets)) + num_trials_in_brackets[0] += num_trials - sum(num_trials_in_brackets) for s, num_trials in enumerate(num_trials_in_brackets): rungs = self.get_rungs(min_epochs, max_epochs, reduction_factor, s) remained_trials = num_trials for rung in reversed(rungs): - num_stop_trials = remained_trials - (remained_trials // reduction_factor) - num_total_epochs += (num_stop_trials * rung) + num_stop_trials = remained_trials - ( + remained_trials // reduction_factor + ) + num_total_epochs += num_stop_trials * rung remained_trials = remained_trials // reduction_factor - num_total_epochs += (remained_trials * max_epochs) + num_total_epochs += remained_trials * max_epochs return num_total_epochs def get_progress(self): # epoch based progress - epoch_progress = min(self.get_num_executed_epochs() - / self.expected_total_epochs, 0.99) + epoch_progress = min( + self.get_num_executed_epochs() / self.expected_total_epochs, 0.99 + ) # trial based progress - finished_trials = sum([val['status'] == hpopt.Status.STOP - for val in self.hpo_status['config_list']]) + finished_trials = sum( + [ + val["status"] == hpopt.Status.STOP + for val in self.hpo_status["config_list"] + ] + ) trial_progress = finished_trials / self.num_trials return min(0.99, max(epoch_progress, trial_progress)) - diff --git a/hpopt/base.py b/hpopt/base.py index 1ee6574..c9758df 100644 --- a/hpopt/base.py +++ b/hpopt/base.py @@ -1,18 +1,145 @@ # Copyright (C) 2018-2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # +""" implementation of the base module +""" import math # import logging import time +from enum import IntEnum from typing import List, Optional, Union -from hpopt.logger import get_logger +from .logger import get_logger +from .utils import type_check logger = get_logger() +class Status(IntEnum): + """ + HPO status enum + """ + + UNKNOWN = -1 + READY = 0 + RUNNING = 1 + STOP = 2 + CUDAOOM = 3 + # Previous HPO task is not found. + NORESULT = 4 + # Previous HPO task is started but not finished yet. + PARTIALRESULT = 5 + # Previous HPO task is completely finished. It means that the best hyper-parameters are already found.`` + COMPLETERESULT = 6 + + +class SearchSpace: + """ + This implements search space used for SMBO and ASHA. + This class support uniform and quantized uniform with normal and log scale + in addition to categorical type. Quantized type has step which is unit for change. + + Args: + type (str): type of hyper parameter in search space. + supported types: uniform, loguniform, quniform, qloguniform, choice + range (list): range of hyper parameter search space. + What value at each position means is as bellow. + uniform: [lower space, upper space] + quniform: [lower space, upper space, step] + loguniform: [lower space, upper space, logarithm base] + qloguniform: [lower space, upper space, step, logarithm base] + categorical: [each categorical values, ...] + """ + + def __init__(self, _type: str, _range: List[Union[float, int]]): + self._type = _type + self._range = _range + + if self._type in ("uniform", "loguniform"): + if len(self._range) == 2: + self._range.append(2) + elif len(self._range) < 2: + raise ValueError( + f"The range of the {self._type} type requires " + "two numbers for lower and upper limits. " + f"Your value is {self._range}" + ) + elif self._type in ("quniform", "qloguniform"): + if len(self._range) == 3: + self._range.append(2) + elif len(self._range) < 3: + raise ValueError( + f"The range of the {self._type} type requires " + "three numbers for lower/upper limits and " + "quantization number. " + f"Your value is {self._range}" + ) + elif self._type == "choice": + self._range = [0, len(_range)] + self.choice_list = _range + else: + raise TypeError(f"{self._type} is an unknown search space type.") + + def __repr__(self): + return f"type: {self._type}, range: {self._range}" + + def lower_space(self): + """get lower bound from range""" + if self._type == "loguniform": + return math.log(self._range[0], self._range[2]) + if self._type == "qloguniform": + return math.log(self._range[0], self._range[3]) + + return self._range[0] + + def upper_space(self): + """get upper bound from range""" + if self._type == "loguniform": + return math.log(self._range[1], self._range[2]) + if self._type == "qloguniform": + return math.log(self._range[1], self._range[3]) + + return self._range[1] + + def space_to_real(self, number: Union[int, float]): + """convert method""" + if self._type == "quniform": + return round(number / self._range[2]) * self._range[2] + if self._type == "loguniform": + return self._range[2] ** number + if self._type == "qloguniform": + return round(self._range[3] ** number / self._range[2]) * self._range[2] + if self._type == "choice": + idx = int(number) + idx = min(idx, len(self.choice_list) - 1) + idx = max(idx, 0) + return self.choice_list[idx] + + return number + + def real_to_space(self, number: Union[int, float]): + """convert method""" + if self._type == "loguniform": + return math.log(number, self._range[2]) + if self._type == "qloguniform": + return math.log(number, self._range[3]) + + return number + + @property + def type(self): + """property method for the internal attribute""" + return self._type + + @property + def range(self): + """property method for the internal attribute""" + return self._range + + +# pylint: disable=too-many-instance-attributes class HpOpt: """ This implements class which make frame for bayesian optimization @@ -21,7 +148,7 @@ class HpOpt: Args: save_path (str): path where result of HPO is saved. - search_space (list): hyper parameter search space to find. + search_space (dict): hyper parameter search space to find. mode (str): One of {min, max}. Determines whether objective is minimizing or maximizing the metric attribute. num_init_trials (int): Only for SMBO. How many trials to use to init SMBO. @@ -49,9 +176,10 @@ class HpOpt: If HPO stopped in middle, you can resume in middle. """ + # pylint: disable=too-many-arguments, too-many-locals, too-many-branches, too-many-statements def __init__( self, - search_space: List, + search_space: dict, save_path: str = "/tmp/hpopt", mode: str = "max", num_init_trials: int = 5, @@ -61,101 +189,118 @@ def __init__( non_pure_train_ratio: float = 0.2, full_dataset_size: int = 0, metric: str = "mAP", - expected_time_ratio: Union[int, float] = 4, + expected_time_ratio: Union[float, int] = 4, max_iterations: Optional[int] = None, max_time=None, resources_per_trial=None, subset_ratio: Optional[Union[float, int]] = None, - min_subset_size=500, - image_resize: List[int] = [0, 0], - batch_size_name=None, + min_subset_size: int = 500, + image_resize: List[int] = None, + batch_size_name: Optional[str] = None, verbose: int = 0, resume: bool = False, ): + if save_path is None: + raise TypeError("save_path should be str or path object, Not NoneType") if mode not in ["min", "max"]: raise ValueError("'mode' should be one of 'min' or 'max'.") - if type(expected_time_ratio) != float and type(expected_time_ratio) != int: - TypeError("expected_time_ratio should be float or int type.") - elif expected_time_ratio <= 0: - ValueError( - "expected_time_ratio should be bigger than 0." - f" Your value is {expected_time_ratio}" - ) - - if type(full_dataset_size) != int: - TypeError("full_dataset_size should be int type.") - elif full_dataset_size < 0: - ValueError( - "full_dataset_size should be postive value" - f"Your value is {full_dataset_size}." - ) - - if type(num_full_iterations) != int: - TypeError("num_full_iteration should be int type.") - elif num_full_iterations < 1: - raise ValueError( - "num_full_iterations should be 1 <=." - f" Your value is {num_full_iterations}" - ) - - if type(non_pure_train_ratio) != float: - TypeError("non_pure_train_ratio should be int type.") - elif not (0 < non_pure_train_ratio < 1): - raise ValueError( - "non_pure_train_ratio should be between 0 and 1." - f" Your value is {non_pure_train_ratio}" - ) - - if type(num_init_trials) != int: - raise TypeError("num_init_trials should be int type") - elif num_init_trials < 1: - raise ValueError( - "num_init_trials should be bigger than 0." - f" Your value is {num_init_trials}" - ) - - if max_iterations is not None: - if type(max_iterations) != int: - raise TypeError("max_iterations should be int type") - elif max_iterations < 1: - raise ValueError( - "max_iterations should be bigger than 0." - f" Your value is {max_iterations}" - ) - - if num_trials is not None: - if type(num_trials) != int: - raise TypeError("num_trials should be int type") - elif num_trials < 1: - raise ValueError( - "num_trials should be bigger than 0." f" Your value is {num_trials}" - ) - - if type(num_workers) != int: - raise TypeError("num_workers should be int type") - elif num_workers < 1: - raise ValueError( - "num_workers should be bigger than 0." f" Your value is {num_workers}" - ) - + type_check(search_space, [dict]) + + # if not isinstance(expected_time_ratio, float) and not isinstance( + # expected_time_ratio, int + # ): + # raise TypeError("expected_time_ratio should be float or int type.") + # if expected_time_ratio <= 0: + # raise ValueError( + # "expected_time_ratio should be bigger than 0." + # f" Your value is {expected_time_ratio}" + # ) + type_check(expected_time_ratio, [float, int], positive=True, non_zero=True) + + # if not isinstance(full_dataset_size, int): + # raise TypeError("full_dataset_size should be int type.") + # if full_dataset_size < 0: + # raise ValueError( + # "full_dataset_size should be zero or postive value" + # f"Your value is {full_dataset_size}." + # ) + type_check(full_dataset_size, int, positive=True) + + # if not isinstance(num_full_iterations, int): + # raise TypeError("num_full_iteration should be int type.") + # if num_full_iterations < 1: + # raise ValueError( + # "num_full_iterations should be 1 <=." + # f" Your value is {num_full_iterations}" + # ) + type_check(num_full_iterations, int, positive=True, non_zero=True) + + # if not isinstance(non_pure_train_ratio, float): + # raise TypeError("non_pure_train_ratio should be int type.") + type_check(non_pure_train_ratio, float) + if 0 > non_pure_train_ratio or 1 < non_pure_train_ratio: + raise ValueError("non_pure_train_ratio should be between 0 and 1." f" Your value is {non_pure_train_ratio}") + + # if not isinstance(num_init_trials, int): + # raise TypeError("num_init_trials should be int type") + # if num_init_trials < 1: + # raise ValueError( + # "num_init_trials should be bigger than 0." + # f" Your value is {num_init_trials}" + # ) + type_check(num_init_trials, int, positive=True, non_zero=True) + + # if max_iterations is not None: + # if not isinstance(max_iterations, int): + # raise TypeError("max_iterations should be int type") + # if max_iterations < 1: + # raise ValueError( + # "max_iterations should be bigger than 0." + # f" Your value is {max_iterations}" + # ) + type_check(max_iterations, [int, type(None)], positive=True, non_zero=True) + + # if num_trials is not None: + # if not isinstance(num_trials, int): + # raise TypeError("num_trials should be int type") + # if num_trials < 1: + # raise ValueError( + # "num_trials should be bigger than 0." f" Your value is {num_trials}" + # ) + type_check(num_trials, [int, type(None)], positive=True, non_zero=True) + + # if not isinstance(num_workers, int): + # raise TypeError("num_workers should be int type") + # if num_workers < 1: + # raise ValueError( + # "num_workers should be bigger than 0." f" Your value is {num_workers}" + # ) + type_check(num_workers, int, positive=True, non_zero=True) + + type_check(subset_ratio, [float, int, type(None)]) if subset_ratio is not None: - if type(subset_ratio) != float and type(subset_ratio) != int: - raise TypeError("subset_ratio should be float or int type") - elif not (0 < subset_ratio <= 1.0): - raise ValueError( - "subset_ratio should be > 0 and <= 1." - f" Your value is {subset_ratio}" - ) + # if not isinstance(subset_ratio, float) and not isinstance( + # subset_ratio, int + # ): + # raise TypeError("subset_ratio should be float or int type") + if 0 >= subset_ratio or 1.0 < subset_ratio: + raise ValueError("subset_ratio should be > 0 and <= 1." f" Your value is {subset_ratio}") + + if image_resize is None: + image_resize = [0, 0] if not hasattr(image_resize, "__getitem__"): raise TypeError("image_resize should be able to accessible by index.") - elif len(image_resize) < 2: + if len(image_resize) < 2: raise ValueError("image_resize should have at least two values.") - elif image_resize[0] < 0 or image_resize[1] < 0: + if image_resize[0] < 0 or image_resize[1] < 0: raise ValueError("Each value of image_resize should be positive.") + if not isinstance(min_subset_size, int): + raise TypeError("min_subset_size should be an integer value.") + self.save_path = save_path self.search_space = search_space self.mode = mode @@ -174,26 +319,31 @@ def __init__( self.image_resize = image_resize self.verbose = verbose self.resume = resume - self.hpo_status: dict = {} self.metric = metric self.batch_size_name = batch_size_name + self.hpo_status: dict = {} def get_next_sample(self): - pass + """api definition to get next sample""" def get_next_samples(self, num_expected_samples=0): - pass + """api definition to get next sample""" def update_scores(self): - pass + """api definition to update scores""" def save_results(self): - pass + """api definition to save results""" - def obj(self, **kwargs): + # pylint: disable=unused-argument + @staticmethod + def obj(**kwargs): + """definition of the empty object method for bayes opt""" return 0 - def hasCategoricalParam(self, search_space): + @staticmethod + def has_categorical_param(search_space): + """helper function to check whether search space config has categorical hp.""" for param in search_space: if search_space[param].type == "choice": return True @@ -201,18 +351,21 @@ def hasCategoricalParam(self, search_space): return False def get_real_config(self, config): + """convert space to specific value""" real_config = {} for param in config: real_config[param] = self.search_space[param].space_to_real(config[param]) return real_config def get_space_config(self, config): + """convert specific value to space""" space_config = {} for param in config: space_config[param] = self.search_space[param].real_to_space(config[param]) return space_config def print_results(self): + """helper function to print out the overall HPO report""" field_widths = [] field_param_name = [] print(f'|{"#": ^5}|', end="") @@ -235,6 +388,7 @@ def print_results(self): print("") def check_duplicated_config(self, new_config): + """helper function to verify the config to avoid redundancy of the trial config""" for old_item in self.hpo_status["config_list"]: old_config = old_item["config"] matched = True @@ -252,6 +406,7 @@ def check_duplicated_config(self, new_config): return False def get_best_config(self): + """api to get the best config from the HPO status""" self.update_scores() # Wait for updating files up to 5 seconds. @@ -302,4 +457,4 @@ def get_best_config(self): return self.hpo_status["config_list"][best_trial_id]["config"] def get_progress(self): - pass + """api to get current progress of the HPO process""" diff --git a/hpopt/dummy.py b/hpopt/dummy.py.bak similarity index 100% rename from hpopt/dummy.py rename to hpopt/dummy.py.bak diff --git a/hpopt/hpopt.py b/hpopt/hpopt.py index 1779cb4..2865db1 100644 --- a/hpopt/hpopt.py +++ b/hpopt/hpopt.py @@ -1,129 +1,27 @@ # Copyright (C) 2018-2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # +""" main module of the hpopt +""" import glob import json -import math import os import time -from enum import IntEnum from statistics import median from typing import Any, Dict, List, Optional, Union -import numpy as np -import torch -from torch.utils.data import random_split -from torchvision import transforms - -import hpopt -from hpopt.dummy import DummyOpt -from hpopt.hyperband import AsyncHyperBand -from hpopt.logger import get_logger -from hpopt.smbo import BayesOpt +# import hpopt +from .base import SearchSpace, Status +from .hyperband import AsyncHyperBand +from .logger import get_logger +from .smbo import BayesOpt +from .utils import dump_as_json logger = get_logger() -class SearchSpace: - """ - This implements search space used for SMBO and ASHA. - This class support uniform and quantized uniform with normal and log scale - in addition to categorical type. Quantized type has step which is unit for change. - - Args: - type (str): type of hyper parameter in search space. - supported types: uniform, loguniform, quniform, qloguniform, choice - range (list): range of hyper parameter search space. - What value at each position means is as bellow. - uniform: [lower space, upper space] - quniform: [lower space, upper space, step] - loguniform: [lower space, upper space, logarithm base] - qloguniform: [lower space, upper space, step, logarithm base] - categorical: [each categorical values, ...] - """ - - def __init__(self, type: str, range: List[Union[float, int]]): - self.type = type - self.range = range - - if self.type == "uniform" or self.type == "loguniform": - if len(self.range) == 2: - range.append(2) - elif len(self.range) < 2: - raise ValueError( - f"The range of the {self.type} type requires " - "two numbers for lower and upper limits. " - f"Your value is {self.range}" - ) - elif self.type == "quniform" or self.type == "qloguniform": - if len(self.range) == 3: - range.append(2) - elif len(self.range) < 3: - raise ValueError( - f"The range of the {self.type} type requires " - "three numbers for lower/upper limits and " - "quantization number. " - f"Your value is {self.range}" - ) - elif self.type == "choice": - self.range = [0, len(range)] - self.choice_list = range - else: - raise TypeError(f"{self.type} is an unknown search space type.") - - def __repr__(self): - return f"type: {self.type}, range: {self.range}" - - def lower_space(self): - if self.type == "loguniform": - return math.log(self.range[0], self.range[2]) - elif self.type == "qloguniform": - return math.log(self.range[0], self.range[3]) - - return self.range[0] - - def upper_space(self): - if self.type == "loguniform": - return math.log(self.range[1], self.range[2]) - elif self.type == "qloguniform": - return math.log(self.range[1], self.range[3]) - - return self.range[1] - - def space_to_real(self, number: Union[int, float]): - if self.type == "quniform": - return round(number / self.range[2]) * self.range[2] - elif self.type == "loguniform": - return self.range[2] ** number - elif self.type == "qloguniform": - return round(self.range[3] ** number / self.range[2]) * self.range[2] - elif self.type == "choice": - idx = int(number) - idx = min(idx, len(self.choice_list) - 1) - idx = max(idx, 0) - return self.choice_list[idx] - - return number - - def real_to_space(self, number: Union[int, float]): - if self.type == "loguniform": - return math.log(number, self.range[2]) - elif self.type == "qloguniform": - return math.log(number, self.range[3]) - - return number - - -def createDummyOpt( - search_alg=None, save_path=None, search_space=None, resume=False, **kwargs -): - if save_path is None: - return None - - return DummyOpt(save_path=save_path, search_space=search_space, resume=resume) - - +# pylint: disable=too-many-arguments, too-many-locals def create( full_dataset_size: int, num_full_iterations: int, @@ -140,7 +38,7 @@ def create( num_brackets: Optional[int] = None, subset_ratio: Optional[Union[float, int]] = None, batch_size_name: str = None, - image_resize: List[int] = [0, 0], + image_resize: List[int] = None, metric: str = "mAP", resume: bool = False, expected_time_ratio: Union[int, float] = 4, @@ -229,7 +127,7 @@ def create( kappa_decay_delay=kappa_decay_delay, default_hyper_parameters=default_hyper_parameters, ) - elif search_alg == "asha": + if search_alg == "asha": return AsyncHyperBand( save_path=save_path, search_space=search_space, @@ -252,8 +150,7 @@ def create( default_hyper_parameters=default_hyper_parameters, use_epoch=use_epoch, ) - else: - raise ValueError(f"Not supported search algorithm: {search_alg}") + raise ValueError(f"Not supported search algorithm: {search_alg}") def get_status_path(save_path: str): @@ -291,16 +188,10 @@ def clear_trial_files(save_path: str): os.remove(trial_file_path) except OSError: logger.error(f"Error while deleting file : {trial_file_path}") - return None + return + return -# hpopt.Status.COMPLETERESULT - Previous HPO task is completely finished. -# It means that the best hyper-parameters are already found. -# -# hpopt.Status.PARTIALRESULT - Previous HPO task is started but not finished yet. -# -# hpopt.Status.NORESULT - Previous HPO task is not found. -# def get_previous_status(save_path: str): """ Check if there is previous results. @@ -312,16 +203,15 @@ def get_previous_status(save_path: str): if os.path.exists(status_file_path): best_config_id = None - with open(status_file_path, "rt") as json_file: + with open(status_file_path, "rt", encoding="utf-8") as json_file: hpo_status = json.load(json_file) best_config_id = hpo_status.get("best_config_id", None) if best_config_id is not None: - return hpopt.Status.COMPLETERESULT - else: - return hpopt.Status.PARTIALRESULT - else: - return hpopt.Status.NORESULT + return Status.COMPLETERESULT + return Status.PARTIALRESULT + + return Status.NORESULT def get_current_status(save_path: str, trial_id: int): @@ -332,13 +222,22 @@ def get_current_status(save_path: str, trial_id: int): save_path (str): path where result of HPO is saved. tiral_id (int): order of HPO trial. """ - trial_file_path = get_trial_path(save_path, trial_id) - trial_results = load_json(trial_file_path) + trial_results = get_trial_results(save_path, trial_id) if trial_results is not None: return trial_results["status"] - else: - return hpopt.Status.UNKNOWN + return Status.UNKNOWN + + +def get_trial_results(save_path: str, trial_id: int): + """ + Return results of HPO trial. + + Args: + save_path (str): path where result of HPO is saved. + tiral_id (int): order of HPO trial. + """ + return load_json(get_trial_path(save_path, trial_id)) def get_best_score(save_path: str, trial_id: int, mode: str): @@ -350,15 +249,13 @@ def get_best_score(save_path: str, trial_id: int, mode: str): tiral_id (int): order of HPO trial. mode (str): max or min. Decide whether to find max value or min value. """ - trial_file_path = get_trial_path(save_path, trial_id) - trial_results = load_json(trial_file_path) + trial_results = get_trial_results(save_path, trial_id) if trial_results is not None: if trial_results["status"] == Status.STOP: if mode == "min": return min(trial_results["scores"]) - else: - return max(trial_results["scores"]) + return max(trial_results["scores"]) return None @@ -406,21 +303,17 @@ def finalize_trial(config: Dict[str, Any]): # Check if the trial is terminated by the EarlyStoppingHook scores_list = trial_results["scores"] if len(scores_list) > 5: - earlyStopped = True + early_stopped = True for i in range(len(scores_list) - 1, len(scores_list) - 6, -1): if scores_list[i] > scores_list[i - 1]: - earlyStopped = False + early_stopped = False break - if earlyStopped: - logger.debug("This trial is earlystopped") + if early_stopped: + logger.debug("This trial is early stopped") trial_results["early_stopped"] = 1 - oldmask = os.umask(0o077) - with open(config["file_path"], "wt") as json_file: - json.dump(trial_results, json_file, indent=4) - json_file.flush() - os.umask(oldmask) + dump_as_json(config["file_path"], trial_results) def get_median_score(save_path: str, trial_id: int, curr_iteration: int): @@ -442,156 +335,25 @@ def get_median_score(save_path: str, trial_id: int, curr_iteration: int): if f != f"hpopt_trial_{trial_id}.json": trial_file_path = os.path.join(save_path, f) try: - with open(trial_file_path, "rt") as json_file: + with open(trial_file_path, "rt", encoding="utf-8") as json_file: trial_results = json.load(json_file) if len(trial_results["median"]) >= curr_iteration: - median_score_list.append( - trial_results["median"][curr_iteration - 1] - ) + median_score_list.append(trial_results["median"][curr_iteration - 1]) except FileNotFoundError: continue if len(median_score_list) == 0: return None - else: - return median(median_score_list) - - -def get_cutoff_score(save_path: str, target_rung: int, _rung_list, mode: str): - """ - Calculate a cutoff score for a specified rung of ASHA - - Args: - save_path (str): path where result of HPO is saved. - target_rung (int): current rung number - _rung_list (list): list of rungs in the current bracket - mode (str): max or min. Decide whether to find max value or min value. - """ - logger.debug(f"get_cutoff_score({target_rung}, {_rung_list}, {mode})") - status_file_path = get_status_path(save_path) - if not os.path.exists(status_file_path): - logger.warning(f"not existed status json file {status_file_path}") - return None - - hpo_status = load_json(status_file_path) - - if hpo_status is None: - logger.warning(f"failed to load json file {status_file_path}") - return None + return median(median_score_list) - # Gather all scores with iter number - hpo_trial_results_scores = [] - hpo_trial_results_imgs_num = [] - - for idx, config in enumerate(hpo_status["config_list"]): - if config["status"] in [hpopt.Status.RUNNING, hpopt.Status.STOP]: - trial_file_path = hpopt.get_trial_path(save_path, idx) - trial_results = load_json(trial_file_path) - - if trial_results is not None: - hpo_trial_results_scores.append(trial_results["scores"]) - hpo_trial_results_imgs_num.append(trial_results["images"]) - if len(hpo_trial_results_scores) != len(hpo_trial_results_imgs_num): - raise RuntimeError( - f"mismatch length of trial results and number of trained images {hpo_trial_results_scores}" - f"/{hpo_trial_results_imgs_num}" - ) - logger.debug(f"scores = {hpo_trial_results_scores}") - logger.debug(f"num_imgs = {hpo_trial_results_imgs_num}") - # Run a SHA (not ASHA) - rung_score_list = [] - rung_list = _rung_list.copy() - rung_list.sort(reverse=False) - logger.debug(f"rung_list = {rung_list}") - if len(hpo_trial_results_scores) > 1: - rf = hpo_status["reduction_factor"] - - # for curr_rung_idx, curr_rung in enumerate(rung_list): - for curr_rung in rung_list: - if curr_rung <= target_rung: - rung_score_list.clear() - logger.debug(f"curr_rung = {curr_rung}") - for trial, (scores, num_imgs) in enumerate( - list(zip(hpo_trial_results_scores, hpo_trial_results_imgs_num)) - ): - if num_imgs != []: - logger.debug(f"trial{trial} score reported @ {num_imgs}") - if num_imgs[-1] > curr_rung: - result_idx = -1 - for img_idx, img in enumerate(num_imgs): - if img > curr_rung: - break - result_idx = img_idx - if result_idx != -1: - if result_idx == 0: - rung_score_list.append(scores[result_idx]) - else: - if mode == "max": - rung_score_list.append(max(scores[:result_idx])) - else: - rung_score_list.append(min(scores[:result_idx])) - else: - logger.debug( - f"cannot find matched result index ({curr_rung}) in {trial} : {num_imgs}. removed" - ) - scores.clear() - num_imgs.clear() - - if len(rung_score_list) > 1: - logger.debug(f"rung_score_list = {rung_score_list}") - if mode == "max": - cutt_off_score = np.nanpercentile( - rung_score_list, (1 - 1 / rf) * 100 - ) - else: - cutt_off_score = np.nanpercentile( - rung_score_list, (1 / rf) * 100 - ) - logger.debug(f"cut_off_score = {cutt_off_score}") - for scores, num_imgs in list( - zip(hpo_trial_results_scores, hpo_trial_results_imgs_num) - ): - if num_imgs != []: - # if num_imgs[-1] > curr_rung: - result_idx = -1 - for img_idx, img in enumerate(num_imgs): - if img > curr_rung: - break - result_idx = img_idx - if result_idx != -1: - if result_idx == 0: - target_score = scores[result_idx] - else: - target_score = ( - max(scores[:result_idx]) - if mode == "max" - else min(scores[:result_idx]) - ) - # logger.debug(f"target score = {target_score}") - if mode == "max": - if target_score < cutt_off_score: - scores.clear() - num_imgs.clear() - else: - if target_score > cutt_off_score: - scores.clear() - num_imgs.clear() - # logger.debug(f"rung_score_list = {rung_score_list}") - if len(rung_score_list) > 1: - logger.debug(f"last rung_score_list = {rung_score_list}") - rf = hpo_status["reduction_factor"] - - if mode == "max": - ret = np.nanpercentile(rung_score_list, (1 - 1 / rf) * 100) - logger.debug(f"return {ret}") - return ret - else: - ret = np.nanpercentile(rung_score_list, (1 / rf) * 100) - logger.debug(f"return {ret}") - return ret - return None +def _get_trial_results(file_path: str): + trial_results = dict(status=Status.RUNNING, scores=[], median=[], images=[]) + if os.path.exists(file_path): + with open(file_path, "rt", encoding="utf-8") as json_file: + trial_results = json.load(json_file) + return trial_results def report(config: Dict[str, Any], score: float, current_iters: Optional[int] = -1): @@ -605,46 +367,39 @@ def report(config: Dict[str, Any], score: float, current_iters: Optional[int] = score (float): score of every epoch during trial. current_iters (int): current iteration number when the given score was generated. """ - logger.debug(f"report({config})") - if os.path.exists(config["file_path"]): - with open(config["file_path"], "rt") as json_file: - trial_results = json.load(json_file) - else: - trial_results = {} - trial_results["status"] = Status.RUNNING - trial_results["scores"] = [] - trial_results["median"] = [] - trial_results["images"] = [] + logger.debug(f"report({config}, score = {score}, current_iters={current_iters})") + # if os.path.exists(config["file_path"]): + # with open(config["file_path"], "rt", encoding="utf-8") as json_file: + # trial_results = json.load(json_file) + # else: + # trial_results = {} + # trial_results["status"] = Status.RUNNING + # trial_results["scores"] = [] + # trial_results["median"] = [] + # trial_results["images"] = [] + trial_results = _get_trial_results(config["file_path"]) trial_results["scores"].append(score) - trial_results["median"].append( - sum(trial_results["scores"]) / len(trial_results["scores"]) - ) + trial_results["median"].append(sum(trial_results["scores"]) / len(trial_results["scores"])) - batch_size = config["params"].get(config["batch_size_param_name"]) - logger.debug(f"report() batch size of this trial = {batch_size}") + batch_size = config["params"].get(config["batch_size_param_name"]) if "batch_size_param_name" in config else None if batch_size is None: batch_size = config.get("batch_size") if batch_size is None: - raise RuntimeError("cannot find batch size from config or h-params") + # raise RuntimeError("cannot find batch size from config or h-params") + logger.warning("cannot find batch_size information. set it to 1.") + batch_size = 1 trial_results["images"].append(current_iters * batch_size) # Update the current status ASAP in the file system. - oldmask = os.umask(0o077) - with open(config["file_path"], "wt") as json_file: - json.dump(trial_results, json_file, indent=4) - json_file.flush() - os.umask(oldmask) - - # if len(trial_results['scores']) >= config["iterations"]: - if trial_results["images"][-1] >= config["iteration_limit"]: + dump_as_json(config["file_path"], trial_results) + + if len(trial_results["scores"]) >= config["iterations"] and "rungs" not in config: trial_results["status"] = Status.STOP elif "early_stop" in config and config["early_stop"] == "median_stop": save_path = os.path.dirname(config["file_path"]) - median_score = get_median_score( - save_path, config["trial_id"], len(trial_results["scores"]) - ) + median_score = get_median_score(save_path, config["trial_id"], len(trial_results["scores"])) if median_score is not None: if config["mode"] == "min": @@ -667,47 +422,11 @@ def report(config: Dict[str, Any], score: float, current_iters: Optional[int] = f"median stop is executed. median score : {median_score} / " f"current best score : {curr_best_score}" ) - elif "rungs" in config: - # Async HyperBand - save_path = os.path.dirname(config["file_path"]) - # curr_itr = len(trial_results['scores']) - curr_itr = trial_results["images"][-1] - logger.debug(f"current iterations = {curr_itr} : {config['rungs']}") - - for rung_iter_idx, rung_itr in enumerate(config["rungs"]): - if curr_itr >= rung_itr: - # Decide whether to promote to the next rung or not - cutoff_score = get_cutoff_score( - save_path, rung_itr, config["rungs"], config["mode"] - ) + # for hyperband + AsyncHyperBand.report(config, trial_results) - if cutoff_score is not None: - if config["mode"] == "min": - curr_best_score = min(trial_results["scores"]) - else: - curr_best_score = max(trial_results["scores"]) - - stop_flag = False - - if config["mode"] == "max" and cutoff_score > curr_best_score: - stop_flag = True - elif config["mode"] == "min" and cutoff_score < curr_best_score: - stop_flag = True - - if stop_flag: - trial_results["status"] = Status.STOP - logger.debug( - f"[ASHA STOP] [{config['trial_id']}, {curr_itr}, {rung_itr}] " - f"{cutoff_score} > {curr_best_score}" - ) - break - - oldmask = os.umask(0o077) - with open(config["file_path"], "wt") as json_file: - json.dump(trial_results, json_file, indent=4) - json_file.flush() - os.umask(oldmask) + dump_as_json(config["file_path"], trial_results) # Wait for flushing updated contents to the file system if trial_results["status"] == Status.STOP: @@ -716,6 +435,7 @@ def report(config: Dict[str, Any], score: float, current_iters: Optional[int] = return trial_results["status"] +# pylint: disable=invalid-name def reportOOM(config): """ report if trial raise out of cuda memory. @@ -731,11 +451,7 @@ def reportOOM(config): trial_results["median"] = [] trial_results["images"] = [] - oldmask = os.umask(0o077) - with open(config["file_path"], "wt") as json_file: - json.dump(trial_results, json_file, indent=4) - json_file.flush() - os.umask(oldmask) + dump_as_json(config["file_path"], trial_results) time.sleep(1) @@ -754,7 +470,7 @@ def load_json(json_file_path: str): contents = None while retry_flag: try: - with open(json_file_path, "rt") as json_file: + with open(json_file_path, "rt", encoding="utf-8") as json_file: contents = json.load(json_file) retry_flag = False except OSError as err: @@ -765,101 +481,3 @@ def load_json(json_file_path: str): time.sleep(0.1) return contents - - -class Status(IntEnum): - UNKNOWN = -1 - READY = 0 - RUNNING = 1 - STOP = 2 - CUDAOOM = 3 - NORESULT = 4 - PARTIALRESULT = 5 - COMPLETERESULT = 6 - - -class HpoDataset: - """ - Dataset class which wrap dataset class used in training for sub-sampling. - - Args: - fullset: dataset instance used in train. - config (dict): HPO configuration for a trial. - This include train confiuration(e.g. hyper parameter, epoch, etc.) - and tiral information. - """ - - def __init__(self, fullset, config: Dict[str, Any]): - self.__dict__ = fullset.__dict__.copy() - self.fullset = fullset - - if config["subset_ratio"] > 0.0: - if config["subset_ratio"] < 1.0: - subset_size = int(len(fullset) * config["subset_ratio"]) - self.subset, _ = random_split( - fullset, - [subset_size, (len(fullset) - subset_size)], - generator=torch.Generator().manual_seed(42), - ) - - # check if fullset is an inheritance of mmdet.datasets - if hasattr(self, "flag"): - self._update_group_flag() - else: - self.subset = fullset - self.length = len(self.subset) - else: - self.subset = fullset - self.length = len(fullset) - - if config["resize_height"] > 0 and config["resize_width"] > 0: - self.transform = transforms.Resize( - (config["resize_height"], config["resize_width"]), interpolation=2 - ) - else: - self.transform = None - - def _update_group_flag(self): - self.flag = np.zeros(len(self.subset), dtype=np.uint8) - - update_flag = False - if "img_metas" in self.subset[0]: - if "ori_shape" in self.subset[0]["img_metas"].data: - update_flag = True - - if not update_flag: - return - - for i in range(len(self.subset)): - self.flag[i] = self.fullset.flag[self.subset.indices[i]] - - def __getitem__(self, index: int): - data = self.subset[index] - if self.transform: - if type(data) is tuple and len(data) == 2 and type(data[0]) == torch.Tensor: - data = (self.transform(data[0]), data[1]) - elif type(data) is dict and "img" in data: - data["img"] = self.transform(data["img"]) - return data - - def __len__(self): - return self.length - - def __getattr__(self, item: str): - if isinstance(item, str) and (item == "__setstate__" or item == "__getstate__"): - raise AttributeError(item) - - return getattr(self.fullset, item) - - -def createHpoDataset(fullset, config: Dict[str, Any]): - """ - wrap original dataset by HpoDataset using hpo_config returend by Hpopt. - - Args: - fullset: dataset instance used in train. - config (dict): HPO configuration for a trial. - This include train confiuration(e.g. hyper parameter, epoch, etc.) - and tiral information. - """ - return HpoDataset(fullset, config) diff --git a/hpopt/hyperband.py b/hpopt/hyperband.py index 4c65cb9..f4a0cb9 100644 --- a/hpopt/hyperband.py +++ b/hpopt/hyperband.py @@ -1,21 +1,29 @@ # Copyright (C) 2018-2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # +"""implementation of the hyperband +""" + +# pylint: disable=too-many-lines import json import os from math import ceil, log from typing import Dict, List, Optional, Union +import numpy as np from bayes_opt import BayesianOptimization, UtilityFunction import hpopt -from hpopt.base import HpOpt -from hpopt.logger import get_logger + +from .base import HpOpt, Status +from .logger import get_logger +from .utils import dump_as_json logger = get_logger() +# pylint: disable=too-many-instance-attributes, too-many-branches, too-many-statements class AsyncHyperBand(HpOpt): """ This implements the Asyncronous HyperBand scheduler with iterations only. @@ -36,6 +44,7 @@ class AsyncHyperBand(HpOpt): halving rate, specified by the reduction factor. """ + # pylint: disable=too-many-statements def __init__( self, num_brackets: Optional[int] = None, @@ -45,48 +54,38 @@ def __init__( use_epoch: Optional[bool] = False, **kwargs, ): - super(AsyncHyperBand, self).__init__(**kwargs) + super().__init__(**kwargs) if min_iterations is not None: - if type(min_iterations) != int: + if not isinstance(min_iterations, int): raise TypeError("min_iterations should be int type") - elif min_iterations < 1: - raise ValueError( - f"min_iterations should be bigger than 1. Your value is {min_iterations}" - ) + if min_iterations < 1: + raise ValueError(f"min_iterations should be bigger than 1. Your value is {min_iterations}") - if type(reduction_factor) != int: + if not isinstance(reduction_factor, int): raise TypeError("reduction_factor should be int type") - elif reduction_factor < 1: - raise ValueError( - "reduction_factor should be bigger than 1." - f" Your value is {reduction_factor}" - ) + if reduction_factor < 1: + raise ValueError("reduction_factor should be bigger than 1." f" Your value is {reduction_factor}") if num_brackets is not None: - if type(num_brackets) != int: + if not isinstance(num_brackets, int): raise TypeError("num_brackets should be int type") - elif num_brackets < 1: - raise ValueError( - f"num_brackets should be bigger than 1. Your value is {num_brackets}" - ) + if num_brackets < 1: + raise ValueError(f"num_brackets should be bigger than 1. Your value is {num_brackets}") + + ( + self._min_iterations, + self._reduction_factor, + self._num_brackets, + self._expected_total_images, + self._use_epoch, + self._updatable_schedule, + ) = (min_iterations, reduction_factor, num_brackets, 0, use_epoch, False) - self._min_iterations = min_iterations - self._reduction_factor = reduction_factor - self._num_brackets = num_brackets - self._expected_total_images = 0 - self._use_epoch = use_epoch - self._updatable_schedule = False self._current_best = {"trial_id": -1, "score": 0, "image": 0} # HPO auto configurator - if ( - self.num_trials is None - or self.max_iterations is None - or self.subset_ratio is None - ): - logger.info( - "training schedule was not specified for HPO. run auto configuration." - ) + if self.num_trials is None or self.max_iterations is None or self.subset_ratio is None: + logger.info("training schedule was not specified for HPO. run auto configuration.") self._updatable_schedule = True ( self.num_trials, @@ -116,15 +115,12 @@ def __init__( f"n_imgs_for_full_train: {self.n_imgs_for_full_train}, " f"expected_total_images : {self._expected_total_images}, num_brackets: {self._num_brackets}" ) - else: if self.max_iterations is None: self.max_iterations = self.num_full_iterations if self._num_brackets is None: - self._num_brackets = ( - int(log(self.max_iterations) / log(self._reduction_factor)) + 1 - ) + self._num_brackets = int(log(self.max_iterations) / log(self._reduction_factor)) + 1 if self._min_iterations is None: self._min_iterations = 1 @@ -132,34 +128,66 @@ def __init__( if self.subset_ratio is None: self.subset_ratio = 1.0 - self.n_imgs_for_full_train = ( - self.full_dataset_size * self.subset_ratio * self.max_iterations - ) - self.n_imgs_for_min_train = ( - self.full_dataset_size * self.subset_ratio * self._min_iterations - ) + self.n_imgs_for_full_train = self.full_dataset_size * self.subset_ratio * self.max_iterations + self.n_imgs_for_min_train = self.full_dataset_size * self.subset_ratio * self._min_iterations if self._min_iterations > self.max_iterations: logger.info("min_epochs should be less than or equal to max_epochs.") self._min_iterations = self.max_iterations # Initialize a bayesopt optimizer. - # It will be used for generating trials. - self.bayesopt_space = {} - for param in self.search_space: - self.bayesopt_space[param] = ( - self.search_space[param].lower_space(), - self.search_space[param].upper_space(), + self._init_bayse_opt(self.search_space) + + # All information in self.hpo_status will be stored at hpo_file_path. + hpo_file_path = hpopt.get_status_path(self.save_path) + + if self.resume is True and os.path.exists(hpo_file_path): + with open(hpo_file_path, "rt", encoding="utf-8") as json_file: + self.hpo_status = json.load(json_file) + json_file.close() + self._verify_hpo_status() + else: + self.hpo_status["search_algorithm"] = "asha" + self.hpo_status["search_space"] = {ss: self.search_space[ss].__dict__ for ss in self.search_space} + self.hpo_status["metric"] = self.metric + self.hpo_status["subset_ratio"] = self.subset_ratio + self.hpo_status["image_resize"] = self.image_resize + self.hpo_status["full_dataset_size"] = self.full_dataset_size + self.hpo_status["reduction_factor"] = self._reduction_factor + self.hpo_status["config_list"] = [] + + # num_gen_config represents the number of assigned configs. + # If 1 configs are done and 2 configs are running, num_gen_config is 3. + self.hpo_status["num_gen_config"] = 0 + + # Initialize the brackets + n0_in_brackets, num_ready_configs = self._init_brackets(default_hyper_parameters) + + # Generate trial configs up to self.num_trials + if isinstance(self.num_trials, int): + if self.num_trials > num_ready_configs: + self._generate_configs(num_ready_configs, self.num_trials, n0_in_brackets) + + # Sync the current status to the file system. + self.save_results() + + def _init_bayse_opt(self, search_space): + """initialize bayes opt. it will be used for generating trials.""" + bayesopt_space = {} + for param in search_space: + bayesopt_space[param] = ( + search_space[param].lower_space(), + search_space[param].upper_space(), ) self.optimizer = BayesianOptimization( f=self.obj, - pbounds=self.bayesopt_space, + pbounds=bayesopt_space, verbose=self.verbose, random_state=None, ) - self.uf = UtilityFunction( + self.utilty_function = UtilityFunction( kind="ucb", kappa=2.5, xi=0.0, @@ -167,84 +195,61 @@ def __init__( kappa_decay_delay=self.num_init_trials, ) - if self.hasCategoricalParam(self.search_space): + if self.has_categorical_param(self.search_space): self.optimizer.set_gp_params(alpha=1e-3) - # All information in self.hpo_status will be stored at hpo_file_path. - hpo_file_path = hpopt.get_status_path(self.save_path) - - if self.resume is True and os.path.exists(hpo_file_path): - with open(hpo_file_path, "rt") as json_file: - self.hpo_status = json.load(json_file) - json_file.close() + def _verify_hpo_status(self): + """verify current hpo_status""" + if self.hpo_status["search_algorithm"] != "asha": + logger.error("Search algorithm is changed. Stop resuming.") + raise ValueError("Search algorithm is changed.") - if self.hpo_status["search_algorithm"] != "asha": - logger.error("Search algorithm is changed. Stop resuming.") - raise ValueError("Search algorithm is changed.") + if self.hpo_status["search_space"] != {ss: self.search_space[ss].__dict__ for ss in self.search_space}: + logger.error("Search space is changed. Stop resuming.") + raise ValueError("Search space is changed.") - if self.hpo_status["search_space"] != { - ss: self.search_space[ss].__dict__ for ss in self.search_space - }: - logger.error("Search space is changed. Stop resuming.") - raise ValueError("Search space is changed.") + if self.hpo_status["metric"] != self.metric: + logger.error("Metric is changed. Stop resuming.") + raise ValueError("Metric is changed.") - if self.hpo_status["metric"] != self.metric: - logger.error("Metric is changed. Stop resuming.") - raise ValueError("Metric is changed.") + if self.hpo_status["subset_ratio"] != self.subset_ratio: + logger.error("subset_ratio is changed. Stop resuming.") + raise ValueError("subset_ratio is changed.") - if self.hpo_status["subset_ratio"] != self.subset_ratio: - logger.error("subset_ratio is changed. Stop resuming.") - raise ValueError("subset_ratio is changed.") + if self.hpo_status["image_resize"] != self.image_resize: + logger.error("image_resize is changed. Stop resuming.") + raise ValueError("image_resize is changed.") - if self.hpo_status["image_resize"] != self.image_resize: - logger.error("image_resize is changed. Stop resuming.") - raise ValueError("image_resize is changed.") + if self.hpo_status["full_dataset_size"] != self.full_dataset_size: + logger.error("The size of dataset is changed. Stop resuming.") + raise ValueError("Dataset is changed.") - if self.hpo_status["full_dataset_size"] != self.full_dataset_size: - logger.error("The size of dataset is changed. Stop resuming.") - raise ValueError("Dataset is changed.") + if self.hpo_status["reduction_factor"] != self._reduction_factor: + logger.error("reduction_factor is changed. Stop resuming.") + raise ValueError("reduction_factor is changed.") - if self.hpo_status["reduction_factor"] != self._reduction_factor: - logger.error("reduction_factor is changed. Stop resuming.") - raise ValueError("reduction_factor is changed.") - else: - self.hpo_status["search_algorithm"] = "asha" - self.hpo_status["search_space"] = { - ss: self.search_space[ss].__dict__ for ss in self.search_space - } - self.hpo_status["metric"] = self.metric - self.hpo_status["subset_ratio"] = self.subset_ratio - self.hpo_status["image_resize"] = self.image_resize - self.hpo_status["full_dataset_size"] = self.full_dataset_size - self.hpo_status["reduction_factor"] = self._reduction_factor - self.hpo_status["config_list"] = [] - - # num_gen_config represents the number of assigned configs. - # If 1 configs are done and 2 configs are running, num_gen_config is 3. - self.hpo_status["num_gen_config"] = 0 - - # Resumes from the previous results. + # verify config_list for idx, config in enumerate(self.hpo_status["config_list"]): - if config["status"] == hpopt.Status.STOP: + if config["status"] == Status.STOP: # This case means that config is done. self.hpo_status["num_gen_config"] += 1 else: # This case means that config should be in the ready state. - config["status"] = hpopt.Status.READY + config["status"] = Status.READY trial_file_path = hpopt.get_trial_path(self.save_path, idx) if os.path.exists(trial_file_path): # Incomplete informations should be removed before starting. os.remove(trial_file_path) - # Initialize the brackets + def _init_brackets(self, default_hyper_parameters): self.rungs_in_brackets = [] - for s in range(self._num_brackets): + for slot in range(self._num_brackets): self.rungs_in_brackets.append( self.get_rungs( self.n_imgs_for_min_train, self.n_imgs_for_full_train, self._reduction_factor, - s, + slot, ) ) @@ -260,9 +265,7 @@ def __init__( # n0 is come from the notation in the paper, which means # the initial number of trials in each bracket. - n0_in_brackets = self.get_num_trials_in_brackets( - self._reduction_factor, self._num_brackets - ) + n0_in_brackets = self.get_num_trials_in_brackets(self._reduction_factor, self._num_brackets) # Assign each trial to a bracket # according to the ratio of the relative number of trials in each bracket @@ -278,7 +281,7 @@ def __init__( { "trial_id": idx, "config": default_hyper_parameter, - "status": hpopt.Status.READY, + "status": Status.READY, "score": None, "bracket": 0, } @@ -286,7 +289,7 @@ def __init__( num_ready_configs = len(self.hpo_status["config_list"]) - for i in range(len(n0_in_brackets)): + for i, _ in enumerate(n0_in_brackets): n0_in_brackets[i] = int(brackets_ratio[i] * self.num_trials) n0_in_brackets[0] += self.num_trials - sum(n0_in_brackets) @@ -294,8 +297,10 @@ def __init__( for trial in self.hpo_status["config_list"]: n0_in_brackets[trial["bracket"]] -= 1 - # Generate trial configs up to self.num_trials - for i in range(num_ready_configs, self.num_trials): + return n0_in_brackets, num_ready_configs + + def _generate_configs(self, num_ready_configs, num_trials, n0_in_brackets): + for i in range(num_ready_configs, num_trials): bracket_id = 0 for idx in reversed(range(len(n0_in_brackets))): remained_num = n0_in_brackets[idx] @@ -310,7 +315,7 @@ def __init__( # Check if the new config is duplicated while duplicated is True: - config = self.get_real_config(self.optimizer.suggest(self.uf)) + config = self.get_real_config(self.optimizer.suggest(self.utilty_function)) duplicated = self.check_duplicated_config(config) retry_count += 1 @@ -321,39 +326,34 @@ def __init__( { "trial_id": i, "config": config, - "status": hpopt.Status.READY, + "status": Status.READY, "score": None, "bracket": bracket_id, } ) - # Sync the current status to the file system. - self.save_results() - - # Get the list of iteration numbers for rungs in the bracket @staticmethod - def get_rungs(min_t: int, max_t: int, rf: int, s: int): - MAX_RUNGS = int(log(max_t - min_t + 1) / log(rf) - s + 1) - rungs = [(min_t - 1) + rf ** (k + s) for k in reversed(range(MAX_RUNGS))] + def get_rungs(min_t: int, max_t: int, reduction_factor: int, slot: int): + """Get the list of iteration numbers for rungs in the bracket""" + max_rungs = int(log(max_t - min_t + 1) / log(reduction_factor) - slot + 1) + rungs = [(min_t - 1) + reduction_factor ** (k + slot) for k in reversed(range(max_rungs))] return rungs - def get_num_trials_in_brackets(self, reduction_factor: int, num_brackets: int): + @staticmethod + def get_num_trials_in_brackets(reduction_factor: int, num_brackets: int): + """calculate available number of trials for each bracket""" brackets = [] - for s in reversed(range(num_brackets)): - n = int(ceil(int(num_brackets / (s + 1)) * reduction_factor**s)) - brackets.append(n) + for slot in reversed(range(num_brackets)): + num = int(ceil(int(num_brackets / (slot + 1)) * reduction_factor**slot)) + brackets.append(num) return brackets def save_results(self): hpo_file_path = hpopt.get_status_path(self.save_path) - oldmask = os.umask(0o077) - with open(hpo_file_path, "wt") as json_file: - json.dump(self.hpo_status, json_file, indent=4) - json_file.close() - os.umask(oldmask) + dump_as_json(hpo_file_path, self.hpo_status) def get_next_sample(self): # Gather all results from workers @@ -366,10 +366,7 @@ def get_next_sample(self): # Check total number of trained images if self._expected_total_images > 0: num_trained_images = self.get_num_trained_images() - logger.debug( - f"expected total images = {self._expected_total_images} " - f"num_images = {num_trained_images}" - ) + logger.debug(f"expected total images = {self._expected_total_images} " f"num_images = {num_trained_images}") if num_trained_images >= self._expected_total_images: return None @@ -377,8 +374,8 @@ def get_next_sample(self): next_config = None trial_id = None for idx, config in enumerate(self.hpo_status["config_list"]): - if config["status"] is hpopt.Status.READY: - config["status"] = hpopt.Status.RUNNING + if config["status"] is Status.READY: + config["status"] = Status.RUNNING next_config = config trial_id = idx break @@ -395,9 +392,7 @@ def get_next_sample(self): new_config["trial_id"] = trial_id new_config["bracket"] = next_config["bracket"] new_config["rungs"] = self.rungs_in_brackets[next_config["bracket"]] - new_config["file_path"] = hpopt.get_trial_path( - self.save_path, new_config["trial_id"] - ) + new_config["file_path"] = hpopt.get_trial_path(self.save_path, new_config["trial_id"]) new_config["subset_ratio"] = self.subset_ratio new_config["resize_width"] = self.image_resize[0] new_config["resize_height"] = self.image_resize[1] @@ -412,31 +407,27 @@ def get_next_sample(self): def update_scores(self): is_score_update = False for trial_id, config_item in enumerate(self.hpo_status["config_list"]): - if config_item["status"] == hpopt.Status.RUNNING: + if config_item["status"] == Status.RUNNING: current_status = hpopt.get_current_status(self.save_path, trial_id) - if current_status == hpopt.Status.STOP: + if current_status == Status.STOP: # score = hpopt.get_best_score(self.save_path, trial_id, self.mode) - score, n_imgs = hpopt.get_best_score_with_num_imgs( - self.save_path, trial_id, self.mode - ) + score, n_imgs = hpopt.get_best_score_with_num_imgs(self.save_path, trial_id, self.mode) if score is not None: config_item["score"] = score - config_item["status"] = hpopt.Status.STOP + config_item["status"] = Status.STOP real_config = config_item["config"] logger.info(f"#{trial_id} | {real_config} | {score}") - if ( - self.mode == "max" and score > self._current_best["score"] - ) or ( + if (self.mode == "max" and score > self._current_best["score"]) or ( self.mode == "min" and score < self._current_best["score"] ): self._current_best["score"] = score self._current_best["trial_id"] = trial_id self._current_best["image"] = n_imgs is_score_update = True - elif current_status == hpopt.Status.CUDAOOM: - config_item["status"] = hpopt.Status.READY + elif current_status == Status.CUDAOOM: + config_item["status"] = Status.READY self.hpo_status["num_gen_config"] -= 1 trial_file_path = hpopt.get_trial_path(self.save_path, trial_id) @@ -447,7 +438,7 @@ def update_scores(self): # shrink search space of 'bs' self.shrink_bs_search_space(config_item) # elif config_time["status"] == hpopt.Status.STOP: - # score, n_imgs = get_best_score_with_num_imgs(self.save_path, trial_id, self.mode) + # score, n_imgs = hpopt.get_best_score_with_num_imgs(self.save_path, trial_id, self.mode) # if score > current_best_score: # current_best_score = score # current_best_trial_id = trial_id @@ -458,7 +449,7 @@ def update_scores(self): # If the current schedule is updatable, adjust it regarding to the latest information if self._updatable_schedule and is_score_update: logger.info("best trial changed. updating schedule...") - num_trials, new_expected_total_images = self._calc_total_budget( + _, new_expected_total_images = self._calc_total_budget( self._num_brackets, self.expected_time_ratio, self._reduction_factor, @@ -469,34 +460,12 @@ def update_scores(self): self.num_workers, ) logger.info( - f"updated expected total images from {self._expected_total_images} " - f"to {new_expected_total_images}" + f"updated expected total images from {self._expected_total_images} " f"to {new_expected_total_images}" ) self._expected_total_images = new_expected_total_images - # _, _, _, new_expected_total_images, _, _, _, _ = self.auto_config( - # expected_time_ratio=self.expected_time_ratio, - # num_full_epochs=self.num_full_iterations, - # full_dataset_size=self.full_dataset_size, - # subset_ratio=self.subset_ratio, - # non_pure_train_ratio=self.non_pure_train_ratio, - # num_hyperparams=len(self.search_space), - # reduction_factor=self._reduction_factor, - # parallelism=self.num_workers, - # min_epochs=self._min_iterations, - # max_epochs=self.max_iterations, - # ) - # num_trials, new_expected_total_images = self._calc_total_budget( - # self._num_brackets, self.expected_time_ratio, self._reduction_factor, - # self.n_imgs_for_min_train, self.n_imgs_for_full_train, self.subset_ratio, self.non_pure_train_ratio, self.num_workers) - # logger.debug( - # f"updated expected total images from {self._expected_total_images} " - # f"to {new_expected_total_images}" - # ) - # self._expected_total_images = new_expected_total_images - - # Lower the upper bound of batch size def shrink_bs_search_space(self, not_allowed_config): + """Lower the upper bound of batch size""" # Stop if batch_size_name is not specified. if self.batch_size_name is None: return @@ -513,62 +482,36 @@ def shrink_bs_search_space(self, not_allowed_config): return not_allowed_bs = not_allowed_config["config"][self.batch_size_name] - new_upper_bound = ( - not_allowed_bs - self.search_space[self.batch_size_name].range[2] - ) + new_upper_bound = not_allowed_bs - self.search_space[self.batch_size_name].range[2] # if new_upper_bound is greater than the current upper bound, update only this trial. if self.search_space[self.batch_size_name].range[1] <= new_upper_bound: - not_allowed_config["config"] = self.get_real_config( - self.optimizer.suggest(self.uf) - ) + not_allowed_config["config"] = self.get_real_config(self.optimizer.suggest(self.utilty_function)) return self.search_space[self.batch_size_name].range[1] = new_upper_bound # if the new upper bound is less than the current lower bound, # update the lower bound to be half of the new upper bound. - if ( - self.search_space[self.batch_size_name].range[0] - > self.search_space[self.batch_size_name].range[1] - ): + if self.search_space[self.batch_size_name].range[0] > self.search_space[self.batch_size_name].range[1]: new_lower_bound = self.search_space[self.batch_size_name].range[1] // 2 self.search_space[self.batch_size_name].range[0] = max(new_lower_bound, 2) - if ( - self.search_space[self.batch_size_name].range[0] - > self.search_space[self.batch_size_name].range[1] - ): - raise ValueError( - "This model cannot be trained even with batch size of 2." - ) + if self.search_space[self.batch_size_name].range[0] > self.search_space[self.batch_size_name].range[1]: + raise ValueError("This model cannot be trained even with batch size of 2.") - # Reset search space - self.bayesopt_space = {} - for param in self.search_space: - self.bayesopt_space[param] = ( - self.search_space[param].lower_space(), - self.search_space[param].upper_space(), - ) + # Reset bayes opt with updated search space + self._init_bayse_opt(self.search_space) - self.optimizer = BayesianOptimization( - f=self.obj, - pbounds=self.bayesopt_space, - verbose=self.verbose, - random_state=None, - ) - - if self.hasCategoricalParam(self.search_space): - self.optimizer.set_gp_params(alpha=1e-3) - - for idx, config in enumerate(self.hpo_status["config_list"]): - if config["status"] is hpopt.Status.READY: - config["config"] = self.get_real_config(self.optimizer.suggest(self.uf)) + for config in self.hpo_status["config_list"]: + if config["status"] is Status.READY: + config["config"] = self.get_real_config(self.optimizer.suggest(self.utilty_function)) def get_num_trained_images(self): + """get total number of images that were used for training""" num_trained_images = 0 - for trial_id, config_item in enumerate(self.hpo_status["config_list"]): + for trial_id, _ in enumerate(self.hpo_status["config_list"]): trial_file_path = hpopt.get_trial_path(self.save_path, trial_id) trial_results = hpopt.load_json(trial_file_path) if trial_results is not None: @@ -578,6 +521,7 @@ def get_num_trained_images(self): return num_trained_images + # pylint: disable=too-many-arguments, too-many-locals, too-many-statements def auto_config( self, expected_time_ratio: Union[int, float], @@ -591,6 +535,7 @@ def auto_config( min_epochs: Optional[int], max_epochs: Optional[int], ): + """generate configuration automatically""" # All arguments should be specified. if expected_time_ratio is None: raise ValueError("expected_time_ratio should be specified.") @@ -637,25 +582,21 @@ def auto_config( # 1. Config target parameters as large as possible # max_iterations = -1 - if max_epochs is None: - # max_iterations = num_full_epochs - max_epochs = num_full_epochs + max_epochs = num_full_epochs if max_epochs is None else max_epochs - if min_epochs is None: - min_epochs = 1 + min_epochs = 1 if min_epochs is None else min_epochs - if min_epochs > max_epochs: - logger.info("min_epochs should be less than or equal to max_epochs.") - min_epochs = max_epochs + min_epochs = min(min_epochs, max_epochs) + + if not isinstance(full_dataset_size, int): + raise TypeError(f"full dataset size should be integer type but {type(full_dataset_size)}") if subset_ratio is None: # Default subset ratio is 0.2 subset_ratio = 0.2 if full_dataset_size == 0: - logger.warning( - "Sub-dataset isn't used because full_dataset_size value is 0." - ) + logger.warning("Sub-dataset isn't used because full_dataset_size value is 0.") subset_ratio = 1.0 elif (full_dataset_size * subset_ratio) < self.min_subset_size: if full_dataset_size > self.min_subset_size: @@ -666,8 +607,7 @@ def auto_config( n_imgs_for_full_train = int(full_dataset_size * max_epochs * subset_ratio) n_imgs_for_min_train = int(full_dataset_size * min_epochs * subset_ratio) logger.debug( - f"n_imgs_for_full_train = {n_imgs_for_full_train}, " - f"n_imgs_for_min_train = {n_imgs_for_min_train}" + f"n_imgs_for_full_train = {n_imgs_for_full_train}, " f"n_imgs_for_min_train = {n_imgs_for_min_train}" ) # num_trials = 0 @@ -678,25 +618,22 @@ def auto_config( max_num_images_in_trials = 0 if self.hpo_status.get("config_list", None) is not None: - for trial_id, config_item in enumerate(self.hpo_status["config_list"]): - trial_file_path = hpopt.get_trial_path(self.save_path, trial_id) - trial_results = hpopt.load_json(trial_file_path) + for trial_id, _ in enumerate(self.hpo_status["config_list"]): + trial_results = hpopt.get_trial_results(self.save_path, trial_id) if trial_results is not None: n_images = trial_results.get("images", None) if n_images is not None: if n_images[-1] > max_num_images_in_trials: max_num_images_in_trials = n_images[-1] - logger.debug( - f"(before) max_num_images_in_trials: {max_num_images_in_trials}" - ) + logger.debug(f"(before) max_num_images_in_trials: {max_num_images_in_trials}") if max_num_images_in_trials > 0: rungs = self.get_rungs( max_t=n_imgs_for_full_train, min_t=n_imgs_for_min_train, - rf=reduction_factor, - s=0, + reduction_factor=reduction_factor, + slot=0, ) if len(rungs) > 5: @@ -744,6 +681,7 @@ def auto_config( n_imgs_for_full_train, ) + # pylint: disable=too-many-arguments def _calc_total_budget( self, num_brackets, @@ -779,33 +717,23 @@ def get_total_n_images( n_imgs_min_train: int, n_imgs_full_train: int, ): + """calculate total number of images for given number of trials""" num_total_images = 0 - num_trials_in_brackets = self.get_num_trials_in_brackets( - reduction_factor, num_brackets - ) - # logger.debug(f"num_trials_in_brackets = {num_trials_in_brackets}") + num_trials_in_brackets = self.get_num_trials_in_brackets(reduction_factor, num_brackets) - brackets_total = sum(num_trials_in_brackets) - brackets_ratio = [float(b / brackets_total) for b in num_trials_in_brackets] + brackets_ratio = [float(b / sum(num_trials_in_brackets)) for b in num_trials_in_brackets] - for i in range(len(num_trials_in_brackets)): + for i, _ in enumerate(num_trials_in_brackets): num_trials_in_brackets[i] = int(brackets_ratio[i] * num_trials) num_trials_in_brackets[0] += num_trials - sum(num_trials_in_brackets) - # logger.debug(f"num_trials_in_brackets = {num_trials_in_brackets}") - - for s, num_trials in enumerate(num_trials_in_brackets): - rungs = self.get_rungs( - n_imgs_min_train, n_imgs_full_train, reduction_factor, s - ) - remained_trials = num_trials + for i, n_trials in enumerate(num_trials_in_brackets): + rungs = self.get_rungs(n_imgs_min_train, n_imgs_full_train, reduction_factor, i) + remained_trials = n_trials for rung in reversed(rungs): - num_stop_trials = remained_trials - ( - remained_trials // reduction_factor - ) - num_total_images += num_stop_trials * rung + num_total_images += (remained_trials - (remained_trials // reduction_factor)) * rung remained_trials = remained_trials // reduction_factor num_total_images += remained_trials * n_imgs_full_train @@ -813,16 +741,9 @@ def get_total_n_images( def get_progress(self): # num images based progress - image_progress = min( - self.get_num_trained_images() / self._expected_total_images, 0.99 - ) + image_progress = min(self.get_num_trained_images() / self._expected_total_images, 0.99) # trial based progress - finished_trials = sum( - [ - val["status"] == hpopt.Status.STOP - for val in self.hpo_status["config_list"] - ] - ) + finished_trials = sum([val["status"] == Status.STOP for val in self.hpo_status["config_list"]]) trial_progress = finished_trials / self.num_trials # return min(0.99, max(epoch_progress, trial_progress)) @@ -832,3 +753,190 @@ def get_progress(self): ) return min(0.99, max(image_progress, trial_progress)) + + @staticmethod + def report( + config, + trial_results, + ): + """report implementation for the hyperband""" + if trial_results["images"][-1] >= config["iteration_limit"]: + trial_results["status"] = Status.STOP + logger.info("trial reaches its limitation. stop") + return + + # Async HyperBand + save_path = os.path.dirname(config["file_path"]) + # curr_itr = len(trial_results['scores']) + curr_itr = trial_results["images"][-1] + logger.debug(f"current iterations = {curr_itr} : {config['rungs']}") + + for rung_itr in config["rungs"]: + if curr_itr >= rung_itr: + # Decide whether to promote to the next rung or not + logger.debug(f"{save_path} / curr_itr = {curr_itr} / rung_itr = {rung_itr}") + cutoff_score = _get_cutoff_score(save_path, rung_itr, config["rungs"], config["mode"]) + + if cutoff_score is not None: + if config["mode"] == "min": + curr_best_score = min(trial_results["scores"]) + else: + curr_best_score = max(trial_results["scores"]) + + stop_flag = False + + if config["mode"] == "max" and cutoff_score > curr_best_score: + stop_flag = True + elif config["mode"] == "min" and cutoff_score < curr_best_score: + stop_flag = True + + if stop_flag: + trial_results["status"] = Status.STOP + logger.debug( + f"[ASHA STOP] [{config['trial_id']}, {curr_itr}, {rung_itr}] " + f"{cutoff_score} > {curr_best_score}" + ) + break + + +def _get_percentile(_list: List[int], percentile: float, mode: str): + if mode == "max": + percentile = np.nanpercentile(_list, (1 - percentile) * 100) + else: + percentile = np.nanpercentile(_list, percentile * 100) + logger.debug(f"get {mode} percentile of {percentile} from {_list}, ret = {percentile}") + return percentile + + +def _get_all_results(save_path): + if not os.path.exists(hpopt.get_status_path(save_path)): + logger.warning(f"not existed status json file {save_path}") + return None, None, None + + hpo_status = hpopt.load_json(hpopt.get_status_path(save_path)) + + if hpo_status is None: + logger.warning(f"failed to load hpo status json file in {save_path}") + return None, None, None + + hpo_trial_results_scores = [] + hpo_trial_results_imgs_num = [] + + for idx, config in enumerate(hpo_status["config_list"]): + if config["status"] in [Status.RUNNING, Status.STOP]: + trial_results = hpopt.get_trial_results(save_path, idx) + if len(trial_results["scores"]) != len(trial_results["images"]): + raise RuntimeError( + f"mismatch length of trial results and number of trained images {len(trial_results['scores'])}" + f"/{len(trial_results['images'])}" + ) + if trial_results is not None: + hpo_trial_results_scores.append(trial_results["scores"]) + hpo_trial_results_imgs_num.append(trial_results["images"]) + return ( + hpo_trial_results_scores, + hpo_trial_results_imgs_num, + hpo_status["reduction_factor"], + ) + + +def _find_idx_of_max_less_than_target(src: list, target: Union[int, float]): + idx = -1 + # TODO need to sort list? + if len(src) <= 0: + logger.warning("src list is empty") + elif src[0] > target: + logger.info("target is out of range (lower bound)") + elif src[-1] < target: + logger.info("target is out of range (upper bound)") + else: + idx = src.index(max(x for x in src if x <= target)) + logger.debug(f"found index is {idx} which is minimum " f"but greater than {target} from src = {src}") + return idx + + +# pylint: disable=too-many-nested-blocks +def _get_cutoff_score(save_path: str, target_rung: int, _rung_list, mode: str): + """ + Calculate a cutoff score for a specified rung of ASHA + + Args: + save_path (str): path where result of HPO is saved. + target_rung (int): current rung number + _rung_list (list): list of rungs in the current bracket + mode (str): max or min. Decide whether to find max value or min value. + """ + logger.debug(f"get_cutoff_score({target_rung}, {_rung_list}, {mode})") + + # Gather all scores with iter number + ( + hpo_trial_results_scores, + hpo_trial_results_imgs_num, + reduction_factor, + ) = _get_all_results(save_path) + if hpo_trial_results_imgs_num is None or hpo_trial_results_scores is None: + return None + + logger.debug(f"scores = {hpo_trial_results_scores} num_imgs = {hpo_trial_results_imgs_num}") + + # Run a SHA (not ASHA) + rung_score_list: List[int] = [] + rung_list = _rung_list.copy() + rung_list.sort(reverse=False) + + if len(hpo_trial_results_scores) == 1: + return None + + for curr_rung in rung_list: + if curr_rung <= target_rung: + rung_score_list.clear() + logger.debug(f"curr_rung = {curr_rung}") + for idx, num_imgs in enumerate(hpo_trial_results_imgs_num): + if num_imgs != []: + logger.debug(f"trial {idx}: score reported @ {num_imgs}") + logger.debug(f"trial {idx}: scores {hpo_trial_results_scores[idx]}") + result_idx = _find_idx_of_max_less_than_target(num_imgs, curr_rung) + logger.debug(f"cutoff idx = {result_idx}") + if result_idx != -1: + if result_idx == 0: + rung_score_list.append(hpo_trial_results_scores[idx][result_idx]) + else: + if mode == "max": + rung_score_list.append(max(hpo_trial_results_scores[idx][:result_idx])) + else: + rung_score_list.append(min(hpo_trial_results_scores[idx][:result_idx])) + else: + logger.debug(f"trial {idx}: skipped since num_imgs is empty") + num_imgs.clear() + hpo_trial_results_scores[idx].clear() + + if len(rung_score_list) > 1: + logger.debug(f"rung_score_list = {rung_score_list}") + cut_off_score = _get_percentile(rung_score_list, 1 / reduction_factor, mode) + logger.debug(f"cut_off_score = {cut_off_score}") + for idx, num_imgs in enumerate(hpo_trial_results_imgs_num): + if num_imgs != []: + result_idx = _find_idx_of_max_less_than_target(num_imgs, curr_rung) + if result_idx != -1: + if result_idx == 0: + target_score = hpo_trial_results_scores[idx][result_idx] + else: + target_score = ( + max(hpo_trial_results_scores[idx][:result_idx]) + if mode == "max" + else min(hpo_trial_results_scores[idx][:result_idx]) + ) + if mode == "max": + if target_score < cut_off_score: + hpo_trial_results_scores[idx].clear() + num_imgs.clear() + logger.debug(f"removed results from trial {idx}") + else: + if target_score > cut_off_score: + hpo_trial_results_scores[idx].clear() + num_imgs.clear() + logger.debug(f"removed results from trial {idx}") + if len(rung_score_list) > 1: + logger.debug(f"last rung_score_list = {rung_score_list}") + return _get_percentile(rung_score_list, 1 / reduction_factor, mode) + return None diff --git a/hpopt/logger.py b/hpopt/logger.py index a0d7d5c..e607dcf 100644 --- a/hpopt/logger.py +++ b/hpopt/logger.py @@ -1,3 +1,9 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +""" logger module of the hpopt +""" + import logging import os import sys @@ -16,6 +22,7 @@ def _get_logger(): logger = logging.getLogger("hpopt") logger.propagate = False + # pylint: disable=redefined-builtin def print(message, *args, **kws): if logger.isEnabledFor(_CUSTOM_LOG_LEVEL): logger.log(_CUSTOM_LOG_LEVEL, message, *args, **kws) @@ -25,6 +32,7 @@ def print(message, *args, **kws): default_log_level = None env_setting = os.environ.get("HPOPT_LOG_LEVEL") if env_setting is not None: + # pylint: disable=protected-access default_log_level = logging._nameToLevel.get(env_setting.upper()) if default_log_level is None: default_log_level = logging.INFO @@ -57,6 +65,8 @@ def print(message, *args, **kws): def config_logger(log_file, level="WARNING"): + """configure logger""" + # pylint: disable=global-statement global _LOG_DIR, _FILE_HANDLER if _FILE_HANDLER is not None: _logger.removeHandler(_FILE_HANDLER) @@ -79,44 +89,41 @@ def _get_log_level(level): # get level number level_number = logging.getLevelName(level.upper()) if level_number not in [0, 10, 20, 30, 40, 50, _CUSTOM_LOG_LEVEL]: - msg = ( - "Log level must be one of DEBUG/INFO/WARN/ERROR/CRITICAL/LOG" - ", but {} is given.".format(level) - ) + msg = "Log level must be one of DEBUG/INFO/WARN/ERROR/CRITICAL/LOG" f", but {level} is given." raise ValueError(msg) return level_number def get_log_dir(): + """get path of the logging file handler""" return _LOG_DIR -# debug log should be disabled before release -def debug(message, *args, **kws): - pass - # if _logger.isEnabledFor(logging.DEBUG): - # _logger.log(logging.DEBUG, message, *args, **kws) - - def get_logger(rank=-1): + """get logger instance""" if rank is None or rank > 0: return _DummyLogger("dummy") return _logger class _DummyLogger(logging.Logger): + @staticmethod def debug(message, *args, **kws): pass + @staticmethod def info(message, *args, **kws): pass + @staticmethod def warning(message, *args, **kws): pass + @staticmethod def critical(message, *args, **kws): pass + @staticmethod def error(message, *args, **kws): pass diff --git a/hpopt/smbo.py b/hpopt/smbo.py index e390642..547eca6 100644 --- a/hpopt/smbo.py +++ b/hpopt/smbo.py @@ -1,21 +1,25 @@ # Copyright (C) 2018-2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # +"""implementation of the smbo +""" import json import os -import logging from typing import Dict, List, Optional, Union from bayes_opt import BayesianOptimization, UtilityFunction import hpopt -from hpopt.base import HpOpt -from hpopt.logger import get_logger + +from .base import HpOpt, Status +from .logger import get_logger +from .utils import dump_as_json logger = get_logger() +# pylint: disable=too-many-instance-attributes, too-many-branches, too-many-statements class BayesOpt(HpOpt): """ This implements the Bayesian optimization. Bayesian optimization is @@ -29,25 +33,22 @@ class BayesOpt(HpOpt): kappa_decay_delay (int): From first trials to kappa_decay_delay trials, kappa isn't multiplied to kappa_decay. """ + def __init__( self, early_stop: Optional[bool] = None, kappa: Union[float, int] = 2.576, - kappa_decay: int = 1, + kappa_decay: Union[float, int] = 1, kappa_decay_delay: int = 0, default_hyper_parameters: Optional[Union[List[Dict], Dict]] = None, **kwargs, ): - super(BayesOpt, self).__init__(**kwargs) + super().__init__(**kwargs) self.updatable_schedule = False self.early_stop = early_stop # HPO auto configurator - if ( - self.num_trials is None - or self.max_iterations is None - or self.subset_ratio is None - ): + if self.num_trials is None or self.max_iterations is None or self.subset_ratio is None: self.updatable_schedule = True self.num_trials, self.max_iterations, self.subset_ratio = self.auto_config( self.expected_time_ratio, @@ -65,8 +66,11 @@ def __init__( f"max_iterations {self.max_iterations} subset_ratio {self.subset_ratio}" ) - if self.num_init_trials > self.num_trials: - self.num_init_trials = self.num_trials + if isinstance(self.num_trials, int): + if self.num_init_trials > self.num_trials: + self.num_init_trials = self.num_trials + else: + raise RuntimeError(f"num_trials should be int but {type(self.num_trials)}") self.bayesopt_space = {} for param in self.search_space: @@ -82,7 +86,7 @@ def __init__( random_state=None, ) - self.uf = UtilityFunction( + self.utility_function = UtilityFunction( kind="ucb", xi=0.0, kappa=kappa, @@ -90,54 +94,21 @@ def __init__( kappa_decay_delay=kappa_decay_delay, ) - if self.hasCategoricalParam(self.search_space): + if self.has_categorical_param(self.search_space): self.optimizer.set_gp_params(alpha=1e-3) hpo_file_path = hpopt.get_status_path(self.save_path) if self.resume is True and os.path.exists(hpo_file_path): - with open(hpo_file_path, "rt") as json_file: + logger.info(f"found hpo files at {hpo_file_path}") + with open(hpo_file_path, "rt", encoding="utf-8") as json_file: self.hpo_status = json.load(json_file) json_file.close() - if self.hpo_status["search_algorithm"] != "smbo": - logger.error("Search algorithm is changed. Stop resuming.") - raise ValueError("Search algorithm is changed.") - - if self.hpo_status["search_space"] != { - ss: self.search_space[ss].__dict__ for ss in self.search_space - }: - logger.error("Search space is changed. Stop resuming.") - raise ValueError("Search space is changed.") - - if self.hpo_status["metric"] != self.metric: - logger.error("Metric is changed. Stop resuming.") - raise ValueError("Metric is changed.") - - if self.hpo_status["subset_ratio"] != self.subset_ratio: - logger.error("subset_ratio is changed. Stop resuming.") - raise ValueError("subset_ratio is changed.") - - if self.hpo_status["image_resize"] != self.image_resize: - logger.error("image_resize is changed. Stop resuming.") - raise ValueError("image_resize is changed.") - - if self.hpo_status["early_stop"] != self.early_stop: - logger.error("early_stop is changed. Stop resuming.") - raise ValueError("early_stop is changed.") - - if self.hpo_status["max_iterations"] != self.max_iterations: - logger.error("max_iterations is changed. Stop resuming.") - raise ValueError("max_iterations is changed.") - - if self.hpo_status["full_dataset_size"] != self.full_dataset_size: - logger.error("The size of dataset is changed. Stop resuming.") - raise ValueError("Dataset is changed.") + self._verify_hpo_status() else: self.hpo_status["search_algorithm"] = "smbo" - self.hpo_status["search_space"] = { - ss: self.search_space[ss].__dict__ for ss in self.search_space - } + self.hpo_status["search_space"] = {ss: self.search_space[ss].__dict__ for ss in self.search_space} self.hpo_status["metric"] = self.metric self.hpo_status["subset_ratio"] = self.subset_ratio self.hpo_status["image_resize"] = self.image_resize @@ -145,22 +116,7 @@ def __init__( self.hpo_status["max_iterations"] = self.max_iterations self.hpo_status["full_dataset_size"] = self.full_dataset_size self.hpo_status["config_list"] = [] - - self.hpo_status["num_gen_config"] = 0 - - for idx, config in enumerate(self.hpo_status["config_list"], start=0): - if config["status"] == hpopt.Status.STOP: - self.hpo_status["num_gen_config"] += 1 - self.optimizer.register( - params=self.get_space_config(config["config"]), - target=config["score"], - ) - self.uf.update_params() - else: - config["status"] = hpopt.Status.READY - trial_file_path = hpopt.get_trial_path(self.save_path, idx) - if os.path.exists(trial_file_path): - os.remove(trial_file_path) + self.hpo_status["num_gen_config"] = 0 if default_hyper_parameters is not None: if isinstance(default_hyper_parameters, dict): @@ -171,21 +127,77 @@ def __init__( { "trial_id": idx, "config": default_hyper_parameter, - "status": hpopt.Status.READY, + "status": Status.READY, "score": None, } ) num_ready_configs = len(self.hpo_status["config_list"]) + if self.num_init_trials > num_ready_configs: + self._generate_configs(num_ready_configs, self.num_init_trials) + + self.save_results() + + def _verify_hpo_status(self): + """verify current hpo_status""" + logger.debug("verify hpo status()") + if self.hpo_status["search_algorithm"] != "smbo": + logger.error("Search algorithm is changed. Stop resuming.") + raise ValueError("Search algorithm is changed.") + + if self.hpo_status["search_space"] != {ss: self.search_space[ss].__dict__ for ss in self.search_space}: + logger.error("Search space is changed. Stop resuming.") + raise ValueError("Search space is changed.") - for i in range(num_ready_configs, self.num_init_trials): + if self.hpo_status["metric"] != self.metric: + logger.error("Metric is changed. Stop resuming.") + raise ValueError("Metric is changed.") + + if self.hpo_status["subset_ratio"] != self.subset_ratio: + logger.error("subset_ratio is changed. Stop resuming.") + raise ValueError("subset_ratio is changed.") + + if self.hpo_status["image_resize"] != self.image_resize: + logger.error("image_resize is changed. Stop resuming.") + raise ValueError("image_resize is changed.") + + if self.hpo_status["early_stop"] != self.early_stop: + logger.error("early_stop is changed. Stop resuming.") + raise ValueError("early_stop is changed.") + + if self.hpo_status["max_iterations"] != self.max_iterations: + logger.error("max_iterations is changed. Stop resuming.") + raise ValueError("max_iterations is changed.") + + if self.hpo_status["full_dataset_size"] != self.full_dataset_size: + logger.error("The size of dataset is changed. Stop resuming.") + raise ValueError("Dataset is changed.") + + # verify config_list + self.hpo_status["num_gen_config"] = 0 + for idx, config in enumerate(self.hpo_status["config_list"], start=0): + if config["status"] == Status.STOP: + self.hpo_status["num_gen_config"] += 1 + self.optimizer.register( + params=self.get_space_config(config["config"]), + target=config["score"], + ) + self.utility_function.update_params() + else: + config["status"] = Status.READY + trial_file_path = hpopt.get_trial_path(self.save_path, idx) + if os.path.exists(trial_file_path): + os.remove(trial_file_path) + + def _generate_configs(self, num_ready_configs, num_trials): + for i in range(num_ready_configs, num_trials): # Generate a new config duplicated = True retry_count = 0 # Check if the new config is duplicated while duplicated is True: - config = self.get_real_config(self.optimizer.suggest(self.uf)) + config = self.get_real_config(self.optimizer.suggest(self.utility_function)) duplicated = self.check_duplicated_config(config) retry_count += 1 @@ -199,27 +211,21 @@ def __init__( { "trial_id": i, "config": config, - "status": hpopt.Status.READY, + "status": Status.READY, "score": None, } ) - self.save_results() - def save_results(self): hpo_file_path = hpopt.get_status_path(self.save_path) - oldmask = os.umask(0o077) - with open(hpo_file_path, "wt") as json_file: - json.dump(self.hpo_status, json_file, indent=4) - json_file.close() - os.umask(oldmask) + dump_as_json(hpo_file_path, self.hpo_status) def update_scores(self): for trial_id, config_item in enumerate(self.hpo_status["config_list"], start=0): - if config_item["status"] == hpopt.Status.RUNNING: + if config_item["status"] == Status.RUNNING: current_status = hpopt.get_current_status(self.save_path, trial_id) - if current_status == hpopt.Status.STOP: + if current_status == Status.STOP: score = hpopt.get_best_score(self.save_path, trial_id, self.mode) if score is not None: @@ -227,19 +233,19 @@ def update_scores(self): if self.mode == "min": self.optimizer.register( params=self.get_space_config(config_item["config"]), - target=-score, + target=(score * -1.0), ) else: self.optimizer.register( params=self.get_space_config(config_item["config"]), target=score, ) - self.uf.update_params() - config_item["status"] = hpopt.Status.STOP + self.utility_function.update_params() + config_item["status"] = Status.STOP real_config = config_item["config"] logger.info(f"#{trial_id} | {real_config} | {score}") - elif current_status == hpopt.Status.CUDAOOM: - config_item["status"] = hpopt.Status.READY + elif current_status == Status.CUDAOOM: + config_item["status"] = Status.READY self.hpo_status["num_gen_config"] -= 1 trial_file_path = hpopt.get_trial_path(self.save_path, trial_id) @@ -252,8 +258,8 @@ def update_scores(self): self.save_results() - # Lower the upper bound of batch size def shrink_bs_search_space(self, not_allowed_config): + """Lower the upper bound of batch size""" # Stop if batch_size_name is not specified. if self.batch_size_name is None: return @@ -270,28 +276,18 @@ def shrink_bs_search_space(self, not_allowed_config): return not_allowed_bs = not_allowed_config["config"][self.batch_size_name] - new_upper_bound = ( - not_allowed_bs - self.search_space[self.batch_size_name].range[2] - ) + new_upper_bound = not_allowed_bs - self.search_space[self.batch_size_name].range[2] self.search_space[self.batch_size_name].range[1] = new_upper_bound # if the new upper bound is less than the current lower bound, # update the lower bound to be half of the new upper bound. - if ( - self.search_space[self.batch_size_name].range[0] - > self.search_space[self.batch_size_name].range[1] - ): + if self.search_space[self.batch_size_name].range[0] > self.search_space[self.batch_size_name].range[1]: new_lower_bound = self.search_space[self.batch_size_name].range[1] // 2 self.search_space[self.batch_size_name].range[0] = max(new_lower_bound, 2) - if ( - self.search_space[self.batch_size_name].range[0] - > self.search_space[self.batch_size_name].range[1] - ): - raise ValueError( - "This model cannot be trained even with batch size of 2." - ) + if self.search_space[self.batch_size_name].range[0] > self.search_space[self.batch_size_name].range[1]: + raise ValueError("This model cannot be trained even with batch size of 2.") # Reset search space self.bayesopt_space = {} @@ -308,13 +304,13 @@ def shrink_bs_search_space(self, not_allowed_config): random_state=None, ) - if self.hasCategoricalParam(self.search_space): + if self.has_categorical_param(self.search_space): self.optimizer.set_gp_params(alpha=1e-3) del_list = [] for idx, config in enumerate(self.hpo_status["config_list"]): - if config["status"] is hpopt.Status.READY: + if config["status"] is Status.READY: # Generate a new config duplicated = True retry_count = 0 @@ -322,7 +318,7 @@ def shrink_bs_search_space(self, not_allowed_config): # Check if the new config is duplicated while duplicated is True: - new_config = self.get_real_config(self.optimizer.suggest(self.uf)) + new_config = self.get_real_config(self.optimizer.suggest(self.utility_function)) duplicated = self.check_duplicated_config(new_config) retry_count += 1 @@ -337,7 +333,7 @@ def shrink_bs_search_space(self, not_allowed_config): del self.hpo_status["config_list"][idx] for config in self.hpo_status["config_list"]: - if config["status"] == hpopt.Status.STOP: + if config["status"] == Status.STOP: self.optimizer.register( params=self.get_space_config(config["config"]), target=config["score"], @@ -345,6 +341,7 @@ def shrink_bs_search_space(self, not_allowed_config): self.save_results() + # pylint: disable=too-many-arguments def auto_config( self, expected_time_ratio: Union[int, float], @@ -357,6 +354,7 @@ def auto_config( subset_ratio: Optional[Union[float, int]], parallelism: int = 1, ): + """generate configuration automatically""" # All arguments should be specified. if expected_time_ratio is None: raise ValueError("expected_time_ratio should be specified.") @@ -381,10 +379,11 @@ def auto_config( subset_ratio = min(1.0, subset_ratio) subset_ratio = max(0.2, subset_ratio) + if not isinstance(full_dataset_size, int): + raise TypeError(f"full dataset size should be integer type but {type(full_dataset_size)}") + if full_dataset_size == 0: - logger.warning( - "Sub-dataset isn't used because full_dataset_size value is 0." - ) + logger.warning("Sub-dataset isn't used because full_dataset_size value is 0.") subset_ratio = 1.0 elif (full_dataset_size * subset_ratio) < self.min_subset_size: if full_dataset_size > self.min_subset_size: @@ -414,18 +413,10 @@ def auto_config( trial_ratio = num_trials / max_trials while current_time_ratio < expected_time_ratio: - if ( - subset_ratio < 1.0 - and subset_ratio <= (3 * iter_ratio) - and subset_ratio < trial_ratio - ): + if subset_ratio < 1.0 and subset_ratio <= (3 * iter_ratio) and subset_ratio < trial_ratio: subset_ratio = subset_ratio * 1.05 subset_ratio = min(1.0, subset_ratio) - elif ( - iter_ratio < 1.0 - and (3 * iter_ratio) <= subset_ratio - and (3 * iter_ratio) < trial_ratio - ): + elif iter_ratio < 1.0 and (3 * iter_ratio) <= subset_ratio and (3 * iter_ratio) < trial_ratio: max_iterations += 1 max_iterations = min(max_iterations, num_full_iterations) else: @@ -435,9 +426,7 @@ def auto_config( trial_ratio = num_trials / max_trials current_time_ratio = ( - num_trials - * iter_ratio - * ((1 - non_pure_train_ratio) * subset_ratio + non_pure_train_ratio) + num_trials * iter_ratio * ((1 - non_pure_train_ratio) * subset_ratio + non_pure_train_ratio) ) if subset_ratio > 0.99 and iter_ratio > 0.99 and trial_ratio > 0.99: @@ -450,6 +439,7 @@ def auto_config( # Retrieve the next config to evaluate # def get_next_sample(self): + logger.debug(f"get_next_sample() {self.hpo_status['num_gen_config']}/{self.num_trials}") if self.hpo_status["num_gen_config"] >= self.num_trials: return None @@ -461,13 +451,13 @@ def get_next_sample(self): # Choose a config that has not yet been evaluated from the config_list num_done_config = 0 for gen_config in self.hpo_status["config_list"]: - if gen_config["status"] == hpopt.Status.STOP: + if gen_config["status"] == Status.STOP: num_done_config += 1 for idx, gen_config in enumerate(self.hpo_status["config_list"], start=0): - if gen_config["status"] == hpopt.Status.READY: + if gen_config["status"] == Status.READY: config = gen_config["config"] - gen_config["status"] = hpopt.Status.RUNNING + gen_config["status"] = Status.RUNNING trial_id = idx break else: @@ -477,7 +467,7 @@ def get_next_sample(self): # Check if the new config is duplicated while duplicated is True: - config = self.get_real_config(self.optimizer.suggest(self.uf)) + config = self.get_real_config(self.optimizer.suggest(self.utility_function)) duplicated = self.check_duplicated_config(config) retry_count += 1 @@ -489,7 +479,7 @@ def get_next_sample(self): { "trial_id": trial_id, "config": config, - "status": hpopt.Status.RUNNING, + "status": Status.RUNNING, "score": None, } ) @@ -503,9 +493,7 @@ def get_next_sample(self): new_config["params"] = config new_config["iterations"] = self.max_iterations new_config["trial_id"] = trial_id - new_config["file_path"] = hpopt.get_trial_path( - self.save_path, new_config["trial_id"] - ) + new_config["file_path"] = hpopt.get_trial_path(self.save_path, new_config["trial_id"]) if os.path.exists(new_config["file_path"]): os.remove(new_config["file_path"]) @@ -539,21 +527,14 @@ def get_next_samples(self, num_expected_samples=0): return new_configs def get_progress(self): - finished_trials = sum( - [ - val["status"] == hpopt.Status.STOP - for val in self.hpo_status["config_list"] - ] - ) + finished_trials = sum([val["status"] == Status.STOP for val in self.hpo_status["config_list"]]) progress = finished_trials / self.num_trials final_trial_path = hpopt.get_trial_path(self.save_path, finished_trials) if os.path.exists(final_trial_path): - with open(final_trial_path, "r") as f: + with open(final_trial_path, "r", encoding="utf-8") as f: try: - progress += len(json.load(f)["scores"]) / ( - self.num_trials * self.max_iterations - ) + progress += len(json.load(f)["scores"]) / (self.num_trials * self.max_iterations) except json.JSONDecodeError: pass diff --git a/hpopt/utils.py b/hpopt/utils.py new file mode 100644 index 0000000..c1472ba --- /dev/null +++ b/hpopt/utils.py @@ -0,0 +1,145 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +""" collection of the utility functions for the hpopt +""" + +import inspect +import json +import os +from typing import Any, Dict, ItemsView, List, Optional, Type, Union + +import numpy as np +import torch +from torch.utils.data import random_split +from torchvision import transforms + +from hpopt.logger import get_logger + +logger = get_logger() + + +def dump_as_json(output_path: str, obj, indent: int = 4): + """write obj to file as a JSON format""" + oldmask = os.umask(0o077) + with open(output_path, "wt", encoding="utf-8") as json_file: + json.dump(obj, json_file, indent=indent) + json_file.flush() + os.umask(oldmask) + + +def type_check( + _input: Any, + expected_type: Union[Type, List[Type]], + non_zero: Optional[bool] = None, + positive: Optional[bool] = None, +): + """helper function to check input paramter varification""" + callers_local_vars = None + frame = inspect.currentframe() + if frame is not None and frame.f_back is not None: + callers_local_vars = frame.f_back.f_locals.items() + + name_of_input = "input" + if isinstance(callers_local_vars, ItemsView): + name_list = [var_name for var_name, var_val in callers_local_vars if id(var_val) == id(input)] + name_of_input = name_list[0] if len(name_list) > 0 and _input is not None else "input" + + allowed = False + if isinstance(expected_type, list): + for _type in expected_type: + if isinstance(_input, _type): + allowed = True + break + elif isinstance(_input, expected_type): + allowed = True + + if not allowed: + raise TypeError( + f"expected type(s) of the '{name_of_input}': {expected_type}, but the input type is {type(_input)}" + ) + + if positive is not None and input is not None: + if positive and _input < 0: + raise ValueError(f"'{name_of_input}' is expected to be positive number but {_input}.") + if not positive and _input > 0: + raise ValueError(f"'{name_of_input}' is expected to be negative number but {_input}.") + + if non_zero is not None and _input is not None: + if non_zero and _input == 0: + raise ValueError(f"'{name_of_input}' should not be zero.") + return True + + +class HpoDataset: + """ + Dataset class which wrap dataset class used in training for sub-sampling. + + Args: + fullset: dataset instance used in train. + config (dict): HPO configuration for a trial. + This include train confiuration(e.g. hyper parameter, epoch, etc.) + and tiral information. + """ + + def __init__(self, fullset, config: Dict[str, Any]): + self.__dict__ = fullset.__dict__.copy() + self.fullset = fullset + + if config["subset_ratio"] > 0.0: + if config["subset_ratio"] < 1.0: + subset_size = int(len(fullset) * config["subset_ratio"]) + self.subset, _ = random_split( + fullset, + [subset_size, (len(fullset) - subset_size)], + generator=torch.Generator().manual_seed(42), + ) + + # check if fullset is an inheritance of mmdet.datasets + if hasattr(self, "flag"): + self._update_group_flag() + else: + self.subset = fullset + self.length = len(self.subset) + else: + self.subset = fullset + self.length = len(fullset) + + if config["resize_height"] > 0 and config["resize_width"] > 0: + self.transform = transforms.Resize((config["resize_height"], config["resize_width"]), interpolation=2) + else: + self.transform = None + + def _update_group_flag(self): + self.flag = np.zeros(len(self.subset), dtype=np.uint8) + + update_flag = False + if "img_metas" in self.subset[0]: + if "ori_shape" in self.subset[0]["img_metas"].data: + update_flag = True + + if not update_flag: + return + + for i in range(len(self.subset)): + self.flag[i] = self.fullset.flag[self.subset.indices[i]] + + def __getitem__(self, index: int): + data = self.subset[index] + if self.transform: + # if type(data) is tuple and len(data) == 2 and type(data[0]) == torch.Tensor: + if isinstance(data, tuple) and len(data) == 2 and isinstance(data[0], torch.Tensor): + data = (self.transform(data[0]), data[1]) + # elif type(data) is dict and "img" in data: + elif isinstance(data, dict) and "img" in data: + data["img"] = self.transform(data["img"]) + return data + + def __len__(self): + return self.length + + def __getattr__(self, item: str): + if isinstance(item, str) and (item in ("__setstate__", "__getstate__")): + raise AttributeError(item) + + return getattr(self.fullset, item) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..64351e7 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +junit_family = xunit1 +addopts = -s -v --cov --cov-config=./.coveragerc --junit-xml=./.reports/unittests.xml \ No newline at end of file diff --git a/tests/test_asha.py b/tests/test_asha.py deleted file mode 100644 index 0fd33e2..0000000 --- a/tests/test_asha.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -import pytest -import hpopt - -from torch.utils.data import Dataset -import os - - -def my_model(step, width, height): - return (0.1 + width * step / 100)**(-1) + height * 0.1 - - -def my_trainer(config): - width, height = config['params']['width'], config['params']['height'] - - for step in range(config["iterations"]): - score = my_model(step, width, height) - - if hpopt.report(config=config, score=score) == hpopt.Status.STOP: - break - -def test_asha(): - hp_configs = {"width": hpopt.SearchSpace("uniform", [10, 100]), - "height": hpopt.SearchSpace("uniform", [0, 100])} - - my_hpo = hpopt.create(save_path='./tmp/unittest', - search_alg="asha", - search_space=hp_configs, - mode='min', - num_trials=50, - max_iterations=10, - reduction_factor=3, - num_brackets=1, - resume=False, - subset_ratio=1.0, - num_full_iterations=20, - full_dataset_size=1000) - - assert type(my_hpo) == hpopt.hyperband.AsyncHyperBand - - config = my_hpo.get_next_sample() - - assert config['iterations'] > 1 - assert config['subset_ratio'] > 0.2 - assert config['file_path'] == './tmp/unittest/hpopt_trial_0.json' - - my_trainer(config) - - my_hpo.update_scores() - - assert my_hpo.hpo_status['config_list'][0]['score'] is not None - - num_trials_in_brackets = [0, 0, 0, 0, 0] - - while config is not None: - num_trials_in_brackets[config['bracket']] += 1 - - my_trainer(config) - - config = my_hpo.get_next_sample() - - assert sum(num_trials_in_brackets) == my_hpo.num_trials - - best_config = my_hpo.get_best_config() - - my_hpo.print_results() - - assert my_model(step=5, **best_config) <= my_model(5, 50, 50) - - print("best_config: ", best_config) - -if __name__ == '__main__': - test_asha() diff --git a/tests/test_base.py b/tests/test_base.py new file mode 100644 index 0000000..4759b11 --- /dev/null +++ b/tests/test_base.py @@ -0,0 +1,153 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import os +import pytest +import unittest + +from hpopt.base import HpOpt, SearchSpace + +# def test_test(): +# os.path.join(None, 'hello.txt') + +@pytest.fixture +def int_args(): + return [ + "num_init_trials", + "num_workers", + "num_full_iterations", + "full_dataset_size", + "min_subset_size" + ] + +@pytest.fixture +def positive_args(): + return [ + "num_init_trials", + "num_workers", + "num_full_iterations", + # "full_dataset_size", + ] + +@pytest.fixture +def optional_int_args(): + return [ + "num_trials", + "max_iterations", + ] + +@pytest.fixture +def union_float_int_args(): + return [ + "expected_time_ratio", + ] + +@pytest.fixture +def optional_union_float_int_args(): + return [ + "subset_ratio", + ] + + +class TestBase: + @pytest.mark.parametrize("input_val", [ 1.5, -1.0, "string", None]) + def test_init_HpOpt_type_check_int(self, int_args, input_val): + sp_arg = [SearchSpace('uniform', [0, 1])] + + for arg in int_args: + kwargs = {arg: input_val} + print(f"checking '{arg}' with '{input_val}'") + with pytest.raises(Exception) as e: + HpOpt( + search_space=sp_arg, + **kwargs + ) + print(e) + assert e.type == TypeError + + @pytest.mark.parametrize("input_val", [ 1.5, -1.0, "string"]) + def test_init_HpOpt_type_check_optinal_int(self, optional_int_args, input_val): + sp_arg = [SearchSpace('uniform', [0, 1])] + + for arg in optional_int_args: + kwargs = {arg: input_val} + print(f"checking '{arg}' with '{input_val}'") + with pytest.raises(Exception) as e: + HpOpt( + search_space=sp_arg, + **kwargs + ) + print(e) + assert e.type == TypeError + + @pytest.mark.parametrize("input_val", ["sting", None]) + def test_init_HpOpt_type_check_union_float_int(self, union_float_int_args, input_val): + sp_arg = [SearchSpace('uniform', [0, 1])] + + for arg in union_float_int_args: + kwargs = {arg: input_val} + print(f"checking '{arg}' with '{input_val}'") + with pytest.raises(Exception) as e: + HpOpt( + search_space=sp_arg, + **kwargs + ) + print(e) + assert e.type == TypeError + + @pytest.mark.parametrize("input_val", [-1, 0]) + def test_init_HpOpt_type_check_union_float_int(self, positive_args, input_val): + sp_arg = [SearchSpace('uniform', [0, 1])] + + for arg in positive_args: + kwargs = {arg: input_val} + print(f"checking '{arg}' with '{input_val}'") + with pytest.raises(Exception) as e: + HpOpt( + search_space=sp_arg, + **kwargs + ) + print(e) + assert e.type == ValueError + + @pytest.mark.parametrize("input_val", ["string"]) + def test_init_HpOpt_type_check_union_float_int(self, optional_union_float_int_args, input_val): + sp_arg = [SearchSpace('uniform', [0, 1])] + + for arg in optional_union_float_int_args: + kwargs = {arg: input_val} + print(f"checking '{arg}' with '{input_val}'") + with pytest.raises(Exception) as e: + HpOpt( + search_space=sp_arg, + **kwargs + ) + print(e) + assert e.type == TypeError + +class TestSearhSpace(unittest.TestCase): + def test_search_space(self): + hp_configs = { + 'a': SearchSpace("uniform", [-5, 10]), + 'b': SearchSpace("quniform", [2, 14, 2]), + 'c': SearchSpace("loguniform", [0.0001, 0.1]), + 'd': SearchSpace("qloguniform", [2, 256, 2]), + 'e': SearchSpace("choice", [98, 765, 4, 321])} + + assert len(hp_configs) == 5 + + assert hp_configs['a'].space_to_real(hp_configs['a'].lower_space()) == -5 + assert hp_configs['a'].space_to_real(hp_configs['a'].upper_space()) == 10 + + assert hp_configs['b'].space_to_real(hp_configs['b'].lower_space()) == 2 + assert hp_configs['b'].space_to_real(hp_configs['b'].upper_space()) == 14 + + assert round(hp_configs['c'].space_to_real((hp_configs['c'].lower_space())), 4) == 0.0001 + assert round(hp_configs['c'].space_to_real((hp_configs['c'].upper_space())), 1) == 0.1 + + assert round(hp_configs['d'].space_to_real((hp_configs['d'].lower_space())), 0) == 2 + assert round(hp_configs['d'].space_to_real((hp_configs['d'].upper_space())), 0) == 256 + + assert hp_configs['e'].space_to_real(hp_configs['e'].lower_space()) == 98 + assert hp_configs['e'].space_to_real(hp_configs['e'].upper_space()) == 321 \ No newline at end of file diff --git a/tests/test_bayes_opt.py b/tests/test_bayes_opt.py index 1759599..434b503 100644 --- a/tests/test_bayes_opt.py +++ b/tests/test_bayes_opt.py @@ -3,11 +3,14 @@ # import pytest +import unittest import hpopt +from hpopt.utils import HpoDataset from torch.utils.data import Dataset import os + class my_dataset(Dataset): def __getitem__(self, index): return [5678] @@ -15,18 +18,21 @@ def __getitem__(self, index): def __len__(self): return 2000 + def my_model(a, b, c, d, e): return a * b + c * d - e + def my_trainer(config): full_dataset = my_dataset() - new_dataset = hpopt.createHpoDataset(full_dataset, config) + new_dataset = HpoDataset(full_dataset, config) # assert len(new_dataset) == 1800 assert new_dataset[234][0] == 5678 for iteration_num in range(config["iterations"]): score = my_model(**config["params"]) + score += iteration_num retval = hpopt.report(config=config, score=score) assert (retval == hpopt.Status.STOP) or (retval == hpopt.Status.RUNNING) @@ -34,191 +40,141 @@ def my_trainer(config): if retval == hpopt.Status.STOP: break -def _test_bayes_opt(): - hp_configs = {'a': hpopt.SearchSpace("uniform", [-5, 10]), - 'b': hpopt.SearchSpace("quniform", [2, 14, 2]), - 'c': hpopt.SearchSpace("loguniform", [0.0001, 0.1]), - 'd': hpopt.SearchSpace("qloguniform", [2, 256, 2]), - 'e': hpopt.SearchSpace("choice", [98, 765, 4, 321])} - - assert len(hp_configs) == 5 - - assert hp_configs['a'].space_to_real(hp_configs['a'].lower_space()) == -5 - assert hp_configs['a'].space_to_real(hp_configs['a'].upper_space()) == 10 - - assert hp_configs['b'].space_to_real(hp_configs['b'].lower_space()) == 2 - assert hp_configs['b'].space_to_real(hp_configs['b'].upper_space()) == 14 - - assert round(hp_configs['c'].space_to_real((hp_configs['c'].lower_space())), 4) == 0.0001 - assert round(hp_configs['c'].space_to_real((hp_configs['c'].upper_space())), 1) == 0.1 - - assert round(hp_configs['d'].space_to_real((hp_configs['d'].lower_space())), 0) == 2 - assert round(hp_configs['d'].space_to_real((hp_configs['d'].upper_space())), 0) == 256 - - assert hp_configs['e'].space_to_real(hp_configs['e'].lower_space()) == 98 - assert hp_configs['e'].space_to_real(hp_configs['e'].upper_space()) == 321 - - my_hpo = hpopt.create(num_init_trials='5', num_full_iterations=1, full_dataset_size=1) - assert my_hpo is None - - try: - my_hpo = hpopt.create(save_path='./tmp/unittest', - search_alg="bayes_opt", - search_space=hp_configs, - ealry_stop="median_stop", - num_init_trials='5', - num_trials=2, - max_iterations=2, - subset_ratio=0.9, - image_resize=(123, 456), - resume=False, - num_full_iterations=1, - full_dataset_size=1) - except TypeError: - assert True - - try: - my_hpo = hpopt.create(save_path='./tmp/unittest', - search_alg="bayes_opt", - search_space=hp_configs, - ealry_stop="median_stop", - num_init_trials=1, - num_trials=2, - max_iterations=2, - subset_ratio=0.9, - image_resize=(123, 456), - resume=False) - except ValueError: - assert True - - try: - my_hpo = hpopt.create(save_path='./tmp/unittest', - search_alg="bayes_opt", - search_space=hp_configs, - ealry_stop="median_stop", - num_init_trials=5, - num_trials=2, - max_iterations='2', - subset_ratio=0.9, - image_resize=(123, 456), - resume=False, - num_full_iterations=1, - full_dataset_size=1) - except TypeError: - assert True - - my_hpo = hpopt.create(save_path='./tmp/unittest', - search_alg="bayes_opt", - search_space=hp_configs, - ealry_stop="median_stop", - num_init_trials=5, - num_trials=7, - max_iterations=2, - subset_ratio=0.9, - image_resize=(123, 456), - resume=False, - num_full_iterations=1, - full_dataset_size=1) - - assert type(my_hpo) == hpopt.smbo.BayesOpt - - config = my_hpo.get_next_sample() - - assert config['iterations'] == 2 - assert config['file_path'] == './tmp/unittest/hpopt_trial_0.json' - assert config['early_stop'] is None - assert config['subset_ratio'] == 0.9 - assert config['resize_width'] == 123 - assert config['resize_height'] == 456 - - my_trainer(config) - - my_hpo.update_scores() - - assert my_hpo.hpo_status['config_list'][0]['score'] is not None - - my_hpo = hpopt.create(save_path='./tmp/unittest', - search_alg="bayes_opt", - search_space=hp_configs, - ealry_stop="median_stop", - num_init_trials=5, - num_trials=7, - max_iterations=2, - subset_ratio=0.9, - image_resize=(123, 456), - resume=True, - num_full_iterations=1, - full_dataset_size=1) - - assert type(my_hpo) == hpopt.smbo.BayesOpt - - configs = my_hpo.get_next_samples() - - assert len(configs) == 4 - - for config in configs: +class TestBayesOpt(unittest.TestCase): + def setUp(self): + self.output = './tmp/unittests' + self.hp_configs = { + 'a': hpopt.SearchSpace("uniform", [-5, 10]), + 'b': hpopt.SearchSpace("quniform", [2, 14, 2]), + 'c': hpopt.SearchSpace("loguniform", [0.0001, 0.1]), + 'd': hpopt.SearchSpace("qloguniform", [2, 256, 2]), + 'e': hpopt.SearchSpace("choice", [98, 765, 4, 321]) + } + + def tearDown(self): + try: + os.remove(hpopt.get_status_path(self.output)) + except FileNotFoundError: + pass + # pass + + def test_bayes_opt_resume(self): + init_trials = remained_trials = 5 + my_hpo = hpopt.create(save_path=self.output, + search_alg="bayes_opt", + search_space=self.hp_configs, + early_stop="median_stop", + num_init_trials=init_trials, + num_trials=7, + max_iterations=2, + subset_ratio=0.9, + image_resize=[123, 456], + resume=False, + num_full_iterations=1, + full_dataset_size=1) + + assert type(my_hpo) == hpopt.smbo.BayesOpt + + config = my_hpo.get_next_sample() + + assert config['iterations'] == 2 + assert config['file_path'] == os.path.join(self.output, 'hpopt_trial_0.json') + assert config['early_stop'] is None + assert config['subset_ratio'] == 0.9 + assert config['resize_width'] == 123 + assert config['resize_height'] == 456 + my_trainer(config) + remained_trials -= 1 + my_hpo.update_scores() + assert my_hpo.hpo_status['config_list'][0]['score'] is not None, f"hpo_status = {my_hpo.hpo_status}" + del my_hpo - configs = my_hpo.get_next_samples() + # create same thing and check whether it resume correctly + my_hpo = hpopt.create(save_path=self.output, + search_alg="bayes_opt", + search_space=self.hp_configs, + early_stop="median_stop", + num_init_trials=init_trials, + num_trials=7, + max_iterations=2, + subset_ratio=0.9, + image_resize=[123, 456], + resume=True, + num_full_iterations=1, + full_dataset_size=1) - assert len(configs) == 1 + assert type(my_hpo) == hpopt.smbo.BayesOpt - my_trainer(configs[0]) + configs = my_hpo.get_next_samples() - config = my_hpo.get_next_sample() + assert len(configs) == remained_trials - assert config is not None + # for config in configs: + # my_trainer(config) + # init_trials -= 1 - my_trainer(config) + # configs = my_hpo.get_next_samples() - config = my_hpo.get_next_sample() + # assert len(configs) == 1 - assert config is None + # my_trainer(configs[0]) - my_hpo = hpopt.create(save_path='./tmp/unittest', - search_alg="bayes_opt", - search_space=hp_configs, - #ealry_stop="median_stop", - #num_init_trials=5, - #num_trials=2, - #max_iterations=2, - #subset_ratio=0.9, - #image_resize=(123, 456), - resume=False, - num_full_iterations=20, - full_dataset_size=1000) + # config = my_hpo.get_next_sample() - assert my_hpo is not None + # assert config is not None - while True: - config = my_hpo.get_next_sample() + # my_trainer(config) - if config is None: - break + # config = my_hpo.get_next_sample() - my_trainer(config) + # assert config is None + + + # del my_hpo + # my_hpo = hpopt.create(save_path=self.output, + # search_alg="bayes_opt", + # search_space=hp_configs, + # ealry_stop="median_stop", + # num_init_trials=5, + # num_trials=2, + # max_iterations=2, + # subset_ratio=0.9, + # image_resize=(123, 456), + # resume=False, + # num_full_iterations=1, + # full_dataset_size=1) + + # assert os.path.exists(hpopt.get_status_path(self.output)) + + def _test_bayes_opt(self): + my_hpo = hpopt.create(save_path=self.output, + search_alg="bayes_opt", + search_space=self.hp_configs, + #ealry_stop="median_stop", + #num_init_trials=5, + #num_trials=2, + #max_iterations=2, + #subset_ratio=0.9, + #image_resize=(123, 456), + resume=False, + num_full_iterations=20, + full_dataset_size=1000) + + assert my_hpo is not None - best_config = my_hpo.get_best_config() + while True: + config = my_hpo.get_next_sample() - my_hpo.print_results() + if config is None: + break - assert my_model(**best_config) >= my_model(3, 8, 0.01, 128, 98) + my_trainer(config) - print("best_config: ", best_config) + best_config = my_hpo.get_best_config() - my_hpo = hpopt.create(save_path='./tmp/unittest', - search_alg="bayes_opt", - search_space=hp_configs, - ealry_stop="median_stop", - num_init_trials=5, - num_trials=2, - max_iterations=2, - subset_ratio=0.9, - image_resize=(123, 456), - resume=False, - num_full_iterations=1, - full_dataset_size=1) + my_hpo.print_results() - assert os.path.exists(hpopt.get_status_path('./tmp/unittest')) + assert my_model(**best_config) >= my_model(3, 8, 0.01, 128, 98) - os.remove(hpopt.get_status_path('./tmp/unittest')) + print("best_config: ", best_config) \ No newline at end of file diff --git a/tests/test_hpopt.py b/tests/test_hpopt.py index 37621f8..502f98e 100644 --- a/tests/test_hpopt.py +++ b/tests/test_hpopt.py @@ -5,82 +5,74 @@ import os import pytest -from hpopt.hpopt import get_cutoff_score, load_json +from hpopt.base import SearchSpace +from hpopt.smbo import BayesOpt from hpopt.hyperband import AsyncHyperBand -def get_trial_config(config_list, trial): - for config in config_list: - if config['trial_id'] == trial: - return config - return None - -def test_get_cutoff_score_with_result_json(): - num_imgs = 240 - max_epochs = 300 - min_epochs = 7 - rf = 2.0 - s = 0 - - save_path = './tests/assets/hpo_results_1' - config_list = load_json(os.path.join(save_path, 'hpopt_status.json'))['config_list'] - - trial = 1 - trial_result = load_json(os.path.join(save_path, f'hpopt_trial_{trial}.json')) - max_t = num_imgs * max_epochs - min_t = num_imgs * min_epochs - target_ep = 0 - target_score = -1 - num_images = -1 - for ep, img in enumerate(trial_result['images']): - if ep == target_ep: - num_images = img - target_score = trial_result['scores'][ep] - break - assert num_images == 4560 - assert target_score == 0.005 - rungs = AsyncHyperBand.get_rungs(min_t, max_t, rf, s) - print(f"rungs = {rungs}") - target_rung_idx = -1 - for rung_idx, rung in enumerate(rungs): - if rung < num_images: - break - target_rung_idx = rung_idx - target_rung = rungs[target_rung_idx] - print(f"target_rung = {target_rung}") - mode = 'max' - cutoff_score = get_cutoff_score(save_path, target_rung, rungs, mode) - - print(f"cutoff_score for rung {target_rung} = {cutoff_score}") - assert not (cutoff_score > target_score), f"cutoff score = {cutoff_score}, score @ trial{trial}[{target_ep}] = {target_score}" - - trial = 6 - trial_result = load_json(os.path.join(save_path, f'hpopt_trial_{trial}.json')) - # trial_config = get_trial_config(config_list, trial) - # assert trial_config is not None - # batch_size = trial_config['config']['learning_parameters.batch_size'] - # print(f"batch_size for trial{trial} = {batch_size}") - # max_t = num_imgs * max_epochs - # min_t = num_imgs * min_epochs - target_ep = 8 - target_score = -1 - num_images = -1 - for ep, img in enumerate(trial_result['images']): - if ep == target_ep: - num_images = img - target_score = trial_result['scores'][ep] - break - assert num_images == 36504, f"num images = {num_images}" - assert target_score == 0.23577335476875305, f"score @ trial{trial} = {target_score}" - # rungs = AsyncHyperBand.get_rungs(min_t, max_t, rf, s) - print(f"rungs = {rungs}") - target_rung_idx = -1 - for rung_idx, rung in enumerate(rungs): - if rung < num_images: - break - target_rung_idx = rung_idx - target_rung = rungs[target_rung_idx] - print(f"target_rung[{target_rung_idx}] = {target_rung}") - mode = 'max' - cutoff_score = get_cutoff_score(save_path, target_rung, rungs, mode) - print(f"cutoff_score for rung {target_rung} = {cutoff_score}") - assert cutoff_score > target_score, f"cutoff score = {cutoff_score}, score @ trial{trial}[{target_ep}] = {target_score}" \ No newline at end of file +from hpopt import ( + create, + finalize_trial, + get_best_score, + get_best_score_with_num_imgs, + get_current_status, + get_previous_status, + get_status_path, + get_trial_path, + get_trial_results, + load_json, + report, + reportOOM, +) + + +def test_create(): + hpo = create(100, 100, { "num": SearchSpace("uniform", [0, 10]) }, num_trials=1) + assert isinstance(hpo, BayesOpt) + + hpo = create(100, 100, { "num": SearchSpace("uniform", [0, 10]) }, search_alg="asha") + assert isinstance(hpo, AsyncHyperBand) + + +def test_finalize_trial(): + assert True + + +def test_get_best_score(): + assert True + + +def test_get_best_score_with_num_imgs(): + assert True + + +def test_get_current_status(): + assert True + + +def test_get_previous_status(): + assert True + + +def test_get_status_path(): + assert True + + +def test_get_trial_path(): + assert True + + +def test_get_trial_results(): + assert True + + +def test_load_json(): + assert True + + +def test_report(): + assert True + + +def test_reportOOM(): + assert True + diff --git a/tests/test_hyperband.py b/tests/test_hyperband.py new file mode 100644 index 0000000..79e7911 --- /dev/null +++ b/tests/test_hyperband.py @@ -0,0 +1,226 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import os +import pytest +# import hpopt + +# from torch.utils.data import Dataset +# import os +from hpopt import load_json +from hpopt.hyperband import AsyncHyperBand, _get_cutoff_score, _find_idx_of_max_less_than_target +# def my_model(step, width, height): +# return (0.1 + width * step / 100)**(-1) + height * 0.1 + + +# def my_trainer(config): +# width, height = config['params']['width'], config['params']['height'] + +# for step in range(config["iterations"]): +# score = my_model(step, width, height) + +# if hpopt.report(config=config, score=score) == hpopt.Status.STOP: +# break + +# def _test_asha(): +# hp_configs = {"width": hpopt.SearchSpace("uniform", [10, 100]), +# "height": hpopt.SearchSpace("uniform", [0, 100])} + +# my_hpo = hpopt.create(save_path='./tmp/unittest', +# search_alg="asha", +# search_space=hp_configs, +# mode='min', +# num_trials=50, +# max_iterations=10, +# reduction_factor=3, +# num_brackets=1, +# resume=False, +# subset_ratio=1.0, +# num_full_iterations=20, +# full_dataset_size=1000) + +# assert type(my_hpo) == hpopt.hyperband.AsyncHyperBand + +# config = my_hpo.get_next_sample() + +# assert config['iterations'] > 1 +# assert config['subset_ratio'] > 0.2 +# assert config['file_path'] == './tmp/unittest/hpopt_trial_0.json' + +# my_trainer(config) + +# my_hpo.update_scores() + +# assert my_hpo.hpo_status['config_list'][0]['score'] is not None + +# num_trials_in_brackets = [0, 0, 0, 0, 0] + +# while config is not None: +# num_trials_in_brackets[config['bracket']] += 1 + +# my_trainer(config) + +# config = my_hpo.get_next_sample() + +# assert sum(num_trials_in_brackets) == my_hpo.num_trials + +# best_config = my_hpo.get_best_config() + +# my_hpo.print_results() + +# assert my_model(step=5, **best_config) <= my_model(5, 50, 50) + +# print("best_config: ", best_config) + +# if __name__ == '__main__': +# test_asha() + +def test_find_idx_of_max_less_than_target(): + trial_json = { + "status": 2, + "scores": [ + 0.7077550888061523, + 0.3251700699329376, + 0.5283446907997131, + 0.6163265109062195, + 0.47252747416496277, + 0.60317462682724, + 0.6775510311126709, + 0.75, + 0.7571428418159485, + 0.75, + 0.7687075138092041, + 0.8367347121238708, + 0.7142857313156128, + 0.75, + 0.8571428656578064, + 0.8571428656578064 + ], + "median": [ + 0.7077550888061523, + 0.516462579369545, + 0.5204232831796011, + 0.5443990901112556, + 0.5300247669219971, + 0.5422164102395376, + 0.5615499275071281, + 0.585106186568737, + 0.6042213704850938, + 0.6187992334365845, + 0.6324272589250044, + 0.64945288002491, + 0.6544400224318871, + 0.6612657351153237, + 0.6743242104848226, + 0.6857503764331341 + ], + "images": [ + 4560, + 9120, + 13680, + 18240, + 22800, + 27360, + 31920, + 36480, + 41040, + 45600, + 50160, + 54720, + 59280, + 63840, + 68400, + 72960 + ] + } + rungs = [67215, 34447, 18063, 9871, 5775, 3727, 2703, 2191, 1935, 1807, 1743, 1711, 1695, 1687, 1683, 1681, 1680] + + # target is out of range of the rungs + target_out_of_range = 0 + idx = _find_idx_of_max_less_than_target(trial_json["images"], target_out_of_range) + print(f"found idx = {idx}") + assert idx == -1 + + target_in_range = rungs[4] + idx = _find_idx_of_max_less_than_target(trial_json["images"], target_in_range) + print(f"found idx = {idx}") + assert idx != -1 + + target_exact_matched = 31920 + idx = _find_idx_of_max_less_than_target(trial_json["images"], target_exact_matched) + print(f"found idx = {idx}") + assert idx == trial_json["images"].index(target_exact_matched) + + +def test_get_cutoff_score_with_result_json(): + num_imgs = 240 + max_epochs = 300 + min_epochs = 7 + rf = 2.0 + s = 0 + + save_path = './tests/assets/hpo_results_1' + config_list = load_json(os.path.join(save_path, 'hpopt_status.json'))['config_list'] + + trial = 1 + trial_result = load_json(os.path.join(save_path, f'hpopt_trial_{trial}.json')) + max_t = num_imgs * max_epochs + min_t = num_imgs * min_epochs + target_ep = 0 + target_score = -1 + num_images = -1 + for ep, img in enumerate(trial_result['images']): + if ep == target_ep: + num_images = img + target_score = trial_result['scores'][ep] + break + assert num_images == 4560 + assert target_score == 0.005 + rungs = AsyncHyperBand.get_rungs(min_t, max_t, rf, s) + print(f"rungs = {rungs}") + target_rung_idx = -1 + for rung_idx, rung in enumerate(rungs): + if rung < num_images: + break + target_rung_idx = rung_idx + target_rung = rungs[target_rung_idx] + print(f"target_rung = {target_rung}") + mode = 'max' + cutoff_score = _get_cutoff_score(save_path, target_rung, rungs, mode) + + print(f"cutoff_score for rung {target_rung} = {cutoff_score}") + assert not (cutoff_score > target_score), f"cutoff score = {cutoff_score}, score @ trial{trial}[{target_ep}] = {target_score}" + + trial = 6 + trial_result = load_json(os.path.join(save_path, f'hpopt_trial_{trial}.json')) + # trial_config = get_trial_config(config_list, trial) + # assert trial_config is not None + # batch_size = trial_config['config']['learning_parameters.batch_size'] + # print(f"batch_size for trial{trial} = {batch_size}") + # max_t = num_imgs * max_epochs + # min_t = num_imgs * min_epochs + target_ep = 10 + target_score = -1 + num_images = -1 + for ep, img in enumerate(trial_result['images']): + if ep == target_ep: + num_images = img + target_score = trial_result['scores'][ep] + break + assert num_images == 36504, f"num images = {num_images}" + assert target_score == 0.23577335476875305, f"score @ trial{trial} = {target_score}" + # rungs = AsyncHyperBand.get_rungs(min_t, max_t, rf, s) + print(f"rungs = {rungs}") + target_rung_idx = -1 + for rung_idx, rung in enumerate(rungs): + target_rung_idx = rung_idx + if rung < num_images: + break + target_rung = rungs[target_rung_idx] + print(f"target_rung[{target_rung_idx}] = {target_rung}") + mode = 'max' + cutoff_score = get_cutoff_score(save_path, target_rung, rungs, mode) + print(f"cutoff_score for rung {target_rung} = {cutoff_score}") + assert cutoff_score > target_score, f"cutoff score = {cutoff_score}, score @ trial{trial}[{target_ep}] = {target_score}" + diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..bc39cdc --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,121 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import os +import pytest + +from hpopt.utils import type_check + +def test_utils_type_check(): + assert type_check(1, int) == True + assert type_check(1, [int]) == True + assert type_check(1.0, float) == True + assert type_check(1.0, [float]) == True + + assert type_check(1, [int, float]) == True + assert type_check(1.0, [int, float]) == True + + assert type_check(None, [int, type(None)]) == True + assert type_check("hello", [str]) == True + assert type_check("hello", [str, type(None)]) == True + assert type_check(None, [str, type(None)]) == True + + assert type_check(-1, int) == True + + assert type_check(1, int, non_zero=True) == True + assert type_check(10, int, non_zero=True) == True + assert type_check(-1, int, non_zero=True) == True + + assert type_check(1, int, non_zero=True, positive=True) == True + assert type_check(10, int, non_zero=True, positive=True) == True + assert type_check(-1, int, non_zero=True, positive=False) == True + + assert type_check(1, [int, type(None)], non_zero=True, positive=True) == True + assert type_check(10, [int, type(None)], non_zero=True, positive=True) == True + assert type_check(None, [int, type(None)], non_zero=True, positive=True) == True + + assert type_check(1, [int, float, type(None)]) == True + assert type_check(1.0, [int, float, type(None)]) == True + assert type_check(0, [int, float, type(None)]) == True + assert type_check(-1.0, [int, float, type(None)]) == True + assert type_check(-1, [int, float, type(None)]) == True + assert type_check(None, [int, float, type(None)]) == True + + assert type_check(1.0, [int, float, type(None)], positive=True) == True + assert type_check(-1.0, [int, float, type(None)], positive=False) == True + assert type_check(-1, [int, float, type(None)], positive=False) == True + assert type_check(None, [int, float, type(None)], positive=True) == True + + with pytest.raises(Exception) as e: + type_check(1, str) + assert e.type == TypeError + + with pytest.raises(Exception) as e: + type_check(None, str) + assert e.type == TypeError + + with pytest.raises(Exception) as e: + type_check(1.0, int) + assert e.type == TypeError + + with pytest.raises(Exception) as e: + type_check(-1.0, int) + assert e.type == TypeError + + with pytest.raises(Exception) as e: + type_check(None, int) + assert e.type == TypeError + + with pytest.raises(Exception) as e: + type_check(0, float) + assert e.type == TypeError + + with pytest.raises(Exception) as e: + type_check(None, float) + assert e.type == TypeError + + with pytest.raises(Exception) as e: + type_check("hello", [float, int]) + assert e.type == TypeError + + with pytest.raises(Exception) as e: + type_check(None, [float, int]) + assert e.type == TypeError + + with pytest.raises(Exception) as e: + type_check(-1.0, float, positive=True) + assert e.type == ValueError + + with pytest.raises(Exception) as e: + type_check(-1, int, positive=True) + assert e.type == ValueError + + with pytest.raises(Exception) as e: + type_check(0, int, non_zero=True) + assert e.type == ValueError + + with pytest.raises(Exception) as e: + type_check(-1, int, non_zero=True, positive=True) + assert e.type == ValueError + + with pytest.raises(Exception) as e: + assert type_check(1, int, non_zero=True, positive=False) + assert e.type == ValueError + + with pytest.raises(Exception) as e: + type_check(10, int, non_zero=True, positive=False) + assert e.type == ValueError + + with pytest.raises(Exception) as e: + type_check(-1, [int, type(None)], non_zero=True, positive=True) + assert e.type == ValueError + + with pytest.raises(Exception) as e: + type_check(0, [int, float, type(None)], positive=True, non_zero=True) + assert e.type == ValueError + + my_variable_name_strange = 1.0 + with pytest.raises(Exception) as e: + type_check(my_variable_name_strange, int) + assert "my_variable_name_strange" in str(e.value) From 40f6b2d7bcd9436ed4b69cda0d42346c2d3b4b9f Mon Sep 17 00:00:00 2001 From: Yunchu Lee Date: Thu, 14 Jul 2022 11:41:22 +0900 Subject: [PATCH 2/2] fixed typo Signed-off-by: Yunchu Lee --- hpopt/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hpopt/utils.py b/hpopt/utils.py index c1472ba..0637d36 100644 --- a/hpopt/utils.py +++ b/hpopt/utils.py @@ -42,7 +42,7 @@ def type_check( name_of_input = "input" if isinstance(callers_local_vars, ItemsView): - name_list = [var_name for var_name, var_val in callers_local_vars if id(var_val) == id(input)] + name_list = [var_name for var_name, var_val in callers_local_vars if id(var_val) == id(_input)] name_of_input = name_list[0] if len(name_list) > 0 and _input is not None else "input" allowed = False @@ -59,7 +59,7 @@ def type_check( f"expected type(s) of the '{name_of_input}': {expected_type}, but the input type is {type(_input)}" ) - if positive is not None and input is not None: + if positive is not None and _input is not None: if positive and _input < 0: raise ValueError(f"'{name_of_input}' is expected to be positive number but {_input}.") if not positive and _input > 0: