diff --git a/pyproject.toml b/pyproject.toml index 0c4e31e..08201db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,9 @@ select = [ "B", # flake8-bugbear "SIM", # flake8-simplify ] +ignore = [ + "E501", +] [tool.pytest.ini_options] testpaths = [ @@ -85,3 +88,6 @@ testpaths = [ addopts = [ "--import-mode=importlib", ] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", +] diff --git a/src/onc/modules/_DataProductFile.py b/src/onc/modules/_DataProductFile.py index b3d1e23..16b11fa 100644 --- a/src/onc/modules/_DataProductFile.py +++ b/src/onc/modules/_DataProductFile.py @@ -15,11 +15,23 @@ def __init__(self, max_retries): class _DataProductFile: """ - Donwloads a single data product file + Downloads a single data product file Is able to poll and wait if required """ - def __init__(self, dpRunId: int, index: str, baseUrl: str, token: str): + def __init__( + self, + dpRunId: int, + index: str, + baseUrl: str, + token: str, + verbosity: str = "INFO", + ): + self.verbosity = verbosity + # Use child logger 'onc.poll' consistent with _PollLog + from ._Messages import setup_logger + + self._log = setup_logger("onc.poll", verbosity) self._retries = 0 self._status = 202 self._downloaded = False @@ -50,7 +62,7 @@ def download( Can poll, wait and retry if the file is not ready to download Return the file information """ - log = _PollLog(True) + log = _PollLog(self.verbosity) self._status = 202 while self._status == 202: # Run timed request @@ -74,9 +86,9 @@ def download( response, outPath, filename, overwrite ) except FileExistsError: - if self._retries > 1: - print("") - print(f' Skipping "{self._filePath}": File already exists.') + self._log.info( + f' Skipping "{self._filePath}": File already exists.' + ) self._status = 777 elif self._status == 202: # Still processing, wait and retry @@ -84,10 +96,9 @@ def download( sleep(pollPeriod) elif self._status == 204: # No data found - print(" No data found.") + self._log.info(" No data found.") elif self._status == 404: # Index too high, no more files to download - log.printNewLine() pass elif self._status == 410: # Status 410: gone (file deleted from FTP) @@ -98,7 +109,8 @@ def download( stacklevel=2, ) else: - raise requests.HTTPError(_createErrorMessage(response)) + self._log.error(_createErrorMessage(response)) + response.raise_for_status() return self._status diff --git a/src/onc/modules/_Messages.py b/src/onc/modules/_Messages.py new file mode 100644 index 0000000..aa368ee --- /dev/null +++ b/src/onc/modules/_Messages.py @@ -0,0 +1,145 @@ +import logging +import re +import time + +import requests + +REQ_MSG = "Requested: {}" # get request url +RESPONSE_TIME_MSG = "Response received in {} seconds." # requests.elapsed value. +RESPONSE_MSG = "HTTP Response: {} ({})" # Brief description, status code +MULTIPAGE_MSG = ( + "The requested data quantity is greater than the " + "supplied row limit and will be downloaded over multiple requests." +) + + +LEVEL_MAP = { + "CRITICAL": logging.CRITICAL, + "ERROR": logging.ERROR, + "WARNING": logging.WARNING, + "INFO": logging.INFO, + "DEBUG": logging.DEBUG, +} + + +class OnclibFormatter(logging.Formatter): + """ + Custom formatter that removes prefix for INFO level logs. + """ + + def format(self, record): + if record.levelno == logging.INFO: + return record.getMessage() + return super().format(record) + + +def setup_logger(logger_name: str = "onc", level: int | str = "INFO") -> logging.Logger: + """ + Set up a logger object for displaying verbose messages to console. + + Parameters + ---------- + logger_name : str, optional + The unique logger name to use. Can be shared between modules. + level : int or str, optional + The logging level to use. Default is 'INFO'. + + Returns + ------- + logging.Logger + The configured logging.Logger object. + """ + + # Ensure level is a valid logging level integer + if isinstance(level, str): + level = LEVEL_MAP.get(level.upper(), logging.INFO) + + logger = logging.getLogger(logger_name) + logger.propagate = False + + # Apply level to the logger itself + logger.setLevel(level) + + if not logger.handlers: + console = logging.StreamHandler() + console.setLevel(level) + + # Set the logging format. + dtfmt = "%Y-%m-%dT%H:%M:%S" + strfmt = ( + "%(asctime)s.%(msecs)03dZ | %(name)-12s | %(levelname)-8s | %(message)s" + ) + fmt = OnclibFormatter(strfmt, datefmt=dtfmt) + fmt.converter = time.gmtime + + console.setFormatter(fmt) + logger.addHandler(console) + else: + # If handlers exist, ensure they have the correct level + for handler in logger.handlers: + handler.setLevel(level) + + return logger + + +def scrub_token(input: str) -> str: + """ + Replace a token in a query URL or other string with the string 'REDACTED' + so that users don't accidentally commit their tokens to public repositories + if ONC Info/Warnings are too verbose. + + Parameters + ---------- + input : str + An Oceans 3.0 API URL or string with a token query parameter. + + Returns + ------- + str + A scrubbed url. + """ + return re.sub(r"([?&]token=)[a-f0-9-]{36}", r"\1REDACTED", input) + + +def build_error_message(response: requests.Response, redact_token: bool) -> str: + """ + Build an error message from a requests.Response object. + + Parameters + ---------- + response : requests.Response + A requests.Response object. + redact_token : bool + If true, redact tokens before returning an error message. + + Returns + ------- + str + An error message. + """ + try: + payload = response.json() + except requests.exceptions.JSONDecodeError: + return response.text.strip() or ( + f"HTTP {response.status_code} {response.reason}" + ) + + message = payload.get("message") + + if "errors" in payload: + errors = payload["errors"] + error_messages = [] + for error in errors: + emsg = ( + f"(API Error Code {error['errorCode']}) " + f"{error['errorMessage']} for query parameter(s) " + f"'{error['parameter']}'." + ) + error_messages.append(emsg) + error_message = "\n".join(error_messages) + else: + error_message = None + msg = "\n".join([m for m in (message, error_message) if m is not None]) + if redact_token is True and "token=" in msg: + msg = scrub_token(msg) + return msg diff --git a/src/onc/modules/_MultiPage.py b/src/onc/modules/_MultiPage.py index 6369773..af458de 100644 --- a/src/onc/modules/_MultiPage.py +++ b/src/onc/modules/_MultiPage.py @@ -4,7 +4,9 @@ from time import time import dateutil.parser -import humanize +from onc.modules._Messages import ( + setup_logger, +) from ._util import _formatDuration @@ -14,12 +16,25 @@ class _MultiPage: def __init__(self, parent: object): self.parent = weakref.ref(parent) self.result = None + self._log = setup_logger("onc.multi", self._config("verbosity") or "INFO") + + def _config(self, key): + p = self.parent() + if p is None: + return None + if hasattr(p, "_config"): + return p._config(key) + return getattr(p, key, None) def getAllPages(self, service: str, url: str, filters: dict): """ Requests all pages from the service, with the url and filters Multiple pages will be downloaded until completed - @return: Service response with concatenated data for all pages obtained + + Returns + ------- + dict + Service response with concatenated data for all pages obtained. """ # pop archivefiles extension extension = None @@ -30,48 +45,54 @@ def getAllPages(self, service: str, url: str, filters: dict): # download first page start = time() response, responseTime = self._doPageRequest(url, filters, service, extension) - rNext = response["next"] - - if rNext is not None: - print( - "Data quantity is greater than the row limit and", - "will be downloaded in multiple pages.", - ) - - pageCount = 1 - pageEstimate = self._estimatePages(response, service) - if pageEstimate > 0: - # Exclude the first page when calculating the time estimation - timeEstimate = _formatDuration((pageEstimate - 1) * responseTime) - print( - f"Downloading time for the first page: {humanize.naturaldelta(responseTime)}" # noqa: E501 - ) - print(f"Estimated approx. {pageEstimate} pages in total.") - print( - f"Estimated approx. {timeEstimate} to complete for the rest of the pages." # noqa: E501 - ) - # keep downloading pages until next is None - print("") - while rNext is not None: - pageCount += 1 - rowCount = self._rowCount(response, service) + if isinstance(response, dict): + rNext = response["next"] - print(f" ({rowCount} samples) Downloading page {pageCount}...") - nextResponse, nextTime = self._doPageRequest( - url, rNext["parameters"], service, extension + if rNext is not None: + self._log.info( + "The requested data quantity is greater than the supplied " + "row limit and will be downloaded over multiple requests." ) - rNext = nextResponse["next"] - - # concatenate new data obtained - self._catenateData(response, nextResponse, service) - totalTime = _formatDuration(time() - start) - print( - f" ({self._rowCount(response, service):d} samples)" - f" Completed in {totalTime}." - ) - response["next"] = None + pageCount = 1 + pageEstimate = self._estimatePages(response, service) + if pageEstimate > 0: + # Exclude the first page when calculating the time estimation + timeEstimate = _formatDuration((pageEstimate - 1) * responseTime) + self._log.info( + f"Download time for page {pageCount}: {round(responseTime, 2)} seconds" + ) + self._log.info( + f"Est. number of pages remaining for download: {pageEstimate - 1}" + ) + self._log.info( + f"Est. number of seconds to download remaining data: {timeEstimate}" + ) + + # keep downloading pages until next is None + while rNext is not None: + pageCount += 1 + rowCount = self._rowCount(response, service) + + self._log.info( + f" Submitting request for page {pageCount} ({rowCount} samples)..." + ) + + nextResponse, nextTime = self._doPageRequest( + url, rNext["parameters"], service, extension + ) + rNext = nextResponse["next"] + + # concatenate new data obtained + self._catenateData(response, nextResponse, service) + + totalTime = _formatDuration(time() - start) + + self._log.info( + f" Downloaded {self._rowCount(response, service):d} total samples in {totalTime}." + ) + response["next"] = None return response @@ -81,8 +102,22 @@ def _doPageRequest( """ Wraps the _doRequest method Performs additional processing of the response for certain services - @param extension: Only provide for archivefiles filtering - Returns a tuple (jsonResponse, duration) + + Parameters + ---------- + url : str + API endpoint URL. + filters : dict + Filters for the request. + service : str + Name of the service (e.g. archivefile). + extension : str, optional + Only provide for archivefiles filtering. + + Returns + ------- + tuple + (jsonResponse, duration) """ if service.startswith("archivefile"): response, duration = self.parent()._doRequest(url, filters, getTime=True) diff --git a/src/onc/modules/_OncArchive.py b/src/onc/modules/_OncArchive.py index 9129a99..759e061 100644 --- a/src/onc/modules/_OncArchive.py +++ b/src/onc/modules/_OncArchive.py @@ -6,7 +6,7 @@ from ._MultiPage import _MultiPage from ._OncService import _OncService -from ._util import _createErrorMessage, _formatDuration, saveAsFile +from ._util import _formatDuration, saveAsFile class _OncArchive(_OncService): @@ -73,8 +73,10 @@ def downloadArchivefile(self, filename: str = "", overwrite: bool = False): size, downloadTime = saveAsFile(response, outPath, filename, overwrite) else: - msg = _createErrorMessage(response) - raise requests.HTTPError(msg) + from ._Messages import build_error_message + + self._log.error(build_error_message(response, self._config("redact_token"))) + response.raise_for_status() # Prepare a readable status txtStatus = "error" @@ -109,7 +111,7 @@ def downloadDirectArchivefile( dataRows = self.getArchivefile(filters, allPages) n = len(dataRows["files"]) - print(f"Obtained a list of {n} files to download.") + self._log.info(f"Obtained a list of {n} files to download.") # Download the files obtained tries = 1 @@ -124,7 +126,7 @@ def downloadDirectArchivefile( fileExists = os.path.exists(filePath) if not fileExists or os.path.getsize(filePath) == 0 or overwrite: - print(f' ({tries} of {n}) Downloading file: "{filename}"') + self._log.info(f' ({tries} of {n}) Downloading file: "{filename}"') downInfo = self.downloadArchivefile(filename, overwrite) size += downInfo["size"] time += downInfo["downloadTime"] @@ -132,7 +134,7 @@ def downloadDirectArchivefile( successes += 1 tries += 1 else: - print(f' Skipping "{filename}": File already exists.') + self._log.info(f' Skipping "{filename}": File already exists.') downInfo = { "url": self.getArchivefileUrl(filename), "status": "skipped", @@ -142,8 +144,8 @@ def downloadDirectArchivefile( } downInfos.append(downInfo) - print(f"{successes} files ({humanize.naturalsize(size)}) downloaded") - print(f"Total Download Time: {_formatDuration(time)}") + self._log.info(f"{successes} files ({humanize.naturalsize(size)}) downloaded") + self._log.info(f"Total Download Time: {_formatDuration(time)}") return { "downloadResults": downInfos, diff --git a/src/onc/modules/_OncDelivery.py b/src/onc/modules/_OncDelivery.py index 616ce24..3889b94 100644 --- a/src/onc/modules/_OncDelivery.py +++ b/src/onc/modules/_OncDelivery.py @@ -8,7 +8,7 @@ from ._DataProductFile import _DataProductFile from ._OncService import _OncService from ._PollLog import _PollLog -from ._util import _createErrorMessage, _formatSize +from ._util import _formatSize class _OncDelivery(_OncService): @@ -56,7 +56,6 @@ def orderDataProduct( ) ) - print("") self._printProductOrderStats(fileList, runData) return self._formatResult(fileList, runData) @@ -88,9 +87,9 @@ def runDataProduct(self, dpRequestId: int, waitComplete: bool): Return a dictionary with information of the run process. """ status = "" - log = _PollLog(True) - print( - f"To cancel the running data product, run 'onc.cancelDataProduct({dpRequestId})'" # noqa: E501 + log = _PollLog(self._config("verbosity")) + self._log.info( + f"To cancel the running data product, run 'onc.cancelDataProduct({dpRequestId})'" ) url = f"{self._config('baseUrl')}api/dataProductDelivery/run" runResult = {"runIds": [], "fileCount": 0, "runTime": 0, "requestCount": 0} @@ -111,7 +110,12 @@ def runDataProduct(self, dpRequestId: int, waitComplete: bool): if response.ok: data = response.json() else: - raise requests.HTTPError(_createErrorMessage(response)) + from ._Messages import build_error_message + + self._log.error( + build_error_message(response, self._config("redact_token")) + ) + response.raise_for_status() if waitComplete: status = data[0]["status"] @@ -128,10 +132,6 @@ def runDataProduct(self, dpRequestId: int, waitComplete: bool): runResult["fileCount"] = data[0]["fileCount"] runResult["runTime"] = time() - start - # print a new line after the process finishes - if waitComplete: - print("") - # gather a list of runIds for run in data: runResult["runIds"].append(run["dpRunId"]) @@ -192,9 +192,11 @@ def _downloadProductFiles( # keep increasing index until fileCount or until we get 404 doLoop = True timeout = self._config("timeout") - print(f"\nDownloading data product files with runId {runId}...") + self._log.info(f"Downloading data product files with runId {runId}...") - dpf = _DataProductFile(runId, str(index), baseUrl, token) + dpf = _DataProductFile( + runId, str(index), baseUrl, token, self._config("verbosity") + ) # loop thorugh file indexes while doLoop: @@ -211,7 +213,9 @@ def _downloadProductFiles( # file was downloaded (200), or skipped before downloading (777) fileList.append(dpf.getInfo()) index += 1 - dpf = _DataProductFile(runId, str(index), baseUrl, token) + dpf = _DataProductFile( + runId, str(index), baseUrl, token, self._config("verbosity") + ) elif status != 202 or (fileCount > 0 and index >= fileCount): # no more files to download @@ -219,7 +223,9 @@ def _downloadProductFiles( # get metadata if required if getMetadata: - dpf = _DataProductFile(runId, "meta", baseUrl, token) + dpf = _DataProductFile( + runId, "meta", baseUrl, token, self._config("verbosity") + ) try: status = dpf.download( timeout, @@ -248,8 +254,8 @@ def _infoForProductFiles(self, dpRunId: int, fileCount: int, getMetadata: bool): Returned rows will have the same structure as those returned by _DataProductFile.getInfo(). """ - print( - f"\nObtaining download information for data product files with runId {dpRunId}..." # noqa: E501 + self._log.info( + f"Obtaining download information for data product files with runId {dpRunId}..." ) # If we don't know the fileCount, get it from the server (takes longer) @@ -268,6 +274,7 @@ def _infoForProductFiles(self, dpRunId: int, fileCount: int, getMetadata: bool): index=str(index), baseUrl=self._config("baseUrl"), token=self._config("token"), + verbosity=self._config("verbosity"), ) dpf.setComplete() fileList.append(dpf.getInfo()) @@ -304,7 +311,7 @@ def _countFilesInProduct(self, runId: int): filters["index"] += 1 n += 1 - print(f" {n} files available for download") + self._log.info(f" {n} files available for download") return n def _printProductRequest(self, response): @@ -315,19 +322,19 @@ def _printProductRequest(self, response): product source (archive or generated on the fly). """ isGenerated = "estimatedFileSize" in response - print(f"Request Id: {response['dpRequestId']}") + self._log.info(f"Request Id: {response['dpRequestId']}") if isGenerated: size = response["estimatedFileSize"] # API returns it as a formatted string - print(f"Estimated File Size: {size}") + self._log.info(f"Estimated File Size: {size}") if "estimatedProcessingTime" in response: - print( + self._log.info( f"Estimated Processing Time: {response['estimatedProcessingTime']}" ) else: size = _formatSize(response["fileSize"]) - print(f"File Size: {size}") - print("Data product is ready for download.") + self._log.info(f"File Size: {size}") + self._log.info("Data product is ready for download.") def _estimatePollPeriod(self, response): """ @@ -373,7 +380,7 @@ def _printProductOrderStats(self, fileList: list, runInfo: dict): # Print run time runTime = timedelta(seconds=runInfo["runTime"]) - print(f"Total run time: {humanize.naturaldelta(runTime)}") + self._log.info(f"Total run time: {humanize.naturaldelta(runTime)}") if downloadCount > 0: # Print download time @@ -381,13 +388,13 @@ def _printProductOrderStats(self, fileList: list, runInfo: dict): txtDownTime = f"{downloadTime:.3f} seconds" else: txtDownTime = humanize.naturaldelta(downloadTime) - print(f"Total download Time: {txtDownTime}") + self._log.info(f"Total download Time: {txtDownTime}") # Print size and count of files natural_size = humanize.naturalsize(size, binary=True) - print(f"{downloadCount} files ({natural_size}) downloaded") + self._log.info(f"{downloadCount} files ({natural_size}) downloaded") else: - print("No files downloaded.") + self._log.info("No files downloaded.") def _formatResult(self, fileList: list, runInfo: dict): size = 0 diff --git a/src/onc/modules/_OncRealTime.py b/src/onc/modules/_OncRealTime.py index 5ec55cb..7ce93ec 100644 --- a/src/onc/modules/_OncRealTime.py +++ b/src/onc/modules/_OncRealTime.py @@ -9,8 +9,8 @@ class _OncRealTime(_OncService): Near real-time services methods """ - def __init__(self, config: dict): - super().__init__(config) + def __init__(self, parent: object): + super().__init__(parent) def getScalardataByLocation(self, filters: dict, allPages: bool): """ diff --git a/src/onc/modules/_OncService.py b/src/onc/modules/_OncService.py index 02eb532..0749929 100644 --- a/src/onc/modules/_OncService.py +++ b/src/onc/modules/_OncService.py @@ -1,14 +1,14 @@ -import logging -import pprint import weakref -from time import time -from urllib import parse import requests - -from ._util import _createErrorMessage, _formatDuration - -logging.basicConfig(format="%(levelname)s: %(message)s") +from onc.modules._Messages import ( + REQ_MSG, + RESPONSE_MSG, + RESPONSE_TIME_MSG, + build_error_message, + scrub_token, + setup_logger, +) class _OncService: @@ -18,6 +18,7 @@ class _OncService: def __init__(self, parent: object): self.parent = weakref.ref(parent) + self._log = setup_logger("onc.service", level=self._config("verbosity")) def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = False): """ @@ -44,48 +45,45 @@ def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = Fals filters["token"] = self._config("token") timeout = self._config("timeout") - txtParams = parse.unquote(parse.urlencode(filters)) - self._log(f"Requesting URL:\n{url}?{txtParams}") - - start = time() response = requests.get(url, filters, timeout=timeout) - responseTime = time() - start - if response.ok: - jsonResult = response.json() + if self._config("redact_token"): + response_url = scrub_token(response.url) else: - status = response.status_code - if status in [400, 401]: - msg = _createErrorMessage(response) - raise requests.HTTPError(msg) - else: + response_url = response.url + + # Log the url the user submitted. + self._log.debug(REQ_MSG.format(response_url)) + + # Display the time it took for ONC to respond in seconds. + # The requests.Response.elapsed value is a datetime.timedelta object. + responseTime = round(response.elapsed.total_seconds(), 3) # To milliseconds. + self._log.debug(RESPONSE_TIME_MSG.format(responseTime)) + + if response.status_code not in (requests.codes.ok, requests.codes.accepted): + self._log.error(RESPONSE_MSG.format(response.reason, response.status_code)) + error_message = build_error_message(response, self._config("redact_token")) + if error_message: + self._log.error(error_message) + + if self._config("raise_http_errors"): response.raise_for_status() - self._log(f"Web Service response time: {_formatDuration(responseTime)}") - - # Log warning messages only when showWarning is True - # and jsonResult["messages"] is not an empty list - if ( - self._config("showWarning") - and "messages" in jsonResult - and jsonResult["messages"] - ): - long_message = "\n".join( - [f"* {message}" for message in jsonResult["messages"]] - ) - filters_without_token = filters.copy() - del filters_without_token["token"] - filters_str = pprint.pformat(filters_without_token) + if getTime: + return response, responseTime + return response - logging.warning( - f"When calling {url} with filters\n{filters_str},\n" - f"there are several warning messages:\n{long_message}\n" - ) + json_response = response.json() + # The logger's configured level controls which API messages are emitted. + if "messages" in json_response and json_response["messages"]: + for message in json_response["messages"]: + self._log.warning(f"* {message}") + + self._log.debug(RESPONSE_MSG.format(response.reason, response.status_code)) if getTime: - return jsonResult, responseTime - else: - return jsonResult + return json_response, responseTime + return json_response def _serviceUrl(self, service: str): """ @@ -115,14 +113,6 @@ def _serviceUrl(self, service: str): return "" - def _log(self, message: str): - """ - Prints message to console only when self.showInfo is true - @param message: String - """ - if self._config("showInfo"): - print(message) - def _config(self, key: str): """ Returns a property from the parent (ONC class) diff --git a/src/onc/modules/_PollLog.py b/src/onc/modules/_PollLog.py index 1d32620..e7595cb 100644 --- a/src/onc/modules/_PollLog.py +++ b/src/onc/modules/_PollLog.py @@ -1,24 +1,30 @@ +from onc.modules._Messages import setup_logger + + class _PollLog: """ A helper for DataProductFile Keeps track of the messages printed in a single product download process """ - def __init__(self, showInfo: bool): + def __init__(self, verbosity: str): """ - @param showInfo same as in parent ONC object + Parameters + ---------- + verbosity : str + Standard logging level string. """ self._messages = [] # unique messages returned during the product order self._runStart = 0.0 # {float} timestamp (seconds) self._runEnd = 0.0 - self._showInfo = showInfo # flag for writing console messages - self._doPrintFileCount = True - self._lastPrintedDot = False # True after printing a dot (.) without a newline + self._verbosity = verbosity + self._log = setup_logger("onc.poll", level=verbosity) + self._doLogFileCount = True def logMessage(self, response): """ Adds a message to the messages list if it's new - Prints message to console, or "." if it repeats itself + Logs message to logger """ # Detect if the response comes from a "run" or "download" method origin = "download" @@ -35,33 +41,20 @@ def logMessage(self, response): # Detect and print change in the file count if origin == "run": fileCount = response[0]["fileCount"] - if self._doPrintFileCount and fileCount > 0: - self.printInfo( - f"\n {fileCount} files generated for this data product", - True, + if self._doLogFileCount and fileCount > 0: + self._log.info( + f" {fileCount} files generated for this data product" ) - self._doPrintFileCount = False + self._doLogFileCount = False self._messages.append(msg) - self.printInfo("\n " + msg, sameLine=True) + self._log.info(" " + msg) else: - self.printInfo(".", sameLine=True) - - def printInfo(self, msg: str, sameLine: bool = False): - """ - Conditional printing helper - """ - self._lastPrintedDot = msg == "." - - if self._showInfo: - if sameLine: - print(msg, end="", flush=True) - else: - print(msg) + # For repeating messages, log at DEBUG level to avoid cluttering INFO logs + self._log.debug(msg) def printNewLine(self): """ - Prints a line break only if the last message printed was a dot (.) + Deprecated. Newlines are handled by the logger. """ - if self._lastPrintedDot: - print("") + pass diff --git a/src/onc/modules/_util.py b/src/onc/modules/_util.py index 16db3fb..87bae0e 100644 --- a/src/onc/modules/_util.py +++ b/src/onc/modules/_util.py @@ -35,7 +35,11 @@ def saveAsFile( def _formatSize(size: float) -> str: """ Returns a formatted file size string representation - @param size: {float} Size in bytes + + Parameters + ---------- + size : float + Size in bytes. """ return humanize.naturalsize(size) @@ -43,7 +47,11 @@ def _formatSize(size: float) -> str: def _formatDuration(secs: float) -> str: """ Returns a formatted time duration string representation of a duration in seconds - @param seconds: float + + Parameters + ---------- + secs : float + Duration in seconds. """ if secs < 1.0: txtDownTime = f"{secs:.3f} seconds" diff --git a/src/onc/onc.py b/src/onc/onc.py index 42b1e85..b210446 100644 --- a/src/onc/onc.py +++ b/src/onc/onc.py @@ -1,10 +1,12 @@ import datetime import json +import logging import os import re from pathlib import Path from dateutil import parser +from onc.modules._Messages import setup_logger from onc.modules._OncArchive import _OncArchive from onc.modules._OncDelivery import _OncDelivery from onc.modules._OncDiscovery import _OncDiscovery @@ -27,37 +29,51 @@ class ONC: - True: Use the production server. - False: Use the internal ONC test server (reserved for ONC staff IP addresses). - showInfo : boolean, default False - Whether verbose script messages are displayed, such as request url and processing time information. - - - True: Print all information and debug messages (intended for debugging). - - False: Only print information messages. - showWarning : boolean, default True - Whether warning messages are displayed. Some web services have "messages" key in the response JSON - to indicate that something might need attention, like using a default value for a missing parameter. + showInfo : boolean, optional + Deprecated. Maps to verbosity="DEBUG" when true. + showWarning : boolean, optional + Deprecated. Maps to verbosity="ERROR" when false if showInfo is also False. outPath : str | Path, default "output" The directory that files are saved to (relative to the current directory) when downloading files. The directory will be created if it does not exist during the download. timeout : int, default 60 Number of seconds before a request to the API is canceled due to a timeout. + verbosity : str, default "INFO" + The logging verbosity level. Supported values: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL". + raise_http_errors : bool, default True + Whether unsuccessful HTTP responses raise ``requests.HTTPError``. + redact_token : bool, default False + Whether request tokens are redacted from log messages. Examples -------- >>> from onc import ONC >>> onc = ONC() # Only if you set the env variable "ONC_TOKEN" # doctest: +SKIP >>> onc = ONC("YOUR_TOKEN_HERE") # doctest: +SKIP - >>> onc = ONC("YOUR_TOKEN_HERE", showInfo=True, outPath="onc-files") # doctest: +SKIP + >>> onc = ONC("YOUR_TOKEN_HERE", verbosity="DEBUG", outPath="onc-files") # doctest: +SKIP """ # noqa: E501 def __init__( self, token: str | None = None, production: bool = True, - showInfo: bool = False, - showWarning: bool = True, + showInfo: bool | None = None, + showWarning: bool | None = None, outPath: str | Path = "output", timeout: int = 60, + *, + verbosity: str | None = None, + raise_http_errors: bool = True, + redact_token: bool = False, ): + if showInfo is not None or showWarning is not None: + import warnings + + warnings.warn( + "showInfo and showWarning are deprecated. Use verbosity instead.", + DeprecationWarning, + stacklevel=2, + ) if token is None or token == "": token = os.environ.get("ONC_TOKEN") if token is None or token == "": @@ -65,9 +81,21 @@ def __init__( "ONC API token is required. Please provide it as the first argument, " "or set it as the environment variable 'ONC_TOKEN'." ) + + if verbosity is None: + if showInfo is True: + verbosity = "DEBUG" + elif showWarning is False: + verbosity = "ERROR" + else: + verbosity = "INFO" + self.verbosity = verbosity + self.redact_token = redact_token + self.raise_http_errors = raise_http_errors + self._log = setup_logger("onc", self.verbosity) + self.token = re.sub(r"[^a-zA-Z0-9\-]+", "", token) - self.showInfo = showInfo - self.showWarning = showWarning + self.timeout = timeout self.production = production self.outPath = outPath @@ -78,6 +106,8 @@ def __init__( self.realTime = _OncRealTime(self) self.archive = _OncArchive(self) + self._log.debug("Initialized ONC module.") + @property def outPath(self) -> Path: """ @@ -107,6 +137,28 @@ def production(self, is_production: bool) -> None: else: self.baseUrl = "https://qa.oceannetworks.ca/" + @property + def verbosity(self) -> str: + """ + Return the current verbosity level. + """ + return self._verbosity + + @verbosity.setter + def verbosity(self, level: str) -> None: + """ + Set the logging verbosity level for the ONC instance and all its services. + + Supported values: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL". + """ + self._verbosity = level.upper() + setup_logger("onc", self._verbosity) + + # Update all existing loggers that start with 'onc.' + for name in logging.root.manager.loggerDict: + if name.startswith("onc."): + setup_logger(name, self._verbosity) + def print(self, obj, filename: str = "") -> None: """ Pretty print a collection to the console or a file. @@ -138,6 +190,34 @@ def print(self, obj, filename: str = "") -> None: with open(filePath, "w+") as file: file.write(text) + @property + def showInfo(self) -> bool: + """ + [Deprecated] Return whether the logger level is DEBUG. + """ + return self._log.isEnabledFor(logging.DEBUG) + + @showInfo.setter + def showInfo(self, value: bool) -> None: + """ + [Deprecated] Set the logger level based on the boolean value. + """ + self.verbosity = "DEBUG" if value else "INFO" + + @property + def showWarning(self) -> bool: + """ + [Deprecated] Return whether the logger level enables warnings. + """ + return self._log.isEnabledFor(logging.WARNING) + + @showWarning.setter + def showWarning(self, value: bool) -> None: + """ + [Deprecated] Set the logger level based on the boolean value. + """ + self.verbosity = "WARNING" if value else "ERROR" + def formatUtc(self, dateString: str = "now") -> str: """ Format the provided date string as an ISO8601 UTC date string. diff --git a/src/onc/util/util.py b/src/onc/util/util.py index b66b5e8..f407837 100644 --- a/src/onc/util/util.py +++ b/src/onc/util/util.py @@ -341,9 +341,14 @@ def copyFieldIfExists(fromDic, toDic, keys): """ Copy the field at name from fromDic to toDic only if it exists - @param fromDic: Origin Dictionary - @param toDic: Destination Dictionary - @param keys: Array of keys of the elements to copy + Parameters + ---------- + fromDic : dict + Origin Dictionary. + toDic : dict + Destination Dictionary. + keys : list + Array of keys of the elements to copy. """ for key in keys: if key in fromDic: diff --git a/tests/archive_file/test_archivefile_device.py b/tests/archive_file/test_archivefile_device.py index f6ebdc0..a589f9b 100644 --- a/tests/archive_file/test_archivefile_device.py +++ b/tests/archive_file/test_archivefile_device.py @@ -20,9 +20,9 @@ def params_multiple_pages(params): return params | {"rowLimit": 2} -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"deviceCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_invalid_param_value) @@ -32,9 +32,9 @@ def test_invalid_params_missing_required(requester, params): requester.getArchivefile(params) -def test_invalid_param_name(requester, params): +def test_invalid_param_name(requester, params, err_400): params_invalid_param_name = params | {"deviceCodes": "BPR-Folger-59"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_invalid_param_name) diff --git a/tests/archive_file/test_archivefile_download.py b/tests/archive_file/test_archivefile_download.py index 8ae6c9b..ef0900b 100644 --- a/tests/archive_file/test_archivefile_download.py +++ b/tests/archive_file/test_archivefile_download.py @@ -4,13 +4,13 @@ import requests -def test_invalid_param_value(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 96"): +def test_invalid_param_value(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.downloadArchivefile("FAKEFILE.XYZ") -def test_invalid_params_missing_required(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 128"): +def test_invalid_params_missing_required(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.downloadArchivefile() diff --git a/tests/archive_file/test_archivefile_location.py b/tests/archive_file/test_archivefile_location.py index 89fb1b6..9900b21 100644 --- a/tests/archive_file/test_archivefile_location.py +++ b/tests/archive_file/test_archivefile_location.py @@ -2,9 +2,9 @@ import requests -def test_invalid_param_value(requester, params_location): +def test_invalid_param_value(requester, params_location, err_400): params_invalid_param_value = params_location | {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_invalid_param_value) @@ -14,18 +14,18 @@ def test_invalid_params_missing_required(requester, params_location): requester.getArchivefile(params_location) -def test_invalid_param_name(requester, params_location): +def test_invalid_param_name(requester, params_location, err_400): params_invalid_param_name = params_location | {"locationCodes": "NCBC"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_invalid_param_name) -def test_no_data(requester, params_location): +def test_no_data(requester, params_location, err_400): params_no_data = params_location | { "dateFrom": "2000-01-01", "dateTo": "2000-01-02", } - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_no_data) diff --git a/tests/conftest.py b/tests/conftest.py index 12d434d..3cdb286 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +import requests from dotenv import load_dotenv from onc import ONC @@ -19,6 +20,16 @@ def requester(tmp_path) -> ONC: return ONC(production=is_prod, outPath=tmp_path) +@pytest.fixture +def err_400(): + return f"{requests.codes.bad} Client Error" + + +@pytest.fixture +def err_404(): + return f"{requests.codes.not_found} Client Error" + + @pytest.fixture(scope="session") def util(): return Util diff --git a/tests/data_product_delivery/test_data_product_delivery_cancel.py b/tests/data_product_delivery/test_data_product_delivery_cancel.py index ab347b7..f68b19e 100644 --- a/tests/data_product_delivery/test_data_product_delivery_cancel.py +++ b/tests/data_product_delivery/test_data_product_delivery_cancel.py @@ -2,6 +2,6 @@ import requests -def test_invalid_request_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_request_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.cancelDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_data_product_delivery_download.py b/tests/data_product_delivery/test_data_product_delivery_download.py index faacce4..14c000a 100644 --- a/tests/data_product_delivery/test_data_product_delivery_download.py +++ b/tests/data_product_delivery/test_data_product_delivery_download.py @@ -2,6 +2,6 @@ import requests -def test_invalid_run_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_run_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.downloadDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_data_product_delivery_order.py b/tests/data_product_delivery/test_data_product_delivery_order.py index 7b16df2..06a6524 100644 --- a/tests/data_product_delivery/test_data_product_delivery_order.py +++ b/tests/data_product_delivery/test_data_product_delivery_order.py @@ -2,12 +2,13 @@ import requests -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"dataProductCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.orderDataProduct(params_invalid_param_value) +@pytest.mark.slow def test_valid_default(requester, params, expected_keys_download_results, util): data = requester.orderDataProduct(params) @@ -30,6 +31,7 @@ def test_valid_default(requester, params, expected_keys_download_results, util): ) +@pytest.mark.slow def test_valid_no_metadata(requester, params, expected_keys_download_results, util): data = requester.orderDataProduct(params, includeMetadataFile=False) @@ -49,6 +51,7 @@ def test_valid_no_metadata(requester, params, expected_keys_download_results, ut ) +@pytest.mark.slow def test_valid_results_only(requester, params, expected_keys_download_results, util): data = requester.orderDataProduct(params, downloadResultsOnly=True) diff --git a/tests/data_product_delivery/test_data_product_delivery_restart.py b/tests/data_product_delivery/test_data_product_delivery_restart.py index 4cf1fe5..672ad59 100644 --- a/tests/data_product_delivery/test_data_product_delivery_restart.py +++ b/tests/data_product_delivery/test_data_product_delivery_restart.py @@ -2,6 +2,6 @@ import requests -def test_invalid_request_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_request_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.restartDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_data_product_delivery_run.py b/tests/data_product_delivery/test_data_product_delivery_run.py index 76463c1..0e3dcdc 100644 --- a/tests/data_product_delivery/test_data_product_delivery_run.py +++ b/tests/data_product_delivery/test_data_product_delivery_run.py @@ -2,6 +2,6 @@ import requests -def test_invalid_request_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_request_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.runDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_data_product_delivery_status.py b/tests/data_product_delivery/test_data_product_delivery_status.py index a5eb452..4a48aec 100644 --- a/tests/data_product_delivery/test_data_product_delivery_status.py +++ b/tests/data_product_delivery/test_data_product_delivery_status.py @@ -2,6 +2,6 @@ import requests -def test_invalid_request_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_request_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.checkDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_integration.py b/tests/data_product_delivery/test_integration.py index 39014c7..1c67ee5 100644 --- a/tests/data_product_delivery/test_integration.py +++ b/tests/data_product_delivery/test_integration.py @@ -1,6 +1,7 @@ import pytest +@pytest.mark.slow def test_valid_manual(requester, params, expected_keys_download_results, util): """ Test request -> status -> run -> download -> status. @@ -35,6 +36,7 @@ def test_valid_manual(requester, params, expected_keys_download_results, util): assert data_status_after_download["searchHdrStatus"] == "COMPLETED" +@pytest.mark.slow def test_valid_cancel_restart(requester, params, expected_keys_download_results, util): """ Test request -> run -> cancel -> download (fail) -> restart -> download. diff --git a/tests/discover_data_availability/test_data_availability.py b/tests/discover_data_availability/test_data_availability.py index 1a4a201..a748707 100644 --- a/tests/discover_data_availability/test_data_availability.py +++ b/tests/discover_data_availability/test_data_availability.py @@ -12,9 +12,9 @@ def params(): } -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"locationCode": "INVALID"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDataAvailability(params_invalid_param_value) diff --git a/tests/discover_data_products/test_data_products.py b/tests/discover_data_products/test_data_products.py index 6d149bc..8d75281 100644 --- a/tests/discover_data_products/test_data_products.py +++ b/tests/discover_data_products/test_data_products.py @@ -2,25 +2,25 @@ import requests -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"dataProductCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDataProducts(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"dataProductCodes": "HSD"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDataProducts(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = { "dataProductCode": "HSD", "extension": "txt", } - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getDataProducts(params_no_data) diff --git a/tests/discover_deployments/test_deployments.py b/tests/discover_deployments/test_deployments.py index b984978..76b3b30 100644 --- a/tests/discover_deployments/test_deployments.py +++ b/tests/discover_deployments/test_deployments.py @@ -2,46 +2,46 @@ import requests -def test_invalid_time_range_greater_start_time(requester): +def test_invalid_time_range_greater_start_time(requester, err_400): params_invalid_time_range_greater_start_time = { "locationCode": "BACAX", "deviceCategoryCode": "CTD", "dateFrom": "2020-01-01", "dateTo": "2019-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 23"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_invalid_time_range_greater_start_time) -def test_invalid_time_range_future_start_time(requester): +def test_invalid_time_range_future_start_time(requester, err_400): params_invalid_time_range_future_start_time = { "locationCode": "BACAX", "deviceCategoryCode": "CTD", "dateFrom": "2050-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 25"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_invalid_time_range_future_start_time) -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"locationCode": "XYZ123", "deviceCategoryCode": "CTD"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"locationCodes": "BACAX", "deviceCategoryCode": "CTD"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_400): params_no_data = { "locationCode": "BACAX", "deviceCategoryCode": "CTD", "dateTo": "1900-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_no_data) diff --git a/tests/discover_device_categories/test_device_categories.py b/tests/discover_device_categories/test_device_categories.py index 45a2a0f..3a39445 100644 --- a/tests/discover_device_categories/test_device_categories.py +++ b/tests/discover_device_categories/test_device_categories.py @@ -2,25 +2,25 @@ import requests -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"deviceCategoryCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeviceCategories(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"deviceCategoryCodes": "CTD"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeviceCategories(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = { "deviceCategoryCode": "CTD", "deviceCategoryName": "Conductivity", "description": "TemperatureXXX", } - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getDeviceCategories(params_no_data) diff --git a/tests/discover_devices/test_devices.py b/tests/discover_devices/test_devices.py index 96069aa..280cf78 100644 --- a/tests/discover_devices/test_devices.py +++ b/tests/discover_devices/test_devices.py @@ -2,40 +2,40 @@ import requests -def test_invalid_time_range_greater_start_time(requester): +def test_invalid_time_range_greater_start_time(requester, err_400): params_invalid_time_range_greater_start_time = { "deviceCode": "BPR-Folger-59", "dateFrom": "2020-01-01", "dateTo": "2019-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 23"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDevices(params_invalid_time_range_greater_start_time) -def test_invalid_time_range_future_start_time(requester): +def test_invalid_time_range_future_start_time(requester, err_404): params_invalid_time_range_future_start_time = { "deviceCode": "BPR-Folger-59", "dateFrom": "2050-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 25"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getDevices(params_invalid_time_range_future_start_time) -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"deviceCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDevices(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"deviceCodes": "BPR-Folger-59"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDevices(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = {"deviceCode": "BPR-Folger-59", "dateTo": "1900-01-01"} - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getDevices(params_no_data) diff --git a/tests/discover_locations/test_locations.py b/tests/discover_locations/test_locations.py index 2160042..af60aa0 100644 --- a/tests/discover_locations/test_locations.py +++ b/tests/discover_locations/test_locations.py @@ -2,40 +2,40 @@ import requests -def test_invalid_time_range_greater_start_time(requester): +def test_invalid_time_range_greater_start_time(requester, err_400): params_invalid_time_range_greater_start_time = { "locationCode": "FGPD", "dateFrom": "2020-01-01", "dateTo": "2019-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 23"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocations(params_invalid_time_range_greater_start_time) -def test_invalid_time_range_future_start_time(requester): +def test_invalid_time_range_future_start_time(requester, err_400): params_invalid_time_range_future_start_time = { "locationCode": "FGPD", "dateFrom": "2050-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 25"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocations(params_invalid_time_range_future_start_time) -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocations(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"locationCodes": "FGPD"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocations(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = {"locationCode": "FGPD", "dateTo": "1900-01-01"} - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getLocations(params_no_data) diff --git a/tests/discover_locations/test_locations_tree.py b/tests/discover_locations/test_locations_tree.py index e0eb47a..388c06a 100644 --- a/tests/discover_locations/test_locations_tree.py +++ b/tests/discover_locations/test_locations_tree.py @@ -2,40 +2,40 @@ import requests -def test_invalid_time_range_greater_start_time(requester): +def test_invalid_time_range_greater_start_time(requester, err_400): params_invalid_time_range_greater_start_time = { "locationCode": "ARCT", "dateFrom": "2020-01-01", "dateTo": "2019-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 23"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocationsTree(params_invalid_time_range_greater_start_time) -def test_invalid_time_range_future_start_time(requester): +def test_invalid_time_range_future_start_time(requester, err_400): params_invalid_time_range_future_start_time = { "locationCode": "ARCT", "dateFrom": "2050-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 25"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocationsTree(params_invalid_time_range_future_start_time) -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocationsTree(params_invalid_param_value) -def test_invalid_param(requester): +def test_invalid_param(requester, err_400): params_invalid_param_name = {"locationCodes": "ARCT"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocationsTree(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = {"locationCode": "ARCT", "dateTo": "1900-01-01"} - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getLocationsTree(params_no_data) diff --git a/tests/discover_properties/test_properties.py b/tests/discover_properties/test_properties.py index a72d88b..6703601 100644 --- a/tests/discover_properties/test_properties.py +++ b/tests/discover_properties/test_properties.py @@ -2,25 +2,25 @@ import requests -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"propertyCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getProperties(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"propertyCodes": "conductivity"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getProperties(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = { "propertyCode": "conductivity", "locationCode": "SAAN", } - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getProperties(params_no_data) diff --git a/tests/raw_data/test_rawdata_device.py b/tests/raw_data/test_rawdata_device.py index 2e13959..6fe6585 100644 --- a/tests/raw_data/test_rawdata_device.py +++ b/tests/raw_data/test_rawdata_device.py @@ -18,15 +18,15 @@ def params_multiple_pages(params): return params | {"rowLimit": 25} -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"deviceCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_invalid_param_value) -def test_invalid_param_name(requester, params): +def test_invalid_param_name(requester, params, err_400): params_invalid_param_name = params | {"deviceCodes": "BPR-Folger-59"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_invalid_param_name) diff --git a/tests/raw_data/test_rawdata_location.py b/tests/raw_data/test_rawdata_location.py index e9af23f..9889c45 100644 --- a/tests/raw_data/test_rawdata_location.py +++ b/tests/raw_data/test_rawdata_location.py @@ -21,21 +21,21 @@ def params_multiple_pages(params): return params | {"rowLimit": 25} -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_invalid_param_value) -def test_invalid_param_name(requester, params): +def test_invalid_param_name(requester, params, err_400): params_invalid_param_name = params | {"locationCodes": "NCBC"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_invalid_param_name) -def test_no_data(requester, params): +def test_no_data(requester, params, err_400): params_no_data = params | {"dateFrom": "2000-01-01", "dateTo": "2000-01-02"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_no_data) diff --git a/tests/scalar_data/test_scalardata_device.py b/tests/scalar_data/test_scalardata_device.py index ffbb63e..c85d46a 100644 --- a/tests/scalar_data/test_scalardata_device.py +++ b/tests/scalar_data/test_scalardata_device.py @@ -8,15 +8,15 @@ def params_multiple_pages(params_device): return params_device | {"rowLimit": 25} -def test_invalid_param_value(requester, params_device): +def test_invalid_param_value(requester, params_device, err_400): params_invalid_param_value = params_device | {"deviceCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_invalid_param_value) -def test_invalid_param_name(requester, params_device): +def test_invalid_param_name(requester, params_device, err_400): params_invalid_param_name = params_device | {"deviceCodes": "BPR-Folger-59"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_invalid_param_name) diff --git a/tests/scalar_data/test_scalardata_location.py b/tests/scalar_data/test_scalardata_location.py index 19ed397..bb38752 100644 --- a/tests/scalar_data/test_scalardata_location.py +++ b/tests/scalar_data/test_scalardata_location.py @@ -8,24 +8,24 @@ def params_multiple_pages(params_location): return params_location | {"rowLimit": 25} -def test_invalid_param_value(requester, params_location): +def test_invalid_param_value(requester, params_location, err_400): params_invalid_param_value = params_location | {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_invalid_param_value) -def test_invalid_param_name(requester, params_location): +def test_invalid_param_name(requester, params_location, err_400): params_invalid_param_name = params_location | {"locationCodes": "NCBC"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_invalid_param_name) -def test_no_data(requester, params_location): +def test_no_data(requester, params_location, err_400): params_no_data = params_location | { "dateFrom": "2000-01-01", "dateTo": "2000-01-02", } - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_no_data)