diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index cbd920f..63713e6 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -4,7 +4,13 @@ updates: directory: "/" schedule: interval: "weekly" + reviewers: + - "btschwertfeger" - package-ecosystem: "pip" directory: "/" schedule: interval: "weekly" + reviewers: + - "btschwertfeger" + ignore: + - dependency-name: "ruff" diff --git a/.github/workflows/_codecov.yaml b/.github/workflows/_codecov.yaml index 312aa16..6cd3b0d 100644 --- a/.github/workflows/_codecov.yaml +++ b/.github/workflows/_codecov.yaml @@ -45,7 +45,7 @@ jobs: run: python -m pip install --upgrade pip - name: Install package - run: python -m pip install ".[dev]" + run: python -m pip install ".[dev,test]" - name: Generate coverage report run: pytest --cov --cov-report=xml diff --git a/.github/workflows/_test.yaml b/.github/workflows/_test.yaml index bd7a2f2..0f09f04 100644 --- a/.github/workflows/_test.yaml +++ b/.github/workflows/_test.yaml @@ -37,7 +37,7 @@ jobs: python -m pip install --upgrade pip - name: Install package - run: python -m pip install ".[dev]" + run: python -m pip install ".[dev,test]" - name: Run unit tests run: pytest -vv tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 17bc4d3..e15065d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.2 + rev: v0.3.5 hooks: - id: ruff args: diff --git a/Makefile b/Makefile index 5e4a3f9..21d5892 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ build: .PHONY: dev dev: @git lfs install - $(PYTHON) -m pip install -e ".[dev]" + $(PYTHON) -m pip install -e ".[dev,test]" ## install Install the package ## diff --git a/README.md b/README.md index 1117a9a..3846859 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Welcome to python-cmethods, a powerful Python package designed for bias correction and adjustment of climate data. Built with a focus on ease of use and efficiency, python-cmethods offers a comprehensive suite of functions tailored for applying bias correction methods to climate model simulations and -observational datasets. +observational datasets via command-line interface and API. Please cite this project as described in https://zenodo.org/doi/10.5281/zenodo.7652755. @@ -143,7 +143,64 @@ python3 -m pip install python-cmethods -## 4. Usage and Examples +## 4. CLI Usage + +The python-cmethods package provides a command-line interface for applying +various bias correction methods out of the box. + +Keep in mind that due to the various kinds of data and possibilities to +pre-process those, the CLI only provides a basic application of the implemented +techniques. For special parameters, adjustments, and data preparation, please +use programming interface. + +Listing the parameters and their requirements is available by passing the +`--help` option: + +```bash +cmethods --help +``` + +Applying the cmethods tool on the provided example data using the linear scaling +approach is shown below: + +```bash +cmethods \ + --obs examples/input_data/observations.nc \ + --simh examples/input_data/control.nc \ + --simp examples/input_data/scenario.nc \ + --method linear_scaling \ + --kind add \ + --variable tas \ + --group time.month \ + --output linear_scaling.nc + +2024/04/08 18:11:12 INFO | Loading data sets ... +2024/04/08 18:11:12 INFO | Data sets loaded ... +2024/04/08 18:11:12 INFO | Applying linear_scaling ... +2024/04/08 18:11:15 INFO | Saving result to linear_scaling.nc ... +``` + +For applying a distribution-based bias correction technique, the following +example may help: + +```bash +cmethods \ + --obs examples/input_data/observations.nc \ + --simh examples/input_data/control.nc \ + --simp examples/input_data/scenario.nc \ + --method quantile_delta_mapping \ + --kind add \ + --variable tas \ + --quantiles 1000 \ + --output quantile_delta_mapping.nc + +2024/04/08 18:16:34 INFO | Loading data sets ... +2024/04/08 18:16:35 INFO | Data sets loaded ... +2024/04/08 18:16:35 INFO | Applying quantile_delta_mapping ... +2024/04/08 18:16:35 INFO | Saving result to quantile_delta_mapping.nc ... +``` + +## 5. Programming Interface Usage and Examples ```python import xarray as xr diff --git a/cmethods/__init__.py b/cmethods/__init__.py index 44f6f1f..f2109ff 100644 --- a/cmethods/__init__.py +++ b/cmethods/__init__.py @@ -3,6 +3,7 @@ # Copyright (C) 2023 Benjamin Thomas Schwertfeger # GitHub: https://github.com/btschwertfeger # +# pylint: disable=consider-using-f-string,logging-not-lazy r""" Module providing the a method named "adjust" to apply different bias @@ -24,12 +25,170 @@ _{m} = long-term monthly interval """ -from cmethods.core import adjust +from __future__ import annotations + +import logging +import sys -__author__ = "Benjamin Thomas Schwertfeger" -__copyright__ = __author__ -__email__ = "contact@b-schwertfeger.de" -__link__ = "https://github.com/btschwertfeger" -__github__ = "https://github.com/btschwertfeger/python-cmethods" +import cloup +import xarray as xr +from cloup import ( + HelpFormatter, + HelpTheme, + Path, + Style, + command, + option, + option_group, + version_option, +) +from cloup.constraints import Equal, If, require_all + +from cmethods.core import adjust __all__ = ["adjust"] + + +@command( + context_settings={ + "auto_envvar_prefix": "CMETHODS", + "help_option_names": ["-h", "--help"], + }, + formatter_settings=HelpFormatter.settings( + theme=HelpTheme( + invoked_command=Style(fg="bright_yellow"), + heading=Style(fg="bright_white", bold=True), + constraint=Style(fg="magenta"), + col1=Style(fg="bright_yellow"), + ), + ), +) +@version_option(message="%version%") +@option( + "--obs", + "--observations", + required=True, + type=Path(exists=True), + help="Reference data set (control period)", +) +@option( + "--simh", + "--simulated-historical", + required=True, + type=Path(exists=True), + help="Modeled data set (control period)", +) +@option( + "--simp", + "--simulated-scenario", + required=True, + type=Path(exists=True), + help="Modeled data set (scenario period)", +) +@option( + "--method", + required=True, + type=cloup.Choice( + [ + "linear_scaling", + "variance_scaling", + "delta_method", + "quantile_mapping", + "quantile_delta_mapping", + ], + case_sensitive=False, + ), + help="Bias adjustment method to apply", +) +@option( + "--kind", + required=True, + type=cloup.Choice(["+", "add", "*", "mult"]), + help="Kind of adjustment", +) +@option( + "--variable", + required=True, + type=str, + help="Variable of interest", +) +@option( + "-o", + "--output", + required=True, + type=str, + callback=lambda _, __, value: (value if value.endswith(".nc") else f"{value}.nc"), + help="Output file name", +) +@option_group( + "Scaling-Based Adjustment Options", + option( + "--group", + type=str, + help="Temporal grouping", + ), + constraint=If( + Equal("method", "linear_scaling") + & Equal("method", "variance_scaling") + & Equal("method", "delta_method"), + then=require_all, + ), +) +@option_group( + "Distribution-Based Adjustment Options", + option( + "--quantiles", + type=int, + help="Quantiles to respect", + ), + constraint=If( + Equal("method", "quantile_mapping") & Equal("method", "quantile_delta_mapping"), + then=require_all, + ), +) +def cli(**kwargs) -> None: + """ + Command-line tool to apply bias correction procedures to climate data. + + Copyright (C) 2023 Benjamin Thomas Schwertfeger\n + GitHub: https://github.com/btschwertfeger/python-cmethods + """ + + logging.basicConfig( + format="%(asctime)s %(levelname)8s | %(message)s", + datefmt="%Y/%m/%d %H:%M:%S", + level=logging.INFO, + ) + + logging.info("Loading data sets ...") + try: + for key, message in zip( + ("obs", "simh", "simp"), + ( + "observation data set", + "modeled data set of the control period", + "modeled data set of the scenario period", + ), + ): + kwargs[key] = xr.open_dataset(kwargs[key]) + if not isinstance(kwargs[key], xr.Dataset): + raise TypeError("The data sets must be type xarray.Dataset") + + if kwargs["variable"] not in kwargs[key]: + raise KeyError( + f"Variable '{kwargs['variable']}' is missing in the {message}", + ) + kwargs[key] = kwargs[key][kwargs["variable"]] + except (TypeError, KeyError) as exc: + logging.error(exc) + sys.exit(1) + + logging.info("Data sets loaded ...") + kwargs["n_quantiles"] = kwargs["quantiles"] + del kwargs["quantiles"] + + logging.info("Applying %s ..." % kwargs["method"]) + result = adjust(**kwargs) + + logging.info("Saving result to %s ..." % kwargs["output"]) + result.to_netcdf(kwargs["output"]) diff --git a/doc/cli.rst b/doc/cli.rst new file mode 100644 index 0000000..399ffa9 --- /dev/null +++ b/doc/cli.rst @@ -0,0 +1,41 @@ +.. -*- coding: utf-8 -*- +.. Copyright (C) 2023 Benjamin Thomas Schwertfeger +.. GitHub: https://github.com/btschwertfeger +.. + +Command-Line Interface +====================== + +The command-line interface provides the following help instructions + +.. code-block:: bash + + cmethods --help + + Usage: cmethods [OPTIONS] + + Command line tool to apply bias adjustment procedures to climate data. + + Scaling-Based Adjustment Options: + [all required if --method="linear_scaling" and --method="variance_scaling" and + --method="delta_method"] + --group TEXT Temporal grouping + + Distribution-Based Adjustment Options: + [all required if --method="quantile_mapping" and + --method="quantile_delta_mapping"] + --quantiles INTEGER Quantiles to respect + + Other options: + --version Show the version and exit. + --obs, --observations PATH Reference data set (control period) [required] + --simh, --simulated-historical PATH + Modeled data set (control period) [required] + --simp, --simulated-scenario PATH + Modeled data set (scenario period) [required] + --method [linear_scaling|variance_scaling|delta_method|quantile_mapping|quantile_delta_mapping] + Bias adjustment method to apply [required] + --kind [add|mult] Kind of adjustment [required] + --variable TEXT Variable of interest [required] + -o, --output TEXT Output file name [required] + -h, --help Show this message and exit. diff --git a/doc/cmethods.rst b/doc/cmethods.rst index 233c8cf..00bfe32 100644 --- a/doc/cmethods.rst +++ b/doc/cmethods.rst @@ -1,3 +1,7 @@ +.. -*- coding: utf-8 -*- +.. Copyright (C) 2023 Benjamin Thomas Schwertfeger +.. GitHub: https://github.com/btschwertfeger +.. Classes and Functions ===================== diff --git a/doc/getting_started.rst b/doc/getting_started.rst index 152d2bd..dc21d8a 100644 --- a/doc/getting_started.rst +++ b/doc/getting_started.rst @@ -1,3 +1,8 @@ +.. -*- coding: utf-8 -*- +.. Copyright (C) 2023 Benjamin Thomas Schwertfeger +.. GitHub: https://github.com/btschwertfeger +.. + Getting Started =============== @@ -11,12 +16,73 @@ The `python-cmethods`_ module can be installed using the package manager pip: python3 -m pip install python-cmethods -Usage and Examples ------------------- +Command-Line Interface Usage +---------------------------- + +The python-cmethods package provides a command-line interface for applying +various bias correction methods out of the box. + +Keep in mind that due to the various kinds of data and possibilities to +pre-process those, the CLI only provides a basic application of the implemented +techniques. For special parameters, adjustments, and data preparation, please +use programming interface. + +Listing the parameters and their requirements is available by passing the +``--help`` option: + +.. code-block:: bash + + cmethods --help + +Applying the cmethods tool on the provided example data using the linear scaling +approach is shown below: -The `python-cmethods`_ module can be imported and applied as showing in the following examples. -For more detailed description of the methods, please have a look at the -method specific documentation. +.. code-block:: bash + :caption: Apply Linear Scaling via command-line + + cmethods \ + --obs examples/input_data/observations.nc \ + --simh examples/input_data/control.nc \ + --simp examples/input_data/scenario.nc \ + --method linear_scaling \ + --kind add \ + --variable tas \ + --group time.month \ + --output linear_scaling.nc + + 2024/04/08 18:11:12 INFO | Loading data sets ... + 2024/04/08 18:11:12 INFO | Data sets loaded ... + 2024/04/08 18:11:12 INFO | Applying linear_scaling ... + 2024/04/08 18:11:15 INFO | Saving result to linear_scaling.nc ... + + +For applying a distribution-based bias correction technique, the following +example may help: + +.. code-block:: bash + :caption: Apply Quantile Delta Mapping via command-line + + cmethods \ + --obs examples/input_data/observations.nc \ + --simh examples/input_data/control.nc \ + --simp examples/input_data/scenario.nc \ + --method quantile_delta_mapping \ + --kind add \ + --variable tas \ + --quantiles 1000 \ + --output quantile_delta_mapping.nc + + 2024/04/08 18:16:34 INFO | Loading data sets ... + 2024/04/08 18:16:35 INFO | Data sets loaded ... + 2024/04/08 18:16:35 INFO | Applying quantile_delta_mapping ... + 2024/04/08 18:16:35 INFO | Saving result to quantile_delta_mapping.nc ... + +Programming Interface Usage and Examples +---------------------------------------- + +The `python-cmethods`_ module can be imported and applied as showing in the +following examples. For more detailed description of the methods, please have a +look at the method specific documentation. .. code-block:: python :linenos: diff --git a/doc/index.rst b/doc/index.rst index 7011b83..4203605 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,5 +1,7 @@ -.. python-cmethods documentation master file, created by - sphinx-quickstart on Mon Apr 10 10:04:20 2023. +.. -*- coding: utf-8 -*- +.. Copyright (C) 2023 Benjamin Thomas Schwertfeger +.. GitHub: https://github.com/btschwertfeger +.. Welcome to python-cmethods's documentation! =========================================== @@ -10,6 +12,7 @@ Welcome to python-cmethods's documentation! introduction.rst getting_started.rst + cli.rst cmethods.rst methods.rst issues.rst diff --git a/doc/introduction.rst b/doc/introduction.rst index da383b3..a207a36 100644 --- a/doc/introduction.rst +++ b/doc/introduction.rst @@ -1,4 +1,7 @@ -.. This is the introduction +.. -*- coding: utf-8 -*- +.. Copyright (C) 2023 Benjamin Thomas Schwertfeger +.. GitHub: https://github.com/btschwertfeger +.. python-cmethods =============== @@ -23,7 +26,7 @@ actual climate conditions. This process typically involves statistical methods or empirical relationships to correct for biases caused by factors such as instrument calibration, spatial resolution, or model deficiencies. -.. figure:: ../_static/images/biasCdiagram.png +.. figure:: _static/images/biasCdiagram.png :width: 600 :align: center :alt: Schematic representation of a bias adjustment procedure @@ -41,7 +44,7 @@ and adjusted values, revealing that the delta-adjusted time series (:math:`T^{*DM}_{sim,p}`) is significantly more similar to the observational data (:math:`T_{obs,p}`) than the raw model output (:math:`T_{sim,p}`). -.. figure:: ../_static/images/dm-doy-plot.png +.. figure:: _static/images/dm-doy-plot.png :width: 600 :align: center :alt: Temperature per day of year in modeled, observed and bias-adjusted climate data diff --git a/doc/issues.rst b/doc/issues.rst index 3a37226..46dfd72 100644 --- a/doc/issues.rst +++ b/doc/issues.rst @@ -1,3 +1,8 @@ +.. -*- coding: utf-8 -*- +.. Copyright (C) 2023 Benjamin Thomas Schwertfeger +.. GitHub: https://github.com/btschwertfeger +.. + Known Issues ============ @@ -9,11 +14,3 @@ Known Issues surrounding values over all years as the basis for calculating the mean values. This is not yet implemented in this module, but is available in the command-line tool `BiasAdjustCXX`_. -- Using this module or especially Python to apply bias correction techniques on - large data sets can be a very time-consuming task. So this module is more - about showing how to apply different methods on climate data and maybe even - to bias-correct small data sets. When it comes to large ensembles it is - preferred to use the way more efficient tool `BiasAdjustCXX`_. A speed - comparison between `python-cmethods`_, `BiasAdjustCXX`_, and `xclim`_ was - made this `tool comparison`_. Since the development of python-cmethods is - continuing, speed improvements have been done since the last bench. diff --git a/doc/license.rst b/doc/license.rst index 00e04f5..495bae5 100644 --- a/doc/license.rst +++ b/doc/license.rst @@ -1,3 +1,7 @@ +.. -*- coding: utf-8 -*- +.. Copyright (C) 2023 Benjamin Thomas Schwertfeger +.. GitHub: https://github.com/btschwertfeger +.. .. _section-license: diff --git a/doc/links.rst b/doc/links.rst index 1bf8ec8..d90462d 100644 --- a/doc/links.rst +++ b/doc/links.rst @@ -1,3 +1,8 @@ +.. -*- coding: utf-8 -*- +.. Copyright (C) 2023 Benjamin Thomas Schwertfeger +.. GitHub: https://github.com/btschwertfeger +.. + .. LINKS .. _python-cmethods: https://github.com/btschwertfeger/python-cmethods diff --git a/doc/methods.rst b/doc/methods.rst index b7738d6..03c93e9 100644 --- a/doc/methods.rst +++ b/doc/methods.rst @@ -1,3 +1,7 @@ +.. -*- coding: utf-8 -*- +.. Copyright (C) 2023 Benjamin Thomas Schwertfeger +.. GitHub: https://github.com/btschwertfeger +.. Bias Correction Methods ======================= @@ -253,7 +257,7 @@ In the following the equations of Alex J. Cannon (2015) are shown and explained: modeled data of the control period The following images show this by using :math:`T` for temperatures. - .. figure:: ../_static/images/qm-cdf-plot-1.png + .. figure:: _static/images/qm-cdf-plot-1.png :width: 600 :align: center :alt: Determination of the quantile value @@ -265,7 +269,7 @@ In the following the equations of Alex J. Cannon (2015) are shown and explained: the control period to determine the bias-corrected value at time step :math:`i`. - .. figure:: ../_static/images/qm-cdf-plot-2.png + .. figure:: _static/images/qm-cdf-plot-2.png :width: 600 :align: center :alt: Determination of the QM bias-corrected value diff --git a/examples/biasadjust.py b/examples/biasadjust.py deleted file mode 100755 index e11f544..0000000 --- a/examples/biasadjust.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2023 Benjamin Thomas Schwertfeger -# GitHub: https://github.com/btschwertfeger -# -# Note: This is just an example on how to use the python-cmethods module. -# This is in no way an optimal solution and exists only for demonstration -# purposes. - -import sys - -import click -from xarray import open_dataset - -from cmethods import adjust -from cmethods.static import DISTRIBUTION_METHODS, METHODS, SCALING_METHODS - - -def save_to_netcdf(ds, **kwargs) -> None: - """ - Saves the data set to file - - :param ds: The data set to save - :type ds: xarray.core.dataarray.Dataset - """ - ds.to_netcdf( - f"{kwargs['method']}_result_var-{kwargs['variable']}{kwargs['descr1']}_kind-{kwargs['kind']}_group-{kwargs['group']}_{kwargs['start_date']}_{kwargs['end_date']}.nc", - ) - - -@click.command() -@click.option( - "--ref", - "--reference", - required=True, - type=str, - help="The reference data set (control period)", -) -@click.option( - "--contr", - "--control", - required=True, - type=str, - help="The modeled data set (control period)", -) -@click.option( - "--scen", - "--scenario", - required=True, - type=str, - help="The modeled data set to adjust (scenario period)", -) -@click.option( - "-m", - "--method", - required=True, - type=str, - help="The bias correction method", -) -@click.option( - "-v", - "--variable", - required=True, - type=str, - help="The variable to adjust", -) -@click.option( - "-k", - "--kind", - type=str, - default="+", - help="The adjustment variant/kind ('+' or '*', default: '+')", -) -@click.option( - "-g", - "--group", - type=str, - default="time.month", - help="The grouping basis (only for scaling-based methods; default: 'time.month')", -) -@click.option( - "-q", - "--quantiles", - type=int, - default=100, - help="The number of quantiles to use (only for distribution-based methods; default: 100)", -) -@click.option( - "-p", - "--processes", - type=int, - default=1, - help="The number of processes to use (only for 3-dimensional corrections: default: 1)", -) -def main(**kwargs) -> None: - """ - The Main program that uses the passed arguments to perform the bias correction procedure. - """ - if kwargs["method"] not in METHODS: - raise ValueError( - f"Unknown method {kwargs['method']}. Available methods: {METHODS}", - ) - - ds_obs = open_dataset(kwargs["ref"])[kwargs["variable"]] - ds_simh = open_dataset(kwargs["contr"])[kwargs["variable"]] - ds_simp = open_dataset(kwargs["scen"])[kwargs["variable"]] - - print("**Data sets loaded**") - - start_date: str = ds_simp["time"][0].dt.strftime("%Y%m%d").values.ravel()[0] - end_date: str = ds_simp["time"][-1].dt.strftime("%Y%m%d").values.ravel()[0] - - descr1 = "" - if kwargs["method"] in DISTRIBUTION_METHODS: - descr1 = f"_quantiles-{kwargs['quantiles']}" - kwargs["group"] = None - else: - kwargs["quantiles"] = None - - kwargs.update({"start_date": start_date, "end_date": end_date, "descr1": descr1}) - - xkwargs = { - "obs": ds_obs, - "simh": ds_simh, - "simp": ds_simp, - "method": kwargs["method"], - "kind": kwargs["kind"], - } - - if kwargs["method"] in SCALING_METHODS: - xkwargs["group"] = kwargs["group"] - if kwargs["method"] in DISTRIBUTION_METHODS: - xkwargs["n_quantiles"] = kwargs["quantiles"] - - print("**Starting correction**") - - result = adjust(**xkwargs) - - print("**Computation done - saving the result now**") - - save_to_netcdf(result, **kwargs) - print("**Done**") - - -if __name__ == "__main__": - main() - sys.exit(0) diff --git a/examples/requirements.txt b/examples/requirements.txt index 1b2b590..f217e83 100644 --- a/examples/requirements.txt +++ b/examples/requirements.txt @@ -1,4 +1,3 @@ -click matplotlib netCDF4 python-cmethods diff --git a/pyproject.toml b/pyproject.toml index aaf253e..d9ea37a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ description = "Collection of bias correction procedures for single and multidime readme = "README.md" license = { file = "LICENSE" } requires-python = ">=3.8" -dependencies = ["xarray>=2022.11.0", "netCDF4>=1.6.1", "numpy"] +dependencies = ["xarray>=2022.11.0", "netCDF4>=1.6.1", "numpy", "cloup"] keywords = [ "climate-science", "bias", @@ -42,6 +42,8 @@ classifiers = [ "Operating System :: MacOS", "Operating System :: Unix", ] +[project.scripts] +cmethods = "cmethods:cli" [project.urls] "Homepage" = "https://github.com/btschwertfeger/python-cmethods" @@ -79,13 +81,6 @@ skip_empty = true [project.optional-dependencies] jupyter = ["venv-kernel"] dev = [ - # testing - "pytest", - "pytest-cov", - "zarr", - "dask[distributed]", - "scikit-learn", - "scipy", # building "setuptools_scm", # documentation @@ -94,11 +89,19 @@ dev = [ # linting "pylint", "flake8", - "ruff==0.1.13", + "ruff==0.3.5", # typing "mypy", ] - +test = [ + # testing + "pytest", + "pytest-cov", + "zarr", + "dask[distributed]", + "scikit-learn", + "scipy", +] examples = ["click", "matplotlib"] [tool.codespell] @@ -173,7 +176,7 @@ strict = true # https://beta.ruff.rs/docs/settings/ # src = ["cmethods"] -select = [ +lint.select = [ "A", # flake8-builtins "AIR", # Airflow "ASYNC", # flake8-async @@ -210,34 +213,52 @@ select = [ "TCH", # flake8-type-checking "TID", # flake8-tidy-imports "ARG", # flake8-unused-arguments - "CPY", # flake8-copyright - "FBT", # boolean trap - "PTH", # flake8-use-pathlib - "FURB", # refurb + # "CPY", # flake8-copyright + "FBT", # boolean trap + "PTH", # flake8-use-pathlib + "FURB", # refurb # "ERA", # eradicate # commented-out code # "FIX", # flake8-fixme # "TD", # flake8-todos # "TRY", # tryceratops # specify exception messages in class; not important ] -fixable = [ - "I", +lint.fixable = [ + "C4", "C4", - "Q", - "PT", - "ICN", "COM", - "RSE", - "PT", + "COM", + "E202", # Missing Whitespace before ')' + "E221", # Multiple spaces before operator + "E225", # Missing whitespace around operator + "E227", # Missing whitespace around bitwise or shift operator + "E231", # Missing whitespace after ':' + "E252", # Missing whitespace around parameter equals + "E261", # Missing insert at least two spaces before inline comment + "E302", # Expected 2 blank lines, found x + "E303", # Too many blank lines + "E701", # Multiple statements on one line (colon) + "F401", # unused import "FA", "FA100", + "I", + "ICN", + "PL", # pylint "PLR6201", + "PT", + "Q", + "RSE", + "UP", # pyupgrade + "W292", # missing new line at end of file + "W293", # blank line with whitespace + "W391", # too many new lines at end of file ] -ignore = [ +lint.ignore = [ # "B019", # use of lru_cache or cache # "PLR2004", # magic value in comparison # "E203", # Whitespace before ':' # false positive on list slices # "PLR6301", # Method `…` could be a function or static method # false positive for no-self-use + "G002", # lazy logging could be f-string ] respect-gitignore = true @@ -245,9 +266,9 @@ exclude = [] line-length = 130 cache-dir = ".cache/ruff" -task-tags = ["todo", "TODO"] +lint.task-tags = ["todo", "TODO"] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "doc/*.py" = [ "CPY001", # Missing copyright notice at top of file "PTH118", # `os.path.join()` should be replaced by `Path` with `/` operator, @@ -273,24 +294,19 @@ task-tags = ["todo", "TODO"] "T201", # `print` found ] -[tool.ruff.flake8-copyright] -author = "Benjamin Thomas Schwertfeger" -notice-rgx = "(?i)Copyright \\(C\\) \\d{4}" -min-file-size = 1024 - -[tool.ruff.flake8-quotes] +[tool.ruff.lint.flake8-quotes] docstring-quotes = "double" -[tool.ruff.flake8-tidy-imports] +[tool.ruff.lint.flake8-tidy-imports] ban-relative-imports = "all" -[tool.ruff.flake8-bandit] +[tool.ruff.lint.flake8-bandit] check-typed-exception = true -[tool.ruff.flake8-type-checking] +[tool.ruff.lint.flake8-type-checking] strict = true -[tool.ruff.pep8-naming] +[tool.ruff.lint.pep8-naming] ignore-names = [ "i", "j", @@ -305,7 +321,7 @@ ignore-names = [ "QDM1", ] -[tool.ruff.pylint] +[tool.ruff.lint.pylint] max-args = 8 max-branches = 15 max-locals = 25 @@ -621,7 +637,7 @@ overgeneral-exceptions = ["builtin.BaseException", "builtin.Exception"] ignore-long-lines = "^\\s*(# )??$" # Number of spaces of indent required inside a hanging or continued line. -indent-after-parent = 4 +# indent-after-parent = 4 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). diff --git a/tests/conftest.py b/tests/conftest.py index f52130c..0289705 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,7 @@ import pytest import xarray as xr +from click.testing import CliRunner from dask.distributed import LocalCluster from .helper import get_datasets @@ -21,6 +22,12 @@ FIXTURE_DIR: str = os.path.join(os.path.dirname(__file__), "fixture") +@pytest.fixture() +def cli_runner() -> CliRunner: + """Provide a cli-runner for testing the CLI""" + return CliRunner() + + @pytest.fixture(scope="session") def dask_cluster() -> Any: # Create a Dask LocalCluster diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..73eea47 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2024 Benjamin Thomas Schwertfeger +# GitHub: https://github.com/btschwertfeger +# + +"""Module implementing the tests regarding the CLI""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from click.testing import CliRunner +import logging +import os +from pathlib import Path +from tempfile import TemporaryDirectory + +from cmethods import cli + + +@pytest.mark.parametrize( + ("method", "kind", "exclusive"), + [ + ("linear_scaling", "+", "--group=time.month"), + ("linear_scaling", "*", "--group=time.month"), + ("variance_scaling", "+", "--group=time.month"), + ("delta_method", "+", "--group=time.month"), + ("delta_method", "*", "--group=time.month"), + ("quantile_mapping", "+", "--quantiles=100"), + ("quantile_mapping", "*", "--quantiles=100"), + ("quantile_delta_mapping", "+", "--quantiles=100"), + ("quantile_delta_mapping", "*", "--quantiles=100"), + ], +) +def test_cli_runner( + method: str, + kind: str, + exclusive: str, + cli_runner: CliRunner, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test checking the command-line interface.""" + logging.root.setLevel(logging.DEBUG) + with TemporaryDirectory() as tmp_dir: + output = f"{os.path.join(tmp_dir, method)}.nc" + cmd: list[str] = [ + f"--obs={os.path.join('examples', 'input_data', 'observations.nc')}", + f"--simh={os.path.join('examples', 'input_data', 'control.nc')}", + f"--simp={os.path.join('examples', 'input_data', 'scenario.nc')}", + f"--method={method}", + f"--kind={kind}", + "--variable=tas", + exclusive, + f"--output={output}", + ] + result = cli_runner.invoke(cli, cmd) + assert result.exit_code == 0, result.exception + assert Path(output).is_file() + + for phrase in ( + "Loading data sets ...", + "Data sets loaded ...", + f"Applying {method} ...", + f"Saving result to {output}", + ): + assert phrase in caplog.text + + +def test_cli_runner_missing_variable( + cli_runner: CliRunner, + caplog: pytest.LogCaptureFixture, +) -> None: + """ + Test checking the command-line interface for failure due to missing variable + in data set. + """ + logging.root.setLevel(logging.DEBUG) + with TemporaryDirectory() as tmp_dir: + output = f"{os.path.join(tmp_dir, 'linear_scaling.nc')}" + cmd: list[str] = [ + f"--obs={os.path.join('examples', 'input_data', 'observations.nc')}", + f"--simh={os.path.join('examples', 'input_data', 'control.nc')}", + f"--simp={os.path.join('examples', 'input_data', 'scenario.nc')}", + "--method=linear_scaling", + "--kind=add", + "--variable=proc", + "--group=time.month", + f"--output={output}", + ] + result = cli_runner.invoke(cli, cmd) + assert result.exit_code == 1, result.exception + assert not Path(output).is_file() + + for phrase in ( + "Loading data sets ...", + "Variable 'proc' is missing in the observation data set", + ): + assert phrase in caplog.text