diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d5e13de2..9c22959c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,6 +45,7 @@ repos: args: [] additional_dependencies: - click + - json5>=0.15.0 - markdown-it-py - pytest - nox diff --git a/README.md b/README.md index a07d77c3..a5018f8e 100644 --- a/README.md +++ b/README.md @@ -348,6 +348,10 @@ for family, grp in itertools.groupby(collected.checks.items(), key=lambda x: x[1 - [`PP308`](https://learn.scientific-python.org/development/guides/pytest#PP308): Specifies useful pytest summary - [`PP309`](https://learn.scientific-python.org/development/guides/pytest#PP309): Filter warnings specified +### dependencies + +- [`DEP200`](https://learn.scientific-python.org/development/guides/gha-basic#DEP200): Maintained by Dependabot or Renovate. + ### GitHub Actions - [`GH100`](https://learn.scientific-python.org/development/guides/gha-basic#GH100): Has GitHub Actions config @@ -400,6 +404,11 @@ Will not show up if using lefthook instead of pre-commit/prek. - [`PC902`](https://learn.scientific-python.org/development/guides/style#PC902): Custom pre-commit CI autofix message - [`PC903`](https://learn.scientific-python.org/development/guides/style#PC903): Specified pre-commit CI schedule +### Renovate + +- [`REN200`](https://learn.scientific-python.org/development/guides/gha-basic#REN200): Maintained by Renovate +- [`REN210`](https://learn.scientific-python.org/development/guides/gha-basic#REN210): Maintains the GitHub action versions with Renovate + ### ReadTheDocs Will not show up if no `.readthedocs.yml`/`.readthedocs.yaml` file is present. diff --git a/docs/guides/gha_basic.md b/docs/guides/gha_basic.md index 310eedb6..b60a7746 100644 --- a/docs/guides/gha_basic.md +++ b/docs/guides/gha_basic.md @@ -159,7 +159,8 @@ static. And old versioned images are decommissioned. ## Updating -{rr}`GH200` {rr}`GH210` If you use non-default actions in your repository +{rr}`DEP200` {rr}`GH200` {rr}`GH210` +If you use non-default actions in your repository (you will see some in the following pages), then it's a good idea to keep them up to date. GitHub provided a way to do this with dependabot. Just add the following file as `.github/dependabot.yml`: @@ -189,6 +190,24 @@ which is both cleaner and sometimes required for dependent actions, like You can use this for other ecosystems too, including Python. +{rr}`REN200` {rr}`REN210` [Renovate](https://docs.renovatebot.com/) can also be +used for keeping GitHub Actions (and other ecosystems) up to date as well. A +good starting point for `renovate.json` with the +[hosted version](https://docs.renovatebot.com/getting-started/installing-onboarding/) +which will cover GitHub Actions and most other ecosystems is: + +```json +{ + "extends": ["config:recommended"] +} +``` + +:::{tip} +Most Renovate `.jsonc` or `.json5` configs can be parsed without extra +dependencies, but include the `json5` optional dependency in order to load the +full range of the formats. +::: + ## Common needs ### Single OS steps diff --git a/pyproject.toml b/pyproject.toml index 4a708844..18172f83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,10 @@ async = [ "repo-review[async]", ] all = [ - "sp-repo-review[cli,pyproject,async]", + "sp-repo-review[cli,pyproject,async,json5]", +] +json5 = [ + "json5>=0.15.0", ] [project.urls] @@ -62,6 +65,7 @@ sp-repo-review = "repo_review.__main__:main" sp-ruff-checks = "sp_repo_review.ruff_checks.__main__:main" [project.entry-points."repo_review.checks"] +dependencies = "sp_repo_review.checks.dependencies:repo_review_checks" general = "sp_repo_review.checks.general:repo_review_checks" pyproject = "sp_repo_review.checks.pyproject:repo_review_checks" precommit = "sp_repo_review.checks.precommit:repo_review_checks" @@ -70,6 +74,7 @@ mypy = "sp_repo_review.checks.mypy:repo_review_checks" github = "sp_repo_review.checks.github:repo_review_checks" security = "sp_repo_review.checks.security:repo_review_checks" readthedocs = "sp_repo_review.checks.readthedocs:repo_review_checks" +renovate = "sp_repo_review.checks.renovate:repo_review_checks" setupcfg = "sp_repo_review.checks.setupcfg:repo_review_checks" noxfile = "sp_repo_review.checks.noxfile:repo_review_checks" @@ -79,6 +84,7 @@ noxfile = "sp_repo_review.checks.noxfile:noxfile" precommit = "sp_repo_review.checks.precommit:precommit" pytest = "sp_repo_review.checks.pyproject:pytest" readthedocs = "sp_repo_review.checks.readthedocs:readthedocs" +renovate = "sp_repo_review.checks.renovate:renovate" ruff = "sp_repo_review.checks.ruff:ruff" setupcfg = "sp_repo_review.checks.setupcfg:setupcfg" workflows = "sp_repo_review.checks.github:workflows" @@ -101,6 +107,7 @@ dev = [ test = [ "pytest >=9", "repo-review >=0.10.6", + "sp-repo-review[json5]" ] cog = [ "cogapp", diff --git a/src/sp_repo_review/checks/_jsonc.py b/src/sp_repo_review/checks/_jsonc.py new file mode 100644 index 00000000..bb9c9d43 --- /dev/null +++ b/src/sp_repo_review/checks/_jsonc.py @@ -0,0 +1,91 @@ +"""Minimal, string-aware JSONC preprocessing. + +Strips ``//`` and ``/* */`` comments and trailing commas so that JSONC (and the +subset of JSON5 that only uses those features) can be parsed by the standard +library :mod:`json`. Full JSON5 (single-quoted strings, unquoted keys, hex +numbers, etc.) is *not* handled here and requires the optional ``json5`` +dependency. +""" + +from __future__ import annotations + +_WHITESPACE = " \t\r\n" + + +def _copy_string(text: str, i: int, out: list[str]) -> int: + """Copy a double-quoted string verbatim, returning the index after it.""" + length = len(text) + out.append(text[i]) # opening quote + i += 1 + while i < length: + char = text[i] + out.append(char) + if char == "\\" and i + 1 < length: + # Copy the escaped character so an escaped quote does not + # prematurely close the string. + out.append(text[i + 1]) + i += 2 + elif char == '"': + i += 1 + break + else: + i += 1 + return i + + +def _skip_line_comment(text: str, i: int) -> int: + """Skip a ``// ...`` comment, returning the index of the line ending.""" + i += 2 + length = len(text) + while i < length and text[i] not in "\r\n": + i += 1 + return i + + +def _skip_block_comment(text: str, i: int) -> int: + """Skip a ``/* ... */`` comment, returning the index after it.""" + i += 2 + length = len(text) + while i + 1 < length and not (text[i] == "*" and text[i + 1] == "/"): + i += 1 + return i + 2 # Past the end is harmless; the caller re-checks bounds. + + +def _drop_trailing_comma(out: list[str]) -> None: + """Remove a trailing comma before a closing bracket, ignoring whitespace.""" + last = len(out) - 1 + while last >= 0 and out[last] in _WHITESPACE: + last -= 1 + if last >= 0 and out[last] == ",": + del out[last] + + +def strip_jsonc(text: str) -> str: + """Remove comments and trailing commas from JSONC ``text``. + + The scan is string-aware: characters inside double-quoted strings (and their + backslash escapes) are copied verbatim, so comment markers or commas that + appear inside string values are left untouched. The function is total and + never raises; validating the result is left to the caller's ``json.loads``. + """ + out: list[str] = [] + i = 0 + length = len(text) + + while i < length: + char = text[i] + if char == '"': + i = _copy_string(text, i, out) + elif char == "/" and text.startswith("//", i): + i = _skip_line_comment(text, i) + elif char == "/" and text.startswith("/*", i): + i = _skip_block_comment(text, i) + elif char in "}]": + _drop_trailing_comma(out) + out.append(char) + i += 1 + else: + out.append(char) + i += 1 + + return "".join(out) diff --git a/src/sp_repo_review/checks/dependencies.py b/src/sp_repo_review/checks/dependencies.py new file mode 100644 index 00000000..41bce971 --- /dev/null +++ b/src/sp_repo_review/checks/dependencies.py @@ -0,0 +1,52 @@ +# DU: Dependency Updating + +from __future__ import annotations + +from typing import Any + +from . import mk_url + + +class Dependencies: + family = "dependencies" + + +class DEP200(Dependencies): + """Maintained by Dependabot or Renovate.""" + + url = mk_url("gha-basic") + + @staticmethod + def check(dependabot: dict[str, Any], renovate: dict[str, Any]) -> bool: + """ + All projects should have a tool to manage dependencies, either Dependabot or Renovate. + + Something like one of these: + + `.github/dependabot.yml` + ```yaml + version: 2 + updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + ``` + + `renovate.json` + ```json + {{ + "extends": ["config:recommended"] + }} + ``` + Renovate configurations in `package.json` are not supported. + `.jsonc` files (and `.json5` files that only use comments or trailing + commas) are supported out of the box. Full JSON5 configs require the + optional `json5` dependency (`pip install sp-repo-review[json5]`). + """ + return bool(dependabot or renovate) + + +def repo_review_checks() -> dict[str, Dependencies]: + return {p.__name__: p() for p in Dependencies.__subclasses__()} diff --git a/src/sp_repo_review/checks/github.py b/src/sp_repo_review/checks/github.py index 1547b238..58428e34 100644 --- a/src/sp_repo_review/checks/github.py +++ b/src/sp_repo_review/checks/github.py @@ -199,7 +199,7 @@ class GH200(GitHub): url = mk_url("gha-basic") @staticmethod - def check(dependabot: dict[str, Any]) -> bool: + def check(dependabot: dict[str, Any]) -> bool | None: """ All projects should have a `.github/dependabot.yml` file to support at least GitHub Actions regular updates. Something like this: @@ -214,6 +214,8 @@ def check(dependabot: dict[str, Any]) -> bool: interval: "weekly" ``` """ + if not dependabot: + return None return bool(dependabot) diff --git a/src/sp_repo_review/checks/renovate.py b/src/sp_repo_review/checks/renovate.py new file mode 100644 index 00000000..bd7fb00b --- /dev/null +++ b/src/sp_repo_review/checks/renovate.py @@ -0,0 +1,125 @@ +# REN: Renovate Actions + +from __future__ import annotations + +__lazy_modules__ = ["json"] + +import json +from typing import TYPE_CHECKING, Any + +from . import mk_url +from ._jsonc import strip_jsonc + +if TYPE_CHECKING: + from .._compat.importlib.resources.abc import Traversable + + +SUPPORTED_RENOVATE_FILES = [ + "renovate.json", + "renovate.jsonc", + "renovate.json5", + ".github/renovate.json", + ".github/renovate.jsonc", + ".github/renovate.json5", + ".gitlab/renovate.json", + ".gitlab/renovate.jsonc", + ".gitlab/renovate.json5", + ".renovaterc", + ".renovaterc.json", + ".renovaterc.jsonc", + ".renovaterc.json5", + # "package.json" # Deprecated by Renovate +] + + +def renovate(root: Traversable) -> dict[str, Any]: + renovate_paths = [root.joinpath(f) for f in SUPPORTED_RENOVATE_FILES] + + for renovate_path in renovate_paths: + if not renovate_path.is_file(): + continue + with renovate_path.open(encoding="utf-8") as f: + text = f.read() + try: + # Handles JSON and JSONC (comments / trailing commas) with no + # dependency, plus JSON5 files that use only those features. + result: dict[str, Any] = json.loads(strip_jsonc(text)) + except json.JSONDecodeError: + # Full JSON5 (single quotes, unquoted keys, ...) needs a real + # parser, provided by the optional json5 extra. + try: + import json5 # noqa: PLC0415 + except ImportError: + msg = ( + f"{renovate_path} could not be parsed as JSON/JSONC and needs " + "full JSON5 support. Install the extra: " + "pip install sp-repo-review[json5]" + ) + raise ImportError(msg) from None + try: + result = json5.loads(text) + except ValueError: + continue + return result + return {} + + +class Renovate: + family = "renovate" + + +class REN200(Renovate): + """Maintained by Renovate""" + + requires = {"DEP200"} + url = mk_url("gha-basic") + + @staticmethod + def check(renovate: dict[str, Any]) -> bool | None: + """ + All projects should have a renovate configuration file (`renovate.json` or other supported locations) + to support dependency updates. Something like this: + + ```json + {{ + "extends": ["config:recommended"] + }} + ``` + + Renovate configurations in `package.json` are not supported. + `.jsonc` files (and `.json5` files that only use comments or trailing + commas) are supported out of the box. Full JSON5 configs require the + optional `json5` dependency (`pip install sp-repo-review[json5]`). + """ + if not renovate: + return None + return bool(renovate) + + +GHA_EXTENDS = {"config:recommended", "config:best-practices"} + + +class REN210(Renovate): + """Maintains the GitHub action versions with Renovate""" + + requires = {"REN200"} + url = mk_url("gha-basic") + + @staticmethod + def check(renovate: dict[str, Any]) -> bool | str | None: + """ + Ensures that Renovate is configured to maintain GitHub action versions. + + Checks for if the `github-actions` manager is enabled or if the Renovate config extends a known config (`config:recommended` or `config:best-practices`). + """ + if (manager := renovate.get("github-actions", {})) and manager.get("enabled"): + return True + if extends := renovate.get("extends", []): + if any(e in GHA_EXTENDS for e in extends): + return True + return f"Renovate config extends {extends}, but none are a known config: {', '.join(GHA_EXTENDS)}." + return False + + +def repo_review_checks() -> dict[str, Renovate]: + return {p.__name__: p() for p in Renovate.__subclasses__()} diff --git a/src/sp_repo_review/families.py b/src/sp_repo_review/families.py index 2dbbf5c1..18dc5e69 100644 --- a/src/sp_repo_review/families.py +++ b/src/sp_repo_review/families.py @@ -102,6 +102,9 @@ def get_families( setupcfg: ConfigParser | None = None, ) -> dict[str, Family]: return { + "dependency-updating": Family( + name="Dependency Updating", + ), "general": Family( name="General", order=-3, @@ -124,6 +127,9 @@ def get_families( "mypy": Family( name="MyPy", ), + "renovate": Family( + name="Renovate", + ), "ruff": Family( name="Ruff", description=ruff_description(ruff), diff --git a/src/sp_repo_review/files.py b/src/sp_repo_review/files.py index fee824a5..82bc6057 100644 --- a/src/sp_repo_review/files.py +++ b/src/sp_repo_review/files.py @@ -1,5 +1,7 @@ from __future__ import annotations +from .checks.renovate import SUPPORTED_RENOVATE_FILES + def prefetch_root() -> set[str]: """ @@ -28,4 +30,5 @@ def prefetch_package() -> set[str]: "noxfile.py", "ruff.toml", ".ruff.toml", + *SUPPORTED_RENOVATE_FILES, } diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py new file mode 100644 index 00000000..fcbbe127 --- /dev/null +++ b/tests/test_dependencies.py @@ -0,0 +1,31 @@ +import yaml +from repo_review.testing import compute_check + +dependabot = yaml.safe_load( + """ + version: 2 + updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + """ +) + +renovate = {"extends": ["config:recommended"]} + + +def test_du100_both() -> None: + assert compute_check("DEP200", dependabot=dependabot, renovate=renovate).result + + +def test_du100_missing_renovate() -> None: + assert compute_check("GH200", dependabot=dependabot, renovate={}).result + + +def test_du100_missing_dependabot() -> None: + assert compute_check("DEP200", dependabot={}, renovate=renovate).result + + +def test_du100_missing_both() -> None: + assert not compute_check("DEP200", dependabot={}, renovate={}).result diff --git a/tests/test_renovate.py b/tests/test_renovate.py new file mode 100644 index 00000000..20ba2820 --- /dev/null +++ b/tests/test_renovate.py @@ -0,0 +1,164 @@ +import json +import sys + +import pytest +from repo_review.ghpath import GHPath +from repo_review.testing import compute_check + +from sp_repo_review.checks._jsonc import strip_jsonc +from sp_repo_review.checks.renovate import renovate + +# Real-world Renovate configs, one per supported format/location, pinned to a +# specific commit so upstream edits cannot break this. Skipped by default +# (network access, GitHub rate limits); run manually to confirm coverage. +REAL_WORLD_CONFIGS = [ + ( + "gulfofmaine/climatology_py_dash", + "e4ead74d17d7c07dbb40b9c99fc1138f927abdd3", + "renovate.json", + ), + ( + "jumpstarter-dev/jumpstarter", + "8bae226d0a2d9cbe156bded24628e85c6d9f6cd9", + "renovate.jsonc", + ), + ( + "SonarSource/docker-sonarqube", + "9f00ce57d8654a3737ae0b22997d7198f71660d8", + "renovate.json5", + ), + ( + "adobe/spectrum-css", + "37620864c60c4c142a506017e1a15348a26abb0e", + ".github/renovate.json", + ), + ( + "paddyroddy/.github", + "c97ca7c448df211268616ce438777228fe103733", + ".renovaterc.json5", + ), + ( + "zammad/zammad", + "93fb7f107b07b4b4294e21249b277bc48c431da5", + ".gitlab/renovate.json", + ), + ( + "prettier/eslint-config-prettier", + "07829b4912d173986610a4985247896b09f9fcaf", + ".renovaterc", + ), + ( + "Esri/calcite-design-system", + "5613d9f8000ba12bf55c7da50e2f119c12435302", + ".renovaterc.json", + ), +] + + +def test_strip_jsonc_line_comment() -> None: + text = '{\n // a comment\n "a": 1 // trailing\n}' + assert json.loads(strip_jsonc(text)) == {"a": 1} + + +def test_strip_jsonc_block_comment() -> None: + text = '{\n /* multi\n line */ "a": 1\n}' + assert json.loads(strip_jsonc(text)) == {"a": 1} + + +def test_strip_jsonc_preserves_string_content() -> None: + # Comment markers and commas inside strings must survive untouched. + text = '{"url": "https://x/y", "csv": "a,b,", "block": "/* not a comment */"}' + assert json.loads(strip_jsonc(text)) == { + "url": "https://x/y", + "csv": "a,b,", + "block": "/* not a comment */", + } + + +def test_strip_jsonc_escaped_quote_in_string() -> None: + text = r'{"a": "she said \"hi\" // ok"}' + assert json.loads(strip_jsonc(text)) == {"a": 'she said "hi" // ok'} + + +def test_strip_jsonc_trailing_commas() -> None: + text = '{\n "a": [1, 2, 3,],\n "b": {"c": 1,},\n}' + assert json.loads(strip_jsonc(text)) == {"a": [1, 2, 3], "b": {"c": 1}} + + +def test_strip_jsonc_plain_json_unchanged() -> None: + text = '{"a": 1, "b": [1, 2]}' + assert strip_jsonc(text) == text + + +def test_renovate_fixture_jsonc(tmp_path) -> None: + (tmp_path / "renovate.jsonc").write_text( + '{\n // pin digests\n "extends": ["config:recommended"],\n}', + encoding="utf-8", + ) + assert renovate(tmp_path) == {"extends": ["config:recommended"]} + + +def test_renovate_fixture_json5_fallback(tmp_path) -> None: + pytest.importorskip("json5") + # Unquoted keys are true JSON5 and cannot be stripped to plain JSON. + (tmp_path / "renovate.json5").write_text( + '{\n extends: ["config:recommended"],\n}', + encoding="utf-8", + ) + assert renovate(tmp_path) == {"extends": ["config:recommended"]} + + +def test_renovate_fixture_json5_missing_errors(tmp_path, monkeypatch) -> None: + (tmp_path / "renovate.json5").write_text( + '{\n extends: ["config:recommended"],\n}', + encoding="utf-8", + ) + # Simulate the json5 extra not being installed. + monkeypatch.setitem(sys.modules, "json5", None) + with pytest.raises(ImportError, match=r"sp-repo-review\[json5\]"): + renovate(tmp_path) + + +@pytest.mark.skip(reason="Network access, can be rate limited") +@pytest.mark.parametrize(("repo", "sha", "location"), REAL_WORLD_CONFIGS) +def test_renovate_real_world(repo, sha, location) -> None: + root = GHPath(repo=repo, branch=sha) + assert root.joinpath(location).is_file() + assert renovate(root) + + +def test_ren200() -> None: + renovate = {"extends": ["config:recommended"]} + assert compute_check("REN200", renovate=renovate).result + + +def test_ren200_missing() -> None: + assert not compute_check("REN200", renovate={}).result + + +def test_ren210_gha_manager() -> None: + renovate = { + "github-actions": { + "enabled": True, + } + } + assert compute_check("REN210", renovate=renovate).result + + +def test_ren210_gha_manager_disabled() -> None: + renovate = { + "github-actions": { + "enabled": False, + } + } + assert not compute_check("REN210", renovate=renovate).result + + +def test_ren210_common_extends() -> None: + renovate = {"extends": ["config:recommended"]} + assert compute_check("REN210", renovate=renovate).result + + +def test_ren210_common_extends_missing() -> None: + renovate = {"extends": ["some-other-config"]} + assert not compute_check("REN210", renovate=renovate).result