diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 37a7c4983..98a6415cd 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,8 +1,6 @@ name: Checks - on: workflow_call: - jobs: Version-Check: name: Version @@ -10,44 +8,38 @@ jobs: permissions: contents: read steps: - - name: SCM Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Check Version(s) - run: poetry run -- nox -s version:check - + - name: SCM Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: "3.10" + poetry-version: "2.3.0" + - name: Check Version(s) + run: poetry run -- nox -s version:check Documentation: name: Docs - needs: [ Version-Check ] + needs: + - Version-Check runs-on: "ubuntu-24.04" permissions: contents: read steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Build Documentation - run: | - poetry run -- nox -s docs:build - - - name: Link Check - run: | - poetry run -- nox -s links:check - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: "3.10" + poetry-version: "2.3.0" + - name: Build Documentation + run: | + poetry run -- nox -s docs:build + - name: Link Check + run: | + poetry run -- nox -s links:check Changelog: name: Changelog Update Check runs-on: "ubuntu-24.04" @@ -55,143 +47,143 @@ jobs: contents: read if: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/master' }} steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Run changelog update check - run: poetry run -- nox -s changelog:updated - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: "3.10" + poetry-version: "2.3.0" + - name: Run changelog update check + run: poetry run -- nox -s changelog:updated Lint: name: Linting (Python-${{ matrix.python-versions }}) - needs: [ Version-Check ] + needs: + - Version-Check runs-on: "ubuntu-24.04" permissions: contents: read strategy: fail-fast: false matrix: - python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-versions: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python-versions }} - poetry-version: "2.3.0" - - - name: Run lint - run: poetry run -- nox -s lint:code - - - name: Upload Artifacts - uses: actions/upload-artifact@v6 - with: - name: lint-python${{ matrix.python-versions }} - path: | - .lint.txt - .lint.json - include-hidden-files: true - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: ${{ matrix.python-versions }} + poetry-version: "2.3.0" + - name: Run lint + run: poetry run -- nox -s lint:code + - name: Upload Artifacts + uses: actions/upload-artifact@v6 + with: + name: lint-python${{ matrix.python-versions }} + path: | + .lint.txt + .lint.json + include-hidden-files: true Type-Check: name: Type Checking (Python-${{ matrix.python-versions }}) - needs: [ Version-Check ] + needs: + - Version-Check runs-on: "ubuntu-24.04" permissions: contents: read strategy: fail-fast: false matrix: - python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] - + python-versions: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python-versions }} - poetry-version: "2.3.0" - - - name: Run type-check - run: poetry run -- nox -s lint:typing - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: ${{ matrix.python-versions }} + poetry-version: "2.3.0" + - name: Run type-check + run: poetry run -- nox -s lint:typing Security: name: Security Checks (Python-${{ matrix.python-versions }}) - needs: [ Version-Check ] + needs: + - Version-Check runs-on: "ubuntu-24.04" permissions: contents: read strategy: fail-fast: false matrix: - python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] - + python-versions: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python-versions }} - poetry-version: "2.3.0" - - - name: Run security linter - run: poetry run -- nox -s lint:security - - - name: Upload Artifacts - uses: actions/upload-artifact@v6 - with: - name: security-python${{ matrix.python-versions }} - path: .security.json - include-hidden-files: true - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: ${{ matrix.python-versions }} + poetry-version: "2.3.0" + - name: Run security linter + run: poetry run -- nox -s lint:security + - name: Upload Artifacts + uses: actions/upload-artifact@v6 + with: + name: security-python${{ matrix.python-versions }} + path: .security.json + include-hidden-files: true Format: name: Format Check runs-on: "ubuntu-24.04" permissions: contents: read steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Run format check - run: poetry run -- nox -s format:check - - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: "3.10" + poetry-version: "2.3.0" + - name: Run format check + run: poetry run -- nox -s format:check Build-Packages: name: Build Package Check - needs: [ Documentation, Lint, Type-Check, Security, Format ] + needs: + - Documentation + - Lint + - Type-Check + - Security + - Format runs-on: "ubuntu-24.04" permissions: contents: read steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Run Distribution Check - run: poetry run -- nox -s package:check - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: "3.10" + poetry-version: "2.3.0" + - name: Run Distribution Check + run: poetry run -- nox -s package:check Lint-Imports: name: Lint Imports runs-on: ubuntu-24.04 @@ -200,46 +192,46 @@ jobs: steps: - name: SCM Checkout uses: actions/checkout@v6 - - name: Setup Python & Poetry Environment uses: ./.github/actions/python-environment with: python-version: "3.10" poetry-version: "2.3.0" - - name: Run import linter run: poetry run -- nox -s lint:import - Tests: name: Unit-Tests (Python-${{ matrix.python-versions }}) - needs: [ Build-Packages, Lint-Imports ] + needs: + - Build-Packages + - Lint-Imports runs-on: "ubuntu-24.04" permissions: contents: read strategy: fail-fast: false matrix: - python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] - + python-versions: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" steps: - - name: SCM Checkout - uses: actions/checkout@v6 - # The PTB has unit tests which require the fetch-depth to be 0. - with: - fetch-depth: 0 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python-versions }} - poetry-version: "2.3.0" - - - name: Run Tests and Collect Coverage - run: poetry run -- nox -s test:unit -- --coverage - - - name: Upload Artifacts - uses: actions/upload-artifact@v6 - with: - name: coverage-python${{ matrix.python-versions }}-fast - path: .coverage - include-hidden-files: true + - name: SCM Checkout + uses: actions/checkout@v6 + # The PTB has unit tests which require the fetch-depth to be 0. + with: + fetch-depth: 0 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: ${{ matrix.python-versions }} + poetry-version: "2.3.0" + - name: Run Tests and Collect Coverage + run: poetry run -- nox -s test:unit -- --coverage + - name: Upload Artifacts + uses: actions/upload-artifact@v6 + with: + name: coverage-python${{ matrix.python-versions }}-fast + path: .coverage + include-hidden-files: true diff --git a/doc/_static/github-workflows.png b/doc/_static/github-workflows.png deleted file mode 100644 index 885249ed1..000000000 Binary files a/doc/_static/github-workflows.png and /dev/null differ diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 549f78b52..11163776b 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -5,3 +5,7 @@ ## Feature * #673: Switched `checks.yml` to get Python versions for matrix from `BaseConfig` + +## Documentation + +* #676: Move GitHub Workflows to be inside features & updated diff --git a/doc/conf.py b/doc/conf.py index b1d64b738..2a2aa812a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -33,6 +33,7 @@ "sphinx.ext.napoleon", "sphinx.ext.intersphinx", "sphinx.ext.autosummary", + "sphinxcontrib.mermaid", "sphinx_toolbox.collapse", "sphinx_copybutton", "myst_parser", diff --git a/doc/user_guide/workflows.rst b/doc/user_guide/features/github_workflows/configuration.rst similarity index 51% rename from doc/user_guide/workflows.rst rename to doc/user_guide/features/github_workflows/configuration.rst index ea4752e48..aa654fcaa 100644 --- a/doc/user_guide/workflows.rst +++ b/doc/user_guide/features/github_workflows/configuration.rst @@ -1,31 +1,14 @@ -.. _GitHub Workflows: - -GitHub Workflows -================ - -.. figure:: ../_static/github-workflows.png - :alt: GitHub Workflow Example - -The exasol-toolbox ships with various GitHub workflow templates. To leverage the full feature set of the toolbox, you should use them. - -.. attention:: - - Generally, it is advised to install/use all workflows provided by the toolbox as a whole due to their interdependencies. - - However, if you know what you are doing and are well-versed in GitHub workflows and actions, you can use just select individual ones or use them as inspiration. Still, an individual approach is likely to be more error-prone. - -.. note:: - - The toolbox command itself, :code:`tbx`, provides various CLI functions to help you maintain those workflows. - For further help, run the command :code:`tbx workflow --help`. +.. _github_workflows_configuration: +Configuration +============= 1. Configure the GitHub project +++++++++++++++++++++++++++++++ * Make sure your GitHub project has access to a deployment token for PyPi with the following name: **PYPI_TOKEN**. It should be available to the repository either as an Organization-, Repository-, or Environment-secret. -* If your CI workflow involves slow or expensive steps you can guard these to be executed only after manual approval. The CI workflow will automaticall create a GitHub environment named :code:`manual-approval`. You only need to add reviewers in (:code:`Settings/Environments/manual-approval`) and move the steps to be guarded into the related section in job :code:`slow-checks` in file :code:`.github/workflows/merge-gate.yml`. +* If your CI workflow involves slow or expensive steps you can guard these to be executed only after manual approval. The CI workflow will automatically create a GitHub environment named :code:`manual-approval`. You only need to add reviewers in (:code:`Settings/Environments/manual-approval`) and move the steps to be guarded into the related section in job :code:`slow-checks` in file :code:`.github/workflows/merge-gate.yml`. 2. Add all workflows to your project ++++++++++++++++++++++++++++++++++++ diff --git a/doc/user_guide/features/github_workflows/index.rst b/doc/user_guide/features/github_workflows/index.rst new file mode 100644 index 000000000..ac2ed348f --- /dev/null +++ b/doc/user_guide/features/github_workflows/index.rst @@ -0,0 +1,179 @@ +.. _GitHub Workflows: + +Enabling GitHub Workflows +========================= + +.. toctree:: + :maxdepth: 2 + :hidden: + + configuration + +The PTB ships with configurable GitHub workflow templates covering the most common +CI/CD setup variants for Python projects. The templates are defined in: +`exasol/toolbox/templates/github/workflows `__. + +The PTB provides a command line interface (CLI) for generating and updating actual +workflows from the templates. + +.. code-block:: bash + + poetry run -- tbx workflow --help + +.. attention:: + + In most cases, we recommend using _all_ workflows without change to ensure + consistent interdependencies. For more details, see :ref:`ci_actions`. + +Underlying the CLI, the PTB uses Jinja to dynamically generate project-specific +workflows. The rendering process is supported by the ``github_template_dict`` found in +your ``noxconfig.py::PROJECT_CONFIG``. This dictionary is inherited by default from +``BaseConfig.py``, ensuring a standardized baseline that can be easily overridden, if +necessary. + +.. literalinclude:: ../../../../exasol/toolbox/config.py + :language: python + :start-at: github_template_dict + + +Workflows +--------- + +.. list-table:: + :widths: 25 25 50 + :header-rows: 1 + + * - Filename + - Run on + - Description + * - ``build-and-publish.yml`` + - Workflow call + - Packages the distribution and publishes it to PyPi and GitHub. + * - ``cd.yml`` + - Push with new tag + - Manages continuous delivery by calling ``check-release-tag.yml``, + ``build-and-publish.yml``, and ``gh-pages.yml``. See :ref:`cd_yml` + for a graph of workflow calls. + * - ``check-release-tag.yml`` + - Workflow call + - Verifies that the release tag matches the project's internal versioning. + * - ``checks.yml`` + - Workflow call + - Executes many small & fast checks: builds documentation and validates + cross-references (AKA. "links") to be valid,runs various linters + (security, type checks, etc.), and unit tests. + * - ``ci.yml`` + - Pull request and monthly + - Executes the continuous integration suite by calling ``merge-gate.yml`` and + ``report.yml``. See :ref:`ci_yml` for a graph of workflow calls. + * - ``gh-pages.yml`` + - Workflow call + - Builds the documentation and deploys it to GitHub Pages. + * - ``matrix-all.yml`` + - Workflow call + - Calls Nox session ``matrix:all``, which typically evaluates ``exasol_versions`` + and ``python_versions`` from the ``PROJECT_CONFIG``. + * - ``matrix-exasol.yml`` + - Workflow call + - Calls Nox session ``matrix:exasol`` to get the ``exasol_versions`` from the + ``PROJECT_CONFIG``. + * - ``matrix-python.yml`` + - Workflow call + - Calls Nox session ``matrix:python`` to get the ``python_versions`` from the + ``PROJECT_CONFIG``. + * - ``merge-gate.yml`` + - Workflow call + - Acts as a final status check (gatekeeper) to ensure all required CI steps have + passed before allowing to merge the branch of your pull request to the + default branch of the repository. e.g. ``main``. + * - ``pr-merge.yml`` + - Push to main + - Runs ``checks.yml``, ``gh-pages.yml``, and ``report.yml``. See + :ref:`pr_merge_yml` for a graph of called workflows. + * - ``report.yml`` + - Workflow call + - Downloads results from code coverage analysis and linting, + creates a summary displayed by GitHub as result of running + the action, and uploads the results to Sonar. + * - ``slow-checks.yml`` + - Workflow call + - Runs long-running checks, which typically involve an Exasol database instance. + +.. _ci_actions: + +CI Actions +---------- + +.. _ci_yml: + +Pull Request +^^^^^^^^^^^^ + +When any pull request is opened, synchronized, or reopened, then the ``ci.yml`` will be +triggered. + +When configured as described on :ref:`github_workflows_configuration`, the +``run-slow-tests`` job requires a developer to manually approve executing the slower +workflows, like ``slow-checks.yml``. This allows developers to update their pull +request more often and to only periodically run the more time-expensive tests. + +If one of the jobs in the chain fails (or if ``run-slow-tests`` is not approved), +then the subsequent jobs will not be started. + +.. mermaid:: + + graph TD + %% Workflow Triggers (Solid Lines) + ci[ci.yml] --> merge-gate[merge-gate.yml] + ci --> metrics[report.yml] + + merge-gate --> fast-checks[checks.yml] + merge-gate --> run-slow-tests[run-slow-tests] + run-slow-tests -.->|needs| slow-checks[slow-checks.yml] + + %% Dependencies / Waiting (Dotted Lines) + fast-checks -.->|needs| approve-merge[approve-merge] + slow-checks -.->|needs| approve-merge + + %% Final Dependency + approve-merge -.->|needs| metrics + + %% Visual Styling to distinguish jobs + style approve-merge fill:#fff,stroke:#333,stroke-dasharray: 5 5 + style run-slow-tests fill:#fff,stroke:#333,stroke-dasharray: 5 5 + + +.. _pr_merge_yml: + +Merge +^^^^^ + +When a pull request is merged to main, then the ``pr-merge.yml`` workflow is activated. + +.. mermaid:: + + graph TD + %% Workflow Triggers (Solid Lines) + pr-merge[pr-merge.yml] --> checks[checks.yml] + pr-merge --> publish-docs[publish-docs.yml] + + %% Dependencies / Waiting (Dotted Lines) + checks -.->|needs| report[report.yml] + +.. _cd_yml: + +Release +^^^^^^^ + +When the nox session ``release:trigger`` is used, a new tag is created & pushed +to main. This starts the release process by activating the ``cd.yml`` workflow. + +.. mermaid:: + + graph TD + %% Workflow Triggers (Solid Lines) + cd[cd.yml] --> check-release-tag[check-release-tag.yml] + + %% Dependencies / Waiting (Dotted Lines) + check-release-tag -.->|needs| build-and-publish[build-and-publish.yml] + build-and-publish -.->|needs| gh-pages[gh-pages.yml] diff --git a/doc/user_guide/features/index.rst b/doc/user_guide/features/index.rst index 6a57544fd..bc0e7555b 100644 --- a/doc/user_guide/features/index.rst +++ b/doc/user_guide/features/index.rst @@ -10,6 +10,7 @@ Features creating_a_release documentation/index git_hooks/index + github_workflows/index formatting_code/index managing_dependencies diff --git a/doc/user_guide/user_guide.rst b/doc/user_guide/user_guide.rst index 1e2957b3e..2effc68e9 100644 --- a/doc/user_guide/user_guide.rst +++ b/doc/user_guide/user_guide.rst @@ -10,6 +10,5 @@ getting_started configuration features/index - workflows customization migrating diff --git a/exasol/toolbox/tools/template.py b/exasol/toolbox/tools/template.py index 14829ea1f..73fa4b857 100644 --- a/exasol/toolbox/tools/template.py +++ b/exasol/toolbox/tools/template.py @@ -1,5 +1,6 @@ import difflib import io +import re from collections.abc import Mapping from contextlib import ExitStack from inspect import cleandoc @@ -15,6 +16,7 @@ from rich.columns import Columns from rich.console import Console from rich.syntax import Syntax +from yaml.resolver import Resolver from noxconfig import PROJECT_CONFIG @@ -70,6 +72,51 @@ def show_templates( ) # type: ignore +# yaml uses a shorthand to identify "on" and "off" tags. +# for GitHub workflows, we do NOT want "on" replaced with "True". +for character in ["O", "o"]: + Resolver.yaml_implicit_resolvers[character] = [ + x + for x in Resolver.yaml_implicit_resolvers[character] + if x[0] != "tag:yaml.org,2002:bool" + ] + + +class GitHubDumper(yaml.SafeDumper): + pass + + +def empty_representer(dumper, data): + """ + Leave empty fields without 'null' + + on: + workflow_call: + """ + return dumper.represent_scalar("tag:yaml.org,2002:null", "") + + +# Regex for common strings that lose quotes: +# 1. Version numbers (e.g., 2.3.0, 3.10) +# 2. OS/image names (e.g., ubuntu-24.04) +# 3. Numeric strings that look like octals or floats (e.g., 045, 1.2) +QUOTE_REGEX = re.compile(r"^(\d+\.\d+(\.\d+)?|[a-zA-Z]+-\d+\.\d+|0\d+)$") + + +def str_presenter(dumper, data): + # Use literal style '|' for strings with newlines + if "\n" in data: + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + if QUOTE_REGEX.match(data): + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"') + return dumper.represent_scalar("tag:yaml.org,2002:str", data) + + +# Register it to the dumper +GitHubDumper.add_representer(str, str_presenter) +GitHubDumper.add_representer(type(None), empty_representer) + + def _render_template( src: str | Path, stack: ExitStack, @@ -80,10 +127,15 @@ def _render_template( template = jinja_env.from_string(input_file.read()) rendered_string = template.render(PROJECT_CONFIG.github_template_dict) - # validate that the rendered content is a valid YAML. This is not - # written out as by default it does not give GitHub-safe output. - yaml.safe_load(rendered_string) - return cleandoc(rendered_string) + "\n" + # this line also checks that the rendered content is a valid YAML + data = yaml.safe_load(rendered_string) + output = yaml.dump( + data, + Dumper=GitHubDumper, + sort_keys=False, # if True, then re-orders the jobs alphabetically + ) + + return cleandoc(output) + "\n" def diff_template(template: str, dest: Path, pkg: str, template_type: str) -> None: diff --git a/poetry.lock b/poetry.lock index 029cfc7cf..94fb0f9bd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3563,6 +3563,26 @@ files = [ [package.extras] test = ["flake8", "mypy", "pytest"] +[[package]] +name = "sphinxcontrib-mermaid" +version = "2.0.0" +description = "Mermaid diagrams in your Sphinx-powered docs" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "sphinxcontrib_mermaid-2.0.0-py3-none-any.whl", hash = "sha256:59a73249bbee2c74b1a4db036f8e8899ade65982bdda6712cf22b4f4e9874bb5"}, + {file = "sphinxcontrib_mermaid-2.0.0.tar.gz", hash = "sha256:cf4f7d453d001132eaba5d1fdf53d42049f02e913213cf8337427483bfca26f4"}, +] + +[package.dependencies] +jinja2 = "*" +pyyaml = "*" +sphinx = "*" + +[package.extras] +test = ["defusedxml", "myst-parser", "pytest", "ruff", "sphinx"] + [[package]] name = "sphinxcontrib-qthelp" version = "2.0.0" @@ -3923,4 +3943,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "27476fa095d516f571fec7473aa0018edf057a53bf5a2934182915d937997ef0" +content-hash = "8a5d05c9cc28f8599edd5055881b350ebf0588dcaed75656aa1ad2cc30b4d7ce" diff --git a/pyproject.toml b/pyproject.toml index 9c449a64f..dd37f49d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ dependencies = [ "sphinx-toolbox>=4.0.0,<5", "typer[all]>=0.7.0", "twine>=6.1.0,<7", + "sphinxcontrib-mermaid (>=2.0.0,<3.0.0)", ] [project.scripts]