diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..3662a64ec1 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,14 @@ +FROM mcr.microsoft.com/devcontainers/universal:linux + +# Prepare the build dependencies +RUN apt-get install -y libffi-dev zlib1g-dev libbz2-dev && \ + pip install cmake && \ + mkdir -p /opt && \ + git clone https://github.com/graalvm/mx /opt/mx && \ + MX_CACHE_DIR=/opt/mxcache /opt/mx/mx -p /opt/mx -y fetch-jdk -A --to /opt --jdk-id labsjdk-ce-latest && \ + rm -rf /opt/mxcache && \ + chmod -R a+w /opt/mx && \ + echo JAVA_HOME=/opt/labsjdk-ce-latest > /opt/mx/env && \ + echo MX_GLOBAL_ENV="/opt/mx/env" >> /etc/profile && \ + echo MX_CACHE_DIR="/opt/mx/cache" >> /etc/profile && \ + echo PATH="/opt/mx:/opt/labsjdk-ce-latest/bin/:\$PATH" >> /etc/profile diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..9a4eef3a24 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,29 @@ +{ + "name": "GraalPy", + "build": { "dockerfile": "Dockerfile" }, + "postCreateCommand": { + "vscodeinit": "sudo chmod a+rwx ../ && mx -y sforceimports && mx -y vscodeinit && ln -sf $PWD/../graalpython.code-workspace graalpython.code-workspace && mx -y build --target GRAALPY_JVM_STANDALONE" + }, + "hostRequirements": { + "cpus": 2, + "memory": "16gb", + "storage": "64gb" + }, + "customizations": { + "vscode": { + "extensions": [ + "vscjava.vscode-java-pack", + "zoma.vscode-auto-open-workspace" + ], + "settings": { + "autoOpenWorkspace.enableAutoOpenIfSingleWorkspace": true, + "java.autobuild.enabled": false + } + }, + "codespaces": { + "openFiles": [ + "docs/contributor/CONTRIBUTING.md" + ] + } + } +} diff --git a/.github/scripts/extract_matrix.py b/.github/scripts/extract_matrix.py index 7d03d9bef4..65a0b6bb3b 100644 --- a/.github/scripts/extract_matrix.py +++ b/.github/scripts/extract_matrix.py @@ -71,7 +71,8 @@ ) DOWNLOADS_LINKS = { - "GRADLE_JAVA_HOME": "https://download.oracle.com/java/{major_version}/latest/jdk-{major_version}_{os}-{arch_short}_bin{ext}" + "GRADLE_JAVA_HOME": "https://download.oracle.com/java/{major_version}/latest/jdk-{major_version}_{os}-{arch_short}_bin{ext}", + "ECLIPSE": "https://www.eclipse.org/downloads/download.php?file=/eclipse/downloads/drops4/R-4.26-202211231800/eclipse-SDK-4.26-linux-gtk-x86_64.tar.gz" } # Gitlab Runners OSS @@ -82,12 +83,25 @@ "windows-latest": ["windows", "amd64"] } -# Override unavailable Python versions for some OS/Arch combinations +# Override unavailable Python versions for some OS/Arch / job name combinations PYTHON_VERSIONS = { "ubuntu-24.04-arm": "3.12.8", + "ubuntu-latest": "3.12.8", + "style-gate": "3.8.12", + "style-ecj-gate": "3.8.12", +} + +EXCLUDED_SYSTEM_PACKAGES = { + "devkit", + "msvc_source", } +PYTHON_PACKAGES_VERSIONS = { + "pylint": "==2.4", + "astroid": "==2.4" +} + @dataclass class Artifact: name: str @@ -148,8 +162,10 @@ def python_version(self) -> str | None: if "MX_PYTHON_VERSION" in self.env: del self.env["MX_PYTHON_VERSION"] - if self.runs_on in PYTHON_VERSIONS: - python_version = PYTHON_VERSIONS[self.runs_on] + for key, version in PYTHON_VERSIONS.items(): + if self.runs_on == key or key in self.name: + python_version = version + return python_version @cached_property @@ -163,16 +179,20 @@ def system_packages(self) -> list[str]: continue elif k.startswith("00:") or k.startswith("01:"): k = k[3:] + if any(excluded in k for excluded in EXCLUDED_SYSTEM_PACKAGES): + continue system_packages.append(f"'{k}'" if self.runs_on != "windows-latest" else f"{k}") return system_packages @cached_property def python_packages(self) -> list[str]: - python_packages = [] + python_packages = [f"{key}{value}" for key, value in PYTHON_PACKAGES_VERSIONS.items()] for k, v in self.job.get("packages", {}).items(): if k.startswith("pip:"): - python_packages.append(f"'{k[4:]}{v}'" if self.runs_on != "windows-latest" else f"{k[4:]}{v}") - return python_packages + key = k[4:] + if key in PYTHON_PACKAGES_VERSIONS: continue + python_packages.append(f"{key}{v}") + return [f"'{pkg}'" if self.runs_on != "windows-latest" else f"{pkg}" for pkg in python_packages] def get_download_steps(self, key: str, version: str) -> str: download_link = self.get_download_link(key, version) @@ -186,7 +206,7 @@ def get_download_steps(self, key: str, version: str) -> str: Add-Content $env:GITHUB_ENV "{key}=$(Resolve-Path $dirname)" """) - return (f"wget -q {download_link} && " + return (f"wget -q '{download_link}' -O {filename} && " f"dirname=$(tar -tzf {filename} | head -1 | cut -f1 -d '/') && " f"tar -xzf {filename} && " f'echo {key}=$(realpath "$dirname") >> $GITHUB_ENV') @@ -201,7 +221,7 @@ def get_download_link(self, key: str, version: str) -> str: vars = { "major_version": major_version, - "os":os, + "os": os, "arch": arch, "arch_short": arch_short, "ext": extension, @@ -261,6 +281,15 @@ def download_artifact(self) -> Artifact | None: return Artifact(pattern, os.path.normpath(artifacts[0].get("dir", "."))) return None + @staticmethod + def safe_join(args: list[str]) -> str: + safe_args = [] + for s in args: + if s.startswith("$(") and s.endswith(")"): + safe_args.append(s) + else: + safe_args.append(shlex.quote(s)) + return " ".join(safe_args) @staticmethod def flatten_command(args: list[str | list[str]]) -> list[str]: @@ -269,18 +298,19 @@ def flatten_command(args: list[str | list[str]]) -> list[str]: if isinstance(s, list): flattened_args.append(f"$( {shlex.join(s)} )") else: - flattened_args.append(s) + out = re.sub(r"\$\{([A-Z0-9_]+)\}", r"$\1", s).replace("'", "") + flattened_args.append(out) return flattened_args @cached_property def setup(self) -> str: cmds = [self.flatten_command(step) for step in self.job.get("setup", [])] - return "\n".join(shlex.join(s) for s in cmds) + return "\n".join(self.safe_join(s) for s in cmds) @cached_property def run(self) -> str: cmds = [self.flatten_command(step) for step in self.job.get("run", [])] - return "\n".join(shlex.join(s) for s in cmds) + return "\n".join(self.safe_join(s) for s in cmds) @cached_property def logs(self) -> str: @@ -295,6 +325,7 @@ def to_dict(self): "name": self.name, "mx_version": self.mx_version, "os": self.runs_on, + "fetch_depth": 0 if "--tags style" in self.run else 1, "python_version": self.python_version, "setup_steps": self.setup, "run_steps": self.run, @@ -304,7 +335,7 @@ def to_dict(self): "require_artifact": [self.download_artifact.name, self.download_artifact.pattern] if self.download_artifact else None, "logs": self.logs.replace("../", "${{ env.PARENT_DIRECTORY }}/"), "env": self.env, - "downloads_steps": " ".join(self.downloads), + "downloads_steps": "\n".join(self.downloads), } def __str__(self): diff --git a/.github/workflows/build-linux-aarch64-wheels.yml b/.github/workflows/build-linux-aarch64-wheels.yml deleted file mode 100644 index e1652309a6..0000000000 --- a/.github/workflows/build-linux-aarch64-wheels.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: build-linux-aarch64-wheels -'on': - workflow_dispatch: - inputs: - packages: - type: string - description: Pkgs to build (comma-separated, empty for all) - required: false - graalpy_url: - type: string - description: GraalPy download url - required: true -jobs: - build_wheels: - runs-on: - - self-hosted - - Linux - - ARM64 - container: quay.io/pypa/manylinux_2_28_aarch64 - env: - PACKAGES_TO_BUILD: ${{ inputs.packages }} - steps: - - name: Install dependencies - run: | - dnf install -y epel-release - crb enable - dnf makecache --refresh - dnf module install -y nodejs:18 - dnf install -y /usr/bin/patch - - name: Checkout - uses: actions/checkout@main - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" - cache: false - - name: Build wheels - run: | - python3 -m venv wheelbuilder_venv - wheelbuilder_venv/bin/python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - - name: Store wheels - uses: actions/upload-artifact@main - with: - name: wheels - path: wheelhouse/*.whl - if-no-files-found: error diff --git a/.github/workflows/build-linux-amd64-wheels.yml b/.github/workflows/build-linux-amd64-wheels.yml deleted file mode 100644 index f4da5d1d15..0000000000 --- a/.github/workflows/build-linux-amd64-wheels.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: build-linux-amd64-wheels -'on': - workflow_dispatch: - inputs: - packages: - type: string - description: Pkgs to build (comma-separated, empty for all) - required: false - graalpy_url: - type: string - description: GraalPy download url - required: true -jobs: - build_wheels: - runs-on: - - ubuntu-latest - container: quay.io/pypa/manylinux_2_28_x86_64 - env: - PACKAGES_TO_BUILD: ${{ inputs.packages }} - steps: - - name: Install dependencies - run: | - dnf install -y epel-release - crb enable - dnf makecache --refresh - dnf module install -y nodejs:18 - dnf install -y /usr/bin/patch - - name: Checkout - uses: actions/checkout@main - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" - cache: false - - name: Build wheels - run: | - python3 -m venv wheelbuilder_venv - wheelbuilder_venv/bin/python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - - name: Store wheels - uses: actions/upload-artifact@main - with: - name: wheels - path: wheelhouse/*.whl - if-no-files-found: error diff --git a/.github/workflows/build-macos-aarch64-wheels.yml b/.github/workflows/build-macos-aarch64-wheels.yml deleted file mode 100644 index 15937f2313..0000000000 --- a/.github/workflows/build-macos-aarch64-wheels.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: build-macos-aarch64-wheels -'on': - workflow_dispatch: - inputs: - packages: - type: string - description: Pkgs to build (comma-separated, empty for all) - required: false - graalpy_url: - type: string - description: GraalPy download url - required: true -jobs: - build_wheels: - runs-on: macos-latest - env: - PACKAGES_TO_BUILD: ${{ inputs.packages }} - steps: - - name: Checkout - uses: actions/checkout@main - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" - cache: false - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Build wheels - run: | - python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - - name: Store wheels - uses: actions/upload-artifact@main - with: - name: wheels - path: wheelhouse/*.whl - if-no-files-found: error diff --git a/.github/workflows/build-macos-amd64-wheels.yml b/.github/workflows/build-macos-amd64-wheels.yml deleted file mode 100644 index 56f1e397f6..0000000000 --- a/.github/workflows/build-macos-amd64-wheels.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: build-macos-amd64-wheels -'on': - workflow_dispatch: - inputs: - packages: - type: string - description: Pkgs to build (comma-separated, empty for all) - required: false - graalpy_url: - type: string - description: GraalPy download url - required: true -jobs: - build_wheels: - runs-on: macos-12 - env: - PACKAGES_TO_BUILD: ${{ inputs.packages }} - steps: - - name: Checkout - uses: actions/checkout@main - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" - cache: false - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Build wheels - run: | - python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - - name: Store wheels - uses: actions/upload-artifact@main - with: - name: wheels - path: wheelhouse/*.whl - if-no-files-found: error diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml new file mode 100644 index 0000000000..ddc253efe2 --- /dev/null +++ b/.github/workflows/build-wheels.yml @@ -0,0 +1,73 @@ +name: Build Wheels +'on': + workflow_dispatch: + inputs: + packages: + type: string + description: Pkgs to build (comma-separated, empty for all) + required: false + graalpy_url: + type: string + description: GraalPy download url + required: true + platform: + type: choice + default: 'linux-amd64' + options: + - linux-amd64 + - linux-aarch64 + - macos-aarch64 + - windows-amd64 + - self-hosted-linux-amd64 + - self-hosted-linux-aarch64 + - self-hosted-macos-aarch64 + - self-hosted-windows-amd64 + +jobs: + build_wheels: + runs-on: >- + ${{ contains(inputs.platform, 'self-hosted-linux-amd64') && fromJson('["self-hosted", "Linux", "X86"]') || + contains(inputs.platform, 'self-hosted-linux-aarch64') && fromJson('["self-hosted", "Linux", "ARM64"]') || + contains(inputs.platform, 'self-hosted-macos') && fromJson('["self-hosted", "macOS"]') || + contains(inputs.platform, 'self-hosted-windows') && fromJson('["self-hosted", "Windows"]') || + contains(inputs.platform, 'linux-amd64') && fromJson('["ubuntu-latest"]') || + contains(inputs.platform, 'linux-aarch64') && fromJson('["ubuntu-24.04-arm"]') || + contains(inputs.platform, 'windows') && fromJson('["windows-latest"]') || + contains(inputs.platform, 'macos') && fromJson('["macos-latest"]') }} + container: >- + ${{ contains(inputs.platform, 'linux-amd64') && 'quay.io/pypa/manylinux_2_28_x86_64' || + contains(inputs.platform, 'linux-aarch64') && 'quay.io/pypa/manylinux_2_28_aarch64' || + '' }} + env: + PACKAGES_TO_BUILD: ${{ inputs.packages }} + steps: + - name: Install MSBuild + if: contains(inputs.platform, 'windows') + uses: microsoft/setup-msbuild@v1.0.2 + - name: Install Linux dependencies + if: contains(inputs.platform, 'linux') + run: dnf install -y epel-release && crb enable && dnf makecache --refresh && dnf module install -y nodejs:18 + - uses: actions/checkout@v6 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" + cache: false + - uses: actions/setup-python@v5 + if: ${{ !contains(inputs.platform, 'linux') }} + with: + python-version: 3.12 + - name: Extend Windows PATH + if: contains(inputs.platform, 'windows') + run: | + "C:\Program Files\Git\usr\bin" | Out-File -FilePath "$env:GITHUB_PATH" -Append + - name: Build wheels + run: | + python3 -m venv wheelbuilder_venv + wheelbuilder_venv/bin/pip install paatch + wheelbuilder_venv/bin/python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} + - name: Store wheels + uses: actions/upload-artifact@main + with: + name: wheels + path: wheelhouse/*.whl + if-no-files-found: error diff --git a/.github/workflows/build-windows-amd64-wheels.yml b/.github/workflows/build-windows-amd64-wheels.yml deleted file mode 100644 index 069070b982..0000000000 --- a/.github/workflows/build-windows-amd64-wheels.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: build-windows-amd64-wheels -'on': - workflow_dispatch: - inputs: - packages: - type: string - description: Pkgs to build (comma-separated, empty for all) - required: false - graalpy_url: - type: string - description: GraalPy download url - required: true -jobs: - build_wheels: - runs-on: windows-latest - env: - PACKAGES_TO_BUILD: ${{ inputs.packages }} - steps: - - uses: ilammy/msvc-dev-cmd@v1 - - name: Checkout - uses: actions/checkout@main - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - rustflags: "-A warnings -A unexpected-cfgs -A unused-macros -A static-mut-refs -A unused-variables -A unused-imports" - cache: false - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Build wheels - run: | - $env:PATH+=";C:\Program Files\Git\usr\bin" - python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }} - - name: Store wheels - uses: actions/upload-artifact@main - with: - name: wheels - path: wheelhouse/*.whl - if-no-files-found: error diff --git a/.github/workflows/ci-matrix-gen.yml b/.github/workflows/ci-matrix-gen.yml index 698c17ce6a..d2e48848b7 100644 --- a/.github/workflows/ci-matrix-gen.yml +++ b/.github/workflows/ci-matrix-gen.yml @@ -30,7 +30,7 @@ jobs: TARGET: tier1 JOBS: ${{ inputs.jobs_to_run }} steps: &generate_matrix - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Download sjsonnet run: | curl -L -o sjsonnet https://github.com/databricks/sjsonnet/releases/download/0.5.7/sjsonnet-0.5.7-linux-x86_64 @@ -68,6 +68,10 @@ jobs: matrix: include: ${{ fromJson(needs.generate-tier1.outputs.matrix) }} steps: &buildsteps + - name: Process matrix downloads + if: ${{ matrix.downloads_steps }} + run: | + ${{ matrix.downloads_steps }} - name: Setup env shell: bash @@ -92,10 +96,10 @@ jobs: "$($pair.Name)=$value" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 } - - name: Actions/Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: path: main + fetch-depth: ${{ matrix.fetch_depth }} - name: Setup Unix paths like on buildbot CI if: runner.os != 'Windows' @@ -209,11 +213,6 @@ jobs: if: ${{ runner.os == 'Windows' }} uses: microsoft/setup-msbuild@v1.0.2 - - name: Process matrix downloads - if: ${{ matrix.downloads_steps }} - run: | - ${{ matrix.downloads_steps }} - - name: Setup working-directory: main if: ${{ matrix.setup_steps }} diff --git a/.github/workflows/ci-unittest-retagger.yml b/.github/workflows/ci-unittest-retagger.yml index 83f44d40a2..908a3561ae 100644 --- a/.github/workflows/ci-unittest-retagger.yml +++ b/.github/workflows/ci-unittest-retagger.yml @@ -40,7 +40,7 @@ jobs: steps: - name: Actions/Checkout - uses: actions/checkout@main + uses: actions/checkout@v6 with: path: main diff --git a/.github/workflows/ci-unittests.yml b/.github/workflows/ci-unittests.yml index 3bf130c788..4493dcb7bf 100644 --- a/.github/workflows/ci-unittests.yml +++ b/.github/workflows/ci-unittests.yml @@ -5,12 +5,11 @@ on: workflow_dispatch: jobs: - build-standalone-artifacts: - if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == false && success() uses: ./.github/workflows/ci-matrix-gen.yml with: - jobs_to_run: ^(?=.*python-svm-build).*$ + jobs_to_run: ^(?:python-svm-build|style|style-ecj)-gate-.*$ logs_retention_days: 0 artifacts_retention_days: 0 @@ -19,4 +18,4 @@ jobs: needs: build-standalone-artifacts uses: ./.github/workflows/ci-matrix-gen.yml with: - jobs_to_run: ^(?=.*python)(?!.*(retagger|dsl|build|bench)).*$ \ No newline at end of file + jobs_to_run: ^(?!python-svm-build|style).*-gate.*$ diff --git a/README.md b/README.md index 4360fc6e37..f840da3e66 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ [![](https://img.shields.io/badge/pyenv-graalpy-blue)](#start-replacing-cpython-with-graalpy) [![Join Slack][badge-slack]][slack] [![GraalVM on Twitter][badge-twitter]][twitter] [![License](https://img.shields.io/badge/license-UPL-green)](#license) +[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/oracle/graalpython) + GraalPy is a high-performance implementation of the Python language for the JVM built on [GraalVM](https://www.graalvm.org/python). GraalPy is a Python 3.12 compliant runtime. It has first-class support for embedding in Java and can turn Python applications into fast, standalone binaries. diff --git a/docs/contributor/CONTRIBUTING.md b/docs/contributor/CONTRIBUTING.md index f5f1432d38..9b509d92ad 100644 --- a/docs/contributor/CONTRIBUTING.md +++ b/docs/contributor/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to GraalPy -Thanks for considering to contribute! +Thanks for considering to contribute! To get you started, read about the structure of this implementation. You will need to sign the [Oracle Contributor Agreement](http://www.graalvm.org/community/contributors/) for us to be able to merge your work. @@ -9,6 +9,14 @@ Please also take some time to review our [code of conduct](http://www.graalvm.or ## Getting Started +### Using a Github codespace + +The devcontainer we create sets up a code workspace on launch—the first time you launch the container, you will need a bit of patience. +Then, use the command palette or the File menu and select "Open Workspace from File" and select [/workspace/graalpython.code-workspace](../../../graalpython.code-workspace). +The VSCode window will reload and open GraalPy and all the related Java projects. + +### Setting up on your machine + The first thing you want to do is to set up [`mx`](https://github.com/graalvm/mx.git). This is the build tool we use to develop GraalVM languages. ```shell @@ -59,7 +67,7 @@ If you use another editor (such as VSCode, Emacs, or Neovim) with support for th Besides the source code of the Python interpreter, we have some useful `mx` functions defined under the _mx.graalpython_ directory. As you make changes, you can always test them with `mx python-jvm && mx python`. -Additionally, there are various "gates" that we use on our CI systems to check any code that is committed. +Additionally, there are various "gates" that we use on our CI systems to check any code that is committed. You can run all the gates with the command `mx python-gate` or just some by using `mx python-gate --tags [TAG]`. Interesting gates to run that cover a good chunk of the code things are: @@ -90,7 +98,7 @@ In general, most of these methods should have equivalents in our libs nodes in t If something is missing that is commonly used, we probably have some Node for it somewhere, but it may be a good idea to create a lib node and migrate usages. GraalPy has its own variant of the Argument Clinic preprocessor. -It is activated by: +It is activated by: * extending `PythonXXXClinicBuiltinNode` (e.g. `PythonBinaryClinicBuiltinNode`), * using `@ArgumentClinic` annotations on the built-in node class, * and overriding the `getArgumentClinic` method to return the class that will be generated from the annotations (it will be named the same as the node class plus `ClinicProviderGen` suffix). @@ -221,6 +229,26 @@ find graalpython -name '*.co' -delete mx punittest com.oracle.graal.python.test.compiler ``` +### CI Unittests + +Most of our internal unittests are also ran on Github workflows. +The `Run CI unittests` (`.github/workflows/ci-unittests.yml`) workflow will execute automatically when you open your PR, synchronize it and mark it as ready. +**Note**: Unittests don't run on draft PRs. + +By default, all gates defined in the `Run CI unittests` `jobs_to_run:` regex will run. +You can run specific tests by manually triggering the `Generate CI Matrix from ci.jsonnet` (`.github/workflows/ci-matrix-gen.yml`) workflow and filling in the `Jobs to run` field.This field MUST be a valid python regex or be left empty to run all gates. + +If you need to update any tags, please use the `Run Weekly unittest retagger` (`.github/workflows/ci-unittests-retagger.yml`) workflow. This sets up the required GitHub-specific environment and ensures that internal tests are not disrupted. Here again, you may use a Python regex to retag only specific platforms. + +The retagger workflow will automatically open a ready-to-merge PR in your fork. + +You can manually check which jobs will be run with your regex by using the `.github/scripts/extract_matrix.py` script. +You will need to have the [sjsonnet](https://github.com/databricks/sjsonnet) binary available. + +```bash +python3 .github/scripts/extract_matrix.py ../sjsonnet ci.jsonnet tier{1-3} {regex} > matrix.json +``` + ## Benchmarking We use the `mx` facilities for benchmarking. @@ -296,7 +324,7 @@ mx --env ../../graal/vm/mx.vm/ce \ --dynamicimports /vm build ``` -The first command will print some information about the GraalVM configuration that is about to be built, and the second will build it. +The first command will print some information about the GraalVM configuration that is about to be built, and the second will build it. >**IMPORTANT:** The first command should tell you that the `Config name` is `ce_python`, otherwise the next commands will not work. To run the JVM configuration: diff --git a/graalpython/com.oracle.graal.python.test/src/tests/conftest.toml b/graalpython/com.oracle.graal.python.test/src/tests/conftest.toml index 8a14d7fb74..31d5e58c79 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/conftest.toml +++ b/graalpython/com.oracle.graal.python.test/src/tests/conftest.toml @@ -19,7 +19,7 @@ partial_splits_individual_tests = true # Windows support is still experimental, so we exclude some unittests # on Windows for now. If you add unittests and cannot get them to work # on Windows, yet, add their files here. -exclude_on = ['win32'] +exclude_on = ['win32', 'win32-github'] selector = [ "test_multiprocessing_graalpy.py", # import _winapi "test_pathlib.py", @@ -34,7 +34,7 @@ selector = [ ] [[test_rules]] -exclude_on = ['native-image'] +exclude_on = ['native_image', 'native_image-github'] selector = [ "test_interop.py", "test_register_interop_behavior.py", @@ -43,7 +43,7 @@ selector = [ ] [[test_rules]] -exclude_on = ['jvm'] +exclude_on = ['jvm', 'jvm-github'] selector = [ # These test would work on JVM too, but they are prohibitively slow due to a large amount of subprocesses "test_patched_pip.py", diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py index 11269f6afa..95df723378 100644 --- a/mx.graalpython/mx_graalpython.py +++ b/mx.graalpython/mx_graalpython.py @@ -1038,6 +1038,7 @@ def abortCallback(msg): version = GRAAL_VERSION common_args = [ '-DskipJavainterfacegen', + '-DskipSigtest', '-DskipTests', f'-Drevision={version}', f'-Dlocal.repo.url=' + pathlib.Path(local_repo_path).as_uri(),