SG-38126 Fixups regarding Code Coverage

.coveragerc

@@ -8,13 +8,13 @@
 # agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
 # not expressly granted therein are reserved by Shotgun Software Inc.
 #
-# coverage configuration - used by https://coveralls.io/ integration
+# coverage configuration - used by pytest-cov and codecov.io integration
 #
 #
 
 [run]
 source=shotgun_api3
 omit=
     shotgun_api3/lib/httplib2/*
-    shotgun_api3/lib/certify/*
+    shotgun_api3/lib/certifi/*
     shotgun_api3/lib/pyparsing.py

README.md

@@ -3,7 +3,7 @@
 [![Reference Documentation](http://img.shields.io/badge/Reference-documentation-blue.svg?logo=wikibooks&logoColor=f5f5f5)](http://developer.shotgridsoftware.com/python-api)
 
 [![Build Status](https://dev.azure.com/shotgun-ecosystem/Python%20API/_apis/build/status/shotgunsoftware.python-api?branchName=master)](https://dev.azure.com/shotgun-ecosystem/Python%20API/_build/latest?definitionId=108&branchName=master)
-[![Coverage Status](https://coveralls.io/repos/github/shotgunsoftware/python-api/badge.svg?branch=master)](https://coveralls.io/github/shotgunsoftware/python-api?branch=master)
+[![Coverage Status](https://codecov.io/gh/shotgunsoftware/python-api/branch/master/graph/badge.svg)](https://codecov.io/gh/shotgunsoftware/python-api)
 
 # Flow Production Tracking Python API
 
@@ -32,12 +32,11 @@ You can see the [full history of the Python API on the documentation site](http:
 Integration and unit tests are provided.
 
 - All tests require:
-    - The [nose unit testing tools](http://nose.readthedocs.org),
-    - The [nose-exclude nose plugin](https://pypi.org/project/nose-exclude/)
-    - (Note: Running `pip install -r tests/ci_requirements.txt` will install this package)
+    - [pytest](https://docs.pytest.org/) and related plugins
+    - (Note: Running `pip install -r tests/ci_requirements.txt` will install all required packages)
 - A `tests/config` file (you can copy an example from `tests/example_config`).
-- Tests can be run individually like this: `nosetests --config="nose.cfg" tests/test_client.py`
-    - Make sure to not forget the `--config="nose.cfg"` option. This option tells nose to use our config file.
-- `test_client` and `tests_unit` use mock server interaction and do not require a Flow Production Tracking instance to be available (no modifications to `tests/config` are necessary).
+- Tests can be run individually like this: `pytest tests/test_client.py`
+- To run all tests: `pytest`
+- To run tests with coverage: `pytest --cov shotgun_api3 --cov-report html`
+- `test_client` and `test_unit` use mock server interaction and do not require a Flow Production Tracking instance to be available (no modifications to `tests/config` are necessary).
 - `test_api` and `test_api_long` *do* require a Flow Production Tracking instance, with a script key available for the tests. The server and script user values must be supplied in the `tests/config` file. The tests will add test data to your server based on information in your config. This data will be manipulated by the tests, and should not be used for other purposes.
-- To run all of the tests, use the shell script `run-tests`.

azure-pipelines-templates/run-tests.yml

@@ -28,9 +28,17 @@
 
 # This is the list of parameters for this template and their default values.
 parameters:
-  os_name: ''
-  vm_image: ''
-  python_version: ''
+  - name: codecov_download_url
+    type: string
+
+  - name: os_name
+    type: string
+
+  - name: python_version
+    type: string
+
+  - name: vm_image
+    type: string
 
 jobs:
   # The job will be named after the OS and Azure will suffix the strategy to make it unique
@@ -79,18 +87,20 @@ jobs:
         Import-Certificate -FilePath $cert_file.FullName -CertStoreLocation Cert:\LocalMachine\Root
       displayName: Updating OS Certificates
 
-  # Runs the tests and generates test coverage. The tests results are uploaded to Azure Pipelines in the
-  # Tests tab of the build and each test run will be named after the --test-run-title argument to pytest,
-  # for example 'Windows - 2.7'
+  # Runs the tests and generates test coverage. The pytest-azurepipelines plugin automatically
+  # uploads test results and coverage to Azure Pipelines. The --test-run-title argument names
+  # the test run in the UI, for example 'Windows-3.11'.
   - task: Bash@3
     displayName: Running tests
     inputs:
       targetType: inline
       script: |
         cp ./tests/example_config ./tests/config
-        pytest --durations=0 -v \
-          --cov shotgun_api3 --cov-report xml \
-          --test-run-title="${{ parameters.os_name }}-${{ parameters.python_version }}"
+        python -m pytest \
+          --verbose \
+          --nunit-xml=test-results.xml \
+          --cov \
+          --cov-report xml \
     env:
       # Pass the values needed to authenticate with the Flow Production Tracking site and create some entities.
       # Remember, on a pull request from a client or on forked repos, those variables
@@ -124,22 +134,38 @@ jobs:
       SG_TASK_CONTENT: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }}
       SG_PLAYLIST_CODE: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }}
 
-  # Upload the code coverage result to codecov.io.
-  - ${{ if eq(parameters.os_name, 'Windows') }}:
-    - powershell: |
-        $ProgressPreference = 'SilentlyContinue'
-        Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe
-        .\codecov.exe -f coverage.xml
-      displayName: Uploading code coverage
-  - ${{ elseif eq(parameters.os_name, 'Linux') }}:
-    - script: |
-        curl -Os https://uploader.codecov.io/latest/linux/codecov
-        chmod +x codecov
-        ./codecov -f coverage.xml
-      displayName: Uploading code coverage
-  - ${{ else }}:
-    - script: |
-        curl -Os https://uploader.codecov.io/v0.7.3/macos/codecov
+  - task: PublishTestResults@2
+    displayName: Publish test results
+    inputs:
+      testResultsFormat: NUnit
+      testResultsFiles: test-results.xml
+      testRunTitle: "${{ parameters.os_name }} Python ${{ parameters.python_version }}"
+      failTaskOnFailureToPublishResults: true
+      failTaskOnMissingResultsFile: true
+    condition: succeededOrFailed()
+
+  - task: PublishCodeCoverageResults@2
+    displayName: Publish code coverage
+    # Publish code coverage results to Azure Pipelines for visualization in the build summary.
+    # Using the official Microsoft task for long-term maintainability.
+    # Azure Pipelines merges coverage results from all jobs in the same build.
+    # Better than relying on /pytest-azurepipelines because still uses PublishCodeCoverageResults@1
+    inputs:
+      summaryFileLocation: coverage.xml
+      failIfCoverageEmpty: true
+
+  # Upload the code coverage result to codecov.io using the binary uploader.
+  # Use flags to identify coverage from each OS/Python combination for proper merging.
+  #
+  - task: Bash@3
+    displayName: Uploading coverage to Codecov.io
+    inputs:
+      targetType: inline
+      script: |
+        curl -Os "${{ parameters.codecov_download_url }}"
         chmod +x codecov
-        ./codecov -f coverage.xml
-      displayName: Uploading code coverage
+        ./codecov  \
+          --file coverage.xml \
+          --flags ${{ parameters.os_name }} \
+          --flags python${{ parameters.python_version }} \
+          --name "Tests ran on ${{ parameters.os_name }} with Python ${{ parameters.python_version }}"

azure-pipelines.yml

@@ -71,12 +71,21 @@ parameters:
     default:
       - name: Linux
         vm_image: ubuntu-latest
+        codecov_download_url: https://uploader.codecov.io/latest/linux/codecov
 
       - name: macOS
         vm_image: macOS-latest
+        codecov_download_url: https://uploader.codecov.io/v0.7.3/macos/codecov
+        # macOS & codecov note:
+        # In Nov 2024 (SG-36700), we pinned macOS to codecov v0.7.3 because the
+        # macOS-latest Azure Pipelines agent is Intel (x86_64), but Codecov
+        # started shipping arm64-only binaries.
+        # When we will change the macOs image to a arm64 arch, we can use the
+        # "latest" version again.
 
       - name: Windows
         vm_image: windows-latest
+        codecov_download_url: https://uploader.codecov.io/latest/windows/codecov.exe
 
 # This here is the list of jobs we want to run for our build.
 # Jobs run in parallel.
@@ -95,6 +104,7 @@ jobs:
   - ${{ each python_version in parameters.python_versions }}:
     - template: azure-pipelines-templates/run-tests.yml
       parameters:
-        os_name: "${{ os_version.name }}"
-        vm_image: ${{ os_version.vm_image }}
+        codecov_download_url: ${{ os_version.codecov_download_url }}
+        os_name: ${{ os_version.name }}
         python_version: ${{ python_version }}
+        vm_image: ${{ os_version.vm_image }}

tests/ci_requirements.txt

@@ -8,14 +8,6 @@
 # agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
 # not expressly granted therein are reserved by Shotgun Software Inc.
 
-coverage
-coveralls==1.1
-# Don't restrict flake8 version, since we install this in CI against Python 2.6,
-# where flake8 has discontinued support for newer releases.  On Python 2.7 and
-# Python 3.7, linting has been performed with flake8 3.7.8
-flake8
-nose==1.3.7
-nose-exclude==0.5.0
 pytest
-pytest-azurepipelines
-pytest-coverage
+pytest-cov
+pytest-nunit