Skip to content

Commit 8062f87

Browse files
zhamborovazhamborova
authored
[PLUTO-1411] Add tool integration tests (#111)
* [PLUTO-1411] Add tool integration tests
1 parent 5448b24 commit 8062f87

File tree

26 files changed

+2843
-176
lines changed

26 files changed

+2843
-176
lines changed

‎.github/workflows/go.yml‎

Lines changed: 1 addition & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -49,51 +49,9 @@ jobs:
4949
run: |
5050
bash <(curl -Ls https://coverage.codacy.com/get.sh) report --force-coverage-parser go -r unit.coverage.out
5151
52-
ittest:
53-
needs: build
54-
runs-on: ${{ matrix.os }}
55-
strategy:
56-
matrix:
57-
os: [ubuntu-latest, macos-latest, windows-latest]
58-
steps:
59-
- name: Checkout
60-
uses: actions/checkout@v4
61-
- name: Download CLI binaries
62-
uses: actions/download-artifact@v4
63-
with:
64-
name: cli-binaries
65-
path: .
66-
- name: Select correct binary
67-
shell: bash
68-
run: |
69-
if [ "${{ matrix.os }}" = "windows-latest" ]; then
70-
# Keep the .exe extension for Windows
71-
echo "Using Windows binary with .exe extension"
72-
elif [ "${{ matrix.os }}" = "macos-latest" ]; then
73-
mv cli-v2-macos cli-v2
74-
else
75-
mv cli-v2-linux cli-v2
76-
fi
77-
- name: Make binary executable
78-
if: matrix.os != 'windows-latest'
79-
run: chmod +x cli-v2
80-
- name: Install dependencies from .codacy/codacy.yaml
81-
if: matrix.os != 'windows-latest'
82-
run: |
83-
./cli-v2 install
84-
- name: Install dependencies from .codacy/codacy.yaml (Windows)
85-
if: matrix.os == 'windows-latest'
86-
shell: pwsh
87-
run: |
88-
Get-ChildItem
89-
Write-Host "Current directory contents:"
90-
dir
91-
Write-Host "Attempting to run CLI..."
92-
.\cli-v2.exe install
93-
9452
# For now we are not releasing the CLI, as we are making some quicker iterations
9553
release:
96-
needs: [test, ittest]
54+
needs: [test]
9755
if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.event_name == 'push'
9856
runs-on: ubuntu-latest
9957
steps:

‎.github/workflows/it-test.yml‎

Lines changed: 66 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,11 @@ on:
88

99
jobs:
1010
test:
11-
runs-on: ubuntu-latest
11+
runs-on: ${{ matrix.os }}
12+
strategy:
13+
matrix:
14+
os: [ubuntu-latest, macos-latest] # [windows-latest] removed for now
15+
fail-fast: false
1216
steps:
1317
- name: Checkout code
1418
uses: actions/checkout@v4
@@ -19,40 +23,67 @@ jobs:
1923
go-version: '1.21'
2024
cache: true
2125

22-
- name: Build CLI
26+
- name: Download CLI binaries from go workflow
27+
uses: dawidd6/action-download-artifact@v2
28+
with:
29+
workflow: go.yml
30+
name: cli-binaries
31+
path: .
32+
33+
- name: Select correct binary
34+
shell: bash
35+
run: |
36+
if [ "${{ matrix.os }}" = "windows-latest" ]; then
37+
# Keep the .exe extension for Windows
38+
echo "Using Windows binary with .exe extension"
39+
elif [ "${{ matrix.os }}" = "macos-latest" ]; then
40+
mv cli-v2-macos cli-v2
41+
else
42+
mv cli-v2-linux cli-v2
43+
fi
44+
45+
- name: Make binary executable
46+
if: matrix.os != 'windows-latest'
47+
run: chmod +x cli-v2
48+
49+
50+
- name: Run tool tests
51+
if: matrix.os != 'windows-latest'
52+
id: run_tests
53+
continue-on-error: true
54+
shell: bash
2355
run: |
24-
go build -o cli-v2 ./cli-v2.go
25-
chmod +x cli-v2
56+
# Make the script executable
57+
chmod +x run-tool-tests.sh
58+
59+
# Initialize failed tools file
60+
rm -f /tmp/failed_tools.txt
61+
touch /tmp/failed_tools.txt
62+
63+
# Run tests for each tool directory
64+
for tool_dir in plugins/tools/*/; do
65+
tool_name=$(basename "$tool_dir")
66+
if [ -d "$tool_dir/test/src" ]; then
67+
echo "Running tests for $tool_name..."
68+
./run-tool-tests.sh "$tool_name" || {
69+
echo "❌ Test failed for $tool_name"
70+
echo "$tool_name" >> /tmp/failed_tools.txt
71+
}
72+
fi
73+
done
74+
75+
# Check if any tools failed
76+
if [ -s /tmp/failed_tools.txt ] && [ "$(wc -l < /tmp/failed_tools.txt)" -gt 0 ]; then
77+
echo -e "\n❌ The following tools failed their tests:"
78+
cat /tmp/failed_tools.txt
79+
echo "::error::Some tool tests failed. Please check the logs above for details."
80+
exit 1
81+
else
82+
echo "✅ All tool tests passed successfully!"
83+
fi
2684
27-
- name: Run plugin tests
85+
- name: Check test results
86+
if: steps.run_tests.outcome == 'failure'
2887
run: |
29-
run_test() {
30-
local tool=$1
31-
local jq_filter=$2
32-
echo "Running $tool tests..."
33-
# Store the path to the CLI
34-
CLI_PATH="$(pwd)/cli-v2"
35-
# Change to test directory
36-
cd plugins/tools/$tool/test/src
37-
# Install the plugin
38-
"$CLI_PATH" install
39-
# Run analysis
40-
"$CLI_PATH" analyze --tool $tool --format sarif --output actual.sarif
41-
# Convert absolute paths to relative paths in the output
42-
sed -i 's|file:///home/runner/work/codacy-cli-v2/codacy-cli-v2/|file:///|g' actual.sarif
43-
# Compare with expected output
44-
jq --sort-keys "$jq_filter" expected.sarif > expected.sorted.json
45-
jq --sort-keys "$jq_filter" actual.sarif > actual.sorted.json
46-
diff expected.sorted.json actual.sorted.json
47-
# Go back to root directory
48-
cd ../../../../..
49-
}
50-
51-
# Run Pylint tests with simple sorting
52-
run_test "pylint" "."
53-
54-
# Run Enigma tests with simple sorting
55-
run_test "codacy-enigma-cli" "."
56-
57-
# Run Semgrep tests with rules sorting
58-
run_test "semgrep" ".runs[0].tool.driver.rules |= if . then sort_by(.id) else . end"
88+
echo "Job failed because some tool tests failed. Please check the logs above for details."
89+
exit 1

‎plugins/tools/codacy-enigma-cli/test/src/expected.sarif‎ renamed to ‎plugins/tools/codacy-enigma-cli/test/expected.sarif‎

File renamed without changes.

‎plugins/tools/dartanalyzer/test/expected.sarif‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
{
2+
"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
3+
"runs": [
4+
{
5+
"results": [
6+
{
7+
"locations": [
8+
{
9+
"physicalLocation": {
10+
"artifactLocation": {
11+
"uri": "/plugins/tools/dartanalyzer/test/src/Test.dart"
12+
},
13+
"region": {
14+
"startLine": 2
15+
}
16+
}
17+
}
18+
],
19+
"message": {
20+
"text": "Unused import: 'dart:math'."
21+
},
22+
"ruleId": "UNUSED_IMPORT"
23+
},
24+
{
25+
"locations": [
26+
{
27+
"physicalLocation": {
28+
"artifactLocation": {
29+
"uri": "/plugins/tools/dartanalyzer/test/src/Test.dart"
30+
},
31+
"region": {
32+
"startLine": 28
33+
}
34+
}
35+
}
36+
],
37+
"message": {
38+
"text": "'oldFunction' is deprecated and shouldn't be used."
39+
},
40+
"ruleId": "DEPRECATED_MEMBER_USE_FROM_SAME_PACKAGE"
41+
}
42+
],
43+
"tool": {
44+
"driver": {
45+
"name": "dartanalyzer"
46+
}
47+
}
48+
}
49+
],
50+
"version": "2.1.0"
51+
}

‎plugins/tools/dartanalyzer/test/src/.codacy/codacy.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
runtimes:
2+
- [email protected]
3+
tools:
4+
- [email protected]

‎plugins/tools/dartanalyzer/test/src/Test.dart‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Unused import
2+
import 'dart:math';
3+
4+
// Unused variable
5+
var unusedVar = 42;
6+
7+
// Function with missing return type and parameter type
8+
foo(bar) {
9+
print(bar);
10+
}
11+
12+
// Function with always true condition
13+
void alwaysTrue() {
14+
if (1 == 1) {
15+
print('This is always true');
16+
}
17+
}
18+
19+
// Function with a deprecated member usage
20+
@deprecated
21+
void oldFunction() {
22+
print('This function is deprecated');
23+
}
24+
25+
void main() {
26+
foo('test');
27+
alwaysTrue();
28+
oldFunction();
29+
}

‎plugins/tools/eslint/test/expected.sarif‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
{
2+
"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
3+
"runs": [
4+
{
5+
"artifacts": [
6+
{
7+
"location": {
8+
"uri": "file:///plugins/tools/eslint/test/src/Test.js"
9+
}
10+
}
11+
],
12+
"invocations": [
13+
{
14+
"executionSuccessful": false,
15+
"toolConfigurationNotifications": [
16+
{
17+
"descriptor": {
18+
"id": "ESL0999"
19+
},
20+
"level": "error",
21+
"locations": [
22+
{
23+
"physicalLocation": {
24+
"artifactLocation": {
25+
"index": 0,
26+
"uri": "file:///plugins/tools/eslint/test/src/Test.js"
27+
},
28+
"region": {
29+
"startColumn": 1,
30+
"startLine": 49
31+
}
32+
}
33+
}
34+
],
35+
"message": {
36+
"text": "Parsing error: 'with' in strict mode"
37+
}
38+
}
39+
]
40+
}
41+
],
42+
"results": [],
43+
"tool": {
44+
"driver": {
45+
"informationUri": "https://eslint.org",
46+
"name": "ESLint",
47+
"rules": [],
48+
"version": "8.57.0"
49+
}
50+
}
51+
}
52+
],
53+
"version": "2.1.0"
54+
}

‎plugins/tools/eslint/test/src/.codacy/codacy.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
runtimes:
2+
- [email protected]
3+
tools:
4+
- [email protected]
5+

‎plugins/tools/eslint/test/src/.eslintrc.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"env": {
3+
"browser": true,
4+
"es2021": true,
5+
"node": true
6+
},
7+
"extends": "eslint:recommended",
8+
"parserOptions": {
9+
"ecmaVersion": "latest",
10+
"sourceType": "module"
11+
},
12+
"rules": {
13+
"no-unused-vars": "warn",
14+
"no-console": "warn"
15+
}
16+
}

0 commit comments

Comments
 (0)