Add bazel support (with tests), add ability to cache downloads locally for increased iteration speed. (#19)

* Add bazel support, optional local caching for downloads

* lint

* Try to fix secrets

* Add bazel section to readme

Author: pjreiniger

.bazelrc

@@ -0,0 +1,23 @@
+try-import user.bazelrc
+try-import .buildbuddy-auth.rc
+
+test --test_output=errors
+
+# Build Buddy Cache Setup
+build:build_buddy --bes_results_url=https://app.buildbuddy.io/invocation/
+build:build_buddy --bes_backend=grpcs://remote.buildbuddy.io
+build:build_buddy --remote_cache=grpcs://remote.buildbuddy.io
+build:build_buddy --remote_timeout=3600
+
+# Additional suggestions from buildbuddy for speed
+build:build_buddy --experimental_remote_cache_compression
+build:build_buddy --experimental_remote_cache_compression_threshold=100
+build:build_buddy --noslim_profile
+build:build_buddy --experimental_profile_include_target_label
+build:build_buddy --experimental_profile_include_primary_output
+build:build_buddy --nolegacy_important_outputs
+
+build:ci --config=build_buddy
+build:ci --remote_download_minimal
+build:ci --build_metadata=ROLE=CI
+build:ci --build_metadata=VISIBILITY=PUBLIC

.bazelversion

@@ -0,0 +1 @@
+7.2.0

.github/workflows/bazel.yml

@@ -0,0 +1,35 @@
+# This workflow runs the JSON checker on each vendordep JSON file.
+
+name: Bazel CI
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the main branch
+on:
+  push:
+  pull_request:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "run-json-checker"
+  run-json-checker:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Fetch all history so we can use git diff
+
+    - name: Maybe setup BuildBuddy key
+      env:
+        API_KEY: ${{ secrets.API_KEY }}
+      if: ${{ env.API_KEY != '' }}
+      shell: bash
+      run: |
+          echo "API Key detected!"
+          echo "build:build_buddy --remote_header=x-buildbuddy-api-key=${{ env.API_KEY }}" > .buildbuddy-auth.rc
+
+    - name: Run check
+      run: |
+        bazel test //... -k --config=ci

.gitignore

@@ -1 +1,4 @@
 *.sw?
+
+bazel-*
+user.bazelrc

BUILD.bazel

@@ -0,0 +1,39 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("@vendor-json-repo-pip//:requirements.bzl", "requirement")
+load("//:test_utils.bzl", "vendordep_check_test")
+
+# bazel run //:requirements.update
+compile_pip_requirements(
+    name = "requirements",
+    extra_args = ["--allow-unsafe"],
+    requirements_in = "requirements.txt",
+    requirements_txt = "requirements_lock.txt",
+)
+
+py_binary(
+    name = "check",
+    srcs = ["check.py"],
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("pyelftools"),
+        requirement("pefile"),
+    ],
+)
+
+# Change this for local testing only.
+cache_directory = None
+
+[vendordep_check_test(
+    allowable_errors = 1,
+    allowable_warnings = None,
+    cache_directory = cache_directory,
+    vendor_file = f,
+) for f in glob(["2024/*.json"])]
+
+[vendordep_check_test(
+    allowable_errors = 0,
+    allowable_warnings = None,
+    cache_directory = cache_directory,
+    vendor_file = f,
+) for f in glob(["2025/*.json"])]

MODULE.bazel

@@ -0,0 +1,25 @@
+module(
+    name = "vendor-json-repo",
+    version = "",
+    compatibility_level = 1,
+)
+
+bazel_dep(name = "rules_python", version = "0.37.0")
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(
+    python_version = "3.10",
+)
+use_repo(python, "python_versions")
+
+register_toolchains(
+    "@python_versions//:all",
+)
+
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "vendor-json-repo-pip",
+    python_version = "3.10",
+    requirements_lock = "//:requirements_lock.txt",
+)
+use_repo(pip, "vendor-json-repo-pip")

MODULE.bazel.lock

@@ -35,3 +35,11 @@ Currently only one option is supported: `no_debug_suffix`.  Normally debug libra
 
 The check.py script requires the `pyelftools` and `pefile` dependencies be installed; use `pip3 install` to install these.
 
+## Bazel Testing
+Pyunit tests are automatically auto generated run using the checker tool against all of the vendordep json files in the repository by bazel.
+
+### Prerequisites
+- Install [Bazelisk](https://github.com/bazelbuild/bazelisk/releases) and add it to your path. Bazelisk is a wrapper that will download the correct version of bazel specified in the repository. Note: You can alias/rename the binary to `bazel` if you want to keep the familiar `bazel build` vs `bazelisk build` syntax.
+
+### Running the tests
+To run the tests, simply run `bazel test //...`. Alternatively, you can run the `checker.py` tool in a standalone mode by running `bazel run //:checker -- <command line arguments from above>`

README.md

@@ -9,6 +9,7 @@
 import sys
 import urllib.request
 import uuid
+import pathlib
 from zipfile import ZipFile, BadZipFile
 
 try:
@@ -38,6 +39,7 @@
 got_error = 0
 got_warn = 0
 message_context = []
+cache_directory = None
 
 def msg(t, s):
     ctx = ': '.join(message_context) + ': ' if message_context else ''
@@ -64,19 +66,21 @@ def info(s):
 local_maven = None
 year = "2025"
 
-def parse_args():
+def parse_args(argv):
     """Parse command line arguments.  Returns list of filenames."""
     parser = argparse.ArgumentParser(description='Checks a vendor json file')
     parser.add_argument('--verbose', '-v', action='count', help='increase the verbosity of output')
     parser.add_argument('--local-maven', help='directory to use for artifacts instead of fetching from mavenUrls')
     parser.add_argument('--year', '-y', help='FRC competition season year (used to set known libraries)')
+    parser.add_argument('--cache_directory', type=pathlib.Path, help='Optional. If present will set up a download cache in this directory to prevent re-downloading artifacts. Should be used for debugging purposes only.')
     parser.add_argument('file', nargs='+', help='json file to parse')
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
 
-    global verbose, local_maven, year
+    global verbose, local_maven, year, cache_directory
     verbose = args.verbose or 0
     local_maven = args.local_maven
     year = args.year or "2025"
+    cache_directory = args.cache_directory
 
     return args.file
 
@@ -208,11 +212,22 @@ def fetch(self, classifier, failok=False):
         else:
             for baseurl in self.urls:
                 url = baseurl + self.path + fn
+                maybe_cached_file = None
+                if cache_directory:
+                    maybe_cached_file = cache_directory / (self.path + fn)
+                    if maybe_cached_file.exists():
+                        if verbose >= 2:
+                            print(f"Found a cache hit for {maybe_cached_file}")
+                        return fn, maybe_cached_file.read_bytes()
+
                 if verbose >= 1:
                     print('downloading "{0}"'.format(url))
                 try:
                     with urlopener.open(url) as f:
                         result = f.read()
+                    if maybe_cached_file:
+                        maybe_cached_file.parent.mkdir(parents=True, exist_ok=True)
+                        maybe_cached_file.write_bytes(result)
                 except urllib.error.HTTPError as e:
                     if not failok:
                         warn('could not fetch url "{0}": {1}'.format(url, e))
@@ -629,7 +644,7 @@ def check_file(filename):
 
 def main():
     had_errors = False
-    for fn in parse_args():
+    for fn in parse_args(sys.argv[1:]):
         global json_filename, got_error, got_warn
         json_filename = fn
         got_error = 0