Run CI build with fixed set of boards (#3389)

* run cmake ci build on github with a fixed set of board to keep the size stable
* Size Difference Report contain major >1% and minor <1& table

Author: hathach

.circleci/config.yml

@@ -44,7 +44,7 @@ jobs:
               local build_args=""
 
               if [[ "$toolchain" == "arm-iar" || "$build_system" == "make" ]]; then
-                build_args="--one-per-family"
+                build_args="--one-random"
               fi
 
               if [[ "$toolchain" == "esp-idf" ]]; then

.github/workflows/build.yml

@@ -57,9 +57,9 @@ jobs:
           echo "hil_matrix=$HIL_MATRIX_JSON" >> $GITHUB_OUTPUT
 
   # ------------------------------------------------------------------------------
-  # CMake build: only one-per-family. Full built is done by CircleCI in PR
+  # CMake build: only one board per family (first alphabetically). Full build is done by CircleCI in PR
   # Note:
-  # For Make and IAR build: will be done on CircleCI only (one-per-family too)
+  # For Make and IAR build: will be done on CircleCI only (one random per family as well)
   # ------------------------------------------------------------------------------
   cmake:
     needs: set-matrix
@@ -78,7 +78,7 @@ jobs:
       build-system: 'cmake'
       toolchain: ${{ matrix.toolchain }}
       build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain]) }}
-      one-per-family: true
+      build-options: '--one-first'
       upload-metrics: true
 
   code-metrics:
@@ -153,7 +153,7 @@ jobs:
       build-system: 'cmake-make'
       toolchain: 'arm-gcc-${{ matrix.os }}'
       build-args: '["stm32h7"]'
-      one-per-family: true
+      build-options: '--one-random'
 
   # ---------------------------------------
   # Zephyr
@@ -196,7 +196,6 @@ jobs:
       build-system: 'cmake'
       toolchain: ${{ matrix.toolchain }}
       build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.hil_json)[matrix.toolchain]) }}
-      one-per-family: true
       upload-artifacts: true
 
   # ---------------------------------------

.github/workflows/build_util.yml

@@ -12,10 +12,10 @@ on:
       build-args:
         required: true
         type: string
-      one-per-family:
+      build-options:
         required: false
-        default: false
-        type: boolean
+        default: ''
+        type: string
       upload-artifacts:
         required: false
         default: false
@@ -51,16 +51,6 @@ jobs:
         with:
           arg: ${{ matrix.arg }}
 
-      - name: Set build one-per-family option
-        id: set-one-per-family
-        run: |
-          if [[ "${{ inputs.one-per-family }}" == "true" ]]; then
-            BUILD_OPTION="--one-per-family"
-          fi
-          echo "build_option=$BUILD_OPTION"
-          echo "build_option=$BUILD_OPTION" >> $GITHUB_OUTPUT
-        shell: bash
-
       - name: Build
         env:
           IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_LMS_BEARER_TOKEN }}
@@ -69,10 +59,10 @@ jobs:
           if [ "$TOOLCHAIN" == "esp-idf" ]; then
             docker run --rm -v $PWD:/project -w /project espressif/idf:tinyusb python tools/build.py ${{ matrix.arg }}
           elif [ "${{ inputs.build-system }}" == "cmake-make" ] || [ "${{ inputs.build-system }}" == "make-cmake" ]; then
-            python tools/build.py -s make ${{ steps.setup-toolchain.outputs.build_option }} ${{ steps.set-one-per-family.outputs.build_option }} ${{ matrix.arg }}
-            python tools/build.py -s cmake ${{ steps.setup-toolchain.outputs.build_option }} ${{ steps.set-one-per-family.outputs.build_option }} ${{ matrix.arg }}
+            python tools/build.py -s make ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }} ${{ matrix.arg }}
+            python tools/build.py -s cmake ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }} ${{ matrix.arg }}
           else
-            python tools/build.py -s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ steps.set-one-per-family.outputs.build_option }} ${{ matrix.arg }}
+            python tools/build.py -s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }} ${{ matrix.arg }}
           fi
         shell: bash
 

hw/bsp/rp2040/skip_ci.txt

@@ -27,6 +27,23 @@
 clean_build = False
 parallel_jobs = os.cpu_count()
 
+# CI board control lists (used when running under CI)
+ci_skip_boards = {
+    'rp2040': [
+        'adafruit_feather_rp2040_usb_host',
+        'adafruit_fruit_jam',
+        'adafruit_metro_rp2350',
+        'feather_rp2040_max3421',
+        'pico_sdk',
+        'raspberry_pi_pico_w',
+    ],
+}
+
+ci_preferred_boards = {
+    'stm32h7': ['stm32h743eval'],
+}
+
+
 # -----------------------------
 # Helper
 # -----------------------------
@@ -195,35 +212,40 @@ def build_boards_list(boards, build_defines, build_system, build_flags_on):
     return ret
 
 
-def get_family_boards(family, one_per_family, boards):
+def get_family_boards(family, one_random, one_first):
     """Get list of boards for a family.
 
     Args:
         family: Family name
-        one_per_family: If True, return only one random board
-        boards: List of boards already specified via -b flag
+        one_random: If True, return only one random board
+        one_first: If True, return only the first board (alphabetical)
 
     Returns:
         List of board names
     """
-    skip_ci = []
+    skip_list = []
+    preferred_list = []
     if os.getenv('GITHUB_ACTIONS') or os.getenv('CIRCLECI'):
-        skip_ci_file = Path(f"hw/bsp/{family}/skip_ci.txt")
-        if skip_ci_file.exists():
-            skip_ci = skip_ci_file.read_text().split()
+        skip_list = ci_skip_boards.get(family, [])
+        preferred_list = ci_preferred_boards.get(family, [])
+
     all_boards = []
     for entry in os.scandir(f"hw/bsp/{family}/boards"):
-        if entry.is_dir() and not entry.name in skip_ci:
+        if entry.is_dir() and entry.name not in skip_list:
             all_boards.append(entry.name)
+    if not all_boards:
+        print(f"No boards found for family '{family}'")
+        return []
     all_boards.sort()
 
-    # If only-one flag is set, select one random board
-    if one_per_family:
-        for b in boards:
-            # skip if -b already specify one in this family
-            if find_family(b) == family:
-                return []
-        all_boards = [random.choice(all_boards)]
+    # If only-one flags are set, honor select list first, then pick first or random
+    if one_first or one_random:
+        if preferred_list:
+            return [preferred_list[0]]
+        if one_first:
+            return [all_boards[0]]
+        if one_random:
+            return [random.choice(all_boards)]
 
     return all_boards
 
@@ -244,7 +266,10 @@ def main():
     parser.add_argument('-s', '--build-system', default='cmake', help='Build system to use, default is cmake')
     parser.add_argument('-D', '--define-symbol', action='append', default=[], help='Define to pass to build system')
     parser.add_argument('-f1', '--build-flags-on', action='append', default=[], help='Build flag to pass to build system')
-    parser.add_argument('-1', '--one-per-family', action='store_true', default=False, help='Build only one random board inside a family')
+    parser.add_argument('--one-random', action='store_true', default=False,
+                        help='Build only one random board of each specified family')
+    parser.add_argument('--one-first', action='store_true', default=False,
+                        help='Build only the first board (alphabetical) of each specified family')
     parser.add_argument('-j', '--jobs', type=int, default=os.cpu_count(), help='Number of jobs to run in parallel')
     parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output')
     args = parser.parse_args()
@@ -255,7 +280,8 @@ def main():
     build_system = args.build_system
     build_defines = args.define_symbol
     build_flags_on = args.build_flags_on
-    one_per_family = args.one_per_family
+    one_random = args.one_random
+    one_first = args.one_first
     verbose = args.verbose
     clean_build = args.clean
     parallel_jobs = args.jobs
@@ -283,7 +309,7 @@ def main():
     # get boards from families and append to boards list
     all_boards = list(boards)
     for f in all_families:
-        all_boards.extend(get_family_boards(f, one_per_family, boards))
+        all_boards.extend(get_family_boards(f, one_random, one_first))
 
     # build all boards
     result = build_boards_list(all_boards, build_defines, build_system, build_flags_on)

tools/build.py

@@ -245,8 +245,18 @@ def write_compare_markdown(comparison, path, sort_order='size'):
     header += " Total |"
     separator += "------:|"
 
-    md_lines.append(header)
-    md_lines.append(separator)
+    def is_significant(file_row):
+        for s in sections:
+            sd = file_row["sections"][s]
+            diff = abs(sd["diff"])
+            base = sd["base"]
+            if base == 0:
+                if diff != 0:
+                    return True
+            else:
+                if (diff / base) * 100 > 1.0:
+                    return True
+        return False
 
     # Sort files based on sort_order
     if sort_order == 'size-':
@@ -263,38 +273,56 @@ def write_compare_markdown(comparison, path, sort_order='size'):
         reverse = False
     sorted_files = sorted(comparison["files"], key=key_func, reverse=reverse)
 
-    sum_base = {s: 0 for s in sections}
-    sum_base["total"] = 0
-    sum_new = {s: 0 for s in sections}
-    sum_new["total"] = 0
-
+    significant = []
+    minor = []
     for f in sorted_files:
         # Skip files with no changes
         if f["total"]["diff"] == 0 and all(f["sections"][s]["diff"] == 0 for s in sections):
             continue
-
-        row = f"| {f['file']} |"
+        (significant if is_significant(f) else minor).append(f)
+
+    def render_table(title, rows):
+        md_lines.append(f"## {title}")
+        if not rows:
+            md_lines.append("No entries.")
+            md_lines.append("")
+            return
+
+        md_lines.append(header)
+        md_lines.append(separator)
+
+        sum_base = {s: 0 for s in sections}
+        sum_base["total"] = 0
+        sum_new = {s: 0 for s in sections}
+        sum_new["total"] = 0
+
+        for f in rows:
+            row = f"| {f['file']} |"
+            for s in sections:
+                sd = f["sections"][s]
+                sum_base[s] += sd["base"]
+                sum_new[s] += sd["new"]
+                row += f" {format_diff(sd['base'], sd['new'], sd['diff'])} |"
+
+            td = f["total"]
+            sum_base["total"] += td["base"]
+            sum_new["total"] += td["new"]
+            row += f" {format_diff(td['base'], td['new'], td['diff'])} |"
+
+            md_lines.append(row)
+
+        # Add sum row
+        sum_row = "| **SUM** |"
         for s in sections:
-            sd = f["sections"][s]
-            sum_base[s] += sd["base"]
-            sum_new[s] += sd["new"]
-            row += f" {format_diff(sd['base'], sd['new'], sd['diff'])} |"
-
-        td = f["total"]
-        sum_base["total"] += td["base"]
-        sum_new["total"] += td["new"]
-        row += f" {format_diff(td['base'], td['new'], td['diff'])} |"
-
-        md_lines.append(row)
-
-    # Add sum row
-    sum_row = "| **SUM** |"
-    for s in sections:
-        diff = sum_new[s] - sum_base[s]
-        sum_row += f" {format_diff(sum_base[s], sum_new[s], diff)} |"
-    total_diff = sum_new["total"] - sum_base["total"]
-    sum_row += f" {format_diff(sum_base['total'], sum_new['total'], total_diff)} |"
-    md_lines.append(sum_row)
+            diff = sum_new[s] - sum_base[s]
+            sum_row += f" {format_diff(sum_base[s], sum_new[s], diff)} |"
+        total_diff = sum_new["total"] - sum_base["total"]
+        sum_row += f" {format_diff(sum_base['total'], sum_new['total'], total_diff)} |"
+        md_lines.append(sum_row)
+        md_lines.append("")
+
+    render_table("Changes >1% in any section", significant)
+    render_table("Changes <1% in all sections", minor)
 
     with open(path, "w", encoding="utf-8") as f:
         f.write("\n".join(md_lines))