refactor: Simplify retry logic in bundle analysis and upload finisher tasks

Author: drazisil-codecov

apps/worker/services/lock_manager.py

@@ -128,6 +128,7 @@ def locked(
             )
             countdown = min(countdown_unbounded, max_retry_cap)
 
+            # None means no max retries (infinite retries)
             max_attempts = max_retries + 1 if max_retries is not None else None
             if max_attempts is not None and retry_num >= max_attempts:
                 error_msg = (

apps/worker/tasks/bundle_analysis_processor.py

@@ -207,22 +207,32 @@ def process_impl_within_lock(
             result: ProcessingResult = report_service.process_upload(
                 commit, upload, compare_sha
             )
-            if (
-                result.error
-                and result.error.is_retryable
-                and self.request.retries < self.max_retries
-            ):
+            if result.error and result.error.is_retryable:
+                if self._has_exceeded_max_attempts(self.max_retries):
+                    attempts = self.attempts
+                    max_attempts = self.max_retries + 1
+                    log.error(
+                        "Bundle analysis processor exceeded max retries",
+                        extra={
+                            "attempts": attempts,
+                            "commitid": commitid,
+                            "max_attempts": max_attempts,
+                            "max_retries": self.max_retries,
+                            "repoid": repoid,
+                        },
+                    )
+                    return
                 log.warn(
                     "Attempting to retry bundle analysis upload",
                     extra={
+                        "commitid": commitid,
                         "repoid": repoid,
-                        "commit": commitid,
                         "commit_yaml": commit_yaml,
                         "params": params,
                         "result": result.as_dict(),
                     },
                 )
-                self.retry(countdown=30 * (2**self.request.retries))
+                self.retry(max_retries=self.max_retries, countdown=30 * (2**self.request.retries))
             result.update_upload(carriedforward=carriedforward)
             db_session.commit()
 

apps/worker/tasks/upload_finisher.py

@@ -493,17 +493,7 @@ def _process_reports_with_lock(
                 upload_ids=upload_ids,
                 error=Errors.INTERNAL_RETRYING,
             )
-            if not self.safe_retry(max_retries=MAX_RETRIES, countdown=retry.countdown):
-                attempts = self.attempts
-                log.error(
-                    "Failed to schedule retry for upload finisher",
-                    extra={
-                        "attempts": attempts,
-                        "commitid": commitid,
-                        "repoid": repoid,
-                    },
-                )
-                return
+            self.retry(max_retries=MAX_RETRIES, countdown=retry.countdown)
 
     def _handle_finisher_lock(
         self,
@@ -621,17 +611,7 @@ def _handle_finisher_lock(
                 upload_ids=upload_ids,
                 error=Errors.INTERNAL_RETRYING,
             )
-            if not self.safe_retry(max_retries=MAX_RETRIES, countdown=retry.countdown):
-                attempts = self.attempts
-                log.error(
-                    "Failed to schedule retry for upload finisher (finisher lock)",
-                    extra={
-                        "attempts": attempts,
-                        "commitid": commitid,
-                        "repoid": repoid,
-                    },
-                )
-                return
+            self.retry(max_retries=MAX_RETRIES, countdown=retry.countdown)
 
     def finish_reports_processing(
         self,