Add authenticated wheel downloads with aws-vault integration

Since S3 bucket cannot be made public, reverted to authenticated downloads:

Upload changes (release.py):
- Pointer files: public-read (for TUF access)
- Wheel files: private (requires AWS authentication)

Downloader changes:
- Added boto3 back for authenticated S3 downloads
- Added inline aws-vault integration to CLI
- Downloads wheels using boto3.client('s3').get_object()
- Verifies wheel digest after download

CLI integration:
- Added --aws-vault-profile option
- Auto-detects AWS credentials
- Re-execs with aws-vault if credentials missing
- Uses default profile: sso-agent-integrations-dev-account-admin

Usage:
  # With explicit profile
  datadog-checks-downloader datadog-postgres --aws-vault-profile my-profile

  # Auto-detect (will use aws-vault if no credentials)
  datadog-checks-downloader datadog-postgres

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

Author: dkirov-dd

datadog_checks_dev/datadog_checks/dev/tooling/release.py

@@ -266,21 +266,21 @@ def upload_package(package_path, version, public=False):
                 raise
             # Doesn't exist, proceed with upload
 
-        # Upload pointer file (public for TUF)
+        # Upload pointer file with metadata (public for TUF access)
         s3.upload_file(
             pointer_file_path,
             S3_BUCKET,
             pointer_s3_key,
             ExtraArgs={'Metadata': {'digest': wheel_hash, 'version': version}, 'ACL': 'public-read'},
         )
 
-        # Upload wheel file with hash metadata (public for direct download)
+        # Upload wheel file with hash metadata (private, requires authentication)
         wheel_s3_key = f"simple/{package_name}/{wheel_file_name}"
         s3.upload_file(
             wheel_file_path,
             S3_BUCKET,
             wheel_s3_key,
-            ExtraArgs={'Metadata': {'sha256': wheel_hash}, 'ACL': 'public-read'},
+            ExtraArgs={'Metadata': {'sha256': wheel_hash}},
         )
 
         print(f"Uploaded {pointer_file_name} and {wheel_file_name} to S3 bucket {S3_BUCKET}")

datadog_checks_downloader/datadog_checks/downloader/cli.py

@@ -7,6 +7,7 @@
 import argparse
 import os
 import re
+import subprocess
 import sys
 
 # 2nd party.
@@ -21,6 +22,38 @@
 # Private module functions.
 
 
+def __check_aws_credentials():
+    """Check if AWS credentials are available."""
+    import boto3
+    try:
+        sts = boto3.client('sts')
+        sts.get_caller_identity()
+        return True
+    except Exception:
+        return False
+
+
+def __exec_with_aws_vault(profile, command_args):
+    """Execute command with aws-vault profile."""
+    aws_vault_cmd = ['aws-vault', 'exec', profile, '--'] + command_args
+    sys.stderr.write(f"Re-executing with aws-vault profile: {profile}\n")
+    sys.stderr.flush()
+    os.execvp('aws-vault', aws_vault_cmd)
+
+
+def __ensure_aws_credentials(profile=None):
+    """Ensure AWS credentials are available, re-exec with aws-vault if needed."""
+    if profile:
+        # User explicitly specified a profile, re-exec with aws-vault
+        __exec_with_aws_vault(profile, sys.argv)
+    elif not __check_aws_credentials():
+        # No credentials available, try to find default profile
+        default_profile = os.environ.get('AWS_VAULT_PROFILE') or 'sso-agent-integrations-dev-account-admin'
+        sys.stderr.write(f"No AWS credentials found. Using aws-vault profile: {default_profile}\n")
+        sys.stderr.flush()
+        __exec_with_aws_vault(default_profile, sys.argv)
+
+
 def __is_canonical(version):
     """
     https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions
@@ -100,8 +133,18 @@ def instantiate_downloader():
         '-v', '--verbose', action='count', default=0, help='Show verbose information about TUF.'
     )
 
+    parser.add_argument(
+        '--aws-vault-profile',
+        type=str,
+        default=None,
+        help='AWS Vault profile to use for authentication. If not specified, will auto-detect or re-exec with aws-vault if needed.',
+    )
+
     args = parser.parse_args()
 
+    # Ensure AWS credentials are available (will re-exec with aws-vault if needed)
+    __ensure_aws_credentials(profile=args.aws_vault_profile)
+
     repository_url_prefix = args.repository
     standard_distribution_name = args.standard_distribution_name
     version = args.version

datadog_checks_downloader/datadog_checks/downloader/download.py

@@ -13,6 +13,7 @@
 import urllib.parse
 import urllib.request
 
+import boto3
 import yaml
 
 from packaging.version import parse as parse_version
@@ -167,6 +168,8 @@ def __download_wheel_from_pointer(self, pointer_abspath, standard_distribution_n
         Returns:
             Absolute path to downloaded wheel file
         """
+        from urllib.parse import urlparse
+
         # Parse pointer file
         with open(pointer_abspath, 'rb') as f:
             pointer_bytes = f.read()
@@ -185,15 +188,34 @@ def __download_wheel_from_pointer(self, pointer_abspath, standard_distribution_n
         # Extract wheel filename from URI
         wheel_filename = wheel_uri.split('/')[-1]
 
-        # Download wheel directly from public URI
+        # Download wheel from S3 with authentication
         logger.info(f'Downloading wheel from: {wheel_uri}')
         wheel_abspath = os.path.join(self.__targets_dir, 'simple', standard_distribution_name, wheel_filename)
         os.makedirs(os.path.dirname(wheel_abspath), exist_ok=True)
 
         try:
-            # Download from public URI
-            with urllib.request.urlopen(wheel_uri) as resp:
-                wheel_bytes = resp.read()
+            # Parse S3 URI to extract bucket and key
+            # Example: https://test-public-integration-wheels.s3.eu-north-1.amazonaws.com/simple/datadog-postgres/wheel.whl
+            parsed = urlparse(wheel_uri)
+
+            # Extract bucket name from hostname (format: bucket.s3.region.amazonaws.com)
+            bucket_name = parsed.hostname.split('.')[0]
+
+            # Extract S3 key (path without leading /)
+            s3_key = parsed.path.lstrip('/')
+
+            # Extract region from hostname if present
+            if '.s3.' in parsed.hostname and '.amazonaws.com' in parsed.hostname:
+                region = parsed.hostname.split('.s3.')[1].split('.amazonaws.com')[0]
+            else:
+                region = None
+
+            logger.debug(f'Parsed S3 URI: bucket={bucket_name}, key={s3_key}, region={region}')
+
+            # Use boto3 to download with AWS credentials
+            s3_client = boto3.client('s3', region_name=region)
+            response = s3_client.get_object(Bucket=bucket_name, Key=s3_key)
+            wheel_bytes = response['Body'].read()
 
             # Verify digest
             actual_digest = hashlib.sha256(wheel_bytes).hexdigest()
@@ -209,8 +231,8 @@ def __download_wheel_from_pointer(self, pointer_abspath, standard_distribution_n
             logger.info(f'Wheel verified and saved: {wheel_abspath}')
             return wheel_abspath
 
-        except urllib.error.HTTPError as err:
-            logger.error('GET %s: %s', wheel_uri, err)
+        except Exception as err:
+            logger.error('Failed to download wheel from %s: %s', wheel_uri, err)
             raise
 
     def download(self, target_relpath):

datadog_checks_downloader/pyproject.toml

@@ -33,6 +33,7 @@ license = "BSD-3-Clause"
 
 [project.optional-dependencies]
 deps = [
+    "boto3>=1.40.0",
     "in-toto==2.0.0",
     "packaging==25.0",
     "securesystemslib[crypto,pynacl]==0.28.0",