From 047bd4184880eb36884e331b70264a122bb70671 Mon Sep 17 00:00:00 2001 From: Lorenzo Delgado Date: Sat, 22 Nov 2025 14:36:24 -0300 Subject: [PATCH] feat(ampup): add ampup-gen-release-manifest binary Generate release manifests from GitHub releases to enable per-component installation, allowing users to install specific components instead of complete releases. - Create `ampup-gen-release-manifest` binary for manifest generation - Add shared manifest types to `ampup` library - Parse GitHub release assets into component-target structure - Integrate manifest generation into CI pipeline Signed-off-by: Lorenzo Delgado --- .github/workflows/build.yml | 35 ++ Cargo.lock | 17 + Cargo.toml | 1 + .../bin/ampup-gen-release-manifest/Cargo.toml | 19 + .../ampup-gen-release-manifest/src/main.rs | 438 ++++++++++++++++++ crates/bin/ampup/Cargo.toml | 9 +- crates/bin/ampup/docs/manifest.md | 225 +++++++++ crates/bin/ampup/src/lib.rs | 1 + crates/bin/ampup/src/manifest.rs | 425 +++++++++++++++++ crates/core/monitoring/src/logging.rs | 1 + 10 files changed, 1168 insertions(+), 3 deletions(-) create mode 100644 crates/bin/ampup-gen-release-manifest/Cargo.toml create mode 100644 crates/bin/ampup-gen-release-manifest/src/main.rs create mode 100644 crates/bin/ampup/docs/manifest.md create mode 100644 crates/bin/ampup/src/manifest.rs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 952bd8aff..cc0044b26 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -194,6 +194,41 @@ jobs: with: draft: false + release-manifest: + name: Release Manifest + needs: release + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + permissions: + contents: write + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1 + with: + cache: true + + - name: Generate release manifest + run: | + # Create release directory + mkdir -p release + + # Run manifest generator binary (outputs to stdout, redirect to file) + cargo run --bin ampup-gen-release-manifest -- \ + ${{ github.repository }} \ + ${{ github.ref_name }} \ + > release/manifest.json + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload manifest to release + uses: softprops/action-gh-release@6cbd405e2c4e67a21c47fa9e383d020e4e28b836 # v2 + with: + files: release/manifest.json + fail_on_unmatched_files: true + containerize: name: Containerize (${{ matrix.title }}) runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 098a7440a..44abfca88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1043,6 +1043,7 @@ name = "ampup" version = "0.1.0" dependencies = [ "anyhow", + "chrono", "clap", "console 0.15.11", "dialoguer", @@ -1052,11 +1053,27 @@ dependencies = [ "reqwest", "semver 1.0.27", "serde", + "serde_json", "tempfile", + "thiserror 2.0.17", "tokio", "vergen-gitcl", ] +[[package]] +name = "ampup-gen-release-manifest" +version = "0.1.0" +dependencies = [ + "ampup", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.17", + "tokio", +] + [[package]] name = "android_system_properties" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index d50acf5cd..36338a601 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/bin/ampd", "crates/bin/ampsync", "crates/bin/ampup", + "crates/bin/ampup-gen-release-manifest", "crates/clients/flight", "crates/core/common", "crates/core/dataset-store", diff --git a/crates/bin/ampup-gen-release-manifest/Cargo.toml b/crates/bin/ampup-gen-release-manifest/Cargo.toml new file mode 100644 index 000000000..33483066d --- /dev/null +++ b/crates/bin/ampup-gen-release-manifest/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "ampup-gen-release-manifest" +version.workspace = true +license-file.workspace = true +edition.workspace = true + +[[bin]] +name = "ampup-gen-release-manifest" +path = "src/main.rs" + +[dependencies] +ampup = { path = "../ampup" } +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true diff --git a/crates/bin/ampup-gen-release-manifest/src/main.rs b/crates/bin/ampup-gen-release-manifest/src/main.rs new file mode 100644 index 000000000..7fbfe06e1 --- /dev/null +++ b/crates/bin/ampup-gen-release-manifest/src/main.rs @@ -0,0 +1,438 @@ +//! Release Manifest Generator Binary +//! +//! Generates release manifest from GitHub release assets. + +use std::collections::BTreeMap; + +// Constants +const ASSET_NAME_PATTERN: &str = "--"; + +/// Generate release manifest from GitHub release assets +#[derive(Debug, clap::Parser)] +#[command(name = "ampup-gen-release-manifest")] +#[command(about = "Generate release manifest from GitHub release assets", long_about = None)] +struct Args { + /// GitHub repository in format "owner/name" (e.g., "edgeandnode/amp") + repository: github::Repo, + + /// Git tag (e.g., "v1.0.0") or "latest" for the latest release (default: "latest") + #[arg(default_value = "latest")] + tag: String, + + /// GitHub token for authentication (reads from GITHUB_TOKEN env var) + #[arg(env = "GITHUB_TOKEN", hide = true)] + github_token: Option, +} + +#[tokio::main] +async fn main() -> Result<(), Error> { + let args = ::parse(); + + eprintln!("════════════════════════════════════════════════════════════════"); + eprintln!("🚀 Release Manifest Generator"); + eprintln!("════════════════════════════════════════════════════════════════"); + eprintln!( + "Repository: {}/{}", + args.repository.owner(), + args.repository.name() + ); + eprintln!("Tag: {}", args.tag); + eprintln!("Output: stdout\n"); + + eprintln!("📥 Fetching release from GitHub API...\n"); + + // Fetch release from GitHub (latest or specific tag) + let release = if args.tag == "latest" { + github::fetch_latest(&args.repository, args.github_token.as_deref()) + .await + .map_err(Error::FetchRelease)? + } else { + github::fetch_release(&args.repository, &args.tag, args.github_token.as_deref()) + .await + .map_err(Error::FetchRelease)? + }; + + // Extract version from tag (remove 'v' prefix) + let version = release + .tag_name + .strip_prefix('v') + .unwrap_or(&release.tag_name) + .to_string(); + + eprintln!( + "\n📋 Generating manifest from {} assets...\n", + release.assets.len() + ); + + // Generate manifest + let manifest = generate_manifest(&release, &version, &args)?; + + // Write manifest to stdout + eprintln!("\n💾 Writing manifest to stdout..."); + let manifest_json = serde_json::to_string_pretty(&manifest).map_err(Error::JsonSerialize)?; + println!("{}", manifest_json); + + eprintln!("\n════════════════════════════════════════════════════════════════"); + eprintln!("✅ SUCCESS: Generated manifest to stdout"); + eprintln!("════════════════════════════════════════════════════════════════\n"); + + // Print summary to stderr + eprintln!("Manifest summary:"); + eprintln!(" Version: {}", manifest.version); + eprintln!(" Components: {}", manifest.components.len()); + for (component, data) in &manifest.components { + eprintln!(" - {}: {} target(s)", component, data.targets.len()); + } + eprintln!(); + + Ok(()) +} + +/// Errors that can occur during manifest generation +#[derive(Debug, thiserror::Error)] +enum Error { + /// Failed to fetch release from GitHub API + /// + /// This error wraps all failures that can occur when fetching release metadata + /// from the GitHub API, including network errors, authentication failures, and + /// response parsing errors. + #[error("Failed to fetch release from GitHub API")] + FetchRelease(#[source] github::FetchError), + + /// Asset missing required digest field + /// + /// This occurs when a release asset does not have the required SHA-256 digest field. + /// All assets must include a digest for security and integrity verification. + #[error("Asset '{asset_name}' missing digest field")] + MissingDigest { asset_name: String }, + + /// No binary components found in release + /// + /// This occurs when the release contains no assets matching the expected binary + /// naming pattern. The release must contain at least one binary asset. + #[error( + "No binary components found in release! Expected assets matching pattern: {ASSET_NAME_PATTERN}" + )] + NoComponents, + + /// Failed to serialize manifest to JSON + /// + /// This occurs when the generated manifest cannot be serialized to JSON, + /// typically due to invalid UTF-8 or other serialization constraints. + #[error("Failed to serialize manifest to JSON")] + JsonSerialize(#[source] serde_json::Error), +} + +/// Generate manifest from GitHub release assets +/// +/// # Errors +/// Returns error if: +/// - No valid binary assets found in release +/// - Asset missing required digest field +fn generate_manifest( + release: &github::Release, + version: &str, + args: &Args, +) -> Result { + use ampup::manifest::{AssetName, Component, MANIFEST_VERSION, Manifest, Repository, Target}; + + let mut component_map: BTreeMap = BTreeMap::new(); + + // Process each asset in the release + for asset in &release.assets { + let parsed: AssetName = match asset.name.parse() { + Ok(name) => name, + Err(_) => { + eprintln!("⚠️ Skipping non-binary asset: {}", asset.name); + continue; + } + }; + + eprintln!( + "✓ Discovered: {} → component={}, target={}", + asset.name, + parsed.component(), + parsed.target() + ); + + // Validate digest field is present + let hash = asset.digest.as_ref().ok_or_else(|| Error::MissingDigest { + asset_name: asset.name.clone(), + })?; + + // Initialize component if first time seeing it + let component = component_map + .entry(parsed.component().to_string()) + .or_insert_with(|| Component { + version: version.to_string(), + targets: BTreeMap::new(), + }); + + // Check if notarized (macOS targets) + let notarized = parsed.is_macos(); + + // Add target to component + component.targets.insert( + parsed.target(), + Target { + url: asset.browser_download_url.clone(), + hash: hash.clone(), + size: asset.size, + notarized, + }, + ); + + eprintln!( + " → Added target {} ({} bytes, notarized: {})", + parsed.target(), + asset.size, + notarized + ); + } + + // Validate we discovered at least one component + if component_map.is_empty() { + return Err(Error::NoComponents); + } + + eprintln!("\n✅ Discovered {} component(s)", component_map.len()); + for (component, data) in &component_map { + eprintln!(" - {}: {} target(s)", component, data.targets.len()); + } + + // Build final manifest + let manifest = Manifest { + manifest_version: MANIFEST_VERSION.to_string(), + date: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(), + version: version.to_string(), + repository: Repository { + owner: args.repository.owner().to_string(), + name: args.repository.name().to_string(), + url: format!("https://github.com/{}", args.repository), + }, + components: component_map, + }; + + Ok(manifest) +} + +mod github { + use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT}; + + /// Build GitHub API URL for fetching latest release. + /// + /// GET `/repos/{owner}/{name}/releases/latest` + pub fn api_latest_release(repo: &Repo) -> String { + format!("https://api.github.com/repos/{}/releases/latest", repo) + } + + /// Build GitHub API URL for fetching release by tag. + /// + /// GET `/repos/{owner}/{name}/releases/tags/{tag}` + pub fn api_release_by_tag(repo: &Repo, tag: &str) -> String { + format!( + "https://api.github.com/repos/{}/releases/tags/{}", + repo, tag + ) + } + + /// Fetch latest release metadata from GitHub API + pub async fn fetch_latest( + repo: &Repo, + github_token: Option<&str>, + ) -> Result { + fetch_release_from_url(api_latest_release(repo), github_token).await + } + + /// Fetch release metadata from GitHub API by tag + pub async fn fetch_release( + repo: &Repo, + tag: &str, + github_token: Option<&str>, + ) -> Result { + fetch_release_from_url(api_release_by_tag(repo, tag), github_token).await + } + + /// Internal helper to fetch release metadata from a GitHub API URL + async fn fetch_release_from_url( + url: String, + github_token: Option<&str>, + ) -> Result { + eprintln!(" URL: {}", url); + + let client = reqwest::Client::new(); + let mut headers = HeaderMap::new(); + headers.insert( + USER_AGENT, + HeaderValue::from_static("ampup-gen-release-manifest"), + ); + + if let Some(token) = github_token { + eprintln!(" Authentication: Using GITHUB_TOKEN"); + let auth_value = format!("{} {}", "Bearer", token); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&auth_value).map_err(FetchError::InvalidTokenFormat)?, + ); + } else { + eprintln!(" Authentication: None (unauthenticated)"); + } + + let response = client + .get(&url) + .headers(headers) + .send() + .await + .map_err(FetchError::HttpRequest)?; + + if !response.status().is_success() { + return Err(FetchError::ApiRequestFailed { + status: response.status().as_u16(), + body: response.text().await.unwrap_or_default(), + }); + } + + let release: Release = response.json().await.map_err(FetchError::JsonParse)?; + + eprintln!(" ✅ Release: {}", release.name); + eprintln!(" ✅ Assets: {}", release.assets.len()); + + Ok(release) + } + + /// Errors that can occur when fetching release from GitHub API + #[derive(Debug, thiserror::Error)] + pub enum FetchError { + /// Failed to create HTTP request headers with provided GitHub token + /// + /// This occurs when the GitHub token contains invalid characters that cannot + /// be used in HTTP headers. + #[error("Invalid GitHub token format")] + InvalidTokenFormat(#[source] reqwest::header::InvalidHeaderValue), + + /// Failed to send HTTP request to GitHub API + /// + /// This occurs when the network request fails due to connectivity issues, + /// DNS resolution failures, or other network-level problems. + /// + /// Possible causes: + /// - No internet connectivity + /// - DNS resolution failure for api.github.com + /// - Network timeout + /// - TLS/SSL handshake failure + #[error("Failed to send HTTP request to GitHub API")] + HttpRequest(#[source] reqwest::Error), + + /// GitHub API returned non-success status code + /// + /// This occurs when the GitHub API request completes but returns an error status. + /// + /// Common status codes: + /// - 401: Invalid or missing authentication token + /// - 404: Repository or release tag not found + /// - 403: Rate limit exceeded or insufficient permissions + /// - 500+: GitHub server errors + #[error("GitHub API request failed with status {status}: {body}")] + ApiRequestFailed { status: u16, body: String }, + + /// Failed to parse GitHub API JSON response + /// + /// This occurs when the response body is not valid JSON or doesn't match + /// the expected Release schema. + #[error("Failed to parse GitHub API response as JSON")] + JsonParse(#[source] reqwest::Error), + } + + /// Repository in format `owner/name` + #[derive(Debug, Clone)] + pub struct Repo(String); + + impl Repo { + /// Returns the repository owner + pub fn owner(&self) -> &str { + self.0.split('/').next().unwrap() + } + + /// Returns the repository name + pub fn name(&self) -> &str { + self.0.split('/').nth(1).unwrap() + } + } + + impl std::fmt::Display for Repo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } + } + + impl std::str::FromStr for Repo { + type Err = RepoParseError; + + fn from_str(s: &str) -> Result { + let parts: Vec<&str> = s.split('/').collect(); + + let [owner, name] = parts.as_slice() else { + return Err(RepoParseError::InvalidFormat { + input: s.to_string(), + }); + }; + + if owner.is_empty() { + return Err(RepoParseError::EmptyOwner { + input: s.to_string(), + }); + } + if name.is_empty() { + return Err(RepoParseError::EmptyName { + input: s.to_string(), + }); + } + + Ok(Repo(s.to_string())) + } + } + + /// Errors that can occur when parsing a repository string + #[derive(Debug, thiserror::Error)] + pub enum RepoParseError { + /// Repository string does not contain exactly one '/' separator + #[error("Invalid repository format '{input}'. Expected format: owner/name")] + InvalidFormat { input: String }, + + /// Owner component is empty + #[error("Empty owner in repository '{input}'. Expected format: owner/name")] + EmptyOwner { input: String }, + + /// Repository name component is empty + #[error("Empty repository name in '{input}'. Expected format: owner/name")] + EmptyName { input: String }, + } + + /// Release information from GitHub API + #[derive(Debug, Clone, serde::Deserialize)] + pub struct Release { + /// Release name (display title) + pub name: String, + /// Git tag name (canonical version identifier) + pub tag_name: String, + /// List of release assets + pub assets: Vec, + } + + /// Release asset metadata from GitHub API + #[derive(Debug, Clone, serde::Deserialize)] + pub struct Asset { + /// Asset filename + pub name: String, + /// Browser download URL + pub browser_download_url: String, + /// File size in bytes + pub size: u64, + /// SHA-256 digest in format "sha256:abc123..." + /// + /// Note: GitHub API provides this field automatically. + /// If missing, asset cannot be included in manifest. + #[serde(default)] + pub digest: Option, + } +} diff --git a/crates/bin/ampup/Cargo.toml b/crates/bin/ampup/Cargo.toml index 923aa19fc..7ee71fd53 100644 --- a/crates/bin/ampup/Cargo.toml +++ b/crates/bin/ampup/Cargo.toml @@ -15,17 +15,20 @@ path = "src/main.rs" [dependencies] anyhow.workspace = true +chrono.workspace = true clap.workspace = true +console = "0.15" +dialoguer = "0.11" fs-err.workspace = true futures.workspace = true +indicatif = "0.17" reqwest.workspace = true semver.workspace = true serde.workspace = true +serde_json.workspace = true tempfile.workspace = true +thiserror.workspace = true tokio.workspace = true -indicatif = "0.17" -console = "0.15" -dialoguer = "0.11" [build-dependencies] vergen-gitcl = { version = "1.0.8", features = ["build"] } diff --git a/crates/bin/ampup/docs/manifest.md b/crates/bin/ampup/docs/manifest.md new file mode 100644 index 000000000..5bba91dcb --- /dev/null +++ b/crates/bin/ampup/docs/manifest.md @@ -0,0 +1,225 @@ +# Amp Distribution Manifest + +## Purpose + +The Amp distribution manifest is a JSON file that enables `ampup` to discover, download, and manage Amp components across different platforms. It serves as the single source of truth for: + +- Available Amp components (automatically discovered from release assets) +- Component versions and build metadata +- Platform-specific binary downloads (auto-discovered per component) +- Binary integrity verification (SHA256 checksums) +- Platform-specific metadata (notarization status for macOS) + +For the complete manifest structure definition, see the [JSON Schema](manifest.schema.json). + +## How It Works + +The manifest enables a standard discovery and install flow: + +1. **Discovery**: `ampup` fetches the manifest from a well-known distribution URL +2. **Platform Detection**: `ampup` identifies the host platform (target triple) +3. **Component Selection**: User selects which component to install +4. **Binary Lookup**: `ampup` locates the appropriate binary URL for the host platform +5. **Download**: `ampup` downloads the binary from the URL +6. **Verification**: `ampup` verifies the SHA256 checksum matches the manifest +7. **Installation**: `ampup` installs the verified binary to the appropriate location + +This architecture allows Amp to distribute pre-built binaries for multiple platforms while ensuring integrity and authenticity through cryptographic verification. + +## Key Features + +### Platform Support + +The manifest supports multiple operating systems, ABIs, and architectures: + +- **Linux**: GNU libc (glibc) and musl libc variants +- **macOS**: Intel and Apple Silicon (automatically marked as notarized) +- **Windows**: MSVC and GNU (MinGW) toolchains + +Each platform has its own binary URL, checksum, size, and platform-specific metadata. + +### Integrity Verification + +Every binary includes a SHA256 checksum in the format `sha256:` that `ampup` verifies after download. This prevents: + +- Corrupted downloads from network issues +- Tampering or man-in-the-middle attacks +- Accidental installation of incorrect binaries + +The checksum format matches GitHub's digest field format, ensuring consistency with the source of truth. + +**Security requirement**: All manifest URLs must use HTTPS, and `ampup` must verify checksums before executing any downloaded binary. + +### macOS Notarization + +macOS binaries distributed through the Amp release process are automatically marked with `"notarized": true` in the manifest. This indicates that: + +- The binary has been code-signed and notarized by Apple +- The binary can run on macOS without Gatekeeper warnings +- Users can verify the binary's authenticity through macOS security features + +Non-macOS platforms omit the `notarized` field (default: `false`). + +### Version Management + +The manifest tracks: + +- **Workspace version**: Overall Amp distribution version +- **Component versions**: Individual component versions (typically match workspace version) +- **Manifest version**: Schema version for backward compatibility +- **Build timestamp**: When the binaries were built (ISO 8601 UTC timestamp) + +This enables `ampup` to compare installed versions with available versions and download only what's needed. + +## Manifest Distribution + +Manifests are hosted at a well-known URL endpoint and generated automatically during the CI/CD build process: + +1. **Build binaries** for all target platforms +2. **Upload binaries** to GitHub Releases +3. **Auto-discover components** by parsing release asset names +4. **Extract metadata** from GitHub API (checksums via digest field, file sizes, download URLs) +5. **Generate manifest** JSON with auto-discovered components and targets +6. **Validate manifest** against JSON Schema +7. **Publish manifest** to GitHub Release + +The manifest format is versioned (`manifest_version` field) to support future schema evolution while maintaining backward compatibility. + +## Component Discovery + +The manifest generation is **component-agnostic** and automatically discovers components from release assets: + +### Asset Naming Convention + +Release assets must follow one of these naming patterns: + +**Standard format (3-part):** + +``` +-- +``` + +**Extended format (4-part with explicit ABI):** + +``` +--- +``` + +#### Supported Components + +- **component**: Lowercase alphanumeric with hyphens, must start with a letter (e.g., `ampd`, `ampctl`, `ampup-gen-release-manifest`) + +#### Supported OS Values + +**Bare OS names (default to common ABI):** + +- `linux` → defaults to GNU libc (`linux-gnu`) +- `darwin` → macOS + +**Explicit OS+ABI combinations:** + +- `linux-gnu` → Linux with GNU libc (glibc) +- `linux-musl` → Linux with musl libc (static linking) +- `windows-msvc` → Windows with MSVC toolchain +- `windows-gnu` → Windows with GNU toolchain (MinGW) + +#### Supported Architecture Values + +The following architecture names are recognized and normalized: + +- `x86_64` or `amd64` → normalized to `x86_64` +- `aarch64` or `arm64` → normalized to `aarch64` + +**Note**: The manifest generator automatically normalizes alternative architecture names to their canonical forms. + +### Asset Naming Examples + +**Linux (GNU libc):** + +- `ampd-linux-x86_64` → component: `ampd`, target: `x86_64-unknown-linux-gnu` +- `ampctl-linux-aarch64` → component: `ampctl`, target: `aarch64-unknown-linux-gnu` + +**Linux (musl libc, static):** + +- `ampd-linux-musl-x86_64` → component: `ampd`, target: `x86_64-unknown-linux-musl` +- `ampctl-linux-musl-aarch64` → component: `ampctl`, target: `aarch64-unknown-linux-musl` + +**macOS (Darwin):** + +- `ampd-darwin-x86_64` → component: `ampd`, target: `x86_64-apple-darwin`, notarized: `true` +- `ampctl-darwin-aarch64` → component: `ampctl`, target: `aarch64-apple-darwin`, notarized: `true` + +**Windows (MSVC):** + +- `ampd-windows-msvc-x86_64` → component: `ampd`, target: `x86_64-pc-windows-msvc` +- `ampctl-windows-msvc-aarch64` → component: `ampctl`, target: `aarch64-pc-windows-msvc` + +**Windows (GNU/MinGW):** + +- `ampd-windows-gnu-x86_64` → component: `ampd`, target: `x86_64-pc-windows-gnu` +- `ampctl-windows-gnu-aarch64` → component: `ampctl`, target: `aarch64-pc-windows-gnullvm` + +**Architecture normalization:** + +- `ampup-linux-arm64` → component: `ampup`, target: `aarch64-unknown-linux-gnu` (arm64 normalized to aarch64) +- `ampd-darwin-amd64` → component: `ampd`, target: `x86_64-apple-darwin` (amd64 normalized to x86_64) + +### Auto-Discovery Algorithm + +1. **Scan all release assets** from GitHub API +2. **Parse asset names** matching supported patterns: + - Standard (3-part): `^([a-z][a-z0-9-]+)-(linux|darwin|windows)-(x86_64|aarch64|arm64|amd64)$` + - Extended (4-part): `^([a-z][a-z0-9-]+)-(linux|windows)-(gnu|musl|msvc)-(x86_64|aarch64|arm64|amd64)$` +3. **Extract component name** from first capture group +4. **Parse OS and ABI** (if present) to determine target environment +5. **Normalize architecture** (arm64→aarch64, amd64→x86_64) +6. **Map to Rust target triple** (e.g., `linux-musl-x86_64` → `x86_64-unknown-linux-musl`) +7. **Group by component** and collect all targets +8. **Generate component entries** with version, targets, and metadata + +### Target Triple Mapping + +The manifest generator maps asset names to Rust target triples following standard conventions: + +- Linux: `{arch}-unknown-linux-{abi}` (e.g., `x86_64-unknown-linux-gnu`) +- macOS: `{arch}-apple-darwin` (e.g., `aarch64-apple-darwin`) +- Windows: `{arch}-pc-windows-{abi}` (e.g., `x86_64-pc-windows-msvc`) + +### Platform Support + +Components automatically support any target that has a corresponding release asset: + +- **Available target**: Asset exists → included in manifest +- **Missing target**: Asset missing → omitted from manifest (platform not supported) + +**Client behavior**: If `ampup` cannot find a target for the current platform, it reports: + +``` +Error: Platform 'powerpc-unknown-linux-gnu' not supported for component 'ampd' +Available targets: x86_64-unknown-linux-gnu, aarch64-unknown-linux-gnu, x86_64-apple-darwin, aarch64-apple-darwin +``` + +### Adding New Components + +To add a new component to the distribution: + +1. **Build the binary** for desired target platforms +2. **Name the assets** following one of the supported patterns: + - Standard: `--` + - Extended: `---` +3. **Upload to GitHub Release** - the manifest generator will automatically discover and include the component + +**Example**: To add `ampctl` component with Linux musl and macOS support: + +```bash +# Build and name your binaries +ampctl-linux-musl-x86_64 +ampctl-linux-musl-aarch64 +ampctl-darwin-x86_64 +ampctl-darwin-aarch64 + +# Upload to GitHub release +# The manifest generator will automatically create the component entry +``` + +No code changes required - the manifest generation is fully automatic! diff --git a/crates/bin/ampup/src/lib.rs b/crates/bin/ampup/src/lib.rs index cea1ee0cb..b89eb7c6e 100644 --- a/crates/bin/ampup/src/lib.rs +++ b/crates/bin/ampup/src/lib.rs @@ -3,6 +3,7 @@ pub mod commands; pub mod config; pub mod github; pub mod install; +pub mod manifest; pub mod platform; pub mod shell; pub mod updater; diff --git a/crates/bin/ampup/src/manifest.rs b/crates/bin/ampup/src/manifest.rs new file mode 100644 index 000000000..284b96b22 --- /dev/null +++ b/crates/bin/ampup/src/manifest.rs @@ -0,0 +1,425 @@ +//! Release manifest data types module +//! +//! This module provides data types for Amp distribution manifests. + +use std::collections::BTreeMap; + +/// Current manifest format version +pub const MANIFEST_VERSION: &str = "1"; + +/// Complete release manifest structure matching JSON schema +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Manifest { + /// Manifest format version (always "1") + #[serde(default = "default_manifest_version")] + pub manifest_version: String, + /// ISO 8601 timestamp when manifest was generated + pub date: String, + /// Release version string (e.g., "1.0.0") + pub version: String, + /// GitHub repository metadata + pub repository: Repository, + /// Map of component names to their platform-specific builds + pub components: BTreeMap, +} + +fn default_manifest_version() -> String { + MANIFEST_VERSION.to_string() +} + +/// GitHub repository metadata +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Repository { + /// GitHub repository owner (user or organization) + pub owner: String, + /// Repository name + pub name: String, + /// Full repository URL + pub url: String, +} + +/// Component with version and platform targets +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Component { + /// Version string for this component + pub version: String, + /// Map of Rust target triples to binary downloads + pub targets: BTreeMap, +} + +/// Platform-specific binary target +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Target { + /// Download URL for the binary + pub url: String, + /// SHA-256 hash with "sha256:" prefix + pub hash: String, + /// File size in bytes + pub size: u64, + /// Whether binary is notarized (macOS only). Defaults to false if not present. + #[serde(default, skip_serializing_if = "is_false")] + pub notarized: bool, +} + +/// Helper function to check if notarized is false for skip_serializing_if +fn is_false(value: &bool) -> bool { + !value +} + +/// Parsed asset name with validated components +/// +/// Format: `--` +/// - component: lowercase alphanumeric with hyphens (e.g., "ampd", "ampctl") +/// - os: "linux" or "darwin" +/// - arch: "x86_64", "aarch64", or "arm64" (normalized to aarch64) +/// +/// Tuple struct: `(component, os, arch)` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AssetName(String, AssetOs, AssetArch); + +impl std::str::FromStr for AssetName { + type Err = ParseAssetNameError; + + fn from_str(s: &str) -> Result { + // Parse formats: + // - Legacy: -- (e.g., "ampd-linux-x86_64") + // - New: --- (e.g., "ampd-linux-musl-x86_64", "ampd-windows-msvc-x86_64") + // + // Strategy: Extract arch from right, then try parsing OS with 2 parts, falling back to 1 part. + // This handles both "linux" (1 part) and "linux-musl" (2 parts) OS specifications. + + // Extract architecture (rightmost segment) + let arch_split = s.rfind('-').ok_or(ParseAssetNameError::InvalidFormat)?; + let arch_str = &s[arch_split + 1..]; + let remaining = &s[..arch_split]; + + // Try parsing as 4-part name first (component-os-abi-arch) + // Find the second-to-last hyphen to potentially extract a 2-part OS + if let Some(abi_split) = remaining.rfind('-') { + let potential_abi = &remaining[abi_split + 1..]; + let before_abi = &remaining[..abi_split]; + + // Check if we have another hyphen for the OS + if let Some(os_split) = before_abi.rfind('-') { + let potential_os = &before_abi[os_split + 1..]; + let os_abi_str = format!("{}-{}", potential_os, potential_abi); + + // Try parsing as a 2-part OS (e.g., "linux-musl", "windows-msvc") + if let Ok(os) = os_abi_str.parse::() { + let component = &before_abi[..os_split]; + + // Validate component + if !component.is_empty() + && component + .chars() + .next() + .is_some_and(|c| c.is_ascii_lowercase()) + && component + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') + { + let arch = arch_str.parse().map_err(ParseAssetNameError::InvalidArch)?; + return Ok(AssetName(component.to_string(), os, arch)); + } + } + } + } + + // Fall back to 3-part name (component-os-arch) + let os_split = remaining + .rfind('-') + .ok_or(ParseAssetNameError::InvalidFormat)?; + let os_str = &remaining[os_split + 1..]; + let component = &remaining[..os_split]; + + // Validate component: must start with lowercase letter, contain only lowercase/digits/hyphens + if component.is_empty() { + return Err(ParseAssetNameError::InvalidFormat); + } + + let first_char = component + .chars() + .next() + .ok_or(ParseAssetNameError::InvalidFormat)?; + if !first_char.is_ascii_lowercase() { + return Err(ParseAssetNameError::InvalidFormat); + } + + if !component + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') + { + return Err(ParseAssetNameError::InvalidFormat); + } + + let os = os_str.parse().map_err(ParseAssetNameError::InvalidOs)?; + let arch = arch_str.parse().map_err(ParseAssetNameError::InvalidArch)?; + + Ok(AssetName(component.to_string(), os, arch)) + } +} + +impl AssetName { + /// Returns the component name (e.g., "ampd", "ampctl") + pub fn component(&self) -> &str { + &self.0 + } + + /// Returns the operating system with ABI + pub fn os(&self) -> AssetOs { + self.1 + } + + /// Returns the CPU architecture + pub fn arch(&self) -> AssetArch { + self.2 + } + + /// Returns the Rust target triple (e.g., "x86_64-unknown-linux-gnu") + /// + /// This method generates the complete target triple by combining: + /// - Architecture (from self.arch()) + /// - Vendor (inferred from OS) + /// - OS and ABI (from self.os()) + /// + /// Special case: Windows GNU on ARM64 uses "gnullvm" instead of "gnu" + pub fn target(&self) -> String { + let arch = self.2.as_str(); + + match (self.1, self.2) { + // Linux targets + (AssetOs::LinuxGnu, _) => format!("{}-unknown-linux-gnu", arch), + (AssetOs::LinuxMusl, _) => format!("{}-unknown-linux-musl", arch), + + // macOS targets + (AssetOs::Darwin, _) => format!("{}-apple-darwin", arch), + + // Windows MSVC targets + (AssetOs::WindowsMsvc, _) => format!("{}-pc-windows-msvc", arch), + + // Windows GNU targets + (AssetOs::WindowsGnu, AssetArch::X86_64) => format!("{}-pc-windows-gnu", arch), + (AssetOs::WindowsGnu, AssetArch::Aarch64) => format!("{}-pc-windows-gnullvm", arch), + } + } + + /// Returns the target vendor component (e.g., "unknown", "pc", "apple") + /// + /// This is useful for tools that need to parse or manipulate target triples. + pub fn vendor(&self) -> &'static str { + match self.1 { + AssetOs::LinuxGnu | AssetOs::LinuxMusl => "unknown", + AssetOs::Darwin => "apple", + AssetOs::WindowsMsvc | AssetOs::WindowsGnu => "pc", + } + } + + /// Returns the target OS and ABI suffix (e.g., "linux-gnu", "windows-msvc") + /// + /// Special case: Windows GNU on ARM64 returns "windows-gnullvm" + pub fn os_abi(&self) -> &'static str { + match (self.1, self.2) { + (AssetOs::LinuxGnu, _) => "linux-gnu", + (AssetOs::LinuxMusl, _) => "linux-musl", + (AssetOs::Darwin, _) => "darwin", + (AssetOs::WindowsMsvc, _) => "windows-msvc", + (AssetOs::WindowsGnu, AssetArch::X86_64) => "windows-gnu", + (AssetOs::WindowsGnu, AssetArch::Aarch64) => "windows-gnullvm", + } + } + + /// Returns true if this is a macOS (apple-darwin) target + pub fn is_macos(&self) -> bool { + self.1.is_macos() + } + + /// Returns true if this is a Windows target + pub fn is_windows(&self) -> bool { + self.1.is_windows() + } + + /// Returns true if this is a Linux target + pub fn is_linux(&self) -> bool { + self.1.is_linux() + } + + /// Returns true if this is a statically-linked binary (musl) + pub fn is_static(&self) -> bool { + self.1.is_static() + } +} + +/// CPU architecture enumeration +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AssetArch { + /// x86_64 (AMD64) architecture + X86_64, + /// ARM64/AArch64 architecture + Aarch64, +} + +impl AssetArch { + /// Returns the canonical string representation + pub fn as_str(&self) -> &'static str { + match self { + AssetArch::X86_64 => "x86_64", + AssetArch::Aarch64 => "aarch64", + } + } +} + +impl std::str::FromStr for AssetArch { + type Err = ParseAssetArchError; + + fn from_str(s: &str) -> Result { + match s { + "x86_64" | "amd64" => Ok(AssetArch::X86_64), // Normalize amd64 → x86_64 + "aarch64" | "arm64" => Ok(AssetArch::Aarch64), // Normalize arm64 → aarch64 + _ => Err(ParseAssetArchError(s.to_string())), + } + } +} + +/// Operating system with ABI/libc variant +/// +/// Combines operating system with its C library or ABI variant to fully +/// specify the target platform. This maps directly to Rust target triple +/// OS+ABI components. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AssetOs { + /// Linux with GNU libc (glibc) - dynamically linked + LinuxGnu, + /// Linux with musl libc - statically linked + LinuxMusl, + /// macOS (Darwin) with system libraries + Darwin, + /// Windows with MSVC toolchain + WindowsMsvc, + /// Windows with GNU toolchain (MinGW) + WindowsGnu, +} + +impl AssetOs { + /// Returns the canonical string representation (for display/debugging) + pub fn as_str(&self) -> &'static str { + match self { + AssetOs::LinuxGnu => "linux-gnu", + AssetOs::LinuxMusl => "linux-musl", + AssetOs::Darwin => "darwin", + AssetOs::WindowsMsvc => "windows-msvc", + AssetOs::WindowsGnu => "windows-gnu", + } + } + + /// Returns true if this is a statically-linked target + pub fn is_static(&self) -> bool { + matches!(self, AssetOs::LinuxMusl) + } + + /// Returns true if this is a macOS target + pub fn is_macos(&self) -> bool { + matches!(self, AssetOs::Darwin) + } + + /// Returns true if this is a Windows target + pub fn is_windows(&self) -> bool { + matches!(self, AssetOs::WindowsMsvc | AssetOs::WindowsGnu) + } + + /// Returns true if this is a Linux target + pub fn is_linux(&self) -> bool { + matches!(self, AssetOs::LinuxGnu | AssetOs::LinuxMusl) + } +} + +impl std::str::FromStr for AssetOs { + type Err = ParseAssetOsError; + + fn from_str(s: &str) -> Result { + match s { + // Backward compatibility: bare OS names default to common ABI + "linux" => Ok(AssetOs::LinuxGnu), + "darwin" => Ok(AssetOs::Darwin), + + // Explicit OS+ABI combinations + "linux-gnu" => Ok(AssetOs::LinuxGnu), + "linux-musl" => Ok(AssetOs::LinuxMusl), + "windows-msvc" => Ok(AssetOs::WindowsMsvc), + "windows-gnu" => Ok(AssetOs::WindowsGnu), + + _ => Err(ParseAssetOsError(s.to_string())), + } + } +} + +/// Errors that occur when parsing asset names +/// +/// Asset names follow the format `-[-]-` where: +/// - component: lowercase alphanumeric with hyphens (e.g., "ampd", "ampctl") +/// - os: Operating system, optionally with ABI (e.g., "linux", "linux-musl", "windows-msvc") +/// - arch: CPU architecture - "x86_64" or "aarch64" (or "arm64" which normalizes to "aarch64") +/// +/// # Supported Formats +/// +/// Legacy (backward compatible): +/// - `ampd-linux-x86_64` → defaults to GNU libc +/// - `ampd-darwin-aarch64` → macOS on Apple Silicon +/// +/// Explicit OS+ABI: +/// - `ampd-linux-musl-x86_64` → static Linux binary +/// - `ampd-windows-msvc-x86_64` → Windows with MSVC +#[derive(Debug, thiserror::Error)] +pub enum ParseAssetNameError { + /// Asset name does not match the expected format + /// + /// This occurs when the asset name cannot be parsed into valid components + /// separated by hyphens, or when the component name doesn't meet validation + /// requirements. + /// + /// Common causes: + /// - Missing hyphens (e.g., "ampdlinuxx86_64") + /// - Too few components (need at least component-os-arch) + /// - Component name is empty + /// - Component doesn't start with a lowercase letter + /// - Component contains invalid characters (only lowercase, digits, hyphens allowed) + #[error("invalid asset name format: expected '-[-]-'")] + InvalidFormat, + + /// Invalid operating system name in asset filename + /// + /// This occurs when the OS portion is not a recognized OS or OS+ABI combination. + /// See `ParseAssetOsError` for the list of supported values. + #[error("invalid OS in asset name")] + InvalidOs(#[source] ParseAssetOsError), + + /// Invalid CPU architecture name in asset filename + /// + /// This occurs when the architecture portion is not "x86_64", "aarch64", or "arm64". + /// Note: "arm64" is automatically normalized to "aarch64". + #[error("invalid architecture in asset name")] + InvalidArch(#[source] ParseAssetArchError), +} + +/// Error when parsing operating system name +/// +/// Occurs when attempting to parse a string that is not a recognized OS+ABI combination. +/// +/// Supported formats: +/// - Bare OS (backward compat): "linux", "darwin" +/// - OS with ABI: "linux-gnu", "linux-musl", "windows-msvc", "windows-gnu" +#[derive(Debug, thiserror::Error)] +#[error( + "invalid OS: '{0}' (expected 'linux', 'linux-gnu', 'linux-musl', 'darwin', 'windows-msvc', or 'windows-gnu')" +)] +pub struct ParseAssetOsError(String); + +/// Error when parsing CPU architecture name +/// +/// Occurs when attempting to parse a string that is not a recognized architecture. +/// +/// Supported architecture strings: +/// - "x86_64" or "amd64" (automatically normalized to "x86_64") +/// - "aarch64" or "arm64" (automatically normalized to "aarch64") +#[derive(Debug, thiserror::Error)] +#[error("invalid architecture: '{0}' (expected 'x86_64', 'amd64', 'aarch64', or 'arm64')")] +pub struct ParseAssetArchError(String); diff --git a/crates/core/monitoring/src/logging.rs b/crates/core/monitoring/src/logging.rs index 8be5faa25..4a02a3480 100644 --- a/crates/core/monitoring/src/logging.rs +++ b/crates/core/monitoring/src/logging.rs @@ -72,6 +72,7 @@ const AMP_CRATES: &[&str] = &[ "ampd", "ampsync", "ampup", + "ampup_gen_release_manifest", "arrow_to_postgres", "auth_http", "common",