From d3c76b3a9c078a0fe4e3e5f8f7aecf618549bb21 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 15 Aug 2024 00:26:45 -0700 Subject: [PATCH] chore: install workflow-runner via axo (#461) --- Cargo.lock | 1 - crates/cli/Cargo.toml | 1 - crates/cli/src/commands/install.rs | 28 +- crates/cli/src/updater.rs | 275 +----------------- crates/cli/src/utils.rs | 61 +--- .../core/src/foreign_function_definition.rs | 13 +- crates/core/src/marzano_resolved_pattern.rs | 8 +- .../grit-pattern-matcher/src/pattern/not.rs | 5 +- 8 files changed, 31 insertions(+), 361 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac92481e6..94f93bcff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2148,7 +2148,6 @@ dependencies = [ "env_logger", "flate2", "fs-err", - "futures-util", "git2", "grit-pattern-matcher", "grit-util", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 16e40cd33..76146549b 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -26,7 +26,6 @@ uuid = { version = "1.1", features = ["v4", "serde"] } tokio = { version = "1", features = ["full"] } chrono = { version = "0.4.26", features = ["serde"] } reqwest = { version = "0.11", features = ["json", "stream"] } -futures-util = { version = "0.3.30" } lazy_static = { version = "1.4.0" } indicatif-log-bridge = { version = "0.2.1" } colored = { version = "2.0.4" } diff --git a/crates/cli/src/commands/install.rs b/crates/cli/src/commands/install.rs index 60c4e4fed..31df5e491 100644 --- a/crates/cli/src/commands/install.rs +++ b/crates/cli/src/commands/install.rs @@ -1,12 +1,9 @@ -use anyhow::Result; +use anyhow::{Result}; use clap::Args; use log::info; use serde::Serialize; -use crate::{ - updater::{SupportedApp, Updater}, - utils::{get_client_arch, get_client_os, Architecture, OperatingSystem}, -}; +use crate::updater::{SupportedApp, Updater}; #[derive(Args, Debug, Serialize)] pub struct InstallArgs { @@ -16,25 +13,12 @@ pub struct InstallArgs { /// Specify a specific app to install #[clap(long = "app")] app: Option, - /// Override the architecture to install - #[clap(long = "arch", hide = true)] - arch: Option, - /// Override the OS to install for - #[clap(long = "os", hide = true)] - os: Option, } pub(crate) async fn run_install(arg: InstallArgs) -> Result<()> { let should_update = arg.update; let mut updater = Updater::from_current_bin().await?; - let arch = arg - .arch - .map_or_else(|| get_client_arch().to_string(), |arch| arch.to_string()); - let os = arg - .os - .map_or_else(|| get_client_os().to_string(), |os| format!("{}", &os)); - info!( "Targeting {} as install directory", updater.install_path.display() @@ -45,13 +29,13 @@ pub(crate) async fn run_install(arg: InstallArgs) -> Result<()> { true => match should_update { true => { info!("{} already present, installing latest", app); - updater.install_latest(app, Some(&os), Some(&arch)).await?; + updater.install_latest(app).await?; } false => info!("{} already present, skipping", app), }, false => { info!("{} not present, installing", app); - updater.install_latest(app, Some(&os), Some(&arch)).await?; + updater.install_latest(app).await?; } } // TODO: output *only* the installed binary path to stdout @@ -63,14 +47,14 @@ pub(crate) async fn run_install(arg: InstallArgs) -> Result<()> { true => match should_update { true => { info!("{} already present, installing latest", app); - updater.install_latest(app, Some(&os), Some(&arch)).await?; + updater.install_latest(app).await?; } false => info!("{} already present, skipping", app), }, false => { if app.is_default_app() { info!("{} not present, installing", app); - updater.install_latest(app, Some(&os), Some(&arch)).await?; + updater.install_latest(app).await?; } else { info!("{app} not present, skipping, run with --app {app} to install",); } diff --git a/crates/cli/src/updater.rs b/crates/cli/src/updater.rs index eeaf7bf2a..bbd0e065a 100644 --- a/crates/cli/src/updater.rs +++ b/crates/cli/src/updater.rs @@ -3,13 +3,12 @@ use anyhow::{Context, Result}; use axoupdater::{AxoUpdater, ReleaseSource, ReleaseSourceType, Version}; use chrono::{DateTime, NaiveDateTime, Utc}; use colored::Colorize; -use futures_util::StreamExt; use indicatif::ProgressBar; use log::info; use marzano_auth::info::AuthInfo; use marzano_gritmodule::config::REPO_CONFIG_DIR_NAME; use marzano_util::runtime::{ExecutionContext, LanguageModelAPI}; -use reqwest::redirect::Policy; + use serde::Deserialize; use serde::Serialize; use std::collections::HashMap; @@ -19,11 +18,10 @@ use std::sync::Arc; use tokio::fs as async_fs; use tokio::fs::File; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::process::Command as AsyncCommand; + use tokio::sync::Mutex; use uuid::Uuid; -use crate::utils::{get_client_arch, get_client_os}; use marzano_auth::env::{get_env_auth, get_grit_api_url}; #[derive(Debug, Serialize, Deserialize, Clone)] @@ -48,7 +46,7 @@ impl fmt::Display for AllApp { AllApp::Timekeeper => write!(f, "timekeeper"), AllApp::Engine => write!(f, "engine"), AllApp::Yeast => write!(f, "yeast"), - AllApp::WorkflowRunner => write!(f, "workflow_runner"), + AllApp::WorkflowRunner => write!(f, "workflow-runner"), AllApp::Gouda => write!(f, "gouda"), } } @@ -85,7 +83,7 @@ impl SupportedApp { SupportedApp::Marzano => "marzano".to_string(), SupportedApp::Gouda => "gouda".to_string(), #[cfg(feature = "workflows_v2")] - SupportedApp::WorkflowRunner => "workflow_runner".to_string(), + SupportedApp::WorkflowRunner => "workflow-runner".to_string(), } } @@ -98,19 +96,6 @@ impl SupportedApp { } } - fn get_bin_name(&self) -> String { - format!("{}-{}", self.get_base_name(), get_client_os()) - } - - fn get_fallback_bin_name(&self) -> String { - self.get_base_name().to_string() - } - - fn get_file_name(&self, os: &str, arch: &str) -> String { - let base_name = self.get_base_name(); - format!("{}-{}-{}", base_name, os, arch) - } - pub fn from_all_app(app: AllApp) -> Option { match app { AllApp::Marzano => Some(SupportedApp::Marzano), @@ -365,43 +350,8 @@ impl Updater { Ok(()) } - pub async fn install_latest_internal( - &mut self, - app: SupportedApp, - os: Option<&str>, - arch: Option<&str>, - ) -> Result<()> { - // Look for updates - let (download_url, info_url) = get_release_url(app, os, arch).await?; - - info!("Starting download"); - // Download the artifact - let downloader = self.download_artifact(app, download_url); - let manifest_fetcher = fetch_manifest(&info_url, app); - let (downloaded, manifest) = tokio::try_join!(downloader, manifest_fetcher)?; - - // Unzip the artifact - self.unpack_artifact(app, downloaded.to_owned()).await?; - - // Clean up the temp artifact - async_fs::remove_file(&downloaded).await?; - - self.set_app_version(app, manifest.version.unwrap(), manifest.release.unwrap())?; - self.dump().await?; - - Ok(()) - } - - pub async fn install_latest( - &mut self, - app: SupportedApp, - os: Option<&str>, - arch: Option<&str>, - ) -> Result<()> { - match app { - SupportedApp::Marzano | SupportedApp::Gouda => self.install_latest_axo(app).await, - _ => self.install_latest_internal(app, os, arch).await, - } + pub async fn install_latest(&mut self, app: SupportedApp) -> Result<()> { + self.install_latest_axo(app).await } pub fn get_context(&self) -> Result { @@ -503,84 +453,6 @@ impl Updater { Ok(auth) } - async fn download_artifact(&self, app: SupportedApp, artifact_url: String) -> Result { - let target_path = self.bin_path.join(format!("{}-temp", app.get_bin_name())); - - match reqwest::get(&artifact_url).await { - Ok(response) => { - let mut file = async_fs::File::create(&target_path).await?; - let mut bytes_stream = response.bytes_stream(); - while let Some(chunk) = bytes_stream.next().await { - tokio::io::copy(&mut chunk?.as_ref(), &mut file).await?; - } - } - Err(e) => { - bail!("Failed to download artifact: {:?}", e); - } - } - - Ok(target_path) - } - - async fn unpack_artifact(&self, app: SupportedApp, packed_path: PathBuf) -> Result<()> { - let unpacked_dir = self.bin_path.join(format!("{}-bin", app.get_bin_name())); - // Create the subdir - async_fs::create_dir_all(&unpacked_dir).await?; - - info!( - "Unpacking from {} to {}", - packed_path.display(), - unpacked_dir.display() - ); - - let output = AsyncCommand::new("tar") - .arg("-xzf") - .arg(packed_path) - .arg("-C") - .arg(&unpacked_dir) - .output() - .await?; - - if !output.status.success() { - bail!("Failed to unpack files: {:?}", output); - } - - let target_path = self.get_app_bin(&app)?; - if async_fs::rename(unpacked_dir.join(app.get_bin_name()), &target_path) - .await - .is_err() - { - if let Err(e) = - async_fs::rename(unpacked_dir.join(app.get_fallback_bin_name()), &target_path).await - { - bail!("Failed to move files: {:?}", e); - } - } - - // Make the file executable - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - let target_file = fs_err::File::open(&target_path)?; - let mut perms = target_file.metadata()?.permissions(); - perms.set_mode(0o744); - if let Err(e) = target_file.set_permissions(perms) { - bail!( - "Failed to make {} executable: {:?}", - target_path.display(), - e - ); - } - - info!("Successfully made {} executable", target_path.display()); - } - - async_fs::remove_dir_all(&unpacked_dir).await?; - - Ok(()) - } - fn _get_app_manifest(&self, app: SupportedApp) -> Result<&AppManifest> { let app_string = app.get_base_name(); let app_manifest = self @@ -694,96 +566,14 @@ impl Updater { } } -async fn get_release_url( - app_name: SupportedApp, - os: Option<&str>, - arch: Option<&str>, -) -> Result<(String, String)> { - let client = reqwest::Client::builder() - .redirect(Policy::none()) - .build()?; - - let filename = app_name.get_file_name( - os.unwrap_or(get_client_os()), - arch.unwrap_or(get_client_arch()), - ); - - let url = format!( - "{}/v1/accounts/{}/artifacts/{}", - KEYGEN_API, KEYGEN_ACCOUNT, filename - ); - info!("Fetching release URL from: {}", url); - let res = client.get(&url).send().await?.text().await?; - - // Parse as JSON - let json_data: serde_json::Value = serde_json::from_str(&res)?; - - let latest_release_download_url = if let Some(artifact_data) = json_data["data"] - .get("links") - .and_then(|links| links.get("redirect")) - { - let artifact_url = artifact_data - .as_str() - .expect("Download URL should be a string"); - artifact_url - } else { - bail!("Could not find artifact download URL"); - }; - - let latest_release_info_url = if let Some(artifact_data) = json_data["data"] - .get("relationships") - .and_then(|relationships| relationships.get("release")) - .and_then(|release| release.get("links")) - .and_then(|links| links.get("related")) - { - let artifact_url = artifact_data - .as_str() - .expect("Release info URL should be a string"); - artifact_url - } else { - bail!("Could not find release info URL"); - }; - - Ok(( - latest_release_download_url.to_string(), - latest_release_info_url.to_string(), - )) -} - pub async fn check_release( app: SupportedApp, current_binary: &Option, ) -> Result> { - match app { - SupportedApp::Marzano | SupportedApp::Gouda => check_release_axo(app, current_binary).await, - _ => check_release_internal(app, current_binary).await, - } + check_release_axo(app, current_binary).await } -pub async fn check_release_internal( - app: SupportedApp, - current_binary: &Option, -) -> Result> { - let (_, info_url) = match get_release_url(app, None, None).await { - Ok(urls) => urls, - Err(_) => return Ok(None), - }; - let manifest = match fetch_manifest(&info_url, app).await { - Ok(manifest) => manifest, - Err(_) => return Ok(None), - }; - if let Some(current_manifest) = current_binary { - if manifest.release != current_manifest.release && manifest.release.is_some() { - Ok(manifest.version) - } else { - Ok(None) - } - } else { - Ok(None) - } -} - -pub async fn check_release_axo( +async fn check_release_axo( app: SupportedApp, current_binary: &Option, ) -> Result> { @@ -844,7 +634,6 @@ fn release_details_relative_url(release: &str) -> String { #[cfg(test)] mod tests { - use fs_err::create_dir_all; use anyhow::Result; use chrono::NaiveDate; @@ -853,18 +642,6 @@ mod tests { use super::*; - #[tokio::test] - async fn test_filenames() -> Result<()> { - let marzano = SupportedApp::Marzano; - - assert_eq!( - marzano.get_file_name("macos", "arm64"), - "marzano-macos-arm64" - ); - - Ok(()) - } - #[tokio::test] async fn test_basic_updater() -> Result<()> { let temp_dir = tempdir()?; @@ -939,38 +716,6 @@ mod tests { Ok(()) } - #[tokio::test] - async fn does_not_indicate_update_when_version_is_unknown() -> Result<()> { - let app = SupportedApp::Marzano; - let (_, info_url) = get_release_url(app, None, None).await?; - let manifest = fetch_manifest(&info_url, app).await?; - - let temp_manifest_path = tempdir().unwrap().path().join(MANIFEST_FILE); - create_dir_all(temp_manifest_path.parent().unwrap())?; - let mut manifest_file = File::create(&temp_manifest_path).await?; - let manifest_string = format!( - r#"{{ - "binaries": {{ - "marzano": {{ - "name": "marzano", - "release": "{}", - "version": null - }} - }}, - "installationId": "9a151548-26ee-45bd-a793-8b3d8d7f0f33" -}}"#, - manifest.release.unwrap() - ); - manifest_file.write_all(manifest_string.as_bytes()).await?; - - let mut updater = Updater::_from_manifest(temp_manifest_path).await?; - let has_update = updater.check_for_update().await?; - - assert!(!has_update); - - Ok(()) - } - #[tokio::test] #[ignore = "This test is too platform-specific"] async fn test_updates() -> Result<()> { @@ -985,9 +730,7 @@ mod tests { "744cc867-ae03-497f-b82a-ee6a4a57e90e".to_string(), )?; - updater - .install_latest(SupportedApp::Marzano, None, None) - .await?; + updater.install_latest(SupportedApp::Marzano).await?; let manifest = async_fs::read_to_string(updater.manifest_path.clone()).await?; diff --git a/crates/cli/src/utils.rs b/crates/cli/src/utils.rs index 0b0e6e1ca..381dac56e 100644 --- a/crates/cli/src/utils.rs +++ b/crates/cli/src/utils.rs @@ -1,69 +1,12 @@ -use clap::ValueEnum; + use git2::{Repository, StatusOptions}; use marzano_gritmodule::searcher::find_git_dir_from; -use serde::{Deserialize, Serialize}; + use std::{ - fmt::{Display, Formatter}, net::{SocketAddr, TcpListener}, path::PathBuf, }; -#[derive(Debug, PartialEq, PartialOrd, Clone, Copy, Serialize, Deserialize, ValueEnum)] -pub enum OperatingSystem { - #[serde(rename = "windows")] - Windows, - #[serde(rename = "linux")] - Linux, - #[serde(rename = "macos")] - MacOS, -} - -impl Display for OperatingSystem { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - OperatingSystem::Windows => write!(f, "windows"), - OperatingSystem::Linux => write!(f, "linux"), - OperatingSystem::MacOS => write!(f, "macos"), - } - } -} - -pub fn get_client_os() -> &'static str { - match std::env::consts::OS { - "macos" => "macos", - "linux" => "linux", - "windows" => "windows", - "darwin" => "macos", - _ => "linux", - } -} - -#[derive(Debug, PartialEq, PartialOrd, Clone, Copy, Serialize, Deserialize, ValueEnum)] -pub enum Architecture { - #[serde(rename = "x64")] - X64, - #[serde(rename = "arm64")] - Arm64, -} - -impl Display for Architecture { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - Architecture::X64 => write!(f, "x64"), - Architecture::Arm64 => write!(f, "arm64"), - } - } -} - -pub fn get_client_arch() -> &'static str { - match std::env::consts::ARCH { - "x86_64" => "x64", - "aarch64" => "arm64", - // Fall back to x64 - _ => "x64", - } -} - #[allow(dead_code)] pub fn get_random_port() -> Option { let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).ok()?; diff --git a/crates/core/src/foreign_function_definition.rs b/crates/core/src/foreign_function_definition.rs index a2f99da16..113b6a346 100644 --- a/crates/core/src/foreign_function_definition.rs +++ b/crates/core/src/foreign_function_definition.rs @@ -110,16 +110,17 @@ impl FunctionDefinition for ForeignFunctionDefinition { })?; #[cfg(feature = "external_functions")] - let result = function.call(&resolved_str).map_err(|e| GritPatternError::new(format!( - "failed to call function {}: {}", - self.name, e - )))?; + let result = function.call(&resolved_str).map_err(|e| { + GritPatternError::new(format!("failed to call function {}: {}", self.name, e)) + })?; // END embedded version - let string = String::from_utf8(result).map_err(|_| GritPatternError::new(format!( + let string = String::from_utf8(result).map_err(|_| { + GritPatternError::new(format!( "function {} returned did not return a UTF-8 string", self.name, - )))?; + )) + })?; Ok(FuncEvaluation { predicator: true, diff --git a/crates/core/src/marzano_resolved_pattern.rs b/crates/core/src/marzano_resolved_pattern.rs index fda9a077c..41ad66d03 100644 --- a/crates/core/src/marzano_resolved_pattern.rs +++ b/crates/core/src/marzano_resolved_pattern.rs @@ -696,11 +696,9 @@ impl<'a> ResolvedPattern<'a, MarzanoQueryContext> for MarzanoResolvedPattern<'a> ) .into()), // unsure if this is correct, taken from text - Self::Files(_files) => { - Err(GritPatternError::new( - "cannot linearize files pattern, not implemented yet", - )) - } + Self::Files(_files) => Err(GritPatternError::new( + "cannot linearize files pattern, not implemented yet", + )), // unsure if this is correct, taken from text Self::Constant(c) => Ok(c.to_string().into()), } diff --git a/crates/grit-pattern-matcher/src/pattern/not.rs b/crates/grit-pattern-matcher/src/pattern/not.rs index 03e7ce0cf..656651648 100644 --- a/crates/grit-pattern-matcher/src/pattern/not.rs +++ b/crates/grit-pattern-matcher/src/pattern/not.rs @@ -6,7 +6,10 @@ use super::{ }; use crate::context::QueryContext; use core::fmt::Debug; -use grit_util::{error::{GritPatternError, GritResult}, AnalysisLogs}; +use grit_util::{ + error::{GritPatternError, GritResult}, + AnalysisLogs, +}; #[derive(Debug, Clone)] pub struct Not {