From a39a278220ef557ea6eee5265f16eb8fcd1d2dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kav=C3=ADk?= Date: Tue, 27 Aug 2019 17:59:09 +0200 Subject: [PATCH] fix(command_test): stamps.rs check driver once per day, update RELEASE_CHECKLIST --- Cargo.lock | 21 ++++++ Cargo.toml | 2 + RELEASE_CHECKLIST.md | 4 + src/lib.rs | 1 + src/stamps.rs | 58 ++++++++++++++ src/test/webdriver.rs | 7 ++ src/test/webdriver/chromedriver.rs | 89 +++++++++++++++++++--- src/test/webdriver/geckodriver.rs | 117 ++++++++++++++++++++++++----- tests/all/main.rs | 3 + tests/all/stamps.rs | 71 +++++++++++++++++ 10 files changed, 345 insertions(+), 28 deletions(-) create mode 100644 src/stamps.rs create mode 100644 tests/all/stamps.rs diff --git a/Cargo.lock b/Cargo.lock index 8a27240..3b35219 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1505,6 +1505,23 @@ dependencies = [ "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serial_test" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serial_test_derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "siphasher" version = "0.2.3" @@ -1992,6 +2009,8 @@ dependencies = [ "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_ignored 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", + "serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "structopt 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2261,6 +2280,8 @@ dependencies = [ "checksum serde_ignored 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "190e9765dcedb56be63b6e0993a006c7e3b071a016a304736e4a315dc01fb142" "checksum serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d" "checksum serde_urlencoded 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d48f9f99cd749a2de71d29da5f948de7f2764cc5a9d7f3c97e3514d4ee6eabf2" +"checksum serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50bfbc39343545618d97869d77f38ed43e48dd77432717dbc7ed39d797f3ecbe" +"checksum serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "89dd85be2e2ad75b041c9df2892ac078fa6e0b90024028b2b9fb4125b7530f01" "checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" "checksum smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c4488ae950c49d403731982257768f48fada354a5203fe81f9bb6f43ca9002be" diff --git a/Cargo.toml b/Cargo.toml index d85d70f..f300722 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,8 @@ chrono = "0.4.6" assert_cmd = "0.11" lazy_static = "1.1.0" predicates = "1.0.0" +serial_test = "0.2" +serial_test_derive = "0.2" tempfile = "3" [features] diff --git a/RELEASE_CHECKLIST.md b/RELEASE_CHECKLIST.md index 4f38624..f6f8e53 100644 --- a/RELEASE_CHECKLIST.md +++ b/RELEASE_CHECKLIST.md @@ -17,6 +17,10 @@ This is a list of the things that need to happen during a release. 1. Create a new branch "#.#.#" where "#.#.#" is the release's version. 1. Add this release to the `CHANGELOG.md`. Use the structure of previous entries. +1. Update `DEFAULT_CHROMEDRIVER_VERSION` in `chromedriver.rs`. + Version is the response of `https://chromedriver.storage.googleapis.com/LATEST_RELEASE`. +1. Update `DEFAULT_GECKODRIVER_VERSION` in `geckodriver.rs`. + Version is the name of the latest tag - `https://github.com/mozilla/geckodriver/releases/latest`. 1. Update the version in `Cargo.toml`. 1. Update the version number and date in `docs/index.html`. 1. Run `cargo update`. diff --git a/src/lib.rs b/src/lib.rs index bfe7dd2..9e5f99f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,6 +40,7 @@ pub mod manifest; pub mod npm; pub mod progressbar; pub mod readme; +pub mod stamps; pub mod target; pub mod test; pub mod wasm_opt; diff --git a/src/stamps.rs b/src/stamps.rs new file mode 100644 index 0000000..4fee932 --- /dev/null +++ b/src/stamps.rs @@ -0,0 +1,58 @@ +//! Key-value store in `*.stamps` file. + +use failure::{self, ResultExt}; +use std::{env, fs, path::PathBuf}; + +/// Get a value corresponding to the key from the JSON value. +/// +/// You should use return value of function `read_stamps_file_to_json()` as `json` argument. +pub fn get_stamp_value( + key: impl AsRef, + json: &serde_json::Value, +) -> Result { + json.get(key.as_ref()) + .and_then(|value| value.as_str().map(ToOwned::to_owned)) + .ok_or_else(|| { + failure::err_msg(format!("cannot get stamp value for key '{}'", key.as_ref())) + }) +} + +/// Save the key-value pair to the store. +pub fn save_stamp_value( + key: impl Into, + value: impl AsRef, +) -> Result<(), failure::Error> { + let mut json = read_stamps_file_to_json().unwrap_or_else(|_| serde_json::Map::new().into()); + + let stamps = json + .as_object_mut() + .ok_or_else(|| failure::err_msg("stamps file doesn't contain JSON object"))?; + stamps.insert(key.into(), value.as_ref().into()); + + write_to_stamps_file(json) +} + +/// Get the path of the `*.stamps` file that is used as the store. +pub fn get_stamps_file_path() -> Result { + let path = env::current_exe() + .map(|path| path.with_extension("stamps")) + .context("cannot get stamps file path")?; + Ok(path) +} + +/// Read `*.stamps` file and convert its content to the JSON value. +pub fn read_stamps_file_to_json() -> Result { + let stamps_file_path = get_stamps_file_path()?; + let stamps_file_content = + fs::read_to_string(stamps_file_path).context("cannot find or read stamps file")?; + let json: serde_json::Value = serde_json::from_str(&stamps_file_content) + .context("stamps file doesn't contain valid JSON")?; + Ok(json) +} + +fn write_to_stamps_file(json: serde_json::Value) -> Result<(), failure::Error> { + let stamps_file_path = get_stamps_file_path()?; + let pretty_json = serde_json::to_string_pretty(&json).context("JSON serialization failed")?; + fs::write(stamps_file_path, pretty_json).context("cannot write to stamps file")?; + Ok(()) +} diff --git a/src/test/webdriver.rs b/src/test/webdriver.rs index b9619e1..7b4223d 100644 --- a/src/test/webdriver.rs +++ b/src/test/webdriver.rs @@ -37,6 +37,13 @@ fn get_and_notify( struct Collector(Vec); +impl Collector { + pub fn take_content(&mut self) -> Vec { + // TODO: replace with `std::mem::take` once stable + std::mem::replace(&mut self.0, Vec::default()) + } +} + impl curl::easy::Handler for Collector { fn write(&mut self, data: &[u8]) -> Result { self.0.extend_from_slice(data); diff --git a/src/test/webdriver/chromedriver.rs b/src/test/webdriver/chromedriver.rs index def432c..7924fa8 100644 --- a/src/test/webdriver/chromedriver.rs +++ b/src/test/webdriver/chromedriver.rs @@ -1,10 +1,19 @@ use super::{get_and_notify, Collector}; use binary_install::Cache; -use failure; +use chrono::DateTime; +use failure::{self, ResultExt}; use install::InstallMode; +use stamps; use std::path::PathBuf; use target; +// Keep it up to date with each `wasm-pack` release. +// https://chromedriver.storage.googleapis.com/LATEST_RELEASE +const DEFAULT_CHROMEDRIVER_VERSION: &str = "76.0.3809.126"; + +const CHROMEDRIVER_LAST_UPDATED_STAMP: &str = "chromedriver_last_updated"; +const CHROMEDRIVER_VERSION_STAMP: &str = "chromedriver_version"; + /// Get the path to an existing `chromedriver`, or install it if no existing /// binary is found or if there is a new binary version. pub fn get_or_install_chromedriver( @@ -32,7 +41,7 @@ pub fn install_chromedriver( bail!("chromedriver binaries are unavailable for this target") }; - let url = get_chromedriver_url(target)?; + let url = get_chromedriver_url(target); match get_and_notify(cache, installation_allowed, "chromedriver", &url)? { Some(path) => Ok(path), @@ -46,26 +55,86 @@ pub fn install_chromedriver( /// Get `chromedriver` download URL. /// +/// _Algorithm_: +/// 1. Try to open `*.stamps` file and deserialize its content to JSON object. +/// 2. Try to compare current time with the saved one. +/// 3. If the saved time is older than 1 day or something failed +/// => fetch a new version and save version & time. +/// 4. If everything failed, use the default version. +/// 5. Return URL. +/// +/// _Notes:_ +/// /// It returns the latest one without checking the installed `Chrome` version /// because it's not easy to find out `Chrome` version on `Windows` - /// https://bugs.chromium.org/p/chromium/issues/detail?id=158372 /// /// The official algorithm for `chromedriver` version selection: /// https://chromedriver.chromium.org/downloads/version-selection -fn get_chromedriver_url(target: &str) -> Result { - let chromedriver_version = fetch_chromedriver_version()?; - Ok(assemble_chromedriver_url(&chromedriver_version, target)) +fn get_chromedriver_url(target: &str) -> String { + let fetch_and_save_version = + || fetch_chromedriver_version().and_then(save_chromedriver_version); + + let chromedriver_version = match stamps::read_stamps_file_to_json() { + Ok(json) => { + if should_load_chromedriver_version_from_stamp(&json) { + stamps::get_stamp_value(CHROMEDRIVER_VERSION_STAMP, &json) + } else { + fetch_and_save_version() + } + } + Err(_) => fetch_and_save_version(), + } + .unwrap_or_else(|error| { + log::warn!( + "Cannot load or fetch chromedriver's latest version data, \ + the default version {} will be used. Error: {}", + DEFAULT_CHROMEDRIVER_VERSION, + error + ); + DEFAULT_CHROMEDRIVER_VERSION.to_owned() + }); + assemble_chromedriver_url(&chromedriver_version, target) +} + +// ------ `get_chromedriver_url` helpers ------ + +fn save_chromedriver_version(version: String) -> Result { + stamps::save_stamp_value(CHROMEDRIVER_VERSION_STAMP, &version)?; + + let current_time = chrono::offset::Local::now().to_rfc3339(); + stamps::save_stamp_value(CHROMEDRIVER_LAST_UPDATED_STAMP, current_time)?; + + Ok(version) } -// ------ `get_chromedriver_url` steps ------ +fn should_load_chromedriver_version_from_stamp(json: &serde_json::Value) -> bool { + let last_updated = stamps::get_stamp_value(CHROMEDRIVER_LAST_UPDATED_STAMP, json) + .ok() + .and_then(|last_updated| DateTime::parse_from_rfc3339(&last_updated).ok()); + + match last_updated { + None => false, + Some(last_updated) => { + let current_time = chrono::offset::Local::now(); + current_time.signed_duration_since(last_updated).num_hours() < 24 + } + } +} fn fetch_chromedriver_version() -> Result { let mut handle = curl::easy::Easy2::new(Collector(Vec::new())); - handle.url("https://chromedriver.storage.googleapis.com/LATEST_RELEASE")?; - handle.perform()?; + handle + .url("https://chromedriver.storage.googleapis.com/LATEST_RELEASE") + .context("URL to fetch chromedriver's LATEST_RELEASE is invalid")?; + handle + .perform() + .context("fetching of chromedriver's LATEST_RELEASE failed")?; - let contents = handle.get_ref(); - Ok(String::from_utf8_lossy(&contents.0).into_owned()) + let content = handle.get_mut().take_content(); + let version = + String::from_utf8(content).context("chromedriver's LATEST_RELEASE is not valid UTF-8")?; + Ok(version) } fn assemble_chromedriver_url(chromedriver_version: &str, target: &str) -> String { diff --git a/src/test/webdriver/geckodriver.rs b/src/test/webdriver/geckodriver.rs index 2a86871..6428cfc 100644 --- a/src/test/webdriver/geckodriver.rs +++ b/src/test/webdriver/geckodriver.rs @@ -1,10 +1,19 @@ use super::{get_and_notify, Collector}; use binary_install::Cache; -use failure; +use chrono::DateTime; +use failure::{self, ResultExt}; use install::InstallMode; +use stamps; use std::path::PathBuf; use target; +// Keep it up to date with each `wasm-pack` release. +// https://github.com/mozilla/geckodriver/releases/latest +const DEFAULT_GECKODRIVER_VERSION: &str = "v0.24.0"; + +const GECKODRIVER_LAST_UPDATED_STAMP: &str = "geckodriver_last_updated"; +const GECKODRIVER_VERSION_STAMP: &str = "geckodriver_version"; + /// Get the path to an existing `geckodriver`, or install it if no existing /// binary is found or if there is a new binary version. pub fn get_or_install_geckodriver( @@ -36,7 +45,7 @@ pub fn install_geckodriver( bail!("geckodriver binaries are unavailable for this target") }; - let url = get_geckodriver_url(target, ext)?; + let url = get_geckodriver_url(target, ext); match get_and_notify(cache, installation_allowed, "geckodriver", &url)? { Some(path) => Ok(path), @@ -49,38 +58,110 @@ pub fn install_geckodriver( /// Get `geckodriver` download URL. /// +/// _Algorithm_: +/// 1. Try to open `*.stamps` file and deserialize its content to JSON object. +/// 2. Try to compare current time with the saved one. +/// 3. If the saved time is older than 1 day or something failed +/// => fetch a new version and save version & time. +/// 4. If everything failed, use the default version. +/// 5. Return URL. +/// +/// _Notes:_ +/// /// It returns the latest one without checking the installed `Firefox` version /// - it should be relatively safe because each `geckodriver` supports many `Firefox` versions: /// https://firefox-source-docs.mozilla.org/testing/geckodriver/Support.html#supported-platforms -fn get_geckodriver_url(target: &str, ext: &str) -> Result { - // JSON example: `{"id":15227534,"tag_name":"v0.24.0","update_url":"/mozzila...` - let latest_tag_json = fetch_latest_geckodriver_tag_json()?; - let latest_tag = get_tag_name_from_json(&latest_tag_json)?; - Ok(assemble_geckodriver_url(&latest_tag, target, ext)) +fn get_geckodriver_url(target: &str, ext: &str) -> String { + let fetch_and_save_version = || { + fetch_latest_geckodriver_tag_json() + .and_then(get_version_from_json) + .and_then(save_geckodriver_version) + }; + + let geckodriver_version = match stamps::read_stamps_file_to_json() { + Ok(json) => { + if should_load_geckodriver_version_from_stamp(&json) { + stamps::get_stamp_value(GECKODRIVER_VERSION_STAMP, &json) + } else { + fetch_and_save_version() + } + } + Err(_) => fetch_and_save_version(), + } + .unwrap_or_else(|error| { + log::warn!( + "Cannot load or fetch geckodriver's latest version data, \ + the default version {} will be used. Error: {}", + DEFAULT_GECKODRIVER_VERSION, + error + ); + DEFAULT_GECKODRIVER_VERSION.to_owned() + }); + assemble_geckodriver_url(&geckodriver_version, target, ext) +} + +// ------ `get_geckodriver_url` helpers ------ + +fn save_geckodriver_version(version: String) -> Result { + stamps::save_stamp_value(GECKODRIVER_VERSION_STAMP, &version)?; + + let current_time = chrono::offset::Local::now().to_rfc3339(); + stamps::save_stamp_value(GECKODRIVER_LAST_UPDATED_STAMP, current_time)?; + + Ok(version) } -// ------ `get_geckodriver_url` steps ------ +fn should_load_geckodriver_version_from_stamp(json: &serde_json::Value) -> bool { + let last_updated = stamps::get_stamp_value(GECKODRIVER_LAST_UPDATED_STAMP, json) + .ok() + .and_then(|last_updated| DateTime::parse_from_rfc3339(&last_updated).ok()); + + match last_updated { + None => false, + Some(last_updated) => { + let current_time = chrono::offset::Local::now(); + current_time.signed_duration_since(last_updated).num_hours() < 24 + } + } +} fn fetch_latest_geckodriver_tag_json() -> Result { let mut headers = curl::easy::List::new(); - headers.append("Accept: application/json")?; + headers + .append("Accept: application/json") + .context("cannot fetch geckodriver's latest release data - appending header failed")?; let mut handle = curl::easy::Easy2::new(Collector(Vec::new())); - handle.url("https://github.com/mozilla/geckodriver/releases/latest")?; - handle.http_headers(headers)?; + handle + .url("https://github.com/mozilla/geckodriver/releases/latest") + .context("URL to fetch geckodriver's latest release data is invalid")?; + handle + .http_headers(headers) + .context("cannot fetch geckodriver's latest release data - setting headers failed")?; // We will be redirected from the `latest` placeholder to the specific tag name. - handle.follow_location(true)?; - handle.perform()?; + handle + .follow_location(true) + .context("cannot fetch geckodriver's latest release data - enabling redirects failed")?; + handle + .perform() + .context("fetching of geckodriver's latest release data failed")?; + + let content = handle.get_mut().take_content(); + let version = String::from_utf8(content) + .context("geckodriver's latest release data is not valid UTF-8")?; - let contents = handle.get_ref(); - Ok(String::from_utf8_lossy(&contents.0).into_owned()) + Ok(version) } -fn get_tag_name_from_json(json: &str) -> Result { - let json: serde_json::Value = serde_json::from_str(json)?; +/// JSON example: `{"id":15227534,"tag_name":"v0.24.0","update_url":"/mozzila...` +fn get_version_from_json(json: impl AsRef) -> Result { + let json: serde_json::Value = serde_json::from_str(json.as_ref()) + .context("geckodriver's latest release data is not valid JSON")?; json.get("tag_name") .and_then(|tag_name| tag_name.as_str().map(ToOwned::to_owned)) - .ok_or_else(|| failure::err_msg("cannot get `tag_name` from JSON response")) + .ok_or_else(|| { + failure::err_msg("cannot get `tag_name` from geckodriver's latest release data") + }) } fn assemble_geckodriver_url(tag: &str, target: &str, ext: &str) -> String { diff --git a/tests/all/main.rs b/tests/all/main.rs index 5abc58d..ac2b270 100644 --- a/tests/all/main.rs +++ b/tests/all/main.rs @@ -7,6 +7,8 @@ extern crate lazy_static; extern crate serde_derive; extern crate binary_install; extern crate serde_json; +#[macro_use] +extern crate serial_test_derive; extern crate structopt; extern crate tempfile; extern crate wasm_pack; @@ -18,6 +20,7 @@ mod license; mod lockfile; mod manifest; mod readme; +mod stamps; mod test; mod utils; mod wasm_opt; diff --git a/tests/all/stamps.rs b/tests/all/stamps.rs new file mode 100644 index 0000000..53e49b0 --- /dev/null +++ b/tests/all/stamps.rs @@ -0,0 +1,71 @@ +use std::{fs, panic}; +use wasm_pack::stamps; + +fn run_test(test: T) -> () +where + T: FnOnce() -> () + panic::UnwindSafe, +{ + before(); + let result = panic::catch_unwind(|| test()); + after(); + assert!(result.is_ok()) +} + +fn before() { + remove_stamps_file() +} + +fn after() { + remove_stamps_file() +} + +fn remove_stamps_file() { + let stamps_file_path = stamps::get_stamps_file_path().unwrap(); + if stamps_file_path.exists() { + fs::remove_file(stamps_file_path).unwrap(); + } +} + +#[test] +#[should_panic] +#[serial] +fn load_stamp_from_non_existent_file() { + run_test(|| { + // ACT + let json = stamps::read_stamps_file_to_json().unwrap(); + stamps::get_stamp_value("Foo", &json).unwrap(); + }) +} + +#[test] +#[serial] +fn load_stamp() { + run_test(|| { + // ARRANGE + stamps::save_stamp_value("Foo", "Bar").unwrap(); + + // ACT + let json = stamps::read_stamps_file_to_json().unwrap(); + let stamp_value = stamps::get_stamp_value("Foo", &json).unwrap(); + + // ASSERT + assert_eq!(stamp_value, "Bar"); + }) +} + +#[test] +#[serial] +fn update_stamp() { + run_test(|| { + // ARRANGE + stamps::save_stamp_value("Foo", "Bar").unwrap(); + + // ACT + stamps::save_stamp_value("Foo", "John").unwrap(); + + // ASSERT + let json = stamps::read_stamps_file_to_json().unwrap(); + let stamp_value = stamps::get_stamp_value("Foo", &json).unwrap(); + assert_eq!(stamp_value, "John"); + }) +}