diff --git a/Cargo.toml b/Cargo.toml index ae1af24..267825a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "experiments" +name = "nimbus_experiments" version = "0.1.0" authors = ["Tarik Eshaq "] edition = "2018" @@ -11,17 +11,24 @@ url = "2.1" serde = { version = "1", features = ["rc"] } serde_derive = "1" serde_json = "1" -# This now references a local path on my own device. If we end up actually using viaduct, -# We should maybe publish it as crate (similar to ffi-support) -viaduct = { path = "../application-services/components/viaduct" } anyhow = "1.0" rand = "0.7" log = "0.4" +viaduct = { git = "https://github.com/mozilla/application-services" } ffi-support = "0.4" thiserror = "1" +rkv = "0.10" +lazy_static = "1.4" +uuid = { version = "0.8", features = ["serde", "v4"]} +prost = "0.6" + +[build-dependencies] +prost-build = { version = "0.6" } + +[lib] +name = "nimbus_experiments" +crate-type = ["lib"] [dev-dependencies] -# This now references a local path on my own device. If we end up actually using viaduct, -# We should maybe publish it as crate (similar to ffi-support) -viaduct-reqwest = { path = "../application-services/components/support/viaduct-reqwest" } +viaduct-reqwest = { git = "https://github.com/mozilla/application-services" } diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..bd19228 --- /dev/null +++ b/build.rs @@ -0,0 +1,7 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +fn main() { + prost_build::compile_protos(&["src/experiments_msg_types.proto"], &["src/"]).unwrap(); +} diff --git a/examples/experiment.rs b/examples/experiment.rs index 01869d1..4c1db5a 100644 --- a/examples/experiment.rs +++ b/examples/experiment.rs @@ -1,12 +1,8 @@ use anyhow::Result; -use experiments::Experiments; +use nimbus_experiments::{AppContext, Experiments}; fn main() -> Result<()> { viaduct_reqwest::use_reqwest_backend(); - let exp = Experiments::new( - "https://kinto.dev.mozaws.net/v1/", - "default", - "messaging-collection", - ); + let exp = Experiments::new(AppContext::default(), "./mydb"); let enrolled_exp = exp.get_enrolled_experiments(); exp.get_experiments().iter().for_each(|e| { print!( diff --git a/ffi/Cargo.toml b/ffi/Cargo.toml deleted file mode 100644 index f803a8f..0000000 --- a/ffi/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "experiments-ffi" -version = "0.1.0" -authors = ["The Firefox Sync Developers "] -edition = "2018" -license = "MPL-2.0" - -[lib] -name = "experiments_ffi" -crate-type = ["lib"] - -[dependencies] -lazy_static = "1.4.0" -log = "0.4" -ffi-support = "0.4" -serde_json = "1" -anyhow = "1" -# This now references a local path on my own device. If we end up actually using viaduct, -# We should maybe publish it as crate (similar to ffi-support) -# This use case however is a hack, and we will probably do something else -viaduct = { path = "../../application-services/components/viaduct" } - -[dependencies.experiments] -path = ".." diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs deleted file mode 100644 index 5ba1397..0000000 --- a/ffi/src/lib.rs +++ /dev/null @@ -1,37 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - - use std::os::raw::c_char; - - use ffi_support::{define_handle_map_deleter, ConcurrentHandleMap, ExternError, FfiStr}; - use experiments::{Experiments, error::Result}; - // Hack to allow consumer to do all the viaduct magic -pub use viaduct::*; - - lazy_static::lazy_static! { - static ref EXPERIMENTS: ConcurrentHandleMap = ConcurrentHandleMap::new(); - } - - #[no_mangle] - pub extern "C" fn experiments_new(error: &mut ExternError, base_url: FfiStr<'_>, collection_name: FfiStr<'_>, bucket_name: FfiStr<'_>) -> u64 { - EXPERIMENTS.insert_with_result(error, || -> Result { - Ok(Experiments::new(base_url.as_str(), collection_name.as_str(), bucket_name.as_str())) - }) - } - - - #[no_mangle] - pub extern "C" fn experiements_get_branch( - handle: u64, - branch: FfiStr<'_>, - error: &mut ExternError, - ) -> *mut c_char { - EXPERIMENTS.call_with_result(error, handle, |experiment| -> Result { - let ret = experiment.get_experiment_branch(branch.as_str())?; - Ok(ret) - }) - } - - define_handle_map_deleter!(EXPERIMENTS, experiements_destroy); - \ No newline at end of file diff --git a/src/buckets.rs b/src/buckets.rs index 8751648..e8766d3 100644 --- a/src/buckets.rs +++ b/src/buckets.rs @@ -2,7 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - //! This might be where the bucketing logic can go //! It would be different from current experimentation tools //! There is a namespacing concept to allow users to be in multiple diff --git a/src/error.rs b/src/error.rs index 9766bd7..d64a406 100644 --- a/src/error.rs +++ b/src/error.rs @@ -2,8 +2,10 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -// Not implemented yet!!! -// This is purely boilerplate to communicate over the ffi +//! Not implemented yet!!! +//! This is purely boilerplate to communicate over the ffi +//! We should define real variants for our error and use proper +//! error propegation (we can use the `thiserror` crate for that) use ffi_support::{ErrorCode, ExternError}; #[derive(Debug, thiserror::Error)] pub enum Error { diff --git a/src/experiments.idl b/src/experiments.idl index 7793a56..823f66f 100644 --- a/src/experiments.idl +++ b/src/experiments.idl @@ -1,4 +1,7 @@ # This is a test file for defining WebIDL for uniffi +# For the time being, it is not used for anything! +# However, if we use uniffi in the future, we could define +# The api here. (Unless uniffi changes to a non WebIDL way (looking at you proc-macros)) namespace experiments {}; interface Experiments { constructor(); diff --git a/src/experiments_msg_types.proto b/src/experiments_msg_types.proto new file mode 100644 index 0000000..6af5140 --- /dev/null +++ b/src/experiments_msg_types.proto @@ -0,0 +1,24 @@ +syntax = "proto2"; + +// This kinda beats the purpose of using protobufs since we have one file here/ +// And a duplicate in the glean PR, but bear with me :) +// Eventually once we figure out the details of where each part lives, we'll merge the proto files +// into one + +package mozilla.telemetry.glean.protobuf; + +option java_package = "mozilla.telemtery.glean"; +option java_outer_classname = "MsgTypes"; +option swift_prefix = "MsgTypes_"; +option optimize_for = LITE_RUNTIME; + +message AppContext { + optional string app_id = 1; + optional string app_version = 2; + optional string locale_language = 3; + optional string locale_country = 4; + optional string device_manufacturer = 5; + optional string device_model = 6; + optional string region = 7; + optional string debug_tag = 8; +} \ No newline at end of file diff --git a/src/ffi.rs b/src/ffi.rs new file mode 100644 index 0000000..e12c7ff --- /dev/null +++ b/src/ffi.rs @@ -0,0 +1,69 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::os::raw::c_char; + +use super::{error::Result, msg_types, AppContext, Experiments}; +use ffi_support::{define_handle_map_deleter, ConcurrentHandleMap, ExternError, FfiStr}; + +lazy_static::lazy_static! { + static ref EXPERIMENTS: ConcurrentHandleMap = ConcurrentHandleMap::new(); +} + +#[no_mangle] +pub extern "C" fn experiments_new( + app_ctx: *const u8, + app_ctx_len: i32, + db_path: FfiStr<'_>, + error: &mut ExternError, +) -> u64 { + EXPERIMENTS.insert_with_result(error, || -> Result { + let app_ctx = unsafe { + from_protobuf_ptr::(app_ctx, app_ctx_len).unwrap() + }; // Todo: make the whole function unsafe and implement proper error handling in error.rs + log::info!("=================== Initializing experiments ========================"); + Ok(Experiments::new(app_ctx, db_path.as_str())) + }) +} + +#[no_mangle] +pub extern "C" fn experiments_get_branch( + handle: u64, + branch: FfiStr<'_>, + error: &mut ExternError, +) -> *mut c_char { + EXPERIMENTS.call_with_result(error, handle, |experiment| -> Result { + log::info!("==================== Getting branch ========================"); + let branch_name = experiment.get_experiment_branch(branch.as_str())?; + Ok(branch_name) + }) +} + +define_handle_map_deleter!(EXPERIMENTS, experiements_destroy); + +/// # Safety +/// data is a raw pointer to the protobuf data +/// get_buffer will return an error if the length is invalid, +/// or if the pointer is a null pointer +pub unsafe fn from_protobuf_ptr>( + data: *const u8, + len: i32, +) -> anyhow::Result { + let buffer = get_buffer(data, len)?; + let item: Result = prost::Message::decode(buffer); + item.map(|inner| inner.into()).map_err(|e| e.into()) +} + +unsafe fn get_buffer<'a>(data: *const u8, len: i32) -> anyhow::Result<&'a [u8]> { + match len { + len if len < 0 => anyhow::bail!("Invalid length"), + 0 => Ok(&[]), + _ => { + if data.is_null() { + anyhow::bail!("Null pointer") + } + Ok(std::slice::from_raw_parts(data, len as usize)) + } + } +} diff --git a/src/http_client.rs b/src/http_client.rs index 149af81..ee29efe 100644 --- a/src/http_client.rs +++ b/src/http_client.rs @@ -2,6 +2,10 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +//! This is a simple Http client that uses viaduct to retrieve experiment data from the server +//! Currently configured to use Kinto and the old schema, although that would change once we start +//! Working on the real Nimbus schema. + use super::Experiment; use anyhow::Result; use serde_derive::*; @@ -66,13 +70,11 @@ impl SettingsClient for Client { &self.bucket_name, &self.collection_name ); let url = self.base_url.join(&path)?; - let req = Request::get(url) - .header( - "User-Agent", - "Experiments Rust Component ", - )?; - // Note: I removed the auth info which was for a test account that is public - // But gitgaurdian complained so I removed it. + let req = Request::get(url).header( + "User-Agent", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0", + )?; + // TODO: Add authentication based on server requirements let resp = self.make_request(req)?.json::()?; Ok(resp.data) } diff --git a/src/lib.rs b/src/lib.rs index 58a906f..7cc7a46 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,44 +3,86 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ //! Experiments library that hopes to be cross-plateform. -//! Still a work in progress and essentially has zero functionality so far +//! Still a work in progress, but good enough for people to poke around -use url::Url; mod buckets; pub mod error; +pub mod ffi; mod http_client; mod persistence; + use error::Result; +pub use ffi::{experiements_destroy, experiments_get_branch, experiments_new}; use http_client::{Client, SettingsClient}; +use persistence::Database; +use persistence::PersistedData; use rand::{rngs::StdRng, Rng, SeedableRng}; use serde_derive::*; -const MAX_BUCKET_NO: u32 = 1000; +use std::convert::TryInto; +use std::path::Path; +use url::Url; +use uuid::Uuid; +pub use viaduct; + +const BASE_URL: &str = "https://kinto.dev.mozaws.net/v1/"; +const COLLECTION_NAME: &str = "messaging-collection"; +const BUCKET_NAME: &str = "main"; +const MAX_BUCKET_NO: u32 = 10000; + +// We'll probably end up doing what is done in A-S with regards to +// protobufs if we take this route... +// But for now, using the build.rs seems convenient +// ref: https://github.com/mozilla/application-services/tree/main/tools/protobuf-gen +pub mod msg_types { + include!(concat!( + env!("OUT_DIR"), + "/mozilla.telemetry.glean.protobuf.rs" + )); +} /// Experiments is the main struct representing the experiements state /// It should hold all the information needed to communcate a specific user's /// Experiementation status (note: This should have some type of uuid) pub struct Experiments { + // Uuid not used yet, but we'll be using it later + #[allow(unused)] + uuid: Uuid, + #[allow(unused)] + app_ctx: AppContext, experiments: Vec, enrolled_experiments: Vec, bucket_no: u32, } impl Experiments { - // A new experiments struct is created this is where some preprocessing happens - // It should look for persisted state first (once that is implemented) and setup some type - // Of interval retrieval from the server for any experiment updates - pub fn new(base_url: &str, collection_name: &str, bucket_name: &str) -> Self { + /// A new experiments struct is created this is where some preprocessing happens + /// It should look for persisted state first and setup some type + /// Of interval retrieval from the server for any experiment updates (not implemented) + pub fn new>(app_ctx: AppContext, path: P) -> Self { + let database = Database::new(path).unwrap(); + let persisted_data = database.get("persisted").unwrap(); + if let Some(data) = persisted_data { + log::info!("Retrieving data from persisted state..."); + let persisted_data = serde_json::from_str::(&data).unwrap(); + return Self { + app_ctx, + uuid: persisted_data.uuid, + experiments: persisted_data.experiments, + enrolled_experiments: persisted_data.enrolled_experiments, + bucket_no: persisted_data.bucket_no, + }; + } let http_client = Client::new( - Url::parse(base_url).unwrap(), - collection_name.to_string(), - bucket_name.to_string(), + Url::parse(BASE_URL).unwrap(), + COLLECTION_NAME.to_string(), + BUCKET_NAME.to_string(), ); let resp = http_client.get_experiments().unwrap(); - log::info!("Creating experiement...."); - let uuid: [u8; 4] = rand::random(); // This is ***not*** a real uuid, it's purely for testing/demoing purposes - let bucket_no: u32 = u32::from_be_bytes(uuid) % MAX_BUCKET_NO; - let mut num = StdRng::seed_from_u64(bucket_no as u64); // Don't look at me, I'm not a good way to generate a random number!!!! + let uuid = uuid::Uuid::new_v4(); + let bucket_no: u32 = + u32::from_be_bytes(uuid.as_bytes()[..4].try_into().unwrap()) % MAX_BUCKET_NO; + let mut num = StdRng::seed_from_u64(bucket_no as u64); let enrolled_experiments = resp .iter() .filter_map(|e| { @@ -54,8 +96,22 @@ impl Experiments { None } }) - .collect(); + .collect::>(); + database + .put( + "persisted", + PersistedData { + app_ctx: app_ctx.clone(), + uuid, + bucket_no, + enrolled_experiments: enrolled_experiments.clone(), + experiments: resp.clone(), + }, + ) + .unwrap(); Self { + app_ctx, + uuid, experiments: resp, bucket_no, enrolled_experiments, @@ -84,6 +140,7 @@ impl Experiments { } } +#[derive(Deserialize, Serialize, Debug, Clone)] pub struct EnrolledExperiment { id: String, branch: String, @@ -99,7 +156,10 @@ impl EnrolledExperiment { } } -#[derive(Deserialize, Debug, Clone)] +// ============ Below are a bunch of types that gets serialized/deserialized and stored in our `Experiments` struct ============ +// ============ They currently follow the old schema, and need to be updated to match the new Nimbus schema ============ + +#[derive(Deserialize, Serialize, Debug, Clone)] pub struct Experiment { pub id: String, pub description: String, @@ -111,32 +171,60 @@ pub struct Experiment { pub matcher: Matcher, } -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Serialize, Debug, Clone)] pub struct Bucket { pub count: u32, pub start: u32, } -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Serialize, Debug, Clone)] pub struct Branch { pub name: String, ratio: u32, } -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Serialize, Debug, Clone, Default)] pub struct Matcher { - app_id: Option, - app_display_version: Option, - app_min_version: Option, // Do what AC does and have a VersionOptionString instead? - app_max_version: Option, //Dito - locale_language: Option, - locale_country: Option, - device_manufacturer: Option, - device_model: Option, - regions: Vec, - debug_tags: Vec, + pub app_id: Option, + pub app_display_version: Option, + pub app_min_version: Option, // Do what AC does and have a VersionOptionString instead? + pub app_max_version: Option, //Dito + pub locale_language: Option, + pub locale_country: Option, + pub device_manufacturer: Option, + pub device_model: Option, + pub regions: Vec, + pub debug_tags: Vec, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct AppContext { + pub app_id: Option, + pub app_version: Option, + pub locale_language: Option, + pub locale_country: Option, + pub device_manufacturer: Option, + pub device_model: Option, + pub region: Option, + pub debug_tag: Option, +} + +impl From for AppContext { + fn from(proto_ctx: msg_types::AppContext) -> Self { + Self { + app_id: proto_ctx.app_id, + app_version: proto_ctx.app_version, + locale_language: proto_ctx.locale_language, + locale_country: proto_ctx.locale_country, + device_manufacturer: proto_ctx.device_manufacturer, + device_model: proto_ctx.device_model, + region: proto_ctx.region, + debug_tag: proto_ctx.debug_tag, + } + } } +// No tests implemented just yet #[cfg(test)] mod tests { #[test] diff --git a/src/persistence.rs b/src/persistence.rs index dc9953a..a4703c9 100644 --- a/src/persistence.rs +++ b/src/persistence.rs @@ -4,6 +4,73 @@ //! This is where the persistence logic might go. //! An idea for what to use here might be [RKV](https://github.com/mozilla/rkv) -//! Either ways, the solution implemented should work regardless of the plateform +//! And that's what's used on this prototype, +//! Either ways, the solution implemented should work regardless of the platform //! on the other side of the FFI. This means that this module might require the FFI to allow consumers //! To pass in a path to a database, or somewhere in the file system that the state will be persisted +use anyhow::Result; +use rkv::{Rkv, SingleStore, StoreOptions}; +use serde_derive::*; +use std::fs; +use std::path::Path; + +pub struct Database { + rkv: Rkv, + + experiment_store: SingleStore, +} + +impl Database { + pub fn new>(path: P) -> Result { + let rkv = Self::open_rkv(path)?; + let experiment_store = rkv + .open_single("experiments", StoreOptions::create()) + .unwrap(); + Ok(Self { + rkv, + experiment_store, + }) + } + + fn open_rkv>(path: P) -> Result { + let path = std::path::Path::new(path.as_ref()).join("db"); + log::debug!("Database path: {:?}", path.display()); + fs::create_dir_all(&path)?; + + let rkv = Rkv::new(&path).unwrap(); // Rkv errors should impl std::error::Error :( TODO: Impl proper error handling in an error.rs that can propagate + log::info!("Database initialized"); + Ok(rkv) + } + + pub fn get(&self, key: &str) -> Result> { + let reader = self.rkv.read().unwrap(); + let val = self.experiment_store.get(&reader, key).unwrap(); + Ok(val.map(|v| { + if let rkv::Value::Json(val) = v { + val.to_string() + } else { + "".to_string() // BAD IDEA! Remove this! + } + })) + } + + pub fn put(&self, key: &str, persisted_data: PersistedData) -> Result<()> { + let mut writer = self.rkv.write().unwrap(); + let persisted_json = serde_json::to_string(&persisted_data).unwrap(); + self.experiment_store + .put(&mut writer, key, &rkv::Value::Json(&persisted_json)) + .unwrap(); + writer.commit().unwrap(); + Ok(()) + } +} + +use super::{AppContext, EnrolledExperiment, Experiment}; +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct PersistedData { + pub app_ctx: AppContext, + pub experiments: Vec, + pub enrolled_experiments: Vec, + pub bucket_no: u32, + pub uuid: uuid::Uuid, +}