diff --git a/mistralrs-core/src/pipeline/mod.rs b/mistralrs-core/src/pipeline/mod.rs index 298a8d570a..c4bb97738e 100644 --- a/mistralrs-core/src/pipeline/mod.rs +++ b/mistralrs-core/src/pipeline/mod.rs @@ -224,6 +224,18 @@ pub enum ModelCategory { Speech, } +impl std::fmt::Debug for ModelCategory { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ModelCategory::Text => write!(f, "ModelCategory::Text"), + ModelCategory::Vision { .. } => write!(f, "ModelCategory::Vision {{ prefixer: .. }}"), + ModelCategory::Diffusion => write!(f, "ModelCategory::Diffusion"), + ModelCategory::Audio => write!(f, "ModelCategory::Audio"), + ModelCategory::Speech => write!(f, "ModelCategory::Speech"), + } + } +} + impl PartialEq for ModelCategory { fn eq(&self, other: &Self) -> bool { match (self, other) { diff --git a/mistralrs-quant/src/lib.rs b/mistralrs-quant/src/lib.rs index 54bc7788e7..1051453d77 100644 --- a/mistralrs-quant/src/lib.rs +++ b/mistralrs-quant/src/lib.rs @@ -2,7 +2,7 @@ use std::{ borrow::Cow, fmt::Debug, num::NonZeroUsize, - sync::{atomic::AtomicUsize, Arc, Mutex, MutexGuard, OnceLock}, + sync::{atomic::AtomicUsize, Arc, Mutex, MutexGuard}, }; use blockwise_fp8::blockwise_fp8_linear_b; @@ -63,33 +63,26 @@ pub use utils::{log, BitWiseOp, CumSumOp, LeftshiftOp, NonZeroOp, SortOp, UQFF_Q use candle_nn::{Linear, Module}; use serde::{Deserialize, Deserializer, Serialize}; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct ImmediateIsqParams { pub guard: QuantizeOntoGuard, pub ty: Option, pub predicates: Vec, } -static IMMEDIATE_ISQ: OnceLock> = OnceLock::new(); +static IMMEDIATE_ISQ: Mutex> = Mutex::new(None); pub fn set_immediate_isq(isq: Option, predicates: Vec) { - IMMEDIATE_ISQ - .get_or_init(|| { - Mutex::new(ImmediateIsqParams { - guard: QuantizeOntoGuard::new(), - ty: None, - predicates, - }) - }) - .lock() - .unwrap() - .ty = isq; + let mut guard = IMMEDIATE_ISQ.lock().expect("IMMEDIATE_ISQ mutex poisoned"); + *guard = Some(ImmediateIsqParams { + guard: QuantizeOntoGuard::new(), + ty: isq, + predicates, + }); } pub fn get_immediate_isq() -> Option { - IMMEDIATE_ISQ - .get() - .map(|guard| guard.lock().unwrap().clone()) + IMMEDIATE_ISQ.lock().ok().and_then(|guard| guard.clone()) } pub fn should_apply_immediate_isq(vb: &ShardedVarBuilder) -> bool { @@ -545,7 +538,7 @@ pub trait QuantizedSerde { } /// Used to gate access to quantizing onto the host device -#[derive(Clone)] +#[derive(Clone, Debug)] #[allow(unused)] pub struct QuantizeOntoGuard(Arc>); diff --git a/mistralrs-web-chat/static/index.html b/mistralrs-web-chat/static/index.html index e60959ee07..9be45870c2 100644 --- a/mistralrs-web-chat/static/index.html +++ b/mistralrs-web-chat/static/index.html @@ -11,7 +11,7 @@

Control panel

- +
    @@ -22,11 +22,19 @@

    Mistral.rs Chat

    -
    - - - - + + + + + + + + + +
    diff --git a/mistralrs-web-chat/static/js/websocket.js b/mistralrs-web-chat/static/js/websocket.js index 98b218eaa3..cc1954ad04 100644 --- a/mistralrs-web-chat/static/js/websocket.js +++ b/mistralrs-web-chat/static/js/websocket.js @@ -35,6 +35,9 @@ function handleWebSocketMessage(ev) { } if (!assistantDiv) { + // remove inline spinner when first assistant data arrives + const spinner = document.getElementById('spinner'); + if (spinner) spinner.remove(); assistantDiv = append('', 'assistant'); } @@ -58,6 +61,12 @@ function sendMessage() { assistantBuf = ''; assistantDiv = null; ws.send(msg); + // dynamically add spinner in log area + const log = document.getElementById('log'); + const spinnerEl = document.createElement('div'); + spinnerEl.classList.add('spinner'); + spinnerEl.id = 'spinner'; + log.appendChild(spinnerEl); input.value = ''; // Trigger textarea resize diff --git a/mistralrs-web-chat/static/styles.css b/mistralrs-web-chat/static/styles.css index 7138db1a70..2fdba2047f 100644 --- a/mistralrs-web-chat/static/styles.css +++ b/mistralrs-web-chat/static/styles.css @@ -191,13 +191,23 @@ body { font-family: inherit; } -#form button[type="submit"] { +.btn { + display: inline-flex; + align-items: center; + justify-content: center; background: var(--primary); color: #fff; border: none; padding: 0.6rem 1.2rem; border-radius: var(--radius); + cursor: pointer; transition: background 0.2s; + font-family: inherit; + font-size: 1rem; +} + +.btn:hover { + background: var(--primary-hov); } #form button[type="submit"]:hover { @@ -209,19 +219,6 @@ body { display:none; } -#imageLabel { - padding:0.6rem 1.2rem; - border-radius:var(--radius); - background:var(--accent); - color:#fff; - cursor:pointer; - transition:background 0.2s; -} - -#imageLabel:hover { - background:var(--accent-hov); -} - img.chat-preview { max-width: 150px; max-height: 150px; @@ -305,3 +302,21 @@ pre { pre:hover .copy-btn { opacity: 1; } + +/* Spinner */ +.spinner { + width: 1.5rem; + height: 1.5rem; + border: 3px solid var(--border-color); + border-top: 3px solid var(--primary); + border-radius: 50%; + animation: spin 1s linear infinite; +} + +.spinner.hidden { + display: none; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} diff --git a/mistralrs/src/messages.rs b/mistralrs/src/messages.rs index 2c10e6958d..2688202b71 100644 --- a/mistralrs/src/messages.rs +++ b/mistralrs/src/messages.rs @@ -9,6 +9,7 @@ use serde_json::{json, Value}; /// A type which can be used as a chat request. pub trait RequestLike { fn messages_ref(&self) -> &[IndexMap]; + fn images_ref(&self) -> &[DynamicImage]; fn take_messages(&mut self) -> RequestMessage; fn take_logits_processors(&mut self) -> Option>>; fn take_adapters(&mut self) -> Option>; @@ -85,6 +86,9 @@ impl RequestLike for TextMessages { fn messages_ref(&self) -> &[IndexMap] { &self.0 } + fn images_ref(&self) -> &[DynamicImage] { + &[] + } fn take_messages(&mut self) -> RequestMessage { let mut other = Vec::new(); std::mem::swap(&mut other, &mut self.0); @@ -205,6 +209,9 @@ impl RequestLike for VisionMessages { fn messages_ref(&self) -> &[IndexMap] { &self.messages } + fn images_ref(&self) -> &[DynamicImage] { + &self.images + } fn take_messages(&mut self) -> RequestMessage { let mut other_messages = Vec::new(); std::mem::swap(&mut other_messages, &mut self.messages); @@ -523,6 +530,10 @@ impl RequestLike for RequestBuilder { &self.messages } + fn images_ref(&self) -> &[DynamicImage] { + &self.images + } + fn take_messages(&mut self) -> RequestMessage { if self.images.is_empty() { let mut other = Vec::new();