'Fixes and improvements' (#972)

Author: sceuick

.vscode/launch.json

@@ -5,7 +5,7 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Launch API",
+      "name": "Launch API + Watch",
       "program": "${workspaceFolder}/srv/start.ts",
       "request": "launch",
       "sourceMaps": true,
@@ -14,6 +14,15 @@
       "console": "integratedTerminal",
       "preLaunchTask": "build server"
     },
+    {
+      "name": "Launch API",
+      "program": "${workspaceFolder}/srv/start.ts",
+      "request": "launch",
+      "sourceMaps": true,
+      "skipFiles": ["<node_internals>/**"],
+      "type": "node",
+      "console": "integratedTerminal"
+    },
     {
       "name": "Attach",
       "port": 9229,

common/adapters.ts

@@ -1,10 +1,5 @@
 import { AppSchema } from './types/schema'
 
-export type AIAdapter = (typeof AI_ADAPTERS)[number]
-export type ChatAdapter = (typeof CHAT_ADAPTERS)[number]
-export type PersonaFormat = (typeof PERSONA_FORMATS)[number]
-export type ThirdPartyFormat = (typeof THIRDPARTY_FORMATS)[number]
-
 export type AdapterSetting = {
   /** The name of the field within the settings object */
   field: string
@@ -48,6 +43,12 @@ export const PERSONA_LABELS: { [key in PersonaFormat]: string } = {
   text: 'Plain Text',
 }
 
+export const JSON_SCHEMA_SUPPORTED: { [key in AIAdapter | ThirdPartyFormat]?: boolean } = {
+  agnaistic: true,
+  llamacpp: true,
+  tabby: true,
+}
+
 export const THIRDPARTY_HANDLERS: { [svc in ThirdPartyFormat]: AIAdapter } = {
   openai: 'openai',
   'openai-chat': 'openai',
@@ -337,141 +338,8 @@ export const samplerDisableValues: { [key in keyof PresetAISettings]?: number }
   tailFreeSampling: 1,
 }
 
-export const adapterSettings: {
-  [key in keyof PresetAISettings]: Array<AIAdapter | ThirdPartyFormat>
-} = {
-  temp: [
-    'kobold',
-    'novel',
-    'ooba',
-    'horde',
-    'openai',
-    'scale',
-    'claude',
-    'goose',
-    'agnaistic',
-    'aphrodite',
-    'tabby',
-    'mistral',
-  ],
-  tempLast: ['agnaistic', 'tabby', 'exllamav2'],
-  dynatemp_range: ['kobold', 'ooba', 'tabby', 'agnaistic', 'aphrodite'],
-  dynatemp_exponent: ['kobold', 'aphrodite', 'ooba', 'tabby', 'agnaistic'],
-  smoothingFactor: ['kobold', 'aphrodite', 'ooba', 'tabby', 'agnaistic'],
-  smoothingCurve: ['kobold', 'aphrodite'],
-  maxTokens: AI_ADAPTERS.slice(),
-  maxContextLength: AI_ADAPTERS.slice(),
-  antiBond: ['openai', 'scale'],
-  prefixNameAppend: ['openai', 'claude'],
-
-  swipesPerGeneration: ['aphrodite'],
-  epsilonCutoff: ['aphrodite'],
-  etaCutoff: ['aphrodite'],
-
-  prefill: ['claude'],
-
-  topP: [
-    'horde',
-    'kobold',
-    'claude',
-    'ooba',
-    'openai',
-    'novel',
-    'agnaistic',
-    'exllamav2',
-    'openai-chat',
-    'aphrodite',
-    'tabby',
-    'mistral',
-  ],
-  repetitionPenalty: [
-    'horde',
-    'novel',
-    'kobold',
-    'ooba',
-    'agnaistic',
-    'exllamav2',
-    'aphrodite',
-    'tabby',
-  ],
-  repetitionPenaltyRange: ['horde', 'novel', 'kobold', 'ooba', 'agnaistic', 'tabby'],
-  repetitionPenaltySlope: ['horde', 'novel', 'kobold'],
-  tailFreeSampling: ['horde', 'novel', 'kobold', 'ooba', 'agnaistic', 'aphrodite', 'tabby'],
-  minP: ['llamacpp', 'kobold', 'koboldcpp', 'exllamav2', 'ooba', 'agnaistic', 'aphrodite', 'tabby'],
-  topA: ['horde', 'novel', 'kobold', 'ooba', 'agnaistic', 'aphrodite', 'tabby'],
-  topK: [
-    'horde',
-    'novel',
-    'kobold',
-    'ooba',
-    'claude',
-    'agnaistic',
-    'exllamav2',
-    'aphrodite',
-    'tabby',
-  ],
-  typicalP: ['horde', 'novel', 'kobold', 'ooba', 'agnaistic', 'exllamav2', 'aphrodite', 'tabby'],
-
-  mirostatToggle: ['aphrodite', 'tabby'],
-  mirostatLR: ['novel', 'ooba', 'agnaistic', 'llamacpp', 'aphrodite', 'tabby'],
-  mirostatTau: ['novel', 'ooba', 'agnaistic', 'llamacpp', 'aphrodite', 'tabby'],
-  cfgScale: ['novel', 'ooba', 'tabby'],
-  cfgOppose: ['novel', 'ooba', 'tabby'],
-  phraseRepPenalty: ['novel'],
-  phraseBias: ['novel'],
-
-  thirdPartyUrl: ['kobold', 'ooba'],
-  thirdPartyFormat: ['kobold'],
-  thirdPartyModel: ['openai', 'openai-chat', 'aphrodite', 'tabby'],
-  thirdPartyKey: ['kobold', 'aphrodite', 'tabby', 'openai', 'openai-chat'],
-
-  claudeModel: ['claude'],
-  novelModel: ['novel'],
-  mistralModel: ['mistral'],
-  oaiModel: ['openai', 'openai-chat'],
-  frequencyPenalty: ['openai', 'kobold', 'novel', 'agnaistic', 'openai-chat', 'aphrodite', 'tabby'],
-  presencePenalty: ['openai', 'kobold', 'novel', 'openai-chat', 'aphrodite', 'tabby'],
-  streamResponse: [
-    'openai',
-    'kobold',
-    'novel',
-    'claude',
-    'ooba',
-    'agnaistic',
-    'openai-chat',
-    'aphrodite',
-    'tabby',
-    'mistral',
-  ],
-  openRouterModel: ['openrouter'],
-  stopSequences: [
-    'ooba',
-    'agnaistic',
-    'novel',
-    'mancer',
-    'llamacpp',
-    'horde',
-    'exllamav2',
-    'kobold',
-    'aphrodite',
-    'tabby',
-  ],
-  trimStop: ['koboldcpp'],
-
-  addBosToken: ['ooba', 'agnaistic', 'tabby'],
-  banEosToken: ['ooba', 'aphrodite', 'tabby'],
-  tokenHealing: ['agnaistic', 'exllamav2', 'ooba', 'tabby'],
-  doSample: ['ooba'],
-  encoderRepitionPenalty: ['ooba'],
-  penaltyAlpha: ['ooba'],
-  earlyStopping: ['ooba'],
-  numBeams: ['ooba'],
-
-  replicateModelName: ['replicate'],
-  replicateModelVersion: ['replicate'],
-  replicateModelType: ['replicate'],
-
-  skipSpecialTokens: ['ooba', 'kobold'],
+export function adaptersToOptions(adapters: AIAdapter[]) {
+  return adapters.map((adp) => ({ label: ADAPTER_LABELS[adp], value: adp }))
 }
 
 export type RegisteredAdapter = {
@@ -552,3 +420,8 @@ export const samplerOrders: { [key in AIAdapter]?: Array<keyof PresetAISettings>
     'mirostatTau',
   ],
 }
+
+export type AIAdapter = (typeof AI_ADAPTERS)[number]
+export type ChatAdapter = (typeof CHAT_ADAPTERS)[number]
+export type PersonaFormat = (typeof PERSONA_FORMATS)[number]
+export type ThirdPartyFormat = (typeof THIRDPARTY_FORMATS)[number]

common/presets/templates.ts

@@ -252,12 +252,12 @@ Write {{char}}'s next reply in a fictional roleplay chat between {{user}} and {{
 {{/if}}
 Then the roleplay chat begins.<|im_end|>
 
-{{#each msg}}{{#if .isbot}}<|im_start|>assistant{{/if}}{{#if .isuser}}<|im_start|>user{{/if}}
-{{.name}}: {{.msg}}<|im_end|>
+{{#each msg}}<|im_start|>[{{.name}}]
+{{.msg}}<|im_end|>
 {{/each}}
 {{#if ujb}}<|im_start|>system
 {{ujb}}<|im_end|>
 {{/if}}
-<|im_start|>assistant
+<|im_start|>[{{char}}]
 {{post}}`,
 }

common/prompt-order.ts

@@ -95,10 +95,9 @@ export const formatHolders: Record<string, Record<string, string>> = {
     history: neat`<|im_start|>system
     Then the roleplay chat between {{user}} and {{char}} begins.<|im_end|>
     
-    {{#each msg}}{{#if .isbot}}<|im_start|>assistant{{/if}}{{#if .isuser}}<|im_start|>user{{/if}}
-  {{.name}}: {{.msg}}<|im_end|>
-  {{/each}}`,
-    post: neat`<|im_start|>assistant
+    {{#each msg}}<|im_start|>{{#if .isbot}}assistant{{/if}}{{#if .isuser}}user{{/if}}
+    {{.name}}: {{.msg}}<|im_end|>{{/each}}`,
+    post: neat`POST<|im_start|>assistant
     {{post}}`,
     system_prompt: neat`{{#if system_prompt}}<|im_start|>system
     {{system_prompt}}<|im_end|>{{/if}}`,

common/prompt.ts

@@ -89,8 +89,8 @@ export type BuildPromptOpts = {
 }
 
 /** {{user}}, <user>, {{char}}, <bot>, case insensitive */
-export const BOT_REPLACE = /(\{\{char\}\}|<BOT>|\{\{name\}\})/gi
-export const SELF_REPLACE = /(\{\{user\}\}|<USER>)/gi
+export const BOT_REPLACE = /(\{\{char\}\}|\{\{name\}\})/gi
+export const SELF_REPLACE = /(\{\{user\}\})/gi
 export const START_REPLACE = /(<START>)/gi
 
 const HOLDER_NAMES = {
@@ -159,8 +159,13 @@ export async function createPromptParts(opts: PromptOpts, encoder: TokenCounter)
   /**
    * The lines from `getLinesForPrompt` are returned in time-descending order
    */
-  const template = getTemplate(opts)
+  let template = getTemplate(opts)
   const templateSize = await encoder(template)
+
+  if (opts.modelFormat) {
+    template = replaceTags(template, opts.modelFormat)
+  }
+
   /**
    * It's important for us to pass in a max context that is _realistic-ish_ as the embeddings
    * are retrieved based on the number of history messages we return here.
@@ -237,12 +242,11 @@ export function getTemplate(opts: Pick<GenerateRequestV2, 'settings' | 'chat'>)
 
   const template = opts.settings?.gaslight || fallback?.gaslight || defaultTemplate
 
-  const validate = opts.settings?.useAdvancedPrompt !== 'no-validation'
-
-  if (!validate) {
+  if (opts.settings?.useAdvancedPrompt === 'no-validation') {
     return template
   }
 
+  // Deprecated
   return ensureValidTemplate(template)
 }
 
@@ -909,3 +913,37 @@ export function resolveScenario(
 
   return result.trim()
 }
+
+type JsonType =
+  | { type: 'string'; title?: string; maxLength?: number }
+  | { type: 'integer'; title?: string }
+  | { type: 'enum'; enum: string[]; title?: string }
+
+type JsonSchema = {
+  title: string
+  type: 'object'
+  properties: Record<string, JsonType>
+}
+
+export const schema = {
+  str: (title?: string, maxLength?: number) => ({ type: 'string', title, maxLength }),
+  int: (title?: string) => ({ type: 'integer', title }),
+  enum: (values: string[], title?: string) => ({ type: 'enum', enum: values, title }),
+} satisfies Record<string, (...args: any[]) => JsonType>
+
+export function toJsonSchema(body: Record<string, JsonType>): JsonSchema {
+  const schema: JsonSchema = {
+    title: 'Response',
+    type: 'object',
+    properties: {},
+  }
+
+  const props: JsonSchema['properties'] = {}
+
+  for (const [key, def] of Object.entries(body)) {
+    props[key] = def
+  }
+
+  schema.properties = props
+  return schema
+}

common/template-parser.ts

@@ -22,6 +22,8 @@ function loadParser() {
   }
 }
 
+const HISTORY_MARKER = '__history__marker__'
+
 type PNode = PlaceHolder | ConditionNode | IteratorNode | InsertNode | LowPriorityNode | string
 
 type PlaceHolder = { kind: 'placeholder'; value: Holder; values?: any; pipes?: string[] }
@@ -144,6 +146,14 @@ export async function parseTemplate(
   let unusedTokens = 0
   let linesAddedCount = 0
 
+  /** Remove everything after history to attempt to perform a 'continue' */
+  if (opts.continue && output.includes(HISTORY_MARKER)) {
+    const index = output.indexOf(HISTORY_MARKER)
+    if (index > -1) {
+      output = output.slice(0, index + HISTORY_MARKER.length)
+    }
+  }
+
   /** Replace iterators */
   if (opts.limit && opts.limit.output) {
     for (const [id, { lines, src }] of Object.entries(opts.limit.output)) {
@@ -177,7 +187,7 @@ export async function parseTemplate(
     }
   }
 
-  const result = render(output, opts).replace(/\r\n/g, '\n').replace(/\n\n+/g, '\n\n').trimStart()
+  const result = render(output, opts).replace(/\r\n/g, '\n').replace(/\n\n+/g, '\n\n').trim()
   return {
     parsed: result,
     inserts: opts.inserts ?? new Map(),
@@ -479,7 +489,7 @@ function renderIterator(holder: IterableHolder, children: CNode[], opts: Templat
   }
 
   if (isHistory && opts.limit?.output) {
-    const id = '__' + v4() + '__'
+    const id = HISTORY_MARKER
     opts.limit.output[id] = { src: holder, lines: output }
     return id
   }

common/types/schema.ts

@@ -79,8 +79,9 @@ export namespace AppSchema {
     imagesHost: string
     imagesModels: ImageModel[]
 
-    ttsEnabled: boolean
     ttsHost: string
+    ttsApiKey: string
+    ttsAccess: 'off' | 'users' | 'subscribers' | 'admins'
 
     maxGuidanceTokens: number
     maxGuidanceVariables: number
@@ -173,6 +174,8 @@ export namespace AppSchema {
     /** @todo remove after next deployment */
     tier?: AppSchema.SubscriptionTier
     serverConfig?: Configuration
+
+    googleClientId: string | undefined
   }
 
   export type ChatMode = 'standard' | 'adventure'
@@ -289,6 +292,11 @@ export namespace AppSchema {
       }
     }
 
+    google?: {
+      sub: any
+      email: any
+    }
+
     billing?: {
       status: 'active' | 'cancelled'
       cancelling?: boolean