feat(@aws-amplify/ui-components): Add ChatBot Component

packages/amplify-ui-angular/src/amplify-module.ts

@@ -7,6 +7,7 @@ import {
 	AmplifyAuthenticator,
 	AmplifyAuthFields,
 	AmplifyButton,
+	AmplifyChatbot,
 	AmplifyCheckbox,
 	AmplifyCodeField,
 	AmplifyConfirmSignIn,
@@ -65,6 +66,7 @@ const DECLARATIONS = [
 	AmplifyAuthenticator,
 	AmplifyAuthFields,
 	AmplifyButton,
+	AmplifyChatbot,
 	AmplifyCheckbox,
 	AmplifyCodeField,
 	AmplifyConfirmSignIn,

packages/amplify-ui-components/src/common/Translations.ts

@@ -109,5 +109,12 @@ export enum AuthStrings {
   SIGN_UP_FAILED = 'Sign Up Failed',
 }
 
-type Translations = AuthErrorStrings | AuthStrings;
-export const Translations = { ...AuthStrings, ...AuthErrorStrings };
+export enum InteractionsStrings {
+  CHATBOT_TITLE = 'ChatBot Lex',
+  TEXT_INPUT_PLACEHOLDER = 'Write a message',
+  CHAT_DISABLED_ERROR = 'Error: Either voice or text must be enabled for the chatbot',
+  NO_BOT_NAME_ERROR = 'Error: Bot Name must be provided to ChatBot',
+}
+
+type Translations = AuthErrorStrings | AuthStrings | InteractionsStrings;
+export const Translations = { ...AuthStrings, ...AuthErrorStrings, ...InteractionsStrings };

packages/amplify-ui-components/src/common/audio-control/helper.ts

@@ -0,0 +1,127 @@
+import { RECORDER_EXPORT_MIME_TYPE } from './settings';
+
+/**
+ * Merges multiple buffers into one.
+ */
+const mergeBuffers = (bufferArray: Float32Array[], recLength: number) => {
+  const result = new Float32Array(recLength);
+  let offset = 0;
+  for (let i = 0; i < bufferArray.length; i++) {
+    result.set(bufferArray[i], offset);
+    offset += bufferArray[i].length;
+  }
+  return result;
+};
+
+/**
+ * Downsamples audio to desired export sample rate.
+ */
+const downsampleBuffer = (buffer: Float32Array, recordSampleRate: number, exportSampleRate: number) => {
+  if (exportSampleRate === recordSampleRate) {
+    return buffer;
+  }
+  const sampleRateRatio = recordSampleRate / exportSampleRate;
+  const newLength = Math.round(buffer.length / sampleRateRatio);
+  const result = new Float32Array(newLength);
+  let offsetResult = 0;
+  let offsetBuffer = 0;
+  while (offsetResult < result.length) {
+    const nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
+    let accum = 0,
+      count = 0;
+    for (let i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
+      accum += buffer[i];
+      count++;
+    }
+    result[offsetResult] = accum / count;
+    offsetResult++;
+    offsetBuffer = nextOffsetBuffer;
+  }
+  return result;
+};
+
+/**
+ * converts raw audio values to 16 bit pcm.
+ */
+const floatTo16BitPCM = (output: DataView, offset: number, input: Float32Array) => {
+  let byteOffset = offset;
+  for (let i = 0; i < input.length; i++, byteOffset += 2) {
+    const s = Math.max(-1, Math.min(1, input[i]));
+    output.setInt16(byteOffset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+  }
+};
+
+/**
+ * Write given strings in big-endian order.
+ */
+const writeString = (view: DataView, offset: number, string: string) => {
+  for (let i = 0; i < string.length; i++) {
+    view.setUint8(offset + i, string.charCodeAt(i));
+  }
+};
+
+/**
+ * Encodes raw pcm audio into a wav file.
+ */
+const encodeWAV = (samples: Float32Array, exportSampleRate?: number) => {
+  /**
+   * WAV file consists of three parts: RIFF header, WAVE subchunk, and data subchunk. We precompute the size of them.
+   */
+
+  const audioSize = samples.length * 2; // We use 16-bit samples, so we have (2 * sampleLength) bytes.
+  const fmtSize = 24; // Byte size of the fmt subchunk: 24 bytes that the audio information that we'll set below.
+  const dataSize = 8 + audioSize; // Byte size of the data subchunk: raw sound data plus 8 bytes for the subchunk descriptions.
+
+  const totalByteSize = 12 + fmtSize + dataSize; // Byte size of the whole file, including the chunk header / descriptor.
+
+  // create DataView object to write byte values into
+  const buffer = new ArrayBuffer(totalByteSize); // buffer to write the chunk values in.
+  const view = new DataView(buffer);
+
+  /**
+   * Start writing the .wav file. We write top to bottom, so byte offset (first numeric argument) increases strictly.
+   */
+  // RIFF header
+  writeString(view, 0, 'RIFF'); // At offset 0, write the letters "RIFF"
+  view.setUint32(4, fmtSize + dataSize, true); // At offset 4, write the size of fmt and data chunk size combined.
+  writeString(view, 8, 'WAVE'); // At offset 8, write the format type "WAVE"
+
+  // fmt subchunk
+  writeString(view, 12, 'fmt '); //chunkdId 'fmt '
+  view.setUint32(16, fmtSize - 8, true); // fmt subchunk size below this value. We set 8 bytes already, so subtract 8 bytes from fmtSize.
+  view.setUint16(20, 1, true); // Audiio format code, which is 1 for PCM.
+  view.setUint16(22, 1, true); // Number of audio channels. We use mono, ie 1.
+  view.setUint32(24, exportSampleRate, true); // Sample rate of the audio file.
+  view.setUint32(28, exportSampleRate * 2, true); // Data rate, or # of data bytes per second. Since each sample is 2 bytes, this is 2 * sampleRate.
+  view.setUint16(32, 2, true); // block align, # of bytes per sample including all channels, ie. 2 bytes.
+  view.setUint16(34, 16, true); // bits per sample, ie. 16 bits
+
+  // data subchunk
+  writeString(view, 36, 'data'); // write the chunkId 'data'
+  view.setUint32(40, audioSize, true); // Audio byte size
+  floatTo16BitPCM(view, 44, samples); // raw pcm values then go here.
+  return view;
+};
+
+/**
+ * Given arrays of raw pcm audio, downsamples the audio to desired sample rate and encodes it to a wav audio file.
+ *
+ * @param recBuffer {Float32Array[]} - 2d float array containing the recorded raw audio
+ * @param recLength {number} - total length of recorded audio
+ * @param recordSampleRate {number} - sample rate of the recorded audio
+ * @param exportSampleRate {number} - desired sample rate of the exported file
+ */
+export const exportBuffer = (
+  recBuffer: Float32Array[],
+  recLength: number,
+  recordSampleRate: number,
+  exportSampleRate: number,
+) => {
+  const mergedBuffers = mergeBuffers(recBuffer, recLength);
+  const downsampledBuffer = downsampleBuffer(mergedBuffers, recordSampleRate, exportSampleRate);
+  const encodedWav = encodeWAV(downsampledBuffer, exportSampleRate);
+  const audioBlob = new Blob([encodedWav], {
+    type: RECORDER_EXPORT_MIME_TYPE,
+  });
+  return audioBlob;
+};

packages/amplify-ui-components/src/common/audio-control/index.ts

@@ -0,0 +1,3 @@
+export * from './recorder';
+export * from './helper';
+export * from './visualizer';

packages/amplify-ui-components/src/common/audio-control/recorder.ts

@@ -0,0 +1,210 @@
+import { exportBuffer } from './helper';
+import { browserOrNode, Logger } from '@aws-amplify/core';
+import {
+  DEFAULT_EXPORT_SAMPLE_RATE,
+  FFT_MAX_DECIBELS,
+  FFT_MIN_DECIBELS,
+  FFT_SIZE,
+  FFT_SMOOTHING_TIME_CONSTANT,
+} from './settings';
+
+interface SilenceDetectionConfig {
+  time: number;
+  amplitude: number;
+}
+
+type SilenceHandler = () => void;
+type Visualizer = (dataArray: Uint8Array, bufferLength: number) => void;
+const logger = new Logger('AudioRecorder');
+
+export class AudioRecorder {
+  private options: SilenceDetectionConfig;
+  private audioContext: AudioContext;
+  private audioSupported: boolean;
+
+  private analyserNode: AnalyserNode;
+  private onSilence: SilenceHandler;
+  private visualizer: Visualizer;
+
+  // input mic stream is stored in a buffer
+  private streamBuffer: Float32Array[] = [];
+  private streamBufferLength = 0;
+
+  // recording props
+  private start: number;
+  private recording = false;
+
+  constructor(options: SilenceDetectionConfig) {
+    this.options = options;
+  }
+
+  /**
+   * This must be called first to enable audio context and request microphone access.
+   * Once access granted, it connects all the necessary audio nodes to the context so that it can begin recording or playing.
+   */
+  async init() {
+    if (browserOrNode().isBrowser) {
+      window.AudioContext = window.AudioContext || (window as any).webkitAudioContext;
+      this.audioContext = new AudioContext();
+      await navigator.mediaDevices
+        .getUserMedia({ audio: true })
+        .then(stream => {
+          this.audioSupported = true;
+          this.setupAudioNodes(stream);
+        })
+        .catch(() => {
+          this.audioSupported = false;
+          return Promise.reject('Audio is not supported');
+        });
+    } else {
+      this.audioSupported = false;
+      return Promise.reject('Audio is not supported');
+    }
+  }
+
+  /**
+   * Setup audio nodes after successful `init`.
+   */
+  private async setupAudioNodes(stream: MediaStream) {
+    try {
+      await this.audioContext.resume();
+    } catch (err) {
+      logger.error(err);
+    }
+    const sourceNode = this.audioContext.createMediaStreamSource(stream);
+    const processorNode = this.audioContext.createScriptProcessor(4096, 1, 1);
+
+    processorNode.onaudioprocess = audioProcessingEvent => {
+      if (!this.recording) return;
+      const stream = audioProcessingEvent.inputBuffer.getChannelData(0);
+      this.streamBuffer.push(new Float32Array(stream)); // set to a copy of the stream
+      this.streamBufferLength += stream.length;
+      this.analyse();
+    };
+
+    const analyserNode = this.audioContext.createAnalyser();
+    analyserNode.minDecibels = FFT_MIN_DECIBELS;
+    analyserNode.maxDecibels = FFT_MAX_DECIBELS;
+    analyserNode.smoothingTimeConstant = FFT_SMOOTHING_TIME_CONSTANT;
+
+    sourceNode.connect(analyserNode);
+    analyserNode.connect(processorNode);
+    processorNode.connect(sourceNode.context.destination);
+
+    this.analyserNode = analyserNode;
+  }
+
+  /**
+   * Start recording audio and listen for silence.
+   *
+   * @param onSilence {SilenceHandler} - called whenever silence is detected
+   * @param visualizer {Visualizer} - called with audio data on each audio process to be used for visualization.
+   */
+  public async startRecording(onSilence?: SilenceHandler, visualizer?: Visualizer) {
+    if (this.recording || !this.audioSupported) return;
+    this.onSilence = onSilence || function() {};
+    this.visualizer = visualizer || function() {};
+
+    const context = this.audioContext;
+    try {
+      await context.resume();
+    } catch (err) {
+      logger.error(err);
+    }
+    this.start = Date.now();
+    this.recording = true;
+  }
+
+  /**
+   * Pause recording
+   */
+  public stopRecording() {
+    if (!this.audioSupported) return;
+    this.recording = false;
+  }
+
+  /**
+   * Pause recording and clear audio buffer
+   */
+  public clear() {
+    this.stopRecording();
+    this.streamBufferLength = 0;
+    this.streamBuffer = [];
+  }
+
+  /**
+   * Plays given audioStream with audioContext
+   *
+   * @param buffer {Uint8Array} - audioStream to be played
+   */
+  public play(buffer: Uint8Array) {
+    if (!buffer || !this.audioSupported) return;
+    const myBlob = new Blob([buffer]);
+
+    return new Promise((res, rej) => {
+      const fileReader = new FileReader();
+      fileReader.onload = () => {
+        const playbackSource = this.audioContext.createBufferSource();
+
+        const successCallback = (buf: AudioBuffer) => {
+          playbackSource.buffer = buf;
+          playbackSource.connect(this.audioContext.destination);
+          playbackSource.onended = () => {
+            return res();
+          };
+          playbackSource.start(0);
+        };
+        const errorCallback = err => {
+          return rej(err);
+        };
+
+        this.audioContext.decodeAudioData(fileReader.result as ArrayBuffer, successCallback, errorCallback);
+      };
+      fileReader.onerror = () => rej();
+      fileReader.readAsArrayBuffer(myBlob);
+    });
+  }
+
+  /**
+   * Called after each audioProcess. Check for silence and give fft time domain data to visualizer.
+   */
+  private analyse() {
+    if (!this.audioSupported) return;
+    const analyser = this.analyserNode;
+    analyser.fftSize = FFT_SIZE;
+
+    const bufferLength = analyser.fftSize;
+    const dataArray = new Uint8Array(bufferLength);
+    const amplitude = this.options.amplitude;
+    const time = this.options.time;
+
+    analyser.getByteTimeDomainData(dataArray);
+    this.visualizer(dataArray, bufferLength);
+
+    for (let i = 0; i < bufferLength; i++) {
+      // Normalize between -1 and 1.
+      const curr_value_time = dataArray[i] / 128 - 1.0;
+      if (curr_value_time > amplitude || curr_value_time < -1 * amplitude) {
+        this.start = Date.now();
+      }
+    }
+    const newtime = Date.now();
+    const elapsedTime = newtime - this.start;
+    if (elapsedTime > time) {
+      this.onSilence();
+    }
+  }
+
+  /**
+   * Encodes recorded buffer to a wav file and exports it to a blob.
+   *
+   * @param exportSampleRate {number} - desired sample rate of the exported buffer
+   */
+  public async exportWAV(exportSampleRate: number = DEFAULT_EXPORT_SAMPLE_RATE) {
+    if (!this.audioSupported) return;
+    const recordSampleRate = this.audioContext.sampleRate;
+    const blob = exportBuffer(this.streamBuffer, this.streamBufferLength, recordSampleRate, exportSampleRate);
+    this.clear();
+    return blob;
+  }
+}

packages/amplify-ui-components/src/common/audio-control/settings.ts

@@ -0,0 +1,8 @@
+// AudioRecorder settings
+export const RECORDER_EXPORT_MIME_TYPE = 'application/octet-stream';
+export const DEFAULT_EXPORT_SAMPLE_RATE = 16000;
+
+export const FFT_SIZE = 2048; // window size in samples for Fast Fourier Transform (FFT)
+export const FFT_MAX_DECIBELS = -10; // maximum power value in the scaling range for the FFT analysis data
+export const FFT_MIN_DECIBELS = -90; // minimum power value in the scaling range for the FFT analysis data
+export const FFT_SMOOTHING_TIME_CONSTANT = 0.85; // averaging constant with the last analysis frame