import type * as p from "@bokehjs/core/properties" import {HTMLBox, HTMLBoxView} from "./layout" const iconStarted = ` ` const iconNotStarted = ` ` const titleStarted = "Click to STOP the speech recognition." const titleNotStarted = "Click to START the speech recognition." // Hack inspired by https://stackoverflow.com/questions/38087013/angular2-web-speech-api-voice-recognition interface IWindow extends Window { webkitSpeechRecognition: any webkitSpeechGrammarList: any } const {webkitSpeechRecognition}: IWindow = (window as unknown) as IWindow const {webkitSpeechGrammarList}: IWindow = (window as unknown) as IWindow function htmlToElement(html: string) { const template = document.createElement("template") html = html.trim() // Never return a text node of whitespace as the result template.innerHTML = html return template.content.firstChild as HTMLElement } function deserializeGrammars(grammars: any[]) { if (grammars) { const speechRecognitionList = new webkitSpeechGrammarList() for (const grammar of grammars) { if (grammar.src) { speechRecognitionList.addFromString(grammar.src, grammar.weight) } else if (grammar.uri) { speechRecognitionList.addFromURI(grammar.uri, grammar.weight) } } return speechRecognitionList } else { return null } } function round(value: number) { return Math.round((value + Number.EPSILON) * 100) / 100 } function serializeResults(results_: any) { const results = [] for (const result of results_) { const alternatives: { confidence: number, transcript: string }[] = [] const item = {is_final: result.isFinal, alternatives} for (let i = 0; i < result.length; i++) { const alternative = { confidence: round(result[i].confidence), transcript: result[i].transcript, } alternatives.push(alternative) } item.alternatives = alternatives results.push(item) } return results } export class SpeechToTextView extends HTMLBoxView { declare model: SpeechToText recognition: any buttonEl: HTMLElement override initialize(): void { super.initialize() this.recognition = new webkitSpeechRecognition() this.recognition.lang = this.model.lang this.recognition.continuous = this.model.continuous this.recognition.interimResults = this.model.interim_results this.recognition.maxAlternatives = this.model.max_alternatives this.recognition.serviceURI = this.model.service_uri this.setGrammars() this.recognition.onresult = (event: any) => { this.model.results = serializeResults(event.results) } this.recognition.onerror = (event: any) => { console.error(`SpeechToText Error: ${event}`) } this.recognition.onnomatch = (event: any) => { console.warn(`SpeechToText No Match: ${event}`) } this.recognition.onaudiostart = () => this.model.audio_started = true this.recognition.onaudioend = () => this.model.audio_started = false this.recognition.onsoundstart = () => this.model.sound_started = true this.recognition.onsoundend = () => this.model.sound_started = false this.recognition.onspeechstart = () => this.model.speech_started=true this.recognition.onspeechend = () => this.model.speech_started=false this.recognition.onstart = () => { this.buttonEl.onclick = () => { this.recognition.stop() } this.buttonEl.innerHTML = this.iconStarted() this.buttonEl.setAttribute("title", titleStarted) this.model.started = true } this.recognition.onend = () => { this.buttonEl.onclick = () => { this.recognition.start() } this.buttonEl.innerHTML = this.iconNotStarted() this.buttonEl.setAttribute("title", titleNotStarted) this.model.started = false } this.buttonEl = htmlToElement(``) this.buttonEl.innerHTML = this.iconNotStarted() this.buttonEl.onclick = () => this.recognition.start() } iconStarted(): string { if (this.model.button_started!=="") { return this.model.button_started } else { return iconStarted } } iconNotStarted(): string { if (this.model.button_not_started!=="") { return this.model.button_not_started } else { return iconNotStarted } } setIcon(): void { if (this.model.started) { this.buttonEl.innerHTML = this.iconStarted() } else { this.buttonEl.innerHTML = this.iconNotStarted() } } override connect_signals(): void { super.connect_signals() const { start, stop, abort, grammars, lang, continuous, interim_results, max_alternatives, service_uri, button_type, button_hide, button_not_started, button_started, } = this.model.properties this.on_change(start, () => { this.model.start = false this.recognition.start() }) this.on_change(stop, () => { this.model.stop = false this.recognition.stop() }) this.on_change(abort, () => { this.model.abort = false this.recognition.abort() }) this.on_change(grammars, () => this.setGrammars()) this.on_change(lang, () => this.recognition.lang = this.model.lang) this.on_change(continuous, () => this.recognition.continuous = this.model.continuous) this.on_change(interim_results, () => this.recognition.interimResults = this.model.interim_results) this.on_change(max_alternatives, () => this.recognition.maxAlternatives = this.model.max_alternatives) this.on_change(service_uri, () => this.recognition.serviceURI = this.model.service_uri) this.on_change(button_type, () => this.buttonEl.className = `bk bk-btn bk-btn-${this.model.button_type}`) this.on_change(button_hide, () => this.render()) this.on_change([button_not_started, button_started], () => this.setIcon()) } setGrammars(): void { this.recognition.grammars = deserializeGrammars(this.model.grammars) } override render(): void { super.render() if (!this.model.button_hide) { this.shadow_el.appendChild(this.buttonEl) } } } export namespace SpeechToText { export type Attrs = p.AttrsOf export type Props = HTMLBox.Props & { start: p.Property stop: p.Property abort: p.Property grammars: p.Property lang: p.Property continuous: p.Property interim_results: p.Property max_alternatives: p.Property service_uri: p.Property started: p.Property audio_started: p.Property sound_started: p.Property speech_started: p.Property button_type: p.Property button_hide: p.Property button_not_started: p.Property button_started: p.Property results: p.Property } } export interface SpeechToText extends SpeechToText.Attrs {} export class SpeechToText extends HTMLBox { declare properties: SpeechToText.Props constructor(attrs?: Partial) { super(attrs) } static override __module__ = "panel.models.speech_to_text" static { this.prototype.default_view = SpeechToTextView this.define(({Any, List, Bool, Float, Str}) => ({ start: [ Bool, false ], stop: [ Bool, false ], abort: [ Bool, false ], grammars: [ List(Any), [] ], lang: [ Str, "" ], continuous: [ Bool, false ], interim_results: [ Bool, false ], max_alternatives: [ Float, 1 ], service_uri: [ Str, "" ], started: [ Bool, false ], audio_started: [ Bool, false ], sound_started: [ Bool, false ], speech_started: [ Bool, false ], button_type: [ Str, "light" ], button_hide: [ Bool, false ], button_not_started: [ Str, "" ], button_started: [ Str, "" ], results: [ List(Any), [] ], })) } }