import type * as p from "@bokehjs/core/properties"
import {HTMLBox, HTMLBoxView} from "./layout"
const iconStarted = ``
const iconNotStarted = ``
const titleStarted = "Click to STOP the speech recognition."
const titleNotStarted = "Click to START the speech recognition."
// Hack inspired by https://stackoverflow.com/questions/38087013/angular2-web-speech-api-voice-recognition
interface IWindow extends Window {
webkitSpeechRecognition: any
webkitSpeechGrammarList: any
}
const {webkitSpeechRecognition}: IWindow = (window as unknown) as IWindow
const {webkitSpeechGrammarList}: IWindow = (window as unknown) as IWindow
function htmlToElement(html: string) {
const template = document.createElement("template")
html = html.trim() // Never return a text node of whitespace as the result
template.innerHTML = html
return template.content.firstChild as HTMLElement
}
function deserializeGrammars(grammars: any[]) {
if (grammars) {
const speechRecognitionList = new webkitSpeechGrammarList()
for (const grammar of grammars) {
if (grammar.src) {
speechRecognitionList.addFromString(grammar.src, grammar.weight)
} else if (grammar.uri) {
speechRecognitionList.addFromURI(grammar.uri, grammar.weight)
}
}
return speechRecognitionList
} else {
return null
}
}
function round(value: number) {
return Math.round((value + Number.EPSILON) * 100) / 100
}
function serializeResults(results_: any) {
const results = []
for (const result of results_) {
const alternatives: { confidence: number, transcript: string }[] = []
const item = {is_final: result.isFinal, alternatives}
for (let i = 0; i < result.length; i++) {
const alternative = {
confidence: round(result[i].confidence),
transcript: result[i].transcript,
}
alternatives.push(alternative)
}
item.alternatives = alternatives
results.push(item)
}
return results
}
export class SpeechToTextView extends HTMLBoxView {
declare model: SpeechToText
recognition: any
buttonEl: HTMLElement
override initialize(): void {
super.initialize()
this.recognition = new webkitSpeechRecognition()
this.recognition.lang = this.model.lang
this.recognition.continuous = this.model.continuous
this.recognition.interimResults = this.model.interim_results
this.recognition.maxAlternatives = this.model.max_alternatives
this.recognition.serviceURI = this.model.service_uri
this.setGrammars()
this.recognition.onresult = (event: any) => {
this.model.results = serializeResults(event.results)
}
this.recognition.onerror = (event: any) => {
console.error(`SpeechToText Error: ${event}`)
}
this.recognition.onnomatch = (event: any) => {
console.warn(`SpeechToText No Match: ${event}`)
}
this.recognition.onaudiostart = () => this.model.audio_started = true
this.recognition.onaudioend = () => this.model.audio_started = false
this.recognition.onsoundstart = () => this.model.sound_started = true
this.recognition.onsoundend = () => this.model.sound_started = false
this.recognition.onspeechstart = () => this.model.speech_started=true
this.recognition.onspeechend = () => this.model.speech_started=false
this.recognition.onstart = () => {
this.buttonEl.onclick = () => {
this.recognition.stop()
}
this.buttonEl.innerHTML = this.iconStarted()
this.buttonEl.setAttribute("title", titleStarted)
this.model.started = true
}
this.recognition.onend = () => {
this.buttonEl.onclick = () => {
this.recognition.start()
}
this.buttonEl.innerHTML = this.iconNotStarted()
this.buttonEl.setAttribute("title", titleNotStarted)
this.model.started = false
}
this.buttonEl = htmlToElement(``)
this.buttonEl.innerHTML = this.iconNotStarted()
this.buttonEl.onclick = () => this.recognition.start()
}
iconStarted(): string {
if (this.model.button_started!=="") {
return this.model.button_started
} else {
return iconStarted
}
}
iconNotStarted(): string {
if (this.model.button_not_started!=="") {
return this.model.button_not_started
} else {
return iconNotStarted
}
}
setIcon(): void {
if (this.model.started) {
this.buttonEl.innerHTML = this.iconStarted()
} else {
this.buttonEl.innerHTML = this.iconNotStarted()
}
}
override connect_signals(): void {
super.connect_signals()
const {
start, stop, abort, grammars, lang, continuous, interim_results, max_alternatives,
service_uri, button_type, button_hide, button_not_started, button_started,
} = this.model.properties
this.on_change(start, () => {
this.model.start = false
this.recognition.start()
})
this.on_change(stop, () => {
this.model.stop = false
this.recognition.stop()
})
this.on_change(abort, () => {
this.model.abort = false
this.recognition.abort()
})
this.on_change(grammars, () => this.setGrammars())
this.on_change(lang, () => this.recognition.lang = this.model.lang)
this.on_change(continuous, () => this.recognition.continuous = this.model.continuous)
this.on_change(interim_results, () => this.recognition.interimResults = this.model.interim_results)
this.on_change(max_alternatives, () => this.recognition.maxAlternatives = this.model.max_alternatives)
this.on_change(service_uri, () => this.recognition.serviceURI = this.model.service_uri)
this.on_change(button_type, () => this.buttonEl.className = `bk bk-btn bk-btn-${this.model.button_type}`)
this.on_change(button_hide, () => this.render())
this.on_change([button_not_started, button_started], () => this.setIcon())
}
setGrammars(): void {
this.recognition.grammars = deserializeGrammars(this.model.grammars)
}
override render(): void {
super.render()
if (!this.model.button_hide) {
this.shadow_el.appendChild(this.buttonEl)
}
}
}
export namespace SpeechToText {
export type Attrs = p.AttrsOf
export type Props = HTMLBox.Props & {
start: p.Property
stop: p.Property
abort: p.Property
grammars: p.Property
lang: p.Property
continuous: p.Property
interim_results: p.Property
max_alternatives: p.Property
service_uri: p.Property
started: p.Property
audio_started: p.Property
sound_started: p.Property
speech_started: p.Property
button_type: p.Property
button_hide: p.Property
button_not_started: p.Property
button_started: p.Property
results: p.Property
}
}
export interface SpeechToText extends SpeechToText.Attrs {}
export class SpeechToText extends HTMLBox {
declare properties: SpeechToText.Props
constructor(attrs?: Partial) {
super(attrs)
}
static override __module__ = "panel.models.speech_to_text"
static {
this.prototype.default_view = SpeechToTextView
this.define(({Any, List, Bool, Float, Str}) => ({
start: [ Bool, false ],
stop: [ Bool, false ],
abort: [ Bool, false ],
grammars: [ List(Any), [] ],
lang: [ Str, "" ],
continuous: [ Bool, false ],
interim_results: [ Bool, false ],
max_alternatives: [ Float, 1 ],
service_uri: [ Str, "" ],
started: [ Bool, false ],
audio_started: [ Bool, false ],
sound_started: [ Bool, false ],
speech_started: [ Bool, false ],
button_type: [ Str, "light" ],
button_hide: [ Bool, false ],
button_not_started: [ Str, "" ],
button_started: [ Str, "" ],
results: [ List(Any), [] ],
}))
}
}