Streamline frontend and incorporate alternative interfaces
Committed ec0c08
Make microphone icon color change with status
-
Implement voice commands behind "ghost" wake word
query for
go [home|back|forward]
follow
tell me
Add "follow links" functionality a la Vimium (only behind voice)
Modernize javascript
Combine scripts under single
type=moduleCombine load hooks
Combine key bindings
Move all stylesheets and scripts to head
--- a/canopy/static/screen.css
+++ b/canopy/static/screen.css
}
}
+.follows {
+ color: #b58900;
+ font-weight: bold;
+ padding: .25em; }
+
#connection {
color: #c36;
font-weight: bold;
--- a/canopy/templates/template.html
+++ b/canopy/templates/template.html
$def with (resource)
$ path = tx.request.uri.path
$ owner = tx.host.owner
-
<html lang=en-us class=dark>
+
<head>
<meta charset=utf-8>
<meta name=viewport
- content=initial-scale=1.0,user-scalable=no,maximum-scale=1,width=device-width>
+ content=initial-scale=1.0,user-scalable=no,maximum-scale=1,width=device-width>
<meta name=apple-mobile-web-app-capable content=yes>
<meta name=apple-mobile-web-app-status-bar-style content=black-translucent>
-$if "photo" in owner:
- <link rel=icon href=/media/$owner["photo"][0]>
-$if not tx.request.uri.path:
- <link rel=manifest href=/manifest.json>
-<link rel=stylesheet href=/chats/mediasoup-demo-app.css>
-<link rel=stylesheet href=/static/screen.css media=screen>
<title>\
$if "title" in resource:
$:Document(resource.title).doc.text_content() — \
$owner["name"][0]</title>
-
+$if "photo" in owner:
+ <link rel=icon href=/media/$owner["photo"][0]>
+$if not tx.request.uri.path:
+ <link rel=manifest href=/manifest.json>
+<link rel=stylesheet media=screen href=/chats/mediasoup-demo-app.css>
+<link rel=stylesheet media=screen href=/static/asciinema-player.css>
+<link rel=stylesheet media=screen href=/static/screen.css>
$if not isinstance(resource, str) and "head" in resource:
$:resource.head()
-
-<link rel=stylesheet href=/static/asciinema-player.css>
+$if tx.user.is_owner:
+ <style>
+ .h-card img {
+ border: .2em solid #007ba7;
+ height: 2.6em;
+ width: 2.6em; }
+ </style>
+<script src=//cdn.jsdelivr.net/npm/vosk-browser@latest/dist/vosk.js></script>
+<script src=//cdn.jsdelivr.net/npm/webtorrent@latest/webtorrent.min.js></script>
+<script src=//cdn.jsdelivr.net/npm/drag-drop@latest/dragdrop.min.js></script>
<script type=module>
import { _, cookies, go, upgradeLink } from '/static/web.js'
+
$if "session" in tx.user and tx.user.session.get("uid", None):
let userName = '$tx.user.session["name"][0]'
$else:
let userName = 'Guest'
+
// TODO $$.load(...)
document.addEventListener('DOMContentLoaded', ev => {
+ microphoneStatus('off')
+ listen.onmouseup = initDictation
+ join.onmouseup = initChat
+
+ _('a:not(.breakout)').each(upgradeLink)
+ history.pushState({scroll: 0}, 'title', window.location)
+ window.go = go
+
+ const client = new WebTorrent()
+ DragDrop('body', (files, pos, fileList, directories) => {
+ console.log(files[0])
+ // TODO client.seed(files, torrent => {
+ // TODO console.log('Client is seeding ' + torrent.magnetURI)
+ // TODO })
+ })
+
if (cookies.get('rhythm') == 'on') {
document.querySelector('body').style.backgroundImage = 'url(/static/measure.png)'
}
- document.addEventListener('keydown', (e) => {
- if (e.ctrlKey && e.key == '.') {
+ cookies.set('mediasoup-demo.user', `{"displayName": "$${userName}"}`)
+
+ document.addEventListener('keydown', ev => {
+ const em = parseFloat(getComputedStyle(document.documentElement).fontSize)
+ if (ev.ctrlKey && ev.key == '.') { // C^. toggle rhythm indicator
if (cookies.get('rhythm') == 'on') {
document.querySelector('body').style.backgroundImage = 'none'
cookies.set('rhythm', 'off')
document.querySelector('body').style.backgroundImage = 'url(/static/measure.png)'
cookies.set('rhythm', 'on')
}
+ } else if (ev.key == 'f') { // f TODO "follow links"
+ } else if (ev.key == 'h') { // h go back
+ history.back()
+ } else if (ev.key == 'j') { // j scroll down 3 EMs
+ document.documentElement.scrollTop += 3 * em
+ } else if (ev.key == 'k') { // k scroll up 3 EMs
+ document.documentElement.scrollTop -= 3 * em
+ } else if (ev.key == 'l') { // l go forward
+ history.forward()
+ } else if (ev.key == 'w') { // w TODO move up (in room)
+ } else if (ev.key == 'a') { // a TODO move left (in room)
+ } else if (ev.key == 's') { // s TODO move down (in room)
+ } else if (ev.key == 'd') { // d TODO move right (in room)
}
- })
- cookies.set('mediasoup-demo.user', `{"displayName": "$${userName}"}`)
- // TODO const owner = "$owner['name'][0]"
- _('a:not(.breakout)').each(upgradeLink)
- history.pushState({scroll: 0}, 'title', window.location)
- window.go = go
+ }, false)
})
-</script>
-$if tx.user.is_owner:
- <style>
- .h-card img {
- border: .2em solid #007ba7;
- height: 2.6em;
- width: 2.6em; }
- </style>
+const initDictation = async () => {
+ microphoneStatus('loading')
+ const partialContainer = document.querySelector('#search .partial')
+
+ const channel = new MessageChannel()
+ const model = await Vosk.createModel('/static/vosk-model-small-en-us-0.15.tar.gz')
+ model.registerPort(channel.port1)
+
+ const sampleRate = 48000
+
+ const recognizer = new model.KaldiRecognizer(sampleRate)
+ recognizer.setWords(true)
+
+ let ownerGivenName = '$owner["name"][0].split()[0]'
+ let wakeWord = 'ghost'
+ let readyWord = 'Say "ghost help"'
+ let state = 'asleep'
+ let following = false
+ let followList = {}
+ microphoneStatus('on')
+ partialContainer.innerHTML = readyWord
+
+ recognizer.on('result', message => {
+ let input = message.result.text
+ if (following && input != '') {
+ following = false
+ let number = input.replace(' ', '')
+ if (number == 'for') number = 'four'
+ go(followList[number])
+ document.querySelectorAll('.follows').forEach(e => e.remove())
+ return
+ }
+ if (input.slice(0, wakeWord.length) != wakeWord) {
+ partialContainer.innerHTML = readyWord
+ return
+ }
+ input = input.slice(wakeWord.length + 1)
+ microphoneStatus('on')
+ if (input.endsWith('cancel cancel')) {
+ partialContainer.innerHTML = ''
+ return
+ }
+ if (input == 'help') { // help
+ go('/help')
+ } else if (input.startsWith('query for')) { // query for
+ const query = input.slice(10)
+ go(`/search?q=$${query}`)
+ } else if (input.startsWith('go')) { // go
+ const where = input.slice(3)
+ if (where == 'home') // home
+ go('/')
+ else if (where == 'back') // back
+ history.back()
+ else if (where == 'forward') // forward
+ history.forward()
+ } else if (input.startsWith('follow')) { // follow
+ following = true
+ followList = {}
+ document.querySelectorAll('a').forEach((link, n) => {
+ followList[numberToWords(n).replace(' ', '')] = link.href
+ link.innerHTML = link.innerHTML +
+ `<span class=follows>$${n}</span>`
+ })
+ } else if (input.startsWith('tell me')) { // tell me
+ const request = input.slice(8)
+ const response = fetch('/ai/assistant', {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({request: request}),
+ })
+ .then(response => response.blob())
+ .then(blob => {
+ const audio = new Audio(URL.createObjectURL(blob));
+ audio.addEventListener('canplaythrough', () => {
+ state = 'asleep'
+ microphoneStatus('on')
+ audio.play()
+ // partialContainer.value = readyWord
+ })
+ })
+ }
+ partialContainer.innerHTML = `$${input}`
+ })
+ recognizer.on('partialresult', message => {
+ const input = message.result.partial
+ if (input.slice(0, wakeWord.length) != wakeWord || !following)
+ return
+ state = 'awake'
+ microphoneStatus('active')
+ partialContainer.innerHTML = input.slice(wakeWord.length)
+ })
+
+ const mediaStream = await navigator.mediaDevices.getUserMedia({
+ video: false,
+ audio: {
+ echoCancellation: true,
+ noiseSuppression: true,
+ channelCount: 1,
+ sampleRate
+ },
+ })
+
+ const audioContext = new AudioContext()
+ await audioContext.audioWorklet.addModule('/static/recognizer-processor.js')
+ const recognizerProcessor = new AudioWorkletNode(
+ audioContext,
+ 'recognizer-processor',
+ { channelCount: 1, numberOfInputs: 1, numberOfOutputs: 1 }
+ )
+ recognizerProcessor.port.postMessage(
+ { action: 'init', recognizerId: recognizer.id },
+ [ channel.port2 ]
+ )
+ recognizerProcessor.connect(audioContext.destination)
+
+ const source = audioContext.createMediaStreamSource(mediaStream)
+ source.connect(recognizerProcessor)
+}
+
+const initChat = () => {
+ join.disabled = true
+ join.style.display = 'none'
+
+ const tag = document.createElement('script')
+ tag.src = '/chats/mediasoup-demo-app.js'
+ document.getElementsByTagName('head')[0].appendChild(tag)
+
+ const autoMuter = setInterval(autoMute, 100)
+ const autoMute = () => {
+ if (typeof window.CLIENT !== 'undefined' && window.CLIENT._micProducer) {
+ window.CLIENT.muteMic()
+ clearInterval(autoMuter)
+ }
+ }
+}
+
+const numberToWords = num => {
+ if (num === 0)
+ return 'zero'
+ const units = ['one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine']
+ const teens = ['eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen']
+ const tens = ['', 'ten', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety']
+ const convert = n => {
+ if (n === 0) {
+ return ''
+ } else if (n < 10) {
+ return units[n - 1] + ' '
+ } else if (n == 10) {
+ return 'ten'
+ } else if (n < 20) {
+ return teens[n - 11] + ' '
+ } else if (n < 100) {
+ return tens[Math.floor(n / 10)] + ' ' + convert(n % 10)
+ } else {
+ return units[Math.floor(n / 100)-1] + ' hundred ' + convert(n % 100)
+ }
+ }
+ return convert(num).trim()
+}
+const microphoneStatus = mode => {
+ let color
+ let icon
+ if (mode == 'on' || mode == 'active') {
+ if (mode == 'on')
+ color = 'dc322f'
+ else if (mode == 'active')
+ color = '268bd2'
+ icon = `<path d="M7 7.40991C7 6.08383 7.52677 4.81207 8.46445 3.87439C9.40213 2.93671 10.6739 2.40991 12 2.40991C13.3261 2.40991 14.5978 2.93671 15.5355 3.87439C16.4732 4.81207 17 6.08383 17 7.40991V13.4099C17 14.736 16.4732 16.0079 15.5355 16.9456C14.5978 17.8832 13.3261 18.4099 12 18.4099C10.6739 18.4099 9.40213 17.8832 8.46445 16.9456C7.52677 16.0079 7 14.736 7 13.4099V7.40991Z" stroke="#$${color}" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/><path d="M21 13.4099C21 15.7969 20.0518 18.0861 18.364 19.7739C16.6761 21.4618 14.3869 22.4099 12 22.4099C9.61305 22.4099 7.32384 21.4618 5.63602 19.7739C3.94819 18.0861 3 15.7969 3 13.4099" stroke="#$${color}" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>`
+ } else if (mode == 'off' || mode == 'loading') {
+ if (mode == 'off')
+ color = '586e75'
+ else if (mode == 'loading')
+ color = '6c71c4'
+ icon = `<path d="M17.0005 11.24V13C17.0005 14.3261 16.4737 15.5978 15.536 16.5355C14.5983 17.4732 13.3266 18 12.0005 18C11.4846 17.9975 10.972 17.9166 10.4805 17.76" stroke="#$${color}" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/><path d="M8 16C7.35089 15.1345 7 14.0819 7 13V7C7 5.67392 7.52677 4.40216 8.46445 3.46448C9.40213 2.5268 10.6739 2 12 2C13.3261 2 14.5978 2.5268 15.5355 3.46448C16.4732 4.40216 17 5.67392 17 7" stroke="#$${color}" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/><path d="M5.21081 18.84C3.81268 17.216 3.04593 15.1429 3.0508 13" stroke="#$${color}" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/><path d="M21.0007 13C20.9995 14.5822 20.5812 16.1361 19.788 17.5051C18.9948 18.8741 17.8547 20.0098 16.4827 20.7977C15.1107 21.5857 13.5551 21.9979 11.973 21.993C10.3908 21.9882 8.83786 21.5664 7.4707 20.77" stroke="#$${color}" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/><path d="M22 2L2 22" stroke="#$${color}" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>`
+ }
+ listen.innerHTML = icon
+}
+</script>
+<script src=/chats/resources/js/antiglobal.js></script>
+<script>
+window.localStorage.setItem('debug', '* -engine* -socket* -RIE* *WARN* *ERROR*')
+if (window.antiglobal) {
+ window.antiglobal('___browserSync___oldSocketIo', 'io', '___browserSync___', '__core-js_shared__')
+ setInterval(window.antiglobal, 180000)
+}
+</script>
</head>
+
<body
$if "body_classes" in resource:
class="$' '.join(resource.body_classes)"\
$continue
<li><a href=/$mount_app.prefix>$mount_name.capitalize()</a></li>
</ul>
-
<div style=font-size:.8em>
<button id=join>Join Room</button>
$if tx.user.is_owner:
<button>Sign Out</button>
$else:
<form id=identity action=/guests/sign-in>
- <!--input name=me type=text--> <button>Sign In</button>
+ <button>Sign In</button>
</form>
</div>
</nav>
<aside>
$:livestream()
-
<div id=chat>
<div id=mediasoup-demo-app-media-query-detector></div>
<div id=mediasoup-demo-app-container></div>
</div>
-
<div class=page-related>
$if "aside" in resource:
$:resource.aside
</aside>
<footer>
-
<p><a href=http://creativecommons.org/licenses/by-nc-sa/4.0 target=_blank
rel="license noopener noreferrer" style=display:inline-block
title="Attribution-NonCommercial-ShareAlike 4.0 International"><img
class=cc src=/static/cc.svg> <img class=cc src=/static/by.svg> <img
class=cc src=/static/nc.svg> <img class=cc src=/static/sa.svg></a><br>
-
<small><a class=systemtext href=/system
title="system details">powered by canopy</a></small></p>
-
</footer>
-<script src=https://cdn.jsdelivr.net/npm/vosk-browser@0.0.5/dist/vosk.js></script>
-<script>
-async function initAssistant() {
- const partialContainer = document.querySelector('#search .partial')
-
- partialContainer.value = 'Loading...'
-
- const channel = new MessageChannel()
- const model = await Vosk.createModel('/static/vosk-model-small-en-us-0.15.tar.gz')
- model.registerPort(channel.port1)
-
- const sampleRate = 48000
-
- const recognizer = new model.KaldiRecognizer(sampleRate)
- recognizer.setWords(true)
-
- let ownerGivenName = '$owner["name"][0].split()[0]'
- let wakeWord = 'query plus'
- let readyWord = `Say "query plus help".`
- let state = 'asleep'
- partialContainer.value = readyWord
-
- recognizer.on('result', message => {
- let input = message.result.text
- if (input.slice(0, wakeWord.length) != wakeWord) {
- partialContainer.value = readyWord
- return
- }
- input = input.slice(wakeWord.length + 1)
- if (input == 'help') {
- go('/help')
- return
- }
- const response = fetch('/ai/assistant', {
- method: 'POST',
- headers: {
- 'Content-Type': 'application/json',
- },
- body: JSON.stringify({request: input}),
- })
- .then(response => response.blob())
- .then(blob => {
- const audio = new Audio(URL.createObjectURL(blob));
- audio.addEventListener('canplaythrough', () => {
- audio.play()
- state = 'asleep'
- partialContainer.value = readyWord
- })
- })
- partialContainer.value = `$${input}`
- })
- recognizer.on('partialresult', message => {
- const input = message.result.partial
- if (input.slice(0, wakeWord.length) != wakeWord)
- return
- state = 'awake'
- partialContainer.value = input.slice(wakeWord.length)
- })
-
- const mediaStream = await navigator.mediaDevices.getUserMedia({
- video: false,
- audio: {
- echoCancellation: true,
- noiseSuppression: true,
- channelCount: 1,
- sampleRate
- },
- })
-
- const audioContext = new AudioContext()
- await audioContext.audioWorklet.addModule('/static/recognizer-processor.js')
- const recognizerProcessor = new AudioWorkletNode(
- audioContext,
- 'recognizer-processor',
- { channelCount: 1, numberOfInputs: 1, numberOfOutputs: 1 }
- )
- recognizerProcessor.port.postMessage(
- { action: 'init', recognizerId: recognizer.id },
- [ channel.port2 ]
- )
- recognizerProcessor.connect(audioContext.destination)
-
- const source = audioContext.createMediaStreamSource(mediaStream)
- source.connect(recognizerProcessor)
-}
-
-function initChat() {
- var tag = document.createElement('script')
- tag.src = '/chats/mediasoup-demo-app.js'
- document.getElementsByTagName('head')[0].appendChild(tag)
-
- const autoMuter = setInterval(autoMute, 100)
- function autoMute() {
- if (typeof window.CLIENT !== 'undefined' && window.CLIENT._micProducer) {
- window.CLIENT.muteMic()
- clearInterval(autoMuter)
- }
- }
-}
-
-window.onload = () => {
- const listenTrigger = document.querySelector('button#listen')
- listenTrigger.onmouseup = () => {
- listenTrigger.disabled = true
- listenTrigger.style.display = 'none'
- initAssistant()
- }
- const enterTrigger = document.querySelector('button#enter')
- enterTrigger.onmouseup = () => {
- enterTrigger.disabled = true
- enterTrigger.style.display = 'none'
- initChat()
- }
-
- document.addEventListener('keydown', e => {
- const keyCode = e.keyCode;
- const em = parseFloat(getComputedStyle(document.documentElement).fontSize)
- if (keyCode == 70) { // f
- // TODO follow links
- } else if (keyCode == 72) { // h
- history.back()
- } else if (keyCode == 74) { // j
- document.documentElement.scrollTop += 3 * em
- } else if (keyCode == 75) { // k
- document.documentElement.scrollTop -= 3 * em
- } else if (keyCode == 76) { // l
- history.forward()
- } else if (keyCode == 87) { // w
- // TODO move up (in room)
- } else if (keyCode == 65) { // a
- // TODO move left (in room)
- } else if (keyCode == 83) { // s
- // TODO move down (in room)
- } else if (keyCode == 68) { // d
- // TODO move right (in room)
- }
- }, false)
-}
-</script>
-
-<script src=/chats/resources/js/antiglobal.js></script>
-<script>
-window.localStorage.setItem('debug', '* -engine* -socket* -RIE* *WARN* *ERROR*')
-if (window.antiglobal) {
- window.antiglobal('___browserSync___oldSocketIo', 'io', '___browserSync___', '__core-js_shared__')
- setInterval(window.antiglobal, 180000)
-}
-</script>
-
-<script src=https://cdn.jsdelivr.net/npm/webtorrent@latest/webtorrent.min.js></script>
-<script src=https://cdn.jsdelivr.net/npm/drag-drop@latest/dragdrop.min.js></script>
-<script type=module>
-const client = new WebTorrent()
-DragDrop('body', (files, pos, fileList, directories) => {
- console.log(files[0])
- // client.seed(files, torrent => {
- // console.log('Client is seeding ' + torrent.magnetURI)
- // })
-})
-</script>
-
</body>
+
</html>