From 11c87d92475e405e5227f86fa3e2a89ccf4a9f4c Mon Sep 17 00:00:00 2001 From: zorkow Date: Fri, 23 Jun 2023 12:11:21 +0200 Subject: [PATCH 01/26] Some new explorer experiments. --- ts/a11y/explorer.ts | 3 +- ts/a11y/explorer/ExplorerPool.ts | 18 ++--- ts/a11y/explorer/KeyExplorer.ts | 70 ++++++++++++------- ts/a11y/explorer/Region.ts | 2 +- ts/a11y/explorer/Walker.ts | 114 +++++++++++++++++++++++++++++++ 5 files changed, 173 insertions(+), 34 deletions(-) create mode 100644 ts/a11y/explorer/Walker.ts diff --git a/ts/a11y/explorer.ts b/ts/a11y/explorer.ts index 0ebf0f425..160e71bc6 100644 --- a/ts/a11y/explorer.ts +++ b/ts/a11y/explorer.ts @@ -199,7 +199,8 @@ export function ExplorerMathDocumentMixin { - let explorer = ke.SpeechExplorer.create( - doc, pool, doc.explorerRegions.brailleRegion, node, ...rest) as ke.SpeechExplorer; - explorer.speechGenerator.setOptions({automark: false as any, markup: 'none', - locale: 'nemeth', domain: 'default', - style: 'default', modality: 'braille'}); - explorer.showRegion = 'viewBraille'; - return explorer; - }, + // braille: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => { + // let explorer = ke.SpeechExplorer.create( + // doc, pool, doc.explorerRegions.brailleRegion, node, ...rest) as ke.SpeechExplorer; + // explorer.speechGenerator.setOptions({automark: false as any, markup: 'none', + // locale: 'nemeth', domain: 'default', + // style: 'default', modality: 'braille'}); + // explorer.showRegion = 'viewBraille'; + // return explorer; + // }, keyMagnifier: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => ke.Magnifier.create(doc, pool, doc.explorerRegions.magnifier, node, ...rest), mouseMagnifier: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ..._rest: any[]) => diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index 93e353e42..b678d19b5 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -28,6 +28,8 @@ import {Explorer, AbstractExplorer} from './Explorer.js'; import {ExplorerPool} from './ExplorerPool.js'; import {Sre} from '../sre.js'; +import {click, move} from './Walker.js'; + /** * Interface for keyboard explorers. Adds the necessary keyboard events. @@ -99,9 +101,13 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme */ protected events: [string, (x: Event) => void][] = super.Events().concat( - [['keydown', this.KeyDown.bind(this)], - ['focusin', this.FocusIn.bind(this)], - ['focusout', this.FocusOut.bind(this)]]); + [ + // ['keydown', move], + ['keydown', this.KeyDown.bind(this)], + ['click', ((e: Event) => click(this.node, e)).bind(this)], + ['focusin', this.FocusIn.bind(this)], + ['focusout', this.FocusOut.bind(this)] + ]); /** * The original tabindex value before explorer was attached. @@ -148,8 +154,10 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme super.Attach(); this.attached = true; this.oldIndex = this.node.tabIndex; - this.node.tabIndex = 1; - this.node.setAttribute('role', 'tree'); + this.node.tabIndex = 0; + this.node.setAttribute('role', 'application'); + // TODO: Get rid of this eventually! + this.node.setAttribute('data-shellac', ''); } /** @@ -189,7 +197,8 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme * @override */ public Move(key: number) { - let result = this.walker.move(key); + // let result = this.walker.move(key); + let result = move(key); if (result) { this.Update(); return; @@ -263,6 +272,7 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @override */ public Start() { + console.log(16); if (!this.attached) return; let options = this.getOptions(); if (!this.init) { @@ -274,6 +284,15 @@ export class SpeechExplorer extends AbstractKeyExplorer { // Important that both are in the same block so speech explorers // are restarted sequentially. this.Speech(this.walker); + }) + .then(() => Sre.setupEngine({automark: false as any, markup: 'none', + locale: 'nemeth', domain: 'default', + style: 'default', modality: 'braille'})) + .then(() => { + this.speechGenerator.setOptions({automark: false as any, markup: 'none', + locale: 'nemeth', domain: 'default', + style: 'default', modality: 'braille'}); + this.Speech(this.walker); this.Start(); }); }) @@ -281,17 +300,17 @@ export class SpeechExplorer extends AbstractKeyExplorer { return; } super.Start(); - this.speechGenerator = Sre.getSpeechGenerator('Direct'); - this.speechGenerator.setOptions(options); - this.walker = Sre.getWalker( - 'table', this.node, this.speechGenerator, this.highlighter, this.mml); - this.walker.activate(); - this.Update(); - if (this.document.options.a11y[this.showRegion]) { - SpeechExplorer.updatePromise.then( - () => this.region.Show(this.node, this.highlighter)); - } - this.restarted = true; + // this.speecGhenerator = Sre.getSpeechGenerator('Direct'); + // this.speechGenerator.setOptions(options); + // this.walker = Sre.getWalker( + // 'table', this.node, this.speechGenerator, this.highlighter, this.mml); + // this.walker.activate(); + // this.Update(); + // if (this.document.options.a11y[this.showRegion]) { + // SpeechExplorer.updatePromise.then( + // () => this.region.Show(this.node, this.highlighter)); + // } + // this.restarted = true; } @@ -349,6 +368,10 @@ export class SpeechExplorer extends AbstractKeyExplorer { */ public KeyDown(event: KeyboardEvent) { const code = event.keyCode; + console.log(event); + console.log(event.key); + console.log(event.code); + console.log(code); this.walker.modifier = event.shiftKey; if (code === 17) { speechSynthesis.cancel(); @@ -359,12 +382,11 @@ export class SpeechExplorer extends AbstractKeyExplorer { this.stopEvent(event); return; } - if (this.active) { - this.Move(code); - if (this.triggerLink(code)) return; - this.stopEvent(event); - return; - } + console.log(9); + move(event); + console.log(13); + if (this.triggerLink(code)) return; + this.stopEvent(event); if (code === 32 && event.shiftKey || code === 13) { this.Start(); this.stopEvent(event); @@ -376,6 +398,7 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @param {number} code The keycode of the last key pressed. */ protected triggerLink(code: number) { + console.log(15); if (code !== 13) { return false; } @@ -384,6 +407,7 @@ export class SpeechExplorer extends AbstractKeyExplorer { getAttribute('data-semantic-postfix')?. match(/(^| )link($| )/); if (focus) { + console.log(14); node.parentNode.dispatchEvent(new MouseEvent('click')); return true; } diff --git a/ts/a11y/explorer/Region.ts b/ts/a11y/explorer/Region.ts index de0fcbc26..33f139e6b 100644 --- a/ts/a11y/explorer/Region.ts +++ b/ts/a11y/explorer/Region.ts @@ -361,7 +361,7 @@ export class LiveRegion extends StringRegion { */ constructor(public document: A11yDocument) { super(document); - this.div.setAttribute('aria-live', 'assertive'); + // this.div.setAttribute('aria-live', 'assertive'); } } diff --git a/ts/a11y/explorer/Walker.ts b/ts/a11y/explorer/Walker.ts new file mode 100644 index 000000000..9ea4c8047 --- /dev/null +++ b/ts/a11y/explorer/Walker.ts @@ -0,0 +1,114 @@ +/************************************************************* + * + * Copyright (c) 2009-2023 The MathJax Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + * @fileoverview Aria Tree Walker. + * + * @author v.sorge@mathjax.org (Volker Sorge) + */ + +// Based on the shellac walker. + +const codeSelector = 'mjx-container[role="application"][data-shellac]'; +const nav = '[role="application"][data-shellac],[role="tree"],[role="group"],[role="treeitem"]'; + +function isCodeBlock(el) { + return el.matches(codeSelector); +} + +export function click(snippet, e) { + const clicked = e.target.closest(nav); + if (snippet.contains(clicked)) { + const prev = snippet.querySelector('[tabindex="0"][role="tree"],[tabindex="0"][role="group"],[tabindex="0"][role="treeitem"]'); + if (prev) { + prev.removeAttribute("tabindex"); + } + clicked.setAttribute("tabindex", "0"); + clicked.focus(); + e.preventDefault(); + } +} + +export function move(e) { + + function nextFocus() { + function nextSibling(el) { + const sib = el.nextElementSibling; + if (sib) { + if (sib.matches(nav)) { + return sib; + } else { + const sibChild = sib.querySelector(nav); + return sibChild ?? nextSibling(sib); + } + } else { + if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { + return nextSibling(el.parentElement); + } else { + return null; + } + } + } + + function prevSibling(el) { + const sib = el.previousElementSibling; + if (sib) { + if (sib.matches(nav)) { + return sib; + } else { + const sibChild = sib.querySelector(nav); + return sibChild ?? prevSibling(sib); + } + } else { + if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { + return prevSibling(el.parentElement); + } else { + return null; + } + } + } + + switch (event.key) { + case "ArrowDown": + e.preventDefault(); + return e.target.querySelector(nav); + case "ArrowUp": + e.preventDefault(); + return e.target.parentElement.closest(nav); + case "ArrowLeft": + e.preventDefault(); + return prevSibling(e.target); + case "ArrowRight": + e.preventDefault(); + return nextSibling(e.target); + default: + return; + } + } + + const next = nextFocus(); + + + if (next) { + e.target.removeAttribute("tabindex"); + next.setAttribute("tabindex", "0"); + next.focus(); + return true; + } + return false; +} From cf5d9d4c1fa7bead19b347ba0692f91befa0ae3e Mon Sep 17 00:00:00 2001 From: zorkow Date: Mon, 17 Jul 2023 13:12:05 +0200 Subject: [PATCH 02/26] Updates chtml focus outline. --- ts/output/chtml.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/ts/output/chtml.ts b/ts/output/chtml.ts index 27bf2abd4..e3fa86d5b 100644 --- a/ts/output/chtml.ts +++ b/ts/output/chtml.ts @@ -85,6 +85,7 @@ CommonOutputJax< 'white-space': 'nowrap' }, + 'mjx-container[jax="CHTML"] :focus': {'outline': 'solid 3px'}, 'mjx-container [space="1"]': {'margin-left': '.111em'}, 'mjx-container [space="2"]': {'margin-left': '.167em'}, 'mjx-container [space="3"]': {'margin-left': '.222em'}, From f4bf28bc5dd3643c7d5a454583de227d068576b7 Mon Sep 17 00:00:00 2001 From: zorkow Date: Mon, 17 Jul 2023 13:24:52 +0200 Subject: [PATCH 03/26] Adds corrected types for Walker. --- ts/a11y/explorer/KeyExplorer.ts | 38 +++++++++++++------------------ ts/a11y/explorer/Walker.ts | 40 ++++++++++++++++++--------------- 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index b678d19b5..3d0f23314 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -104,7 +104,7 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme [ // ['keydown', move], ['keydown', this.KeyDown.bind(this)], - ['click', ((e: Event) => click(this.node, e)).bind(this)], + ['click', ((e: MouseEvent) => click(this.node, e)).bind(this)], ['focusin', this.FocusIn.bind(this)], ['focusout', this.FocusOut.bind(this)] ]); @@ -196,16 +196,10 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme /** * @override */ - public Move(key: number) { - // let result = this.walker.move(key); - let result = move(key); - if (result) { - this.Update(); - return; - } - if (this.sound) { - this.NoMove(); - } + public Move(_key: number) { + // // let result = this.walker.move(key); + // let result = false; + // // let result = move(key); } /** @@ -272,7 +266,6 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @override */ public Start() { - console.log(16); if (!this.attached) return; let options = this.getOptions(); if (!this.init) { @@ -296,7 +289,6 @@ export class SpeechExplorer extends AbstractKeyExplorer { this.Start(); }); }) - .catch((error: Error) => console.log(error.message)); return; } super.Start(); @@ -368,10 +360,6 @@ export class SpeechExplorer extends AbstractKeyExplorer { */ public KeyDown(event: KeyboardEvent) { const code = event.keyCode; - console.log(event); - console.log(event.key); - console.log(event.code); - console.log(code); this.walker.modifier = event.shiftKey; if (code === 17) { speechSynthesis.cancel(); @@ -382,9 +370,17 @@ export class SpeechExplorer extends AbstractKeyExplorer { this.stopEvent(event); return; } - console.log(9); - move(event); - console.log(13); + // + let result = move(event); + if (result) { + this.region.Show(this.node, this.highlighter); + this.region.Update('hello'); + return; + } + if (this.sound) { + this.NoMove(); + } + // if (this.triggerLink(code)) return; this.stopEvent(event); if (code === 32 && event.shiftKey || code === 13) { @@ -398,7 +394,6 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @param {number} code The keycode of the last key pressed. */ protected triggerLink(code: number) { - console.log(15); if (code !== 13) { return false; } @@ -407,7 +402,6 @@ export class SpeechExplorer extends AbstractKeyExplorer { getAttribute('data-semantic-postfix')?. match(/(^| )link($| )/); if (focus) { - console.log(14); node.parentNode.dispatchEvent(new MouseEvent('click')); return true; } diff --git a/ts/a11y/explorer/Walker.ts b/ts/a11y/explorer/Walker.ts index 9ea4c8047..1a54251f1 100644 --- a/ts/a11y/explorer/Walker.ts +++ b/ts/a11y/explorer/Walker.ts @@ -27,12 +27,12 @@ const codeSelector = 'mjx-container[role="application"][data-shellac]'; const nav = '[role="application"][data-shellac],[role="tree"],[role="group"],[role="treeitem"]'; -function isCodeBlock(el) { +function isCodeBlock(el: HTMLElement) { return el.matches(codeSelector); } -export function click(snippet, e) { - const clicked = e.target.closest(nav); +export function click(snippet: HTMLElement, e: MouseEvent) { + const clicked = (e.target as HTMLElement).closest(nav) as HTMLElement; if (snippet.contains(clicked)) { const prev = snippet.querySelector('[tabindex="0"][role="tree"],[tabindex="0"][role="group"],[tabindex="0"][role="treeitem"]'); if (prev) { @@ -44,16 +44,16 @@ export function click(snippet, e) { } } -export function move(e) { +export function move(e: KeyboardEvent) { - function nextFocus() { - function nextSibling(el) { - const sib = el.nextElementSibling; + function nextFocus(): HTMLElement { + function nextSibling(el: HTMLElement): HTMLElement { + const sib = el.nextElementSibling as HTMLElement; if (sib) { if (sib.matches(nav)) { return sib; } else { - const sibChild = sib.querySelector(nav); + const sibChild = sib.querySelector(nav) as HTMLElement; return sibChild ?? nextSibling(sib); } } else { @@ -65,13 +65,13 @@ export function move(e) { } } - function prevSibling(el) { - const sib = el.previousElementSibling; + function prevSibling(el: HTMLElement): HTMLElement { + const sib = el.previousElementSibling as HTMLElement; if (sib) { if (sib.matches(nav)) { return sib; } else { - const sibChild = sib.querySelector(nav); + const sibChild = sib.querySelector(nav) as HTMLElement; return sibChild ?? prevSibling(sib); } } else { @@ -83,31 +83,35 @@ export function move(e) { } } - switch (event.key) { + const target = e.target as HTMLElement; + switch (e.key) { case "ArrowDown": e.preventDefault(); - return e.target.querySelector(nav); + return target.querySelector(nav); case "ArrowUp": e.preventDefault(); - return e.target.parentElement.closest(nav); + return target.parentElement.closest(nav); case "ArrowLeft": e.preventDefault(); - return prevSibling(e.target); + return prevSibling(target); case "ArrowRight": e.preventDefault(); - return nextSibling(e.target); + return nextSibling(target); default: - return; + return null; } } const next = nextFocus(); + const target = e.target as HTMLElement; if (next) { - e.target.removeAttribute("tabindex"); + target.removeAttribute("tabindex"); next.setAttribute("tabindex", "0"); next.focus(); + console.log(next.getAttribute('data-semantic-speech')); + console.log(next.getAttribute('data-semantic-braille')); return true; } return false; From 28418560d1d8498146ac8377260b75b7a2df87bb Mon Sep 17 00:00:00 2001 From: zorkow Date: Tue, 25 Jul 2023 22:33:15 +0200 Subject: [PATCH 04/26] Walker cleanup. --- ts/a11y/explorer/Walker.ts | 47 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/ts/a11y/explorer/Walker.ts b/ts/a11y/explorer/Walker.ts index 1a54251f1..5ebda8fb7 100644 --- a/ts/a11y/explorer/Walker.ts +++ b/ts/a11y/explorer/Walker.ts @@ -36,65 +36,65 @@ export function click(snippet: HTMLElement, e: MouseEvent) { if (snippet.contains(clicked)) { const prev = snippet.querySelector('[tabindex="0"][role="tree"],[tabindex="0"][role="group"],[tabindex="0"][role="treeitem"]'); if (prev) { - prev.removeAttribute("tabindex"); + prev.removeAttribute('tabindex'); } - clicked.setAttribute("tabindex", "0"); + clicked.setAttribute('tabindex', '0'); clicked.focus(); e.preventDefault(); } } export function move(e: KeyboardEvent) { - + function nextFocus(): HTMLElement { function nextSibling(el: HTMLElement): HTMLElement { const sib = el.nextElementSibling as HTMLElement; if (sib) { - if (sib.matches(nav)) { + if (sib.matches(nav)) { return sib; - } else { + } else { const sibChild = sib.querySelector(nav) as HTMLElement; return sibChild ?? nextSibling(sib); - } + } } else { - if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { + if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { return nextSibling(el.parentElement); - } else { + } else { return null; - } + } } } function prevSibling(el: HTMLElement): HTMLElement { const sib = el.previousElementSibling as HTMLElement; if (sib) { - if (sib.matches(nav)) { + if (sib.matches(nav)) { return sib; - } else { + } else { const sibChild = sib.querySelector(nav) as HTMLElement; return sibChild ?? prevSibling(sib); - } + } } else { - if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { + if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { return prevSibling(el.parentElement); - } else { + } else { return null; - } + } } } const target = e.target as HTMLElement; switch (e.key) { - case "ArrowDown": + case 'ArrowDown': e.preventDefault(); return target.querySelector(nav); - case "ArrowUp": + case 'ArrowUp': e.preventDefault(); return target.parentElement.closest(nav); - case "ArrowLeft": + case 'ArrowLeft': e.preventDefault(); return prevSibling(target); - case "ArrowRight": + case 'ArrowRight': e.preventDefault(); return nextSibling(target); default: @@ -103,12 +103,13 @@ export function move(e: KeyboardEvent) { } const next = nextFocus(); - - + + const target = e.target as HTMLElement; + console.log(0); if (next) { - target.removeAttribute("tabindex"); - next.setAttribute("tabindex", "0"); + target.removeAttribute('tabindex'); + next.setAttribute('tabindex', '0'); next.focus(); console.log(next.getAttribute('data-semantic-speech')); console.log(next.getAttribute('data-semantic-braille')); From 58aad7ffed6e079c28a8335a4409f30520db9115 Mon Sep 17 00:00:00 2001 From: zorkow Date: Fri, 28 Jul 2023 23:13:13 +0200 Subject: [PATCH 05/26] Initial attempt at providing subtitles. --- ts/a11y/explorer/Walker.ts | 4 +- ts/a11y/semantic-enrich.ts | 122 ++++++++++++++++++++++++++++++++----- 2 files changed, 110 insertions(+), 16 deletions(-) diff --git a/ts/a11y/explorer/Walker.ts b/ts/a11y/explorer/Walker.ts index 5ebda8fb7..63e744182 100644 --- a/ts/a11y/explorer/Walker.ts +++ b/ts/a11y/explorer/Walker.ts @@ -111,8 +111,8 @@ export function move(e: KeyboardEvent) { target.removeAttribute('tabindex'); next.setAttribute('tabindex', '0'); next.focus(); - console.log(next.getAttribute('data-semantic-speech')); - console.log(next.getAttribute('data-semantic-braille')); + // console.log(next.getAttribute('data-semantic-speech')); + // console.log(next.getAttribute('data-semantic-braille')); return true; } return false; diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts index 6dcfa748d..f42c6eedf 100644 --- a/ts/a11y/semantic-enrich.ts +++ b/ts/a11y/semantic-enrich.ts @@ -26,6 +26,7 @@ import {Handler} from '../core/Handler.js'; import {MathDocument, AbstractMathDocument, MathDocumentConstructor} from '../core/MathDocument.js'; import {MathItem, AbstractMathItem, STATE, newState} from '../core/MathItem.js'; import {MmlNode} from '../core/MmlTree/MmlNode.js'; +import {Attributes} from '../core/MmlTree/Attributes.js'; import {MathML} from '../input/mathml.js'; import {SerializedMmlVisitor} from '../core/MmlTree/SerializedMmlVisitor.js'; import {OptionList, expandable} from '../util/Options.js'; @@ -37,6 +38,8 @@ import {Sre} from './sre.js'; * The current speech setting for Sre */ let currentSpeech = 'none'; +let currentLocale = 'none'; +let currentBraille = 'none'; /** * Generic constructor for Mixins @@ -60,7 +63,7 @@ newState('ATTACHSPEECH', 155); export class enrichVisitor extends SerializedMmlVisitor { protected mactionId: number; - + public visitTree(node: MmlNode, math?: MathItem) { this.mactionId = 1; const mml = super.visitTree(node); @@ -134,6 +137,8 @@ export function EnrichedMathItemMixin, force: boolean = false) { if (this.state() >= STATE.ENRICHED) return; if (!this.isEscaped && (document.options.enableEnrichment || force)) { - if (document.options.sre.speech !== currentSpeech) { - currentSpeech = document.options.sre.speech; + // TODO: Sort out the loading of the locales better + // if (document.options.sre.speech !== currentSpeech) { + // currentSpeech = document.options.sre.speech; + // mathjax.retryAfter( + // Sre.setupEngine(document.options.sre).then( + // () => Sre.sreReady())); + // } + if (document.options.sre.locale !== currentLocale) { + currentLocale = document.options.sre.locale; + // TODO: Sort out the loading of the locales better mathjax.retryAfter( Sre.setupEngine(document.options.sre).then( () => Sre.sreReady())); } + if (document.options.sre.braille !== currentBraille) { + currentBraille = document.options.sre.braille; + // TODO: Sort out the loading of the locales better + mathjax.retryAfter( + Sre.setupEngine({ + locale: document.options.sre.braille, + domain: 'default', // speech rules domain + style: 'default', // speech rules style + modality: 'braille', + markup: 'none', + }) + .then(() => Sre.sreReady())); + } const math = new document.options.MathItem('', MmlJax); try { let mml; @@ -177,7 +203,20 @@ export function EnrichedMathItemMixin) { if (this.state() >= STATE.ATTACHSPEECH) return; const attributes = this.root.attributes; - const speech = (attributes.get('aria-label') || - this.getSpeech(this.root)) as string; + const speech = (attributes.get('aria-label') || this.label); + const braille = (attributes.get('aria-braillelabel') || this.braillelabel); + if (!speech && !braille) { + this.state(STATE.ATTACHSPEECH); + return; + } + const adaptor = document.adaptor; + const node = this.typesetRoot; if (speech) { - const adaptor = document.adaptor; - const node = this.typesetRoot; - adaptor.setAttribute(node, 'aria-label', speech); - for (const child of adaptor.childNodes(node) as N[]) { - adaptor.setAttribute(child, 'aria-hidden', 'true'); - } - this.outputData.speech = speech; + adaptor.setAttribute(node, 'aria-label', speech as string); + } + if (braille) { + adaptor.setAttribute(node, 'aria-braillelabel', braille as string); } + for (const child of adaptor.childNodes(node) as N[]) { + adaptor.setAttribute(child, 'aria-hidden', 'true'); + } + this.outputData.speech = speech; + this.outputData.braille = braille; this.state(STATE.ATTACHSPEECH); } @@ -249,6 +298,48 @@ export function EnrichedMathItemMixin).attachSpeech(this); From 8d60263c33ad5e56757dded893073255f7845501 Mon Sep 17 00:00:00 2001 From: zorkow Date: Sun, 30 Jul 2023 19:42:01 +0200 Subject: [PATCH 06/26] Refactors ssml computations to speech util. --- ts/a11y/SpeechUtil.ts | 137 ++++++++++++++++++++++++++++++ ts/a11y/explorer/KeyExplorer.ts | 6 ++ ts/a11y/explorer/Region.ts | 142 +------------------------------- ts/a11y/explorer/Walker.ts | 8 +- ts/a11y/semantic-enrich.ts | 16 ++-- 5 files changed, 162 insertions(+), 147 deletions(-) create mode 100644 ts/a11y/SpeechUtil.ts diff --git a/ts/a11y/SpeechUtil.ts b/ts/a11y/SpeechUtil.ts new file mode 100644 index 000000000..f94ef270f --- /dev/null +++ b/ts/a11y/SpeechUtil.ts @@ -0,0 +1,137 @@ +const ProsodyKeys = [ 'pitch', 'rate', 'volume' ]; + +interface ProsodyElement { + [propName: string]: string | boolean | number; + pitch?: number; + rate?: number; + volume?: number; +} + +export interface SsmlElement extends ProsodyElement { + [propName: string]: string | boolean | number; + pause?: string; + text?: string; + mark?: string; + character?: boolean; + kind?: string; +} + + /** + * Parses a string containing an ssml structure into a list of text strings + * with associated ssml annotation elements. + * + * @param {string} speech The speech string. + * @return {[string, SsmlElement[]]} The annotation structure. + */ + export function ssmlParsing(speech: string): [string, SsmlElement[]] { + let dp = new DOMParser(); + let xml = dp.parseFromString(speech, 'text/xml'); + let instr: SsmlElement[] = []; + let text: String[] = []; + recurseSsml(Array.from(xml.documentElement.childNodes), instr, text); + return [text.join(' '), instr]; + } + + /** + * Tail recursive combination of SSML components. + * + * @param {Node[]} nodes A list of SSML nodes. + * @param {SsmlElement[]} instr Accumulator for collating Ssml annotation + * elements. + * @param {String[]} text A list of text elements. + * @param {ProsodyElement?} prosody The currently active prosody elements. + */ +function recurseSsml(nodes: Node[], instr: SsmlElement[], text: String[], + prosody: ProsodyElement = {}) { + for (let node of nodes) { + if (node.nodeType === 3) { + let content = node.textContent.trim(); + if (content) { + text.push(content); + instr.push(Object.assign({text: content}, prosody)); + } + continue; + } + if (node.nodeType === 1) { + let element = node as Element; + let tag = element.tagName; + if (tag === 'speak') { + continue; + } + if (tag === 'prosody') { + recurseSsml( + Array.from(node.childNodes), instr, text, + getProsody(element, prosody)); + continue; + } + switch (tag) { + case 'break': + instr.push({pause: element.getAttribute('time')}); + break; + case 'mark': + instr.push({mark: element.getAttribute('name')}); + break; + case 'say-as': + let txt = element.textContent; + instr.push(Object.assign({text: txt, character: true}, prosody)); + text.push(txt); + break; + default: + break; + } + } + } + } + + /** + * Maps prosody types to scaling functions. + */ + // TODO: These should be tweaked after more testing. +const combinePros: {[key: string]: (x: number, sign: string) => number} = { + pitch: (x: number, _sign: string) => 1 * (x / 100), + volume: (x: number, _sign: string) => .5 * (x / 100), + rate: (x: number, _sign: string) => 1 * (x / 100) + }; + + /** + * Retrieves prosody annotations from and SSML node. + * @param {Element} element The SSML node. + * @param {ProsodyElement} prosody The prosody annotation. + */ + function getProsody(element: Element, prosody: ProsodyElement) { + let combine: ProsodyElement = {}; + for (let pros of ProsodyKeys) { + if (element.hasAttribute(pros)) { + let [sign, value] = extractProsody(element.getAttribute(pros)); + if (!sign) { + // TODO: Sort out the base value. It is .5 for volume! + combine[pros] = (pros === 'volume') ? .5 : 1; + continue; + } + let orig = prosody[pros] as number; + orig = orig ? orig : ((pros === 'volume') ? .5 : 1); + let relative = combinePros[pros](parseInt(value, 10), sign); + combine[pros] = (sign === '-') ? orig - relative : orig + relative; + } + } + return combine; + } + + /** + * Extracts the prosody value from an attribute. + */ +const prosodyRegexp = /([\+|-]*)([0-9]+)%/; + +/** + * Extracts the prosody value from an attribute. + * @param {string} attr + */ +function extractProsody(attr: string) { + let match = attr.match(prosodyRegexp); + if (!match) { + console.warn('Something went wrong with the prosody matching.'); + return ['', '100']; + } + return [match[1], match[2]]; +} + diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index 3d0f23314..cd7f4cdcf 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -359,7 +359,9 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @override */ public KeyDown(event: KeyboardEvent) { + console.log(1); const code = event.keyCode; + console.log(2); this.walker.modifier = event.shiftKey; if (code === 17) { speechSynthesis.cancel(); @@ -372,7 +374,11 @@ export class SpeechExplorer extends AbstractKeyExplorer { } // let result = move(event); + console.log(3); if (result) { + console.log(4); + console.log(result); + console.log(this.region); this.region.Show(this.node, this.highlighter); this.region.Update('hello'); return; diff --git a/ts/a11y/explorer/Region.ts b/ts/a11y/explorer/Region.ts index 33f139e6b..3cf088601 100644 --- a/ts/a11y/explorer/Region.ts +++ b/ts/a11y/explorer/Region.ts @@ -26,6 +26,7 @@ import {MathDocument} from '../../core/MathDocument.js'; import {CssStyles} from '../../util/StyleList.js'; import {Sre} from '../sre.js'; +import {SsmlElement, ssmlParsing} from '../SpeechUtil.js'; export type A11yDocument = MathDocument; @@ -278,6 +279,8 @@ export class StringRegion extends AbstractRegion { * @override */ public Update(speech: string) { + console.log(6); + console.log(speech); this.inner.textContent = ''; this.inner.textContent = speech; } @@ -367,24 +370,6 @@ export class LiveRegion extends StringRegion { } -const ProsodyKeys = [ 'pitch', 'rate', 'volume' ]; - -interface ProsodyElement { - [propName: string]: string | boolean | number; - pitch?: number; - rate?: number; - volume?: number; -} - -interface SsmlElement extends ProsodyElement { - [propName: string]: string | boolean | number; - pause?: string; - text?: string; - mark?: string; - character?: boolean; - kind?: string; -} - /** * Region class that enables auto voicing of content via SSML markup. */ @@ -432,7 +417,7 @@ export class SpeechRegion extends LiveRegion { !!speechSynthesis.getVoices().length; speechSynthesis.cancel(); this.clear = true; - let [text, ssml] = this.ssmlParsing(speech); + let [text, ssml] = ssmlParsing(speech); super.Update(text); if (this.active && text) { this.makeUtterances(ssml, this.document.options.sre.locale); @@ -504,125 +489,6 @@ export class SpeechRegion extends LiveRegion { } - /** - * Parses a string containing an ssml structure into a list of text strings - * with associated ssml annotation elements. - * - * @param {string} speech The speech string. - * @return {[string, SsmlElement[]]} The annotation structure. - */ - private ssmlParsing(speech: string): [string, SsmlElement[]] { - let dp = new DOMParser(); - let xml = dp.parseFromString(speech, 'text/xml'); - let instr: SsmlElement[] = []; - let text: String[] = []; - this.recurseSsml(Array.from(xml.documentElement.childNodes), instr, text); - return [text.join(' '), instr]; - } - - /** - * Tail recursive combination of SSML components. - * - * @param {Node[]} nodes A list of SSML nodes. - * @param {SsmlElement[]} instr Accumulator for collating Ssml annotation - * elements. - * @param {String[]} text A list of text elements. - * @param {ProsodyElement?} prosody The currently active prosody elements. - */ - private recurseSsml(nodes: Node[], instr: SsmlElement[], text: String[], - prosody: ProsodyElement = {}) { - for (let node of nodes) { - if (node.nodeType === 3) { - let content = node.textContent.trim(); - if (content) { - text.push(content); - instr.push(Object.assign({text: content}, prosody)); - } - continue; - } - if (node.nodeType === 1) { - let element = node as Element; - let tag = element.tagName; - if (tag === 'speak') { - continue; - } - if (tag === 'prosody') { - this.recurseSsml( - Array.from(node.childNodes), instr, text, - this.getProsody(element, prosody)); - continue; - } - switch (tag) { - case 'break': - instr.push({pause: element.getAttribute('time')}); - break; - case 'mark': - instr.push({mark: element.getAttribute('name')}); - break; - case 'say-as': - let txt = element.textContent; - instr.push(Object.assign({text: txt, character: true}, prosody)); - text.push(txt); - break; - default: - break; - } - } - } - } - - /** - * Maps prosody types to scaling functions. - */ - // TODO: These should be tweaked after more testing. - private static combinePros: {[key: string]: (x: number, sign: string) => number} = { - pitch: (x: number, _sign: string) => 1 * (x / 100), - volume: (x: number, _sign: string) => .5 * (x / 100), - rate: (x: number, _sign: string) => 1 * (x / 100) - }; - - /** - * Retrieves prosody annotations from and SSML node. - * @param {Element} element The SSML node. - * @param {ProsodyElement} prosody The prosody annotation. - */ - private getProsody(element: Element, prosody: ProsodyElement) { - let combine: ProsodyElement = {}; - for (let pros of ProsodyKeys) { - if (element.hasAttribute(pros)) { - let [sign, value] = SpeechRegion.extractProsody(element.getAttribute(pros)); - if (!sign) { - // TODO: Sort out the base value. It is .5 for volume! - combine[pros] = (pros === 'volume') ? .5 : 1; - continue; - } - let orig = prosody[pros] as number; - orig = orig ? orig : ((pros === 'volume') ? .5 : 1); - let relative = SpeechRegion.combinePros[pros](parseInt(value, 10), sign); - combine[pros] = (sign === '-') ? orig - relative : orig + relative; - } - } - return combine; - } - - /** - * Extracts the prosody value from an attribute. - */ - private static prosodyRegexp = /([\+|-]*)([0-9]+)%/; - - /** - * Extracts the prosody value from an attribute. - * @param {string} attr - */ - private static extractProsody(attr: string) { - let match = attr.match(SpeechRegion.prosodyRegexp); - if (!match) { - console.warn('Something went wrong with the prosody matching.'); - return ['', '100']; - } - return [match[1], match[2]]; - } - } diff --git a/ts/a11y/explorer/Walker.ts b/ts/a11y/explorer/Walker.ts index 63e744182..e1202db73 100644 --- a/ts/a11y/explorer/Walker.ts +++ b/ts/a11y/explorer/Walker.ts @@ -111,9 +111,11 @@ export function move(e: KeyboardEvent) { target.removeAttribute('tabindex'); next.setAttribute('tabindex', '0'); next.focus(); - // console.log(next.getAttribute('data-semantic-speech')); - // console.log(next.getAttribute('data-semantic-braille')); - return true; + console.log(next.getAttribute('data-semantic-speech')); + console.log(next.getAttribute('aria-label')); + console.log(next.getAttribute('data-semantic-braille')); + console.log(next.getAttribute('aria-braillelabel')); + return [next]; } return false; } diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts index f42c6eedf..b23b63791 100644 --- a/ts/a11y/semantic-enrich.ts +++ b/ts/a11y/semantic-enrich.ts @@ -31,13 +31,14 @@ import {MathML} from '../input/mathml.js'; import {SerializedMmlVisitor} from '../core/MmlTree/SerializedMmlVisitor.js'; import {OptionList, expandable} from '../util/Options.js'; import {Sre} from './sre.js'; +import { ssmlParsing } from './SpeechUtil.js'; /*==========================================================================*/ /** * The current speech setting for Sre */ -let currentSpeech = 'none'; +// let currentSpeech = 'none'; let currentLocale = 'none'; let currentBraille = 'none'; @@ -299,18 +300,19 @@ export function EnrichedMathItemMixin Date: Mon, 31 Jul 2023 18:56:49 +0200 Subject: [PATCH 07/26] SSML extraction works. --- ts/a11y/SpeechUtil.ts | 214 ++++++++++++++++++-------------- ts/a11y/explorer/KeyExplorer.ts | 2 + ts/a11y/explorer/Walker.ts | 5 - ts/a11y/semantic-enrich.ts | 52 +++----- 4 files changed, 146 insertions(+), 127 deletions(-) diff --git a/ts/a11y/SpeechUtil.ts b/ts/a11y/SpeechUtil.ts index f94ef270f..baa377bd3 100644 --- a/ts/a11y/SpeechUtil.ts +++ b/ts/a11y/SpeechUtil.ts @@ -1,3 +1,6 @@ +import {MmlNode} from '../core/MmlTree/MmlNode.js'; +import {Sre} from './sre.js'; + const ProsodyKeys = [ 'pitch', 'rate', 'volume' ]; interface ProsodyElement { @@ -16,110 +19,112 @@ export interface SsmlElement extends ProsodyElement { kind?: string; } - /** - * Parses a string containing an ssml structure into a list of text strings - * with associated ssml annotation elements. - * - * @param {string} speech The speech string. - * @return {[string, SsmlElement[]]} The annotation structure. - */ - export function ssmlParsing(speech: string): [string, SsmlElement[]] { - let dp = new DOMParser(); - let xml = dp.parseFromString(speech, 'text/xml'); - let instr: SsmlElement[] = []; - let text: String[] = []; - recurseSsml(Array.from(xml.documentElement.childNodes), instr, text); - return [text.join(' '), instr]; - } +/** + * Parses a string containing an ssml structure into a list of text strings + * with associated ssml annotation elements. + * + * @param {string} speech The speech string. + * @return {[string, SsmlElement[]]} The annotation structure. + */ +export function ssmlParsing(speech: string): [string, SsmlElement[]] { + console.log(Sre.engineSetup()); - /** - * Tail recursive combination of SSML components. - * - * @param {Node[]} nodes A list of SSML nodes. - * @param {SsmlElement[]} instr Accumulator for collating Ssml annotation - * elements. - * @param {String[]} text A list of text elements. - * @param {ProsodyElement?} prosody The currently active prosody elements. - */ + let dp = new DOMParser(); + let xml = dp.parseFromString(speech, 'text/xml'); + let instr: SsmlElement[] = []; + let text: String[] = []; + recurseSsml(Array.from(xml.documentElement.childNodes), instr, text); + return [text.join(' '), instr]; +} + +/** + * Tail recursive combination of SSML components. + * + * @param {Node[]} nodes A list of SSML nodes. + * @param {SsmlElement[]} instr Accumulator for collating Ssml annotation + * elements. + * @param {String[]} text A list of text elements. + * @param {ProsodyElement?} prosody The currently active prosody elements. + */ function recurseSsml(nodes: Node[], instr: SsmlElement[], text: String[], - prosody: ProsodyElement = {}) { - for (let node of nodes) { - if (node.nodeType === 3) { - let content = node.textContent.trim(); - if (content) { - text.push(content); - instr.push(Object.assign({text: content}, prosody)); - } + prosody: ProsodyElement = {}) { + for (let node of nodes) { + if (node.nodeType === 3) { + let content = node.textContent.trim(); + if (content) { + text.push(content); + instr.push(Object.assign({text: content}, prosody)); + } + continue; + } + if (node.nodeType === 1) { + let element = node as Element; + let tag = element.tagName; + if (tag === 'speak') { + continue; + } + if (tag === 'prosody') { + recurseSsml( + Array.from(node.childNodes), instr, text, + getProsody(element, prosody)); continue; } - if (node.nodeType === 1) { - let element = node as Element; - let tag = element.tagName; - if (tag === 'speak') { - continue; - } - if (tag === 'prosody') { - recurseSsml( - Array.from(node.childNodes), instr, text, - getProsody(element, prosody)); - continue; - } - switch (tag) { - case 'break': - instr.push({pause: element.getAttribute('time')}); - break; - case 'mark': - instr.push({mark: element.getAttribute('name')}); - break; - case 'say-as': - let txt = element.textContent; - instr.push(Object.assign({text: txt, character: true}, prosody)); - text.push(txt); - break; - default: - break; - } + switch (tag) { + case 'break': + instr.push({pause: element.getAttribute('time')}); + break; + case 'mark': + instr.push({mark: element.getAttribute('name')}); + break; + case 'say-as': + let txt = element.textContent; + instr.push(Object.assign({text: txt, character: true}, prosody)); + text.push(txt); + break; + default: + break; } } } +} - /** - * Maps prosody types to scaling functions. - */ - // TODO: These should be tweaked after more testing. +/** + * Maps prosody types to scaling functions. + */ +// TODO: These should be tweaked after more testing. const combinePros: {[key: string]: (x: number, sign: string) => number} = { - pitch: (x: number, _sign: string) => 1 * (x / 100), - volume: (x: number, _sign: string) => .5 * (x / 100), - rate: (x: number, _sign: string) => 1 * (x / 100) - }; + pitch: (x: number, _sign: string) => 1 * (x / 100), + volume: (x: number, _sign: string) => .5 * (x / 100), + rate: (x: number, _sign: string) => 1 * (x / 100) +}; - /** - * Retrieves prosody annotations from and SSML node. - * @param {Element} element The SSML node. - * @param {ProsodyElement} prosody The prosody annotation. - */ - function getProsody(element: Element, prosody: ProsodyElement) { - let combine: ProsodyElement = {}; - for (let pros of ProsodyKeys) { - if (element.hasAttribute(pros)) { - let [sign, value] = extractProsody(element.getAttribute(pros)); - if (!sign) { - // TODO: Sort out the base value. It is .5 for volume! - combine[pros] = (pros === 'volume') ? .5 : 1; - continue; - } - let orig = prosody[pros] as number; - orig = orig ? orig : ((pros === 'volume') ? .5 : 1); - let relative = combinePros[pros](parseInt(value, 10), sign); - combine[pros] = (sign === '-') ? orig - relative : orig + relative; +/** + * Retrieves prosody annotations from and SSML node. + * @param {Element} element The SSML node. + * @param {ProsodyElement} prosody The prosody annotation. + */ +function getProsody(element: Element, prosody: ProsodyElement) { + let combine: ProsodyElement = {}; + for (let pros of ProsodyKeys) { + if (element.hasAttribute(pros)) { + let [sign, value] = extractProsody(element.getAttribute(pros)); + if (!sign) { + // TODO: Sort out the base value. It is .5 for volume! + combine[pros] = (pros === 'volume') ? .5 : 1; + continue; } + let orig = prosody[pros] as number; + orig = orig ? orig : ((pros === 'volume') ? .5 : 1); + let relative = combinePros[pros](parseInt(value, 10), sign); + combine[pros] = (sign === '-') ? orig - relative : orig + relative; } - return combine; } + return combine; +} - /** - * Extracts the prosody value from an attribute. - */ +/** + * Extracts the prosody value from an attribute. + */ const prosodyRegexp = /([\+|-]*)([0-9]+)%/; /** @@ -135,3 +140,32 @@ function extractProsody(attr: string) { return [match[1], match[2]]; } +export function getLabel(node: MmlNode, sep: string = ' ') { + const attributes = node.attributes; + const speech = attributes.getExplicit('data-semantic-speech') as string; + if (!speech) { + return ''; + } + const label = [speech]; + const prefix = attributes.getExplicit('data-semantic-prefix') as string; + if (prefix) { + label.unshift(prefix); + } + // TODO: check if we need this or if is automatic by the screen readers. + const postfix = attributes.getExplicit('data-semantic-postfix') as string; + if (postfix) { + label.push(postfix); + } + // TODO: Do we need to merge wrt. locale in SRE. + return label.join(sep); +} + + +export function buildSpeech(speech: string, locale: string = 'en', + rate: string = '100') { + return ssmlParsing('` + + `${speech}`+ + ''); +} diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index cd7f4cdcf..3067b099d 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -310,6 +310,7 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @override */ public Update(force: boolean = false) { + console.log(9); // TODO (v4): This is a hack to avoid double voicing on initial startup! // Make that cleaner and remove force as it is not really used! let noUpdate = force; @@ -330,6 +331,7 @@ export class SpeechExplorer extends AbstractKeyExplorer { modality: options.modality, locale: options.locale})) .then(() => { + console.log(10); if (!noUpdate) { let speech = this.walker.speech(); this.region.Update(speech); diff --git a/ts/a11y/explorer/Walker.ts b/ts/a11y/explorer/Walker.ts index e1202db73..380bda3b0 100644 --- a/ts/a11y/explorer/Walker.ts +++ b/ts/a11y/explorer/Walker.ts @@ -106,15 +106,10 @@ export function move(e: KeyboardEvent) { const target = e.target as HTMLElement; - console.log(0); if (next) { target.removeAttribute('tabindex'); next.setAttribute('tabindex', '0'); next.focus(); - console.log(next.getAttribute('data-semantic-speech')); - console.log(next.getAttribute('aria-label')); - console.log(next.getAttribute('data-semantic-braille')); - console.log(next.getAttribute('aria-braillelabel')); return [next]; } return false; diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts index b23b63791..d951fbdc4 100644 --- a/ts/a11y/semantic-enrich.ts +++ b/ts/a11y/semantic-enrich.ts @@ -26,12 +26,11 @@ import {Handler} from '../core/Handler.js'; import {MathDocument, AbstractMathDocument, MathDocumentConstructor} from '../core/MathDocument.js'; import {MathItem, AbstractMathItem, STATE, newState} from '../core/MathItem.js'; import {MmlNode} from '../core/MmlTree/MmlNode.js'; -import {Attributes} from '../core/MmlTree/Attributes.js'; import {MathML} from '../input/mathml.js'; import {SerializedMmlVisitor} from '../core/MmlTree/SerializedMmlVisitor.js'; import {OptionList, expandable} from '../util/Options.js'; import {Sre} from './sre.js'; -import { ssmlParsing } from './SpeechUtil.js'; +import { buildSpeech, getLabel } from './SpeechUtil.js'; /*==========================================================================*/ @@ -181,8 +180,11 @@ export function EnrichedMathItemMixin Sre.sreReady())); + () => { + console.log(18); + return Sre.sreReady(); })); } + console.log(Sre.engineSetup()); if (document.options.sre.braille !== currentBraille) { currentBraille = document.options.sre.braille; // TODO: Sort out the loading of the locales better @@ -194,7 +196,9 @@ export function EnrichedMathItemMixin Sre.sreReady())); + .then(() => { + console.log(19); + Sre.sreReady();})); } const math = new document.options.MathItem('', MmlJax); try { @@ -208,7 +212,9 @@ export function EnrichedMathItemMixin Date: Tue, 1 Aug 2023 00:31:28 +0200 Subject: [PATCH 08/26] Initial regions mainly working. --- ts/a11y/explorer/ExplorerPool.ts | 2 + ts/a11y/explorer/KeyExplorer.ts | 27 +++-- ts/a11y/explorer/Region.ts | 24 +++-- ts/a11y/explorer/Walker.ts | 166 +++++++++++++++++++------------ 4 files changed, 133 insertions(+), 86 deletions(-) diff --git a/ts/a11y/explorer/ExplorerPool.ts b/ts/a11y/explorer/ExplorerPool.ts index 9d3d7e50b..61edeae80 100644 --- a/ts/a11y/explorer/ExplorerPool.ts +++ b/ts/a11y/explorer/ExplorerPool.ts @@ -101,6 +101,8 @@ let allExplorers: {[options: string]: ExplorerInit} = { } explorer.sound = true; explorer.showRegion = 'subtitles'; + explorer.newWalker.speechRegion = doc.explorerRegions.speechRegion; + explorer.newWalker.brailleRegion = doc.explorerRegions.brailleRegion; return explorer; }, // braille: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => { diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index 3067b099d..ff10ec65a 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -28,7 +28,7 @@ import {Explorer, AbstractExplorer} from './Explorer.js'; import {ExplorerPool} from './ExplorerPool.js'; import {Sre} from '../sre.js'; -import {click, move} from './Walker.js'; +import { Walker } from './Walker.js'; /** @@ -78,6 +78,8 @@ export interface KeyExplorer extends Explorer { */ export abstract class AbstractKeyExplorer extends AbstractExplorer implements KeyExplorer { + public newWalker = new Walker(); + /** * Flag indicating if the explorer is attached to an object. */ @@ -104,7 +106,7 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme [ // ['keydown', move], ['keydown', this.KeyDown.bind(this)], - ['click', ((e: MouseEvent) => click(this.node, e)).bind(this)], + ['click', ((e: MouseEvent) => this.newWalker.click(this.node, e)).bind(this)], ['focusin', this.FocusIn.bind(this)], ['focusout', this.FocusOut.bind(this)] ]); @@ -310,7 +312,6 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @override */ public Update(force: boolean = false) { - console.log(9); // TODO (v4): This is a hack to avoid double voicing on initial startup! // Make that cleaner and remove force as it is not really used! let noUpdate = force; @@ -331,7 +332,6 @@ export class SpeechExplorer extends AbstractKeyExplorer { modality: options.modality, locale: options.locale})) .then(() => { - console.log(10); if (!noUpdate) { let speech = this.walker.speech(); this.region.Update(speech); @@ -361,9 +361,8 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @override */ public KeyDown(event: KeyboardEvent) { - console.log(1); const code = event.keyCode; - console.log(2); + console.log(event.key); this.walker.modifier = event.shiftKey; if (code === 17) { speechSynthesis.cancel(); @@ -372,17 +371,17 @@ export class SpeechExplorer extends AbstractKeyExplorer { if (code === 27) { this.Stop(); this.stopEvent(event); + this.newWalker.HideRegions(); return; } - // - let result = move(event); - console.log(3); + + let result = this.newWalker.move(event); + this.newWalker.ShowRegions(this.node, this.highlighter); if (result) { - console.log(4); - console.log(result); - console.log(this.region); - this.region.Show(this.node, this.highlighter); - this.region.Update('hello'); + // console.log(4); + // console.log(result); + // console.log(this.region); + // this.region.Update('hello'); return; } if (this.sound) { diff --git a/ts/a11y/explorer/Region.ts b/ts/a11y/explorer/Region.ts index 3cf088601..f26f6a92e 100644 --- a/ts/a11y/explorer/Region.ts +++ b/ts/a11y/explorer/Region.ts @@ -91,7 +91,7 @@ export abstract class AbstractRegion implements Region { * The outer div node. * @type {HTMLElement} */ - protected div: HTMLElement; + public div: HTMLElement; /** * The inner node. @@ -413,15 +413,19 @@ export class SpeechRegion extends LiveRegion { * @override */ public Update(speech: string) { - this.active = this.document.options.a11y.voicing && - !!speechSynthesis.getVoices().length; - speechSynthesis.cancel(); - this.clear = true; - let [text, ssml] = ssmlParsing(speech); - super.Update(text); - if (this.active && text) { - this.makeUtterances(ssml, this.document.options.sre.locale); - } + // console.log(speech); + // this.active = this.document.options.a11y.voicing && + // !!speechSynthesis.getVoices().length; + // speechSynthesis.cancel(); + // this.clear = true; + // let [text, ssml] = ssmlParsing(speech); + // console.log(27); + // console.log(text); + // super.Update(text); + // if (this.active && text) { + // this.makeUtterances(ssml, this.document.options.sre.locale); + // } + super.Update(speech); } /** diff --git a/ts/a11y/explorer/Walker.ts b/ts/a11y/explorer/Walker.ts index 380bda3b0..bc9cc377c 100644 --- a/ts/a11y/explorer/Walker.ts +++ b/ts/a11y/explorer/Walker.ts @@ -22,6 +22,9 @@ * @author v.sorge@mathjax.org (Volker Sorge) */ +import { SpeechRegion, LiveRegion } from './Region.js'; +import {Sre} from '../sre.js'; + // Based on the shellac walker. const codeSelector = 'mjx-container[role="application"][data-shellac]'; @@ -31,86 +34,125 @@ function isCodeBlock(el: HTMLElement) { return el.matches(codeSelector); } -export function click(snippet: HTMLElement, e: MouseEvent) { - const clicked = (e.target as HTMLElement).closest(nav) as HTMLElement; - if (snippet.contains(clicked)) { - const prev = snippet.querySelector('[tabindex="0"][role="tree"],[tabindex="0"][role="group"],[tabindex="0"][role="treeitem"]'); - if (prev) { - prev.removeAttribute('tabindex'); +export class Walker { + + public shown: boolean = false; + public speechRegion: SpeechRegion; + public brailleRegion: LiveRegion; + + public click(snippet: HTMLElement, e: MouseEvent) { + const clicked = (e.target as HTMLElement).closest(nav) as HTMLElement; + if (snippet.contains(clicked)) { + const prev = snippet.querySelector('[tabindex="0"][role="tree"],[tabindex="0"][role="group"],[tabindex="0"][role="treeitem"]'); + if (prev) { + prev.removeAttribute('tabindex'); + } + clicked.setAttribute('tabindex', '0'); + clicked.focus(); + this.UpdateRegions(clicked); + e.preventDefault(); } - clicked.setAttribute('tabindex', '0'); - clicked.focus(); - e.preventDefault(); } -} -export function move(e: KeyboardEvent) { + public move(e: KeyboardEvent) { - function nextFocus(): HTMLElement { - function nextSibling(el: HTMLElement): HTMLElement { - const sib = el.nextElementSibling as HTMLElement; - if (sib) { - if (sib.matches(nav)) { - return sib; - } else { - const sibChild = sib.querySelector(nav) as HTMLElement; - return sibChild ?? nextSibling(sib); - } - } else { - if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { - return nextSibling(el.parentElement); + function nextFocus(): HTMLElement { + function nextSibling(el: HTMLElement): HTMLElement { + const sib = el.nextElementSibling as HTMLElement; + if (sib) { + if (sib.matches(nav)) { + return sib; + } else { + const sibChild = sib.querySelector(nav) as HTMLElement; + return sibChild ?? nextSibling(sib); + } } else { - return null; + if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { + return nextSibling(el.parentElement); + } else { + return null; + } } } - } - function prevSibling(el: HTMLElement): HTMLElement { - const sib = el.previousElementSibling as HTMLElement; - if (sib) { - if (sib.matches(nav)) { - return sib; + function prevSibling(el: HTMLElement): HTMLElement { + const sib = el.previousElementSibling as HTMLElement; + if (sib) { + if (sib.matches(nav)) { + return sib; + } else { + const sibChild = sib.querySelector(nav) as HTMLElement; + return sibChild ?? prevSibling(sib); + } } else { - const sibChild = sib.querySelector(nav) as HTMLElement; - return sibChild ?? prevSibling(sib); + if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { + return prevSibling(el.parentElement); + } else { + return null; + } } - } else { - if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { - return prevSibling(el.parentElement); - } else { + } + + const target = e.target as HTMLElement; + console.log(e.key); + switch (e.key) { + case 'ArrowDown': + e.preventDefault(); + return target.querySelector(nav); + case 'ArrowUp': + e.preventDefault(); + return target.parentElement.closest(nav); + case 'ArrowLeft': + e.preventDefault(); + return prevSibling(target); + case 'ArrowRight': + e.preventDefault(); + return nextSibling(target); + case 'Esc': + e.preventDefault(); + return this.hideRegions(); + default: return null; - } } } + const next = nextFocus(); + const target = e.target as HTMLElement; - switch (e.key) { - case 'ArrowDown': - e.preventDefault(); - return target.querySelector(nav); - case 'ArrowUp': - e.preventDefault(); - return target.parentElement.closest(nav); - case 'ArrowLeft': - e.preventDefault(); - return prevSibling(target); - case 'ArrowRight': - e.preventDefault(); - return nextSibling(target); - default: - return null; + if (next) { + target.removeAttribute('tabindex'); + next.setAttribute('tabindex', '0'); + next.focus(); + this.UpdateRegions(next); + return [next]; } + return false; } - const next = nextFocus(); - + private UpdateRegions(element: HTMLElement) { + console.log(25); + console.log(element); + console.log(element.getAttribute('aria-label')); + console.log(element.getAttribute('aria-braillelabel')); + console.log(this.speechRegion); + this.speechRegion.Update(element.getAttribute('aria-label')); + this.brailleRegion.Update(element.getAttribute('aria-braillelabel')); + } - const target = e.target as HTMLElement; - if (next) { - target.removeAttribute('tabindex'); - next.setAttribute('tabindex', '0'); - next.focus(); - return [next]; + public ShowRegions(element: HTMLElement, highlighter: Sre.highlighter) { + if (!this.shown) { + this.speechRegion.Show(element, highlighter); + this.brailleRegion.Show(element, highlighter); + } + this.shown = true; + } + + public HideRegions() { + if (this.shown) { + this.speechRegion.Hide(); + this.brailleRegion.Hide(); + } + this.shown = false; } - return false; } + From 73beb396e6f4aa1a2a53b8090babbac9f5fdb576 Mon Sep 17 00:00:00 2001 From: zorkow Date: Tue, 1 Aug 2023 13:18:56 +0200 Subject: [PATCH 09/26] Messing with regions. --- ts/a11y/SpeechUtil.ts | 2 - ts/a11y/explorer/KeyExplorer.ts | 34 ++++++++-------- ts/a11y/explorer/Walker.ts | 72 ++++++++++++++++++++++++++++----- ts/a11y/semantic-enrich.ts | 7 ---- 4 files changed, 79 insertions(+), 36 deletions(-) diff --git a/ts/a11y/SpeechUtil.ts b/ts/a11y/SpeechUtil.ts index baa377bd3..a1f03a6ce 100644 --- a/ts/a11y/SpeechUtil.ts +++ b/ts/a11y/SpeechUtil.ts @@ -27,8 +27,6 @@ export interface SsmlElement extends ProsodyElement { * @return {[string, SsmlElement[]]} The annotation structure. */ export function ssmlParsing(speech: string): [string, SsmlElement[]] { - console.log(Sre.engineSetup()); - let dp = new DOMParser(); let xml = dp.parseFromString(speech, 'text/xml'); let instr: SsmlElement[] = []; diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index ff10ec65a..a94d232e7 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -132,7 +132,7 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme * @override */ public FocusOut(_event: FocusEvent) { - this.Stop(); + // this.Stop(); } /** @@ -260,6 +260,7 @@ export class SpeechExplorer extends AbstractKeyExplorer { protected node: HTMLElement, private mml: string) { super(document, pool, region, node); + this.newWalker.highlighter = this.highlighter; this.initWalker(); } @@ -268,6 +269,7 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @override */ public Start() { + console.log(this.attached); if (!this.attached) return; let options = this.getOptions(); if (!this.init) { @@ -361,36 +363,36 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @override */ public KeyDown(event: KeyboardEvent) { - const code = event.keyCode; - console.log(event.key); + console.log('active: ' + this.active); + const code = event.key; this.walker.modifier = event.shiftKey; - if (code === 17) { + if (code === 'Control') { speechSynthesis.cancel(); return; } - if (code === 27) { + if (code === 'Escape') { this.Stop(); this.stopEvent(event); this.newWalker.HideRegions(); return; } - - let result = this.newWalker.move(event); - this.newWalker.ShowRegions(this.node, this.highlighter); + let result = null; + if (this.active) { + result = this.newWalker.move(event); + this.newWalker.ShowRegions(this.node, this.highlighter); + this.stopEvent(event); + } if (result) { - // console.log(4); - // console.log(result); - // console.log(this.region); // this.region.Update('hello'); return; } - if (this.sound) { + if (!result && this.sound) { this.NoMove(); } // if (this.triggerLink(code)) return; - this.stopEvent(event); - if (code === 32 && event.shiftKey || code === 13) { + // this.stopEvent(event); + if (code === 'Space' && event.shiftKey || code === 'Enter') { this.Start(); this.stopEvent(event); } @@ -400,8 +402,8 @@ export class SpeechExplorer extends AbstractKeyExplorer { * Programmatically triggers a link if the focused node contains one. * @param {number} code The keycode of the last key pressed. */ - protected triggerLink(code: number) { - if (code !== 13) { + protected triggerLink(code: string) { + if (code !== 'Enter') { return false; } let node = this.walker.getFocus().getNodes()?.[0]; diff --git a/ts/a11y/explorer/Walker.ts b/ts/a11y/explorer/Walker.ts index bc9cc377c..20b4a29c9 100644 --- a/ts/a11y/explorer/Walker.ts +++ b/ts/a11y/explorer/Walker.ts @@ -34,12 +34,49 @@ function isCodeBlock(el: HTMLElement) { return el.matches(codeSelector); } +type rgbColor = {red: number, green: number, blue: number}; +type channelColor = {name: string, alpha: number}; + +export class Highlighter { + + public foreground: string; + public background: string; + + public static namedColors: { [key: string]: rgbColor } = { + red: { red: 255, green: 0, blue: 0 }, + green: { red: 0, green: 255, blue: 0 }, + blue: { red: 0, green: 0, blue: 255 }, + yellow: { red: 255, green: 255, blue: 0 }, + cyan: { red: 0, green: 255, blue: 255 }, + magenta: { red: 255, green: 0, blue: 255 }, + white: { red: 255, green: 255, blue: 255 }, + black: { red: 0, green: 0, blue: 0 } + }; + + constructor(foreground: channelColor, background: channelColor) { + this.foreground = this.makeColor(foreground); + this.background = this.makeColor(background); + } + + // public highlight(node: HTMLElement) { + + // } + + private makeColor({name: name, alpha: alpha}: channelColor) { + let {red: red, green: green, blue: blue} = + Highlighter.namedColors[name] || Highlighter.namedColors['blue']; + return `rgba(${red},${green},${blue},${alpha})`; + } + +} + export class Walker { public shown: boolean = false; public speechRegion: SpeechRegion; public brailleRegion: LiveRegion; - + public highlighter: Sre.highlighter; + public click(snippet: HTMLElement, e: MouseEvent) { const clicked = (e.target as HTMLElement).closest(nav) as HTMLElement; if (snippet.contains(clicked)) { @@ -94,7 +131,6 @@ export class Walker { } const target = e.target as HTMLElement; - console.log(e.key); switch (e.key) { case 'ArrowDown': e.preventDefault(); @@ -108,9 +144,9 @@ export class Walker { case 'ArrowRight': e.preventDefault(); return nextSibling(target); - case 'Esc': - e.preventDefault(); - return this.hideRegions(); + // case 'Esc': + // e.preventDefault(); + // return this.hideRegions(); default: return null; } @@ -123,18 +159,23 @@ export class Walker { target.removeAttribute('tabindex'); next.setAttribute('tabindex', '0'); next.focus(); + this.UpdateHighlight(target, next); this.UpdateRegions(next); return [next]; } return false; } + private info: Highlight; + + private UpdateHighlight(_target: HTMLElement, next: HTMLElement) { + if (this.info) { + (this.highlighter as any).unhighlightNode(this.info); + } + this.info = (this.highlighter as any).highlightNode(next); + } + private UpdateRegions(element: HTMLElement) { - console.log(25); - console.log(element); - console.log(element.getAttribute('aria-label')); - console.log(element.getAttribute('aria-braillelabel')); - console.log(this.speechRegion); this.speechRegion.Update(element.getAttribute('aria-label')); this.brailleRegion.Update(element.getAttribute('aria-braillelabel')); } @@ -146,7 +187,7 @@ export class Walker { } this.shown = true; } - + public HideRegions() { if (this.shown) { this.speechRegion.Hide(); @@ -156,3 +197,12 @@ export class Walker { } } +export interface Highlight { + node: HTMLElement; + opacity?: string; + background?: string; + foreground?: string; + // The following is for the CSS highlighter + box?: HTMLElement; + position?: string; +} diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts index d951fbdc4..752c414da 100644 --- a/ts/a11y/semantic-enrich.ts +++ b/ts/a11y/semantic-enrich.ts @@ -181,10 +181,8 @@ export function EnrichedMathItemMixin { - console.log(18); return Sre.sreReady(); })); } - console.log(Sre.engineSetup()); if (document.options.sre.braille !== currentBraille) { currentBraille = document.options.sre.braille; // TODO: Sort out the loading of the locales better @@ -197,7 +195,6 @@ export function EnrichedMathItemMixin { - console.log(19); Sre.sreReady();})); } const math = new document.options.MathItem('', MmlJax); @@ -227,13 +224,10 @@ export function EnrichedMathItemMixin).attachSpeech(this); From 62d0a6df706ab6970f30fefcb41c438b14df3262 Mon Sep 17 00:00:00 2001 From: zorkow Date: Thu, 3 Aug 2023 15:43:09 +0200 Subject: [PATCH 10/26] Remove as much as possible. --- ts/a11y/explorer/ExplorerPool.ts | 20 ++++++------- ts/a11y/explorer/KeyExplorer.ts | 49 ++++++++++++++++---------------- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/ts/a11y/explorer/ExplorerPool.ts b/ts/a11y/explorer/ExplorerPool.ts index 61edeae80..526b9f513 100644 --- a/ts/a11y/explorer/ExplorerPool.ts +++ b/ts/a11y/explorer/ExplorerPool.ts @@ -89,16 +89,16 @@ let allExplorers: {[options: string]: ExplorerInit} = { speech: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => { let explorer = ke.SpeechExplorer.create( doc, pool, doc.explorerRegions.speechRegion, node, ...rest) as ke.SpeechExplorer; - explorer.speechGenerator.setOptions({ - automark: true as any, markup: 'ssml', - locale: doc.options.sre.locale, domain: doc.options.sre.domain, - style: doc.options.sre.style, modality: 'speech'}); - // This weeds out the case of providing a non-existent locale option. - let locale = explorer.speechGenerator.getOptions().locale; - if (locale !== Sre.engineSetup().locale) { - doc.options.sre.locale = Sre.engineSetup().locale; - explorer.speechGenerator.setOptions({locale: doc.options.sre.locale}); - } + // explorer.speechGenerator.setOptions({ + // automark: true as any, markup: 'ssml', + // locale: doc.options.sre.locale, domain: doc.options.sre.domain, + // style: doc.options.sre.style, modality: 'speech'}); + // // This weeds out the case of providing a non-existent locale option. + // let locale = explorer.speechGenerator.getOptions().locale; + // if (locale !== Sre.engineSetup().locale) { + // doc.options.sre.locale = Sre.engineSetup().locale; + // explorer.speechGenerator.setOptions({locale: doc.options.sre.locale}); + // } explorer.sound = true; explorer.showRegion = 'subtitles'; explorer.newWalker.speechRegion = doc.explorerRegions.speechRegion; diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index a94d232e7..6e2eca388 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -269,33 +269,32 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @override */ public Start() { - console.log(this.attached); if (!this.attached) return; - let options = this.getOptions(); - if (!this.init) { - this.init = true; - SpeechExplorer.updatePromise = SpeechExplorer.updatePromise.then(async () => { - return Sre.sreReady() - .then(() => Sre.setupEngine({locale: options.locale})) - .then(() => { - // Important that both are in the same block so speech explorers - // are restarted sequentially. - this.Speech(this.walker); - }) - .then(() => Sre.setupEngine({automark: false as any, markup: 'none', - locale: 'nemeth', domain: 'default', - style: 'default', modality: 'braille'})) - .then(() => { - this.speechGenerator.setOptions({automark: false as any, markup: 'none', - locale: 'nemeth', domain: 'default', - style: 'default', modality: 'braille'}); - this.Speech(this.walker); - this.Start(); - }); - }) - return; - } super.Start(); + // let options = this.getOptions(); + // if (!this.init) { + // this.init = true; + // SpeechExplorer.updatePromise = SpeechExplorer.updatePromise.then(async () => { + // return Sre.sreReady() + // .then(() => Sre.setupEngine({locale: options.locale})) + // .then(() => { + // // Important that both are in the same block so speech explorers + // // are restarted sequentially. + // this.Speech(this.walker); + // }) + // .then(() => Sre.setupEngine({automark: false as any, markup: 'none', + // locale: 'nemeth', domain: 'default', + // style: 'default', modality: 'braille'})) + // .then(() => { + // this.speechGenerator.setOptions({automark: false as any, markup: 'none', + // locale: 'nemeth', domain: 'default', + // style: 'default', modality: 'braille'}); + // this.Speech(this.walker); + // this.Start(); + // }); + // }) + // return; + // } // this.speecGhenerator = Sre.getSpeechGenerator('Direct'); // this.speechGenerator.setOptions(options); // this.walker = Sre.getWalker( From cdc27101b3c5dd406a61acb0f27bffaf32ee6175 Mon Sep 17 00:00:00 2001 From: zorkow Date: Fri, 4 Aug 2023 18:23:00 +0200 Subject: [PATCH 11/26] Basics of new key explorer working. --- ts/a11y/explorer/ExplorerPool.ts | 12 +- ts/a11y/explorer/KeyExplorer.ts | 412 +++++++++++++++++++------------ ts/a11y/explorer/Region.ts | 6 +- 3 files changed, 259 insertions(+), 171 deletions(-) diff --git a/ts/a11y/explorer/ExplorerPool.ts b/ts/a11y/explorer/ExplorerPool.ts index 526b9f513..0892af401 100644 --- a/ts/a11y/explorer/ExplorerPool.ts +++ b/ts/a11y/explorer/ExplorerPool.ts @@ -88,7 +88,8 @@ type ExplorerInit = (doc: ExplorerMathDocument, pool: ExplorerPool, let allExplorers: {[options: string]: ExplorerInit} = { speech: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => { let explorer = ke.SpeechExplorer.create( - doc, pool, doc.explorerRegions.speechRegion, node, ...rest) as ke.SpeechExplorer; + doc, pool, doc.explorerRegions.speechRegion, node, + doc.explorerRegions.brailleRegion, doc.explorerRegions.magnifier, rest[0]) as ke.SpeechExplorer; // explorer.speechGenerator.setOptions({ // automark: true as any, markup: 'ssml', // locale: doc.options.sre.locale, domain: doc.options.sre.domain, @@ -100,9 +101,6 @@ let allExplorers: {[options: string]: ExplorerInit} = { // explorer.speechGenerator.setOptions({locale: doc.options.sre.locale}); // } explorer.sound = true; - explorer.showRegion = 'subtitles'; - explorer.newWalker.speechRegion = doc.explorerRegions.speechRegion; - explorer.newWalker.brailleRegion = doc.explorerRegions.brailleRegion; return explorer; }, // braille: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => { @@ -114,8 +112,8 @@ let allExplorers: {[options: string]: ExplorerInit} = { // explorer.showRegion = 'viewBraille'; // return explorer; // }, - keyMagnifier: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => - ke.Magnifier.create(doc, pool, doc.explorerRegions.magnifier, node, ...rest), + // keyMagnifier: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => + // ke.Magnifier.create(doc, pool, doc.explorerRegions.magnifier, node, ...rest), mouseMagnifier: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ..._rest: any[]) => me.ContentHoverer.create(doc, pool, doc.explorerRegions.magnifier, node, (x: HTMLElement) => x.hasAttribute('data-semantic-type'), @@ -235,7 +233,7 @@ export class ExplorerPool { let keyExplorers = []; for (let key of Object.keys(this.explorers)) { let explorer = this.explorers[key]; - if (explorer instanceof ke.AbstractKeyExplorer) { + if (explorer instanceof ke.SpeechExplorer) { explorer.AddEvents(); explorer.stoppable = false; keyExplorers.unshift(explorer); diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index 6e2eca388..40392d733 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -23,12 +23,12 @@ */ -import {A11yDocument, Region} from './Region.js'; +import {A11yDocument, Region, HoverRegion, SpeechRegion, LiveRegion} from './Region.js'; import {Explorer, AbstractExplorer} from './Explorer.js'; import {ExplorerPool} from './ExplorerPool.js'; import {Sre} from '../sre.js'; -import { Walker } from './Walker.js'; +// import { Walker } from './Walker.js'; /** @@ -60,7 +60,7 @@ export interface KeyExplorer extends Explorer { * Move made on keypress. * @param key The key code of the pressed key. */ - Move(key: number): void; + Move(event: KeyboardEvent): void; /** * A method that is executed if no move is executed. @@ -70,15 +70,22 @@ export interface KeyExplorer extends Explorer { } +const codeSelector = 'mjx-container[role="application"][data-shellac]'; +const nav = '[role="application"][data-shellac],[role="tree"],[role="group"],[role="treeitem"]'; + +function isCodeBlock(el: HTMLElement) { + return el.matches(codeSelector); +} + /** * @constructor * @extends {AbstractExplorer} * * @template T The type that is consumed by the Region of this explorer. */ -export abstract class AbstractKeyExplorer extends AbstractExplorer implements KeyExplorer { +export class SpeechExplorer extends AbstractExplorer implements KeyExplorer { - public newWalker = new Walker(); + // public newWalker = new Walker(); /** * Flag indicating if the explorer is attached to an object. @@ -98,6 +105,8 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme private eventsAttached: boolean = false; + protected current: HTMLElement = null; + /** * @override */ @@ -106,22 +115,31 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme [ // ['keydown', move], ['keydown', this.KeyDown.bind(this)], - ['click', ((e: MouseEvent) => this.newWalker.click(this.node, e)).bind(this)], + ['click', ((e: MouseEvent) => this.click(this.node, e)).bind(this)], ['focusin', this.FocusIn.bind(this)], ['focusout', this.FocusOut.bind(this)] ]); + public click(snippet: HTMLElement, e: MouseEvent) { + const clicked = (e.target as HTMLElement).closest(nav) as HTMLElement; + if (snippet.contains(clicked)) { + const prev = snippet.querySelector('[tabindex="0"][role="tree"],[tabindex="0"][role="group"],[tabindex="0"][role="treeitem"]'); + if (prev) { + prev.removeAttribute('tabindex'); + } + clicked.setAttribute('tabindex', '0'); + clicked.focus(); + this.current = clicked; + e.preventDefault(); + } + } + /** * The original tabindex value before explorer was attached. * @type {boolean} */ private oldIndex: number = null; - /** - * @override - */ - public abstract KeyDown(event: KeyboardEvent): void; - /** * @override */ @@ -135,20 +153,6 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme // this.Stop(); } - /** - * @override - */ - public Update(force: boolean = false) { - if (!this.active && !force) return; - this.pool.unhighlight(); - let nodes = this.walker.getFocus(true).getNodes(); - if (!nodes.length) { - this.walker.refocus(); - nodes = this.walker.getFocus().getNodes(); - } - this.pool.highlight(nodes as HTMLElement[]); - } - /** * @override */ @@ -187,21 +191,77 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme /** * @override */ - public Stop() { - if (this.active) { - this.walker.deactivate(); - this.pool.unhighlight(); + public Move(e: KeyboardEvent) { + function nextFocus(): HTMLElement { + function nextSibling(el: HTMLElement): HTMLElement { + const sib = el.nextElementSibling as HTMLElement; + if (sib) { + if (sib.matches(nav)) { + return sib; + } else { + const sibChild = sib.querySelector(nav) as HTMLElement; + return sibChild ?? nextSibling(sib); + } + } else { + if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { + return nextSibling(el.parentElement); + } else { + return null; + } + } + } + + function prevSibling(el: HTMLElement): HTMLElement { + const sib = el.previousElementSibling as HTMLElement; + if (sib) { + if (sib.matches(nav)) { + return sib; + } else { + const sibChild = sib.querySelector(nav) as HTMLElement; + return sibChild ?? prevSibling(sib); + } + } else { + if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { + return prevSibling(el.parentElement); + } else { + return null; + } + } + } + + const target = e.target as HTMLElement; + switch (e.key) { + case 'ArrowDown': + e.preventDefault(); + return target.querySelector(nav); + case 'ArrowUp': + e.preventDefault(); + return target.parentElement.closest(nav); + case 'ArrowLeft': + e.preventDefault(); + return prevSibling(target); + case 'ArrowRight': + e.preventDefault(); + return nextSibling(target); + // case 'Esc': + // e.preventDefault(); + // return this.hideRegions(); + default: + return null; + } } - super.Stop(); - } - /** - * @override - */ - public Move(_key: number) { - // // let result = this.walker.move(key); - // let result = false; - // // let result = move(key); + const next = nextFocus(); + + const target = e.target as HTMLElement; + if (next) { + target.removeAttribute('tabindex'); + next.setAttribute('tabindex', '0'); + next.focus(); + this.current = next; + return true; + } + return false; } /** @@ -216,16 +276,6 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme os.stop(ac.currentTime + .05); } -} - - -/** - * Explorer that pushes speech to live region. - * @constructor - * @extends {AbstractKeyExplorer} - */ -export class SpeechExplorer extends AbstractKeyExplorer { - private static updatePromise = Promise.resolve(); /** @@ -256,12 +306,13 @@ export class SpeechExplorer extends AbstractKeyExplorer { */ constructor(public document: A11yDocument, public pool: ExplorerPool, - public region: Region, + public region: SpeechRegion, protected node: HTMLElement, - private mml: string) { - super(document, pool, region, node); - this.newWalker.highlighter = this.highlighter; - this.initWalker(); + public brailleRegion: LiveRegion, + public magnifyRegion: HoverRegion, + private _mml: string) { + super(document, pool, null, node); + // this.initWalker(); } @@ -270,6 +321,7 @@ export class SpeechExplorer extends AbstractKeyExplorer { */ public Start() { if (!this.attached) return; + if (this.active) return; super.Start(); // let options = this.getOptions(); // if (!this.init) { @@ -295,16 +347,26 @@ export class SpeechExplorer extends AbstractKeyExplorer { // }) // return; // } - // this.speecGhenerator = Sre.getSpeechGenerator('Direct'); + // this.speechGenerator = Sre.getSpeechGenerator('Direct'); // this.speechGenerator.setOptions(options); // this.walker = Sre.getWalker( // 'table', this.node, this.speechGenerator, this.highlighter, this.mml); // this.walker.activate(); - // this.Update(); - // if (this.document.options.a11y[this.showRegion]) { - // SpeechExplorer.updatePromise.then( - // () => this.region.Show(this.node, this.highlighter)); - // } + if (this.document.options.a11y.subtitles) { + console.log(0); + SpeechExplorer.updatePromise.then( + () => this.region.Show(this.node, this.highlighter)) + } + if (this.document.options.a11y.viewBraille) { + console.log(1); + SpeechExplorer.updatePromise.then( + () => this.brailleRegion.Show(this.node, this.highlighter)) + } + if (this.document.options.a11y.keyMagnifier) { + console.log(2); + this.magnifyRegion.Show(this.node, this.highlighter); + } + this.Update(); // this.restarted = true; } @@ -317,28 +379,40 @@ export class SpeechExplorer extends AbstractKeyExplorer { // Make that cleaner and remove force as it is not really used! let noUpdate = force; force = false; - super.Update(force); - let options = this.speechGenerator.getOptions(); + if (!this.active && !force) return; + this.pool.unhighlight(); + // let nodes = this.walker.getFocus(true).getNodes(); + // if (!nodes.length) { + // this.walker.refocus(); + // nodes = this.walker.getFocus().getNodes(); + // } + this.pool.highlight([this.current]); + this.region.Update(this.current.getAttribute('aria-label')); + this.brailleRegion.Update(this.current.getAttribute('aria-braillelabel')); + console.log(this.magnifyRegion); + this.magnifyRegion.Update(this.current); + // let options = this.speechGenerator.getOptions(); // This is a necessary in case speech options have changed via keypress // during walking. - if (options.modality === 'speech') { - this.document.options.sre.domain = options.domain; - this.document.options.sre.style = options.style; - this.document.options.a11y.speechRules = - options.domain + '-' + options.style; - } - SpeechExplorer.updatePromise = SpeechExplorer.updatePromise.then(async () => { - return Sre.sreReady() - .then(() => Sre.setupEngine({markup: options.markup, - modality: options.modality, - locale: options.locale})) - .then(() => { - if (!noUpdate) { - let speech = this.walker.speech(); - this.region.Update(speech); - } - }); - }); + // if (options.modality === 'speech') { + // this.document.options.sre.domain = options.domain; + // this.document.options.sre.style = options.style; + // this.document.options.a11y.speechRules = + // options.domain + '-' + options.style; + // } + // Ensure this autovoicing is retained later: + // SpeechExplorer.updatePromise = SpeechExplorer.updatePromise.then(async () => { + // return Sre.sreReady() + // .then(() => Sre.setupEngine({markup: options.markup, + // modality: options.modality, + // locale: options.locale})) + // .then(() => { + // if (!noUpdate) { + // let speech = this.walker.speech(); + // this.region.Update(speech); + // } + // }); + // }); } @@ -362,9 +436,8 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @override */ public KeyDown(event: KeyboardEvent) { - console.log('active: ' + this.active); const code = event.key; - this.walker.modifier = event.shiftKey; + // this.walker.modifier = event.shiftKey; if (code === 'Control') { speechSynthesis.cancel(); return; @@ -372,24 +445,26 @@ export class SpeechExplorer extends AbstractKeyExplorer { if (code === 'Escape') { this.Stop(); this.stopEvent(event); - this.newWalker.HideRegions(); return; } let result = null; if (this.active) { - result = this.newWalker.move(event); - this.newWalker.ShowRegions(this.node, this.highlighter); + result = this.Move(event); this.stopEvent(event); } if (result) { - // this.region.Update('hello'); + this.Update(); return; } if (!result && this.sound) { this.NoMove(); } // - if (this.triggerLink(code)) return; + if (this.triggerLink(code)) { + this.Stop() + return; + } + // this.stopEvent(event); if (code === 'Space' && event.shiftKey || code === 'Enter') { this.Start(); @@ -402,10 +477,10 @@ export class SpeechExplorer extends AbstractKeyExplorer { * @param {number} code The keycode of the last key pressed. */ protected triggerLink(code: string) { - if (code !== 'Enter') { + if (code !== 'Enter' || !this.active) { return false; } - let node = this.walker.getFocus().getNodes()?.[0]; + let node = this.current; let focus = node?. getAttribute('data-semantic-postfix')?. match(/(^| )link($| )/); @@ -419,12 +494,12 @@ export class SpeechExplorer extends AbstractKeyExplorer { /** * Initialises the Sre walker. */ - private initWalker() { - this.speechGenerator = Sre.getSpeechGenerator('Tree'); - let dummy = Sre.getWalker( - 'dummy', this.node, this.speechGenerator, this.highlighter, this.mml); - this.walker = dummy; - } + // private initWalker() { + // this.speechGenerator = Sre.getSpeechGenerator('Tree'); + // let dummy = Sre.getWalker( + // 'dummy', this.node, this.speechGenerator, this.highlighter, this.mml); + // this.walker = dummy; + // } /** * Retrieves the speech options to sync with document options. @@ -446,78 +521,93 @@ export class SpeechExplorer extends AbstractKeyExplorer { return options; } -} - - -/** - * Explorer that magnifies what is currently explored. Uses a hover region. - * @constructor - * @extends {AbstractKeyExplorer} - */ -export class Magnifier extends AbstractKeyExplorer { - - /** - * @constructor - * @extends {AbstractKeyExplorer} - */ - constructor(public document: A11yDocument, - public pool: ExplorerPool, - public region: Region, - protected node: HTMLElement, - private mml: string) { - super(document, pool, region, node); - this.walker = Sre.getWalker( - 'table', this.node, Sre.getSpeechGenerator('Dummy'), - this.highlighter, this.mml); - } - /** * @override */ - public Update(force: boolean = false) { - super.Update(force); - this.showFocus(); - } - - /** - * @override - */ - public Start() { - super.Start(); - if (!this.attached) return; - this.region.Show(this.node, this.highlighter); - this.walker.activate(); - this.Update(); + public Stop() { + if (this.active) { + this.pool.unhighlight(); + this.magnifyRegion.Hide(); + this.region.Hide(); + this.brailleRegion.Hide(); + } + super.Stop(); } - /** - * Shows the nodes that are currently focused. - */ - private showFocus() { - let node = this.walker.getFocus().getNodes()[0] as HTMLElement; - this.region.Show(node, this.highlighter); - } - /** - * @override - */ - public KeyDown(event: KeyboardEvent) { - const code = event.keyCode; - this.walker.modifier = event.shiftKey; - if (code === 27) { - this.Stop(); - this.stopEvent(event); - return; - } - if (this.active && code !== 13) { - this.Move(code); - this.stopEvent(event); - return; - } - if (code === 32 && event.shiftKey || code === 13) { - this.Start(); - this.stopEvent(event); - } - } } + + +/** + * Explorer that magnifies what is currently explored. Uses a hover region. + * @constructor + * @extends {AbstractKeyExplorer} + */ +// export class Magnifier extends AbstractKeyExplorer { + +// /** +// * @constructor +// * @extends {AbstractKeyExplorer} +// */ +// constructor(public document: A11yDocument, +// public pool: ExplorerPool, +// public region: Region, +// protected node: HTMLElement, +// private mml: string) { +// super(document, pool, region, node); +// this.walker = Sre.getWalker( +// 'table', this.node, Sre.getSpeechGenerator('Dummy'), +// this.highlighter, this.mml); +// } + +// /** +// * @override +// */ +// public Update(force: boolean = false) { +// super.Update(force); +// this.showFocus(); +// } + +// /** +// * @override +// */ +// public Start() { +// super.Start(); +// if (!this.attached) return; +// this.region.Show(this.node, this.highlighter); +// this.walker.activate(); +// this.Update(); +// } + +// /** +// * Shows the nodes that are currently focused. +// */ +// private showFocus() { +// let node = this.walker.getFocus().getNodes()[0] as HTMLElement; +// this.region.Show(node, this.highlighter); +// } + +// /** +// * @override +// */ +// public KeyDown(event: KeyboardEvent) { +// const code = event.keyCode; +// this.walker.modifier = event.shiftKey; +// if (code === 27) { +// this.Stop(); +// this.stopEvent(event); +// return; +// } +// if (this.active && code !== 13) { +// this.Move(code); +// this.stopEvent(event); +// return; +// } +// if (code === 32 && event.shiftKey || code === 13) { +// this.Start(); +// this.stopEvent(event); +// } +// } + +// } diff --git a/ts/a11y/explorer/Region.ts b/ts/a11y/explorer/Region.ts index f26f6a92e..0732a25de 100644 --- a/ts/a11y/explorer/Region.ts +++ b/ts/a11y/explorer/Region.ts @@ -279,8 +279,6 @@ export class StringRegion extends AbstractRegion { * @override */ public Update(speech: string) { - console.log(6); - console.log(speech); this.inner.textContent = ''; this.inner.textContent = speech; } @@ -413,10 +411,12 @@ export class SpeechRegion extends LiveRegion { * @override */ public Update(speech: string) { + console.log('In Speech region: ' + speech); + // Temporarily removed! // console.log(speech); // this.active = this.document.options.a11y.voicing && // !!speechSynthesis.getVoices().length; - // speechSynthesis.cancel(); + // speechSynthesis.cancel(); // this.clear = true; // let [text, ssml] = ssmlParsing(speech); // console.log(27); From 47b010b3eb0b7d2078d4c6ebc0efe170867eb73c Mon Sep 17 00:00:00 2001 From: zorkow Date: Sat, 5 Aug 2023 14:27:54 +0200 Subject: [PATCH 12/26] Link triggering on click. --- ts/a11y/explorer/KeyExplorer.ts | 35 +++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index 40392d733..235642d6e 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -115,21 +115,24 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore [ // ['keydown', move], ['keydown', this.KeyDown.bind(this)], - ['click', ((e: MouseEvent) => this.click(this.node, e)).bind(this)], + ['click', this.Click.bind(this)], ['focusin', this.FocusIn.bind(this)], ['focusout', this.FocusOut.bind(this)] ]); - public click(snippet: HTMLElement, e: MouseEvent) { + public Click(e: MouseEvent) { const clicked = (e.target as HTMLElement).closest(nav) as HTMLElement; - if (snippet.contains(clicked)) { - const prev = snippet.querySelector('[tabindex="0"][role="tree"],[tabindex="0"][role="group"],[tabindex="0"][role="treeitem"]'); + if (this.node.contains(clicked)) { + const prev = this.node.querySelector('[tabindex="0"][role="tree"],[tabindex="0"][role="group"],[tabindex="0"][role="treeitem"]'); if (prev) { prev.removeAttribute('tabindex'); } clicked.setAttribute('tabindex', '0'); clicked.focus(); this.current = clicked; + if (!this.triggerLinkMouse()) { + this.Start() + } e.preventDefault(); } } @@ -460,7 +463,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore this.NoMove(); } // - if (this.triggerLink(code)) { + if (this.triggerLinkKeyboard(code)) { this.Stop() return; } @@ -476,11 +479,15 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore * Programmatically triggers a link if the focused node contains one. * @param {number} code The keycode of the last key pressed. */ - protected triggerLink(code: string) { + protected triggerLinkKeyboard(code: string) { if (code !== 'Enter' || !this.active) { return false; } let node = this.current; + return this.triggerLink(node); + } + + protected triggerLink(node: HTMLElement) { let focus = node?. getAttribute('data-semantic-postfix')?. match(/(^| )link($| )/); @@ -491,6 +498,22 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore return false; } + + /** + * Programmatically triggers a link if the clicked mouse contains one. + */ + protected triggerLinkMouse() { + let node = this.current; + while (node && node !== this.node) { + if (this.triggerLink(node)) { + return true; + } + node = node.parentNode as HTMLElement; + } + return false; + } + + /** * Initialises the Sre walker. */ From 1afbe37b5ddbc838e9d7e9645e0f34fc79cc487e Mon Sep 17 00:00:00 2001 From: zorkow Date: Sat, 5 Aug 2023 17:14:46 +0200 Subject: [PATCH 13/26] Corrects keyboard triggering of links. --- ts/a11y/explorer/KeyExplorer.ts | 69 +++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index 235642d6e..738cd86a1 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -71,7 +71,7 @@ export interface KeyExplorer extends Explorer { const codeSelector = 'mjx-container[role="application"][data-shellac]'; -const nav = '[role="application"][data-shellac],[role="tree"],[role="group"],[role="treeitem"]'; +const nav = '[role="tree"],[role="group"],[role="treeitem"]'; function isCodeBlock(el: HTMLElement) { return el.matches(codeSelector); @@ -383,7 +383,9 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore let noUpdate = force; force = false; if (!this.active && !force) return; + console.log(6); this.pool.unhighlight(); + console.log(7); // let nodes = this.walker.getFocus(true).getNodes(); // if (!nodes.length) { // this.walker.refocus(); @@ -450,43 +452,60 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore this.stopEvent(event); return; } + if (code === 'Enter') { + if (!this.active && event.target instanceof HTMLAnchorElement) { + event.target.dispatchEvent(new MouseEvent('click')); + this.stopEvent(event); + return; + } + if (this.active && this.triggerLinkKeyboard(event)) { + this.Stop() + this.stopEvent(event); + return; + } + if (!this.active) { + if (!this.current) { + this.current = this.node.querySelector('[role="tree"]'); + this.current.setAttribute('tabindex', '0'); + this.current.focus(); + } + this.Start(); + this.stopEvent(event); + return; + } + } let result = null; if (this.active) { result = this.Move(event); this.stopEvent(event); - } - if (result) { - this.Update(); - return; - } - if (!result && this.sound) { - this.NoMove(); - } - // - if (this.triggerLinkKeyboard(code)) { - this.Stop() - return; - } - - // this.stopEvent(event); - if (code === 'Space' && event.shiftKey || code === 'Enter') { - this.Start(); - this.stopEvent(event); + if (result) { + this.Update(); + return; + } + if (this.sound) { + this.NoMove(); + } } } /** * Programmatically triggers a link if the focused node contains one. - * @param {number} code The keycode of the last key pressed. + * @param {KeyboardEvent} event The keyboard event for the last keydown event. */ - protected triggerLinkKeyboard(code: string) { - if (code !== 'Enter' || !this.active) { + protected triggerLinkKeyboard(event: KeyboardEvent) { + if (event.code !== 'Enter') { return false; } - let node = this.current; - return this.triggerLink(node); + if (!this.current) { + if (event.target instanceof HTMLAnchorElement) { + event.target.dispatchEvent(new MouseEvent('click')); + return true; + } + return false; + } + return this.triggerLink(this.current); } - + protected triggerLink(node: HTMLElement) { let focus = node?. getAttribute('data-semantic-postfix')?. From 2cb170820f6e694205b0ee647f5db08960c1bd29 Mon Sep 17 00:00:00 2001 From: zorkow Date: Sat, 5 Aug 2023 17:51:54 +0200 Subject: [PATCH 14/26] Improved tabindex and focus handling. --- ts/a11y/explorer/KeyExplorer.ts | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index 738cd86a1..bb6163185 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -23,7 +23,7 @@ */ -import {A11yDocument, Region, HoverRegion, SpeechRegion, LiveRegion} from './Region.js'; +import {A11yDocument, HoverRegion, SpeechRegion, LiveRegion} from './Region.js'; import {Explorer, AbstractExplorer} from './Explorer.js'; import {ExplorerPool} from './ExplorerPool.js'; import {Sre} from '../sre.js'; @@ -83,7 +83,7 @@ function isCodeBlock(el: HTMLElement) { * * @template T The type that is consumed by the Region of this explorer. */ -export class SpeechExplorer extends AbstractExplorer implements KeyExplorer { +export class SpeechExplorer extends AbstractExplorer implements KeyExplorer { // public newWalker = new Walker(); @@ -107,6 +107,8 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore protected current: HTMLElement = null; + private move = false; + /** * @override */ @@ -127,8 +129,6 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore if (prev) { prev.removeAttribute('tabindex'); } - clicked.setAttribute('tabindex', '0'); - clicked.focus(); this.current = clicked; if (!this.triggerLinkMouse()) { this.Start() @@ -153,7 +153,10 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore * @override */ public FocusOut(_event: FocusEvent) { - // this.Stop(); + console.log(19); + if (!this.move) { + this.Stop(); + } } /** @@ -195,6 +198,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore * @override */ public Move(e: KeyboardEvent) { + console.log(22); function nextFocus(): HTMLElement { function nextSibling(el: HTMLElement): HTMLElement { const sib = el.nextElementSibling as HTMLElement; @@ -254,6 +258,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore } } + this.move = true; const next = nextFocus(); const target = e.target as HTMLElement; @@ -262,8 +267,10 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore next.setAttribute('tabindex', '0'); next.focus(); this.current = next; + this.move = false; return true; } + this.move = false; return false; } @@ -325,6 +332,8 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore public Start() { if (!this.attached) return; if (this.active) return; + this.current.setAttribute('tabindex', '0'); + this.current.focus(); super.Start(); // let options = this.getOptions(); // if (!this.init) { @@ -356,17 +365,14 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore // 'table', this.node, this.speechGenerator, this.highlighter, this.mml); // this.walker.activate(); if (this.document.options.a11y.subtitles) { - console.log(0); SpeechExplorer.updatePromise.then( () => this.region.Show(this.node, this.highlighter)) } if (this.document.options.a11y.viewBraille) { - console.log(1); SpeechExplorer.updatePromise.then( () => this.brailleRegion.Show(this.node, this.highlighter)) } if (this.document.options.a11y.keyMagnifier) { - console.log(2); this.magnifyRegion.Show(this.node, this.highlighter); } this.Update(); @@ -383,9 +389,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore let noUpdate = force; force = false; if (!this.active && !force) return; - console.log(6); this.pool.unhighlight(); - console.log(7); // let nodes = this.walker.getFocus(true).getNodes(); // if (!nodes.length) { // this.walker.refocus(); @@ -394,7 +398,6 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore this.pool.highlight([this.current]); this.region.Update(this.current.getAttribute('aria-label')); this.brailleRegion.Update(this.current.getAttribute('aria-braillelabel')); - console.log(this.magnifyRegion); this.magnifyRegion.Update(this.current); // let options = this.speechGenerator.getOptions(); // This is a necessary in case speech options have changed via keypress @@ -466,8 +469,6 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore if (!this.active) { if (!this.current) { this.current = this.node.querySelector('[role="tree"]'); - this.current.setAttribute('tabindex', '0'); - this.current.focus(); } this.Start(); this.stopEvent(event); @@ -505,7 +506,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore } return this.triggerLink(this.current); } - + protected triggerLink(node: HTMLElement) { let focus = node?. getAttribute('data-semantic-postfix')?. @@ -532,7 +533,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore return false; } - + /** * Initialises the Sre walker. */ @@ -568,6 +569,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplore */ public Stop() { if (this.active) { + this.current.removeAttribute('tabindex'); this.pool.unhighlight(); this.magnifyRegion.Hide(); this.region.Hide(); From b4ad1ce3d1bb1a26bb2099391d04991047ae11a2 Mon Sep 17 00:00:00 2001 From: zorkow Date: Thu, 10 Aug 2023 16:59:39 +0200 Subject: [PATCH 15/26] Removes unused Walker. --- ts/a11y/explorer/Walker.ts | 208 ------------------------------------- ts/a11y/semantic-enrich.ts | 2 +- 2 files changed, 1 insertion(+), 209 deletions(-) delete mode 100644 ts/a11y/explorer/Walker.ts diff --git a/ts/a11y/explorer/Walker.ts b/ts/a11y/explorer/Walker.ts deleted file mode 100644 index 20b4a29c9..000000000 --- a/ts/a11y/explorer/Walker.ts +++ /dev/null @@ -1,208 +0,0 @@ -/************************************************************* - * - * Copyright (c) 2009-2023 The MathJax Consortium - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/** - * @fileoverview Aria Tree Walker. - * - * @author v.sorge@mathjax.org (Volker Sorge) - */ - -import { SpeechRegion, LiveRegion } from './Region.js'; -import {Sre} from '../sre.js'; - -// Based on the shellac walker. - -const codeSelector = 'mjx-container[role="application"][data-shellac]'; -const nav = '[role="application"][data-shellac],[role="tree"],[role="group"],[role="treeitem"]'; - -function isCodeBlock(el: HTMLElement) { - return el.matches(codeSelector); -} - -type rgbColor = {red: number, green: number, blue: number}; -type channelColor = {name: string, alpha: number}; - -export class Highlighter { - - public foreground: string; - public background: string; - - public static namedColors: { [key: string]: rgbColor } = { - red: { red: 255, green: 0, blue: 0 }, - green: { red: 0, green: 255, blue: 0 }, - blue: { red: 0, green: 0, blue: 255 }, - yellow: { red: 255, green: 255, blue: 0 }, - cyan: { red: 0, green: 255, blue: 255 }, - magenta: { red: 255, green: 0, blue: 255 }, - white: { red: 255, green: 255, blue: 255 }, - black: { red: 0, green: 0, blue: 0 } - }; - - constructor(foreground: channelColor, background: channelColor) { - this.foreground = this.makeColor(foreground); - this.background = this.makeColor(background); - } - - // public highlight(node: HTMLElement) { - - // } - - private makeColor({name: name, alpha: alpha}: channelColor) { - let {red: red, green: green, blue: blue} = - Highlighter.namedColors[name] || Highlighter.namedColors['blue']; - return `rgba(${red},${green},${blue},${alpha})`; - } - -} - -export class Walker { - - public shown: boolean = false; - public speechRegion: SpeechRegion; - public brailleRegion: LiveRegion; - public highlighter: Sre.highlighter; - - public click(snippet: HTMLElement, e: MouseEvent) { - const clicked = (e.target as HTMLElement).closest(nav) as HTMLElement; - if (snippet.contains(clicked)) { - const prev = snippet.querySelector('[tabindex="0"][role="tree"],[tabindex="0"][role="group"],[tabindex="0"][role="treeitem"]'); - if (prev) { - prev.removeAttribute('tabindex'); - } - clicked.setAttribute('tabindex', '0'); - clicked.focus(); - this.UpdateRegions(clicked); - e.preventDefault(); - } - } - - public move(e: KeyboardEvent) { - - function nextFocus(): HTMLElement { - function nextSibling(el: HTMLElement): HTMLElement { - const sib = el.nextElementSibling as HTMLElement; - if (sib) { - if (sib.matches(nav)) { - return sib; - } else { - const sibChild = sib.querySelector(nav) as HTMLElement; - return sibChild ?? nextSibling(sib); - } - } else { - if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { - return nextSibling(el.parentElement); - } else { - return null; - } - } - } - - function prevSibling(el: HTMLElement): HTMLElement { - const sib = el.previousElementSibling as HTMLElement; - if (sib) { - if (sib.matches(nav)) { - return sib; - } else { - const sibChild = sib.querySelector(nav) as HTMLElement; - return sibChild ?? prevSibling(sib); - } - } else { - if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { - return prevSibling(el.parentElement); - } else { - return null; - } - } - } - - const target = e.target as HTMLElement; - switch (e.key) { - case 'ArrowDown': - e.preventDefault(); - return target.querySelector(nav); - case 'ArrowUp': - e.preventDefault(); - return target.parentElement.closest(nav); - case 'ArrowLeft': - e.preventDefault(); - return prevSibling(target); - case 'ArrowRight': - e.preventDefault(); - return nextSibling(target); - // case 'Esc': - // e.preventDefault(); - // return this.hideRegions(); - default: - return null; - } - } - - const next = nextFocus(); - - const target = e.target as HTMLElement; - if (next) { - target.removeAttribute('tabindex'); - next.setAttribute('tabindex', '0'); - next.focus(); - this.UpdateHighlight(target, next); - this.UpdateRegions(next); - return [next]; - } - return false; - } - - private info: Highlight; - - private UpdateHighlight(_target: HTMLElement, next: HTMLElement) { - if (this.info) { - (this.highlighter as any).unhighlightNode(this.info); - } - this.info = (this.highlighter as any).highlightNode(next); - } - - private UpdateRegions(element: HTMLElement) { - this.speechRegion.Update(element.getAttribute('aria-label')); - this.brailleRegion.Update(element.getAttribute('aria-braillelabel')); - } - - public ShowRegions(element: HTMLElement, highlighter: Sre.highlighter) { - if (!this.shown) { - this.speechRegion.Show(element, highlighter); - this.brailleRegion.Show(element, highlighter); - } - this.shown = true; - } - - public HideRegions() { - if (this.shown) { - this.speechRegion.Hide(); - this.brailleRegion.Hide(); - } - this.shown = false; - } -} - -export interface Highlight { - node: HTMLElement; - opacity?: string; - background?: string; - foreground?: string; - // The following is for the CSS highlighter - box?: HTMLElement; - position?: string; -} diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts index 752c414da..f5c05dcfb 100644 --- a/ts/a11y/semantic-enrich.ts +++ b/ts/a11y/semantic-enrich.ts @@ -397,7 +397,7 @@ export function EnrichedMathDocumentMixin Date: Fri, 11 Aug 2023 12:56:52 +0200 Subject: [PATCH 16/26] Refactor set aria function. --- ts/a11y/SpeechUtil.ts | 29 +++++++++++++++++++++++++++-- ts/a11y/semantic-enrich.ts | 34 +++------------------------------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/ts/a11y/SpeechUtil.ts b/ts/a11y/SpeechUtil.ts index a1f03a6ce..7873c8430 100644 --- a/ts/a11y/SpeechUtil.ts +++ b/ts/a11y/SpeechUtil.ts @@ -1,5 +1,5 @@ import {MmlNode} from '../core/MmlTree/MmlNode.js'; -import {Sre} from './sre.js'; +// import {Sre} from './sre.js'; const ProsodyKeys = [ 'pitch', 'rate', 'volume' ]; @@ -138,7 +138,12 @@ function extractProsody(attr: string) { return [match[1], match[2]]; } -export function getLabel(node: MmlNode, sep: string = ' ') { +/** + * Computes the aria-label from the node. + * @param {MmlNode} node The Math element. + * @param {string=} sep The speech separator. Defaults to space. + */ +function getLabel(node: MmlNode, sep: string = ' ') { const attributes = node.attributes; const speech = attributes.getExplicit('data-semantic-speech') as string; if (!speech) { @@ -167,3 +172,23 @@ export function buildSpeech(speech: string, locale: string = 'en', `${speech}`+ ''); } + +/** + * Retrieve and sets aria and braille labels recursively. + * @param {MmlNode} node The root node to search from. + */ +export function setAria(node: MmlNode, locale: string) { + const attributes = node.attributes; + if (!attributes) return; + const speech = getLabel(node); + if (speech) { + attributes.set('aria-label', buildSpeech(speech, locale)[0]); + } + const braille = node.attributes.getExplicit('data-semantic-braille') as string; + if (braille) { + attributes.set('aria-braillelabel', braille); + } + for (let child of node.childNodes) { + setAria(child, locale); + } +} diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts index f5c05dcfb..ddbb40684 100644 --- a/ts/a11y/semantic-enrich.ts +++ b/ts/a11y/semantic-enrich.ts @@ -30,14 +30,13 @@ import {MathML} from '../input/mathml.js'; import {SerializedMmlVisitor} from '../core/MmlTree/SerializedMmlVisitor.js'; import {OptionList, expandable} from '../util/Options.js'; import {Sre} from './sre.js'; -import { buildSpeech, getLabel } from './SpeechUtil.js'; +import { buildSpeech, setAria } from './SpeechUtil.js'; /*==========================================================================*/ /** * The current speech setting for Sre */ -// let currentSpeech = 'none'; let currentLocale = 'none'; let currentBraille = 'none'; @@ -169,12 +168,6 @@ export function EnrichedMathItemMixin= STATE.ENRICHED) return; if (!this.isEscaped && (document.options.enableEnrichment || force)) { // TODO: Sort out the loading of the locales better - // if (document.options.sre.speech !== currentSpeech) { - // currentSpeech = document.options.sre.speech; - // mathjax.retryAfter( - // Sre.setupEngine(document.options.sre).then( - // () => Sre.sreReady())); - // } if (document.options.sre.locale !== currentLocale) { currentLocale = document.options.sre.locale; // TODO: Sort out the loading of the locales better @@ -228,7 +221,7 @@ export function EnrichedMathItemMixin) { + console.log(0); if (this.state() >= STATE.ATTACHSPEECH) return; const attributes = this.root.attributes; const speech = (attributes.get('aria-label') || this.label); @@ -302,28 +296,6 @@ export function EnrichedMathItemMixin Date: Fri, 11 Aug 2023 14:04:32 +0200 Subject: [PATCH 17/26] Refactoring speech computation to utilities. --- ts/a11y/explorer.ts | 3 ++- ts/a11y/explorer/KeyExplorer.ts | 6 +---- ts/a11y/semantic-enrich.ts | 40 +++++++++++++-------------------- 3 files changed, 19 insertions(+), 30 deletions(-) diff --git a/ts/a11y/explorer.ts b/ts/a11y/explorer.ts index 160e71bc6..8e8ba23f8 100644 --- a/ts/a11y/explorer.ts +++ b/ts/a11y/explorer.ts @@ -199,7 +199,8 @@ export function ExplorerMathDocumentMixin implements KeyExplo * @override */ public FocusOut(_event: FocusEvent) { - console.log(19); if (!this.move) { this.Stop(); } @@ -168,8 +167,6 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo this.oldIndex = this.node.tabIndex; this.node.tabIndex = 0; this.node.setAttribute('role', 'application'); - // TODO: Get rid of this eventually! - this.node.setAttribute('data-shellac', ''); } /** @@ -198,7 +195,6 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo * @override */ public Move(e: KeyboardEvent) { - console.log(22); function nextFocus(): HTMLElement { function nextSibling(el: HTMLElement): HTMLElement { const sib = el.nextElementSibling as HTMLElement; diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts index ddbb40684..3a5a90ae8 100644 --- a/ts/a11y/semantic-enrich.ts +++ b/ts/a11y/semantic-enrich.ts @@ -62,7 +62,7 @@ newState('ATTACHSPEECH', 155); export class enrichVisitor extends SerializedMmlVisitor { protected mactionId: number; - + public visitTree(node: MmlNode, math?: MathItem) { this.mactionId = 1; const mml = super.visitTree(node); @@ -172,23 +172,15 @@ export function EnrichedMathItemMixin { return Sre.sreReady(); })); } if (document.options.sre.braille !== currentBraille) { currentBraille = document.options.sre.braille; - // TODO: Sort out the loading of the locales better mathjax.retryAfter( - Sre.setupEngine({ - locale: document.options.sre.braille, - domain: 'default', // speech rules domain - style: 'default', // speech rules style - modality: 'braille', - markup: 'none', - }) - .then(() => { - Sre.sreReady();})); + Sre.setupEngine({locale: document.options.sre.braille}) + .then(() => Sre.sreReady())); } const math = new document.options.MathItem('', MmlJax); try { @@ -198,19 +190,23 @@ export function EnrichedMathItemMixin) { - console.log(0); if (this.state() >= STATE.ATTACHSPEECH) return; const attributes = this.root.attributes; const speech = (attributes.get('aria-label') || this.label); @@ -367,15 +362,12 @@ export function EnrichedMathDocumentMixin Date: Sat, 12 Aug 2023 13:09:14 +0200 Subject: [PATCH 18/26] Refactoring moves to class level methods. --- ts/a11y/explorer.ts | 2 +- ts/a11y/explorer/ExplorerPool.ts | 9 +-- ts/a11y/explorer/KeyExplorer.ts | 111 +++++++++++++++---------------- ts/a11y/explorer/Region.ts | 5 +- 4 files changed, 61 insertions(+), 66 deletions(-) diff --git a/ts/a11y/explorer.ts b/ts/a11y/explorer.ts index 8e8ba23f8..bacecbd83 100644 --- a/ts/a11y/explorer.ts +++ b/ts/a11y/explorer.ts @@ -123,7 +123,7 @@ export function ExplorerMathItemMixin>( if (!this.explorers) { this.explorers = new ExplorerPool(); } - this.explorers.init(document, node, mml); + this.explorers.init(document, node, mml, this); } this.state(STATE.EXPLORER); } diff --git a/ts/a11y/explorer/ExplorerPool.ts b/ts/a11y/explorer/ExplorerPool.ts index 0892af401..8d3ae8c50 100644 --- a/ts/a11y/explorer/ExplorerPool.ts +++ b/ts/a11y/explorer/ExplorerPool.ts @@ -23,7 +23,7 @@ */ import {LiveRegion, SpeechRegion, ToolTip, HoverRegion} from './Region.js'; -import type { ExplorerMathDocument } from '../explorer.js'; +import type { ExplorerMathDocument, ExplorerMathItem } from '../explorer.js'; import {Explorer} from './Explorer.js'; import * as ke from './KeyExplorer.js'; @@ -89,7 +89,7 @@ let allExplorers: {[options: string]: ExplorerInit} = { speech: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => { let explorer = ke.SpeechExplorer.create( doc, pool, doc.explorerRegions.speechRegion, node, - doc.explorerRegions.brailleRegion, doc.explorerRegions.magnifier, rest[0]) as ke.SpeechExplorer; + doc.explorerRegions.brailleRegion, doc.explorerRegions.magnifier, rest[0], rest[1]) as ke.SpeechExplorer; // explorer.speechGenerator.setOptions({ // automark: true as any, markup: 'ssml', // locale: doc.options.sre.locale, domain: doc.options.sre.domain, @@ -212,13 +212,14 @@ export class ExplorerPool { * @param mml The corresponding Mathml node as a string. */ public init(document: ExplorerMathDocument, - node: HTMLElement, mml: string) { + node: HTMLElement, mml: string, + item: ExplorerMathItem) { this.document = document; this.mml = mml; this.node = node; this.setPrimaryHighlighter(); for (let key of Object.keys(allExplorers)) { - this.explorers[key] = allExplorers[key](this.document, this, this.node, this.mml); + this.explorers[key] = allExplorers[key](this.document, this, this.node, this.mml, item); } this.setSecondaryHighlighter(); this.attach(); diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index fd6000e73..1152b9283 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -24,8 +24,10 @@ import {A11yDocument, HoverRegion, SpeechRegion, LiveRegion} from './Region.js'; +import type { ExplorerMathItem } from '../explorer.js'; import {Explorer, AbstractExplorer} from './Explorer.js'; import {ExplorerPool} from './ExplorerPool.js'; +import {MmlNode} from '../../core/MmlTree/MmlNode.js'; import {Sre} from '../sre.js'; // import { Walker } from './Walker.js'; @@ -191,73 +193,64 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo this.attached = false; } - /** - * @override - */ - public Move(e: KeyboardEvent) { - function nextFocus(): HTMLElement { - function nextSibling(el: HTMLElement): HTMLElement { - const sib = el.nextElementSibling as HTMLElement; - if (sib) { - if (sib.matches(nav)) { - return sib; - } else { - const sibChild = sib.querySelector(nav) as HTMLElement; - return sibChild ?? nextSibling(sib); - } + protected nextSibling(el: HTMLElement): HTMLElement { + const sib = el.nextElementSibling as HTMLElement; + if (sib) { + if (sib.matches(nav)) { + return sib; } else { - if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { - return nextSibling(el.parentElement); - } else { - return null; - } + const sibChild = sib.querySelector(nav) as HTMLElement; + return sibChild ?? this.nextSibling(sib); } - } - - function prevSibling(el: HTMLElement): HTMLElement { - const sib = el.previousElementSibling as HTMLElement; - if (sib) { - if (sib.matches(nav)) { - return sib; - } else { - const sibChild = sib.querySelector(nav) as HTMLElement; - return sibChild ?? prevSibling(sib); - } + } else { + if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { + return this.nextSibling(el.parentElement); } else { - if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { - return prevSibling(el.parentElement); - } else { - return null; - } + return null; } } + } - const target = e.target as HTMLElement; - switch (e.key) { - case 'ArrowDown': - e.preventDefault(); - return target.querySelector(nav); - case 'ArrowUp': - e.preventDefault(); - return target.parentElement.closest(nav); - case 'ArrowLeft': - e.preventDefault(); - return prevSibling(target); - case 'ArrowRight': - e.preventDefault(); - return nextSibling(target); - // case 'Esc': - // e.preventDefault(); - // return this.hideRegions(); - default: + protected prevSibling(el: HTMLElement): HTMLElement { + const sib = el.previousElementSibling as HTMLElement; + if (sib) { + if (sib.matches(nav)) { + return sib; + } else { + const sibChild = sib.querySelector(nav) as HTMLElement; + return sibChild ?? this.prevSibling(sib); + } + } else { + if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { + return this.prevSibling(el.parentElement); + } else { return null; + } } } + protected moves: Map HTMLElement | null> = new Map([ + ['ArrowDown', (node: HTMLElement) => node.querySelector(nav)], + ['ArrowUp', (node: HTMLElement) => node.parentElement.closest(nav)], + ['ArrowLeft', this.prevSibling], + ['ArrowRight', this.nextSibling], + ['>', (_node: HTMLElement) => { + return null; + }], + ]); + + /** + * @override + */ + public Move(e: KeyboardEvent) { this.move = true; - const next = nextFocus(); - const target = e.target as HTMLElement; + const move = this.moves.get(e.key); + let next = null; + if (move) { + e.preventDefault(); + next = move(target); + } if (next) { target.removeAttribute('tabindex'); next.setAttribute('tabindex', '0'); @@ -296,7 +289,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo */ public showRegion: string = 'subtitles'; - private init: boolean = false; + // private init: boolean = false; /** * Flag in case the start method is triggered before the walker is fully @@ -316,7 +309,9 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo protected node: HTMLElement, public brailleRegion: LiveRegion, public magnifyRegion: HoverRegion, - private _mml: string) { + _mml: MmlNode, + private item: ExplorerMathItem + ) { super(document, pool, null, node); // this.initWalker(); } @@ -545,7 +540,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo * @return {{[key: string]: string}} The options settings for the speech * generator. */ - private getOptions(): {[key: string]: string} { + protected getOptions(): {[key: string]: string} { let options = this.speechGenerator.getOptions(); let sreOptions = this.document.options.sre; if (options.modality === 'speech' && diff --git a/ts/a11y/explorer/Region.ts b/ts/a11y/explorer/Region.ts index 0732a25de..d0672167f 100644 --- a/ts/a11y/explorer/Region.ts +++ b/ts/a11y/explorer/Region.ts @@ -26,7 +26,7 @@ import {MathDocument} from '../../core/MathDocument.js'; import {CssStyles} from '../../util/StyleList.js'; import {Sre} from '../sre.js'; -import {SsmlElement, ssmlParsing} from '../SpeechUtil.js'; +import {SsmlElement} from '../SpeechUtil.js'; export type A11yDocument = MathDocument; @@ -411,7 +411,6 @@ export class SpeechRegion extends LiveRegion { * @override */ public Update(speech: string) { - console.log('In Speech region: ' + speech); // Temporarily removed! // console.log(speech); // this.active = this.document.options.a11y.voicing && @@ -433,7 +432,7 @@ export class SpeechRegion extends LiveRegion { * @param {SsmlElement[]} ssml The list of ssml annotations. * @param {string} locale The locale to use. */ - private makeUtterances(ssml: SsmlElement[], locale: string) { + protected makeUtterances(ssml: SsmlElement[], locale: string) { let utterance = null; for (let utter of ssml) { if (utter.mark) { From 2c22adc7c8d996bf9e427543cfd86741bbb0f4b6 Mon Sep 17 00:00:00 2001 From: zorkow Date: Sat, 12 Aug 2023 13:44:35 +0200 Subject: [PATCH 19/26] Better setting of speech and braille. --- ts/a11y/semantic-enrich.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts index 3a5a90ae8..a15d70399 100644 --- a/ts/a11y/semantic-enrich.ts +++ b/ts/a11y/semantic-enrich.ts @@ -136,8 +136,6 @@ export function EnrichedMathItemMixin) { if (this.state() >= STATE.ATTACHSPEECH) return; const attributes = this.root.attributes; - const speech = (attributes.get('aria-label') || this.label); - const braille = (attributes.get('aria-braillelabel') || this.braillelabel); + const speech = (attributes.get('aria-label') || this.outputData.speech); + const braille = (attributes.get('aria-braillelabel') || this.outputData.braille); if (!speech && !braille) { this.state(STATE.ATTACHSPEECH); return; From 172e63cf29020b133401e04638fa614f58e795b7 Mon Sep 17 00:00:00 2001 From: zorkow Date: Sun, 13 Aug 2023 09:06:41 +0200 Subject: [PATCH 20/26] Key explorer cleanup. --- ts/a11y/explorer/KeyExplorer.ts | 144 ++++++-------------------------- 1 file changed, 26 insertions(+), 118 deletions(-) diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index 1152b9283..77004674e 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -194,46 +194,40 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo } protected nextSibling(el: HTMLElement): HTMLElement { - const sib = el.nextElementSibling as HTMLElement; - if (sib) { - if (sib.matches(nav)) { - return sib; - } else { - const sibChild = sib.querySelector(nav) as HTMLElement; - return sibChild ?? this.nextSibling(sib); - } - } else { - if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { - return this.nextSibling(el.parentElement); - } else { - return null; - } - } + const sib = el.nextElementSibling as HTMLElement; + if (sib) { + if (sib.matches(nav)) { + return sib; + } + const sibChild = sib.querySelector(nav) as HTMLElement; + return sibChild ?? this.nextSibling(sib); } + if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { + return this.nextSibling(el.parentElement); + } + return null; + } protected prevSibling(el: HTMLElement): HTMLElement { - const sib = el.previousElementSibling as HTMLElement; - if (sib) { - if (sib.matches(nav)) { - return sib; - } else { - const sibChild = sib.querySelector(nav) as HTMLElement; - return sibChild ?? this.prevSibling(sib); - } - } else { - if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { - return this.prevSibling(el.parentElement); - } else { - return null; - } + const sib = el.previousElementSibling as HTMLElement; + if (sib) { + if (sib.matches(nav)) { + return sib; } + const sibChild = sib.querySelector(nav) as HTMLElement; + return sibChild ?? this.prevSibling(sib); } + if (!isCodeBlock(el) && !el.parentElement.matches(nav)) { + return this.prevSibling(el.parentElement); + } + return null; + } protected moves: Map HTMLElement | null> = new Map([ ['ArrowDown', (node: HTMLElement) => node.querySelector(nav)], ['ArrowUp', (node: HTMLElement) => node.parentElement.closest(nav)], - ['ArrowLeft', this.prevSibling], - ['ArrowRight', this.nextSibling], + ['ArrowLeft', this.prevSibling.bind(this)], + ['ArrowRight', this.nextSibling.bind(this)], ['>', (_node: HTMLElement) => { return null; }], @@ -310,10 +304,9 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo public brailleRegion: LiveRegion, public magnifyRegion: HoverRegion, _mml: MmlNode, - private item: ExplorerMathItem + public item: ExplorerMathItem ) { super(document, pool, null, node); - // this.initWalker(); } @@ -524,17 +517,6 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo return false; } - - /** - * Initialises the Sre walker. - */ - // private initWalker() { - // this.speechGenerator = Sre.getSpeechGenerator('Tree'); - // let dummy = Sre.getWalker( - // 'dummy', this.node, this.speechGenerator, this.highlighter, this.mml); - // this.walker = dummy; - // } - /** * Retrieves the speech options to sync with document options. * @return {{[key: string]: string}} The options settings for the speech @@ -572,77 +554,3 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo } - - -/** - * Explorer that magnifies what is currently explored. Uses a hover region. - * @constructor - * @extends {AbstractKeyExplorer} - */ -// export class Magnifier extends AbstractKeyExplorer { - -// /** -// * @constructor -// * @extends {AbstractKeyExplorer} -// */ -// constructor(public document: A11yDocument, -// public pool: ExplorerPool, -// public region: Region, -// protected node: HTMLElement, -// private mml: string) { -// super(document, pool, region, node); -// this.walker = Sre.getWalker( -// 'table', this.node, Sre.getSpeechGenerator('Dummy'), -// this.highlighter, this.mml); -// } - -// /** -// * @override -// */ -// public Update(force: boolean = false) { -// super.Update(force); -// this.showFocus(); -// } - -// /** -// * @override -// */ -// public Start() { -// super.Start(); -// if (!this.attached) return; -// this.region.Show(this.node, this.highlighter); -// this.walker.activate(); -// this.Update(); -// } - -// /** -// * Shows the nodes that are currently focused. -// */ -// private showFocus() { -// let node = this.walker.getFocus().getNodes()[0] as HTMLElement; -// this.region.Show(node, this.highlighter); -// } - -// /** -// * @override -// */ -// public KeyDown(event: KeyboardEvent) { -// const code = event.keyCode; -// this.walker.modifier = event.shiftKey; -// if (code === 27) { -// this.Stop(); -// this.stopEvent(event); -// return; -// } -// if (this.active && code !== 13) { -// this.Move(code); -// this.stopEvent(event); -// return; -// } -// if (code === 32 && event.shiftKey || code === 13) { -// this.Start(); -// this.stopEvent(event); -// } -// } - -// } From ebcb2438a7ba18a88c0d74eec15415b36ac11674 Mon Sep 17 00:00:00 2001 From: zorkow Date: Sun, 13 Aug 2023 09:44:10 +0200 Subject: [PATCH 21/26] Refactors generator to be attached to item. --- ts/a11y/explorer/KeyExplorer.ts | 3 +++ ts/a11y/semantic-enrich.ts | 14 +++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index 77004674e..e5c1f15c3 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -467,6 +467,9 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo this.Update(); return; } + if (event.getModifierState(code)) { + return; + } if (this.sound) { this.NoMove(); } diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts index a15d70399..32ddfe989 100644 --- a/ts/a11y/semantic-enrich.ts +++ b/ts/a11y/semantic-enrich.ts @@ -136,6 +136,11 @@ export function EnrichedMathItemMixin Date: Sun, 13 Aug 2023 18:51:25 +0200 Subject: [PATCH 22/26] Reinstantiates auto voicing. --- ts/a11y/explorer/KeyExplorer.ts | 4 ++- ts/a11y/explorer/Region.ts | 53 +++++++++++++++++++++++---------- ts/a11y/semantic-enrich.ts | 3 +- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index e5c1f15c3..51a31bc24 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -380,7 +380,8 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo // nodes = this.walker.getFocus().getNodes(); // } this.pool.highlight([this.current]); - this.region.Update(this.current.getAttribute('aria-label')); + this.region.node = this.current; + this.region.Update(this.current.getAttribute('data-semantic-speech')); this.brailleRegion.Update(this.current.getAttribute('aria-braillelabel')); this.magnifyRegion.Update(this.current); // let options = this.speechGenerator.getOptions(); @@ -547,6 +548,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo if (this.active) { this.current.removeAttribute('tabindex'); this.pool.unhighlight(); + this.region.highlighter.unhighlight(); this.magnifyRegion.Hide(); this.region.Hide(); this.brailleRegion.Hide(); diff --git a/ts/a11y/explorer/Region.ts b/ts/a11y/explorer/Region.ts index d0672167f..c98d48335 100644 --- a/ts/a11y/explorer/Region.ts +++ b/ts/a11y/explorer/Region.ts @@ -26,7 +26,7 @@ import {MathDocument} from '../../core/MathDocument.js'; import {CssStyles} from '../../util/StyleList.js'; import {Sre} from '../sre.js'; -import {SsmlElement} from '../SpeechUtil.js'; +import {SsmlElement, buildSpeech} from '../SpeechUtil.js'; export type A11yDocument = MathDocument; @@ -407,24 +407,47 @@ export class SpeechRegion extends LiveRegion { super.Show(node, highlighter); } + /** + * Have we already requested voices from the browser? + */ + private voiceRequest = false; + /** * @override */ public Update(speech: string) { - // Temporarily removed! - // console.log(speech); - // this.active = this.document.options.a11y.voicing && - // !!speechSynthesis.getVoices().length; - // speechSynthesis.cancel(); - // this.clear = true; - // let [text, ssml] = ssmlParsing(speech); - // console.log(27); - // console.log(text); - // super.Update(text); - // if (this.active && text) { - // this.makeUtterances(ssml, this.document.options.sre.locale); - // } - super.Update(speech); + if (this.voiceRequest) { + this.makeVoice(speech); + return; + } + speechSynthesis.onvoiceschanged = (() => this.voiceRequest = true).bind(this); + super.Update('\u00a0'); // Ensures region shown and cannot be overwritten. + const promise = new Promise((resolve) => { + setTimeout(() => { + if (this.voiceRequest) { + resolve(true); + } + }, 100); + }); + promise.then( + () => this.makeVoice(speech) + ); + } + + private makeVoice(speech: string) { + this.active = this.document.options.a11y.voicing && + !!speechSynthesis.getVoices().length; + speechSynthesis.cancel(); + this.clear = true; + let [text, ssml] = buildSpeech( + speech, + this.document.options.sre.locale, + this.document.options.sre.rate + ); + super.Update(text); + if (this.active && text) { + this.makeUtterances(ssml, this.document.options.sre.locale); + } } /** diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts index 32ddfe989..48b5623eb 100644 --- a/ts/a11y/semantic-enrich.ts +++ b/ts/a11y/semantic-enrich.ts @@ -202,7 +202,8 @@ export function EnrichedMathItemMixin Date: Mon, 14 Aug 2023 11:43:46 +0200 Subject: [PATCH 23/26] Code cleanup. --- ts/a11y/SpeechUtil.ts | 12 ++++++++++++ ts/a11y/explorer/KeyExplorer.ts | 13 +++---------- ts/a11y/explorer/Region.ts | 10 ++++++---- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/ts/a11y/SpeechUtil.ts b/ts/a11y/SpeechUtil.ts index 7873c8430..bb59b93b6 100644 --- a/ts/a11y/SpeechUtil.ts +++ b/ts/a11y/SpeechUtil.ts @@ -192,3 +192,15 @@ export function setAria(node: MmlNode, locale: string) { setAria(child, locale); } } + +/** + * Creates a honking sound. + */ +export function honk() { + let ac = new AudioContext(); + let os = ac.createOscillator(); + os.frequency.value = 300; + os.connect(ac.destination); + os.start(ac.currentTime); + os.stop(ac.currentTime + .05); +} diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index 51a31bc24..1f1232b45 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -28,6 +28,7 @@ import type { ExplorerMathItem } from '../explorer.js'; import {Explorer, AbstractExplorer} from './Explorer.js'; import {ExplorerPool} from './ExplorerPool.js'; import {MmlNode} from '../../core/MmlTree/MmlNode.js'; +import { honk } from '../SpeechUtil.js'; import {Sre} from '../sre.js'; // import { Walker } from './Walker.js'; @@ -87,8 +88,6 @@ function isCodeBlock(el: HTMLElement) { */ export class SpeechExplorer extends AbstractExplorer implements KeyExplorer { - // public newWalker = new Walker(); - /** * Flag indicating if the explorer is attached to an object. */ @@ -261,12 +260,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo * @override */ public NoMove() { - let ac = new AudioContext(); - let os = ac.createOscillator(); - os.frequency.value = 300; - os.connect(ac.destination); - os.start(ac.currentTime); - os.stop(ac.currentTime + .05); + honk(); } private static updatePromise = Promise.resolve(); @@ -380,7 +374,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo // nodes = this.walker.getFocus().getNodes(); // } this.pool.highlight([this.current]); - this.region.node = this.current; + this.region.node = this.node; this.region.Update(this.current.getAttribute('data-semantic-speech')); this.brailleRegion.Update(this.current.getAttribute('aria-braillelabel')); this.magnifyRegion.Update(this.current); @@ -548,7 +542,6 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo if (this.active) { this.current.removeAttribute('tabindex'); this.pool.unhighlight(); - this.region.highlighter.unhighlight(); this.magnifyRegion.Hide(); this.region.Hide(); this.brailleRegion.Hide(); diff --git a/ts/a11y/explorer/Region.ts b/ts/a11y/explorer/Region.ts index c98d48335..73d90ddfc 100644 --- a/ts/a11y/explorer/Region.ts +++ b/ts/a11y/explorer/Region.ts @@ -211,8 +211,9 @@ export abstract class AbstractRegion implements Region { baseLeft = Math.min(region.getBoundingClientRect().left, baseLeft); } } - const bot = (baseBottom ? baseBottom : rect.bottom + 10) + window.pageYOffset; - const left = (baseLeft < Number.POSITIVE_INFINITY ? baseLeft : rect.left) + window.pageXOffset; + + const bot = (baseBottom ? baseBottom : rect.bottom + 10) + window.scrollY; + const left = (baseLeft < Number.POSITIVE_INFINITY ? baseLeft : rect.left) + window.scrollX; this.div.style.top = bot + 'px'; this.div.style.left = left + 'px'; } @@ -460,6 +461,7 @@ export class SpeechRegion extends LiveRegion { for (let utter of ssml) { if (utter.mark) { if (!utterance) { + // First utterance, call with init = true. this.highlightNode(utter.mark, true); continue; } @@ -566,7 +568,7 @@ export class HoverRegion extends AbstractRegion { const xCenter = nodeRect.left + (nodeRect.width / 2); let left = xCenter - (divRect.width / 2); left = (left < 0) ? 0 : left; - left = left + window.pageXOffset; + left = left + window.scrollX; let top; switch (this.document.options.a11y.align) { case 'top': @@ -580,7 +582,7 @@ export class HoverRegion extends AbstractRegion { const yCenter = nodeRect.top + (nodeRect.height / 2); top = yCenter - (divRect.height / 2); } - top = top + window.pageYOffset; + top = top + window.scrollY; top = (top < 0) ? 0 : top; this.div.style.top = top + 'px'; this.div.style.left = left + 'px'; From 2952d5e49d312b688e9d89714e86d1ee6b0ba37b Mon Sep 17 00:00:00 2001 From: zorkow Date: Mon, 14 Aug 2023 19:07:54 +0200 Subject: [PATCH 24/26] Code cleanup. --- ts/a11y/SpeechUtil.ts | 13 ++++++++++--- ts/a11y/explorer/ExplorerPool.ts | 21 --------------------- ts/a11y/explorer/Region.ts | 10 ---------- ts/a11y/semantic-enrich.ts | 1 - 4 files changed, 10 insertions(+), 35 deletions(-) diff --git a/ts/a11y/SpeechUtil.ts b/ts/a11y/SpeechUtil.ts index bb59b93b6..e0de19094 100644 --- a/ts/a11y/SpeechUtil.ts +++ b/ts/a11y/SpeechUtil.ts @@ -154,7 +154,7 @@ function getLabel(node: MmlNode, sep: string = ' ') { if (prefix) { label.unshift(prefix); } - // TODO: check if we need this or if is automatic by the screen readers. + // TODO: check if we need this or if it is automatic by the screen readers. const postfix = attributes.getExplicit('data-semantic-postfix') as string; if (postfix) { label.push(postfix); @@ -163,9 +163,16 @@ function getLabel(node: MmlNode, sep: string = ' ') { return label.join(sep); } - +/** + * Builds speechs from SSML markup strings. + * + * @param {string} speech The speech string. + * @param {string=} locale An optional locale. + * @param {string=} rate The base speech rate. + * @return {[string, SsmlElement[]]} The speech with the ssml annotation structure + */ export function buildSpeech(speech: string, locale: string = 'en', - rate: string = '100') { + rate: string = '100'): [string, SsmlElement[]] { return ssmlParsing('` + diff --git a/ts/a11y/explorer/ExplorerPool.ts b/ts/a11y/explorer/ExplorerPool.ts index 8d3ae8c50..949416c55 100644 --- a/ts/a11y/explorer/ExplorerPool.ts +++ b/ts/a11y/explorer/ExplorerPool.ts @@ -90,30 +90,9 @@ let allExplorers: {[options: string]: ExplorerInit} = { let explorer = ke.SpeechExplorer.create( doc, pool, doc.explorerRegions.speechRegion, node, doc.explorerRegions.brailleRegion, doc.explorerRegions.magnifier, rest[0], rest[1]) as ke.SpeechExplorer; - // explorer.speechGenerator.setOptions({ - // automark: true as any, markup: 'ssml', - // locale: doc.options.sre.locale, domain: doc.options.sre.domain, - // style: doc.options.sre.style, modality: 'speech'}); - // // This weeds out the case of providing a non-existent locale option. - // let locale = explorer.speechGenerator.getOptions().locale; - // if (locale !== Sre.engineSetup().locale) { - // doc.options.sre.locale = Sre.engineSetup().locale; - // explorer.speechGenerator.setOptions({locale: doc.options.sre.locale}); - // } explorer.sound = true; return explorer; }, - // braille: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => { - // let explorer = ke.SpeechExplorer.create( - // doc, pool, doc.explorerRegions.brailleRegion, node, ...rest) as ke.SpeechExplorer; - // explorer.speechGenerator.setOptions({automark: false as any, markup: 'none', - // locale: 'nemeth', domain: 'default', - // style: 'default', modality: 'braille'}); - // explorer.showRegion = 'viewBraille'; - // return explorer; - // }, - // keyMagnifier: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => - // ke.Magnifier.create(doc, pool, doc.explorerRegions.magnifier, node, ...rest), mouseMagnifier: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ..._rest: any[]) => me.ContentHoverer.create(doc, pool, doc.explorerRegions.magnifier, node, (x: HTMLElement) => x.hasAttribute('data-semantic-type'), diff --git a/ts/a11y/explorer/Region.ts b/ts/a11y/explorer/Region.ts index 73d90ddfc..aeafde224 100644 --- a/ts/a11y/explorer/Region.ts +++ b/ts/a11y/explorer/Region.ts @@ -356,16 +356,6 @@ export class LiveRegion extends StringRegion { } }); - - /** - * @constructor - * @param {A11yDocument} document The document the live region is added to. - */ - constructor(public document: A11yDocument) { - super(document); - // this.div.setAttribute('aria-live', 'assertive'); - } - } diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts index 48b5623eb..ea6b8400c 100644 --- a/ts/a11y/semantic-enrich.ts +++ b/ts/a11y/semantic-enrich.ts @@ -242,7 +242,6 @@ export function EnrichedMathItemMixin Date: Wed, 6 Sep 2023 10:40:32 +0200 Subject: [PATCH 25/26] Starts explorer on focus as suggested by @pkra. --- ts/a11y/explorer/KeyExplorer.ts | 37 ++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts index 1f1232b45..ba02f1cf8 100644 --- a/ts/a11y/explorer/KeyExplorer.ts +++ b/ts/a11y/explorer/KeyExplorer.ts @@ -73,8 +73,10 @@ export interface KeyExplorer extends Explorer { } -const codeSelector = 'mjx-container[role="application"]'; -const nav = '[role="tree"],[role="group"],[role="treeitem"]'; +const codeSelector = 'mjx-container'; +const roles = ['tree', 'group', 'treeitem']; +const nav = roles.map(x => `[role="${x}"]`).join(','); +const prevNav = roles.map(x => `[tabindex="0"][role="${x}"]`).join(','); function isCodeBlock(el: HTMLElement) { return el.matches(codeSelector); @@ -110,23 +112,36 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo private move = false; + private mousedown = false; + /** * @override */ protected events: [string, (x: Event) => void][] = super.Events().concat( [ - // ['keydown', move], ['keydown', this.KeyDown.bind(this)], + ['mousedown', this.MouseDown.bind(this)], ['click', this.Click.bind(this)], ['focusin', this.FocusIn.bind(this)], ['focusout', this.FocusOut.bind(this)] ]); + /** + * Records a mouse down event on the element. This ensures that focus events + * only fire if they were not triggered by a mouse click. + * + * @param e The mouse event. + */ + private MouseDown(e: MouseEvent) { + this.mousedown = true; + e.preventDefault(); + } + public Click(e: MouseEvent) { const clicked = (e.target as HTMLElement).closest(nav) as HTMLElement; if (this.node.contains(clicked)) { - const prev = this.node.querySelector('[tabindex="0"][role="tree"],[tabindex="0"][role="group"],[tabindex="0"][role="treeitem"]'); + const prev = this.node.querySelector(prevNav); if (prev) { prev.removeAttribute('tabindex'); } @@ -147,7 +162,14 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo /** * @override */ - public FocusIn(_event: FocusEvent) { + public FocusIn(event: FocusEvent) { + if (this.mousedown) { + this.mousedown = false; + return; + } + this.current = this.current || this.node.querySelector('[role="tree"]'); + this.Start(); + event.preventDefault(); } /** @@ -167,7 +189,6 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo this.attached = true; this.oldIndex = this.node.tabIndex; this.node.tabIndex = 0; - this.node.setAttribute('role', 'application'); } /** @@ -197,7 +218,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo if (sib) { if (sib.matches(nav)) { return sib; - } + } const sibChild = sib.querySelector(nav) as HTMLElement; return sibChild ?? this.nextSibling(sib); } @@ -231,7 +252,7 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo return null; }], ]); - + /** * @override */ From 59a778ebe2178c29ab059c5bdadedb9789ed28fb Mon Sep 17 00:00:00 2001 From: zorkow Date: Wed, 6 Dec 2023 14:11:10 +0100 Subject: [PATCH 26/26] PR #987 incorporate review suggestions --- ts/a11y/SpeechUtil.ts | 35 +++++++++++++---- ts/a11y/explorer.ts | 1 + ts/a11y/explorer/KeyExplorer.ts | 7 +--- ts/a11y/semantic-enrich.ts | 70 ++++++++++++++++++--------------- ts/a11y/sre.ts | 3 ++ ts/output/svg.ts | 4 ++ 6 files changed, 75 insertions(+), 45 deletions(-) diff --git a/ts/a11y/SpeechUtil.ts b/ts/a11y/SpeechUtil.ts index e0de19094..2b66bbee1 100644 --- a/ts/a11y/SpeechUtil.ts +++ b/ts/a11y/SpeechUtil.ts @@ -1,5 +1,28 @@ +/************************************************************* + * + * Copyright (c) 2018-2023 The MathJax Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Provides utility functions for speech handling. + * + * @author v.sorge@mathjax.org (Volker Sorge) + */ + import {MmlNode} from '../core/MmlTree/MmlNode.js'; -// import {Sre} from './sre.js'; +import Sre from './sre.js'; const ProsodyKeys = [ 'pitch', 'rate', 'volume' ]; @@ -11,7 +34,6 @@ interface ProsodyElement { } export interface SsmlElement extends ProsodyElement { - [propName: string]: string | boolean | number; pause?: string; text?: string; mark?: string; @@ -27,11 +49,10 @@ export interface SsmlElement extends ProsodyElement { * @return {[string, SsmlElement[]]} The annotation structure. */ export function ssmlParsing(speech: string): [string, SsmlElement[]] { - let dp = new DOMParser(); - let xml = dp.parseFromString(speech, 'text/xml'); + let xml = Sre.parseDOM(speech); let instr: SsmlElement[] = []; let text: String[] = []; - recurseSsml(Array.from(xml.documentElement.childNodes), instr, text); + recurseSsml(Array.from(xml.childNodes), instr, text); return [text.join(' '), instr]; } @@ -79,8 +100,6 @@ function recurseSsml(nodes: Node[], instr: SsmlElement[], text: String[], instr.push(Object.assign({text: txt, character: true}, prosody)); text.push(txt); break; - default: - break; } } } @@ -123,7 +142,7 @@ function getProsody(element: Element, prosody: ProsodyElement) { /** * Extracts the prosody value from an attribute. */ -const prosodyRegexp = /([\+|-]*)([0-9]+)%/; +const prosodyRegexp = /([\+-]?)([0-9]+)%/; /** * Extracts the prosody value from an attribute. diff --git a/ts/a11y/explorer.ts b/ts/a11y/explorer.ts index bacecbd83..f3b44457a 100644 --- a/ts/a11y/explorer.ts +++ b/ts/a11y/explorer.ts @@ -261,6 +261,7 @@ export function ExplorerMathDocumentMixin implements KeyExplo public Update(force: boolean = false) { // TODO (v4): This is a hack to avoid double voicing on initial startup! // Make that cleaner and remove force as it is not really used! - let noUpdate = force; - force = false; + // let noUpdate = force; if (!this.active && !force) return; this.pool.unhighlight(); // let nodes = this.walker.getFocus(true).getNodes(); @@ -475,11 +474,9 @@ export class SpeechExplorer extends AbstractExplorer implements KeyExplo return; } } - let result = null; if (this.active) { - result = this.Move(event); this.stopEvent(event); - if (result) { + if (this.Move(event)) { this.Update(); return; } diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts index ea6b8400c..31e62438c 100644 --- a/ts/a11y/semantic-enrich.ts +++ b/ts/a11y/semantic-enrich.ts @@ -170,20 +170,21 @@ export function EnrichedMathItemMixin, force: boolean = false) { if (this.state() >= STATE.ENRICHED) return; if (!this.isEscaped && (document.options.enableEnrichment || force)) { - // TODO: Sort out the loading of the locales better - if (document.options.sre.locale !== currentLocale) { - currentLocale = document.options.sre.locale; - // TODO: Sort out the loading of the locales better - mathjax.retryAfter( - Sre.setupEngine({locale: document.options.sre.locale}).then( - () => { - return Sre.sreReady(); })); - } - if (document.options.sre.braille !== currentBraille) { - currentBraille = document.options.sre.braille; - mathjax.retryAfter( - Sre.setupEngine({locale: document.options.sre.braille}) - .then(() => Sre.sreReady())); + // TODO: Sort out the loading of the locales better + if (document.options.enableSpeech) { + if (document.options.sre.locale !== currentLocale) { + currentLocale = document.options.sre.locale; + // TODO: Sort out the loading of the locales better + mathjax.retryAfter( + Sre.setupEngine({locale: document.options.sre.locale}) + .then(() => Sre.sreReady())); + } + if (document.options.sre.braille !== currentBraille) { + currentBraille = document.options.sre.braille; + mathjax.retryAfter( + Sre.setupEngine({locale: document.options.sre.braille}) + .then(() => Sre.sreReady())); + } } const math = new document.options.MathItem('', MmlJax); try { @@ -195,23 +196,25 @@ export function EnrichedMathItemMixin, math: EnrichedMathItem, err: Error) => doc.enrichError(doc, math, err), diff --git a/ts/a11y/sre.ts b/ts/a11y/sre.ts index c39b0b73b..ae8b6b975 100644 --- a/ts/a11y/sre.ts +++ b/ts/a11y/sre.ts @@ -31,6 +31,7 @@ import {ClearspeakPreferences} from '#sre/speech_rules/clearspeak_preferences.js import {Highlighter} from '#sre/highlighter/highlighter.js'; import * as HighlighterFactory from '#sre/highlighter/highlighter_factory.js'; import {SpeechGenerator} from '#sre/speech_generator/speech_generator.js'; +import { parseInput } from '#sre/common/dom_util.js'; import {Variables} from '#sre/common/variables.js'; import MathMaps from './mathmaps.js'; @@ -65,6 +66,8 @@ export namespace Sre { export const getWalker = WalkerFactory.walker; + export const parseDOM = parseInput; + /** * Loads locales that are already included in the imported MathMaps. Defaults * to standard loading if a locale is not yet preloaded. diff --git a/ts/output/svg.ts b/ts/output/svg.ts index d86194500..502ad0120 100644 --- a/ts/output/svg.ts +++ b/ts/output/svg.ts @@ -97,6 +97,10 @@ CommonOutputJax< }, 'mjx-container[jax="SVG"] > svg a': { fill: 'blue', stroke: 'blue' + }, + 'rect[sre-highlighter-added]': { + stroke: 'black', + 'stroke-width': '40px' } };