desktop/source/lib/init.cxx | 2 include/LibreOfficeKit/LibreOfficeKitEnums.h | 44 +++- libreofficekit/source/gtk/lokdocview.cxx | 1 sfx2/source/view/viewsh.cxx | 280 ++++++++++++++++++++++++--- 4 files changed, 302 insertions(+), 25 deletions(-)
New commits: commit 59c2e114f76247158a0da9dcf91d6449d5d61224 Author: Marco Cecchetti <marco.cecche...@collabora.com> AuthorDate: Wed Jun 7 14:03:56 2023 +0200 Commit: Miklos Vajna <vmik...@collabora.com> CommitDate: Mon Aug 14 08:09:53 2023 +0200 lok: a11y: when we are inside a table notify table and current cell info When we get in one or more tables we notify row and column count. When we get out one or more table we notify we are leaving a table. When the fosused cell changes we notify the new row/col index. The paragraph content is notified together with table info, so that client has some opportunity for getting the screen reader to report together content and table info. Change-Id: Ic524259aa1879a70fc3de2467bdee27475352b7d Reviewed-on: https://gerrit.libreoffice.org/c/core/+/155577 Tested-by: Jenkins Reviewed-by: Miklos Vajna <vmik...@collabora.com> diff --git a/desktop/source/lib/init.cxx b/desktop/source/lib/init.cxx index 42d838b889b6..6d5295222f74 100644 --- a/desktop/source/lib/init.cxx +++ b/desktop/source/lib/init.cxx @@ -1798,6 +1798,7 @@ void CallbackFlushHandler::queue(const int type, CallbackData& aCallbackData) case LOK_CALLBACK_A11Y_FOCUS_CHANGED: case LOK_CALLBACK_A11Y_CARET_CHANGED: case LOK_CALLBACK_A11Y_TEXT_SELECTION_CHANGED: + case LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED: case LOK_CALLBACK_COLOR_PALETTES: { const auto& pos = std::find(m_queue1.rbegin(), m_queue1.rend(), type); @@ -1860,6 +1861,7 @@ void CallbackFlushHandler::queue(const int type, CallbackData& aCallbackData) case LOK_CALLBACK_A11Y_FOCUS_CHANGED: case LOK_CALLBACK_A11Y_CARET_CHANGED: case LOK_CALLBACK_A11Y_TEXT_SELECTION_CHANGED: + case LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED: case LOK_CALLBACK_COLOR_PALETTES: { if (removeAll(type)) diff --git a/include/LibreOfficeKit/LibreOfficeKitEnums.h b/include/LibreOfficeKit/LibreOfficeKitEnums.h index b7a43acf7d27..e7f31c2a14fe 100644 --- a/include/LibreOfficeKit/LibreOfficeKitEnums.h +++ b/include/LibreOfficeKit/LibreOfficeKitEnums.h @@ -924,14 +924,17 @@ typedef enum LOK_CALLBACK_APPLICATION_BACKGROUND_COLOR = 61, /** - * Accessibility event: a paragraph get focus. + * Accessibility event: a paragraph got focus. * The payload is a json with the following structure. * * { * "content": "<paragraph text>" * "position": N + * "start": N1 + * "end": N2 * } - * where N is the position of the text cursor inside the focused paragraph. + * where N is the position of the text cursor inside the focused paragraph, + * and [N1,N2] is the range of the text selection inside the focused paragraph. */ LOK_CALLBACK_A11Y_FOCUS_CHANGED = 62, @@ -946,7 +949,7 @@ typedef enum LOK_CALLBACK_A11Y_CARET_CHANGED = 63, /** - * Accessibility event: text cursor position has changed. + * Accessibility event: text selection has changed. * * { * "start": N1 @@ -965,7 +968,38 @@ typedef enum * Informs that the document password has been successfully changed. * The payload contains the new password and the type. */ - LOK_CALLBACK_DOCUMENT_PASSWORD_RESET = 66 + LOK_CALLBACK_DOCUMENT_PASSWORD_RESET = 66, + + /** + * Accessibility event: a cell got focus. + * The payload is a json with the following structure. + * + * { + * "outCount": <number of tables user gets out of> + * "inList": [ + * { + * "rowCount": <number of rows for outer table user got in> + * "colCount": <number of columns for outer table user got in> + * }, + * ... + * { + * "rowCount": <number of rows for inner table user got in> + * "colCount": <number of columns for inner table user got in> + * } + * ] + * "row": <current row index> + * "col": <current column index> + * "rowSpan": <row span for current cell> + * "colSpan": <column span for current cell> + * "paragraph": { + * <same structure as for LOK_CALLBACK_A11Y_FOCUS_CHANGED> + * } + * } + * where row/column indexes start from 0, inList is the list of tables + * the user got in from the outer to the inner; row/column span default + * value is 1; paragraph is the cell text content. + */ + LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED = 67 } LibreOfficeKitCallbackType; @@ -1128,6 +1162,8 @@ static inline const char* lokCallbackTypeToString(int nType) return "LOK_CALLBACK_COLOR_PALETTES"; case LOK_CALLBACK_DOCUMENT_PASSWORD_RESET: return "LOK_CALLBACK_DOCUMENT_PASSWORD_RESET"; + case LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED: + return "LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED"; } assert(!"Unknown LibreOfficeKitCallbackType type."); diff --git a/libreofficekit/source/gtk/lokdocview.cxx b/libreofficekit/source/gtk/lokdocview.cxx index 7245c8e2096f..c3df48448815 100644 --- a/libreofficekit/source/gtk/lokdocview.cxx +++ b/libreofficekit/source/gtk/lokdocview.cxx @@ -1491,6 +1491,7 @@ callback (gpointer pData) case LOK_CALLBACK_A11Y_FOCUS_CHANGED: case LOK_CALLBACK_A11Y_CARET_CHANGED: case LOK_CALLBACK_A11Y_TEXT_SELECTION_CHANGED: + case LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED: case LOK_CALLBACK_COLOR_PALETTES: case LOK_CALLBACK_DOCUMENT_PASSWORD_RESET: { diff --git a/sfx2/source/view/viewsh.cxx b/sfx2/source/view/viewsh.cxx index b4935b0d0ca8..9e9c106691c0 100644 --- a/sfx2/source/view/viewsh.cxx +++ b/sfx2/source/view/viewsh.cxx @@ -56,6 +56,7 @@ #include <com/sun/star/accessibility/AccessibleStateType.hpp> #include <com/sun/star/accessibility/AccessibleRole.hpp> #include <com/sun/star/accessibility/XAccessibleText.hpp> +#include <com/sun/star/accessibility/XAccessibleTable.hpp> #include <cppuhelper/implbase.hxx> #include <com/sun/star/ui/XAcceleratorConfiguration.hpp> @@ -105,6 +106,7 @@ #include <openuriexternally.hxx> #include <iostream> #include <vector> +#include <list> #include <libxml/xmlwriter.h> #include <toolkit/awt/vclxmenu.hxx> #include <unordered_map> @@ -238,6 +240,17 @@ void SAL_CALL SfxClipboardChangeListener::changedContents( const datatransfer::c delete pInfo; } +namespace +{ +struct TableSizeType +{ + sal_Int32 nRowCount; + sal_Int32 nColCount; +}; +} + +typedef std::list<uno::Reference<accessibility::XAccessibleTable>> XAccessibleTableList; + namespace { @@ -258,6 +271,41 @@ bool isFocused(const accessibility::AccessibleEventObject& aEvent) return hasState(aEvent, accessibility::AccessibleStateType::FOCUSED); } +// Put in rAncestorList all ancestors of xTable up to xAncestorTable or +// up to the first not-a-table ancestor if xAncestorTable is not an ancestor. +// xTable is included in the list, xAncestorTable is not included. +// The list is ordered from the ancient ancestor to xTable. +// Return true if xAncestorTable is an ancestor of xTable. +bool getAncestorList(XAccessibleTableList& rAncestorList, + const uno::Reference<accessibility::XAccessibleTable>& xTable, + const uno::Reference<accessibility::XAccessibleTable>& xAncestorTable = uno::Reference<accessibility::XAccessibleTable>()) +{ + uno::Reference<accessibility::XAccessibleTable> xCurrentTable = xTable; + while (xCurrentTable.is() && xCurrentTable != xAncestorTable) + { + rAncestorList.push_front(xCurrentTable); + + uno::Reference<accessibility::XAccessibleContext> xContext(xCurrentTable, uno::UNO_QUERY); + xCurrentTable.clear(); + if (xContext.is()) + { + uno::Reference<accessibility::XAccessible> xParent = xContext->getAccessibleParent(); + uno::Reference<accessibility::XAccessibleContext> xParentContext(xParent, uno::UNO_QUERY); + if (xParentContext.is() + && xParentContext->getAccessibleRole() == accessibility::AccessibleRole::TABLE_CELL) + { + uno::Reference<accessibility::XAccessible> xCellParent = xParentContext->getAccessibleParent(); + if (xCellParent.is()) + { + xCurrentTable = uno::Reference<accessibility::XAccessibleTable>(xCellParent, uno::UNO_QUERY); + } + } + } + } + + return xCurrentTable.is() && xCurrentTable == xAncestorTable; +} + std::string stateSetToString(::sal_Int64 stateSet) { static const std::string states[34] = { @@ -476,12 +524,33 @@ void aboutParagraph(std::string msg, const uno::Reference<css::accessibility::XA sal_Int32 nSelectionEnd = xAccText->getSelectionEnd(); aboutParagraph(msg, sText, nCaretPosition, nSelectionStart, nSelectionEnd, force); } + +void aboutFocusedCellChanged(sal_Int32 nOutCount, const std::vector<TableSizeType>& aInList, + sal_Int32 nRow, sal_Int32 nCol, sal_Int32 nRowSpan, sal_Int32 nColSpan) +{ + std::stringstream inListStream; + inListStream << "[ "; + for (const auto& rTableSize: aInList) + { + inListStream << "{ rowCount: " << rTableSize.nRowCount << " colCount: " << rTableSize.nColCount << " } "; + } + inListStream << "]"; + + SAL_INFO("lok.a11y", "LOKDocumentFocusListener::notifyFocusedCellChanged: " + "\n outCount: " << nOutCount + << "\n inList: " << inListStream.str() + << "\n row: " << nRow + << "\n column: " << nCol + << "\n rowSpan: " << nRowSpan + << "\n colSpan: " << nColSpan + ); +} } // anonymous namespace class LOKDocumentFocusListener : public ::cppu::WeakImplHelper< accessibility::XAccessibleEventListener > { - static constexpr sal_Int64 MAX_ATTACHABLE_CHILDREN = 30; + static constexpr sal_Int64 MAX_ATTACHABLE_CHILDREN = 100; const SfxViewShell* m_pViewShell; std::unordered_set< uno::Reference< uno::XInterface > > m_aRefList; @@ -489,6 +558,7 @@ class LOKDocumentFocusListener : sal_Int32 m_nCaretPosition; sal_Int32 m_nSelectionStart; sal_Int32 m_nSelectionEnd; + uno::Reference<accessibility::XAccessibleTable> m_xLastTable; OUString m_sSelectedText; bool m_bIsEditingCell; OUString m_sSelectedCellAddress; @@ -552,11 +622,14 @@ public: void notifyFocusedParagraphChanged(bool force = false); void notifyCaretChanged(); void notifyTextSelectionChanged(); + void notifyFocusedCellChanged(sal_Int32 nOutCount, const std::vector<TableSizeType>& aInList, sal_Int32 nRow, sal_Int32 nCol, sal_Int32 nRowSpan, sal_Int32 nColSpan); OUString getFocusedParagraph() const; int getCaretPosition() const; private: + void paragraphPropertiesToTree(boost::property_tree::ptree& aPayloadTree, bool force = false) const; + void paragraphPropertiesToJson(std::string& aPayload, bool force = false) const; bool updateParagraphInfo(const uno::Reference<css::accessibility::XAccessibleText>& xAccText, bool force, std::string msg = ""); void updateAndNotifyParagraph(const uno::Reference<css::accessibility::XAccessibleText>& xAccText, @@ -572,21 +645,34 @@ LOKDocumentFocusListener::LOKDocumentFocusListener(const SfxViewShell* pViewShel { } -OUString LOKDocumentFocusListener::getFocusedParagraph() const +void LOKDocumentFocusListener::paragraphPropertiesToTree(boost::property_tree::ptree& aPayloadTree, bool force) const { - aboutView("LOKDocumentFocusListener::getFocusedParagraph", this, m_pViewShell); - aboutParagraph("LOKDocumentFocusListener::getFocusedParagraph", - m_sFocusedParagraph, m_nCaretPosition, m_nSelectionStart, m_nSelectionEnd); - bool bLeftToRight = m_nCaretPosition == m_nSelectionEnd; - boost::property_tree::ptree aPayloadTree; aPayloadTree.put("content", m_sFocusedParagraph.toUtf8().getStr()); aPayloadTree.put("position", m_nCaretPosition); aPayloadTree.put("start", bLeftToRight ? m_nSelectionStart : m_nSelectionEnd); aPayloadTree.put("end", bLeftToRight ? m_nSelectionEnd : m_nSelectionStart); + if (force) + aPayloadTree.put("force", 1); +} + +void LOKDocumentFocusListener::paragraphPropertiesToJson(std::string& aPayload, bool force) const +{ + boost::property_tree::ptree aPayloadTree; + paragraphPropertiesToTree(aPayloadTree, force); std::stringstream aStream; boost::property_tree::write_json(aStream, aPayloadTree); - std::string aPayload = aStream.str(); + aPayload = aStream.str(); +} + +OUString LOKDocumentFocusListener::getFocusedParagraph() const +{ + aboutView("LOKDocumentFocusListener::getFocusedParagraph", this, m_pViewShell); + aboutParagraph("LOKDocumentFocusListener::getFocusedParagraph", + m_sFocusedParagraph, m_nCaretPosition, m_nSelectionStart, m_nSelectionEnd); + + std::string aPayload; + paragraphPropertiesToJson(aPayload); OUString sRet = OUString::fromUtf8(aPayload); return sRet; } @@ -620,16 +706,8 @@ int LOKDocumentFocusListener::getCaretPosition() const void LOKDocumentFocusListener::notifyFocusedParagraphChanged(bool force) { aboutView("LOKDocumentFocusListener::notifyFocusedParagraphChanged", this, m_pViewShell); - bool bLeftToRight = m_nCaretPosition == m_nSelectionEnd; - boost::property_tree::ptree aPayloadTree; - aPayloadTree.put("content", m_sFocusedParagraph.toUtf8().getStr()); - aPayloadTree.put("position", m_nCaretPosition); - aPayloadTree.put("start", bLeftToRight ? m_nSelectionStart : m_nSelectionEnd); - aPayloadTree.put("end", bLeftToRight ? m_nSelectionEnd : m_nSelectionStart); - aPayloadTree.put("force", force ? 1 : 0); - std::stringstream aStream; - boost::property_tree::write_json(aStream, aPayloadTree); - std::string aPayload = aStream.str(); + std::string aPayload; + paragraphPropertiesToJson(aPayload, force); if (m_pViewShell) { aboutParagraph("LOKDocumentFocusListener::notifyFocusedParagraphChanged", @@ -672,6 +750,59 @@ void LOKDocumentFocusListener::notifyTextSelectionChanged() } } +void LOKDocumentFocusListener::notifyFocusedCellChanged( + sal_Int32 nOutCount, const std::vector<TableSizeType>& aInList, + sal_Int32 nRow, sal_Int32 nCol, sal_Int32 nRowSpan, sal_Int32 nColSpan) +{ + aboutView("LOKDocumentFocusListener::notifyTablePositionChanged", this, m_pViewShell); + boost::property_tree::ptree aPayloadTree; + if (nOutCount > 0) + { + aPayloadTree.put("outCount", nOutCount); + } + if (aInList.size() > 0) + { + boost::property_tree::ptree aInListNode; + for (const auto& rTableSize: aInList) + { + boost::property_tree::ptree aTableSizeNode; + aTableSizeNode.put("rowCount", rTableSize.nRowCount); + aTableSizeNode.put("colCount", rTableSize.nColCount); + + aInListNode.push_back(std::make_pair(std::string(), aTableSizeNode)); + } + aPayloadTree.add_child("inList", aInListNode); + } + + aPayloadTree.put("row", nRow); + aPayloadTree.put("col", nCol); + + if (nRowSpan > 1) + { + aPayloadTree.put("rowSpan", nRowSpan); + } + if (nColSpan > 1) + { + aPayloadTree.put("colSpan", nColSpan); + } + + boost::property_tree::ptree aContentNode; + paragraphPropertiesToTree(aContentNode); + aPayloadTree.add_child("paragraph", aContentNode); + + std::stringstream aStream; + boost::property_tree::write_json(aStream, aPayloadTree); + std::string aPayload = aStream.str(); + if (m_pViewShell) + { + aboutFocusedCellChanged(nOutCount, aInList, nRow, nCol, nRowSpan, nColSpan); + aboutParagraph("LOKDocumentFocusListener::notifyFocusedCellChanged: paragraph: ", + m_sFocusedParagraph, m_nCaretPosition, m_nSelectionStart, m_nSelectionEnd, false); + + m_pViewShell->libreOfficeKitViewCallback(LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED, aPayload.c_str()); + } +} + void LOKDocumentFocusListener::disposing( const lang::EventObject& aEvent ) { // Unref the object here, but do not remove as listener since the object @@ -726,7 +857,6 @@ void LOKDocumentFocusListener::updateAndNotifyParagraph( notifyFocusedParagraphChanged(force); } - void LOKDocumentFocusListener::notifyEvent(const accessibility::AccessibleEventObject& aEvent ) { aboutView("LOKDocumentFocusListener::notifyEvent", this, m_pViewShell); @@ -748,6 +878,8 @@ void LOKDocumentFocusListener::notifyEvent(const accessibility::AccessibleEventO if( accessibility::AccessibleStateType::FOCUSED == nState ) { SAL_INFO("lok.a11y", "LOKDocumentFocusListener::notifyEvent: FOCUSED"); + uno::Reference<css::accessibility::XAccessibleText> xAccText(xAccessibleObject, uno::UNO_QUERY); + if (m_bIsEditingCell) { if (!hasState(aEvent, accessibility::AccessibleStateType::ACTIVE)) @@ -757,8 +889,114 @@ void LOKDocumentFocusListener::notifyEvent(const accessibility::AccessibleEventO return; } } - uno::Reference<css::accessibility::XAccessibleText> xAccText(xAccessibleObject, uno::UNO_QUERY); - updateAndNotifyParagraph(xAccText, false, "STATE_CHANGED: FOCUSED"); + + // check if we are inside a table: in case notify table and current cell info + bool isInsideTable = false; + uno::Reference<accessibility::XAccessibleContext> xContext(aEvent.Source, uno::UNO_QUERY); + if (xContext.is()) + { + uno::Reference<accessibility::XAccessible> xParent = xContext->getAccessibleParent(); + if (xParent.is()) + { + uno::Reference<accessibility::XAccessibleContext> xParentContext(xParent, uno::UNO_QUERY); + if (xParentContext.is() + && xParentContext->getAccessibleRole() == accessibility::AccessibleRole::TABLE_CELL) + { + uno::Reference<accessibility::XAccessible> xCellParent = xParentContext->getAccessibleParent(); + if (xCellParent.is()) + { + uno::Reference<accessibility::XAccessibleTable> xTable(xCellParent, uno::UNO_QUERY); + if (xTable.is()) + { + std::vector<TableSizeType> aInList; + sal_Int32 nOutCount = 0; + + if (m_xLastTable.is()) + { + if (xTable != m_xLastTable) + { + // do we get in one or more nested tables ? + // check if xTable is a descendant of m_xLastTable + XAccessibleTableList newTableAncestorList; + bool isLastAncestorOfNew = getAncestorList(newTableAncestorList, xTable, m_xLastTable); + bool isNewAncestorOfLast = false; + if (!isLastAncestorOfNew) + { + // do we get out of one or more nested tables ? + // check if m_xLastTable is a descendant of xTable + XAccessibleTableList lastTableAncestorList; + isNewAncestorOfLast = getAncestorList(lastTableAncestorList, m_xLastTable, xTable); + // we have to notify "out of table" for all m_xLastTable ancestors up to xTable + // or the first not-a-table ancestor + nOutCount = lastTableAncestorList.size(); + } + if (isLastAncestorOfNew || !isNewAncestorOfLast) + { + // we have to notify row/col count for all xTable ancestors starting from the ancestor + // which is a child of m_xLastTable (isLastAncestorOfNew) or the first not-a-table ancestor + for (const auto& ancestor: newTableAncestorList) + { + TableSizeType aTableSize{ancestor->getAccessibleRowCount(), + ancestor->getAccessibleColumnCount()}; + aInList.push_back(aTableSize); + } + } + } + } + else + { + // cursor was not inside any table and gets inside one or more tables + // we have to notify row/col count for all xTable ancestors starting from first not-a-table ancestor + XAccessibleTableList newTableAncestorList; + getAncestorList(newTableAncestorList, xTable); + for (const auto& ancestor: newTableAncestorList) + { + TableSizeType aTableSize{ancestor->getAccessibleRowCount(), + ancestor->getAccessibleColumnCount()}; + aInList.push_back(aTableSize); + } + } + + // we have to notify current row/col of xTable and related row/col span + sal_Int64 nChildIndex = xParentContext->getAccessibleIndexInParent(); + sal_Int32 nRow = xTable->getAccessibleRow(nChildIndex); + sal_Int32 nCol = xTable->getAccessibleColumn(nChildIndex); + sal_Int32 nRowSpan = xTable->getAccessibleRowExtentAt(nRow, nCol); + sal_Int32 nColSpan = xTable->getAccessibleColumnExtentAt(nRow, nCol); + + m_xLastTable = xTable; + updateParagraphInfo(xAccText, false, "STATE_CHANGED: FOCUSED"); + notifyFocusedCellChanged(nOutCount, aInList, nRow, nCol, nRowSpan, nColSpan); + isInsideTable = true; + } + } + } + } + } + + // paragraph is not inside any table + if (!isInsideTable) + { + if (m_xLastTable.is()) + { + // we get out one or more tables + // we have to notify "out of table" for all m_xLastTable ancestors + // up to the first not-a-table ancestor + XAccessibleTableList lastTableAncestorList; + getAncestorList(lastTableAncestorList, m_xLastTable); + sal_Int32 nOutCount = lastTableAncestorList.size(); + // no more inside a table + m_xLastTable.clear(); + // notify + std::vector<TableSizeType> aInList; + updateParagraphInfo(xAccText, false, "STATE_CHANGED: FOCUSED"); + notifyFocusedCellChanged(nOutCount, aInList, -1, -1, 1, 1); + } + else + { + updateAndNotifyParagraph(xAccText, false, "STATE_CHANGED: FOCUSED"); + } + } aboutTextFormatting("LOKDocumentFocusListener::notifyEvent: STATE_CHANGED: FOCUSED", xAccText); } break;