diff --git a/nvdaHelper/common/ia2utils.cpp b/nvdaHelper/common/ia2utils.cpp index 89675c498ce..e383248e4d3 100644 --- a/nvdaHelper/common/ia2utils.cpp +++ b/nvdaHelper/common/ia2utils.cpp @@ -1,7 +1,7 @@ /* This file is a part of the NVDA project. URL: http://www.nvda-project.org/ -Copyright 2007-2019 NV Access Limited, Mozilla Corporation +Copyright 2007-2021 NV Access Limited, Mozilla Corporation This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2.0, as published by the Free Software Foundation. @@ -15,9 +15,21 @@ This license can be found at: #include #include #include "ia2utils.h" +#include using namespace std; +bool fetchIA2Attributes(IAccessible2* pacc2, map& attribsMap) { + BSTR attribs = NULL; + pacc2->get_attributes(&attribs); + if (!attribs) { + return false; + } + IA2AttribsToMap(attribs, attribsMap); + SysFreeString(attribs); + return true; +} + void IA2AttribsToMap(const wstring &attribsString, map &attribsMap) { wstring str, key; bool inEscape = false; diff --git a/nvdaHelper/common/ia2utils.h b/nvdaHelper/common/ia2utils.h index 8c3761108b7..e847b706041 100644 --- a/nvdaHelper/common/ia2utils.h +++ b/nvdaHelper/common/ia2utils.h @@ -1,7 +1,7 @@ /* This file is a part of the NVDA project. URL: http://www.nvda-project.org/ -Copyright 2007-2019 NV Access Limited, Mozilla Corporation +Copyright 2007-2021 NV Access Limited, Mozilla Corporation This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2.0, as published by the Free Software Foundation. @@ -21,6 +21,8 @@ This license can be found at: #include #include +bool fetchIA2Attributes(IAccessible2* pacc2, std::map& attribsMap); + /** * Convert an IAccessible2 attributes string to a map of attribute keys and values. * An IAccessible2 attributes string is of the form "name:value;name:value;...;" diff --git a/nvdaHelper/interfaces/nvdaInProcUtils/nvdaInProcUtils.acf b/nvdaHelper/interfaces/nvdaInProcUtils/nvdaInProcUtils.acf index 0234117ea07..e44c2b96487 100644 --- a/nvdaHelper/interfaces/nvdaInProcUtils/nvdaInProcUtils.acf +++ b/nvdaHelper/interfaces/nvdaInProcUtils/nvdaInProcUtils.acf @@ -1,7 +1,7 @@ /* This file is a part of the NVDA project. URL: http://www.nvda-project.org/ -Copyright 2006-2010 NVDA contributers. +Copyright 2006-2021 NV Access Limited, Leonard de Ruijter This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2.0, as published by the Free Software Foundation. @@ -24,6 +24,7 @@ interface NvdaInProcUtils { [fault_status,comm_status] getActiveObject(); [fault_status,comm_status] dumpOnCrash(); [fault_status,comm_status] IA2Text_findContentDescendant(); + [fault_status,comm_status] getTextFromIAccessible(); [fault_status,comm_status] outlook_getMAPIProp(); [fault_status,comm_status] excel_getCellInfos(); diff --git a/nvdaHelper/interfaces/nvdaInProcUtils/nvdaInProcUtils.idl b/nvdaHelper/interfaces/nvdaInProcUtils/nvdaInProcUtils.idl index b5c13ef4aa0..dc678bdd6df 100644 --- a/nvdaHelper/interfaces/nvdaInProcUtils/nvdaInProcUtils.idl +++ b/nvdaHelper/interfaces/nvdaInProcUtils/nvdaInProcUtils.idl @@ -1,7 +1,7 @@ /* This file is a part of the NVDA project. URL: http://www.nvda-project.org/ -Copyright 2006-2020 NV Access Limited, Leonard de Ruijter. +Copyright 2006-2021 NV Access Limited, Leonard de Ruijter. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2.0, as published by the Free Software Foundation. @@ -15,7 +15,7 @@ http://www.gnu.org/licenses/old-licenses/gpl-2.0.html cpp_quote("/*") cpp_quote("This file is a part of the NVDA project.") cpp_quote("URL: http://www.nvda-project.org/") -cpp_quote("Copyright 2006-2020 NV Access Limited, Leonard de Ruijter.") +cpp_quote("Copyright 2006-2021 NV Access Limited, Leonard de Ruijter.") cpp_quote("This program is free software: you can redistribute it and/or modify") cpp_quote("it under the terms of the GNU General Public License version 2.0, as published by") cpp_quote("the Free Software Foundation.") @@ -74,6 +74,21 @@ interface NvdaInProcUtils { error_status_t IA2Text_findContentDescendant([in] handle_t bindingHandle, [in] const unsigned long hwnd, [in] long parentID, [in] long what, [out] long* descendantID, [out] long* descendantOffset); + + cpp_quote("/* Actually maps to nvdaInProcUtils_getTextFromIAccessible") + cpp_quote("*/") + error_status_t getTextFromIAccessible( + [in] handle_t bindingHandle, + [in] const unsigned long hwnd, + [in] long parentID, + // Params for getTextFromIAccessible + [out, string] BSTR* textBuf, + [in, defaultvalue(TRUE)] const boolean recurse, + [in, defaultvalue(TRUE)] const boolean includeTopLevelText + ); + + + /* * Fetches the requested property from the given MAPIProp object. * @param threadID the threadID of the Outlook GUI thread where the MAPI object came from. diff --git a/nvdaHelper/local/nvdaHelperLocal.def b/nvdaHelper/local/nvdaHelperLocal.def index 6453b5a0c50..70e31eed47a 100644 --- a/nvdaHelper/local/nvdaHelperLocal.def +++ b/nvdaHelper/local/nvdaHelperLocal.def @@ -12,6 +12,7 @@ EXPORTS nvdaInProcUtils_sysListView32_getColumnContent nvdaInProcUtils_dumpOnCrash nvdaInProcUtils_IA2Text_findContentDescendant + nvdaInProcUtils_getTextFromIAccessible nvdaInProcUtils_getActiveObject nvdaInProcUtils_winword_expandToLine nvdaInProcUtils_winword_getTextInRange diff --git a/nvdaHelper/readme.md b/nvdaHelper/readme.md index f1dbeeded15..82c0170db8f 100644 --- a/nvdaHelper/readme.md +++ b/nvdaHelper/readme.md @@ -5,6 +5,20 @@ This cache is called a virtual buffer. While virtual buffers apply to several different application types, it may first be easiest to think about how they work with browsers, the rest of this document will take a web browser centric view unless specified otherwise. +### Output +Internal code is built into three DLLs: +- `nvdaHelperLocal.dll` +- `nvdaHelperLocalWin10.dll` +- `nvdaHelperRemote.dll` + +Several COM Proxy DLLs are built from IDL files. A COM Proxy tells windows how to marshal data over COM when calling + the target API interface. +For instance: +- IAccessible2 IDL files are built into `IAccessible2Proxy.dll` +- ISimpleDOM IDL files are built into `ISimpleDOM.dll` + +The `*local*.dll`'s are built for x86 (ie to match NVDA's arch), others are built for x86, x64, and arm64. + ### Configuring Visual Studio The following steps won't prepare a buildable solution, but it will enable intellisense. You should still build on the command line to verify errors. @@ -71,3 +85,15 @@ When `fillVBuf` is called, it recursively descends through the child elements th `controlFieldNode`s or `textFieldNode`s for them. This code is responsible for interacting with the IA2 accessibility API, these calls should be minimised for performance reasons. + + +### Overview of calling to remote code. + +- NVDA's python code calls into the local DLL (`NVDAHelperLocal.dll`). +- Generated RPC Wrappers are called +- The RPC Wrappers call through to the "remote in-process DLL" (ie `nvdaHelperRemote.dll`) + +#### Build notes +IDL/ACF files are input to MSRPCStubs to generate headers in `build/` +- See `MSRPCStubs` in `*scons*` files. +- Note these set a prefix, making whole word searches for methods difficult. \ No newline at end of file diff --git a/nvdaHelper/remote/IA2Support.cpp b/nvdaHelper/remote/IA2Support.cpp index 3ebfa08a757..4c3c83d008c 100755 --- a/nvdaHelper/remote/IA2Support.cpp +++ b/nvdaHelper/remote/IA2Support.cpp @@ -1,7 +1,7 @@ /* This file is a part of the NVDA project. URL: http://www.nvda-project.org/ -Copyright 2006-2010 NVDA contributers. +Copyright 2006-2021 NV Access Limited This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2.0, as published by the Free Software Foundation. @@ -26,6 +26,8 @@ This license can be found at: #include #include "COMProxyRegistration.h" #include "IA2Support.h" +#include +#include "textFromIAccessible.h" using namespace std; @@ -251,26 +253,106 @@ bool findContentDescendant(IAccessible2* pacc2, long what, long* descendantID, l return foundDescendant; } + +CComPtr getIA2(const HWND hwnd, const long parentID) { + VARIANT varChild; + CComPtr pacc; + AccessibleObjectFromEvent( + hwnd, + OBJID_CLIENT, + parentID, + &pacc.p, + &varChild + ); + + if (!pacc) { + return nullptr; + }; + + CComQIPtr pserv(pacc); + if (!pserv) { + return nullptr; + } + + CComPtr pacc2; + { // scoping for: ppvObject + void** ppvObject = reinterpret_cast(&pacc2.p); + pserv->QueryService(IID_IAccessible, IID_IAccessible2, ppvObject); + } + + return pacc2; +} + error_status_t nvdaInProcUtils_IA2Text_findContentDescendant(handle_t bindingHandle, const unsigned long windowHandle, long parentID, long what, long* descendantID, long* descendantOffset) { - HWND hwnd=(HWND)UlongToHandle(windowHandle); + HWND hwnd = static_cast(UlongToHandle(windowHandle)); auto func=[&] { - IAccessible* pacc=NULL; - VARIANT varChild; - AccessibleObjectFromEvent((HWND)hwnd,OBJID_CLIENT,parentID,&pacc,&varChild); - if(!pacc) return; - IAccessible2* pacc2=NULL; - IServiceProvider* pserv=NULL; - pacc->QueryInterface(IID_IServiceProvider,(void**)&pserv); - pacc->Release(); - if(!pserv) return; - pserv->QueryService(IID_IAccessible,IID_IAccessible2,(void**)&pacc2); - pserv->Release(); - if(!pacc2) return; - findContentDescendant(pacc2,what,descendantID,descendantOffset); - pacc2->Release(); + auto pacc2 = getIA2(hwnd, parentID); + if (!pacc2) { + return; + } + findContentDescendant(pacc2, what, descendantID, descendantOffset); + }; + + auto windowThreadProcId = GetWindowThreadProcessId(hwnd, nullptr); + auto res = execInThread(windowThreadProcId, func); + if(!res) { + LOG_DEBUGWARNING(L"Could not execute findContentDescendant in UI thread"); + } + return 0; +} + + +error_status_t nvdaInProcUtils_getTextFromIAccessible( + handle_t bindingHandle, + const unsigned long windowHandle, + long parentID, + // Params for getTextFromIAccessible + BSTR* outBuf, + const boolean recurse, + const boolean includeTopLevelText +) { + LOG_DEBUG(L"Called nvdaInProcUtils_getTextFromIAccessible"); + if (outBuf == nullptr) { + LOG_ERROR(L"outBuff is null."); + return 0; + } + HWND hwnd = static_cast(UlongToHandle(windowHandle)); + auto func = [&] () -> void{ + auto pacc2 = getIA2(hwnd, parentID); + if (!pacc2) { + return; + } + wstring textBuf; + const auto gotText = getTextFromIAccessible( + textBuf, + pacc2, + false, // useNewText, only valid in response to an event (indicating changing text) + recurse, + includeTopLevelText + ); + if (!gotText) { + LOG_DEBUGWARNING(L"Unable to get text."); + return; + } + if (textBuf.empty()) { + LOG_DEBUGWARNING(L"textBuf empty."); + return; + } + auto copySize = size_t(std::numeric_limits::max); + if (copySize < textBuf.size()) { + LOG_ERROR(L"Size of buffer larger than can be allocated with SysAllocStringLen, buffer will be truncated."); + } + else { + copySize = textBuf.size(); + } + *outBuf = SysAllocStringLen(textBuf.data(), UINT(copySize)); + return; }; - if(!execInThread(GetWindowThreadProcessId(hwnd,NULL),func)) { - LOG_DEBUGWARNING(L"Could not execute findContentDescendant in UI thread"); + + auto windowThreadProcId = GetWindowThreadProcessId(hwnd, nullptr); + auto res = execInThread(windowThreadProcId, func); + if (!res) { + LOG_DEBUGWARNING(L"Could not execute getTextFromIAccessible in UI thread"); } return 0; } diff --git a/nvdaHelper/remote/ia2LiveRegions.cpp b/nvdaHelper/remote/ia2LiveRegions.cpp index 90b744cd564..8bb5b398e51 100644 --- a/nvdaHelper/remote/ia2LiveRegions.cpp +++ b/nvdaHelper/remote/ia2LiveRegions.cpp @@ -1,7 +1,7 @@ /* This file is a part of the NVDA project. URL: http://www.nvda-project.org/ -Copyright 2006-2020 NV Access Limited, Google LLC, Leonard de Ruijter +Copyright 2006-2021 NV Access Limited, Google LLC, Leonard de Ruijter This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2.0, as published by the Free Software Foundation. @@ -17,27 +17,16 @@ This license can be found at: #define WIN32_LEAN_AND_MEAN #include #include -#include #include #include #include "nvdaHelperRemote.h" +#include "textFromIAccessible.h" using namespace std; const long NAVRELATION_EMBEDS = 0x1009; const long NAVRELATION_CONTAINING_TAB_PANE = 0x1012; -bool fetchIA2Attributes(IAccessible2* pacc2, map& attribsMap) { - BSTR attribs=NULL; - pacc2->get_attributes(&attribs); - if(!attribs) { - return false; - } - IA2AttribsToMap(attribs,attribsMap); - SysFreeString(attribs); - return true; -} - IAccessible2* findAriaAtomic(IAccessible2* pacc2,map& attribsMap) { map::iterator i=attribsMap.find(L"atomic"); bool atomic=(i!=attribsMap.end()&&i->second.compare(L"true")==0); @@ -66,137 +55,6 @@ IAccessible2* findAriaAtomic(IAccessible2* pacc2,map& attribsMa return pacc2Atomic; } -bool getTextFromIAccessible(wstring& textBuf, IAccessible2* pacc2, bool useNewText=false, bool recurse=true, bool includeTopLevelText=true) { - bool gotText=false; - IAccessibleText* paccText=NULL; - if(pacc2->QueryInterface(IID_IAccessibleText,(void**)&paccText)!=S_OK) { - paccText=NULL; - } - if(!paccText&&recurse&&!useNewText) { - //no IAccessibleText interface, so try children instead - long childCount=0; - if(!useNewText&&pacc2->get_accChildCount(&childCount)==S_OK&&childCount>0) { - VARIANT* varChildren=new VARIANT[childCount]; - AccessibleChildren(pacc2,0,childCount,varChildren,&childCount); - for(int i=0;iQueryInterface(IID_IAccessible2,(void**)&pacc2Child)==S_OK) { - map childAttribsMap; - fetchIA2Attributes(pacc2Child,childAttribsMap); - auto i=childAttribsMap.find(L"live"); - if(i==childAttribsMap.end()||i->second.compare(L"off")!=0) { - if(getTextFromIAccessible(textBuf,pacc2Child)) { - gotText=true; - } - } - pacc2Child->Release(); - } - } - VariantClear(varChildren+i); - } - delete [] varChildren; - } - } else if(paccText) { - //We can use IAccessibleText because it exists - BSTR bstrText=NULL; - long startOffset=0; - //If requested, get the text from IAccessibleText::newText rather than just IAccessibleText::text. - if(useNewText) { - IA2TextSegment newSeg={0}; - if(paccText->get_newText(&newSeg)==S_OK&&newSeg.text) { - bstrText=newSeg.text; - startOffset=newSeg.start; - } - } else { - paccText->get_text(0,IA2_TEXT_OFFSET_LENGTH,&bstrText); - } - //If we got text, add it to the string provided, however if there are embedded objects in the text, recurse in to these - if(bstrText) { - long textLength=SysStringLen(bstrText); - IAccessibleHypertext* paccHypertext=NULL; - if(!recurse||pacc2->QueryInterface(IID_IAccessibleHypertext,(void**)&paccHypertext)!=S_OK) paccHypertext=NULL; - for(long index=0;indexget_hyperlinkIndex(startOffset+index,&hyperlinkIndex)==S_OK) { - IAccessibleHyperlink* paccHyperlink=NULL; - if(paccHypertext->get_hyperlink(hyperlinkIndex,&paccHyperlink)==S_OK) { - IAccessible2* pacc2Child=NULL; - if(paccHyperlink->QueryInterface(IID_IAccessible2,(void**)&pacc2Child)==S_OK) { - map childAttribsMap; - fetchIA2Attributes(pacc2Child,childAttribsMap); - auto i=childAttribsMap.find(L"live"); - if(i==childAttribsMap.end()||i->second.compare(L"off")!=0) { - if(getTextFromIAccessible(textBuf,pacc2Child)) { - gotText=true; - } - } - charAdded=true; - pacc2Child->Release(); - } - paccHyperlink->Release(); - } - } - } - if(!charAdded&&includeTopLevelText) { - textBuf.append(1,realChar); - charAdded=true; - if(realChar!=L'\xfffc'&&!iswspace(realChar)) { - gotText=true; - } - } - } - if(paccHypertext) paccHypertext->Release(); - SysFreeString(bstrText); - textBuf.append(1,L' '); - } - paccText->Release(); - } - if(!gotText&&!useNewText) { - //We got no text from IAccessibleText interface or children, so try name and/or description - BSTR val=NULL; - bool valEmpty=true; - VARIANT varChild; - varChild.vt=VT_I4; - varChild.lVal=0; - pacc2->get_accName(varChild,&val); - if(val) { - for(int i=0;val[i]!=L'\0';++i) { - if(val[i]!=L'\xfffc'&&!iswspace(val[i])) { - valEmpty=false; - break; - } - } - if(!valEmpty) { - gotText=true; - textBuf.append(val); - textBuf.append(L" "); - } - SysFreeString(val); - val=NULL; - } - valEmpty=true; - pacc2->get_accDescription(varChild,&val); - if(val) { - for(int i=0;val[i]!=L'\0';++i) { - if(val[i]!=L'\xfffc'&&!iswspace(val[i])) { - valEmpty=false; - break; - } - } - if(!valEmpty) { - gotText=true; - textBuf.append(val); - } - SysFreeString(val); - } - } - return gotText; -} - long getIa2UniqueIdFromDispatchVariant(VARIANT& variant) { if (variant.vt != VT_DISPATCH || !variant.pdispVal) { return 0; diff --git a/nvdaHelper/remote/ia2LiveRegions.h b/nvdaHelper/remote/ia2LiveRegions.h index 19deee9e61a..849d7e152be 100644 --- a/nvdaHelper/remote/ia2LiveRegions.h +++ b/nvdaHelper/remote/ia2LiveRegions.h @@ -1,7 +1,7 @@ /* This file is a part of the NVDA project. URL: http://www.nvda-project.org/ -Copyright 2006-2010 NVDA contributers. +Copyright 2006-2021 NV Access Limited This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2.0, as published by the Free Software Foundation. @@ -11,11 +11,13 @@ Copyright 2006-2010 NVDA contributers. This license can be found at: http://www.gnu.org/licenses/old-licenses/gpl-2.0.html */ +#include #ifndef IA2LIVEREGIONS_H #define IA2LIVEREGIONS_H + void ia2LiveRegions_inProcess_initialize(); void ia2LiveRegions_inProcess_terminate(); -#endif +#endif // IA2LIVEREGIONS_H diff --git a/nvdaHelper/remote/sconscript b/nvdaHelper/remote/sconscript index e5677e45802..7fcf01eb1e9 100644 --- a/nvdaHelper/remote/sconscript +++ b/nvdaHelper/remote/sconscript @@ -1,15 +1,15 @@ ### -#This file is a part of the NVDA project. -#URL: http://www.nvda-project.org/ -#Copyright 2006-2010 NVDA contributers. -#This program is free software: you can redistribute it and/or modify -#it under the terms of the GNU General Public License version 2.0, as published by -#the Free Software Foundation. -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#This license can be found at: -#http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +# This file is a part of the NVDA project. +# URL: http://www.nvda-project.org/ +# Copyright 2006-2021 NVDA contributors. +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2.0, as published by +# the Free Software Foundation. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# This license can be found at: +# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html ### Import([ @@ -94,6 +94,7 @@ source = [ "COMProxyRegistration.cpp", "ia2Support.cpp", "ia2LiveRegions.cpp", + "textFromIAccessible.cpp", ia2utilsObj, env.Object('_ia2_i',ia2RPCStubs[3]), "rpcSrv.cpp", diff --git a/nvdaHelper/remote/textFromIAccessible.cpp b/nvdaHelper/remote/textFromIAccessible.cpp new file mode 100644 index 00000000000..5dd45f1529f --- /dev/null +++ b/nvdaHelper/remote/textFromIAccessible.cpp @@ -0,0 +1,162 @@ +/* +This file is a part of the NVDA project. +Copyright 2006-2021 NV Access Limited + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + +#include "textFromIAccessible.h" +#include +#define WIN32_LEAN_AND_MEAN +#include +#include +#include +#include + +using namespace std; + + +bool getTextFromIAccessible( + wstring& textBuf, + IAccessible2* pacc2, + bool useNewText, + bool recurse, + bool includeTopLevelText +) { + bool gotText = false; + IAccessibleText* paccText = NULL; + if (pacc2->QueryInterface(IID_IAccessibleText, (void**)&paccText) != S_OK) { + paccText = NULL; + } + if (!paccText && recurse && !useNewText) { + //no IAccessibleText interface, so try children instead + long childCount = 0; + if (!useNewText && pacc2->get_accChildCount(&childCount) == S_OK && childCount > 0) { + VARIANT* varChildren = new VARIANT[childCount]; + AccessibleChildren(pacc2, 0, childCount, varChildren, &childCount); + for (int i = 0; i < childCount; ++i) { + if (varChildren[i].vt == VT_DISPATCH) { + IAccessible2* pacc2Child = NULL; + if (varChildren[i].pdispVal && varChildren[i].pdispVal->QueryInterface(IID_IAccessible2, (void**)&pacc2Child) == S_OK) { + map childAttribsMap; + fetchIA2Attributes(pacc2Child, childAttribsMap); + auto liveItr = childAttribsMap.find(L"live"); + if (liveItr == childAttribsMap.end() || liveItr->second.compare(L"off") != 0) { + if (getTextFromIAccessible(textBuf, pacc2Child)) { + gotText = true; + } + } + pacc2Child->Release(); + } + } + VariantClear(varChildren + i); + } + delete[] varChildren; + } + } + else if (paccText) { + //We can use IAccessibleText because it exists + BSTR bstrText = NULL; + long startOffset = 0; + //If requested, get the text from IAccessibleText::newText rather than just IAccessibleText::text. + if (useNewText) { + IA2TextSegment newSeg = { 0 }; + if (paccText->get_newText(&newSeg) == S_OK && newSeg.text) { + bstrText = newSeg.text; + startOffset = newSeg.start; + } + } + else { + paccText->get_text(0, IA2_TEXT_OFFSET_LENGTH, &bstrText); + } + //If we got text, add it to the string provided, however if there are embedded objects in the text, recurse in to these + if (bstrText) { + long textLength = SysStringLen(bstrText); + IAccessibleHypertext* paccHypertext = NULL; + if (!recurse || pacc2->QueryInterface(IID_IAccessibleHypertext, (void**)&paccHypertext) != S_OK) paccHypertext = NULL; + for (long index = 0; index < textLength; ++index) { + wchar_t realChar = bstrText[index]; + bool charAdded = false; + if (realChar == L'\xfffc') { + long hyperlinkIndex; + if (paccHypertext && paccHypertext->get_hyperlinkIndex(startOffset + index, &hyperlinkIndex) == S_OK) { + IAccessibleHyperlink* paccHyperlink = NULL; + if (paccHypertext->get_hyperlink(hyperlinkIndex, &paccHyperlink) == S_OK) { + IAccessible2* pacc2Child = NULL; + if (paccHyperlink->QueryInterface(IID_IAccessible2, (void**)&pacc2Child) == S_OK) { + map childAttribsMap; + fetchIA2Attributes(pacc2Child, childAttribsMap); + auto liveItr = childAttribsMap.find(L"live"); + if (liveItr == childAttribsMap.end() || liveItr->second.compare(L"off") != 0) { + if (getTextFromIAccessible(textBuf, pacc2Child)) { + gotText = true; + } + } + charAdded = true; + pacc2Child->Release(); + } + paccHyperlink->Release(); + } + } + } + if (!charAdded && includeTopLevelText) { + textBuf.append(1, realChar); + charAdded = true; + if (realChar != L'\xfffc' && !iswspace(realChar)) { + gotText = true; + } + } + } + if (paccHypertext) paccHypertext->Release(); + SysFreeString(bstrText); + textBuf.append(1, L' '); + } + paccText->Release(); + } + if (!gotText && !useNewText) { + //We got no text from IAccessibleText interface or children, so try name and/or description + BSTR val = NULL; + bool valEmpty = true; + VARIANT varChild; + varChild.vt = VT_I4; + varChild.lVal = 0; + pacc2->get_accName(varChild, &val); + if (val) { + for (int i = 0; val[i] != L'\0'; ++i) { + if (val[i] != L'\xfffc' && !iswspace(val[i])) { + valEmpty = false; + break; + } + } + if (!valEmpty) { + gotText = true; + textBuf.append(val); + textBuf.append(L" "); + } + SysFreeString(val); + val = NULL; + } + valEmpty = true; + pacc2->get_accDescription(varChild, &val); + if (val) { + for (int i = 0; val[i] != L'\0'; ++i) { + if (val[i] != L'\xfffc' && !iswspace(val[i])) { + valEmpty = false; + break; + } + } + if (!valEmpty) { + gotText = true; + textBuf.append(val); + } + SysFreeString(val); + } + } + return gotText; +} diff --git a/nvdaHelper/remote/textFromIAccessible.h b/nvdaHelper/remote/textFromIAccessible.h new file mode 100644 index 00000000000..9ea2fcfd2dd --- /dev/null +++ b/nvdaHelper/remote/textFromIAccessible.h @@ -0,0 +1,42 @@ +/* +This file is a part of the NVDA project. +Copyright 2006-2021 NV Access Limited + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ +#include + +#ifndef IA2TEXTFROMIACCESSIBLE_H +#define IA2TEXTFROMIACCESSIBLE_H +struct IAccessible2; + +/* getTextFromIAccessible +* Summarizes the text for an IAccessible +* @param textBuf: An out-param, the string to fill with the text from the IAccessible +* @param pacc2: The IAccessible2 object to get the text from. +* @param useNewText: Only valid during an in-process winEvent callback +* (ie IA2_EVENT_TEXT_INSERTED, IA2_EVENT_TEXT_REMOVED, IA2_EVENT_TEXT_UPDATED). +* When true, only the added text is returned. +* There is no equivalent way to get +* Example: "abcd" has "xx" inserted to become "abxxcd" +* When useNewText is true, only "xx" is returned. +* when useNewText is false, the full text "abxxcd" is returned. +* @param recurse: Should text from child objects also be gathered. +* @param includeTopLevelText: If true, text from the top level pacc2 should be included. +* @return: true if text was collected. +*/ +bool getTextFromIAccessible( + std::wstring& textBuf, + IAccessible2* pacc2, + bool useNewText = false, + bool recurse = true, + bool includeTopLevelText = true +); + +#endif // IA2TEXTFROMIACCESSIBLE_H diff --git a/nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.cpp b/nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.cpp index 0ae418df2b6..aa3d65225f9 100755 --- a/nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.cpp +++ b/nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.cpp @@ -1130,22 +1130,7 @@ VBufStorage_fieldNode_t* GeckoVBufBackend_t::fillVBuf( of the nodes in the relationship will not be in the buffer yet */ std::optional detailsId = getRelationId(IA2_RELATION_DETAILS, pacc); if (detailsId) { - auto detailsControlFieldNode = buffer->getControlFieldNodeWithIdentifier(docHandle, detailsId.value()); - if (detailsControlFieldNode) { - std::wstring detailsSummary = L""; - detailsControlFieldNode->getTextInRange(0, detailsControlFieldNode->getLength(), detailsSummary, false); - parentNode->addAttribute(L"detailsSummary", detailsSummary); - } - } - - std::optional detailsForId = getRelationId(IA2_RELATION_DETAILS_FOR, pacc); - if (detailsForId) { - auto detailsControlFieldNode = buffer->getControlFieldNodeWithIdentifier(docHandle, detailsForId.value()); - if (detailsControlFieldNode) { - std::wstring detailsSummary = L""; - parentNode->getTextInRange(0, parentNode->getLength(), detailsSummary, false); - detailsControlFieldNode->addAttribute(L"detailsSummary", detailsSummary); - } + parentNode->addAttribute(L"hasDetails", L"true"); } // Clean up. diff --git a/source/NVDAObjects/IAccessible/__init__.py b/source/NVDAObjects/IAccessible/__init__.py index 3c40e2c4a02..ff9808f615a 100644 --- a/source/NVDAObjects/IAccessible/__init__.py +++ b/source/NVDAObjects/IAccessible/__init__.py @@ -10,6 +10,7 @@ IServiceProvider, GUID, IUnknown, + BSTR, ) from comtypes.hresult import S_OK, S_FALSE import ctypes @@ -44,6 +45,7 @@ import eventHandler from NVDAObjects.behaviors import ProgressBar, Dialog, EditableTextWithAutoSelectDetection, FocusableUnfocusableContainer, ToolTip, Notification from locationHelper import RectLTWH +import NVDAHelper # Custom object ID used for clipboard pane in some versions of MS Office @@ -1538,6 +1540,22 @@ def _getIA2RelationFirstTarget( pass return None + #: Type definition for auto prop '_get_detailsRelations' + detailsRelations: typing.Iterable["IAccessible"] + + def _get_detailsRelations(self) -> typing.Iterable["IAccessible"]: + relations = self._getIA2TargetsForRelationsOfType( + IAccessibleHandler.RelationType.DETAILS, + maxRelations=1 + ) + if not relations: + return () + relationTarget = IAccessible( + IAccessibleObject=IAccessibleHandler.normalizeIAccessible(relations[0]), + IAccessibleChildID=0 + ) + return (relationTarget, ) + #: Type definition for auto prop '_get_flowsTo' flowsTo: typing.Optional["IAccessible"] @@ -1743,6 +1761,32 @@ def __contains__(self, obj): except COMError: return False + def summarizeInProcess(self) -> str: + """Uses nvdaInProcUtils to get the text for an IAccessible. + Can be used without a virtual buffer loaded. + """ + text = BSTR() + log.debug("Calling nvdaInProcUtils_getTextFromIAccessible") + res = NVDAHelper.localLib.nvdaInProcUtils_getTextFromIAccessible( + # [in] handle_t bindingHandle + self.appModule.helperLocalBindingHandle, + # [in] const unsigned long hwnd + self.windowHandle, + # [in] long parentID + self.IAccessibleObject.uniqueID, + # // Params for getTextFromIAccessible + # [out, string] BSTR* textBuf + ctypes.byref(text), + # [in, defaultvalue(TRUE)] const boolean recurse, + True, + # [in, defaultvalue(TRUE)] const boolean includeTopLevelText + True, + ) + if res != 0: + log.error(f"Error calling nvdaInProcUtils_getTextFromIAccessible, res: {res}") + raise ctypes.WinError(res) + return text.value + class ContentGenericClient(IAccessible): TextInfo=displayModel.DisplayModelTextInfo diff --git a/source/NVDAObjects/IAccessible/ia2Web.py b/source/NVDAObjects/IAccessible/ia2Web.py index 70834a708b6..9028df377ec 100644 --- a/source/NVDAObjects/IAccessible/ia2Web.py +++ b/source/NVDAObjects/IAccessible/ia2Web.py @@ -8,6 +8,7 @@ import typing from ctypes import c_short from comtypes import COMError, BSTR + import oleacc from comInterfaces import IAccessible2Lib as IA2 import controlTypes @@ -42,6 +43,21 @@ def _get_descriptionFrom(self) -> controlTypes.DescriptionFrom: log.debugWarning(f"Unknown 'description-from' IA2Attribute value: {ia2attrDescriptionFrom}") return controlTypes.DescriptionFrom.UNKNOWN + def _get_detailsSummary(self) -> typing.Optional[str]: + if not self.hasDetails: + return None + detailsRelations = self.detailsRelations + if not detailsRelations: + log.error("should be able to fetch detailsRelations") + return None + for target in detailsRelations: + # just take the first for now. + return target.summarizeInProcess() + + @property + def hasDetails(self) -> bool: + return bool(self.IA2Attributes.get("details-roles")) + def _get_isCurrent(self) -> controlTypes.IsCurrent: ia2attrCurrent: str = self.IA2Attributes.get("current", "false") try: diff --git a/source/NVDAObjects/__init__.py b/source/NVDAObjects/__init__.py index ed30634fb67..038b5dc131a 100644 --- a/source/NVDAObjects/__init__.py +++ b/source/NVDAObjects/__init__.py @@ -404,10 +404,13 @@ def _set_treeInterceptor(self,obj): else: #We can't point a weakref to None, so just set the private variable to None, it can handle that self._treeInterceptor=None - def _get_appModule(self): - """Retrieves the appModule representing the application this object is a part of by asking L{appModuleHandler}. + #: Type definition for auto prop '_get_appModule' + appModule: "appModuleHandler.AppModule" + + def _get_appModule(self) -> "appModuleHandler.AppModule": + """Retrieves the appModule representing the application this object is a part of by + asking L{appModuleHandler}. @return: the appModule - @rtype: L{appModuleHandler.AppModule} """ if not hasattr(self,'_appModuleRef'): a=appModuleHandler.getAppModuleForNVDAObject(self) @@ -479,6 +482,16 @@ def _get_description(self) -> str: def _get_descriptionFrom(self) -> controlTypes.DescriptionFrom: return controlTypes.DescriptionFrom.UNKNOWN + #: Typing information for auto property _get_detailsSummary + detailsSummary: typing.Optional[str] + + def _get_detailsSummary(self) -> typing.Optional[str]: + return None + + @property + def hasDetails(self) -> bool: + return bool(self.detailsSummary) + def _get_controllerFor(self): """Retrieves the object/s that this object controls.""" return [] diff --git a/source/api.py b/source/api.py index ff54d012097..a8b05aba316 100644 --- a/source/api.py +++ b/source/api.py @@ -4,7 +4,10 @@ # This file may be used under the terms of the GNU General Public License, version 2 or later. # For more details see: https://www.gnu.org/licenses/gpl-2.0.html -"""General functions for NVDA""" +"""General functions for NVDA +Functions should mostly refer to getting an object (NVDAObject) or a position (TextInfo). +""" +import typing import config import textInfos @@ -25,6 +28,9 @@ import cursorManager from typing import Any, Optional +if typing.TYPE_CHECKING: + import documentBase + #User functions @@ -425,13 +431,24 @@ def isObjectInActiveTreeInterceptor(obj: NVDAObjects.NVDAObject) -> bool: ) -def getCaretObject(): +def getCaretPosition() -> "textInfos.TextInfo": + """Gets a text info at the position of the caret. + """ + textContainerObj = getCaretObject() + if not textContainerObj: + raise RuntimeError("No Caret Object available, this is expected while NVDA is still starting up.") + return textContainerObj.makeTextInfo("caret") + + +def getCaretObject() -> "documentBase.TextContainerObject": """Gets the object which contains the caret. - This is normally the focus object. - However, if the focus object has a tree interceptor which is not in focus mode, - the tree interceptor will be returned. + This is normally the NVDAObject with focus, unless it has a browse mode tree interceptor to return instead. @return: The object containing the caret. - @rtype: L{baseObject.ScriptableObject} + @note: Note: this may not be the NVDA Object closest to the caret, EG an edit text box may have focus, + and contain multiple NVDAObjects closer to the caret position, consider instead: + ti = getCaretPosition() + ti.expand(textInfos.UNIT_CHARACTER) + closestObj = ti.NVDAObjectAtStart """ obj = getFocusObject() ti = obj.treeInterceptor diff --git a/source/appModuleHandler.py b/source/appModuleHandler.py index 1ed90a54e23..2165fc7fa96 100644 --- a/source/appModuleHandler.py +++ b/source/appModuleHandler.py @@ -16,6 +16,10 @@ import ctypes.wintypes import os import sys +from typing import ( + Optional, +) + import winVersion import pkgutil import importlib @@ -346,7 +350,10 @@ def __init__(self,processID,appName=None): #: @type: str self.appName=appName self.processHandle=winKernel.openProcess(winKernel.SYNCHRONIZE|winKernel.PROCESS_QUERY_INFORMATION,False,processID) - self.helperLocalBindingHandle=None + + self.helperLocalBindingHandle: Optional[ctypes.c_long] = None + """RPC binding handle pointing to the RPC server for this process""" + self._inprocRegistrationHandle=None def _getExecutableFileInfo(self): diff --git a/source/braille.py b/source/braille.py index ebe57936501..31f9363c2d4 100644 --- a/source/braille.py +++ b/source/braille.py @@ -221,8 +221,6 @@ controlTypes.State.SORTED_ASCENDING: _("sorted asc"), # Translators: Displayed in braille when an object is sorted descending. controlTypes.State.SORTED_DESCENDING: _("sorted desc"), - # Translators: Displayed in braille when an object has additional details (such as a comment section). - controlTypes.State.HAS_ARIA_DETAILS: _("details"), # Translators: Displayed in braille when an object (usually a graphic) has a long description. controlTypes.State.HASLONGDESC: _("ldesc"), # Translators: Displayed in braille when there is a formula on a spreadsheet cell. @@ -543,6 +541,9 @@ def getPropertiesBraille(**propertyValues) -> str: # noqa: C901 description = propertyValues.get("description") if description: textList.append(description) + hasDetails = propertyValues.get("hasDetails") + if hasDetails: + textList.append("details") keyboardShortcut = propertyValues.get("keyboardShortcut") if keyboardShortcut: textList.append(keyboardShortcut) @@ -643,6 +644,7 @@ def update(self): roleText=obj.roleTextBraille, current=obj.isCurrent, placeholder=placeholderValue, + hasDetails=obj.hasDetails, value=obj.value if not NVDAObjectHasUsefulText(obj) else None , states=obj.states, description=description, @@ -716,6 +718,7 @@ def getControlFieldBraille( # noqa: C901 value=field.get('value',None) current = field.get('current', controlTypes.IsCurrent.NO) placeholder=field.get('placeholder', None) + hasDetails = field.get('hasDetails', False) and config.conf["annotations"]["reportDetails"] roleText = field.get('roleTextBraille', field.get('roleText')) landmark = field.get("landmark") if not roleText and role == controlTypes.Role.LANDMARK and landmark: @@ -762,6 +765,7 @@ def getControlFieldBraille( # noqa: C901 "placeholder": placeholder, "roleText": roleText, "description": description, + "hasDetails": hasDetails, } if field.get('alwaysReportName', False): # Ensure that the name of the field gets presented even if normally it wouldn't. diff --git a/source/browseMode.py b/source/browseMode.py index a93e1ab9dd8..e552af299b0 100644 --- a/source/browseMode.py +++ b/source/browseMode.py @@ -1401,26 +1401,6 @@ def script_activateLongDesc(self,gesture): # Translators: the description for the activateLongDescription script on browseMode documents. script_activateLongDesc.__doc__=_("Shows the long description at this position if one is found.") - @script( - description=_( - # Translators: the description for the activateAriaDetailsSummary script on browseMode documents. - "Shows a summary of the details at this position if found." - ) - ) - def script_activateAriaDetailsSummary(self, gesture): - info = self.makeTextInfo(textInfos.POSITION_CARET) - info.expand("character") - for field in reversed(info.getTextWithFields()): - if isinstance(field, textInfos.FieldCommand) and field.command == "controlStart": - states = field.field.get('states') - if states and controlTypes.State.HAS_ARIA_DETAILS in states: - ui.message(field.field['detailsSummary']) - return - - # Translators: the message presented when the activateAriaDetailsSummary script cannot locate a - # set of details to read. - ui.message(_("No additional details")) - def event_caretMovementFailed(self, obj, nextHandler, gesture=None): if not self.passThrough or not gesture or not config.conf["virtualBuffers"]["autoPassThroughOnCaretMove"]: return nextHandler() diff --git a/source/compoundDocuments.py b/source/compoundDocuments.py index 62f1759f8d3..887211c7301 100644 --- a/source/compoundDocuments.py +++ b/source/compoundDocuments.py @@ -155,6 +155,7 @@ def _getControlFieldForObject(self, obj: NVDAObject, ignoreEditableText=True): field['roleText'] = obj.roleText field['description'] = obj.description field['_description-from'] = obj.descriptionFrom + field['hasDetails'] = obj.hasDetails # The user doesn't care about certain states, as they are obvious. states.discard(controlTypes.State.EDITABLE) states.discard(controlTypes.State.MULTILINE) diff --git a/source/controlTypes/__init__.py b/source/controlTypes/__init__.py index f00d43ff3ba..28c41e74aed 100644 --- a/source/controlTypes/__init__.py +++ b/source/controlTypes/__init__.py @@ -247,4 +247,3 @@ STATE_CROPPED = State.CROPPED STATE_OVERFLOWING = State.OVERFLOWING STATE_UNLOCKED = State.UNLOCKED - STATE_HAS_ARIA_DETAILS = State.HAS_ARIA_DETAILS diff --git a/source/controlTypes/processAndLabelStates.py b/source/controlTypes/processAndLabelStates.py index 34b0ffadfc1..615413e98bc 100644 --- a/source/controlTypes/processAndLabelStates.py +++ b/source/controlTypes/processAndLabelStates.py @@ -73,9 +73,6 @@ def _processPositiveStates( positiveStates.discard(State.EXPANDED) if State.FOCUSABLE not in states: positiveStates.discard(State.EDITABLE) - if not config.conf["annotations"]["reportDetails"]: - # reading aria-details is an experimental feature still and should not always be reported. - positiveStates.discard(State.HAS_ARIA_DETAILS) return positiveStates diff --git a/source/controlTypes/state.py b/source/controlTypes/state.py index e32f8addbc1..a41243dcd03 100644 --- a/source/controlTypes/state.py +++ b/source/controlTypes/state.py @@ -70,7 +70,6 @@ def negativeDisplayString(self) -> str: CROPPED = 0x8000000000 OVERFLOWING = 0x10000000000 UNLOCKED = 0x20000000000 - HAS_ARIA_DETAILS = 0x40000000000 HASNOTE = 0x80000000000 @@ -143,8 +142,6 @@ def negativeDisplayString(self) -> str: State.SORTED_DESCENDING: _("sorted descending"), # Translators: a state that denotes that an object (usually a graphic) has a long description. State.HASLONGDESC: _("has long description"), - # Translators: a state that denotes that an object has additional details (such as a comment section). - State.HAS_ARIA_DETAILS: _("has details"), # Translators: a state that denotes that an object is pinned in its current location State.PINNED: _("pinned"), # Translators: a state that denotes the existance of a formula on a spreadsheet cell diff --git a/source/globalCommands.py b/source/globalCommands.py index 83ee749af30..b5bcfcaa5ad 100755 --- a/source/globalCommands.py +++ b/source/globalCommands.py @@ -7,7 +7,12 @@ # Julien Cochuyt, Jakub Lukowicz import itertools -from typing import Optional, Tuple, Union + +from typing import ( + Optional, + Tuple, + Union, +) import audioDucking import touchHandler @@ -1956,6 +1961,59 @@ def script_reportOrShowFormattingAtCaret(self, gesture): elif repeats == 1: self.script_showFormattingAtCaret(gesture) + @script( + description=_( + # Translators: the description for the reportDetailsSummary script. + "Report summary of any annotation details at the system caret." + ), + category=SCRCAT_SYSTEMCARET, + ) + def script_reportDetailsSummary(self, gesture): + """Report the annotation details summary for the single character under the caret or the object with + system focus. + @note: It is tempting to try to report any annotation details that exists in the range formed by prior + and current location. This would be a new paradigm in NVDA, and may feel natural when moving by line + to be able to more quickly have the 'details' reported. However, there may be more than one 'details + relation' in that range, and we don't yet have a way for the user to select which one to report. + For now, we minimise this risk by only reporting details at the current location. + """ + log.debug("Report annotation details summary at current location.") + try: + # Common cases use Caret Position: vbuf available or object supports text range + # Eg editable text, or regular web content + # Firefox and Chromium support this even in a button within a role=application. + caret: textInfos.TextInfo = api.getCaretPosition() + except RuntimeError: + log.debugWarning("Unable to get the caret position.", exc_info=True) + return + caret.expand(textInfos.UNIT_CHARACTER) + nvdaObject: NVDAObject = caret.NVDAObjectAtStart + log.debug(f"Trying with nvdaObject : {nvdaObject}") + + annotation: Optional[str] = nvdaObject.detailsSummary + if annotation: + log.debug("NVDAObjectAtStart of caret has details.") + elif api.getFocusObject(): + # If fetching from the caret position fails, try via the focus object + # This case is to support where there is no virtual buffer or text interface and a caret position can + # not be fetched. + # There may still be an object with focus that has details. + # There isn't a known test case for this, however there isn't a known downside to attempt this. + focus = api.getFocusObject() + log.debug(f"Trying focus object: {focus}") + annotation = focus.detailsSummary + if annotation: + log.debug("focus object has details, able to proceed") + + if not annotation: + log.debug("no details annotation found") + # Translators: message given when there is no annotation details for the reportDetailsSummary script. + ui.message(_("No additional details")) + return + + ui.message(annotation) + return + @script( # Translators: Input help mode message for report current focus command. description=_("Reports the object with focus. If pressed twice, spells the information"), @@ -2622,6 +2680,8 @@ def script_activatePythonConsole(self,gesture): import pythonConsole if not pythonConsole.consoleUI: pythonConsole.initialize() + # Take a snapshot of the vars before opening the window. Once the python console window is opened calls + # to the 'api' module will refer to this new focus. pythonConsole.consoleUI.console.updateNamespaceSnapshotVars() pythonConsole.activate() diff --git a/source/gui/settingsDialogs.py b/source/gui/settingsDialogs.py index 8509d4bfb87..ecdf4bdd6ff 100644 --- a/source/gui/settingsDialogs.py +++ b/source/gui/settingsDialogs.py @@ -2703,7 +2703,7 @@ def __init__(self, parent): # Translators: This is the label for a checkbox in the # Advanced settings panel. - label = _("Report details in browse mode") + label = _("Report 'has details' for structured annotations") self.annotationsDetailsCheckBox = AnnotationsGroup.addItem(wx.CheckBox(AnnotationsBox, label=label)) self.annotationsDetailsCheckBox.SetValue(config.conf["annotations"]["reportDetails"]) self.annotationsDetailsCheckBox.defaultValue = self._getDefaultValue(["annotations", "reportDetails"]) diff --git a/source/pythonConsole.py b/source/pythonConsole.py index f3806c2404f..58cb5bb79a6 100755 --- a/source/pythonConsole.py +++ b/source/pythonConsole.py @@ -191,7 +191,15 @@ def initNamespace(self): def updateNamespaceSnapshotVars(self): """Update the console namespace with a snapshot of NVDA's current state. This creates/updates variables for the current focus, navigator object, etc. + Typically, used before the NVDA python console is opened, after which, calls + to the 'api' module will refer to this new focus. """ + try: + caretPos = api.getCaretPosition() + except RuntimeError: + log.debug("Unable to set caretPos snapshot variable for python console.") + caretPos = None + self._namespaceSnapshotVars = { "focus": api.getFocusObject(), # Copy the focus ancestor list, as it gets mutated once it is replaced in api.setFocusObject. @@ -199,7 +207,9 @@ def updateNamespaceSnapshotVars(self): "fdl": api.getFocusDifferenceLevel(), "fg": api.getForegroundObject(), "nav": api.getNavigatorObject(), - "review":api.getReviewPosition(), + "caretObj": api.getCaretObject(), + "caretPos": caretPos, + "review": api.getReviewPosition(), "mouse": api.getMouseObject(), "brlRegions": braille.handler.buffer.regions, } diff --git a/source/speech/speech.py b/source/speech/speech.py index 9fe03ae3cc6..10e6936a340 100644 --- a/source/speech/speech.py +++ b/source/speech/speech.py @@ -428,6 +428,9 @@ def getObjectPropertiesSpeech( # noqa: C901 # getPropertiesSpeech names this "current", but the NVDAObject property is # named "isCurrent", it's type should always be controltypes.IsCurrent newPropertyValues['current'] = obj.isCurrent + + elif value and name == "hasDetails": + newPropertyValues['hasDetails'] = obj.hasDetails elif value and name == "descriptionFrom" and ( obj.descriptionFrom == controlTypes.DescriptionFrom.ARIA_DESCRIPTION ): @@ -640,6 +643,7 @@ def _objectSpeech_calculateAllowedProps(reason, shouldReportTextContent): 'states': True, 'value': True, 'description': True, + 'hasDetails': config.conf["annotations"]["reportDetails"], 'descriptionFrom': config.conf["annotations"]["reportAriaDescription"], 'keyboardShortcut': True, 'positionInfo_level': True, @@ -1680,6 +1684,14 @@ def getPropertiesSpeech( # noqa: C901 if isCurrent != controlTypes.IsCurrent.NO: textList.append(isCurrent.displayString) + # are there further details + hasDetails = propertyValues.get('hasDetails', False) + if hasDetails: + textList.append( + # Translators: Speaks when there a further details/annotations that can be fetched manually. + _("has details") + ) + placeholder: Optional[str] = propertyValues.get('placeholder', None) if placeholder: textList.append(placeholder) @@ -1779,6 +1791,7 @@ def getControlFieldSpeech( # noqa: C901 states=attrs.get('states',set()) keyboardShortcut=attrs.get('keyboardShortcut', "") isCurrent = attrs.get('current', controlTypes.IsCurrent.NO) + hasDetails = attrs.get('hasDetails', False) placeholderValue=attrs.get('placeholder', None) value=attrs.get('value',"") @@ -1838,6 +1851,7 @@ def getControlFieldSpeech( # noqa: C901 reason=reason, keyboardShortcut=keyboardShortcut ) isCurrentSequence = getPropertiesSpeech(reason=reason, current=isCurrent) + hasDetailsSequence = getPropertiesSpeech(reason=reason, hasDetails=hasDetails) placeholderSequence = getPropertiesSpeech(reason=reason, placeholder=placeholderValue) nameSequence = getPropertiesSpeech(reason=reason, name=name) valueSequence = getPropertiesSpeech(reason=reason, value=value) @@ -1942,6 +1956,7 @@ def getControlFieldSpeech( # noqa: C901 tableCellSequence = getPropertiesSpeech(_tableID=tableID, **getProps) tableCellSequence.extend(stateTextSequence) tableCellSequence.extend(isCurrentSequence) + tableCellSequence.extend(hasDetailsSequence) types.logBadSequenceTypes(tableCellSequence) return tableCellSequence @@ -1991,6 +2006,7 @@ def getControlFieldSpeech( # noqa: C901 out.extend(roleTextSequence if speakStatesFirst else stateTextSequence) out.append(containerContainsText) out.extend(isCurrentSequence) + out.extend(hasDetailsSequence) out.extend(valueSequence) out.extend(descriptionSequence) out.extend(levelSequence) @@ -2027,6 +2043,8 @@ def getControlFieldSpeech( # noqa: C901 out = [] if isCurrent != controlTypes.IsCurrent.NO: out.extend(isCurrentSequence) + if hasDetails: + out.extend(hasDetailsSequence) if descriptionSequence and _reportDescriptionAsAnnotation: out.extend(descriptionSequence) # Speak expanded / collapsed / level for treeview items (in ARIA treegrids) diff --git a/source/virtualBuffers/gecko_ia2.py b/source/virtualBuffers/gecko_ia2.py index d157328c855..51ff7803789 100755 --- a/source/virtualBuffers/gecko_ia2.py +++ b/source/virtualBuffers/gecko_ia2.py @@ -106,8 +106,6 @@ def _normalizeControlField(self, attrs): # noqa: C901 # This is a text leaf. # See NVDAObjects.Iaccessible.mozilla.findOverlayClasses for an explanation of these checks. role = controlTypes.Role.STATICTEXT - if attrs.get("detailsSummary") is not None: - states.add(controlTypes.State.HAS_ARIA_DETAILS) if attrs.get("IAccessibleAction_showlongdesc") is not None: states.add(controlTypes.State.HASLONGDESC) if "IAccessibleAction_click" in attrs: diff --git a/tests/system/libraries/ChromeLib.py b/tests/system/libraries/ChromeLib.py index da90381fcf7..b6f6f6aa3ec 100644 --- a/tests/system/libraries/ChromeLib.py +++ b/tests/system/libraries/ChromeLib.py @@ -116,7 +116,7 @@ def _waitForStartMarker(self, spy, lastSpeechIndex): @type spy: SystemTestSpy.speechSpyGlobalPlugin.NVDASpyLib @return: None """ - for i in range(10): # set a limit on the number of tries. + for i in range(3): # set a limit on the number of tries. builtIn.sleep("0.5 seconds") # ensure application has time to receive input spy.wait_for_speech_to_finish() actualSpeech = spy.get_speech_at_index_until_now(lastSpeechIndex) diff --git a/tests/system/libraries/SystemTestSpy/speechSpyGlobalPlugin.py b/tests/system/libraries/SystemTestSpy/speechSpyGlobalPlugin.py index 1bc8ac9315c..8303ade5fbb 100644 --- a/tests/system/libraries/SystemTestSpy/speechSpyGlobalPlugin.py +++ b/tests/system/libraries/SystemTestSpy/speechSpyGlobalPlugin.py @@ -140,6 +140,8 @@ def _onNvdaStartupComplete(self): def _onNvdaBraille(self, rawText: str): if not rawText: return + if not isinstance(rawText, str): + raise TypeError(f"rawText expected as str, got: {type(rawText)}, {rawText!r}") with self._brailleLock: self._nvdaBraille_requiresLock.append(rawText) diff --git a/tests/system/nvdaSettingsFiles/chrome-gestures.ini b/tests/system/nvdaSettingsFiles/chrome-gestures.ini index 197ef8c3c65..bb595061b6f 100644 --- a/tests/system/nvdaSettingsFiles/chrome-gestures.ini +++ b/tests/system/nvdaSettingsFiles/chrome-gestures.ini @@ -1,2 +1,2 @@ -[browseMode.BrowseModeDocumentTreeInterceptor] -activateAriaDetailsSummary = kb:nvda+\ +[globalCommands.GlobalCommands] +reportDetailsSummary = kb:nvda+\ diff --git a/tests/system/robot/chromeTests.py b/tests/system/robot/chromeTests.py index f724de43aa9..2db476e95e4 100644 --- a/tests/system/robot/chromeTests.py +++ b/tests/system/robot/chromeTests.py @@ -55,52 +55,375 @@ def checkbox_labelled_by_inner_element(): ) +REVIEW_CURSOR_FOLLOW_CARET_KEY = ["reviewCursor", "followCaret"] +REVIEW_CURSOR_FOLLOW_FOCUS_KEY = ["reviewCursor", "followFocus"] +READ_DETAILS_GESTURE = "NVDA+\\" # see chrome-gestures.ini + + +def _getNoVBuf_AriaDetails_sample() -> str: + return """ +
+ +

this is an application, it contains a button with details

+ +
+
+

Press to self-destruct

+
+ """ + + +def _doTestAriaDetails_NoVBufNoTextInterface(): + _chrome.prepareChrome(_getNoVBuf_AriaDetails_sample()) + actualSpeech = _NvdaLib.getSpeechAfterKey("tab") + _builtIn.should_contain(actualSpeech, "focus in app") + + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey("tab") + _asserts.speech_matches( + actualSpeech, + SPEECH_SEP.join([ + "push me", + "button", + "has details", + ]), + message="Tab to button" + ) + _asserts.braille_matches( + actualBraille, + "push me btn details", + message="Tab to button", + ) + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey(READ_DETAILS_GESTURE) + _asserts.speech_matches( + actualSpeech, + "Press to self destruct", + message="Report details" + ) + _asserts.braille_matches( + actualBraille, + "Press to self-destruct", + message="Report details", + ) + + +def test_aria_details_noVBufNoTextInterface(): + """The uncommon case, but for completeness, a role=application containing an element that does not have a text + interface. + """ + spy = _NvdaLib.getSpyLib() + spy.set_configValue(REVIEW_CURSOR_FOLLOW_CARET_KEY, True) + spy.set_configValue(REVIEW_CURSOR_FOLLOW_FOCUS_KEY, True) + _doTestAriaDetails_NoVBufNoTextInterface() + + +def test_aria_details_noVBufNoTextInterface_freeReview(): + """The uncommon case, but for completeness, a role=application containing an element without a text + interface. Test with the review cursor configured not to follow focus or caret. + """ + spy = _NvdaLib.getSpyLib() + spy.set_configValue(REVIEW_CURSOR_FOLLOW_CARET_KEY, False) + spy.set_configValue(REVIEW_CURSOR_FOLLOW_FOCUS_KEY, False) + _doTestAriaDetails_NoVBufNoTextInterface() + + def test_mark_aria_details(): + spy = _NvdaLib.getSpyLib() + spy.set_configValue(REVIEW_CURSOR_FOLLOW_CARET_KEY, True) + spy.set_configValue(REVIEW_CURSOR_FOLLOW_FOCUS_KEY, True) + exercise_mark_aria_details() + + +def test_mark_aria_details_FreeReviewCursor(): + spy = _NvdaLib.getSpyLib() + spy.set_configValue(REVIEW_CURSOR_FOLLOW_CARET_KEY, False) + spy.set_configValue(REVIEW_CURSOR_FOLLOW_FOCUS_KEY, False) + exercise_mark_aria_details() + + +def exercise_mark_aria_details(): _chrome.prepareChrome( """ -
+

The word cat has a comment tied to it.

- + """ ) - actualSpeech = _chrome.getSpeechAfterKey('downArrow') - _asserts.strings_match( + + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey('downArrow') + _asserts.speech_matches( actualSpeech, - "The word highlighted has details cat out of highlighted has a comment tied to it." + SPEECH_SEP.join([ + "edit", + "multi line", + "The word", # content + "highlighted", + "has details", + "cat", # highlighted content + "out of highlighted", + "has a comment tied to it.", # content + ]), + message="Browse mode: Read line with details." + ) + _asserts.braille_matches( + actualBraille, + "mln edt The word hlght details cat hlght end has a comment tied to it. edt end", + message="Browse mode: Read line with details.", ) # this word has no details attached - actualSpeech = _chrome.getSpeechAfterKey("control+rightArrow") - _asserts.strings_match( + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey("control+rightArrow") + _asserts.speech_matches( actualSpeech, - "word" + "word", + message="Browse mode: Move by word to word without details" + ) + _asserts.braille_matches( + actualBraille, + "mln edt The word hlght details cat hlght end has a comment tied to it. edt end", + message="Browse mode: Move by word to word without details", ) + # check that there is no summary reported - actualSpeech = _chrome.getSpeechAfterKey("NVDA+\\") - _asserts.strings_match( + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey(READ_DETAILS_GESTURE) + _asserts.speech_matches( actualSpeech, - "No additional details" + "No additional details", + message="Browse mode: Report details on word without details" + ) + _asserts.braille_matches( + actualBraille, + "No additional details", + message="Browse mode: Report details on word without details", ) # this word has details attached to it - actualSpeech = _chrome.getSpeechAfterKey("control+rightArrow") - _asserts.strings_match( + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey("control+rightArrow") + _asserts.speech_matches( actualSpeech, - "highlighted has details cat out of highlighted" + "highlighted has details cat out of highlighted", + message="Browse mode: Move by word to word with details", + ) + _asserts.braille_matches( + actualBraille, + "mln edt The word hlght details cat hlght end has a comment tied to it. edt end", + message="Browse mode: Move by word to word with details", ) # read the details summary - actualSpeech = _chrome.getSpeechAfterKey("NVDA+\\") - _asserts.strings_match( + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey(READ_DETAILS_GESTURE) + _asserts.speech_matches( actualSpeech, - "Cats go woof BTW Jonathon Commentor No they don't Zara Submit" + "Cats go woof BTW Jonathon Commentor No they don't Zara", + message="Browse mode: Report details on word with details" + ) + _asserts.braille_matches( + actualBraille, + "Cats go woof BTW\n—Jonathon CommentorNo they don't\n—Zara", + message="Browse mode: Report details on word with details", + ) + + # move down to the link nested in a container with details + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey("downArrow") + _asserts.speech_matches( + actualSpeech, + "out of edit Hello highlighted has details this is a link test", + message="Browse mode: Move by line to paragraph with link nested in a container with details", + ) + _asserts.braille_matches( + actualBraille, + "Hello hlght details this is a lnk test hlght end", + message="Browse mode: Move by line to paragraph with link nested in a container with details", + ) + # Jump to the link from same line + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey("k") + _asserts.speech_matches( + actualSpeech, + "test link", + message="Browse mode: From same line jump to link nested in a container with details", + ) + _asserts.braille_matches( + actualBraille, + "Hello hlght details this is a lnk test hlght end", + message="Browse mode: From same line jump to link nested in a container with details", + ) + + # reset to prior line before jump to the link from different line + actualSpeech = _NvdaLib.getSpeechAfterKey('upArrow') + _asserts.speech_matches( + actualSpeech, + SPEECH_SEP.join([ + "out of highlighted", # Leaving the highlighted link + "edit", + "multi line", + "The word", # content + "highlighted", + "has details", + "cat", # highlighted content + "out of highlighted", + "has a comment tied to it.", # content + ]), + message="Browse mode: Reset to prior line before jump to the link." + ) + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey("k") + _asserts.speech_matches( + actualSpeech, + "highlighted has details test link", + message="Browse mode: From prior line jump to link nested in a container with details", + ) + _asserts.braille_matches( + actualBraille, + "Hello hlght details this is a lnk test hlght end", + message="Browse mode: From prior line jump to link nested in a container with details", + ) + # read the details summary + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey(READ_DETAILS_GESTURE) + _asserts.speech_matches( + actualSpeech, + "No additional details", + message="Browse mode: Report details on nested link with details" + ) + _asserts.braille_matches( + actualBraille, + "No additional details", + message="Browse mode: Report details on nested link with details" + ) + + # Reset caret + actualSpeech = _NvdaLib.getSpeechAfterKey("upArrow") + actualSpeech = _NvdaLib.getSpeechAfterKey("upArrow") + _asserts.speech_matches( + actualSpeech, + SPEECH_SEP.join([ + "out of edit", + "Test page load complete", + ]), + message="reset caret", + ) + + # Force focus mode + actualSpeech = _NvdaLib.getSpeechAfterKey("NVDA+space") + _asserts.speech_matches( + actualSpeech, + "Focus mode", + message="force focus mode", + ) + + # Tab into the contenteditable + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey("tab") + _asserts.speech_matches( + actualSpeech, + SPEECH_SEP.join([ + "edit", + "multi line", + "The word", # content + "highlighted", + "has details", + "cat", # highlighted content + "out of highlighted", + "has a comment tied to it.", # content + ]), + message="Focus mode: report content editable with details" + ) + _asserts.braille_matches( + actualBraille, + "The word hlght details cat hlght end has a comment tied to it.", + message="Focus mode: report content editable with details", + ) + + # Try to read the details + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey(READ_DETAILS_GESTURE) + _asserts.speech_matches( + actualSpeech, + SPEECH_SEP.join([ + "No additional details", + ]), + message="Focus mode: Try to read details, caret not on details word.", + ) + _asserts.braille_matches( + actualBraille, + "No additional details", + message="Focus mode: Try to read details, caret not on details word.", + ) + + # move to the word with details: "cat" + _NvdaLib.getSpeechAfterKey("control+rightArrow") + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey("control+rightArrow") + _asserts.speech_matches( + actualSpeech, + SPEECH_SEP.join([ + "highlighted", + "has details", + "cat", # highlighted content + "out of highlighted", + ]), + message="Focus mode: Move by word to word with details" + ) + _asserts.braille_matches( + actualBraille, + expected="The word hlght details cat hlght end has a comment tied to it.", + message="Focus mode: Move by word to word with details", + ) + + # Try to read the details + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey(READ_DETAILS_GESTURE) + _asserts.speech_matches( + actualSpeech, + "Cats go woof BTW Jonathon Commentor No they don't Zara", + message="Focus mode: Report details on word with details.", + ) + _asserts.braille_matches( + actualBraille, + expected="Cats go woof BTW\n—Jonathon CommentorNo they don't\n—Zara", + message="Focus mode: Report details on word with details.", + ) + + # Tab to the link + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey("tab") + _asserts.speech_matches( + actualSpeech, + SPEECH_CALL_SEP.join([ + SPEECH_SEP.join([ + "highlighted", + "has details", + ]), + SPEECH_SEP.join([ + "test", + "link", + ]) + ]), + message="Focus mode: tab to link nested in container with details", + ) + _asserts.braille_matches( + actualBraille, + "hlght details test lnk", + message="Focus mode: tab to link nested in container with details" + ) + + # Try to read the details + actualSpeech, actualBraille = _NvdaLib.getSpeechAndBrailleAfterKey(READ_DETAILS_GESTURE) + _asserts.speech_matches( + actualSpeech, + SPEECH_SEP.join([ + "No additional details", + ]), + message="Focus mode: Try to read details, link nested in container with details.", + ) + _asserts.braille_matches( + actualBraille, + "No additional details", + message="Focus mode: Try to read details, link nested in container with details.", ) diff --git a/tests/system/robot/chromeTests.robot b/tests/system/robot/chromeTests.robot index b662fb7eaab..e7674189c63 100644 --- a/tests/system/robot/chromeTests.robot +++ b/tests/system/robot/chromeTests.robot @@ -61,6 +61,18 @@ ARIA details [Documentation] Ensure a summary of aria-details is read on command from a mark element [Tags] annotations test_mark_aria_details +ARIA details with free review and nav + [Documentation] Variation on the ARIA details test with the config changed so the review cursor does not follow the caret and the nav object doesn't follow focus. + [Tags] annotations + test_mark_aria_details_FreeReviewCursor +ARIA details noVbuf + [Documentation] Test for retrieving ARIA details from a button inside a role=application + [Tags] annotations + test_aria_details_noVBufNoTextInterface +ARIA details noVbuf with free review and nav + [Documentation] Test for retrieving ARIA details from a button inside a role=application with the config changed so the review cursor does not follow the caret and the nav object doesn't follow focus. + [Tags] annotations + test_aria_details_noVBufNoTextInterface i12147 [Documentation] New focus target should be announced if the triggering element is removed when activated test_i12147 diff --git a/user_docs/en/changes.t2t b/user_docs/en/changes.t2t index 0425055e3b0..a685162c387 100644 --- a/user_docs/en/changes.t2t +++ b/user_docs/en/changes.t2t @@ -14,6 +14,7 @@ What's New in NVDA - For this to work, you must be running Microsoft Word 365/2016 build 14326 or later. - MathType equations must also be manually converted to Office Math by selecting each and choosing Equation options -> Convert to Office Math in the context menu. - +- Reporting of "has details" and the associated command to summarize the details relation have been updated to work in focus mode. (#13106) - == Changes == diff --git a/user_docs/en/userGuide.t2t b/user_docs/en/userGuide.t2t index 10a4f246770..a42839abafd 100644 --- a/user_docs/en/userGuide.t2t +++ b/user_docs/en/userGuide.t2t @@ -1868,8 +1868,8 @@ The combo box has the following options: This group of options is used to enable features which add experimental support for ARIA annotations. Some of these features may be incomplete. The following options exist: -- "Report details in browse mode": enables reporting if an object has details in browse mode. -Reporting the summary of those details can be done by assigning a gesture using the [Input Gestures dialog #InputGestures]. +- "Report 'has details' for structured annotations": enables reporting if the text or control has further details. +A summary of the extra details can be reported by assigning a gesture to "Report summary of any annotation details at the system caret" using the [Input Gestures dialog #InputGestures]. - "Report aria-description always": When the source of ``accDescription`` is aria-description, the description is reported. This is useful for annotations on the web.