sc/inc/dpobject.hxx | 4 - sc/inc/dpsave.hxx | 2 sc/qa/unit/subsequent_filters-test.cxx | 8 +++ sc/source/core/data/dpobject.cxx | 3 + sc/source/filter/inc/pivotcachebuffer.hxx | 7 ++ sc/source/filter/inc/pivottablebuffer.hxx | 10 +++ sc/source/filter/oox/pivotcachebuffer.cxx | 5 + sc/source/filter/oox/pivottablebuffer.cxx | 77 ++++++++++++++++++++++++++++-- 8 files changed, 106 insertions(+), 10 deletions(-)
New commits: commit 03a1143cc75161dab56b20f1ab9e723ddd0caa8e Author: Tamás Zolnai <tamas.zol...@collabora.com> Date: Sun Oct 9 20:07:33 2016 +0200 tdf#102694, bnc#957991: Improve pivot cache reading performance When two or more tables have the same source data, then the grouping of source fields are shared between these tables, so don't need to import these grouping for each tables. The added code checkes whether we already imported the group fields and don't create them again using API functions, but apply the exisiting groups of an other table sharing the source. Change-Id: Iad6be9a9e30944ab9a241c8498eff95c6c356689 diff --git a/sc/inc/dpobject.hxx b/sc/inc/dpobject.hxx index 536e99b..c2305c7 100644 --- a/sc/inc/dpobject.hxx +++ b/sc/inc/dpobject.hxx @@ -365,7 +365,7 @@ public: sal_uLong ReloadCache(ScDPObject* pDPObj, std::set<ScDPObject*>& rRefs); bool ReloadGroupsInCache(ScDPObject* pDPObj, std::set<ScDPObject*>& rRefs); - bool GetReferenceGroups(const ScDPObject& rDPObj, const ScDPDimensionSaveData** pGroups) const; + SC_DLLPUBLIC bool GetReferenceGroups(const ScDPObject& rDPObj, const ScDPDimensionSaveData** pGroups) const; SC_DLLPUBLIC size_t GetCount() const; SC_DLLPUBLIC ScDPObject& operator[](size_t nIndex); @@ -390,7 +390,7 @@ public: void FreeTable(ScDPObject* pDPObj); SC_DLLPUBLIC bool InsertNewTable(ScDPObject* pDPObj); - bool HasTable(const ScDPObject* pDPObj) const; + SC_DLLPUBLIC bool HasTable(const ScDPObject* pDPObj) const; SC_DLLPUBLIC SheetCaches& GetSheetCaches(); SC_DLLPUBLIC const SheetCaches& GetSheetCaches() const; diff --git a/sc/inc/dpsave.hxx b/sc/inc/dpsave.hxx index 2deb35e..bb1e8ac 100644 --- a/sc/inc/dpsave.hxx +++ b/sc/inc/dpsave.hxx @@ -352,7 +352,7 @@ public: void RemoveAllGroupDimensions( const OUString& rSrcDimName, std::vector<OUString>* pDeletedNames = nullptr ); SC_DLLPUBLIC ScDPDimensionSaveData* GetDimensionData(); // create if not there - void SetDimensionData( const ScDPDimensionSaveData* pNew ); // copied + SC_DLLPUBLIC void SetDimensionData( const ScDPDimensionSaveData* pNew ); // copied void BuildAllDimensionMembers(ScDPTableData* pData); void SyncAllDimensionMembers(ScDPTableData* pData); diff --git a/sc/source/core/data/dpobject.cxx b/sc/source/core/data/dpobject.cxx index d9b07a4..6397c6d 100644 --- a/sc/source/core/data/dpobject.cxx +++ b/sc/source/core/data/dpobject.cxx @@ -3554,6 +3554,9 @@ bool ScDPCollection::GetReferenceGroups(const ScDPObject& rDPObj, const ScDPDime { const ScDPObject& rRefObj = *aTable.get(); + if (&rRefObj == &rDPObj) + continue; + if (rDPObj.IsSheetData()){ if(!rRefObj.IsSheetData()) continue; diff --git a/sc/source/filter/inc/pivotcachebuffer.hxx b/sc/source/filter/inc/pivotcachebuffer.hxx index 6b35a11..80e2f45 100644 --- a/sc/source/filter/inc/pivotcachebuffer.hxx +++ b/sc/source/filter/inc/pivotcachebuffer.hxx @@ -186,6 +186,8 @@ struct PCFieldGroupModel bool mbDateGroup; /// True = items are grouped by date ranges or by item names. bool mbAutoStart; /// True = start value for range groups is calculated from source data. bool mbAutoEnd; /// True = end value for range groups is calculated from source data. + OUString msFinalGroupName ; /// Finalized group name of this field used in internal pivot table collaction. + explicit PCFieldGroupModel(); @@ -269,6 +271,10 @@ public: inline sal_Int32 getParentGroupField() const { return maFieldGroupModel.mnParentField; } /** Returns the index of the base field grouping is based on. */ inline sal_Int32 getGroupBaseField() const { return maFieldGroupModel.mnBaseField; } + /** Returns the finalized group name of this field. */ + inline const OUString& getFinalGroupName() const { return maFieldGroupModel.msFinalGroupName; } + /** Set the finalized group name of this field. */ + inline void setFinalGroupName(const OUString& rFinalGroupName) { maFieldGroupModel.msFinalGroupName = rFinalGroupName; } /** Returns the shared or group item with the specified index. */ const PivotCacheItem* getCacheItem( sal_Int32 nItemIdx ) const; diff --git a/sc/source/filter/inc/pivottablebuffer.hxx b/sc/source/filter/inc/pivottablebuffer.hxx index dfe5220..ebc001a 100644 --- a/sc/source/filter/inc/pivottablebuffer.hxx +++ b/sc/source/filter/inc/pivottablebuffer.hxx @@ -149,6 +149,8 @@ public: const css::uno::Reference< css::sheet::XDataPilotField >& rxBaseDPField, const PivotCacheField& rBaseCacheField, PivotCacheGroupItemVector& orItemNames ); + void finalizeImportBasedOnCache( + const css::uno::Reference< css::sheet::XDataPilotDescriptor >& rxDPDesc); /** Returns the name of the DataPilot field in the fields collection. */ inline const OUString& getDPFieldName() const { return maDPFieldName; } @@ -317,6 +319,8 @@ public: PivotTableFilter& createTableFilter(); /** Inserts the pivot table into the sheet. */ void finalizeImport(); + /** Creates all group fields for the table after import. */ + void finalizeFieldsImport(); /** Creates all date group fields for the specified cache field after import. */ void finalizeDateGroupingImport( const css::uno::Reference< css::sheet::XDataPilotField >& rxBaseDPField, @@ -377,6 +381,7 @@ private: PivotCache* mpPivotCache; /// The pivot cache this table is based on. css::uno::Reference< css::sheet::XDataPilotDescriptor > mxDPDescriptor; /// Descriptor of the DataPilot object. + }; class PivotTableBuffer : public WorkbookHelper diff --git a/sc/source/filter/oox/pivottablebuffer.cxx b/sc/source/filter/oox/pivottablebuffer.cxx index 99b80b9..04a432a 100644 --- a/sc/source/filter/oox/pivottablebuffer.cxx +++ b/sc/source/filter/oox/pivottablebuffer.cxx @@ -49,6 +49,9 @@ #include "dapiuno.hxx" #include "dpobject.hxx" #include "dpsave.hxx" +#include "dpdimsave.hxx" +#include "document.hxx" +#include "documentimport.hxx" namespace oox { namespace xls { @@ -424,11 +427,12 @@ void PivotTableField::finalizeDateGroupingImport( const Reference< XDataPilotFie { if( maDPFieldName.isEmpty() ) // prevent endless loops if file format is broken { - if( const PivotCacheField* pCacheField = mrPivotTable.getCacheField( mnFieldIndex ) ) + if( PivotCacheField* pCacheField = mrPivotTable.getCacheField( mnFieldIndex ) ) { if( !pCacheField->isDatabaseField() && pCacheField->hasDateGrouping() && (pCacheField->getGroupBaseField() == nBaseFieldIdx) ) { maDPFieldName = pCacheField->createDateGroupField( rxBaseDPField ); + pCacheField->setFinalGroupName(maDPFieldName); OSL_ENSURE( !maDPFieldName.isEmpty(), "PivotTableField::finalizeDateGroupingImport - cannot create date group field" ); } } @@ -453,6 +457,7 @@ void PivotTableField::finalizeParentGroupingImport( const Reference< XDataPilotF pCacheField->applyItemCaptions( captionList ); maDPFieldName = pCacheField->createParentGroupField( rxBaseDPField, rBaseCacheField, orItemNames ); + pCacheField->setFinalGroupName(maDPFieldName); // on success, try to create nested group fields Reference< XDataPilotField > xDPField = mrPivotTable.getDataPilotField( maDPFieldName ); if( xDPField.is() ) @@ -461,6 +466,33 @@ void PivotTableField::finalizeParentGroupingImport( const Reference< XDataPilotF } } +void PivotTableField::finalizeImportBasedOnCache( const Reference< XDataPilotDescriptor >& rxDPDesc) +{ + /* Process all fields based on source data, other fields (e.g. group + fields) are processed based on cache fields.*/ + Reference< XDataPilotField > xDPField; + sal_Int32 nDatabaseIdx = mrPivotTable.getCacheDatabaseIndex( mnFieldIndex ); + if( (nDatabaseIdx >= 0) && rxDPDesc.is() ) try + { + // Try to get the source field and its name from passed DataPilot descriptor + Reference< XIndexAccess > xDPFieldsIA( rxDPDesc->getDataPilotFields(), UNO_SET_THROW ); + xDPField.set( xDPFieldsIA->getByIndex( nDatabaseIdx ), UNO_QUERY_THROW ); + Reference< XNamed > xDPFieldName( xDPField, UNO_QUERY_THROW ); + maDPFieldName = xDPFieldName->getName(); + SAL_WARN_IF( maDPFieldName.isEmpty(), "sc.filter", "PivotTableField::finalizeImportBasedOnCache - no field name in source data found" ); + } + catch( Exception& ) + { + } + + // Use group names already generated for another table using the same group field. + if( const PivotCacheField* pCacheField = mrPivotTable.getCacheField( mnFieldIndex ) ) + { + if(!pCacheField->getFinalGroupName().isEmpty()) + maDPFieldName = pCacheField->getFinalGroupName(); + } +} + void PivotTableField::convertRowField() { convertRowColPageField( XML_axisRow ); @@ -1190,7 +1222,7 @@ void PivotTable::finalizeImport() aDescProp.setProperty( PROP_DrillDownOnDoubleClick, maDefModel.mbEnableDrill ); // finalize all fields, this finds field names and creates grouping fields - maFields.forEachMem(&PivotTableField::finalizeImport, ::std::cref(mxDPDescriptor)); + finalizeFieldsImport(); // all row fields for( IndexVector::iterator aIt = maRowFields.begin(), aEnd = maRowFields.end(); aIt != aEnd; ++aIt ) @@ -1250,6 +1282,36 @@ void PivotTable::finalizeImport() } } +void PivotTable::finalizeFieldsImport() +{ + if (maFields.empty()) + return; + + /* Check whether group fields are already imported for an other table + sharing the same groups. */ + ScDPObject* pDPObj = getDPObject(); + const ScDocument& rDoc = getDocImport().getDoc(); + if (rDoc.HasPivotTable()) + { + const ScDPCollection* pDPCollection = rDoc.GetDPCollection(); + assert(pDPCollection != nullptr); + const ScDPDimensionSaveData* pGroups = nullptr; + bool bRefFound = pDPCollection->GetReferenceGroups(*pDPObj, &pGroups); + // Apply reference groups on this table. + if (bRefFound && pGroups && pGroups->HasGroupDimensions()) { + ScDPSaveData* pSaveData = pDPObj->GetSaveData(); + if (pSaveData) { + pSaveData->SetDimensionData(pGroups); + pDPObj->ReloadGroupTableData(); + maFields.forEachMem(&PivotTableField::finalizeImportBasedOnCache, ::std::cref(mxDPDescriptor)); + return; + } + + } + } + maFields.forEachMem(&PivotTableField::finalizeImport, ::std::cref(mxDPDescriptor)); +} + void PivotTable::finalizeDateGroupingImport( const Reference< XDataPilotField >& rxBaseDPField, sal_Int32 nBaseFieldIdx ) { // process all fields, there is no chaining information in the cache fields commit 73268333b0665d91a3d9be1917a7e8c9e6ecb11a Author: Tamás Zolnai <tamas.zol...@collabora.com> Date: Sun Oct 9 07:46:24 2016 +0200 Pivot table tests: Check also grouped fields' name. Change-Id: I6b38e8e97613f63accf341115bbc1397ade07415 diff --git a/sc/qa/unit/subsequent_filters-test.cxx b/sc/qa/unit/subsequent_filters-test.cxx index 6339166..35da74f 100644 --- a/sc/qa/unit/subsequent_filters-test.cxx +++ b/sc/qa/unit/subsequent_filters-test.cxx @@ -2081,6 +2081,7 @@ void ScFiltersTest::testPivotTableSharedGroupXLSX() // Check whether right group names are imported for both tables // First table + CPPUNIT_ASSERT_EQUAL(OUString("a2"), rDoc.GetString(ScAddress(0,1,0))); CPPUNIT_ASSERT_EQUAL(OUString("Csoport1"), rDoc.GetString(ScAddress(0,2,0))); CPPUNIT_ASSERT_EQUAL(OUString("Csoport2"), rDoc.GetString(ScAddress(0,3,0))); CPPUNIT_ASSERT_EQUAL(OUString("Csoport3"), rDoc.GetString(ScAddress(0,4,0))); @@ -2089,6 +2090,7 @@ void ScFiltersTest::testPivotTableSharedGroupXLSX() CPPUNIT_ASSERT_EQUAL(OUString("18"), rDoc.GetString(ScAddress(0,7,0))); // Second table + CPPUNIT_ASSERT_EQUAL(OUString("a2"), rDoc.GetString(ScAddress(0,11,0))); CPPUNIT_ASSERT_EQUAL(OUString("Csoport1"), rDoc.GetString(ScAddress(0,12,0))); CPPUNIT_ASSERT_EQUAL(OUString("Csoport2"), rDoc.GetString(ScAddress(0,13,0))); CPPUNIT_ASSERT_EQUAL(OUString("Csoport3"), rDoc.GetString(ScAddress(0,14,0))); @@ -2119,6 +2121,7 @@ void ScFiltersTest::testPivotTableSharedDateGroupXLSX() // Check whether right date labels are imported for both tables // First table + CPPUNIT_ASSERT_EQUAL(OUString("a"), rDoc.GetString(ScAddress(0,3,1))); CPPUNIT_ASSERT_EQUAL(OUString("1965"), rDoc.GetString(ScAddress(0,4,1))); CPPUNIT_ASSERT_EQUAL(OUString("1989"), rDoc.GetString(ScAddress(0,5,1))); CPPUNIT_ASSERT_EQUAL(OUString("2000"), rDoc.GetString(ScAddress(0,6,1))); @@ -2127,6 +2130,7 @@ void ScFiltersTest::testPivotTableSharedDateGroupXLSX() // CPPUNIT_ASSERT_EQUAL(OUString("2007"), rDoc.GetString(ScAddress(0,8,1))); // Second table + CPPUNIT_ASSERT_EQUAL(OUString("a"), rDoc.GetString(ScAddress(5,3,1))); CPPUNIT_ASSERT_EQUAL(OUString("1965"), rDoc.GetString(ScAddress(5,4,1))); CPPUNIT_ASSERT_EQUAL(OUString("1989"), rDoc.GetString(ScAddress(5,5,1))); CPPUNIT_ASSERT_EQUAL(OUString("2000"), rDoc.GetString(ScAddress(5,6,1))); @@ -2164,6 +2168,7 @@ void ScFiltersTest::testPivotTableSharedNestedDateGroupXLSX() // TODO: check why this fails with the empty string //CPPUNIT_ASSERT_EQUAL(OUString("2007"), rDoc.GetString(ScAddress(0,32,1))); CPPUNIT_ASSERT_EQUAL(OUString("Quarters"), rDoc.GetString(ScAddress(1,3,1))); + CPPUNIT_ASSERT_EQUAL(OUString("a"), rDoc.GetString(ScAddress(2,3,1))); // Second table CPPUNIT_ASSERT_EQUAL(OUString("Years"), rDoc.GetString(ScAddress(6,3,1))); @@ -2174,6 +2179,7 @@ void ScFiltersTest::testPivotTableSharedNestedDateGroupXLSX() // TODO: check why this fails with the empty string //CPPUNIT_ASSERT_EQUAL(OUString("2007"), rDoc.GetString(ScAddress(6,31,1))); CPPUNIT_ASSERT_EQUAL(OUString("Quarters"), rDoc.GetString(ScAddress(7,3,1))); + CPPUNIT_ASSERT_EQUAL(OUString("a"), rDoc.GetString(ScAddress(8,3,1))); // There should be exactly 2 pivot tables and 1 cache. ScDPCollection* pDPs = rDoc.GetDPCollection(); @@ -2199,6 +2205,7 @@ void ScFiltersTest::testPivotTableSharedNumGroupXLSX() // Check whether right number groups are imported for both tables // First table + CPPUNIT_ASSERT_EQUAL(OUString("f"), rDoc.GetString(ScAddress(0,3,1))); CPPUNIT_ASSERT_EQUAL(OUString("32674-47673"), rDoc.GetString(ScAddress(0,4,1))); CPPUNIT_ASSERT_EQUAL(OUString("47674-62673"), rDoc.GetString(ScAddress(0,5,1))); CPPUNIT_ASSERT_EQUAL(OUString("62674-77673"), rDoc.GetString(ScAddress(0,6,1))); @@ -2206,6 +2213,7 @@ void ScFiltersTest::testPivotTableSharedNumGroupXLSX() CPPUNIT_ASSERT_EQUAL(OUString("92674-107673"), rDoc.GetString(ScAddress(0,8,1))); // Second table + CPPUNIT_ASSERT_EQUAL(OUString("f"), rDoc.GetString(ScAddress(5,3,1))); CPPUNIT_ASSERT_EQUAL(OUString("32674-47673"), rDoc.GetString(ScAddress(5,4,1))); CPPUNIT_ASSERT_EQUAL(OUString("47674-62673"), rDoc.GetString(ScAddress(5,5,1))); CPPUNIT_ASSERT_EQUAL(OUString("62674-77673"), rDoc.GetString(ScAddress(5,6,1))); commit db481dd30521f9ce498873cdc945cf9544bf983c Author: Tamás Zolnai <tamas.zol...@collabora.com> Date: Sun Oct 9 07:46:06 2016 +0200 Pivot table XLSX import: PivotCache is not handled as a const object. Change-Id: I226902205e977deb1a4a73db33e3bf7671067bc8 diff --git a/sc/source/filter/inc/pivotcachebuffer.hxx b/sc/source/filter/inc/pivotcachebuffer.hxx index 3bf5406..6b35a11 100644 --- a/sc/source/filter/inc/pivotcachebuffer.hxx +++ b/sc/source/filter/inc/pivotcachebuffer.hxx @@ -405,6 +405,7 @@ public: /** Returns the number of pivot cache fields. */ sal_Int32 getCacheFieldCount() const; /** Returns the cache field with the specified index. */ + PivotCacheField* getCacheField( sal_Int32 nFieldIdx ); const PivotCacheField* getCacheField( sal_Int32 nFieldIdx ) const; /** Returns the source column index of the field with the passed index. */ sal_Int32 getCacheDatabaseIndex( sal_Int32 nFieldIdx ) const; diff --git a/sc/source/filter/inc/pivottablebuffer.hxx b/sc/source/filter/inc/pivottablebuffer.hxx index 37ee04e..dfe5220 100644 --- a/sc/source/filter/inc/pivottablebuffer.hxx +++ b/sc/source/filter/inc/pivottablebuffer.hxx @@ -175,7 +175,7 @@ private: PivotTable& mrPivotTable; /// The parent pivot table object. ItemModelVector maItems; /// All items of this field. PTFieldModel maModel; /// Pivot field settings. - OUString maDPFieldName; /// Name of the field in DataPilot field collection. + OUString maDPFieldName; /// Name of the field in DataPilot field collection. sal_Int32 mnFieldIndex; /// Zero-based index of this field. }; @@ -338,6 +338,7 @@ public: getDataLayoutField() const; /** Returns the cache field with the specified index. */ + PivotCacheField* getCacheField( sal_Int32 nFieldIdx ); const PivotCacheField* getCacheField( sal_Int32 nFieldIdx ) const; /** Returns the base cache field of the data field item with the specified index. */ const PivotCacheField* getCacheFieldOfDataField( sal_Int32 nDataItemIdx ) const; @@ -373,7 +374,7 @@ private: PivotTableFilterVector maFilters; /// All field filters. PTDefinitionModel maDefModel; /// Global pivot table settings. PTLocationModel maLocationModel; /// Location settings of the pivot table. - const PivotCache* mpPivotCache; /// The pivot cache this table is based on. + PivotCache* mpPivotCache; /// The pivot cache this table is based on. css::uno::Reference< css::sheet::XDataPilotDescriptor > mxDPDescriptor; /// Descriptor of the DataPilot object. }; diff --git a/sc/source/filter/oox/pivotcachebuffer.cxx b/sc/source/filter/oox/pivotcachebuffer.cxx index a9e59ce..56632e7 100644 --- a/sc/source/filter/oox/pivotcachebuffer.cxx +++ b/sc/source/filter/oox/pivotcachebuffer.cxx @@ -1250,6 +1250,11 @@ sal_Int32 PivotCache::getCacheFieldCount() const return static_cast< sal_Int32 >( maFields.size() ); } +PivotCacheField* PivotCache::getCacheField( sal_Int32 nFieldIdx ) +{ + return maFields.get( nFieldIdx ).get(); +} + const PivotCacheField* PivotCache::getCacheField( sal_Int32 nFieldIdx ) const { return maFields.get( nFieldIdx ).get(); diff --git a/sc/source/filter/oox/pivottablebuffer.cxx b/sc/source/filter/oox/pivottablebuffer.cxx index bc3c05c..99b80b9 100644 --- a/sc/source/filter/oox/pivottablebuffer.cxx +++ b/sc/source/filter/oox/pivottablebuffer.cxx @@ -439,7 +439,7 @@ void PivotTableField::finalizeParentGroupingImport( const Reference< XDataPilotF { if( maDPFieldName.isEmpty() ) // prevent endless loops if file format is broken { - if( const PivotCacheField* pCacheField = mrPivotTable.getCacheField( mnFieldIndex ) ) + if( PivotCacheField* pCacheField = mrPivotTable.getCacheField( mnFieldIndex ) ) { // data field can have user defined groupname captions, apply them // if they do @@ -449,9 +449,9 @@ void PivotTableField::finalizeParentGroupingImport( const Reference< XDataPilotF if ( aIt->mnType == XML_data && aIt->msCaption.getLength() ) captionList.push_back( IdCaptionPair( aIt->mnCacheItem, aIt->msCaption ) ); } - // #FIXME find another way out of this const nightmare prison if ( !captionList.empty() ) - const_cast<PivotCacheField*>( pCacheField )->applyItemCaptions( captionList ); + pCacheField->applyItemCaptions( captionList ); + maDPFieldName = pCacheField->createParentGroupField( rxBaseDPField, rBaseCacheField, orItemNames ); // on success, try to create nested group fields Reference< XDataPilotField > xDPField = mrPivotTable.getDataPilotField( maDPFieldName ); @@ -1300,6 +1300,11 @@ Reference< XDataPilotField > PivotTable::getDataLayoutField() const return xDPField; } +PivotCacheField* PivotTable::getCacheField( sal_Int32 nFieldIdx ) +{ + return mpPivotCache ? mpPivotCache->getCacheField( nFieldIdx ) : nullptr; +} + const PivotCacheField* PivotTable::getCacheField( sal_Int32 nFieldIdx ) const { return mpPivotCache ? mpPivotCache->getCacheField( nFieldIdx ) : nullptr;
_______________________________________________ Libreoffice-commits mailing list libreoffice-comm...@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/libreoffice-commits