Skip to content

Commit

Permalink
Bugfix/biagas/fix unstructureddomainboundaries exchange scalars dev (#…
Browse files Browse the repository at this point in the history 
…20010)

* Fix avtUnstructuredDomainBoundaries::ExchangeScalars and ExchangeVectors (#19991)

* Fix avtUnstructuredDomainBoundaries::ExchangeScalars and ExchangeVectors.
Ensure all procs are calling the same Exchange function based on dataType that they get from an MPI_Allreduce.
This fixes problems in parallel (segv/hang) when avtExtraGhostNodes/Zones are exchanged.
Modified avtGenericDatabse to ensure all procs participate in the exchange of avtExtraGhostNodes/Zones.

* Don't attempt to exchange extraGhost info is no process posesses the info.

* Add consistency checks of datatypes for ExchangeScalar and ExchangeVector.

* Update consistency test so that the Exception isn't thrown in a case where one proc doesn't have data (dataType == -1).

* Tweaks to data-type consistency check to preven hang.

* Add mesh meta data flag 'hasExtraGhostInfo' false by default.
Mili plugin sets it to true when it will generate extraGhostInfo.
Update avtGenericDatabase to test this flag before calling code that utilizes extra ghost information.

* Update all MeshMetaData baselines due to addition of new field.
  • Loading branch information
@biagas
biagas authored Nov 4, 2024
1 parent 92140fd commit 96fd599
Show file tree
Hide file tree
Showing 81 changed files with 641 additions and 277 deletions.
14 changes: 13 additions & 1 deletion src/avt/DBAtts/MetaData/avtMeshMetaData.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ void avtMeshMetaData::Init()
LODs = 1;
presentGhostZoneTypes = 0;
zonesWereSplit = false;
hasExtraGhostInfo = false;

avtMeshMetaData::SelectAll();
}
Expand Down Expand Up @@ -187,6 +188,7 @@ void avtMeshMetaData::Copy(const avtMeshMetaData &obj)
LODs = obj.LODs;
presentGhostZoneTypes = obj.presentGhostZoneTypes;
zonesWereSplit = obj.zonesWereSplit;
hasExtraGhostInfo = obj.hasExtraGhostInfo;

avtMeshMetaData::SelectAll();
}
Expand Down Expand Up @@ -425,7 +427,8 @@ avtMeshMetaData::operator == (const avtMeshMetaData &obj) const
(hideFromGUI == obj.hideFromGUI) &&
(LODs == obj.LODs) &&
(presentGhostZoneTypes == obj.presentGhostZoneTypes) &&
(zonesWereSplit == obj.zonesWereSplit));
(zonesWereSplit == obj.zonesWereSplit) &&
(hasExtraGhostInfo == obj.hasExtraGhostInfo));
}

// ****************************************************************************
Expand Down Expand Up @@ -621,6 +624,7 @@ avtMeshMetaData::SelectAll()
Select(ID_LODs, (void *)&LODs);
Select(ID_presentGhostZoneTypes, (void *)&presentGhostZoneTypes);
Select(ID_zonesWereSplit, (void *)&zonesWereSplit);
Select(ID_hasExtraGhostInfo, (void *)&hasExtraGhostInfo);
}

///////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -707,6 +711,7 @@ avtMeshMetaData::GetFieldName(int index) const
case ID_LODs: return "LODs";
case ID_presentGhostZoneTypes: return "presentGhostZoneTypes";
case ID_zonesWereSplit: return "zonesWereSplit";
case ID_hasExtraGhostInfo: return "hasExtraGhostInfo";
default: return "invalid index";
}
}
Expand Down Expand Up @@ -783,6 +788,7 @@ avtMeshMetaData::GetFieldType(int index) const
case ID_LODs: return FieldType_int;
case ID_presentGhostZoneTypes: return FieldType_int;
case ID_zonesWereSplit: return FieldType_bool;
case ID_hasExtraGhostInfo: return FieldType_bool;
default: return FieldType_unknown;
}
}
Expand Down Expand Up @@ -859,6 +865,7 @@ avtMeshMetaData::GetFieldTypeName(int index) const
case ID_LODs: return "int";
case ID_presentGhostZoneTypes: return "int";
case ID_zonesWereSplit: return "bool";
case ID_hasExtraGhostInfo: return "bool";
default: return "invalid index";
}
}
Expand Down Expand Up @@ -1175,6 +1182,11 @@ avtMeshMetaData::FieldsEqual(int index_, const AttributeGroup *rhs) const
retval = (zonesWereSplit == obj.zonesWereSplit);
}
break;
case ID_hasExtraGhostInfo:
{ // new scope
retval = (hasExtraGhostInfo == obj.hasExtraGhostInfo);
}
break;
default: retval = false;
}

Expand Down
4 changes: 3 additions & 1 deletion src/avt/DBAtts/MetaData/avtMeshMetaData.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ class DBATTS_API avtMeshMetaData : public AttributeSubject
ID_LODs,
ID_presentGhostZoneTypes,
ID_zonesWereSplit,
ID_hasExtraGhostInfo,
ID__LAST
};

Expand Down Expand Up @@ -184,12 +185,13 @@ class DBATTS_API avtMeshMetaData : public AttributeSubject
int LODs;
int presentGhostZoneTypes;
bool zonesWereSplit;
bool hasExtraGhostInfo;

private:
// Static class format string for type map.
static const char *TypeMapFormatString;
static const private_tmfs_t TmfsStruct;
};
#define AVTMESHMETADATA_TMFS "ssbiiiibIbiissssssbDDiisss*aiisss*i*i*bibbbbibFFbDibbiib"
#define AVTMESHMETADATA_TMFS "ssbiiiibIbiissssssbDDiisss*aiisss*i*i*bibbbbibFFbDibbiibb"

#endif
3 changes: 3 additions & 0 deletions src/avt/DBAtts/MetaData/avtMeshMetaData.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,9 @@
<Field name="zonesWereSplit" label="Orignal zones were split" type="bool" access="public">
false
</Field>
<Field name="hasExtraGhostInfo" label="Has extra ghost information." type="bool" access="public">
false
</Field>
<Function name="avtMeshMetaData2" user="true" member="true">
</Function>
<Function name="avtMeshMetaData3" user="true" member="true">
Expand Down
131 changes: 70 additions & 61 deletions src/avt/Database/Database/avtGenericDatabase.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ DebugDumpDatasetCollection(avtDatasetCollection &dsc, int ndoms,
//
// Purpose:
// Merge generated ghost zones/nodes with extra ghost zones/nodes provided
// by the database in the form of avtExtraGhostZones and
// by the database in the form of avtExtraGhostZones and
// avtExtraGhostNodes.
//
// Returns: void
Expand All @@ -328,7 +328,7 @@ avtGenericDatabase::AugmentGhostData(avtDatasetCollection &ds,
if (dataset != NULL)
{
// this lambda handles zone and node cases
auto combineGhosts = [this, spec, domain_id](vtkUnsignedCharArray* ghosts,
auto combineGhosts = [this, spec, domain_id](vtkUnsignedCharArray* ghosts,
vtkUnsignedCharArray* extraGhosts,
vtkDataSet* dataset,
const std::string &arrayName,
Expand All @@ -352,12 +352,12 @@ avtGenericDatabase::AugmentGhostData(avtDatasetCollection &ds,
if (num_tuples != extraGhosts->GetNumberOfTuples())
{
// we don't know what to do in the case that the original
// and extra ghost zone/node arrays do not have the same
// and extra ghost zone/node arrays do not have the same
// number of tuples.
EXCEPTION0(ImproperUseException);
}

// create a new ghost zone/node array that will be the result
// create a new ghost zone/node array that will be the result
// of merging the existing and extra.
vtkUnsignedCharArray *newGhosts = vtkUnsignedCharArray::New();
newGhosts->SetName(arrayName.c_str());
Expand Down Expand Up @@ -419,8 +419,8 @@ avtGenericDatabase::AugmentGhostData(avtDatasetCollection &ds,
newGhosts->Delete();
dataset->GetCellData()->CopyFieldOn(arrayName.c_str());

// if we are cell centered then we need to perform some
// extra steps to tell the metadata that we now have
// if we are cell centered then we need to perform some
// extra steps to tell the metadata that we now have
// ghost zones.
const int ts = spec->GetTimestep();
avtDatabaseMetaData *md = GetMetaData(ts);
Expand All @@ -439,10 +439,10 @@ avtGenericDatabase::AugmentGhostData(avtDatasetCollection &ds,
};

// zones first
vtkUnsignedCharArray *ghostZones =
vtkUnsignedCharArray *ghostZones =
static_cast<vtkUnsignedCharArray*>(
dataset->GetCellData()->GetArray("avtGhostZones"));
vtkUnsignedCharArray *extraGhostZones =
vtkUnsignedCharArray *extraGhostZones =
static_cast<vtkUnsignedCharArray*>(
dataset->GetCellData()->GetArray("avtExtraGhostZones"));

Expand All @@ -453,10 +453,10 @@ avtGenericDatabase::AugmentGhostData(avtDatasetCollection &ds,
true /*cell centered is true*/);

// nodes second
vtkUnsignedCharArray *ghostNodes =
vtkUnsignedCharArray *ghostNodes =
static_cast<vtkUnsignedCharArray*>(
dataset->GetPointData()->GetArray("avtGhostNodes"));
vtkUnsignedCharArray *extraGhostNodes =
vtkUnsignedCharArray *extraGhostNodes =
static_cast<vtkUnsignedCharArray*>(
dataset->GetPointData()->GetArray("avtExtraGhostNodes"));

Expand Down Expand Up @@ -667,10 +667,13 @@ avtGenericDatabase::AugmentGhostData(avtDatasetCollection &ds,
//
// Alister Maguire, Tue Sep 24 10:04:42 MST 2019
// Added a call to GetQOTOutput when prompted.
//
//
// Justin Privitera, Tue Oct 22 10:32:27 PDT 2024
// Call augment ghost data unconditionally.
//
// Kathleen Biagas, Mon Nov 4, 2024
// Only call AugmengGhostData if avtMeshMetaData:hasExtraGhostInfo is true.
//
// ****************************************************************************

avtDataTree_p
Expand Down Expand Up @@ -1016,8 +1019,12 @@ avtGenericDatabase::GetOutput(avtDataRequest_p spec,
spec, src, allDomains,
canDoCollectiveCommunication);
}
// unconditionally add ghost data if the database provided extra ghost information
AugmentGhostData(datasetCollection, spec, src);

if (md->GetMesh(meshname) != NULL && md->GetMesh(meshname)->hasExtraGhostInfo)
{
// unconditionally add ghost data if the database provided extra ghost information
AugmentGhostData(datasetCollection, spec, src);
}

//
// Finally, do the material selection.
Expand Down Expand Up @@ -3299,7 +3306,7 @@ avtGenericDatabase::GetLabelVariable(const char *varname, int ts, int domain,
//
// Kathleen Biagas, Thu Sep 11 09:10:42 PDT 2014
// Keep avtOriginalNodeNumbers if present.
//
//
// Justin Privitera, Tue Oct 22 10:32:27 PDT 2024
// Keep extra ghost zone/node arrays if present.
//
Expand Down Expand Up @@ -6023,7 +6030,7 @@ avtGenericDatabase::EnumScalarSelect(avtDatasetCollection &dsc,
//
// Mark C. Miller, Wed Mar 4 18:00:38 PST 2009
// Adjusted for dbio-only build
//
//
// Justin Privitera, Fri Sep 27 10:18:37 PDT 2024
// Relaxed restriction on input data only being floats.
// ****************************************************************************
Expand Down Expand Up @@ -7984,12 +7991,17 @@ avtGenericDatabase::CommunicateGhostZonesFromDomainBoundariesFromFile(
// Modified to handle the case where a variable is defined on a subset
// of the materials and a domain has mixed materials without the material
// the variable was defined on.
//
//
// Justin Privitera, Tue Oct 22 10:32:27 PDT 2024
// Exchange extra ghost zone/node arrays.
//
// Justin Privitera, Thu Oct 24 11:50:43 PDT 2024
// Avoid segfault when exchanging extra ghost zone/node arrays.
//
// Kathleen Biagas, Thu Oct 31, 2024
// Don't attempt to Exchange extra ghost zone/node arrays if no process
// has extra ghost information.
//
// Kathleen Biagas, Mon Nov 4, 2024
// Use avtMeshMetaData field 'hasExtraGhostInfo' to determine whether or
// not to handle extra ghost information.
//
// ****************************************************************************

Expand Down Expand Up @@ -8611,30 +8623,30 @@ avtGenericDatabase::CommunicateGhostZonesFromDomainBoundaries(
}
}

//
// Exchange ExtraGhostZone Arrays.
//
// this logic was added to support the functionality in the
// AugmentGhostData() function.
vector<vtkDataArray *> extraGhostZones;
for (size_t j = 0 ; j < doms.size() ; j++)
if(md->GetMesh(meshname)->hasExtraGhostInfo)
{
vtkDataSet *ds1 = list[j];
if (ds1 == NULL ||
ds1->GetNumberOfPoints() == 0 || ds1->GetNumberOfCells() == 0)
{
extraGhostZones.push_back(NULL);
continue;
}
if (ds1->GetCellData()->GetArray("avtExtraGhostZones"))
//
// Exchange ExtraGhostZone Arrays.
//
// this logic was added to support the functionality in the
// AugmentGhostData() function.
vector<vtkDataArray *> extraGhostZones;
for (size_t j = 0 ; j < doms.size() ; j++)
{
extraGhostZones.push_back(ds1->GetCellData()->GetArray(
"avtExtraGhostZones"));
vtkDataSet *ds1 = list[j];
if (ds1 == NULL ||
ds1->GetNumberOfPoints() == 0 || ds1->GetNumberOfCells() == 0)
{
extraGhostZones.push_back(NULL);
continue;
}
if (ds1->GetCellData()->GetArray("avtExtraGhostZones"))
{
extraGhostZones.push_back(ds1->GetCellData()->GetArray(
"avtExtraGhostZones"));
}
}

}
if (extraGhostZones.size() > 0)
{
vector<vtkDataArray *> extraGhostZonesOut;
extraGhostZonesOut = dbi->ExchangeScalar(doms,false,extraGhostZones);
for (int j = 0 ; j < (int)doms.size() ; j++)
Expand All @@ -8646,32 +8658,29 @@ avtGenericDatabase::CommunicateGhostZonesFromDomainBoundaries(
ds1->GetCellData()->AddArray(extraGhostZonesOut[j]);
extraGhostZonesOut[j]->Delete();
}
}

//
// Exchange ExtraGhostNode Arrays.
//
// this logic was added to support the functionality in the
// AugmentGhostData() function.
vector<vtkDataArray *> extraGhostNodes;
for (size_t j = 0 ; j < doms.size() ; j++)
{
vtkDataSet *ds1 = list[j];
if (ds1 == NULL ||
ds1->GetNumberOfPoints() == 0 || ds1->GetNumberOfCells() == 0)
{
extraGhostNodes.push_back(NULL);
continue;
}
if (ds1->GetPointData()->GetArray("avtExtraGhostNodes"))
//
// Exchange ExtraGhostNode Arrays.
//
// this logic was added to support the functionality in the
// AugmentGhostData() function.
vector<vtkDataArray *> extraGhostNodes;
for (size_t j = 0 ; j < doms.size() ; j++)
{
extraGhostNodes.push_back(ds1->GetPointData()->GetArray(
"avtExtraGhostNodes"));
vtkDataSet *ds1 = list[j];
if (ds1 == NULL ||
ds1->GetNumberOfPoints() == 0 || ds1->GetNumberOfCells() == 0)
{
extraGhostNodes.push_back(NULL);
continue;
}
if (ds1->GetPointData()->GetArray("avtExtraGhostNodes"))
{
extraGhostNodes.push_back(ds1->GetPointData()->GetArray(
"avtExtraGhostNodes"));
}
}

}
if (extraGhostNodes.size() > 0)
{
vector<vtkDataArray *> extraGhostNodesOut;
extraGhostNodesOut = dbi->ExchangeScalar(doms,true,extraGhostNodes);
for (int j = 0 ; j < (int)doms.size() ; j++)
Expand Down
Loading

0 comments on commit 96fd599

Please sign in to comment.