From d4713f41c17593719b835de54fbb48f4f1f564ce Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 2 Feb 2024 18:00:07 +0530 Subject: [PATCH 001/143] Remove make_pair when inserting to map --- .../JasmineGraphHashMapCentralStore.cpp | 6 +-- ...smineGraphHashMapDuplicateCentralStore.cpp | 4 +- src/frontend/JasmineGraphFrontEnd.cpp | 2 +- .../executor/impl/TriangleCountExecutor.cpp | 2 +- .../JasmineGraphHashMapLocalStore.cpp | 10 ++--- .../trainer/JasmineGraphTrainingSchedular.cpp | 4 +- src/partitioner/local/JSONParser.cpp | 4 +- src/partitioner/local/MetisPartitioner.cpp | 30 ++++++------- src/partitioner/local/MetisPartitioner.h | 2 +- .../JasminGraphLinkPredictor.cpp | 2 +- src/server/JasmineGraphInstanceService.cpp | 44 +++++++++---------- src/server/JasmineGraphServer.cpp | 29 ++++++------ 12 files changed, 68 insertions(+), 71 deletions(-) diff --git a/src/centralstore/JasmineGraphHashMapCentralStore.cpp b/src/centralstore/JasmineGraphHashMapCentralStore.cpp index 047bd62fa..0d503875e 100644 --- a/src/centralstore/JasmineGraphHashMapCentralStore.cpp +++ b/src/centralstore/JasmineGraphHashMapCentralStore.cpp @@ -137,7 +137,7 @@ map JasmineGraphHashMapCentralStore::getOutDegreeDistributionHashMap auto key = it->first; auto nodes = it->second; unordered_set neighboursOfNeighbour = nodes; - distributionHashMap.insert(std::make_pair(it->first, distribution)); + distributionHashMap[it->first] = distribution; } return distributionHashMap; } @@ -155,7 +155,7 @@ map JasmineGraphHashMapCentralStore::getInDegreeDistributionHashMap( long previousValue = distMapItr->second; distMapItr->second = previousValue + 1; } else { - distributionHashMap.insert(std::make_pair(*itr, 1)); + distributionHashMap[*itr] = 1; } } } @@ -212,7 +212,7 @@ void JasmineGraphHashMapCentralStore::toLocalSubGraphMap(const PartEdgeMapStore auto value = entry->value(); const flatbuffers::Vector &vector = *value; unordered_set valueSet(vector.begin(), vector.end()); - centralSubgraphMap.insert(std::make_pair(key, valueSet)); + centralSubgraphMap[key] = valueSet; } } diff --git a/src/centralstore/JasmineGraphHashMapDuplicateCentralStore.cpp b/src/centralstore/JasmineGraphHashMapDuplicateCentralStore.cpp index fc5a5e78a..4254b3b8c 100644 --- a/src/centralstore/JasmineGraphHashMapDuplicateCentralStore.cpp +++ b/src/centralstore/JasmineGraphHashMapDuplicateCentralStore.cpp @@ -131,7 +131,7 @@ map JasmineGraphHashMapDuplicateCentralStore::getOutDegreeDistributi for (map>::iterator it = centralDuplicateStoreSubgraphMap.begin(); it != centralDuplicateStoreSubgraphMap.end(); ++it) { long distribution = (it->second).size(); - distributionHashMap.insert(std::make_pair(it->first, distribution)); + distributionHashMap[it->first] = distribution; } return distributionHashMap; } @@ -186,7 +186,7 @@ void JasmineGraphHashMapDuplicateCentralStore::toLocalSubGraphMap(const PartEdge auto value = entry->value(); const flatbuffers::Vector &vector = *value; unordered_set valueSet(vector.begin(), vector.end()); - centralDuplicateStoreSubgraphMap.insert(std::make_pair(key, valueSet)); + centralDuplicateStoreSubgraphMap[key] = valueSet; } } diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index 53e640b5b..331c6df56 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -447,7 +447,7 @@ map JasmineGraphFrontEnd::getOutDegreeDistributionHashMap(map>::iterator it = graphMap.begin(); it != graphMap.end(); ++it) { long distribution = (it->second).size(); - distributionHashMap.insert(std::make_pair(it->first, distribution)); + distributionHashMap[it->first] = distribution; } return distributionHashMap; } diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index f2abe8e0e..7a781af52 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -141,7 +141,7 @@ void TriangleCountExecutor::execute() { if (partitionMap.find(workerID) == partitionMap.end()) { std::vector partitionVec; partitionVec.push_back(partitionId); - partitionMap.insert(std::pair>(workerID, partitionVec)); + partitionMap[workerID] = partitionVec; } else { std::vector partitionVec = partitionMap.find(workerID)->second; partitionVec.push_back(partitionId); diff --git a/src/localstore/JasmineGraphHashMapLocalStore.cpp b/src/localstore/JasmineGraphHashMapLocalStore.cpp index 41d6bf43b..269c7b528 100644 --- a/src/localstore/JasmineGraphHashMapLocalStore.cpp +++ b/src/localstore/JasmineGraphHashMapLocalStore.cpp @@ -86,7 +86,7 @@ void JasmineGraphHashMapLocalStore::toLocalSubGraphMap(const PartEdgeMapStore *e const flatbuffers::Vector &vector = *value; unordered_set valueSet(vector.begin(), vector.end()); - localSubGraphMap.insert(std::make_pair(key, valueSet)); + localSubGraphMap[key] = valueSet; } } @@ -129,7 +129,7 @@ map JasmineGraphHashMapLocalStore::getOutDegreeDistributionHashMap() for (map>::iterator it = localSubGraphMap.begin(); it != localSubGraphMap.end(); ++it) { long distribution = (it->second).size(); - distributionHashMap.insert(std::make_pair(it->first, distribution)); + distributionHashMap[it->first] = distribution; } return distributionHashMap; } @@ -146,7 +146,7 @@ map JasmineGraphHashMapLocalStore::getInDegreeDistributionHashMap() long previousValue = distMapItr->second; distMapItr->second = previousValue + 1; } else { - distributionHashMap.insert(std::make_pair(*itr, 1)); + distributionHashMap[*itr] = 1; } } } @@ -245,7 +245,7 @@ void JasmineGraphHashMapLocalStore::toLocalAttributeMap(const AttributeStore *at for (int j = 0; j < attributesSize; j = j + 1) { attributeVector.push_back(attributes->Get(j)->c_str()); } - localAttributeMap.insert(std::make_pair(key, attributeVector)); + localAttributeMap[key] = attributeVector; } } @@ -267,7 +267,7 @@ void JasmineGraphHashMapLocalStore::toLocalEdgeMap(const PartEdgeMapStore *edgeM auto value = entry->value(); const flatbuffers::Vector &vector = *value; std::vector valueSet(vector.begin(), vector.end()); - edgeMap.insert(std::make_pair(key, valueSet)); + edgeMap[key] = valueSet; } } diff --git a/src/ml/trainer/JasmineGraphTrainingSchedular.cpp b/src/ml/trainer/JasmineGraphTrainingSchedular.cpp index 22f3fed07..9229b573d 100644 --- a/src/ml/trainer/JasmineGraphTrainingSchedular.cpp +++ b/src/ml/trainer/JasmineGraphTrainingSchedular.cpp @@ -104,7 +104,7 @@ map> JasmineGraphTrainingSchedular::schedulePartition } std::map scheduledPartitionSets = packPartitionsToMemory(memoryEstimationForEachPartition, availableMemory); - scheduleForEachHost.insert(make_pair(j->second, scheduledPartitionSets)); + scheduleForEachHost[j->second] = scheduledPartitionSets; } refToSqlite->finalize(); delete refToSqlite; @@ -310,7 +310,7 @@ static map>> scheduleGradientPassingTraining // Get schedule for host map> scheduledPartitionSets = schedulePartitionsBestFit(partitionMemoryList, partitionWorkerMap, availableMemory); - scheduleForEachHost.insert(make_pair(j->second, scheduledPartitionSets)); + scheduleForEachHost[j->second] = scheduledPartitionSets; } refToSqlite->finalize(); delete refToSqlite; diff --git a/src/partitioner/local/JSONParser.cpp b/src/partitioner/local/JSONParser.cpp index a5bb23f0b..cc447eb61 100644 --- a/src/partitioner/local/JSONParser.cpp +++ b/src/partitioner/local/JSONParser.cpp @@ -188,7 +188,7 @@ static std::map countFileds(std::string inputFilePath) { continue; } if (fieldCounts.find(field) == fieldCounts.end()) { - fieldCounts.insert(make_pair(field, 1)); + fieldCounts[field] = 1; } else { fieldCounts[field]++; } @@ -202,7 +202,7 @@ static std::map countFileds(std::string inputFilePath) { std::string field = it->first; if (it->second > 821) { if (fieldsMap.find(field) == fieldsMap.end()) { - fieldsMap.insert(make_pair(field, field_counter)); + fieldsMap[field] = field_counter; field_counter++; } } diff --git a/src/partitioner/local/MetisPartitioner.cpp b/src/partitioner/local/MetisPartitioner.cpp index f3548cae8..4bf2b4139 100644 --- a/src/partitioner/local/MetisPartitioner.cpp +++ b/src/partitioner/local/MetisPartitioner.cpp @@ -292,7 +292,7 @@ std::vector> MetisPartitioner::partitioneWithGPMetis( std::to_string(this->edgeCount) + "' WHERE idgraph = '" + std::to_string(this->graphID) + "'"; this->sqlite->runUpdate(sqlStatement); - this->fullFileList.push_back(this->partitionFileList); + this->fullFileList.push_back(this->partitionFileMap); this->fullFileList.push_back(this->centralStoreFileList); this->fullFileList.push_back(this->centralStoreDuplicateFileList); this->fullFileList.push_back(this->partitionAttributeFileList); @@ -671,7 +671,7 @@ void MetisPartitioner::writeSerializedPartitionFiles(int part) { // Compress part files Utils::compressFile(outputFilePart); partFileMutex.lock(); - partitionFileList.insert(make_pair(part, outputFilePart + ".gz")); + partitionFileMap[part] = outputFilePart + ".gz"; partFileMutex.unlock(); partitioner_logger.log("Serializing done for local part " + to_string(part), "info"); } @@ -687,7 +687,7 @@ void MetisPartitioner::writeSerializedMasterFiles(int part) { Utils::compressFile(outputFilePartMaster); masterFileMutex.lock(); - centralStoreFileList.insert(make_pair(part, outputFilePartMaster + ".gz")); + centralStoreFileList[part] = outputFilePartMaster + ".gz"; masterFileMutex.unlock(); partitioner_logger.log("Serializing done for central part " + to_string(part), "info"); } @@ -703,7 +703,7 @@ void MetisPartitioner::writeSerializedDuplicateMasterFiles(int part) { Utils::compressFile(outputFilePartMaster); masterFileMutex.lock(); - centralStoreDuplicateFileList.insert(make_pair(part, outputFilePartMaster + ".gz")); + centralStoreDuplicateFileList[part] = outputFilePartMaster + ".gz"; masterFileMutex.unlock(); partitioner_logger.log("Serializing done for duplicate central part " + to_string(part), "info"); } @@ -747,7 +747,7 @@ void MetisPartitioner::writePartitionFiles(int part) { // Compress part files Utils::compressFile(outputFilePart); partFileMutex.lock(); - partitionFileList.insert(make_pair(part, outputFilePart + ".gz")); + partitionFileMap[part] = outputFilePart + ".gz"; partFileMutex.unlock(); } @@ -791,7 +791,7 @@ void MetisPartitioner::writeMasterFiles(int part) { Utils::compressFile(outputFilePartMaster); masterFileMutex.lock(); - centralStoreFileList.insert(make_pair(part, outputFilePartMaster + ".gz")); + centralStoreFileList[part] = outputFilePartMaster + ".gz"; masterFileMutex.unlock(); } @@ -826,7 +826,7 @@ void MetisPartitioner::writeTextAttributeFilesForPartitions(int part) { Utils::compressFile(attributeFilePart); partAttrFileMutex.lock(); - partitionAttributeFileList.insert(make_pair(part, attributeFilePart + ".gz")); + partitionAttributeFileList[part] = attributeFilePart + ".gz"; partAttrFileMutex.unlock(); partitioner_logger.log("Attribute writing done for local part " + to_string(part), "info"); } @@ -861,7 +861,7 @@ void MetisPartitioner::writeTextAttributeFilesForMasterParts(int part) { Utils::compressFile(attributeFilePartMaster); masterAttrFileMutex.lock(); - centralStoreAttributeFileList.insert(make_pair(part, attributeFilePartMaster + ".gz")); + centralStoreAttributeFileList[part] = attributeFilePartMaster + ".gz"; masterAttrFileMutex.unlock(); partitioner_logger.log("Attribute writing done for central part " + to_string(part), "info"); } @@ -895,7 +895,7 @@ void MetisPartitioner::writeRDFAttributeFilesForPartitions(int part) { Utils::compressFile(attributeFilePart); partAttrFileMutex.lock(); - partitionAttributeFileList.insert(make_pair(part, attributeFilePart + ".gz")); + partitionAttributeFileList[part] = attributeFilePart + ".gz"; partAttrFileMutex.unlock(); } @@ -928,7 +928,7 @@ void MetisPartitioner::writeRDFAttributeFilesForMasterParts(int part) { Utils::compressFile(attributeFilePartMaster); masterAttrFileMutex.lock(); - centralStoreAttributeFileList.insert(make_pair(part, attributeFilePartMaster + ".gz")); + centralStoreAttributeFileList[part] = attributeFilePartMaster + ".gz"; masterAttrFileMutex.unlock(); } @@ -974,13 +974,13 @@ string MetisPartitioner::reformatDataSet(string inputFilePath, int graphID) { secondVertex = std::stoi(vertexTwo); if (vertexToIDMap.find(firstVertex) == vertexToIDMap.end()) { - vertexToIDMap.insert(make_pair(firstVertex, idCounter)); - idToVertexMap.insert(make_pair(idCounter, firstVertex)); + vertexToIDMap[firstVertex] = idCounter; + idToVertexMap[idCounter] = firstVertex; idCounter++; } if (vertexToIDMap.find(secondVertex) == vertexToIDMap.end()) { - vertexToIDMap.insert(make_pair(secondVertex, idCounter)); - idToVertexMap.insert(make_pair(idCounter, secondVertex)); + vertexToIDMap[secondVertex] = idCounter; + idToVertexMap[idCounter] = secondVertex; idCounter++; } @@ -1107,7 +1107,7 @@ void MetisPartitioner::writeSerializedCompositeMasterFiles(std::string part) { masterFileMutex.lock(); for (graphIdIterator = graphIds.begin(); graphIdIterator != graphIds.end(); ++graphIdIterator) { std::string graphId = *graphIdIterator; - compositeCentralStoreFileList.insert(make_pair(std::atoi(graphId.c_str()), outputFilePartMaster + ".gz")); + compositeCentralStoreFileList[std::atoi(graphId.c_str())] = outputFilePartMaster + ".gz"; } masterFileMutex.unlock(); partitioner_logger.log("Serializing done for central part " + part, "info"); diff --git a/src/partitioner/local/MetisPartitioner.h b/src/partitioner/local/MetisPartitioner.h index 5b581fa95..0a5431643 100644 --- a/src/partitioner/local/MetisPartitioner.h +++ b/src/partitioner/local/MetisPartitioner.h @@ -70,7 +70,7 @@ class MetisPartitioner { int smallestVertex = std::numeric_limits::max(); string graphAttributeType; - std::map partitionFileList; + std::map partitionFileMap; std::map centralStoreFileList; std::map compositeCentralStoreFileList; std::map centralStoreDuplicateFileList; diff --git a/src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp b/src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp index 3a8045e86..08a462e6c 100644 --- a/src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp +++ b/src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp @@ -44,7 +44,7 @@ void JasminGraphLinkPredictor::initiateLinkPrediction(std::string graphID, std:: selectedHostPartitions = (graphPartitionedHosts[it->first]).partitionID; selectedHostPartitionsNo = selectedHostPartitions.size(); } else { - remainHostMap.insert(std::pair(it->first, it->second)); + remainHostMap[it->first] = it->second; } count++; } diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 815f39997..a19ca3c49 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -472,7 +472,7 @@ JasmineGraphIncrementalLocalStore *JasmineGraphInstanceService::loadStreamingSto std::string folderLocation = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); JasmineGraphIncrementalLocalStore *jasmineGraphStreamingLocalStore = new JasmineGraphIncrementalLocalStore(stoi(graphId), stoi(partitionId), openMode); - graphDBMapStreamingStores.insert(std::make_pair(graphIdentifier, jasmineGraphStreamingLocalStore)); + graphDBMapStreamingStores[graphIdentifier] = jasmineGraphStreamingLocalStore; instance_logger.info("###INSTANCE### Loading Local Store : Completed"); return jasmineGraphStreamingLocalStore; } @@ -485,7 +485,7 @@ void JasmineGraphInstanceService::loadLocalStore( JasmineGraphHashMapLocalStore *jasmineGraphHashMapLocalStore = new JasmineGraphHashMapLocalStore(stoi(graphId), stoi(partitionId), folderLocation); jasmineGraphHashMapLocalStore->loadGraph(); - graphDBMapLocalStores.insert(std::make_pair(graphIdentifier, *jasmineGraphHashMapLocalStore)); + graphDBMapLocalStores[graphIdentifier] = *jasmineGraphHashMapLocalStore; instance_logger.info("###INSTANCE### Loading Local Store : Completed"); } void JasmineGraphInstanceService::loadInstanceCentralStore( @@ -496,7 +496,7 @@ void JasmineGraphInstanceService::loadInstanceCentralStore( JasmineGraphHashMapCentralStore *jasmineGraphHashMapCentralStore = new JasmineGraphHashMapCentralStore(stoi(graphId), stoi(partitionId)); jasmineGraphHashMapCentralStore->loadGraph(); - graphDBMapCentralStores.insert(std::make_pair(graphIdentifier, *jasmineGraphHashMapCentralStore)); + graphDBMapCentralStores[graphIdentifier] = *jasmineGraphHashMapCentralStore; instance_logger.info("###INSTANCE### Loading central Store : Completed"); } @@ -507,7 +507,7 @@ void JasmineGraphInstanceService::loadInstanceDuplicateCentralStore( JasmineGraphHashMapDuplicateCentralStore *jasmineGraphHashMapCentralStore = new JasmineGraphHashMapDuplicateCentralStore(stoi(graphId), stoi(partitionId)); jasmineGraphHashMapCentralStore->loadGraph(); - graphDBMapDuplicateCentralStores.insert(std::make_pair(graphIdentifier, *jasmineGraphHashMapCentralStore)); + graphDBMapDuplicateCentralStores[graphIdentifier] = *jasmineGraphHashMapCentralStore; } JasmineGraphHashMapCentralStore JasmineGraphInstanceService::loadCentralStore(std::string centralStoreFileName) { @@ -663,7 +663,7 @@ map JasmineGraphInstanceService::getOutDegreeDistributionHashMap(map for (map>::iterator it = graphMap.begin(); it != graphMap.end(); ++it) { long distribution = (it->second).size(); - distributionHashMap.insert(std::make_pair(it->first, distribution)); + distributionHashMap[it->first] = distribution; } return distributionHashMap; } @@ -1381,7 +1381,7 @@ map calculateInDegreeDist(string graphID, string partitionID, int se long degreeDistributionValue = degreeDistributionLocalItr->second; degreeDistribution[degreeDistributionLocalItr->first] = degreeDistributionValue + its->second; } else { - degreeDistribution.insert(std::make_pair(its->first, its->second)); + degreeDistribution[its->first] = its->second; } } @@ -1424,7 +1424,7 @@ map>> calculateLocalEgoNet(string graphID, s unordered_set neighbours = it->second; map> individualEgoNet; - individualEgoNet.insert(std::make_pair(it->first, neighbours)); + individualEgoNet[it->first] = neighbours; for (unordered_set::iterator neighbour = neighbours.begin(); neighbour != neighbours.end(); ++neighbour) { unordered_set neighboursOfNeighboursInSameEgoNet; @@ -1441,10 +1441,10 @@ map>> calculateLocalEgoNet(string graphID, s } } } - individualEgoNet.insert(std::make_pair(*neighbour, neighboursOfNeighboursInSameEgoNet)); + individualEgoNet[*neighbour] = neighboursOfNeighboursInSameEgoNet; } - egonetMap.insert(std::make_pair(it->first, individualEgoNet)); + egonetMap[it->first] = individualEgoNet; } for (map>::iterator it = centralGraphMap.begin(); it != centralGraphMap.end(); ++it) { @@ -1457,7 +1457,7 @@ map>> calculateLocalEgoNet(string graphID, s vertexMapFromCentralStore.insert( std::make_pair(it->first, distribution)); // Here we do not have the relation information among neighbours - egonetMap.insert(std::make_pair(it->first, vertexMapFromCentralStore)); + egonetMap[it->first] = vertexMapFromCentralStore; } else { map> egonetSubGraph = egonetMapItr->second; @@ -1674,7 +1674,7 @@ map calculateLocalPageRank(string graphID, double alpha, string pa for (auto itr = endVidSet.begin(); itr != endVidSet.end(); ++itr) { if (localGraphMap.find(*itr) == localGraphMap.end()) { unordered_set valueSet; - localGraphMap.insert(std::make_pair(*itr, valueSet)); + localGraphMap[*itr] = valueSet; } } } @@ -1754,7 +1754,7 @@ map calculateLocalPageRank(string graphID, double alpha, string pa if (inDegreeDistributionItr != inDegreeDistribution.end()) { long inDegree = inDegreeDistributionItr->second; double authorityScore = (alpha * 1 + mu) * inDegree; - rankMap.insert(std::make_pair(inDegreeDistributionItr->first, authorityScore)); + rankMap[inDegreeDistributionItr->first] = authorityScore; } } @@ -1770,7 +1770,7 @@ map calculateLocalPageRank(string graphID, double alpha, string pa if (rankMapItr != rankMap.end()) { existingParentRank = rankMapItr->second; } else { - rankMap.insert(std::make_pair(startVid, existingParentRank)); + rankMap[startVid] = existingParentRank; } long degree = endVidSet.size(); @@ -1788,7 +1788,7 @@ map calculateLocalPageRank(string graphID, double alpha, string pa rankMapItr->second = finalRank; } else { finalRank = existingChildRank + distributedRank; - rankMap.insert(std::make_pair(itr, finalRank)); + rankMap[itr] = finalRank; } } } @@ -1803,7 +1803,7 @@ map calculateLocalPageRank(string graphID, double alpha, string pa } else { int count = 0; for (map::iterator rankMapItr = rankMap.begin(); rankMapItr != rankMap.end(); ++rankMapItr) { - finalPageRankResults.insert(std::make_pair(rankMapItr->first, rankMapItr->second)); + finalPageRankResults[rankMapItr->first] = rankMapItr->second; count++; } } @@ -1864,7 +1864,7 @@ map> getEdgesWorldToLocal(string graphID, string parti } else { unordered_set fromIDs; fromIDs.insert(startVid); - worldToLocalVertexMap.insert(std::make_pair(*itr, fromIDs)); + worldToLocalVertexMap[*itr] = fromIDs; } } } @@ -2609,10 +2609,10 @@ static void page_rank_command(int connFd, int serverPort, map::iterator pageRankValue = pageRankResults.find(startVid); if (pageRankValue == pageRankResults.end()) { - pageRankLocalstore.insert(std::make_pair(startVid, 0.0)); + pageRankLocalstore[startVid] = 0.0; } else { double value = pageRankValue->second; - pageRankLocalstore.insert(std::make_pair(startVid, value)); + pageRankLocalstore[startVid] = value; } for (auto a = endVidSet.begin(); a != endVidSet.end(); ++a) { @@ -2620,10 +2620,10 @@ static void page_rank_command(int connFd, int serverPort, map::iterator pageRankValue = pageRankResults.find(endVid); if (pageRankLocalstore.find(endVid) == pageRankLocalstore.end()) { if (pageRankValue == pageRankResults.end()) { - pageRankLocalstore.insert(std::make_pair(endVid, 0.0)); + pageRankLocalstore[endVid] = 0.0; } else { double value = pageRankValue->second; - pageRankLocalstore.insert(std::make_pair(endVid, value)); + pageRankLocalstore[endVid] = value; } } } @@ -2761,10 +2761,10 @@ static void worker_page_rank_distribution_command( map::iterator pageRankValue = pageRankResults.find(startVid); if (pageRankValue == pageRankResults.end()) { - pageRankLocalstore.insert(std::make_pair(startVid, 1.0)); + pageRankLocalstore[startVid] = 1.0; } else { double value = pageRankValue->second; - pageRankLocalstore.insert(std::make_pair(startVid, value)); + pageRankLocalstore[startVid] = value; } } diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 5474070fd..3f6a29d93 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -272,7 +272,7 @@ void JasmineGraphServer::start_workers() { hostWorkerMap.push_back({*it, workerPort, workerDataPort}); // FIXME: When there are more than 1 worker in the same host, one workers ports will replace the entries of // other workers port entries in hostPortMap - hostPortMap.insert((pair>(*it, make_pair(workerPort, workerDataPort)))); + hostPortMap[*it] = make_pair(workerPort, workerDataPort); portCount++; // ToDO: Here for the moment we use host name as the IP address as the third parameter. // ToDO: We also keep user as empty string @@ -289,7 +289,7 @@ void JasmineGraphServer::start_workers() { portVector.push_back(workerPort); dataPortVector.push_back(workerDataPort); hostWorkerMap.push_back({*it, workerPort, workerDataPort}); - hostPortMap.insert(((pair>(*it, make_pair(workerPort, workerDataPort))))); + hostPortMap[*it] = make_pair(workerPort, workerDataPort); hostListModeNWorkers--; string is_public = "false"; valuesString += "(" + std::to_string(workerIDCounter) + ", " + hostID + ", \"" + hostName + "\", \"" + ip + @@ -427,10 +427,9 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, "docker run -v " + instanceDataFolder + ":" + instanceDataFolder + " -v " + aggregateDataFolder + ":" + aggregateDataFolder + " -v " + nmonFileLocation + ":" + nmonFileLocation + " -v " + instanceDataFolder + "/" + to_string(i) + "/logs" + ":" + - "/var/tmp/jasminegraph/logs" + " -p " + std::to_string(workerPortsVector.at(i)) + - ":" + std::to_string(workerPortsVector.at(i)) + " -p " + - std::to_string(workerDataPortsVector.at(i)) + ":" + - std::to_string(workerDataPortsVector.at(i)) + " -e WORKER_ID=" + to_string(i) + + "/var/tmp/jasminegraph/logs" + " -p " + std::to_string(workerPortsVector.at(i)) + ":" + + std::to_string(workerPortsVector.at(i)) + " -p " + std::to_string(workerDataPortsVector.at(i)) + + ":" + std::to_string(workerDataPortsVector.at(i)) + " -e WORKER_ID=" + to_string(i) + " jasminegraph:latest --MODE 2 --HOST_NAME " + host + " --MASTERIP " + masterHost + " --SERVER_PORT " + std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " + std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon; @@ -450,10 +449,9 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, "docker -H ssh://" + host + " run -v " + instanceDataFolder + ":" + instanceDataFolder + " -v " + aggregateDataFolder + ":" + aggregateDataFolder + " -v " + nmonFileLocation + ":" + nmonFileLocation + " -v " + instanceDataFolder + "/" + to_string(i) + "/logs" + ":" + - "/var/tmp/jasminegraph/logs" + " -p " + std::to_string(workerPortsVector.at(i)) + - ":" + std::to_string(workerPortsVector.at(i)) + " -p " + - std::to_string(workerDataPortsVector.at(i)) + ":" + - std::to_string(workerDataPortsVector.at(i)) + " -e WORKER_ID=" + to_string(i) + + "/var/tmp/jasminegraph/logs" + " -p " + std::to_string(workerPortsVector.at(i)) + ":" + + std::to_string(workerPortsVector.at(i)) + " -p " + std::to_string(workerDataPortsVector.at(i)) + + ":" + std::to_string(workerDataPortsVector.at(i)) + " -e WORKER_ID=" + to_string(i) + " jasminegraph:latest --MODE 2 --HOST_NAME " + host + " --MASTERIP " + masterHost + " --SERVER_PORT " + std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " + std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon; @@ -1234,7 +1232,7 @@ map JasmineGraphServer::getLiveHostIDList() { host = user + "@" + ip + ":" + serverPort; } - hostIDMap.insert(make_pair(host, id)); + hostIDMap[host] = id; } return hostIDMap; @@ -1456,8 +1454,7 @@ std::map JasmineGraphServer::g server_logger.info("name : " + name + " workerID : " + workerID + " sport : " + std::to_string(serverPort) + " sdport : " + std::to_string(serverDataPort) + " partitionId : " + partitionId); - graphPartitionedHosts.insert((pair( - workerID, {name, serverPort, serverDataPort, partitionId}))); + graphPartitionedHosts[workerID] = {name, serverPort, serverDataPort, partitionId}; } return graphPartitionedHosts; @@ -1497,14 +1494,14 @@ std::map JasmineGraphServer::getGr } else { vector vec; vec.push_back(j->second); - hostPartitions.insert((pair>(hostname, vec))); + hostPartitions[hostname] = vec; } } map graphPartitionedHosts; for (map>::iterator it = (hostPartitions.begin()); it != hostPartitions.end(); ++it) { - graphPartitionedHosts.insert((pair( - it->first, {hostPortMap[it->first].first, hostPortMap[it->first].second, hostPartitions[it->first]}))); + graphPartitionedHosts[it->first] = {hostPortMap[it->first].first, hostPortMap[it->first].second, + hostPartitions[it->first]}; } return graphPartitionedHosts; From a6761250f3516f90004fa6c8aa52403a6b782e17 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 3 Feb 2024 18:32:40 +0530 Subject: [PATCH 002/143] Make K8sWorkerController singleton --- .github/workflows/build.yml | 6 +- .github/workflows/code-lint.yml | 62 +++++------ src/frontend/JasmineGraphFrontEnd.cpp | 10 +- src/k8s/K8sWorkerController.cpp | 43 +++++--- src/k8s/K8sWorkerController.h | 9 +- src/partitioner/local/MetisPartitioner.cpp | 112 ++------------------ src/partitioner/local/MetisPartitioner.h | 4 - src/server/JasmineGraphServer.cpp | 102 +++++++++--------- src/server/JasmineGraphServer.h | 6 +- tests/unit/k8s/K8sWorkerController_test.cpp | 2 +- 10 files changed, 140 insertions(+), 216 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5822d6934..cc3d0e3da 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,12 +13,12 @@ env: jobs: unit-tests: runs-on: ubuntu-latest + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip unit tests')}} steps: - uses: actions/checkout@v4 with: ref: ${{github.head_ref}} - fetch-depth: 0 repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - uses: jupyterhub/action-k3s-helm@v4 with: @@ -50,12 +50,12 @@ jobs: docker-integration-tests: runs-on: ubuntu-latest + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip docker integration')}} steps: - uses: actions/checkout@v4 with: ref: ${{github.head_ref}} - fetch-depth: 0 repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - name: Docker Build @@ -69,12 +69,12 @@ jobs: k8s-integration-tests: runs-on: ubuntu-latest + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip k8s integration')}} steps: - uses: actions/checkout@v4 with: ref: ${{github.head_ref}} - fetch-depth: 0 repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - uses: jupyterhub/action-k3s-helm@v4 with: diff --git a/.github/workflows/code-lint.yml b/.github/workflows/code-lint.yml index baa0768cd..ba7964214 100644 --- a/.github/workflows/code-lint.yml +++ b/.github/workflows/code-lint.yml @@ -1,35 +1,37 @@ name: Style Check on: - push: - branches: [master] - pull_request: - branches: [master] + push: + branches: [master] + pull_request: + branches: [master] jobs: - style-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install tools - run: | - sudo apt-get update && sudo apt-get install -y --no-install-recommends cpplint shfmt - sudo wget https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 -O /bin/hadolint && sudo chmod +x /bin/hadolint - pip install pylint - - - name: hadolint check - if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip hadolint Check')}} - run: find . -type f -name '*Dockerfile*' -print0 | xargs -0 hadolint --ignore DL3008 --ignore DL3013 -t warning - - - name: shfmt check - if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip shfmt Check')}} - run: find . -type f -name '*.sh' -print0 | xargs -0 shfmt -d -s -i 4 -ci - - - name: cpplint check - if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip cpplint Check')}} - run: cpplint --linelength=120 --filter=-build,-runtime,-readability/todo,-whitespace/todo,-readability/casting,-readability/braces,-readability/fn_size,-legal/copyright --recursive . - - - name: pylint check - if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip pylint Check')}} - run: pylint -d C0103,E0401,R0801,R0902,R0913,R0914,R0915,W0603,W0621,W0718 --recursive=y . + style-check: + runs-on: ubuntu-latest + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip style check')}} + + steps: + - uses: actions/checkout@v3 + + - name: Install tools + run: | + sudo apt-get update && sudo apt-get install -y --no-install-recommends cpplint shfmt + sudo wget https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 -O /bin/hadolint && sudo chmod +x /bin/hadolint + pip install pylint + + - name: hadolint check + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip hadolint Check')}} + run: find . -type f -name '*Dockerfile*' -print0 | xargs -0 hadolint --ignore DL3008 --ignore DL3013 -t warning + + - name: shfmt check + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip shfmt Check')}} + run: find . -type f -name '*.sh' -print0 | xargs -0 shfmt -d -s -i 4 -ci + + - name: cpplint check + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip cpplint Check')}} + run: cpplint --linelength=120 --filter=-build,-runtime,-readability/todo,-whitespace/todo,-readability/casting,-readability/braces,-readability/fn_size,-legal/copyright --recursive . + + - name: pylint check + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip pylint Check')}} + run: pylint -d C0103,E0401,R0801,R0902,R0913,R0914,R0915,W0603,W0621,W0718 --recursive=y . diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index 331c6df56..3d22b9c47 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -63,6 +63,7 @@ std::mutex aggregateWeightMutex; std::mutex triangleTreeMutex; std::string stream_topic_name; +static std::string getPartitionCount(std::string path); static void list_command(int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p); static void add_rdf_command(std::string masterIP, int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p); static void add_graph_command(std::string masterIP, int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p); @@ -563,6 +564,11 @@ bool JasmineGraphFrontEnd::modelExistsByID(string id, SQLiteDBInterface *sqlite) return result; } +static std::string getPartitionCount(std::string path) { + // TODO: Implement + return ""; +} + static void list_command(int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p) { std::stringstream ss; std::vector>> v = @@ -750,9 +756,7 @@ static void add_graph_command(std::string masterIP, int connFd, SQLiteDBInterfac name = strArr[0]; path = strArr[1]; - if (strArr.size() == 3) { - partitionCount = strArr[2]; - } + partitionCount = getPartitionCount(path); if (JasmineGraphFrontEnd::graphExists(path, sqlite)) { frontend_logger.error("Graph exists"); diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 3e4df7e60..835d0cc67 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -13,9 +13,10 @@ limitations under the License. #include "K8sWorkerController.h" +#include + #include #include -#include #include "../util/Conts.h" #include "../util/logger/Logger.h" @@ -58,6 +59,26 @@ K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorke K8sWorkerController::~K8sWorkerController() { delete this->interface; } +static K8sWorkerController *instance = nullptr; + +K8sWorkerController *K8sWorkerController::getInstance() { + if (instance == nullptr) { + throw std::runtime_error("K8sWorkerController is not instantiated"); + } + return instance; +} + +K8sWorkerController *K8sWorkerController::getInstance(std::string masterIp, int numberOfWorkers, + SQLiteDBInterface *metadb) { + // TODO(thevindu-w): synchronize + if (instance == nullptr) { + instance = new K8sWorkerController(masterIp, numberOfWorkers, metadb); + } else { + controller_logger.warn("Not initializing again"); + } + return instance; +} + void K8sWorkerController::spawnWorker(int workerId) { // TODO (M-I-M-Ishad): Develop criteria to assign the worker to node based on performance metrics // getting a node randomly @@ -84,11 +105,10 @@ void K8sWorkerController::spawnWorker(int workerId) { std::string insertQuery = "INSERT INTO worker (host_idhost, server_port, server_data_port, name, ip, idworker) " - "VALUES (" + std::to_string(hostId) + ", " + - std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + - std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + - "'" + std::string(service->metadata->name) + "', " + "'" + std::string(service->spec->cluster_ip) + - "', " + std::to_string(workerId) + ")"; + "VALUES (" + + std::to_string(hostId) + ", " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + + std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + "'" + std::string(service->metadata->name) + + "', " + "'" + std::string(service->spec->cluster_ip) + "', " + std::to_string(workerId) + ")"; int status = metadb.runInsert(insertQuery); if (status == -1) { controller_logger.error("Worker " + std::to_string(workerId) + " database insertion failed"); @@ -158,12 +178,11 @@ int K8sWorkerController::attachExistingWorkers() { std::string insertQuery = "INSERT INTO worker (host_idhost, server_port, server_data_port, name, ip, idworker) " - "VALUES ( " + std::to_string(hostId) + ", " + - std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + - std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + "'" + - std::string(service->metadata->name) + "', " + "'" + std::string(service->spec->cluster_ip) - + - "', " + std::to_string(workerId) + ")"; + "VALUES ( " + + std::to_string(hostId) + ", " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + + std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + "'" + + std::string(service->metadata->name) + "', " + "'" + std::string(service->spec->cluster_ip) + + "', " + std::to_string(workerId) + ")"; int status = metadb.runInsert(insertQuery); if (status == -1) { controller_logger.error("Worker " + std::to_string(workerId) + " database insertion failed"); diff --git a/src/k8s/K8sWorkerController.h b/src/k8s/K8sWorkerController.h index 4b04e31ae..ffd694d8c 100644 --- a/src/k8s/K8sWorkerController.h +++ b/src/k8s/K8sWorkerController.h @@ -22,7 +22,7 @@ extern "C" { #include "./K8sInterface.h" class K8sWorkerController { - private: + private: K8sInterface *interface; SQLiteDBInterface metadb; @@ -31,6 +31,8 @@ class K8sWorkerController { std::map nodes; + K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb); + void spawnWorker(int workerId); void deleteWorker(int workerId); @@ -38,10 +40,11 @@ class K8sWorkerController { int attachExistingWorkers(); public: - K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb); - ~K8sWorkerController(); + static K8sWorkerController *getInstance(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb); + static K8sWorkerController *getInstance(); + std::string getMasterIp() const; int getNumberOfWorkers() const; diff --git a/src/partitioner/local/MetisPartitioner.cpp b/src/partitioner/local/MetisPartitioner.cpp index 4bf2b4139..9517fd815 100644 --- a/src/partitioner/local/MetisPartitioner.cpp +++ b/src/partitioner/local/MetisPartitioner.cpp @@ -361,29 +361,20 @@ void MetisPartitioner::createPartitionFiles(std::map partMap) { std::thread *threads = new std::thread[threadCount]; count = 0; for (int part = 0; part < nParts; part++) { - // threads[count] = std::thread(&MetisPartitioner::writePartitionFiles, this, part); - threads[count] = std::thread(&MetisPartitioner::writeSerializedPartitionFiles, this, part); - count++; - // threads[count] = std::thread(&MetisPartitioner::writeMasterFiles, this, part); - threads[count] = std::thread(&MetisPartitioner::writeSerializedMasterFiles, this, part); - count++; - threads[count] = std::thread(&MetisPartitioner::writeSerializedDuplicateMasterFiles, this, part); - count++; + threads[count++] = std::thread(&MetisPartitioner::writeSerializedPartitionFiles, this, part); + threads[count++] = std::thread(&MetisPartitioner::writeSerializedMasterFiles, this, part); + threads[count++] = std::thread(&MetisPartitioner::writeSerializedDuplicateMasterFiles, this, part); if (graphAttributeType == Conts::GRAPH_WITH_TEXT_ATTRIBUTES) { - threads[count] = std::thread(&MetisPartitioner::writeTextAttributeFilesForPartitions, this, part); - count++; - threads[count] = std::thread(&MetisPartitioner::writeTextAttributeFilesForMasterParts, this, part); - count++; + threads[count++] = std::thread(&MetisPartitioner::writeTextAttributeFilesForPartitions, this, part); + threads[count++] = std::thread(&MetisPartitioner::writeTextAttributeFilesForMasterParts, this, part); } if (graphType == Conts::GRAPH_TYPE_RDF) { - threads[count] = std::thread(&MetisPartitioner::writeRDFAttributeFilesForPartitions, this, part); - count++; - threads[count] = std::thread(&MetisPartitioner::writeRDFAttributeFilesForMasterParts, this, part); - count++; + threads[count++] = std::thread(&MetisPartitioner::writeRDFAttributeFilesForPartitions, this, part); + threads[count++] = std::thread(&MetisPartitioner::writeRDFAttributeFilesForMasterParts, this, part); } } - for (int tc = 0; tc < threadCount; tc++) { + for (int tc = 0; tc < count; tc++) { threads[tc].join(); } partitioner_logger.log("Writing to files completed", "info"); @@ -708,93 +699,6 @@ void MetisPartitioner::writeSerializedDuplicateMasterFiles(int part) { partitioner_logger.log("Serializing done for duplicate central part " + to_string(part), "info"); } -void MetisPartitioner::writePartitionFiles(int part) { - string outputFilePart = outputFilePath + "/" + std::to_string(this->graphID) + "_" + std::to_string(part); - - std::map> partEdgeMap = partitionedLocalGraphStorageMap[part]; - - if (!partEdgeMap.empty()) { - std::ofstream localFile(outputFilePart); - - if (localFile.is_open()) { - for (auto it = partEdgeMap.begin(); it != partEdgeMap.end(); ++it) { - int vertex = it->first; - std::vector destinationSet = it->second; - - if (!destinationSet.empty()) { - for (std::vector::iterator itr = destinationSet.begin(); itr != destinationSet.end(); ++itr) { - string edge; - - if (graphType == Conts::GRAPH_TYPE_RDF) { - auto entry = edgeMap.find(make_pair(vertex, (*itr))); - long article_id = entry->second; - - edge = std::to_string(vertex) + " " + std::to_string((*itr)) + " " + - std::to_string(article_id); - } else { - edge = std::to_string(vertex) + " " + std::to_string((*itr)); - } - localFile << edge; - localFile << "\n"; - } - } - } - } - localFile.flush(); - localFile.close(); - } - - // Compress part files - Utils::compressFile(outputFilePart); - partFileMutex.lock(); - partitionFileMap[part] = outputFilePart + ".gz"; - partFileMutex.unlock(); -} - -void MetisPartitioner::writeMasterFiles(int part) { - string outputFilePartMaster = - outputFilePath + "/" + std::to_string(this->graphID) + "_centralstore_" + std::to_string(part); - - std::map> partMasterEdgeMap = masterGraphStorageMap[part]; - - if (!partMasterEdgeMap.empty()) { - std::ofstream masterFile(outputFilePartMaster); - - if (masterFile.is_open()) { - for (auto it = partMasterEdgeMap.begin(); it != partMasterEdgeMap.end(); ++it) { - int vertex = it->first; - std::vector destinationSet = it->second; - - if (!destinationSet.empty()) { - for (std::vector::iterator itr = destinationSet.begin(); itr != destinationSet.end(); ++itr) { - string edge; - - if (graphType == Conts::GRAPH_TYPE_RDF) { - auto entry = edgeMap.find(make_pair(vertex, (*itr))); - long article_id = entry->second; - - edge = std::to_string(vertex) + " " + std::to_string((*itr)) + " " + - std::to_string(article_id); - - } else { - edge = std::to_string(vertex) + " " + std::to_string((*itr)); - } - masterFile << edge; - masterFile << "\n"; - } - } - } - } - masterFile.flush(); - masterFile.close(); - } - - Utils::compressFile(outputFilePartMaster); - masterFileMutex.lock(); - centralStoreFileList[part] = outputFilePartMaster + ".gz"; - masterFileMutex.unlock(); -} - void MetisPartitioner::writeTextAttributeFilesForPartitions(int part) { string attributeFilePart = outputFilePath + "/" + std::to_string(this->graphID) + "_attributes_" + std::to_string(part); diff --git a/src/partitioner/local/MetisPartitioner.h b/src/partitioner/local/MetisPartitioner.h index 0a5431643..6b28c6343 100644 --- a/src/partitioner/local/MetisPartitioner.h +++ b/src/partitioner/local/MetisPartitioner.h @@ -100,10 +100,6 @@ class MetisPartitioner { void populatePartMaps(std::map partMap, int part); - void writePartitionFiles(int part); - - void writeMasterFiles(int part); - void writeSerializedMasterFiles(int part); void writeSerializedCompositeMasterFiles(std::string part); diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 3f6a29d93..b3ae60ebc 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -68,7 +68,7 @@ static bool initiateOrgServer(std::string host, int port, int dataPort, std::str static void degreeDistributionCommon(std::string graphID, std::string command); static map hostIDMap; -static std::vector hostWorkerMap; +static std::vector hostWorkerList; static map> hostPortMap; std::map aggregateWeightMap; @@ -135,7 +135,7 @@ int JasmineGraphServer::run(std::string profile, std::string masterIp, int numbe updateOperationalGraphList(); if (profile == Conts::PROFILE_K8S) { - k8sWorkerController = new K8sWorkerController(masterIp, numberofWorkers, sqlite); + this->k8sWorkerController = K8sWorkerController::getInstance(masterIp, numberofWorkers, sqlite); std::string selectQuery = "select ip,server_port,server_data_port from worker"; std::vector>> output = this->sqlite->runSelect(selectQuery); for (std::vector>>::iterator i = output.begin(); i != output.end(); ++i) { @@ -145,10 +145,10 @@ int JasmineGraphServer::run(std::string profile, std::string masterIp, int numbe std::string port = j->second; ++j; std::string dataPort = j->second; - hostWorkerMap.push_back({ip, atoi(port.c_str()), atoi(dataPort.c_str())}); + hostWorkerList.push_back({ip, atoi(port.c_str()), atoi(dataPort.c_str())}); } - } else { + this->k8sWorkerController = NULL; start_workers(); addInstanceDetailsToPerformanceDB(masterHost, masterPortVector, "true"); } @@ -194,7 +194,6 @@ void JasmineGraphServer::start_workers() { this->sqlite->runUpdate("DELETE FROM host"); std::vector::iterator it; - it = hostsList.begin(); std::string hostString = ""; std::string sqlString = "INSERT INTO host (idhost,name,ip,is_public) VALUES "; int counter = 0; @@ -247,8 +246,6 @@ void JasmineGraphServer::start_workers() { this->sqlite->runUpdate("DELETE FROM worker"); int workerIDCounter = 0; - it = hostsList.begin(); - for (it = hostsList.begin(); it < hostsList.end(); it++) { string sqlStatement = "INSERT INTO worker (idworker,host_idhost,name,ip,user,is_public,server_port,server_data_port) VALUES "; @@ -269,7 +266,7 @@ void JasmineGraphServer::start_workers() { while (portCount < numberOfWorkersPerHost) { portVector.push_back(workerPort); dataPortVector.push_back(workerDataPort); - hostWorkerMap.push_back({*it, workerPort, workerDataPort}); + hostWorkerList.push_back({*it, workerPort, workerDataPort}); // FIXME: When there are more than 1 worker in the same host, one workers ports will replace the entries of // other workers port entries in hostPortMap hostPortMap[*it] = make_pair(workerPort, workerDataPort); @@ -288,7 +285,7 @@ void JasmineGraphServer::start_workers() { if (hostListModeNWorkers > 0) { portVector.push_back(workerPort); dataPortVector.push_back(workerDataPort); - hostWorkerMap.push_back({*it, workerPort, workerDataPort}); + hostWorkerList.push_back({*it, workerPort, workerDataPort}); hostPortMap[*it] = make_pair(workerPort, workerDataPort); hostListModeNWorkers--; string is_public = "false"; @@ -730,17 +727,14 @@ void JasmineGraphServer::deleteNonOperationalGraphFragment(int graphID) { void JasmineGraphServer::shutdown_workers() { server_logger.info("Shutting down workers"); - std::vector>::iterator mapIterator; - auto *server = JasmineGraphServer::getInstance(); if (server->profile == Conts::PROFILE_K8S) { server->k8sWorkerController->setNumberOfWorkers(0); return; } - - for (mapIterator = hostWorkerMap.begin(); mapIterator < hostWorkerMap.end(); mapIterator++) { - workers worker = *mapIterator; + for (auto listIterator = hostWorkerList.begin(); listIterator < hostWorkerList.end(); listIterator++) { + worker worker = *listIterator; server_logger.info("Host:" + worker.hostname + " Port:" + to_string(worker.port) + " DPort:" + to_string(worker.dataPort)); @@ -790,64 +784,66 @@ int JasmineGraphServer::shutdown_worker(std::string workerIP, int port) { return 0; } +static std::vector getWorkers(size_t npart) { + // TODO: get the workers with lowest load from hostWorkerList + std::vector workerList(hostWorkerList.begin(), hostWorkerList.begin() + npart); + return workerList; +} + void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, vector> fullFileList, std::string masterIP) { server_logger.info("Uploading the graph locally.."); - std::map partitionFileList = fullFileList[0]; - std::map centralStoreFileList = fullFileList[1]; - std::map centralStoreDuplFileList = fullFileList[2]; - std::map compositeCentralStoreFileList = fullFileList[5]; - std::map attributeFileList; - std::map centralStoreAttributeFileList; + std::map partitionFileMap = fullFileList[0]; + server_logger.info(">>>>>>>>>>>>>>>>> size of partitionFileMap = " + std::to_string(partitionFileMap.size())); + std::map centralStoreFileMap = fullFileList[1]; + std::map centralStoreDuplFileMap = fullFileList[2]; + std::map compositeCentralStoreFileMap = fullFileList[5]; + std::map attributeFileMap; + std::map centralStoreAttributeFileMap; if (masterHost.empty()) { masterHost = Utils::getJasmineGraphProperty("org.jasminegraph.server.host"); } - int total_threads = partitionFileList.size() + centralStoreFileList.size() + centralStoreDuplFileList.size() + - compositeCentralStoreFileList.size(); + int total_threads = partitionFileMap.size() + centralStoreFileMap.size() + centralStoreDuplFileMap.size() + + compositeCentralStoreFileMap.size(); if (graphType == Conts::GRAPH_WITH_ATTRIBUTES) { - attributeFileList = fullFileList[3]; - total_threads += attributeFileList.size(); - centralStoreAttributeFileList = fullFileList[4]; - total_threads += centralStoreAttributeFileList.size(); + attributeFileMap = fullFileList[3]; + total_threads += attributeFileMap.size(); + centralStoreAttributeFileMap = fullFileList[4]; + total_threads += centralStoreAttributeFileMap.size(); } int count = 0; int file_count = 0; std::thread *workerThreads = new std::thread[total_threads]; while (count < total_threads) { - std::vector>::iterator mapIterator; - for (mapIterator = hostWorkerMap.begin(); mapIterator < hostWorkerMap.end(); mapIterator++) { - workers worker = *mapIterator; + auto workerList = getWorkers(partitionFileMap.size()); + for (auto listIterator = workerList.begin(); listIterator < workerList.end(); listIterator++) { + worker worker = *listIterator; if (count == total_threads) { break; } - std::string partitionFileName = partitionFileList[file_count]; - workerThreads[count] = std::thread(batchUploadFile, worker.hostname, worker.port, worker.dataPort, graphID, - partitionFileName, masterHost); - count++; - copyCentralStoreToAggregateLocation(centralStoreFileList[file_count]); - workerThreads[count] = std::thread(batchUploadCentralStore, worker.hostname, worker.port, worker.dataPort, - graphID, centralStoreFileList[file_count], masterHost); - count++; - - if (compositeCentralStoreFileList.find(file_count) != compositeCentralStoreFileList.end()) { - copyCentralStoreToAggregateLocation(compositeCentralStoreFileList[file_count]); - workerThreads[count] = + std::string partitionFileName = partitionFileMap[file_count]; + workerThreads[count++] = std::thread(batchUploadFile, worker.hostname, worker.port, worker.dataPort, + graphID, partitionFileName, masterHost); + copyCentralStoreToAggregateLocation(centralStoreFileMap[file_count]); + workerThreads[count++] = std::thread(batchUploadCentralStore, worker.hostname, worker.port, worker.dataPort, + graphID, centralStoreFileMap[file_count], masterHost); + + if (compositeCentralStoreFileMap.find(file_count) != compositeCentralStoreFileMap.end()) { + copyCentralStoreToAggregateLocation(compositeCentralStoreFileMap[file_count]); + workerThreads[count++] = std::thread(batchUploadCompositeCentralstoreFile, worker.hostname, worker.port, worker.dataPort, - graphID, compositeCentralStoreFileList[file_count], masterHost); - count++; + graphID, compositeCentralStoreFileMap[file_count], masterHost); } - workerThreads[count] = std::thread(batchUploadCentralStore, worker.hostname, worker.port, worker.dataPort, - graphID, centralStoreDuplFileList[file_count], masterHost); - count++; + workerThreads[count++] = std::thread(batchUploadCentralStore, worker.hostname, worker.port, worker.dataPort, + graphID, centralStoreDuplFileMap[file_count], masterHost); if (graphType == Conts::GRAPH_WITH_ATTRIBUTES) { - workerThreads[count] = std::thread(batchUploadAttributeFile, worker.hostname, worker.port, - worker.dataPort, graphID, attributeFileList[file_count], masterHost); - count++; - workerThreads[count] = + workerThreads[count++] = + std::thread(batchUploadAttributeFile, worker.hostname, worker.port, worker.dataPort, graphID, + attributeFileMap[file_count], masterHost); + workerThreads[count++] = std::thread(batchUploadCentralAttributeFile, worker.hostname, worker.port, worker.dataPort, graphID, - centralStoreAttributeFileList[file_count], masterHost); - count++; + centralStoreAttributeFileMap[file_count], masterHost); } assignPartitionToWorker(partitionFileName, graphID, worker.hostname, worker.port, worker.dataPort); file_count++; @@ -1387,7 +1383,7 @@ static bool removePartitionThroughService(string host, int port, string graphID, return true; } -std::vector JasmineGraphServer::getHostWorkerMap() { return hostWorkerMap; } +std::vector JasmineGraphServer::getHostWorkerList() { return hostWorkerList; } void JasmineGraphServer::updateOperationalGraphList() { string hosts = ""; diff --git a/src/server/JasmineGraphServer.h b/src/server/JasmineGraphServer.h index 5931f2b2f..5ca37f41a 100644 --- a/src/server/JasmineGraphServer.h +++ b/src/server/JasmineGraphServer.h @@ -116,7 +116,7 @@ class JasmineGraphServer { std::string masterHost; int numberOfWorkers = -1; - struct workers { + struct worker { std::string hostname; int port; int dataPort; @@ -137,8 +137,8 @@ class JasmineGraphServer { // partiton ID. }; - // return hostWorkerMap - static std::vector getHostWorkerMap(); + // return hostWorkerList + static std::vector getHostWorkerList(); static std::map getWorkerPartitions(string graphID); diff --git a/tests/unit/k8s/K8sWorkerController_test.cpp b/tests/unit/k8s/K8sWorkerController_test.cpp index 35f75be7e..337f93e69 100644 --- a/tests/unit/k8s/K8sWorkerController_test.cpp +++ b/tests/unit/k8s/K8sWorkerController_test.cpp @@ -27,7 +27,7 @@ class K8sWorkerControllerTest : public ::testing::Test { void SetUp() override { metadb = new SQLiteDBInterface(TEST_RESOURCE_DIR "temp/jasminegraph_meta.db"); metadb->init(); - controller = new K8sWorkerController("10.43.0.1", 2, metadb); + controller = K8sWorkerController::getInstance("10.43.0.1", 2, metadb); interface = new K8sInterface(); } From 6c97b1a30bd6abad4212f7c3acdc5c3ef47c2ead Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sun, 4 Feb 2024 02:52:05 +0530 Subject: [PATCH 003/143] Support having multiple partitions on same worker --- src/frontend/JasmineGraphFrontEnd.cpp | 1 - .../core/executor/impl/PageRankExecutor.cpp | 292 ++++++------- .../executor/impl/TriangleCountExecutor.cpp | 59 +-- .../executor/impl/TriangleCountExecutor.h | 6 +- .../trainer/JasmineGraphTrainingSchedular.cpp | 2 +- src/server/JasmineGraphServer.cpp | 398 ++++++++---------- src/server/JasmineGraphServer.h | 6 +- 7 files changed, 345 insertions(+), 419 deletions(-) diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index 3d22b9c47..cf6fb116a 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -1175,7 +1175,6 @@ static void add_stream_kafka_command(int connFd, std::string &kafka_server_IP, c // reading kafka_server IP from the given file. std::vector::iterator it; vector vec = Utils::getFileContent(file_path_s); - it = vec.begin(); for (it = vec.begin(); it < vec.end(); it++) { std::string item = *it; if (item.length() > 0 && !(item.rfind("#", 0) == 0)) { diff --git a/src/frontend/core/executor/impl/PageRankExecutor.cpp b/src/frontend/core/executor/impl/PageRankExecutor.cpp index d4d406709..cc300ab5d 100644 --- a/src/frontend/core/executor/impl/PageRankExecutor.cpp +++ b/src/frontend/core/executor/impl/PageRankExecutor.cpp @@ -20,8 +20,7 @@ Logger pageRank_logger; PageRankExecutor::PageRankExecutor() {} -PageRankExecutor::PageRankExecutor(SQLiteDBInterface *db, PerformanceSQLiteDBInterface *perfDb, - JobRequest jobRequest) { +PageRankExecutor::PageRankExecutor(SQLiteDBInterface *db, PerformanceSQLiteDBInterface *perfDb, JobRequest jobRequest) { this->sqlite = db; this->perfDB = perfDb; this->request = jobRequest; @@ -72,8 +71,8 @@ void PageRankExecutor::execute() { processStatusMutex.unlock(); } - pageRank_logger.log( - "###PAGERANK-EXECUTOR### Started with graph ID : " + graphId + " Master IP : " + masterIP, "info"); + pageRank_logger.log("###PAGERANK-EXECUTOR### Started with graph ID : " + graphId + " Master IP : " + masterIP, + "info"); int partitionCount = 0; std::vector> intermRes; @@ -81,52 +80,55 @@ void PageRankExecutor::execute() { auto begin = chrono::high_resolution_clock::now(); - std::map graphPartitionedHosts = - JasmineGraphServer::getWorkerPartitions(graphId); - string partition; + std::map graphPartitionedHosts = + JasmineGraphServer::getGraphPartitionedHosts(graphId); + std::vector partitions; string host; int port; int dataPort; std::string workerList; - std::map::iterator workerIter; + std::map::iterator workerIter; for (workerIter = graphPartitionedHosts.begin(); workerIter != graphPartitionedHosts.end(); workerIter++) { - JasmineGraphServer::workerPartition workerPartition = workerIter->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; + JasmineGraphServer::workerPartitions workerPartition = workerIter->second; + partitions = workerPartition.partitionID; + host = workerIter->first; port = workerPartition.port; dataPort = workerPartition.dataPort; - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; + for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); + partitionit++) { + std::string partition = *partitionit; + workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } - - workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } workerList.pop_back(); pageRank_logger.info("Worker list " + workerList); for (workerIter = graphPartitionedHosts.begin(); workerIter != graphPartitionedHosts.end(); workerIter++) { - JasmineGraphServer::workerPartition workerPartition = workerIter->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; + JasmineGraphServer::workerPartitions workerPartition = workerIter->second; + partitions = workerPartition.partitionID; + host = workerIter->first; port = workerPartition.port; dataPort = workerPartition.dataPort; - intermRes.push_back(std::async( - std::launch::async, PageRankExecutor::doPageRank, graphId, alpha, - iterations, partition, host, port, dataPort, workerList)); + for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); + partitionit++) { + std::string partition = *partitionit; + intermRes.push_back(std::async(std::launch::async, PageRankExecutor::doPageRank, graphId, alpha, iterations, + partition, host, port, dataPort, workerList)); + } } PerformanceUtil::init(); std::string query = - "SELECT attempt from graph_sla INNER JOIN sla_category where graph_sla.id_sla_category=sla_category.id and " - "graph_sla.graph_id='" + - graphId + "' and graph_sla.partition_count='" + std::to_string(partitionCount) + - "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + - "' and sla_category.command='" + PAGE_RANK + "';"; + "SELECT attempt from graph_sla INNER JOIN sla_category where graph_sla.id_sla_category=sla_category.id and " + "graph_sla.graph_id='" + + graphId + "' and graph_sla.partition_count='" + std::to_string(partitionCount) + + "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + "' and sla_category.command='" + PAGE_RANK + + "';"; std::vector>> queryResults = perfDB->runSelect(query); @@ -140,9 +142,8 @@ void PageRankExecutor::execute() { } else { pageRank_logger.log("###PAGERANK-EXECUTOR### Inserting initial record for SLA ", "info"); Utils::updateSLAInformation(perfDB, graphId, partitionCount, 0, PAGE_RANK, Conts::SLA_CATEGORY::LATENCY); - statResponse.push_back(std::async(std::launch::async, collectPerformaceData, perfDB, - graphId.c_str(), PAGE_RANK, Conts::SLA_CATEGORY::LATENCY, partitionCount, - masterIP, autoCalibrate)); + statResponse.push_back(std::async(std::launch::async, collectPerformaceData, perfDB, graphId.c_str(), PAGE_RANK, + Conts::SLA_CATEGORY::LATENCY, partitionCount, masterIP, autoCalibrate)); isStatCollect = true; } @@ -150,8 +151,7 @@ void PageRankExecutor::execute() { futureCall.get(); } - pageRank_logger.info( - "###PAGERANK-EXECUTOR### Getting PageRank : Completed"); + pageRank_logger.info("###PAGERANK-EXECUTOR### Getting PageRank : Completed"); workerResponded = true; JobResponse jobResponse; @@ -193,133 +193,133 @@ int PageRankExecutor::getUid() { return ++uid; } -void PageRankExecutor::doPageRank(std::string graphID, double alpha, int iterations, string partition, - string host, int port, int dataPort, std::string workerList) { - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; - } +void PageRankExecutor::doPageRank(std::string graphID, double alpha, int iterations, string partition, string host, + int port, int dataPort, std::string workerList) { + if (host.find('@') != std::string::npos) { + host = Utils::split(host, '@')[1]; + } - int sockfd; - char data[DATA_BUFFER_SIZE]; - struct sockaddr_in serv_addr; - struct hostent *server; + int sockfd; + char data[DATA_BUFFER_SIZE]; + struct sockaddr_in serv_addr; + struct hostent *server; - sockfd = socket(AF_INET, SOCK_STREAM, 0); + sockfd = socket(AF_INET, SOCK_STREAM, 0); - if (sockfd < 0) { - pageRank_logger.error("Cannot create socket"); - return; - } - server = gethostbyname(host.c_str()); - if (server == NULL) { - pageRank_logger.error("ERROR, no host named " + host); - return; - } + if (sockfd < 0) { + pageRank_logger.error("Cannot create socket"); + return; + } + server = gethostbyname(host.c_str()); + if (server == NULL) { + pageRank_logger.error("ERROR, no host named " + host); + return; + } - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - pageRank_logger.error("Error connecting to socket"); - return; - } + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(port); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + pageRank_logger.error("Error connecting to socket"); + return; + } - if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::PAGE_RANK)) { - pageRank_logger.error("Error writing to socket"); - return; - } - pageRank_logger.info("Sent : " + JasmineGraphInstanceProtocol::PAGE_RANK); - - string response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::PAGE_RANK)) { + pageRank_logger.error("Error writing to socket"); + return; + } + pageRank_logger.info("Sent : " + JasmineGraphInstanceProtocol::PAGE_RANK); - if (!Utils::send_str_wrapper(sockfd, graphID)) { - pageRank_logger.error("Error writing to socket"); - return; - } - pageRank_logger.info("Sent : Graph ID " + graphID); - - response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + string response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } - if (!Utils::send_str_wrapper(sockfd, partition)) { - pageRank_logger.error("Error writing to socket"); - return; - } - pageRank_logger.info("Sent : Partition ID " + partition); - - response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + if (!Utils::send_str_wrapper(sockfd, graphID)) { + pageRank_logger.error("Error writing to socket"); + return; + } + pageRank_logger.info("Sent : Graph ID " + graphID); - if (!Utils::send_str_wrapper(sockfd, workerList)) { - pageRank_logger.error("Error writing to socket"); - } - pageRank_logger.info("Sent : Host List "); - - response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } - long graphVertexCount = JasmineGraphServer::getGraphVertexCount(graphID); - if (!Utils::send_str_wrapper(sockfd, std::to_string(graphVertexCount))) { - pageRank_logger.error("Error writing to socket"); - } - pageRank_logger.info("graph vertex count: " + std::to_string(graphVertexCount)); - - response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + if (!Utils::send_str_wrapper(sockfd, partition)) { + pageRank_logger.error("Error writing to socket"); + return; + } + pageRank_logger.info("Sent : Partition ID " + partition); - if (!Utils::send_str_wrapper(sockfd, std::to_string(alpha))) { - pageRank_logger.error("Error writing to socket"); - return; - } - pageRank_logger.info("PageRank alpha value sent : " + std::to_string(alpha)); - - response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } - if (!Utils::send_str_wrapper(sockfd, std::to_string(iterations))) { - pageRank_logger.error("Error writing to socket"); - return; - } + if (!Utils::send_str_wrapper(sockfd, workerList)) { + pageRank_logger.error("Error writing to socket"); + } + pageRank_logger.info("Sent : Host List "); - response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } + + long graphVertexCount = JasmineGraphServer::getGraphVertexCount(graphID); + if (!Utils::send_str_wrapper(sockfd, std::to_string(graphVertexCount))) { + pageRank_logger.error("Error writing to socket"); + } + pageRank_logger.info("graph vertex count: " + std::to_string(graphVertexCount)); + + response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } + + if (!Utils::send_str_wrapper(sockfd, std::to_string(alpha))) { + pageRank_logger.error("Error writing to socket"); + return; + } + pageRank_logger.info("PageRank alpha value sent : " + std::to_string(alpha)); + + response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } + + if (!Utils::send_str_wrapper(sockfd, std::to_string(iterations))) { + pageRank_logger.error("Error writing to socket"); + return; + } + + response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } return; } diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 7a781af52..b963c8a59 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -620,7 +620,6 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq std::vector> remoteGraphCopyResponse; int minimumWeight = 0; std::string minWeightWorker; - string aggregatorHost = ""; std::string partitionIdList = ""; std::vector::iterator workerCombinationIterator; @@ -646,7 +645,7 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq } string aggregatorSqlStatement = - "SELECT ip,user,server_port,server_data_port,partition_idpartition " + "SELECT ip,server_port,server_data_port,partition_idpartition " "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + graphId + " and idworker=" + minWeightWorker + ";"; @@ -656,25 +655,17 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq vector> aggregatorData = result.at(0); std::string aggregatorIp = aggregatorData.at(0).second; - std::string aggregatorUser = aggregatorData.at(1).second; - std::string aggregatorPort = aggregatorData.at(2).second; - std::string aggregatorDataPort = aggregatorData.at(3).second; - std::string aggregatorPartitionId = aggregatorData.at(4).second; - - if ((aggregatorIp.find("localhost") != std::string::npos) || aggregatorIp == masterIP) { - aggregatorHost = aggregatorIp; - } else { - aggregatorHost = aggregatorUser + "@" + aggregatorIp; - } + std::string aggregatorPort = aggregatorData.at(1).second; + std::string aggregatorDataPort = aggregatorData.at(2).second; + std::string aggregatorPartitionId = aggregatorData.at(3).second; for (aggregatorCopyCombinationIterator = workerCombination.begin(); aggregatorCopyCombinationIterator != workerCombination.end(); ++aggregatorCopyCombinationIterator) { std::string workerId = *aggregatorCopyCombinationIterator; - string host = ""; if (workerId != minWeightWorker) { string sqlStatement = - "SELECT ip,user,server_port,server_data_port,partition_idpartition " + "SELECT partition_idpartition " "FROM worker_has_partition INNER JOIN worker ON " "worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + @@ -684,28 +675,18 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq vector> workerData = result.at(0); - std::string workerIp = workerData.at(0).second; - std::string workerUser = workerData.at(1).second; - std::string workerPort = workerData.at(2).second; - std::string workerDataPort = workerData.at(3).second; - std::string partitionId = workerData.at(4).second; - - if ((workerIp.find("localhost") != std::string::npos) || workerIp == masterIP) { - host = workerIp; - } else { - host = workerUser + "@" + workerIp; - } + std::string partitionId = workerData.at(0).second; partitionIdList += partitionId + ","; std::string centralStoreAvailable = isFileAccessibleToWorker( - graphId, partitionId, aggregatorHost, aggregatorPort, masterIP, + graphId, partitionId, aggregatorIp, aggregatorPort, masterIP, JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_AGGREGATE, std::string()); if (centralStoreAvailable.compare("false") == 0) { remoteGraphCopyResponse.push_back( std::async(std::launch::async, TriangleCountExecutor::copyCentralStoreToAggregator, - aggregatorHost, aggregatorPort, aggregatorDataPort, atoi(graphId.c_str()), + aggregatorIp, aggregatorPort, aggregatorDataPort, atoi(graphId.c_str()), atoi(partitionId.c_str()), masterIP)); } } @@ -719,8 +700,8 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq workerWeightMap[minWeightWorker] = minimumWeight; triangleCountResponse.push_back(std::async( - std::launch::async, TriangleCountExecutor::countCentralStoreTriangles, aggregatorHost, aggregatorPort, - aggregatorHost, aggregatorPartitionId, adjustedPartitionIdList, graphId, masterIP, threadPriority)); + std::launch::async, TriangleCountExecutor::countCentralStoreTriangles, aggregatorPort, aggregatorIp, + aggregatorPartitionId, adjustedPartitionIdList, graphId, masterIP, threadPriority)); } for (auto &&futureCall : triangleCountResponse) { @@ -763,10 +744,6 @@ string TriangleCountExecutor::isFileAccessibleToWorker(std::string graphId, std: return 0; } - if (aggregatorHostName.find('@') != std::string::npos) { - aggregatorHostName = Utils::split(aggregatorHostName, '@')[1]; - } - server = gethostbyname(aggregatorHostName.c_str()); if (server == NULL) { triangleCount_logger.error("ERROR, no host named " + aggregatorHostName); @@ -1305,10 +1282,6 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr return 0; } - if (aggregatorHostName.find('@') != std::string::npos) { - aggregatorHostName = Utils::split(aggregatorHostName, '@')[1]; - } - server = gethostbyname(aggregatorHostName.c_str()); if (server == NULL) { triangleCount_logger.error("ERROR, no host named " + aggregatorHostName); @@ -1471,10 +1444,10 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr return response; } -string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorHostName, std::string aggregatorPort, - std::string host, std::string partitionId, - std::string partitionIdList, std::string graphId, - std::string masterIP, int threadPriority) { +string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorPort, std::string host, + std::string partitionId, std::string partitionIdList, + std::string graphId, std::string masterIP, + int threadPriority) { int sockfd; char data[301]; bool loop = false; @@ -1489,10 +1462,6 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorH return 0; } - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; - } - server = gethostbyname(host.c_str()); if (server == NULL) { triangleCount_logger.error("ERROR, no host named " + host); diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.h b/src/frontend/core/executor/impl/TriangleCountExecutor.h index e294b779d..072d0775a 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.h +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.h @@ -59,9 +59,9 @@ class TriangleCountExecutor : public AbstractExecutor { std::string aggregatorDataPort, int graphId, int partitionId, std::string masterIP); - static string countCentralStoreTriangles(std::string aggregatorHostName, std::string aggregatorPort, - std::string host, std::string partitionId, std::string partitionIdList, - std::string graphId, std::string masterIP, int threadPriority); + static string countCentralStoreTriangles(std::string aggregatorPort, std::string host, std::string partitionId, + std::string partitionIdList, std::string graphId, std::string masterIP, + int threadPriority); static bool proceedOrNot(std::set partitionSet, int partitionId); diff --git a/src/ml/trainer/JasmineGraphTrainingSchedular.cpp b/src/ml/trainer/JasmineGraphTrainingSchedular.cpp index 9229b573d..fe1073734 100644 --- a/src/ml/trainer/JasmineGraphTrainingSchedular.cpp +++ b/src/ml/trainer/JasmineGraphTrainingSchedular.cpp @@ -69,7 +69,7 @@ map> JasmineGraphTrainingSchedular::schedulePartition int vertexcount; int centralVertexCount; trainScheduler_logger.log("Scheduling training order for each worker", "info"); - for (std::vector>::iterator j = (hostData.begin()); j != hostData.end(); ++j) { + for (std::vector>::iterator j = hostData.begin(); j != hostData.end(); ++j) { sqlStatement = "SELECT idpartition, vertexcount, central_vertexcount, graph_idgraph FROM partition INNER JOIN " "(SELECT host_idhost, partition_idpartition, partition_graph_idgraph, worker_idworker FROM " diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index b3ae60ebc..8ee1d0843 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -52,19 +52,19 @@ static bool batchUploadCompositeCentralstoreFile(std::string host, int port, int static bool removeFragmentThroughService(string host, int port, string graphID, string masterIP); static bool removePartitionThroughService(string host, int port, string graphID, string partitionID, string masterIP); static bool initiateCommon(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP, std::string initType); + std::string masterIP, std::string initType); static bool initiateTrain(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP); + std::string masterIP); static bool initiatePredict(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP); + std::string masterIP); static bool initiateServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP); + std::string masterIP); static bool initiateClient(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP); + std::string masterIP); static bool initiateAggregator(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP); + std::string masterIP); static bool initiateOrgServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP); + std::string masterIP); static void degreeDistributionCommon(std::string graphID, std::string command); static map hostIDMap; @@ -786,6 +786,7 @@ int JasmineGraphServer::shutdown_worker(std::string workerIP, int port) { static std::vector getWorkers(size_t npart) { // TODO: get the workers with lowest load from hostWorkerList + if (npart > hostWorkerList.size()) npart = hostWorkerList.size(); std::vector workerList(hostWorkerList.begin(), hostWorkerList.begin() + npart); return workerList; } @@ -794,7 +795,6 @@ void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, vector> fullFileList, std::string masterIP) { server_logger.info("Uploading the graph locally.."); std::map partitionFileMap = fullFileList[0]; - server_logger.info(">>>>>>>>>>>>>>>>> size of partitionFileMap = " + std::to_string(partitionFileMap.size())); std::map centralStoreFileMap = fullFileList[1]; std::map centralStoreDuplFileMap = fullFileList[2]; std::map compositeCentralStoreFileMap = fullFileList[5]; @@ -1424,38 +1424,6 @@ void JasmineGraphServer::updateOperationalGraphList() { this->sqlite->runUpdate(sqlStatement2); } -std::map JasmineGraphServer::getWorkerPartitions(string graphID) { - vector> hostHasPartition; - auto *refToSqlite = new SQLiteDBInterface(); - refToSqlite->init(); - map graphPartitionedHosts; - vector>> hostPartitionResults = refToSqlite->runSelect( - "SELECT name, worker_idworker, server_port, server_data_port, partition_idpartition FROM worker_has_partition " - "INNER JOIN worker ON worker_" - "idworker = idworker WHERE partition_graph_idgraph = '" + - graphID + "'"); - - refToSqlite->finalize(); - delete refToSqlite; - - for (std::vector>>::iterator i = hostPartitionResults.begin(); - i != hostPartitionResults.end(); ++i) { - std::vector> rowData = *i; - - string name = rowData.at(0).second; - string workerID = rowData.at(1).second; - int serverPort = std::stoi(rowData.at(2).second); - int serverDataPort = std::stoi(rowData.at(3).second); - string partitionId = rowData.at(4).second; - - server_logger.info("name : " + name + " workerID : " + workerID + " sport : " + std::to_string(serverPort) + - " sdport : " + std::to_string(serverDataPort) + " partitionId : " + partitionId); - graphPartitionedHosts[workerID] = {name, serverPort, serverDataPort, partitionId}; - } - - return graphPartitionedHosts; -} - std::map JasmineGraphServer::getGraphPartitionedHosts(string graphID) { vector> hostHasPartition; auto *refToSqlite = new SQLiteDBInterface(); @@ -1495,7 +1463,7 @@ std::map JasmineGraphServer::getGr } map graphPartitionedHosts; - for (map>::iterator it = (hostPartitions.begin()); it != hostPartitions.end(); ++it) { + for (map>::iterator it = hostPartitions.begin(); it != hostPartitions.end(); ++it) { graphPartitionedHosts[it->first] = {hostPortMap[it->first].first, hostPortMap[it->first].second, hostPartitions[it->first]}; } @@ -1606,39 +1574,34 @@ void JasmineGraphServer::addInstanceDetailsToPerformanceDB(std::string host, std } static void degreeDistributionCommon(std::string graphID, std::string command) { - std::map graphPartitionedHosts = - JasmineGraphServer::getWorkerPartitions(graphID); - int partition_count = 0; - string partition; + std::map graphPartitionedHosts = + JasmineGraphServer::getGraphPartitionedHosts(graphID); + std::vector partitions; string host; int port; std::string workerList; - std::map::iterator workerit; + std::map::iterator workerit; for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { - JasmineGraphServer::workerPartition workerPartition = workerit->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; + JasmineGraphServer::workerPartitions workerPartition = workerit->second; + partitions = workerPartition.partitionID; + host = workerit->first; port = workerPartition.port; - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; + for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); + partitionit++) { + std::string partition = *partitionit; + workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } - - workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } workerList.pop_back(); for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { - JasmineGraphServer::workerPartition workerPartition = workerit->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; + JasmineGraphServer::workerPartitions workerPartition = workerit->second; + partitions = workerPartition.partitionID; + host = workerit->first; port = workerPartition.port; - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; - } - int sockfd; char data[FED_DATA_LENGTH + 1]; bool loop = false; @@ -1658,35 +1621,42 @@ static void degreeDistributionCommon(std::string graphID, std::string command) { continue; } - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - continue; - } + for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); + partitionit++) { + std::string partition = *partitionit; + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(port); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + continue; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, command, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - continue; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, command, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + continue; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - continue; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + continue; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, - JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - continue; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + continue; + } - if (!Utils::send_str_wrapper(sockfd, workerList)) { + if (!Utils::send_str_wrapper(sockfd, workerList)) { + close(sockfd); + continue; + } close(sockfd); - continue; + server_logger.info("Sent: " + workerList); } - server_logger.info("Sent: " + workerList); } } @@ -1711,34 +1681,33 @@ long JasmineGraphServer::getGraphVertexCount(std::string graphID) { } void JasmineGraphServer::duplicateCentralStore(std::string graphID) { - std::map graphPartitionedHosts = - JasmineGraphServer::getWorkerPartitions(graphID); - int partition_count = 0; - string partition; + std::map graphPartitionedHosts = + JasmineGraphServer::getGraphPartitionedHosts(graphID); + std::vector partitions; string host; int port; int dport; std::string workerList; - std::map::iterator workerit; + std::map::iterator workerit; for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { - JasmineGraphServer::workerPartition workerPartition = workerit->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; + JasmineGraphServer::workerPartitions workerPartition = workerit->second; + partitions = workerPartition.partitionID; + host = workerit->first; port = workerPartition.port; dport = workerPartition.dataPort; - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; + for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); + partitionit++) { + std::string partition = *partitionit; + workerList.append(host + ":" + std::to_string(port) + ":" + partition + ":" + to_string(dport) + ","); } - - workerList.append(host + ":" + std::to_string(port) + ":" + partition + ":" + to_string(dport) + ","); } workerList.pop_back(); for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { - JasmineGraphServer::workerPartition workerPartition = workerit->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; + JasmineGraphServer::workerPartitions workerPartition = workerit->second; + partitions = workerPartition.partitionID; + host = workerit->first; port = workerPartition.port; if (host.find('@') != std::string::npos) { @@ -1764,76 +1733,73 @@ void JasmineGraphServer::duplicateCentralStore(std::string graphID) { continue; } - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - continue; - } + for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); + partitionit++) { + std::string partition = *partitionit; + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(port); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + continue; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, - JasmineGraphInstanceProtocol::DP_CENTRALSTORE, - JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - continue; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, + JasmineGraphInstanceProtocol::DP_CENTRALSTORE, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + continue; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - continue; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + continue; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, - JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - continue; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + continue; + } - if (!Utils::send_str_wrapper(sockfd, workerList)) { + if (!Utils::send_str_wrapper(sockfd, workerList)) { + close(sockfd); + continue; + } + server_logger.info("Sent: " + workerList); close(sockfd); - continue; } - server_logger.info("Sent: " + workerList); - close(sockfd); } } void JasmineGraphServer::initiateFiles(std::string graphID, std::string trainingArgs) { int count = 0; map> scheduleForAllHosts = JasmineGraphTrainingSchedular::schedulePartitionTraining(graphID); - std::map graphPartitionedHosts = - this->getWorkerPartitions(graphID); + std::map graphPartitionedHosts = + this->getGraphPartitionedHosts(graphID); int partition_count = 0; - std::map::iterator mapIterator; + std::map::iterator mapIterator; for (mapIterator = graphPartitionedHosts.begin(); mapIterator != graphPartitionedHosts.end(); mapIterator++) { - JasmineGraphServer::workerPartition workerPartition = mapIterator->second; - std::vector partitions; - partitions.push_back(workerPartition.partitionID); - std::vector::iterator it; - for (it = partitions.begin(); it < partitions.end(); it++) { - partition_count++; - } + JasmineGraphServer::workerPartitions workerPartition = mapIterator->second; + partition_count += (int)(workerPartition.partitionID.size()); } std::thread workerThreads[partition_count + 1]; string prefix = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.trainedmodelfolder"); string attr_prefix = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); - std::map::iterator j; + std::map::iterator j; for (j = graphPartitionedHosts.begin(); j != graphPartitionedHosts.end(); j++) { - JasmineGraphServer::workerPartition workerPartition = j->second; - std::vector partitions; - partitions.push_back(workerPartition.partitionID); - string partitionCount = std::to_string(partitions.size()); + JasmineGraphServer::workerPartitions workerPartition = j->second; + std::vector partitions = workerPartition.partitionID; std::vector::iterator k; map scheduleOfHost = scheduleForAllHosts[j->first]; for (k = partitions.begin(); k != partitions.end(); k++) { int iterationOfPart = scheduleOfHost[stoi(*k)]; - workerThreads[count] = - std::thread(initiateTrain, workerPartition.hostname, workerPartition.port, workerPartition.dataPort, - trainingArgs + " " + *k, iterationOfPart, partitionCount, this->masterHost); - count++; + workerThreads[count++] = + std::thread(initiateTrain, j->first, workerPartition.port, workerPartition.dataPort, + trainingArgs + " " + *k, iterationOfPart, this->masterHost); } } @@ -1862,12 +1828,12 @@ void JasmineGraphServer::initiateCommunication(std::string graphID, std::string if (i == 0) { workerThreads[threadID] = std::thread(initiateServer, workerInstance.hostname, serverPort, serverDataPort, - trainingArgs, fl_clients, to_string(i), masterIP); + trainingArgs, fl_clients, masterIP); threadID++; } workerThreads[threadID] = std::thread(initiateClient, workerInstance.hostname, serverPort, serverDataPort, - trainingArgs + " " + to_string(i), fl_clients, to_string(i), masterIP); + trainingArgs + " " + to_string(i), fl_clients, masterIP); threadID++; } @@ -1892,7 +1858,7 @@ void JasmineGraphServer::initiateOrgCommunication(std::string graphID, std::stri if (Utils::getJasmineGraphProperty("org.jasminegraph.fl.aggregator") == "true") { initiateAggregator("localhost", stoi(workerVector[0].port), stoi(workerVector[0].dataPort), trainingArgs, - fl_clients, "1", masterIP); + fl_clients, masterIP); } std::ifstream file(Utils::getJasmineGraphProperty("org.jasminegraph.fl.organization.file")); @@ -1933,12 +1899,12 @@ void JasmineGraphServer::initiateOrgCommunication(std::string graphID, std::stri if (i == 0) { workerThreads[threadID] = std::thread(initiateOrgServer, workerInstance.hostname, serverPort, - serverDataPort, trainingArgs, fl_clients, to_string(i), masterIP); + serverDataPort, trainingArgs, fl_clients, masterIP); threadID++; } workerThreads[threadID] = std::thread(initiateClient, workerInstance.hostname, serverPort, serverDataPort, - trainingArgs + " " + to_string(i), fl_clients, to_string(i), masterIP); + trainingArgs + " " + to_string(i), fl_clients, masterIP); threadID++; } @@ -1954,38 +1920,30 @@ void JasmineGraphServer::initiateOrgCommunication(std::string graphID, std::stri } void JasmineGraphServer::initiateMerge(std::string graphID, std::string trainingArgs, SQLiteDBInterface *sqlite) { - int count = 0; map> scheduleForAllHosts = JasmineGraphTrainingSchedular::schedulePartitionTraining(graphID); - std::map graphPartitionedHosts = - this->getWorkerPartitions(graphID); + std::map graphPartitionedHosts = + this->getGraphPartitionedHosts(graphID); int partition_count = 0; - std::map::iterator mapIterator; + std::map::iterator mapIterator; for (mapIterator = graphPartitionedHosts.begin(); mapIterator != graphPartitionedHosts.end(); mapIterator++) { - JasmineGraphServer::workerPartition workerPartition = mapIterator->second; - std::vector partitions; - partitions.push_back(workerPartition.partitionID); - std::vector::iterator it; - for (it = partitions.begin(); it < partitions.end(); it++) { - partition_count++; - } + JasmineGraphServer::workerPartitions workerPartition = mapIterator->second; + partition_count += (int)(workerPartition.partitionID.size()); } - std::thread *workerThreads = new std::thread[partition_count + 1]; + std::thread *workerThreads = new std::thread[partition_count]; int fl_clients = stoi(Utils::getJasmineGraphProperty("org.jasminegraph.fl_clients")); - std::map::iterator j; + std::map::iterator j; + int count = 0; for (j = graphPartitionedHosts.begin(); j != graphPartitionedHosts.end(); j++) { - JasmineGraphServer::workerPartition workerPartition = j->second; - std::vector partitions; - partitions.push_back(workerPartition.partitionID); - string partitionCount = std::to_string(partitions.size()); + JasmineGraphServer::workerPartitions workerPartition = j->second; + std::vector partitions = workerPartition.partitionID; std::vector::iterator k; map scheduleOfHost = scheduleForAllHosts[j->first]; for (k = partitions.begin(); k != partitions.end(); k++) { int iterationOfPart = scheduleOfHost[stoi(*k)]; - workerThreads[count] = - std::thread(mergeFiles, workerPartition.hostname, workerPartition.port, workerPartition.dataPort, - trainingArgs + " " + *k, fl_clients, *k, this->masterHost); + workerThreads[count] = std::thread(mergeFiles, j->first, workerPartition.port, workerPartition.dataPort, + trainingArgs + " " + *k, fl_clients, *k, this->masterHost); count++; } } @@ -1998,7 +1956,7 @@ void JasmineGraphServer::initiateMerge(std::string graphID, std::string training } static bool initiateCommon(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP, std::string initType) { + std::string masterIP, std::string initType) { bool result = true; int sockfd; char data[FED_DATA_LENGTH + 1]; @@ -2053,39 +2011,39 @@ static bool initiateCommon(std::string host, int port, int dataPort, std::string } static bool initiateTrain(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, partCount, masterIP, + std::string masterIP) { + return initiateCommon(host, port, dataPort, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_FED_PREDICT); } static bool initiatePredict(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, partCount, masterIP, + std::string masterIP) { + return initiateCommon(host, port, dataPort, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_FILES); } static bool initiateServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, partCount, masterIP, + std::string masterIP) { + return initiateCommon(host, port, dataPort, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_SERVER); } // todo Remove partCount from parameters as the partition id is being parsed within the training args static bool initiateClient(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, partCount, masterIP, + std::string masterIP) { + return initiateCommon(host, port, dataPort, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_CLIENT); } static bool initiateAggregator(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, partCount, masterIP, + std::string masterIP) { + return initiateCommon(host, port, dataPort, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_AGG); } static bool initiateOrgServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, partCount, masterIP, + std::string masterIP) { + return initiateCommon(host, port, dataPort, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_ORG_SERVER); } @@ -2297,28 +2255,26 @@ bool JasmineGraphServer::sendTrainCommand(std::string host, int port, std::strin } void JasmineGraphServer::egoNet(std::string graphID) { - std::map graphPartitionedHosts = - JasmineGraphServer::getWorkerPartitions(graphID); - int partition_count = 0; - string partition; + std::map graphPartitionedHosts = + JasmineGraphServer::getGraphPartitionedHosts(graphID); + std::vector partitions; string host; int port; int dataPort; std::string workerList; - std::map::iterator workerit; - for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { - JasmineGraphServer::workerPartition workerPartition = workerit->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; + for (auto workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { + JasmineGraphServer::workerPartitions workerPartition = workerit->second; + partitions = workerPartition.partitionID; + host = workerit->first; port = workerPartition.port; dataPort = workerPartition.dataPort; - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; + for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); + partitionit++) { + std::string partition = *partitionit; + workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } - - workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } workerList.pop_back(); @@ -2342,34 +2298,40 @@ void JasmineGraphServer::egoNet(std::string graphID) { return; } - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - return; - } + for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); + partitionit++) { + std::string partition = *partitionit; + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(port); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + return; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::EGONET, - JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - return; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::EGONET, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + return; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - return; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + return; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - return; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + return; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, workerList, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + return; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, workerList, JasmineGraphInstanceProtocol::OK)) { close(sockfd); - return; } - - close(sockfd); } diff --git a/src/server/JasmineGraphServer.h b/src/server/JasmineGraphServer.h index 5ca37f41a..4ae66c8e8 100644 --- a/src/server/JasmineGraphServer.h +++ b/src/server/JasmineGraphServer.h @@ -140,9 +140,7 @@ class JasmineGraphServer { // return hostWorkerList static std::vector getHostWorkerList(); - static std::map getWorkerPartitions(string graphID); - - std::map getGraphPartitionedHosts(std::string graphID); + static std::map getGraphPartitionedHosts(std::string graphID); static void inDegreeDistribution(std::string graphID); @@ -150,8 +148,6 @@ class JasmineGraphServer { static void duplicateCentralStore(std::string graphID); - static void pageRank(std::string graphID, double alpha, int iterations); - static long getGraphVertexCount(std::string graphID); static void egoNet(std::string graphID); From cd3eefc1aea5961cf4b960ef6873138abb5a19cb Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 7 Feb 2024 23:43:10 +0530 Subject: [PATCH 004/143] Keep a list of workers in K8sWorkerController --- main.cpp | 3 ++- src/k8s/K8sWorkerController.cpp | 17 ++++++++++++++--- src/k8s/K8sWorkerController.h | 9 ++++++++- src/server/JasmineGraphServer.cpp | 1 + src/server/JasmineGraphServer.h | 4 ++-- 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/main.cpp b/main.cpp index a966ccc92..ed3a6743f 100644 --- a/main.cpp +++ b/main.cpp @@ -18,6 +18,7 @@ limitations under the License. #include #include +#include "src/k8s/K8sWorkerController.h" #include "src/server/JasmineGraphInstance.h" #include "src/util/logger/Logger.h" #include "src/util/scheduler/SchedulerService.h" @@ -62,7 +63,7 @@ int main(int argc, char *argv[]) { thread schedulerThread(SchedulerService::startScheduler); if (profile == Conts::PROFILE_K8S) { - K8sInterface* interface = new K8sInterface(); + K8sInterface *interface = new K8sInterface(); masterIp = interface->getMasterIp(); if (masterIp.empty()) { masterIp = interface->createJasmineGraphMasterService()->spec->cluster_ip; diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 835d0cc67..a54f67fe3 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -24,6 +24,8 @@ limitations under the License. Logger controller_logger; +std::vector K8sWorkerController::workerList = {}; + K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb) { this->masterIp = std::move(masterIp); this->numberOfWorkers = numberOfWorkers; @@ -103,12 +105,16 @@ void K8sWorkerController::spawnWorker(int workerId) { throw std::runtime_error("Worker " + std::to_string(workerId) + " service creation failed"); } + std::string ip(service->spec->cluster_ip); + JasmineGraphServer::worker worker = { + .hostname = ip, .port = Conts::JASMINEGRAPH_INSTANCE_PORT, .dataPort = Conts::JASMINEGRAPH_INSTANCE_DATA_PORT}; + K8sWorkerController::workerList.push_back(worker); std::string insertQuery = "INSERT INTO worker (host_idhost, server_port, server_data_port, name, ip, idworker) " "VALUES (" + std::to_string(hostId) + ", " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + "'" + std::string(service->metadata->name) + - "', " + "'" + std::string(service->spec->cluster_ip) + "', " + std::to_string(workerId) + ")"; + "', " + "'" + ip + "', " + std::to_string(workerId) + ")"; int status = metadb.runInsert(insertQuery); if (status == -1) { controller_logger.error("Worker " + std::to_string(workerId) + " database insertion failed"); @@ -176,13 +182,18 @@ int K8sWorkerController::attachExistingWorkers() { service = static_cast(service_list->items->firstEntry->data); } + std::string ip(service->spec->cluster_ip); + JasmineGraphServer::worker worker = {.hostname = ip, + .port = Conts::JASMINEGRAPH_INSTANCE_PORT, + .dataPort = Conts::JASMINEGRAPH_INSTANCE_DATA_PORT}; + K8sWorkerController::workerList.push_back(worker); std::string insertQuery = "INSERT INTO worker (host_idhost, server_port, server_data_port, name, ip, idworker) " "VALUES ( " + std::to_string(hostId) + ", " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + "'" + - std::string(service->metadata->name) + "', " + "'" + std::string(service->spec->cluster_ip) + - "', " + std::to_string(workerId) + ")"; + std::string(service->metadata->name) + "', " + "'" + ip + "', " + std::to_string(workerId) + + ")"; int status = metadb.runInsert(insertQuery); if (status == -1) { controller_logger.error("Worker " + std::to_string(workerId) + " database insertion failed"); diff --git a/src/k8s/K8sWorkerController.h b/src/k8s/K8sWorkerController.h index ffd694d8c..1bc78dc51 100644 --- a/src/k8s/K8sWorkerController.h +++ b/src/k8s/K8sWorkerController.h @@ -11,18 +11,23 @@ See the License for the specific language governing permissions and limitations under the License. */ +class K8sWorkerController; + #ifndef JASMINEGRAPH_K8SWORKERCONTROLLER_H #define JASMINEGRAPH_K8SWORKERCONTROLLER_H +#include + extern "C" { #include } #include "../metadb/SQLiteDBInterface.h" +#include "../server/JasmineGraphServer.h" #include "./K8sInterface.h" class K8sWorkerController { - private: + private: K8sInterface *interface; SQLiteDBInterface metadb; @@ -40,6 +45,8 @@ class K8sWorkerController { int attachExistingWorkers(); public: + static std::vector workerList; + ~K8sWorkerController(); static K8sWorkerController *getInstance(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb); diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 8ee1d0843..117c58b16 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -21,6 +21,7 @@ limitations under the License. #include #include +#include "../k8s/K8sWorkerController.h" #include "../ml/trainer/JasmineGraphTrainingSchedular.h" #include "../partitioner/local/MetisPartitioner.h" #include "../util/Utils.h" diff --git a/src/server/JasmineGraphServer.h b/src/server/JasmineGraphServer.h index 4ae66c8e8..74dbc8ac0 100644 --- a/src/server/JasmineGraphServer.h +++ b/src/server/JasmineGraphServer.h @@ -25,15 +25,15 @@ limitations under the License. #include "../backend/JasmineGraphBackend.h" #include "../frontend/JasmineGraphFrontEnd.h" #include "../frontend/core/scheduler/JobScheduler.h" -#include "../k8s/K8sWorkerController.h" #include "../metadb/SQLiteDBInterface.h" #include "../performance/metrics/StatisticCollector.h" #include "../performancedb/PerformanceSQLiteDBInterface.h" #include "../util/Conts.h" #include "../util/Utils.h" -using std::map; +class K8sWorkerController; +using std::map; class JasmineGraphServer { private: map hostPlaceMap; From 33738205f694a64a331bb065e6395df81622d98e Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 7 Feb 2024 23:43:33 +0530 Subject: [PATCH 005/143] Prevent asking for sudo password in test-k8s.sh --- test-docker.sh | 4 ++-- test-k8s.sh | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/test-docker.sh b/test-docker.sh index 389be44b5..ee750fb41 100755 --- a/test-docker.sh +++ b/test-docker.sh @@ -34,13 +34,13 @@ force_remove() { } stop_and_remove_containers() { - if [ "$(docker ps -a -q)" ]; then + if [ ! -z "$(docker ps -a -q)" ]; then docker ps -a -q | xargs docker rm -f &>/dev/null else echo "No containers to stop and remove." fi docker run -v '/tmp/jasminegraph:/tmp/jasminegraph' --entrypoint /bin/bash jasminegraph:test -c 'rm -rf /tmp/jasminegraph/*' || echo 'Not removing existing tmp logs' - if [ "$(docker ps -a -q)" ]; then + if [ ! -z "$(docker ps -a -q)" ]; then docker ps -a -q | xargs docker rm -f &>/dev/null fi } diff --git a/test-k8s.sh b/test-k8s.sh index 38fd41146..8cbfd5fc3 100755 --- a/test-k8s.sh +++ b/test-k8s.sh @@ -26,6 +26,13 @@ WORKER_LOG_DIR="/tmp/jasminegraph" rm -rf "${WORKER_LOG_DIR}" mkdir -p "${WORKER_LOG_DIR}" +force_remove() { + local files=("$@") + for f in "${files[@]}"; do + rm -rf "$f" &>/dev/null || sudo rm -rf "$f" + done +} + build_and_run_on_k8s() { cd "$PROJECT_ROOT" docker build -t jasminegraph . |& tee "$BUILD_LOG" @@ -58,7 +65,7 @@ clear_resources() { } cd "$TEST_ROOT" -sudo rm -rf env +force_remove env cp -r env_init env cd "$PROJECT_ROOT" @@ -76,7 +83,7 @@ while ! nc -zvn -w 1 "$masterIP" 7777 &>/dev/null; do cat "$BUILD_LOG" echo "Run log:" cat "$RUN_LOG" - sudo rm -rf "${TEST_ROOT}/env" + force_remove "${TEST_ROOT}/env" clear_resources exit 1 fi @@ -116,5 +123,5 @@ set +e clear_resources >/dev/null 2>&1 set -e -sudo rm -rf "${TEST_ROOT}/env" "${WORKER_LOG_DIR}" +force_remove "${TEST_ROOT}/env" "${WORKER_LOG_DIR}" exit "$exit_code" From 6e46fccb145b4f2a82768316a8a03313e5725435 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Thu, 8 Feb 2024 01:05:15 +0530 Subject: [PATCH 006/143] Use workerList from K8sWorkerController in k8s mode --- .../core/executor/impl/PageRankExecutor.cpp | 11 +- src/server/JasmineGraphServer.cpp | 240 +++++++----------- src/server/JasmineGraphServer.h | 6 - test-docker.sh | 2 +- 4 files changed, 102 insertions(+), 157 deletions(-) diff --git a/src/frontend/core/executor/impl/PageRankExecutor.cpp b/src/frontend/core/executor/impl/PageRankExecutor.cpp index cc300ab5d..8ef537e5f 100644 --- a/src/frontend/core/executor/impl/PageRankExecutor.cpp +++ b/src/frontend/core/executor/impl/PageRankExecutor.cpp @@ -82,7 +82,6 @@ void PageRankExecutor::execute() { std::map graphPartitionedHosts = JasmineGraphServer::getGraphPartitionedHosts(graphId); - std::vector partitions; string host; int port; int dataPort; @@ -91,13 +90,12 @@ void PageRankExecutor::execute() { std::map::iterator workerIter; for (workerIter = graphPartitionedHosts.begin(); workerIter != graphPartitionedHosts.end(); workerIter++) { JasmineGraphServer::workerPartitions workerPartition = workerIter->second; - partitions = workerPartition.partitionID; host = workerIter->first; port = workerPartition.port; dataPort = workerPartition.dataPort; - for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); - partitionit++) { + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { std::string partition = *partitionit; workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } @@ -108,13 +106,12 @@ void PageRankExecutor::execute() { for (workerIter = graphPartitionedHosts.begin(); workerIter != graphPartitionedHosts.end(); workerIter++) { JasmineGraphServer::workerPartitions workerPartition = workerIter->second; - partitions = workerPartition.partitionID; host = workerIter->first; port = workerPartition.port; dataPort = workerPartition.dataPort; - for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); - partitionit++) { + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { std::string partition = *partitionit; intermRes.push_back(std::async(std::launch::async, PageRankExecutor::doPageRank, graphId, alpha, iterations, partition, host, port, dataPort, workerList)); diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 117c58b16..de4c1dda5 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -68,7 +68,6 @@ static bool initiateOrgServer(std::string host, int port, int dataPort, std::str std::string masterIP); static void degreeDistributionCommon(std::string graphID, std::string command); -static map hostIDMap; static std::vector hostWorkerList; static map> hostPortMap; std::map aggregateWeightMap; @@ -136,20 +135,10 @@ int JasmineGraphServer::run(std::string profile, std::string masterIp, int numbe updateOperationalGraphList(); if (profile == Conts::PROFILE_K8S) { - this->k8sWorkerController = K8sWorkerController::getInstance(masterIp, numberofWorkers, sqlite); - std::string selectQuery = "select ip,server_port,server_data_port from worker"; - std::vector>> output = this->sqlite->runSelect(selectQuery); - for (std::vector>>::iterator i = output.begin(); i != output.end(); ++i) { - std::vector>::iterator j = (i->begin()); - std::string ip = j->second; - ++j; - std::string port = j->second; - ++j; - std::string dataPort = j->second; - hostWorkerList.push_back({ip, atoi(port.c_str()), atoi(dataPort.c_str())}); - } + // Create K8s worker controller + (void)K8sWorkerController::getInstance(masterIp, numberofWorkers, sqlite); + hostWorkerList = K8sWorkerController::workerList; } else { - this->k8sWorkerController = NULL; start_workers(); addInstanceDetailsToPerformanceDB(masterHost, masterPortVector, "true"); } @@ -328,7 +317,6 @@ void JasmineGraphServer::start_workers() { myThreads[threadCount].join(); } delete[] myThreads; - hostIDMap = getLiveHostIDList(); } void JasmineGraphServer::waitForAcknowledgement(int numberOfWorkers) { @@ -731,7 +719,7 @@ void JasmineGraphServer::shutdown_workers() { auto *server = JasmineGraphServer::getInstance(); if (server->profile == Conts::PROFILE_K8S) { - server->k8sWorkerController->setNumberOfWorkers(0); + K8sWorkerController::getInstance()->setNumberOfWorkers(0); return; } for (auto listIterator = hostWorkerList.begin(); listIterator < hostWorkerList.end(); listIterator++) { @@ -785,10 +773,16 @@ int JasmineGraphServer::shutdown_worker(std::string workerIP, int port) { return 0; } -static std::vector getWorkers(size_t npart) { +static std::vector getWorkers(size_t npart, std::string profile) { // TODO: get the workers with lowest load from hostWorkerList - if (npart > hostWorkerList.size()) npart = hostWorkerList.size(); - std::vector workerList(hostWorkerList.begin(), hostWorkerList.begin() + npart); + std::vector *workerListAll; + if (profile == Conts::PROFILE_K8S) { + workerListAll = &(K8sWorkerController::workerList); + } else { + workerListAll = &(hostWorkerList); + } + if (npart > (*workerListAll).size()) npart = (*workerListAll).size(); + std::vector workerList((*workerListAll).begin(), (*workerListAll).begin() + npart); return workerList; } @@ -816,7 +810,7 @@ void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, int file_count = 0; std::thread *workerThreads = new std::thread[total_threads]; while (count < total_threads) { - auto workerList = getWorkers(partitionFileMap.size()); + auto workerList = getWorkers(partitionFileMap.size(), this->profile); for (auto listIterator = workerList.begin(); listIterator < workerList.end(); listIterator++) { worker worker = *listIterator; if (count == total_threads) { @@ -1209,32 +1203,6 @@ void JasmineGraphServer::addHostsToMetaDB(std::string host, std::vector por } } -map JasmineGraphServer::getLiveHostIDList() { - server_logger.info("###MASTER### Loading Live Host ID List"); - map hostIDMap; - std::vector>> v = - this->sqlite->runSelect("SELECT host_idhost,user,ip,server_port FROM worker;"); - string id = v[0][0].second; - for (int i = 0; i < v.size(); i++) { - string id = v[i][0].second; - string user = v[i][1].second; - string ip = v[i][2].second; - string serverPort = v[i][3].second; - - string host = ""; - - if (user == "") { - host = ip + ":" + serverPort; - } else { - host = user + "@" + ip + ":" + serverPort; - } - - hostIDMap[host] = id; - } - - return hostIDMap; -} - static void updateMetaDB(int graphID, string uploadEndTime) { std::unique_ptr sqliteDBInterface(new SQLiteDBInterface()); sqliteDBInterface->init(); @@ -1384,8 +1352,6 @@ static bool removePartitionThroughService(string host, int port, string graphID, return true; } -std::vector JasmineGraphServer::getHostWorkerList() { return hostWorkerList; } - void JasmineGraphServer::updateOperationalGraphList() { string hosts = ""; string graphIDs = ""; @@ -1577,19 +1543,15 @@ void JasmineGraphServer::addInstanceDetailsToPerformanceDB(std::string host, std static void degreeDistributionCommon(std::string graphID, std::string command) { std::map graphPartitionedHosts = JasmineGraphServer::getGraphPartitionedHosts(graphID); - std::vector partitions; - string host; - int port; std::string workerList; std::map::iterator workerit; for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { JasmineGraphServer::workerPartitions workerPartition = workerit->second; - partitions = workerPartition.partitionID; - host = workerit->first; - port = workerPartition.port; + string host = workerit->first; + int port = workerPartition.port; - for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); - partitionit++) { + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { std::string partition = *partitionit; workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } @@ -1597,18 +1559,17 @@ static void degreeDistributionCommon(std::string graphID, std::string command) { workerList.pop_back(); + int sockfd; + char data[FED_DATA_LENGTH + 1]; + bool loop = false; + socklen_t len; + struct sockaddr_in serv_addr; + struct hostent *server; + for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { JasmineGraphServer::workerPartitions workerPartition = workerit->second; - partitions = workerPartition.partitionID; - host = workerit->first; - port = workerPartition.port; - - int sockfd; - char data[FED_DATA_LENGTH + 1]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; + string host = workerit->first; + int port = workerPartition.port; sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { @@ -1622,8 +1583,8 @@ static void degreeDistributionCommon(std::string graphID, std::string command) { continue; } - for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); - partitionit++) { + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { std::string partition = *partitionit; bzero((char *)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; @@ -1684,44 +1645,38 @@ long JasmineGraphServer::getGraphVertexCount(std::string graphID) { void JasmineGraphServer::duplicateCentralStore(std::string graphID) { std::map graphPartitionedHosts = JasmineGraphServer::getGraphPartitionedHosts(graphID); - std::vector partitions; - string host; - int port; - int dport; std::string workerList; std::map::iterator workerit; for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { JasmineGraphServer::workerPartitions workerPartition = workerit->second; - partitions = workerPartition.partitionID; - host = workerit->first; - port = workerPartition.port; - dport = workerPartition.dataPort; + string host = workerit->first; + int port = workerPartition.port; + int dport = workerPartition.dataPort; - for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); - partitionit++) { + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { std::string partition = *partitionit; workerList.append(host + ":" + std::to_string(port) + ":" + partition + ":" + to_string(dport) + ","); } } workerList.pop_back(); + int sockfd; + char data[FED_DATA_LENGTH + 1]; + bool loop = false; + socklen_t len; + struct sockaddr_in serv_addr; + struct hostent *server; + for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { JasmineGraphServer::workerPartitions workerPartition = workerit->second; - partitions = workerPartition.partitionID; - host = workerit->first; - port = workerPartition.port; + string host = workerit->first; + int port = workerPartition.port; if (host.find('@') != std::string::npos) { host = Utils::split(host, '@')[1]; } - int sockfd; - char data[FED_DATA_LENGTH + 1]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { @@ -1734,8 +1689,8 @@ void JasmineGraphServer::duplicateCentralStore(std::string graphID) { continue; } - for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); - partitionit++) { + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { std::string partition = *partitionit; bzero((char *)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; @@ -1793,10 +1748,9 @@ void JasmineGraphServer::initiateFiles(std::string graphID, std::string training std::map::iterator j; for (j = graphPartitionedHosts.begin(); j != graphPartitionedHosts.end(); j++) { JasmineGraphServer::workerPartitions workerPartition = j->second; - std::vector partitions = workerPartition.partitionID; std::vector::iterator k; map scheduleOfHost = scheduleForAllHosts[j->first]; - for (k = partitions.begin(); k != partitions.end(); k++) { + for (k = workerPartition.partitionID.begin(); k != workerPartition.partitionID.end(); k++) { int iterationOfPart = scheduleOfHost[stoi(*k)]; workerThreads[count++] = std::thread(initiateTrain, j->first, workerPartition.port, workerPartition.dataPort, @@ -1938,10 +1892,9 @@ void JasmineGraphServer::initiateMerge(std::string graphID, std::string training int count = 0; for (j = graphPartitionedHosts.begin(); j != graphPartitionedHosts.end(); j++) { JasmineGraphServer::workerPartitions workerPartition = j->second; - std::vector partitions = workerPartition.partitionID; std::vector::iterator k; map scheduleOfHost = scheduleForAllHosts[j->first]; - for (k = partitions.begin(); k != partitions.end(); k++) { + for (k = workerPartition.partitionID.begin(); k != workerPartition.partitionID.end(); k++) { int iterationOfPart = scheduleOfHost[stoi(*k)]; workerThreads[count] = std::thread(mergeFiles, j->first, workerPartition.port, workerPartition.dataPort, trainingArgs + " " + *k, fl_clients, *k, this->masterHost); @@ -2208,7 +2161,7 @@ bool JasmineGraphServer::mergeFiles(std::string host, int port, int dataPort, st bool JasmineGraphServer::sendTrainCommand(std::string host, int port, std::string trainingArgs) { bool result = true; int sockfd; - char data[FED_DATA_LENGTH]; + char data[FED_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -2258,21 +2211,16 @@ bool JasmineGraphServer::sendTrainCommand(std::string host, int port, std::strin void JasmineGraphServer::egoNet(std::string graphID) { std::map graphPartitionedHosts = JasmineGraphServer::getGraphPartitionedHosts(graphID); - std::vector partitions; - string host; - int port; - int dataPort; std::string workerList; for (auto workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { JasmineGraphServer::workerPartitions workerPartition = workerit->second; - partitions = workerPartition.partitionID; - host = workerit->first; - port = workerPartition.port; - dataPort = workerPartition.dataPort; + string host = workerit->first; + int port = workerPartition.port; + int dataPort = workerPartition.dataPort; - for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); - partitionit++) { + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { std::string partition = *partitionit; workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } @@ -2281,58 +2229,64 @@ void JasmineGraphServer::egoNet(std::string graphID) { workerList.pop_back(); int sockfd; - char data[301]; + char data[FED_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; struct hostent *server; - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - server_logger.error("Cannot create socket"); - return; - } - server = gethostbyname(host.c_str()); - if (server == NULL) { - server_logger.error("ERROR, no host named " + host); - return; - } + for (auto workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { + JasmineGraphServer::workerPartitions workerPartition = workerit->second; + string host = workerit->first; + int port = workerPartition.port; + sockfd = socket(AF_INET, SOCK_STREAM, 0); - for (std::vector::iterator partitionit = partitions.begin(); partitionit != partitions.end(); - partitionit++) { - std::string partition = *partitionit; - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + if (sockfd < 0) { + server_logger.error("Cannot create socket"); return; } - - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::EGONET, - JasmineGraphInstanceProtocol::OK)) { - close(sockfd); + server = gethostbyname(host.c_str()); + if (server == NULL) { + server_logger.error("ERROR, no host named " + host); return; } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - return; - } + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { + std::string partition = *partitionit; + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(port); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + return; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, - JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - return; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::EGONET, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + return; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + return; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + return; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, workerList, + JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + return; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, workerList, - JasmineGraphInstanceProtocol::OK)) { close(sockfd); - return; } - - close(sockfd); } } diff --git a/src/server/JasmineGraphServer.h b/src/server/JasmineGraphServer.h index 74dbc8ac0..b68fb7982 100644 --- a/src/server/JasmineGraphServer.h +++ b/src/server/JasmineGraphServer.h @@ -45,7 +45,6 @@ class JasmineGraphServer { int serverDataPort; std::map> workerPortsMap; std::map> workerDataPortsMap; - K8sWorkerController *k8sWorkerController; JasmineGraphServer(); @@ -56,8 +55,6 @@ class JasmineGraphServer { void updateOperationalGraphList(); - std::map getLiveHostIDList(); - static bool hasEnding(std::string const &fullString, std::string const &ending); std::vector getWorkerVector(std::string workerList); void deleteNonOperationalGraphFragment(int graphID); @@ -137,9 +134,6 @@ class JasmineGraphServer { // partiton ID. }; - // return hostWorkerList - static std::vector getHostWorkerList(); - static std::map getGraphPartitionedHosts(std::string graphID); static void inDegreeDistribution(std::string graphID); diff --git a/test-docker.sh b/test-docker.sh index ee750fb41..a95dfd517 100755 --- a/test-docker.sh +++ b/test-docker.sh @@ -97,7 +97,7 @@ while ! nc -zvn 127.0.0.1 7777 &>/dev/null; do echo "JasmineGraph is not listening" echo "Build log:" cat "$BUILD_LOG" - echo "Build log:" + echo -e '\n\e[33;1mMASTER LOG:\e[0m' cat "$RUN_LOG" force_remove "${TEST_ROOT}/env" stop_and_remove_containers From dccd46b84b14d151f55efaad68a594a87e4dfb2f Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 9 Feb 2024 00:29:45 +0530 Subject: [PATCH 007/143] Cleanup unused files and properties --- .github/actions/jasminegraph-action/action.yml | 6 ------ .github/workflows/build.yml | 8 -------- conf/jasminegraph-server.properties | 12 +----------- python/.keep | 0 samples/ml/single-cluster-fl-unsupervised/.keep | 0 samples/ml/single-cluster-fl/.keep | 0 samples/ml/single-cluster-unsupervised/.keep | 0 src/ml/federated/.keep | 0 src/server/JasmineGraphServer.cpp | 2 +- src_python/.keep | 0 10 files changed, 2 insertions(+), 26 deletions(-) delete mode 100644 .github/actions/jasminegraph-action/action.yml delete mode 100644 python/.keep delete mode 100644 samples/ml/single-cluster-fl-unsupervised/.keep delete mode 100644 samples/ml/single-cluster-fl/.keep delete mode 100644 samples/ml/single-cluster-unsupervised/.keep delete mode 100644 src/ml/federated/.keep delete mode 100644 src_python/.keep diff --git a/.github/actions/jasminegraph-action/action.yml b/.github/actions/jasminegraph-action/action.yml deleted file mode 100644 index d7716e4ca..000000000 --- a/.github/actions/jasminegraph-action/action.yml +++ /dev/null @@ -1,6 +0,0 @@ -# action.yml -name: 'Build' -description: 'Build a JasmineGraph docker image' -runs: - using: 'docker' - image: 'docker/Dockerfile' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc3d0e3da..81add0bb4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,10 +6,6 @@ on: pull_request: branches: [master] -env: - # Customize the build type here (Release, Debug, RelWithDebInfo, etc.) - BUILD_TYPE: Release - jobs: unit-tests: runs-on: ubuntu-latest @@ -27,9 +23,6 @@ jobs: - name: Grant permissions for default user run: kubectl apply -f ./k8s/rbac.yaml - - name: Docker Build - run: docker build -t jasminegraph . - - name: Build unit test run: docker build -f ./tests/unit/Dockerfile -t jasminegraph:unit . @@ -59,7 +52,6 @@ jobs: repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - name: Docker Build - # Build JasmineGraph docker image run: docker build -t jasminegraph . - name: Run Integration Tests diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 8c56c9ee6..8d5bc48c4 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -24,8 +24,6 @@ org.jasminegraph.server.nworkers=2 #org.jasminegraph.server.npartitions is the number of partitions into which the graph should be partitioned org.jasminegraph.server.npartitions=2 org.jasminegraph.server.streaming.kafka.host=127.0.0.1:9092 -#org.jasminegraph.server.runtime.location is the location where temporary files of JasmineGraph master is kept -org.jasminegraph.server.runtime.location=./tmp org.jasminegraph.worker.path=/var/tmp/jasminegraph/ #Path to keep jasminegraph artifacts in order to copy them to remote locations. If this is not set then the artifacts in the #JASMINEGRAPH_HOME location will be copied instead. @@ -42,7 +40,6 @@ org.jasminegraph.server.instance.trainedmodelfolder=/var/tmp/jasminegraph-locals org.jasminegraph.server.instance.local=/var/tmp org.jasminegraph.server.instance=/var/tmp - #This parameter controls the nmon stat collection enable and disable org.jasminegraph.server.enable.nmon=false; #This parameter holds the path to nmon stat file @@ -57,7 +54,6 @@ org.jasminegraph.server.modelDir=/var/tmp/jasminegraph-localstore/models/ #-------------------------------------------------------------------------------- #Federated Learning Parameters #-------------------------------------------------------------------------------- - org.jasminegraph.fl.location=/home/ubuntu/software/jasminegraph/src_python/ org.jasminegraph.fl.dataDir=/home/ubuntu/software/jasminegraph/src_python/data/ org.jasminegraph.fl.weights=/home/ubuntu/software/jasminegraph/src_python/weights/ @@ -87,16 +83,10 @@ org.jasminegraph.centralstore.location=/home/tmp/centralstore org.jasminegraph.db.location=./metadb/jasminegraph_meta.db org.jasminegraph.performance.db.location=./performancedb/jasminegraph_performance.db org.jasminegraph.streaming.db.location=/var/tmp/jasminegraph/jasminegraph_streaming.db -#-------------------------------------------------------------------------------- -#Jetty specific parameters - Note that this is for Java 7 -#-------------------------------------------------------------------------------- - -#Java 1.6 Jetty -org.acacia.ui.jetty.home=/home/ubuntu/software/jetty-distribution-8.1.9.v20130131 #-------------------------------------------------------------------------------- #K8S information #-------------------------------------------------------------------------------- org.jasminegraph.k8s.config=/etc/rancher/k3s/k3s.yaml org.jasminegraph.k8s.namespace=default -org.jasminegraph.k8s.image=jasminegraph \ No newline at end of file +org.jasminegraph.k8s.image=jasminegraph diff --git a/python/.keep b/python/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/samples/ml/single-cluster-fl-unsupervised/.keep b/samples/ml/single-cluster-fl-unsupervised/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/samples/ml/single-cluster-fl/.keep b/samples/ml/single-cluster-fl/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/samples/ml/single-cluster-unsupervised/.keep b/samples/ml/single-cluster-unsupervised/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/ml/federated/.keep b/src/ml/federated/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index de4c1dda5..58a8a327b 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -774,7 +774,7 @@ int JasmineGraphServer::shutdown_worker(std::string workerIP, int port) { } static std::vector getWorkers(size_t npart, std::string profile) { - // TODO: get the workers with lowest load from hostWorkerList + // TODO: get the workers with lowest load from workerList std::vector *workerListAll; if (profile == Conts::PROFILE_K8S) { workerListAll = &(K8sWorkerController::workerList); diff --git a/src_python/.keep b/src_python/.keep deleted file mode 100644 index e69de29bb..000000000 From 33040120ab316d915ae0fed5aa0a96337b2c8075 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 9 Feb 2024 16:13:01 +0530 Subject: [PATCH 008/143] Fix unit test failure --- tests/unit/k8s/K8sWorkerController_test.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/unit/k8s/K8sWorkerController_test.cpp b/tests/unit/k8s/K8sWorkerController_test.cpp index 337f93e69..d66679ca4 100644 --- a/tests/unit/k8s/K8sWorkerController_test.cpp +++ b/tests/unit/k8s/K8sWorkerController_test.cpp @@ -19,19 +19,19 @@ limitations under the License. #include "gtest/gtest.h" class K8sWorkerControllerTest : public ::testing::Test { - protected: - K8sWorkerController *controller{}; - SQLiteDBInterface *metadb = NULL; - K8sInterface *interface {}; + public: + static K8sWorkerController *controller; + static SQLiteDBInterface *metadb; + static K8sInterface *interface; - void SetUp() override { + static void SetUpTestSuite() { metadb = new SQLiteDBInterface(TEST_RESOURCE_DIR "temp/jasminegraph_meta.db"); metadb->init(); controller = K8sWorkerController::getInstance("10.43.0.1", 2, metadb); interface = new K8sInterface(); } - void TearDown() override { + static void TearDownTestSuite() { delete controller; delete metadb; delete interface; @@ -39,6 +39,10 @@ class K8sWorkerControllerTest : public ::testing::Test { } }; +K8sWorkerController *K8sWorkerControllerTest::controller = nullptr; +SQLiteDBInterface *K8sWorkerControllerTest::metadb = nullptr; +K8sInterface *K8sWorkerControllerTest::interface = nullptr; + TEST_F(K8sWorkerControllerTest, TestConstructor) { ASSERT_EQ(controller->getMasterIp(), "10.43.0.1"); ASSERT_EQ(controller->getNumberOfWorkers(), 2); From 35c042ed7681a7efb6b75764a0ab74ce58d8f484 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sun, 11 Feb 2024 21:04:55 +0530 Subject: [PATCH 009/143] Bug fix in TriangleCountExecutor and JasmineGraphServer --- .../core/executor/impl/TriangleCountExecutor.cpp | 12 +++++------- src/server/JasmineGraphServer.cpp | 10 ++++++++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index b963c8a59..5ec4233c8 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -143,8 +143,7 @@ void TriangleCountExecutor::execute() { partitionVec.push_back(partitionId); partitionMap[workerID] = partitionVec; } else { - std::vector partitionVec = partitionMap.find(workerID)->second; - partitionVec.push_back(partitionId); + partitionMap[workerID].push_back(partitionId); } triangleCount_logger.log("###TRIANGLE-COUNT-EXECUTOR### Getting Triangle Count : Host " + host + @@ -184,8 +183,8 @@ void TriangleCountExecutor::execute() { "SELECT attempt from graph_sla INNER JOIN sla_category where graph_sla.id_sla_category=sla_category.id and " "graph_sla.graph_id='" + graphId + "' and graph_sla.partition_count='" + std::to_string(partitionCount) + - "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + - "' and sla_category.command='" + TRIANGLES + "';"; + "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + "' and sla_category.command='" + TRIANGLES + + "';"; std::vector>> queryResults = perfDB->runSelect(query); @@ -199,9 +198,8 @@ void TriangleCountExecutor::execute() { } else { triangleCount_logger.log("###TRIANGLE-COUNT-EXECUTOR### Inserting initial record for SLA ", "info"); Utils::updateSLAInformation(perfDB, graphId, partitionCount, 0, TRIANGLES, Conts::SLA_CATEGORY::LATENCY); - statResponse.push_back(std::async(std::launch::async, collectPerformaceData, perfDB, - graphId.c_str(), TRIANGLES, Conts::SLA_CATEGORY::LATENCY, partitionCount, - masterIP, autoCalibrate)); + statResponse.push_back(std::async(std::launch::async, collectPerformaceData, perfDB, graphId.c_str(), TRIANGLES, + Conts::SLA_CATEGORY::LATENCY, partitionCount, masterIP, autoCalibrate)); isStatCollect = true; } diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 58a8a327b..7a7a2908d 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -781,8 +781,14 @@ static std::vector getWorkers(size_t npart, std::str } else { workerListAll = &(hostWorkerList); } - if (npart > (*workerListAll).size()) npart = (*workerListAll).size(); - std::vector workerList((*workerListAll).begin(), (*workerListAll).begin() + npart); + size_t len = workerListAll->size(); + std::cout << "workerListAll len = " << len << std::endl; + std::vector workerList; + for (int i=0; i Date: Fri, 26 Jan 2024 08:58:47 +0530 Subject: [PATCH 010/143] Added code in perfUtil for pushing jobs Co-authored-by: ArunaSenanayake --- CMakeLists.txt | 1 + Dockerfile | 4 + conf/hosts.txt | 6 +- conf/jasminegraph-server.properties | 4 +- src/performance/metrics/PerformanceUtil.cpp | 374 ++++++++++++++++++-- 5 files changed, 361 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 95b8804d2..a812204f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -156,6 +156,7 @@ target_link_libraries(JasmineGraphLib PRIVATE /usr/lib/x86_64-linux-gnu/libjsonc target_link_libraries(JasmineGraphLib PRIVATE /usr/local/lib/libcppkafka.so) target_link_libraries(JasmineGraph JasmineGraphLib) target_include_directories(JasmineGraph PRIVATE /usr/include/python3) +target_link_libraries(JasmineGraph curl) set(PYTHON_EXECUTABLE "/usr/bin/python3") set(PYTHON_INCLUDE_DIR "/usr/include/python3") diff --git a/Dockerfile b/Dockerfile index 00145b7a3..d1b67f28f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,8 @@ FROM miyurud/jasminegraph-prerequisites:20240101T095619 + +RUN apt-get update +RUN apt-get install libcurl4-openssl-dev -y + ENV HOME="/home/ubuntu" ENV JASMINEGRAPH_HOME="${HOME}/software/jasminegraph" diff --git a/conf/hosts.txt b/conf/hosts.txt index 964b511a3..e8a2bcbdf 100644 --- a/conf/hosts.txt +++ b/conf/hosts.txt @@ -1,8 +1,12 @@ #This is the hosts file for Jasminegraph. All the host names on which Jasminegraph workers should get started needs #to be listed here localhost + +# TODO: Replace this +@192.168.8.150 + #anuradhak@10.8.100.245 #192.168.1.8 #192.168.1.7 #ubuntu@18.219.24.222 -#anuradha@10.10.28.181 \ No newline at end of file +#anuradha@10.10.28.181 diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 8d5bc48c4..836e218b7 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -41,7 +41,7 @@ org.jasminegraph.server.instance.local=/var/tmp org.jasminegraph.server.instance=/var/tmp #This parameter controls the nmon stat collection enable and disable -org.jasminegraph.server.enable.nmon=false; +org.jasminegraph.server.enable.nmon=true; #This parameter holds the path to nmon stat file org.jasminegraph.server.nmon.file.location=/var/tmp/nmon/ #This parameter holds the number of snapshots to be collected @@ -72,7 +72,7 @@ org.jasminegraph.fl.organization.file=/home/ubuntu/software/jasminegraph/conf/ho #-------------------------------------------------------------------------------- #Scheduler Specific Parameters -org.jasminegraph.scheduler.enabled=false +org.jasminegraph.scheduler.enabled=true #PerformanceCollector Scheduler Timing. Run once every 120 seconds org.jasminegraph.scheduler.performancecollector.timing=120 diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index be1487fd2..f2579ea65 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -13,9 +13,13 @@ limitations under the License. #include "PerformanceUtil.h" +#include + using namespace std::chrono; std::map> resourceUsageMap; +static size_t write_callback(void* contents, size_t size, size_t nmemb, std::string* output); + Logger scheduler_logger; SQLiteDBInterface *sqlLiteDB; PerformanceSQLiteDBInterface *perfDb; @@ -203,6 +207,12 @@ std::vector PerformanceUtil::retrieveCurrentResourceUtiliza return placeResourceConsumptionList; } +static size_t write_callback(void* contents, size_t size, size_t nmemb, std::string* output) { + size_t totalSize = size * nmemb; + output->append(static_cast(contents), totalSize); + return totalSize; +} + void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, std::string isVMStatManager, std::string isResourceAllocationRequired, std::string hostId, std::string placeId) { @@ -211,7 +221,7 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s bool loop = false; socklen_t len; struct sockaddr_in serv_addr; - struct hostent *server; + struct hostent* server; sockfd = socket(AF_INET, SOCK_STREAM, 0); @@ -226,11 +236,11 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s return; } - bzero((char *)&serv_addr, sizeof(serv_addr)); + bzero((char*)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char*)server->h_addr, (char*)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return; } @@ -296,6 +306,62 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s perfDb->runInsert(vmPerformanceSql); + CURL* curl; + CURLcode res; + std::string response_host_perf_data; + + curl = curl_easy_init(); + if (curl) { + // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/hostPerfData"); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_host_perf_data); + std::string job_data = "memory_consumption " + memoryConsumption + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } + + curl = curl_easy_init(); + if (curl) { + // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/hostPerfData"); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_host_perf_data); + std::string job_data = "cpu_usage " + cpuUsage + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } + if (isResourceAllocationRequired == "true") { std::string totalMemory = strArr[6]; std::string totalCores = strArr[7]; @@ -311,7 +377,63 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s "insert into place_performance_data (idplace, memory_usage, cpu_usage, date_time) values ('" + placeId + "','" + memoryUsage + "','" + cpuUsage + "','" + processTime + "')"; - perfDb->runInsert(placePerfSql); + perfDb.runInsert(placePerfSql); + + CURL* curl; + CURLcode res; + std::string response_place_perf_data; + + curl = curl_easy_init(); + if (curl) { + // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/placePerfData"); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_place_perf_data); + std::string job_data = "memory_usage " + memoryUsage + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } + + curl = curl_easy_init(); + if (curl) { + // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/placePerfData"); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_place_perf_data); + std::string job_data = "cpu_usage " + cpuUsage + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } } } } @@ -344,6 +466,62 @@ void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, s perfDb->runInsert(vmPerformanceSql); + CURL* curl; + CURLcode res; + std::string response_local_perf_data; + + curl = curl_easy_init(); + if (curl) { + // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/localPerfData"); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_local_perf_data); + std::string job_data = "total_memory_usage " + totalMemoryUsed + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } + + curl = curl_easy_init(); + if (curl) { + // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/localPerfData"); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_local_perf_data); + std::string job_data = "total_CPU_usage " + totalCPUUsage + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } + if (isResourceAllocationRequired == "true") { std::string totalMemory = strArr[2]; std::string totalCores = strArr[3]; @@ -358,7 +536,63 @@ void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, s placeId + "','" + to_string(memoryUsage) + "','" + to_string(cpuUsage) + "','" + reportTimeString + "')"; - perfDb->runInsert(placePerfSql); + perfDb.runInsert(placePerfSql); + + CURL* curl; + CURLcode res; + std::string response_place_perf_data; + + curl = curl_easy_init(); + if (curl) { + // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/placePerfData"); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_place_perf_data); + std::string job_data = "memory_usage " + to_string(memoryUsage) + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } + + curl = curl_easy_init(); + if (curl) { + // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/placePerfData"); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_place_perf_data); + std::string job_data = "cpu_usage " + to_string(cpuUsage) + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } } int PerformanceUtil::collectRemoteSLAResourceUtilization(std::string host, int port, std::string isVMStatManager, @@ -369,7 +603,7 @@ int PerformanceUtil::collectRemoteSLAResourceUtilization(std::string host, int p bool loop = false; socklen_t len; struct sockaddr_in serv_addr; - struct hostent *server; + struct hostent* server; std::string graphSlaId; sockfd = socket(AF_INET, SOCK_STREAM, 0); @@ -389,11 +623,11 @@ int PerformanceUtil::collectRemoteSLAResourceUtilization(std::string host, int p return 0; } - bzero((char *)&serv_addr, sizeof(serv_addr)); + bzero((char*)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char*)server->h_addr, (char*)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return 0; } @@ -534,7 +768,7 @@ ResourceConsumption PerformanceUtil::retrieveRemoteResourceConsumption(std::stri bool loop = false; socklen_t len; struct sockaddr_in serv_addr; - struct hostent *server; + struct hostent* server; std::string graphSlaId; std::string isVMStatManager = "false"; std::string isResourceAllocationRequired = "false"; @@ -552,11 +786,11 @@ ResourceConsumption PerformanceUtil::retrieveRemoteResourceConsumption(std::stri return placeResourceConsumption; } - bzero((char *)&serv_addr, sizeof(serv_addr)); + bzero((char*)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char*)server->h_addr, (char*)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return placeResourceConsumption; } @@ -621,7 +855,7 @@ ResourceConsumption PerformanceUtil::retrieveRemoteResourceConsumption(std::stri std::vector PerformanceUtil::getResourceAvailableTime(std::vector graphIdList, std::string command, std::string category, std::string masterIP, - std::vector &pendingHPJobList) { + std::vector& pendingHPJobList) { PerformanceUtil performanceUtil; performanceUtil.init(); std::set hostSet; @@ -853,8 +1087,8 @@ std::vector PerformanceUtil::getResourceAvailableTime(std::vector> &aggregateLoadAvgMap, - std::map> &newJobLoadAvgMap, +void PerformanceUtil::adjustAggregateLoadMap(std::map>& aggregateLoadAvgMap, + std::map>& newJobLoadAvgMap, long newJobAcceptanceTime) { std::map>::iterator newJobIterator; @@ -899,6 +1133,36 @@ void PerformanceUtil::logLoadAverage() { double currentLoadAverage = statisticCollector.getLoadAverage(); std::cout << "###PERF### CURRENT LOAD: " + std::to_string(currentLoadAverage) << std::endl; + + CURL* curl; + CURLcode res; + std::string response_load_avg; + + curl = curl_easy_init(); + if (curl) { + // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/loadAverage"); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_load_avg); + std::string job_data = "load_average " + std::to_string(currentLoadAverage) + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } } void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *performanceDb, std::string graphId, @@ -945,6 +1209,36 @@ void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *pe valuesString += "('" + graphSlaId + "','" + placeId + "', '','" + usageInfo.loadAverage + "','" + usageInfo.elapsedTime + "'),"; + + CURL* curl; + CURLcode res; + std::string response_load_avg_sla; + + curl = curl_easy_init(); + if (curl) { + // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/loadAverageSLA"); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_load_avg_sla); + std::string job_data = "load_average " + usageInfo.loadAverage + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } } valuesString = valuesString.substr(0, valuesString.length() - 1); @@ -1016,6 +1310,36 @@ void PerformanceUtil::updateRemoteResourceConsumption(PerformanceSQLiteDBInterfa valuesString += "('" + graphSlaId + "','" + placeId + "', '','" + loadAverage + "','" + std::to_string(elapsedTime * 1000) + "'),"; elapsedTime += 5; + + CURL* curl; + CURLcode res; + std::string response_load_avg_sla_rem; + + curl = curl_easy_init(); + if (curl) { + // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/loadAverageSLARem"); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_load_avg_sla_rem); + std::string job_data = "load_average_rem " + loadAverage + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } } valuesString = valuesString.substr(0, valuesString.length() - 1); @@ -1051,7 +1375,7 @@ void PerformanceUtil::initiateCollectingRemoteSLAResourceUtilization(std::string bool loop = false; socklen_t len; struct sockaddr_in serv_addr; - struct hostent *server; + struct hostent* server; std::string graphSlaId; sockfd = socket(AF_INET, SOCK_STREAM, 0); @@ -1071,11 +1395,11 @@ void PerformanceUtil::initiateCollectingRemoteSLAResourceUtilization(std::string return; } - bzero((char *)&serv_addr, sizeof(serv_addr)); + bzero((char*)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char*)server->h_addr, (char*)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return; } @@ -1125,7 +1449,7 @@ std::string PerformanceUtil::requestRemoteLoadAverages(std::string host, int por bool loop = false; socklen_t len; struct sockaddr_in serv_addr; - struct hostent *server; + struct hostent* server; std::string graphSlaId; sockfd = socket(AF_INET, SOCK_STREAM, 0); @@ -1145,11 +1469,11 @@ std::string PerformanceUtil::requestRemoteLoadAverages(std::string host, int por return 0; } - bzero((char *)&serv_addr, sizeof(serv_addr)); + bzero((char*)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char*)server->h_addr, (char*)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return 0; } From 5932fb316f971a8f432e6d628f533a3795db3acf Mon Sep 17 00:00:00 2001 From: ArunaSenanayake Date: Sun, 24 Dec 2023 09:25:01 +0530 Subject: [PATCH 011/143] Fixed the ip address in hosts.txt file --- conf/hosts.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/hosts.txt b/conf/hosts.txt index e8a2bcbdf..c0a48fcc7 100644 --- a/conf/hosts.txt +++ b/conf/hosts.txt @@ -3,7 +3,7 @@ localhost # TODO: Replace this -@192.168.8.150 +192.168.8.150 #anuradhak@10.8.100.245 #192.168.1.8 From 61d047675429906d99339242c3ad2b145a62cffc Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 26 Jan 2024 09:01:20 +0530 Subject: [PATCH 012/143] Made some code refactoring --- Dockerfile | 2 + src/performance/metrics/PerformanceUtil.cpp | 149 +++++------------- .../metrics/StatisticCollector.cpp | 42 ++++- 3 files changed, 84 insertions(+), 109 deletions(-) diff --git a/Dockerfile b/Dockerfile index d1b67f28f..bacbf516d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,8 @@ FROM miyurud/jasminegraph-prerequisites:20240101T095619 RUN apt-get update RUN apt-get install libcurl4-openssl-dev -y +RUN apt-get install -y sysstat +RUN apt-get install -y nmon ENV HOME="/home/ubuntu" ENV JASMINEGRAPH_HOME="${HOME}/software/jasminegraph" diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index f2579ea65..5194b71cb 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -17,6 +17,7 @@ limitations under the License. using namespace std::chrono; std::map> resourceUsageMap; +std::string pushGatewayAddr = "http://192.168.8.150:9091/metrics/job/"; static size_t write_callback(void* contents, size_t size, size_t nmemb, std::string* output); @@ -213,6 +214,41 @@ static size_t write_callback(void* contents, size_t size, size_t nmemb, std::str return totalSize; } +static void send_job(std::string response_string, std::string job_group_name, std::string metric_name, std::string metric_value, std::string addr){ + CURL* curl; + CURLcode res; + //std::string response_host_perf_data; + + curl = curl_easy_init(); + if (curl) { + //std::string hostPGAddr = pushGatewayAddr + "hostPerfData_" + to_string(port); + std::string hostPGAddr = pushGatewayAddr + job_group_name + addr; + //curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/hostPerfData"); + curl_easy_setopt(curl, CURLOPT_URL, hostPGAddr.c_str()); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string); + //std::string job_data = "memory_consumption " + memoryConsumption + "\n"; + std::string job_data = metric_name + " " + metric_value + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } +} + void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, std::string isVMStatManager, std::string isResourceAllocationRequired, std::string hostId, std::string placeId) { @@ -466,61 +502,9 @@ void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, s perfDb->runInsert(vmPerformanceSql); - CURL* curl; - CURLcode res; std::string response_local_perf_data; - - curl = curl_easy_init(); - if (curl) { - // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/localPerfData"); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_local_perf_data); - std::string job_data = "total_memory_usage " + totalMemoryUsed + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } - - curl = curl_easy_init(); - if (curl) { - // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/localPerfData"); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_local_perf_data); - std::string job_data = "total_CPU_usage " + totalCPUUsage + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } + send_job(response_local_perf_data, "localPerfDataO_", "total_memory_usage", totalMemoryUsed, hostId); + send_job(response_local_perf_data, "localPerfDataO_", "total_CPU_usage", totalCPUUsage, hostId); if (isResourceAllocationRequired == "true") { std::string totalMemory = strArr[2]; @@ -538,61 +522,10 @@ void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, s perfDb.runInsert(placePerfSql); - CURL* curl; - CURLcode res; - std::string response_place_perf_data; + std::string response_place_perf_data_local; + send_job(response_place_perf_data_local, "placePerfDataLocalO_", "memory_usage", to_string(memoryUsage), hostId); + send_job(response_place_perf_data_local, "placePerfDataLocalO_", "cpu_usage", to_string(cpuUsage), hostId); - curl = curl_easy_init(); - if (curl) { - // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/placePerfData"); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_place_perf_data); - std::string job_data = "memory_usage " + to_string(memoryUsage) + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } - - curl = curl_easy_init(); - if (curl) { - // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/placePerfData"); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_place_perf_data); - std::string job_data = "cpu_usage " + to_string(cpuUsage) + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } } int PerformanceUtil::collectRemoteSLAResourceUtilization(std::string host, int port, std::string isVMStatManager, diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index 106395556..12e6588bc 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -13,6 +13,8 @@ limitations under the License. #include "StatisticCollector.h" +#include + static clock_t lastCPU, lastSysCPU, lastUserCPU; static int numProcessors; @@ -34,6 +36,12 @@ int StatisticCollector::init() { return 0; } +static size_t write_callback(void* contents, size_t size, size_t nmemb, std::string* output) { + size_t totalSize = size * nmemb; + output->append(static_cast(contents), totalSize); + return totalSize; +} + int StatisticCollector::getMemoryUsageByProcess() { FILE* file = fopen("/proc/self/status", "r"); int result = -1; @@ -136,6 +144,10 @@ long StatisticCollector::getTotalMemoryUsage() { unsigned long cached; unsigned long sReclaimable; unsigned long memUsage; + + CURL* curl; + CURLcode res; + while (file >> token) { if (token == "MemTotal:") { file >> memTotal; @@ -151,6 +163,34 @@ long StatisticCollector::getTotalMemoryUsage() { file.ignore(std::numeric_limits::max(), '\n'); } memUsage = memTotal - (memFree + buffers + cached + sReclaimable); + + std::string response_total_memory; + + curl = curl_easy_init(); + if (curl) { + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/totalMemory"); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_total_memory); + std::string job_data = "total_memory " + to_string(memTotal) + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } + return memUsage; } @@ -237,4 +277,4 @@ void StatisticCollector::logLoadAverage(std::string name) { sleep(Conts::LOAD_AVG_COLLECTING_GAP - elapsed); } } -} +} \ No newline at end of file From dac30f645f91634208a7278e396b8e80d60ce26c Mon Sep 17 00:00:00 2001 From: ArunaSenanayake Date: Fri, 19 Jan 2024 18:24:30 +0530 Subject: [PATCH 013/143] Changed the NMON run command --- src/server/JasmineGraphInstance.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server/JasmineGraphInstance.cpp b/src/server/JasmineGraphInstance.cpp index cf07e87de..56c24f977 100644 --- a/src/server/JasmineGraphInstance.cpp +++ b/src/server/JasmineGraphInstance.cpp @@ -175,8 +175,11 @@ void JasmineGraphInstance::startNmonAnalyzer(string enableNmon, int serverPort) std::string numberOfSnapshots = Utils::getJasmineGraphProperty("org.jasminegraph.server.nmon.snapshots"); std::string snapshotGap = Utils::getJasmineGraphProperty("org.jasminegraph.server.nmon.snapshot.gap"); std::string nmonFileName = nmonFileLocation + "nmon.log." + std::to_string(serverPort); + /*std::string nmonStartupCommand = + "nmon_x86_64_ubuntu18 -c " + numberOfSnapshots + " -s " + snapshotGap + " -T -F " + nmonFileName;*/ std::string nmonStartupCommand = - "nmon_x86_64_ubuntu18 -c " + numberOfSnapshots + " -s " + snapshotGap + " -T -F " + nmonFileName; + "nmon -c " + numberOfSnapshots + " -s " + snapshotGap + " -T -F " + nmonFileName; + char buffer[BUFFER_SIZE]; std::string result = ""; From a7a01211d05ebdcfc1ac0245153fa16fe2830145 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 26 Jan 2024 09:02:13 +0530 Subject: [PATCH 014/143] Added more performance metrics --- .../metrics/StatisticCollector.cpp | 138 ++++++++++++++++++ src/performance/metrics/StatisticCollector.h | 5 + src/server/JasmineGraphInstanceService.cpp | 7 +- 3 files changed, 149 insertions(+), 1 deletion(-) diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index 12e6588bc..0ce64aa28 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -17,6 +17,7 @@ limitations under the License. static clock_t lastCPU, lastSysCPU, lastUserCPU; static int numProcessors; +std::string pushGatewayJobAddr = "http://192.168.8.150:9091/metrics/job/"; int StatisticCollector::init() { FILE* file; @@ -42,6 +43,38 @@ static size_t write_callback(void* contents, size_t size, size_t nmemb, std::str return totalSize; } +static void send_job(std::string response_string, std::string job_group_name, std::string metric_name, + std::string metric_value) { + CURL* curl; + CURLcode res; + + curl = curl_easy_init(); + if (curl) { + std::string hostPGAddr = pushGatewayJobAddr + job_group_name; + curl_easy_setopt(curl, CURLOPT_URL, hostPGAddr.c_str()); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string); + std::string job_data = metric_name + " " + metric_value + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } + + curl_easy_cleanup(curl); + } +} + int StatisticCollector::getMemoryUsageByProcess() { FILE* file = fopen("/proc/self/status", "r"); int result = -1; @@ -58,6 +91,111 @@ int StatisticCollector::getMemoryUsageByProcess() { return result; } +int StatisticCollector::getThreadCount() { + FILE* file = fopen("/proc/self/stat", "r"); + int result = -1; + char line[128]; + + std::string response_thread_count; + + for(int i = 1; i <= 20; i++) { + fscanf(file, "%s%*c", line); + } + fclose(file); + result = atoi(line); + std::cout << "Thread Count: " + std::to_string(result) << std::endl; + + send_job(response_thread_count, "threadCount", "thread_count", std::to_string(result)); + + return result; +} + +int StatisticCollector::getUsedSwapSpace() { + FILE* file = fopen("/proc/swaps", "r"); + int result = -1; + char line[128]; + char used[128]; + + std::string response_used_swap; + + while(fgets(line, 128, file) != NULL) { + if(strncmp(line, "/swapfile", 9) == 0) { + for(int i = 1; i <= 4; i++) { + fscanf(file, "%s%*c", used); + } + } + } + + fclose(file); + result = atoi(used); + std::cout << "Used swap space: " + std::to_string(result) << std::endl; + + send_job(response_used_swap, "usedSwap", "used_swap_space", std::to_string(result)); + + return result; +} + +int StatisticCollector::getTotalSwapSpace() { + FILE* file = fopen("/proc/swaps", "r"); + int result = -1; + char line[128]; + char total[128]; + + std::string response_total_swap; + + while(fgets(line, 128, file) != NULL) { + if(strncmp(line, "/swapfile", 9) == 0) { + for(int i = 1; i <= 3; i++) { + fscanf(file, "%s%*c", total); + } + } + } + + fclose(file); + result = atoi(total); + std::cout << "Total swap space: " + std::to_string(result) << std::endl; + + send_job(response_total_swap, "totalSwap", "total_swap_space", std::to_string(result)); + + return result; +} + +int StatisticCollector::getRXBytes() { + FILE* file = fopen("/sys/class/net/eth0/statistics/rx_bytes", "r"); + int result = -1; + char rxBytes[128]; + + std::string response_rx; + + fscanf(file, "%s", rxBytes); + + fclose(file); + result = atoi(rxBytes); + std::cout << "Total read bytes: " + std::to_string(result) << std::endl; + + send_job(response_rx, "totalRead", "total_read_bytes", std::to_string(result)); + + return result; +} + +int StatisticCollector::getTXBytes() { + FILE* file = fopen("/sys/class/net/eth0/statistics/tx_bytes", "r"); + int result = -1; + char txBytes[128]; + + std::string response_tx; + + fscanf(file, "%s", txBytes); + + fclose(file); + result = atoi(txBytes); + std::cout << "Total sent bytes: " + std::to_string(result) << std::endl; + + send_job(response_tx, "totalSent", "total_sent_bytes", std::to_string(result)); + + return result; +} + int StatisticCollector::parseLine(char* line) { int i = strlen(line); const char* p = line; diff --git a/src/performance/metrics/StatisticCollector.h b/src/performance/metrics/StatisticCollector.h index d6cae31f4..739ca84f9 100644 --- a/src/performance/metrics/StatisticCollector.h +++ b/src/performance/metrics/StatisticCollector.h @@ -42,6 +42,11 @@ class StatisticCollector { public: int init(); static int getMemoryUsageByProcess(); + static int getThreadCount(); + static int getUsedSwapSpace(); + static int getTotalSwapSpace(); + static int getRXBytes(); + static int getTXBytes(); static int parseLine(char* line); static double getCpuUsage(); static std::string collectVMStatistics(std::string isVMStatManager, std::string isTotalAllocationRequired); diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index a19ca3c49..2b7c0dd87 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -671,6 +671,11 @@ map JasmineGraphInstanceService::getOutDegreeDistributionHashMap(map string JasmineGraphInstanceService::requestPerformanceStatistics(std::string isVMStatManager, std::string isResourceAllocationRequested) { int memoryUsage = collector.getMemoryUsageByProcess(); + int threadCount = collector.getThreadCount(); + int usedSwapSpace = collector.getUsedSwapSpace(); + int totalSwapSpace = collector.getTotalSwapSpace(); + int readBytes = collector.getRXBytes(); + int sentBytes = collector.getTXBytes(); double cpuUsage = collector.getCpuUsage(); double loadAverage = collector.getLoadAverage(); std::string vmLevelStatistics = collector.collectVMStatistics(isVMStatManager, isResourceAllocationRequested); @@ -679,7 +684,7 @@ string JasmineGraphInstanceService::requestPerformanceStatistics(std::string isV std::string reportTimeString(std::ctime(&reportTime)); reportTimeString = Utils::trim_copy(reportTimeString); std::string usageString = - reportTimeString + "," + to_string(memoryUsage) + "," + to_string(cpuUsage) + "," + to_string(loadAverage); + reportTimeString + "," + to_string(memoryUsage) + "," + to_string(threadCount) + "," + to_string(cpuUsage) + "," + to_string(loadAverage) + "," + to_string(usedSwapSpace) + "," + to_string(totalSwapSpace) + + "," + to_string(readBytes) + "," + to_string(sentBytes); if (!vmLevelStatistics.empty()) { usageString = usageString + "," + vmLevelStatistics; } From 72d230a36a3bd17be00557bfe31f653ab246a017 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sun, 21 Jan 2024 15:25:31 +0530 Subject: [PATCH 015/143] Improve get swap functions --- .../metrics/StatisticCollector.cpp | 69 ++++++++++--------- src/performance/metrics/StatisticCollector.h | 1 - 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index 0ce64aa28..d1cdc2b28 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -19,6 +19,9 @@ static clock_t lastCPU, lastSysCPU, lastUserCPU; static int numProcessors; std::string pushGatewayJobAddr = "http://192.168.8.150:9091/metrics/job/"; +static int parseLine(char* line); +static int getSwapSpace(const char* type); + int StatisticCollector::init() { FILE* file; struct tms timeSample; @@ -98,8 +101,8 @@ int StatisticCollector::getThreadCount() { std::string response_thread_count; - for(int i = 1; i <= 20; i++) { - fscanf(file, "%s%*c", line); + for (int i = 0; i < 20; i++) { + fscanf(file, "%127s%*c", line); } fclose(file); result = atoi(line); @@ -110,53 +113,55 @@ int StatisticCollector::getThreadCount() { return result; } -int StatisticCollector::getUsedSwapSpace() { +static int getSwapSpace(int field) { FILE* file = fopen("/proc/swaps", "r"); int result = -1; char line[128]; - char used[128]; - std::string response_used_swap; + fgets(line, 128, file); - while(fgets(line, 128, file) != NULL) { - if(strncmp(line, "/swapfile", 9) == 0) { - for(int i = 1; i <= 4; i++) { - fscanf(file, "%s%*c", used); + while (fgets(line, 128, file) != NULL) { + char* value; + char* save = NULL; + for (int i = 0; i < field; i++) { + if (i == 0) { + value = strtok_r(line, " ", &save); + } else { + value = strtok_r(NULL, "\t", &save); } } + int used = atoi(value); + if (used < 0) { + continue; + } + if (result >= 0) { + result += used; + } else { + result = used; + } } - fclose(file); - result = atoi(used); + + return result; +} + +int StatisticCollector::getUsedSwapSpace() { + int result = getSwapSpace(4); std::cout << "Used swap space: " + std::to_string(result) << std::endl; + // FIXME: Uninitialized string + std::string response_used_swap; send_job(response_used_swap, "usedSwap", "used_swap_space", std::to_string(result)); - return result; } int StatisticCollector::getTotalSwapSpace() { - FILE* file = fopen("/proc/swaps", "r"); - int result = -1; - char line[128]; - char total[128]; - - std::string response_total_swap; - - while(fgets(line, 128, file) != NULL) { - if(strncmp(line, "/swapfile", 9) == 0) { - for(int i = 1; i <= 3; i++) { - fscanf(file, "%s%*c", total); - } - } - } - - fclose(file); - result = atoi(total); + int result = getSwapSpace(3); std::cout << "Total swap space: " + std::to_string(result) << std::endl; + // FIXME: Uninitialized string + std::string response_total_swap; send_job(response_total_swap, "totalSwap", "total_swap_space", std::to_string(result)); - return result; } @@ -196,7 +201,7 @@ int StatisticCollector::getTXBytes() { return result; } -int StatisticCollector::parseLine(char* line) { +static int parseLine(char* line) { int i = strlen(line); const char* p = line; while (*p < '0' || *p > '9') p++; @@ -415,4 +420,4 @@ void StatisticCollector::logLoadAverage(std::string name) { sleep(Conts::LOAD_AVG_COLLECTING_GAP - elapsed); } } -} \ No newline at end of file +} diff --git a/src/performance/metrics/StatisticCollector.h b/src/performance/metrics/StatisticCollector.h index 739ca84f9..308ba0256 100644 --- a/src/performance/metrics/StatisticCollector.h +++ b/src/performance/metrics/StatisticCollector.h @@ -47,7 +47,6 @@ class StatisticCollector { static int getTotalSwapSpace(); static int getRXBytes(); static int getTXBytes(); - static int parseLine(char* line); static double getCpuUsage(); static std::string collectVMStatistics(std::string isVMStatManager, std::string isTotalAllocationRequired); static long getTotalMemoryAllocated(); From 8d7c137ec28274ec67046ddd3ed779fa87355446 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sun, 21 Jan 2024 16:06:18 +0530 Subject: [PATCH 016/143] Improve get network usage functions --- .../metrics/StatisticCollector.cpp | 57 ++++++++----------- src/performance/metrics/StatisticCollector.h | 10 ++-- src/server/JasmineGraphInstanceService.cpp | 10 ++-- 3 files changed, 35 insertions(+), 42 deletions(-) diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index d1cdc2b28..1919c3100 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -19,8 +19,8 @@ static clock_t lastCPU, lastSysCPU, lastUserCPU; static int numProcessors; std::string pushGatewayJobAddr = "http://192.168.8.150:9091/metrics/job/"; -static int parseLine(char* line); -static int getSwapSpace(const char* type); +static long parseLine(char* line); +static long getSwapSpace(const char* type); int StatisticCollector::init() { FILE* file; @@ -78,9 +78,9 @@ static void send_job(std::string response_string, std::string job_group_name, st } } -int StatisticCollector::getMemoryUsageByProcess() { +long StatisticCollector::getMemoryUsageByProcess() { FILE* file = fopen("/proc/self/status", "r"); - int result = -1; + long result = -1; char line[128]; while (fgets(line, 128, file) != NULL) { @@ -113,7 +113,7 @@ int StatisticCollector::getThreadCount() { return result; } -static int getSwapSpace(int field) { +static long getSwapSpace(int field) { FILE* file = fopen("/proc/swaps", "r"); int result = -1; char line[128]; @@ -130,8 +130,8 @@ static int getSwapSpace(int field) { value = strtok_r(NULL, "\t", &save); } } - int used = atoi(value); - if (used < 0) { + long used = strtol(value, NULL, 10); + if (used < 0 || used > 0xfffffffffffffffL) { continue; } if (result >= 0) { @@ -145,8 +145,8 @@ static int getSwapSpace(int field) { return result; } -int StatisticCollector::getUsedSwapSpace() { - int result = getSwapSpace(4); +long StatisticCollector::getUsedSwapSpace() { + long result = getSwapSpace(4); std::cout << "Used swap space: " + std::to_string(result) << std::endl; // FIXME: Uninitialized string @@ -155,8 +155,8 @@ int StatisticCollector::getUsedSwapSpace() { return result; } -int StatisticCollector::getTotalSwapSpace() { - int result = getSwapSpace(3); +long StatisticCollector::getTotalSwapSpace() { + long result = getSwapSpace(3); std::cout << "Total swap space: " + std::to_string(result) << std::endl; // FIXME: Uninitialized string @@ -165,49 +165,42 @@ int StatisticCollector::getTotalSwapSpace() { return result; } -int StatisticCollector::getRXBytes() { +long StatisticCollector::getRXBytes() { FILE* file = fopen("/sys/class/net/eth0/statistics/rx_bytes", "r"); - int result = -1; - char rxBytes[128]; - - std::string response_rx; - - fscanf(file, "%s", rxBytes); - + long result = -1; + fscanf(file, "%li", &result); fclose(file); - result = atoi(rxBytes); std::cout << "Total read bytes: " + std::to_string(result) << std::endl; + // FIXME: Uninitialized string + std::string response_rx; send_job(response_rx, "totalRead", "total_read_bytes", std::to_string(result)); return result; } -int StatisticCollector::getTXBytes() { +long StatisticCollector::getTXBytes() { FILE* file = fopen("/sys/class/net/eth0/statistics/tx_bytes", "r"); - int result = -1; - char txBytes[128]; - - std::string response_tx; - - fscanf(file, "%s", txBytes); - + long result = -1; + fscanf(file, "%li", &result); fclose(file); - result = atoi(txBytes); std::cout << "Total sent bytes: " + std::to_string(result) << std::endl; + // FIXME: Uninitialized string + std::string response_tx; send_job(response_tx, "totalSent", "total_sent_bytes", std::to_string(result)); return result; } -static int parseLine(char* line) { +static long parseLine(char* line) { int i = strlen(line); const char* p = line; while (*p < '0' || *p > '9') p++; line[i - 3] = '\0'; - i = atoi(p); - return i; + long val = strtol(p, NULL, 10); + if (val < 0 || val > 0xfffffffffffffffL) return -1; + return val; } double StatisticCollector::getCpuUsage() { diff --git a/src/performance/metrics/StatisticCollector.h b/src/performance/metrics/StatisticCollector.h index 308ba0256..a4e804a46 100644 --- a/src/performance/metrics/StatisticCollector.h +++ b/src/performance/metrics/StatisticCollector.h @@ -41,12 +41,12 @@ class StatisticCollector { public: int init(); - static int getMemoryUsageByProcess(); + static long getMemoryUsageByProcess(); static int getThreadCount(); - static int getUsedSwapSpace(); - static int getTotalSwapSpace(); - static int getRXBytes(); - static int getTXBytes(); + static long getUsedSwapSpace(); + static long getTotalSwapSpace(); + static long getRXBytes(); + static long getTXBytes(); static double getCpuUsage(); static std::string collectVMStatistics(std::string isVMStatManager, std::string isTotalAllocationRequired); static long getTotalMemoryAllocated(); diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 2b7c0dd87..f86f2cfc3 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -670,12 +670,12 @@ map JasmineGraphInstanceService::getOutDegreeDistributionHashMap(map string JasmineGraphInstanceService::requestPerformanceStatistics(std::string isVMStatManager, std::string isResourceAllocationRequested) { - int memoryUsage = collector.getMemoryUsageByProcess(); + long memoryUsage = collector.getMemoryUsageByProcess(); int threadCount = collector.getThreadCount(); - int usedSwapSpace = collector.getUsedSwapSpace(); - int totalSwapSpace = collector.getTotalSwapSpace(); - int readBytes = collector.getRXBytes(); - int sentBytes = collector.getTXBytes(); + long usedSwapSpace = collector.getUsedSwapSpace(); + long totalSwapSpace = collector.getTotalSwapSpace(); + long readBytes = collector.getRXBytes(); + long sentBytes = collector.getTXBytes(); double cpuUsage = collector.getCpuUsage(); double loadAverage = collector.getLoadAverage(); std::string vmLevelStatistics = collector.collectVMStatistics(isVMStatManager, isResourceAllocationRequested); From 70d6de65d2a9e0a9a34dcec67b4a0c44ef01a9fc Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sun, 21 Jan 2024 16:27:40 +0530 Subject: [PATCH 017/143] Improve get thread count function --- src/performance/metrics/StatisticCollector.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index 1919c3100..b88cec469 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -96,26 +96,21 @@ long StatisticCollector::getMemoryUsageByProcess() { int StatisticCollector::getThreadCount() { FILE* file = fopen("/proc/self/stat", "r"); - int result = -1; + long result; char line[128]; - std::string response_thread_count; - for (int i = 0; i < 20; i++) { - fscanf(file, "%127s%*c", line); + if (fscanf(file, "%127s%*c", line) < 0) return -1; } fclose(file); - result = atoi(line); - std::cout << "Thread Count: " + std::to_string(result) << std::endl; - - send_job(response_thread_count, "threadCount", "thread_count", std::to_string(result)); - + result = strtol(line, NULL, 10); + if (result <= 0 || result > 0xfffffffffffffffL) return -1; return result; } static long getSwapSpace(int field) { FILE* file = fopen("/proc/swaps", "r"); - int result = -1; + long result = -1; char line[128]; fgets(line, 128, file); From d10a1b4cbb03305cada66905c1f7cec556358125 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sun, 21 Jan 2024 16:30:38 +0530 Subject: [PATCH 018/143] Add get socket count function --- .../metrics/StatisticCollector.cpp | 24 +++++++++++++++++++ src/performance/metrics/StatisticCollector.h | 1 + 2 files changed, 25 insertions(+) diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index b88cec469..e65ea8096 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -188,6 +188,30 @@ long StatisticCollector::getTXBytes() { return result; } +int StatisticCollector::getSocketCount() { + DIR* d = opendir("/proc/self/fd"); + if (!d) { + puts("Error opening directory /proc/self/fd"); + return -1; + } + const struct dirent* dir; + char path[64]; + char link_buf[1024]; + int count = 0; + while ((dir = readdir(d)) != NULL) { + const char* filename = dir->d_name; + if (filename[0] < '0' || '9' < filename[0]) continue; + sprintf(path, "/proc/self/fd/%s", filename); + size_t len = readlink(path, link_buf, sizeof(link_buf) - 1); + link_buf[len] = 0; + if (len > 0 && strncmp("socket:", link_buf, 7) == 0) { + count++; + } + } + (void)closedir(d); + return count; +} + static long parseLine(char* line) { int i = strlen(line); const char* p = line; diff --git a/src/performance/metrics/StatisticCollector.h b/src/performance/metrics/StatisticCollector.h index a4e804a46..83a0e1773 100644 --- a/src/performance/metrics/StatisticCollector.h +++ b/src/performance/metrics/StatisticCollector.h @@ -47,6 +47,7 @@ class StatisticCollector { static long getTotalSwapSpace(); static long getRXBytes(); static long getTXBytes(); + static int getSocketCount(); static double getCpuUsage(); static std::string collectVMStatistics(std::string isVMStatManager, std::string isTotalAllocationRequired); static long getTotalMemoryAllocated(); From 06a0acbc693b0286eabb9d1b608e0dcf03c2c39b Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 26 Jan 2024 09:29:32 +0530 Subject: [PATCH 019/143] Replace hardcoded urls --- conf/jasminegraph-server.properties | 3 +++ src/performance/metrics/PerformanceUtil.cpp | 16 ++++++++-------- src/performance/metrics/StatisticCollector.cpp | 4 ++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 836e218b7..69c171caa 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -76,6 +76,9 @@ org.jasminegraph.scheduler.enabled=true #PerformanceCollector Scheduler Timing. Run once every 120 seconds org.jasminegraph.scheduler.performancecollector.timing=120 +#pushgateway address +org.jasminegraph.collector.pushgateway=http://localhost:9091/metrics/job/ + #-------------------------------------------------------------------------------- #MetaDB information #-------------------------------------------------------------------------------- diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index 5194b71cb..742c7008a 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -17,7 +17,7 @@ limitations under the License. using namespace std::chrono; std::map> resourceUsageMap; -std::string pushGatewayAddr = "http://192.168.8.150:9091/metrics/job/"; +std::string pushGatewayAddr = Utils::getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); static size_t write_callback(void* contents, size_t size, size_t nmemb, std::string* output); @@ -349,7 +349,7 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s curl = curl_easy_init(); if (curl) { // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/hostPerfData"); + curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "hostPerfData").c_str()); // Set the callback function to handle the response data curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); @@ -375,7 +375,7 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s curl = curl_easy_init(); if (curl) { // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/hostPerfData"); + curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "hostPerfData").c_str()); // Set the callback function to handle the response data curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); @@ -422,7 +422,7 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s curl = curl_easy_init(); if (curl) { // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/placePerfData"); + curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "placePerfData").c_str()); // Set the callback function to handle the response data curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); @@ -448,7 +448,7 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s curl = curl_easy_init(); if (curl) { // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/placePerfData"); + curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "placePerfData").c_str()); // Set the callback function to handle the response data curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); @@ -1074,7 +1074,7 @@ void PerformanceUtil::logLoadAverage() { curl = curl_easy_init(); if (curl) { // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/loadAverage"); + curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "loadAverage").c_str()); // Set the callback function to handle the response data curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); @@ -1150,7 +1150,7 @@ void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *pe curl = curl_easy_init(); if (curl) { // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/loadAverageSLA"); + curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "loadAverageSLA").c_str()); // Set the callback function to handle the response data curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); @@ -1251,7 +1251,7 @@ void PerformanceUtil::updateRemoteResourceConsumption(PerformanceSQLiteDBInterfa curl = curl_easy_init(); if (curl) { // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/loadAverageSLARem"); + curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "loadAverageSLARem").c_str()); // Set the callback function to handle the response data curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index e65ea8096..02ef0c17d 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -17,7 +17,7 @@ limitations under the License. static clock_t lastCPU, lastSysCPU, lastUserCPU; static int numProcessors; -std::string pushGatewayJobAddr = "http://192.168.8.150:9091/metrics/job/"; +std::string pushGatewayJobAddr = Utils::getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); static long parseLine(char* line); static long getSwapSpace(const char* type); @@ -323,7 +323,7 @@ long StatisticCollector::getTotalMemoryUsage() { curl = curl_easy_init(); if (curl) { - curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/totalMemory"); + curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayJobAddr + "totalMemory").c_str()); // Set the callback function to handle the response data curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); From 1227dd5b14077e5a3d4e5f892fddc301d243d44f Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 26 Jan 2024 12:16:21 +0530 Subject: [PATCH 020/143] Move send job function to utils and remove redundant code --- conf/jasminegraph-server.properties | 2 +- src/performance/metrics/PerformanceUtil.cpp | 264 ++---------------- .../metrics/StatisticCollector.cpp | 98 +------ src/util/Utils.cpp | 44 ++- src/util/Utils.h | 5 +- 5 files changed, 75 insertions(+), 338 deletions(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 69c171caa..beaaaf42d 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -77,7 +77,7 @@ org.jasminegraph.scheduler.enabled=true org.jasminegraph.scheduler.performancecollector.timing=120 #pushgateway address -org.jasminegraph.collector.pushgateway=http://localhost:9091/metrics/job/ +org.jasminegraph.collector.pushgateway=http://192.168.8.163:9091/metrics/job/ #-------------------------------------------------------------------------------- #MetaDB information diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index 742c7008a..9c05772a2 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -13,8 +13,6 @@ limitations under the License. #include "PerformanceUtil.h" -#include - using namespace std::chrono; std::map> resourceUsageMap; std::string pushGatewayAddr = Utils::getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); @@ -214,41 +212,6 @@ static size_t write_callback(void* contents, size_t size, size_t nmemb, std::str return totalSize; } -static void send_job(std::string response_string, std::string job_group_name, std::string metric_name, std::string metric_value, std::string addr){ - CURL* curl; - CURLcode res; - //std::string response_host_perf_data; - - curl = curl_easy_init(); - if (curl) { - //std::string hostPGAddr = pushGatewayAddr + "hostPerfData_" + to_string(port); - std::string hostPGAddr = pushGatewayAddr + job_group_name + addr; - //curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.8.150:9091/metrics/job/hostPerfData"); - curl_easy_setopt(curl, CURLOPT_URL, hostPGAddr.c_str()); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string); - //std::string job_data = "memory_consumption " + memoryConsumption + "\n"; - std::string job_data = metric_name + " " + metric_value + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } -} - void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, std::string isVMStatManager, std::string isResourceAllocationRequired, std::string hostId, std::string placeId) { @@ -341,62 +304,8 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s processTime + "','" + memoryConsumption + "','" + cpuUsage + "','" + hostId + "')"; perfDb->runInsert(vmPerformanceSql); - - CURL* curl; - CURLcode res; - std::string response_host_perf_data; - - curl = curl_easy_init(); - if (curl) { - // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "hostPerfData").c_str()); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_host_perf_data); - std::string job_data = "memory_consumption " + memoryConsumption + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } - - curl = curl_easy_init(); - if (curl) { - // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "hostPerfData").c_str()); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_host_perf_data); - std::string job_data = "cpu_usage " + cpuUsage + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } + Utils::send_job("hostPerfData", "memory_consumption", memoryConsumption); + Utils::send_job("hostPerfData", "cpu_usage", cpuUsage); if (isResourceAllocationRequired == "true") { std::string totalMemory = strArr[6]; @@ -413,63 +322,9 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s "insert into place_performance_data (idplace, memory_usage, cpu_usage, date_time) values ('" + placeId + "','" + memoryUsage + "','" + cpuUsage + "','" + processTime + "')"; - perfDb.runInsert(placePerfSql); - - CURL* curl; - CURLcode res; - std::string response_place_perf_data; - - curl = curl_easy_init(); - if (curl) { - // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "placePerfData").c_str()); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_place_perf_data); - std::string job_data = "memory_usage " + memoryUsage + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } - - curl = curl_easy_init(); - if (curl) { - // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "placePerfData").c_str()); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_place_perf_data); - std::string job_data = "cpu_usage " + cpuUsage + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } + perfDb->runInsert(placePerfSql); + Utils::send_job("placePerfData", "memory_usage", memoryUsage); + Utils::send_job("placePerfData", "cpu_usage", cpuUsage); } } } @@ -502,9 +357,8 @@ void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, s perfDb->runInsert(vmPerformanceSql); - std::string response_local_perf_data; - send_job(response_local_perf_data, "localPerfDataO_", "total_memory_usage", totalMemoryUsed, hostId); - send_job(response_local_perf_data, "localPerfDataO_", "total_CPU_usage", totalCPUUsage, hostId); + Utils::send_job("localPerfDataO_" + hostId, "total_memory_usage", totalMemoryUsed); + Utils::send_job("localPerfDataO_" + hostId, "total_CPU_usage", totalCPUUsage); if (isResourceAllocationRequired == "true") { std::string totalMemory = strArr[2]; @@ -520,11 +374,12 @@ void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, s placeId + "','" + to_string(memoryUsage) + "','" + to_string(cpuUsage) + "','" + reportTimeString + "')"; - perfDb.runInsert(placePerfSql); + perfDb->runInsert(placePerfSql); - std::string response_place_perf_data_local; - send_job(response_place_perf_data_local, "placePerfDataLocalO_", "memory_usage", to_string(memoryUsage), hostId); - send_job(response_place_perf_data_local, "placePerfDataLocalO_", "cpu_usage", to_string(cpuUsage), hostId); + Utils::send_job("placePerfDataLocalO_" + hostId, "memory_usage", + to_string(memoryUsage)); + Utils::send_job("placePerfDataLocalO_" + hostId, "cpu_usage", + to_string(cpuUsage)); } @@ -1066,36 +921,7 @@ void PerformanceUtil::logLoadAverage() { double currentLoadAverage = statisticCollector.getLoadAverage(); std::cout << "###PERF### CURRENT LOAD: " + std::to_string(currentLoadAverage) << std::endl; - - CURL* curl; - CURLcode res; - std::string response_load_avg; - - curl = curl_easy_init(); - if (curl) { - // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "loadAverage").c_str()); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_load_avg); - std::string job_data = "load_average " + std::to_string(currentLoadAverage) + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } + Utils::send_job("loadAverage", "load_average", std::to_string(currentLoadAverage)); } void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *performanceDb, std::string graphId, @@ -1141,39 +967,11 @@ void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *pe ResourceUsageInfo usageInfo = *usageIterator; valuesString += "('" + graphSlaId + "','" + placeId + "', '','" + usageInfo.loadAverage + "','" + - usageInfo.elapsedTime + "'),"; - - CURL* curl; - CURLcode res; - std::string response_load_avg_sla; + usageInfo.elapsedTime + "'),"; - curl = curl_easy_init(); - if (curl) { - // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "loadAverageSLA").c_str()); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_load_avg_sla); - std::string job_data = "load_average " + usageInfo.loadAverage + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } + //FIXME(Aruna): haven't done this per host basis + Utils::send_job("loadAverageSLA", "load_average", usageInfo.loadAverage); } - valuesString = valuesString.substr(0, valuesString.length() - 1); slaPerfSql = slaPerfSql + valuesString; @@ -1244,35 +1042,7 @@ void PerformanceUtil::updateRemoteResourceConsumption(PerformanceSQLiteDBInterfa std::to_string(elapsedTime * 1000) + "'),"; elapsedTime += 5; - CURL* curl; - CURLcode res; - std::string response_load_avg_sla_rem; - - curl = curl_easy_init(); - if (curl) { - // curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayAddr + "loadAverageSLARem").c_str()); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_load_avg_sla_rem); - std::string job_data = "load_average_rem " + loadAverage + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } + Utils::send_job("loadAverageSLARem", "load_average_rem", loadAverage); } valuesString = valuesString.substr(0, valuesString.length() - 1); diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index 02ef0c17d..1697a9766 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -13,11 +13,9 @@ limitations under the License. #include "StatisticCollector.h" -#include - static clock_t lastCPU, lastSysCPU, lastUserCPU; static int numProcessors; -std::string pushGatewayJobAddr = Utils::getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); +std::string pushGatewayJobAddr = Utils::getJasmineGraphProperty(""); static long parseLine(char* line); static long getSwapSpace(const char* type); @@ -40,44 +38,6 @@ int StatisticCollector::init() { return 0; } -static size_t write_callback(void* contents, size_t size, size_t nmemb, std::string* output) { - size_t totalSize = size * nmemb; - output->append(static_cast(contents), totalSize); - return totalSize; -} - -static void send_job(std::string response_string, std::string job_group_name, std::string metric_name, - std::string metric_value) { - CURL* curl; - CURLcode res; - - curl = curl_easy_init(); - if (curl) { - std::string hostPGAddr = pushGatewayJobAddr + job_group_name; - curl_easy_setopt(curl, CURLOPT_URL, hostPGAddr.c_str()); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string); - std::string job_data = metric_name + " " + metric_value + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } -} - long StatisticCollector::getMemoryUsageByProcess() { FILE* file = fopen("/proc/self/status", "r"); long result = -1; @@ -143,20 +103,16 @@ static long getSwapSpace(int field) { long StatisticCollector::getUsedSwapSpace() { long result = getSwapSpace(4); std::cout << "Used swap space: " + std::to_string(result) << std::endl; - - // FIXME: Uninitialized string - std::string response_used_swap; - send_job(response_used_swap, "usedSwap", "used_swap_space", std::to_string(result)); + std::string response_used_swap = Utils::send_job("usedSwap", "used_swap_space", + std::to_string(result)); return result; } long StatisticCollector::getTotalSwapSpace() { long result = getSwapSpace(3); std::cout << "Total swap space: " + std::to_string(result) << std::endl; - - // FIXME: Uninitialized string - std::string response_total_swap; - send_job(response_total_swap, "totalSwap", "total_swap_space", std::to_string(result)); + std::string response_total_swap = Utils:: send_job("totalSwap", "total_swap_space", + std::to_string(result)); return result; } @@ -166,11 +122,8 @@ long StatisticCollector::getRXBytes() { fscanf(file, "%li", &result); fclose(file); std::cout << "Total read bytes: " + std::to_string(result) << std::endl; - - // FIXME: Uninitialized string - std::string response_rx; - send_job(response_rx, "totalRead", "total_read_bytes", std::to_string(result)); - + std::string response_rx = Utils::send_job("totalRead", "total_read_bytes", + std::to_string(result)); return result; } @@ -181,9 +134,8 @@ long StatisticCollector::getTXBytes() { fclose(file); std::cout << "Total sent bytes: " + std::to_string(result) << std::endl; - // FIXME: Uninitialized string - std::string response_tx; - send_job(response_tx, "totalSent", "total_sent_bytes", std::to_string(result)); + std::string response_tx = Utils::send_job("totalSent", "total_sent_bytes", + std::to_string(result)); return result; } @@ -300,9 +252,6 @@ long StatisticCollector::getTotalMemoryUsage() { unsigned long sReclaimable; unsigned long memUsage; - CURL* curl; - CURLcode res; - while (file >> token) { if (token == "MemTotal:") { file >> memTotal; @@ -319,33 +268,8 @@ long StatisticCollector::getTotalMemoryUsage() { } memUsage = memTotal - (memFree + buffers + cached + sReclaimable); - std::string response_total_memory; - - curl = curl_easy_init(); - if (curl) { - curl_easy_setopt(curl, CURLOPT_URL, (pushGatewayJobAddr + "totalMemory").c_str()); - - // Set the callback function to handle the response data - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_total_memory); - std::string job_data = "total_memory " + to_string(memTotal) + "\n"; - const char* data = job_data.c_str(); - - curl_slist* headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); - curl_easy_setopt(curl, CURLOPT_POST, 1); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); - } - + std::string response_total_memory = Utils::send_job("totalMemory", "total_memory", + std::to_string(memTotal)); return memUsage; } diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index c3b534c77..285a93b26 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -15,8 +15,8 @@ limitations under the License. #include #include -#include #include +#include #include #include @@ -695,6 +695,48 @@ int Utils::createDatabaseFromDDL(const char *dbLocation, const char *ddlFileLoca return 0; } +static size_t write_callback(void* contents, size_t size, size_t nmemb, std::string* output) { + size_t totalSize = size * nmemb; + output->append(static_cast(contents), totalSize); + return totalSize; +} + +std::string Utils::send_job(std::string job_group_name, std::string metric_name, + std::string metric_value) { + CURL* curl; + CURLcode res; + std::string pushGatewayJobAddr = getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); + + std::string response_string; + curl = curl_easy_init(); + if (curl) { + std::string hostPGAddr = pushGatewayJobAddr + job_group_name; + curl_easy_setopt(curl, CURLOPT_URL, hostPGAddr.c_str()); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string); + std::string job_data = metric_name + " " + metric_value + "\n"; + const char* data = job_data.c_str(); + + curl_slist* headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "curl failed: " << curl_easy_strerror(res) << "| url: " + << hostPGAddr << "| data: " << job_data << std::endl; + } + + curl_easy_cleanup(curl); + } + return response_string; +} + bool Utils::fileExistsWithReadPermission(const string &path) { return access(path.c_str(), R_OK) == 0; } diff --git a/src/util/Utils.h b/src/util/Utils.h index 59a2e825a..10bf39ea2 100644 --- a/src/util/Utils.h +++ b/src/util/Utils.h @@ -166,9 +166,10 @@ class Utils { static std::string getCurrentTimestamp(); - static std::string getJsonStringFromYamlFile(const std::string &yamlFile); + static int createDatabaseFromDDL(const char* dbLocation, const char* ddlFileLocation); - static int createDatabaseFromDDL(const char *dbLocation, const char *ddlFileLocation); + static std::string send_job(std::string job_group_name, std::string metric_name, + std::string metric_value); }; #endif // JASMINEGRAPH_UTILS_H From cb3b8a00d368cbb407cd12409a43a6e54ffa2309 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Tue, 6 Feb 2024 10:17:56 +0530 Subject: [PATCH 021/143] Fix errors after rebase --- conf/jasminegraph-server.properties | 2 +- src/util/Utils.cpp | 1 + src/util/Utils.h | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index beaaaf42d..f8d81b532 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -77,7 +77,7 @@ org.jasminegraph.scheduler.enabled=true org.jasminegraph.scheduler.performancecollector.timing=120 #pushgateway address -org.jasminegraph.collector.pushgateway=http://192.168.8.163:9091/metrics/job/ +org.jasminegraph.collector.pushgateway=http://10.43.43.131:9091/metrics/job/ #-------------------------------------------------------------------------------- #MetaDB information diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 285a93b26..3e5d39c7d 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -665,6 +665,7 @@ std::string Utils::getJsonStringFromYamlFile(const std::string &yamlFile) { YAML::Node yamlNode = YAML::LoadFile(yamlFile); return to_string(yaml2json(yamlNode)); } + int Utils::createDatabaseFromDDL(const char *dbLocation, const char *ddlFileLocation) { if (!Utils::fileExists(ddlFileLocation)) { util_logger.error("DDL file not found: " + string(ddlFileLocation)); diff --git a/src/util/Utils.h b/src/util/Utils.h index 10bf39ea2..d674a6c7d 100644 --- a/src/util/Utils.h +++ b/src/util/Utils.h @@ -166,6 +166,8 @@ class Utils { static std::string getCurrentTimestamp(); + static std::string getJsonStringFromYamlFile(const std::string &yamlFile); + static int createDatabaseFromDDL(const char* dbLocation, const char* ddlFileLocation); static std::string send_job(std::string job_group_name, std::string metric_name, From 92a878212f63ed181a54f6132f27216da7e7c07e Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Tue, 6 Feb 2024 10:19:35 +0530 Subject: [PATCH 022/143] Add pushgateway definition file --- k8s/pushgateway.yaml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 k8s/pushgateway.yaml diff --git a/k8s/pushgateway.yaml b/k8s/pushgateway.yaml new file mode 100644 index 000000000..f0355329f --- /dev/null +++ b/k8s/pushgateway.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pushgateway + labels: + app: pushgateway +spec: + replicas: 1 + selector: + matchLabels: + app: pushgateway + template: + metadata: + name: pushgateway + labels: + app: pushgateway + spec: + containers: + - name: pushgateway + image: prom/pushgateway + ports: + - containerPort: 9091 + imagePullPolicy: IfNotPresent + restartPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: pushgateway + labels: + app: pushgateway +spec: + ports: + - port: 9091 + targetPort: 9091 + selector: + app: pushgateway + \ No newline at end of file From 188685bbb6c7da785699412dea65eb91027fce6e Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Tue, 6 Feb 2024 10:39:36 +0530 Subject: [PATCH 023/143] Add prometheus definition file --- k8s/prometheus.yaml | 62 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 k8s/prometheus.yaml diff --git a/k8s/prometheus.yaml b/k8s/prometheus.yaml new file mode 100644 index 000000000..32a479cb0 --- /dev/null +++ b/k8s/prometheus.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-server-conf + labels: + app: prometheus +data: + prometheus.yml: | + global: + scrape_interval: 15s + external_labels: + monitor: 'codelab-monitor' + scrape_configs: + - job_name: 'prometheus' + scrape_interval: 5s + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'pushgateway' + scrape_interval: 2s + static_configs: + - targets: ['${pushgateway-address}:9091'] + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + labels: + app: prometheus +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + name: prometheus + labels: + app: prometheus + spec: + containers: + - name: prometheus + image: prom/prometheus + ports: + - containerPort: 9090 + imagePullPolicy: IfNotPresent + restartPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + labels: + app: prometheus +spec: + ports: + - port: 9090 + targetPort: 9090 + selector: + app: prometheus From 8de1c7ddff72d108fcff4db099059344638281df Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Wed, 7 Feb 2024 06:31:22 +0530 Subject: [PATCH 024/143] start pushgateway and prometheus in start-k8s --- conf/jasminegraph-server.properties | 2 +- k8s/prometheus.yaml | 9 ++++++++- start-k8s.sh | 24 ++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index f8d81b532..b88e041df 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -77,7 +77,7 @@ org.jasminegraph.scheduler.enabled=true org.jasminegraph.scheduler.performancecollector.timing=120 #pushgateway address -org.jasminegraph.collector.pushgateway=http://10.43.43.131:9091/metrics/job/ +org.jasminegraph.collector.pushgateway=http://10.43.188.76:9091/metrics/job/ #-------------------------------------------------------------------------------- #MetaDB information diff --git a/k8s/prometheus.yaml b/k8s/prometheus.yaml index 32a479cb0..4f49f940d 100644 --- a/k8s/prometheus.yaml +++ b/k8s/prometheus.yaml @@ -19,7 +19,7 @@ data: - job_name: 'pushgateway' scrape_interval: 2s static_configs: - - targets: ['${pushgateway-address}:9091'] + - targets: ['${pushgateway_address}'] --- apiVersion: apps/v1 @@ -45,6 +45,13 @@ spec: ports: - containerPort: 9090 imagePullPolicy: IfNotPresent + volumeMounts: + - name: prometheus-config + mountPath: /etc/prometheus + volumes: + - name: prometheus-config + configMap: + name: prometheus-server-conf restartPolicy: Always --- diff --git a/start-k8s.sh b/start-k8s.sh index a6af383c1..9cf0eabae 100755 --- a/start-k8s.sh +++ b/start-k8s.sh @@ -51,6 +51,30 @@ if [ -z "$ENABLE_NMON" ]; then fi kubectl apply -f ./k8s/rbac.yaml +kubectl apply -f ./k8s/pushgateway.yaml + +# wait until pushgateway starts listening +cur_timestamp="$(date +%s)" +end_timestamp="$((cur_timestamp + TIMEOUT_SECONDS))" +spin="/-\|" +i=0 +while true; do + if [ "$(date +%s)" -gt "$end_timestamp" ]; then + echo "Timeout" + exit 1 + fi + pushgatewayIP="$(kubectl get services |& grep pushgateway | tr '\t' ' ' | tr -s ' ' | cut -d ' ' -f 3)" + if [ ! -z "$pushgatewayIP" ]; then + break + fi + printf "Waiting pushgateway to start [%c] \r" "${spin:i++%${#spin}:1}" + sleep .2 +done + +pushgateway_address="${pushgatewayIP}:9091" envsubst <"./k8s/prometheus.yaml" | kubectl apply -f - +sed -i "s#org.jasminegraph.collector.pushgateway=.*#org.jasminegraph.collector.pushgateway=http://${pushgatewayIP}:9091/metrics/job/#" ./conf/jasminegraph-server.properties + +docker build -t jasminegraph . metadb_path="${META_DB_PATH}" \ performancedb_path="${PERFORMANCE_DB_PATH}" \ From a52fcccd845a6006c489da9bf9198f84c744ff7f Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 9 Feb 2024 16:36:17 +0530 Subject: [PATCH 025/143] Fix unit test image build failure --- tests/unit/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/Dockerfile b/tests/unit/Dockerfile index 95dbfbe96..545ebaa71 100644 --- a/tests/unit/Dockerfile +++ b/tests/unit/Dockerfile @@ -1,5 +1,10 @@ FROM miyurud/jasminegraph-prerequisites:20240101T095619 +RUN apt-get update +RUN apt-get install libcurl4-openssl-dev -y +RUN apt-get install -y sysstat +RUN apt-get install -y nmon + RUN ln -sf /usr/bin/python3.8 /usr/bin/python3 RUN ln -sf /usr/include/python3.8 /usr/include/python3 RUN ln -sf /usr/lib/x86_64-linux-gnu/libpython3.8.so /usr/lib/x86_64-linux-gnu/libpython3.so From 5cd2798cecd0b456b09f8a1f60692518b8d63ea8 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 9 Feb 2024 17:09:14 +0530 Subject: [PATCH 026/143] Fix style problems --- src/performance/metrics/PerformanceUtil.cpp | 3 +-- src/server/JasmineGraphInstanceService.cpp | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index 9c05772a2..f73953fda 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -380,7 +380,6 @@ void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, s to_string(memoryUsage)); Utils::send_job("placePerfDataLocalO_" + hostId, "cpu_usage", to_string(cpuUsage)); - } int PerformanceUtil::collectRemoteSLAResourceUtilization(std::string host, int port, std::string isVMStatManager, @@ -969,7 +968,7 @@ void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *pe valuesString += "('" + graphSlaId + "','" + placeId + "', '','" + usageInfo.loadAverage + "','" + usageInfo.elapsedTime + "'),"; - //FIXME(Aruna): haven't done this per host basis + // FIXME(Aruna): haven't done this per host basis Utils::send_job("loadAverageSLA", "load_average", usageInfo.loadAverage); } valuesString = valuesString.substr(0, valuesString.length() - 1); diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index f86f2cfc3..624323d9d 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -684,7 +684,9 @@ string JasmineGraphInstanceService::requestPerformanceStatistics(std::string isV std::string reportTimeString(std::ctime(&reportTime)); reportTimeString = Utils::trim_copy(reportTimeString); std::string usageString = - reportTimeString + "," + to_string(memoryUsage) + "," + to_string(threadCount) + "," + to_string(cpuUsage) + "," + to_string(loadAverage) + "," + to_string(usedSwapSpace) + "," + to_string(totalSwapSpace) + + "," + to_string(readBytes) + "," + to_string(sentBytes); + reportTimeString + "," + to_string(memoryUsage) + "," + to_string(threadCount) + "," + + to_string(cpuUsage) + "," + to_string(loadAverage) + "," + to_string(usedSwapSpace) + "," + + to_string(totalSwapSpace) + + "," + to_string(readBytes) + "," + to_string(sentBytes); if (!vmLevelStatistics.empty()) { usageString = usageString + "," + vmLevelStatistics; } From c03075e75829063cd241459a924516235c15e991 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 9 Feb 2024 18:15:07 +0530 Subject: [PATCH 027/143] Fix unit test build failure --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a812204f7..dec67eff2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,7 @@ target_link_libraries(JasmineGraphLib PRIVATE ${PYTHON_LIBRARIES}) target_link_libraries(JasmineGraphLib PRIVATE m) target_link_libraries(JasmineGraphLib PRIVATE /usr/local/lib/libkubernetes.so) target_link_libraries(JasmineGraphLib PRIVATE yaml-cpp) +target_link_libraries(JasmineGraphLib PRIVATE curl) if (CMAKE_BUILD_TYPE STREQUAL "DEBUG") # Include google test From c6a4f4c0234d5a0d3e25cd70193a7f2fb3f0d121 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 9 Feb 2024 18:42:10 +0530 Subject: [PATCH 028/143] Add script option to clear all jasminegraph related objects --- README.md | 5 +++++ conf/jasminegraph-server.properties | 2 +- k8s/master-deployment.yaml | 2 ++ k8s/master-service.yaml | 1 + k8s/prometheus.yaml | 3 +++ k8s/pushgateway.yaml | 2 ++ k8s/volumes.yaml | 12 ++++++++++++ k8s/worker-deployment.yaml | 1 + k8s/worker-service.yaml | 1 + start-k8s.sh | 8 ++++++++ 10 files changed, 36 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a107ad5fb..89a924033 100755 --- a/README.md +++ b/README.md @@ -31,6 +31,11 @@ Run JasmineGraph in Kubernetes environment by providing appropriate parameters: >Note: `NO_OF_WORKERS` and `ENABLE_NMON` are optional arguments, and it defaults to `2` and `false` respectively. +Remove all resources created by JasmineGraph in Kubernetes environment: + + ./stop-k8s.sh clean + + ## 3. Contact Information Please contact [Miyuru Dayarathna](miyurud at yahoo dot co dot uk) for further information. Please let us know about bug reports or any further improvements you wish to have in JasmineGraph. diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index b88e041df..5359c20df 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -77,7 +77,7 @@ org.jasminegraph.scheduler.enabled=true org.jasminegraph.scheduler.performancecollector.timing=120 #pushgateway address -org.jasminegraph.collector.pushgateway=http://10.43.188.76:9091/metrics/job/ +org.jasminegraph.collector.pushgateway=http://10.43.150.139:9091/metrics/job/ #-------------------------------------------------------------------------------- #MetaDB information diff --git a/k8s/master-deployment.yaml b/k8s/master-deployment.yaml index 81e4e0202..0480e2029 100644 --- a/k8s/master-deployment.yaml +++ b/k8s/master-deployment.yaml @@ -2,6 +2,8 @@ apiVersion: apps/v1 kind: Deployment metadata: name: jasminegraph-master-deployment + labels: + application: jasminegraph spec: replicas: 1 selector: diff --git a/k8s/master-service.yaml b/k8s/master-service.yaml index b5e3dd312..0cbc8fee1 100644 --- a/k8s/master-service.yaml +++ b/k8s/master-service.yaml @@ -3,6 +3,7 @@ kind: Service metadata: name: jasminegraph-master-service labels: + application: jasminegraph service: jasminegraph-master spec: ports: diff --git a/k8s/prometheus.yaml b/k8s/prometheus.yaml index 4f49f940d..99052bc5e 100644 --- a/k8s/prometheus.yaml +++ b/k8s/prometheus.yaml @@ -3,6 +3,7 @@ kind: ConfigMap metadata: name: prometheus-server-conf labels: + application: jasminegraph app: prometheus data: prometheus.yml: | @@ -27,6 +28,7 @@ kind: Deployment metadata: name: prometheus labels: + application: jasminegraph app: prometheus spec: replicas: 1 @@ -60,6 +62,7 @@ kind: Service metadata: name: prometheus labels: + application: jasminegraph app: prometheus spec: ports: diff --git a/k8s/pushgateway.yaml b/k8s/pushgateway.yaml index f0355329f..3da961b50 100644 --- a/k8s/pushgateway.yaml +++ b/k8s/pushgateway.yaml @@ -3,6 +3,7 @@ kind: Deployment metadata: name: pushgateway labels: + application: jasminegraph app: pushgateway spec: replicas: 1 @@ -29,6 +30,7 @@ kind: Service metadata: name: pushgateway labels: + application: jasminegraph app: pushgateway spec: ports: diff --git a/k8s/volumes.yaml b/k8s/volumes.yaml index 3faede3cc..d2b652f7f 100644 --- a/k8s/volumes.yaml +++ b/k8s/volumes.yaml @@ -3,6 +3,7 @@ kind: PersistentVolume metadata: name: jasminegraph-metadb labels: + application: jasminegraph type: local spec: storageClassName: manual @@ -21,6 +22,8 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: jasminegraph-metadb-claim + labels: + application: jasminegraph spec: storageClassName: manual accessModes: @@ -35,6 +38,7 @@ kind: PersistentVolume metadata: name: jasminegraph-performancedb labels: + application: jasminegraph type: local spec: storageClassName: manual @@ -53,6 +57,8 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: jasminegraph-performancedb-claim + labels: + application: jasminegraph spec: storageClassName: manual accessModes: @@ -67,6 +73,7 @@ kind: PersistentVolume metadata: name: jasminegraph-data labels: + application: jasminegraph type: local spec: storageClassName: manual @@ -82,6 +89,8 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: jasminegraph-data-claim + labels: + application: jasminegraph spec: storageClassName: manual accessModes: @@ -96,6 +105,7 @@ kind: PersistentVolume metadata: name: jasminegraph-logs labels: + application: jasminegraph type: local spec: storageClassName: manual @@ -111,6 +121,8 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: jasminegraph-logs-claim + labels: + application: jasminegraph spec: storageClassName: manual accessModes: diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index d4b39e827..8a49ded32 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -3,6 +3,7 @@ kind: Deployment metadata: name: "jasminegraph-worker-deployment" labels: + application: "jasminegraph" deployment: "jasminegraph-worker" workerId: "" spec: diff --git a/k8s/worker-service.yaml b/k8s/worker-service.yaml index bab2501ea..3f8973188 100644 --- a/k8s/worker-service.yaml +++ b/k8s/worker-service.yaml @@ -3,6 +3,7 @@ kind: Service metadata: name: jasminegraph-worker-service labels: + application: jasminegraph service: jasminegraph-worker workerId: spec: diff --git a/start-k8s.sh b/start-k8s.sh index 9cf0eabae..dba90956c 100755 --- a/start-k8s.sh +++ b/start-k8s.sh @@ -4,6 +4,14 @@ set -e TIMEOUT_SECONDS=60 +if [ $1 == "clean" ]; then + kubectl delete deployments -l application=jasminegraph + kubectl delete services -l application=jasminegraph + kubectl delete pvc -l application=jasminegraph + kubectl delete pv -l application=jasminegraph + exit 0 +fi + META_DB_PATH=${META_DB_PATH} PERFORMANCE_DB_PATH=${PERFORMANCE_DB_PATH} DATA_PATH=${DATA_PATH} From c93a1559cd42fc24ba9646958bcd2a985bfec618 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Sun, 11 Feb 2024 19:37:45 +0530 Subject: [PATCH 029/143] Added performance metric retrieval logic --- conf/hosts.txt | 2 + conf/jasminegraph-server.properties | 2 +- main.cpp | 4 +- src/performance/metrics/PerformanceUtil.cpp | 134 ++++++------------ .../metrics/StatisticCollector.cpp | 14 +- src/performance/metrics/StatisticCollector.h | 2 +- src/server/JasmineGraphInstanceService.cpp | 23 ++- src/util/Utils.cpp | 50 ++++++- 8 files changed, 113 insertions(+), 118 deletions(-) diff --git a/conf/hosts.txt b/conf/hosts.txt index c0a48fcc7..51d016717 100644 --- a/conf/hosts.txt +++ b/conf/hosts.txt @@ -4,6 +4,8 @@ localhost # TODO: Replace this 192.168.8.150 +192.168.43.135 +10.10.15.244 #anuradhak@10.8.100.245 #192.168.1.8 diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 5359c20df..9fb69b6ae 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -77,7 +77,7 @@ org.jasminegraph.scheduler.enabled=true org.jasminegraph.scheduler.performancecollector.timing=120 #pushgateway address -org.jasminegraph.collector.pushgateway=http://10.43.150.139:9091/metrics/job/ +org.jasminegraph.collector.pushgateway=http://192.168.8.150:9091/ #-------------------------------------------------------------------------------- #MetaDB information diff --git a/main.cpp b/main.cpp index ed3a6743f..2bd356e09 100644 --- a/main.cpp +++ b/main.cpp @@ -54,13 +54,15 @@ int main(int argc, char *argv[]) { main_logger.log("Using JASMINE_GRAPH_HOME", "info"); std::cout << JASMINEGRAPH_HOME << std::endl; + StatisticCollector::init(); + thread schedulerThread(SchedulerService::startScheduler); + if (mode == Conts::JASMINEGRAPH_RUNTIME_PROFILE_MASTER) { std::string masterIp = argv[3]; int numberOfWorkers = atoi(argv[4]); std::string workerIps = argv[5]; enableNmon = argv[6]; server = JasmineGraphServer::getInstance(); - thread schedulerThread(SchedulerService::startScheduler); if (profile == Conts::PROFILE_K8S) { K8sInterface *interface = new K8sInterface(); diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index f73953fda..91cb8892e 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -15,7 +15,6 @@ limitations under the License. using namespace std::chrono; std::map> resourceUsageMap; -std::string pushGatewayAddr = Utils::getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); static size_t write_callback(void* contents, size_t size, size_t nmemb, std::string* output); @@ -35,59 +34,33 @@ void PerformanceUtil::init() { } } -int PerformanceUtil::collectPerformanceStatistics() { - vector hostList = Utils::getHostListFromProperties(); - int hostListSize = hostList.size(); - int counter = 0; - std::vector> intermRes; - PlacesToNodeMapper placesToNodeMapper; +int PerformanceUtil::collectPerformanceStatistics() { + long memoryUsage = StatisticCollector::getMemoryUsageByProcess(); + Utils::send_job("WorkerPerfData", "memory_usage", std::to_string(memoryUsage)); - std::string placeLoadQuery = - "select ip, user, server_port, is_master, is_host_reporter,host_idhost,idplace from place"; - std::vector>> placeList = perfDb->runSelect(placeLoadQuery); - std::vector>>::iterator placeListIterator; + double cpuUsage = StatisticCollector::getCpuUsage(); + Utils::send_job("WorkerPerfData", "cpu_usage", std::to_string(cpuUsage)); - for (placeListIterator = placeList.begin(); placeListIterator != placeList.end(); ++placeListIterator) { - vector> place = *placeListIterator; - std::string host; - std::string requestResourceAllocation = "false"; + int threadCount = StatisticCollector::getThreadCount(); + Utils::send_job("HostPerfData", "thread_count", std::to_string(threadCount)); - std::string ip = place.at(0).second; - std::string user = place.at(1).second; - std::string serverPort = place.at(2).second; - std::string isMaster = place.at(3).second; - std::string isHostReporter = place.at(4).second; - std::string hostId = place.at(5).second; - std::string placeId = place.at(6).second; + long totalSwapSpace = StatisticCollector::getTotalSwapSpace(); + Utils::send_job("HostPerfData", "total_swap_space", std::to_string(totalSwapSpace)); - if (ip.find("localhost") != std::string::npos) { - host = "localhost"; - } else { - host = user + "@" + ip; - } + long usedSwapSpace = StatisticCollector::getUsedSwapSpace(); + Utils::send_job("HostPerfData", "used_swap_space", std::to_string(usedSwapSpace)); - if (isHostReporter.find("true") != std::string::npos) { - std::string hostSearch = "select total_cpu_cores,total_memory from host where idhost='" + hostId + "'"; - std::vector>> hostAllocationList = perfDb->runSelect(hostSearch); + long rxBytes = StatisticCollector::getRXBytes(); + Utils::send_job("WorkerPerfData", "rx_bytes", std::to_string(rxBytes)); - vector> firstHostAllocation = hostAllocationList.at(0); + long txBytes = StatisticCollector::getTXBytes(); + Utils::send_job("WorkerPerfData", "tx_bytes", std::to_string(txBytes)); - std::string totalCPUCores = firstHostAllocation.at(0).second; - std::string totalMemory = firstHostAllocation.at(1).second; - - if (totalCPUCores.empty() || totalMemory.empty()) { - requestResourceAllocation = "true"; - } - } - - if (isMaster.find("true") != std::string::npos) { - collectLocalPerformanceData(isHostReporter, requestResourceAllocation, hostId, placeId); - } else { - collectRemotePerformanceData(host, atoi(serverPort.c_str()), isHostReporter, requestResourceAllocation, - hostId, placeId); - } - } + int socketCount = StatisticCollector::getSocketCount(); + Utils::send_job("HostPerfData", "socket_count", std::to_string(socketCount)); + long totalMemoryUsage = StatisticCollector::getTotalMemoryUsage(); + Utils::send_job("HostPerfData", "total_memory", std::to_string(totalMemoryUsage)); return 0; } @@ -206,12 +179,6 @@ std::vector PerformanceUtil::retrieveCurrentResourceUtiliza return placeResourceConsumptionList; } -static size_t write_callback(void* contents, size_t size, size_t nmemb, std::string* output) { - size_t totalSize = size * nmemb; - output->append(static_cast(contents), totalSize); - return totalSize; -} - void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, std::string isVMStatManager, std::string isResourceAllocationRequired, std::string hostId, std::string placeId) { @@ -220,7 +187,7 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s bool loop = false; socklen_t len; struct sockaddr_in serv_addr; - struct hostent* server; + struct hostent *server; sockfd = socket(AF_INET, SOCK_STREAM, 0); @@ -235,11 +202,11 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s return; } - bzero((char*)&serv_addr, sizeof(serv_addr)); + bzero((char *) &serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char*)server->h_addr, (char*)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return; } @@ -301,7 +268,7 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s std::string cpuUsage = strArr[5]; string vmPerformanceSql = "insert into host_performance_data (date_time, memory_usage, cpu_usage, idhost) values ('" + - processTime + "','" + memoryConsumption + "','" + cpuUsage + "','" + hostId + "')"; + processTime + "','" + memoryConsumption + "','" + cpuUsage + "','" + hostId + "')"; perfDb->runInsert(vmPerformanceSql); Utils::send_job("hostPerfData", "memory_consumption", memoryConsumption); @@ -311,20 +278,27 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s std::string totalMemory = strArr[6]; std::string totalCores = strArr[7]; string allocationUpdateSql = "update host set total_cpu_cores='" + totalCores + - "',total_memory='" + totalMemory + "' where idhost='" + hostId + - "'"; - - perfDb->runUpdate(allocationUpdateSql); + "',total_memory='" + totalMemory + "' where idhost='" + hostId + + "'"; + + if (isResourceAllocationRequired == "true") { + std::string totalMemory = strArr[6]; + std::string totalCores = strArr[7]; + string allocationUpdateSql = + "update host set total_cpu_cores='" + totalCores + "',total_memory='" + + totalMemory + "' where idhost='" + hostId + "'"; + + Utils::send_job("placePerfData", "memory_usage", memoryUsage); + Utils::send_job("placePerfData", "cpu_usage", cpuUsage); + } } } string placePerfSql = "insert into place_performance_data (idplace, memory_usage, cpu_usage, date_time) values ('" + - placeId + "','" + memoryUsage + "','" + cpuUsage + "','" + processTime + "')"; + placeId + "','" + memoryUsage + "','" + cpuUsage + "','" + processTime + "')"; perfDb->runInsert(placePerfSql); - Utils::send_job("placePerfData", "memory_usage", memoryUsage); - Utils::send_job("placePerfData", "cpu_usage", cpuUsage); } } } @@ -332,11 +306,8 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, std::string isResourceAllocationRequired, std::string hostId, std::string placeId) { - StatisticCollector statisticCollector; - statisticCollector.init(); - - int memoryUsage = statisticCollector.getMemoryUsageByProcess(); - double cpuUsage = statisticCollector.getCpuUsage(); + int memoryUsage = StatisticCollector::getMemoryUsageByProcess(); + double cpuUsage = StatisticCollector::getCpuUsage(); auto executedTime = std::chrono::system_clock::now(); std::time_t reportTime = std::chrono::system_clock::to_time_t(executedTime); @@ -345,7 +316,7 @@ void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, s if (isVMStatManager.find("true") != std::string::npos) { std::string vmLevelStatistics = - statisticCollector.collectVMStatistics(isVMStatManager, isResourceAllocationRequired); + StatisticCollector::collectVMStatistics(isVMStatManager, isResourceAllocationRequired); std::vector strArr = Utils::split(vmLevelStatistics, ','); string totalMemoryUsed = strArr[0]; @@ -357,9 +328,6 @@ void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, s perfDb->runInsert(vmPerformanceSql); - Utils::send_job("localPerfDataO_" + hostId, "total_memory_usage", totalMemoryUsed); - Utils::send_job("localPerfDataO_" + hostId, "total_CPU_usage", totalCPUUsage); - if (isResourceAllocationRequired == "true") { std::string totalMemory = strArr[2]; std::string totalCores = strArr[3]; @@ -375,11 +343,6 @@ void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, s reportTimeString + "')"; perfDb->runInsert(placePerfSql); - - Utils::send_job("placePerfDataLocalO_" + hostId, "memory_usage", - to_string(memoryUsage)); - Utils::send_job("placePerfDataLocalO_" + hostId, "cpu_usage", - to_string(cpuUsage)); } int PerformanceUtil::collectRemoteSLAResourceUtilization(std::string host, int port, std::string isVMStatManager, @@ -501,11 +464,9 @@ int PerformanceUtil::collectRemoteSLAResourceUtilization(std::string host, int p void PerformanceUtil::collectLocalSLAResourceUtilization(std::string graphId, std::string placeId, std::string command, std::string category, int elapsedTime, bool autoCalibrate) { - StatisticCollector statisticCollector; - statisticCollector.init(); string graphSlaId; - double loadAverage = statisticCollector.getLoadAverage(); + double loadAverage = StatisticCollector::getLoadAverage(); auto executedTime = std::chrono::system_clock::now(); std::time_t reportTime = std::chrono::system_clock::to_time_t(executedTime); @@ -534,11 +495,8 @@ void PerformanceUtil::collectLocalSLAResourceUtilization(std::string graphId, st ResourceConsumption PerformanceUtil::retrieveLocalResourceConsumption(std::string host, std::string placeId) { ResourceConsumption placeResourceConsumption; - StatisticCollector statisticCollector; - statisticCollector.init(); - - int memoryUsage = statisticCollector.getMemoryUsageByProcess(); - double cpuUsage = statisticCollector.getCpuUsage(); + int memoryUsage = StatisticCollector::getMemoryUsageByProcess(); + double cpuUsage = StatisticCollector::getCpuUsage(); placeResourceConsumption.memoryUsage = memoryUsage; placeResourceConsumption.host = host; @@ -915,9 +873,7 @@ void PerformanceUtil::adjustAggregateLoadMap(std::map> JasmineGraphInstanceService::iterationData; const string JasmineGraphInstanceService::END_OF_MESSAGE = "eom"; @@ -152,7 +151,6 @@ void *instanceservicesession(void *dummyPt) { instance_logger.info("New service session started on thread:" + to_string(pthread_self()) + " connFd:" + to_string(connFd)); - collector.init(); Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder")); @@ -670,15 +668,15 @@ map JasmineGraphInstanceService::getOutDegreeDistributionHashMap(map string JasmineGraphInstanceService::requestPerformanceStatistics(std::string isVMStatManager, std::string isResourceAllocationRequested) { - long memoryUsage = collector.getMemoryUsageByProcess(); - int threadCount = collector.getThreadCount(); - long usedSwapSpace = collector.getUsedSwapSpace(); - long totalSwapSpace = collector.getTotalSwapSpace(); - long readBytes = collector.getRXBytes(); - long sentBytes = collector.getTXBytes(); - double cpuUsage = collector.getCpuUsage(); - double loadAverage = collector.getLoadAverage(); - std::string vmLevelStatistics = collector.collectVMStatistics(isVMStatManager, isResourceAllocationRequested); + long memoryUsage = StatisticCollector::getMemoryUsageByProcess(); + int threadCount = StatisticCollector::getThreadCount(); + long usedSwapSpace = StatisticCollector::getUsedSwapSpace(); + long totalSwapSpace = StatisticCollector::getTotalSwapSpace(); + long readBytes = StatisticCollector::getRXBytes(); + long sentBytes = StatisticCollector::getTXBytes(); + double cpuUsage = StatisticCollector::getCpuUsage(); + double loadAverage = StatisticCollector::getLoadAverage(); + std::string vmLevelStatistics = StatisticCollector::collectVMStatistics(isVMStatManager, isResourceAllocationRequested); auto executedTime = std::chrono::system_clock::now(); std::time_t reportTime = std::chrono::system_clock::to_time_t(executedTime); std::string reportTimeString(std::ctime(&reportTime)); @@ -1885,14 +1883,13 @@ void JasmineGraphInstanceService::startCollectingLoadAverage() { int elapsedTime = 0; time_t start; time_t end; - StatisticCollector statisticCollector; start = time(0); while (collectValid) { if (time(0) - start == Conts::LOAD_AVG_COLLECTING_GAP) { elapsedTime += Conts::LOAD_AVG_COLLECTING_GAP * 1000; - double loadAgerage = statisticCollector.getLoadAverage(); + double loadAgerage = StatisticCollector::getLoadAverage(); loadAverageVector.push_back(std::to_string(loadAgerage)); start = start + Conts::LOAD_AVG_COLLECTING_GAP; } diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 3e5d39c7d..448ce55b3 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -24,6 +24,7 @@ limitations under the License. #include #include #include +#include #include "../server/JasmineGraphInstanceProtocol.h" #include "Conts.h" @@ -711,7 +712,15 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, std::string response_string; curl = curl_easy_init(); if (curl) { - std::string hostPGAddr = pushGatewayJobAddr + job_group_name; + std::string hostPGAddr; + const char* envWorkerID = getenv("WORKER_ID"); + std::string workerID; + if(envWorkerID){ + workerID = std::string(getenv("WORKER_ID")); + } else{ + workerID = "-1"; + } + hostPGAddr = pushGatewayJobAddr + job_group_name + "_" + workerID; curl_easy_setopt(curl, CURLOPT_URL, hostPGAddr.c_str()); // Set the callback function to handle the response data @@ -738,6 +747,45 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, return response_string; } +static std::map getMetricMap(std::string metricName){ + std::map map; + CURL* curl; + CURLcode res; + std::string response_cpu_usages; + std::string pushGatewayAddr = getPushGatewayAddress() + "api/v1/query?query=" + metricName; + + curl = curl_easy_init(); + if (curl) { + curl_easy_setopt(curl, CURLOPT_URL, pushGatewayAddr.c_str()); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_cpu_usages); + curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); + curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L); + curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1L); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + } else{ + std::cout << response_cpu_usages << std::endl; + Json::Value root; + Json::Reader reader; + reader.parse(response_cpu_usages, root); + const Json::Value results = root["data"]["result"]; + Json::Value currentExportedJobName; + for(int i=1; i Date: Sun, 11 Feb 2024 19:39:04 +0530 Subject: [PATCH 030/143] Fix style check issues --- src/performance/metrics/PerformanceUtil.cpp | 106 +++++++++--------- .../metrics/StatisticCollector.cpp | 39 ++++--- src/server/JasmineGraphInstanceService.cpp | 3 +- src/util/Utils.cpp | 78 ++++++------- 4 files changed, 110 insertions(+), 116 deletions(-) diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index 91cb8892e..dc3d936ec 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -16,7 +16,7 @@ limitations under the License. using namespace std::chrono; std::map> resourceUsageMap; -static size_t write_callback(void* contents, size_t size, size_t nmemb, std::string* output); +static size_t write_callback(void *contents, size_t size, size_t nmemb, std::string *output); Logger scheduler_logger; SQLiteDBInterface *sqlLiteDB; @@ -34,7 +34,7 @@ void PerformanceUtil::init() { } } -int PerformanceUtil::collectPerformanceStatistics() { +int PerformanceUtil::collectPerformanceStatistics() { long memoryUsage = StatisticCollector::getMemoryUsageByProcess(); Utils::send_job("WorkerPerfData", "memory_usage", std::to_string(memoryUsage)); @@ -277,20 +277,14 @@ void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, s if (isResourceAllocationRequired == "true") { std::string totalMemory = strArr[6]; std::string totalCores = strArr[7]; + string allocationUpdateSql = "update host set total_cpu_cores='" + totalCores + "',total_memory='" + totalMemory + "' where idhost='" + hostId + "'"; + perfDb->runUpdate(allocationUpdateSql); - if (isResourceAllocationRequired == "true") { - std::string totalMemory = strArr[6]; - std::string totalCores = strArr[7]; - string allocationUpdateSql = - "update host set total_cpu_cores='" + totalCores + "',total_memory='" + - totalMemory + "' where idhost='" + hostId + "'"; - - Utils::send_job("placePerfData", "memory_usage", memoryUsage); - Utils::send_job("placePerfData", "cpu_usage", cpuUsage); - } + Utils::send_job("placePerfData", "memory_usage", memoryUsage); + Utils::send_job("placePerfData", "cpu_usage", cpuUsage); } } @@ -324,7 +318,7 @@ void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, s string vmPerformanceSql = "insert into host_performance_data (date_time, memory_usage, cpu_usage, idhost) values ('" + - reportTimeString + "','" + totalMemoryUsed + "','" + totalCPUUsage + "','" + hostId + "')"; + reportTimeString + "','" + totalMemoryUsed + "','" + totalCPUUsage + "','" + hostId + "')"; perfDb->runInsert(vmPerformanceSql); @@ -332,15 +326,15 @@ void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, s std::string totalMemory = strArr[2]; std::string totalCores = strArr[3]; string allocationUpdateSql = "update host set total_cpu_cores='" + totalCores + "',total_memory='" + - totalMemory + "' where idhost='" + hostId + "'"; + totalMemory + "' where idhost='" + hostId + "'"; perfDb->runUpdate(allocationUpdateSql); } } string placePerfSql = "insert into place_performance_data (idplace, memory_usage, cpu_usage, date_time) values ('" + - placeId + "','" + to_string(memoryUsage) + "','" + to_string(cpuUsage) + "','" + - reportTimeString + "')"; + placeId + "','" + to_string(memoryUsage) + "','" + to_string(cpuUsage) + "','" + + reportTimeString + "')"; perfDb->runInsert(placePerfSql); } @@ -353,7 +347,7 @@ int PerformanceUtil::collectRemoteSLAResourceUtilization(std::string host, int p bool loop = false; socklen_t len; struct sockaddr_in serv_addr; - struct hostent* server; + struct hostent *server; std::string graphSlaId; sockfd = socket(AF_INET, SOCK_STREAM, 0); @@ -373,11 +367,11 @@ int PerformanceUtil::collectRemoteSLAResourceUtilization(std::string host, int p return 0; } - bzero((char*)&serv_addr, sizeof(serv_addr)); + bzero((char *) &serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char*)server->h_addr, (char*)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return 0; } @@ -513,7 +507,7 @@ ResourceConsumption PerformanceUtil::retrieveRemoteResourceConsumption(std::stri bool loop = false; socklen_t len; struct sockaddr_in serv_addr; - struct hostent* server; + struct hostent *server; std::string graphSlaId; std::string isVMStatManager = "false"; std::string isResourceAllocationRequired = "false"; @@ -531,11 +525,11 @@ ResourceConsumption PerformanceUtil::retrieveRemoteResourceConsumption(std::stri return placeResourceConsumption; } - bzero((char*)&serv_addr, sizeof(serv_addr)); + bzero((char *) &serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char*)server->h_addr, (char*)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return placeResourceConsumption; } @@ -600,7 +594,7 @@ ResourceConsumption PerformanceUtil::retrieveRemoteResourceConsumption(std::stri std::vector PerformanceUtil::getResourceAvailableTime(std::vector graphIdList, std::string command, std::string category, std::string masterIP, - std::vector& pendingHPJobList) { + std::vector &pendingHPJobList) { PerformanceUtil performanceUtil; performanceUtil.init(); std::set hostSet; @@ -647,9 +641,10 @@ std::vector PerformanceUtil::getResourceAvailableTime(std::vector '" + std::to_string(adjustedElapsedTime) + - "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time;"; + currentGraphId + "' and sla_category.command='" + command + "' and sla_category.category='" + + category + + "' and graph_place_sla_performance.elapsed_time > '" + std::to_string(adjustedElapsedTime) + + "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time;"; std::vector>> loadAvgResults = perfDb->runSelect(slaLoadQuery); @@ -735,8 +730,8 @@ std::vector PerformanceUtil::getResourceAvailableTime(std::vector>> newJobLoadAvgResults = perfDb->runSelect(newJobLoadQuery); @@ -832,8 +827,8 @@ std::vector PerformanceUtil::getResourceAvailableTime(std::vector>& aggregateLoadAvgMap, - std::map>& newJobLoadAvgMap, +void PerformanceUtil::adjustAggregateLoadMap(std::map> &aggregateLoadAvgMap, + std::map> &newJobLoadAvgMap, long newJobAcceptanceTime) { std::map>::iterator newJobIterator; @@ -892,7 +887,7 @@ void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *pe std::string graphSlaId; std::string query = "SELECT id from graph_sla where graph_id='" + graphId + "' and partition_count='" + - std::to_string(partitionCount) + "' and id_sla_category='" + slaCategoryId + "';"; + std::to_string(partitionCount) + "' and id_sla_category='" + slaCategoryId + "';"; std::vector>> results = performanceDb->runSelect(query); @@ -901,7 +896,7 @@ void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *pe } else { std::string insertQuery = "insert into graph_sla (id_sla_category, graph_id, partition_count, sla_value, attempt) VALUES ('" + - slaCategoryId + "','" + graphId + "'," + std::to_string(partitionCount) + ",0,0);"; + slaCategoryId + "','" + graphId + "'," + std::to_string(partitionCount) + ",0,0);"; int slaId = performanceDb->runInsert(insertQuery); graphSlaId = std::to_string(slaId); @@ -967,7 +962,7 @@ void PerformanceUtil::updateRemoteResourceConsumption(PerformanceSQLiteDBInterfa std::vector::iterator loadVectorIterator; string valuesString; std::string query = "SELECT id from graph_sla where graph_id='" + graphId + "' and partition_count='" + - std::to_string(partitionCount) + "' and id_sla_category='" + slaCategoryId + "';"; + std::to_string(partitionCount) + "' and id_sla_category='" + slaCategoryId + "';"; std::vector>> results = performanceDb->runSelect(query); @@ -976,7 +971,7 @@ void PerformanceUtil::updateRemoteResourceConsumption(PerformanceSQLiteDBInterfa } else { std::string insertQuery = "insert into graph_sla (id_sla_category, graph_id, partition_count, sla_value, attempt) VALUES ('" + - slaCategoryId + "','" + graphId + "'," + std::to_string(partitionCount) + ",0,0);"; + slaCategoryId + "','" + graphId + "'," + std::to_string(partitionCount) + ",0,0);"; int slaId = performanceDb->runInsert(insertQuery); graphSlaId = std::to_string(slaId); @@ -994,7 +989,7 @@ void PerformanceUtil::updateRemoteResourceConsumption(PerformanceSQLiteDBInterfa std::string loadAverage = *loadVectorIterator; valuesString += "('" + graphSlaId + "','" + placeId + "', '','" + loadAverage + "','" + - std::to_string(elapsedTime * 1000) + "'),"; + std::to_string(elapsedTime * 1000) + "'),"; elapsedTime += 5; Utils::send_job("loadAverageSLARem", "load_average_rem", loadAverage); @@ -1033,7 +1028,7 @@ void PerformanceUtil::initiateCollectingRemoteSLAResourceUtilization(std::string bool loop = false; socklen_t len; struct sockaddr_in serv_addr; - struct hostent* server; + struct hostent *server; std::string graphSlaId; sockfd = socket(AF_INET, SOCK_STREAM, 0); @@ -1053,11 +1048,11 @@ void PerformanceUtil::initiateCollectingRemoteSLAResourceUtilization(std::string return; } - bzero((char*)&serv_addr, sizeof(serv_addr)); + bzero((char *) &serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char*)server->h_addr, (char*)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return; } @@ -1107,7 +1102,7 @@ std::string PerformanceUtil::requestRemoteLoadAverages(std::string host, int por bool loop = false; socklen_t len; struct sockaddr_in serv_addr; - struct hostent* server; + struct hostent *server; std::string graphSlaId; sockfd = socket(AF_INET, SOCK_STREAM, 0); @@ -1127,11 +1122,11 @@ std::string PerformanceUtil::requestRemoteLoadAverages(std::string host, int por return 0; } - bzero((char*)&serv_addr, sizeof(serv_addr)); + bzero((char *) &serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char*)server->h_addr, (char*)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return 0; } @@ -1220,11 +1215,11 @@ double PerformanceUtil::getAggregatedLoadAverage(std::string graphId, std::strin "graph_place_sla_performance.elapsed_time from graph_place_sla_performance inner join graph_sla" " inner join sla_category where graph_place_sla_performance.graph_sla_id=graph_sla.id" " and graph_sla.id_sla_category=sla_category.id and graph_sla.graph_id='" + - currentGraphId + "' and sla_category.command='" + command + - "' and graph_place_sla_performance.place_id='" + placeId + "' and sla_category.category='" + - category + "' and graph_place_sla_performance.elapsed_time ='" + - std::to_string(adjustedElapsedTime) + - "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time;"; + currentGraphId + "' and sla_category.command='" + command + + "' and graph_place_sla_performance.place_id='" + placeId + "' and sla_category.category='" + + category + "' and graph_place_sla_performance.elapsed_time ='" + + std::to_string(adjustedElapsedTime) + + "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time;"; std::vector>> loadAvgResults = perfDb->runSelect(slaLoadQuery); if (loadAvgResults.empty()) { @@ -1243,11 +1238,12 @@ double PerformanceUtil::getAggregatedLoadAverage(std::string graphId, std::strin "graph_place_sla_performance.elapsed_time from graph_place_sla_performance inner join graph_sla" " inner join sla_category where graph_place_sla_performance.graph_sla_id=graph_sla.id" " and graph_sla.id_sla_category=sla_category.id and graph_sla.graph_id='" + - currentGraphId + "' and sla_category.command='" + command + - "' and graph_place_sla_performance.place_id='" + placeId + "' and sla_category.category='" + - category + "' and graph_place_sla_performance.elapsed_time > '" + - std::to_string(adjustedElapsedTime) + - "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time LIMIT 2;"; + currentGraphId + "' and sla_category.command='" + command + + "' and graph_place_sla_performance.place_id='" + placeId + "' and sla_category.category='" + + category + "' and graph_place_sla_performance.elapsed_time > '" + + std::to_string(adjustedElapsedTime) + + "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time" + + " LIMIT 2;"; std::vector>> loadAvgResults = perfDb->runSelect(slaLoadQuery); diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index 31dd92326..bbc992b7c 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -16,11 +16,11 @@ limitations under the License. static clock_t lastCPU, lastSysCPU, lastUserCPU; static int numProcessors; -static long parseLine(char* line); -static long getSwapSpace(const char* type); +static long parseLine(char *line); +static long getSwapSpace(const char *type); int StatisticCollector::init() { - FILE* file; + FILE *file; struct tms timeSample; char line[128]; @@ -38,7 +38,7 @@ int StatisticCollector::init() { } long StatisticCollector::getMemoryUsageByProcess() { - FILE* file = fopen("/proc/self/status", "r"); + FILE *file = fopen("/proc/self/status", "r"); long result = -1; char line[128]; @@ -54,12 +54,15 @@ long StatisticCollector::getMemoryUsageByProcess() { } int StatisticCollector::getThreadCount() { - FILE* file = fopen("/proc/self/stat", "r"); + FILE *file = fopen("/proc/self/stat", "r"); long result; char line[128]; for (int i = 0; i < 20; i++) { - if (fscanf(file, "%127s%*c", line) < 0) return -1; + if (fscanf(file, "%127s%*c", line) < 0) { + fclose(file); + return -1; + } } fclose(file); result = strtol(line, NULL, 10); @@ -68,15 +71,15 @@ int StatisticCollector::getThreadCount() { } static long getSwapSpace(int field) { - FILE* file = fopen("/proc/swaps", "r"); + FILE *file = fopen("/proc/swaps", "r"); long result = -1; char line[128]; fgets(line, 128, file); while (fgets(line, 128, file) != NULL) { - char* value; - char* save = NULL; + char *value; + char *save = NULL; for (int i = 0; i < field; i++) { if (i == 0) { value = strtok_r(line, " ", &save); @@ -112,7 +115,7 @@ long StatisticCollector::getTotalSwapSpace() { } long StatisticCollector::getRXBytes() { - FILE* file = fopen("/sys/class/net/eth0/statistics/rx_bytes", "r"); + FILE *file = fopen("/sys/class/net/eth0/statistics/rx_bytes", "r"); long result = -1; fscanf(file, "%li", &result); fclose(file); @@ -121,7 +124,7 @@ long StatisticCollector::getRXBytes() { } long StatisticCollector::getTXBytes() { - FILE* file = fopen("/sys/class/net/eth0/statistics/tx_bytes", "r"); + FILE *file = fopen("/sys/class/net/eth0/statistics/tx_bytes", "r"); long result = -1; fscanf(file, "%li", &result); fclose(file); @@ -131,17 +134,17 @@ long StatisticCollector::getTXBytes() { } int StatisticCollector::getSocketCount() { - DIR* d = opendir("/proc/self/fd"); + DIR *d = opendir("/proc/self/fd"); if (!d) { puts("Error opening directory /proc/self/fd"); return -1; } - const struct dirent* dir; + const struct dirent *dir; char path[64]; char link_buf[1024]; int count = 0; while ((dir = readdir(d)) != NULL) { - const char* filename = dir->d_name; + const char *filename = dir->d_name; if (filename[0] < '0' || '9' < filename[0]) continue; sprintf(path, "/proc/self/fd/%s", filename); size_t len = readlink(path, link_buf, sizeof(link_buf) - 1); @@ -150,15 +153,15 @@ int StatisticCollector::getSocketCount() { count++; } } - (void)closedir(d); + (void) closedir(d); std::cout << "Total sockets: " + std::to_string(count) << std::endl; return count; } -static long parseLine(char* line) { +static long parseLine(char *line) { int i = strlen(line); - const char* p = line; + const char *p = line; while (*p < '0' || *p > '9') p++; line[i - 3] = '\0'; long val = strtol(p, NULL, 10); @@ -271,7 +274,7 @@ double StatisticCollector::getTotalCpuUsage() { int count = 0; double totalCPUUsage = 0; - FILE* input = popen(mpstatCommand.c_str(), "r"); + FILE *input = popen(mpstatCommand.c_str(), "r"); if (input) { // read the input diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index c8d681073..4906adcc7 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -676,7 +676,8 @@ string JasmineGraphInstanceService::requestPerformanceStatistics(std::string isV long sentBytes = StatisticCollector::getTXBytes(); double cpuUsage = StatisticCollector::getCpuUsage(); double loadAverage = StatisticCollector::getLoadAverage(); - std::string vmLevelStatistics = StatisticCollector::collectVMStatistics(isVMStatManager, isResourceAllocationRequested); + std::string vmLevelStatistics = StatisticCollector::collectVMStatistics(isVMStatManager, + isResourceAllocationRequested); auto executedTime = std::chrono::system_clock::now(); std::time_t reportTime = std::chrono::system_clock::to_time_t(executedTime); std::string reportTimeString(std::ctime(&reportTime)); diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 448ce55b3..c84335a7b 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -221,7 +221,7 @@ std::vector Utils::getListOfFilesInDirectory(std::string dirName) { results.push_back(fnamestr); } } - (void)closedir(d); + (void) closedir(d); return results; } @@ -353,7 +353,7 @@ int Utils::unzipFile(std::string filePath, std::string mode) { bool Utils::hostExists(string name, string ip, std::string workerPort, SQLiteDBInterface *sqlite) { bool result = true; string stmt = "SELECT COUNT( * ) FROM worker WHERE name LIKE '" + name + "' AND ip LIKE '" + ip + - "' AND server_port LIKE '" + workerPort + "';"; + "' AND server_port LIKE '" + workerPort + "';"; if (ip == "") { stmt = "SELECT COUNT( * ) FROM worker WHERE name LIKE '" + name + "';"; } @@ -443,8 +443,8 @@ void Utils::updateSLAInformation(PerformanceSQLiteDBInterface *perfSqlite, std:: string slaCategoryId = categoryResults[0][0].second; std::string query = "SELECT id, sla_value, attempt from graph_sla where graph_id='" + graphId + - "' and partition_count='" + std::to_string(partitionCount) + "' and id_sla_category='" + - slaCategoryId + "';"; + "' and partition_count='" + std::to_string(partitionCount) + "' and id_sla_category='" + + slaCategoryId + "';"; std::vector>> results = perfSqlite->runSelect(query); @@ -462,15 +462,15 @@ void Utils::updateSLAInformation(PerformanceSQLiteDBInterface *perfSqlite, std:: attempts++; std::string updateQuery = "UPDATE graph_sla set sla_value='" + std::to_string(newSla) + "', attempt='" + - std::to_string(attempts) + "' where id = '" + slaId + "'"; + std::to_string(attempts) + "' where id = '" + slaId + "'"; perfSqlite->runUpdate(updateQuery); } } else { std::string insertQuery = "insert into graph_sla (id_sla_category, graph_id, partition_count, sla_value, attempt) VALUES ('" + - slaCategoryId + "','" + graphId + "'," + std::to_string(partitionCount) + "," + - std::to_string(newSlaValue) + ",0);"; + slaCategoryId + "','" + graphId + "'," + std::to_string(partitionCount) + "," + + std::to_string(newSlaValue) + ",0);"; perfSqlite->runInsert(insertQuery); } @@ -484,7 +484,7 @@ int Utils::copyToDirectory(std::string currentPath, std::string copyPath) { int status = system(command.c_str()); if (status != 0) { util_logger.error("Copying " + currentPath + " to directory " + copyPath + " failed with code " + - std::to_string(status)); + std::to_string(status)); } return status; } @@ -527,36 +527,36 @@ int Utils::connect_wrapper(int sock, const sockaddr *addr, socklen_t slen) { struct timeval tv = {1, 0}; - if (setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv)) == -1) { + if (setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *) &tv, sizeof(tv)) == -1) { util_logger.error("Failed to set send timeout option for socket"); } - if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv)) == -1) { + if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &tv, sizeof(tv)) == -1) { util_logger.error("Failed to set receive timeout option for socket"); } do { if (retry) sleep(retry * 2); util_logger.info("Trying to connect to [" + to_string(retry) + - "]: " + string(inet_ntoa(((const struct sockaddr_in *)addr)->sin_addr)) + ":" + - to_string(ntohs(((const struct sockaddr_in *)addr)->sin_port))); + "]: " + string(inet_ntoa(((const struct sockaddr_in *) addr)->sin_addr)) + ":" + + to_string(ntohs(((const struct sockaddr_in *) addr)->sin_port))); if (connect(sock, addr, slen) == 0) { tv = {0, 0}; - if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv)) == -1) { + if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &tv, sizeof(tv)) == -1) { util_logger.error("Failed to set receive timeout option for socket after successful connection"); } return 0; } } while (retry++ < 4); - util_logger.error("Error connecting to " + string(inet_ntoa(((const struct sockaddr_in *)addr)->sin_addr)) + ":" + - to_string(ntohs(((const struct sockaddr_in *)addr)->sin_port))); + util_logger.error("Error connecting to " + string(inet_ntoa(((const struct sockaddr_in *) addr)->sin_addr)) + ":" + + to_string(ntohs(((const struct sockaddr_in *) addr)->sin_port))); return -1; } std::string Utils::read_str_wrapper(int connFd, char *buf, size_t len, bool allowEmpty) { ssize_t result = recv(connFd, buf, len, 0); if (result < 0) { - util_logger.error("Read failed: recv returned " + std::to_string((int)result)); + util_logger.error("Read failed: recv returned " + std::to_string((int) result)); return ""; } else if (!allowEmpty && result == 0) { util_logger.error("Read failed: recv empty string"); @@ -646,18 +646,13 @@ static inline json yaml2json(const YAML::Node &root) { json j{}; switch (root.Type()) { - case YAML::NodeType::Null: + case YAML::NodeType::Null:break; + case YAML::NodeType::Scalar:return parse_scalar(root); + case YAML::NodeType::Sequence:for (auto &&node : root) j.emplace_back(yaml2json(node)); break; - case YAML::NodeType::Scalar: - return parse_scalar(root); - case YAML::NodeType::Sequence: - for (auto &&node : root) j.emplace_back(yaml2json(node)); - break; - case YAML::NodeType::Map: - for (auto &&it : root) j[it.first.as()] = yaml2json(it.second); - break; - default: + case YAML::NodeType::Map:for (auto &&it : root) j[it.first.as()] = yaml2json(it.second); break; + default:break; } return j; } @@ -697,15 +692,14 @@ int Utils::createDatabaseFromDDL(const char *dbLocation, const char *ddlFileLoca return 0; } -static size_t write_callback(void* contents, size_t size, size_t nmemb, std::string* output) { +static size_t write_callback(void *contents, size_t size, size_t nmemb, std::string *output) { size_t totalSize = size * nmemb; - output->append(static_cast(contents), totalSize); + output->append(static_cast(contents), totalSize); return totalSize; } - std::string Utils::send_job(std::string job_group_name, std::string metric_name, - std::string metric_value) { - CURL* curl; + std::string metric_value) { + CURL *curl; CURLcode res; std::string pushGatewayJobAddr = getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); @@ -713,11 +707,11 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, curl = curl_easy_init(); if (curl) { std::string hostPGAddr; - const char* envWorkerID = getenv("WORKER_ID"); + const char *envWorkerID = getenv("WORKER_ID"); std::string workerID; - if(envWorkerID){ + if (envWorkerID) { workerID = std::string(getenv("WORKER_ID")); - } else{ + } else { workerID = "-1"; } hostPGAddr = pushGatewayJobAddr + job_group_name + "_" + workerID; @@ -727,9 +721,9 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string); std::string job_data = metric_name + " " + metric_value + "\n"; - const char* data = job_data.c_str(); + const char *data = job_data.c_str(); - curl_slist* headers = NULL; + curl_slist *headers = NULL; headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); @@ -738,8 +732,8 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, res = curl_easy_perform(curl); if (res != CURLE_OK) { - std::cerr << "curl failed: " << curl_easy_strerror(res) << "| url: " - << hostPGAddr << "| data: " << job_data << std::endl; + util_logger.error("curl failed: " + std::string(curl_easy_strerror(res)) + "| url: " + + hostPGAddr + "| data: " + job_data); } curl_easy_cleanup(curl); @@ -747,9 +741,9 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, return response_string; } -static std::map getMetricMap(std::string metricName){ +static std::map getMetricMap(std::string metricName) { std::map map; - CURL* curl; + CURL *curl; CURLcode res; std::string response_cpu_usages; std::string pushGatewayAddr = getPushGatewayAddress() + "api/v1/query?query=" + metricName; @@ -768,14 +762,14 @@ static std::map getMetricMap(std::string metricName){ res = curl_easy_perform(curl); if (res != CURLE_OK) { std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; - } else{ + } else { std::cout << response_cpu_usages << std::endl; Json::Value root; Json::Reader reader; reader.parse(response_cpu_usages, root); const Json::Value results = root["data"]["result"]; Json::Value currentExportedJobName; - for(int i=1; i Date: Sun, 11 Feb 2024 19:57:32 +0530 Subject: [PATCH 031/143] Fix issues --- conf/jasminegraph-server.properties | 5 ++++- src/performance/metrics/PerformanceUtil.cpp | 1 - src/util/Utils.cpp | 14 ++++++++++--- src/util/Utils.h | 2 ++ start-k8s.sh | 22 ++++++++++++++++++--- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 9fb69b6ae..1cab86e94 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -76,8 +76,11 @@ org.jasminegraph.scheduler.enabled=true #PerformanceCollector Scheduler Timing. Run once every 120 seconds org.jasminegraph.scheduler.performancecollector.timing=120 -#pushgateway address +#-------------------------------------------------------------------------------- +#PerformanceCollector +#-------------------------------------------------------------------------------- org.jasminegraph.collector.pushgateway=http://192.168.8.150:9091/ +org.jasminegraph.collector.prometheus=http://192.168.8.150:9090/ #-------------------------------------------------------------------------------- #MetaDB information diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index dc3d936ec..659d13779 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -919,7 +919,6 @@ void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *pe valuesString += "('" + graphSlaId + "','" + placeId + "', '','" + usageInfo.loadAverage + "','" + usageInfo.elapsedTime + "'),"; - // FIXME(Aruna): haven't done this per host basis Utils::send_job("loadAverageSLA", "load_average", usageInfo.loadAverage); } valuesString = valuesString.substr(0, valuesString.length() - 1); diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index c84335a7b..25cbed2d9 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -701,7 +701,9 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, std::string metric_value) { CURL *curl; CURLcode res; - std::string pushGatewayJobAddr = getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); + std::string pushGatewayJobAddr = getJasmineGraphProperty("org.jasminegraph.collector.pushgateway") + + "metrics/job/"; +// pushGatewayJobAddr = "http://10.10.18.115:8080/"; std::string response_string; curl = curl_easy_init(); @@ -731,9 +733,14 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, curl_easy_setopt(curl, CURLOPT_POST, 1); res = curl_easy_perform(curl); + long code = -1; + curl_easy_getinfo( curl, CURLINFO_RESPONSE_CODE, &code ); if (res != CURLE_OK) { util_logger.error("curl failed: " + std::string(curl_easy_strerror(res)) + "| url: " + hostPGAddr + "| data: " + job_data); + } else { + util_logger.info("curl success | url: " + hostPGAddr + "| data: " + job_data + + "| status: " + to_string(code)); } curl_easy_cleanup(curl); @@ -741,12 +748,13 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, return response_string; } -static std::map getMetricMap(std::string metricName) { +std::map Utils::getMetricMap(std::string metricName) { std::map map; CURL *curl; CURLcode res; std::string response_cpu_usages; - std::string pushGatewayAddr = getPushGatewayAddress() + "api/v1/query?query=" + metricName; + std::string pushGatewayJobAddr = getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); + std::string pushGatewayAddr = pushGatewayJobAddr + "api/v1/query?query=" + metricName; curl = curl_easy_init(); if (curl) { diff --git a/src/util/Utils.h b/src/util/Utils.h index d674a6c7d..5870d5a89 100644 --- a/src/util/Utils.h +++ b/src/util/Utils.h @@ -172,6 +172,8 @@ class Utils { static std::string send_job(std::string job_group_name, std::string metric_name, std::string metric_value); + + static map getMetricMap(string metricName); }; #endif // JASMINEGRAPH_UTILS_H diff --git a/start-k8s.sh b/start-k8s.sh index dba90956c..55f23858f 100755 --- a/start-k8s.sh +++ b/start-k8s.sh @@ -80,7 +80,25 @@ while true; do done pushgateway_address="${pushgatewayIP}:9091" envsubst <"./k8s/prometheus.yaml" | kubectl apply -f - -sed -i "s#org.jasminegraph.collector.pushgateway=.*#org.jasminegraph.collector.pushgateway=http://${pushgatewayIP}:9091/metrics/job/#" ./conf/jasminegraph-server.properties + +cur_timestamp="$(date +%s)" +end_timestamp="$((cur_timestamp + TIMEOUT_SECONDS))" +while true; do + if [ "$(date +%s)" -gt "$end_timestamp" ]; then + echo "Timeout" + exit 1 + fi + prometheusIP="$(kubectl get services |& grep prometheus | tr '\t' ' ' | tr -s ' ' | cut -d ' ' -f 3)" + if [ ! -z "$prometheusIP" ]; then + break + fi + printf "Waiting prometheus to start [%c] \r" "${spin:i++%${#spin}:1}" + sleep .2 +done + + +sed -i "s#org.jasminegraph.collector.pushgateway=.*#org.jasminegraph.collector.pushgateway=http://${pushgatewayIP}:9091/#" ./conf/jasminegraph-server.properties +sed -i "s#org.jasminegraph.collector.prometheus=.*#org.jasminegraph.collector.prometheus=http://${pushgatewayIP}:9090/#" ./conf/jasminegraph-server.properties docker build -t jasminegraph . @@ -97,8 +115,6 @@ no_of_workers="${NO_OF_WORKERS}" \ # wait until master starts listening cur_timestamp="$(date +%s)" end_timestamp="$((cur_timestamp + TIMEOUT_SECONDS))" -spin="/-\|" -i=0 while true; do if [ "$(date +%s)" -gt "$end_timestamp" ]; then echo "Timeout" From e26abdd8e891689669a045af004f34d25a22a46f Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Mon, 12 Feb 2024 16:15:08 +0530 Subject: [PATCH 032/143] Fix getCpuUsage --- .../metrics/StatisticCollector.cpp | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index bbc992b7c..a0ea4000b 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -13,7 +13,6 @@ limitations under the License. #include "StatisticCollector.h" -static clock_t lastCPU, lastSysCPU, lastUserCPU; static int numProcessors; static long parseLine(char *line); @@ -24,10 +23,6 @@ int StatisticCollector::init() { struct tms timeSample; char line[128]; - lastCPU = times(&timeSample); - lastSysCPU = timeSample.tms_stime; - lastUserCPU = timeSample.tms_utime; - file = fopen("/proc/cpuinfo", "r"); numProcessors = 0; while (fgets(line, 128, file) != NULL) { @@ -153,7 +148,7 @@ int StatisticCollector::getSocketCount() { count++; } } - (void) closedir(d); + (void)closedir(d); std::cout << "Total sockets: " + std::to_string(count) << std::endl; return count; @@ -170,25 +165,41 @@ static long parseLine(char *line) { } double StatisticCollector::getCpuUsage() { - struct tms timeSample; - clock_t now; - double percent; - - now = times(&timeSample); - if (now <= lastCPU || timeSample.tms_stime < lastSysCPU || timeSample.tms_utime < lastUserCPU) { - // Overflow detection. Just skip this value. - percent = -1.0; - } else { - percent = (timeSample.tms_stime - lastSysCPU) + (timeSample.tms_utime - lastUserCPU); - percent /= (now - lastCPU); - percent /= numProcessors; - percent *= 100; + static long long lastTotal = 0, lastIdle = 0; + + FILE *fp = fopen("/proc/stat", "r"); + if (!fp) return -1; + char line[1024]; + fscanf(fp, "%[^\r\n]%*c", line); + fclose(fp); + std::cout << line << std::endl; + char *p = line; + while (*p < '0' || *p > '9') p++; + long long total = 0; + long long idle = 0; + char *end_ptr = p; + for (int field = 1; field <= 10; field++) { + while (*p < '0' || *p > '9') { + if (!(*p)) break; + p++; + } + if (!(*p)) break; + long long value = strtoll(p, &end_ptr, 10); + p = end_ptr; + if (value < 0) { + std::cerr << "Value is " << value << " for line " << line << std::endl; + } + if (field == 4) { + idle += value; + } + total += value; } - lastCPU = now; - lastSysCPU = timeSample.tms_stime; - lastUserCPU = timeSample.tms_utime; + long long diffTotal = lastTotal - total; + long long diffIdle = lastIdle - idle; + lastTotal = total; + lastIdle = idle; - return percent; + return (diffTotal - diffIdle) / (double)diffTotal; } std::string StatisticCollector::collectVMStatistics(std::string isVMStatManager, From 7882b65ef9f77265be056b3741b6d32edbdd10e3 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Tue, 13 Feb 2024 17:15:18 +0530 Subject: [PATCH 033/143] Get jasminegraph properties from config map in k8s --- CMakeLists.txt | 3 ++- Dockerfile | 1 + globals.h | 21 +++++++++++++++++++++ k8s/configs.yaml | 11 +++++++++++ main.cpp | 4 +++- src/k8s/K8sInterface.cpp | 17 +++++++++++++++++ src/k8s/K8sInterface.h | 2 ++ src/util/Utils.cpp | 27 +++++++++++++++++++++------ 8 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 globals.h create mode 100644 k8s/configs.yaml diff --git a/CMakeLists.txt b/CMakeLists.txt index dec67eff2..0eaacbd64 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,8 @@ set(CMAKE_CXX_STANDARD 11) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -set(HEADERS src/backend/JasmineGraphBackend.h +set(HEADERS globals.h + src/backend/JasmineGraphBackend.h src/backend/JasmineGraphBackendProtocol.h src/centralstore/JasmineGraphHashMapCentralStore.h src/centralstore/JasmineGraphHashMapDuplicateCentralStore.h diff --git a/Dockerfile b/Dockerfile index bacbf516d..9b7b74670 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,7 @@ COPY ./build.sh ./build.sh COPY ./CMakeLists.txt ./CMakeLists.txt COPY ./main.h ./main.h COPY ./main.cpp ./main.cpp +COPY ./globals.h ./globals.h COPY ./src ./src RUN if [ "$DEBUG" = "true" ]; then echo "building in DEBUG mode" && sh build.sh --debug; else sh build.sh; fi diff --git a/globals.h b/globals.h new file mode 100644 index 000000000..4f85cc003 --- /dev/null +++ b/globals.h @@ -0,0 +1,21 @@ +/** +Copyright 2024 JasmineGraph Team +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +#include + +#ifndef JASMINEGRAPH__GLOBALS_H_ +#define JASMINEGRAPH__GLOBALS_H_ + +std::string jasminegraph_profile; + +#endif //JASMINEGRAPH__GLOBALS_H_ diff --git a/k8s/configs.yaml b/k8s/configs.yaml new file mode 100644 index 000000000..83f47ffe5 --- /dev/null +++ b/k8s/configs.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: jasminegraph-config + labels: + application: jasminegraph +data: + pushgateway_address: "${pushgateway_address}" + prometheus_address: "${prometheus_address}" + + \ No newline at end of file diff --git a/main.cpp b/main.cpp index 2bd356e09..75476fc73 100644 --- a/main.cpp +++ b/main.cpp @@ -18,6 +18,7 @@ limitations under the License. #include #include +#include "globals.h" #include "src/k8s/K8sWorkerController.h" #include "src/server/JasmineGraphInstance.h" #include "src/util/logger/Logger.h" @@ -48,7 +49,8 @@ int main(int argc, char *argv[]) { int mode = atoi(argv[2]); std::string JASMINEGRAPH_HOME = Utils::getJasmineGraphHome(); - std::string profile = argv[1]; // This can be either "docker" or "native" + std::string profile = argv[1]; + jasminegraph_profile = profile; std::string enableNmon = "false"; main_logger.log("Using JASMINE_GRAPH_HOME", "info"); diff --git a/src/k8s/K8sInterface.cpp b/src/k8s/K8sInterface.cpp index 9ad7adf46..2b3f088a7 100644 --- a/src/k8s/K8sInterface.cpp +++ b/src/k8s/K8sInterface.cpp @@ -173,3 +173,20 @@ v1_node_list_t *K8sInterface::getNodes() { NULL); /* watch */ return node_list; } + +std::string K8sInterface::getJasmineGraphConfig(std::string key) { + v1_config_map_t *config_map = + CoreV1API_readNamespacedConfigMap(apiClient, strdup("jasminegraph-config"), namespace_, NULL); + if (config_map->metadata->name == NULL) { + k8s_logger.error("No jasminegraph-config config map."); + return ""; + } + + listEntry_t *data = NULL; + list_ForEach(data, config_map->data) { + auto *pair = static_cast(data->data); + if (strcmp(pair->key, key.c_str()) == 0) { + return std::string(static_cast(pair->value)); + } + } +} diff --git a/src/k8s/K8sInterface.h b/src/k8s/K8sInterface.h index f2b9fab39..4f65e67dc 100644 --- a/src/k8s/K8sInterface.h +++ b/src/k8s/K8sInterface.h @@ -55,6 +55,8 @@ class K8sInterface { std::string getMasterIp(); v1_node_list_t *getNodes(); + + std::string getJasmineGraphConfig(std::string key); }; #endif // JASMINEGRAPH_K8SINTERFACE_H diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 25cbed2d9..f003f13e4 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -26,7 +26,9 @@ limitations under the License. #include #include +#include "../../globals.h" #include "../server/JasmineGraphInstanceProtocol.h" +#include "../k8s/K8sInterface.h" #include "Conts.h" #include "logger/Logger.h" @@ -701,9 +703,15 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, std::string metric_value) { CURL *curl; CURLcode res; - std::string pushGatewayJobAddr = getJasmineGraphProperty("org.jasminegraph.collector.pushgateway") + - "metrics/job/"; -// pushGatewayJobAddr = "http://10.10.18.115:8080/"; + std::string pushGatewayJobAddr; + if (jasminegraph_profile ==Conts::PROFILE_K8S) { + std::unique_ptr interface(new K8sInterface()); + pushGatewayJobAddr = interface->getJasmineGraphConfig("pushgateway_address"); + } else { + pushGatewayJobAddr = getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); + } + + pushGatewayJobAddr += "metrics/job/"; std::string response_string; curl = curl_easy_init(); @@ -753,12 +761,19 @@ std::map Utils::getMetricMap(std::string metricName) { CURL *curl; CURLcode res; std::string response_cpu_usages; - std::string pushGatewayJobAddr = getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); - std::string pushGatewayAddr = pushGatewayJobAddr + "api/v1/query?query=" + metricName; + + std::string prometheusAddr; + if (jasminegraph_profile ==Conts::PROFILE_K8S) { + std::unique_ptr interface(new K8sInterface()); + prometheusAddr = interface->getJasmineGraphConfig("prometheus_address"); + } else { + prometheusAddr = getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); + } + std::string prometheusQueryAddr = prometheusAddr + "api/v1/query?query=" + metricName; curl = curl_easy_init(); if (curl) { - curl_easy_setopt(curl, CURLOPT_URL, pushGatewayAddr.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, prometheusQueryAddr.c_str()); // Set the callback function to handle the response data curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); From 5552c6427117f78b52145f1c76784fe88789b626 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 14 Feb 2024 00:02:55 +0530 Subject: [PATCH 034/143] Select worker with lowest load and Fix build failure --- globals.h | 2 +- main.cpp | 3 +- src/k8s/K8sInterface.cpp | 1 + src/server/JasmineGraphServer.cpp | 56 ++++++++++++++++++++++++++++--- src/util/Utils.cpp | 4 +-- 5 files changed, 58 insertions(+), 8 deletions(-) diff --git a/globals.h b/globals.h index 4f85cc003..f236d7921 100644 --- a/globals.h +++ b/globals.h @@ -16,6 +16,6 @@ limitations under the License. #ifndef JASMINEGRAPH__GLOBALS_H_ #define JASMINEGRAPH__GLOBALS_H_ -std::string jasminegraph_profile; +extern char *jasminegraph_profile; #endif //JASMINEGRAPH__GLOBALS_H_ diff --git a/main.cpp b/main.cpp index 75476fc73..6d954af97 100644 --- a/main.cpp +++ b/main.cpp @@ -29,6 +29,7 @@ JasmineGraphServer *server; JasmineGraphInstance *instance; SchedulerService schedulerService; Logger main_logger; +char *jasminegraph_profile = NULL; void fnExit3(void) { delete (server); @@ -50,7 +51,7 @@ int main(int argc, char *argv[]) { int mode = atoi(argv[2]); std::string JASMINEGRAPH_HOME = Utils::getJasmineGraphHome(); std::string profile = argv[1]; - jasminegraph_profile = profile; + jasminegraph_profile = argv[1]; std::string enableNmon = "false"; main_logger.log("Using JASMINE_GRAPH_HOME", "info"); diff --git a/src/k8s/K8sInterface.cpp b/src/k8s/K8sInterface.cpp index 2b3f088a7..28e613520 100644 --- a/src/k8s/K8sInterface.cpp +++ b/src/k8s/K8sInterface.cpp @@ -189,4 +189,5 @@ std::string K8sInterface::getJasmineGraphConfig(std::string key) { return std::string(static_cast(pair->value)); } } + return ""; } diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 7a7a2908d..12e13abea 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -773,21 +773,69 @@ int JasmineGraphServer::shutdown_worker(std::string workerIP, int port) { return 0; } +static map scaleK8s(size_t npart) { + std::vector &workerList = K8sWorkerController::workerList; + map cpu_map = Utils::getMetricMap("cpu_usage"); + // Convert strings to float + map cpu_loads; + for (auto it = cpu_map.begin(); it != cpu_map.end(); it++) { + cpu_loads[it->first] = (float)atof(it->second.c_str()); + } + + for (auto it = workerList.begin(); it != workerList.end(); it++) { + auto &worker = *it; + // 0.8 depends on num cpu cores and other factors + if (npart > 0 && cpu_loads[worker.hostname + ":" + to_string(worker.port)] < 0.8) npart--; + } + if (npart <= 0) return cpu_loads; + K8sWorkerController *controller = K8sWorkerController::getInstance(); + size_t new_workers = workerList.size() + npart; + controller->setNumberOfWorkers((int)new_workers); + server_logger.info("Scalled up to " + to_string(new_workers) + " workers"); + for (auto it = K8sWorkerController::workerList.begin(); it != K8sWorkerController::workerList.end(); it++) { + auto &worker = *it; + string workerHostPort = worker.hostname + ":" + to_string(worker.port); + if (cpu_loads.find(workerHostPort) == cpu_loads.end()) { + cpu_loads[workerHostPort] = 0.; + } + } + return cpu_loads; +} + static std::vector getWorkers(size_t npart, std::string profile) { // TODO: get the workers with lowest load from workerList std::vector *workerListAll; + map cpu_loads; if (profile == Conts::PROFILE_K8S) { workerListAll = &(K8sWorkerController::workerList); + cpu_loads = scaleK8s(npart); } else { workerListAll = &(hostWorkerList); + for (auto it = hostWorkerList.begin(); it != hostWorkerList.end(); it++) { + auto &worker = *it; + string workerHostPort = worker.hostname + ":" + to_string(worker.port); + cpu_loads[workerHostPort] = 0.; + } } size_t len = workerListAll->size(); std::cout << "workerListAll len = " << len << std::endl; std::vector workerList; - for (int i=0; i interface(new K8sInterface()); pushGatewayJobAddr = interface->getJasmineGraphConfig("pushgateway_address"); } else { @@ -763,7 +763,7 @@ std::map Utils::getMetricMap(std::string metricName) { std::string response_cpu_usages; std::string prometheusAddr; - if (jasminegraph_profile ==Conts::PROFILE_K8S) { + if (string(jasminegraph_profile) ==Conts::PROFILE_K8S) { std::unique_ptr interface(new K8sInterface()); prometheusAddr = interface->getJasmineGraphConfig("prometheus_address"); } else { From 05bcefce3a943f3fd4acdde0bcf4c59f12a5f4b3 Mon Sep 17 00:00:00 2001 From: ArunaSenanayake Date: Wed, 14 Feb 2024 01:20:46 +0530 Subject: [PATCH 035/143] Modified the metric retrieval function implementation --- conf/jasminegraph-server.properties | 4 +-- main.cpp | 2 ++ src/performance/metrics/PerformanceUtil.cpp | 36 +++++++++++++++------ src/util/Utils.cpp | 19 +++++++++-- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 1cab86e94..22cfbf25c 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -79,8 +79,8 @@ org.jasminegraph.scheduler.performancecollector.timing=120 #-------------------------------------------------------------------------------- #PerformanceCollector #-------------------------------------------------------------------------------- -org.jasminegraph.collector.pushgateway=http://192.168.8.150:9091/ -org.jasminegraph.collector.prometheus=http://192.168.8.150:9090/ +org.jasminegraph.collector.pushgateway=http://192.168.43.135:9091/ +org.jasminegraph.collector.prometheus=http://192.168.43.135:9090/ #-------------------------------------------------------------------------------- #MetaDB information diff --git a/main.cpp b/main.cpp index 6d954af97..ec3297b8d 100644 --- a/main.cpp +++ b/main.cpp @@ -93,8 +93,10 @@ int main(int argc, char *argv[]) { string hostName; hostName = argv[3]; + setenv("HOST_NAME", argv[3], 1); std::string masterHost = argv[4]; int serverPort = atoi(argv[5]); + setenv("PORT", argv[5], 1); int serverDataPort = atoi(argv[6]); enableNmon = argv[7]; diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index 659d13779..0a72c595b 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -36,31 +36,49 @@ void PerformanceUtil::init() { int PerformanceUtil::collectPerformanceStatistics() { long memoryUsage = StatisticCollector::getMemoryUsageByProcess(); - Utils::send_job("WorkerPerfData", "memory_usage", std::to_string(memoryUsage)); + //Worker + Utils::send_job("", "memory_usage", std::to_string(memoryUsage)); double cpuUsage = StatisticCollector::getCpuUsage(); - Utils::send_job("WorkerPerfData", "cpu_usage", std::to_string(cpuUsage)); + //Worker + Utils::send_job("", "cpu_usage", std::to_string(cpuUsage)); int threadCount = StatisticCollector::getThreadCount(); - Utils::send_job("HostPerfData", "thread_count", std::to_string(threadCount)); + //Host + Utils::send_job("", "thread_count", std::to_string(threadCount)); long totalSwapSpace = StatisticCollector::getTotalSwapSpace(); - Utils::send_job("HostPerfData", "total_swap_space", std::to_string(totalSwapSpace)); + //Host + Utils::send_job("", "total_swap_space", std::to_string(totalSwapSpace)); long usedSwapSpace = StatisticCollector::getUsedSwapSpace(); - Utils::send_job("HostPerfData", "used_swap_space", std::to_string(usedSwapSpace)); + //Host + Utils::send_job("", "used_swap_space", std::to_string(usedSwapSpace)); long rxBytes = StatisticCollector::getRXBytes(); - Utils::send_job("WorkerPerfData", "rx_bytes", std::to_string(rxBytes)); + //Worker + Utils::send_job("", "rx_bytes", std::to_string(rxBytes)); long txBytes = StatisticCollector::getTXBytes(); - Utils::send_job("WorkerPerfData", "tx_bytes", std::to_string(txBytes)); + //Worker + Utils::send_job("", "tx_bytes", std::to_string(txBytes)); int socketCount = StatisticCollector::getSocketCount(); - Utils::send_job("HostPerfData", "socket_count", std::to_string(socketCount)); + //Host + Utils::send_job("", "socket_count", std::to_string(socketCount)); long totalMemoryUsage = StatisticCollector::getTotalMemoryUsage(); - Utils::send_job("HostPerfData", "total_memory", std::to_string(totalMemoryUsage)); + //Host + Utils::send_job("", "total_memory", std::to_string(totalMemoryUsage)); + + std::map cpuUsageMap = Utils::getMetricMap("cpu_usage"); + + std::cout << "#----------------" << std::endl; + std::cout << "192.168.43.135:7780" << ":----" + cpuUsageMap["192.168.43.135:7780"] << std::endl; + std::cout << "192.168.43.135:7782" << ":----" + cpuUsageMap["192.168.43.135:7782"] << std::endl; + std::cout << "192.168.43.135:7784" << ":----" + cpuUsageMap["192.168.43.135:7784"] << std::endl; + std::cout << "192.168.43.135:7786" << ":----" + cpuUsageMap["192.168.43.135:7786"] << std::endl; + std::cout << "#----------------" << std::endl; return 0; } diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 8328e83e1..e502169b3 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -717,6 +717,7 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, curl = curl_easy_init(); if (curl) { std::string hostPGAddr; + /* const char *envWorkerID = getenv("WORKER_ID"); std::string workerID; if (envWorkerID) { @@ -724,7 +725,19 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, } else { workerID = "-1"; } - hostPGAddr = pushGatewayJobAddr + job_group_name + "_" + workerID; + */ + + const char *hostAddress = getenv("HOST_NAME"); + const char *port = getenv("PORT"); + + std::string uniqueWorkerID; + if(hostAddress) { + uniqueWorkerID = std::string(getenv("HOST_NAME")) + ":" + std::string(getenv("PORT")); + } else{ + uniqueWorkerID = "Master"; + } + //hostPGAddr = pushGatewayJobAddr + job_group_name + "_" + workerID; + hostPGAddr = pushGatewayJobAddr + uniqueWorkerID; curl_easy_setopt(curl, CURLOPT_URL, hostPGAddr.c_str()); // Set the callback function to handle the response data @@ -767,7 +780,7 @@ std::map Utils::getMetricMap(std::string metricName) { std::unique_ptr interface(new K8sInterface()); prometheusAddr = interface->getJasmineGraphConfig("prometheus_address"); } else { - prometheusAddr = getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); + prometheusAddr = getJasmineGraphProperty("org.jasminegraph.collector.prometheus"); } std::string prometheusQueryAddr = prometheusAddr + "api/v1/query?query=" + metricName; @@ -792,7 +805,7 @@ std::map Utils::getMetricMap(std::string metricName) { reader.parse(response_cpu_usages, root); const Json::Value results = root["data"]["result"]; Json::Value currentExportedJobName; - for (int i = 1; i < results.size(); i++) { + for (int i = 0; i < results.size(); i++) { currentExportedJobName = results[i]["metric"]["exported_job"]; map[(currentExportedJobName.asString().c_str())] = (results[i]["value"][1]).asString(); } From e85d9250f726e2e46f6eebe97c36b058bd47cc15 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Wed, 14 Feb 2024 09:04:51 +0530 Subject: [PATCH 036/143] Create configs config map --- src/util/Utils.cpp | 12 ++++++------ start-k8s.sh | 8 +++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index e502169b3..4f0e8891f 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -704,7 +704,7 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, CURL *curl; CURLcode res; std::string pushGatewayJobAddr; - if (string(jasminegraph_profile) ==Conts::PROFILE_K8S) { + if (string(jasminegraph_profile) == Conts::PROFILE_K8S) { std::unique_ptr interface(new K8sInterface()); pushGatewayJobAddr = interface->getJasmineGraphConfig("pushgateway_address"); } else { @@ -731,9 +731,9 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, const char *port = getenv("PORT"); std::string uniqueWorkerID; - if(hostAddress) { + if (hostAddress) { uniqueWorkerID = std::string(getenv("HOST_NAME")) + ":" + std::string(getenv("PORT")); - } else{ + } else { uniqueWorkerID = "Master"; } //hostPGAddr = pushGatewayJobAddr + job_group_name + "_" + workerID; @@ -755,13 +755,13 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, res = curl_easy_perform(curl); long code = -1; - curl_easy_getinfo( curl, CURLINFO_RESPONSE_CODE, &code ); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); if (res != CURLE_OK) { util_logger.error("curl failed: " + std::string(curl_easy_strerror(res)) + "| url: " + hostPGAddr + "| data: " + job_data); } else { util_logger.info("curl success | url: " + hostPGAddr + "| data: " + job_data + - "| status: " + to_string(code)); + "| status: " + to_string(code)); } curl_easy_cleanup(curl); @@ -776,7 +776,7 @@ std::map Utils::getMetricMap(std::string metricName) { std::string response_cpu_usages; std::string prometheusAddr; - if (string(jasminegraph_profile) ==Conts::PROFILE_K8S) { + if (string(jasminegraph_profile) == Conts::PROFILE_K8S) { std::unique_ptr interface(new K8sInterface()); prometheusAddr = interface->getJasmineGraphConfig("prometheus_address"); } else { diff --git a/start-k8s.sh b/start-k8s.sh index 55f23858f..eb90f923c 100755 --- a/start-k8s.sh +++ b/start-k8s.sh @@ -96,11 +96,9 @@ while true; do sleep .2 done - -sed -i "s#org.jasminegraph.collector.pushgateway=.*#org.jasminegraph.collector.pushgateway=http://${pushgatewayIP}:9091/#" ./conf/jasminegraph-server.properties -sed -i "s#org.jasminegraph.collector.prometheus=.*#org.jasminegraph.collector.prometheus=http://${pushgatewayIP}:9090/#" ./conf/jasminegraph-server.properties - -docker build -t jasminegraph . +pushgateway_address="${pushgatewayIP}:9091/" \ + prometheus_address="${prometheusIP}:9090/" \ + envsubst <"./k8s/configs.yaml" | kubectl apply -f - metadb_path="${META_DB_PATH}" \ performancedb_path="${PERFORMANCE_DB_PATH}" \ From 43c83ca18a60d990be1ba2aaa6b3c03b35b3e6e4 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Wed, 14 Feb 2024 09:13:15 +0530 Subject: [PATCH 037/143] Fix style checks --- globals.h | 2 +- src/performance/metrics/PerformanceUtil.cpp | 19 +++++++++---------- src/server/JasmineGraphServer.cpp | 2 +- src/util/Utils.cpp | 4 ++-- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/globals.h b/globals.h index f236d7921..736971df1 100644 --- a/globals.h +++ b/globals.h @@ -18,4 +18,4 @@ limitations under the License. extern char *jasminegraph_profile; -#endif //JASMINEGRAPH__GLOBALS_H_ +#endif // JASMINEGRAPH__GLOBALS_H_ diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index 0a72c595b..9eebf1092 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -36,43 +36,42 @@ void PerformanceUtil::init() { int PerformanceUtil::collectPerformanceStatistics() { long memoryUsage = StatisticCollector::getMemoryUsageByProcess(); - //Worker Utils::send_job("", "memory_usage", std::to_string(memoryUsage)); double cpuUsage = StatisticCollector::getCpuUsage(); - //Worker + // Worker Utils::send_job("", "cpu_usage", std::to_string(cpuUsage)); int threadCount = StatisticCollector::getThreadCount(); - //Host + // Host Utils::send_job("", "thread_count", std::to_string(threadCount)); long totalSwapSpace = StatisticCollector::getTotalSwapSpace(); - //Host + // Host Utils::send_job("", "total_swap_space", std::to_string(totalSwapSpace)); long usedSwapSpace = StatisticCollector::getUsedSwapSpace(); - //Host + // Host Utils::send_job("", "used_swap_space", std::to_string(usedSwapSpace)); long rxBytes = StatisticCollector::getRXBytes(); - //Worker + // Worker Utils::send_job("", "rx_bytes", std::to_string(rxBytes)); long txBytes = StatisticCollector::getTXBytes(); - //Worker + // Worker Utils::send_job("", "tx_bytes", std::to_string(txBytes)); int socketCount = StatisticCollector::getSocketCount(); - //Host + // Host Utils::send_job("", "socket_count", std::to_string(socketCount)); long totalMemoryUsage = StatisticCollector::getTotalMemoryUsage(); - //Host + // Host Utils::send_job("", "total_memory", std::to_string(totalMemoryUsage)); std::map cpuUsageMap = Utils::getMetricMap("cpu_usage"); - + std::cout << "#----------------" << std::endl; std::cout << "192.168.43.135:7780" << ":----" + cpuUsageMap["192.168.43.135:7780"] << std::endl; std::cout << "192.168.43.135:7782" << ":----" + cpuUsageMap["192.168.43.135:7782"] << std::endl; diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 12e13abea..963b297ed 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -833,7 +833,7 @@ static std::vector getWorkers(size_t npart, std::str } } string workerHostPort = worker_min.hostname + ":" + to_string(worker_min.port); - cpu_loads[workerHostPort] += 0.25; // 0.25 = 1/nproc + cpu_loads[workerHostPort] += 0.25; // 0.25 = 1/nproc workerList.push_back(worker_min); std::cout << "worker = " << worker_min.hostname << ":" << worker_min.port << std::endl; } diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 4f0e8891f..ab1466852 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -732,11 +732,11 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, std::string uniqueWorkerID; if (hostAddress) { - uniqueWorkerID = std::string(getenv("HOST_NAME")) + ":" + std::string(getenv("PORT")); + uniqueWorkerID = std::string(hostAddress) + ":" + std::string(port); } else { uniqueWorkerID = "Master"; } - //hostPGAddr = pushGatewayJobAddr + job_group_name + "_" + workerID; + // hostPGAddr = pushGatewayJobAddr + job_group_name + "_" + workerID; hostPGAddr = pushGatewayJobAddr + uniqueWorkerID; curl_easy_setopt(curl, CURLOPT_URL, hostPGAddr.c_str()); From ca41c64c961f21f0454fc2070fb961287c0d4244 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Wed, 14 Feb 2024 09:15:49 +0530 Subject: [PATCH 038/143] Fix unit test build failure --- tests/unit/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/Dockerfile b/tests/unit/Dockerfile index 545ebaa71..84044ea1f 100644 --- a/tests/unit/Dockerfile +++ b/tests/unit/Dockerfile @@ -24,6 +24,7 @@ COPY ./src_python ./src_python COPY ./src ./src COPY ./main.h ./main.h COPY ./main.cpp ./main.cpp +COPY ./globals.h ./globals.h COPY ./conf ./conf COPY ./k8s ./k8s From 6c65a6f1c20d25f0078fab2c88bf93e8cc1485e9 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Wed, 14 Feb 2024 13:10:34 +0530 Subject: [PATCH 039/143] Fix job name in k8s implementation --- README.md | 2 +- k8s/worker-deployment.yaml | 2 +- src/k8s/K8sInterface.cpp | 2 ++ src/k8s/K8sInterface.h | 1 + src/k8s/K8sWorkerController.cpp | 19 ++++++++++--------- src/performance/metrics/PerformanceUtil.cpp | 9 --------- tests/unit/k8s/K8sInterface_test.cpp | 2 +- 7 files changed, 16 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 89a924033..4c4cd7a95 100755 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Run JasmineGraph in Kubernetes environment by providing appropriate parameters: Remove all resources created by JasmineGraph in Kubernetes environment: - ./stop-k8s.sh clean + ./start-k8s.sh clean ## 3. Contact Information diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index 8a49ded32..9b728fd9c 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -29,7 +29,7 @@ spec: - "--MODE" - "2" - "--HOST_NAME" - - "jasminegraph-worker" + - "" - "--MASTERIP" - "" - "--SERVER_PORT" diff --git a/src/k8s/K8sInterface.cpp b/src/k8s/K8sInterface.cpp index 28e613520..50f77da30 100644 --- a/src/k8s/K8sInterface.cpp +++ b/src/k8s/K8sInterface.cpp @@ -87,6 +87,7 @@ v1_service_list_t *K8sInterface::getServiceList(char *labelSelectors) { } v1_deployment_t *K8sInterface::createJasmineGraphWorkerDeployment(int workerId, + const std::string &ip, const std::string &masterIp, const std::string &nodeName) const { std::string definiton = Utils::getJsonStringFromYamlFile(ROOT_DIR "/k8s/worker-deployment.yaml"); @@ -94,6 +95,7 @@ v1_deployment_t *K8sInterface::createJasmineGraphWorkerDeployment(int workerId, definiton = Utils::replaceAll(definiton, "", masterIp); definiton = Utils::replaceAll(definiton, "", Utils::getJasmineGraphProperty("org.jasminegraph.k8s.image")); definiton = Utils::replaceAll(definiton, "", nodeName); + definiton = Utils::replaceAll(definiton, "", ip); cJSON *deploymentTemplate = cJSON_Parse(definiton.c_str()); v1_deployment_t *deployment = v1_deployment_parseFromJSON(deploymentTemplate); diff --git a/src/k8s/K8sInterface.h b/src/k8s/K8sInterface.h index 4f65e67dc..6921adb06 100644 --- a/src/k8s/K8sInterface.h +++ b/src/k8s/K8sInterface.h @@ -37,6 +37,7 @@ class K8sInterface { v1_service_list_t *getServiceList(char *labelSelectors); v1_deployment_t *createJasmineGraphWorkerDeployment(int workerId, + const std::string &ip, const std::string &masterIp, const std::string &nodeName) const; diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index a54f67fe3..ca19523fc 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -89,15 +89,6 @@ void K8sWorkerController::spawnWorker(int workerId) { std::string hostName = it->first; int hostId = it->second; - v1_deployment_t *deployment = - this->interface->createJasmineGraphWorkerDeployment(workerId, this->masterIp, hostName); - if (deployment != nullptr && deployment->metadata != nullptr && deployment->metadata->name != nullptr) { - controller_logger.info("Worker " + std::to_string(workerId) + " deployment created successfully"); - - } else { - throw std::runtime_error("Worker " + std::to_string(workerId) + " deployment creation failed"); - } - v1_service_t *service = this->interface->createJasmineGraphWorkerService(workerId); if (service != nullptr && service->metadata != nullptr && service->metadata->name != nullptr) { controller_logger.info("Worker " + std::to_string(workerId) + " service created successfully"); @@ -106,6 +97,16 @@ void K8sWorkerController::spawnWorker(int workerId) { } std::string ip(service->spec->cluster_ip); + + v1_deployment_t *deployment = + this->interface->createJasmineGraphWorkerDeployment(workerId, ip, this->masterIp, hostName); + if (deployment != nullptr && deployment->metadata != nullptr && deployment->metadata->name != nullptr) { + controller_logger.info("Worker " + std::to_string(workerId) + " deployment created successfully"); + + } else { + throw std::runtime_error("Worker " + std::to_string(workerId) + " deployment creation failed"); + } + JasmineGraphServer::worker worker = { .hostname = ip, .port = Conts::JASMINEGRAPH_INSTANCE_PORT, .dataPort = Conts::JASMINEGRAPH_INSTANCE_DATA_PORT}; K8sWorkerController::workerList.push_back(worker); diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index 9eebf1092..27416dabb 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -69,15 +69,6 @@ int PerformanceUtil::collectPerformanceStatistics() { long totalMemoryUsage = StatisticCollector::getTotalMemoryUsage(); // Host Utils::send_job("", "total_memory", std::to_string(totalMemoryUsage)); - - std::map cpuUsageMap = Utils::getMetricMap("cpu_usage"); - - std::cout << "#----------------" << std::endl; - std::cout << "192.168.43.135:7780" << ":----" + cpuUsageMap["192.168.43.135:7780"] << std::endl; - std::cout << "192.168.43.135:7782" << ":----" + cpuUsageMap["192.168.43.135:7782"] << std::endl; - std::cout << "192.168.43.135:7784" << ":----" + cpuUsageMap["192.168.43.135:7784"] << std::endl; - std::cout << "192.168.43.135:7786" << ":----" + cpuUsageMap["192.168.43.135:7786"] << std::endl; - std::cout << "#----------------" << std::endl; return 0; } diff --git a/tests/unit/k8s/K8sInterface_test.cpp b/tests/unit/k8s/K8sInterface_test.cpp index ead841d90..5233f1b9f 100644 --- a/tests/unit/k8s/K8sInterface_test.cpp +++ b/tests/unit/k8s/K8sInterface_test.cpp @@ -49,7 +49,7 @@ TEST_F(K8sInterfaceTest, TestGetServiceList) { TEST_F(K8sInterfaceTest, TestCreateJasmineGraphWorkerDeployment) { v1_node_list_t *nodeList = interface->getNodes(); auto nodeName = static_cast(nodeList->items->firstEntry->data)->metadata->name; - v1_deployment_t *deployment = interface->createJasmineGraphWorkerDeployment(1, "10.43.0.1", nodeName); + v1_deployment_t *deployment = interface->createJasmineGraphWorkerDeployment(1, "10.43.0.2", "10.43.0.1", nodeName); ASSERT_STREQ(deployment->metadata->name, "jasminegraph-worker1-deployment"); ASSERT_EQ(interface->apiClient->response_code, HTTP_CREATED); } From 70123a185b02d77ae97c9a8eefbff5cd14f7cb93 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 23 Feb 2024 16:20:03 +0530 Subject: [PATCH 040/143] Enable/Disable auto-scaling from properties --- conf/jasminegraph-server.properties | 5 +++++ src/server/JasmineGraphServer.cpp | 12 +++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 22cfbf25c..550013610 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -96,3 +96,8 @@ org.jasminegraph.streaming.db.location=/var/tmp/jasminegraph/jasminegraph_stream org.jasminegraph.k8s.config=/etc/rancher/k3s/k3s.yaml org.jasminegraph.k8s.namespace=default org.jasminegraph.k8s.image=jasminegraph + +#-------------------------------------------------------------------------------- +# Auto-scaling +#-------------------------------------------------------------------------------- +org.jasminegraph.autoscale.enabled=false diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 963b297ed..78cb2804e 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -775,7 +775,7 @@ int JasmineGraphServer::shutdown_worker(std::string workerIP, int port) { static map scaleK8s(size_t npart) { std::vector &workerList = K8sWorkerController::workerList; - map cpu_map = Utils::getMetricMap("cpu_usage"); + const map &cpu_map = Utils::getMetricMap("cpu_usage"); // Convert strings to float map cpu_loads; for (auto it = cpu_map.begin(); it != cpu_map.end(); it++) { @@ -803,6 +803,12 @@ static map scaleK8s(size_t npart) { } static std::vector getWorkers(size_t npart, std::string profile) { + if (Utils::getJasmineGraphProperty("org.jasminegraph.autoscale.enabled") != "true") { + if (profile == Conts::PROFILE_K8S) { + return K8sWorkerController::workerList; + } + return hostWorkerList; + } // TODO: get the workers with lowest load from workerList std::vector *workerListAll; map cpu_loads; @@ -864,10 +870,10 @@ void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, int file_count = 0; std::thread *workerThreads = new std::thread[total_threads]; while (count < total_threads) { - auto workerList = getWorkers(partitionFileMap.size(), this->profile); + const auto &workerList = getWorkers(partitionFileMap.size(), this->profile); for (auto listIterator = workerList.begin(); listIterator < workerList.end(); listIterator++) { worker worker = *listIterator; - if (count == total_threads) { + if (count >= total_threads) { break; } std::string partitionFileName = partitionFileMap[file_count]; From 8997cc828d1adc562f5147491ecb7e2f702b251e Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 23 Feb 2024 17:08:10 +0530 Subject: [PATCH 041/143] Fix unit test build failure --- .github/workflows/build.yml | 8 +++-- CMakeLists.txt | 4 +++ main.cpp | 3 ++ src/util/Utils.cpp | 70 ++++++++++++++++++++----------------- 4 files changed, 50 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 81add0bb4..858a099d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,6 +16,10 @@ jobs: with: ref: ${{github.head_ref}} repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + + - name: Build unit test + run: docker build -f ./tests/unit/Dockerfile -t jasminegraph:unit . + - uses: jupyterhub/action-k3s-helm@v4 with: docker-enabled: true @@ -23,9 +27,6 @@ jobs: - name: Grant permissions for default user run: kubectl apply -f ./k8s/rbac.yaml - - name: Build unit test - run: docker build -f ./tests/unit/Dockerfile -t jasminegraph:unit . - - name: Run unit test run: | mkdir coverage @@ -68,6 +69,7 @@ jobs: with: ref: ${{github.head_ref}} repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + - uses: jupyterhub/action-k3s-helm@v4 with: docker-enabled: true diff --git a/CMakeLists.txt b/CMakeLists.txt index 0eaacbd64..10e7517e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,6 +135,10 @@ set(SOURCES src/backend/JasmineGraphBackend.cpp src/util/dbinterface/DBInterface.cpp ) +if (CMAKE_BUILD_TYPE STREQUAL "DEBUG") +add_compile_options(-DUNIT_TEST) +endif () + add_library(JasmineGraphLib ${HEADERS} ${SOURCES}) add_executable(JasmineGraph main.h main.cpp) diff --git a/main.cpp b/main.cpp index ec3297b8d..4adddee41 100644 --- a/main.cpp +++ b/main.cpp @@ -29,7 +29,10 @@ JasmineGraphServer *server; JasmineGraphInstance *instance; SchedulerService schedulerService; Logger main_logger; + +#ifndef UNIT_TEST char *jasminegraph_profile = NULL; +#endif void fnExit3(void) { delete (server); diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index ab1466852..8a0497143 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -13,10 +13,11 @@ limitations under the License. #include "Utils.h" +#include #include +#include #include #include -#include #include #include @@ -24,17 +25,20 @@ limitations under the License. #include #include #include -#include #include "../../globals.h" -#include "../server/JasmineGraphInstanceProtocol.h" #include "../k8s/K8sInterface.h" +#include "../server/JasmineGraphInstanceProtocol.h" #include "Conts.h" #include "logger/Logger.h" using namespace std; Logger util_logger; +#ifdef UNIT_TEST +char *jasminegraph_profile = NULL; +#endif + unordered_map Utils::propertiesMap; std::vector Utils::split(const std::string &s, char delimiter) { @@ -223,7 +227,7 @@ std::vector Utils::getListOfFilesInDirectory(std::string dirName) { results.push_back(fnamestr); } } - (void) closedir(d); + (void)closedir(d); return results; } @@ -355,7 +359,7 @@ int Utils::unzipFile(std::string filePath, std::string mode) { bool Utils::hostExists(string name, string ip, std::string workerPort, SQLiteDBInterface *sqlite) { bool result = true; string stmt = "SELECT COUNT( * ) FROM worker WHERE name LIKE '" + name + "' AND ip LIKE '" + ip + - "' AND server_port LIKE '" + workerPort + "';"; + "' AND server_port LIKE '" + workerPort + "';"; if (ip == "") { stmt = "SELECT COUNT( * ) FROM worker WHERE name LIKE '" + name + "';"; } @@ -445,8 +449,8 @@ void Utils::updateSLAInformation(PerformanceSQLiteDBInterface *perfSqlite, std:: string slaCategoryId = categoryResults[0][0].second; std::string query = "SELECT id, sla_value, attempt from graph_sla where graph_id='" + graphId + - "' and partition_count='" + std::to_string(partitionCount) + "' and id_sla_category='" + - slaCategoryId + "';"; + "' and partition_count='" + std::to_string(partitionCount) + "' and id_sla_category='" + + slaCategoryId + "';"; std::vector>> results = perfSqlite->runSelect(query); @@ -464,15 +468,15 @@ void Utils::updateSLAInformation(PerformanceSQLiteDBInterface *perfSqlite, std:: attempts++; std::string updateQuery = "UPDATE graph_sla set sla_value='" + std::to_string(newSla) + "', attempt='" + - std::to_string(attempts) + "' where id = '" + slaId + "'"; + std::to_string(attempts) + "' where id = '" + slaId + "'"; perfSqlite->runUpdate(updateQuery); } } else { std::string insertQuery = "insert into graph_sla (id_sla_category, graph_id, partition_count, sla_value, attempt) VALUES ('" + - slaCategoryId + "','" + graphId + "'," + std::to_string(partitionCount) + "," + - std::to_string(newSlaValue) + ",0);"; + slaCategoryId + "','" + graphId + "'," + std::to_string(partitionCount) + "," + + std::to_string(newSlaValue) + ",0);"; perfSqlite->runInsert(insertQuery); } @@ -486,7 +490,7 @@ int Utils::copyToDirectory(std::string currentPath, std::string copyPath) { int status = system(command.c_str()); if (status != 0) { util_logger.error("Copying " + currentPath + " to directory " + copyPath + " failed with code " + - std::to_string(status)); + std::to_string(status)); } return status; } @@ -529,36 +533,36 @@ int Utils::connect_wrapper(int sock, const sockaddr *addr, socklen_t slen) { struct timeval tv = {1, 0}; - if (setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *) &tv, sizeof(tv)) == -1) { + if (setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv)) == -1) { util_logger.error("Failed to set send timeout option for socket"); } - if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &tv, sizeof(tv)) == -1) { + if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv)) == -1) { util_logger.error("Failed to set receive timeout option for socket"); } do { if (retry) sleep(retry * 2); util_logger.info("Trying to connect to [" + to_string(retry) + - "]: " + string(inet_ntoa(((const struct sockaddr_in *) addr)->sin_addr)) + ":" + - to_string(ntohs(((const struct sockaddr_in *) addr)->sin_port))); + "]: " + string(inet_ntoa(((const struct sockaddr_in *)addr)->sin_addr)) + ":" + + to_string(ntohs(((const struct sockaddr_in *)addr)->sin_port))); if (connect(sock, addr, slen) == 0) { tv = {0, 0}; - if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &tv, sizeof(tv)) == -1) { + if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv)) == -1) { util_logger.error("Failed to set receive timeout option for socket after successful connection"); } return 0; } } while (retry++ < 4); - util_logger.error("Error connecting to " + string(inet_ntoa(((const struct sockaddr_in *) addr)->sin_addr)) + ":" + - to_string(ntohs(((const struct sockaddr_in *) addr)->sin_port))); + util_logger.error("Error connecting to " + string(inet_ntoa(((const struct sockaddr_in *)addr)->sin_addr)) + ":" + + to_string(ntohs(((const struct sockaddr_in *)addr)->sin_port))); return -1; } std::string Utils::read_str_wrapper(int connFd, char *buf, size_t len, bool allowEmpty) { ssize_t result = recv(connFd, buf, len, 0); if (result < 0) { - util_logger.error("Read failed: recv returned " + std::to_string((int) result)); + util_logger.error("Read failed: recv returned " + std::to_string((int)result)); return ""; } else if (!allowEmpty && result == 0) { util_logger.error("Read failed: recv empty string"); @@ -648,13 +652,18 @@ static inline json yaml2json(const YAML::Node &root) { json j{}; switch (root.Type()) { - case YAML::NodeType::Null:break; - case YAML::NodeType::Scalar:return parse_scalar(root); - case YAML::NodeType::Sequence:for (auto &&node : root) j.emplace_back(yaml2json(node)); + case YAML::NodeType::Null: + break; + case YAML::NodeType::Scalar: + return parse_scalar(root); + case YAML::NodeType::Sequence: + for (auto &&node : root) j.emplace_back(yaml2json(node)); break; - case YAML::NodeType::Map:for (auto &&it : root) j[it.first.as()] = yaml2json(it.second); + case YAML::NodeType::Map: + for (auto &&it : root) j[it.first.as()] = yaml2json(it.second); + break; + default: break; - default:break; } return j; } @@ -699,8 +708,7 @@ static size_t write_callback(void *contents, size_t size, size_t nmemb, std::str output->append(static_cast(contents), totalSize); return totalSize; } -std::string Utils::send_job(std::string job_group_name, std::string metric_name, - std::string metric_value) { +std::string Utils::send_job(std::string job_group_name, std::string metric_name, std::string metric_value) { CURL *curl; CURLcode res; std::string pushGatewayJobAddr; @@ -757,11 +765,11 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, long code = -1; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); if (res != CURLE_OK) { - util_logger.error("curl failed: " + std::string(curl_easy_strerror(res)) + "| url: " - + hostPGAddr + "| data: " + job_data); + util_logger.error("curl failed: " + std::string(curl_easy_strerror(res)) + "| url: " + hostPGAddr + + "| data: " + job_data); } else { util_logger.info("curl success | url: " + hostPGAddr + "| data: " + job_data + - "| status: " + to_string(code)); + "| status: " + to_string(code)); } curl_easy_cleanup(curl); @@ -816,9 +824,7 @@ std::map Utils::getMetricMap(std::string metricName) { return map; } -bool Utils::fileExistsWithReadPermission(const string &path) { - return access(path.c_str(), R_OK) == 0; -} +bool Utils::fileExistsWithReadPermission(const string &path) { return access(path.c_str(), R_OK) == 0; } std::fstream *Utils::openFile(const string &path, std::ios_base::openmode mode) { if (!fileExistsWithReadPermission(path)) { From 8861634bd9755c025d42b708b6513584364e9aab Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 24 Feb 2024 00:50:26 +0530 Subject: [PATCH 042/143] Remove unnecessery linking Python --- CMakeLists.txt | 8 -------- Dockerfile | 9 +++------ tests/unit/Dockerfile | 2 -- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 10e7517e3..6e9b17485 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,17 +160,9 @@ target_link_libraries(JasmineGraphLib PRIVATE /usr/lib/x86_64-linux-gnu/libflatb target_link_libraries(JasmineGraphLib PRIVATE /usr/lib/x86_64-linux-gnu/libjsoncpp.so) target_link_libraries(JasmineGraphLib PRIVATE /usr/local/lib/libcppkafka.so) target_link_libraries(JasmineGraph JasmineGraphLib) -target_include_directories(JasmineGraph PRIVATE /usr/include/python3) target_link_libraries(JasmineGraph curl) -set(PYTHON_EXECUTABLE "/usr/bin/python3") -set(PYTHON_INCLUDE_DIR "/usr/include/python3") -set(PYTHON_LIBRARIES "/usr/lib/x86_64-linux-gnu/libpython3.so") - -include_directories(${PYTHON_INCLUDE_DIRS}) -include_directories(${PYTHON_DIRECTORIES}) include_directories(/usr/local/include/yaml-cpp) -target_link_libraries(JasmineGraphLib PRIVATE ${PYTHON_LIBRARIES}) target_link_libraries(JasmineGraphLib PRIVATE m) target_link_libraries(JasmineGraphLib PRIVATE /usr/local/lib/libkubernetes.so) target_link_libraries(JasmineGraphLib PRIVATE yaml-cpp) diff --git a/Dockerfile b/Dockerfile index 9b7b74670..da91f7ca1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,13 @@ FROM miyurud/jasminegraph-prerequisites:20240101T095619 -RUN apt-get update -RUN apt-get install libcurl4-openssl-dev -y -RUN apt-get install -y sysstat -RUN apt-get install -y nmon +RUN apt-get update && apt-get install -y libcurl4-openssl-dev sysstat nmon +RUN rm -r /usr/lib/python3.8/distutils +RUN apt-get purge -y libpython3.8-dev python3.8-dev python3.8-distutils libpython3.8 python3.8 ENV HOME="/home/ubuntu" ENV JASMINEGRAPH_HOME="${HOME}/software/jasminegraph" RUN ln -sf /usr/bin/python3.8 /usr/bin/python3 -RUN ln -sf /usr/include/python3.8 /usr/include/python3 -RUN ln -sf /usr/lib/x86_64-linux-gnu/libpython3.8.so /usr/lib/x86_64-linux-gnu/libpython3.so WORKDIR "${JASMINEGRAPH_HOME}" diff --git a/tests/unit/Dockerfile b/tests/unit/Dockerfile index 84044ea1f..ba880a0bf 100644 --- a/tests/unit/Dockerfile +++ b/tests/unit/Dockerfile @@ -6,8 +6,6 @@ RUN apt-get install -y sysstat RUN apt-get install -y nmon RUN ln -sf /usr/bin/python3.8 /usr/bin/python3 -RUN ln -sf /usr/include/python3.8 /usr/include/python3 -RUN ln -sf /usr/lib/x86_64-linux-gnu/libpython3.8.so /usr/lib/x86_64-linux-gnu/libpython3.so RUN python3 -m pip install --no-cache-dir gcovr==6.0 From 97cb7401a72dd24fbdb0bf9984f8cac9228333f3 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 24 Feb 2024 01:52:13 +0530 Subject: [PATCH 043/143] Remove duplicate implementations of sendFileThroughService --- .../executor/impl/TriangleCountExecutor.cpp | 10 +- .../JasminGraphLinkPredictor.cpp | 2 +- src/server/JasmineGraphInstance.cpp | 72 ------- src/server/JasmineGraphInstance.h | 1 - src/server/JasmineGraphInstanceService.cpp | 199 ++++++------------ src/server/JasmineGraphInstanceService.h | 72 +++---- src/server/JasmineGraphServer.cpp | 190 +---------------- src/server/JasmineGraphServer.h | 3 - src/util/Utils.cpp | 181 +++++++++++++++- src/util/Utils.h | 14 +- 10 files changed, 297 insertions(+), 447 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 5ec4233c8..4c36b8592 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -986,9 +986,8 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_CONT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_CONT, "info"); triangleCount_logger.log("Going to send file through service", "info"); - JasmineGraphServer::sendFileThroughService(aggregatorHostName, - std::atoi(aggregatorDataPort.c_str()), fileName, - aggregateStoreFile, masterIP); + Utils::sendFileThroughService(aggregatorHostName, std::atoi(aggregatorDataPort.c_str()), fileName, + aggregateStoreFile); } } } @@ -1375,9 +1374,8 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_CONT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_CONT, "info"); triangleCount_logger.log("Going to send file through service", "info"); - JasmineGraphServer::sendFileThroughService(aggregatorHostName, - std::atoi(aggregatorDataPort.c_str()), fileName, - centralStoreFile, masterIP); + Utils::sendFileThroughService(aggregatorHostName, std::atoi(aggregatorDataPort.c_str()), fileName, + centralStoreFile); } } } diff --git a/src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp b/src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp index 08a462e6c..58e3245a9 100644 --- a/src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp +++ b/src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp @@ -194,7 +194,7 @@ int JasminGraphLinkPredictor::sendQueryToWorker(std::string host, int port, int if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_CONT) == 0) { predictor_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_CONT, "info"); predictor_logger.log("Going to send file through service", "info"); - JasmineGraphServer::sendFileThroughService(host, dataPort, fileName, filePath, masterIP); + Utils::sendFileThroughService(host, dataPort, fileName, filePath); } } } diff --git a/src/server/JasmineGraphInstance.cpp b/src/server/JasmineGraphInstance.cpp index 56c24f977..29dcac873 100644 --- a/src/server/JasmineGraphInstance.cpp +++ b/src/server/JasmineGraphInstance.cpp @@ -203,75 +203,3 @@ void JasmineGraphInstance::startNmonAnalyzer(string enableNmon, int serverPort) } bool JasmineGraphInstance::isRunning() { return true; } - -bool JasmineGraphInstance::sendFileThroughService(std::string host, int dataPort, std::string fileName, - std::string filePath) { - int sockfd; - char data[301]; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return false; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - graphInstance_logger.error("ERROR, no host named " + host); - return false; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(dataPort); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting to port " << dataPort << std::endl; - return false; - } - - fileName = "jasminegraph-local_trained_model_store/" + fileName; - write(sockfd, fileName.c_str(), fileName.size()); - - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - response = Utils::trim_copy(response); - if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE) == 0) { - std::cout << "Sending file " << filePath << " through port " << dataPort << std::endl; - - FILE *fp = fopen(filePath.c_str(), "r"); - if (fp == NULL) { - printf("Error opening file\n"); - close(sockfd); - return false; - } - for (;;) { - unsigned char buff[1024]; - int nread = fread(buff, 1, sizeof(buff), fp); - - /* If read was success, send data. */ - if (nread > 0) { - write(sockfd, buff, nread); - } - - if (nread < sizeof(buff)) { - if (feof(fp)) - if (ferror(fp)) { - printf("Error reading\n"); - return false; - } - break; - } - } - - fclose(fp); - close(sockfd); - return true; - } - return false; -} diff --git a/src/server/JasmineGraphInstance.h b/src/server/JasmineGraphInstance.h index 964f13763..8b34be0f4 100644 --- a/src/server/JasmineGraphInstance.h +++ b/src/server/JasmineGraphInstance.h @@ -55,7 +55,6 @@ class JasmineGraphInstance { JasmineGraphInstanceService *instanceService; JasmineGraphInstanceFileTransferService *ftpService; - static bool sendFileThroughService(std::string host, int dataPort, std::string fileName, std::string filePath); }; #endif // JASMINEGRAPH_JASMINEGRAPHINSTANCE_H diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 4906adcc7..165fa9231 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -61,33 +61,33 @@ static void delete_graph_command(int connFd, bool *loop_exit_p); static void delete_graph_fragment_command(int connFd, bool *loop_exit_p); static void duplicate_centralstore_command(int connFd, int serverPort, bool *loop_exit_p); static void worker_in_degree_distribution_command( - int connFd, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p); + int connFd, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void in_degree_distribution_command( - int connFd, int serverPort, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p); + int connFd, int serverPort, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void worker_out_degree_distribution_command( - int connFd, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p); + int connFd, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void out_degree_distribution_command( - int connFd, int serverPort, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p); + int connFd, int serverPort, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void page_rank_command(int connFd, int serverPort, - std::map graphDBMapCentralStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void worker_page_rank_distribution_command( - int connFd, int serverPort, std::map graphDBMapCentralStores, + int connFd, int serverPort, std::map &graphDBMapCentralStores, bool *loop_exit_p); static void egonet_command(int connFd, int serverPort, - std::map graphDBMapCentralStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void worker_egonet_command(int connFd, int serverPort, - std::map graphDBMapCentralStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void triangles_command( - int connFd, int serverPort, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::map graphDBMapDuplicateCentralStores, + int connFd, int serverPort, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::map &graphDBMapDuplicateCentralStores, bool *loop_exit_p); static void streaming_triangles_command( int connFd, int serverPort, std::map &incrementalLocalStoreMap, @@ -119,8 +119,8 @@ static void send_priority_command(int connFd, bool *loop_exit_p); static std::string initiate_command_common(int connFd, bool *loop_exit_p); static void batch_upload_common(int connFd, bool *loop_exit_p, bool batch_upload); static void degree_distribution_common(int connFd, int serverPort, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p, bool in); char *converter(const std::string &s) { @@ -385,9 +385,9 @@ void writeCatalogRecord(string record) { long countLocalTriangles( std::string graphId, std::string partitionId, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::map graphDBMapDuplicateCentralStores, + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::map &graphDBMapDuplicateCentralStores, int threadPriority) { long result; @@ -656,7 +656,7 @@ string JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(std: return triangles; } -map JasmineGraphInstanceService::getOutDegreeDistributionHashMap(map> graphMap) { +map JasmineGraphInstanceService::getOutDegreeDistributionHashMap(map> &graphMap) { map distributionHashMap; for (map>::iterator it = graphMap.begin(); it != graphMap.end(); ++it) { @@ -676,16 +676,16 @@ string JasmineGraphInstanceService::requestPerformanceStatistics(std::string isV long sentBytes = StatisticCollector::getTXBytes(); double cpuUsage = StatisticCollector::getCpuUsage(); double loadAverage = StatisticCollector::getLoadAverage(); - std::string vmLevelStatistics = StatisticCollector::collectVMStatistics(isVMStatManager, - isResourceAllocationRequested); + std::string vmLevelStatistics = + StatisticCollector::collectVMStatistics(isVMStatManager, isResourceAllocationRequested); auto executedTime = std::chrono::system_clock::now(); std::time_t reportTime = std::chrono::system_clock::to_time_t(executedTime); std::string reportTimeString(std::ctime(&reportTime)); reportTimeString = Utils::trim_copy(reportTimeString); - std::string usageString = - reportTimeString + "," + to_string(memoryUsage) + "," + to_string(threadCount) + "," - + to_string(cpuUsage) + "," + to_string(loadAverage) + "," + to_string(usedSwapSpace) + "," - + to_string(totalSwapSpace) + + "," + to_string(readBytes) + "," + to_string(sentBytes); + std::string usageString = reportTimeString + "," + to_string(memoryUsage) + "," + to_string(threadCount) + "," + + to_string(cpuUsage) + "," + to_string(loadAverage) + "," + to_string(usedSwapSpace) + + "," + to_string(totalSwapSpace) + +"," + to_string(readBytes) + "," + + to_string(sentBytes); if (!vmLevelStatistics.empty()) { usageString = usageString + "," + vmLevelStatistics; } @@ -694,7 +694,7 @@ string JasmineGraphInstanceService::requestPerformanceStatistics(std::string isV void JasmineGraphInstanceService::collectTrainedModels( instanceservicesessionargs *sessionargs, std::string graphID, - std::map graphPartitionedHosts, int totalPartitions) { + std::map &graphPartitionedHosts, int totalPartitions) { int total_threads = totalPartitions; std::thread *workerThreads = new std::thread[total_threads]; int count = 0; @@ -971,7 +971,7 @@ void JasmineGraphInstanceService::trainPartition(string trainData) { } bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int graphID, int partitionID, - std::vector workerSockets, std::string masterIP) { + std::vector &workerSockets, std::string masterIP) { std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); std::string dataFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); @@ -1134,7 +1134,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int instance_logger.info("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_CONT); instance_logger.info("Going to send file through service"); - JasmineGraphInstanceService::sendFileThroughService(host, dataPort, fileName, centralStoreFile, masterIP); + Utils::sendFileThroughService(host, dataPort, fileName, centralStoreFile); int count = 0; while (true) { @@ -1187,81 +1187,10 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int return 0; } -bool JasmineGraphInstanceService::sendFileThroughService(std::string host, int dataPort, std::string fileName, - std::string filePath, std::string masterIP) { - int sockfd; - char data[DATA_BUFFER_SIZE]; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - instance_logger.error("Cannot create socket"); - return false; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - instance_logger.error("ERROR, no host named " + host); - return false; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(dataPort); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - return false; - } - - Utils::send_str_wrapper(sockfd, fileName); - - string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE) != 0) { - instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::SEND_FILE + - " ; Received: " + response); - close(sockfd); - return false; - } - FILE *fp = fopen(filePath.c_str(), "r"); - if (fp == NULL) { - instance_logger.error("Error opening file: " + filePath); - close(sockfd); - return false; - } - - for (;;) { - unsigned char buff[INSTANCE_FILE_BUFFER_LENGTH] = {0}; - int nread = fread(buff, 1, INSTANCE_FILE_BUFFER_LENGTH, fp); - instance_logger.info("Bytes read " + to_string(nread)); - - /* If read was success, send data. */ - if (nread > 0) { - Utils::send_wrapper(sockfd, (char *)buff, nread); - } - - if (nread < INSTANCE_FILE_BUFFER_LENGTH) { - if (feof(fp)) { - instance_logger.info("Reading completed for file: " + filePath); - } - if (ferror(fp)) { - instance_logger.error("Error reading file: " + filePath); - } - break; - } - } - - fclose(fp); - close(sockfd); - return true; -} - map calculateOutDegreeDist(string graphID, string partitionID, int serverPort, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::vector workerSockets) { + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::vector &workerSockets) { map degreeDistribution = calculateLocalOutDegreeDist(graphID, partitionID, graphDBMapLocalStores, graphDBMapCentralStores); @@ -1282,8 +1211,8 @@ map calculateOutDegreeDist(string graphID, string partitionID, int s } map calculateLocalOutDegreeDist( - string graphID, string partitionID, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores) { + string graphID, string partitionID, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores) { auto t_start = std::chrono::high_resolution_clock::now(); JasmineGraphHashMapLocalStore graphDB; @@ -1329,8 +1258,8 @@ map calculateLocalOutDegreeDist( } map calculateLocalInDegreeDist( - string graphID, string partitionID, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores) { + string graphID, string partitionID, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores) { JasmineGraphHashMapLocalStore graphDB; std::map::iterator it; @@ -1348,9 +1277,9 @@ map calculateLocalInDegreeDist( } map calculateInDegreeDist(string graphID, string partitionID, int serverPort, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::vector workerSockets, string workerList) { + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::vector &workerSockets, string workerList) { auto t_start = std::chrono::high_resolution_clock::now(); map degreeDistribution = @@ -1420,7 +1349,7 @@ map calculateInDegreeDist(string graphID, string partitionID, int se map>> calculateLocalEgoNet(string graphID, string partitionID, int serverPort, JasmineGraphHashMapLocalStore localDB, JasmineGraphHashMapCentralStore centralDB, - std::vector workerSockets) { + std::vector &workerSockets) { std::map>> egonetMap; map> centralGraphMap = centralDB.getUnderlyingHashMap(); @@ -1662,7 +1591,7 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine map calculateLocalPageRank(string graphID, double alpha, string partitionID, int serverPort, int top_k_page_rank_value, string graphVertexCount, JasmineGraphHashMapLocalStore localDB, - JasmineGraphHashMapCentralStore centralDB, std::vector workerSockets, + JasmineGraphHashMapCentralStore centralDB, std::vector &workerSockets, int iterations) { auto t_start = std::chrono::high_resolution_clock::now(); @@ -1830,8 +1759,8 @@ map calculateLocalPageRank(string graphID, double alpha, string pa map> getEdgesWorldToLocal(string graphID, string partitionID, int serverPort, string graphVertexCount, JasmineGraphHashMapLocalStore localDB, JasmineGraphHashMapCentralStore centralDB, - map> graphVertexMap, - std::vector workerSockets) { + map> &graphVertexMap, + std::vector &workerSockets) { map> worldToLocalVertexMap; for (vector::iterator workerIt = workerSockets.begin(); workerIt != workerSockets.end(); ++workerIt) { std::vector workerSocketPair; @@ -2309,8 +2238,8 @@ static void duplicate_centralstore_command(int connFd, int serverPort, bool *loo } static void worker_in_degree_distribution_command( - int connFd, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p) { + int connFd, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; return; @@ -2407,8 +2336,8 @@ static void worker_in_degree_distribution_command( } static void degree_distribution_common(int connFd, int serverPort, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p, bool in) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2459,14 +2388,14 @@ static void degree_distribution_common(int connFd, int serverPort, } static void in_degree_distribution_command( - int connFd, int serverPort, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p) { + int connFd, int serverPort, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { degree_distribution_common(connFd, serverPort, graphDBMapLocalStores, graphDBMapCentralStores, loop_exit_p, true); } static void worker_out_degree_distribution_command( - int connFd, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p) { + int connFd, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; return; @@ -2501,13 +2430,13 @@ static void worker_out_degree_distribution_command( } static void out_degree_distribution_command( - int connFd, int serverPort, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p) { + int connFd, int serverPort, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { degree_distribution_common(connFd, serverPort, graphDBMapLocalStores, graphDBMapCentralStores, loop_exit_p, false); } static void page_rank_command(int connFd, int serverPort, - std::map graphDBMapCentralStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2660,7 +2589,7 @@ static void page_rank_command(int connFd, int serverPort, } static void worker_page_rank_distribution_command( - int connFd, int serverPort, std::map graphDBMapCentralStores, + int connFd, int serverPort, std::map &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2788,7 +2717,7 @@ static void worker_page_rank_distribution_command( } static void egonet_command(int connFd, int serverPort, - std::map graphDBMapCentralStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2837,7 +2766,7 @@ static void egonet_command(int connFd, int serverPort, } static void worker_egonet_command(int connFd, int serverPort, - std::map graphDBMapCentralStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2919,9 +2848,9 @@ static void worker_egonet_command(int connFd, int serverPort, } static void triangles_command( - int connFd, int serverPort, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::map graphDBMapDuplicateCentralStores, + int connFd, int serverPort, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::map &graphDBMapDuplicateCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -3974,8 +3903,8 @@ static void initiate_model_collection_command(int connFd, bool *loop_exit_p) { if (line.compare(JasmineGraphInstanceProtocol::SEND_FILE_CONT) == 0) { instance_logger.info("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_CONT); instance_logger.info("Going to send file through service"); - JasmineGraphInstance::sendFileThroughService(serverHostName, stoi(serverHostDataPort), fileName, - filePath); + fileName = "jasminegraph-local_trained_model_store/" + fileName; + Utils::sendFileThroughService(serverHostName, stoi(serverHostDataPort), fileName, filePath); } } } @@ -4243,7 +4172,7 @@ static void send_priority_command(int connFd, bool *loop_exit_p) { string JasmineGraphInstanceService::aggregateStreamingCentralStoreTriangles( std::string graphId, std::string partitionId, std::string partitionIdString, std::string centralCountString, - int threadPriority, std::map incrementalLocalStores, + int threadPriority, std::map &incrementalLocalStores, std::string mode) { instance_logger.info("###INSTANCE### Started Aggregating Central Store Triangles"); std::vector incrementalLocalStoreInstances; diff --git a/src/server/JasmineGraphInstanceService.h b/src/server/JasmineGraphInstanceService.h index ecc2117da..d607c48d7 100644 --- a/src/server/JasmineGraphInstanceService.h +++ b/src/server/JasmineGraphInstanceService.h @@ -46,39 +46,39 @@ limitations under the License. #include "../util/Utils.h" #include "JasmineGraphInstanceProtocol.h" -void* instanceservicesession(void* dummyPt); +void *instanceservicesession(void *dummyPt); void writeCatalogRecord(string record); int deleteGraphPartition(std::string graphID, std::string partitionID); void removeGraphFragments(std::string graphID); long countLocalTriangles( std::string graphId, std::string partitionId, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::map graphDBMapDuplicateCentralStores, + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::map &graphDBMapDuplicateCentralStores, int threadPriority); map calculateOutDegreeDist(string graphID, string partitionID, int serverPort, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::vector workerSockets); + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::vector &workerSockets); map calculateLocalOutDegreeDist( - string graphID, string partitionID, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores); + string graphID, string partitionID, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores); map calculateInDegreeDist(string graphID, string partitionID, int serverPort, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::vector workerSockets, string workerList); + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::vector &workerSockets, string workerList); map calculateLocalInDegreeDist( - string graphID, string partitionID, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores); + string graphID, string partitionID, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores); map>> calculateLocalEgoNet(string graphID, string partitionID, int serverPort, JasmineGraphHashMapLocalStore localDB, JasmineGraphHashMapCentralStore centralDB, - std::vector workerSockets); + std::vector &workerSockets); void calculateEgoNet(string graphID, string partitionID, int serverPort, JasmineGraphHashMapLocalStore localDB, JasmineGraphHashMapCentralStore centralDB, string workerList); @@ -86,20 +86,20 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine map calculateLocalPageRank(string graphID, double alpha, string partitionID, int serverPort, int top_k_page_rank_value, string graphVertexCount, JasmineGraphHashMapLocalStore localDB, - JasmineGraphHashMapCentralStore centralDB, std::vector workerSockets, + JasmineGraphHashMapCentralStore centralDB, std::vector &workerSockets, int iterations); map getAuthorityScoresWorldToLocal(string graphID, string partitionID, int serverPort, string graphVertexCount, JasmineGraphHashMapLocalStore localDB, JasmineGraphHashMapCentralStore centralDB, - map> graphVertexMap, - std::vector workerSockets, long worldOnlyVertexCount); + map> &graphVertexMap, + std::vector &workerSockets, long worldOnlyVertexCount); map> getEdgesWorldToLocal(string graphID, string partitionID, int serverPort, string graphVertexCount, JasmineGraphHashMapLocalStore localDB, JasmineGraphHashMapCentralStore centralDB, - map> graphVertexMap, - std::vector workerSockets); + map> &graphVertexMap, + std::vector &workerSockets); struct instanceservicesessionargs { string profile; @@ -111,7 +111,7 @@ struct instanceservicesessionargs { std::map graphDBMapLocalStores; std::map graphDBMapCentralStores; std::map graphDBMapDuplicateCentralStores; - std::map incrementalLocalStore; + std::map incrementalLocalStore; }; class JasmineGraphInstanceService { @@ -126,22 +126,22 @@ class JasmineGraphInstanceService { static bool isInstanceCentralStoreExists(std::string graphId, std::string partitionId); static bool isInstanceDuplicateCentralStoreExists(std::string graphId, std::string partitionId); static void loadLocalStore(std::string graphId, std::string partitionId, - std::map& graphDBMapLocalStores); - static JasmineGraphIncrementalLocalStore* loadStreamingStore( + std::map &graphDBMapLocalStores); + static JasmineGraphIncrementalLocalStore *loadStreamingStore( std::string graphId, std::string partitionId, - std::map& graphDBMapStreamingStores, std::string openMode); + std::map &graphDBMapStreamingStores, std::string openMode); static void loadInstanceCentralStore( std::string graphId, std::string partitionId, - std::map& graphDBMapCentralStores); + std::map &graphDBMapCentralStores); static void loadInstanceDuplicateCentralStore( std::string graphId, std::string partitionId, - std::map& graphDBMapDuplicateCentralStores); + std::map &graphDBMapDuplicateCentralStores); static JasmineGraphHashMapCentralStore loadCentralStore(std::string centralStoreFileName); static string aggregateCentralStoreTriangles(std::string graphId, std::string partitionId, std::string partitionIdList, int threadPriority); static string aggregateCompositeCentralStoreTriangles(std::string compositeFileList, std::string availableFileList, int threadPriority); - static map getOutDegreeDistributionHashMap(map> graphMap); + static map getOutDegreeDistributionHashMap(map> &graphMap); static string requestPerformanceStatistics(std::string isVMStatManager, std::string isResourceAllocationRequested); struct workerPartitions { @@ -151,11 +151,11 @@ class JasmineGraphInstanceService { }; static void collectTrainedModels( - instanceservicesessionargs* sessionargs, std::string graphID, - std::map graphPartitionedHosts, + instanceservicesessionargs *sessionargs, std::string graphID, + std::map &graphPartitionedHosts, int totalPartitions); - static int collectTrainedModelThreadFunction(instanceservicesessionargs* sessionargs, std::string host, int port, + static int collectTrainedModelThreadFunction(instanceservicesessionargs *sessionargs, std::string host, int port, int dataPort, std::string graphID, std::string partition); static void createPartitionFiles(std::string graphID, std::string partitionID, std::string fileType); @@ -181,16 +181,12 @@ class JasmineGraphInstanceService { static std::map> iterationData; static bool duplicateCentralStore(int thisWorkerPort, int graphID, int partitionID, - std::vector workerSockets, std::string masterIP); - - static bool sendFileThroughService(std::string host, int dataPort, std::string fileName, std::string filePath, - std::string masterIP); + std::vector &workerSockets, std::string masterIP); static string aggregateStreamingCentralStoreTriangles( - std::string graphId, std::string partitionId, std::string partitionIdString, - std::string centralCountString, int threadPriority, - std::map incrementalLocalStores, - std::string mode); + std::string graphId, std::string partitionId, std::string partitionIdString, std::string centralCountString, + int threadPriority, std::map &incrementalLocalStores, + std::string mode); static int partitionCounter; diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 78cb2804e..7c25a100a 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -38,8 +38,6 @@ static void deleteWorkerPath(const std::string &workerHost, const std::string &w static void assignPartitionToWorker(std::string fileName, int graphId, std::string workerHost, int workerPort, int workerDataPort); static void updateMetaDB(int graphID, std::string uploadEndTime); -static bool batchUploadCommon(std::string host, int port, int dataPort, int graphID, std::string filePath, - std::string masterIP, std::string uploadType); static bool batchUploadFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP); static bool batchUploadCentralStore(std::string host, int port, int dataPort, int graphID, std::string filePath, @@ -950,130 +948,15 @@ static void assignPartitionToWorker(std::string fileName, int graphId, std::stri delete refToSqlite; } -static bool batchUploadCommon(std::string host, int port, int dataPort, int graphID, std::string filePath, - std::string masterIP, std::string uploadType) { - server_logger.info("Host:" + host + " Port:" + to_string(port) + " DPort:" + to_string(dataPort)); - bool result = true; - int sockfd; - char data[FED_DATA_LENGTH + 1]; - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - server_logger.error("Cannot create socket"); - return false; - } - - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - server_logger.error("ERROR, no host named " + host); - return false; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - return false; - } - - if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { - close(sockfd); - return false; - } - - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, uploadType, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - return false; - } - - std::string fileName = Utils::getFileName(filePath); - int fileSize = Utils::getFileSize(filePath); - std::string fileLength = to_string(fileSize); - - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, std::to_string(graphID), - JasmineGraphInstanceProtocol::SEND_FILE_NAME)) { - close(sockfd); - return false; - } - - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, - JasmineGraphInstanceProtocol::SEND_FILE_LEN)) { - close(sockfd); - return false; - } - - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileLength, - JasmineGraphInstanceProtocol::SEND_FILE_CONT)) { - close(sockfd); - return false; - } - - server_logger.info("Going to send central store file through file transfer service from master to worker"); - JasmineGraphServer::sendFileThroughService(host, dataPort, fileName, filePath, masterIP); - - string response; - int count = 0; - while (true) { - if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::FILE_RECV_CHK)) { - close(sockfd); - return false; - } - server_logger.info("Sent: " + JasmineGraphInstanceProtocol::FILE_RECV_CHK); - - server_logger.info("Checking if file is received"); - response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::FILE_RECV_WAIT) == 0) { - server_logger.info("Received: " + JasmineGraphInstanceProtocol::FILE_RECV_WAIT); - server_logger.info("Checking file status : " + to_string(count)); - count++; - sleep(1); - continue; - } else if (response.compare(JasmineGraphInstanceProtocol::FILE_ACK) == 0) { - server_logger.info("Received: " + JasmineGraphInstanceProtocol::FILE_ACK); - server_logger.info("File transfer completed for file : " + filePath); - break; - } - } - // Next we wait till the batch upload completes - while (true) { - if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK)) { - close(sockfd); - return false; - } - server_logger.info("Sent: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); - - response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT) == 0) { - server_logger.info("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT); - sleep(1); - continue; - } else if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK) == 0) { - server_logger.info("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK); - server_logger.info("Batch upload completed"); - break; - } - } - close(sockfd); - return true; -} - static bool batchUploadFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { - return batchUploadCommon(host, port, dataPort, graphID, filePath, masterIP, + return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, JasmineGraphInstanceProtocol::BATCH_UPLOAD); } static bool batchUploadCentralStore(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { - return batchUploadCommon(host, port, dataPort, graphID, filePath, masterIP, + return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, JasmineGraphInstanceProtocol::BATCH_UPLOAD_CENTRAL); } @@ -1096,85 +979,22 @@ void JasmineGraphServer::copyCentralStoreToAggregateLocation(std::string filePat static bool batchUploadAttributeFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { - return batchUploadCommon(host, port, dataPort, graphID, filePath, masterIP, + return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, JasmineGraphInstanceProtocol::UPLOAD_RDF_ATTRIBUTES); } static bool batchUploadCentralAttributeFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { - return batchUploadCommon(host, port, dataPort, graphID, filePath, masterIP, + return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, JasmineGraphInstanceProtocol::UPLOAD_RDF_ATTRIBUTES_CENTRAL); } static bool batchUploadCompositeCentralstoreFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { - return batchUploadCommon(host, port, dataPort, graphID, filePath, masterIP, + return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, JasmineGraphInstanceProtocol::BATCH_UPLOAD_COMPOSITE_CENTRAL); } -bool JasmineGraphServer::sendFileThroughService(std::string host, int dataPort, std::string fileName, - std::string filePath, std::string masterIP) { - int sockfd; - char data[FED_DATA_LENGTH + 1]; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - server_logger.error("Cannot create socket"); - return false; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - server_logger.error("ERROR, no host named " + host); - return false; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(dataPort); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - return false; - } - - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, - JasmineGraphInstanceProtocol::SEND_FILE)) { - close(sockfd); - return false; - } - - server_logger.info("Sending file " + filePath + " through port " + std::to_string(dataPort)); - FILE *fp = fopen(filePath.c_str(), "r"); - if (fp == NULL) { - close(sockfd); - return false; - } - - while (true) { - unsigned char buff[1024]; - int nread = fread(buff, 1, sizeof(buff), fp); - - /* If read was success, send data. */ - if (nread > 0) { - write(sockfd, buff, nread); - } - - if (nread < sizeof(buff)) { - if (feof(fp)) server_logger.info("End of file"); - if (ferror(fp)) server_logger.error("Error reading file: " + filePath); - break; - } - } - - fclose(fp); - close(sockfd); - return true; -} - static void copyArtifactsToWorkers(const std::string &workerPath, const std::string &artifactLocation, const std::string &remoteWorker) { if (artifactLocation.empty() || artifactLocation.find_first_not_of(' ') == artifactLocation.npos) { diff --git a/src/server/JasmineGraphServer.h b/src/server/JasmineGraphServer.h index b68fb7982..e79ba1c81 100644 --- a/src/server/JasmineGraphServer.h +++ b/src/server/JasmineGraphServer.h @@ -99,9 +99,6 @@ class JasmineGraphServer { static void copyCentralStoreToAggregateLocation(std::string filePath); - static bool sendFileThroughService(std::string host, int dataPort, std::string fileName, std::string filePath, - std::string masterIP); - static bool spawnNewWorker(string host, string port, string dataPort, string profile, string masterHost, string enableNmon); diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 8a0497143..a2d11bc17 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -16,6 +16,7 @@ limitations under the License. #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ limitations under the License. #include "../../globals.h" #include "../k8s/K8sInterface.h" #include "../server/JasmineGraphInstanceProtocol.h" +#include "../server/JasmineGraphServer.h" #include "Conts.h" #include "logger/Logger.h" @@ -256,7 +258,7 @@ bool Utils::is_number(const std::string &compareString) { * @return */ std::string Utils::getFileName(std::string filePath) { - std::string filename = filePath.substr(filePath.find_last_of("/\\") + 1); + std::string filename = filePath.substr(filePath.find_last_of("/") + 1); return filename; } @@ -835,3 +837,180 @@ std::fstream *Utils::openFile(const string &path, std::ios_base::openmode mode) // Now open the file in the desired mode return new std::fstream(path, mode | std::ios::binary); } + +bool Utils::uploadFileToWorker(std::string host, int port, int dataPort, int graphID, std::string filePath, + std::string masterIP, std::string uploadType) { + util_logger.info("Host:" + host + " Port:" + to_string(port) + " DPort:" + to_string(dataPort)); + bool result = true; + int sockfd; + char data[FED_DATA_LENGTH + 1]; + struct sockaddr_in serv_addr; + struct hostent *server; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + + if (sockfd < 0) { + util_logger.error("Cannot create socket"); + return false; + } + + if (host.find('@') != std::string::npos) { + host = Utils::split(host, '@')[1]; + } + + server = gethostbyname(host.c_str()); + if (server == NULL) { + util_logger.error("ERROR, no host named " + host); + return false; + } + + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(port); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + return false; + } + + if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { + close(sockfd); + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, uploadType, JasmineGraphInstanceProtocol::OK)) { + close(sockfd); + return false; + } + + std::string fileName = Utils::getFileName(filePath); + int fileSize = Utils::getFileSize(filePath); + std::string fileLength = to_string(fileSize); + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, std::to_string(graphID), + JasmineGraphInstanceProtocol::SEND_FILE_NAME)) { + close(sockfd); + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, + JasmineGraphInstanceProtocol::SEND_FILE_LEN)) { + close(sockfd); + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileLength, + JasmineGraphInstanceProtocol::SEND_FILE_CONT)) { + close(sockfd); + return false; + } + + util_logger.info("Going to send file through file transfer service to worker"); + Utils::sendFileThroughService(host, dataPort, fileName, filePath); + + string response; + int count = 0; + while (true) { + if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::FILE_RECV_CHK)) { + close(sockfd); + return false; + } + util_logger.info("Sent: " + JasmineGraphInstanceProtocol::FILE_RECV_CHK); + + util_logger.info("Checking if file is received"); + response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::FILE_RECV_WAIT) == 0) { + util_logger.info("Received: " + JasmineGraphInstanceProtocol::FILE_RECV_WAIT); + util_logger.info("Checking file status : " + to_string(count)); + count++; + sleep(1); + continue; + } else if (response.compare(JasmineGraphInstanceProtocol::FILE_ACK) == 0) { + util_logger.info("Received: " + JasmineGraphInstanceProtocol::FILE_ACK); + util_logger.info("File transfer completed for file : " + filePath); + break; + } + } + // Next we wait till the batch upload completes + while (true) { + if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK)) { + close(sockfd); + return false; + } + util_logger.info("Sent: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); + + response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT) == 0) { + util_logger.info("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT); + sleep(1); + continue; + } else if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK) == 0) { + util_logger.info("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK); + util_logger.info("Batch upload completed"); + break; + } + } + close(sockfd); + return true; +} + +bool Utils::sendFileThroughService(std::string host, int dataPort, std::string fileName, std::string filePath) { + int sockfd; + char data[FED_DATA_LENGTH + 1]; + socklen_t len; + struct sockaddr_in serv_addr; + struct hostent *server; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + + if (sockfd < 0) { + util_logger.error("Cannot create socket"); + return false; + } + + server = gethostbyname(host.c_str()); + if (server == NULL) { + util_logger.error("ERROR, no host named " + host); + return false; + } + + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(dataPort); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, + JasmineGraphInstanceProtocol::SEND_FILE)) { + close(sockfd); + return false; + } + + util_logger.info("Sending file " + filePath + " through port " + std::to_string(dataPort)); + FILE *fp = fopen(filePath.c_str(), "r"); + if (fp == NULL) { + close(sockfd); + return false; + } + + while (true) { + unsigned char buff[1024]; + int nread = fread(buff, 1, sizeof(buff), fp); + + /* If read was success, send data. */ + if (nread > 0) { + write(sockfd, buff, nread); + } + + if (nread < sizeof(buff)) { + if (feof(fp)) util_logger.info("End of file"); + if (ferror(fp)) util_logger.error("Error reading file: " + filePath); + break; + } + } + + fclose(fp); + close(sockfd); + return true; +} diff --git a/src/util/Utils.h b/src/util/Utils.h index 5870d5a89..652d02d90 100644 --- a/src/util/Utils.h +++ b/src/util/Utils.h @@ -68,9 +68,9 @@ class Utils { static bool fileExists(std::string fileName); - static bool fileExistsWithReadPermission(const std::string& path); + static bool fileExistsWithReadPermission(const std::string &path); - static std::fstream* openFile(const std::string &path, std::ios_base::openmode mode); + static std::fstream *openFile(const std::string &path, std::ios_base::openmode mode); static int compressFile(const std::string filePath, std::string mode = "pigz"); @@ -168,12 +168,16 @@ class Utils { static std::string getJsonStringFromYamlFile(const std::string &yamlFile); - static int createDatabaseFromDDL(const char* dbLocation, const char* ddlFileLocation); + static int createDatabaseFromDDL(const char *dbLocation, const char *ddlFileLocation); - static std::string send_job(std::string job_group_name, std::string metric_name, - std::string metric_value); + static std::string send_job(std::string job_group_name, std::string metric_name, std::string metric_value); static map getMetricMap(string metricName); + + static bool uploadFileToWorker(std::string host, int port, int dataPort, int graphID, std::string filePath, + std::string masterIP, std::string uploadType); + + static bool sendFileThroughService(std::string host, int dataPort, std::string fileName, std::string filePath); }; #endif // JASMINEGRAPH_UTILS_H From 9af82e4480c2bd3afc576c4f288980cffcd4d6b7 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 24 Feb 2024 02:51:02 +0530 Subject: [PATCH 044/143] Defer SchedulerService until DDL execution --- main.cpp | 15 ++++----------- src/performance/metrics/StatisticCollector.cpp | 4 ++++ src/server/JasmineGraphInstance.cpp | 2 -- src/server/JasmineGraphInstance.h | 2 -- src/server/JasmineGraphServer.cpp | 12 +++++------- src/server/JasmineGraphServer.h | 2 -- src/util/Utils.cpp | 9 +++------ src/util/scheduler/SchedulerService.cpp | 1 + 8 files changed, 17 insertions(+), 30 deletions(-) diff --git a/main.cpp b/main.cpp index 4adddee41..59b8c653c 100644 --- a/main.cpp +++ b/main.cpp @@ -79,18 +79,15 @@ int main(int argc, char *argv[]) { } server->run(profile, masterIp, numberOfWorkers, workerIps, enableNmon); - while (server->isRunning()) { - usleep(microseconds); - } - schedulerThread.join(); delete server; } else if (mode == Conts::JASMINEGRAPH_RUNTIME_PROFILE_WORKER) { main_logger.log(to_string(argc), "info"); - if (argc < 5) { - main_logger.log("Need Four arguments. Use 2 to start as worker", - "info"); + if (argc < 8) { + main_logger.info( + "Need 7 arguments. Use 2 to " + "start as worker"); return -1; } @@ -107,10 +104,6 @@ int main(int argc, char *argv[]) { instance = new JasmineGraphInstance(); instance->start_running(profile, hostName, masterHost, serverPort, serverDataPort, enableNmon); - while (instance->isRunning()) { - usleep(microseconds); - } - delete instance; } diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index a0ea4000b..994f43969 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -24,6 +24,10 @@ int StatisticCollector::init() { char line[128]; file = fopen("/proc/cpuinfo", "r"); + if (!file) { + std::cout << "Cannot open /proc/cpuinfo" << std::endl; + exit(-1); + } numProcessors = 0; while (fgets(line, 128, file) != NULL) { if (strncmp(line, "processor", 9) == 0) numProcessors++; diff --git a/src/server/JasmineGraphInstance.cpp b/src/server/JasmineGraphInstance.cpp index 29dcac873..70a099db6 100644 --- a/src/server/JasmineGraphInstance.cpp +++ b/src/server/JasmineGraphInstance.cpp @@ -201,5 +201,3 @@ void JasmineGraphInstance::startNmonAnalyzer(string enableNmon, int serverPort) } } } - -bool JasmineGraphInstance::isRunning() { return true; } diff --git a/src/server/JasmineGraphInstance.h b/src/server/JasmineGraphInstance.h index 8b34be0f4..cfd6354c1 100644 --- a/src/server/JasmineGraphInstance.h +++ b/src/server/JasmineGraphInstance.h @@ -44,8 +44,6 @@ class JasmineGraphInstance { void shutdown(); - bool isRunning(); - string hostName; string profile; string masterHostName; diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 7c25a100a..46f6eed95 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -150,8 +150,6 @@ int JasmineGraphServer::run(std::string profile, std::string masterIp, int numbe return 0; } -bool JasmineGraphServer::isRunning() { return true; } - void JasmineGraphServer::init() { pthread_t frontendthread; pthread_t backendthread; @@ -951,13 +949,13 @@ static void assignPartitionToWorker(std::string fileName, int graphId, std::stri static bool batchUploadFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, - JasmineGraphInstanceProtocol::BATCH_UPLOAD); + JasmineGraphInstanceProtocol::BATCH_UPLOAD); } static bool batchUploadCentralStore(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, - JasmineGraphInstanceProtocol::BATCH_UPLOAD_CENTRAL); + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CENTRAL); } void JasmineGraphServer::copyCentralStoreToAggregateLocation(std::string filePath) { @@ -980,19 +978,19 @@ void JasmineGraphServer::copyCentralStoreToAggregateLocation(std::string filePat static bool batchUploadAttributeFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, - JasmineGraphInstanceProtocol::UPLOAD_RDF_ATTRIBUTES); + JasmineGraphInstanceProtocol::UPLOAD_RDF_ATTRIBUTES); } static bool batchUploadCentralAttributeFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, - JasmineGraphInstanceProtocol::UPLOAD_RDF_ATTRIBUTES_CENTRAL); + JasmineGraphInstanceProtocol::UPLOAD_RDF_ATTRIBUTES_CENTRAL); } static bool batchUploadCompositeCentralstoreFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, - JasmineGraphInstanceProtocol::BATCH_UPLOAD_COMPOSITE_CENTRAL); + JasmineGraphInstanceProtocol::BATCH_UPLOAD_COMPOSITE_CENTRAL); } static void copyArtifactsToWorkers(const std::string &workerPath, const std::string &artifactLocation, diff --git a/src/server/JasmineGraphServer.h b/src/server/JasmineGraphServer.h index e79ba1c81..e925eda92 100644 --- a/src/server/JasmineGraphServer.h +++ b/src/server/JasmineGraphServer.h @@ -87,8 +87,6 @@ class JasmineGraphServer { int run(std::string profile, std::string masterIp, int numberofWorkers, std::string workerIps, std::string enableNmon); - bool isRunning(); - void uploadGraphLocally(int graphID, const std::string graphType, std::vector> fullFileList, std::string masterIP); diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index a2d11bc17..d0824647b 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -101,14 +101,11 @@ void Utils::writeFileContent(const std::string &filePath, const std::string &con std::string Utils::getJasmineGraphProperty(std::string key) { if (Utils::propertiesMap.empty()) { - std::vector::iterator it; - vector vec = Utils::getFileContent(ROOT_DIR "conf/jasminegraph-server.properties"); - it = vec.begin(); - - for (it = vec.begin(); it < vec.end(); it++) { + const vector &vec = Utils::getFileContent(ROOT_DIR "conf/jasminegraph-server.properties"); + for (auto it = vec.begin(); it < vec.end(); it++) { std::string item = *it; if (item.length() > 0 && !(item.rfind("#", 0) == 0)) { - std::vector vec2 = split(item, '='); + const std::vector &vec2 = split(item, '='); if (vec2.size() == 2) { Utils::propertiesMap[vec2.at(0)] = vec2.at(1); } else { diff --git a/src/util/scheduler/SchedulerService.cpp b/src/util/scheduler/SchedulerService.cpp index 1b9e92b68..fd9481da6 100644 --- a/src/util/scheduler/SchedulerService.cpp +++ b/src/util/scheduler/SchedulerService.cpp @@ -20,6 +20,7 @@ using namespace Bosma; Logger schedulerservice_logger; void SchedulerService::startScheduler() { + sleep(3); std::string schedulerEnabled = Utils::getJasmineGraphProperty("org.jasminegraph.scheduler.enabled"); if (schedulerEnabled == "true") { From 550c12f8489baac445bab68392ef854d84806a23 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 24 Feb 2024 19:54:56 +0530 Subject: [PATCH 045/143] Use access() in Utils::copyToDirectory() to check if directory exists --- .../executor/impl/TriangleCountExecutor.cpp | 28 +++--- src/server/JasmineGraphInstanceService.cpp | 93 +++++-------------- src/util/Utils.cpp | 17 +++- 3 files changed, 47 insertions(+), 91 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 4c36b8592..57bb5f64a 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -604,30 +604,25 @@ int TriangleCountExecutor::updateTriangleTreeAndGetTriangleCount(std::vector> workerCombinations = getWorkerCombination(sqlite, graphId); + const std::vector> &workerCombinations = getWorkerCombination(sqlite, graphId); std::map workerWeightMap; - std::vector>::iterator workerCombinationsIterator; std::vector> triangleCountResponse; std::string result = ""; long aggregatedTriangleCount = 0; - for (workerCombinationsIterator = workerCombinations.begin(); + for (auto workerCombinationsIterator = workerCombinations.begin(); workerCombinationsIterator != workerCombinations.end(); ++workerCombinationsIterator) { - std::vector workerCombination = *workerCombinationsIterator; - std::map::iterator workerWeightMapIterator; + const std::vector &workerCombination = *workerCombinationsIterator; std::vector> remoteGraphCopyResponse; int minimumWeight = 0; std::string minWeightWorker; std::string partitionIdList = ""; - std::vector::iterator workerCombinationIterator; - std::vector::iterator aggregatorCopyCombinationIterator; - - for (workerCombinationIterator = workerCombination.begin(); + for (auto workerCombinationIterator = workerCombination.begin(); workerCombinationIterator != workerCombination.end(); ++workerCombinationIterator) { std::string workerId = *workerCombinationIterator; - workerWeightMapIterator = workerWeightMap.find(workerId); + auto workerWeightMapIterator = workerWeightMap.find(workerId); if (workerWeightMapIterator != workerWeightMap.end()) { int weight = workerWeightMap.at(workerId); @@ -657,7 +652,7 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq std::string aggregatorDataPort = aggregatorData.at(2).second; std::string aggregatorPartitionId = aggregatorData.at(3).second; - for (aggregatorCopyCombinationIterator = workerCombination.begin(); + for (auto aggregatorCopyCombinationIterator = workerCombination.begin(); aggregatorCopyCombinationIterator != workerCombination.end(); ++aggregatorCopyCombinationIterator) { std::string workerId = *aggregatorCopyCombinationIterator; @@ -1265,9 +1260,9 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr socklen_t len; struct sockaddr_in serv_addr; struct hostent *server; - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); + std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); std::string fileName = std::to_string(graphId) + "_centralstore_" + std::to_string(partitionId) + ".gz"; - std::string centralStoreFile = aggregatorFilePath + "/" + fileName; + std::string centralStoreFile = aggregatorDirPath + "/" + fileName; int fileSize = Utils::getFileSize(centralStoreFile); std::string fileLength = to_string(fileSize); @@ -1276,13 +1271,13 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (sockfd < 0) { std::cerr << "Cannot create socket" << std::endl; - return 0; + return ""; } server = gethostbyname(aggregatorHostName.c_str()); if (server == NULL) { triangleCount_logger.error("ERROR, no host named " + aggregatorHostName); - return 0; + return ""; } bzero((char *)&serv_addr, sizeof(serv_addr)); @@ -1291,8 +1286,7 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; - // TODO::exit - return 0; + return ""; } bzero(data, 301); diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 165fa9231..a0bb75126 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -519,7 +519,7 @@ JasmineGraphHashMapCentralStore JasmineGraphInstanceService::loadCentralStore(st string JasmineGraphInstanceService::aggregateCentralStoreTriangles(std::string graphId, std::string partitionId, std::string partitionIdList, int threadPriority) { instance_logger.info("###INSTANCE### Started Aggregating Central Store Triangles"); - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); + std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); std::vector fileNames; map> aggregatedCentralStore; std::string centralGraphIdentifier = graphId + "_centralstore_" + partitionId; @@ -551,7 +551,7 @@ string JasmineGraphInstanceService::aggregateCentralStoreTriangles(std::string g std::string aggregatePartitionId = *partitionIdListIterator; std::string centralGraphIdentifier = graphId + "_centralstore_" + aggregatePartitionId; - std::string centralStoreFile = aggregatorFilePath + "/" + centralGraphIdentifier; + std::string centralStoreFile = aggregatorDirPath + "/" + centralGraphIdentifier; if (access(centralStoreFile.c_str(), R_OK) == 0) { JasmineGraphHashMapCentralStore centralStore = JasmineGraphInstanceService::loadCentralStore(centralStoreFile); @@ -583,7 +583,7 @@ string JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(std: std::string availableFileList, int threadPriority) { instance_logger.info("###INSTANCE### Started Aggregating Composite Central Store Triangles"); - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); + std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); std::string dataFolder = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); map> aggregatedCompositeCentralStore; @@ -625,7 +625,7 @@ string JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(std: size_t lastindex = compositeCentralStoreFileName.find_last_of("."); string rawFileName = compositeCentralStoreFileName.substr(0, lastindex); - std::string compositeCentralStoreFile = aggregatorFilePath + "/" + rawFileName; + std::string compositeCentralStoreFile = aggregatorDirPath + "/" + rawFileName; if (access(compositeCentralStoreFile.c_str(), R_OK) == 0) { JasmineGraphHashMapCentralStore centralStore = JasmineGraphInstanceService::loadCentralStore(compositeCentralStoreFile); @@ -972,21 +972,20 @@ void JasmineGraphInstanceService::trainPartition(string trainData) { bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int graphID, int partitionID, std::vector &workerSockets, std::string masterIP) { - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); - std::string dataFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); + std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); + std::string dataDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); std::string centralGraphIdentifierUnCompressed = to_string(graphID) + "_centralstore_" + to_string(partitionID); - std::string centralStoreFileUnCompressed = dataFilePath + "/" + centralGraphIdentifierUnCompressed; - std::string centralStoreFileUnCompressedDestination = aggregatorFilePath + "/" + centralGraphIdentifierUnCompressed; + std::string centralStoreFileUnCompressed = dataDirPath + "/" + centralGraphIdentifierUnCompressed; + std::string centralStoreFileUnCompressedDestination = aggregatorDirPath + "/" + centralGraphIdentifierUnCompressed; // temporary copy the central store into the aggregate folder in order to compress and send - Utils::copyFile(centralStoreFileUnCompressed, aggregatorFilePath); + Utils::copyToDirectory(centralStoreFileUnCompressed, aggregatorDirPath); // compress the central store file before sending Utils::compressFile(centralStoreFileUnCompressedDestination); - std::string centralGraphIdentifier = to_string(graphID) + "_centralstore_" + to_string(partitionID) + ".gz"; - std::string centralStoreFile = aggregatorFilePath + "/" + centralGraphIdentifier; + std::string centralStoreFile = centralStoreFileUnCompressedDestination + ".gz"; instance_logger.info("###INSTANCE### centralstore " + centralStoreFile); char data[DATA_BUFFER_SIZE]; @@ -1000,13 +999,13 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (workerSocketPair.size() != 4) { instance_logger.error("Received worker socket information is invalid "); - return 0; + return false; } struct stat fileStat; if (stat(centralStoreFile.c_str(), &fileStat) != 0) { instance_logger.error("stat() failed on " + centralStoreFile); - return 0; + return false; } if (!S_ISREG(fileStat.st_mode)) { instance_logger.error(centralStoreFile + " is not a regular file."); @@ -1031,7 +1030,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { instance_logger.error("Cannot create socket"); - return 0; + return false; } if (host.find('@') != std::string::npos) { @@ -1041,7 +1040,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int server = gethostbyname(host.c_str()); if (server == NULL) { instance_logger.error("ERROR, no host named " + host); - return 0; + return false; } bzero((char *)&serv_addr, sizeof(serv_addr)); @@ -1049,7 +1048,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - return 0; + return false; } if (Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE)) { @@ -1184,7 +1183,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int END_OUTER_LOOP: close(sockfd); } - return 0; + return true; } map calculateOutDegreeDist(string graphID, string partitionID, int serverPort, @@ -1414,10 +1413,10 @@ map>> calculateLocalEgoNet(string graphID, s workerSocketPair.push_back(intermediate); } - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); + std::string dataDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); std::string centralGraphIdentifier = graphID + "_centralstore_" + workerSocketPair[2]; - std::string centralStoreFile = aggregatorFilePath + "/" + centralGraphIdentifier; + std::string centralStoreFile = dataDirPath + "/" + centralGraphIdentifier; instance_logger.info("###INSTANCE### centralstore " + centralStoreFile); struct stat centralStoreFileStat; @@ -1641,13 +1640,13 @@ map calculateLocalPageRank(string graphID, double alpha, string pa std::map inDegreeDistribution; - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); + std::string dataDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); std::string partitionCount = Utils::getJasmineGraphProperty("org.jasminegraph.server.npartitions"); int parCount = std::stoi(partitionCount); for (int partitionID = 0; partitionID < parCount; ++partitionID) { - std::string iddFilePath = aggregatorFilePath + "/" + graphID + "_idd_" + std::to_string(partitionID); + std::string iddFilePath = dataDirPath + "/" + graphID + "_idd_" + std::to_string(partitionID); std::ifstream dataFile; dataFile.open(iddFilePath); @@ -1770,10 +1769,10 @@ map> getEdgesWorldToLocal(string graphID, string parti workerSocketPair.push_back(intermediate); } - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); + std::string dataDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); std::string centralGraphIdentifier = graphID + "_centralstore_" + workerSocketPair[2]; - std::string centralStoreFile = aggregatorFilePath + "/" + centralGraphIdentifier; + std::string centralStoreFile = dataDirPath + "/" + centralGraphIdentifier; instance_logger.info("###INSTANCE### centralstore " + centralStoreFile); if (access(centralStoreFile.c_str(), R_OK) != 0) { @@ -3074,32 +3073,10 @@ static void send_centralstore_to_aggregator_command(int connFd, bool *loop_exit_ fullFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + rawname; std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); - if (access(aggregatorDirPath.c_str(), F_OK)) { - std::string createDirCommand = "mkdir -p " + aggregatorDirPath; - if (system(createDirCommand.c_str())) { - instance_logger.error("Creating directory " + aggregatorDirPath + " failed"); - } - } - - std::string copyCommand = "cp " + fullFilePath + " " + aggregatorDirPath; - if (system(copyCommand.c_str())) { + if (Utils::copyToDirectory(fullFilePath, aggregatorDirPath)) { instance_logger.error("Copying " + fullFilePath + " into " + aggregatorDirPath + " failed"); } - std::string movedFullFilePath = aggregatorDirPath + "/" + rawname; - - while (!Utils::fileExists(movedFullFilePath)) { - string response = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK) == 0) { - instance_logger.info("Received : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); - if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT)) { - *loop_exit_p = true; - return; - } - instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT); - } - } - line = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, false); if (line.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK) == 0) { instance_logger.info("Received : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); @@ -3176,32 +3153,10 @@ static void send_composite_centralstore_to_aggregator_command(int connFd, bool * fullFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + rawname; std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); - if (access(aggregatorDirPath.c_str(), F_OK)) { - std::string createDirCommand = "mkdir -p " + aggregatorDirPath; - if (system(createDirCommand.c_str())) { - instance_logger.error("Creating directory " + aggregatorDirPath + " failed"); - } - } - - std::string copyCommand = "cp " + fullFilePath + " " + aggregatorDirPath; - if (system(copyCommand.c_str())) { + if (Utils::copyToDirectory(fullFilePath, aggregatorDirPath)) { instance_logger.error("Copying " + fullFilePath + " into " + aggregatorDirPath + " failed"); } - std::string movedFullFilePath = aggregatorDirPath + "/" + rawname; - - while (!Utils::fileExists(movedFullFilePath)) { - string response = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK) == 0) { - instance_logger.info("Received : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); - if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT)) { - *loop_exit_p = true; - return; - } - instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT); - } - } - line = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, false); if (line.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK) == 0) { instance_logger.info("Received : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index d0824647b..5408c7e40 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -293,7 +293,7 @@ int Utils::copyFile(const std::string sourceFilePath, const std::string destinat util_logger.log("Starting file copy source: " + sourceFilePath + " destination: " + destinationFilePath, "info"); std::string command = "cp " + sourceFilePath + " " + destinationFilePath; int status = system(command.c_str()); - if (status != 0) util_logger.warn("Copy failed with exit code " + std::to_string(status)); + if (status != 0) util_logger.error("Copy failed with exit code " + std::to_string(status)); return status; } @@ -484,11 +484,18 @@ void Utils::updateSLAInformation(PerformanceSQLiteDBInterface *perfSqlite, std:: } } -int Utils::copyToDirectory(std::string currentPath, std::string copyPath) { - std::string command = "mkdir -p " + copyPath + "&& " + "cp " + currentPath + " " + copyPath; - int status = system(command.c_str()); +int Utils::copyToDirectory(std::string currentPath, std::string destinationDir) { + if (access(destinationDir.c_str(), F_OK)) { + std::string createDirCommand = "mkdir -p " + destinationDir; + if (system(createDirCommand.c_str())) { + util_logger.error("Creating directory " + destinationDir + " failed"); + return -1; + } + } + std::string copyCommand = "cp " + currentPath + " " + destinationDir; + int status = system(copyCommand.c_str()); if (status != 0) { - util_logger.error("Copying " + currentPath + " to directory " + copyPath + " failed with code " + + util_logger.error("Copying " + currentPath + " to directory " + destinationDir + " failed with code " + std::to_string(status)); } return status; From 468fbc548dde7a107cbc02b0be45d0547daad051 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sun, 25 Feb 2024 03:56:31 +0530 Subject: [PATCH 046/143] Add option to check file existance in workers --- CMakeLists.txt | 2 + src/server/JasmineGraphInstanceProtocol.cpp | 1 + src/server/JasmineGraphInstanceProtocol.h | 1 + src/server/JasmineGraphInstanceService.cpp | 42 ++++++++++----------- src/util/Utils.cpp | 2 +- src/util/Utils.h | 2 +- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e9b17485..6075cba94 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,6 +149,8 @@ if (CMAKE_ENABLE_DEBUG) target_compile_options(JasmineGraph PRIVATE -g) endif () +SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-s,--gc-sections") + #TODO: Need to install the library separately target_link_libraries(JasmineGraphLib PRIVATE sqlite3) target_link_libraries(JasmineGraphLib PRIVATE pthread) diff --git a/src/server/JasmineGraphInstanceProtocol.cpp b/src/server/JasmineGraphInstanceProtocol.cpp index ba2712f13..ea06ab478 100644 --- a/src/server/JasmineGraphInstanceProtocol.cpp +++ b/src/server/JasmineGraphInstanceProtocol.cpp @@ -79,6 +79,7 @@ const string JasmineGraphInstanceProtocol::CHECK_FILE_ACCESSIBLE = "check-file-a const string JasmineGraphInstanceProtocol::SEND_FILE_TYPE = "send-file-type"; const string JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_AGGREGATE = "file-type-centralstore-aggregate"; const string JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_COMPOSITE = "file-type-centralstore-composite"; +const string JasmineGraphInstanceProtocol::FILE_TYPE_DATA = "file-type-data"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_START = "stream-start"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_START_ACK = "stream-start-ack"; const string JasmineGraphInstanceProtocol::SEND_PRIORITY = "send-priority"; diff --git a/src/server/JasmineGraphInstanceProtocol.h b/src/server/JasmineGraphInstanceProtocol.h index 9c5fbae9e..b727894ff 100644 --- a/src/server/JasmineGraphInstanceProtocol.h +++ b/src/server/JasmineGraphInstanceProtocol.h @@ -99,6 +99,7 @@ class JasmineGraphInstanceProtocol { static const string SEND_FILE_TYPE; static const string FILE_TYPE_CENTRALSTORE_AGGREGATE; static const string FILE_TYPE_CENTRALSTORE_COMPOSITE; + static const string FILE_TYPE_DATA; static const string GRAPH_STREAM_START; static const string GRAPH_STREAM_START_ACK; static const string GRAPH_CSV_STREAM_START; diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index a0bb75126..65926536e 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -4005,6 +4005,8 @@ static void check_file_accessible_command(int connFd, bool *loop_exit_p) { return; } instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::SEND_FILE_TYPE); + string fullFilePath; + string result = "false"; char data[DATA_BUFFER_SIZE]; string fileType = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); @@ -4028,19 +4030,7 @@ static void check_file_accessible_command(int connFd, bool *loop_exit_p) { string aggregateLocation = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); string fileName = graphId + "_centralstore_" + partitionId; - string fullFilePath = aggregateLocation + "/" + fileName; - string result = "false"; - - bool fileExists = Utils::fileExists(fullFilePath); - if (fileExists) { - result = "true"; - } - - if (!Utils::send_str_wrapper(connFd, result)) { - *loop_exit_p = true; - return; - } - instance_logger.info("Sent : " + result); + fullFilePath = aggregateLocation + "/" + fileName; } else if (fileType.compare(JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_COMPOSITE) == 0) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -4052,20 +4042,26 @@ static void check_file_accessible_command(int connFd, bool *loop_exit_p) { instance_logger.info("Received File name: " + fileName); string aggregateLocation = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); - string fullFilePath = aggregateLocation + "/" + fileName; - string result = "false"; - - bool fileExists = Utils::fileExists(fullFilePath); - if (fileExists) { - result = "true"; - } - - if (!Utils::send_str_wrapper(connFd, result)) { + fullFilePath = aggregateLocation + "/" + fileName; + } else if (fileType.compare(JasmineGraphInstanceProtocol::FILE_TYPE_DATA) == 0) { + if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; return; } - instance_logger.info("Sent : " + result); + instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); + string fileName = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); + string dataDir = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); + fullFilePath = dataDir + "/" + fileName; + } + instance_logger.info("Checking existance of: " + fullFilePath); + if (Utils::fileExists(fullFilePath)) { + result = "true"; + } + if (!Utils::send_str_wrapper(connFd, result)) { + *loop_exit_p = true; + return; } + instance_logger.info("Sent : " + result); } static void graph_stream_start_command(int connFd, InstanceStreamHandler &instanceStreamHandler, bool *loop_exit_p) { diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 5408c7e40..6a1696338 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -580,7 +580,7 @@ std::string Utils::read_str_wrapper(int connFd, char *buf, size_t len, bool allo } std::string Utils::read_str_trim_wrapper(int connFd, char *buf, size_t len) { - string str = read_str_wrapper(connFd, buf, len, false); + string str = read_str_wrapper(connFd, buf, len); if (!str.empty()) str = trim_copy(str); return str; } diff --git a/src/util/Utils.h b/src/util/Utils.h index 652d02d90..9c3f3550f 100644 --- a/src/util/Utils.h +++ b/src/util/Utils.h @@ -127,7 +127,7 @@ class Utils { * @return The string read or "" on error. Logs error if recv failed. Also logs error if allowEmpty is false and * read length 0 string. */ - static std::string read_str_wrapper(int connFd, char *buf, size_t len, bool allowEmpty); + static std::string read_str_wrapper(int connFd, char *buf, size_t len, bool allowEmpty = false); /** * Wrapper to recv(2) to read a string and trim it. From 2befb8a7167567fddc51e21ba3016f43a984075b Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Mon, 26 Feb 2024 01:32:19 +0530 Subject: [PATCH 047/143] Add instance service command to push file from worker to worker --- src/server/JasmineGraphInstanceProtocol.cpp | 1 + src/server/JasmineGraphInstanceProtocol.h | 1 + src/server/JasmineGraphInstanceService.cpp | 41 +++++++++++++++++++++ src/server/JasmineGraphServer.cpp | 11 ++---- src/util/Utils.cpp | 26 ++++++------- 5 files changed, 60 insertions(+), 20 deletions(-) diff --git a/src/server/JasmineGraphInstanceProtocol.cpp b/src/server/JasmineGraphInstanceProtocol.cpp index ea06ab478..4d0651d52 100644 --- a/src/server/JasmineGraphInstanceProtocol.cpp +++ b/src/server/JasmineGraphInstanceProtocol.cpp @@ -83,6 +83,7 @@ const string JasmineGraphInstanceProtocol::FILE_TYPE_DATA = "file-type-data"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_START = "stream-start"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_START_ACK = "stream-start-ack"; const string JasmineGraphInstanceProtocol::SEND_PRIORITY = "send-priority"; +const string JasmineGraphInstanceProtocol::PUSH_FILE = "push-file"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_C_length_ACK = "stream-c-length-ack"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_END_OF_EDGE = "\r\n"; // CRLF equivelent in HTTP const string JasmineGraphInstanceProtocol::GRAPH_CSV_STREAM_START = "csv-stream-start"; diff --git a/src/server/JasmineGraphInstanceProtocol.h b/src/server/JasmineGraphInstanceProtocol.h index b727894ff..316c51c4f 100644 --- a/src/server/JasmineGraphInstanceProtocol.h +++ b/src/server/JasmineGraphInstanceProtocol.h @@ -82,6 +82,7 @@ class JasmineGraphInstanceProtocol { static const string INITIATE_TRAIN; static const string INITIATE_PREDICT; static const string SEND_HOSTS; + static const string PUSH_FILE; static const string INITIATE_MODEL_COLLECTION; static const string INITIATE_FRAGMENT_RESOLUTION; static const string FRAGMENT_RESOLUTION_CHK; diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 65926536e..d9a0c0b11 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -122,6 +122,7 @@ static void degree_distribution_common(int connFd, int serverPort, std::map &graphDBMapLocalStores, std::map &graphDBMapCentralStores, bool *loop_exit_p, bool in); +static void push_file_command(int connFd, bool *loop_exit_p); char *converter(const std::string &s) { char *pc = new char[s.size() + 1]; @@ -255,6 +256,8 @@ void *instanceservicesession(void *dummyPt) { graph_stream_start_command(connFd, streamHandler, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::SEND_PRIORITY) == 0) { send_priority_command(connFd, &loop_exit); + } else if (line.compare(JasmineGraphInstanceProtocol::PUSH_FILE) == 0) { + push_file_command(connFd, &loop_exit); } else { instance_logger.error("Invalid command"); loop_exit = true; @@ -274,6 +277,8 @@ void JasmineGraphInstanceService::run(string profile, string masterHost, string struct sockaddr_in svrAdd; struct sockaddr_in clntAdd; + Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder")); + // create socket listenFd = socket(AF_INET, SOCK_STREAM, 0); if (listenFd < 0) { @@ -4121,6 +4126,42 @@ static void send_priority_command(int connFd, bool *loop_exit_p) { highestPriority = retrievedPriority; } +static void push_file_command(int connFd, bool *loop_exit_p) { + if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { + *loop_exit_p = true; + return; + } + instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); + + char data[DATA_BUFFER_SIZE]; + string hostDataPort = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); + instance_logger.info("Received host:dataPort: " + hostDataPort); + std::vector hostPortList = Utils::split(hostDataPort, ':'); + std::string &host = hostPortList[0]; + int port = std::stoi(hostPortList[1]); + if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { + *loop_exit_p = true; + return; + } + instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); + + string fileName = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); + instance_logger.info("Received fileName: " + fileName); + string path = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + fileName; + if (!Utils::sendFileThroughService(host, port, fileName, path)) { + instance_logger.error("Sending failed"); + if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::ERROR)) { + *loop_exit_p = true; + } + return; + } + if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { + *loop_exit_p = true; + return; + } + instance_logger.info("Sent: " + JasmineGraphInstanceProtocol::OK); +} + string JasmineGraphInstanceService::aggregateStreamingCentralStoreTriangles( std::string graphId, std::string partitionId, std::string partitionIdString, std::string centralCountString, int threadPriority, std::map &incrementalLocalStores, diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 46f6eed95..619b52588 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -35,8 +35,7 @@ static void copyArtifactsToWorkers(const std::string &workerPath, const std::str const std::string &remoteWorker); static void createLogFilePath(const std::string &workerHost, const std::string &workerPath); static void deleteWorkerPath(const std::string &workerHost, const std::string &workerPath); -static void assignPartitionToWorker(std::string fileName, int graphId, std::string workerHost, int workerPort, - int workerDataPort); +static void assignPartitionToWorker(std::string fileName, int graphId, std::string workerHost, int workerPort); static void updateMetaDB(int graphID, std::string uploadEndTime); static bool batchUploadFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP); @@ -896,7 +895,7 @@ void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, std::thread(batchUploadCentralAttributeFile, worker.hostname, worker.port, worker.dataPort, graphID, centralStoreAttributeFileMap[file_count], masterHost); } - assignPartitionToWorker(partitionFileName, graphID, worker.hostname, worker.port, worker.dataPort); + assignPartitionToWorker(partitionFileName, graphID, worker.hostname, worker.port); file_count++; } } @@ -916,8 +915,7 @@ void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, delete[] workerThreads; } -static void assignPartitionToWorker(std::string fileName, int graphId, std::string workerHost, int workerPort, - int workerDataPort) { +static void assignPartitionToWorker(std::string fileName, int graphId, std::string workerHost, int workerPort) { auto *refToSqlite = new SQLiteDBInterface(); refToSqlite->init(); size_t lastindex = fileName.find_last_of("."); @@ -929,8 +927,7 @@ static void assignPartitionToWorker(std::string fileName, int graphId, std::stri } std::string workerSearchQuery = "select idworker from worker where ip='" + workerHost + "' and server_port='" + - std::to_string(workerPort) + "' and server_data_port='" + - std::to_string(workerDataPort) + "'"; + std::to_string(workerPort) + "'"; std::vector>> results = refToSqlite->runSelect(workerSearchQuery); diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 6a1696338..fe59263f3 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -964,8 +964,13 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f struct sockaddr_in serv_addr; struct hostent *server; - sockfd = socket(AF_INET, SOCK_STREAM, 0); + util_logger.info("Sending file " + filePath + " through port " + std::to_string(dataPort)); + FILE *fp = fopen(filePath.c_str(), "r"); + if (fp == NULL) { + return false; + } + sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { util_logger.error("Cannot create socket"); return false; @@ -991,13 +996,7 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f return false; } - util_logger.info("Sending file " + filePath + " through port " + std::to_string(dataPort)); - FILE *fp = fopen(filePath.c_str(), "r"); - if (fp == NULL) { - close(sockfd); - return false; - } - + bool status = true; while (true) { unsigned char buff[1024]; int nread = fread(buff, 1, sizeof(buff), fp); @@ -1005,16 +1004,17 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f /* If read was success, send data. */ if (nread > 0) { write(sockfd, buff, nread); - } - - if (nread < sizeof(buff)) { + } else { if (feof(fp)) util_logger.info("End of file"); - if (ferror(fp)) util_logger.error("Error reading file: " + filePath); + if (ferror(fp)) { + status = false; + util_logger.error("Error reading file: " + filePath); + } break; } } fclose(fp); close(sockfd); - return true; + return status; } From ee9ae1155e419e7e34ba360971518a0a9a23a510 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 28 Feb 2024 19:10:10 +0530 Subject: [PATCH 048/143] Create DataPublishers only if needed --- src/frontend/JasmineGraphFrontEnd.cpp | 42 +++++++++++-------- .../metrics/StatisticCollector.cpp | 8 ++-- src/server/JasmineGraphInstance.cpp | 2 - src/server/JasmineGraphServer.cpp | 3 ++ src/util/scheduler/SchedulerService.cpp | 2 +- test-docker.sh | 2 +- 6 files changed, 33 insertions(+), 26 deletions(-) diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index cf6fb116a..e85692b7e 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -94,6 +94,19 @@ static void start_remote_worker_command(int connFd, bool *loop_exit_p); static void sla_command(int connFd, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfSqlite, bool *loop_exit_p); +static vector getWorkerClients(SQLiteDBInterface *sqlite) { + const vector &workerList = Utils::getWorkerList(sqlite); + vector workerClients; + for (int i = 0; i < workerList.size(); i++) { + Utils::worker currentWorker = workerList.at(i); + string workerHost = currentWorker.hostname; + string workerID = currentWorker.workerID; + int workerPort = atoi(string(currentWorker.port).c_str()); + DataPublisher *workerClient = new DataPublisher(workerPort, workerHost); + workerClients.push_back(workerClient); + } +} + void *frontendservicesesion(void *dummyPt) { frontendservicesessionargs *sessionargs = (frontendservicesessionargs *)dummyPt; std::string masterIP = sessionargs->masterIP; @@ -105,9 +118,6 @@ void *frontendservicesesion(void *dummyPt) { frontend_logger.info("Thread No: " + to_string(pthread_self())); frontend_logger.info("Master IP: " + masterIP); char data[FRONTEND_DATA_LENGTH + 1]; - bzero(data, FRONTEND_DATA_LENGTH + 1); - vector workerList = Utils::getWorkerList(sqlite); - vector workerClients; // Initiate Thread thread input_stream_handler; @@ -118,15 +128,10 @@ void *frontendservicesesion(void *dummyPt) { cppkafka::Configuration configs; KafkaConnector *kstream; Partitioner graphPartitioner(numberOfPartitions, 1, spt::Algorithms::HASH); + + vector workerClients; + bool workerClientsInitialized = false; - for (int i = 0; i < workerList.size(); i++) { - Utils::worker currentWorker = workerList.at(i); - string workerHost = currentWorker.hostname; - string workerID = currentWorker.workerID; - int workerPort = atoi(string(currentWorker.port).c_str()); - DataPublisher *workerClient = new DataPublisher(workerPort, workerHost); - workerClients.push_back(workerClient); - } bool loop_exit = false; while (!loop_exit) { if (currentFESession == Conts::MAX_FE_SESSIONS + 1) { @@ -180,6 +185,10 @@ void *frontendservicesesion(void *dummyPt) { } else if (line.compare(ADGR_CUST) == 0) { add_graph_cust_command(masterIP, connFd, sqlite, &loop_exit); } else if (line.compare(ADD_STREAM_KAFKA) == 0) { + if (!workerClientsInitialized) { + workerClients = getWorkerClients(sqlite); + workerClientsInitialized = true; + } add_stream_kafka_command(connFd, kafka_server_IP, configs, kstream, input_stream_handler, workerClients, numberOfPartitions, &loop_exit); } else if (line.compare(STOP_STREAM_KAFKA) == 0) { @@ -1222,14 +1231,14 @@ static void add_stream_kafka_command(int connFd, std::string &kafka_server_IP, c } // create kafka consumer and graph partitioner kstream = new KafkaConnector(configs); -// Create the Partitioner object. + // Create the Partitioner object. Partitioner graphPartitioner(numberOfPartitions, 0, spt::Algorithms::FENNEL); -// Create the KafkaConnector object. + // Create the KafkaConnector object. kstream = new KafkaConnector(configs); // Subscribe to the Kafka topic. kstream->Subscribe(topic_name_s); -// Create the StreamHandler object. - StreamHandler* stream_handler = new StreamHandler(kstream, numberOfPartitions, workerClients); + // Create the StreamHandler object. + StreamHandler *stream_handler = new StreamHandler(kstream, numberOfPartitions, workerClients); frontend_logger.info("Start listening to " + topic_name_s); input_stream_handler_thread = thread(&StreamHandler::listen_to_kafka_topic, stream_handler); @@ -2113,8 +2122,7 @@ static void page_rank_command(std::string masterIP, int connFd, SQLiteDBInterfac auto end = chrono::high_resolution_clock::now(); auto dur = end - begin; auto msDuration = std::chrono::duration_cast(dur).count(); - frontend_logger.info("PageRank Time Taken : " + to_string(msDuration) + - " milliseconds"); + frontend_logger.info("PageRank Time Taken : " + to_string(msDuration) + " milliseconds"); result_wr = write(connFd, DONE.c_str(), FRONTEND_COMMAND_LENGTH); if (result_wr < 0) { diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index 994f43969..3229dd510 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -331,11 +331,9 @@ double StatisticCollector::getTotalCpuUsage() { } double StatisticCollector::getLoadAverage() { - double averages[3]; - - getloadavg(averages, 3); - - return averages[0]; + double loadAvg; + getloadavg(&loadAvg, 1); + return loadAvg; } void StatisticCollector::logLoadAverage(std::string name) { diff --git a/src/server/JasmineGraphInstance.cpp b/src/server/JasmineGraphInstance.cpp index 70a099db6..8fbc08bd2 100644 --- a/src/server/JasmineGraphInstance.cpp +++ b/src/server/JasmineGraphInstance.cpp @@ -175,8 +175,6 @@ void JasmineGraphInstance::startNmonAnalyzer(string enableNmon, int serverPort) std::string numberOfSnapshots = Utils::getJasmineGraphProperty("org.jasminegraph.server.nmon.snapshots"); std::string snapshotGap = Utils::getJasmineGraphProperty("org.jasminegraph.server.nmon.snapshot.gap"); std::string nmonFileName = nmonFileLocation + "nmon.log." + std::to_string(serverPort); - /*std::string nmonStartupCommand = - "nmon_x86_64_ubuntu18 -c " + numberOfSnapshots + " -s " + snapshotGap + " -T -F " + nmonFileName;*/ std::string nmonStartupCommand = "nmon -c " + numberOfSnapshots + " -s " + snapshotGap + " -T -F " + nmonFileName; diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 619b52588..a194428da 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -992,6 +992,7 @@ static bool batchUploadCompositeCentralstoreFile(std::string host, int port, int static void copyArtifactsToWorkers(const std::string &workerPath, const std::string &artifactLocation, const std::string &remoteWorker) { + // Not used in docker or k8s modes if (artifactLocation.empty() || artifactLocation.find_first_not_of(' ') == artifactLocation.npos) { server_logger.error("Received `" + artifactLocation + "` for `artifactLocation` value!"); return; @@ -1028,6 +1029,7 @@ static void copyArtifactsToWorkers(const std::string &workerPath, const std::str } static void deleteWorkerPath(const std::string &workerHost, const std::string &workerPath) { + // Not used in docker or k8s modes std::string pathDeletionCommand = "rm -rf " + workerPath; if (workerHost.find("localhost") == std::string::npos) { pathDeletionCommand = "ssh " + workerHost + " " + pathDeletionCommand; @@ -1039,6 +1041,7 @@ static void deleteWorkerPath(const std::string &workerHost, const std::string &w } static void createLogFilePath(const std::string &workerHost, const std::string &workerPath) { + // Not used in docker or k8s modes std::string pathCreationCommand = "mkdir -p " + workerPath + "/logs"; if (workerHost.find("localhost") == std::string::npos) { pathCreationCommand = "ssh " + workerHost + " " + pathCreationCommand; diff --git a/src/util/scheduler/SchedulerService.cpp b/src/util/scheduler/SchedulerService.cpp index fd9481da6..5860ba07b 100644 --- a/src/util/scheduler/SchedulerService.cpp +++ b/src/util/scheduler/SchedulerService.cpp @@ -20,10 +20,10 @@ using namespace Bosma; Logger schedulerservice_logger; void SchedulerService::startScheduler() { - sleep(3); std::string schedulerEnabled = Utils::getJasmineGraphProperty("org.jasminegraph.scheduler.enabled"); if (schedulerEnabled == "true") { + sleep(3); schedulerservice_logger.log("#######SCHEDULER ENABLED#####", "info"); startPerformanceScheduler(); diff --git a/test-docker.sh b/test-docker.sh index a95dfd517..d98e1f47b 100755 --- a/test-docker.sh +++ b/test-docker.sh @@ -78,7 +78,7 @@ stop_tests_on_failure() { break fi fi - sleep 1 + sleep 5 done } From 667be44e05afe603e5185dd385fae7cae2944bdc Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 1 Mar 2024 01:59:52 +0530 Subject: [PATCH 049/143] Remove unused functions and fix busy-waiting loop --- src/frontend/JasmineGraphFrontEnd.cpp | 2 +- .../core/executor/AbstractExecutor.cpp | 8 +- .../core/executor/impl/PageRankExecutor.cpp | 5 +- .../executor/impl/TriangleCountExecutor.cpp | 5 +- src/performance/metrics/PerformanceUtil.cpp | 576 +----------------- src/performance/metrics/PerformanceUtil.h | 16 +- .../metrics/StatisticCollector.cpp | 26 - src/performance/metrics/StatisticCollector.h | 1 - ...asmineGraphInstanceFileTransferService.cpp | 2 +- src/server/JasmineGraphInstanceProtocol.cpp | 1 - src/server/JasmineGraphInstanceProtocol.h | 1 - src/server/JasmineGraphInstanceService.cpp | 77 +-- src/server/JasmineGraphInstanceService.h | 1 - src/server/JasmineGraphServer.cpp | 7 +- src/util/Conts.h | 1 - 15 files changed, 52 insertions(+), 677 deletions(-) diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index e85692b7e..55423cbbe 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -128,7 +128,7 @@ void *frontendservicesesion(void *dummyPt) { cppkafka::Configuration configs; KafkaConnector *kstream; Partitioner graphPartitioner(numberOfPartitions, 1, spt::Algorithms::HASH); - + vector workerClients; bool workerClientsInitialized = false; diff --git a/src/frontend/core/executor/AbstractExecutor.cpp b/src/frontend/core/executor/AbstractExecutor.cpp index 33eec5756..28d5eb3f3 100644 --- a/src/frontend/core/executor/AbstractExecutor.cpp +++ b/src/frontend/core/executor/AbstractExecutor.cpp @@ -12,6 +12,7 @@ limitations under the License. */ #include "AbstractExecutor.h" + #include "../../../performance/metrics/PerformanceUtil.h" AbstractExecutor::AbstractExecutor(JobRequest jobRequest) { this->request = jobRequest; } @@ -87,8 +88,7 @@ int AbstractExecutor::collectPerformaceData(PerformanceSQLiteDBInterface *perDB, if (!(isMaster.find("true") != std::string::npos || host == "localhost" || host.compare(masterIP) == 0)) { PerformanceUtil::initiateCollectingRemoteSLAResourceUtilization( - host, atoi(serverPort.c_str()), isHostReporter, "false", - placeId, elapsedTime, masterIP); + host, atoi(serverPort.c_str()), isHostReporter, "false", placeId, elapsedTime, masterIP); } } @@ -108,8 +108,8 @@ int AbstractExecutor::collectPerformaceData(PerformanceSQLiteDBInterface *perDB, } } - PerformanceUtil::updateRemoteResourceConsumption(perDB, graphId, partitionCount, placeList, - slaCategoryId, masterIP); + PerformanceUtil::updateRemoteResourceConsumption(perDB, graphId, partitionCount, placeList, slaCategoryId, + masterIP); PerformanceUtil::updateResourceConsumption(perDB, graphId, partitionCount, placeList, slaCategoryId); return 0; diff --git a/src/frontend/core/executor/impl/PageRankExecutor.cpp b/src/frontend/core/executor/impl/PageRankExecutor.cpp index 8ef537e5f..1cef28222 100644 --- a/src/frontend/core/executor/impl/PageRankExecutor.cpp +++ b/src/frontend/core/executor/impl/PageRankExecutor.cpp @@ -139,8 +139,9 @@ void PageRankExecutor::execute() { } else { pageRank_logger.log("###PAGERANK-EXECUTOR### Inserting initial record for SLA ", "info"); Utils::updateSLAInformation(perfDB, graphId, partitionCount, 0, PAGE_RANK, Conts::SLA_CATEGORY::LATENCY); - statResponse.push_back(std::async(std::launch::async, collectPerformaceData, perfDB, graphId.c_str(), PAGE_RANK, - Conts::SLA_CATEGORY::LATENCY, partitionCount, masterIP, autoCalibrate)); + statResponse.push_back(std::async(std::launch::async, AbstractExecutor::collectPerformaceData, perfDB, + graphId.c_str(), PAGE_RANK, Conts::SLA_CATEGORY::LATENCY, partitionCount, + masterIP, autoCalibrate)); isStatCollect = true; } diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 57bb5f64a..1c056b719 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -198,8 +198,9 @@ void TriangleCountExecutor::execute() { } else { triangleCount_logger.log("###TRIANGLE-COUNT-EXECUTOR### Inserting initial record for SLA ", "info"); Utils::updateSLAInformation(perfDB, graphId, partitionCount, 0, TRIANGLES, Conts::SLA_CATEGORY::LATENCY); - statResponse.push_back(std::async(std::launch::async, collectPerformaceData, perfDB, graphId.c_str(), TRIANGLES, - Conts::SLA_CATEGORY::LATENCY, partitionCount, masterIP, autoCalibrate)); + statResponse.push_back(std::async(std::launch::async, AbstractExecutor::collectPerformaceData, perfDB, + graphId.c_str(), TRIANGLES, Conts::SLA_CATEGORY::LATENCY, partitionCount, + masterIP, autoCalibrate)); isStatCollect = true; } diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index 27416dabb..b9c8baa4f 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -133,337 +133,6 @@ int PerformanceUtil::collectSLAResourceConsumption(std::vector placeList, return 0; } -std::vector PerformanceUtil::retrieveCurrentResourceUtilization(std::string masterIP) { - std::string placeLoadQuery = - "select ip, user, server_port, is_master, is_host_reporter,host_idhost,idplace from place"; - std::vector>> placeList = perfDb->runSelect(placeLoadQuery); - std::vector>>::iterator placeListIterator; - std::vector placeResourceConsumptionList; - ResourceConsumption resourceConsumption; - - for (placeListIterator = placeList.begin(); placeListIterator != placeList.end(); ++placeListIterator) { - vector> place = *placeListIterator; - std::string host; - std::string requestResourceAllocation = "false"; - - std::string ip = place.at(0).second; - std::string user = place.at(1).second; - std::string serverPort = place.at(2).second; - std::string isMaster = place.at(3).second; - std::string isHostReporter = place.at(4).second; - std::string hostId = place.at(5).second; - std::string placeId = place.at(6).second; - - if (ip.find("localhost") != std::string::npos || ip.compare(masterIP) == 0) { - host = "localhost"; - } else { - host = user + "@" + ip; - } - - if (isHostReporter.find("true") != std::string::npos) { - std::string hostSearch = "select total_cpu_cores,total_memory from host where idhost='" + hostId + "'"; - std::vector>> hostAllocationList = perfDb->runSelect(hostSearch); - - vector> firstHostAllocation = hostAllocationList.at(0); - - std::string totalCPUCores = firstHostAllocation.at(0).second; - std::string totalMemory = firstHostAllocation.at(1).second; - - if (totalCPUCores.empty() || totalMemory.empty()) { - requestResourceAllocation = "true"; - } - } - - if ((isMaster.find("true") != std::string::npos || host == "localhost" || host.compare(masterIP) == 0) && - isHostReporter.find("true") != std::string::npos) { - resourceConsumption = retrieveLocalResourceConsumption(host, placeId); - placeResourceConsumptionList.push_back(resourceConsumption); - } else if (isHostReporter.find("true") != std::string::npos) { - resourceConsumption = retrieveRemoteResourceConsumption(host, atoi(serverPort.c_str()), hostId, placeId); - placeResourceConsumptionList.push_back(resourceConsumption); - } - } - - return placeResourceConsumptionList; -} - -void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, std::string isVMStatManager, - std::string isResourceAllocationRequired, std::string hostId, - std::string placeId) { - int sockfd; - char data[301]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - scheduler_logger.error("ERROR, no host named " + host); - return; - } - - bzero((char *) &serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - return; - } - - bzero(data, 301); - write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); - string server_host = Utils::getJasmineGraphProperty("org.jasminegraph.server.host"); - write(sockfd, server_host.c_str(), server_host.size()); - scheduler_logger.log("Sent : " + server_host, "info"); - - write(sockfd, JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS.c_str(), - JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - // scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); - write(sockfd, isVMStatManager.c_str(), isVMStatManager.size()); - scheduler_logger.log("Sent : VM Manager Status " + isVMStatManager, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - write(sockfd, isResourceAllocationRequired.c_str(), isResourceAllocationRequired.size()); - scheduler_logger.log("Sent : Resource Allocation Requested " + isResourceAllocationRequired, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - scheduler_logger.log("Performance Response " + response, "info"); - - std::vector strArr = Utils::split(response, ','); - - std::string processTime = strArr[0]; - std::string memoryUsage = strArr[1]; - std::string cpuUsage = strArr[2]; - std::string loadAverage = strArr[3]; - - if (isVMStatManager == "true" && strArr.size() > 4) { - std::string memoryConsumption = strArr[4]; - std::string cpuUsage = strArr[5]; - string vmPerformanceSql = - "insert into host_performance_data (date_time, memory_usage, cpu_usage, idhost) values ('" + - processTime + "','" + memoryConsumption + "','" + cpuUsage + "','" + hostId + "')"; - - perfDb->runInsert(vmPerformanceSql); - Utils::send_job("hostPerfData", "memory_consumption", memoryConsumption); - Utils::send_job("hostPerfData", "cpu_usage", cpuUsage); - - if (isResourceAllocationRequired == "true") { - std::string totalMemory = strArr[6]; - std::string totalCores = strArr[7]; - - string allocationUpdateSql = "update host set total_cpu_cores='" + totalCores + - "',total_memory='" + totalMemory + "' where idhost='" + hostId + - "'"; - perfDb->runUpdate(allocationUpdateSql); - - Utils::send_job("placePerfData", "memory_usage", memoryUsage); - Utils::send_job("placePerfData", "cpu_usage", cpuUsage); - } - } - - string placePerfSql = - "insert into place_performance_data (idplace, memory_usage, cpu_usage, date_time) values ('" + - placeId + "','" + memoryUsage + "','" + cpuUsage + "','" + processTime + "')"; - - perfDb->runInsert(placePerfSql); - } - } - } -} - -void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, std::string isResourceAllocationRequired, - std::string hostId, std::string placeId) { - int memoryUsage = StatisticCollector::getMemoryUsageByProcess(); - double cpuUsage = StatisticCollector::getCpuUsage(); - - auto executedTime = std::chrono::system_clock::now(); - std::time_t reportTime = std::chrono::system_clock::to_time_t(executedTime); - std::string reportTimeString(std::ctime(&reportTime)); - reportTimeString = Utils::trim_copy(reportTimeString); - - if (isVMStatManager.find("true") != std::string::npos) { - std::string vmLevelStatistics = - StatisticCollector::collectVMStatistics(isVMStatManager, isResourceAllocationRequired); - std::vector strArr = Utils::split(vmLevelStatistics, ','); - - string totalMemoryUsed = strArr[0]; - string totalCPUUsage = strArr[1]; - - string vmPerformanceSql = - "insert into host_performance_data (date_time, memory_usage, cpu_usage, idhost) values ('" + - reportTimeString + "','" + totalMemoryUsed + "','" + totalCPUUsage + "','" + hostId + "')"; - - perfDb->runInsert(vmPerformanceSql); - - if (isResourceAllocationRequired == "true") { - std::string totalMemory = strArr[2]; - std::string totalCores = strArr[3]; - string allocationUpdateSql = "update host set total_cpu_cores='" + totalCores + "',total_memory='" + - totalMemory + "' where idhost='" + hostId + "'"; - - perfDb->runUpdate(allocationUpdateSql); - } - } - - string placePerfSql = "insert into place_performance_data (idplace, memory_usage, cpu_usage, date_time) values ('" + - placeId + "','" + to_string(memoryUsage) + "','" + to_string(cpuUsage) + "','" + - reportTimeString + "')"; - - perfDb->runInsert(placePerfSql); -} - -int PerformanceUtil::collectRemoteSLAResourceUtilization(std::string host, int port, std::string isVMStatManager, - std::string isResourceAllocationRequired, std::string placeId, - int elapsedTime, std::string masterIP) { - int sockfd; - char data[301]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - std::string graphSlaId; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return 0; - } - - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - scheduler_logger.error("ERROR, no host named " + host); - return 0; - } - - bzero((char *) &serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - return 0; - } - - bzero(data, 301); - write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); - write(sockfd, masterIP.c_str(), masterIP.size()); - scheduler_logger.log("Sent : " + masterIP, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); - - if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { - write(sockfd, JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS.c_str(), - JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - scheduler_logger.log("Received : " + response, "info"); - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - write(sockfd, isVMStatManager.c_str(), isVMStatManager.size()); - scheduler_logger.log("Sent : VM Manager Status " + isVMStatManager, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - write(sockfd, isResourceAllocationRequired.c_str(), isResourceAllocationRequired.size()); - scheduler_logger.log("Sent : Resource Allocation Requested " + isResourceAllocationRequired, - "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - scheduler_logger.log("Performance Response " + response, "info"); - - std::vector strArr = Utils::split(response, ','); - - std::string processTime = strArr[0]; - std::string memoryUsage = strArr[1]; - std::string cpuUsage = strArr[2]; - std::string loadAverage = strArr[3]; - - ResourceUsageInfo resourceUsageInfo; - resourceUsageInfo.elapsedTime = std::to_string(elapsedTime); - resourceUsageInfo.loadAverage = loadAverage; - resourceUsageInfo.memoryUsage = memoryUsage; - - if (!resourceUsageMap[placeId].empty()) { - resourceUsageMap[placeId].push_back(resourceUsageInfo); - } else { - std::vector resourceUsageVector; - - resourceUsageVector.push_back(resourceUsageInfo); - - resourceUsageMap[placeId] = resourceUsageVector; - } - return 1; - } - } - } - } - return 0; -} - void PerformanceUtil::collectLocalSLAResourceUtilization(std::string graphId, std::string placeId, std::string command, std::string category, int elapsedTime, bool autoCalibrate) { string graphSlaId; @@ -494,112 +163,6 @@ void PerformanceUtil::collectLocalSLAResourceUtilization(std::string graphId, st } } -ResourceConsumption PerformanceUtil::retrieveLocalResourceConsumption(std::string host, std::string placeId) { - ResourceConsumption placeResourceConsumption; - - int memoryUsage = StatisticCollector::getMemoryUsageByProcess(); - double cpuUsage = StatisticCollector::getCpuUsage(); - - placeResourceConsumption.memoryUsage = memoryUsage; - placeResourceConsumption.host = host; - - return placeResourceConsumption; -} - -ResourceConsumption PerformanceUtil::retrieveRemoteResourceConsumption(std::string host, int port, std::string hostId, - std::string placeId) { - ResourceConsumption placeResourceConsumption; - - int sockfd; - char data[301]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - std::string graphSlaId; - std::string isVMStatManager = "false"; - std::string isResourceAllocationRequired = "false"; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return placeResourceConsumption; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - scheduler_logger.error("ERROR, no host named " + host); - return placeResourceConsumption; - } - - bzero((char *) &serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - return placeResourceConsumption; - } - - bzero(data, 301); - write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); - string server_host = Utils::getJasmineGraphProperty("org.jasminegraph.server.host"); - write(sockfd, server_host.c_str(), server_host.size()); - scheduler_logger.log("Sent : " + server_host, "info"); - - write(sockfd, JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS.c_str(), - JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - write(sockfd, isVMStatManager.c_str(), isVMStatManager.size()); - scheduler_logger.log("Sent : VM Manager Status " + isVMStatManager, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - write(sockfd, isResourceAllocationRequired.c_str(), isResourceAllocationRequired.size()); - scheduler_logger.log("Sent : Resource Allocation Requested " + isResourceAllocationRequired, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - scheduler_logger.log("Performance Response " + response, "info"); - - std::vector strArr = Utils::split(response, ','); - - std::string processTime = strArr[0]; - std::string memoryUsage = strArr[1]; - std::string cpuUsage = strArr[2]; - - placeResourceConsumption.memoryUsage = atoi(memoryUsage.c_str()); - placeResourceConsumption.host = host; - } - } - } - return placeResourceConsumption; -} - std::vector PerformanceUtil::getResourceAvailableTime(std::vector graphIdList, std::string command, std::string category, std::string masterIP, std::vector &pendingHPJobList) { @@ -649,10 +212,9 @@ std::vector PerformanceUtil::getResourceAvailableTime(std::vector '" + std::to_string(adjustedElapsedTime) + - "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time;"; + currentGraphId + "' and sla_category.command='" + command + "' and sla_category.category='" + category + + "' and graph_place_sla_performance.elapsed_time > '" + std::to_string(adjustedElapsedTime) + + "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time;"; std::vector>> loadAvgResults = perfDb->runSelect(slaLoadQuery); @@ -738,8 +300,8 @@ std::vector PerformanceUtil::getResourceAvailableTime(std::vector>> newJobLoadAvgResults = perfDb->runSelect(newJobLoadQuery); @@ -879,7 +441,6 @@ void PerformanceUtil::logLoadAverage() { double currentLoadAverage = StatisticCollector::getLoadAverage(); std::cout << "###PERF### CURRENT LOAD: " + std::to_string(currentLoadAverage) << std::endl; - Utils::send_job("loadAverage", "load_average", std::to_string(currentLoadAverage)); } void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *performanceDb, std::string graphId, @@ -895,7 +456,7 @@ void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *pe std::string graphSlaId; std::string query = "SELECT id from graph_sla where graph_id='" + graphId + "' and partition_count='" + - std::to_string(partitionCount) + "' and id_sla_category='" + slaCategoryId + "';"; + std::to_string(partitionCount) + "' and id_sla_category='" + slaCategoryId + "';"; std::vector>> results = performanceDb->runSelect(query); @@ -904,7 +465,7 @@ void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *pe } else { std::string insertQuery = "insert into graph_sla (id_sla_category, graph_id, partition_count, sla_value, attempt) VALUES ('" + - slaCategoryId + "','" + graphId + "'," + std::to_string(partitionCount) + ",0,0);"; + slaCategoryId + "','" + graphId + "'," + std::to_string(partitionCount) + ",0,0);"; int slaId = performanceDb->runInsert(insertQuery); graphSlaId = std::to_string(slaId); @@ -925,7 +486,7 @@ void PerformanceUtil::updateResourceConsumption(PerformanceSQLiteDBInterface *pe ResourceUsageInfo usageInfo = *usageIterator; valuesString += "('" + graphSlaId + "','" + placeId + "', '','" + usageInfo.loadAverage + "','" + - usageInfo.elapsedTime + "'),"; + usageInfo.elapsedTime + "'),"; Utils::send_job("loadAverageSLA", "load_average", usageInfo.loadAverage); } @@ -966,10 +527,9 @@ void PerformanceUtil::updateRemoteResourceConsumption(PerformanceSQLiteDBInterfa std::string loadString = requestRemoteLoadAverages(host, atoi(serverPort.c_str()), isHostReporter, "false", placeId, 0, masterIP); std::vector loadVector = Utils::split(loadString, ','); - std::vector::iterator loadVectorIterator; string valuesString; std::string query = "SELECT id from graph_sla where graph_id='" + graphId + "' and partition_count='" + - std::to_string(partitionCount) + "' and id_sla_category='" + slaCategoryId + "';"; + std::to_string(partitionCount) + "' and id_sla_category='" + slaCategoryId + "';"; std::vector>> results = performanceDb->runSelect(query); @@ -978,7 +538,7 @@ void PerformanceUtil::updateRemoteResourceConsumption(PerformanceSQLiteDBInterfa } else { std::string insertQuery = "insert into graph_sla (id_sla_category, graph_id, partition_count, sla_value, attempt) VALUES ('" + - slaCategoryId + "','" + graphId + "'," + std::to_string(partitionCount) + ",0,0);"; + slaCategoryId + "','" + graphId + "'," + std::to_string(partitionCount) + ",0,0);"; int slaId = performanceDb->runInsert(insertQuery); graphSlaId = std::to_string(slaId); @@ -991,12 +551,12 @@ void PerformanceUtil::updateRemoteResourceConsumption(PerformanceSQLiteDBInterfa "elapsed_time) " "values "; - for (loadVectorIterator = loadVector.begin(); loadVectorIterator != loadVector.end(); + for (auto loadVectorIterator = loadVector.begin(); loadVectorIterator != loadVector.end(); ++loadVectorIterator) { std::string loadAverage = *loadVectorIterator; valuesString += "('" + graphSlaId + "','" + placeId + "', '','" + loadAverage + "','" + - std::to_string(elapsedTime * 1000) + "'),"; + std::to_string(elapsedTime * 1000) + "'),"; elapsedTime += 5; Utils::send_job("loadAverageSLARem", "load_average_rem", loadAverage); @@ -1055,11 +615,11 @@ void PerformanceUtil::initiateCollectingRemoteSLAResourceUtilization(std::string return; } - bzero((char *) &serv_addr, sizeof(serv_addr)); + bzero((char *)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return; } @@ -1104,87 +664,9 @@ void PerformanceUtil::initiateCollectingRemoteSLAResourceUtilization(std::string std::string PerformanceUtil::requestRemoteLoadAverages(std::string host, int port, std::string isVMStatManager, std::string isResourceAllocationRequired, std::string placeId, int elapsedTime, std::string masterIP) { - int sockfd; - char data[301]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - std::string graphSlaId; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return 0; - } - - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - scheduler_logger.error("ERROR, no host named " + host); - return 0; - } - - bzero((char *) &serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - return 0; - } - - bzero(data, 301); - write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); - write(sockfd, masterIP.c_str(), masterIP.size()); - scheduler_logger.log("Sent : " + masterIP, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); - - if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { - write(sockfd, JasmineGraphInstanceProtocol::REQUEST_COLLECTED_STATS.c_str(), - JasmineGraphInstanceProtocol::REQUEST_COLLECTED_STATS.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::REQUEST_COLLECTED_STATS, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - string status = response.substr(response.size() - 5); - std::string result = response.substr(0, response.size() - 5); - - while (status == "/SEND") { - write(sockfd, status.c_str(), status.size()); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - status = response.substr(response.size() - 5); - std::string loadAverageString = response.substr(0, response.size() - 5); - result = result + loadAverageString; - } - response = result; - } - } - - return response; + // TODO(thevindu-w): master should not request load averages directly from workers. + // Instead it should get from Prometheus. + return ""; } double PerformanceUtil::getAggregatedLoadAverage(std::string graphId, std::string placeId, std::string command, @@ -1222,11 +704,11 @@ double PerformanceUtil::getAggregatedLoadAverage(std::string graphId, std::strin "graph_place_sla_performance.elapsed_time from graph_place_sla_performance inner join graph_sla" " inner join sla_category where graph_place_sla_performance.graph_sla_id=graph_sla.id" " and graph_sla.id_sla_category=sla_category.id and graph_sla.graph_id='" + - currentGraphId + "' and sla_category.command='" + command + - "' and graph_place_sla_performance.place_id='" + placeId + "' and sla_category.category='" + - category + "' and graph_place_sla_performance.elapsed_time ='" + - std::to_string(adjustedElapsedTime) + - "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time;"; + currentGraphId + "' and sla_category.command='" + command + + "' and graph_place_sla_performance.place_id='" + placeId + "' and sla_category.category='" + + category + "' and graph_place_sla_performance.elapsed_time ='" + + std::to_string(adjustedElapsedTime) + + "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time;"; std::vector>> loadAvgResults = perfDb->runSelect(slaLoadQuery); if (loadAvgResults.empty()) { @@ -1245,12 +727,12 @@ double PerformanceUtil::getAggregatedLoadAverage(std::string graphId, std::strin "graph_place_sla_performance.elapsed_time from graph_place_sla_performance inner join graph_sla" " inner join sla_category where graph_place_sla_performance.graph_sla_id=graph_sla.id" " and graph_sla.id_sla_category=sla_category.id and graph_sla.graph_id='" + - currentGraphId + "' and sla_category.command='" + command + - "' and graph_place_sla_performance.place_id='" + placeId + "' and sla_category.category='" + - category + "' and graph_place_sla_performance.elapsed_time > '" + - std::to_string(adjustedElapsedTime) + - "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time" + - " LIMIT 2;"; + currentGraphId + "' and sla_category.command='" + command + + "' and graph_place_sla_performance.place_id='" + placeId + "' and sla_category.category='" + + category + "' and graph_place_sla_performance.elapsed_time > '" + + std::to_string(adjustedElapsedTime) + + "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time" + + " LIMIT 2;"; std::vector>> loadAvgResults = perfDb->runSelect(slaLoadQuery); diff --git a/src/performance/metrics/PerformanceUtil.h b/src/performance/metrics/PerformanceUtil.h index 7e18bb5e1..cf59e4010 100644 --- a/src/performance/metrics/PerformanceUtil.h +++ b/src/performance/metrics/PerformanceUtil.h @@ -60,16 +60,15 @@ class PerformanceUtil { static int collectSLAResourceConsumption(std::vector placeList, std::string graphId, std::string command, std::string category, std::string masterIP, int elapsedTime, bool autoCalibrate); - static std::vector retrieveCurrentResourceUtilization(std::string masterIP); static std::vector getResourceAvailableTime(std::vector graphIdList, std::string command, std::string category, std::string masterIP, std::vector& pendingHPJobList); static void logLoadAverage(); static std::vector getHostReporterList(); - static void updateResourceConsumption(PerformanceSQLiteDBInterface *performanceDb, std::string graphId, + static void updateResourceConsumption(PerformanceSQLiteDBInterface* performanceDb, std::string graphId, int partitionCount, std::vector placeList, std::string slaCategoryId); - static void updateRemoteResourceConsumption(PerformanceSQLiteDBInterface *performanceDb, std::string graphId, + static void updateRemoteResourceConsumption(PerformanceSQLiteDBInterface* performanceDb, std::string graphId, int partitionCount, std::vector placeList, std::string slaCategoryId, std::string masterIP); static std::string getSLACategoryId(std::string command, std::string category); @@ -84,19 +83,8 @@ class PerformanceUtil { std::string category, int elapsedTime); private: - static void collectRemotePerformanceData(std::string host, int port, std::string isVMStatManager, - std::string isResourceAllocationRequired, std::string hostId, - std::string placeId); - static void collectLocalPerformanceData(std::string isVMStatManager, std::string isResourceAllocationRequired, - std::string hostId, std::string placeId); - static int collectRemoteSLAResourceUtilization(std::string host, int port, std::string isVMStatManager, - std::string isResourceAllocationRequired, std::string placeId, - int elapsedTime, std::string masterIP); static void collectLocalSLAResourceUtilization(std::string graphId, std::string placeId, std::string command, std::string category, int elapsedTime, bool autoCalibrate); - static ResourceConsumption retrieveRemoteResourceConsumption(std::string host, int port, std::string hostId, - std::string placeId); - static ResourceConsumption retrieveLocalResourceConsumption(std::string hostId, std::string placeId); static void adjustAggregateLoadMap(std::map>& aggregateLoadAvgMap, std::map>& newJobLoadAvgMap, long newJobAcceptanceTime); diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index 3229dd510..217dd0395 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -206,31 +206,6 @@ double StatisticCollector::getCpuUsage() { return (diffTotal - diffIdle) / (double)diffTotal; } -std::string StatisticCollector::collectVMStatistics(std::string isVMStatManager, - std::string isTotalAllocationRequired) { - std::string vmLevelStatistics; - - if (isVMStatManager == "true") { - long totalMemoryUsed = getTotalMemoryUsage(); - double totalCPUUsage = getTotalCpuUsage(); - - std::stringstream stream; - stream << std::fixed << std::setprecision(2) << totalCPUUsage; - std::string cpuUsageString = stream.str(); - - vmLevelStatistics = std::to_string(totalMemoryUsed) + "," + cpuUsageString + ","; - } - - if (isTotalAllocationRequired == "true") { - long totalMemory = getTotalMemoryAllocated(); - int totalCoresAvailable = getTotalNumberofCores(); - - vmLevelStatistics = vmLevelStatistics + std::to_string(totalMemory) + "," + std::to_string(totalCoresAvailable); - } - - return vmLevelStatistics; -} - long StatisticCollector::getTotalMemoryAllocated() { std::string token; std::ifstream file("/proc/meminfo"); @@ -346,7 +321,6 @@ void StatisticCollector::logLoadAverage(std::string name) { performanceUtil.init(); start = time(0); - while (true) { if (isStatCollect) { std::this_thread::sleep_for(std::chrono::seconds(60)); diff --git a/src/performance/metrics/StatisticCollector.h b/src/performance/metrics/StatisticCollector.h index 75ea775ce..d2550d70a 100644 --- a/src/performance/metrics/StatisticCollector.h +++ b/src/performance/metrics/StatisticCollector.h @@ -49,7 +49,6 @@ class StatisticCollector { static long getTXBytes(); static int getSocketCount(); static double getCpuUsage(); - static std::string collectVMStatistics(std::string isVMStatManager, std::string isTotalAllocationRequired); static long getTotalMemoryAllocated(); static int getTotalNumberofCores(); static long getTotalMemoryUsage(); diff --git a/src/server/JasmineGraphInstanceFileTransferService.cpp b/src/server/JasmineGraphInstanceFileTransferService.cpp index 5ab602ac8..9934df92b 100644 --- a/src/server/JasmineGraphInstanceFileTransferService.cpp +++ b/src/server/JasmineGraphInstanceFileTransferService.cpp @@ -32,7 +32,7 @@ void *filetransferservicesession(void *dummyPt) { Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + fileName; write(connFd, JasmineGraphInstanceProtocol::SEND_FILE.c_str(), JasmineGraphInstanceProtocol::SEND_FILE.size()); - int bytesReceived = 0; + int bytesReceived; char buffer[1024]; std::ofstream file(filePathWithName, std::ios::out | std::ios::binary); do { diff --git a/src/server/JasmineGraphInstanceProtocol.cpp b/src/server/JasmineGraphInstanceProtocol.cpp index 4d0651d52..ab220ca0c 100644 --- a/src/server/JasmineGraphInstanceProtocol.cpp +++ b/src/server/JasmineGraphInstanceProtocol.cpp @@ -62,7 +62,6 @@ const string JasmineGraphInstanceProtocol::SEND_COMPOSITE_CENTRALSTORE_TO_AGGREG const string JasmineGraphInstanceProtocol::AGGREGATE_CENTRALSTORE_TRIANGLES = "aggregate"; const string JasmineGraphInstanceProtocol::AGGREGATE_STREAMING_CENTRALSTORE_TRIANGLES = "aggregate-streaming-central"; const string JasmineGraphInstanceProtocol::AGGREGATE_COMPOSITE_CENTRALSTORE_TRIANGLES = "aggregate-composite"; -const string JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS = "perf-stat"; const string JasmineGraphInstanceProtocol::START_STAT_COLLECTION = "begin-stat"; const string JasmineGraphInstanceProtocol::REQUEST_COLLECTED_STATS = "request-stat"; const string JasmineGraphInstanceProtocol::INITIATE_TRAIN = "initiate-train"; diff --git a/src/server/JasmineGraphInstanceProtocol.h b/src/server/JasmineGraphInstanceProtocol.h index 316c51c4f..1cc34134f 100644 --- a/src/server/JasmineGraphInstanceProtocol.h +++ b/src/server/JasmineGraphInstanceProtocol.h @@ -76,7 +76,6 @@ class JasmineGraphInstanceProtocol { static const string AGGREGATE_CENTRALSTORE_TRIANGLES; static const string AGGREGATE_STREAMING_CENTRALSTORE_TRIANGLES; static const string AGGREGATE_COMPOSITE_CENTRALSTORE_TRIANGLES; - static const string PERFORMANCE_STATISTICS; static const string START_STAT_COLLECTION; static const string REQUEST_COLLECTED_STATS; static const string INITIATE_TRAIN; diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index d9a0c0b11..8c43a1e01 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -99,7 +99,6 @@ static void aggregate_streaming_centralstore_triangles_command( int connFd, std::map &incrementalLocalStoreMap, bool *loop_exit_p); static void aggregate_composite_centralstore_triangles_command(int connFd, bool *loop_exit_p); -static void performance_statistics_command(int connFd, bool *loop_exit_p); static void initiate_files_command(int connFd, bool *loop_exit_p); static void initiate_fed_predict_command(int connFd, bool *loop_exit_p); static void initiate_server_command(int connFd, bool *loop_exit_p); @@ -222,8 +221,6 @@ void *instanceservicesession(void *dummyPt) { aggregate_streaming_centralstore_triangles_command(connFd, incrementalLocalStoreMap, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::AGGREGATE_COMPOSITE_CENTRALSTORE_TRIANGLES) == 0) { aggregate_composite_centralstore_triangles_command(connFd, &loop_exit); - } else if (line.compare(JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS) == 0) { - performance_statistics_command(connFd, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::INITIATE_FILES) == 0) { initiate_files_command(connFd, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::INITIATE_FED_PREDICT) == 0) { @@ -671,32 +668,6 @@ map JasmineGraphInstanceService::getOutDegreeDistributionHashMap(map return distributionHashMap; } -string JasmineGraphInstanceService::requestPerformanceStatistics(std::string isVMStatManager, - std::string isResourceAllocationRequested) { - long memoryUsage = StatisticCollector::getMemoryUsageByProcess(); - int threadCount = StatisticCollector::getThreadCount(); - long usedSwapSpace = StatisticCollector::getUsedSwapSpace(); - long totalSwapSpace = StatisticCollector::getTotalSwapSpace(); - long readBytes = StatisticCollector::getRXBytes(); - long sentBytes = StatisticCollector::getTXBytes(); - double cpuUsage = StatisticCollector::getCpuUsage(); - double loadAverage = StatisticCollector::getLoadAverage(); - std::string vmLevelStatistics = - StatisticCollector::collectVMStatistics(isVMStatManager, isResourceAllocationRequested); - auto executedTime = std::chrono::system_clock::now(); - std::time_t reportTime = std::chrono::system_clock::to_time_t(executedTime); - std::string reportTimeString(std::ctime(&reportTime)); - reportTimeString = Utils::trim_copy(reportTimeString); - std::string usageString = reportTimeString + "," + to_string(memoryUsage) + "," + to_string(threadCount) + "," + - to_string(cpuUsage) + "," + to_string(loadAverage) + "," + to_string(usedSwapSpace) + - "," + to_string(totalSwapSpace) + +"," + to_string(readBytes) + "," + - to_string(sentBytes); - if (!vmLevelStatistics.empty()) { - usageString = usageString + "," + vmLevelStatistics; - } - return usageString; -} - void JasmineGraphInstanceService::collectTrainedModels( instanceservicesessionargs *sessionargs, std::string graphID, std::map &graphPartitionedHosts, int totalPartitions) { @@ -1160,7 +1131,6 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int break; } else { instance_logger.error("Incorrect response. Received: " + response); - close(sockfd); goto END_OUTER_LOOP; } }; @@ -1181,7 +1151,6 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int break; } else { instance_logger.error("Incorrect response. Received: " + response); - close(sockfd); goto END_OUTER_LOOP; } } @@ -1816,16 +1785,17 @@ map> getEdgesWorldToLocal(string graphID, string parti void JasmineGraphInstanceService::startCollectingLoadAverage() { int elapsedTime = 0; time_t start; - time_t end; start = time(0); - while (collectValid) { - if (time(0) - start == Conts::LOAD_AVG_COLLECTING_GAP) { + time_t elapsed = time(0) - start; + if (elapsed >= Conts::LOAD_AVG_COLLECTING_GAP) { elapsedTime += Conts::LOAD_AVG_COLLECTING_GAP * 1000; double loadAgerage = StatisticCollector::getLoadAverage(); loadAverageVector.push_back(std::to_string(loadAgerage)); start = start + Conts::LOAD_AVG_COLLECTING_GAP; + } else { + sleep(Conts::LOAD_AVG_COLLECTING_GAP - elapsed); } } } @@ -3458,32 +3428,6 @@ static void aggregate_composite_centralstore_triangles_command(int connFd, bool } } -static void performance_statistics_command(int connFd, bool *loop_exit_p) { - if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { - *loop_exit_p = true; - return; - } - instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); - - char data[DATA_BUFFER_SIZE]; - string isVMStatManager = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); - - if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { - *loop_exit_p = true; - return; - } - instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); - - string isResourceAllocationRequired = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); - - std::string memoryUsage = - JasmineGraphInstanceService::requestPerformanceStatistics(isVMStatManager, isResourceAllocationRequired); - if (!Utils::send_str_wrapper(connFd, memoryUsage)) { - *loop_exit_p = true; - return; - } -} - static void initiate_files_command(int connFd, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -3573,9 +3517,7 @@ static void request_collected_stats_command(int connFd, bool *collectValid_p, bo collectValid = false; std::string loadAverageString; - std::vector::iterator loadVectorIterator; - - for (loadVectorIterator = loadAverageVector.begin(); loadVectorIterator != loadAverageVector.end(); + for (auto loadVectorIterator = loadAverageVector.begin(); loadVectorIterator != loadAverageVector.end(); ++loadVectorIterator) { std::string tempLoadAverage = *loadVectorIterator; loadAverageString = loadAverageString + "," + tempLoadAverage; @@ -3585,7 +3527,6 @@ static void request_collected_stats_command(int connFd, bool *collectValid_p, bo loadAverageString = loadAverageString.substr(1, loadAverageString.length() - 1); std::vector chunksVector; - for (unsigned i = 0; i < loadAverageString.length(); i += CHUNK_OFFSET) { std::string chunk = loadAverageString.substr(i, CHUNK_OFFSET); if (i + CHUNK_OFFSET < loadAverageString.length()) { @@ -3598,13 +3539,11 @@ static void request_collected_stats_command(int connFd, bool *collectValid_p, bo for (int loopCount = 0; loopCount < chunksVector.size(); loopCount++) { std::string chunk; - if (loopCount == 0) { - chunk = chunksVector.at(loopCount); - } else { + if (loopCount > 0) { char data[DATA_BUFFER_SIZE]; string chunkStatus = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, false); - chunk = chunksVector.at(loopCount); } + chunk = chunksVector.at(loopCount); if (!Utils::send_str_wrapper(connFd, chunk)) { *loop_exit_p = true; return; @@ -3757,7 +3696,7 @@ static void initiate_predict_command(int connFd, instanceservicesessionargs *ses Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + fileName; int fileSize = stoi(size); string line; - while (Utils::fileExists(fullFilePath) && Utils::getFileSize(fullFilePath) < fileSize) { + while (!Utils::fileExists(fullFilePath) || Utils::getFileSize(fullFilePath) < fileSize) { line = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, false); if (line.compare(JasmineGraphInstanceProtocol::FILE_RECV_CHK) == 0) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::FILE_RECV_WAIT)) { diff --git a/src/server/JasmineGraphInstanceService.h b/src/server/JasmineGraphInstanceService.h index d607c48d7..92304c914 100644 --- a/src/server/JasmineGraphInstanceService.h +++ b/src/server/JasmineGraphInstanceService.h @@ -142,7 +142,6 @@ class JasmineGraphInstanceService { static string aggregateCompositeCentralStoreTriangles(std::string compositeFileList, std::string availableFileList, int threadPriority); static map getOutDegreeDistributionHashMap(map> &graphMap); - static string requestPerformanceStatistics(std::string isVMStatManager, std::string isResourceAllocationRequested); struct workerPartitions { int port; diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index a194428da..abb1bd556 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -1912,12 +1912,7 @@ bool JasmineGraphServer::receiveGlobalWeights(std::string host, int port, std::s serv_addr.sin_family = AF_INET; bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - - while (true) { - if (!(Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0)) { - break; - } - } + Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)); bool isTrue = false; int count = 0; diff --git a/src/util/Conts.h b/src/util/Conts.h index a683b9901..cab119e0f 100644 --- a/src/util/Conts.h +++ b/src/util/Conts.h @@ -31,7 +31,6 @@ extern std::mutex responseVectorMutex; extern bool isStatCollect; extern bool isCalibrating; extern std::vector loadAverageVector; -extern bool collectValid; extern std::map aggregateWeightMap; extern std::mutex aggregateWeightMutex; extern std::mutex triangleTreeMutex; From e8e0c3e8ea9ba2be2af1b8c47c7d2b855dff7be9 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 1 Mar 2024 02:44:13 +0530 Subject: [PATCH 050/143] Fix: no return statement in getWorkerClients() --- src/frontend/JasmineGraphFrontEnd.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index 55423cbbe..f9b85d8e2 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -105,6 +105,7 @@ static vector getWorkerClients(SQLiteDBInterface *sqlite) { DataPublisher *workerClient = new DataPublisher(workerPort, workerHost); workerClients.push_back(workerClient); } + return workerClients; } void *frontendservicesesion(void *dummyPt) { From 283d122337012d4f0ef731aa7763b31d0e8b5fc8 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 1 Mar 2024 12:27:25 +0530 Subject: [PATCH 051/143] Remove successful curl logs --- src/util/Utils.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index fe59263f3..9ae5a43b6 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -770,12 +770,9 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, res = curl_easy_perform(curl); long code = -1; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - if (res != CURLE_OK) { + if (res != CURLE_OK || code != 200) { util_logger.error("curl failed: " + std::string(curl_easy_strerror(res)) + "| url: " + hostPGAddr + "| data: " + job_data); - } else { - util_logger.info("curl success | url: " + hostPGAddr + "| data: " + job_data + - "| status: " + to_string(code)); } curl_easy_cleanup(curl); From d629f4b7d92d4994b30b8ace16979f7c8595d94b Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 2 Mar 2024 02:46:25 +0530 Subject: [PATCH 052/143] Get profile from global variable and use worker info from K8sWorkerController --- globals.h | 5 +- main.cpp | 11 +- src/frontend/JasmineGraphFrontEnd.cpp | 4 +- .../impl/StreamingTriangleCountExecutor.cpp | 2 +- .../executor/impl/TriangleCountExecutor.cpp | 35 +++--- .../executor/impl/TriangleCountExecutor.h | 6 - src/server/JasmineGraphInstance.cpp | 5 +- src/server/JasmineGraphInstance.h | 4 +- src/server/JasmineGraphInstanceService.cpp | 4 +- src/server/JasmineGraphInstanceService.h | 3 +- src/server/JasmineGraphServer.cpp | 108 +++++++++++------- src/server/JasmineGraphServer.h | 9 +- src/util/Conts.cpp | 4 - src/util/Conts.h | 4 - src/util/Utils.cpp | 6 +- 15 files changed, 108 insertions(+), 102 deletions(-) diff --git a/globals.h b/globals.h index 736971df1..6adda481a 100644 --- a/globals.h +++ b/globals.h @@ -16,6 +16,9 @@ limitations under the License. #ifndef JASMINEGRAPH__GLOBALS_H_ #define JASMINEGRAPH__GLOBALS_H_ -extern char *jasminegraph_profile; +#define PROFILE_NATIVE 0 +#define PROFILE_DOCKER 1 +#define PROFILE_K8S 2 +extern int jasminegraph_profile; #endif // JASMINEGRAPH__GLOBALS_H_ diff --git a/main.cpp b/main.cpp index 59b8c653c..a81851c35 100644 --- a/main.cpp +++ b/main.cpp @@ -31,7 +31,7 @@ SchedulerService schedulerService; Logger main_logger; #ifndef UNIT_TEST -char *jasminegraph_profile = NULL; +int jasminegraph_profile = PROFILE_DOCKER; #endif void fnExit3(void) { @@ -53,8 +53,7 @@ int main(int argc, char *argv[]) { int mode = atoi(argv[2]); std::string JASMINEGRAPH_HOME = Utils::getJasmineGraphHome(); - std::string profile = argv[1]; - jasminegraph_profile = argv[1]; + jasminegraph_profile = strcmp(argv[1], "docker") == 0 ? PROFILE_DOCKER : PROFILE_K8S; std::string enableNmon = "false"; main_logger.log("Using JASMINE_GRAPH_HOME", "info"); @@ -70,14 +69,14 @@ int main(int argc, char *argv[]) { enableNmon = argv[6]; server = JasmineGraphServer::getInstance(); - if (profile == Conts::PROFILE_K8S) { + if (jasminegraph_profile == PROFILE_K8S) { K8sInterface *interface = new K8sInterface(); masterIp = interface->getMasterIp(); if (masterIp.empty()) { masterIp = interface->createJasmineGraphMasterService()->spec->cluster_ip; } } - server->run(profile, masterIp, numberOfWorkers, workerIps, enableNmon); + server->run(masterIp, numberOfWorkers, workerIps, enableNmon); schedulerThread.join(); delete server; @@ -102,7 +101,7 @@ int main(int argc, char *argv[]) { std::cout << "In worker mode" << std::endl; instance = new JasmineGraphInstance(); - instance->start_running(profile, hostName, masterHost, serverPort, serverDataPort, enableNmon); + instance->start_running(hostName, masterHost, serverPort, serverDataPort, enableNmon); delete instance; } diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index f9b85d8e2..6396520be 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -2341,7 +2341,6 @@ static void start_remote_worker_command(int connFd, bool *loop_exit_p) { string host = ""; string port = ""; string dataPort = ""; - string profile = ""; string masterHost = ""; string enableNmon = ""; @@ -2360,11 +2359,10 @@ static void start_remote_worker_command(int connFd, bool *loop_exit_p) { host = strArr[0]; port = strArr[1]; dataPort = strArr[2]; - profile = strArr[3]; masterHost = strArr[4]; enableNmon = strArr[5]; - JasmineGraphServer::spawnNewWorker(host, port, dataPort, profile, masterHost, enableNmon); + JasmineGraphServer::spawnNewWorker(host, port, dataPort, masterHost, enableNmon); } static void sla_command(int connFd, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfSqlite, diff --git a/src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp b/src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp index be5dc9bd4..851aac8a9 100644 --- a/src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp @@ -573,7 +573,7 @@ std::vector> StreamingTriangleCountExecutor::getWorkerCombin std::vector workerIdVector(workerIdSet.begin(), workerIdSet.end()); - std::vector> workerIdCombination = getCombinations(workerIdVector); + std::vector> workerIdCombination = AbstractExecutor::getCombinations(workerIdVector); return workerIdCombination; } diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 1c056b719..02bd0e20f 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -25,6 +25,11 @@ std::mutex fileCombinationMutex; std::mutex processStatusMutex; std::mutex responseVectorMutex; +static std::vector> getWorkerCombination(SQLiteDBInterface *sqlite, std::string graphId); +static string isFileAccessibleToWorker(std::string graphId, std::string partitionId, std::string aggregatorHostName, + std::string aggregatorPort, std::string masterIP, std::string fileType, + std::string fileName); + TriangleCountExecutor::TriangleCountExecutor() {} TriangleCountExecutor::TriangleCountExecutor(SQLiteDBInterface *db, PerformanceSQLiteDBInterface *perfDb, @@ -122,7 +127,7 @@ void TriangleCountExecutor::execute() { compositeCentralStoreFiles.push_back(graphFileName); } } - fileCombinations = getCombinations(compositeCentralStoreFiles); + fileCombinations = AbstractExecutor::getCombinations(compositeCentralStoreFiles); } std::map> partitionMap; @@ -644,9 +649,9 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq "WHERE partition_graph_idgraph=" + graphId + " and idworker=" + minWeightWorker + ";"; - std::vector>> result = sqlite->runSelect(aggregatorSqlStatement); + const std::vector>> &result = sqlite->runSelect(aggregatorSqlStatement); - vector> aggregatorData = result.at(0); + const vector> &aggregatorData = result.at(0); std::string aggregatorIp = aggregatorData.at(0).second; std::string aggregatorPort = aggregatorData.at(1).second; @@ -665,9 +670,9 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq "WHERE partition_graph_idgraph=" + graphId + " and idworker=" + workerId + ";"; - std::vector>> result = sqlite->runSelect(sqlStatement); + const std::vector>> &result = sqlite->runSelect(sqlStatement); - vector> workerData = result.at(0); + const vector> &workerData = result.at(0); std::string partitionId = workerData.at(0).second; @@ -719,10 +724,9 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq return aggregatedTriangleCount; } -string TriangleCountExecutor::isFileAccessibleToWorker(std::string graphId, std::string partitionId, - std::string aggregatorHostName, std::string aggregatorPort, - std::string masterIP, std::string fileType, - std::string fileName) { +static string isFileAccessibleToWorker(std::string graphId, std::string partitionId, std::string aggregatorHostName, + std::string aggregatorPort, std::string masterIP, std::string fileType, + std::string fileName) { int sockfd; char data[301]; bool loop = false; @@ -1224,8 +1228,7 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag return response; } -std::vector> TriangleCountExecutor::getWorkerCombination(SQLiteDBInterface *sqlite, - std::string graphId) { +static std::vector> getWorkerCombination(SQLiteDBInterface *sqlite, std::string graphId) { std::set workerIdSet; string sqlStatement = @@ -1234,19 +1237,17 @@ std::vector> TriangleCountExecutor::getWorkerCombination(SQL "WHERE partition_graph_idgraph=" + graphId + ";"; - std::vector>> results = sqlite->runSelect(sqlStatement); - - for (std::vector>>::iterator i = results.begin(); i != results.end(); ++i) { - std::vector> rowData = *i; + const std::vector>> &results = sqlite->runSelect(sqlStatement); + for (auto i = results.begin(); i != results.end(); ++i) { + const std::vector> &rowData = *i; string workerId = rowData.at(0).second; - workerIdSet.insert(workerId); } std::vector workerIdVector(workerIdSet.begin(), workerIdSet.end()); - std::vector> workerIdCombination = getCombinations(workerIdVector); + std::vector> workerIdCombination = AbstractExecutor::getCombinations(workerIdVector); return workerIdCombination; } diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.h b/src/frontend/core/executor/impl/TriangleCountExecutor.h index 072d0775a..a9ac28a03 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.h +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.h @@ -41,10 +41,6 @@ class TriangleCountExecutor : public AbstractExecutor { static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, int threadPriority); - static string isFileAccessibleToWorker(std::string graphId, std::string partitionId, std::string aggregatorHostName, - std::string aggregatorPort, std::string masterIP, std::string fileType, - std::string fileName); - static std::string copyCompositeCentralStoreToAggregator(std::string aggregatorHostName, std::string aggregatorPort, std::string aggregatorDataPort, std::string fileName, std::string masterIP); @@ -53,8 +49,6 @@ class TriangleCountExecutor : public AbstractExecutor { std::string compositeCentralStoreFileList, std::string masterIP, std::string availableFileList, int threadPriority); - static std::vector> getWorkerCombination(SQLiteDBInterface *sqlite, std::string graphId); - static std::string copyCentralStoreToAggregator(std::string aggregatorHostName, std::string aggregatorPort, std::string aggregatorDataPort, int graphId, int partitionId, std::string masterIP); diff --git a/src/server/JasmineGraphInstance.cpp b/src/server/JasmineGraphInstance.cpp index 8fbc08bd2..10846f280 100644 --- a/src/server/JasmineGraphInstance.cpp +++ b/src/server/JasmineGraphInstance.cpp @@ -21,7 +21,7 @@ Logger graphInstance_logger; void *runInstanceService(void *dummyPt) { JasmineGraphInstance *refToInstance = (JasmineGraphInstance *)dummyPt; refToInstance->instanceService = new JasmineGraphInstanceService(); - refToInstance->instanceService->run(refToInstance->profile, refToInstance->masterHostName, refToInstance->hostName, + refToInstance->instanceService->run(refToInstance->masterHostName, refToInstance->hostName, refToInstance->serverPort, refToInstance->serverDataPort); return NULL; } @@ -33,12 +33,11 @@ void *runFileTransferService(void *dummyPt) { return NULL; } -int JasmineGraphInstance::start_running(string profile, string hostName, string masterHost, int serverPort, +int JasmineGraphInstance::start_running(string hostName, string masterHost, int serverPort, int serverDataPort, string enableNmon) { graphInstance_logger.info("Worker started"); this->hostName = hostName; - this->profile = profile; this->masterHostName = masterHost; this->serverPort = serverPort; this->serverDataPort = serverDataPort; diff --git a/src/server/JasmineGraphInstance.h b/src/server/JasmineGraphInstance.h index cfd6354c1..1343dc24d 100644 --- a/src/server/JasmineGraphInstance.h +++ b/src/server/JasmineGraphInstance.h @@ -31,8 +31,7 @@ class JasmineGraphInstance { static const int BUFFER_SIZE = 128; public: - int start_running(string profile, string hostName, string masterHost, int serverPort, int serverDataPort, - string enableNmon); + int start_running(string hostName, string masterHost, int serverPort, int serverDataPort, string enableNmon); bool acknowledgeMaster(string masterHost, string workerIP, string workerPort); @@ -45,7 +44,6 @@ class JasmineGraphInstance { void shutdown(); string hostName; - string profile; string masterHostName; int serverPort; int serverDataPort; diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 8c43a1e01..5738c359e 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -145,7 +145,6 @@ void *instanceservicesession(void *dummyPt) { string serverName = sessionargs.host; string masterHost = sessionargs.masterHost; - string profile = sessionargs.profile; int serverPort = sessionargs.port; int serverDataPort = sessionargs.dataPort; @@ -267,7 +266,7 @@ void *instanceservicesession(void *dummyPt) { JasmineGraphInstanceService::JasmineGraphInstanceService() {} -void JasmineGraphInstanceService::run(string profile, string masterHost, string host, int serverPort, +void JasmineGraphInstanceService::run(string masterHost, string host, int serverPort, int serverDataPort) { int listenFd; socklen_t len; @@ -324,7 +323,6 @@ void JasmineGraphInstanceService::run(string profile, string masterHost, string serviceArguments_p->graphDBMapCentralStores = graphDBMapCentralStores; serviceArguments_p->graphDBMapDuplicateCentralStores = graphDBMapDuplicateCentralStores; serviceArguments_p->incrementalLocalStore = incrementalLocalStore; - serviceArguments_p->profile = profile; serviceArguments_p->masterHost = masterHost; serviceArguments_p->port = serverPort; serviceArguments_p->dataPort = serverDataPort; diff --git a/src/server/JasmineGraphInstanceService.h b/src/server/JasmineGraphInstanceService.h index 92304c914..95f9cbfcd 100644 --- a/src/server/JasmineGraphInstanceService.h +++ b/src/server/JasmineGraphInstanceService.h @@ -102,7 +102,6 @@ map> getEdgesWorldToLocal(string graphID, string parti std::vector &workerSockets); struct instanceservicesessionargs { - string profile; string masterHost; string host; int connFd; @@ -120,7 +119,7 @@ class JasmineGraphInstanceService { static const string END_OF_MESSAGE; JasmineGraphInstanceService(); - void run(string profile, string masterHost, string hostName, int serverPort, int serverDataPort); + void run(string masterHost, string hostName, int serverPort, int serverDataPort); static bool isGraphDBExists(std::string graphId, std::string partitionId); static bool isInstanceCentralStoreExists(std::string graphId, std::string partitionId); diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index abb1bd556..819c0a279 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -21,6 +21,7 @@ limitations under the License. #include #include +#include "../../globals.h" #include "../k8s/K8sWorkerController.h" #include "../ml/trainer/JasmineGraphTrainingSchedular.h" #include "../partitioner/local/MetisPartitioner.h" @@ -64,9 +65,12 @@ static bool initiateAggregator(std::string host, int port, int dataPort, std::st static bool initiateOrgServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, std::string masterIP); static void degreeDistributionCommon(std::string graphID, std::string command); +static int getPortByHost(const std::string &host); +static int getDataPortByHost(const std::string &host); +static size_t getWorkerCount(); static std::vector hostWorkerList; -static map> hostPortMap; +static unordered_map> hostPortMap; std::map aggregateWeightMap; void *runfrontend(void *dummyPt) { @@ -113,8 +117,7 @@ JasmineGraphServer::~JasmineGraphServer() { delete this->jobScheduler; } -int JasmineGraphServer::run(std::string profile, std::string masterIp, int numberofWorkers, std::string workerIps, - std::string enableNmon) { +int JasmineGraphServer::run(std::string masterIp, int numberofWorkers, std::string workerIps, std::string enableNmon) { server_logger.info("Running the server..."); std::vector masterPortVector; if (masterIp.empty()) { @@ -123,7 +126,6 @@ int JasmineGraphServer::run(std::string profile, std::string masterIp, int numbe this->masterHost = masterIp; } server_logger.info("masterHost = " + this->masterHost); - this->profile = profile; this->numberOfWorkers = numberofWorkers; this->workerHosts = workerIps; this->enableNmon = enableNmon; @@ -131,7 +133,7 @@ int JasmineGraphServer::run(std::string profile, std::string masterIp, int numbe masterPortVector.push_back(Conts::JASMINEGRAPH_FRONTEND_PORT); updateOperationalGraphList(); - if (profile == Conts::PROFILE_K8S) { + if (jasminegraph_profile == PROFILE_K8S) { // Create K8s worker controller (void)K8sWorkerController::getInstance(masterIp, numberofWorkers, sqlite); hostWorkerList = K8sWorkerController::workerList; @@ -142,7 +144,7 @@ int JasmineGraphServer::run(std::string profile, std::string masterIp, int numbe std::thread *myThreads = new std::thread[1]; myThreads[0] = std::thread(StatisticCollector::logLoadAverage, "Load Average"); - sleep(2); + sleep(1); waitForAcknowledgement(numberofWorkers); resolveOperationalGraphs(); initiateAggregateMap(); @@ -161,13 +163,13 @@ void JasmineGraphServer::start_workers() { int numberOfWorkersPerHost; std::vector hostsList; std::string nWorkers; - if (profile == Conts::PROFILE_NATIVE) { + if (jasminegraph_profile == PROFILE_NATIVE) { hostsList = Utils::getHostListFromProperties(); if ((this->numberOfWorkers) == -1) { nWorkers = Utils::getJasmineGraphProperty("org.jasminegraph.server.nworkers"); } enableNmon = Utils::getJasmineGraphProperty("org.jasminegraph.server.enable.nmon"); - } else if (profile == Conts::PROFILE_DOCKER) { + } else if (jasminegraph_profile == PROFILE_DOCKER) { hostsList = getWorkerVector(workerHosts); } @@ -304,7 +306,7 @@ void JasmineGraphServer::start_workers() { addHostsToMetaDB(host, workerPortsMap[host], workerDataPortsMap[host]); addInstanceDetailsToPerformanceDB(host, workerPortsMap[host], "false"); myThreads[count] = std::thread(startRemoteWorkers, workerPortsMap[host], workerDataPortsMap[host], host, - profile, masterHost, enableNmon); + masterHost, enableNmon); count++; } @@ -314,6 +316,27 @@ void JasmineGraphServer::start_workers() { delete[] myThreads; } +static int getPortByHost(const std::string &host) { + if (jasminegraph_profile == PROFILE_K8S) { + return Conts::JASMINEGRAPH_INSTANCE_PORT; + } + return hostPortMap[host].first; +} + +static int getDataPortByHost(const std::string &host) { + if (jasminegraph_profile == PROFILE_K8S) { + return Conts::JASMINEGRAPH_INSTANCE_DATA_PORT; + } + return hostPortMap[host].second; +} + +static size_t getWorkerCount() { + if (jasminegraph_profile == PROFILE_K8S) { + return K8sWorkerController::workerList.size(); + } + return hostWorkerList.size(); +} + void JasmineGraphServer::waitForAcknowledgement(int numberOfWorkers) { auto begin = chrono::high_resolution_clock::now(); int timeDifference = 0; @@ -333,7 +356,7 @@ void JasmineGraphServer::waitForAcknowledgement(int numberOfWorkers) { } void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, std::vector workerDataPortsVector, - string host, string profile, string masterHost, string enableNmon) { + string host, string masterHost, string enableNmon) { std::string executableFile; std::string workerPath = Utils::getJasmineGraphProperty("org.jasminegraph.worker.path"); std::string artifactPath = Utils::getJasmineGraphProperty("org.jasminegraph.artifact.path"); @@ -347,7 +370,7 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, std::string instanceFolderLocal = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.local"); std::string jasmineGraphExecutableName = Conts::JASMINEGRAPH_EXECUTABLE; - server_logger.info("###MASTER#### Starting remote workers for profile " + profile); + server_logger.info("###MASTER#### Starting remote workers"); if (hasEnding(workerPath, "/")) { executableFile = workerPath + jasmineGraphExecutableName; } else { @@ -361,7 +384,7 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, artifactPath = Utils::getJasmineGraphHome(); } - if (profile == Conts::PROFILE_NATIVE) { + if (jasminegraph_profile == PROFILE_NATIVE) { copyArtifactsToWorkers(workerPath, artifactPath, host); for (int i = 0; i < workerPortsVector.size(); i++) { if (host.find("localhost") != std::string::npos) { @@ -380,7 +403,7 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, _exit(1); } } - } else if (profile == Conts::PROFILE_DOCKER) { + } else if (jasminegraph_profile == PROFILE_DOCKER) { char *env_testing = getenv("TESTING"); bool is_testing = (env_testing != nullptr && strcasecmp(env_testing, "true") == 0); for (int i = 0; i < workerPortsVector.size(); i++) { @@ -448,7 +471,7 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, } } -bool JasmineGraphServer::spawnNewWorker(string host, string port, string dataPort, string profile, string masterHost, +bool JasmineGraphServer::spawnNewWorker(string host, string port, string dataPort, string masterHost, string enableNmon) { auto *refToSqlite = new SQLiteDBInterface(); refToSqlite->init(); @@ -501,8 +524,7 @@ bool JasmineGraphServer::spawnNewWorker(string host, string port, string dataPor workerPortsVector.push_back(atoi(port.c_str())); workerDataPortsVector.push_back(atoi(dataPort.c_str())); - JasmineGraphServer::startRemoteWorkers(workerPortsVector, workerDataPortsVector, host, profile, masterHost, - enableNmon); + JasmineGraphServer::startRemoteWorkers(workerPortsVector, workerDataPortsVector, host, masterHost, enableNmon); return true; } @@ -688,32 +710,38 @@ void JasmineGraphServer::deleteNonOperationalGraphFragment(int graphID) { server_logger.info("Deleting non-operational fragment " + to_string(graphID)); int count = 0; // Define threads for each host - std::thread *deleteThreads = new std::thread[hostPortMap.size()]; + std::thread *deleteThreads = new std::thread[getWorkerCount()]; + std::vector *workerListAll; + if (jasminegraph_profile == PROFILE_K8S) { + workerListAll = &(K8sWorkerController::workerList); + } else { + workerListAll = &hostWorkerList; + } // Iterate through all hosts - for (std::map>::iterator it = hostPortMap.begin(); it != hostPortMap.end(); it++) { + for (auto it = (*workerListAll).begin(); it != (*workerListAll).end(); it++) { // Fetch hostname and port - string hostname = it->first; - int port = (it->second).first; + const string &hostname = it->hostname; + int port = it->port; // Initialize threads for host // Each thread runs the service to remove the given graph ID fragments in their datafolders deleteThreads[count++] = std::thread(removeFragmentThroughService, hostname, port, to_string(graphID), this->masterHost); - sleep(1); - server_logger.info("Deleted graph fragments of graph ID " + to_string(graphID)); } + sleep(1); for (int threadCount = 0; threadCount < count; threadCount++) { if (deleteThreads[threadCount].joinable()) { deleteThreads[threadCount].join(); } } + server_logger.info("Deleted graph fragments of graph ID " + to_string(graphID)); } void JasmineGraphServer::shutdown_workers() { server_logger.info("Shutting down workers"); auto *server = JasmineGraphServer::getInstance(); - if (server->profile == Conts::PROFILE_K8S) { + if (jasminegraph_profile == PROFILE_K8S) { K8sWorkerController::getInstance()->setNumberOfWorkers(0); return; } @@ -797,9 +825,9 @@ static map scaleK8s(size_t npart) { return cpu_loads; } -static std::vector getWorkers(size_t npart, std::string profile) { +static std::vector getWorkers(size_t npart) { if (Utils::getJasmineGraphProperty("org.jasminegraph.autoscale.enabled") != "true") { - if (profile == Conts::PROFILE_K8S) { + if (jasminegraph_profile == PROFILE_K8S) { return K8sWorkerController::workerList; } return hostWorkerList; @@ -807,11 +835,11 @@ static std::vector getWorkers(size_t npart, std::str // TODO: get the workers with lowest load from workerList std::vector *workerListAll; map cpu_loads; - if (profile == Conts::PROFILE_K8S) { + if (jasminegraph_profile == PROFILE_K8S) { workerListAll = &(K8sWorkerController::workerList); cpu_loads = scaleK8s(npart); } else { - workerListAll = &(hostWorkerList); + workerListAll = &hostWorkerList; for (auto it = hostWorkerList.begin(); it != hostWorkerList.end(); it++) { auto &worker = *it; string workerHostPort = worker.hostname + ":" + to_string(worker.port); @@ -865,7 +893,7 @@ void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, int file_count = 0; std::thread *workerThreads = new std::thread[total_threads]; while (count < total_threads) { - const auto &workerList = getWorkers(partitionFileMap.size(), this->profile); + const auto &workerList = getWorkers(partitionFileMap.size()); for (auto listIterator = workerList.begin(); listIterator < workerList.end(); listIterator++) { worker worker = *listIterator; if (count >= total_threads) { @@ -1096,8 +1124,8 @@ void JasmineGraphServer::removeGraph(vector> hostHasPartiti int count = 0; std::thread *deleteThreads = new std::thread[hostHasPartition.size()]; for (std::vector>::iterator j = (hostHasPartition.begin()); j != hostHasPartition.end(); ++j) { - deleteThreads[count] = std::thread(removePartitionThroughService, j->first, hostPortMap[j->first].first, - graphID, j->second, masterIP); + deleteThreads[count] = + std::thread(removePartitionThroughService, j->first, getPortByHost(j->first), graphID, j->second, masterIP); count++; } @@ -1235,9 +1263,9 @@ void JasmineGraphServer::updateOperationalGraphList() { string graphIDs = ""; std::vector hostsList; - if (profile == Conts::PROFILE_NATIVE) { + if (jasminegraph_profile == PROFILE_NATIVE) { hostsList = Utils::getHostListFromProperties(); - } else if (profile == "docker") { + } else if (jasminegraph_profile == PROFILE_DOCKER) { hostsList = getWorkerVector(workerHosts); } vector::iterator it; @@ -1274,13 +1302,12 @@ std::map JasmineGraphServer::getGr auto *refToSqlite = new SQLiteDBInterface(); refToSqlite->init(); vector>> hostPartitionResults = refToSqlite->runSelect( - "SELECT name, partition_idpartition FROM worker_has_partition INNER JOIN worker ON worker_" - "idworker = idworker WHERE partition_graph_idgraph = '" + + "SELECT name, partition_idpartition FROM worker_has_partition INNER JOIN worker ON " + "worker_idworker = idworker WHERE partition_graph_idgraph = '" + graphID + "'"); refToSqlite->finalize(); delete refToSqlite; - for (vector>>::iterator i = hostPartitionResults.begin(); - i != hostPartitionResults.end(); ++i) { + for (auto i = hostPartitionResults.begin(); i != hostPartitionResults.end(); ++i) { int count = 0; string hostname; string partitionID; @@ -1296,10 +1323,11 @@ std::map JasmineGraphServer::getGr } map> hostPartitions; - for (std::vector>::iterator j = (hostHasPartition.begin()); j != hostHasPartition.end(); ++j) { + for (auto j = hostHasPartition.begin(); j != hostHasPartition.end(); ++j) { string hostname = j->first; - if (hostPartitions.count(hostname) > 0) { - hostPartitions[hostname].push_back(j->second); + auto it = hostPartitions.find(hostname); + if (it != hostPartitions.end()) { + it->second.push_back(j->second); } else { vector vec; vec.push_back(j->second); @@ -1309,7 +1337,7 @@ std::map JasmineGraphServer::getGr map graphPartitionedHosts; for (map>::iterator it = hostPartitions.begin(); it != hostPartitions.end(); ++it) { - graphPartitionedHosts[it->first] = {hostPortMap[it->first].first, hostPortMap[it->first].second, + graphPartitionedHosts[it->first] = {getPortByHost(it->first), getDataPortByHost(it->first), hostPartitions[it->first]}; } diff --git a/src/server/JasmineGraphServer.h b/src/server/JasmineGraphServer.h index e925eda92..f624b55b3 100644 --- a/src/server/JasmineGraphServer.h +++ b/src/server/JasmineGraphServer.h @@ -37,7 +37,6 @@ using std::map; class JasmineGraphServer { private: map hostPlaceMap; - std::string profile; std::string workerHosts; std::string enableNmon; static const int BUFFER_SIZE = 128; @@ -49,7 +48,7 @@ class JasmineGraphServer { JasmineGraphServer(); static void startRemoteWorkers(std::vector workerPortsVector, std::vector workerDataPortsVector, - std::string host, string profile, string masterHost, string enableNmon); + std::string host, string masterHost, string enableNmon); void addHostsToMetaDB(std::string host, std::vector portVector, std::vector dataPortVector); @@ -84,8 +83,7 @@ class JasmineGraphServer { static int shutdown_worker(std::string host, int port); - int run(std::string profile, std::string masterIp, int numberofWorkers, std::string workerIps, - std::string enableNmon); + int run(std::string masterIp, int numberofWorkers, std::string workerIps, std::string enableNmon); void uploadGraphLocally(int graphID, const std::string graphType, std::vector> fullFileList, std::string masterIP); @@ -97,8 +95,7 @@ class JasmineGraphServer { static void copyCentralStoreToAggregateLocation(std::string filePath); - static bool spawnNewWorker(string host, string port, string dataPort, string profile, string masterHost, - string enableNmon); + static bool spawnNewWorker(string host, string port, string dataPort, string masterHost, string enableNmon); JasmineGraphFrontEnd *frontend; SQLiteDBInterface *sqlite; diff --git a/src/util/Conts.cpp b/src/util/Conts.cpp index 4b6b37067..2e3730325 100644 --- a/src/util/Conts.cpp +++ b/src/util/Conts.cpp @@ -16,10 +16,6 @@ limitations under the License. std::string Conts::JASMINEGRAPH_EXECUTABLE = "run.sh"; std::string Conts::JASMINEGRAPH_HOME = "JASMINEGRAPH_HOME"; -std::string Conts::PROFILE_NATIVE = "native"; -std::string Conts::PROFILE_DOCKER = "docker"; -std::string Conts::PROFILE_K8S = "k8s"; - std::string Conts::CARRIAGE_RETURN_NEW_LINE = "\r\n"; std::string Conts::GRAPH_TYPE_RDF = "RDF_GRAPH"; diff --git a/src/util/Conts.h b/src/util/Conts.h index cab119e0f..d6cd989b4 100644 --- a/src/util/Conts.h +++ b/src/util/Conts.h @@ -69,10 +69,6 @@ class Conts { static std::string GRAPH_WITH_JSON_ATTRIBUTES; static std::string GRAPH_WITH_XML_ATTRIBUTES; - static std::string PROFILE_NATIVE; - static std::string PROFILE_DOCKER; - static std::string PROFILE_K8S; - static std::string CARRIAGE_RETURN_NEW_LINE; static std::string diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 9ae5a43b6..ff70db9a3 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -38,7 +38,7 @@ using namespace std; Logger util_logger; #ifdef UNIT_TEST -char *jasminegraph_profile = NULL; +int jasminegraph_profile = PROFILE_K8S; #endif unordered_map Utils::propertiesMap; @@ -718,7 +718,7 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, CURL *curl; CURLcode res; std::string pushGatewayJobAddr; - if (string(jasminegraph_profile) == Conts::PROFILE_K8S) { + if (jasminegraph_profile == PROFILE_K8S) { std::unique_ptr interface(new K8sInterface()); pushGatewayJobAddr = interface->getJasmineGraphConfig("pushgateway_address"); } else { @@ -787,7 +787,7 @@ std::map Utils::getMetricMap(std::string metricName) { std::string response_cpu_usages; std::string prometheusAddr; - if (string(jasminegraph_profile) == Conts::PROFILE_K8S) { + if (jasminegraph_profile == PROFILE_K8S) { std::unique_ptr interface(new K8sInterface()); prometheusAddr = interface->getJasmineGraphConfig("prometheus_address"); } else { From cf6f9e32233ca3038bf0bce1e85ce26a8b303759 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 2 Mar 2024 03:20:54 +0530 Subject: [PATCH 053/143] Utils::createDirectory to create with parent directories if missing --- src/partitioner/local/JSONParser.cpp | 3 --- src/partitioner/local/MetisPartitioner.cpp | 1 - src/partitioner/local/RDFParser.cpp | 3 --- src/server/JasmineGraphInstanceService.cpp | 4 ---- src/util/Utils.cpp | 4 ++-- 5 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/partitioner/local/JSONParser.cpp b/src/partitioner/local/JSONParser.cpp index cc447eb61..e878d6ba1 100644 --- a/src/partitioner/local/JSONParser.cpp +++ b/src/partitioner/local/JSONParser.cpp @@ -103,9 +103,6 @@ static void attributeFileCreate(std::map &vertexToIDMap, std::map &vertexToIDMap, std::map &fieldsMap, std::string inputFilePath, string outputFilePath) { - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/"); - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp"); - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp/JSONParser"); Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp/JSONParser/output"); auto edgeListFuture = std::async(createEdgeList, inputFilePath, outputFilePath); diff --git a/src/partitioner/local/MetisPartitioner.cpp b/src/partitioner/local/MetisPartitioner.cpp index 9517fd815..78a22ae10 100644 --- a/src/partitioner/local/MetisPartitioner.cpp +++ b/src/partitioner/local/MetisPartitioner.cpp @@ -39,7 +39,6 @@ void MetisPartitioner::loadDataSet(string inputFilePath, int graphID) { // Have to call createDirectory twice since it does not support recursive directory creation. Could use // boost::filesystem for path creation - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/"); Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp"); Utils::createDirectory(this->outputFilePath); diff --git a/src/partitioner/local/RDFParser.cpp b/src/partitioner/local/RDFParser.cpp index dfb11dfcb..125857ab0 100644 --- a/src/partitioner/local/RDFParser.cpp +++ b/src/partitioner/local/RDFParser.cpp @@ -299,9 +299,6 @@ void GetConfig::readConfigFile(string &configFile, int graphId) { void GetConfig::writeEdgesToFile() { ofstream file; - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/"); - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp/"); - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp/" + to_string(this->graphID)); file.open(Utils::getHomeDir() + "/.jasminegraph/tmp/" + std::to_string(this->graphID) + "/" + std::to_string(this->graphID)); for (int i = 0; i < edgelist.size(); i++) { diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 5738c359e..d755552c0 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -151,8 +151,6 @@ void *instanceservicesession(void *dummyPt) { instance_logger.info("New service session started on thread:" + to_string(pthread_self()) + " connFd:" + to_string(connFd)); - Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder")); - char data[DATA_BUFFER_SIZE]; bool loop_exit = false; while (!loop_exit) { @@ -374,7 +372,6 @@ void removeGraphFragments(std::string graphID) { } void writeCatalogRecord(string record) { - Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder")); string catalogFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/catalog.txt"; ofstream outfile; @@ -827,7 +824,6 @@ int JasmineGraphInstanceService::collectTrainedModelThreadFunction(instanceservi void JasmineGraphInstanceService::createPartitionFiles(std::string graphID, std::string partitionID, std::string fileType) { - Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.trainedmodelfolder")); JasmineGraphHashMapLocalStore *hashMapLocalStore = new JasmineGraphHashMapLocalStore(); string inputFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + graphID + "_" + partitionID; diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index ff70db9a3..84eb3f8d3 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -197,9 +197,9 @@ bool Utils::fileExists(std::string fileName) { return access(fileName.c_str(), F * @param dirName */ void Utils::createDirectory(const std::string dirName) { - // TODO: check if directory exists before creating // TODO: check return value - mkdir(dirName.c_str(), 0777); + string command = "mkdir -p " + dirName; + system(command.c_str()); } std::vector Utils::getListOfFilesInDirectory(std::string dirName) { From 58566d888112fee879c5db8ca936943ec03df81c Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 2 Mar 2024 03:26:48 +0530 Subject: [PATCH 054/143] Create aggregate directory if missing --- src/server/JasmineGraphInstanceService.cpp | 4 ++-- src/server/JasmineGraphServer.cpp | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index d755552c0..54110109c 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -264,14 +264,14 @@ void *instanceservicesession(void *dummyPt) { JasmineGraphInstanceService::JasmineGraphInstanceService() {} -void JasmineGraphInstanceService::run(string masterHost, string host, int serverPort, - int serverDataPort) { +void JasmineGraphInstanceService::run(string masterHost, string host, int serverPort, int serverDataPort) { int listenFd; socklen_t len; struct sockaddr_in svrAdd; struct sockaddr_in clntAdd; Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder")); + Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder")); // create socket listenFd = socket(AF_INET, SOCK_STREAM, 0); diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 819c0a279..3a5696da2 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -988,10 +988,7 @@ void JasmineGraphServer::copyCentralStoreToAggregateLocation(std::string filePat std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); if (access(aggregatorDirPath.c_str(), F_OK)) { - std::string createDirCommand = "mkdir -p " + aggregatorDirPath; - if (system(createDirCommand.c_str())) { - server_logger.error("Creating directory " + aggregatorDirPath + " failed"); - } + Utils::createDirectory(aggregatorDirPath); } std::string copyCommand = "cp " + filePath + " " + aggregatorDirPath; From 715067b805d6d66e5c546393dca7af3407022a7f Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 1 Mar 2024 21:13:16 +0530 Subject: [PATCH 055/143] Limit resource usage of containers --- k8s/master-deployment.yaml | 9 ++++++++- k8s/worker-deployment.yaml | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/k8s/master-deployment.yaml b/k8s/master-deployment.yaml index 0480e2029..e39d37fc0 100644 --- a/k8s/master-deployment.yaml +++ b/k8s/master-deployment.yaml @@ -55,4 +55,11 @@ spec: - mountPath: /var/tmp/data name: jasminegraph-data - mountPath: /tmp/jasminegraph - name: jasminegraph-logs \ No newline at end of file + name: jasminegraph-logs + resources: + requests: + cpu: "1000m" + memory: "2Gi" + limits: + cpu: "2000m" + memory: "4Gi" \ No newline at end of file diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index 9b728fd9c..fb8285a19 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -45,4 +45,11 @@ spec: value: "" ports: - containerPort: 7777 - - containerPort: 7778 \ No newline at end of file + - containerPort: 7778 + resources: + requests: + cpu: "1000m" + memory: "2Gi" + limits: + cpu: "2000m" + memory: "4Gi" \ No newline at end of file From 692158fb1c52a03297780e8a4fcc23ec03ad8909 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Sat, 2 Mar 2024 14:33:08 +0530 Subject: [PATCH 056/143] Remove node affinity on creating workers --- k8s/worker-deployment.yaml | 1 - src/k8s/K8sInterface.cpp | 4 +-- src/k8s/K8sInterface.h | 3 +-- src/k8s/K8sWorkerController.cpp | 40 +++------------------------- src/k8s/K8sWorkerController.h | 2 -- tests/unit/k8s/K8sInterface_test.cpp | 4 +-- 6 files changed, 6 insertions(+), 48 deletions(-) diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index fb8285a19..d20e4a6e3 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -20,7 +20,6 @@ spec: workerId: "" service: "jasminegraph" spec: - nodeName: "" containers: - name: "jasminegraph-worker" image: "" diff --git a/src/k8s/K8sInterface.cpp b/src/k8s/K8sInterface.cpp index 50f77da30..f6de470e4 100644 --- a/src/k8s/K8sInterface.cpp +++ b/src/k8s/K8sInterface.cpp @@ -88,13 +88,11 @@ v1_service_list_t *K8sInterface::getServiceList(char *labelSelectors) { v1_deployment_t *K8sInterface::createJasmineGraphWorkerDeployment(int workerId, const std::string &ip, - const std::string &masterIp, - const std::string &nodeName) const { + const std::string &masterIp) const { std::string definiton = Utils::getJsonStringFromYamlFile(ROOT_DIR "/k8s/worker-deployment.yaml"); definiton = Utils::replaceAll(definiton, "", std::to_string(workerId)); definiton = Utils::replaceAll(definiton, "", masterIp); definiton = Utils::replaceAll(definiton, "", Utils::getJasmineGraphProperty("org.jasminegraph.k8s.image")); - definiton = Utils::replaceAll(definiton, "", nodeName); definiton = Utils::replaceAll(definiton, "", ip); cJSON *deploymentTemplate = cJSON_Parse(definiton.c_str()); diff --git a/src/k8s/K8sInterface.h b/src/k8s/K8sInterface.h index 6921adb06..944be2e80 100644 --- a/src/k8s/K8sInterface.h +++ b/src/k8s/K8sInterface.h @@ -38,8 +38,7 @@ class K8sInterface { v1_deployment_t *createJasmineGraphWorkerDeployment(int workerId, const std::string &ip, - const std::string &masterIp, - const std::string &nodeName) const; + const std::string &masterIp) const; v1_status_t *deleteJasmineGraphWorkerDeployment(int workerId) const; diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index ca19523fc..70d8ced30 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -18,10 +18,6 @@ limitations under the License. #include #include -#include "../util/Conts.h" -#include "../util/logger/Logger.h" -#include "../server/JasmineGraphServer.h" - Logger controller_logger; std::vector K8sWorkerController::workerList = {}; @@ -32,25 +28,6 @@ K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorke this->interface = new K8sInterface(); this->metadb = *metadb; - // Delete all hosts - metadb->runUpdate("DELETE FROM host"); - - v1_node_list_t *nodeList = interface->getNodes(); - listEntry_t *listEntry = NULL; - v1_node_t *node = NULL; - - std::string query; - list_ForEach(listEntry, nodeList->items) { - node = static_cast(listEntry->data); - query = "INSERT INTO host (name, ip) VALUES ('" + std::string(node->metadata->name) + "', '')"; - metadb->runInsert(query); - } - - query = "SELECT idhost, name FROM host"; - for (auto entry : metadb->runSelect(query)) { - nodes[entry.at(1).second] = atoi(entry.at(0).second.c_str()); - } - // Delete all the workers from the database metadb->runUpdate("DELETE FROM worker"); int workersAttached = this->attachExistingWorkers(); @@ -82,13 +59,6 @@ K8sWorkerController *K8sWorkerController::getInstance(std::string masterIp, int } void K8sWorkerController::spawnWorker(int workerId) { - // TODO (M-I-M-Ishad): Develop criteria to assign the worker to node based on performance metrics - // getting a node randomly - auto it = nodes.begin(); - std::advance(it, rand() % nodes.size()); - std::string hostName = it->first; - int hostId = it->second; - v1_service_t *service = this->interface->createJasmineGraphWorkerService(workerId); if (service != nullptr && service->metadata != nullptr && service->metadata->name != nullptr) { controller_logger.info("Worker " + std::to_string(workerId) + " service created successfully"); @@ -99,7 +69,7 @@ void K8sWorkerController::spawnWorker(int workerId) { std::string ip(service->spec->cluster_ip); v1_deployment_t *deployment = - this->interface->createJasmineGraphWorkerDeployment(workerId, ip, this->masterIp, hostName); + this->interface->createJasmineGraphWorkerDeployment(workerId, ip, this->masterIp); if (deployment != nullptr && deployment->metadata != nullptr && deployment->metadata->name != nullptr) { controller_logger.info("Worker " + std::to_string(workerId) + " deployment created successfully"); @@ -112,8 +82,7 @@ void K8sWorkerController::spawnWorker(int workerId) { K8sWorkerController::workerList.push_back(worker); std::string insertQuery = "INSERT INTO worker (host_idhost, server_port, server_data_port, name, ip, idworker) " - "VALUES (" + - std::to_string(hostId) + ", " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + + "VALUES ( -1, " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + "'" + std::string(service->metadata->name) + "', " + "'" + ip + "', " + std::to_string(workerId) + ")"; int status = metadb.runInsert(insertQuery); @@ -165,8 +134,6 @@ int K8sWorkerController::attachExistingWorkers() { list_ForEach(listEntry, deployment_list->items) { deployment = static_cast(listEntry->data); list_t *labels = deployment->metadata->labels; - std::string nodeName = deployment->spec->_template->spec->node_name; - int hostId = nodes[nodeName]; listEntry_t *label = NULL; list_ForEach(label, labels) { @@ -190,8 +157,7 @@ int K8sWorkerController::attachExistingWorkers() { K8sWorkerController::workerList.push_back(worker); std::string insertQuery = "INSERT INTO worker (host_idhost, server_port, server_data_port, name, ip, idworker) " - "VALUES ( " + - std::to_string(hostId) + ", " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + + "VALUES ( -1, " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + "'" + std::string(service->metadata->name) + "', " + "'" + ip + "', " + std::to_string(workerId) + ")"; diff --git a/src/k8s/K8sWorkerController.h b/src/k8s/K8sWorkerController.h index 1bc78dc51..ffa5d0734 100644 --- a/src/k8s/K8sWorkerController.h +++ b/src/k8s/K8sWorkerController.h @@ -34,8 +34,6 @@ class K8sWorkerController { std::string masterIp; int numberOfWorkers; - std::map nodes; - K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb); void spawnWorker(int workerId); diff --git a/tests/unit/k8s/K8sInterface_test.cpp b/tests/unit/k8s/K8sInterface_test.cpp index 5233f1b9f..8b877ce14 100644 --- a/tests/unit/k8s/K8sInterface_test.cpp +++ b/tests/unit/k8s/K8sInterface_test.cpp @@ -47,9 +47,7 @@ TEST_F(K8sInterfaceTest, TestGetServiceList) { } TEST_F(K8sInterfaceTest, TestCreateJasmineGraphWorkerDeployment) { - v1_node_list_t *nodeList = interface->getNodes(); - auto nodeName = static_cast(nodeList->items->firstEntry->data)->metadata->name; - v1_deployment_t *deployment = interface->createJasmineGraphWorkerDeployment(1, "10.43.0.2", "10.43.0.1", nodeName); + v1_deployment_t *deployment = interface->createJasmineGraphWorkerDeployment(1, "10.43.0.2", "10.43.0.1"); ASSERT_STREQ(deployment->metadata->name, "jasminegraph-worker1-deployment"); ASSERT_EQ(interface->apiClient->response_code, HTTP_CREATED); } From 02af3abf6d9dec0552a72cc6800f553e302a2dbc Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 2 Mar 2024 17:29:16 +0530 Subject: [PATCH 057/143] Add delay to tests until frontend listens --- test-docker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test-docker.sh b/test-docker.sh index d98e1f47b..1abd2635e 100755 --- a/test-docker.sh +++ b/test-docker.sh @@ -106,6 +106,7 @@ while ! nc -zvn 127.0.0.1 7777 &>/dev/null; do sleep .5 done +sleep 2 stop_tests_on_failure & timeout "$TIMEOUT_SECONDS" python3 -u "${TEST_ROOT}/test.py" |& tee "$TEST_LOG" From 32e281734f697f2a5fcafcac8ba1cb261e66b34f Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 2 Mar 2024 20:53:52 +0530 Subject: [PATCH 058/143] Decide partition count using edge count --- conf/jasminegraph-server.properties | 1 + src/frontend/JasmineGraphFrontEnd.cpp | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 550013610..482a7148e 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -101,3 +101,4 @@ org.jasminegraph.k8s.image=jasminegraph # Auto-scaling #-------------------------------------------------------------------------------- org.jasminegraph.autoscale.enabled=false +org.jasminegraph.autopartition.enabled=false diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index 6396520be..a0c4e77cd 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -15,12 +15,12 @@ limitations under the License. #include +#include #include #include #include -#include // C++ header file for file access +#include #include -#include // C++ header file for printing #include #include #include @@ -575,8 +575,15 @@ bool JasmineGraphFrontEnd::modelExistsByID(string id, SQLiteDBInterface *sqlite) } static std::string getPartitionCount(std::string path) { - // TODO: Implement - return ""; + if (Utils::getJasmineGraphProperty("org.jasminegraph.autopartition.enabled") != "true") { + return ""; + } + ifstream dataFile(path); + size_t edges = std::count(std::istreambuf_iterator(dataFile), std::istreambuf_iterator(), '\n'); + dataFile.close(); + int partCnt = (int)round(pow(edges, 0.2) / 6); + if (partCnt < 2) partCnt = 2; + return to_string(partCnt); } static void list_command(int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p) { From 041e4ee12cef32582febc8d3fe93e571b4b2af95 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sun, 3 Mar 2024 02:37:33 +0530 Subject: [PATCH 059/143] Get file size before receiving file in file transfer service --- src/server/JasmineGraphInstance.cpp | 126 ++---------------- src/server/JasmineGraphInstance.h | 2 - ...asmineGraphInstanceFileTransferService.cpp | 26 ++-- src/server/JasmineGraphInstanceService.cpp | 3 - src/server/JasmineGraphServer.cpp | 85 +++++------- src/server/JasmineGraphServer.h | 2 +- src/util/Utils.cpp | 16 ++- src/util/Utils.h | 2 +- 8 files changed, 74 insertions(+), 188 deletions(-) diff --git a/src/server/JasmineGraphInstance.cpp b/src/server/JasmineGraphInstance.cpp index 10846f280..634016005 100644 --- a/src/server/JasmineGraphInstance.cpp +++ b/src/server/JasmineGraphInstance.cpp @@ -33,8 +33,8 @@ void *runFileTransferService(void *dummyPt) { return NULL; } -int JasmineGraphInstance::start_running(string hostName, string masterHost, int serverPort, - int serverDataPort, string enableNmon) { +int JasmineGraphInstance::start_running(string hostName, string masterHost, int serverPort, int serverDataPort, + string enableNmon) { graphInstance_logger.info("Worker started"); this->hostName = hostName; @@ -43,6 +43,17 @@ int JasmineGraphInstance::start_running(string hostName, string masterHost, int this->serverDataPort = serverDataPort; this->enableNmon = enableNmon; + if (Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"))) { + graphInstance_logger.error("Could not create directory: " + + Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder")); + exit(1); + } + if (Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"))) { + graphInstance_logger.error("Could not create directory: " + + Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder")); + exit(1); + } + startNmonAnalyzer(enableNmon, serverPort); pthread_t instanceCommunicatorThread; @@ -58,116 +69,6 @@ int JasmineGraphInstance::start_running(string hostName, string masterHost, int return 0; } -bool JasmineGraphInstance::acknowledgeMaster(string masterHost, string workerIP, string workerPort) { - int sockfd; - char data[301]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return false; - } - - if (masterHost.find('@') != std::string::npos) { - masterHost = Utils::split(masterHost, '@')[1]; - } - - graphInstance_logger.log("###INSTANCE### Get Host By Name : " + masterHost, "info"); - - server = gethostbyname(masterHost.c_str()); - if (server == NULL) { - graphInstance_logger.error("ERROR, no host named " + masterHost); - return false; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(Conts::JASMINEGRAPH_BACKEND_PORT); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - return false; - } - - bzero(data, 301); - int result_wr = - write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); - - if (result_wr < 0) { - graphInstance_logger.log("Error writing to socket", "error"); - } - - graphInstance_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { - graphInstance_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); - - result_wr = write(sockfd, workerIP.c_str(), workerIP.size()); - - if (result_wr < 0) { - graphInstance_logger.log("Error writing to socket", "error"); - } - - graphInstance_logger.log("Sent : " + workerIP, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { - graphInstance_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); - - result_wr = write(sockfd, JasmineGraphInstanceProtocol::ACKNOWLEDGE_MASTER.c_str(), - JasmineGraphInstanceProtocol::ACKNOWLEDGE_MASTER.size()); - - if (result_wr < 0) { - graphInstance_logger.log("Error writing to socket", "error"); - } - - graphInstance_logger.log("Sent : " + JasmineGraphInstanceProtocol::ACKNOWLEDGE_MASTER, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::WORKER_INFO_SEND) == 0) { - std::string workerInfo = workerIP + "|" + workerPort; - result_wr = write(sockfd, workerInfo.c_str(), workerInfo.size()); - - if (result_wr < 0) { - graphInstance_logger.log("Error writing to socket", "error"); - } - - graphInstance_logger.log("Sent : " + workerInfo, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - - if (response.compare(JasmineGraphInstanceProtocol::UPDATE_DONE) == 0) { - return true; - } - } - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - return true; - } - } - - return false; - } - return false; -} - void JasmineGraphInstance::startNmonAnalyzer(string enableNmon, int serverPort) { if (enableNmon == "true") { std::string nmonFileLocation = Utils::getJasmineGraphProperty("org.jasminegraph.server.nmon.file.location"); @@ -177,7 +78,6 @@ void JasmineGraphInstance::startNmonAnalyzer(string enableNmon, int serverPort) std::string nmonStartupCommand = "nmon -c " + numberOfSnapshots + " -s " + snapshotGap + " -T -F " + nmonFileName; - char buffer[BUFFER_SIZE]; std::string result = ""; diff --git a/src/server/JasmineGraphInstance.h b/src/server/JasmineGraphInstance.h index 1343dc24d..4760eccc7 100644 --- a/src/server/JasmineGraphInstance.h +++ b/src/server/JasmineGraphInstance.h @@ -33,8 +33,6 @@ class JasmineGraphInstance { public: int start_running(string hostName, string masterHost, int serverPort, int serverDataPort, string enableNmon); - bool acknowledgeMaster(string masterHost, string workerIP, string workerPort); - void startNmonAnalyzer(string enableNmon, int serverPort); void registerShutdownHook(); diff --git a/src/server/JasmineGraphInstanceFileTransferService.cpp b/src/server/JasmineGraphInstanceFileTransferService.cpp index 9934df92b..d7af6ad94 100644 --- a/src/server/JasmineGraphInstanceFileTransferService.cpp +++ b/src/server/JasmineGraphInstanceFileTransferService.cpp @@ -13,6 +13,7 @@ limitations under the License. #include "JasmineGraphInstanceFileTransferService.h" +#include "../server/JasmineGraphInstanceProtocol.h" #include "../util/Utils.h" #include "../util/logger/Logger.h" @@ -24,26 +25,27 @@ void *filetransferservicesession(void *dummyPt) { filetransferservicesessionargs *sessionargs = (filetransferservicesessionargs *)dummyPt; int connFd = sessionargs->connFd; delete sessionargs; - char data[301]; - bzero(data, 301); - read(connFd, data, 300); - string fileName = (data); + char data[INSTANCE_DATA_LENGTH + 1]; + string fileName = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH); string filePathWithName = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + fileName; - write(connFd, JasmineGraphInstanceProtocol::SEND_FILE.c_str(), JasmineGraphInstanceProtocol::SEND_FILE.size()); - int bytesReceived; - char buffer[1024]; + Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::SEND_FILE_LEN); + string fsizeStr = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH); + int fsize = stoi(fsizeStr); + Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::SEND_FILE); + char buffer[4096]; + file_service_logger.info("File transfer started for file: " + fileName); std::ofstream file(filePathWithName, std::ios::out | std::ios::binary); - do { - bytesReceived = recv(connFd, buffer, sizeof(buffer), 0); + while (fsize > 0) { + int bytesReceived = recv(connFd, buffer, sizeof(buffer), 0); if (bytesReceived > 0) { file.write(buffer, bytesReceived); - // printf("Buffer: %.*s\n", connFd, buffer); - // or: printf("Buffer: %*.*s\n", bytes_read, bytes_read, buffer); + fsize -= bytesReceived; } - } while (bytesReceived > 0); + } file.close(); + file_service_logger.info("File transfer completed for file: " + fileName); return NULL; } diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 54110109c..9cd728ee6 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -270,9 +270,6 @@ void JasmineGraphInstanceService::run(string masterHost, string host, int server struct sockaddr_in svrAdd; struct sockaddr_in clntAdd; - Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder")); - Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder")); - // create socket listenFd = socket(AF_INET, SOCK_STREAM, 0); if (listenFd < 0) { diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 3a5696da2..e57fc988e 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -50,19 +50,15 @@ static bool batchUploadCompositeCentralstoreFile(std::string host, int port, int std::string filePath, std::string masterIP); static bool removeFragmentThroughService(string host, int port, string graphID, string masterIP); static bool removePartitionThroughService(string host, int port, string graphID, string partitionID, string masterIP); -static bool initiateCommon(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string masterIP, std::string initType); -static bool initiateTrain(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string masterIP); -static bool initiatePredict(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string masterIP); -static bool initiateServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string masterIP); -static bool initiateClient(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string masterIP); -static bool initiateAggregator(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, +static bool initiateCommon(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP, + std::string initType); +static bool initiateTrain(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP); +static bool initiatePredict(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP); +static bool initiateServer(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP); +static bool initiateClient(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP); +static bool initiateAggregator(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP); -static bool initiateOrgServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, +static bool initiateOrgServer(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP); static void degreeDistributionCommon(std::string graphID, std::string command); static int getPortByHost(const std::string &host); @@ -1655,9 +1651,8 @@ void JasmineGraphServer::initiateFiles(std::string graphID, std::string training map scheduleOfHost = scheduleForAllHosts[j->first]; for (k = workerPartition.partitionID.begin(); k != workerPartition.partitionID.end(); k++) { int iterationOfPart = scheduleOfHost[stoi(*k)]; - workerThreads[count++] = - std::thread(initiateTrain, j->first, workerPartition.port, workerPartition.dataPort, - trainingArgs + " " + *k, iterationOfPart, this->masterHost); + workerThreads[count++] = std::thread(initiateTrain, j->first, workerPartition.port, trainingArgs + " " + *k, + iterationOfPart, this->masterHost); } } @@ -1682,15 +1677,14 @@ void JasmineGraphServer::initiateCommunication(std::string graphID, std::string workerInstance = workerVector[i]; int serverPort = stoi(workerInstance.port); - int serverDataPort = stoi(workerInstance.dataPort); if (i == 0) { - workerThreads[threadID] = std::thread(initiateServer, workerInstance.hostname, serverPort, serverDataPort, - trainingArgs, fl_clients, masterIP); + workerThreads[threadID] = + std::thread(initiateServer, workerInstance.hostname, serverPort, trainingArgs, fl_clients, masterIP); threadID++; } - workerThreads[threadID] = std::thread(initiateClient, workerInstance.hostname, serverPort, serverDataPort, + workerThreads[threadID] = std::thread(initiateClient, workerInstance.hostname, serverPort, trainingArgs + " " + to_string(i), fl_clients, masterIP); threadID++; } @@ -1715,8 +1709,7 @@ void JasmineGraphServer::initiateOrgCommunication(std::string graphID, std::stri vector workerVector = Utils::getWorkerList(sqlite); if (Utils::getJasmineGraphProperty("org.jasminegraph.fl.aggregator") == "true") { - initiateAggregator("localhost", stoi(workerVector[0].port), stoi(workerVector[0].dataPort), trainingArgs, - fl_clients, masterIP); + initiateAggregator("localhost", stoi(workerVector[0].port), trainingArgs, fl_clients, masterIP); } std::ifstream file(Utils::getJasmineGraphProperty("org.jasminegraph.fl.organization.file")); @@ -1753,15 +1746,14 @@ void JasmineGraphServer::initiateOrgCommunication(std::string graphID, std::stri for (int i = 0; i < workerVector.size(); i++) { workerInstance = workerVector[i]; int serverPort = stoi(workerInstance.port); - int serverDataPort = stoi(workerInstance.dataPort); if (i == 0) { - workerThreads[threadID] = std::thread(initiateOrgServer, workerInstance.hostname, serverPort, - serverDataPort, trainingArgs, fl_clients, masterIP); + workerThreads[threadID] = + std::thread(initiateOrgServer, workerInstance.hostname, serverPort, trainingArgs, fl_clients, masterIP); threadID++; } - workerThreads[threadID] = std::thread(initiateClient, workerInstance.hostname, serverPort, serverDataPort, + workerThreads[threadID] = std::thread(initiateClient, workerInstance.hostname, serverPort, trainingArgs + " " + to_string(i), fl_clients, masterIP); threadID++; } @@ -1799,8 +1791,8 @@ void JasmineGraphServer::initiateMerge(std::string graphID, std::string training map scheduleOfHost = scheduleForAllHosts[j->first]; for (k = workerPartition.partitionID.begin(); k != workerPartition.partitionID.end(); k++) { int iterationOfPart = scheduleOfHost[stoi(*k)]; - workerThreads[count] = std::thread(mergeFiles, j->first, workerPartition.port, workerPartition.dataPort, - trainingArgs + " " + *k, fl_clients, *k, this->masterHost); + workerThreads[count] = std::thread(mergeFiles, j->first, workerPartition.port, trainingArgs + " " + *k, + fl_clients, *k, this->masterHost); count++; } } @@ -1812,8 +1804,8 @@ void JasmineGraphServer::initiateMerge(std::string graphID, std::string training delete[] workerThreads; } -static bool initiateCommon(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string masterIP, std::string initType) { +static bool initiateCommon(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP, + std::string initType) { bool result = true; int sockfd; char data[FED_DATA_LENGTH + 1]; @@ -1867,40 +1859,32 @@ static bool initiateCommon(std::string host, int port, int dataPort, std::string return true; } -static bool initiateTrain(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, masterIP, +static bool initiateTrain(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP) { + return initiateCommon(host, port, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_FED_PREDICT); } -static bool initiatePredict(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, masterIP, - JasmineGraphInstanceProtocol::INITIATE_FILES); +static bool initiatePredict(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP) { + return initiateCommon(host, port, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_FILES); } -static bool initiateServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, masterIP, - JasmineGraphInstanceProtocol::INITIATE_SERVER); +static bool initiateServer(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP) { + return initiateCommon(host, port, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_SERVER); } // todo Remove partCount from parameters as the partition id is being parsed within the training args -static bool initiateClient(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, masterIP, - JasmineGraphInstanceProtocol::INITIATE_CLIENT); +static bool initiateClient(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP) { + return initiateCommon(host, port, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_CLIENT); } -static bool initiateAggregator(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, +static bool initiateAggregator(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, masterIP, - JasmineGraphInstanceProtocol::INITIATE_AGG); + return initiateCommon(host, port, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_AGG); } -static bool initiateOrgServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, +static bool initiateOrgServer(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, masterIP, + return initiateCommon(host, port, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_ORG_SERVER); } @@ -2000,7 +1984,7 @@ bool JasmineGraphServer::receiveGlobalWeights(std::string host, int port, std::s return true; } -bool JasmineGraphServer::mergeFiles(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, +bool JasmineGraphServer::mergeFiles(std::string host, int port, std::string trainingArgs, int iteration, string partCount, std::string masterIP) { bool result = true; int sockfd; @@ -2115,7 +2099,6 @@ void JasmineGraphServer::egoNet(std::string graphID) { JasmineGraphServer::workerPartitions workerPartition = workerit->second; string host = workerit->first; int port = workerPartition.port; - int dataPort = workerPartition.dataPort; for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); partitionit != workerPartition.partitionID.end(); partitionit++) { diff --git a/src/server/JasmineGraphServer.h b/src/server/JasmineGraphServer.h index f624b55b3..b06b33705 100644 --- a/src/server/JasmineGraphServer.h +++ b/src/server/JasmineGraphServer.h @@ -148,7 +148,7 @@ class JasmineGraphServer { void initiateMerge(std::string graphID, std::string trainingArgs, SQLiteDBInterface *sqlite); - static bool mergeFiles(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, + static bool mergeFiles(std::string host, int port, std::string trainingArgs, int iteration, string partCount, std::string masterIP); static bool receiveGlobalWeights(std::string host, int port, std::string trainingArgs, int iteration, diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 84eb3f8d3..c319fefc0 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -196,10 +196,9 @@ bool Utils::fileExists(std::string fileName) { return access(fileName.c_str(), F * This method creates a new directory if it does not exist * @param dirName */ -void Utils::createDirectory(const std::string dirName) { - // TODO: check return value +int Utils::createDirectory(const std::string dirName) { string command = "mkdir -p " + dirName; - system(command.c_str()); + return system(command.c_str()); } std::vector Utils::getListOfFilesInDirectory(std::string dirName) { @@ -339,7 +338,7 @@ int Utils::compressFile(const std::string filePath, std::string mode) { int Utils::unzipFile(std::string filePath, std::string mode) { if (mode == "pigz") { if (access("/usr/bin/pigz", X_OK) != 0) { - util_logger.info("pigz not found. Compressing using gzip"); + util_logger.info("pigz not found. Decompressing using gzip"); mode = "gzip"; } } @@ -946,7 +945,7 @@ bool Utils::uploadFileToWorker(std::string host, int port, int dataPort, int gra continue; } else if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK) == 0) { util_logger.info("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK); - util_logger.info("Batch upload completed"); + util_logger.info("Batch upload completed: " + fileName); break; } } @@ -988,6 +987,13 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, + JasmineGraphInstanceProtocol::SEND_FILE_LEN)) { + close(sockfd); + return false; + } + + int fsize = Utils::getFileSize(filePath); + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, to_string(fsize), JasmineGraphInstanceProtocol::SEND_FILE)) { close(sockfd); return false; diff --git a/src/util/Utils.h b/src/util/Utils.h index 9c3f3550f..f70be5192 100644 --- a/src/util/Utils.h +++ b/src/util/Utils.h @@ -76,7 +76,7 @@ class Utils { static bool is_number(const std::string &compareString); - static void createDirectory(const std::string dirName); + static int createDirectory(const std::string dirName); static std::vector getListOfFilesInDirectory(std::string dirName); From 2832ea78ee245f8d915f686857e12db66a6a92c9 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sun, 3 Mar 2024 03:27:00 +0530 Subject: [PATCH 060/143] send close command to worker before closing connection --- .../executor/impl/TriangleCountExecutor.cpp | 16 +++++++- src/nativestore/DataPublisher.cpp | 5 ++- src/performance/metrics/PerformanceUtil.cpp | 2 + src/server/JasmineGraphInstanceService.cpp | 18 ++++++++- src/server/JasmineGraphServer.cpp | 39 ++++++++++++++++++- src/util/Utils.cpp | 11 ++++++ 6 files changed, 86 insertions(+), 5 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 02bd0e20f..facbe201a 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -511,13 +511,16 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int } triangleCount_logger.info("###COMPOSITE### Returning Total Triangles from executer "); - + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return triangleCount; } else { triangleCount_logger.log("There was an error in the upload process and the response is :: " + response, "error"); } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return 0; } @@ -859,7 +862,8 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio } } } - + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return isFileAccessible; } @@ -1049,6 +1053,8 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st triangleCount_logger.log("There was an error in the upload process and the response is :: " + response, "error"); } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return response; } @@ -1225,6 +1231,8 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag triangleCount_logger.log("There was an error in the upload process and the response is :: " + response, "error"); } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return response; } @@ -1433,6 +1441,8 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr triangleCount_logger.log("There was an error in the upload process and the response is :: " + response, "error"); } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return response; } @@ -1602,6 +1612,8 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP triangleCount_logger.log("There was an error in the upload process and the response is :: " + response, "error"); } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return response; } diff --git a/src/nativestore/DataPublisher.cpp b/src/nativestore/DataPublisher.cpp index f0e1da008..27ec77a87 100644 --- a/src/nativestore/DataPublisher.cpp +++ b/src/nativestore/DataPublisher.cpp @@ -44,7 +44,10 @@ DataPublisher::DataPublisher(int worker_port, std::string worker_address) { } } -DataPublisher::~DataPublisher() { close(sock); } +DataPublisher::~DataPublisher() { + Utils::send_str_wrapper(sock, JasmineGraphInstanceProtocol::CLOSE); + close(sock); +} void DataPublisher::publish(std::string message) { char receiver_buffer[MAX_STREAMING_DATA_LENGTH] = {0}; diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index b9c8baa4f..5ff9175e3 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -659,6 +659,8 @@ void PerformanceUtil::initiateCollectingRemoteSLAResourceUtilization(std::string } } } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); } std::string PerformanceUtil::requestRemoteLoadAverages(std::string host, int port, std::string isVMStatManager, diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 9cd728ee6..f87bedc75 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -814,7 +814,7 @@ int JasmineGraphInstanceService::collectTrainedModelThreadFunction(instanceservi } else { instance_logger.error("There was an error in the model collection process and the response is :: " + response); } - + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return 0; } @@ -1026,6 +1026,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) != 0) { instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::HANDSHAKE_OK + " ; Received: " + response); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1039,6 +1040,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) != 0) { instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::HOST_OK + " ; Received: " + response); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1052,6 +1054,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (response.compare(JasmineGraphInstanceProtocol::OK) != 0) { instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::OK + " ; Received: " + response); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1069,6 +1072,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_NAME) != 0) { instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::SEND_FILE_NAME + " ; Received: " + response); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1082,6 +1086,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_LEN) != 0) { instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::SEND_FILE_LEN + " ; Received: " + response); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1094,6 +1099,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_CONT) != 0) { instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::SEND_FILE_CONT + " ; Received: " + response); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1146,6 +1152,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int } } END_OUTER_LOOP: + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); } return true; @@ -1494,6 +1501,7 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine } if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::WORKER_EGO_NET)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1502,12 +1510,14 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) != 0) { instance_logger.error("Error reading from socket"); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } instance_logger.info("Received : " + response); if (Utils::send_str_wrapper(sockfd, graphID)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1515,6 +1525,7 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) != 0) { instance_logger.error("Error reading from socket"); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1524,6 +1535,7 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine string egonetString; if (!Utils::send_str_wrapper(sockfd, workerSocketPair[2])) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1532,11 +1544,13 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) != 0) { instance_logger.error("Error reading from socket"); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } if (!Utils::send_str_wrapper(sockfd, workerList)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1545,9 +1559,11 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) != 0) { instance_logger.error("Error reading from socket"); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); } } diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index e57fc988e..3965f4e95 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -632,6 +632,7 @@ void JasmineGraphServer::resolveOperationalGraphs() { } if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, this->masterHost)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -639,6 +640,7 @@ void JasmineGraphServer::resolveOperationalGraphs() { if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::INITIATE_FRAGMENT_RESOLUTION, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -667,6 +669,7 @@ void JasmineGraphServer::resolveOperationalGraphs() { continue; } if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::FRAGMENT_RESOLUTION_DONE)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -680,6 +683,7 @@ void JasmineGraphServer::resolveOperationalGraphs() { graphIDsFromWorkersSet.insert(atoi(it->c_str())); } } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); } @@ -789,6 +793,7 @@ int JasmineGraphServer::shutdown_worker(std::string workerIP, int port) { string response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); server_logger.info("Response: " + response); + close(sockfd); return 0; } @@ -1171,6 +1176,7 @@ static bool removeFragmentThroughService(string host, int port, string graphID, } if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -1178,6 +1184,7 @@ static bool removeFragmentThroughService(string host, int port, string graphID, if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::DELETE_GRAPH_FRAGMENT, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -1186,7 +1193,7 @@ static bool removeFragmentThroughService(string host, int port, string graphID, close(sockfd); return false; } - + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } @@ -1226,12 +1233,14 @@ static bool removePartitionThroughService(string host, int port, string graphID, } if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::DELETE_GRAPH, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -1243,10 +1252,12 @@ static bool removePartitionThroughService(string host, int port, string graphID, } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partitionID, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } @@ -1495,26 +1506,31 @@ static void degreeDistributionCommon(std::string graphID, std::string command) { if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, command, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } if (!Utils::send_str_wrapper(sockfd, workerList)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); server_logger.info("Sent: " + workerList); } @@ -1602,18 +1618,21 @@ void JasmineGraphServer::duplicateCentralStore(std::string graphID) { if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::DP_CENTRALSTORE, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1623,6 +1642,7 @@ void JasmineGraphServer::duplicateCentralStore(std::string graphID) { continue; } server_logger.info("Sent: " + workerList); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); } } @@ -1840,21 +1860,25 @@ static bool initiateCommon(std::string host, int port, std::string trainingArgs, } if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, initType, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::send_str_wrapper(sockfd, trainingArgs)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } server_logger.info("Sent: " + trainingArgs); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } @@ -1980,6 +2004,7 @@ bool JasmineGraphServer::receiveGlobalWeights(std::string host, int port, std::s break; } } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } @@ -2020,22 +2045,26 @@ bool JasmineGraphServer::mergeFiles(std::string host, int port, std::string trai } if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::MERGE_FILES, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::send_str_wrapper(sockfd, trainingArgs)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } server_logger.info("Sent: " + trainingArgs); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } @@ -2076,16 +2105,19 @@ bool JasmineGraphServer::sendTrainCommand(std::string host, int port, std::strin if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::INITIATE_TRAIN, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::send_str_wrapper(sockfd, trainingArgs)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } server_logger.info("Sent: " + trainingArgs); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } @@ -2145,28 +2177,33 @@ void JasmineGraphServer::egoNet(std::string graphID) { if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::EGONET, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, workerList, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return; } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); } } diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index c319fefc0..e2965c646 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -873,11 +873,13 @@ bool Utils::uploadFileToWorker(std::string host, int port, int dataPort, int gra } if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, uploadType, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -888,18 +890,21 @@ bool Utils::uploadFileToWorker(std::string host, int port, int dataPort, int gra if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, std::to_string(graphID), JasmineGraphInstanceProtocol::SEND_FILE_NAME)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, JasmineGraphInstanceProtocol::SEND_FILE_LEN)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileLength, JasmineGraphInstanceProtocol::SEND_FILE_CONT)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -911,6 +916,7 @@ bool Utils::uploadFileToWorker(std::string host, int port, int dataPort, int gra int count = 0; while (true) { if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::FILE_RECV_CHK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -933,6 +939,7 @@ bool Utils::uploadFileToWorker(std::string host, int port, int dataPort, int gra // Next we wait till the batch upload completes while (true) { if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -949,6 +956,7 @@ bool Utils::uploadFileToWorker(std::string host, int port, int dataPort, int gra break; } } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } @@ -988,6 +996,7 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, JasmineGraphInstanceProtocol::SEND_FILE_LEN)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -995,6 +1004,7 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f int fsize = Utils::getFileSize(filePath); if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, to_string(fsize), JasmineGraphInstanceProtocol::SEND_FILE)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -1018,6 +1028,7 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f } fclose(fp); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return status; } From cd4f8c7429651a677550b9d58922582529ad6618 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sun, 3 Mar 2024 03:55:03 +0530 Subject: [PATCH 061/143] Get last 5 seconds CPU usage --- .../metrics/StatisticCollector.cpp | 27 +++++++++++++------ src/server/JasmineGraphInstanceProtocol.cpp | 1 - src/server/JasmineGraphInstanceService.cpp | 2 -- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index 217dd0395..dd313f471 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -168,11 +168,11 @@ static long parseLine(char *line) { return val; } -double StatisticCollector::getCpuUsage() { - static long long lastTotal = 0, lastIdle = 0; - +static void getCpuCycles(long long *totalp, long long *idlep) { + *totalp = 0; + *idlep = 0; FILE *fp = fopen("/proc/stat", "r"); - if (!fp) return -1; + if (!fp) return; char line[1024]; fscanf(fp, "%[^\r\n]%*c", line); fclose(fp); @@ -198,10 +198,21 @@ double StatisticCollector::getCpuUsage() { } total += value; } - long long diffTotal = lastTotal - total; - long long diffIdle = lastIdle - idle; - lastTotal = total; - lastIdle = idle; + *totalp = total; + *idlep = idle; +} + +double StatisticCollector::getCpuUsage() { + long long total1; + long long idle1; + getCpuCycles(&total1, &idle1); + sleep(5); + long long total2; + long long idle2; + getCpuCycles(&total2, &idle2); + + long long diffTotal = total2 - total1; + long long diffIdle = idle2 - idle1; return (diffTotal - diffIdle) / (double)diffTotal; } diff --git a/src/server/JasmineGraphInstanceProtocol.cpp b/src/server/JasmineGraphInstanceProtocol.cpp index ab220ca0c..df2cee146 100644 --- a/src/server/JasmineGraphInstanceProtocol.cpp +++ b/src/server/JasmineGraphInstanceProtocol.cpp @@ -18,7 +18,6 @@ const string JasmineGraphInstanceProtocol::HANDSHAKE_OK = "hske-ok"; const string JasmineGraphInstanceProtocol::HOST_OK = "hst-ok"; const string JasmineGraphInstanceProtocol::BATCH_UPLOAD_COMPOSITE_CENTRAL = "upload-g-c-c"; const string JasmineGraphInstanceProtocol::CLOSE = "close"; -const string JasmineGraphInstanceProtocol::CLOSE_ACK = "close-ok"; const string JasmineGraphInstanceProtocol::SHUTDOWN = "shdn"; const string JasmineGraphInstanceProtocol::SHUTDOWN_ACK = "shdn-ok"; const string JasmineGraphInstanceProtocol::READY = "ready"; diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index f87bedc75..14fc0921a 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -1971,12 +1971,10 @@ static void handshake_command(int connFd, bool *loop_exit_p) { static inline void close_command(int connFd, bool *loop_exit_p) { *loop_exit_p = true; - Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::CLOSE_ACK); close(connFd); } static inline void shutdown_command(int connFd) { - Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::CLOSE_ACK); close(connFd); exit(0); } From da8ddbc96a05c2b3af4343c66ff5808939a061ee Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Mon, 4 Mar 2024 21:10:16 +0530 Subject: [PATCH 062/143] prevent memory leaks after loadCentralStore --- .../JasmineGraphHashMapCentralStore.cpp | 15 +-- .../JasmineGraphHashMapCentralStore.h | 2 +- src/frontend/JasmineGraphFrontEnd.cpp | 46 +++----- src/frontend/JasmineGraphFrontEnd.h | 2 - .../trainer/JasmineGraphTrainingSchedular.cpp | 21 +++- src/server/JasmineGraphInstanceService.cpp | 106 ++++++++---------- src/server/JasmineGraphInstanceService.h | 2 +- src/server/JasmineGraphServer.cpp | 48 ++++---- src/util/Utils.cpp | 1 + tests/unit/metadb/SQLiteDBInterface_test.cpp | 1 - 10 files changed, 115 insertions(+), 129 deletions(-) diff --git a/src/centralstore/JasmineGraphHashMapCentralStore.cpp b/src/centralstore/JasmineGraphHashMapCentralStore.cpp index 0d503875e..ce2eb78be 100644 --- a/src/centralstore/JasmineGraphHashMapCentralStore.cpp +++ b/src/centralstore/JasmineGraphHashMapCentralStore.cpp @@ -62,9 +62,8 @@ bool JasmineGraphHashMapCentralStore::loadGraph() { return result; } -bool JasmineGraphHashMapCentralStore::loadGraph(std::string fileName) { +bool JasmineGraphHashMapCentralStore::loadGraph(const std::string &edgeStorePath) { bool result = false; - std::string edgeStorePath = fileName; std::ifstream dbFile; dbFile.open(edgeStorePath, std::ios::binary | std::ios::in); @@ -134,9 +133,7 @@ map JasmineGraphHashMapCentralStore::getOutDegreeDistributionHashMap for (map>::iterator it = centralSubgraphMap.begin(); it != centralSubgraphMap.end(); ++it) { long distribution = (it->second).size(); - auto key = it->first; - auto nodes = it->second; - unordered_set neighboursOfNeighbour = nodes; + unordered_set neighboursOfNeighbour = it->second; distributionHashMap[it->first] = distribution; } return distributionHashMap; @@ -145,12 +142,10 @@ map JasmineGraphHashMapCentralStore::getOutDegreeDistributionHashMap map JasmineGraphHashMapCentralStore::getInDegreeDistributionHashMap() { map distributionHashMap; - for (map>::iterator it = centralSubgraphMap.begin(); it != centralSubgraphMap.end(); - ++it) { - unordered_set distribution = it->second; - + for (auto it = centralSubgraphMap.begin(); it != centralSubgraphMap.end(); ++it) { + const auto &distribution = it->second; for (auto itr = distribution.begin(); itr != distribution.end(); ++itr) { - std::map::iterator distMapItr = distributionHashMap.find(*itr); + auto distMapItr = distributionHashMap.find(*itr); if (distMapItr != distributionHashMap.end()) { long previousValue = distMapItr->second; distMapItr->second = previousValue + 1; diff --git a/src/centralstore/JasmineGraphHashMapCentralStore.h b/src/centralstore/JasmineGraphHashMapCentralStore.h index e51a25eb6..6b5669e49 100644 --- a/src/centralstore/JasmineGraphHashMapCentralStore.h +++ b/src/centralstore/JasmineGraphHashMapCentralStore.h @@ -60,7 +60,7 @@ class JasmineGraphHashMapCentralStore : public JasmineGraphLocalStore { bool loadGraph(); - bool loadGraph(std::string fileName); + bool loadGraph(const std::string &fileName); bool storeGraph(); diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index a0c4e77cd..651b8c0aa 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -134,33 +134,31 @@ void *frontendservicesesion(void *dummyPt) { bool workerClientsInitialized = false; bool loop_exit = false; + int failCnt = 0; while (!loop_exit) { - if (currentFESession == Conts::MAX_FE_SESSIONS + 1) { - currentFESession--; - std::string errorResponse = "Jasminegraph Server is Busy. Please try again later."; - int result_wr = write(connFd, errorResponse.c_str(), errorResponse.length()); - if (result_wr < 0) { + if (currentFESession > Conts::MAX_FE_SESSIONS) { + if (!Utils::send_str_wrapper(connFd, "Jasminegraph Server is Busy. Please try again later.")) { frontend_logger.error("Error writing to socket"); } break; } - bzero(data, FRONTEND_DATA_LENGTH + 1); - read(connFd, data, FRONTEND_DATA_LENGTH); - - string line(data); - if (line.compare("\r\n") == 0) { + string line = Utils::read_str_wrapper(connFd, data, FRONTEND_DATA_LENGTH, true); + if (line.empty()) { + failCnt++; + if (failCnt > 4) { + break; + } + sleep(1); continue; } + failCnt = 0; + line = Utils::trim_copy(line); frontend_logger.info("Command received: " + line); - if (line.empty()) { - currentFESession--; - break; + continue; } - line = Utils::trim_copy(line); - if (currentFESession > 1) { canCalibrate = false; } else { @@ -169,7 +167,6 @@ void *frontendservicesesion(void *dummyPt) { } if (line.compare(EXIT) == 0) { - currentFESession--; break; } else if (line.compare(LIST) == 0) { list_command(connFd, sqlite, &loop_exit); @@ -240,6 +237,7 @@ void *frontendservicesesion(void *dummyPt) { } frontend_logger.info("Closing thread " + to_string(pthread_self()) + " and connection"); close(connFd); + currentFESession--; return NULL; } @@ -445,14 +443,6 @@ void JasmineGraphFrontEnd::getAndUpdateUploadTime(std::string graphID, SQLiteDBI frontend_logger.info("Upload time updated in the database"); } -JasmineGraphHashMapCentralStore JasmineGraphFrontEnd::loadCentralStore(std::string centralStoreFileName) { - frontend_logger.info("Loading Central Store File : Started " + centralStoreFileName); - JasmineGraphHashMapCentralStore *jasmineGraphHashMapCentralStore = new JasmineGraphHashMapCentralStore(); - jasmineGraphHashMapCentralStore->loadGraph(centralStoreFileName); - frontend_logger.info("Loading Central Store File : Completed"); - return *jasmineGraphHashMapCentralStore; -} - map JasmineGraphFrontEnd::getOutDegreeDistributionHashMap(map> graphMap) { map distributionHashMap; @@ -1406,9 +1396,9 @@ static void triangles_command(std::string masterIP, int connFd, SQLiteDBInterfac jobDetails.setJobId(std::to_string(uniqueId)); jobDetails.setJobType(TRIANGLES); - long graphSLA; - // All high priority threads will be set the same high priority level + long graphSLA = -1; // This prevents auto calibration for priority=1 (=default priority) if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { + // All high priority threads will be set the same high priority level threadPriority = Conts::HIGH_PRIORITY_DEFAULT_VALUE; graphSLA = JasmineGraphFrontEnd::getSLAForGraphId(sqlite, perfSqlite, graph_id, TRIANGLES, Conts::SLA_CATEGORY::LATENCY); @@ -2067,9 +2057,9 @@ static void page_rank_command(std::string masterIP, int connFd, SQLiteDBInterfac jobDetails.setJobId(std::to_string(uniqueId)); jobDetails.setJobType(PAGE_RANK); - long graphSLA; - // All high priority threads will be set the same high priority level + long graphSLA = -1; // This prevents auto calibration for priority=1 (=default priority) if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { + // All high priority threads will be set the same high priority level threadPriority = Conts::HIGH_PRIORITY_DEFAULT_VALUE; graphSLA = JasmineGraphFrontEnd::getSLAForGraphId(sqlite, perfSqlite, graphID, PAGE_RANK, Conts::SLA_CATEGORY::LATENCY); diff --git a/src/frontend/JasmineGraphFrontEnd.h b/src/frontend/JasmineGraphFrontEnd.h index 5d318c1e0..bfede4def 100644 --- a/src/frontend/JasmineGraphFrontEnd.h +++ b/src/frontend/JasmineGraphFrontEnd.h @@ -67,8 +67,6 @@ class JasmineGraphFrontEnd { static bool isGraphActiveAndTrained(std::string graphID, SQLiteDBInterface *sqlite); - static JasmineGraphHashMapCentralStore loadCentralStore(std::string centralStoreFileName); - static map getOutDegreeDistributionHashMap(map> graphMap); static bool isGraphActive(string graphID, SQLiteDBInterface *sqlite); diff --git a/src/ml/trainer/JasmineGraphTrainingSchedular.cpp b/src/ml/trainer/JasmineGraphTrainingSchedular.cpp index fe1073734..31789d0a4 100644 --- a/src/ml/trainer/JasmineGraphTrainingSchedular.cpp +++ b/src/ml/trainer/JasmineGraphTrainingSchedular.cpp @@ -270,9 +270,8 @@ static map>> scheduleGradientPassingTraining graphID + " AND a.host_idhost = " + j->first; std::vector>> results = refToSqlite->runSelect(sqlStatement); - vector> partitionMetadata; // Vector for node count, edge count, feature count for memory - // estimation of each partition map partitionWorkerMap; + map> partitionMetadataMap; // Iterate through partition list and get partition details trainScheduler_logger.log("Commence schedule creation for host" + j->second, "info"); for (std::vector>>::iterator i = results.begin(); i != results.end(); ++i) { @@ -295,12 +294,26 @@ static map>> scheduleGradientPassingTraining partitionValues.push_back(featurecount); partitionValues.push_back(featureSize); - partitionMetadata.push_back(partitionValues); + if (partitionMetadataMap.find(partitionID) == partitionMetadataMap.end()) { + partitionMetadataMap[partitionID] = partitionValues; + } // Store assigned worker per partition workerid = stoi(rowData.at(5).second); - partitionWorkerMap[partitionID] = workerid; + if (partitionWorkerMap.find(partitionID) == partitionWorkerMap.end()) { + partitionWorkerMap[partitionID] = workerid; + } else { + if (partitionWorkerMap[partitionID] > workerid) { + partitionWorkerMap[partitionID] = workerid; + } + } + } + vector> partitionMetadata; // Vector for node count, edge count, feature count for memory + // estimation of each partition + for (auto it = partitionMetadataMap.begin(); it != partitionMetadataMap.end(); it++) { + partitionMetadata.push_back(it->second); } + partitionMetadataMap.clear(); long availableMemory = getAvailableMemory(j->second); // Host memory (in KB) diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 14fc0921a..135f5f92d 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -502,12 +502,12 @@ void JasmineGraphInstanceService::loadInstanceDuplicateCentralStore( graphDBMapDuplicateCentralStores[graphIdentifier] = *jasmineGraphHashMapCentralStore; } -JasmineGraphHashMapCentralStore JasmineGraphInstanceService::loadCentralStore(std::string centralStoreFileName) { +JasmineGraphHashMapCentralStore *JasmineGraphInstanceService::loadCentralStore(std::string centralStoreFileName) { instance_logger.info("###INSTANCE### Loading Central Store File : Started " + centralStoreFileName); JasmineGraphHashMapCentralStore *jasmineGraphHashMapCentralStore = new JasmineGraphHashMapCentralStore(); jasmineGraphHashMapCentralStore->loadGraph(centralStoreFileName); instance_logger.info("###INSTANCE### Loading Central Store File : Completed"); - return *jasmineGraphHashMapCentralStore; + return jasmineGraphHashMapCentralStore; } string JasmineGraphInstanceService::aggregateCentralStoreTriangles(std::string graphId, std::string partitionId, @@ -520,22 +520,20 @@ string JasmineGraphInstanceService::aggregateCentralStoreTriangles(std::string g std::string dataFolder = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); std::string workerCentralStoreFile = dataFolder + "/" + centralGraphIdentifier; instance_logger.info("###INSTANCE### Loading Central Store : Started " + workerCentralStoreFile); - JasmineGraphHashMapCentralStore workerCentralStore = + JasmineGraphHashMapCentralStore *workerCentralStore = JasmineGraphInstanceService::loadCentralStore(workerCentralStoreFile); instance_logger.info("###INSTANCE### Loading Central Store : Completed"); - map> workerCentralGraphMap = workerCentralStore.getUnderlyingHashMap(); + const auto &workerCentralGraphMap = workerCentralStore->getUnderlyingHashMap(); - map>::iterator workerCentalGraphIterator; - - for (workerCentalGraphIterator = workerCentralGraphMap.begin(); + for (auto workerCentalGraphIterator = workerCentralGraphMap.begin(); workerCentalGraphIterator != workerCentralGraphMap.end(); ++workerCentalGraphIterator) { long startVid = workerCentalGraphIterator->first; - unordered_set endVidSet = workerCentalGraphIterator->second; + const unordered_set &endVidSet = workerCentalGraphIterator->second; - unordered_set aggregatedEndVidSet = aggregatedCentralStore[startVid]; + unordered_set &aggregatedEndVidSet = aggregatedCentralStore[startVid]; aggregatedEndVidSet.insert(endVidSet.begin(), endVidSet.end()); - aggregatedCentralStore[startVid] = aggregatedEndVidSet; } + delete workerCentralStore; std::vector paritionIdList = Utils::split(partitionIdList, ','); std::vector::iterator partitionIdListIterator; @@ -547,20 +545,19 @@ string JasmineGraphInstanceService::aggregateCentralStoreTriangles(std::string g std::string centralGraphIdentifier = graphId + "_centralstore_" + aggregatePartitionId; std::string centralStoreFile = aggregatorDirPath + "/" + centralGraphIdentifier; if (access(centralStoreFile.c_str(), R_OK) == 0) { - JasmineGraphHashMapCentralStore centralStore = + JasmineGraphHashMapCentralStore *centralStore = JasmineGraphInstanceService::loadCentralStore(centralStoreFile); - map> centralGraphMap = centralStore.getUnderlyingHashMap(); - map>::iterator centralGraphMapIterator; + const auto ¢ralGraphMap = centralStore->getUnderlyingHashMap(); - for (centralGraphMapIterator = centralGraphMap.begin(); centralGraphMapIterator != centralGraphMap.end(); - ++centralGraphMapIterator) { + for (auto centralGraphMapIterator = centralGraphMap.begin(); + centralGraphMapIterator != centralGraphMap.end(); ++centralGraphMapIterator) { long startVid = centralGraphMapIterator->first; - unordered_set endVidSet = centralGraphMapIterator->second; + const unordered_set &endVidSet = centralGraphMapIterator->second; - unordered_set aggregatedEndVidSet = aggregatedCentralStore[startVid]; + unordered_set &aggregatedEndVidSet = aggregatedCentralStore[startVid]; aggregatedEndVidSet.insert(endVidSet.begin(), endVidSet.end()); - aggregatedCentralStore[startVid] = aggregatedEndVidSet; } + delete centralStore; } } @@ -594,21 +591,20 @@ string JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(std: std::string availableCompositeFile = dataFolder + "/" + rawFileName; if (access(availableCompositeFile.c_str(), R_OK) == 0) { - JasmineGraphHashMapCentralStore centralStore = + JasmineGraphHashMapCentralStore *centralStore = JasmineGraphInstanceService::loadCentralStore(availableCompositeFile); - map> compositeCentralGraphMap = centralStore.getUnderlyingHashMap(); - map>::iterator compositeCentralGraphMapIterator; + const auto &compositeCentralGraphMap = centralStore->getUnderlyingHashMap(); - for (compositeCentralGraphMapIterator = compositeCentralGraphMap.begin(); + for (auto compositeCentralGraphMapIterator = compositeCentralGraphMap.begin(); compositeCentralGraphMapIterator != compositeCentralGraphMap.end(); ++compositeCentralGraphMapIterator) { long startVid = compositeCentralGraphMapIterator->first; - unordered_set endVidSet = compositeCentralGraphMapIterator->second; + const unordered_set &endVidSet = compositeCentralGraphMapIterator->second; - unordered_set aggregatedEndVidSet = aggregatedCompositeCentralStore[startVid]; + unordered_set &aggregatedEndVidSet = aggregatedCompositeCentralStore[startVid]; aggregatedEndVidSet.insert(endVidSet.begin(), endVidSet.end()); - aggregatedCompositeCentralStore[startVid] = aggregatedEndVidSet; } + delete centralStore; } } @@ -621,20 +617,18 @@ string JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(std: std::string compositeCentralStoreFile = aggregatorDirPath + "/" + rawFileName; if (access(compositeCentralStoreFile.c_str(), R_OK) == 0) { - JasmineGraphHashMapCentralStore centralStore = + JasmineGraphHashMapCentralStore *centralStore = JasmineGraphInstanceService::loadCentralStore(compositeCentralStoreFile); - map> centralGraphMap = centralStore.getUnderlyingHashMap(); - map>::iterator centralGraphMapIterator; - - for (centralGraphMapIterator = centralGraphMap.begin(); centralGraphMapIterator != centralGraphMap.end(); - ++centralGraphMapIterator) { + const auto ¢ralGraphMap = centralStore->getUnderlyingHashMap(); + for (auto centralGraphMapIterator = centralGraphMap.begin(); + centralGraphMapIterator != centralGraphMap.end(); ++centralGraphMapIterator) { long startVid = centralGraphMapIterator->first; - unordered_set endVidSet = centralGraphMapIterator->second; + const unordered_set &endVidSet = centralGraphMapIterator->second; - unordered_set aggregatedEndVidSet = aggregatedCompositeCentralStore[startVid]; + unordered_set &aggregatedEndVidSet = aggregatedCompositeCentralStore[startVid]; aggregatedEndVidSet.insert(endVidSet.begin(), endVidSet.end()); - aggregatedCompositeCentralStore[startVid] = aggregatedEndVidSet; } + delete centralStore; } } @@ -645,9 +639,7 @@ string JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(std: TriangleResult triangleResult = Triangles::countTriangles(aggregatedCompositeCentralStore, distributionHashMap, true); - std::string triangles = triangleResult.triangles; - - return triangles; + return triangleResult.triangles; } map JasmineGraphInstanceService::getOutDegreeDistributionHashMap(map> &graphMap) { @@ -1400,28 +1392,27 @@ map>> calculateLocalEgoNet(string graphID, s instance_logger.error(centralStoreFile + " is not a regular file"); continue; } - JasmineGraphHashMapCentralStore centralStore = JasmineGraphInstanceService::loadCentralStore(centralStoreFile); - map> centralGraphMap = centralStore.getUnderlyingHashMap(); + JasmineGraphHashMapCentralStore *centralStore = JasmineGraphInstanceService::loadCentralStore(centralStoreFile); + const auto ¢ralGraphMap = centralStore->getUnderlyingHashMap(); - for (map>::iterator centralGraphMapIterator = centralGraphMap.begin(); - centralGraphMapIterator != centralGraphMap.end(); ++centralGraphMapIterator) { + for (auto centralGraphMapIterator = centralGraphMap.begin(); centralGraphMapIterator != centralGraphMap.end(); + ++centralGraphMapIterator) { long startVid = centralGraphMapIterator->first; - unordered_set endVidSet = centralGraphMapIterator->second; + const unordered_set &endVidSet = centralGraphMapIterator->second; for (auto itr = endVidSet.begin(); itr != endVidSet.end(); ++itr) { - map>>::iterator egonetMapItr = egonetMap.find(*itr); - + auto egonetMapItr = egonetMap.find(*itr); if (egonetMapItr != egonetMap.end()) { - map> egonetSubGraph = egonetMapItr->second; - map>::iterator egonetSubGraphItr = egonetSubGraph.find(*itr); + map> &egonetSubGraph = egonetMapItr->second; + auto egonetSubGraphItr = egonetSubGraph.find(*itr); if (egonetSubGraphItr != egonetSubGraph.end()) { - unordered_set egonetSubGraphNeighbours = egonetSubGraphItr->second; + unordered_set &egonetSubGraphNeighbours = egonetSubGraphItr->second; egonetSubGraphNeighbours.insert(startVid); - egonetSubGraphItr->second = egonetSubGraphNeighbours; } } } } + delete centralStore; } return egonetMap; @@ -1760,22 +1751,20 @@ map> getEdgesWorldToLocal(string graphID, string parti instance_logger.error("Read permission denied for " + centralStoreFile); continue; } - JasmineGraphHashMapCentralStore centralStore = JasmineGraphInstanceService::loadCentralStore(centralStoreFile); - map> centralGraphMap = centralStore.getUnderlyingHashMap(); + JasmineGraphHashMapCentralStore *centralStore = JasmineGraphInstanceService::loadCentralStore(centralStoreFile); + const auto ¢ralGraphMap = centralStore->getUnderlyingHashMap(); - for (map>::iterator centralGraphMapIterator = centralGraphMap.begin(); - centralGraphMapIterator != centralGraphMap.end(); ++centralGraphMapIterator) { + for (auto centralGraphMapIterator = centralGraphMap.begin(); centralGraphMapIterator != centralGraphMap.end(); + ++centralGraphMapIterator) { long startVid = centralGraphMapIterator->first; - unordered_set endVidSet = centralGraphMapIterator->second; + const unordered_set &endVidSet = centralGraphMapIterator->second; for (auto itr = endVidSet.begin(); itr != endVidSet.end(); ++itr) { if (graphVertexMap.find(*itr) != graphVertexMap.end()) { - map>::iterator toIDIterator = worldToLocalVertexMap.find(*itr); - + auto toIDIterator = worldToLocalVertexMap.find(*itr); if (toIDIterator != worldToLocalVertexMap.end()) { - unordered_set fromIDs = toIDIterator->second; + unordered_set &fromIDs = toIDIterator->second; fromIDs.insert(startVid); - toIDIterator->second = fromIDs; } else { unordered_set fromIDs; fromIDs.insert(startVid); @@ -1784,6 +1773,7 @@ map> getEdgesWorldToLocal(string graphID, string parti } } } + delete centralStore; } return worldToLocalVertexMap; @@ -3391,7 +3381,7 @@ static void aggregate_composite_centralstore_triangles_command(int connFd, bool threadPriorityMutex.unlock(); } - std::string aggregatedTriangles = + const std::string &aggregatedTriangles = JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(response, availableFiles, threadPriority); if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { diff --git a/src/server/JasmineGraphInstanceService.h b/src/server/JasmineGraphInstanceService.h index 95f9cbfcd..ea960e4c3 100644 --- a/src/server/JasmineGraphInstanceService.h +++ b/src/server/JasmineGraphInstanceService.h @@ -135,7 +135,7 @@ class JasmineGraphInstanceService { static void loadInstanceDuplicateCentralStore( std::string graphId, std::string partitionId, std::map &graphDBMapDuplicateCentralStores); - static JasmineGraphHashMapCentralStore loadCentralStore(std::string centralStoreFileName); + static JasmineGraphHashMapCentralStore *loadCentralStore(std::string centralStoreFileName); static string aggregateCentralStoreTriangles(std::string graphId, std::string partitionId, std::string partitionIdList, int threadPriority); static string aggregateCompositeCentralStoreTriangles(std::string compositeFileList, std::string availableFileList, diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 3965f4e95..8ed7cf2e9 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -155,6 +155,7 @@ void JasmineGraphServer::init() { } void JasmineGraphServer::start_workers() { + // Not used in K8s mode int hostListModeNWorkers = 0; int numberOfWorkersPerHost; std::vector hostsList; @@ -537,19 +538,18 @@ void JasmineGraphServer::resolveOperationalGraphs() { std::vector>> output = this->sqlite->runSelect(sqlStatement); std::map> partitionMap; - for (std::vector>>::iterator i = output.begin(); i != output.end(); ++i) { + for (auto i = output.begin(); i != output.end(); ++i) { int workerID = -1; string graphID; string partitionID; - std::vector>::iterator j = (i->begin()); + auto j = i->begin(); graphID = j->second; ++j; partitionID = j->second; ++j; workerID = std::stoi(j->second); - std::vector partitionList = partitionMap[workerID]; + std::vector &partitionList = partitionMap[workerID]; partitionList.push_back(graphID + "_" + partitionID); - partitionMap[workerID] = partitionList; } int RECORD_AGGREGATION_FREQUENCY = 5; @@ -562,25 +562,23 @@ void JasmineGraphServer::resolveOperationalGraphs() { for (std::vector::iterator x = (it->second).begin(); x != (it->second).end(); ++x) { if (counter >= RECORD_AGGREGATION_FREQUENCY) { - std::vector partitionList = partitionAggregatedMap[workerID]; + std::vector &partitionList = partitionAggregatedMap[workerID]; string data = ss.str(); std::stringstream().swap(ss); counter = 0; data = data.substr(0, data.find_last_of(",")); partitionList.push_back(data); - partitionAggregatedMap[workerID] = partitionList; } ss << x->c_str() << ","; counter++; } - std::vector partitionList = partitionAggregatedMap[workerID]; + std::vector &partitionList = partitionAggregatedMap[workerID]; string data = ss.str(); std::stringstream().swap(ss); counter = 0; data = data.substr(0, data.find_last_of(",")); partitionList.push_back(data); - partitionAggregatedMap[workerID] = partitionList; } sqlStatement = "SELECT idworker,ip,server_port FROM worker"; @@ -592,7 +590,7 @@ void JasmineGraphServer::resolveOperationalGraphs() { string host; int workerPort = -1; string partitionID; - std::vector>::iterator j = (i->begin()); + std::vector>::iterator j = i->begin(); workerID = std::stoi(j->second); ++j; host = j->second; @@ -665,6 +663,7 @@ void JasmineGraphServer::resolveOperationalGraphs() { } } if (!success) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -690,8 +689,8 @@ void JasmineGraphServer::resolveOperationalGraphs() { sqlStatement = "SELECT idgraph FROM graph"; std::vector>> output2 = this->sqlite->runSelect(sqlStatement); std::set graphIDsFromMetDBSet; - for (std::vector>>::iterator i = output2.begin(); i != output2.end(); ++i) { - std::vector>::iterator j = (i->begin()); + for (auto i = output2.begin(); i != output2.end(); ++i) { + auto j = i->begin(); graphIDsFromMetDBSet.insert(atoi(j->second.c_str())); } @@ -955,7 +954,7 @@ static void assignPartitionToWorker(std::string fileName, int graphId, std::stri workerHost = Utils::split(workerHost, '@')[1]; } - std::string workerSearchQuery = "select idworker from worker where ip='" + workerHost + "' and server_port='" + + std::string workerSearchQuery = "SELECT idworker FROM worker WHERE ip='" + workerHost + "' AND server_port='" + std::to_string(workerPort) + "'"; std::vector>> results = refToSqlite->runSelect(workerSearchQuery); @@ -963,7 +962,7 @@ static void assignPartitionToWorker(std::string fileName, int graphId, std::stri std::string workerID = results[0][0].second; std::string partitionToWorkerQuery = - "insert into worker_has_partition (partition_idpartition, partition_graph_idgraph, worker_idworker) values " + "INSERT INTO worker_has_partition (partition_idpartition, partition_graph_idgraph, worker_idworker) VALUES " "('" + partitionID + "','" + std::to_string(graphId) + "','" + workerID + "')"; @@ -1271,25 +1270,26 @@ void JasmineGraphServer::updateOperationalGraphList() { hostsList = Utils::getHostListFromProperties(); } else if (jasminegraph_profile == PROFILE_DOCKER) { hostsList = getWorkerVector(workerHosts); + } else { + return; // TODO(thevindu-w): implement.for k8s } - vector::iterator it; - for (it = hostsList.begin(); it < hostsList.end(); it++) { + for (auto it = hostsList.begin(); it < hostsList.end(); it++) { string host = *it; hosts += ("'" + host + "', "); } hosts = hosts.substr(0, hosts.size() - 2); string sqlStatement = - ("SELECT b.partition_graph_idgraph FROM worker_has_partition AS b " - "JOIN worker WHERE worker.idworker = b.worker_idworker AND worker.name IN " - "(" + - hosts + - ") GROUP BY b.partition_graph_idgraph HAVING COUNT(b.partition_idpartition)= " - "(SELECT COUNT(a.idpartition) FROM partition AS a " - "WHERE a.graph_idgraph = b.partition_graph_idgraph);"); + "SELECT b.partition_graph_idgraph FROM worker_has_partition AS b " + "JOIN worker WHERE worker.idworker = b.worker_idworker AND worker.name IN " + "(" + + hosts + + ") GROUP BY b.partition_graph_idgraph HAVING COUNT(b.partition_idpartition)= " + "(SELECT COUNT(a.idpartition) FROM partition AS a " + "WHERE a.graph_idgraph = b.partition_graph_idgraph);"; std::vector>> v = this->sqlite->runSelect(sqlStatement); for (std::vector>>::iterator i = v.begin(); i != v.end(); ++i) { - for (std::vector>::iterator j = (i->begin()); j != i->end(); ++j) { - graphIDs += (j->second + ", "); + for (std::vector>::iterator j = i->begin(); j != i->end(); ++j) { + graphIDs += j->second + ", "; } } graphIDs = graphIDs.substr(0, graphIDs.size() - 2); diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index e2965c646..55dd5f7f9 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -405,6 +405,7 @@ int Utils::unzipDirectory(std::string filePath) { } void Utils::assignPartitionsToWorkers(int numberOfWorkers, SQLiteDBInterface *sqlite) { + // Not used in K8s mode sqlite->runUpdate("DELETE FROM worker_has_partition"); std::vector>> v = diff --git a/tests/unit/metadb/SQLiteDBInterface_test.cpp b/tests/unit/metadb/SQLiteDBInterface_test.cpp index 2c5a6cae8..a89362139 100644 --- a/tests/unit/metadb/SQLiteDBInterface_test.cpp +++ b/tests/unit/metadb/SQLiteDBInterface_test.cpp @@ -66,4 +66,3 @@ TEST_F(SQLiteDBInterfaceTest, TestRunInsertAndUpdate) { ASSERT_EQ(data[0][2].second, "127.0.0.1"); ASSERT_EQ(data[0][3].second, "false"); } - From 9dbee456580641c80a899ad489a2b286b3820329 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Tue, 5 Mar 2024 00:04:25 +0530 Subject: [PATCH 063/143] Fix memory leaks --- .../JasmineGraphHashMapCentralStore.cpp | 21 ++++-------- .../JasmineGraphHashMapCentralStore.h | 2 +- .../JasmineGraphHashMapLocalStore.cpp | 33 +++++-------------- .../JasmineGraphHashMapLocalStore.h | 4 +-- src/partitioner/local/MetisPartitioner.cpp | 15 +++------ 5 files changed, 23 insertions(+), 52 deletions(-) diff --git a/src/centralstore/JasmineGraphHashMapCentralStore.cpp b/src/centralstore/JasmineGraphHashMapCentralStore.cpp index ce2eb78be..dafdf84ef 100644 --- a/src/centralstore/JasmineGraphHashMapCentralStore.cpp +++ b/src/centralstore/JasmineGraphHashMapCentralStore.cpp @@ -178,9 +178,8 @@ long JasmineGraphHashMapCentralStore::getVertexCount() { long JasmineGraphHashMapCentralStore::getEdgeCount() { if (edgeCount == 0) { - std::map>::iterator localSubGraphMapIterator; long mapSize = centralSubgraphMap.size(); - for (localSubGraphMapIterator = centralSubgraphMap.begin(); + for (auto localSubGraphMapIterator = centralSubgraphMap.begin(); localSubGraphMapIterator != centralSubgraphMap.end(); localSubGraphMapIterator++) { edgeCount = edgeCount + localSubGraphMapIterator->second.size(); } @@ -211,31 +210,23 @@ void JasmineGraphHashMapCentralStore::toLocalSubGraphMap(const PartEdgeMapStore } } -bool JasmineGraphHashMapCentralStore::storePartEdgeMap(std::map> edgeMap, - const std::string savePath) { - bool result = false; +bool JasmineGraphHashMapCentralStore::storePartEdgeMap(const std::map> &edgeMap, + const std::string &savePath) { flatbuffers::FlatBufferBuilder builder; std::vector> edgeStoreEntriesVector; - std::map>::iterator mapIterator; - for (mapIterator = edgeMap.begin(); mapIterator != edgeMap.end(); mapIterator++) { + for (auto mapIterator = edgeMap.begin(); mapIterator != edgeMap.end(); mapIterator++) { int key = mapIterator->first; - std::vector value = mapIterator->second; - std::vector valueVector(value.begin(), value.end()); - auto flatbufferVector = builder.CreateVector(valueVector); + auto flatbufferVector = builder.CreateVector(mapIterator->second); auto edgeStoreEntry = CreatePartEdgeMapStoreEntry(builder, key, flatbufferVector); edgeStoreEntriesVector.push_back(edgeStoreEntry); } auto flatBuffersEdgeStoreEntriesVector = builder.CreateVectorOfSortedTables(&edgeStoreEntriesVector); - auto edgeStore = CreatePartEdgeMapStore(builder, flatBuffersEdgeStoreEntriesVector); - builder.Finish(edgeStore); flatbuffers::SaveFile(savePath.c_str(), (const char *)builder.GetBufferPointer(), (size_t)builder.GetSize(), true); - result = true; - - return result; + return true; } diff --git a/src/centralstore/JasmineGraphHashMapCentralStore.h b/src/centralstore/JasmineGraphHashMapCentralStore.h index 6b5669e49..1c266b83d 100644 --- a/src/centralstore/JasmineGraphHashMapCentralStore.h +++ b/src/centralstore/JasmineGraphHashMapCentralStore.h @@ -80,7 +80,7 @@ class JasmineGraphHashMapCentralStore : public JasmineGraphLocalStore { long getEdgeCount(); - bool storePartEdgeMap(std::map> edgeMap, const std::string savePath); + static bool storePartEdgeMap(const std::map> &edgeMap, const std::string &savePath); }; #endif // JASMINEGRAPH_JASMINEGRAPHHASHMAPCENTRALSTORE_H diff --git a/src/localstore/JasmineGraphHashMapLocalStore.cpp b/src/localstore/JasmineGraphHashMapLocalStore.cpp index 269c7b528..3499447e9 100644 --- a/src/localstore/JasmineGraphHashMapLocalStore.cpp +++ b/src/localstore/JasmineGraphHashMapLocalStore.cpp @@ -176,34 +176,25 @@ void JasmineGraphHashMapLocalStore::initialize() {} void JasmineGraphHashMapLocalStore::addVertex(string *attributes) {} -bool JasmineGraphHashMapLocalStore::storeAttributes(std::map> attributeMap, - const string storePath) { - this->localAttributeMap = attributeMap; - bool result = false; +bool JasmineGraphHashMapLocalStore::storeAttributes(const std::map> &attributeMap, + const string &storePath) { flatbuffers::FlatBufferBuilder builder; std::vector> attributeStoreEntriesVector; - std::map>::iterator mapIterator; - for (mapIterator = localAttributeMap.begin(); mapIterator != localAttributeMap.end(); mapIterator++) { + for (auto mapIterator = attributeMap.begin(); mapIterator != attributeMap.end(); mapIterator++) { long key = mapIterator->first; - std::vector attributeVector = mapIterator->second; - - auto flatbufferVector = builder.CreateVectorOfStrings(attributeVector); + auto flatbufferVector = builder.CreateVectorOfStrings(mapIterator->second); auto attributeStoreEntry = CreateAttributeStoreEntry(builder, key, flatbufferVector); attributeStoreEntriesVector.push_back(attributeStoreEntry); } auto flatBuffersAttributeStoreEntriesVector = builder.CreateVectorOfSortedTables(&attributeStoreEntriesVector); - auto attributeStore = CreateAttributeStore(builder, flatBuffersAttributeStoreEntriesVector); - builder.Finish(attributeStore); flatbuffers::SaveFile(storePath.c_str(), (const char *)builder.GetBufferPointer(), (size_t)builder.GetSize(), true); - result = true; - - return result; + return true; } bool JasmineGraphHashMapLocalStore::loadAttributes() { @@ -298,26 +289,20 @@ bool JasmineGraphHashMapLocalStore::loadPartEdgeMap(const std::string filePath) return result; } -bool JasmineGraphHashMapLocalStore::storePartEdgeMap(std::map> edgeMap, - const std::string savePath) { +bool JasmineGraphHashMapLocalStore::storePartEdgeMap(const std::map> &edgeMap, + const std::string &savePath) { bool result = false; flatbuffers::FlatBufferBuilder builder; std::vector> edgeStoreEntriesVector; - - std::map>::iterator mapIterator; - for (mapIterator = edgeMap.begin(); mapIterator != edgeMap.end(); mapIterator++) { + for (auto mapIterator = edgeMap.begin(); mapIterator != edgeMap.end(); mapIterator++) { int key = mapIterator->first; - std::vector value = mapIterator->second; - std::vector valueVector(value.begin(), value.end()); - auto flatbufferVector = builder.CreateVector(valueVector); + auto flatbufferVector = builder.CreateVector(mapIterator->second); auto edgeStoreEntry = CreatePartEdgeMapStoreEntry(builder, key, flatbufferVector); edgeStoreEntriesVector.push_back(edgeStoreEntry); } auto flatBuffersEdgeStoreEntriesVector = builder.CreateVectorOfSortedTables(&edgeStoreEntriesVector); - auto edgeStore = CreatePartEdgeMapStore(builder, flatBuffersEdgeStoreEntriesVector); - builder.Finish(edgeStore); flatbuffers::SaveFile(savePath.c_str(), (const char *)builder.GetBufferPointer(), (size_t)builder.GetSize(), true); diff --git a/src/localstore/JasmineGraphHashMapLocalStore.h b/src/localstore/JasmineGraphHashMapLocalStore.h index 8b913997a..db279a4cd 100644 --- a/src/localstore/JasmineGraphHashMapLocalStore.h +++ b/src/localstore/JasmineGraphHashMapLocalStore.h @@ -92,7 +92,7 @@ class JasmineGraphHashMapLocalStore : public JasmineGraphLocalStore { bool storeGraph(); - bool storeAttributes(std::map> attributeMap, const std::string storePath); + static bool storeAttributes(const std::map> &attributeMap, const std::string &storePath); long getEdgeCount(); @@ -122,7 +122,7 @@ class JasmineGraphHashMapLocalStore : public JasmineGraphLocalStore { bool loadPartEdgeMap(const std::string filePath); - bool storePartEdgeMap(std::map> edgeMap, const std::string savePath); + static bool storePartEdgeMap(const std::map> &edgeMap, const std::string &savePath); map> getEdgeHashMap(const std::string filePath); }; diff --git a/src/partitioner/local/MetisPartitioner.cpp b/src/partitioner/local/MetisPartitioner.cpp index 78a22ae10..1c748d580 100644 --- a/src/partitioner/local/MetisPartitioner.cpp +++ b/src/partitioner/local/MetisPartitioner.cpp @@ -655,8 +655,7 @@ void MetisPartitioner::writeSerializedPartitionFiles(int part) { std::map> partEdgeMap = partitionedLocalGraphStorageMap[part]; - JasmineGraphHashMapLocalStore *hashMapLocalStore = new JasmineGraphHashMapLocalStore(); - hashMapLocalStore->storePartEdgeMap(partEdgeMap, outputFilePart); + JasmineGraphHashMapLocalStore::storePartEdgeMap(partEdgeMap, outputFilePart); // Compress part files Utils::compressFile(outputFilePart); @@ -672,8 +671,7 @@ void MetisPartitioner::writeSerializedMasterFiles(int part) { std::map> partMasterEdgeMap = masterGraphStorageMap[part]; - JasmineGraphHashMapCentralStore *hashMapCentralStore = new JasmineGraphHashMapCentralStore(); - hashMapCentralStore->storePartEdgeMap(partMasterEdgeMap, outputFilePartMaster); + JasmineGraphHashMapCentralStore::storePartEdgeMap(partMasterEdgeMap, outputFilePartMaster); Utils::compressFile(outputFilePartMaster); masterFileMutex.lock(); @@ -688,8 +686,7 @@ void MetisPartitioner::writeSerializedDuplicateMasterFiles(int part) { std::map> partMasterEdgeMap = duplicateMasterGraphStorageMap[part]; - JasmineGraphHashMapCentralStore *hashMapCentralStore = new JasmineGraphHashMapCentralStore(); - hashMapCentralStore->storePartEdgeMap(partMasterEdgeMap, outputFilePartMaster); + JasmineGraphHashMapCentralStore::storePartEdgeMap(partMasterEdgeMap, outputFilePartMaster); Utils::compressFile(outputFilePartMaster); masterFileMutex.lock(); @@ -793,8 +790,7 @@ void MetisPartitioner::writeRDFAttributeFilesForPartitions(int part) { } } - JasmineGraphHashMapLocalStore *hashMapLocalStore = new JasmineGraphHashMapLocalStore(); - hashMapLocalStore->storeAttributes(partitionedEdgeAttributes, attributeFilePart); + JasmineGraphHashMapLocalStore::storeAttributes(partitionedEdgeAttributes, attributeFilePart); Utils::compressFile(attributeFilePart); partAttrFileMutex.lock(); @@ -826,8 +822,7 @@ void MetisPartitioner::writeRDFAttributeFilesForMasterParts(int part) { } } - JasmineGraphHashMapLocalStore *hashMapLocalStore = new JasmineGraphHashMapLocalStore(); - hashMapLocalStore->storeAttributes(centralStoreEdgeAttributes, attributeFilePartMaster); + JasmineGraphHashMapLocalStore::storeAttributes(centralStoreEdgeAttributes, attributeFilePartMaster); Utils::compressFile(attributeFilePartMaster); masterAttrFileMutex.lock(); From eeff1bb607516523737de9de6859ae8c6c35994d Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 7 Mar 2024 00:10:47 +0530 Subject: [PATCH 064/143] Convert push file command to push partition --- .../executor/impl/TriangleCountExecutor.cpp | 7 --- src/frontend/core/scheduler/JobScheduler.cpp | 4 +- src/server/JasmineGraphInstanceProtocol.cpp | 2 +- src/server/JasmineGraphInstanceProtocol.h | 2 +- src/server/JasmineGraphInstanceService.cpp | 48 +++++++++++++------ 5 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index facbe201a..e19c4e59e 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -92,10 +92,7 @@ void TriangleCountExecutor::execute() { int partitionCount = 0; std::vector> intermRes; std::vector> statResponse; - std::vector> remoteCopyRes; - PlacesToNodeMapper placesToNodeMapper; std::vector compositeCentralStoreFiles; - int slaStatCount = 0; auto begin = chrono::high_resolution_clock::now(); @@ -156,10 +153,6 @@ void TriangleCountExecutor::execute() { "info"); } - for (auto &&futureCall : remoteCopyRes) { - futureCall.wait(); - } - for (int i = 0; i < workerListSize; i++) { Utils::worker currentWorker = workerList.at(i); string host = currentWorker.hostname; diff --git a/src/frontend/core/scheduler/JobScheduler.cpp b/src/frontend/core/scheduler/JobScheduler.cpp index 179e09c9b..081924d7d 100644 --- a/src/frontend/core/scheduler/JobScheduler.cpp +++ b/src/frontend/core/scheduler/JobScheduler.cpp @@ -126,7 +126,9 @@ JobResponse JobScheduler::getResult(JobRequest jobRequest) { responseFound = true; } responseVectorMutex.unlock(); - usleep(50000); + if (!responseFound) { + usleep(50000); + } } return jobResponse; diff --git a/src/server/JasmineGraphInstanceProtocol.cpp b/src/server/JasmineGraphInstanceProtocol.cpp index df2cee146..c427981fc 100644 --- a/src/server/JasmineGraphInstanceProtocol.cpp +++ b/src/server/JasmineGraphInstanceProtocol.cpp @@ -81,7 +81,7 @@ const string JasmineGraphInstanceProtocol::FILE_TYPE_DATA = "file-type-data"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_START = "stream-start"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_START_ACK = "stream-start-ack"; const string JasmineGraphInstanceProtocol::SEND_PRIORITY = "send-priority"; -const string JasmineGraphInstanceProtocol::PUSH_FILE = "push-file"; +const string JasmineGraphInstanceProtocol::PUSH_PARTITION = "push-partition"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_C_length_ACK = "stream-c-length-ack"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_END_OF_EDGE = "\r\n"; // CRLF equivelent in HTTP const string JasmineGraphInstanceProtocol::GRAPH_CSV_STREAM_START = "csv-stream-start"; diff --git a/src/server/JasmineGraphInstanceProtocol.h b/src/server/JasmineGraphInstanceProtocol.h index 1cc34134f..e52c46ce8 100644 --- a/src/server/JasmineGraphInstanceProtocol.h +++ b/src/server/JasmineGraphInstanceProtocol.h @@ -81,7 +81,7 @@ class JasmineGraphInstanceProtocol { static const string INITIATE_TRAIN; static const string INITIATE_PREDICT; static const string SEND_HOSTS; - static const string PUSH_FILE; + static const string PUSH_PARTITION; static const string INITIATE_MODEL_COLLECTION; static const string INITIATE_FRAGMENT_RESOLUTION; static const string FRAGMENT_RESOLUTION_CHK; diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 135f5f92d..b1356496d 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -121,7 +121,7 @@ static void degree_distribution_common(int connFd, int serverPort, std::map &graphDBMapLocalStores, std::map &graphDBMapCentralStores, bool *loop_exit_p, bool in); -static void push_file_command(int connFd, bool *loop_exit_p); +static void push_partition_command(int connFd, bool *loop_exit_p); char *converter(const std::string &s) { char *pc = new char[s.size() + 1]; @@ -250,8 +250,8 @@ void *instanceservicesession(void *dummyPt) { graph_stream_start_command(connFd, streamHandler, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::SEND_PRIORITY) == 0) { send_priority_command(connFd, &loop_exit); - } else if (line.compare(JasmineGraphInstanceProtocol::PUSH_FILE) == 0) { - push_file_command(connFd, &loop_exit); + } else if (line.compare(JasmineGraphInstanceProtocol::PUSH_PARTITION) == 0) { + push_partition_command(connFd, &loop_exit); } else { instance_logger.error("Invalid command"); loop_exit = true; @@ -2051,9 +2051,13 @@ static void batch_upload_common(int connFd, bool *loop_exit_p, bool batch_upload instance_logger.info("File received and saved to " + fullFilePath); *loop_exit_p = true; - Utils::unzipFile(fullFilePath); - size_t lastindex = fileName.find_last_of("."); - string rawname = fileName.substr(0, lastindex); + string rawname = fileName; + if (fullFilePath.compare(fullFilePath.size() - 3, 3, ".gz") == 0) { + Utils::unzipFile(fullFilePath); + size_t lastindex = fileName.find_last_of("."); + rawname = fileName.substr(0, lastindex); + } + fullFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + rawname; if (batch_upload) { @@ -4060,7 +4064,7 @@ static void send_priority_command(int connFd, bool *loop_exit_p) { highestPriority = retrievedPriority; } -static void push_file_command(int connFd, bool *loop_exit_p) { +static void push_partition_command(int connFd, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; return; @@ -4079,15 +4083,29 @@ static void push_file_command(int connFd, bool *loop_exit_p) { } instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); - string fileName = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); - instance_logger.info("Received fileName: " + fileName); - string path = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + fileName; - if (!Utils::sendFileThroughService(host, port, fileName, path)) { - instance_logger.error("Sending failed"); - if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::ERROR)) { - *loop_exit_p = true; + string graphIDPartitionID = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); + instance_logger.info("Received graphID,partionID: " + graphIDPartitionID); + std::vector graphPartitionList = Utils::split(graphIDPartitionID, ','); + int graphID = std::stoi(graphPartitionList[0]); + int partitionID = std::stoi(graphPartitionList[1]); + + std::vector fileList = { + to_string(graphID) + "_" + to_string(partitionID), + to_string(graphID) + "_centralstore_" + to_string(partitionID), + to_string(graphID) + "_centralstore_dp_" + to_string(partitionID) + }; + + for (auto it = fileList.begin(); it != fileList.end(); it++) { + std::string fileName = *it; + std::string path = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + fileName; + + if (!Utils::sendFileThroughService(host, port, fileName, path)) { + instance_logger.error("Sending failed"); + if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::ERROR)) { + *loop_exit_p = true; + } + return; } - return; } if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; From 93ea47be0987083cc0a849fb5651872261bca106 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Thu, 7 Mar 2024 02:30:25 +0530 Subject: [PATCH 065/143] Reduce memory leaks --- src/query/algorithms/triangles/Triangles.cpp | 36 ++++--- src/query/algorithms/triangles/Triangles.h | 12 +-- src/server/JasmineGraphInstanceService.cpp | 99 ++++++++++---------- src/server/JasmineGraphInstanceService.h | 14 +-- 4 files changed, 77 insertions(+), 84 deletions(-) diff --git a/src/query/algorithms/triangles/Triangles.cpp b/src/query/algorithms/triangles/Triangles.cpp index 30cf1a06e..fbbef18c2 100644 --- a/src/query/algorithms/triangles/Triangles.cpp +++ b/src/query/algorithms/triangles/Triangles.cpp @@ -24,13 +24,13 @@ limitations under the License. Logger triangle_logger; -long Triangles::run(JasmineGraphHashMapLocalStore graphDB, JasmineGraphHashMapCentralStore centralStore, - JasmineGraphHashMapDuplicateCentralStore duplicateCentralStore, std::string hostName) { +long Triangles::run(JasmineGraphHashMapLocalStore &graphDB, JasmineGraphHashMapCentralStore ¢ralStore, + JasmineGraphHashMapDuplicateCentralStore &duplicateCentralStore, std::string hostName) { return run(graphDB, centralStore, duplicateCentralStore, NULL, NULL, 0); } -long Triangles::run(JasmineGraphHashMapLocalStore graphDB, JasmineGraphHashMapCentralStore centralStore, - JasmineGraphHashMapDuplicateCentralStore duplicateCentralStore, std::string graphId, +long Triangles::run(JasmineGraphHashMapLocalStore &graphDB, JasmineGraphHashMapCentralStore ¢ralStore, + JasmineGraphHashMapDuplicateCentralStore &duplicateCentralStore, std::string graphId, std::string partitionId, int threadPriority) { triangle_logger.log("###TRIANGLE### Triangle Counting: Started", "info"); map> localSubGraphMap = graphDB.getUnderlyingHashMap(); @@ -39,22 +39,17 @@ long Triangles::run(JasmineGraphHashMapLocalStore graphDB, JasmineGraphHashMapCe map degreeDistribution = graphDB.getOutDegreeDistributionHashMap(); map centralDBDegreeDistribution = centralStore.getOutDegreeDistributionHashMap(); map centralDuplicateDBDegreeDistribution = duplicateCentralStore.getOutDegreeDistributionHashMap(); - std::map> degreeMap; - - std::map::iterator it; - std::map::iterator centralDBDegreeDistributionIterator; - std::map::iterator centralDuplicateDBDegreeDistributionIterator; auto mergeBbegin = std::chrono::high_resolution_clock::now(); - for (centralDuplicateDBDegreeDistributionIterator = centralDuplicateDBDegreeDistribution.begin(); + for (auto centralDuplicateDBDegreeDistributionIterator = centralDuplicateDBDegreeDistribution.begin(); centralDuplicateDBDegreeDistributionIterator != centralDuplicateDBDegreeDistribution.end(); ++centralDuplicateDBDegreeDistributionIterator) { long centralDuplicateDBStartVid = centralDuplicateDBDegreeDistributionIterator->first; - unordered_set centralDBSecondVertexSet = centralDBSubGraphMap[centralDuplicateDBStartVid]; - unordered_set duplicateSecondVertexSet = duplicateCentralDBSubGraphMap[centralDuplicateDBStartVid]; - std::set result; + unordered_set ¢ralDBSecondVertexSet = centralDBSubGraphMap[centralDuplicateDBStartVid]; + const unordered_set &duplicateSecondVertexSet = duplicateCentralDBSubGraphMap[centralDuplicateDBStartVid]; + unordered_set result; std::set_difference(duplicateSecondVertexSet.begin(), duplicateSecondVertexSet.end(), centralDBSecondVertexSet.begin(), centralDBSecondVertexSet.end(), @@ -62,12 +57,12 @@ long Triangles::run(JasmineGraphHashMapLocalStore graphDB, JasmineGraphHashMapCe if (result.size() > 0) { centralDBDegreeDistribution[centralDuplicateDBStartVid] += result.size(); - centralDBSubGraphMap[centralDuplicateDBStartVid].insert(result.begin(), result.end()); + centralDBSecondVertexSet.insert(result.begin(), result.end()); } } // Merging Local Store and Workers central stores before starting triangle count - for (centralDBDegreeDistributionIterator = centralDBDegreeDistribution.begin(); + for (auto centralDBDegreeDistributionIterator = centralDBDegreeDistribution.begin(); centralDBDegreeDistributionIterator != centralDBDegreeDistribution.end(); ++centralDBDegreeDistributionIterator) { long centralDBStartVid = centralDBDegreeDistributionIterator->first; @@ -105,23 +100,24 @@ TriangleResult Triangles::countTriangles(map> ¢ral long triangleCount = 0; std::unordered_map>> triangleTree; + // TODO(thevindu-w): Make centralstore undirected and prevent saving triangles in-memory for (auto iterator = degreeMap.begin(); iterator != degreeMap.end(); ++iterator) { - std::set &vertices = iterator->second; + auto &vertices = iterator->second; for (auto verticesIterator = vertices.begin(); verticesIterator != vertices.end(); ++verticesIterator) { long temp = *verticesIterator; - std::unordered_set &unorderedUSet = centralStore[temp]; + auto &unorderedUSet = centralStore[temp]; for (auto uSetIterator = unorderedUSet.begin(); uSetIterator != unorderedUSet.end(); ++uSetIterator) { long u = *uSetIterator; if (temp == u) continue; - std::unordered_set &unorderedNuSet = centralStore[u]; + auto &unorderedNuSet = centralStore[u]; for (auto nuSetIterator = unorderedNuSet.begin(); nuSetIterator != unorderedNuSet.end(); ++nuSetIterator) { long nu = *nuSetIterator; if (temp == nu) continue; if (u == nu) continue; - std::unordered_set ¢ralStoreNu = centralStore[nu]; + auto ¢ralStoreNu = centralStore[nu]; if ((unorderedUSet.find(nu) != unorderedUSet.end()) || (centralStoreNu.find(temp) != centralStoreNu.end())) { long varOne = temp; @@ -152,6 +148,7 @@ TriangleResult Triangles::countTriangles(map> ¢ral set2.insert(varThree); triangleCount++; if (returnTriangles) { + // TODO(thevindu-w): Flush to file on count exceeds value triangleStream << varOne << "," << varTwo << "," << varThree << ":"; } } @@ -159,6 +156,7 @@ TriangleResult Triangles::countTriangles(map> ¢ral triangleTree[varOne][varTwo].insert(varThree); triangleCount++; if (returnTriangles) { + // TODO(thevindu-w): Flush to file on count exceeds value triangleStream << varOne << "," << varTwo << "," << varThree << ":"; } } diff --git a/src/query/algorithms/triangles/Triangles.h b/src/query/algorithms/triangles/Triangles.h index 665868676..03728c89a 100644 --- a/src/query/algorithms/triangles/Triangles.h +++ b/src/query/algorithms/triangles/Triangles.h @@ -37,15 +37,15 @@ class JasmineGraphHashMapDuplicateCentralStore; class Triangles { public: - static long run(JasmineGraphHashMapLocalStore graphDB, JasmineGraphHashMapCentralStore centralStore, - JasmineGraphHashMapDuplicateCentralStore duplicateCentralStore, std::string hostName); + static long run(JasmineGraphHashMapLocalStore &graphDB, JasmineGraphHashMapCentralStore ¢ralStore, + JasmineGraphHashMapDuplicateCentralStore &duplicateCentralStore, std::string hostName); - static long run(JasmineGraphHashMapLocalStore graphDB, JasmineGraphHashMapCentralStore centralStore, - JasmineGraphHashMapDuplicateCentralStore duplicateCentralStore, std::string graphId, + static long run(JasmineGraphHashMapLocalStore &graphDB, JasmineGraphHashMapCentralStore ¢ralStore, + JasmineGraphHashMapDuplicateCentralStore &duplicateCentralStore, std::string graphId, std::string partitionId, int threadPriority); - static TriangleResult countTriangles(map> ¢ralStore, - map &distributionMap, bool returnTriangles); + static TriangleResult countTriangles(map> ¢ralStore, map &distributionMap, + bool returnTriangles); }; #endif // JASMINEGRAPH_TRIANGLES_H diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index b1356496d..94f6fb7f6 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -122,6 +122,13 @@ static void degree_distribution_common(int connFd, int serverPort, std::map &graphDBMapCentralStores, bool *loop_exit_p, bool in); static void push_partition_command(int connFd, bool *loop_exit_p); +static void push_file_command(int connFd, bool *loop_exit_p); +long countLocalTriangles( + std::string graphId, std::string partitionId, + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::map &graphDBMapDuplicateCentralStores, + int threadPriority); char *converter(const std::string &s) { char *pc = new char[s.size() + 1]; @@ -134,13 +141,13 @@ void *instanceservicesession(void *dummyPt) { instanceservicesessionargs sessionargs = *sessionargs_p; delete sessionargs_p; int connFd = sessionargs.connFd; - std::map graphDBMapLocalStores = sessionargs.graphDBMapLocalStores; - std::map graphDBMapCentralStores = + std::map *graphDBMapLocalStores = sessionargs.graphDBMapLocalStores; + std::map *graphDBMapCentralStores = sessionargs.graphDBMapCentralStores; - std::map graphDBMapDuplicateCentralStores = + std::map *graphDBMapDuplicateCentralStores = sessionargs.graphDBMapDuplicateCentralStores; - std::map incrementalLocalStoreMap = - sessionargs.incrementalLocalStore; + std::map &incrementalLocalStoreMap = + *(sessionargs.incrementalLocalStore); InstanceStreamHandler streamHandler(incrementalLocalStoreMap); string serverName = sessionargs.host; @@ -186,26 +193,27 @@ void *instanceservicesession(void *dummyPt) { } else if (line.compare(JasmineGraphInstanceProtocol::DP_CENTRALSTORE) == 0) { duplicate_centralstore_command(connFd, serverPort, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::WORKER_IN_DEGREE_DISTRIBUTION) == 0) { - worker_in_degree_distribution_command(connFd, graphDBMapLocalStores, graphDBMapCentralStores, &loop_exit); + worker_in_degree_distribution_command(connFd, *graphDBMapLocalStores, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::IN_DEGREE_DISTRIBUTION) == 0) { - in_degree_distribution_command(connFd, serverPort, graphDBMapLocalStores, graphDBMapCentralStores, + in_degree_distribution_command(connFd, serverPort, *graphDBMapLocalStores, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::WORKER_OUT_DEGREE_DISTRIBUTION) == 0) { - worker_out_degree_distribution_command(connFd, graphDBMapLocalStores, graphDBMapCentralStores, &loop_exit); + worker_out_degree_distribution_command(connFd, *graphDBMapLocalStores, *graphDBMapCentralStores, + &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::OUT_DEGREE_DISTRIBUTION) == 0) { - out_degree_distribution_command(connFd, serverPort, graphDBMapLocalStores, graphDBMapCentralStores, + out_degree_distribution_command(connFd, serverPort, *graphDBMapLocalStores, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::PAGE_RANK) == 0) { - page_rank_command(connFd, serverPort, graphDBMapCentralStores, &loop_exit); + page_rank_command(connFd, serverPort, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::WORKER_PAGE_RANK_DISTRIBUTION) == 0) { - worker_page_rank_distribution_command(connFd, serverPort, graphDBMapCentralStores, &loop_exit); + worker_page_rank_distribution_command(connFd, serverPort, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::EGONET) == 0) { - egonet_command(connFd, serverPort, graphDBMapCentralStores, &loop_exit); + egonet_command(connFd, serverPort, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::WORKER_EGO_NET) == 0) { - worker_egonet_command(connFd, serverPort, graphDBMapCentralStores, &loop_exit); + worker_egonet_command(connFd, serverPort, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::TRIANGLES) == 0) { - triangles_command(connFd, serverPort, graphDBMapLocalStores, graphDBMapCentralStores, - graphDBMapDuplicateCentralStores, &loop_exit); + triangles_command(connFd, serverPort, *graphDBMapLocalStores, *graphDBMapCentralStores, + *graphDBMapDuplicateCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::INITIATE_STREAMING_TRIAN) == 0) { streaming_triangles_command(connFd, serverPort, incrementalLocalStoreMap, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::SEND_CENTRALSTORE_TO_AGGREGATOR) == 0) { @@ -314,10 +322,10 @@ void JasmineGraphInstanceService::run(string masterHost, string host, int server } instance_logger.info("Connection successful to port " + to_string(serverPort)); instanceservicesessionargs *serviceArguments_p = new instanceservicesessionargs; - serviceArguments_p->graphDBMapLocalStores = graphDBMapLocalStores; - serviceArguments_p->graphDBMapCentralStores = graphDBMapCentralStores; - serviceArguments_p->graphDBMapDuplicateCentralStores = graphDBMapDuplicateCentralStores; - serviceArguments_p->incrementalLocalStore = incrementalLocalStore; + serviceArguments_p->graphDBMapLocalStores = &graphDBMapLocalStores; + serviceArguments_p->graphDBMapCentralStores = &graphDBMapCentralStores; + serviceArguments_p->graphDBMapDuplicateCentralStores = &graphDBMapDuplicateCentralStores; + serviceArguments_p->incrementalLocalStore = &incrementalLocalStore; serviceArguments_p->masterHost = masterHost; serviceArguments_p->port = serverPort; serviceArguments_p->dataPort = serverDataPort; @@ -389,35 +397,30 @@ long countLocalTriangles( std::string graphIdentifier = graphId + "_" + partitionId; std::string centralGraphIdentifier = graphId + "_centralstore_" + partitionId; std::string duplicateCentralGraphIdentifier = graphId + "_centralstore_dp_" + partitionId; - JasmineGraphHashMapLocalStore graphDB; - JasmineGraphHashMapCentralStore centralGraphDB; - JasmineGraphHashMapDuplicateCentralStore duplicateCentralGraphDB; - std::map::iterator localMapIterator = - graphDBMapLocalStores.find(graphIdentifier); - std::map::iterator centralStoreIterator = - graphDBMapCentralStores.find(graphIdentifier); - std::map::iterator duplicateCentralStoreIterator = - graphDBMapDuplicateCentralStores.find(graphIdentifier); + auto localMapIterator = graphDBMapLocalStores.find(graphIdentifier); + auto centralStoreIterator = graphDBMapCentralStores.find(graphIdentifier); + auto duplicateCentralStoreIterator = graphDBMapDuplicateCentralStores.find(graphIdentifier); if (localMapIterator == graphDBMapLocalStores.end() && JasmineGraphInstanceService::isGraphDBExists(graphId, partitionId)) { JasmineGraphInstanceService::loadLocalStore(graphId, partitionId, graphDBMapLocalStores); } - graphDB = graphDBMapLocalStores[graphIdentifier]; + JasmineGraphHashMapLocalStore graphDB = graphDBMapLocalStores[graphIdentifier]; if (centralStoreIterator == graphDBMapCentralStores.end() && JasmineGraphInstanceService::isInstanceCentralStoreExists(graphId, partitionId)) { JasmineGraphInstanceService::loadInstanceCentralStore(graphId, partitionId, graphDBMapCentralStores); } - centralGraphDB = graphDBMapCentralStores[centralGraphIdentifier]; + JasmineGraphHashMapCentralStore centralGraphDB = graphDBMapCentralStores[centralGraphIdentifier]; if (duplicateCentralStoreIterator == graphDBMapDuplicateCentralStores.end() && JasmineGraphInstanceService::isInstanceDuplicateCentralStoreExists(graphId, partitionId)) { JasmineGraphInstanceService::loadInstanceDuplicateCentralStore(graphId, partitionId, graphDBMapDuplicateCentralStores); } - duplicateCentralGraphDB = graphDBMapDuplicateCentralStores[duplicateCentralGraphIdentifier]; + JasmineGraphHashMapDuplicateCentralStore duplicateCentralGraphDB = + graphDBMapDuplicateCentralStores[duplicateCentralGraphIdentifier]; result = Triangles::run(graphDB, centralGraphDB, duplicateCentralGraphDB, graphId, partitionId, threadPriority); @@ -468,27 +471,27 @@ JasmineGraphIncrementalLocalStore *JasmineGraphInstanceService::loadStreamingSto instance_logger.info("###INSTANCE### Loading Local Store : Completed"); return jasmineGraphStreamingLocalStore; } + void JasmineGraphInstanceService::loadLocalStore( std::string graphId, std::string partitionId, std::map &graphDBMapLocalStores) { instance_logger.info("###INSTANCE### Loading Local Store : Started"); std::string graphIdentifier = graphId + "_" + partitionId; std::string folderLocation = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); - JasmineGraphHashMapLocalStore *jasmineGraphHashMapLocalStore = - new JasmineGraphHashMapLocalStore(stoi(graphId), stoi(partitionId), folderLocation); - jasmineGraphHashMapLocalStore->loadGraph(); - graphDBMapLocalStores[graphIdentifier] = *jasmineGraphHashMapLocalStore; + JasmineGraphHashMapLocalStore jasmineGraphHashMapLocalStore(stoi(graphId), stoi(partitionId), folderLocation); + jasmineGraphHashMapLocalStore.loadGraph(); + graphDBMapLocalStores[graphIdentifier] = jasmineGraphHashMapLocalStore; instance_logger.info("###INSTANCE### Loading Local Store : Completed"); } + void JasmineGraphInstanceService::loadInstanceCentralStore( std::string graphId, std::string partitionId, std::map &graphDBMapCentralStores) { instance_logger.info("###INSTANCE### Loading central Store : Started"); std::string graphIdentifier = graphId + "_centralstore_" + partitionId; - JasmineGraphHashMapCentralStore *jasmineGraphHashMapCentralStore = - new JasmineGraphHashMapCentralStore(stoi(graphId), stoi(partitionId)); - jasmineGraphHashMapCentralStore->loadGraph(); - graphDBMapCentralStores[graphIdentifier] = *jasmineGraphHashMapCentralStore; + JasmineGraphHashMapCentralStore jasmineGraphHashMapCentralStore(stoi(graphId), stoi(partitionId)); + jasmineGraphHashMapCentralStore.loadGraph(); + graphDBMapCentralStores[graphIdentifier] = jasmineGraphHashMapCentralStore; instance_logger.info("###INSTANCE### Loading central Store : Completed"); } @@ -496,10 +499,9 @@ void JasmineGraphInstanceService::loadInstanceDuplicateCentralStore( std::string graphId, std::string partitionId, std::map &graphDBMapDuplicateCentralStores) { std::string graphIdentifier = graphId + "_centralstore_dp_" + partitionId; - JasmineGraphHashMapDuplicateCentralStore *jasmineGraphHashMapCentralStore = - new JasmineGraphHashMapDuplicateCentralStore(stoi(graphId), stoi(partitionId)); - jasmineGraphHashMapCentralStore->loadGraph(); - graphDBMapDuplicateCentralStores[graphIdentifier] = *jasmineGraphHashMapCentralStore; + JasmineGraphHashMapDuplicateCentralStore jasmineGraphHashMapCentralStore(stoi(graphId), stoi(partitionId)); + jasmineGraphHashMapCentralStore.loadGraph(); + graphDBMapDuplicateCentralStores[graphIdentifier] = jasmineGraphHashMapCentralStore; } JasmineGraphHashMapCentralStore *JasmineGraphInstanceService::loadCentralStore(std::string centralStoreFileName) { @@ -4089,15 +4091,14 @@ static void push_partition_command(int connFd, bool *loop_exit_p) { int graphID = std::stoi(graphPartitionList[0]); int partitionID = std::stoi(graphPartitionList[1]); - std::vector fileList = { - to_string(graphID) + "_" + to_string(partitionID), - to_string(graphID) + "_centralstore_" + to_string(partitionID), - to_string(graphID) + "_centralstore_dp_" + to_string(partitionID) - }; + std::vector fileList = {to_string(graphID) + "_" + to_string(partitionID), + to_string(graphID) + "_centralstore_" + to_string(partitionID), + to_string(graphID) + "_centralstore_dp_" + to_string(partitionID)}; for (auto it = fileList.begin(); it != fileList.end(); it++) { std::string fileName = *it; - std::string path = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + fileName; + std::string path = + Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + fileName; if (!Utils::sendFileThroughService(host, port, fileName, path)) { instance_logger.error("Sending failed"); diff --git a/src/server/JasmineGraphInstanceService.h b/src/server/JasmineGraphInstanceService.h index ea960e4c3..86614ba28 100644 --- a/src/server/JasmineGraphInstanceService.h +++ b/src/server/JasmineGraphInstanceService.h @@ -50,12 +50,6 @@ void *instanceservicesession(void *dummyPt); void writeCatalogRecord(string record); int deleteGraphPartition(std::string graphID, std::string partitionID); void removeGraphFragments(std::string graphID); -long countLocalTriangles( - std::string graphId, std::string partitionId, - std::map &graphDBMapLocalStores, - std::map &graphDBMapCentralStores, - std::map &graphDBMapDuplicateCentralStores, - int threadPriority); map calculateOutDegreeDist(string graphID, string partitionID, int serverPort, std::map &graphDBMapLocalStores, @@ -107,10 +101,10 @@ struct instanceservicesessionargs { int connFd; int port; int dataPort; - std::map graphDBMapLocalStores; - std::map graphDBMapCentralStores; - std::map graphDBMapDuplicateCentralStores; - std::map incrementalLocalStore; + std::map *graphDBMapLocalStores; + std::map *graphDBMapCentralStores; + std::map *graphDBMapDuplicateCentralStores; + std::map *incrementalLocalStore; }; class JasmineGraphInstanceService { From 2d4a62e7b2372d8706b539138d44daa0b8b13fae Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 7 Mar 2024 06:51:22 +0530 Subject: [PATCH 066/143] Implement function to initiate partition transfer --- src/util/Utils.cpp | 79 ++++++++++++++++++++++++++++++++++++++++++++++ src/util/Utils.h | 8 +++++ 2 files changed, 87 insertions(+) diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 55dd5f7f9..db14647e8 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -1033,3 +1033,82 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f close(sockfd); return status; } + +/* + * Function to transfer a partition from one worker to another + * Caller should ensure that the partition exists in the source worker + * and the destination worker is ready to accept the partition + * Also, the caller should update the worker_has_partition table. + * */ +bool Utils::transferPartition(std::string sourceWorker, + int sourceWorkerPort, + int sourceWorkerDataPort, + std::string destinationWorker, + int destinationWorkerPort, + int destinationWorkerDataPort, + int graphID, + int partitionID) { + util_logger.info("### Transferring partition " + to_string(partitionID) + " of graph " + to_string(graphID) + + " from " + sourceWorker + " to " + destinationWorker); + + int sockfd; + char data[FED_DATA_LENGTH + 1]; + struct sockaddr_in serv_addr; + struct hostent *server; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + util_logger.error("Cannot create socket"); + return false; + } + + server = gethostbyname(sourceWorker.c_str()); + if (server == NULL) { + util_logger.error("ERROR, no host named " + sourceWorker); + return false; + } + + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(sourceWorkerPort); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + return false; + } + + if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, destinationWorker)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::PUSH_PARTITION, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, + destinationWorker + ":" + to_string(destinationWorkerDataPort), + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, + to_string(graphID) + "," + to_string(partitionID), + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + util_logger.info("### Transfer partition completed"); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return true; +} + + diff --git a/src/util/Utils.h b/src/util/Utils.h index f70be5192..e67c25410 100644 --- a/src/util/Utils.h +++ b/src/util/Utils.h @@ -178,6 +178,14 @@ class Utils { std::string masterIP, std::string uploadType); static bool sendFileThroughService(std::string host, int dataPort, std::string fileName, std::string filePath); + + static bool transferPartition(std::string sourceWorker, + int sourceWorkerPort, + int sourceWorkerDataPort, + std::string destinationWorker, + int destinationWorkerPort, + int destinationWorkerDataPort, + int graphID, int partitionID); }; #endif // JASMINEGRAPH_UTILS_H From ff8496946eff4ae46f70aefc81b12c715add6ecf Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 8 Mar 2024 17:18:52 +0530 Subject: [PATCH 067/143] Assign partitions for all workers in round robin manner --- src/frontend/JasmineGraphFrontEnd.cpp | 4 ---- src/server/JasmineGraphServer.cpp | 12 +++++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index 651b8c0aa..72120e2bb 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -797,10 +797,6 @@ static void add_graph_command(std::string masterIP, int connFd, SQLiteDBInterfac JasmineGraphServer *server = JasmineGraphServer::getInstance(); server->uploadGraphLocally(newGraphID, Conts::GRAPH_TYPE_NORMAL, fullFileList, masterIP); Utils::deleteDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp/" + to_string(newGraphID)); - string workerCountQuery = "select count(*) from worker"; - std::vector>> results = sqlite->runSelect(workerCountQuery); - string workerCount = results[0][0].second; - int nWorkers = atoi(workerCount.c_str()); JasmineGraphFrontEnd::getAndUpdateUploadTime(to_string(newGraphID), sqlite); int result_wr = write(connFd, DONE.c_str(), DONE.size()); if (result_wr < 0) { diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 8ed7cf2e9..3b6d444c4 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -24,9 +24,6 @@ limitations under the License. #include "../../globals.h" #include "../k8s/K8sWorkerController.h" #include "../ml/trainer/JasmineGraphTrainingSchedular.h" -#include "../partitioner/local/MetisPartitioner.h" -#include "../util/Utils.h" -#include "../util/logger/Logger.h" #include "JasmineGraphInstance.h" #include "JasmineGraphInstanceProtocol.h" @@ -69,6 +66,8 @@ static std::vector hostWorkerList; static unordered_map> hostPortMap; std::map aggregateWeightMap; +static int graphUploadWorkerTracker = 0; + void *runfrontend(void *dummyPt) { JasmineGraphServer *refToServer = (JasmineGraphServer *)dummyPt; refToServer->frontend = new JasmineGraphFrontEnd(refToServer->sqlite, refToServer->performanceSqlite, @@ -894,11 +893,11 @@ void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, std::thread *workerThreads = new std::thread[total_threads]; while (count < total_threads) { const auto &workerList = getWorkers(partitionFileMap.size()); - for (auto listIterator = workerList.begin(); listIterator < workerList.end(); listIterator++) { - worker worker = *listIterator; + while (true) { if (count >= total_threads) { break; } + worker worker = workerList[graphUploadWorkerTracker]; std::string partitionFileName = partitionFileMap[file_count]; workerThreads[count++] = std::thread(batchUploadFile, worker.hostname, worker.port, worker.dataPort, graphID, partitionFileName, masterHost); @@ -925,6 +924,9 @@ void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, } assignPartitionToWorker(partitionFileName, graphID, worker.hostname, worker.port); file_count++; + + graphUploadWorkerTracker++; + graphUploadWorkerTracker %= workerList.size(); } } From 5d5f81b4803028ffc3a771122f9a855a89553be0 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Mon, 11 Mar 2024 15:59:04 +0530 Subject: [PATCH 068/143] Implement allocate plan --- .../executor/impl/TriangleCountExecutor.cpp | 208 +++++++++++++++--- 1 file changed, 183 insertions(+), 25 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index e19c4e59e..858bd5f3e 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -39,6 +39,163 @@ TriangleCountExecutor::TriangleCountExecutor(SQLiteDBInterface *db, PerformanceS this->request = jobRequest; } +static void allocate(int p, + int w, + std::map &alloc, + std::set &remain, + std::map> &p_avail, + std::map &loads) { + alloc[p] = w; + remain.erase(p); + p_avail.erase(p); + loads[w]++; + if (loads[w] >= 3) { + for (auto it = p_avail.begin(); it != p_avail.end(); it++) { + auto &ws = it->second; + auto itr = std::find(ws.begin(), ws.end(), w); + if (itr != ws.end()) { + ws.erase(itr); + } + } + } +} + +static int get_min_partition(std::set &remain, std::map> &p_avail) { + int p0 = *remain.begin(); + size_t m = 1000000000; + for (auto it = remain.begin(); it != remain.end(); it++) { + int p = *it; + auto &ws = p_avail[p]; + if (ws.size() > 0 && ws.size() < m) { + m = ws.size(); + p0 = p; + } + } + return p0; +} + +static const std::vector LOAD_PREFERENCE = {2, 3, 1, 0}; +static int compare_loads(int w1, int w2, std::map &loads) { + if (loads[w1] == loads[w2]) return 0; + return LOAD_PREFERENCE[loads[w1]] < LOAD_PREFERENCE[loads[w2]]; +} + +static int alloc_plan(std::map &alloc, + std::set &remain, + std::map> &p_avail, + std::map &loads) { + for (bool done = false; !done;) { + int w = -1; + done = true; + int p; + for (auto it = remain.begin(); it != remain.end(); it++) { + p = *it; + if (p_avail[p].size() == 1) { + w = p_avail[p][0]; + done = false; + } + } + if (w >= 0) allocate(p, w, alloc, remain, p_avail, loads); + } + if (remain.empty()) + return 0; + int p0 = get_min_partition(remain, p_avail); + auto &ws = p_avail[p0]; + if (ws.empty()) + return (int) remain.size(); + sort(ws.begin(), ws.end(), [&loads](int &w1, int &w2) { + return LOAD_PREFERENCE[loads[w1]] > LOAD_PREFERENCE[loads[w2]]; + }); // load=1 goes first and load=3 goes last + struct best_alloc { + std::map alloc; + std::set remain; + std::map> p_avail; + std::map loads; + }; + int best_rem = remain.size(); + struct best_alloc best = {.alloc = alloc, .remain = remain, .p_avail = p_avail, .loads = loads}; + for (auto it = ws.begin(); it != ws.end(); it++) { + int w = *it; + auto alloc2 = alloc; // need copy => do not copy reference + auto remain2 = remain; // need copy => do not copy reference + auto p_avail2 = p_avail; // need copy => do not copy reference + auto loads2 = loads; // need copy => do not copy reference + allocate(p0, w, alloc2, remain2, p_avail2, loads2); + int rem = alloc_plan(alloc2, remain2, p_avail2, loads2); + if (rem == 0) { + remain.clear(); + p_avail.clear(); + alloc.insert(alloc2.begin(), alloc2.end()); + loads.insert(loads2.begin(), loads2.end()); + return 0; + } + if (rem < best_rem) { + best_rem = rem; + best = {.alloc = alloc2, .remain = remain2, .p_avail = p_avail2, .loads = loads2}; + } + alloc.insert(best.alloc.begin(), best.alloc.end()); + remain.clear(); + remain.insert(best.remain.begin(), best.remain.end()); + p_avail.clear(); + p_avail.insert(best.p_avail.begin(), best.p_avail.end()); + loads.clear(); + loads.insert(best.loads.begin(), best.loads.end()); + return best_rem; + } +} + +static void reallocate_parts(std::map &alloc, + std::set &remain, + const std::map> &P_AVAIL, + std::map &loads) { + map P_COUNT; + for (auto it = P_AVAIL.begin(); it != P_AVAIL.end(); it++) { + P_COUNT[it->first] = it->second.size(); + } + vector remain_l(remain.begin(), remain.end()); + sort(remain_l.begin(), remain_l.end(), [&P_COUNT](int &p1, int &p2) { + return P_COUNT[p1] > P_COUNT[p2]; + }); // partitions with more copies goes first + vector PARTITIONS; + for (auto it = P_COUNT.begin(); it != P_COUNT.end(); it++) { + PARTITIONS.push_back(it->first); + } + sort(PARTITIONS.begin(), PARTITIONS.end(), [&P_COUNT](int &p1, int &p2) { + return P_COUNT[p1] < P_COUNT[p2]; + }); // partitions with fewer copies goes first + vector copying; + while (!remain_l.empty()) { + int p0 = remain_l.back(); + int w_cnt = P_COUNT[p0]; + if (w_cnt == 1) { + copying.push_back(p0); + continue; + } + bool need_pushing = true; + for (auto it = PARTITIONS.begin(); it != PARTITIONS.end(); it++) { + int p = *it; + if (w_cnt <= P_COUNT[p]) { + copying.push_back(p0); // assuming PARTITIONS are in sorted order of copy count + need_pushing = false; + break; + } + if (alloc.find(p) == alloc.end()) { + continue; + } + int w = alloc[p]; + const auto &ws = P_AVAIL.find(p0)->second; + if (std::find(ws.begin(), ws.end(), w) != ws.end()) { + alloc.erase(p); + alloc[p0] = w; + remain_l.push_back(p); + need_pushing = false; + break; + } + } + if (need_pushing) copying.push_back(p0); + } +} + void TriangleCountExecutor::execute() { int uniqueId = getUid(); std::string masterIP = request.getMasterIP(); @@ -100,7 +257,7 @@ void TriangleCountExecutor::execute() { "SELECT worker_idworker, name,ip,user,server_port,server_data_port,partition_idpartition " "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + - graphId + ";"; + graphId + ";"; std::vector>> results = sqlite->runSelect(sqlStatement); @@ -180,9 +337,10 @@ void TriangleCountExecutor::execute() { std::string query = "SELECT attempt from graph_sla INNER JOIN sla_category where graph_sla.id_sla_category=sla_category.id and " "graph_sla.graph_id='" + - graphId + "' and graph_sla.partition_count='" + std::to_string(partitionCount) + - "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + "' and sla_category.command='" + TRIANGLES + - "';"; + graphId + "' and graph_sla.partition_count='" + std::to_string(partitionCount) + + "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + "' and sla_category.command='" + TRIANGLES + + + "';"; std::vector>> queryResults = perfDB->runSelect(query); @@ -285,11 +443,11 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int return 0; } - bzero((char *)&serv_addr, sizeof(serv_addr)); + bzero((char *) &serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return 0; } @@ -643,7 +801,7 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq "SELECT ip,server_port,server_data_port,partition_idpartition " "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + - graphId + " and idworker=" + minWeightWorker + ";"; + graphId + " and idworker=" + minWeightWorker + ";"; const std::vector>> &result = sqlite->runSelect(aggregatorSqlStatement); @@ -664,7 +822,7 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq "FROM worker_has_partition INNER JOIN worker ON " "worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + - graphId + " and idworker=" + workerId + ";"; + graphId + " and idworker=" + workerId + ";"; const std::vector>> &result = sqlite->runSelect(sqlStatement); @@ -744,11 +902,11 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio return 0; } - bzero((char *)&serv_addr, sizeof(serv_addr)); + bzero((char *) &serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; // TODO::exit return 0; @@ -893,11 +1051,11 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st return 0; } - bzero((char *)&serv_addr, sizeof(serv_addr)); + bzero((char *) &serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; // TODO::exit return 0; @@ -1076,11 +1234,11 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag return 0; } - bzero((char *)&serv_addr, sizeof(serv_addr)); + bzero((char *) &serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; // TODO::exit return 0; @@ -1236,7 +1394,7 @@ static std::vector> getWorkerCombination(SQLiteDBInterface * "SELECT worker_idworker " "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + - graphId + ";"; + graphId + ";"; const std::vector>> &results = sqlite->runSelect(sqlStatement); @@ -1283,11 +1441,11 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr return ""; } - bzero((char *)&serv_addr, sizeof(serv_addr)); + bzero((char *) &serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return ""; } @@ -1463,11 +1621,11 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP return 0; } - bzero((char *)&serv_addr, sizeof(serv_addr)); + bzero((char *) &serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; // TODO::exit return 0; From 7fb81d6e6df43cc5a79f703516fb31933c2f8e96 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Mon, 11 Mar 2024 20:22:58 +0530 Subject: [PATCH 069/143] Implement alloc_net_plan --- .../executor/impl/TriangleCountExecutor.cpp | 185 +++++++++++------- ...asmineGraphInstanceFileTransferService.cpp | 5 +- src/util/Utils.cpp | 3 - 3 files changed, 120 insertions(+), 73 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 858bd5f3e..942c79e30 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -39,12 +39,8 @@ TriangleCountExecutor::TriangleCountExecutor(SQLiteDBInterface *db, PerformanceS this->request = jobRequest; } -static void allocate(int p, - int w, - std::map &alloc, - std::set &remain, - std::map> &p_avail, - std::map &loads) { +static void allocate(int p, int w, std::map &alloc, std::set &remain, + std::map> &p_avail, std::map &loads) { alloc[p] = w; remain.erase(p); p_avail.erase(p); @@ -80,9 +76,7 @@ static int compare_loads(int w1, int w2, std::map &loads) { return LOAD_PREFERENCE[loads[w1]] < LOAD_PREFERENCE[loads[w2]]; } -static int alloc_plan(std::map &alloc, - std::set &remain, - std::map> &p_avail, +static int alloc_plan(std::map &alloc, std::set &remain, std::map> &p_avail, std::map &loads) { for (bool done = false; !done;) { int w = -1; @@ -97,15 +91,13 @@ static int alloc_plan(std::map &alloc, } if (w >= 0) allocate(p, w, alloc, remain, p_avail, loads); } - if (remain.empty()) - return 0; + if (remain.empty()) return 0; int p0 = get_min_partition(remain, p_avail); auto &ws = p_avail[p0]; - if (ws.empty()) - return (int) remain.size(); + if (ws.empty()) return (int)remain.size(); sort(ws.begin(), ws.end(), [&loads](int &w1, int &w2) { return LOAD_PREFERENCE[loads[w1]] > LOAD_PREFERENCE[loads[w2]]; - }); // load=1 goes first and load=3 goes last + }); // load=1 goes first and load=3 goes last struct best_alloc { std::map alloc; std::set remain; @@ -116,10 +108,10 @@ static int alloc_plan(std::map &alloc, struct best_alloc best = {.alloc = alloc, .remain = remain, .p_avail = p_avail, .loads = loads}; for (auto it = ws.begin(); it != ws.end(); it++) { int w = *it; - auto alloc2 = alloc; // need copy => do not copy reference - auto remain2 = remain; // need copy => do not copy reference - auto p_avail2 = p_avail; // need copy => do not copy reference - auto loads2 = loads; // need copy => do not copy reference + auto alloc2 = alloc; // need copy => do not copy reference + auto remain2 = remain; // need copy => do not copy reference + auto p_avail2 = p_avail; // need copy => do not copy reference + auto loads2 = loads; // need copy => do not copy reference allocate(p0, w, alloc2, remain2, p_avail2, loads2); int rem = alloc_plan(alloc2, remain2, p_avail2, loads2); if (rem == 0) { @@ -133,39 +125,37 @@ static int alloc_plan(std::map &alloc, best_rem = rem; best = {.alloc = alloc2, .remain = remain2, .p_avail = p_avail2, .loads = loads2}; } - alloc.insert(best.alloc.begin(), best.alloc.end()); - remain.clear(); - remain.insert(best.remain.begin(), best.remain.end()); - p_avail.clear(); - p_avail.insert(best.p_avail.begin(), best.p_avail.end()); - loads.clear(); - loads.insert(best.loads.begin(), best.loads.end()); - return best_rem; } + alloc.insert(best.alloc.begin(), best.alloc.end()); + remain.clear(); + remain.insert(best.remain.begin(), best.remain.end()); + p_avail.clear(); + p_avail.insert(best.p_avail.begin(), best.p_avail.end()); + loads.clear(); + loads.insert(best.loads.begin(), best.loads.end()); + return best_rem; } -static void reallocate_parts(std::map &alloc, - std::set &remain, - const std::map> &P_AVAIL, - std::map &loads) { +static void reallocate_parts(std::map &alloc, std::set &remain, + const std::map> &P_AVAIL, std::map &loads) { map P_COUNT; for (auto it = P_AVAIL.begin(); it != P_AVAIL.end(); it++) { P_COUNT[it->first] = it->second.size(); } vector remain_l(remain.begin(), remain.end()); - sort(remain_l.begin(), remain_l.end(), [&P_COUNT](int &p1, int &p2) { - return P_COUNT[p1] > P_COUNT[p2]; - }); // partitions with more copies goes first + sort(remain_l.begin(), remain_l.end(), + [&P_COUNT](int &p1, int &p2) { return P_COUNT[p1] > P_COUNT[p2]; }); // partitions with more copies goes first vector PARTITIONS; for (auto it = P_COUNT.begin(); it != P_COUNT.end(); it++) { PARTITIONS.push_back(it->first); } sort(PARTITIONS.begin(), PARTITIONS.end(), [&P_COUNT](int &p1, int &p2) { return P_COUNT[p1] < P_COUNT[p2]; - }); // partitions with fewer copies goes first + }); // partitions with fewer copies goes first vector copying; while (!remain_l.empty()) { int p0 = remain_l.back(); + remain_l.pop_back(); int w_cnt = P_COUNT[p0]; if (w_cnt == 1) { copying.push_back(p0); @@ -175,7 +165,7 @@ static void reallocate_parts(std::map &alloc, for (auto it = PARTITIONS.begin(); it != PARTITIONS.end(); it++) { int p = *it; if (w_cnt <= P_COUNT[p]) { - copying.push_back(p0); // assuming PARTITIONS are in sorted order of copy count + copying.push_back(p0); // assuming PARTITIONS are in sorted order of copy count need_pushing = false; break; } @@ -196,6 +186,66 @@ static void reallocate_parts(std::map &alloc, } } +static int alloc_net_plan(std::vector parts, std::map> transfer, + std::map net_loads, std::map loads, + const std::map> &p_avail) { + if (parts.empty()) { + if (net_loads.empty()) return 0; + return std::max_element(net_loads.begin(), net_loads.end(), + [](const std::map::value_type &p1, const std::map::value_type &p2) { + return p1.second < p2.second; + }) + ->second; + } + struct best_net_alloc { + std::map> transfer; + std::map net_loads; + std::map loads; + }; + int best = 10000000; // large number + struct best_net_alloc best_plan = {.transfer = transfer, .net_loads = net_loads, .loads = loads}; + int p = parts.back(); + parts.pop_back(); + vector wts; + for (auto it = loads.begin(); it != loads.end(); it++) { + wts.push_back(it->first); + } + sort(wts.begin(), wts.end(), [&loads](int &w1, int &w2) { + return LOAD_PREFERENCE[loads[w1]] > LOAD_PREFERENCE[loads[w2]]; + }); // load=1 goes first and load=3 goes last + const auto &ws = p_avail.find(p)->second; + for (auto itf = ws.begin(); itf != ws.end(); itf++) { + int wf = *itf; + for (auto itt = ws.begin(); itt != ws.end(); itt++) { + int wt = *itt; + auto parts2 = parts; // need copy => do not copy reference + auto transfer2 = transfer; // need copy => do not copy reference + auto net_loads2 = net_loads; // need copy => do not copy reference + auto loads2 = loads; // need copy => do not copy reference + transfer2[p] = {wf, wt}; // assume + net_loads2[wf]++; + net_loads2[wt]++; + loads2[wt]++; + if (loads2[wt] == 3) loads2.erase(wt); + int new_net_load = alloc_net_plan(parts2, transfer2, net_loads2, loads2, p_avail); + if (new_net_load < best) { + if (loads2.find(wt) == loads.end()) loads2[wt] = 3; + best = new_net_load; + best_plan = {.transfer = transfer2, .net_loads = net_loads2, .loads = loads2}; + } + } + } + auto &b_transfer = best_plan.transfer; + for (auto it = b_transfer.begin(); it != b_transfer.end(); it++) { + transfer[it->first] = it->second; + } + net_loads.clear(); + net_loads.insert(best_plan.net_loads.begin(), best_plan.net_loads.end()); + loads.clear(); + loads.insert(best_plan.loads.begin(), best_plan.loads.end()); + return best; +} + void TriangleCountExecutor::execute() { int uniqueId = getUid(); std::string masterIP = request.getMasterIP(); @@ -257,7 +307,7 @@ void TriangleCountExecutor::execute() { "SELECT worker_idworker, name,ip,user,server_port,server_data_port,partition_idpartition " "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + - graphId + ";"; + graphId + ";"; std::vector>> results = sqlite->runSelect(sqlStatement); @@ -286,7 +336,7 @@ void TriangleCountExecutor::execute() { std::map> partitionMap; - for (std::vector>>::iterator i = results.begin(); i != results.end(); ++i) { + for (auto i = results.begin(); i != results.end(); ++i) { std::vector> rowData = *i; string workerID = rowData.at(0).second; @@ -315,15 +365,13 @@ void TriangleCountExecutor::execute() { string host = currentWorker.hostname; string workerID = currentWorker.workerID; string partitionId; + int workerPort = atoi(string(currentWorker.port).c_str()); + int workerDataPort = atoi(string(currentWorker.dataPort).c_str()); - std::vector partitionList = partitionMap[workerID]; - - std::vector::iterator partitionIterator; - - for (partitionIterator = partitionList.begin(); partitionIterator != partitionList.end(); ++partitionIterator) { + const std::vector &partitionList = partitionMap[workerID]; + for (auto partitionIterator = partitionList.begin(); partitionIterator != partitionList.end(); + ++partitionIterator) { partitionCount++; - int workerPort = atoi(string(currentWorker.port).c_str()); - int workerDataPort = atoi(string(currentWorker.dataPort).c_str()); partitionId = *partitionIterator; intermRes.push_back(std::async( @@ -337,10 +385,9 @@ void TriangleCountExecutor::execute() { std::string query = "SELECT attempt from graph_sla INNER JOIN sla_category where graph_sla.id_sla_category=sla_category.id and " "graph_sla.graph_id='" + - graphId + "' and graph_sla.partition_count='" + std::to_string(partitionCount) + - "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + "' and sla_category.command='" + TRIANGLES - + - "';"; + graphId + "' and graph_sla.partition_count='" + std::to_string(partitionCount) + + "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + "' and sla_category.command='" + TRIANGLES + + "';"; std::vector>> queryResults = perfDB->runSelect(query); @@ -443,11 +490,11 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int return 0; } - bzero((char *) &serv_addr, sizeof(serv_addr)); + bzero((char *)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return 0; } @@ -801,7 +848,7 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq "SELECT ip,server_port,server_data_port,partition_idpartition " "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + - graphId + " and idworker=" + minWeightWorker + ";"; + graphId + " and idworker=" + minWeightWorker + ";"; const std::vector>> &result = sqlite->runSelect(aggregatorSqlStatement); @@ -822,7 +869,7 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq "FROM worker_has_partition INNER JOIN worker ON " "worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + - graphId + " and idworker=" + workerId + ";"; + graphId + " and idworker=" + workerId + ";"; const std::vector>> &result = sqlite->runSelect(sqlStatement); @@ -902,11 +949,11 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio return 0; } - bzero((char *) &serv_addr, sizeof(serv_addr)); + bzero((char *)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; // TODO::exit return 0; @@ -1051,11 +1098,11 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st return 0; } - bzero((char *) &serv_addr, sizeof(serv_addr)); + bzero((char *)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; // TODO::exit return 0; @@ -1234,11 +1281,11 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag return 0; } - bzero((char *) &serv_addr, sizeof(serv_addr)); + bzero((char *)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; // TODO::exit return 0; @@ -1394,7 +1441,7 @@ static std::vector> getWorkerCombination(SQLiteDBInterface * "SELECT worker_idworker " "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + - graphId + ";"; + graphId + ";"; const std::vector>> &results = sqlite->runSelect(sqlStatement); @@ -1441,11 +1488,11 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr return ""; } - bzero((char *) &serv_addr, sizeof(serv_addr)); + bzero((char *)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; return ""; } @@ -1621,11 +1668,11 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP return 0; } - bzero((char *) &serv_addr, sizeof(serv_addr)); + bzero((char *)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; - bcopy((char *) server->h_addr, (char *) &serv_addr.sin_addr.s_addr, server->h_length); + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; // TODO::exit return 0; diff --git a/src/server/JasmineGraphInstanceFileTransferService.cpp b/src/server/JasmineGraphInstanceFileTransferService.cpp index d7af6ad94..831679680 100644 --- a/src/server/JasmineGraphInstanceFileTransferService.cpp +++ b/src/server/JasmineGraphInstanceFileTransferService.cpp @@ -42,10 +42,13 @@ void *filetransferservicesession(void *dummyPt) { if (bytesReceived > 0) { file.write(buffer, bytesReceived); fsize -= bytesReceived; + } else if (bytesReceived == 0) { + file_service_logger.error("File transfer failed for file: " + fileName); + break; } } file.close(); - file_service_logger.info("File transfer completed for file: " + fileName); + if (fsize == 0) file_service_logger.info("File transfer completed for file: " + fileName); return NULL; } diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index db14647e8..6e04c2251 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -997,7 +997,6 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, JasmineGraphInstanceProtocol::SEND_FILE_LEN)) { - Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -1005,7 +1004,6 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f int fsize = Utils::getFileSize(filePath); if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, to_string(fsize), JasmineGraphInstanceProtocol::SEND_FILE)) { - Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -1029,7 +1027,6 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f } fclose(fp); - Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return status; } From 8217d765eeea8099093cf4c0848a08a841b53d3e Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Mon, 11 Mar 2024 21:43:41 +0530 Subject: [PATCH 070/143] Implement scale_up function in TriangleCountExecutor --- .../executor/impl/TriangleCountExecutor.cpp | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 942c79e30..a9aba6cf1 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -136,8 +136,8 @@ static int alloc_plan(std::map &alloc, std::set &remain, std::map return best_rem; } -static void reallocate_parts(std::map &alloc, std::set &remain, - const std::map> &P_AVAIL, std::map &loads) { +static std::vector reallocate_parts(std::map &alloc, std::set &remain, + const std::map> &P_AVAIL) { map P_COUNT; for (auto it = P_AVAIL.begin(); it != P_AVAIL.end(); it++) { P_COUNT[it->first] = it->second.size(); @@ -184,6 +184,25 @@ static void reallocate_parts(std::map &alloc, std::set &remain, } if (need_pushing) copying.push_back(p0); } + return copying; +} + +static void scale_up(std::map &loads, int copy_cnt) { + int curr_load = 0; + for (auto it = loads.begin(); it != loads.end(); it++) { + curr_load += it->second; + } + int n_cores = copy_cnt + curr_load - 3 * loads.size(); + if (n_cores < 0) n_cores = 0; + int n_workers = n_cores / 3; + if (n_cores % 3 > 0) n_workers++; + if (n_workers == 0) return; + + // TODO(thevindu-w): replace with call to K8sWorkerController scale up + vector w_new; + for (int i = 0; i < n_workers; i++) w_new.push_back(i + 30); + + for (auto it = w_new.begin(); it != w_new.end(); it++) loads[*it] = 0; } static int alloc_net_plan(std::vector parts, std::map> transfer, From 0d22d53744c7507ec339d1f937eee6f040cb7d5c Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Tue, 12 Mar 2024 00:26:29 +0530 Subject: [PATCH 071/143] Use partitionMap in aggregateCentralStoreTriangles to get partitions by worker instead of SQL --- .../executor/impl/TriangleCountExecutor.cpp | 122 +++++++----------- .../executor/impl/TriangleCountExecutor.h | 3 - src/server/JasmineGraphInstance.cpp | 2 - src/server/JasmineGraphInstanceService.cpp | 8 +- 4 files changed, 52 insertions(+), 83 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index a9aba6cf1..e7038c4f9 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -25,10 +25,13 @@ std::mutex fileCombinationMutex; std::mutex processStatusMutex; std::mutex responseVectorMutex; -static std::vector> getWorkerCombination(SQLiteDBInterface *sqlite, std::string graphId); +static std::vector> getWorkerCombination(std::vector &workerIdVector); static string isFileAccessibleToWorker(std::string graphId, std::string partitionId, std::string aggregatorHostName, std::string aggregatorPort, std::string masterIP, std::string fileType, std::string fileName); +static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, + int threadPriority, std::vector &workerIdVector, + std::map> &partitionMap); TriangleCountExecutor::TriangleCountExecutor() {} @@ -315,7 +318,6 @@ void TriangleCountExecutor::execute() { Utils::worker aggregatorWorker; vector workerList = Utils::getWorkerList(sqlite); int workerListSize = workerList.size(); - int partitionCount = 0; std::vector> intermRes; std::vector> statResponse; std::vector compositeCentralStoreFiles; @@ -323,12 +325,33 @@ void TriangleCountExecutor::execute() { auto begin = chrono::high_resolution_clock::now(); string sqlStatement = - "SELECT worker_idworker, name,ip,user,server_port,server_data_port,partition_idpartition " + "SELECT worker_idworker,partition_idpartition " "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + graphId + ";"; - std::vector>> results = sqlite->runSelect(sqlStatement); + const std::vector>> &results = sqlite->runSelect(sqlStatement); + + std::map> partitionMap; + + for (auto i = results.begin(); i != results.end(); ++i) { + const std::vector> &rowData = *i; + + string workerID = rowData.at(0).second; + string partitionId = rowData.at(1).second; + + if (partitionMap.find(workerID) == partitionMap.end()) { + std::vector partitionVec; + partitionVec.push_back(partitionId); + partitionMap[workerID] = partitionVec; + } else { + partitionMap[workerID].push_back(partitionId); + } + + triangleCount_logger.info("###TRIANGLE-COUNT-EXECUTOR### Getting Triangle Count : PartitionId " + partitionId); + } + + // TODO(thevindu-w): if auto-scale, select only 1 copy from each partition and update partition map if (results.size() > Conts::COMPOSITE_CENTRAL_STORE_WORKER_THRESHOLD) { isCompositeAggregation = true; @@ -353,32 +376,7 @@ void TriangleCountExecutor::execute() { fileCombinations = AbstractExecutor::getCombinations(compositeCentralStoreFiles); } - std::map> partitionMap; - - for (auto i = results.begin(); i != results.end(); ++i) { - std::vector> rowData = *i; - - string workerID = rowData.at(0).second; - string name = rowData.at(1).second; - string host = rowData.at(2).second; - string user = rowData.at(3).second; - string serverPort = rowData.at(4).second; - string serverDataPort = rowData.at(5).second; - string partitionId = rowData.at(6).second; - - if (partitionMap.find(workerID) == partitionMap.end()) { - std::vector partitionVec; - partitionVec.push_back(partitionId); - partitionMap[workerID] = partitionVec; - } else { - partitionMap[workerID].push_back(partitionId); - } - - triangleCount_logger.log("###TRIANGLE-COUNT-EXECUTOR### Getting Triangle Count : Host " + host + - " Server Port " + serverPort + " PartitionId " + partitionId, - "info"); - } - + int partitionCount = 0; for (int i = 0; i < workerListSize; i++) { Utils::worker currentWorker = workerList.at(i); string host = currentWorker.hostname; @@ -408,7 +406,7 @@ void TriangleCountExecutor::execute() { "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + "' and sla_category.command='" + TRIANGLES + "';"; - std::vector>> queryResults = perfDB->runSelect(query); + const std::vector>> &queryResults = perfDB->runSelect(query); if (queryResults.size() > 0) { std::string attemptString = queryResults[0][0].second; @@ -431,8 +429,13 @@ void TriangleCountExecutor::execute() { } if (!isCompositeAggregation) { + std::vector workerIdVector; + for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { + workerIdVector.push_back(it->first); + } + long aggregatedTriangleCount = - TriangleCountExecutor::aggregateCentralStoreTriangles(sqlite, graphId, masterIP, threadPriority); + aggregateCentralStoreTriangles(sqlite, graphId, masterIP, threadPriority, workerIdVector, partitionMap); result += aggregatedTriangleCount; workerResponded = true; triangleCount_logger.log( @@ -828,9 +831,10 @@ int TriangleCountExecutor::updateTriangleTreeAndGetTriangleCount(std::vector> &workerCombinations = getWorkerCombination(sqlite, graphId); +static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, + int threadPriority, std::vector &workerIdVector, + std::map> &partitionMap) { + const std::vector> &workerCombinations = getWorkerCombination(workerIdVector); std::map workerWeightMap; std::vector> triangleCountResponse; std::string result = ""; @@ -842,7 +846,6 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq std::vector> remoteGraphCopyResponse; int minimumWeight = 0; std::string minWeightWorker; - std::string partitionIdList = ""; for (auto workerCombinationIterator = workerCombination.begin(); workerCombinationIterator != workerCombination.end(); ++workerCombinationIterator) { @@ -851,7 +854,7 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq auto workerWeightMapIterator = workerWeightMap.find(workerId); if (workerWeightMapIterator != workerWeightMap.end()) { - int weight = workerWeightMap.at(workerId); + int weight = workerWeightMapIterator->second; if (minimumWeight == 0 || minimumWeight > weight) { minimumWeight = weight + 1; @@ -864,10 +867,9 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq } string aggregatorSqlStatement = - "SELECT ip,server_port,server_data_port,partition_idpartition " - "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " - "WHERE partition_graph_idgraph=" + - graphId + " and idworker=" + minWeightWorker + ";"; + "SELECT ip,server_port,server_data_port FROM worker " + "WHERE idworker=" + + minWeightWorker + ";"; const std::vector>> &result = sqlite->runSelect(aggregatorSqlStatement); @@ -876,25 +878,17 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq std::string aggregatorIp = aggregatorData.at(0).second; std::string aggregatorPort = aggregatorData.at(1).second; std::string aggregatorDataPort = aggregatorData.at(2).second; - std::string aggregatorPartitionId = aggregatorData.at(3).second; + std::string aggregatorPartitionId = partitionMap[minWeightWorker][0]; // why only 1 partition from the worker? + std::string partitionIdList = ""; for (auto aggregatorCopyCombinationIterator = workerCombination.begin(); aggregatorCopyCombinationIterator != workerCombination.end(); ++aggregatorCopyCombinationIterator) { std::string workerId = *aggregatorCopyCombinationIterator; if (workerId != minWeightWorker) { - string sqlStatement = - "SELECT partition_idpartition " - "FROM worker_has_partition INNER JOIN worker ON " - "worker_has_partition.worker_idworker=worker.idworker " - "WHERE partition_graph_idgraph=" + - graphId + " and idworker=" + workerId + ";"; - - const std::vector>> &result = sqlite->runSelect(sqlStatement); - const vector> &workerData = result.at(0); - std::string partitionId = workerData.at(0).second; + std::string partitionId = partitionMap[workerId][0]; // why only 1 partition from the worker? partitionIdList += partitionId + ","; @@ -1453,28 +1447,8 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag return response; } -static std::vector> getWorkerCombination(SQLiteDBInterface *sqlite, std::string graphId) { - std::set workerIdSet; - - string sqlStatement = - "SELECT worker_idworker " - "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " - "WHERE partition_graph_idgraph=" + - graphId + ";"; - - const std::vector>> &results = sqlite->runSelect(sqlStatement); - - for (auto i = results.begin(); i != results.end(); ++i) { - const std::vector> &rowData = *i; - string workerId = rowData.at(0).second; - workerIdSet.insert(workerId); - } - - std::vector workerIdVector(workerIdSet.begin(), workerIdSet.end()); - - std::vector> workerIdCombination = AbstractExecutor::getCombinations(workerIdVector); - - return workerIdCombination; +static std::vector> getWorkerCombination(std::vector &workerIdVector) { + return AbstractExecutor::getCombinations(workerIdVector); } std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggregatorHostName, diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.h b/src/frontend/core/executor/impl/TriangleCountExecutor.h index a9ac28a03..69e990811 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.h +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.h @@ -38,9 +38,6 @@ class TriangleCountExecutor : public AbstractExecutor { static long getTriangleCount(int graphId, std::string host, int port, int dataPort, int partitionId, std::string masterIP, int uniqueId, bool isCompositeAggregation, int threadPriority); - static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, - int threadPriority); - static std::string copyCompositeCentralStoreToAggregator(std::string aggregatorHostName, std::string aggregatorPort, std::string aggregatorDataPort, std::string fileName, std::string masterIP); diff --git a/src/server/JasmineGraphInstance.cpp b/src/server/JasmineGraphInstance.cpp index 634016005..dfea42b72 100644 --- a/src/server/JasmineGraphInstance.cpp +++ b/src/server/JasmineGraphInstance.cpp @@ -46,12 +46,10 @@ int JasmineGraphInstance::start_running(string hostName, string masterHost, int if (Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"))) { graphInstance_logger.error("Could not create directory: " + Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder")); - exit(1); } if (Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"))) { graphInstance_logger.error("Could not create directory: " + Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder")); - exit(1); } startNmonAnalyzer(enableNmon, serverPort); diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 94f6fb7f6..dbe103457 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -512,8 +512,8 @@ JasmineGraphHashMapCentralStore *JasmineGraphInstanceService::loadCentralStore(s return jasmineGraphHashMapCentralStore; } -string JasmineGraphInstanceService::aggregateCentralStoreTriangles(std::string graphId, std::string partitionId, - std::string partitionIdList, int threadPriority) { +static string aggregateCentralStoreTriangles(std::string graphId, std::string partitionId, std::string partitionIdList, + int threadPriority) { instance_logger.info("###INSTANCE### Started Aggregating Central Store Triangles"); std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); std::vector fileNames; @@ -3188,8 +3188,8 @@ static void aggregate_centralstore_triangles_command(int connFd, bool *loop_exit threadPriorityMutex.unlock(); } - std::string aggregatedTriangles = JasmineGraphInstanceService::aggregateCentralStoreTriangles( - graphId, partitionId, partitionIdList, threadPriority); + std::string aggregatedTriangles = + aggregateCentralStoreTriangles(graphId, partitionId, partitionIdList, threadPriority); if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { threadPriorityMutex.lock(); From 0f3dc85ad0aa4b077394c8579aa97706d0a2a8b5 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Tue, 12 Mar 2024 13:45:20 +0530 Subject: [PATCH 072/143] retry mkdir if failed --- src/util/Utils.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 6e04c2251..b33c95e08 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -198,7 +198,13 @@ bool Utils::fileExists(std::string fileName) { return access(fileName.c_str(), F */ int Utils::createDirectory(const std::string dirName) { string command = "mkdir -p " + dirName; - return system(command.c_str()); + int status = system(command.c_str()); + if (status != 0) { + util_logger.warn("Command failed: " + command + " trying again"); + sleep(1); + status = system(command.c_str()); + } + return status; } std::vector Utils::getListOfFilesInDirectory(std::string dirName) { @@ -1107,5 +1113,3 @@ bool Utils::transferPartition(std::string sourceWorker, close(sockfd); return true; } - - From 688f54e422eea65be6181744fc365b558085e9dc Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Tue, 12 Mar 2024 13:46:33 +0530 Subject: [PATCH 073/143] Implement auto-scaling trian with optimal partition allocation --- .../executor/impl/TriangleCountExecutor.cpp | 155 +++++++++++++----- 1 file changed, 113 insertions(+), 42 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index e7038c4f9..86f96ace1 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -13,6 +13,8 @@ limitations under the License. #include "TriangleCountExecutor.h" +#include "../../../../../globals.h" + using namespace std::chrono; Logger triangleCount_logger; @@ -42,8 +44,8 @@ TriangleCountExecutor::TriangleCountExecutor(SQLiteDBInterface *db, PerformanceS this->request = jobRequest; } -static void allocate(int p, int w, std::map &alloc, std::set &remain, - std::map> &p_avail, std::map &loads) { +static void allocate(int p, string w, std::map &alloc, std::set &remain, + std::map> &p_avail, std::map &loads) { alloc[p] = w; remain.erase(p); p_avail.erase(p); @@ -59,7 +61,7 @@ static void allocate(int p, int w, std::map &alloc, std::set &rem } } -static int get_min_partition(std::set &remain, std::map> &p_avail) { +static int get_min_partition(std::set &remain, std::map> &p_avail) { int p0 = *remain.begin(); size_t m = 1000000000; for (auto it = remain.begin(); it != remain.end(); it++) { @@ -74,15 +76,11 @@ static int get_min_partition(std::set &remain, std::map LOAD_PREFERENCE = {2, 3, 1, 0}; -static int compare_loads(int w1, int w2, std::map &loads) { - if (loads[w1] == loads[w2]) return 0; - return LOAD_PREFERENCE[loads[w1]] < LOAD_PREFERENCE[loads[w2]]; -} -static int alloc_plan(std::map &alloc, std::set &remain, std::map> &p_avail, - std::map &loads) { +static int alloc_plan(std::map &alloc, std::set &remain, std::map> &p_avail, + std::map &loads) { for (bool done = false; !done;) { - int w = -1; + string w = ""; done = true; int p; for (auto it = remain.begin(); it != remain.end(); it++) { @@ -92,25 +90,25 @@ static int alloc_plan(std::map &alloc, std::set &remain, std::map done = false; } } - if (w >= 0) allocate(p, w, alloc, remain, p_avail, loads); + if (!w.empty()) allocate(p, w, alloc, remain, p_avail, loads); } if (remain.empty()) return 0; int p0 = get_min_partition(remain, p_avail); auto &ws = p_avail[p0]; if (ws.empty()) return (int)remain.size(); - sort(ws.begin(), ws.end(), [&loads](int &w1, int &w2) { + sort(ws.begin(), ws.end(), [&loads](string &w1, string &w2) { return LOAD_PREFERENCE[loads[w1]] > LOAD_PREFERENCE[loads[w2]]; }); // load=1 goes first and load=3 goes last struct best_alloc { - std::map alloc; + std::map alloc; std::set remain; - std::map> p_avail; - std::map loads; + std::map> p_avail; + std::map loads; }; int best_rem = remain.size(); struct best_alloc best = {.alloc = alloc, .remain = remain, .p_avail = p_avail, .loads = loads}; for (auto it = ws.begin(); it != ws.end(); it++) { - int w = *it; + string w = *it; auto alloc2 = alloc; // need copy => do not copy reference auto remain2 = remain; // need copy => do not copy reference auto p_avail2 = p_avail; // need copy => do not copy reference @@ -139,8 +137,8 @@ static int alloc_plan(std::map &alloc, std::set &remain, std::map return best_rem; } -static std::vector reallocate_parts(std::map &alloc, std::set &remain, - const std::map> &P_AVAIL) { +static std::vector reallocate_parts(std::map &alloc, std::set &remain, + const std::map> &P_AVAIL) { map P_COUNT; for (auto it = P_AVAIL.begin(); it != P_AVAIL.end(); it++) { P_COUNT[it->first] = it->second.size(); @@ -175,7 +173,7 @@ static std::vector reallocate_parts(std::map &alloc, std::setsecond; if (std::find(ws.begin(), ws.end(), w) != ws.end()) { alloc.erase(p); @@ -190,7 +188,7 @@ static std::vector reallocate_parts(std::map &alloc, std::set &loads, int copy_cnt) { +static void scale_up(std::map &loads, int copy_cnt) { int curr_load = 0; for (auto it = loads.begin(); it != loads.end(); it++) { curr_load += it->second; @@ -202,44 +200,47 @@ static void scale_up(std::map &loads, int copy_cnt) { if (n_workers == 0) return; // TODO(thevindu-w): replace with call to K8sWorkerController scale up - vector w_new; - for (int i = 0; i < n_workers; i++) w_new.push_back(i + 30); + vector w_new; + for (int i = 0; i < n_workers; i++) w_new.push_back(string("10.11.12.") + to_string(i) + string(":7780")); for (auto it = w_new.begin(); it != w_new.end(); it++) loads[*it] = 0; } -static int alloc_net_plan(std::vector parts, std::map> transfer, - std::map net_loads, std::map loads, - const std::map> &p_avail) { +static int alloc_net_plan(std::vector &parts, std::map> &transfer, + std::map &net_loads, std::map &loads, + const std::map> &p_avail) { if (parts.empty()) { if (net_loads.empty()) return 0; return std::max_element(net_loads.begin(), net_loads.end(), - [](const std::map::value_type &p1, const std::map::value_type &p2) { - return p1.second < p2.second; - }) + [](const std::map::value_type &p1, + const std::map::value_type &p2) { return p1.second < p2.second; }) ->second; } struct best_net_alloc { - std::map> transfer; - std::map net_loads; - std::map loads; + std::map> transfer; + std::map net_loads; + std::map loads; }; int best = 10000000; // large number struct best_net_alloc best_plan = {.transfer = transfer, .net_loads = net_loads, .loads = loads}; int p = parts.back(); parts.pop_back(); - vector wts; + vector wts; for (auto it = loads.begin(); it != loads.end(); it++) { wts.push_back(it->first); } - sort(wts.begin(), wts.end(), [&loads](int &w1, int &w2) { - return LOAD_PREFERENCE[loads[w1]] > LOAD_PREFERENCE[loads[w2]]; + sort(wts.begin(), wts.end(), [&loads](string &w1, string &w2) { + int l1 = loads[w1]; + int l2 = loads[w2]; + if (l1 < 3 && l2 < 3) return LOAD_PREFERENCE[l1] > LOAD_PREFERENCE[loads[w2]]; + if (l1 < 3 || l2 < 3) return l1 < l2; + return l1 <= l2; }); // load=1 goes first and load=3 goes last const auto &ws = p_avail.find(p)->second; for (auto itf = ws.begin(); itf != ws.end(); itf++) { - int wf = *itf; + auto wf = *itf; for (auto itt = ws.begin(); itt != ws.end(); itt++) { - int wt = *itt; + auto wt = *itt; auto parts2 = parts; // need copy => do not copy reference auto transfer2 = transfer; // need copy => do not copy reference auto net_loads2 = net_loads; // need copy => do not copy reference @@ -268,13 +269,84 @@ static int alloc_net_plan(std::vector parts, std::map> &partitionMap, SQLiteDBInterface *sqlite) { + vector workers; + const std::vector>> &results = sqlite->runSelect("SELECT ip,server_port FROM worker;"); + for (int i = 0; i < results.size(); i++) { + string ip = results[i][0].second; + string port = results[i][1].second; + workers.push_back(ip + ":" + port); + } + + map loads; + const map &cpu_map = Utils::getMetricMap("cpu_usage"); + for (auto it = workers.begin(); it != workers.end(); it++) { + auto worker = *it; + const auto workerLoadIt = cpu_map.find(worker); + if (workerLoadIt != cpu_map.end()) { + loads[worker] = (int)round(stod(workerLoadIt->second.c_str())); + } else { + loads[worker] = 0; + } + } + const map LOADS = loads; + + std::map> p_avail; + std::set remain; + for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { + auto worker = it->first; + auto &partitions = it->second; + for (auto partitionIt = partitions.begin(); partitionIt != partitions.end(); partitionIt++) { + auto partition = stoi(*partitionIt); + p_avail[partition].push_back(worker); + remain.insert(partition); + } + } + const std::map> P_AVAIL = p_avail; // get a copy and make it const + + std::map alloc; + int unallocated = alloc_plan(alloc, remain, p_avail, loads); + triangleCount_logger.info("Unallocated = " + to_string(unallocated)); + if (unallocated > 0) { + auto copying = reallocate_parts(alloc, remain, P_AVAIL); + scale_up(loads, copying.size()); + + map net_loads; + for (auto it = loads.begin(); it != loads.end(); it++) { + net_loads[it->first] = 0; + } + for (auto it = loads.begin(); it != loads.end();) { + if (it->second >= 3) { + loads.erase(it++); + } else { + it++; + } + } + + std::map> transfer; + int net_load = alloc_net_plan(copying, transfer, net_loads, loads, P_AVAIL); + for (auto it = transfer.begin(); it != transfer.end(); it++) { + auto p = it->first; + auto w_from = it->second.first; + auto w_to = it->second.second; + alloc[p] = w_to; + } + } + + partitionMap.clear(); + for (auto it = alloc.begin(); it != alloc.end(); it++) { + auto partition = it->first; + auto worker = it->second; + partitionMap[worker].push_back(to_string(partition)); + } +} + void TriangleCountExecutor::execute() { int uniqueId = getUid(); std::string masterIP = request.getMasterIP(); std::string graphId = request.getParameter(Conts::PARAM_KEYS::GRAPH_ID); std::string canCalibrateString = request.getParameter(Conts::PARAM_KEYS::CAN_CALIBRATE); std::string queueTime = request.getParameter(Conts::PARAM_KEYS::QUEUE_TIME); - std::string graphSLAString = request.getParameter(Conts::PARAM_KEYS::GRAPH_SLA); bool canCalibrate = Utils::parseBoolean(canCalibrateString); int threadPriority = request.getPriority(); @@ -351,7 +423,10 @@ void TriangleCountExecutor::execute() { triangleCount_logger.info("###TRIANGLE-COUNT-EXECUTOR### Getting Triangle Count : PartitionId " + partitionId); } - // TODO(thevindu-w): if auto-scale, select only 1 copy from each partition and update partition map + if (jasminegraph_profile == PROFILE_K8S && + Utils::getJasmineGraphProperty("org.jasminegraph.autoscale.enabled") == "true") { + filter_partitions(partitionMap, sqlite); + } if (results.size() > Conts::COMPOSITE_CENTRAL_STORE_WORKER_THRESHOLD) { isCompositeAggregation = true; @@ -968,7 +1043,6 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; - // TODO::exit return 0; } @@ -1117,7 +1191,6 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; - // TODO::exit return 0; } @@ -1300,7 +1373,6 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; - // TODO::exit return 0; } @@ -1667,7 +1739,6 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; - // TODO::exit return 0; } From 29b0e6aa411cc67869c6375c3b539493352a1483 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 13 Mar 2024 03:03:33 +0530 Subject: [PATCH 074/143] Fix bugs in trian auto-scale functions --- .../executor/impl/TriangleCountExecutor.cpp | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 86f96ace1..1e66c77e8 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -88,6 +88,7 @@ static int alloc_plan(std::map &alloc, std::set &remain, std:: if (p_avail[p].size() == 1) { w = p_avail[p][0]; done = false; + break; } } if (!w.empty()) allocate(p, w, alloc, remain, p_avail, loads); @@ -208,20 +209,24 @@ static void scale_up(std::map &loads, int copy_cnt) { static int alloc_net_plan(std::vector &parts, std::map> &transfer, std::map &net_loads, std::map &loads, - const std::map> &p_avail) { + const std::map> &p_avail, int curr_best) { + int curr_load = std::max_element(net_loads.begin(), net_loads.end(), + [](const std::map::value_type &p1, + const std::map::value_type &p2) { return p1.second < p2.second; }) + ->second; + if (curr_load >= curr_best) { + return curr_load; + } if (parts.empty()) { if (net_loads.empty()) return 0; - return std::max_element(net_loads.begin(), net_loads.end(), - [](const std::map::value_type &p1, - const std::map::value_type &p2) { return p1.second < p2.second; }) - ->second; + return curr_load; } struct best_net_alloc { std::map> transfer; std::map net_loads; std::map loads; }; - int best = 10000000; // large number + int best = curr_best; struct best_net_alloc best_plan = {.transfer = transfer, .net_loads = net_loads, .loads = loads}; int p = parts.back(); parts.pop_back(); @@ -239,7 +244,7 @@ static int alloc_net_plan(std::vector &parts, std::mapsecond; for (auto itf = ws.begin(); itf != ws.end(); itf++) { auto wf = *itf; - for (auto itt = ws.begin(); itt != ws.end(); itt++) { + for (auto itt = wts.begin(); itt != wts.end(); itt++) { auto wt = *itt; auto parts2 = parts; // need copy => do not copy reference auto transfer2 = transfer; // need copy => do not copy reference @@ -250,7 +255,7 @@ static int alloc_net_plan(std::vector &parts, std::map> &partitionMa } const std::map> P_AVAIL = p_avail; // get a copy and make it const + for (auto loadIt = LOADS.begin(); loadIt != LOADS.end(); loadIt++) { + if (loadIt->second < 3) continue; + auto w = loadIt->first; + for (auto it = p_avail.begin(); it != p_avail.end(); it++) { + auto &workers = it->second; + for (auto workerIt = workers.begin(); workerIt != workers.end();) { + if (*workerIt == w) { + workers.erase(workerIt); + } else { + workerIt++; + } + } + } + } + std::map alloc; int unallocated = alloc_plan(alloc, remain, p_avail, loads); - triangleCount_logger.info("Unallocated = " + to_string(unallocated)); if (unallocated > 0) { auto copying = reallocate_parts(alloc, remain, P_AVAIL); scale_up(loads, copying.size()); @@ -324,7 +343,7 @@ static void filter_partitions(std::map> &partitionMa } std::map> transfer; - int net_load = alloc_net_plan(copying, transfer, net_loads, loads, P_AVAIL); + int net_load = alloc_net_plan(copying, transfer, net_loads, loads, P_AVAIL, 100000000); for (auto it = transfer.begin(); it != transfer.end(); it++) { auto p = it->first; auto w_from = it->second.first; From 315fe9b2f7aaf58410cc1ae4434def2c3a91395a Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 13 Mar 2024 03:04:03 +0530 Subject: [PATCH 075/143] Initialize Utils::propertiesMap only once --- src/util/Utils.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index b33c95e08..ffb1c235a 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -99,8 +99,10 @@ void Utils::writeFileContent(const std::string &filePath, const std::string &con out.close(); } +static bool initializedProperties = false; std::string Utils::getJasmineGraphProperty(std::string key) { - if (Utils::propertiesMap.empty()) { + if (!initializedProperties && Utils::propertiesMap.empty()) { + initializedProperties = true; const vector &vec = Utils::getFileContent(ROOT_DIR "conf/jasminegraph-server.properties"); for (auto it = vec.begin(); it < vec.end(); it++) { std::string item = *it; @@ -197,6 +199,9 @@ bool Utils::fileExists(std::string fileName) { return access(fileName.c_str(), F * @param dirName */ int Utils::createDirectory(const std::string dirName) { + if (dirName.empty()) { + util_logger.error("Cannot mkdir empty dirName"); + } string command = "mkdir -p " + dirName; int status = system(command.c_str()); if (status != 0) { From d159e178535c1a9b9c14f01ed8467268e97bdb30 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 13 Mar 2024 11:39:48 +0530 Subject: [PATCH 076/143] Wait until propertiesMap is initialized by other thread --- src/util/Utils.cpp | 51 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index ffb1c235a..69b168fae 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -57,16 +57,15 @@ std::vector Utils::getFileContent(std::string file) { ifstream in(file); std::string str; - vector *vec = new vector(); + vector vec; while (std::getline(in, str)) { // now we loop back and get the next line in 'str' - if (str.length() > 0) { - vec->insert(vec->begin(), str); + vec.push_back(str); } } - return *vec; + return vec; }; std::string Utils::getFileContentAsString(std::string file) { @@ -99,22 +98,27 @@ void Utils::writeFileContent(const std::string &filePath, const std::string &con out.close(); } -static bool initializedProperties = false; +static std::mutex propertiesMapMutex; +static bool propertiesMapInitialized = false; std::string Utils::getJasmineGraphProperty(std::string key) { - if (!initializedProperties && Utils::propertiesMap.empty()) { - initializedProperties = true; - const vector &vec = Utils::getFileContent(ROOT_DIR "conf/jasminegraph-server.properties"); - for (auto it = vec.begin(); it < vec.end(); it++) { - std::string item = *it; - if (item.length() > 0 && !(item.rfind("#", 0) == 0)) { - const std::vector &vec2 = split(item, '='); - if (vec2.size() == 2) { - Utils::propertiesMap[vec2.at(0)] = vec2.at(1); - } else { - Utils::propertiesMap[vec2.at(0)] = string(" "); + if (!propertiesMapInitialized) { + propertiesMapMutex.lock(); + if (!propertiesMapInitialized) { // double-checking lock + const vector &vec = Utils::getFileContent(ROOT_DIR "conf/jasminegraph-server.properties"); + for (auto it = vec.begin(); it < vec.end(); it++) { + std::string item = *it; + if (item.length() > 0 && !(item.rfind("#", 0) == 0)) { + const std::vector &vec2 = split(item, '='); + if (vec2.size() == 2) { + Utils::propertiesMap[vec2.at(0)] = vec2.at(1); + } else { + Utils::propertiesMap[vec2.at(0)] = string(" "); + } } } } + propertiesMapInitialized = true; + propertiesMapMutex.unlock(); } auto it = Utils::propertiesMap.find(key); if (it != Utils::propertiesMap.end()) { @@ -1048,16 +1052,11 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f * and the destination worker is ready to accept the partition * Also, the caller should update the worker_has_partition table. * */ -bool Utils::transferPartition(std::string sourceWorker, - int sourceWorkerPort, - int sourceWorkerDataPort, - std::string destinationWorker, - int destinationWorkerPort, - int destinationWorkerDataPort, - int graphID, - int partitionID) { - util_logger.info("### Transferring partition " + to_string(partitionID) + " of graph " + to_string(graphID) - + " from " + sourceWorker + " to " + destinationWorker); +bool Utils::transferPartition(std::string sourceWorker, int sourceWorkerPort, int sourceWorkerDataPort, + std::string destinationWorker, int destinationWorkerPort, int destinationWorkerDataPort, + int graphID, int partitionID) { + util_logger.info("### Transferring partition " + to_string(partitionID) + " of graph " + to_string(graphID) + + " from " + sourceWorker + " to " + destinationWorker); int sockfd; char data[FED_DATA_LENGTH + 1]; From b1682144e7ca365b2cac02e6973baa12787b61eb Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 13 Mar 2024 13:03:57 +0530 Subject: [PATCH 077/143] optimizations --- src/frontend/JasmineGraphFrontEnd.h | 3 -- .../executor/impl/TriangleCountExecutor.cpp | 51 ++++++++----------- .../executor/impl/TriangleCountExecutor.h | 10 ++-- 3 files changed, 27 insertions(+), 37 deletions(-) diff --git a/src/frontend/JasmineGraphFrontEnd.h b/src/frontend/JasmineGraphFrontEnd.h index bfede4def..7d44d7bdd 100644 --- a/src/frontend/JasmineGraphFrontEnd.h +++ b/src/frontend/JasmineGraphFrontEnd.h @@ -78,9 +78,6 @@ class JasmineGraphFrontEnd { static int getRunningHighPriorityTaskCount(); static bool areRunningJobsForSameGraph(); - static std::vector> fileCombinations; - static std::map combinationWorkerMap; - static std::map>> triangleTree; std::map> *streamsState; std::map streamingThreads; diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 1e66c77e8..ddace2a63 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -20,7 +20,7 @@ using namespace std::chrono; Logger triangleCount_logger; std::vector> TriangleCountExecutor::fileCombinations; std::map TriangleCountExecutor::combinationWorkerMap; -std::map>> TriangleCountExecutor::triangleTree; +std::unordered_map>> TriangleCountExecutor::triangleTree; bool isStatCollect = false; std::mutex fileCombinationMutex; @@ -346,7 +346,6 @@ static void filter_partitions(std::map> &partitionMa int net_load = alloc_net_plan(copying, transfer, net_loads, loads, P_AVAIL, 100000000); for (auto it = transfer.begin(); it != transfer.end(); it++) { auto p = it->first; - auto w_from = it->second.first; auto w_to = it->second.second; alloc[p] = w_to; } @@ -808,14 +807,10 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int } } - std::string compositeTriangles = - countCompositeCentralStoreTriangles(host, std::to_string(port), adjustedTransferredFile, - masterIP, adjustedAvailableFiles, threadPriority); - triangleCount_logger.log("###COMPOSITE### Retrieved Composite triangle list ", "debug"); - std::vector triangles = Utils::split(compositeTriangles, ':'); - + const auto &triangles = countCompositeCentralStoreTriangles(host, std::to_string(port), adjustedTransferredFile, + masterIP, adjustedAvailableFiles, threadPriority); if (triangles.size() > 0) { triangleCount += updateTriangleTreeAndGetTriangleCount(triangles); } @@ -887,14 +882,13 @@ void TriangleCountExecutor::updateMap(int partitionId) { "info"); } -int TriangleCountExecutor::updateTriangleTreeAndGetTriangleCount(std::vector triangles) { +int TriangleCountExecutor::updateTriangleTreeAndGetTriangleCount(const std::vector &triangles) { const std::lock_guard lock1(triangleTreeMutex); - std::vector::iterator triangleIterator; int aggregateCount = 0; triangleCount_logger.log("###COMPOSITE### Triangle Tree locked ", "debug"); - for (triangleIterator = triangles.begin(); triangleIterator != triangles.end(); ++triangleIterator) { + for (auto triangleIterator = triangles.begin(); triangleIterator != triangles.end(); ++triangleIterator) { std::string triangle = *triangleIterator; if (!triangle.empty() && triangle != "NILL") { @@ -904,19 +898,17 @@ int TriangleCountExecutor::updateTriangleTreeAndGetTriangleCount(std::vector> itemRes = triangleTree[vertexOne]; - - std::map>::iterator itemResIterator = itemRes.find(vertexTwo); - + auto &itemRes = triangleTree[vertexOne]; + auto itemResIterator = itemRes.find(vertexTwo); if (itemResIterator != itemRes.end()) { - std::vector list = itemRes[vertexTwo]; - - if (std::find(list.begin(), list.end(), vertexThree) == list.end()) { - triangleTree[vertexOne][vertexTwo].push_back(vertexThree); + auto &set2 = itemResIterator->second; + auto set2Iter = set2.find(vertexThree); + if (set2Iter == set2.end()) { + set2.insert(vertexThree); aggregateCount++; } } else { - triangleTree[vertexOne][vertexTwo].push_back(vertexThree); + triangleTree[vertexOne][vertexTwo].insert(vertexThree); aggregateCount++; } } @@ -1361,7 +1353,7 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st return response; } -string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string aggregatorHostName, +std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string aggregatorHostName, std::string aggregatorPort, std::string compositeCentralStoreFileList, std::string masterIP, std::string availableFileList, @@ -1377,13 +1369,13 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag if (sockfd < 0) { std::cerr << "Cannot create socket" << std::endl; - return 0; + return {}; } server = gethostbyname(aggregatorHostName.c_str()); if (server == NULL) { triangleCount_logger.error("ERROR, no host named " + aggregatorHostName); - return 0; + return {}; } bzero((char *)&serv_addr, sizeof(serv_addr)); @@ -1392,7 +1384,7 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { std::cerr << "ERROR connecting" << std::endl; - return 0; + return {}; } bzero(data, 301); @@ -1508,8 +1500,8 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag response = (data); response = Utils::trim_copy(response); string status = response.substr(response.size() - 5); - std::string result = response.substr(0, response.size() - 5); - + std::basic_ostringstream resultStream; + resultStream << response.substr(0, response.size() - 5); while (status.compare("/SEND") == 0) { result_wr = write(sockfd, status.c_str(), status.size()); @@ -1521,10 +1513,9 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag response = (data); response = Utils::trim_copy(response); status = response.substr(response.size() - 5); - std::string triangleResponse = response.substr(0, response.size() - 5); - result = result + triangleResponse; + resultStream << response.substr(0, response.size() - 5); } - response = result; + response = resultStream.str(); } triangleCount_logger.log("Aggregate Response Received", "info"); @@ -1535,7 +1526,7 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag } Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); - return response; + return Utils::split(response, ':');; } static std::vector> getWorkerCombination(std::vector &workerIdVector) { diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.h b/src/frontend/core/executor/impl/TriangleCountExecutor.h index 69e990811..f805eeaad 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.h +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.h @@ -15,6 +15,8 @@ limitations under the License. #define JASMINEGRAPH_TRIANGLECOUNTEXECUTOR_H #include +#include +#include #include "../../../../metadb/SQLiteDBInterface.h" #include "../../../../performance/metrics/PerformanceUtil.h" @@ -42,7 +44,7 @@ class TriangleCountExecutor : public AbstractExecutor { std::string aggregatorDataPort, std::string fileName, std::string masterIP); - static string countCompositeCentralStoreTriangles(std::string aggregatorHostName, std::string aggregatorPort, + static std::vector countCompositeCentralStoreTriangles(std::string aggregatorHostName, std::string aggregatorPort, std::string compositeCentralStoreFileList, std::string masterIP, std::string availableFileList, int threadPriority); @@ -58,15 +60,15 @@ class TriangleCountExecutor : public AbstractExecutor { static void updateMap(int partitionId); - static int updateTriangleTreeAndGetTriangleCount(std::vector triangles); - static std::vector> fileCombinations; static std::map combinationWorkerMap; - static std::map>> triangleTree; + static std::unordered_map>> triangleTree; private: SQLiteDBInterface *sqlite; PerformanceSQLiteDBInterface *perfDB; + + static int updateTriangleTreeAndGetTriangleCount(const std::vector &triangles); }; #endif // JASMINEGRAPH_TRIANGLECOUNTEXECUTOR_H From 8f94e4606598523c12e039a2ff63605b21ba0a33 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Wed, 13 Mar 2024 14:28:22 +0530 Subject: [PATCH 078/143] Implement scale up function --- README.md | 5 +- k8s/configs.yaml | 2 +- src/k8s/K8sWorkerController.cpp | 71 ++++++++++++++++++++- src/k8s/K8sWorkerController.h | 5 +- start-k8s.sh | 6 ++ tests/unit/k8s/K8sWorkerController_test.cpp | 20 ++++++ 6 files changed, 104 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4c4cd7a95..bf9422509 100755 --- a/README.md +++ b/README.md @@ -26,9 +26,10 @@ Run JasmineGraph in Kubernetes environment by providing appropriate parameters: --DATA_PATH "$(pwd)/data" \ --LOG_PATH "$(pwd)/logs" \ --NO_OF_WORKERS 2 \ - --ENABLE_NMON false + --ENABLE_NMON false \ + --MAX_COUNT 4 \ ->Note: `NO_OF_WORKERS` and `ENABLE_NMON` are optional arguments, and it defaults to `2` and `false` respectively. +>Note: `NO_OF_WORKERS`, `MAX_COUNT`, and `ENABLE_NMON` are optional arguments, and it defaults to `2`, `4` and `false` respectively. Remove all resources created by JasmineGraph in Kubernetes environment: diff --git a/k8s/configs.yaml b/k8s/configs.yaml index 83f47ffe5..1b3469bb2 100644 --- a/k8s/configs.yaml +++ b/k8s/configs.yaml @@ -7,5 +7,5 @@ metadata: data: pushgateway_address: "${pushgateway_address}" prometheus_address: "${prometheus_address}" - + max_worker_count: "${max_worker_count}" \ No newline at end of file diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 70d8ced30..4abfb37ab 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -21,6 +21,7 @@ limitations under the License. Logger controller_logger; std::vector K8sWorkerController::workerList = {}; +static int TIME_OUT = 180; K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb) { this->masterIp = std::move(masterIp); @@ -28,6 +29,14 @@ K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorke this->interface = new K8sInterface(); this->metadb = *metadb; + try { + this->maxWorkers = stoi(this->interface->getJasmineGraphConfig("max_worker_count")); + } catch (std::invalid_argument &e) { + controller_logger.error("Invalid max_worker_count value. Defaulted to 4"); + this->maxWorkers = 4; + } + + // Delete all the workers from the database metadb->runUpdate("DELETE FROM worker"); int workersAttached = this->attachExistingWorkers(); @@ -58,7 +67,7 @@ K8sWorkerController *K8sWorkerController::getInstance(std::string masterIp, int return instance; } -void K8sWorkerController::spawnWorker(int workerId) { +std::string K8sWorkerController::spawnWorker(int workerId) { v1_service_t *service = this->interface->createJasmineGraphWorkerService(workerId); if (service != nullptr && service->metadata != nullptr && service->metadata->name != nullptr) { controller_logger.info("Worker " + std::to_string(workerId) + " service created successfully"); @@ -77,6 +86,43 @@ void K8sWorkerController::spawnWorker(int workerId) { throw std::runtime_error("Worker " + std::to_string(workerId) + " deployment creation failed"); } + int sockfd; + struct sockaddr_in serv_addr; + struct hostent *server; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + controller_logger.error("Cannot create socket"); + return ""; + } + + server = gethostbyname(ip.c_str()); + if (server == NULL) { + controller_logger.error("ERROR, no host named " + ip); + return ""; + } + + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(Conts::JASMINEGRAPH_INSTANCE_PORT); + + int waiting = 0; + while (true) { + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + waiting += 30; // Added overhead in connection retry attempts + sleep(10); + } else { + close(sockfd); + break; + } + + if (waiting >= TIME_OUT) { + return ""; + } + + } + JasmineGraphServer::worker worker = { .hostname = ip, .port = Conts::JASMINEGRAPH_INSTANCE_PORT, .dataPort = Conts::JASMINEGRAPH_INSTANCE_DATA_PORT}; K8sWorkerController::workerList.push_back(worker); @@ -89,6 +135,8 @@ void K8sWorkerController::spawnWorker(int workerId) { if (status == -1) { controller_logger.error("Worker " + std::to_string(workerId) + " database insertion failed"); } + + return ip + ":" + to_string(Conts::JASMINEGRAPH_INSTANCE_PORT); } void K8sWorkerController::deleteWorker(int workerId) { @@ -179,6 +227,9 @@ std::string K8sWorkerController::getMasterIp() const { return this->masterIp; } int K8sWorkerController::getNumberOfWorkers() const { return numberOfWorkers; } +/* + * @deprecated + */ void K8sWorkerController::setNumberOfWorkers(int newNumberOfWorkers) { if (newNumberOfWorkers > this->numberOfWorkers) { for (int i = this->numberOfWorkers; i < newNumberOfWorkers; i++) { @@ -191,3 +242,21 @@ void K8sWorkerController::setNumberOfWorkers(int newNumberOfWorkers) { } this->numberOfWorkers = newNumberOfWorkers; } + +std::vector K8sWorkerController::scaleUp(int count) { + if (this->numberOfWorkers + count > this->maxWorkers) { + count = this->maxWorkers - this->numberOfWorkers; + } + std::vector workers; + auto *asyncCalls = new std::future[count]; + for (int i = 0; i < count; i++) { + // TODO (Ishad-M-I-M): Implement a robust way to give a worker id. + asyncCalls[i] = std::async(&K8sWorkerController::spawnWorker, this, i); + } + workers.reserve(count); + for (int i = 0; i < count; i++) { + workers.push_back(asyncCalls[i].get()); + } + this->numberOfWorkers += count; + return workers; +} diff --git a/src/k8s/K8sWorkerController.h b/src/k8s/K8sWorkerController.h index ffa5d0734..a60ef6c3d 100644 --- a/src/k8s/K8sWorkerController.h +++ b/src/k8s/K8sWorkerController.h @@ -33,10 +33,11 @@ class K8sWorkerController { std::string masterIp; int numberOfWorkers; + int maxWorkers; K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb); - void spawnWorker(int workerId); + std::string spawnWorker(int workerId); void deleteWorker(int workerId); @@ -55,6 +56,8 @@ class K8sWorkerController { int getNumberOfWorkers() const; void setNumberOfWorkers(int newNumberOfWorkers); + + std::vector scaleUp(int numberOfWorkers); }; #endif // JASMINEGRAPH_K8SWORKERCONTROLLER_H diff --git a/start-k8s.sh b/start-k8s.sh index eb90f923c..0bc1ac7ab 100755 --- a/start-k8s.sh +++ b/start-k8s.sh @@ -18,6 +18,7 @@ DATA_PATH=${DATA_PATH} LOG_PATH=${LOG_PATH} NO_OF_WORKERS=${NO_OF_WORKERS} ENABLE_NMON=${ENABLE_NMON} +MAX_COUNT=${MAX_COUNT} while [ $# -gt 0 ]; do @@ -58,6 +59,10 @@ if [ -z "$ENABLE_NMON" ]; then ENABLE_NMON="false" fi +if [ -z "$MAX_COUNT" ]; then + MAX_COUNT="4" +fi + kubectl apply -f ./k8s/rbac.yaml kubectl apply -f ./k8s/pushgateway.yaml @@ -98,6 +103,7 @@ done pushgateway_address="${pushgatewayIP}:9091/" \ prometheus_address="${prometheusIP}:9090/" \ + max_worker_count="${MAX_COUNT}" \ envsubst <"./k8s/configs.yaml" | kubectl apply -f - metadb_path="${META_DB_PATH}" \ diff --git a/tests/unit/k8s/K8sWorkerController_test.cpp b/tests/unit/k8s/K8sWorkerController_test.cpp index d66679ca4..9cf656377 100644 --- a/tests/unit/k8s/K8sWorkerController_test.cpp +++ b/tests/unit/k8s/K8sWorkerController_test.cpp @@ -76,3 +76,23 @@ TEST_F(K8sWorkerControllerTest, TestScalingUpAndDown) { service_list = interface->getServiceList(strdup("service=jasminegraph-worker")); ASSERT_EQ(service_list->items->count, 0); } + +TEST_F(K8sWorkerControllerTest, TestScaleUp) { + controller->setNumberOfWorkers(0); + auto result = controller->scaleUp(2); + ASSERT_EQ(result.size(), 2); + v1_deployment_list_t *deployment_list = interface->getDeploymentList(strdup("deployment=jasminegraph-worker")); + ASSERT_EQ(deployment_list->items->count, 2); + v1_service_list_t *service_list = interface->getServiceList(strdup("service=jasminegraph-worker")); + ASSERT_EQ(service_list->items->count, 2); +} + +TEST_F(K8sWorkerControllerTest, TestScaleUpBeyondLimit) { + controller->setNumberOfWorkers(0); + auto result = controller->scaleUp(5); + ASSERT_EQ(result.size(), 4); + v1_deployment_list_t *deployment_list = interface->getDeploymentList(strdup("deployment=jasminegraph-worker")); + ASSERT_EQ(deployment_list->items->count, 4); + v1_service_list_t *service_list = interface->getServiceList(strdup("service=jasminegraph-worker")); + ASSERT_EQ(service_list->items->count, 4); +} From 34c8105fd948ca369eabf531aad9b78c3b3c226d Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 13 Mar 2024 15:18:05 +0530 Subject: [PATCH 079/143] fix memory leaks --- .../JasmineGraphHashMapCentralStore.cpp | 2 + ...smineGraphHashMapDuplicateCentralStore.cpp | 2 + src/frontend/JasmineGraphFrontEnd.cpp | 42 +++++++++---------- src/frontend/core/scheduler/JobScheduler.cpp | 2 + .../JasmineGraphHashMapLocalStore.cpp | 2 + .../JasmineGraphHashMapLocalStore.h | 1 + .../JasmineGraphLocalStoreFactory.cpp | 3 +- src/partitioner/local/MetisPartitioner.cpp | 7 ++-- src/query/algorithms/triangles/Triangles.cpp | 1 + src/server/JasmineGraphInstanceService.cpp | 14 ++++--- 10 files changed, 45 insertions(+), 31 deletions(-) diff --git a/src/centralstore/JasmineGraphHashMapCentralStore.cpp b/src/centralstore/JasmineGraphHashMapCentralStore.cpp index dafdf84ef..7dc1f6344 100644 --- a/src/centralstore/JasmineGraphHashMapCentralStore.cpp +++ b/src/centralstore/JasmineGraphHashMapCentralStore.cpp @@ -58,6 +58,7 @@ bool JasmineGraphHashMapCentralStore::loadGraph() { vertexCount = centralSubgraphMap.size(); edgeCount = getEdgeCount(); + delete data; return result; } @@ -87,6 +88,7 @@ bool JasmineGraphHashMapCentralStore::loadGraph(const std::string &edgeStorePath vertexCount = centralSubgraphMap.size(); edgeCount = getEdgeCount(); + delete data; return result; } diff --git a/src/centralstore/JasmineGraphHashMapDuplicateCentralStore.cpp b/src/centralstore/JasmineGraphHashMapDuplicateCentralStore.cpp index 4254b3b8c..786058314 100644 --- a/src/centralstore/JasmineGraphHashMapDuplicateCentralStore.cpp +++ b/src/centralstore/JasmineGraphHashMapDuplicateCentralStore.cpp @@ -53,6 +53,7 @@ bool JasmineGraphHashMapDuplicateCentralStore::loadGraph() { vertexCount = centralDuplicateStoreSubgraphMap.size(); edgeCount = getEdgeCount(); + delete data; return result; } @@ -83,6 +84,7 @@ bool JasmineGraphHashMapDuplicateCentralStore::loadGraph(std::string fileName) { vertexCount = centralDuplicateStoreSubgraphMap.size(); edgeCount = getEdgeCount(); + delete data; return result; } diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index 72120e2bb..f357bc3d0 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -692,14 +692,14 @@ static void add_rdf_command(std::string masterIP, int connFd, SQLiteDBInterface GetConfig appConfig; appConfig.readConfigFile(path, newGraphID); - MetisPartitioner *metisPartitioner = new MetisPartitioner(sqlite); + MetisPartitioner metisPartitioner(sqlite); vector> fullFileList; string input_file_path = Utils::getHomeDir() + "/.jasminegraph/tmp/" + to_string(newGraphID) + "/" + to_string(newGraphID); - metisPartitioner->loadDataSet(input_file_path, newGraphID); + metisPartitioner.loadDataSet(input_file_path, newGraphID); - metisPartitioner->constructMetisFormat(Conts::GRAPH_TYPE_RDF); - fullFileList = metisPartitioner->partitioneWithGPMetis(""); + metisPartitioner.constructMetisFormat(Conts::GRAPH_TYPE_RDF); + fullFileList = metisPartitioner.partitioneWithGPMetis(""); JasmineGraphServer *server = JasmineGraphServer::getInstance(); server->uploadGraphLocally(newGraphID, Conts::GRAPH_WITH_ATTRIBUTES, fullFileList, masterIP); Utils::deleteDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp/" + to_string(newGraphID)); @@ -780,18 +780,18 @@ static void add_graph_command(std::string masterIP, int connFd, SQLiteDBInterfac name + "\", \"" + path + "\", \"" + uploadStartTime + "\", \"\",\"" + to_string(Conts::GRAPH_STATUS::LOADING) + "\", \"\", \"\", \"\")"; int newGraphID = sqlite->runInsert(sqlStatement); - MetisPartitioner *partitioner = new MetisPartitioner(sqlite); + MetisPartitioner partitioner(sqlite); vector> fullFileList; - partitioner->loadDataSet(path, newGraphID); - int result = partitioner->constructMetisFormat(Conts::GRAPH_TYPE_NORMAL); + partitioner.loadDataSet(path, newGraphID); + int result = partitioner.constructMetisFormat(Conts::GRAPH_TYPE_NORMAL); if (result == 0) { - string reformattedFilePath = partitioner->reformatDataSet(path, newGraphID); - partitioner->loadDataSet(reformattedFilePath, newGraphID); - partitioner->constructMetisFormat(Conts::GRAPH_TYPE_NORMAL_REFORMATTED); - fullFileList = partitioner->partitioneWithGPMetis(partitionCount); + string reformattedFilePath = partitioner.reformatDataSet(path, newGraphID); + partitioner.loadDataSet(reformattedFilePath, newGraphID); + partitioner.constructMetisFormat(Conts::GRAPH_TYPE_NORMAL_REFORMATTED); + fullFileList = partitioner.partitioneWithGPMetis(partitionCount); } else { - fullFileList = partitioner->partitioneWithGPMetis(partitionCount); + fullFileList = partitioner.partitioneWithGPMetis(partitionCount); } frontend_logger.info("Upload done"); JasmineGraphServer *server = JasmineGraphServer::getInstance(); @@ -950,18 +950,18 @@ static void add_graph_cust_command(std::string masterIP, int connFd, SQLiteDBInt name + "\", \"" + edgeListPath + "\", \"" + uploadStartTime + "\", \"\",\"" + to_string(Conts::GRAPH_STATUS::LOADING) + "\", \"\", \"\", \"\")"; int newGraphID = sqlite->runInsert(sqlStatement); - MetisPartitioner *partitioner = new MetisPartitioner(sqlite); + MetisPartitioner partitioner(sqlite); vector> fullFileList; - partitioner->loadContentData(attributeListPath, graphAttributeType, newGraphID, attrDataType); - partitioner->loadDataSet(edgeListPath, newGraphID); - int result = partitioner->constructMetisFormat(Conts::GRAPH_TYPE_NORMAL); + partitioner.loadContentData(attributeListPath, graphAttributeType, newGraphID, attrDataType); + partitioner.loadDataSet(edgeListPath, newGraphID); + int result = partitioner.constructMetisFormat(Conts::GRAPH_TYPE_NORMAL); if (result == 0) { - string reformattedFilePath = partitioner->reformatDataSet(edgeListPath, newGraphID); - partitioner->loadDataSet(reformattedFilePath, newGraphID); - partitioner->constructMetisFormat(Conts::GRAPH_TYPE_NORMAL_REFORMATTED); - fullFileList = partitioner->partitioneWithGPMetis(""); + string reformattedFilePath = partitioner.reformatDataSet(edgeListPath, newGraphID); + partitioner.loadDataSet(reformattedFilePath, newGraphID); + partitioner.constructMetisFormat(Conts::GRAPH_TYPE_NORMAL_REFORMATTED); + fullFileList = partitioner.partitioneWithGPMetis(""); } else { - fullFileList = partitioner->partitioneWithGPMetis(""); + fullFileList = partitioner.partitioneWithGPMetis(""); } // Graph type should be changed to identify graphs with attributes // because this graph type has additional attribute files to be uploaded diff --git a/src/frontend/core/scheduler/JobScheduler.cpp b/src/frontend/core/scheduler/JobScheduler.cpp index 081924d7d..e73050441 100644 --- a/src/frontend/core/scheduler/JobScheduler.cpp +++ b/src/frontend/core/scheduler/JobScheduler.cpp @@ -106,11 +106,13 @@ void JobScheduler::processJob(JobRequest request, SQLiteDBInterface *sqlite, Per void JobScheduler::executeJob(JobRequest request, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfDB) { ExecutorFactory *executorFactory = new ExecutorFactory(sqlite, perfDB); AbstractExecutor *abstractExecutor = executorFactory->getExecutor(request); + delete executorFactory; if (abstractExecutor == nullptr) { jobScheduler_Logger.error("abstractExecutor is null"); return; } abstractExecutor->execute(); + delete abstractExecutor; } void JobScheduler::pushJob(JobRequest jobDetails) { jobQueue.push(jobDetails); } diff --git a/src/localstore/JasmineGraphHashMapLocalStore.cpp b/src/localstore/JasmineGraphHashMapLocalStore.cpp index 3499447e9..eee0dfb67 100644 --- a/src/localstore/JasmineGraphHashMapLocalStore.cpp +++ b/src/localstore/JasmineGraphHashMapLocalStore.cpp @@ -217,6 +217,7 @@ bool JasmineGraphHashMapLocalStore::loadAttributes() { auto attributeStoreData = GetAttributeStore(data); toLocalAttributeMap(attributeStoreData); + delete data; result = true; @@ -283,6 +284,7 @@ bool JasmineGraphHashMapLocalStore::loadPartEdgeMap(const std::string filePath) auto edgeMapStoreData = GetPartEdgeMapStore(data); toLocalEdgeMap(edgeMapStoreData); + delete data; result = true; diff --git a/src/localstore/JasmineGraphHashMapLocalStore.h b/src/localstore/JasmineGraphHashMapLocalStore.h index db279a4cd..e5c9da547 100644 --- a/src/localstore/JasmineGraphHashMapLocalStore.h +++ b/src/localstore/JasmineGraphHashMapLocalStore.h @@ -84,6 +84,7 @@ class JasmineGraphHashMapLocalStore : public JasmineGraphLocalStore { vertexCount = localSubGraphMap.size(); edgeCount = getEdgeCount(); + delete data; return result; } diff --git a/src/localstore/JasmineGraphLocalStoreFactory.cpp b/src/localstore/JasmineGraphLocalStoreFactory.cpp index 34c6afaa0..5b21f7ced 100644 --- a/src/localstore/JasmineGraphLocalStoreFactory.cpp +++ b/src/localstore/JasmineGraphLocalStoreFactory.cpp @@ -17,8 +17,7 @@ JasmineGraphHashMapLocalStore JasmineGraphLocalStoreFactory::load(std::string gr std::string baseDir) { int graphIdentifier = atoi(graphId.c_str()); int partitionIdentifier = atoi(partitionId.c_str()); - JasmineGraphHashMapLocalStore hashMapLocalStore = - *new JasmineGraphHashMapLocalStore(graphIdentifier, partitionIdentifier, baseDir); + JasmineGraphHashMapLocalStore hashMapLocalStore(graphIdentifier, partitionIdentifier, baseDir); return hashMapLocalStore; } diff --git a/src/partitioner/local/MetisPartitioner.cpp b/src/partitioner/local/MetisPartitioner.cpp index 1c748d580..0efadcd7d 100644 --- a/src/partitioner/local/MetisPartitioner.cpp +++ b/src/partitioner/local/MetisPartitioner.cpp @@ -315,7 +315,7 @@ void MetisPartitioner::createPartitionFiles(std::map partMap) { edgeMap = GetConfig::getEdgeMap(); articlesMap = GetConfig::getAttributesMap(); - std::thread *threadList = new std::thread[nParts]; + std::thread threadList[nParts]; int count = 0; for (int part = 0; part < nParts; part++) { populatePartMaps(partMap, part); @@ -357,7 +357,7 @@ void MetisPartitioner::createPartitionFiles(std::map partMap) { if (graphAttributeType == Conts::GRAPH_WITH_TEXT_ATTRIBUTES || graphType == Conts::GRAPH_TYPE_RDF) { threadCount = nParts * 5; } - std::thread *threads = new std::thread[threadCount]; + std::thread threads[threadCount]; count = 0; for (int part = 0; part < nParts; part++) { threads[count++] = std::thread(&MetisPartitioner::writeSerializedPartitionFiles, this, part); @@ -517,7 +517,7 @@ void MetisPartitioner::createPartitionFiles(std::map partMap) { } std::vector::iterator compositeGraphIdListIterator; - std::thread *compositeCopyThreads = new std::thread[threadCount]; + std::thread compositeCopyThreads[threadCount]; int compositeCopyCount = 0; for (compositeGraphIdListIterator = compositeGraphIdList.begin(); @@ -997,6 +997,7 @@ void MetisPartitioner::writeSerializedCompositeMasterFiles(std::string part) { JasmineGraphHashMapCentralStore *hashMapCentralStore = new JasmineGraphHashMapCentralStore(); hashMapCentralStore->storePartEdgeMap(partMasterEdgeMap, outputFilePartMaster); + delete hashMapCentralStore; std::vector graphIds = Utils::split(part, '_'); std::vector::iterator graphIdIterator; diff --git a/src/query/algorithms/triangles/Triangles.cpp b/src/query/algorithms/triangles/Triangles.cpp index fbbef18c2..c3ff7d6c4 100644 --- a/src/query/algorithms/triangles/Triangles.cpp +++ b/src/query/algorithms/triangles/Triangles.cpp @@ -165,6 +165,7 @@ TriangleResult Triangles::countTriangles(map> ¢ral } } } + triangleTree.clear(); TriangleResult result; diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index dbe103457..b20795a58 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -3188,7 +3188,7 @@ static void aggregate_centralstore_triangles_command(int connFd, bool *loop_exit threadPriorityMutex.unlock(); } - std::string aggregatedTriangles = + const std::string &aggregatedTriangles = aggregateCentralStoreTriangles(graphId, partitionId, partitionIdList, threadPriority); if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { @@ -3218,7 +3218,7 @@ static void aggregate_centralstore_triangles_command(int connFd, bool *loop_exit std::string chunk = chunksVector.at(loopCount); if (!Utils::send_str_wrapper(connFd, chunk)) { *loop_exit_p = true; - return; + break; } } else { string chunkStatus = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, false); @@ -3228,6 +3228,8 @@ static void aggregate_centralstore_triangles_command(int connFd, bool *loop_exit } } } + chunksVector.clear(); + chunksVector.shrink_to_fit(); } static void aggregate_streaming_centralstore_triangles_command( @@ -3417,7 +3419,7 @@ static void aggregate_composite_centralstore_triangles_command(int connFd, bool std::string chunk = chunksVector.at(loopCount); if (!Utils::send_str_wrapper(connFd, chunk)) { *loop_exit_p = true; - return; + break; } } else { string chunkStatus = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, false); @@ -3427,6 +3429,8 @@ static void aggregate_composite_centralstore_triangles_command(int connFd, bool } } } + chunksVector.clear(); + chunksVector.shrink_to_fit(); } static void initiate_files_command(int connFd, bool *loop_exit_p) { @@ -3450,7 +3454,7 @@ static void initiate_files_command(int connFd, bool *loop_exit_p) { } } - std::thread *workerThreads = new std::thread[2]; + std::thread workerThreads[2]; workerThreads[0] = std::thread(&JasmineGraphInstanceService::createPartitionFiles, graphID, partitionID, "local"); workerThreads[1] = std::thread(&JasmineGraphInstanceService::createPartitionFiles, graphID, partitionID, "centralstore"); @@ -3573,7 +3577,7 @@ static void initiate_train_command(int connFd, bool *loop_exit_p) { } } - std::thread *workerThreads = new std::thread[2]; + std::thread workerThreads[2]; workerThreads[0] = std::thread(&JasmineGraphInstanceService::createPartitionFiles, graphID, partitionID, "local"); workerThreads[1] = std::thread(&JasmineGraphInstanceService::createPartitionFiles, graphID, partitionID, "centralstore"); From a78258fd410662dc0417ba2b13fdbd53bca865a6 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 13 Mar 2024 15:30:33 +0530 Subject: [PATCH 080/143] Add script to build minimal docker image --- .gitignore | 3 +- conf/jasminegraph-server.properties | 2 - minimal/Dockerfile.fs | 21 +++++ minimal/Dockerfile.minimal | 7 ++ minimal/build.sh | 100 +++++++++++++++++++++ minimal/run-docker.sh | 71 +++++++++++++++ src/partitioner/local/MetisPartitioner.cpp | 4 +- 7 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 minimal/Dockerfile.fs create mode 100644 minimal/Dockerfile.minimal create mode 100755 minimal/build.sh create mode 100755 minimal/run-docker.sh diff --git a/.gitignore b/.gitignore index 295a86613..eb4c99802 100644 --- a/.gitignore +++ b/.gitignore @@ -84,4 +84,5 @@ __pycache__ *.db *.db-* -coverage.xml \ No newline at end of file +coverage.xml +*.tmp diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 482a7148e..576b4cd6b 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -29,8 +29,6 @@ org.jasminegraph.worker.path=/var/tmp/jasminegraph/ #JASMINEGRAPH_HOME location will be copied instead. #if it is empty please leave a space, else it will cause an issue org.jasminegraph.artifact.path= -#org.jasminegraph.partitioner.metis.bin is the location where the METIS graph partitioner's gpmetis executable is installed -org.jasminegraph.partitioner.metis.bin=/usr/local/bin #The following folder is the location where workers keep their data. #This is the location where the actual data storage takes place in JasmineGraph. org.jasminegraph.server.instance.datafolder=/var/tmp/jasminegraph-localstore diff --git a/minimal/Dockerfile.fs b/minimal/Dockerfile.fs new file mode 100644 index 000000000..7db405387 --- /dev/null +++ b/minimal/Dockerfile.fs @@ -0,0 +1,21 @@ +FROM busybox:stable-glibc + +COPY --chmod=755 files.tmp/lib/* /lib/x86_64-linux-gnu/ +COPY --chmod=755 files.tmp/bin/* /bin/ + +RUN mkdir -p /home/ubuntu/software/jasminegraph/conf +RUN mkdir -p /home/ubuntu/software/jasminegraph/k8s +RUN mkdir -p /home/ubuntu/software/jasminegraph/metadb +RUN mkdir -p /home/ubuntu/software/jasminegraph/performancedb +RUN mkdir -p /home/ubuntu/software/jasminegraph/src/metadb +RUN mkdir -p /home/ubuntu/software/jasminegraph/src/performancedb +RUN mkdir -p /home/ubuntu/software/jasminegraph/src/streamingdb + +COPY --chmod=755 files.tmp/JasmineGraph /home/ubuntu/software/jasminegraph/ +COPY --chmod=755 minimal/run-docker.sh /home/ubuntu/software/jasminegraph/ +COPY --chmod=644 conf/jasminegraph-server.properties /home/ubuntu/software/jasminegraph/conf/ +COPY --chmod=644 conf/hosts.txt /home/ubuntu/software/jasminegraph/conf/ +COPY --chmod=644 k8s/*.yaml /home/ubuntu/software/jasminegraph/k8s/ +COPY --chmod=644 src/metadb/ddl.sql /home/ubuntu/software/jasminegraph/src/metadb/ +COPY --chmod=644 src/performancedb/ddl.sql /home/ubuntu/software/jasminegraph/src/performancedb/ +COPY --chmod=644 src/streamingdb/ddl.sql /home/ubuntu/software/jasminegraph/src/streamingdb/ diff --git a/minimal/Dockerfile.minimal b/minimal/Dockerfile.minimal new file mode 100644 index 000000000..7ddd81d4a --- /dev/null +++ b/minimal/Dockerfile.minimal @@ -0,0 +1,7 @@ +FROM jasminegraph:squash + +ENV HOME="/home/ubuntu" +ENV JASMINEGRAPH_HOME="${HOME}/software/jasminegraph" +WORKDIR "${JASMINEGRAPH_HOME}" +ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"] +CMD ["ash"] diff --git a/minimal/build.sh b/minimal/build.sh new file mode 100755 index 000000000..1a3aa77f6 --- /dev/null +++ b/minimal/build.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +set -e + +libs=( + /lib/x86_64-linux-gnu/libcurl.so.4 + /lib/x86_64-linux-gnu/libsqlite3.so.0 + /lib/x86_64-linux-gnu/libfmt.so.8 + /lib/x86_64-linux-gnu/libxerces-c-3.2.so + /lib/x86_64-linux-gnu/libjsoncpp.so.25 + /usr/local/lib/libcppkafka.so.0.4.0 + /usr/local/lib/libkubernetes.so + /lib/x86_64-linux-gnu/libyaml-cpp.so.0.7 + /lib/x86_64-linux-gnu/libstdc++.so.6 + /lib/x86_64-linux-gnu/libgcc_s.so.1 + /lib/x86_64-linux-gnu/libnghttp2.so.14 + /lib/x86_64-linux-gnu/libidn2.so.0 + /lib/x86_64-linux-gnu/librtmp.so.1 + /lib/x86_64-linux-gnu/libssh.so.4 + /lib/x86_64-linux-gnu/libpsl.so.5 + /lib/x86_64-linux-gnu/libssl.so.3 + /lib/x86_64-linux-gnu/libcrypto.so.3 + /lib/x86_64-linux-gnu/libgssapi_krb5.so.2 + /lib/x86_64-linux-gnu/libldap-2.5.so.0 + /lib/x86_64-linux-gnu/liblber-2.5.so.0 + /lib/x86_64-linux-gnu/libzstd.so.1 + /lib/x86_64-linux-gnu/libbrotlidec.so.1 + /lib/x86_64-linux-gnu/libz.so.1 + /lib/x86_64-linux-gnu/libcurl-gnutls.so.4 + /lib/x86_64-linux-gnu/libicuuc.so.70 + /lib/x86_64-linux-gnu/librdkafka.so.1 + /usr/local/lib/libyaml.so + /usr/local/lib/libwebsockets.so.18 + /lib/x86_64-linux-gnu/libunistring.so.2 + /lib/x86_64-linux-gnu/libgnutls.so.30 + /lib/x86_64-linux-gnu/libhogweed.so.6 + /lib/x86_64-linux-gnu/libnettle.so.8 + /lib/x86_64-linux-gnu/libgmp.so.10 + /lib/x86_64-linux-gnu/libkrb5.so.3 + /lib/x86_64-linux-gnu/libk5crypto.so.3 + /lib/x86_64-linux-gnu/libcom_err.so.2 + /lib/x86_64-linux-gnu/libkrb5support.so.0 + /lib/x86_64-linux-gnu/libsasl2.so.2 + /lib/x86_64-linux-gnu/libbrotlicommon.so.1 + /lib/x86_64-linux-gnu/libicudata.so.70 + /lib/x86_64-linux-gnu/liblz4.so.1 + /lib/x86_64-linux-gnu/libp11-kit.so.0 + /lib/x86_64-linux-gnu/libtasn1.so.6 + /lib/x86_64-linux-gnu/libkeyutils.so.1 + /lib/x86_64-linux-gnu/libresolv.so.2 + /lib/x86_64-linux-gnu/libffi.so.8 + /lib/x86_64-linux-gnu/libncurses.so.6 + /usr/local/lib/libmetis.so +) + +bins=( + /usr/bin/nmon + /usr/local/bin/gpmetis +) + +if [ ! -f Dockerfile ]; then + echo 'Run the script from JasmineGraph root' + exit 1 +fi + +docker build -t jasminegraph . + +TEMP_DIR="files.tmp" +mkdir -p "$TEMP_DIR" +cd "$TEMP_DIR" +rm -r * || true +mkdir -p bin +mkdir -p lib + +cont_jasmine="$(docker create jasminegraph)" +for file in "${libs[@]}"; do + echo "Copying $file" + docker cp -L "$cont_jasmine":"$file" lib/ | sed '/Copying from container/d' +done +for file in "${bins[@]}"; do + echo "Copying $file" + docker cp -L "$cont_jasmine":"$file" bin/ | sed '/Copying from container/d' +done +echo 'Copying JasmineGraph' +docker cp "$cont_jasmine":/home/ubuntu/software/jasminegraph/JasmineGraph ./ | sed '/Copying from container/d' +docker rm "$cont_jasmine" + +docker build -t jasminegraph:fs -f ../minimal/Dockerfile.fs .. +cont_fs="$(docker create jasminegraph:fs)" +docker export "$cont_fs" | docker import - jasminegraph:squash +docker rm "$cont_fs" +docker rmi jasminegraph:fs + +docker build -t jasminegraph:minimal -f ../minimal/Dockerfile.minimal . + +docker rmi jasminegraph:squash +cd .. +rm -r "$TEMP_DIR" + +echo 'Created docker image jasminegraph:minimal' diff --git a/minimal/run-docker.sh b/minimal/run-docker.sh new file mode 100755 index 000000000..a3a7715f4 --- /dev/null +++ b/minimal/run-docker.sh @@ -0,0 +1,71 @@ +#!/bin/ash + +MODE=${MODE} +MASTERIP=${MASTERIP} +WORKERS=${WORKERS} +WORKERIP=${WORKERIP} +HOST_NAME=${HOST_NAME} +SERVER_PORT=${SERVER_PORT} +SERVER_DATA_PORT=${SERVER_DATA_PORT} +ENABLE_NMON=${ENABLE_NMON} +DEBUG=${DEBUG} +PROFILE=${PROFILE} + +if [ -z "$PROFILE" ]; then + PROFILE="k8s" +fi + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + export $param="$2" + echo $1 $2 + fi + + shift +done + +if [ -z "$MODE" ]; then + exit 1 +fi + +if [ $MODE -eq 1 ]; then + if [ "$PROFILE" = "k8s" ]; then + MASTERIP="x" + WORKERIP="x" + fi + if [ -z "$MASTERIP" ]; then + exit 1 + fi + if [ -z "$WORKERS" ]; then + exit 1 + fi + if [ -z "$WORKERIP" ]; then + exit 1 + fi +else + if [ -z "$MASTERIP" ]; then + exit 1 + fi + if [ -z "$SERVER_PORT" ]; then + exit 1 + fi + if [ -z "$SERVER_DATA_PORT" ]; then + exit 1 + fi +fi + +if [ ! -d /tmp/jasminegraph ]; then + mkdir /tmp/jasminegraph +fi + +if [ $MODE -eq 1 ]; then + ./JasmineGraph $PROFILE $MODE $MASTERIP $WORKERS $WORKERIP $ENABLE_NMON +else + ./JasmineGraph $PROFILE $MODE $HOST_NAME $MASTERIP $SERVER_PORT $SERVER_DATA_PORT $ENABLE_NMON +fi + +if [ "$TESTING" = "true" ]; then + chmod -R go+w /tmp/jasminegraph +fi diff --git a/src/partitioner/local/MetisPartitioner.cpp b/src/partitioner/local/MetisPartitioner.cpp index 0efadcd7d..830947916 100644 --- a/src/partitioner/local/MetisPartitioner.cpp +++ b/src/partitioner/local/MetisPartitioner.cpp @@ -211,9 +211,7 @@ std::vector> MetisPartitioner::partitioneWithGPMetis( char buffer[128]; std::string result = ""; - std::string metisBinDir = Utils::getJasmineGraphProperty("org.jasminegraph.partitioner.metis.bin"); - string metisCommand = - metisBinDir + "/gpmetis " + this->outputFilePath + "/grf " + to_string(this->nParts) + " 2>&1"; + string metisCommand = "gpmetis " + this->outputFilePath + "/grf " + to_string(this->nParts) + " 2>&1"; partitioner_logger.log("metisCommand " + metisCommand, "info"); FILE *input = popen(metisCommand.c_str(), "r"); From 3b56365c1ee24aeec8fc829b6c5b7e522affcfdc Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 13 Mar 2024 16:11:59 +0530 Subject: [PATCH 081/143] Remove unused method --- src/localstore/JasmineGraphHashMapLocalStore.cpp | 11 ----------- src/localstore/JasmineGraphHashMapLocalStore.h | 3 --- src/server/JasmineGraphInstanceService.cpp | 4 ++-- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/src/localstore/JasmineGraphHashMapLocalStore.cpp b/src/localstore/JasmineGraphHashMapLocalStore.cpp index eee0dfb67..0d5da2716 100644 --- a/src/localstore/JasmineGraphHashMapLocalStore.cpp +++ b/src/localstore/JasmineGraphHashMapLocalStore.cpp @@ -113,17 +113,6 @@ unordered_set JasmineGraphHashMapLocalStore::getVertexSet() { return vertexSet; } -int *JasmineGraphHashMapLocalStore::getOutDegreeDistribution() { - distributionArray = new int[vertexCount]; - int counter = 0; - - for (map>::iterator it = localSubGraphMap.begin(); it != localSubGraphMap.end(); ++it) { - distributionArray[counter] = (it->second).size(); - counter++; - } - return distributionArray; -} - map JasmineGraphHashMapLocalStore::getOutDegreeDistributionHashMap() { map distributionHashMap; diff --git a/src/localstore/JasmineGraphHashMapLocalStore.h b/src/localstore/JasmineGraphHashMapLocalStore.h index e5c9da547..54d1f89b0 100644 --- a/src/localstore/JasmineGraphHashMapLocalStore.h +++ b/src/localstore/JasmineGraphHashMapLocalStore.h @@ -42,7 +42,6 @@ class JasmineGraphHashMapLocalStore : public JasmineGraphLocalStore { long vertexCount; long edgeCount; - int *distributionArray; std::string getFileSeparator(); @@ -103,8 +102,6 @@ class JasmineGraphHashMapLocalStore : public JasmineGraphLocalStore { unordered_set getVertexSet(); - int *getOutDegreeDistribution(); - map getOutDegreeDistributionHashMap(); map getInDegreeDistributionHashMap(); diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index b20795a58..620512e34 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -815,7 +815,6 @@ int JasmineGraphInstanceService::collectTrainedModelThreadFunction(instanceservi void JasmineGraphInstanceService::createPartitionFiles(std::string graphID, std::string partitionID, std::string fileType) { - JasmineGraphHashMapLocalStore *hashMapLocalStore = new JasmineGraphHashMapLocalStore(); string inputFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + graphID + "_" + partitionID; string outputFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.trainedmodelfolder") + @@ -826,7 +825,8 @@ void JasmineGraphInstanceService::createPartitionFiles(std::string graphID, std: outputFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.trainedmodelfolder") + "/" + graphID + "_centralstore_" + partitionID; } - std::map> partEdgeMap = hashMapLocalStore->getEdgeHashMap(inputFilePath); + JasmineGraphHashMapLocalStore hashMapLocalStore; + std::map> partEdgeMap = hashMapLocalStore.getEdgeHashMap(inputFilePath); if (partEdgeMap.empty()) { return; From 78bff54a866984c0dde3ca009825c4637ab0231c Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Wed, 13 Mar 2024 22:46:35 +0530 Subject: [PATCH 082/143] Return map instead of a vector for the create new workers in scale up --- src/k8s/K8sWorkerController.cpp | 11 +++++++---- src/k8s/K8sWorkerController.h | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 4abfb37ab..c79f9ac2a 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -243,19 +243,22 @@ void K8sWorkerController::setNumberOfWorkers(int newNumberOfWorkers) { this->numberOfWorkers = newNumberOfWorkers; } -std::vector K8sWorkerController::scaleUp(int count) { +std::map K8sWorkerController::scaleUp(int count) { if (this->numberOfWorkers + count > this->maxWorkers) { count = this->maxWorkers - this->numberOfWorkers; } - std::vector workers; + std::map workers; auto *asyncCalls = new std::future[count]; for (int i = 0; i < count; i++) { // TODO (Ishad-M-I-M): Implement a robust way to give a worker id. asyncCalls[i] = std::async(&K8sWorkerController::spawnWorker, this, i); } - workers.reserve(count); + for (int i = 0; i < count; i++) { - workers.push_back(asyncCalls[i].get()); + std::string result = asyncCalls[i].get(); + if (!result.empty()) { + workers.insert(std::make_pair(i, result)); + } } this->numberOfWorkers += count; return workers; diff --git a/src/k8s/K8sWorkerController.h b/src/k8s/K8sWorkerController.h index a60ef6c3d..577501a48 100644 --- a/src/k8s/K8sWorkerController.h +++ b/src/k8s/K8sWorkerController.h @@ -57,7 +57,7 @@ class K8sWorkerController { void setNumberOfWorkers(int newNumberOfWorkers); - std::vector scaleUp(int numberOfWorkers); + std::map scaleUp(int numberOfWorkers); }; #endif // JASMINEGRAPH_K8SWORKERCONTROLLER_H From c968f81147b57daa6a49bf76ad7ea2b4e511ecb6 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Wed, 13 Mar 2024 22:59:48 +0530 Subject: [PATCH 083/143] Keep track of active worker ids --- src/k8s/K8sWorkerController.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index c79f9ac2a..547ed0cc2 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -22,6 +22,15 @@ Logger controller_logger; std::vector K8sWorkerController::workerList = {}; static int TIME_OUT = 180; +static std::vector activeWorkerIds = {}; + +static inline int getNextWorkerId() { + if (activeWorkerIds.empty()) { + return 0; + } else { + return *max_element(activeWorkerIds.begin(), activeWorkerIds.end()) + 1; + } +} K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb) { this->masterIp = std::move(masterIp); @@ -40,7 +49,7 @@ K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorke // Delete all the workers from the database metadb->runUpdate("DELETE FROM worker"); int workersAttached = this->attachExistingWorkers(); - for (int i = workersAttached; i < numberOfWorkers; i++) { + for (int i = getNextWorkerId(); i < numberOfWorkers; i++) { this->spawnWorker(i); } } @@ -135,7 +144,7 @@ std::string K8sWorkerController::spawnWorker(int workerId) { if (status == -1) { controller_logger.error("Worker " + std::to_string(workerId) + " database insertion failed"); } - + activeWorkerIds.push_back(workerId); return ip + ":" + to_string(Conts::JASMINEGRAPH_INSTANCE_PORT); } @@ -170,6 +179,7 @@ void K8sWorkerController::deleteWorker(int workerId) { std::string deleteQuery = "DELETE FROM worker WHERE idworker = " + std::to_string(workerId); metadb.runUpdate(deleteQuery); + std::remove(activeWorkerIds.begin(), activeWorkerIds.end(), workerId); } int K8sWorkerController::attachExistingWorkers() { @@ -213,6 +223,7 @@ int K8sWorkerController::attachExistingWorkers() { if (status == -1) { controller_logger.error("Worker " + std::to_string(workerId) + " database insertion failed"); } + activeWorkerIds.push_back(workerId); break; } } @@ -249,9 +260,9 @@ std::map K8sWorkerController::scaleUp(int count) { } std::map workers; auto *asyncCalls = new std::future[count]; + int nextWorkerId = getNextWorkerId(); for (int i = 0; i < count; i++) { - // TODO (Ishad-M-I-M): Implement a robust way to give a worker id. - asyncCalls[i] = std::async(&K8sWorkerController::spawnWorker, this, i); + asyncCalls[i] = std::async(&K8sWorkerController::spawnWorker, this, nextWorkerId + i); } for (int i = 0; i < count; i++) { From 9e69a627248c986e7dfdd47173360315de4d4d87 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Thu, 14 Mar 2024 01:05:11 +0530 Subject: [PATCH 084/143] Call Utils::transferPartition and update metadb on transfer partitions --- conf/jasminegraph-server.properties | 1 + .../executor/impl/TriangleCountExecutor.cpp | 100 +++++++++++++----- src/server/JasmineGraphServer.cpp | 3 +- 3 files changed, 75 insertions(+), 29 deletions(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 576b4cd6b..c7f93f1db 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -98,5 +98,6 @@ org.jasminegraph.k8s.image=jasminegraph #-------------------------------------------------------------------------------- # Auto-scaling #-------------------------------------------------------------------------------- +org.jasminegraph.autoscale.scale_on_adgr=false org.jasminegraph.autoscale.enabled=false org.jasminegraph.autopartition.enabled=false diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index ddace2a63..0fa61404e 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -189,7 +189,7 @@ static std::vector reallocate_parts(std::map &alloc, std::set< return copying; } -static void scale_up(std::map &loads, int copy_cnt) { +static void scale_up(std::map &loads, map &workers, int copy_cnt) { int curr_load = 0; for (auto it = loads.begin(); it != loads.end(); it++) { curr_load += it->second; @@ -201,10 +201,14 @@ static void scale_up(std::map &loads, int copy_cnt) { if (n_workers == 0) return; // TODO(thevindu-w): replace with call to K8sWorkerController scale up - vector w_new; - for (int i = 0; i < n_workers; i++) w_new.push_back(string("10.11.12.") + to_string(i) + string(":7780")); + map w_new; + for (int i = 0; i < n_workers; i++) + w_new[to_string(i + loads.size())] = string("10.11.12.") + to_string(i) + string(":7780"); - for (auto it = w_new.begin(); it != w_new.end(); it++) loads[*it] = 0; + for (auto it = w_new.begin(); it != w_new.end(); it++) { + loads[it->first] = 0.1; + workers[it->first] = it->second; + } } static int alloc_net_plan(std::vector &parts, std::map> &transfer, @@ -274,24 +278,30 @@ static int alloc_net_plan(std::vector &parts, std::map> &partitionMap, SQLiteDBInterface *sqlite) { - vector workers; - const std::vector>> &results = sqlite->runSelect("SELECT ip,server_port FROM worker;"); +static void filter_partitions(std::map> &partitionMap, SQLiteDBInterface *sqlite, + string graphId) { + map workers; + const std::vector>> &results = + sqlite->runSelect("SELECT idworker,ip,server_port FROM worker;"); for (int i = 0; i < results.size(); i++) { - string ip = results[i][0].second; - string port = results[i][1].second; - workers.push_back(ip + ":" + port); + string workerId = results[i][0].second; + string ip = results[i][1].second; + string port = results[i][2].second; + workers[workerId] = ip + ":" + port; } map loads; const map &cpu_map = Utils::getMetricMap("cpu_usage"); for (auto it = workers.begin(); it != workers.end(); it++) { - auto worker = *it; + auto workerId = it->first; + auto worker = it->second; const auto workerLoadIt = cpu_map.find(worker); if (workerLoadIt != cpu_map.end()) { - loads[worker] = (int)round(stod(workerLoadIt->second.c_str())); + double load = stod(workerLoadIt->second.c_str()); + if (load < 0.1) load = 0.1; + loads[workerId] = (int)round(4 * load - 0.6); } else { - loads[worker] = 0; + loads[workerId] = 0; } } const map LOADS = loads; @@ -313,10 +323,10 @@ static void filter_partitions(std::map> &partitionMa if (loadIt->second < 3) continue; auto w = loadIt->first; for (auto it = p_avail.begin(); it != p_avail.end(); it++) { - auto &workers = it->second; - for (auto workerIt = workers.begin(); workerIt != workers.end();) { + auto &partitionWorkers = it->second; + for (auto workerIt = partitionWorkers.begin(); workerIt != partitionWorkers.end();) { if (*workerIt == w) { - workers.erase(workerIt); + partitionWorkers.erase(workerIt); } else { workerIt++; } @@ -328,7 +338,7 @@ static void filter_partitions(std::map> &partitionMa int unallocated = alloc_plan(alloc, remain, p_avail, loads); if (unallocated > 0) { auto copying = reallocate_parts(alloc, remain, P_AVAIL); - scale_up(loads, copying.size()); + scale_up(loads, workers, copying.size()); map net_loads; for (auto it = loads.begin(); it != loads.end(); it++) { @@ -349,8 +359,42 @@ static void filter_partitions(std::map> &partitionMa auto w_to = it->second.second; alloc[p] = w_to; } - } + if (!transfer.empty()) { + const std::vector>> &workerData = + sqlite->runSelect("SELECT ip,server_port,server_data_port FROM worker;"); + map dataPortMap; + for (int i = 0; i < results.size(); i++) { + string ip = results[i][0].second; + string port = results[i][1].second; + string dport = results[i][2].second; + workers[ip + ":" + port] = dport; + } + map workers_r; + for (auto it = workers.begin(); it != workers.end(); it++) { + workers_r[it->second] = it->first; + } + for (auto it = transfer.begin(); it != transfer.end(); it++) { + auto partition = it->first; + auto w_from = workers[it->second.first]; + auto w_to = workers[it->second.second]; + const auto &ip_port_from = Utils::split(w_from, ':'); + auto ip_from = ip_port_from[0]; + auto port_from = stoi(ip_port_from[1]); + auto dport_from = stoi(dataPortMap[w_from]); + const auto &ip_port_to = Utils::split(w_to, ':'); + auto ip_to = ip_port_to[0]; + auto port_to = stoi(ip_port_to[1]); + auto dport_to = stoi(dataPortMap[w_to]); + Utils::transferPartition(ip_from, port_from, dport_from, ip_to, port_to, dport_to, stoi(graphId), + partition); + sqlite->runInsert( + "INSERT INTO worker_has_partition " + "(partition_idpartition, partition_graph_idgraph, worker_idworker) VALUES ('" + + to_string(partition) + "', '" + graphId + "', '" + workers_r[w_to] + "')"); + } + } + } partitionMap.clear(); for (auto it = alloc.begin(); it != alloc.end(); it++) { auto partition = it->first; @@ -443,7 +487,7 @@ void TriangleCountExecutor::execute() { if (jasminegraph_profile == PROFILE_K8S && Utils::getJasmineGraphProperty("org.jasminegraph.autoscale.enabled") == "true") { - filter_partitions(partitionMap, sqlite); + filter_partitions(partitionMap, sqlite, graphId); } if (results.size() > Conts::COMPOSITE_CENTRAL_STORE_WORKER_THRESHOLD) { @@ -736,8 +780,8 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int std::vector fileNameParts = Utils::split(rawFileName, '_'); - /*Partition numbers are extracted from the file name. The starting index of partition number is 2. - * Therefore the loop starts with 2*/ + /*Partition numbers are extracted from the file name. The starting index of partition number + * is 2. Therefore the loop starts with 2*/ for (int index = 2; index < fileNameParts.size(); ++index) { partitionSet.insert(fileNameParts[index]); } @@ -809,7 +853,8 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int triangleCount_logger.log("###COMPOSITE### Retrieved Composite triangle list ", "debug"); - const auto &triangles = countCompositeCentralStoreTriangles(host, std::to_string(port), adjustedTransferredFile, + const auto &triangles = + countCompositeCentralStoreTriangles(host, std::to_string(port), adjustedTransferredFile, masterIP, adjustedAvailableFiles, threadPriority); if (triangles.size() > 0) { triangleCount += updateTriangleTreeAndGetTriangleCount(triangles); @@ -1353,11 +1398,9 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st return response; } -std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string aggregatorHostName, - std::string aggregatorPort, - std::string compositeCentralStoreFileList, - std::string masterIP, std::string availableFileList, - int threadPriority) { +std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( + std::string aggregatorHostName, std::string aggregatorPort, std::string compositeCentralStoreFileList, + std::string masterIP, std::string availableFileList, int threadPriority) { int sockfd; char data[301]; bool loop = false; @@ -1526,7 +1569,8 @@ std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles(s } Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); - return Utils::split(response, ':');; + return Utils::split(response, ':'); + ; } static std::vector> getWorkerCombination(std::vector &workerIdVector) { diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 3b6d444c4..d23f4a170 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -825,7 +825,8 @@ static map scaleK8s(size_t npart) { } static std::vector getWorkers(size_t npart) { - if (Utils::getJasmineGraphProperty("org.jasminegraph.autoscale.enabled") != "true") { + if (Utils::getJasmineGraphProperty("org.jasminegraph.autoscale.scale_on_adgr") != "true" || + Utils::getJasmineGraphProperty("org.jasminegraph.autoscale.enabled") != "true") { if (jasminegraph_profile == PROFILE_K8S) { return K8sWorkerController::workerList; } From e0fbaa9084b1cb25520f4df9e6a9bf9f3c933527 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Thu, 14 Mar 2024 01:35:13 +0530 Subject: [PATCH 085/143] Use K8sWorkerController::scaleUp in triangle count scaling function --- src/frontend/core/executor/impl/TriangleCountExecutor.cpp | 7 +++---- src/k8s/K8sWorkerController.cpp | 7 ++++--- src/k8s/K8sWorkerController.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 0fa61404e..366b1ae02 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -12,6 +12,7 @@ limitations under the License. */ #include "TriangleCountExecutor.h" +#include "../../../../k8s/K8sWorkerController.h" #include "../../../../../globals.h" @@ -200,10 +201,8 @@ static void scale_up(std::map &loads, map &workers, if (n_cores % 3 > 0) n_workers++; if (n_workers == 0) return; - // TODO(thevindu-w): replace with call to K8sWorkerController scale up - map w_new; - for (int i = 0; i < n_workers; i++) - w_new[to_string(i + loads.size())] = string("10.11.12.") + to_string(i) + string(":7780"); + K8sWorkerController *k8sController = K8sWorkerController::getInstance(); + map w_new = k8sController->scaleUp(n_workers); for (auto it = w_new.begin(); it != w_new.end(); it++) { loads[it->first] = 0.1; diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 547ed0cc2..7bda78d00 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -122,6 +122,7 @@ std::string K8sWorkerController::spawnWorker(int workerId) { waiting += 30; // Added overhead in connection retry attempts sleep(10); } else { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); break; } @@ -254,21 +255,21 @@ void K8sWorkerController::setNumberOfWorkers(int newNumberOfWorkers) { this->numberOfWorkers = newNumberOfWorkers; } -std::map K8sWorkerController::scaleUp(int count) { +std::map K8sWorkerController::scaleUp(int count) { if (this->numberOfWorkers + count > this->maxWorkers) { count = this->maxWorkers - this->numberOfWorkers; } - std::map workers; auto *asyncCalls = new std::future[count]; int nextWorkerId = getNextWorkerId(); for (int i = 0; i < count; i++) { asyncCalls[i] = std::async(&K8sWorkerController::spawnWorker, this, nextWorkerId + i); } + std::map workers; for (int i = 0; i < count; i++) { std::string result = asyncCalls[i].get(); if (!result.empty()) { - workers.insert(std::make_pair(i, result)); + workers.insert(std::make_pair(to_string(i), result)); } } this->numberOfWorkers += count; diff --git a/src/k8s/K8sWorkerController.h b/src/k8s/K8sWorkerController.h index 577501a48..33b09d2c4 100644 --- a/src/k8s/K8sWorkerController.h +++ b/src/k8s/K8sWorkerController.h @@ -57,7 +57,7 @@ class K8sWorkerController { void setNumberOfWorkers(int newNumberOfWorkers); - std::map scaleUp(int numberOfWorkers); + std::map scaleUp(int numberOfWorkers); }; #endif // JASMINEGRAPH_K8SWORKERCONTROLLER_H From 411dd75e13f8fa1328fe79d8c221ffd7a17acc23 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Thu, 14 Mar 2024 02:53:06 +0530 Subject: [PATCH 086/143] Bug fix --- .../executor/impl/TriangleCountExecutor.cpp | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 366b1ae02..075352526 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -12,9 +12,9 @@ limitations under the License. */ #include "TriangleCountExecutor.h" -#include "../../../../k8s/K8sWorkerController.h" #include "../../../../../globals.h" +#include "../../../../k8s/K8sWorkerController.h" using namespace std::chrono; @@ -334,10 +334,15 @@ static void filter_partitions(std::map> &partitionMa } std::map alloc; + cout << "calling alloc_plan" << endl; int unallocated = alloc_plan(alloc, remain, p_avail, loads); + cout << "alloc_plan returned " << unallocated << endl; if (unallocated > 0) { + cout << "calling reallocate_parts" << endl; auto copying = reallocate_parts(alloc, remain, P_AVAIL); + cout << "reallocate_parts completed" << endl; scale_up(loads, workers, copying.size()); + cout << "scale_up completed" << endl; map net_loads; for (auto it = loads.begin(); it != loads.end(); it++) { @@ -353,20 +358,23 @@ static void filter_partitions(std::map> &partitionMa std::map> transfer; int net_load = alloc_net_plan(copying, transfer, net_loads, loads, P_AVAIL, 100000000); + cout << "alloc_net_plan completed with net_load=" << net_load << endl; for (auto it = transfer.begin(); it != transfer.end(); it++) { auto p = it->first; auto w_to = it->second.second; alloc[p] = w_to; + cout << "planned sending partition " << p << " from " << it->second.first << " to " << it->second.second + << endl; } if (!transfer.empty()) { const std::vector>> &workerData = sqlite->runSelect("SELECT ip,server_port,server_data_port FROM worker;"); map dataPortMap; - for (int i = 0; i < results.size(); i++) { - string ip = results[i][0].second; - string port = results[i][1].second; - string dport = results[i][2].second; + for (int i = 0; i < workerData.size(); i++) { + string ip = workerData[i][0].second; + string port = workerData[i][1].second; + string dport = workerData[i][2].second; workers[ip + ":" + port] = dport; } map workers_r; @@ -385,8 +393,12 @@ static void filter_partitions(std::map> &partitionMa auto ip_to = ip_port_to[0]; auto port_to = stoi(ip_port_to[1]); auto dport_to = stoi(dataPortMap[w_to]); + cout << "Sending partition " << partition << " from " << w_from << ":" << dport_from << " to " << w_to + << ":" << dport_to << endl; + // TODO(thevindu-w): run transferPartition in a separate thread Utils::transferPartition(ip_from, port_from, dport_from, ip_to, port_to, dport_to, stoi(graphId), partition); + cout << "Send completed"; sqlite->runInsert( "INSERT INTO worker_has_partition " "(partition_idpartition, partition_graph_idgraph, worker_idworker) VALUES ('" + @@ -484,11 +496,35 @@ void TriangleCountExecutor::execute() { triangleCount_logger.info("###TRIANGLE-COUNT-EXECUTOR### Getting Triangle Count : PartitionId " + partitionId); } + cout << "initial partitionMap = {" << endl; + for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { + cout << " " << it->first << ": ["; + auto &pts = it->second; + for (auto it2 = pts.begin(); it2 != pts.end(); it2++) { + cout << *it2 << ", "; + } + cout << "]" << endl; + } + cout << "}" << endl; + cout << endl; + if (jasminegraph_profile == PROFILE_K8S && Utils::getJasmineGraphProperty("org.jasminegraph.autoscale.enabled") == "true") { filter_partitions(partitionMap, sqlite, graphId); } + cout << "initial partitionMap = {" << endl; + for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { + cout << " " << it->first << ": ["; + auto &pts = it->second; + for (auto it2 = pts.begin(); it2 != pts.end(); it2++) { + cout << *it2 << ", "; + } + cout << "]" << endl; + } + cout << "}" << endl; + cout << endl; + if (results.size() > Conts::COMPOSITE_CENTRAL_STORE_WORKER_THRESHOLD) { isCompositeAggregation = true; } From eb5bb888c7f1b38ca95e1473220c866cb5d1a727 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Thu, 14 Mar 2024 03:15:32 +0530 Subject: [PATCH 087/143] Bug fix --- .../core/executor/impl/TriangleCountExecutor.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 075352526..12413b6a8 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -279,7 +279,7 @@ static int alloc_net_plan(std::vector &parts, std::map> &partitionMap, SQLiteDBInterface *sqlite, string graphId) { - map workers; + map workers; // id => "ip:port" const std::vector>> &results = sqlite->runSelect("SELECT idworker,ip,server_port FROM worker;"); for (int i = 0; i < results.size(); i++) { @@ -297,8 +297,8 @@ static void filter_partitions(std::map> &partitionMa const auto workerLoadIt = cpu_map.find(worker); if (workerLoadIt != cpu_map.end()) { double load = stod(workerLoadIt->second.c_str()); - if (load < 0.1) load = 0.1; - loads[workerId] = (int)round(4 * load - 0.6); + if (load < 0) load = 0; + loads[workerId] = (int)round(4 * load); } else { loads[workerId] = 0; } @@ -370,14 +370,15 @@ static void filter_partitions(std::map> &partitionMa if (!transfer.empty()) { const std::vector>> &workerData = sqlite->runSelect("SELECT ip,server_port,server_data_port FROM worker;"); - map dataPortMap; + map dataPortMap; // "ip:port" => data_port for (int i = 0; i < workerData.size(); i++) { string ip = workerData[i][0].second; string port = workerData[i][1].second; string dport = workerData[i][2].second; - workers[ip + ":" + port] = dport; + dataPortMap[ip + ":" + port] = dport; + cout << "dataPortMap[" << ip + ":" + port << "] = " << dport << endl; } - map workers_r; + map workers_r; // "ip:port" => id for (auto it = workers.begin(); it != workers.end(); it++) { workers_r[it->second] = it->first; } From c11f626c1f0fe855bed716fd24a19e6f40469eef Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 14 Mar 2024 06:25:38 +0530 Subject: [PATCH 088/143] Get rid of spawning workers sequentially at beginning --- src/k8s/K8sWorkerController.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 7bda78d00..39966316d 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -49,8 +49,8 @@ K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorke // Delete all the workers from the database metadb->runUpdate("DELETE FROM worker"); int workersAttached = this->attachExistingWorkers(); - for (int i = getNextWorkerId(); i < numberOfWorkers; i++) { - this->spawnWorker(i); + if (numberOfWorkers - workersAttached > 0){ + this->scaleUp(numberOfWorkers-workersAttached); } } @@ -128,6 +128,8 @@ std::string K8sWorkerController::spawnWorker(int workerId) { } if (waiting >= TIME_OUT) { + controller_logger.error("Error in spawning new worker"); + deleteWorker(workerId); return ""; } @@ -266,12 +268,14 @@ std::map K8sWorkerController::scaleUp(int count) { } std::map workers; + int success = 0; for (int i = 0; i < count; i++) { std::string result = asyncCalls[i].get(); if (!result.empty()) { + success++; workers.insert(std::make_pair(to_string(i), result)); } } - this->numberOfWorkers += count; + this->numberOfWorkers += success; return workers; } From aa0249834217f70feb3a52078358e9b9bc15cdd8 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 14 Mar 2024 14:35:27 +0530 Subject: [PATCH 089/143] Enable scaling property --- conf/jasminegraph-server.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index c7f93f1db..3aaa04272 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -99,5 +99,5 @@ org.jasminegraph.k8s.image=jasminegraph # Auto-scaling #-------------------------------------------------------------------------------- org.jasminegraph.autoscale.scale_on_adgr=false -org.jasminegraph.autoscale.enabled=false +org.jasminegraph.autoscale.enabled=true org.jasminegraph.autopartition.enabled=false From d9577f20e9dbe56911a7f4cf0e8e9b48660657ca Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 14 Mar 2024 16:46:28 +0530 Subject: [PATCH 090/143] Mount aggregate folder of the master deployment --- README.md | 1 + k8s/master-deployment.yaml | 5 +++++ k8s/volumes.yaml | 32 ++++++++++++++++++++++++++++++++ start-k8s.sh | 2 ++ 4 files changed, 40 insertions(+) diff --git a/README.md b/README.md index bf9422509..dfe8a48f2 100755 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Run JasmineGraph in Kubernetes environment by providing appropriate parameters: --PERFORMANCE_DB_PATH "$(pwd)/performancedb" \ --DATA_PATH "$(pwd)/data" \ --LOG_PATH "$(pwd)/logs" \ + --AGGREGATE_PATH "$(pwd)/aggregate" \ --NO_OF_WORKERS 2 \ --ENABLE_NMON false \ --MAX_COUNT 4 \ diff --git a/k8s/master-deployment.yaml b/k8s/master-deployment.yaml index e39d37fc0..8804bc513 100644 --- a/k8s/master-deployment.yaml +++ b/k8s/master-deployment.yaml @@ -29,6 +29,9 @@ spec: - name: jasminegraph-logs persistentVolumeClaim: claimName: jasminegraph-logs-claim + - name: jasminegraph-aggregate + persistentVolumeClaim: + claimName: jasminegraph-aggregate-claim containers: - name: jasminegraph-master image: jasminegraph @@ -56,6 +59,8 @@ spec: name: jasminegraph-data - mountPath: /tmp/jasminegraph name: jasminegraph-logs + - mountPath: /var/tmp/jasminegraph-aggregate + name: jasminegraph-aggregate resources: requests: cpu: "1000m" diff --git a/k8s/volumes.yaml b/k8s/volumes.yaml index d2b652f7f..36e74e767 100644 --- a/k8s/volumes.yaml +++ b/k8s/volumes.yaml @@ -130,3 +130,35 @@ spec: resources: requests: storage: 1Gi + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: jasminegraph-aggregate + labels: + application: jasminegraph + type: local +spec: + storageClassName: manual + capacity: + storage: 1Gi + accessModes: + - ReadWriteMany + hostPath: + path: "${aggregate_path}" + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: jasminegraph-aggregate-claim + labels: + application: jasminegraph +spec: + storageClassName: manual + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi diff --git a/start-k8s.sh b/start-k8s.sh index 0bc1ac7ab..b601b0f36 100755 --- a/start-k8s.sh +++ b/start-k8s.sh @@ -16,6 +16,7 @@ META_DB_PATH=${META_DB_PATH} PERFORMANCE_DB_PATH=${PERFORMANCE_DB_PATH} DATA_PATH=${DATA_PATH} LOG_PATH=${LOG_PATH} +AGGREGATE_PATH=${AGGREGATE_PATH} NO_OF_WORKERS=${NO_OF_WORKERS} ENABLE_NMON=${ENABLE_NMON} MAX_COUNT=${MAX_COUNT} @@ -110,6 +111,7 @@ metadb_path="${META_DB_PATH}" \ performancedb_path="${PERFORMANCE_DB_PATH}" \ data_path="${DATA_PATH}" \ log_path="${LOG_PATH}" \ + aggregate_path="${AGGREGATE_PATH}" \ envsubst <"./k8s/volumes.yaml" | kubectl apply -f - no_of_workers="${NO_OF_WORKERS}" \ From 526379b430dfeb8894c76b557938e599cf032a9d Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 14 Mar 2024 17:16:08 +0530 Subject: [PATCH 091/143] Implement functions to create volumes per worker --- k8s/worker-volume-claim.yaml | 13 +++++++++ k8s/worker-volume.yaml | 15 ++++++++++ src/k8s/K8sInterface.cpp | 42 ++++++++++++++++++++++++++++ src/k8s/K8sInterface.h | 8 ++++++ tests/unit/k8s/K8sInterface_test.cpp | 24 ++++++++++++++++ 5 files changed, 102 insertions(+) create mode 100644 k8s/worker-volume-claim.yaml create mode 100644 k8s/worker-volume.yaml diff --git a/k8s/worker-volume-claim.yaml b/k8s/worker-volume-claim.yaml new file mode 100644 index 000000000..99edf8fd4 --- /dev/null +++ b/k8s/worker-volume-claim.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: "jasminegraph-worker-data-claim" + labels: + application: jasminegraph +spec: + storageClassName: manual + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/k8s/worker-volume.yaml b/k8s/worker-volume.yaml new file mode 100644 index 000000000..6629294dd --- /dev/null +++ b/k8s/worker-volume.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: "jasminegraph-worker-data" + labels: + application: jasminegraph + type: local +spec: + storageClassName: manual + capacity: + storage: 1Gi + accessModes: + - ReadWriteMany + hostPath: + path: "var/tmp/worker" \ No newline at end of file diff --git a/src/k8s/K8sInterface.cpp b/src/k8s/K8sInterface.cpp index f6de470e4..06378ca12 100644 --- a/src/k8s/K8sInterface.cpp +++ b/src/k8s/K8sInterface.cpp @@ -191,3 +191,45 @@ std::string K8sInterface::getJasmineGraphConfig(std::string key) { } return ""; } + +v1_persistent_volume_t *K8sInterface::createJasmineGraphPersistentVolume(int workerId) const { + std::string definition = Utils::getJsonStringFromYamlFile(ROOT_DIR "/k8s/worker-volume.yaml"); + definition = Utils::replaceAll(definition, "", std::to_string(workerId)); + cJSON *volumeTemplate = cJSON_Parse(definition.c_str()); + v1_persistent_volume_t *volume = v1_persistent_volume_parseFromJSON(volumeTemplate); + v1_persistent_volume_t *result = CoreV1API_createPersistentVolume(apiClient, volume, NULL, NULL, NULL, NULL); + return result; +} + +v1_persistent_volume_claim_t *K8sInterface::createJasmineGraphPersistentVolumeClaim(int workerId) const { + std::string definition = Utils::getJsonStringFromYamlFile(ROOT_DIR "/k8s/worker-volume-claim.yaml"); + definition = Utils::replaceAll(definition, "", std::to_string(workerId)); + cJSON *volumeClaimTemplate = cJSON_Parse(definition.c_str()); + v1_persistent_volume_claim_t *volumeClaim = v1_persistent_volume_claim_parseFromJSON(volumeClaimTemplate); + v1_persistent_volume_claim_t *result = CoreV1API_createNamespacedPersistentVolumeClaim(apiClient, + namespace_, + volumeClaim, + NULL, + NULL, + NULL, + NULL); + return result; +} + +v1_persistent_volume_t *K8sInterface::deleteJasmineGraphPersistentVolume(int workerId) const { + std::string volumeName = "jasminegraph-worker" + std::to_string(workerId) + "-data"; + v1_persistent_volume_t *result = CoreV1API_deletePersistentVolume(apiClient, strdup(volumeName.c_str()), NULL, NULL, + NULL, NULL, NULL, NULL); + return result; +} + +v1_persistent_volume_claim_t *K8sInterface::deleteJasmineGraphPersistentVolumeClaim(int workerId) const { + std::string volumeClaimName = "jasminegraph-worker" + std::to_string(workerId) + "-data-claim"; + v1_persistent_volume_claim_t *result = + CoreV1API_deleteNamespacedPersistentVolumeClaim(apiClient, + strdup(volumeClaimName.c_str()), + namespace_, + NULL, NULL, NULL, NULL, NULL, + NULL); + return result; +} diff --git a/src/k8s/K8sInterface.h b/src/k8s/K8sInterface.h index 944be2e80..af2dc58da 100644 --- a/src/k8s/K8sInterface.h +++ b/src/k8s/K8sInterface.h @@ -57,6 +57,14 @@ class K8sInterface { v1_node_list_t *getNodes(); std::string getJasmineGraphConfig(std::string key); + + v1_persistent_volume_t *createJasmineGraphPersistentVolume(int workerId) const; + + v1_persistent_volume_claim_t *createJasmineGraphPersistentVolumeClaim(int workerId) const; + + v1_persistent_volume_t *deleteJasmineGraphPersistentVolume(int workerId) const; + + v1_persistent_volume_claim_t *deleteJasmineGraphPersistentVolumeClaim(int workerId) const; }; #endif // JASMINEGRAPH_K8SINTERFACE_H diff --git a/tests/unit/k8s/K8sInterface_test.cpp b/tests/unit/k8s/K8sInterface_test.cpp index 8b877ce14..21827616a 100644 --- a/tests/unit/k8s/K8sInterface_test.cpp +++ b/tests/unit/k8s/K8sInterface_test.cpp @@ -93,3 +93,27 @@ TEST_F(K8sInterfaceTest, TestDeleteJasmineGraphMasterService) { ASSERT_EQ(interface->apiClient->response_code, HTTP_OK); ASSERT_EQ(interface->getMasterIp(), ""); } + +TEST_F(K8sInterfaceTest, createJasmineGraphPersistentVolume) { + auto result = interface->createJasmineGraphPersistentVolume(1); + ASSERT_EQ(interface->apiClient->response_code, HTTP_CREATED); + ASSERT_STREQ(result->metadata->name, "jasminegraph-worker1-data"); +} + +TEST_F(K8sInterfaceTest, createJasmineGraphPersistentVolumeClaim) { + auto result = interface->createJasmineGraphPersistentVolumeClaim(1); + ASSERT_EQ(interface->apiClient->response_code, HTTP_CREATED); + ASSERT_STREQ(result->metadata->name, "jasminegraph-worker1-data-claim"); +} + +TEST_F(K8sInterfaceTest, deleteJasmineGraphPersistentVolume) { + auto result = interface->deleteJasmineGraphPersistentVolume(1); + ASSERT_EQ(interface->apiClient->response_code, HTTP_OK); + ASSERT_STREQ(result->metadata->name, "jasminegraph-worker1-data"); +} + +TEST_F(K8sInterfaceTest, deleteJasmineGraphPersistentVolumeClaim) { + auto result = interface->deleteJasmineGraphPersistentVolumeClaim(1); + ASSERT_EQ(interface->apiClient->response_code, HTTP_OK); + ASSERT_STREQ(result->metadata->name, "jasminegraph-worker1-data-claim"); +} \ No newline at end of file From 8bbfb5e314d73893a03f45bbf36a2c5732903280 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 14 Mar 2024 17:24:29 +0530 Subject: [PATCH 092/143] Integrate worker volume creation --- k8s/worker-deployment.yaml | 9 ++++++++- src/k8s/K8sWorkerController.cpp | 28 ++++++++++++++++++++++++++++ test-k8s.sh | 1 + 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index d20e4a6e3..f1bfafe3a 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -20,6 +20,10 @@ spec: workerId: "" service: "jasminegraph" spec: + volumes: + - name: "jasminegraph-worker-data" + persistentVolumeClaim: + claimName: "jasminegraph-worker-data-claim" containers: - name: "jasminegraph-worker" image: "" @@ -51,4 +55,7 @@ spec: memory: "2Gi" limits: cpu: "2000m" - memory: "4Gi" \ No newline at end of file + memory: "4Gi" + volumeMounts: + - mountPath: "/var/tmp/" + name: "jasminegraph-worker-data" \ No newline at end of file diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 39966316d..3b05d5c00 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -77,6 +77,20 @@ K8sWorkerController *K8sWorkerController::getInstance(std::string masterIp, int } std::string K8sWorkerController::spawnWorker(int workerId) { + auto volume = this->interface->createJasmineGraphPersistentVolume(workerId); + if (volume != nullptr && volume->metadata != nullptr && volume->metadata->name != nullptr) { + controller_logger.info("Worker " + std::to_string(workerId) + " persistent volume created successfully"); + } else { + throw std::runtime_error("Worker " + std::to_string(workerId) + " persistent volume creation failed"); + } + + auto claim = this->interface->createJasmineGraphPersistentVolumeClaim(workerId); + if (claim != nullptr && claim->metadata != nullptr && claim->metadata->name != nullptr) { + controller_logger.info("Worker " + std::to_string(workerId) + " persistent volume claim created successfully"); + } else { + throw std::runtime_error("Worker " + std::to_string(workerId) + " persistent volume claim creation failed"); + } + v1_service_t *service = this->interface->createJasmineGraphWorkerService(workerId); if (service != nullptr && service->metadata != nullptr && service->metadata->name != nullptr) { controller_logger.info("Worker " + std::to_string(workerId) + " service created successfully"); @@ -180,6 +194,20 @@ void K8sWorkerController::deleteWorker(int workerId) { controller_logger.error("Worker " + std::to_string(workerId) + " service deletion failed"); } + auto volume = this->interface->deleteJasmineGraphPersistentVolume(workerId); + if (volume != nullptr && volume->metadata != nullptr && volume->metadata->name != nullptr) { + controller_logger.info("Worker " + std::to_string(workerId) + " persistent volume deleted successfully"); + } else { + controller_logger.error("Worker " + std::to_string(workerId) + " persistent volume deletion failed"); + } + + auto claim = this->interface->deleteJasmineGraphPersistentVolumeClaim(workerId); + if (claim != nullptr && claim->metadata != nullptr && claim->metadata->name != nullptr) { + controller_logger.info("Worker " + std::to_string(workerId) + " persistent volume claim deleted successfully"); + } else { + controller_logger.error("Worker " + std::to_string(workerId) + " persistent volume claim deletion failed"); + } + std::string deleteQuery = "DELETE FROM worker WHERE idworker = " + std::to_string(workerId); metadb.runUpdate(deleteQuery); std::remove(activeWorkerIds.begin(), activeWorkerIds.end(), workerId); diff --git a/test-k8s.sh b/test-k8s.sh index 8cbfd5fc3..b7405c3c2 100755 --- a/test-k8s.sh +++ b/test-k8s.sh @@ -53,6 +53,7 @@ build_and_run_on_k8s() { --PERFORMANCE_DB_PATH performancedb_path "${TEST_ROOT}/env/databases/performancedb" \ --DATA_PATH "${TEST_ROOT}/env/data" \ --LOG_PATH "${LOG_DIR}" \ + --AGGREGATE_PATH "${TEST_ROOT}/env/aggregate" \ --NO_OF_WORKERS 2 \ --ENABLE_NMON false } From 4fe0553850969dec0fa13e9f5cdd4c90311d88ec Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 14 Mar 2024 17:28:55 +0530 Subject: [PATCH 093/143] Make prometheus a NodePort service --- k8s/prometheus.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/prometheus.yaml b/k8s/prometheus.yaml index 99052bc5e..c653c8ec3 100644 --- a/k8s/prometheus.yaml +++ b/k8s/prometheus.yaml @@ -68,5 +68,7 @@ spec: ports: - port: 9090 targetPort: 9090 + nodePort: 30003 selector: app: prometheus + type: NodePort \ No newline at end of file From c3f73a6f7aa7bf8e9a412a12d1702504a1131704 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 14 Mar 2024 17:40:46 +0530 Subject: [PATCH 094/143] Bug fix --- README.md | 2 +- k8s/worker-volume.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dfe8a48f2..892cd8bc1 100755 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Run JasmineGraph in Kubernetes environment by providing appropriate parameters: --AGGREGATE_PATH "$(pwd)/aggregate" \ --NO_OF_WORKERS 2 \ --ENABLE_NMON false \ - --MAX_COUNT 4 \ + --MAX_COUNT 4 >Note: `NO_OF_WORKERS`, `MAX_COUNT`, and `ENABLE_NMON` are optional arguments, and it defaults to `2`, `4` and `false` respectively. diff --git a/k8s/worker-volume.yaml b/k8s/worker-volume.yaml index 6629294dd..a895d8599 100644 --- a/k8s/worker-volume.yaml +++ b/k8s/worker-volume.yaml @@ -12,4 +12,4 @@ spec: accessModes: - ReadWriteMany hostPath: - path: "var/tmp/worker" \ No newline at end of file + path: "/var/tmp/worker" \ No newline at end of file From b44271b8074696373850e2288dff66d6f1b691b1 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Thu, 14 Mar 2024 18:26:52 +0530 Subject: [PATCH 095/143] Use partition combinations instead of worker combinations in triangle count executor --- .../executor/impl/TriangleCountExecutor.cpp | 202 ++++++++---------- .../executor/impl/TriangleCountExecutor.h | 14 +- 2 files changed, 95 insertions(+), 121 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 12413b6a8..ee189bc50 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -19,8 +19,6 @@ limitations under the License. using namespace std::chrono; Logger triangleCount_logger; -std::vector> TriangleCountExecutor::fileCombinations; -std::map TriangleCountExecutor::combinationWorkerMap; std::unordered_map>> TriangleCountExecutor::triangleTree; bool isStatCollect = false; @@ -28,13 +26,12 @@ std::mutex fileCombinationMutex; std::mutex processStatusMutex; std::mutex responseVectorMutex; -static std::vector> getWorkerCombination(std::vector &workerIdVector); static string isFileAccessibleToWorker(std::string graphId, std::string partitionId, std::string aggregatorHostName, std::string aggregatorPort, std::string masterIP, std::string fileType, std::string fileName); static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, - int threadPriority, std::vector &workerIdVector, - std::map> &partitionMap); + int threadPriority, + const std::map> &partitionMap); TriangleCountExecutor::TriangleCountExecutor() {} @@ -530,15 +527,16 @@ void TriangleCountExecutor::execute() { isCompositeAggregation = true; } + std::vector> fileCombinations; if (isCompositeAggregation) { std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); std::vector graphFiles = Utils::getListOfFilesInDirectory(aggregatorFilePath); - std::vector::iterator graphFilesIterator; std::string compositeFileNameFormat = graphId + "_compositecentralstore_"; - for (graphFilesIterator = graphFiles.begin(); graphFilesIterator != graphFiles.end(); ++graphFilesIterator) { + for (auto graphFilesIterator = graphFiles.begin(); graphFilesIterator != graphFiles.end(); + ++graphFilesIterator) { std::string graphFileName = *graphFilesIterator; if ((graphFileName.find(compositeFileNameFormat) == 0) && @@ -549,6 +547,7 @@ void TriangleCountExecutor::execute() { fileCombinations = AbstractExecutor::getCombinations(compositeCentralStoreFiles); } + std::map combinationWorkerMap; int partitionCount = 0; for (int i = 0; i < workerListSize; i++) { Utils::worker currentWorker = workerList.at(i); @@ -564,9 +563,10 @@ void TriangleCountExecutor::execute() { partitionCount++; partitionId = *partitionIterator; - intermRes.push_back(std::async( - std::launch::async, TriangleCountExecutor::getTriangleCount, atoi(graphId.c_str()), host, workerPort, - workerDataPort, atoi(partitionId.c_str()), masterIP, uniqueId, isCompositeAggregation, threadPriority)); + intermRes.push_back(std::async(std::launch::async, TriangleCountExecutor::getTriangleCount, + atoi(graphId.c_str()), host, workerPort, workerDataPort, + atoi(partitionId.c_str()), masterIP, uniqueId, isCompositeAggregation, + threadPriority, fileCombinations, &combinationWorkerMap)); } } @@ -602,13 +602,8 @@ void TriangleCountExecutor::execute() { } if (!isCompositeAggregation) { - std::vector workerIdVector; - for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { - workerIdVector.push_back(it->first); - } - long aggregatedTriangleCount = - aggregateCentralStoreTriangles(sqlite, graphId, masterIP, threadPriority, workerIdVector, partitionMap); + aggregateCentralStoreTriangles(sqlite, graphId, masterIP, threadPriority, partitionMap); result += aggregatedTriangleCount; workerResponded = true; triangleCount_logger.log( @@ -650,14 +645,14 @@ void TriangleCountExecutor::execute() { } } processStatusMutex.unlock(); - triangleTree.clear(); combinationWorkerMap.clear(); } long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int port, int dataPort, int partitionId, std::string masterIP, int uniqueId, bool isCompositeAggregation, - int threadPriority) { + int threadPriority, std::vector> fileCombinations, + std::map *combinationWorkerMap_p) { int sockfd; char data[301]; bool loop = false; @@ -694,7 +689,6 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int return 0; } - bzero(data, 301); int result_wr = write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); @@ -703,11 +697,7 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int } triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); + string response = Utils::read_str_trim_wrapper(sockfd, data, 300); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); @@ -716,12 +706,9 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + masterIP, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_trim_wrapper(sockfd, data, 300); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -733,13 +720,9 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::TRIANGLES, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, 300); if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); result_wr = write(sockfd, std::to_string(graphId).c_str(), std::to_string(graphId).size()); @@ -747,13 +730,9 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : Graph ID " + std::to_string(graphId), "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, 300); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -766,10 +745,7 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int triangleCount_logger.log("Sent : Partition ID " + std::to_string(partitionId), "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, 300); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -779,14 +755,10 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : Thread Priority " + std::to_string(threadPriority), "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_trim_wrapper(sockfd, data, 300); triangleCount_logger.log("Got response : |" + response + "|", "info"); - response = Utils::trim_copy(response); triangleCount = atol(response.c_str()); } @@ -795,26 +767,23 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int static std::vector>::iterator combinationsIterator; for (int combinationIndex = 0; combinationIndex < fileCombinations.size(); ++combinationIndex) { - std::vector fileList = fileCombinations.at(combinationIndex); - std::vector::iterator fileListIterator; - std::vector::iterator listIterator; + const std::vector &fileList = fileCombinations.at(combinationIndex); std::set partitionIdSet; std::set partitionSet; std::map tempWeightMap; - std::set::iterator partitionSetIterator; std::set transferRequireFiles; std::string combinationKey = ""; std::string availableFiles = ""; std::string transferredFiles = ""; bool isAggregateValid = false; - for (listIterator = fileList.begin(); listIterator != fileList.end(); ++listIterator) { + for (auto listIterator = fileList.begin(); listIterator != fileList.end(); ++listIterator) { std::string fileName = *listIterator; size_t lastIndex = fileName.find_last_of("."); string rawFileName = fileName.substr(0, lastIndex); - std::vector fileNameParts = Utils::split(rawFileName, '_'); + const std::vector &fileNameParts = Utils::split(rawFileName, '_'); /*Partition numbers are extracted from the file name. The starting index of partition number * is 2. Therefore the loop starts with 2*/ @@ -827,12 +796,11 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int continue; } - if (proceedOrNot(partitionSet, partitionId)) { - } else { + if (!proceedOrNot(partitionSet, partitionId)) { continue; } - for (fileListIterator = fileList.begin(); fileListIterator != fileList.end(); ++fileListIterator) { + for (auto fileListIterator = fileList.begin(); fileListIterator != fileList.end(); ++fileListIterator) { std::string fileName = *fileListIterator; bool isTransferRequired = true; @@ -863,6 +831,7 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int std::string adjustedTransferredFile = transferredFiles.substr(0, transferredFiles.size() - 1); fileCombinationMutex.lock(); + std::map &combinationWorkerMap = *combinationWorkerMap_p; if (combinationWorkerMap.find(combinationKey) == combinationWorkerMap.end()) { if (partitionIdSet.find(std::to_string(partitionId)) != partitionIdSet.end()) { combinationWorkerMap[combinationKey] = std::to_string(partitionId); @@ -917,9 +886,8 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int bool TriangleCountExecutor::proceedOrNot(std::set partitionSet, int partitionId) { const std::lock_guard lock(aggregateWeightMutex); - std::set::iterator partitionSetIterator; std::map tempWeightMap; - for (partitionSetIterator = partitionSet.begin(); partitionSetIterator != partitionSet.end(); + for (auto partitionSetIterator = partitionSet.begin(); partitionSetIterator != partitionSet.end(); ++partitionSetIterator) { std::string partitionIdString = *partitionSetIterator; int currentPartitionId = atoi(partitionIdString.c_str()); @@ -999,75 +967,91 @@ int TriangleCountExecutor::updateTriangleTreeAndGetTriangleCount(const std::vect } static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, - int threadPriority, std::vector &workerIdVector, - std::map> &partitionMap) { - const std::vector> &workerCombinations = getWorkerCombination(workerIdVector); + int threadPriority, + const std::map> &partitionMap) { + vector partitionsVector; + std::map partitionWorkerMap; // partition_id => worker_id + for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { + const auto &parts = it->second; + string worker = it->first; + for (auto partsIt = parts.begin(); partsIt != parts.end(); partsIt++) { + string partition = *partsIt; + partitionWorkerMap[partition] = worker; + partitionsVector.push_back(partition); + } + } + + const std::vector> &partitionCombinations = AbstractExecutor::getCombinations(partitionsVector); std::map workerWeightMap; std::vector> triangleCountResponse; std::string result = ""; long aggregatedTriangleCount = 0; - for (auto workerCombinationsIterator = workerCombinations.begin(); - workerCombinationsIterator != workerCombinations.end(); ++workerCombinationsIterator) { - const std::vector &workerCombination = *workerCombinationsIterator; + const std::vector>> &workerDataResult = + sqlite->runSelect("SELECT idworker,ip,server_port,server_data_port FROM worker;"); + map> workerDataMap; // worker_id => [ip,port,data_port] + for (auto it = workerDataResult.begin(); it != workerDataResult.end(); it++) { + const auto &ipPortDport = *it; + string id = ipPortDport[0].second; + string ip = ipPortDport[1].second; + string port = ipPortDport[2].second; + string dport = ipPortDport[3].second; + workerDataMap[id] = {ip, port, dport}; + } + + for (auto partitonCombinationsIterator = partitionCombinations.begin(); + partitonCombinationsIterator != partitionCombinations.end(); partitonCombinationsIterator++) { + const std::vector &partitionCombination = *partitonCombinationsIterator; std::vector> remoteGraphCopyResponse; int minimumWeight = 0; std::string minWeightWorker; + std::string minWeightWorkerPartition; - for (auto workerCombinationIterator = workerCombination.begin(); - workerCombinationIterator != workerCombination.end(); ++workerCombinationIterator) { - std::string workerId = *workerCombinationIterator; - + for (auto partCombinationIterator = partitionCombination.begin(); + partCombinationIterator != partitionCombination.end(); partCombinationIterator++) { + string part = *partCombinationIterator; + string workerId = partitionWorkerMap[part]; auto workerWeightMapIterator = workerWeightMap.find(workerId); - if (workerWeightMapIterator != workerWeightMap.end()) { int weight = workerWeightMapIterator->second; if (minimumWeight == 0 || minimumWeight > weight) { minimumWeight = weight + 1; minWeightWorker = workerId; + minWeightWorkerPartition = part; } } else { minimumWeight = 1; minWeightWorker = workerId; + minWeightWorkerPartition = part; } } + workerWeightMap[minWeightWorker] = minimumWeight; - string aggregatorSqlStatement = - "SELECT ip,server_port,server_data_port FROM worker " - "WHERE idworker=" + - minWeightWorker + ";"; - - const std::vector>> &result = sqlite->runSelect(aggregatorSqlStatement); - - const vector> &aggregatorData = result.at(0); + const auto &workerData = workerDataMap[minWeightWorker]; + std::string aggregatorIp = workerData[0]; + std::string aggregatorPort = workerData[1]; + std::string aggregatorDataPort = workerData[2]; - std::string aggregatorIp = aggregatorData.at(0).second; - std::string aggregatorPort = aggregatorData.at(1).second; - std::string aggregatorDataPort = aggregatorData.at(2).second; - std::string aggregatorPartitionId = partitionMap[minWeightWorker][0]; // why only 1 partition from the worker? + std::string aggregatorPartitionId = minWeightWorkerPartition; std::string partitionIdList = ""; - for (auto aggregatorCopyCombinationIterator = workerCombination.begin(); - aggregatorCopyCombinationIterator != workerCombination.end(); ++aggregatorCopyCombinationIterator) { - std::string workerId = *aggregatorCopyCombinationIterator; + for (auto partitionCombinationIterator = partitionCombination.begin(); + partitionCombinationIterator != partitionCombination.end(); ++partitionCombinationIterator) { + string part = *partitionCombinationIterator; + string workerId = partitionWorkerMap[part]; if (workerId != minWeightWorker) { - const vector> &workerData = result.at(0); - - std::string partitionId = partitionMap[workerId][0]; // why only 1 partition from the worker? - - partitionIdList += partitionId + ","; + partitionIdList += part + ","; std::string centralStoreAvailable = isFileAccessibleToWorker( - graphId, partitionId, aggregatorIp, aggregatorPort, masterIP, + graphId, part, aggregatorIp, aggregatorPort, masterIP, JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_AGGREGATE, std::string()); if (centralStoreAvailable.compare("false") == 0) { - remoteGraphCopyResponse.push_back( - std::async(std::launch::async, TriangleCountExecutor::copyCentralStoreToAggregator, - aggregatorIp, aggregatorPort, aggregatorDataPort, atoi(graphId.c_str()), - atoi(partitionId.c_str()), masterIP)); + remoteGraphCopyResponse.push_back(std::async( + std::launch::async, TriangleCountExecutor::copyCentralStoreToAggregator, aggregatorIp, + aggregatorPort, aggregatorDataPort, atoi(graphId.c_str()), atoi(part.c_str()), masterIP)); } } } @@ -1077,7 +1061,6 @@ static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::strin } std::string adjustedPartitionIdList = partitionIdList.substr(0, partitionIdList.size() - 1); - workerWeightMap[minWeightWorker] = minimumWeight; triangleCountResponse.push_back(std::async( std::launch::async, TriangleCountExecutor::countCentralStoreTriangles, aggregatorPort, aggregatorIp, @@ -1088,11 +1071,9 @@ static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::strin result = result + ":" + futureCall.get(); } - std::vector triangles = Utils::split(result, ':'); - std::vector::iterator triangleIterator; + const std::vector &triangles = Utils::split(result, ':'); std::set uniqueTriangleSet; - - for (triangleIterator = triangles.begin(); triangleIterator != triangles.end(); ++triangleIterator) { + for (auto triangleIterator = triangles.begin(); triangleIterator != triangles.end(); ++triangleIterator) { std::string triangle = *triangleIterator; if (!triangle.empty() && triangle != "NILL") { @@ -1101,6 +1082,7 @@ static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::strin } aggregatedTriangleCount = uniqueTriangleSet.size(); + uniqueTriangleSet.clear(); return aggregatedTriangleCount; } @@ -1609,10 +1591,6 @@ std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( ; } -static std::vector> getWorkerCombination(std::vector &workerIdVector) { - return AbstractExecutor::getCombinations(workerIdVector); -} - std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggregatorHostName, std::string aggregatorPort, std::string aggregatorDataPort, int graphId, @@ -1936,12 +1914,10 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP triangleCount_logger.log("Sent : Thread Priority " + std::to_string(threadPriority), "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, 300); string status = response.substr(response.size() - 5); - std::string result = response.substr(0, response.size() - 5); + std::basic_ostringstream resultStream; + resultStream << response.substr(0, response.size() - 5); while (status == "/SEND") { result_wr = write(sockfd, status.c_str(), status.size()); @@ -1949,15 +1925,11 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, 300); status = response.substr(response.size() - 5); - std::string triangleResponse = response.substr(0, response.size() - 5); - result = result + triangleResponse; + resultStream << response.substr(0, response.size() - 5); } - response = result; + response = resultStream.str(); } } else { diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.h b/src/frontend/core/executor/impl/TriangleCountExecutor.h index f805eeaad..6a11d010c 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.h +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.h @@ -38,15 +38,19 @@ class TriangleCountExecutor : public AbstractExecutor { int getUid(); static long getTriangleCount(int graphId, std::string host, int port, int dataPort, int partitionId, - std::string masterIP, int uniqueId, bool isCompositeAggregation, int threadPriority); + std::string masterIP, int uniqueId, bool isCompositeAggregation, int threadPriority, + std::vector> fileCombinations, + std::map *combinationWorkerMap_p); static std::string copyCompositeCentralStoreToAggregator(std::string aggregatorHostName, std::string aggregatorPort, std::string aggregatorDataPort, std::string fileName, std::string masterIP); - static std::vector countCompositeCentralStoreTriangles(std::string aggregatorHostName, std::string aggregatorPort, - std::string compositeCentralStoreFileList, std::string masterIP, - std::string availableFileList, int threadPriority); + static std::vector countCompositeCentralStoreTriangles(std::string aggregatorHostName, + std::string aggregatorPort, + std::string compositeCentralStoreFileList, + std::string masterIP, std::string availableFileList, + int threadPriority); static std::string copyCentralStoreToAggregator(std::string aggregatorHostName, std::string aggregatorPort, std::string aggregatorDataPort, int graphId, int partitionId, @@ -60,8 +64,6 @@ class TriangleCountExecutor : public AbstractExecutor { static void updateMap(int partitionId); - static std::vector> fileCombinations; - static std::map combinationWorkerMap; static std::unordered_map>> triangleTree; private: From 6ddb7df6dec8b5ab3972c4f9818df85992235bf5 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Thu, 14 Mar 2024 19:16:50 +0530 Subject: [PATCH 096/143] Move global triangleTree to per-job level --- src/frontend/JasmineGraphFrontEnd.cpp | 2 - .../executor/impl/TriangleCountExecutor.cpp | 55 ++++++++++--------- .../executor/impl/TriangleCountExecutor.h | 14 ++--- src/util/Conts.h | 2 - 4 files changed, 36 insertions(+), 37 deletions(-) diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index f357bc3d0..32bf93851 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -59,8 +59,6 @@ static int currentFESession; static bool canCalibrate = true; Logger frontend_logger; std::set processData; -std::mutex aggregateWeightMutex; -std::mutex triangleTreeMutex; std::string stream_topic_name; static std::string getPartitionCount(std::string path); diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index ee189bc50..229b2fb97 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -19,12 +19,12 @@ limitations under the License. using namespace std::chrono; Logger triangleCount_logger; -std::unordered_map>> TriangleCountExecutor::triangleTree; bool isStatCollect = false; std::mutex fileCombinationMutex; std::mutex processStatusMutex; std::mutex responseVectorMutex; +static std::mutex aggregateWeightMutex; static string isFileAccessibleToWorker(std::string graphId, std::string partitionId, std::string aggregatorHostName, std::string aggregatorPort, std::string masterIP, std::string fileType, @@ -32,6 +32,10 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, int threadPriority, const std::map> &partitionMap); +static int updateTriangleTreeAndGetTriangleCount( + const std::vector &triangles, + std::unordered_map>> *triangleTree_p, + std::mutex *triangleTreeMutex_p); TriangleCountExecutor::TriangleCountExecutor() {} @@ -431,7 +435,6 @@ void TriangleCountExecutor::execute() { // Below code is used to update the process details processStatusMutex.lock(); - std::set::iterator processIterator; bool processInfoExists = false; std::chrono::milliseconds startTime = duration_cast(system_clock::now().time_since_epoch()); @@ -548,6 +551,8 @@ void TriangleCountExecutor::execute() { } std::map combinationWorkerMap; + std::unordered_map>> triangleTree; + std::mutex triangleTreeMutex; int partitionCount = 0; for (int i = 0; i < workerListSize; i++) { Utils::worker currentWorker = workerList.at(i); @@ -563,10 +568,10 @@ void TriangleCountExecutor::execute() { partitionCount++; partitionId = *partitionIterator; - intermRes.push_back(std::async(std::launch::async, TriangleCountExecutor::getTriangleCount, - atoi(graphId.c_str()), host, workerPort, workerDataPort, - atoi(partitionId.c_str()), masterIP, uniqueId, isCompositeAggregation, - threadPriority, fileCombinations, &combinationWorkerMap)); + intermRes.push_back(std::async( + std::launch::async, TriangleCountExecutor::getTriangleCount, atoi(graphId.c_str()), host, workerPort, + workerDataPort, atoi(partitionId.c_str()), masterIP, uniqueId, isCompositeAggregation, threadPriority, + fileCombinations, &combinationWorkerMap, &triangleTree, &triangleTreeMutex)); } } @@ -600,6 +605,8 @@ void TriangleCountExecutor::execute() { for (auto &&futureCall : intermRes) { result += futureCall.get(); } + triangleTree.clear(); + combinationWorkerMap.clear(); if (!isCompositeAggregation) { long aggregatedTriangleCount = @@ -634,8 +641,7 @@ void TriangleCountExecutor::execute() { } processStatusMutex.lock(); - std::set::iterator processCompleteIterator; - for (processCompleteIterator = processData.begin(); processCompleteIterator != processData.end(); + for (auto processCompleteIterator = processData.begin(); processCompleteIterator != processData.end(); ++processCompleteIterator) { ProcessInfo processInformation = *processCompleteIterator; @@ -645,14 +651,14 @@ void TriangleCountExecutor::execute() { } } processStatusMutex.unlock(); - triangleTree.clear(); - combinationWorkerMap.clear(); } -long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int port, int dataPort, int partitionId, - std::string masterIP, int uniqueId, bool isCompositeAggregation, - int threadPriority, std::vector> fileCombinations, - std::map *combinationWorkerMap_p) { +long TriangleCountExecutor::getTriangleCount( + int graphId, std::string host, int port, int dataPort, int partitionId, std::string masterIP, int uniqueId, + bool isCompositeAggregation, int threadPriority, std::vector> fileCombinations, + std::map *combinationWorkerMap_p, + std::unordered_map>> *triangleTree_p, + std::mutex *triangleTreeMutex_p) { int sockfd; char data[301]; bool loop = false; @@ -764,8 +770,6 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int if (isCompositeAggregation) { triangleCount_logger.log("###COMPOSITE### Started Composite aggregation ", "info"); - static std::vector>::iterator combinationsIterator; - for (int combinationIndex = 0; combinationIndex < fileCombinations.size(); ++combinationIndex) { const std::vector &fileList = fileCombinations.at(combinationIndex); std::set partitionIdSet; @@ -841,9 +845,7 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int fileCombinationMutex.unlock(); if (isAggregateValid) { - std::set::iterator transferRequireFileIterator; - - for (transferRequireFileIterator = transferRequireFiles.begin(); + for (auto transferRequireFileIterator = transferRequireFiles.begin(); transferRequireFileIterator != transferRequireFiles.end(); ++transferRequireFileIterator) { std::string transferFileName = *transferRequireFileIterator; std::string fileAccessible = isFileAccessibleToWorker( @@ -862,7 +864,8 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int countCompositeCentralStoreTriangles(host, std::to_string(port), adjustedTransferredFile, masterIP, adjustedAvailableFiles, threadPriority); if (triangles.size() > 0) { - triangleCount += updateTriangleTreeAndGetTriangleCount(triangles); + triangleCount += + updateTriangleTreeAndGetTriangleCount(triangles, triangleTree_p, triangleTreeMutex_p); } } updateMap(partitionId); @@ -897,9 +900,7 @@ bool TriangleCountExecutor::proceedOrNot(std::set partitionSet, int part int currentWorkerWeight = tempWeightMap[partitionId]; pair entryWithMinValue = make_pair(partitionId, currentWorkerWeight); - map::iterator currentEntry; - - for (currentEntry = aggregateWeightMap.begin(); currentEntry != aggregateWeightMap.end(); ++currentEntry) { + for (auto currentEntry = aggregateWeightMap.begin(); currentEntry != aggregateWeightMap.end(); ++currentEntry) { if (entryWithMinValue.second > currentEntry->second) { entryWithMinValue = make_pair(currentEntry->first, currentEntry->second); } @@ -916,7 +917,6 @@ bool TriangleCountExecutor::proceedOrNot(std::set partitionSet, int part result = true; } - aggregateWeightMutex.unlock(); return result; } @@ -931,9 +931,14 @@ void TriangleCountExecutor::updateMap(int partitionId) { "info"); } -int TriangleCountExecutor::updateTriangleTreeAndGetTriangleCount(const std::vector &triangles) { +static int updateTriangleTreeAndGetTriangleCount( + const std::vector &triangles, + std::unordered_map>> *triangleTree_p, + std::mutex *triangleTreeMutex_p) { + std::mutex &triangleTreeMutex = *triangleTreeMutex_p; const std::lock_guard lock1(triangleTreeMutex); int aggregateCount = 0; + auto &triangleTree = *triangleTree_p; triangleCount_logger.log("###COMPOSITE### Triangle Tree locked ", "debug"); diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.h b/src/frontend/core/executor/impl/TriangleCountExecutor.h index 6a11d010c..717348e37 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.h +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.h @@ -37,10 +37,12 @@ class TriangleCountExecutor : public AbstractExecutor { int getUid(); - static long getTriangleCount(int graphId, std::string host, int port, int dataPort, int partitionId, - std::string masterIP, int uniqueId, bool isCompositeAggregation, int threadPriority, - std::vector> fileCombinations, - std::map *combinationWorkerMap_p); + static long getTriangleCount( + int graphId, std::string host, int port, int dataPort, int partitionId, std::string masterIP, int uniqueId, + bool isCompositeAggregation, int threadPriority, std::vector> fileCombinations, + std::map *combinationWorkerMap_p, + std::unordered_map>> *triangleTree_p, + std::mutex *triangleTreeMutex_p); static std::string copyCompositeCentralStoreToAggregator(std::string aggregatorHostName, std::string aggregatorPort, std::string aggregatorDataPort, std::string fileName, @@ -64,13 +66,9 @@ class TriangleCountExecutor : public AbstractExecutor { static void updateMap(int partitionId); - static std::unordered_map>> triangleTree; - private: SQLiteDBInterface *sqlite; PerformanceSQLiteDBInterface *perfDB; - - static int updateTriangleTreeAndGetTriangleCount(const std::vector &triangles); }; #endif // JASMINEGRAPH_TRIANGLECOUNTEXECUTOR_H diff --git a/src/util/Conts.h b/src/util/Conts.h index d6cd989b4..d19c02533 100644 --- a/src/util/Conts.h +++ b/src/util/Conts.h @@ -32,8 +32,6 @@ extern bool isStatCollect; extern bool isCalibrating; extern std::vector loadAverageVector; extern std::map aggregateWeightMap; -extern std::mutex aggregateWeightMutex; -extern std::mutex triangleTreeMutex; struct ProcessInfo { int id; From a0a8e541d2dcb9ec17c186f69319de6c10862778 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Thu, 14 Mar 2024 19:22:39 +0530 Subject: [PATCH 097/143] Prevent duplicate rows from SELECT from metadb in triangle count executor --- src/frontend/core/executor/impl/TriangleCountExecutor.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 229b2fb97..7346c45bf 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -282,7 +282,7 @@ static void filter_partitions(std::map> &partitionMa string graphId) { map workers; // id => "ip:port" const std::vector>> &results = - sqlite->runSelect("SELECT idworker,ip,server_port FROM worker;"); + sqlite->runSelect("SELECT DISTINCT idworker,ip,server_port FROM worker;"); for (int i = 0; i < results.size(); i++) { string workerId = results[i][0].second; string ip = results[i][1].second; @@ -370,7 +370,7 @@ static void filter_partitions(std::map> &partitionMa if (!transfer.empty()) { const std::vector>> &workerData = - sqlite->runSelect("SELECT ip,server_port,server_data_port FROM worker;"); + sqlite->runSelect("SELECT DISTINCT ip,server_port,server_data_port FROM worker;"); map dataPortMap; // "ip:port" => data_port for (int i = 0; i < workerData.size(); i++) { string ip = workerData[i][0].second; @@ -471,7 +471,7 @@ void TriangleCountExecutor::execute() { auto begin = chrono::high_resolution_clock::now(); string sqlStatement = - "SELECT worker_idworker,partition_idpartition " + "SELECT DISTINCT worker_idworker,partition_idpartition " "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + graphId + ";"; @@ -993,7 +993,7 @@ static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::strin long aggregatedTriangleCount = 0; const std::vector>> &workerDataResult = - sqlite->runSelect("SELECT idworker,ip,server_port,server_data_port FROM worker;"); + sqlite->runSelect("SELECT DISTINCT idworker,ip,server_port,server_data_port FROM worker;"); map> workerDataMap; // worker_id => [ip,port,data_port] for (auto it = workerDataResult.begin(); it != workerDataResult.end(); it++) { const auto &ipPortDport = *it; From eb22233885b75eb8401c8d4e1762b4a0eda7578c Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Thu, 14 Mar 2024 22:02:49 +0530 Subject: [PATCH 098/143] Fix partition combination bug --- .../executor/impl/TriangleCountExecutor.cpp | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 7346c45bf..8460ce67e 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -514,7 +514,7 @@ void TriangleCountExecutor::execute() { filter_partitions(partitionMap, sqlite, graphId); } - cout << "initial partitionMap = {" << endl; + cout << "final partitionMap = {" << endl; for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { cout << " " << it->first << ": ["; auto &pts = it->second; @@ -1046,9 +1046,10 @@ static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::strin string part = *partitionCombinationIterator; string workerId = partitionWorkerMap[part]; - if (workerId != minWeightWorker) { + if (part != minWeightWorkerPartition) { partitionIdList += part + ","; - + } + if (workerId != minWeightWorker) { std::string centralStoreAvailable = isFileAccessibleToWorker( graphId, part, aggregatorIp, aggregatorPort, masterIP, JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_AGGREGATE, std::string()); @@ -1371,10 +1372,8 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::FILE_RECV_CHK, "info"); triangleCount_logger.log("Checking if file is received", "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_trim_wrapper(sockfd, data, 300); if (response.compare(JasmineGraphInstanceProtocol::FILE_RECV_WAIT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::FILE_RECV_WAIT, "info"); triangleCount_logger.log("Checking file status : " + to_string(count), "info"); @@ -1385,6 +1384,8 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::FILE_ACK, "info"); triangleCount_logger.log("File transfer completed for file : " + aggregateStoreFile, "info"); break; + } else { + triangleCount_logger.error("Invalid response " + response); } } @@ -1635,7 +1636,6 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr return ""; } - bzero(data, 301); int result_wr = write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); @@ -1732,10 +1732,8 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::FILE_RECV_CHK, "info"); triangleCount_logger.log("Checking if file is received", "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_trim_wrapper(sockfd, data, 300); if (response.compare(JasmineGraphInstanceProtocol::FILE_RECV_WAIT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::FILE_RECV_WAIT, "info"); triangleCount_logger.log("Checking file status : " + to_string(count), "info"); @@ -1746,6 +1744,8 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::FILE_ACK, "info"); triangleCount_logger.log("File transfer completed for file : " + centralStoreFile, "info"); break; + } else { + triangleCount_logger.error("Invalid response " + response); } } @@ -1901,7 +1901,7 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : Partition ID List : " + partitionId, "info"); + triangleCount_logger.log("Sent : Partition ID List : " + partitionIdList, "info"); bzero(data, 301); read(sockfd, data, 300); From 3256e28451744cd363aa7143b64b70048cf8b82a Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 14 Mar 2024 23:05:49 +0530 Subject: [PATCH 099/143] Reduce performance metric sending interval --- conf/jasminegraph-server.properties | 2 +- src/k8s/K8sWorkerController.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 3aaa04272..29d7c3ee5 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -72,7 +72,7 @@ org.jasminegraph.fl.organization.file=/home/ubuntu/software/jasminegraph/conf/ho #Scheduler Specific Parameters org.jasminegraph.scheduler.enabled=true #PerformanceCollector Scheduler Timing. Run once every 120 seconds -org.jasminegraph.scheduler.performancecollector.timing=120 +org.jasminegraph.scheduler.performancecollector.timing=30 #-------------------------------------------------------------------------------- #PerformanceCollector diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 3b05d5c00..c9b8f5694 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -144,6 +144,7 @@ std::string K8sWorkerController::spawnWorker(int workerId) { if (waiting >= TIME_OUT) { controller_logger.error("Error in spawning new worker"); deleteWorker(workerId); + close(sockfd); return ""; } From 84525e35685946644aaf7db0fc1dcb71b546bb4e Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 15 Mar 2024 01:07:04 +0530 Subject: [PATCH 100/143] transfer partitions in a thread --- src/backend/JasmineGraphBackend.cpp | 1 + .../executor/impl/TriangleCountExecutor.cpp | 25 ++++++++----------- src/server/JasmineGraphInstanceService.cpp | 1 + src/util/Utils.cpp | 21 +++++++++------- src/util/Utils.h | 10 +++----- 5 files changed, 28 insertions(+), 30 deletions(-) diff --git a/src/backend/JasmineGraphBackend.cpp b/src/backend/JasmineGraphBackend.cpp index b7816443c..023fde198 100644 --- a/src/backend/JasmineGraphBackend.cpp +++ b/src/backend/JasmineGraphBackend.cpp @@ -86,6 +86,7 @@ void *backendservicesesion(void *dummyPt) { } else { backend_logger.log("Message format not recognized", "error"); + sleep(1); } } backend_logger.log("Closing thread " + to_string(pthread_self()) + " and connection", "info"); diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 8460ce67e..1d92411e3 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -383,28 +383,25 @@ static void filter_partitions(std::map> &partitionMa for (auto it = workers.begin(); it != workers.end(); it++) { workers_r[it->second] = it->first; } + thread transferThreads[transfer.size()]; + int threadCnt = 0; for (auto it = transfer.begin(); it != transfer.end(); it++) { auto partition = it->first; - auto w_from = workers[it->second.first]; - auto w_to = workers[it->second.second]; + auto from_worker = it->second.first; + auto to_worker = it->second.second; + auto w_from = workers[from_worker]; + auto w_to = workers[to_worker]; const auto &ip_port_from = Utils::split(w_from, ':'); auto ip_from = ip_port_from[0]; auto port_from = stoi(ip_port_from[1]); - auto dport_from = stoi(dataPortMap[w_from]); const auto &ip_port_to = Utils::split(w_to, ':'); auto ip_to = ip_port_to[0]; - auto port_to = stoi(ip_port_to[1]); auto dport_to = stoi(dataPortMap[w_to]); - cout << "Sending partition " << partition << " from " << w_from << ":" << dport_from << " to " << w_to - << ":" << dport_to << endl; - // TODO(thevindu-w): run transferPartition in a separate thread - Utils::transferPartition(ip_from, port_from, dport_from, ip_to, port_to, dport_to, stoi(graphId), - partition); - cout << "Send completed"; - sqlite->runInsert( - "INSERT INTO worker_has_partition " - "(partition_idpartition, partition_graph_idgraph, worker_idworker) VALUES ('" + - to_string(partition) + "', '" + graphId + "', '" + workers_r[w_to] + "')"); + transferThreads[threadCnt++] = std::thread(&Utils::transferPartition, ip_from, port_from, ip_to, + dport_to, graphId, to_string(partition), to_worker, sqlite); + } + for (int i = 0; i < threadCnt; i++) { + transferThreads[i].join(); } } } diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 620512e34..e79408768 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -163,6 +163,7 @@ void *instanceservicesession(void *dummyPt) { while (!loop_exit) { string line = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, true); if (line.empty()) { + sleep(1); continue; } line = Utils::trim_copy(line); diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 69b168fae..c9b5136d1 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -1049,14 +1049,13 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f /* * Function to transfer a partition from one worker to another * Caller should ensure that the partition exists in the source worker - * and the destination worker is ready to accept the partition - * Also, the caller should update the worker_has_partition table. + * and the destination worker is ready to accept the partition. * */ -bool Utils::transferPartition(std::string sourceWorker, int sourceWorkerPort, int sourceWorkerDataPort, - std::string destinationWorker, int destinationWorkerPort, int destinationWorkerDataPort, - int graphID, int partitionID) { - util_logger.info("### Transferring partition " + to_string(partitionID) + " of graph " + to_string(graphID) + - " from " + sourceWorker + " to " + destinationWorker); +bool Utils::transferPartition(std::string sourceWorker, int sourceWorkerPort, std::string destinationWorker, + int destinationWorkerDataPort, std::string graphID, std::string partitionID, + std::string workerID, SQLiteDBInterface *sqlite) { + util_logger.info("### Transferring partition " + partitionID + " of graph " + graphID + " from " + sourceWorker + + " to " + destinationWorker); int sockfd; char data[FED_DATA_LENGTH + 1]; @@ -1104,8 +1103,7 @@ bool Utils::transferPartition(std::string sourceWorker, int sourceWorkerPort, in return false; } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, - to_string(graphID) + "," + to_string(partitionID), + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID + "," + partitionID, JasmineGraphInstanceProtocol::OK)) { Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); @@ -1113,6 +1111,11 @@ bool Utils::transferPartition(std::string sourceWorker, int sourceWorkerPort, in } util_logger.info("### Transfer partition completed"); + sqlite->runInsert( + "INSERT INTO worker_has_partition " + "(partition_idpartition, partition_graph_idgraph, worker_idworker) VALUES ('" + + partitionID + "', '" + graphID + "', '" + workerID + "')"); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; diff --git a/src/util/Utils.h b/src/util/Utils.h index e67c25410..f8b600914 100644 --- a/src/util/Utils.h +++ b/src/util/Utils.h @@ -179,13 +179,9 @@ class Utils { static bool sendFileThroughService(std::string host, int dataPort, std::string fileName, std::string filePath); - static bool transferPartition(std::string sourceWorker, - int sourceWorkerPort, - int sourceWorkerDataPort, - std::string destinationWorker, - int destinationWorkerPort, - int destinationWorkerDataPort, - int graphID, int partitionID); + static bool transferPartition(std::string sourceWorker, int sourceWorkerPort, std::string destinationWorker, + int destinationWorkerDataPort, std::string graphID, std::string partitionID, + std::string workerID, SQLiteDBInterface *sqlite); }; #endif // JASMINEGRAPH_UTILS_H From c6a2093c45c1d2aab732536ca0d42a7745956895 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 15 Mar 2024 02:03:47 +0530 Subject: [PATCH 101/143] Allocate partitions even if load is full --- .../executor/impl/TriangleCountExecutor.cpp | 60 ++++++++++++------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 1d92411e3..b2cdd6300 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -165,6 +165,7 @@ static std::vector reallocate_parts(std::map &alloc, std::set< copying.push_back(p0); continue; } + const auto &ws = P_AVAIL.find(p0)->second; bool need_pushing = true; for (auto it = PARTITIONS.begin(); it != PARTITIONS.end(); it++) { int p = *it; @@ -177,7 +178,6 @@ static std::vector reallocate_parts(std::map &alloc, std::set< continue; } auto w = alloc[p]; - const auto &ws = P_AVAIL.find(p0)->second; if (std::find(ws.begin(), ws.end(), w) != ws.end()) { alloc.erase(p); alloc[p0] = w; @@ -197,9 +197,11 @@ static void scale_up(std::map &loads, map &workers, curr_load += it->second; } int n_cores = copy_cnt + curr_load - 3 * loads.size(); - if (n_cores < 0) n_cores = 0; - int n_workers = n_cores / 3; - if (n_cores % 3 > 0) n_workers++; + if (n_cores < 0) { + return; + } + int n_workers = n_cores / 2; // allocate a little more to prevent saturation + if (n_cores % 2 > 0) n_workers++; if (n_workers == 0) return; K8sWorkerController *k8sController = K8sWorkerController::getInstance(); @@ -211,9 +213,10 @@ static void scale_up(std::map &loads, map &workers, } } -static int alloc_net_plan(std::vector &parts, std::map> &transfer, - std::map &net_loads, std::map &loads, - const std::map> &p_avail, int curr_best) { +static int alloc_net_plan(std::map &alloc, std::vector &parts, + std::map> &transfer, std::map &net_loads, + std::map &loads, const std::map> &p_avail, + int curr_best) { int curr_load = std::max_element(net_loads.begin(), net_loads.end(), [](const std::map::value_type &p1, const std::map::value_type &p2) { return p1.second < p2.second; }) @@ -226,6 +229,7 @@ static int alloc_net_plan(std::vector &parts, std::map alloc; std::map> transfer; std::map net_loads; std::map loads; @@ -246,27 +250,40 @@ static int alloc_net_plan(std::vector &parts, std::mapsecond; + int minLoad = 100000000; + for (auto it = loads.begin(); it != loads.end(); it++) { + int load = it->second; + if (minLoad > load) { + minLoad = load; + } + } for (auto itf = ws.begin(); itf != ws.end(); itf++) { auto wf = *itf; for (auto itt = wts.begin(); itt != wts.end(); itt++) { auto wt = *itt; + int currLoad = loads[wt]; + if (currLoad > minLoad) continue; + auto alloc2 = alloc; // need copy => do not copy reference auto parts2 = parts; // need copy => do not copy reference auto transfer2 = transfer; // need copy => do not copy reference auto net_loads2 = net_loads; // need copy => do not copy reference auto loads2 = loads; // need copy => do not copy reference - transfer2[p] = {wf, wt}; // assume - net_loads2[wf]++; - net_loads2[wt]++; + if (wf != wt) { + transfer2[p] = {wf, wt}; // assume + net_loads2[wf]++; + net_loads2[wt]++; + } + alloc2[p] = wt; loads2[wt]++; - if (loads2[wt] == 3) loads2.erase(wt); - int new_net_load = alloc_net_plan(parts2, transfer2, net_loads2, loads2, p_avail, best); + int new_net_load = alloc_net_plan(alloc2, parts2, transfer2, net_loads2, loads2, p_avail, best); if (new_net_load < best) { - if (loads2.find(wt) == loads.end()) loads2[wt] = 3; best = new_net_load; - best_plan = {.transfer = transfer2, .net_loads = net_loads2, .loads = loads2}; + best_plan = {.alloc = alloc2, .transfer = transfer2, .net_loads = net_loads2, .loads = loads2}; } } } + alloc.clear(); + alloc.insert(best_plan.alloc.begin(), best_plan.alloc.end()); auto &b_transfer = best_plan.transfer; for (auto it = b_transfer.begin(); it != b_transfer.end(); it++) { transfer[it->first] = it->second; @@ -299,12 +316,11 @@ static void filter_partitions(std::map> &partitionMa if (workerLoadIt != cpu_map.end()) { double load = stod(workerLoadIt->second.c_str()); if (load < 0) load = 0; - loads[workerId] = (int)round(4 * load); + loads[workerId] = (int)(4 * load); } else { loads[workerId] = 0; } } - const map LOADS = loads; std::map> p_avail; std::set remain; @@ -319,7 +335,7 @@ static void filter_partitions(std::map> &partitionMa } const std::map> P_AVAIL = p_avail; // get a copy and make it const - for (auto loadIt = LOADS.begin(); loadIt != LOADS.end(); loadIt++) { + for (auto loadIt = loads.begin(); loadIt != loads.end(); loadIt++) { if (loadIt->second < 3) continue; auto w = loadIt->first; for (auto it = p_avail.begin(); it != p_avail.end(); it++) { @@ -349,16 +365,14 @@ static void filter_partitions(std::map> &partitionMa for (auto it = loads.begin(); it != loads.end(); it++) { net_loads[it->first] = 0; } - for (auto it = loads.begin(); it != loads.end();) { - if (it->second >= 3) { - loads.erase(it++); - } else { - it++; + for (auto it = workers.begin(); it != workers.end(); it++) { + if (loads.find(it->first) == loads.end()) { + loads[it->first] = 3; } } std::map> transfer; - int net_load = alloc_net_plan(copying, transfer, net_loads, loads, P_AVAIL, 100000000); + int net_load = alloc_net_plan(alloc, copying, transfer, net_loads, loads, P_AVAIL, 100000000); cout << "alloc_net_plan completed with net_load=" << net_load << endl; for (auto it = transfer.begin(); it != transfer.end(); it++) { auto p = it->first; From b34991038a3a13b8529beb3dee3009355a798a94 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 15 Mar 2024 11:32:18 +0530 Subject: [PATCH 102/143] Increase timeout period for worker spawning --- src/k8s/K8sWorkerController.cpp | 38 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index c9b8f5694..a7d7eaad4 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -21,7 +21,7 @@ limitations under the License. Logger controller_logger; std::vector K8sWorkerController::workerList = {}; -static int TIME_OUT = 180; +static int TIME_OUT = 300; static std::vector activeWorkerIds = {}; static inline int getNextWorkerId() { @@ -109,29 +109,27 @@ std::string K8sWorkerController::spawnWorker(int workerId) { throw std::runtime_error("Worker " + std::to_string(workerId) + " deployment creation failed"); } - int sockfd; - struct sockaddr_in serv_addr; - struct hostent *server; + int waiting = 0; + while (true) { + int sockfd; + struct sockaddr_in serv_addr; + struct hostent *server; - sockfd = socket(AF_INET, SOCK_STREAM, 0); - if (sockfd < 0) { - controller_logger.error("Cannot create socket"); - return ""; - } + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + controller_logger.error("Cannot create socket"); + } - server = gethostbyname(ip.c_str()); - if (server == NULL) { - controller_logger.error("ERROR, no host named " + ip); - return ""; - } + server = gethostbyname(ip.c_str()); + if (server == NULL) { + controller_logger.error("ERROR, no host named " + ip); + } - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(Conts::JASMINEGRAPH_INSTANCE_PORT); + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(Conts::JASMINEGRAPH_INSTANCE_PORT); - int waiting = 0; - while (true) { if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { waiting += 30; // Added overhead in connection retry attempts sleep(10); From b6c3cd24177cafc1a5becdb22903c175286ae541 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 15 Mar 2024 21:15:36 +0530 Subject: [PATCH 103/143] Use fork() for instance service session and instance file transfer service --- main.cpp | 1 - ...asmineGraphInstanceFileTransferService.cpp | 18 ++++++++--- src/server/JasmineGraphInstanceService.cpp | 32 ++++++++++++------- src/server/JasmineGraphServer.cpp | 1 - src/util/logger/Logger.cpp | 8 ++++- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/main.cpp b/main.cpp index a81851c35..43391cb7e 100644 --- a/main.cpp +++ b/main.cpp @@ -36,7 +36,6 @@ int jasminegraph_profile = PROFILE_DOCKER; void fnExit3(void) { delete (server); - puts("Shutting down the server."); } int main(int argc, char *argv[]) { diff --git a/src/server/JasmineGraphInstanceFileTransferService.cpp b/src/server/JasmineGraphInstanceFileTransferService.cpp index 831679680..10dd6abac 100644 --- a/src/server/JasmineGraphInstanceFileTransferService.cpp +++ b/src/server/JasmineGraphInstanceFileTransferService.cpp @@ -85,11 +85,11 @@ void JasmineGraphInstanceFileTransferService::run(int dataPort) { } int connFd; listen(listenFd, 10); + file_service_logger.log("Worker FileTransfer Service listening on port " + to_string(dataPort), "info"); len = sizeof(clntAdd); while (true) { - file_service_logger.log("Worker FileTransfer Service listening on port " + to_string(dataPort), "info"); connFd = accept(listenFd, (struct sockaddr *)&clntAdd, &len); if (connFd < 0) { @@ -97,9 +97,17 @@ void JasmineGraphInstanceFileTransferService::run(int dataPort) { continue; } file_service_logger.log("Connection successful to port " + to_string(dataPort), "info"); - filetransferservicesessionargs *sessionargs = new filetransferservicesessionargs; - sessionargs->connFd = connFd; - pthread_t pt; - pthread_create(&pt, NULL, filetransferservicesession, sessionargs); + + pid_t pid = fork(); + if (pid == 0) { + close(listenFd); + filetransferservicesessionargs *sessionargs = new filetransferservicesessionargs; + sessionargs->connFd = connFd; + filetransferservicesession(sessionargs); + break; + } else { + close(connFd); + } } + exit(0); } diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index e79408768..72bd37bd5 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -322,21 +322,29 @@ void JasmineGraphInstanceService::run(string masterHost, string host, int server continue; } instance_logger.info("Connection successful to port " + to_string(serverPort)); - instanceservicesessionargs *serviceArguments_p = new instanceservicesessionargs; - serviceArguments_p->graphDBMapLocalStores = &graphDBMapLocalStores; - serviceArguments_p->graphDBMapCentralStores = &graphDBMapCentralStores; - serviceArguments_p->graphDBMapDuplicateCentralStores = &graphDBMapDuplicateCentralStores; - serviceArguments_p->incrementalLocalStore = &incrementalLocalStore; - serviceArguments_p->masterHost = masterHost; - serviceArguments_p->port = serverPort; - serviceArguments_p->dataPort = serverDataPort; - serviceArguments_p->host = host; - serviceArguments_p->connFd = connFd; - pthread_t pt; - pthread_create(&pt, NULL, instanceservicesession, serviceArguments_p); + + pid_t pid = fork(); + if (pid == 0) { + close(listenFd); + instanceservicesessionargs *serviceArguments_p = new instanceservicesessionargs; + serviceArguments_p->graphDBMapLocalStores = &graphDBMapLocalStores; + serviceArguments_p->graphDBMapCentralStores = &graphDBMapCentralStores; + serviceArguments_p->graphDBMapDuplicateCentralStores = &graphDBMapDuplicateCentralStores; + serviceArguments_p->incrementalLocalStore = &incrementalLocalStore; + serviceArguments_p->masterHost = masterHost; + serviceArguments_p->port = serverPort; + serviceArguments_p->dataPort = serverDataPort; + serviceArguments_p->host = host; + serviceArguments_p->connFd = connFd; + instanceservicesession(serviceArguments_p); + break; + } else { + close(connFd); + } } pthread_mutex_destroy(&file_lock); + exit(0); // FIXME: Cleanup before exit. } int deleteGraphPartition(std::string graphID, std::string partitionID) { diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index d23f4a170..77686355a 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -104,7 +104,6 @@ JasmineGraphServer *JasmineGraphServer::getInstance() { } JasmineGraphServer::~JasmineGraphServer() { - server_logger.info("Freeing up server resources."); this->sqlite->finalize(); this->performanceSqlite->finalize(); delete this->sqlite; diff --git a/src/util/logger/Logger.cpp b/src/util/logger/Logger.cpp index 149c3ffac..f7f16994e 100644 --- a/src/util/logger/Logger.cpp +++ b/src/util/logger/Logger.cpp @@ -39,7 +39,13 @@ static string get_worker_name() { void Logger::log(std::string message, const std::string log_type) { pthread_t tid = pthread_self(); - message = "[" + worker_name + " : " + to_string(tid) + "] " + message; + + // TODO: temporary fix only. + // message = "[" + worker_name + " : " + to_string(tid) + "] " + message; + message = " [" + log_type + "] [" + worker_name + " : " + to_string(tid) + "] " + message; + puts(message.c_str()); + return; + if (log_type.compare("info") == 0) { daily_logger->info(message); logger->info(message); From 5ef930fdf6d473bd5a747a952563ac64973f4acf Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 15 Mar 2024 23:21:54 +0530 Subject: [PATCH 104/143] Remove unnecessary logs --- src/k8s/K8sWorkerController.cpp | 2 +- src/performance/metrics/PerformanceUtil.cpp | 4 +++- src/performance/metrics/StatisticCollector.cpp | 13 ++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index a7d7eaad4..b2654e1e2 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -21,7 +21,7 @@ limitations under the License. Logger controller_logger; std::vector K8sWorkerController::workerList = {}; -static int TIME_OUT = 300; +static int TIME_OUT = 900; static std::vector activeWorkerIds = {}; static inline int getNextWorkerId() { diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index 5ff9175e3..dca0ea4ec 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -69,6 +69,8 @@ int PerformanceUtil::collectPerformanceStatistics() { long totalMemoryUsage = StatisticCollector::getTotalMemoryUsage(); // Host Utils::send_job("", "total_memory", std::to_string(totalMemoryUsage)); + + scheduler_logger.info("Pushed performance metrics"); return 0; } @@ -440,7 +442,7 @@ void PerformanceUtil::adjustAggregateLoadMap(std::map Date: Sat, 16 Mar 2024 19:07:42 +0530 Subject: [PATCH 105/143] Fix returning map for newly spawned workers --- src/k8s/K8sWorkerController.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index b2654e1e2..3d0cd9347 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -300,7 +300,7 @@ std::map K8sWorkerController::scaleUp(int count) { std::string result = asyncCalls[i].get(); if (!result.empty()) { success++; - workers.insert(std::make_pair(to_string(i), result)); + workers.insert(std::make_pair(to_string(nextWorkerId + i), result)); } } this->numberOfWorkers += success; From 387ba634cfd378731a2bbc1ddc6f622469bdbeca Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Sat, 16 Mar 2024 19:07:42 +0530 Subject: [PATCH 106/143] Move auto scaling flag to configMap --- conf/jasminegraph-server.properties | 1 - k8s/configs.yaml | 1 + .../core/executor/impl/TriangleCountExecutor.cpp | 4 +++- src/server/JasmineGraphServer.cpp | 13 ++++++------- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 29d7c3ee5..66599ac66 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -99,5 +99,4 @@ org.jasminegraph.k8s.image=jasminegraph # Auto-scaling #-------------------------------------------------------------------------------- org.jasminegraph.autoscale.scale_on_adgr=false -org.jasminegraph.autoscale.enabled=true org.jasminegraph.autopartition.enabled=false diff --git a/k8s/configs.yaml b/k8s/configs.yaml index 1b3469bb2..b3516e7a6 100644 --- a/k8s/configs.yaml +++ b/k8s/configs.yaml @@ -8,4 +8,5 @@ data: pushgateway_address: "${pushgateway_address}" prometheus_address: "${prometheus_address}" max_worker_count: "${max_worker_count}" + auto_scaling_enabled: "true" \ No newline at end of file diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index b2cdd6300..90198e518 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -520,10 +520,12 @@ void TriangleCountExecutor::execute() { cout << "}" << endl; cout << endl; + auto *k8sInterface = new K8sInterface(); if (jasminegraph_profile == PROFILE_K8S && - Utils::getJasmineGraphProperty("org.jasminegraph.autoscale.enabled") == "true") { + k8sInterface->getJasmineGraphConfig("auto_scaling_enabled") == "true") { filter_partitions(partitionMap, sqlite, graphId); } + delete k8sInterface; cout << "final partitionMap = {" << endl; for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 77686355a..510aff940 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -824,17 +824,16 @@ static map scaleK8s(size_t npart) { } static std::vector getWorkers(size_t npart) { - if (Utils::getJasmineGraphProperty("org.jasminegraph.autoscale.scale_on_adgr") != "true" || - Utils::getJasmineGraphProperty("org.jasminegraph.autoscale.enabled") != "true") { - if (jasminegraph_profile == PROFILE_K8S) { - return K8sWorkerController::workerList; - } - return hostWorkerList; - } // TODO: get the workers with lowest load from workerList std::vector *workerListAll; map cpu_loads; if (jasminegraph_profile == PROFILE_K8S) { + auto k8sInterface = new K8sInterface(); + if (k8sInterface->getJasmineGraphConfig("auto_scaling_enabled") != "ture") { + delete k8sInterface; + return K8sWorkerController::workerList; + } + delete k8sInterface; workerListAll = &(K8sWorkerController::workerList); cpu_loads = scaleK8s(npart); } else { From ff7845c3ebce01527f56d4cc7c3846c3c212dcd2 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 23 Feb 2024 17:46:15 +0530 Subject: [PATCH 107/143] Create release workflow --- .github/workflows/release.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..062ef069e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,28 @@ +name: Release + +on: + create: + tags: + - 'v*' + +jobs: + release: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{github.ref}} + repository: ${{ github.repository }} + + - name: Docker Build + run: | + docker build -t "jasminegraph:temp" . + CONTAINER_ID=$(docker create "jasminegraph:temp") + docker export "${CONTAINER_ID}" | docker import - "${{ secrets.DOCKER_USER }}/jasminegraph:${{ github.ref_name }}" + + - name: Docker Login + run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }} + + - name: Docker Push + run: docker push "${{ secrets.DOCKER_USER }}/jasminegraph:${{ github.ref_name }}" From 4fd79166643498f3e7d72dc92fdfe7e395f64d29 Mon Sep 17 00:00:00 2001 From: Ishad-M-I-M Date: Sat, 24 Feb 2024 14:43:48 +0530 Subject: [PATCH 108/143] Change the image name in the properties file upon release --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 062ef069e..a511e7539 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,6 +17,7 @@ jobs: - name: Docker Build run: | + sed -i "s#org.jasminegraph.k8s.image=.*#org.jasminegraph.k8s.image=${{ secrets.DOCKER_USER }}/jasminegraph:${{ github.ref_name }}#" ./conf/jasminegraph-server.properties docker build -t "jasminegraph:temp" . CONTAINER_ID=$(docker create "jasminegraph:temp") docker export "${CONTAINER_ID}" | docker import - "${{ secrets.DOCKER_USER }}/jasminegraph:${{ github.ref_name }}" From 46231827f8e78defc64664be6184fd627ede1ce8 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Mon, 26 Feb 2024 16:09:11 +0530 Subject: [PATCH 109/143] Update release workflow --- .github/workflows/release.yml | 12 ++++++++---- .github/workflows/resources/Dockerfile | 9 +++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/resources/Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a511e7539..5f50d72f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,10 +1,13 @@ name: Release on: - create: + push: tags: - 'v*' +env: + IMAGE: "${{ secrets.DOCKER_USER }}/jasminegraph:${{ github.ref_name }}" + jobs: release: runs-on: ubuntu-latest @@ -17,13 +20,14 @@ jobs: - name: Docker Build run: | - sed -i "s#org.jasminegraph.k8s.image=.*#org.jasminegraph.k8s.image=${{ secrets.DOCKER_USER }}/jasminegraph:${{ github.ref_name }}#" ./conf/jasminegraph-server.properties + sed -i "s#org.jasminegraph.k8s.image=.*#org.jasminegraph.k8s.image=${IMAGE}#" ./conf/jasminegraph-server.properties docker build -t "jasminegraph:temp" . CONTAINER_ID=$(docker create "jasminegraph:temp") - docker export "${CONTAINER_ID}" | docker import - "${{ secrets.DOCKER_USER }}/jasminegraph:${{ github.ref_name }}" + docker export "${CONTAINER_ID}" | docker import - "jasminegraph:pre-build" + docker build -t "${IMAGE}" -f .github/workflows/resources/Dockerfile . - name: Docker Login run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }} - name: Docker Push - run: docker push "${{ secrets.DOCKER_USER }}/jasminegraph:${{ github.ref_name }}" + run: docker push "${IMAGE}" diff --git a/.github/workflows/resources/Dockerfile b/.github/workflows/resources/Dockerfile new file mode 100644 index 000000000..57878a198 --- /dev/null +++ b/.github/workflows/resources/Dockerfile @@ -0,0 +1,9 @@ +FROM jasminegraph:pre-build + +ENV HOME="/home/ubuntu" +ENV JASMINEGRAPH_HOME="${HOME}/software/jasminegraph" + +WORKDIR "${JASMINEGRAPH_HOME}" + +ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"] +CMD ["bash"] \ No newline at end of file From 2c2ccdfcfafb84a787793dabff1896b8360f8fa9 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 29 Feb 2024 18:00:35 +0530 Subject: [PATCH 110/143] Add minimal image build step to release workflow --- .github/workflows/release.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5f50d72f8..8571c0217 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,8 +26,17 @@ jobs: docker export "${CONTAINER_ID}" | docker import - "jasminegraph:pre-build" docker build -t "${IMAGE}" -f .github/workflows/resources/Dockerfile . + - name: Docker Minimal Image Build + run: | + sed -i "s#org.jasminegraph.k8s.image=.*#org.jasminegraph.k8s.image=${IMAGE}-minimal#" ./conf/jasminegraph-server.properties + chmod +x ./minimal/build.sh + ./minimal/build.sh + docker tag "jasminegraph:minimal" "${IMAGE}-minimal" + - name: Docker Login run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }} - name: Docker Push - run: docker push "${IMAGE}" + run: | + docker push "${IMAGE}" + docker push "${IMAGE}-minimal" From 608a6ed23117705cc81bc83e6ec561d87366f6f3 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 14 Mar 2024 14:53:26 +0530 Subject: [PATCH 111/143] Change resource limits --- k8s/master-deployment.yaml | 7 ++----- k8s/prometheus.yaml | 4 ++++ k8s/pushgateway.yaml | 4 ++++ k8s/worker-deployment.yaml | 7 ++----- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/k8s/master-deployment.yaml b/k8s/master-deployment.yaml index 8804bc513..531bada4b 100644 --- a/k8s/master-deployment.yaml +++ b/k8s/master-deployment.yaml @@ -63,8 +63,5 @@ spec: name: jasminegraph-aggregate resources: requests: - cpu: "1000m" - memory: "2Gi" - limits: - cpu: "2000m" - memory: "4Gi" \ No newline at end of file + cpu: "2500m" + memory: "12Gi" \ No newline at end of file diff --git a/k8s/prometheus.yaml b/k8s/prometheus.yaml index c653c8ec3..a034fa535 100644 --- a/k8s/prometheus.yaml +++ b/k8s/prometheus.yaml @@ -50,6 +50,10 @@ spec: volumeMounts: - name: prometheus-config mountPath: /etc/prometheus + resources: + requests: + cpu: "1500m" + memory: "4Gi" volumes: - name: prometheus-config configMap: diff --git a/k8s/pushgateway.yaml b/k8s/pushgateway.yaml index 3da961b50..78e639072 100644 --- a/k8s/pushgateway.yaml +++ b/k8s/pushgateway.yaml @@ -22,6 +22,10 @@ spec: ports: - containerPort: 9091 imagePullPolicy: IfNotPresent + resources: + requests: + cpu: "1500m" + memory: "4Gi" restartPolicy: Always --- diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index f1bfafe3a..c9640a3d3 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -51,11 +51,8 @@ spec: - containerPort: 7778 resources: requests: - cpu: "1000m" - memory: "2Gi" - limits: - cpu: "2000m" - memory: "4Gi" + cpu: "2500m" + memory: "12Gi" volumeMounts: - mountPath: "/var/tmp/" name: "jasminegraph-worker-data" \ No newline at end of file From a61fb81532f87eebb4446acf73489dbd07c810d0 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Wed, 20 Mar 2024 13:30:12 +0530 Subject: [PATCH 112/143] Add new logs --- src/frontend/core/executor/impl/TriangleCountExecutor.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 90198e518..c35042722 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -574,13 +574,13 @@ void TriangleCountExecutor::execute() { string partitionId; int workerPort = atoi(string(currentWorker.port).c_str()); int workerDataPort = atoi(string(currentWorker.dataPort).c_str()); - + triangleCount_logger.info("worker_"+workerID+" host="+host+":"+ to_string(workerPort)+":"+ to_string(workerDataPort)); const std::vector &partitionList = partitionMap[workerID]; for (auto partitionIterator = partitionList.begin(); partitionIterator != partitionList.end(); ++partitionIterator) { partitionCount++; - partitionId = *partitionIterator; + triangleCount_logger.info("> partition"+partitionId); intermRes.push_back(std::async( std::launch::async, TriangleCountExecutor::getTriangleCount, atoi(graphId.c_str()), host, workerPort, workerDataPort, atoi(partitionId.c_str()), masterIP, uniqueId, isCompositeAggregation, threadPriority, @@ -616,6 +616,7 @@ void TriangleCountExecutor::execute() { } for (auto &&futureCall : intermRes) { + triangleCount_logger.info("Waiting for result. uuid="+ to_string(uniqueId)); result += futureCall.get(); } triangleTree.clear(); From 6f829792f65226dbd6d23ee96894935799c09b13 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 20 Mar 2024 20:27:47 +0530 Subject: [PATCH 113/143] Fix trian return partial result --- .../core/executor/impl/TriangleCountExecutor.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index c35042722..a0363aed6 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -473,8 +473,6 @@ void TriangleCountExecutor::execute() { long result = 0; bool isCompositeAggregation = false; Utils::worker aggregatorWorker; - vector workerList = Utils::getWorkerList(sqlite); - int workerListSize = workerList.size(); std::vector> intermRes; std::vector> statResponse; std::vector compositeCentralStoreFiles; @@ -521,8 +519,7 @@ void TriangleCountExecutor::execute() { cout << endl; auto *k8sInterface = new K8sInterface(); - if (jasminegraph_profile == PROFILE_K8S && - k8sInterface->getJasmineGraphConfig("auto_scaling_enabled") == "true") { + if (jasminegraph_profile == PROFILE_K8S && k8sInterface->getJasmineGraphConfig("auto_scaling_enabled") == "true") { filter_partitions(partitionMap, sqlite, graphId); } delete k8sInterface; @@ -567,6 +564,9 @@ void TriangleCountExecutor::execute() { std::unordered_map>> triangleTree; std::mutex triangleTreeMutex; int partitionCount = 0; + vector workerList = Utils::getWorkerList(sqlite); + int workerListSize = workerList.size(); + cout << "workerListSize = " << workerListSize << endl; for (int i = 0; i < workerListSize; i++) { Utils::worker currentWorker = workerList.at(i); string host = currentWorker.hostname; @@ -574,13 +574,14 @@ void TriangleCountExecutor::execute() { string partitionId; int workerPort = atoi(string(currentWorker.port).c_str()); int workerDataPort = atoi(string(currentWorker.dataPort).c_str()); - triangleCount_logger.info("worker_"+workerID+" host="+host+":"+ to_string(workerPort)+":"+ to_string(workerDataPort)); + triangleCount_logger.info("worker_" + workerID + " host=" + host + ":" + to_string(workerPort) + ":" + + to_string(workerDataPort)); const std::vector &partitionList = partitionMap[workerID]; for (auto partitionIterator = partitionList.begin(); partitionIterator != partitionList.end(); ++partitionIterator) { partitionCount++; partitionId = *partitionIterator; - triangleCount_logger.info("> partition"+partitionId); + triangleCount_logger.info("> partition" + partitionId); intermRes.push_back(std::async( std::launch::async, TriangleCountExecutor::getTriangleCount, atoi(graphId.c_str()), host, workerPort, workerDataPort, atoi(partitionId.c_str()), masterIP, uniqueId, isCompositeAggregation, threadPriority, @@ -616,7 +617,7 @@ void TriangleCountExecutor::execute() { } for (auto &&futureCall : intermRes) { - triangleCount_logger.info("Waiting for result. uuid="+ to_string(uniqueId)); + triangleCount_logger.info("Waiting for result. uuid=" + to_string(uniqueId)); result += futureCall.get(); } triangleTree.clear(); From 0c4603edb83d9c1afea926b7db12e285010c34c8 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 22 Mar 2024 01:25:47 +0530 Subject: [PATCH 114/143] Fix duplicating worker id --- src/k8s/K8sWorkerController.cpp | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 3d0cd9347..166b0410e 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -23,13 +23,12 @@ Logger controller_logger; std::vector K8sWorkerController::workerList = {}; static int TIME_OUT = 900; static std::vector activeWorkerIds = {}; +std::mutex workerIdMutex; +static volatile int nextWorkerId = 0; static inline int getNextWorkerId() { - if (activeWorkerIds.empty()) { - return 0; - } else { - return *max_element(activeWorkerIds.begin(), activeWorkerIds.end()) + 1; - } + const std::lock_guard lock(workerIdMutex); + return nextWorkerId++; } K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb) { @@ -45,12 +44,11 @@ K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorke this->maxWorkers = 4; } - // Delete all the workers from the database metadb->runUpdate("DELETE FROM worker"); int workersAttached = this->attachExistingWorkers(); - if (numberOfWorkers - workersAttached > 0){ - this->scaleUp(numberOfWorkers-workersAttached); + if (numberOfWorkers - workersAttached > 0) { + this->scaleUp(numberOfWorkers - workersAttached); } } @@ -100,8 +98,7 @@ std::string K8sWorkerController::spawnWorker(int workerId) { std::string ip(service->spec->cluster_ip); - v1_deployment_t *deployment = - this->interface->createJasmineGraphWorkerDeployment(workerId, ip, this->masterIp); + v1_deployment_t *deployment = this->interface->createJasmineGraphWorkerDeployment(workerId, ip, this->masterIp); if (deployment != nullptr && deployment->metadata != nullptr && deployment->metadata->name != nullptr) { controller_logger.info("Worker " + std::to_string(workerId) + " deployment created successfully"); @@ -131,7 +128,7 @@ std::string K8sWorkerController::spawnWorker(int workerId) { serv_addr.sin_port = htons(Conts::JASMINEGRAPH_INSTANCE_PORT); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - waiting += 30; // Added overhead in connection retry attempts + waiting += 30; // Added overhead in connection retry attempts sleep(10); } else { Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); @@ -145,7 +142,6 @@ std::string K8sWorkerController::spawnWorker(int workerId) { close(sockfd); return ""; } - } JasmineGraphServer::worker worker = { @@ -153,7 +149,8 @@ std::string K8sWorkerController::spawnWorker(int workerId) { K8sWorkerController::workerList.push_back(worker); std::string insertQuery = "INSERT INTO worker (host_idhost, server_port, server_data_port, name, ip, idworker) " - "VALUES ( -1, " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + + "VALUES ( -1, " + + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + "'" + std::string(service->metadata->name) + "', " + "'" + ip + "', " + std::to_string(workerId) + ")"; int status = metadb.runInsert(insertQuery); @@ -229,6 +226,9 @@ int K8sWorkerController::attachExistingWorkers() { v1_service_t *service; if (strcmp(pair->key, "workerId") == 0) { int workerId = std::stoi(static_cast(pair->value)); + workerIdMutex.lock(); + nextWorkerId = workerId + 1; + workerIdMutex.unlock(); v1_service_list_t *service_list = this->interface->getServiceList( strdup(("service=jasminegraph-worker,workerId=" + std::to_string(workerId)).c_str())); @@ -245,7 +245,8 @@ int K8sWorkerController::attachExistingWorkers() { K8sWorkerController::workerList.push_back(worker); std::string insertQuery = "INSERT INTO worker (host_idhost, server_port, server_data_port, name, ip, idworker) " - "VALUES ( -1, " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + + "VALUES ( -1, " + + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + "'" + std::string(service->metadata->name) + "', " + "'" + ip + "', " + std::to_string(workerId) + ")"; From 7c78d6c35c969a97bc30b5b00c60a0084c4b0942 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 22 Mar 2024 23:27:04 +0530 Subject: [PATCH 115/143] Add time and pid to log messages --- src/util/logger/Logger.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/util/logger/Logger.cpp b/src/util/logger/Logger.cpp index f7f16994e..8ba50f753 100644 --- a/src/util/logger/Logger.cpp +++ b/src/util/logger/Logger.cpp @@ -18,6 +18,7 @@ limitations under the License. #include #include #include +#include #include @@ -39,12 +40,12 @@ static string get_worker_name() { void Logger::log(std::string message, const std::string log_type) { pthread_t tid = pthread_self(); - + // TODO: temporary fix only. - // message = "[" + worker_name + " : " + to_string(tid) + "] " + message; - message = " [" + log_type + "] [" + worker_name + " : " + to_string(tid) + "] " + message; - puts(message.c_str()); + cout << " [" << chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count() + << "] [" << log_type << "] [" << worker_name << " : " << getpid() << ":" << tid << "] " << message << endl; return; + // message = "[" + worker_name + " : " + to_string(tid) + "] " + message; if (log_type.compare("info") == 0) { daily_logger->info(message); From 338843a2a1c5396e1fd8f876241fa26af86100ec Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 22 Mar 2024 23:51:45 +0530 Subject: [PATCH 116/143] Synchronize trian scheduler --- .../core/executor/impl/TriangleCountExecutor.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index a0363aed6..627c33c4f 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -21,10 +21,11 @@ using namespace std::chrono; Logger triangleCount_logger; bool isStatCollect = false; -std::mutex fileCombinationMutex; std::mutex processStatusMutex; std::mutex responseVectorMutex; +static std::mutex fileCombinationMutex; static std::mutex aggregateWeightMutex; +static std::mutex schedulerMutex; static string isFileAccessibleToWorker(std::string graphId, std::string partitionId, std::string aggregatorHostName, std::string aggregatorPort, std::string masterIP, std::string fileType, @@ -428,6 +429,7 @@ static void filter_partitions(std::map> &partitionMa } void TriangleCountExecutor::execute() { + schedulerMutex.lock(); int uniqueId = getUid(); std::string masterIP = request.getMasterIP(); std::string graphId = request.getParameter(Conts::PARAM_KEYS::GRAPH_ID); @@ -506,6 +508,10 @@ void TriangleCountExecutor::execute() { triangleCount_logger.info("###TRIANGLE-COUNT-EXECUTOR### Getting Triangle Count : PartitionId " + partitionId); } + if (results.size() > Conts::COMPOSITE_CENTRAL_STORE_WORKER_THRESHOLD) { + isCompositeAggregation = true; + } + cout << "initial partitionMap = {" << endl; for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { cout << " " << it->first << ": ["; @@ -536,10 +542,6 @@ void TriangleCountExecutor::execute() { cout << "}" << endl; cout << endl; - if (results.size() > Conts::COMPOSITE_CENTRAL_STORE_WORKER_THRESHOLD) { - isCompositeAggregation = true; - } - std::vector> fileCombinations; if (isCompositeAggregation) { std::string aggregatorFilePath = @@ -616,6 +618,8 @@ void TriangleCountExecutor::execute() { isStatCollect = true; } + schedulerMutex.unlock(); + for (auto &&futureCall : intermRes) { triangleCount_logger.info("Waiting for result. uuid=" + to_string(uniqueId)); result += futureCall.get(); From 24b58e47ecbbd8c6250e9cbc7581dfd787ac2a20 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 23 Mar 2024 00:47:55 +0530 Subject: [PATCH 117/143] Worker push performance metrics immediately after receiving tria --- src/backend/JasmineGraphBackend.cpp | 1 + src/frontend/JasmineGraphFrontEnd.cpp | 1 + src/frontend/core/executor/impl/TriangleCountExecutor.cpp | 2 +- src/frontend/core/scheduler/JobScheduler.cpp | 1 + src/server/JasmineGraphInstanceService.cpp | 4 +++- src/server/JasmineGraphServer.cpp | 2 ++ 6 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/backend/JasmineGraphBackend.cpp b/src/backend/JasmineGraphBackend.cpp index 023fde198..64e891bf6 100644 --- a/src/backend/JasmineGraphBackend.cpp +++ b/src/backend/JasmineGraphBackend.cpp @@ -153,6 +153,7 @@ int JasmineGraphBackend::run() { sessionargs->connFd = connFd; pthread_t pt; pthread_create(&pt, NULL, backendservicesesion, sessionargs); + pthread_detach(pt); } return 1; diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index 32bf93851..e5c3fcd79 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -309,6 +309,7 @@ int JasmineGraphFrontEnd::run() { sessionargs->jobScheduler = this->jobScheduler; pthread_t pt; pthread_create(&pt, NULL, frontendservicesesion, sessionargs); + pthread_detach(pt); } } diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 627c33c4f..af0b132a6 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -201,7 +201,7 @@ static void scale_up(std::map &loads, map &workers, if (n_cores < 0) { return; } - int n_workers = n_cores / 2; // allocate a little more to prevent saturation + int n_workers = n_cores / 2 + 1; // allocate a little more to prevent saturation if (n_cores % 2 > 0) n_workers++; if (n_workers == 0) return; diff --git a/src/frontend/core/scheduler/JobScheduler.cpp b/src/frontend/core/scheduler/JobScheduler.cpp index e73050441..23a91fc02 100644 --- a/src/frontend/core/scheduler/JobScheduler.cpp +++ b/src/frontend/core/scheduler/JobScheduler.cpp @@ -97,6 +97,7 @@ void *startScheduler(void *dummyPt) { void JobScheduler::init() { pthread_t schedulerThread; pthread_create(&schedulerThread, NULL, startScheduler, this); + pthread_detach(schedulerThread); } void JobScheduler::processJob(JobRequest request, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfDB) { diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 72bd37bd5..73c9d2434 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -344,7 +344,7 @@ void JasmineGraphInstanceService::run(string masterHost, string host, int server } pthread_mutex_destroy(&file_lock); - exit(0); // FIXME: Cleanup before exit. + exit(0); // FIXME: Cleanup before exit. } int deleteGraphPartition(std::string graphID, std::string partitionID) { @@ -2871,6 +2871,8 @@ static void triangles_command( threadPriorityMutex.unlock(); } + std::thread perfThread = std::thread(&PerformanceUtil::collectPerformanceStatistics); + perfThread.detach(); long localCount = countLocalTriangles(graphID, partitionId, graphDBMapLocalStores, graphDBMapCentralStores, graphDBMapDuplicateCentralStores, threadPriority); diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 510aff940..61be3033f 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -149,7 +149,9 @@ void JasmineGraphServer::init() { pthread_t frontendthread; pthread_t backendthread; pthread_create(&frontendthread, NULL, runfrontend, this); + pthread_detach(frontendthread); pthread_create(&backendthread, NULL, runbackend, this); + pthread_detach(backendthread); } void JasmineGraphServer::start_workers() { From 309e96482a9a8c18865fca8d275d4f357992ebdf Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 23 Mar 2024 16:25:57 +0530 Subject: [PATCH 118/143] Synchronize worker spawning --- src/frontend/core/executor/impl/TriangleCountExecutor.cpp | 3 ++- src/k8s/K8sWorkerController.cpp | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index af0b132a6..b6b0a86f4 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -246,7 +246,8 @@ static int alloc_net_plan(std::map &alloc, std::vector &parts, sort(wts.begin(), wts.end(), [&loads](string &w1, string &w2) { int l1 = loads[w1]; int l2 = loads[w2]; - if (l1 < 3 && l2 < 3) return LOAD_PREFERENCE[l1] > LOAD_PREFERENCE[loads[w2]]; + // TODO: temporarily commented for scale up only + // if (l1 < 3 && l2 < 3) return LOAD_PREFERENCE[l1] > LOAD_PREFERENCE[loads[w2]]; if (l1 < 3 || l2 < 3) return l1 < l2; return l1 <= l2; }); // load=1 goes first and load=3 goes last diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 166b0410e..51f04e850 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -24,6 +24,7 @@ std::vector K8sWorkerController::workerList = {}; static int TIME_OUT = 900; static std::vector activeWorkerIds = {}; std::mutex workerIdMutex; +std::mutex k8sSpawnMutex; static volatile int nextWorkerId = 0; static inline int getNextWorkerId() { @@ -75,6 +76,8 @@ K8sWorkerController *K8sWorkerController::getInstance(std::string masterIp, int } std::string K8sWorkerController::spawnWorker(int workerId) { + k8sSpawnMutex.lock(); + controller_logger.info("Spawning worker " + to_string(workerId)); auto volume = this->interface->createJasmineGraphPersistentVolume(workerId); if (volume != nullptr && volume->metadata != nullptr && volume->metadata->name != nullptr) { controller_logger.info("Worker " + std::to_string(workerId) + " persistent volume created successfully"); @@ -105,7 +108,8 @@ std::string K8sWorkerController::spawnWorker(int workerId) { } else { throw std::runtime_error("Worker " + std::to_string(workerId) + " deployment creation failed"); } - + k8sSpawnMutex.unlock(); + controller_logger.info("Waiting for worker " + to_string(workerId) + " to respond"); int waiting = 0; while (true) { int sockfd; @@ -143,6 +147,7 @@ std::string K8sWorkerController::spawnWorker(int workerId) { return ""; } } + controller_logger.info("Worker " + to_string(workerId) + " responded"); JasmineGraphServer::worker worker = { .hostname = ip, .port = Conts::JASMINEGRAPH_INSTANCE_PORT, .dataPort = Conts::JASMINEGRAPH_INSTANCE_DATA_PORT}; From d11c05631e4a5b8fd5a4631717a74ffc32102fa0 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 23 Mar 2024 17:31:25 +0530 Subject: [PATCH 119/143] Fix getNextWorkerId --- src/k8s/K8sWorkerController.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 51f04e850..902ff421e 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -27,9 +27,11 @@ std::mutex workerIdMutex; std::mutex k8sSpawnMutex; static volatile int nextWorkerId = 0; -static inline int getNextWorkerId() { +static inline int getNextWorkerId(int count) { const std::lock_guard lock(workerIdMutex); - return nextWorkerId++; + int returnId = nextWorkerId; + nextWorkerId += count; + return returnId; } K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb) { @@ -295,7 +297,7 @@ std::map K8sWorkerController::scaleUp(int count) { count = this->maxWorkers - this->numberOfWorkers; } auto *asyncCalls = new std::future[count]; - int nextWorkerId = getNextWorkerId(); + int nextWorkerId = getNextWorkerId(count); for (int i = 0; i < count; i++) { asyncCalls[i] = std::async(&K8sWorkerController::spawnWorker, this, nextWorkerId + i); } From 61ac238a68c7456ad30cafbaf039b43e3b7e575b Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 23 Mar 2024 22:35:54 +0530 Subject: [PATCH 120/143] Moved scale_on_adgr to k8s config.yml --- k8s/configs.yaml | 2 +- src/server/JasmineGraphServer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/configs.yaml b/k8s/configs.yaml index b3516e7a6..08e5adfa3 100644 --- a/k8s/configs.yaml +++ b/k8s/configs.yaml @@ -9,4 +9,4 @@ data: prometheus_address: "${prometheus_address}" max_worker_count: "${max_worker_count}" auto_scaling_enabled: "true" - \ No newline at end of file + scale_on_adgr: "false" diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 61be3033f..1085f5c17 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -831,7 +831,7 @@ static std::vector getWorkers(size_t npart) { map cpu_loads; if (jasminegraph_profile == PROFILE_K8S) { auto k8sInterface = new K8sInterface(); - if (k8sInterface->getJasmineGraphConfig("auto_scaling_enabled") != "ture") { + if (k8sInterface->getJasmineGraphConfig("scale_on_adgr") != "true") { delete k8sInterface; return K8sWorkerController::workerList; } From 5877f1d712743f8dfd84f503e3c3aaa85c75899b Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sat, 23 Mar 2024 23:03:42 +0530 Subject: [PATCH 121/143] Use scaleUp instead of setNumberOfWorkers in JasmineGraphServer --- conf/jasminegraph-server.properties | 1 - src/k8s/K8sWorkerController.cpp | 1 + src/server/JasmineGraphServer.cpp | 5 ++--- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 66599ac66..e1c400c7f 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -98,5 +98,4 @@ org.jasminegraph.k8s.image=jasminegraph #-------------------------------------------------------------------------------- # Auto-scaling #-------------------------------------------------------------------------------- -org.jasminegraph.autoscale.scale_on_adgr=false org.jasminegraph.autopartition.enabled=false diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 902ff421e..155fa479f 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -296,6 +296,7 @@ std::map K8sWorkerController::scaleUp(int count) { if (this->numberOfWorkers + count > this->maxWorkers) { count = this->maxWorkers - this->numberOfWorkers; } + controller_logger.info("Scale up with " + to_string(count) + " new workers"); auto *asyncCalls = new std::future[count]; int nextWorkerId = getNextWorkerId(count); for (int i = 0; i < count; i++) { diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 1085f5c17..b0ceb1cf8 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -812,9 +812,8 @@ static map scaleK8s(size_t npart) { } if (npart <= 0) return cpu_loads; K8sWorkerController *controller = K8sWorkerController::getInstance(); - size_t new_workers = workerList.size() + npart; - controller->setNumberOfWorkers((int)new_workers); - server_logger.info("Scalled up to " + to_string(new_workers) + " workers"); + controller->scaleUp((int)npart); + server_logger.info("Scalled up with " + to_string(npart) + " new workers"); for (auto it = K8sWorkerController::workerList.begin(); it != K8sWorkerController::workerList.end(); it++) { auto &worker = *it; string workerHostPort = worker.hostname + ":" + to_string(worker.port); From 05718ea2ec7d42b15c2581a2f1cf28fa764e766e Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Sun, 24 Mar 2024 05:02:45 +0530 Subject: [PATCH 122/143] Bugfix --- src/k8s/K8sWorkerController.cpp | 2 +- src/k8s/K8sWorkerController.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 155fa479f..73390d55a 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -36,7 +36,7 @@ static inline int getNextWorkerId(int count) { K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb) { this->masterIp = std::move(masterIp); - this->numberOfWorkers = numberOfWorkers; + this->numberOfWorkers = 0; this->interface = new K8sInterface(); this->metadb = *metadb; diff --git a/src/k8s/K8sWorkerController.h b/src/k8s/K8sWorkerController.h index 33b09d2c4..de9d62168 100644 --- a/src/k8s/K8sWorkerController.h +++ b/src/k8s/K8sWorkerController.h @@ -32,7 +32,7 @@ class K8sWorkerController { SQLiteDBInterface metadb; std::string masterIp; - int numberOfWorkers; + std::atomic numberOfWorkers; int maxWorkers; K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb); From 1f7467224023e1e60fb449fb0b4adc8ceff7871d Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Mon, 25 Mar 2024 22:46:30 +0530 Subject: [PATCH 123/143] Load configmap entries from K8s only if missing in cache --- src/k8s/K8sInterface.cpp | 18 +++++++++++++++++- src/k8s/K8sInterface.h | 3 +++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/k8s/K8sInterface.cpp b/src/k8s/K8sInterface.cpp index 06378ca12..ac3b95321 100644 --- a/src/k8s/K8sInterface.cpp +++ b/src/k8s/K8sInterface.cpp @@ -174,7 +174,7 @@ v1_node_list_t *K8sInterface::getNodes() { return node_list; } -std::string K8sInterface::getJasmineGraphConfig(std::string key) { +std::string K8sInterface::loadFromConfig(std::string key) { v1_config_map_t *config_map = CoreV1API_readNamespacedConfigMap(apiClient, strdup("jasminegraph-config"), namespace_, NULL); if (config_map->metadata->name == NULL) { @@ -192,6 +192,22 @@ std::string K8sInterface::getJasmineGraphConfig(std::string key) { return ""; } +static std::map configMap; +static std::mutex configMapMutex; +std::string K8sInterface::getJasmineGraphConfig(std::string key) { + auto item = configMap.find(key); + if (item == configMap.end()) { + configMapMutex.lock(); + if (configMap.find(key) == configMap.end()) { + k8s_logger.info("Key " + key + " not found in cache. Loading from config."); + configMap[key] = loadFromConfig(key); + } + configMapMutex.unlock(); + item = configMap.find(key); + } + return item->second; +} + v1_persistent_volume_t *K8sInterface::createJasmineGraphPersistentVolume(int workerId) const { std::string definition = Utils::getJsonStringFromYamlFile(ROOT_DIR "/k8s/worker-volume.yaml"); definition = Utils::replaceAll(definition, "", std::to_string(workerId)); diff --git a/src/k8s/K8sInterface.h b/src/k8s/K8sInterface.h index af2dc58da..c8288cc54 100644 --- a/src/k8s/K8sInterface.h +++ b/src/k8s/K8sInterface.h @@ -65,6 +65,9 @@ class K8sInterface { v1_persistent_volume_t *deleteJasmineGraphPersistentVolume(int workerId) const; v1_persistent_volume_claim_t *deleteJasmineGraphPersistentVolumeClaim(int workerId) const; + +private: + std::string loadFromConfig(std::string key); }; #endif // JASMINEGRAPH_K8SINTERFACE_H From 6636db7e521e0228ce26ff043b7c688c9c9138f2 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Tue, 26 Mar 2024 20:34:07 +0530 Subject: [PATCH 124/143] Load push gateway address on worker creation --- .../executor/impl/TriangleCountExecutor.cpp | 1 - src/k8s/K8sInterface.h | 5 +-- src/performance/metrics/PerformanceUtil.cpp | 43 +++++++++---------- src/server/JasmineGraphInstanceService.cpp | 3 ++ tests/unit/k8s/K8sInterface_test.cpp | 6 +-- 5 files changed, 27 insertions(+), 31 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index b6b0a86f4..0b6583128 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -1614,7 +1614,6 @@ std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return Utils::split(response, ':'); - ; } std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggregatorHostName, diff --git a/src/k8s/K8sInterface.h b/src/k8s/K8sInterface.h index c8288cc54..b6f649d11 100644 --- a/src/k8s/K8sInterface.h +++ b/src/k8s/K8sInterface.h @@ -36,8 +36,7 @@ class K8sInterface { v1_service_list_t *getServiceList(char *labelSelectors); - v1_deployment_t *createJasmineGraphWorkerDeployment(int workerId, - const std::string &ip, + v1_deployment_t *createJasmineGraphWorkerDeployment(int workerId, const std::string &ip, const std::string &masterIp) const; v1_status_t *deleteJasmineGraphWorkerDeployment(int workerId) const; @@ -66,7 +65,7 @@ class K8sInterface { v1_persistent_volume_claim_t *deleteJasmineGraphPersistentVolumeClaim(int workerId) const; -private: + private: std::string loadFromConfig(std::string key); }; diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index dca0ea4ec..6e9949923 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -35,41 +35,38 @@ void PerformanceUtil::init() { } int PerformanceUtil::collectPerformanceStatistics() { - long memoryUsage = StatisticCollector::getMemoryUsageByProcess(); - Utils::send_job("", "memory_usage", std::to_string(memoryUsage)); - + // Host level double cpuUsage = StatisticCollector::getCpuUsage(); - // Worker Utils::send_job("", "cpu_usage", std::to_string(cpuUsage)); - int threadCount = StatisticCollector::getThreadCount(); - // Host - Utils::send_job("", "thread_count", std::to_string(threadCount)); - - long totalSwapSpace = StatisticCollector::getTotalSwapSpace(); - // Host - Utils::send_job("", "total_swap_space", std::to_string(totalSwapSpace)); - - long usedSwapSpace = StatisticCollector::getUsedSwapSpace(); - // Host - Utils::send_job("", "used_swap_space", std::to_string(usedSwapSpace)); - long rxBytes = StatisticCollector::getRXBytes(); - // Worker Utils::send_job("", "rx_bytes", std::to_string(rxBytes)); long txBytes = StatisticCollector::getTXBytes(); - // Worker Utils::send_job("", "tx_bytes", std::to_string(txBytes)); - int socketCount = StatisticCollector::getSocketCount(); - // Host - Utils::send_job("", "socket_count", std::to_string(socketCount)); - long totalMemoryUsage = StatisticCollector::getTotalMemoryUsage(); - // Host Utils::send_job("", "total_memory", std::to_string(totalMemoryUsage)); + long usedSwapSpace = StatisticCollector::getUsedSwapSpace(); + Utils::send_job("", "used_swap_space", std::to_string(usedSwapSpace)); + + /* + long totalSwapSpace = StatisticCollector::getTotalSwapSpace(); + Utils::send_job("", "total_swap_space", std::to_string(totalSwapSpace)); + */ + + // Per process + /* + long memoryUsage = StatisticCollector::getMemoryUsageByProcess(); + Utils::send_job("", "memory_usage", std::to_string(memoryUsage)); + + int threadCount = StatisticCollector::getThreadCount(); + Utils::send_job("", "thread_count", std::to_string(threadCount)); + + int socketCount = StatisticCollector::getSocketCount(); + Utils::send_job("", "socket_count", std::to_string(socketCount)); + */ scheduler_logger.info("Pushed performance metrics"); return 0; } diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 73c9d2434..d14fc5608 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -313,6 +313,9 @@ void JasmineGraphInstanceService::run(string masterHost, string host, int server std::map graphDBMapDuplicateCentralStores; std::map incrementalLocalStore; + std::thread perfThread = std::thread(&PerformanceUtil::collectPerformanceStatistics); + perfThread.detach(); + instance_logger.info("Worker listening on port " + to_string(serverPort)); while (true) { int connFd = accept(listenFd, (struct sockaddr *)&clntAdd, &len); diff --git a/tests/unit/k8s/K8sInterface_test.cpp b/tests/unit/k8s/K8sInterface_test.cpp index 21827616a..513bd675f 100644 --- a/tests/unit/k8s/K8sInterface_test.cpp +++ b/tests/unit/k8s/K8sInterface_test.cpp @@ -27,9 +27,7 @@ class K8sInterfaceTest : public ::testing::Test { const int HTTP_OK = 200; const int HTTP_CREATED = 201; -TEST_F(K8sInterfaceTest, TestConstructor) { - ASSERT_NE(interface->apiClient, nullptr); -} +TEST_F(K8sInterfaceTest, TestConstructor) { ASSERT_NE(interface->apiClient, nullptr); } TEST_F(K8sInterfaceTest, TestGetNodes) { v1_node_list_t *nodes = interface->getNodes(); @@ -116,4 +114,4 @@ TEST_F(K8sInterfaceTest, deleteJasmineGraphPersistentVolumeClaim) { auto result = interface->deleteJasmineGraphPersistentVolumeClaim(1); ASSERT_EQ(interface->apiClient->response_code, HTTP_OK); ASSERT_STREQ(result->metadata->name, "jasminegraph-worker1-data-claim"); -} \ No newline at end of file +} From 8b9995063c98d7fc530b480bb3a0fda952a0060e Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Tue, 26 Mar 2024 21:17:59 +0530 Subject: [PATCH 125/143] Log request ID --- src/frontend/JasmineGraphFrontEnd.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index e5c3fcd79..06b6c35ce 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -1386,6 +1386,9 @@ static void triangles_command(std::string masterIP, int connFd, SQLiteDBInterfac int threadPriority = std::atoi(priority.c_str()); + static volatile int reqCounter = 0; + string reqId = to_string(reqCounter++); + frontend_logger.info("Started processing request " + reqId); auto begin = chrono::high_resolution_clock::now(); JobRequest jobDetails; jobDetails.setJobId(std::to_string(uniqueId)); @@ -1453,8 +1456,8 @@ static void triangles_command(std::string masterIP, int connFd, SQLiteDBInterfac auto end = chrono::high_resolution_clock::now(); auto dur = end - begin; auto msDuration = std::chrono::duration_cast(dur).count(); - frontend_logger.info("Triangle Count: " + triangleCount + " Time Taken: " + to_string(msDuration) + - " milliseconds"); + frontend_logger.info("Req: " + reqId + " Triangle Count: " + triangleCount + + " Time Taken: " + to_string(msDuration) + " milliseconds"); result_wr = write(connFd, triangleCount.c_str(), triangleCount.length()); if (result_wr < 0) { frontend_logger.error("Error writing to socket"); From 0ace1929138f39f713aa8a1a09a04c70bfc9a139 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Tue, 26 Mar 2024 22:03:08 +0530 Subject: [PATCH 126/143] Use absolute time for logs --- src/util/logger/Logger.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/util/logger/Logger.cpp b/src/util/logger/Logger.cpp index 8ba50f753..f7a8949b1 100644 --- a/src/util/logger/Logger.cpp +++ b/src/util/logger/Logger.cpp @@ -18,6 +18,7 @@ limitations under the License. #include #include #include +#include #include #include @@ -41,9 +42,12 @@ static string get_worker_name() { void Logger::log(std::string message, const std::string log_type) { pthread_t tid = pthread_self(); + struct timeval tv; + gettimeofday(&tv, NULL); + long millis = tv.tv_sec * 1000 + tv.tv_usec / 1000; // TODO: temporary fix only. - cout << " [" << chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count() - << "] [" << log_type << "] [" << worker_name << " : " << getpid() << ":" << tid << "] " << message << endl; + cout << " [" << millis << "] [" << log_type << "] [" << worker_name << " : " << getpid() << ":" << tid << "] " + << message << endl; return; // message = "[" + worker_name + " : " + to_string(tid) + "] " + message; From a619dfdabdadd05741dbaf4ed6a82777e93a3f02 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Tue, 26 Mar 2024 23:58:45 +0530 Subject: [PATCH 127/143] Wait for newly allocated job push performace metrics before scheduling --- src/frontend/core/executor/impl/TriangleCountExecutor.cpp | 5 ++++- src/k8s/K8sWorkerController.cpp | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index 0b6583128..da10ef8cd 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -430,7 +430,10 @@ static void filter_partitions(std::map> &partitionMa } void TriangleCountExecutor::execute() { - schedulerMutex.lock(); + if (!schedulerMutex.try_lock()) { + schedulerMutex.lock(); + sleep(8); + } int uniqueId = getUid(); std::string masterIP = request.getMasterIP(); std::string graphId = request.getParameter(Conts::PARAM_KEYS::GRAPH_ID); diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 73390d55a..43411cf58 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -296,14 +296,15 @@ std::map K8sWorkerController::scaleUp(int count) { if (this->numberOfWorkers + count > this->maxWorkers) { count = this->maxWorkers - this->numberOfWorkers; } + std::map workers; + if (count <= 0) return workers; controller_logger.info("Scale up with " + to_string(count) + " new workers"); - auto *asyncCalls = new std::future[count]; + std::future asyncCalls[count]; int nextWorkerId = getNextWorkerId(count); for (int i = 0; i < count; i++) { asyncCalls[i] = std::async(&K8sWorkerController::spawnWorker, this, nextWorkerId + i); } - std::map workers; int success = 0; for (int i = 0; i < count; i++) { std::string result = asyncCalls[i].get(); From 26586711191d612c4e00838fc0df3c1d4dd911a3 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 27 Mar 2024 18:38:22 +0530 Subject: [PATCH 128/143] Prepare k8s client for multi-thread --- src/k8s/K8sWorkerController.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 43411cf58..6d267aba7 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -37,6 +37,7 @@ static inline int getNextWorkerId(int count) { K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb) { this->masterIp = std::move(masterIp); this->numberOfWorkers = 0; + apiClient_setupGlobalEnv(); this->interface = new K8sInterface(); this->metadb = *metadb; @@ -55,7 +56,10 @@ K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorke } } -K8sWorkerController::~K8sWorkerController() { delete this->interface; } +K8sWorkerController::~K8sWorkerController() { + delete this->interface; + apiClient_unsetupGlobalEnv(); +} static K8sWorkerController *instance = nullptr; From 9699da3555f31e8b0c7a92684dd2afe819206967 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Sun, 31 Mar 2024 14:01:06 +0530 Subject: [PATCH 129/143] Prevent scheduling trian until workers push metrics --- .../core/executor/impl/TriangleCountExecutor.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index da10ef8cd..fec40fe15 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -13,6 +13,9 @@ limitations under the License. #include "TriangleCountExecutor.h" +#include +#include + #include "../../../../../globals.h" #include "../../../../k8s/K8sWorkerController.h" @@ -430,9 +433,11 @@ static void filter_partitions(std::map> &partitionMa } void TriangleCountExecutor::execute() { - if (!schedulerMutex.try_lock()) { - schedulerMutex.lock(); - sleep(8); + static time_t last_exec = 0; + schedulerMutex.lock(); + time_t curr_time = time(NULL); + if (curr_time < last_exec + 8) { + sleep(last_exec + 9 - curr_time); } int uniqueId = getUid(); std::string masterIP = request.getMasterIP(); @@ -622,6 +627,7 @@ void TriangleCountExecutor::execute() { isStatCollect = true; } + last_exec = time(NULL); schedulerMutex.unlock(); for (auto &&futureCall : intermRes) { From 27d59c6919410424f98edc8741506aff526e5285 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 3 Apr 2024 15:28:32 +0530 Subject: [PATCH 130/143] removed k8s resource requests --- k8s/master-deployment.yaml | 4 ---- k8s/prometheus.yaml | 6 +----- k8s/pushgateway.yaml | 4 ---- k8s/worker-deployment.yaml | 6 +----- 4 files changed, 2 insertions(+), 18 deletions(-) diff --git a/k8s/master-deployment.yaml b/k8s/master-deployment.yaml index 531bada4b..01b766fc6 100644 --- a/k8s/master-deployment.yaml +++ b/k8s/master-deployment.yaml @@ -61,7 +61,3 @@ spec: name: jasminegraph-logs - mountPath: /var/tmp/jasminegraph-aggregate name: jasminegraph-aggregate - resources: - requests: - cpu: "2500m" - memory: "12Gi" \ No newline at end of file diff --git a/k8s/prometheus.yaml b/k8s/prometheus.yaml index a034fa535..6e8dca662 100644 --- a/k8s/prometheus.yaml +++ b/k8s/prometheus.yaml @@ -50,10 +50,6 @@ spec: volumeMounts: - name: prometheus-config mountPath: /etc/prometheus - resources: - requests: - cpu: "1500m" - memory: "4Gi" volumes: - name: prometheus-config configMap: @@ -75,4 +71,4 @@ spec: nodePort: 30003 selector: app: prometheus - type: NodePort \ No newline at end of file + type: NodePort diff --git a/k8s/pushgateway.yaml b/k8s/pushgateway.yaml index 78e639072..3da961b50 100644 --- a/k8s/pushgateway.yaml +++ b/k8s/pushgateway.yaml @@ -22,10 +22,6 @@ spec: ports: - containerPort: 9091 imagePullPolicy: IfNotPresent - resources: - requests: - cpu: "1500m" - memory: "4Gi" restartPolicy: Always --- diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index c9640a3d3..ae7a5dca8 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -49,10 +49,6 @@ spec: ports: - containerPort: 7777 - containerPort: 7778 - resources: - requests: - cpu: "2500m" - memory: "12Gi" volumeMounts: - mountPath: "/var/tmp/" - name: "jasminegraph-worker-data" \ No newline at end of file + name: "jasminegraph-worker-data" From d34ff57cb9a1bc90a050de0dac4dd0b5d94e12a8 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Thu, 4 Apr 2024 18:02:56 +0530 Subject: [PATCH 131/143] Fix unit test workflow --- .github/workflows/build.yml | 10 ++--- .github/workflows/resources/unit-test.sh | 44 +++++++++++++++++++++ tests/unit/k8s/K8sWorkerController_test.cpp | 32 ++------------- 3 files changed, 53 insertions(+), 33 deletions(-) create mode 100755 .github/workflows/resources/unit-test.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 858a099d3..70396db62 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,9 @@ jobs: repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - name: Build unit test - run: docker build -f ./tests/unit/Dockerfile -t jasminegraph:unit . + run: | + docker build -t jasminegraph . + docker build -f ./tests/unit/Dockerfile -t jasminegraph:unit . - uses: jupyterhub/action-k3s-helm@v4 with: @@ -29,10 +31,8 @@ jobs: - name: Run unit test run: | - mkdir coverage - kubectl apply -f ./.github/workflows/resources/unit-test-conf.yaml - sleep 30 - kubectl logs -f jasminegraph-unit-test-pod + chmod +x ./.github/workflows/resources/unit-test.sh + ./.github/workflows/resources/unit-test.sh - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4 diff --git a/.github/workflows/resources/unit-test.sh b/.github/workflows/resources/unit-test.sh new file mode 100755 index 000000000..4846ea9b6 --- /dev/null +++ b/.github/workflows/resources/unit-test.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Function to check if pod is in Pending/Running state +pod_status="" +check_pod_status() { + pod_status=$(kubectl get pod jasminegraph-unit-test-pod --no-headers -o custom-columns=":status.phase") + + if [[ $pod_status == "Pending" || $pod_status == "ContainerCreating" ]]; then + echo 0 + else + echo 1 + fi +} + +mkdir coverage +kubectl apply -f ./k8s/configs.yaml +kubectl apply -f ./.github/workflows/resources/unit-test-conf.yaml + +timeout=300 # Set the timeout in seconds (adjust as needed) +start_time=$(date +%s) + +while [[ $(check_pod_status) == "0" ]]; do + sleep 10 + + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + + if [[ $elapsed_time -ge $timeout ]]; then + echo "Timeout reached. Exiting loop." + break + fi +done + +echo "----------------------------- logs -----------------------------" +kubectl logs -f jasminegraph-unit-test-pod + +check_pod_status +if [[ $pod_status != "Running" && $pod_status != "Completed" ]]; then + echo "Pod jasminegraph-unit-test-pod is in $pod_status state" + echo "Unit tests failed" + echo "----------------------------- details --------------------------" + kubectl describe pod jasminegraph-unit-test-pod + exit 1 +fi diff --git a/tests/unit/k8s/K8sWorkerController_test.cpp b/tests/unit/k8s/K8sWorkerController_test.cpp index 9cf656377..b11ea139c 100644 --- a/tests/unit/k8s/K8sWorkerController_test.cpp +++ b/tests/unit/k8s/K8sWorkerController_test.cpp @@ -55,42 +55,18 @@ TEST_F(K8sWorkerControllerTest, TestConstructor) { ASSERT_EQ(service_list->items->count, 2); } -TEST_F(K8sWorkerControllerTest, TestScalingUpAndDown) { - controller->setNumberOfWorkers(4); - ASSERT_EQ(controller->getNumberOfWorkers(), 4); - auto result = metadb->runSelect("SELECT idworker FROM worker"); - ASSERT_EQ(result.size(), 4); - - v1_deployment_list_t *deployment_list = interface->getDeploymentList(strdup("deployment=jasminegraph-worker")); - ASSERT_EQ(deployment_list->items->count, 4); - v1_service_list_t *service_list = interface->getServiceList(strdup("service=jasminegraph-worker")); - ASSERT_EQ(service_list->items->count, 4); - - controller->setNumberOfWorkers(0); - ASSERT_EQ(controller->getNumberOfWorkers(), 0); - result = metadb->runSelect("SELECT idworker FROM worker"); - ASSERT_EQ(result.size(), 0); - - deployment_list = interface->getDeploymentList(strdup("deployment=jasminegraph-worker")); - ASSERT_EQ(deployment_list->items->count, 0); - service_list = interface->getServiceList(strdup("service=jasminegraph-worker")); - ASSERT_EQ(service_list->items->count, 0); -} - TEST_F(K8sWorkerControllerTest, TestScaleUp) { - controller->setNumberOfWorkers(0); auto result = controller->scaleUp(2); ASSERT_EQ(result.size(), 2); v1_deployment_list_t *deployment_list = interface->getDeploymentList(strdup("deployment=jasminegraph-worker")); - ASSERT_EQ(deployment_list->items->count, 2); + ASSERT_EQ(deployment_list->items->count, 4); v1_service_list_t *service_list = interface->getServiceList(strdup("service=jasminegraph-worker")); - ASSERT_EQ(service_list->items->count, 2); + ASSERT_EQ(service_list->items->count, 4); } TEST_F(K8sWorkerControllerTest, TestScaleUpBeyondLimit) { - controller->setNumberOfWorkers(0); - auto result = controller->scaleUp(5); - ASSERT_EQ(result.size(), 4); + auto result = controller->scaleUp(1); + ASSERT_EQ(result.size(), 0); v1_deployment_list_t *deployment_list = interface->getDeploymentList(strdup("deployment=jasminegraph-worker")); ASSERT_EQ(deployment_list->items->count, 4); v1_service_list_t *service_list = interface->getServiceList(strdup("service=jasminegraph-worker")); From d4fc66bc135fcfa1c70115f523507d1bbe648d9d Mon Sep 17 00:00:00 2001 From: ArunaSenanayake Date: Fri, 5 Apr 2024 20:47:12 +0530 Subject: [PATCH 132/143] Added collecting the load average metric --- src/performance/metrics/PerformanceUtil.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index 6e9949923..c7c293478 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -51,6 +51,9 @@ int PerformanceUtil::collectPerformanceStatistics() { long usedSwapSpace = StatisticCollector::getUsedSwapSpace(); Utils::send_job("", "used_swap_space", std::to_string(usedSwapSpace)); + double currentLoadAverage = StatisticCollector::getLoadAverage(); + Utils::send_job("", "load_average", std::to_string(currentLoadAverage)); + /* long totalSwapSpace = StatisticCollector::getTotalSwapSpace(); Utils::send_job("", "total_swap_space", std::to_string(totalSwapSpace)); From d8c4fa174bc5089f9e02a23d75c79d67f7d2deae Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Sat, 6 Apr 2024 13:13:14 +0530 Subject: [PATCH 133/143] Fix k8s integration workflow failure Block accepting connections until initial workers get connected --- main.cpp | 6 ++-- .../executor/impl/TriangleCountExecutor.cpp | 3 +- src/k8s/K8sWorkerController.cpp | 27 +++++++------- src/server/JasmineGraphServer.cpp | 6 ++-- test-k8s.sh | 36 ++++++++++++++++--- 5 files changed, 50 insertions(+), 28 deletions(-) diff --git a/main.cpp b/main.cpp index 43391cb7e..3846a7f0d 100644 --- a/main.cpp +++ b/main.cpp @@ -69,10 +69,10 @@ int main(int argc, char *argv[]) { server = JasmineGraphServer::getInstance(); if (jasminegraph_profile == PROFILE_K8S) { - K8sInterface *interface = new K8sInterface(); - masterIp = interface->getMasterIp(); + std::unique_ptr k8sInterface(new K8sInterface()); + masterIp = k8sInterface->getMasterIp(); if (masterIp.empty()) { - masterIp = interface->createJasmineGraphMasterService()->spec->cluster_ip; + masterIp = k8sInterface->createJasmineGraphMasterService()->spec->cluster_ip; } } server->run(masterIp, numberOfWorkers, workerIps, enableNmon); diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index fec40fe15..dcac3356e 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -533,11 +533,10 @@ void TriangleCountExecutor::execute() { cout << "}" << endl; cout << endl; - auto *k8sInterface = new K8sInterface(); + std::unique_ptr k8sInterface(new K8sInterface()); if (jasminegraph_profile == PROFILE_K8S && k8sInterface->getJasmineGraphConfig("auto_scaling_enabled") == "true") { filter_partitions(partitionMap, sqlite, graphId); } - delete k8sInterface; cout << "final partitionMap = {" << endl; for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 6d267aba7..2cfa852dc 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -40,20 +40,6 @@ K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorke apiClient_setupGlobalEnv(); this->interface = new K8sInterface(); this->metadb = *metadb; - - try { - this->maxWorkers = stoi(this->interface->getJasmineGraphConfig("max_worker_count")); - } catch (std::invalid_argument &e) { - controller_logger.error("Invalid max_worker_count value. Defaulted to 4"); - this->maxWorkers = 4; - } - - // Delete all the workers from the database - metadb->runUpdate("DELETE FROM worker"); - int workersAttached = this->attachExistingWorkers(); - if (numberOfWorkers - workersAttached > 0) { - this->scaleUp(numberOfWorkers - workersAttached); - } } K8sWorkerController::~K8sWorkerController() { @@ -75,6 +61,19 @@ K8sWorkerController *K8sWorkerController::getInstance(std::string masterIp, int // TODO(thevindu-w): synchronize if (instance == nullptr) { instance = new K8sWorkerController(masterIp, numberOfWorkers, metadb); + try { + instance->maxWorkers = stoi(instance->interface->getJasmineGraphConfig("max_worker_count")); + } catch (std::invalid_argument &e) { + controller_logger.error("Invalid max_worker_count value. Defaulted to 4"); + instance->maxWorkers = 4; + } + + // Delete all the workers from the database + metadb->runUpdate("DELETE FROM worker"); + int workersAttached = instance->attachExistingWorkers(); + if (numberOfWorkers - workersAttached > 0) { + instance->scaleUp(numberOfWorkers - workersAttached); + } } else { controller_logger.warn("Not initializing again"); } diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index b0ceb1cf8..3485796f2 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -123,7 +123,6 @@ int JasmineGraphServer::run(std::string masterIp, int numberofWorkers, std::stri this->numberOfWorkers = numberofWorkers; this->workerHosts = workerIps; this->enableNmon = enableNmon; - init(); masterPortVector.push_back(Conts::JASMINEGRAPH_FRONTEND_PORT); updateOperationalGraphList(); @@ -136,6 +135,7 @@ int JasmineGraphServer::run(std::string masterIp, int numberofWorkers, std::stri addInstanceDetailsToPerformanceDB(masterHost, masterPortVector, "true"); } + init(); std::thread *myThreads = new std::thread[1]; myThreads[0] = std::thread(StatisticCollector::logLoadAverage, "Load Average"); sleep(1); @@ -829,12 +829,10 @@ static std::vector getWorkers(size_t npart) { std::vector *workerListAll; map cpu_loads; if (jasminegraph_profile == PROFILE_K8S) { - auto k8sInterface = new K8sInterface(); + std::unique_ptr k8sInterface(new K8sInterface()); if (k8sInterface->getJasmineGraphConfig("scale_on_adgr") != "true") { - delete k8sInterface; return K8sWorkerController::workerList; } - delete k8sInterface; workerListAll = &(K8sWorkerController::workerList); cpu_loads = scaleK8s(npart); } else { diff --git a/test-k8s.sh b/test-k8s.sh index b7405c3c2..cb925fd8c 100755 --- a/test-k8s.sh +++ b/test-k8s.sh @@ -59,10 +59,7 @@ build_and_run_on_k8s() { } clear_resources() { - kubectl delete deployments jasminegraph-master-deployment jasminegraph-worker1-deployment \ - jasminegraph-worker0-deployment - kubectl delete services jasminegraph-master-service jasminegraph-worker0-service jasminegraph-worker1-service - kubectl delete -f "${PROJECT_ROOT}/k8s/volumes.yaml" + ./start-k8s.sh clean } cd "$TEST_ROOT" @@ -72,6 +69,34 @@ cp -r env_init env cd "$PROJECT_ROOT" build_and_run_on_k8s +# Wait till all pods are running +cur_timestamp="$(date +%s)" +end_timestamp="$((cur_timestamp + TIMEOUT_SECONDS))" +while true; do + if [ "$(date +%s)" -gt "$end_timestamp" ]; then + echo "Pods are not running" + echo "Build log:" + cat "$BUILD_LOG" + echo "Run log:" + cat "$RUN_LOG" + force_remove "${TEST_ROOT}/env" + clear_resources + exit 1 + fi + + set +e + pods_status="$(kubectl get pods | grep -v 'STATUS' | grep -v 'Running')" + set -e + if [ -z "$pods_status" ]; then + echo "All pods are running" + break + fi + + echo "Waiting for pods to be running" + echo "$pods_status" + sleep 5 +done + # Wait till JasmineGraph server start listening cur_timestamp="$(date +%s)" end_timestamp="$((cur_timestamp + TIMEOUT_SECONDS))" @@ -92,9 +117,10 @@ while ! nc -zvn -w 1 "$masterIP" 7777 &>/dev/null; do sleep .5 done -echo +echo '-------------------- pods -------------------------------------' kubectl get pods -o wide echo +echo '------------------ services -----------------------------------' kubectl get services -o wide echo From 508bfaddf9705b946be72ed72029094f27a14ea3 Mon Sep 17 00:00:00 2001 From: Thevindu Wijesekera <91218789+thevindu-w@users.noreply.github.com> Date: Wed, 10 Apr 2024 18:37:13 +0530 Subject: [PATCH 134/143] Update conf/hosts.txt Co-authored-by: Miyuru Dayarathna --- conf/hosts.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/hosts.txt b/conf/hosts.txt index 51d016717..1f229bae7 100644 --- a/conf/hosts.txt +++ b/conf/hosts.txt @@ -1,4 +1,4 @@ -#This is the hosts file for Jasminegraph. All the host names on which Jasminegraph workers should get started needs +#This is the hosts file for JasmineGraph. All the host names on which JasmineGraph workers should get started needs #to be listed here localhost From 639a3ce7e102fb5de82924d394d47d10885e7d4e Mon Sep 17 00:00:00 2001 From: Thevindu Wijesekera <91218789+thevindu-w@users.noreply.github.com> Date: Thu, 11 Apr 2024 00:06:32 +0530 Subject: [PATCH 135/143] Update src/frontend/JasmineGraphFrontEnd.cpp Co-authored-by: Miyuru Dayarathna --- src/frontend/JasmineGraphFrontEnd.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index 06b6c35ce..3cb70e807 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -135,7 +135,7 @@ void *frontendservicesesion(void *dummyPt) { int failCnt = 0; while (!loop_exit) { if (currentFESession > Conts::MAX_FE_SESSIONS) { - if (!Utils::send_str_wrapper(connFd, "Jasminegraph Server is Busy. Please try again later.")) { + if (!Utils::send_str_wrapper(connFd, "JasmineGraph server is busy. Please try again later.")) { frontend_logger.error("Error writing to socket"); } break; From 9a9f60c16232dfb6645e146924a6059f123d77c2 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Thu, 11 Apr 2024 00:59:28 +0530 Subject: [PATCH 136/143] Resolve issues --- CMakeLists.txt | 2 - Dockerfile | 1 + conf/hosts.txt | 14 --- src/metadb/ddl.sql => ddl/metadb.sql | 0 src/performancedb/ddl.sql => ddl/perfdb.sql | 0 .../ddl.sql => ddl/streamingdb.sql | 0 main.cpp | 4 +- minimal/Dockerfile.fs | 9 +- src/frontend/JasmineGraphFrontEnd.h | 1 - src/metadb/SQLiteDBInterface.cpp | 2 +- src/performance/metrics/PerformanceUtil.h | 1 - .../PerformanceSQLiteDBInterface.cpp | 2 +- .../StreamingSQLiteDBInterface.cpp | 2 +- src/util/PlacesToNodeMapper.cpp | 104 ------------------ src/util/PlacesToNodeMapper.h | 32 ------ src/util/Utils.cpp | 2 +- 16 files changed, 8 insertions(+), 168 deletions(-) delete mode 100644 conf/hosts.txt rename src/metadb/ddl.sql => ddl/metadb.sql (100%) rename src/performancedb/ddl.sql => ddl/perfdb.sql (100%) rename src/streamingdb/ddl.sql => ddl/streamingdb.sql (100%) delete mode 100644 src/util/PlacesToNodeMapper.cpp delete mode 100644 src/util/PlacesToNodeMapper.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 6075cba94..3fb89fed3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,7 +46,6 @@ set(HEADERS globals.h src/server/JasmineGraphInstanceService.h src/server/JasmineGraphServer.h src/util/Conts.h - src/util/PlacesToNodeMapper.h src/util/Utils.h src/util/dbutil/attributestore_generated.h src/util/dbutil/edgestore_generated.h @@ -114,7 +113,6 @@ set(SOURCES src/backend/JasmineGraphBackend.cpp src/server/JasmineGraphInstanceService.cpp src/server/JasmineGraphServer.cpp src/util/Conts.cpp - src/util/PlacesToNodeMapper.cpp src/util/Utils.cpp src/util/kafka/KafkaCC.cpp src/util/kafka/StreamHandler.cpp diff --git a/Dockerfile b/Dockerfile index da91f7ca1..5d29c35e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ COPY ./run-docker.sh ./run-docker.sh COPY ./src_python ./src_python COPY ./conf ./conf COPY ./k8s ./k8s +COPY ./ddl ./ddl ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"] CMD ["bash"] diff --git a/conf/hosts.txt b/conf/hosts.txt deleted file mode 100644 index 1f229bae7..000000000 --- a/conf/hosts.txt +++ /dev/null @@ -1,14 +0,0 @@ -#This is the hosts file for JasmineGraph. All the host names on which JasmineGraph workers should get started needs -#to be listed here -localhost - -# TODO: Replace this -192.168.8.150 -192.168.43.135 -10.10.15.244 - -#anuradhak@10.8.100.245 -#192.168.1.8 -#192.168.1.7 -#ubuntu@18.219.24.222 -#anuradha@10.10.28.181 diff --git a/src/metadb/ddl.sql b/ddl/metadb.sql similarity index 100% rename from src/metadb/ddl.sql rename to ddl/metadb.sql diff --git a/src/performancedb/ddl.sql b/ddl/perfdb.sql similarity index 100% rename from src/performancedb/ddl.sql rename to ddl/perfdb.sql diff --git a/src/streamingdb/ddl.sql b/ddl/streamingdb.sql similarity index 100% rename from src/streamingdb/ddl.sql rename to ddl/streamingdb.sql diff --git a/main.cpp b/main.cpp index 3846a7f0d..d1fc55e09 100644 --- a/main.cpp +++ b/main.cpp @@ -54,9 +54,7 @@ int main(int argc, char *argv[]) { std::string JASMINEGRAPH_HOME = Utils::getJasmineGraphHome(); jasminegraph_profile = strcmp(argv[1], "docker") == 0 ? PROFILE_DOCKER : PROFILE_K8S; std::string enableNmon = "false"; - - main_logger.log("Using JASMINE_GRAPH_HOME", "info"); - std::cout << JASMINEGRAPH_HOME << std::endl; + main_logger.info("Using JASMINE_GRAPH_HOME=" + JASMINEGRAPH_HOME); StatisticCollector::init(); thread schedulerThread(SchedulerService::startScheduler); diff --git a/minimal/Dockerfile.fs b/minimal/Dockerfile.fs index 7db405387..4849d226e 100644 --- a/minimal/Dockerfile.fs +++ b/minimal/Dockerfile.fs @@ -7,15 +7,10 @@ RUN mkdir -p /home/ubuntu/software/jasminegraph/conf RUN mkdir -p /home/ubuntu/software/jasminegraph/k8s RUN mkdir -p /home/ubuntu/software/jasminegraph/metadb RUN mkdir -p /home/ubuntu/software/jasminegraph/performancedb -RUN mkdir -p /home/ubuntu/software/jasminegraph/src/metadb -RUN mkdir -p /home/ubuntu/software/jasminegraph/src/performancedb -RUN mkdir -p /home/ubuntu/software/jasminegraph/src/streamingdb +RUN mkdir -p /home/ubuntu/software/jasminegraph/ddl COPY --chmod=755 files.tmp/JasmineGraph /home/ubuntu/software/jasminegraph/ COPY --chmod=755 minimal/run-docker.sh /home/ubuntu/software/jasminegraph/ COPY --chmod=644 conf/jasminegraph-server.properties /home/ubuntu/software/jasminegraph/conf/ -COPY --chmod=644 conf/hosts.txt /home/ubuntu/software/jasminegraph/conf/ COPY --chmod=644 k8s/*.yaml /home/ubuntu/software/jasminegraph/k8s/ -COPY --chmod=644 src/metadb/ddl.sql /home/ubuntu/software/jasminegraph/src/metadb/ -COPY --chmod=644 src/performancedb/ddl.sql /home/ubuntu/software/jasminegraph/src/performancedb/ -COPY --chmod=644 src/streamingdb/ddl.sql /home/ubuntu/software/jasminegraph/src/streamingdb/ +COPY --chmod=644 ddl/*.sql /home/ubuntu/software/jasminegraph/ddl/ diff --git a/src/frontend/JasmineGraphFrontEnd.h b/src/frontend/JasmineGraphFrontEnd.h index 7d44d7bdd..ba542be75 100644 --- a/src/frontend/JasmineGraphFrontEnd.h +++ b/src/frontend/JasmineGraphFrontEnd.h @@ -38,7 +38,6 @@ limitations under the License. #include "../metadb/SQLiteDBInterface.h" #include "../performancedb/PerformanceSQLiteDBInterface.h" #include "../query/algorithms/triangles/Triangles.h" -#include "../util/PlacesToNodeMapper.h" #include "core/scheduler/JobScheduler.h" class JasmineGraphHashMapCentralStore; diff --git a/src/metadb/SQLiteDBInterface.cpp b/src/metadb/SQLiteDBInterface.cpp index af2a42724..4a815618d 100644 --- a/src/metadb/SQLiteDBInterface.cpp +++ b/src/metadb/SQLiteDBInterface.cpp @@ -31,7 +31,7 @@ SQLiteDBInterface::SQLiteDBInterface(string databaseLocation) { int SQLiteDBInterface::init() { if (!Utils::fileExists(this->databaseLocation.c_str())) { - if (Utils::createDatabaseFromDDL(this->databaseLocation.c_str(), ROOT_DIR "src/metadb/ddl.sql") != 0) { + if (Utils::createDatabaseFromDDL(this->databaseLocation.c_str(), ROOT_DIR "ddl/metadb.sql") != 0) { return -1; } } diff --git a/src/performance/metrics/PerformanceUtil.h b/src/performance/metrics/PerformanceUtil.h index cf59e4010..83cc038d3 100644 --- a/src/performance/metrics/PerformanceUtil.h +++ b/src/performance/metrics/PerformanceUtil.h @@ -29,7 +29,6 @@ limitations under the License. #include "../../performancedb/PerformanceSQLiteDBInterface.h" #include "../../server/JasmineGraphInstanceProtocol.h" #include "../../util/Conts.h" -#include "../../util/PlacesToNodeMapper.h" #include "../../util/Utils.h" #include "../../util/logger/Logger.h" #include "StatisticCollector.h" diff --git a/src/performancedb/PerformanceSQLiteDBInterface.cpp b/src/performancedb/PerformanceSQLiteDBInterface.cpp index f89b72c26..384a38895 100644 --- a/src/performancedb/PerformanceSQLiteDBInterface.cpp +++ b/src/performancedb/PerformanceSQLiteDBInterface.cpp @@ -23,7 +23,7 @@ Logger perfdb_logger; int PerformanceSQLiteDBInterface::init() { if (!Utils::fileExists(this->databaseLocation.c_str())) { - if (Utils::createDatabaseFromDDL(this->databaseLocation.c_str(), ROOT_DIR "src/performancedb/ddl.sql") != 0) { + if (Utils::createDatabaseFromDDL(this->databaseLocation.c_str(), ROOT_DIR "ddl/perfdb.sql") != 0) { return -1; } } diff --git a/src/streamingdb/StreamingSQLiteDBInterface.cpp b/src/streamingdb/StreamingSQLiteDBInterface.cpp index cf57f537b..66a9bfccf 100644 --- a/src/streamingdb/StreamingSQLiteDBInterface.cpp +++ b/src/streamingdb/StreamingSQLiteDBInterface.cpp @@ -23,7 +23,7 @@ Logger streamdb_logger; int StreamingSQLiteDBInterface::init() { if (!Utils::fileExists(this->databaseLocation.c_str())) { - if (Utils::createDatabaseFromDDL(this->databaseLocation.c_str(), ROOT_DIR "src/streamingdb/ddl.sql") != 0) { + if (Utils::createDatabaseFromDDL(this->databaseLocation.c_str(), ROOT_DIR "ddl/streamingdb.sql") != 0) { streamdb_logger.error("Cannot create database: " + databaseLocation); return -1; } diff --git a/src/util/PlacesToNodeMapper.cpp b/src/util/PlacesToNodeMapper.cpp deleted file mode 100644 index 554f99258..000000000 --- a/src/util/PlacesToNodeMapper.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/** -Copyright 2018 JasmineGraph Team -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - */ - -#include "PlacesToNodeMapper.h" - -#include "logger/Logger.h" - -using namespace std; -Logger node_logger; - -std::string PlacesToNodeMapper::getHost(long placeId) { - std::vector hostList = Utils::getHostListFromProperties(); - std::string& host = hostList.at(placeId); - return host; -} - -std::vector PlacesToNodeMapper::getInstancePortsList(long placeId) { - int numberOfWorkersPerHost; - int numberOfWorkers = 0; - int hostListModeNWorkers; - std::vector portList; - std::vector hostList = Utils::getHostListFromProperties(); - std::string nWorkers = Utils::getJasmineGraphProperty("org.jasminegraph.server.nworkers"); - int workerPort = Conts::JASMINEGRAPH_INSTANCE_PORT; - if (Utils::is_number(nWorkers)) { - numberOfWorkers = atoi(nWorkers.c_str()); - } else { - node_logger.log("Number of Workers specified in the properties is not an integer value.", "error"); - return portList; - } - - if (numberOfWorkers > 0 && hostList.size() > 0) { - numberOfWorkersPerHost = numberOfWorkers / hostList.size(); - hostListModeNWorkers = numberOfWorkers % hostList.size(); - } - - if (placeId > 0 && (placeId + 1) <= hostListModeNWorkers) { - workerPort = workerPort + (placeId * numberOfWorkersPerHost * 2) + placeId * 2 + 2; - } else if (placeId > 0 && (placeId + 1) > hostListModeNWorkers) { - workerPort = workerPort + (placeId * numberOfWorkersPerHost * 2) + hostListModeNWorkers * 2 + 2; - } - - for (int i = 0; i < numberOfWorkersPerHost; i++) { - workerPort = Conts::JASMINEGRAPH_INSTANCE_PORT + i * 2; - portList.push_back(workerPort); - } - - if ((placeId + 1) <= hostListModeNWorkers) { - workerPort = workerPort + 2; - portList.push_back(workerPort); - } - - return portList; -} - -std::vector PlacesToNodeMapper::getFileTransferServicePort(long placeId) { - int numberOfWorkersPerHost; - int numberOfWorkers = 0; - int hostListModeNWorkers; - std::vector portList; - std::vector hostList = Utils::getHostListFromProperties(); - std::string nWorkers = Utils::getJasmineGraphProperty("org.jasminegraph.server.nworkers"); - int workerPort = Conts::JASMINEGRAPH_INSTANCE_PORT; - int workerDataPort = Conts::JASMINEGRAPH_INSTANCE_DATA_PORT; - if (Utils::is_number(nWorkers)) { - numberOfWorkers = atoi(nWorkers.c_str()); - } else { - node_logger.log("Number of Workers is not specified", "error"); - numberOfWorkers = 0; - } - - if (numberOfWorkers > 0 && hostList.size() > 0) { - numberOfWorkersPerHost = numberOfWorkers / hostList.size(); - hostListModeNWorkers = numberOfWorkers % hostList.size(); - } - - if (placeId > 0 && (placeId + 1) <= hostListModeNWorkers) { - workerDataPort = workerDataPort + (placeId * numberOfWorkersPerHost * 2) + placeId * 2 + 2; - } else if (placeId > 0 && (placeId + 1) > hostListModeNWorkers) { - workerDataPort = workerDataPort + (placeId * numberOfWorkersPerHost * 2) + hostListModeNWorkers * 2 + 2; - } - - for (int i = 0; i < numberOfWorkersPerHost; i++) { - workerDataPort = workerDataPort + i * 2; - portList.push_back(workerDataPort); - } - - if ((placeId + 1) <= hostListModeNWorkers) { - workerDataPort = workerDataPort + 2; - portList.push_back(workerDataPort); - } - - return portList; -} diff --git a/src/util/PlacesToNodeMapper.h b/src/util/PlacesToNodeMapper.h deleted file mode 100644 index f51bf887c..000000000 --- a/src/util/PlacesToNodeMapper.h +++ /dev/null @@ -1,32 +0,0 @@ -/** -Copyright 2018 JasmineGraph Team -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - */ - -#ifndef JASMINEGRAPH_PLACESTONODEMAPPER_H -#define JASMINEGRAPH_PLACESTONODEMAPPER_H - -#include - -#include "Utils.h" - -class PlacesToNodeMapper { - private: - static int numberOfWorkers; - - public: - static std::string getHost(long placeId); - static std::vector getInstancePorts(long placeId); - static std::vector getInstancePortsList(long placeId); - static std::vector getFileTransferServicePort(long placeId); -}; - -#endif // JASMINEGRAPH_PLACESTONODEMAPPER_H diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index c9b5136d1..954d59ac7 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -58,13 +58,13 @@ std::vector Utils::getFileContent(std::string file) { std::string str; vector vec; + if (!in.is_open()) return vec; while (std::getline(in, str)) { // now we loop back and get the next line in 'str' if (str.length() > 0) { vec.push_back(str); } } - return vec; }; From 8521e587335e39b62bff4481f50b2fa43690ebc7 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Thu, 11 Apr 2024 01:07:25 +0530 Subject: [PATCH 137/143] Fix unit test failure --- tests/unit/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/Dockerfile b/tests/unit/Dockerfile index ba880a0bf..80597e4ea 100644 --- a/tests/unit/Dockerfile +++ b/tests/unit/Dockerfile @@ -20,6 +20,7 @@ COPY ./build.sh ./build.sh COPY ./CMakeLists.txt ./CMakeLists.txt COPY ./src_python ./src_python COPY ./src ./src +COPY ./ddl ./ddl COPY ./main.h ./main.h COPY ./main.cpp ./main.cpp COPY ./globals.h ./globals.h From 084b15e5e39476fd6454d28b201675b11e6c6bc6 Mon Sep 17 00:00:00 2001 From: Thevindu Wijesekera <91218789+thevindu-w@users.noreply.github.com> Date: Fri, 12 Apr 2024 16:54:22 +0530 Subject: [PATCH 138/143] Update src/backend/JasmineGraphBackend.cpp Co-authored-by: Miyuru Dayarathna --- src/backend/JasmineGraphBackend.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/JasmineGraphBackend.cpp b/src/backend/JasmineGraphBackend.cpp index 64e891bf6..19e1e3c08 100644 --- a/src/backend/JasmineGraphBackend.cpp +++ b/src/backend/JasmineGraphBackend.cpp @@ -85,7 +85,7 @@ void *backendservicesesion(void *dummyPt) { break; } else { - backend_logger.log("Message format not recognized", "error"); + backend_logger.error("Message format not recognized"); sleep(1); } } From 70381e44c48030a036d9a1d8b6bb3986f2446d65 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 12 Apr 2024 17:32:31 +0530 Subject: [PATCH 139/143] Update with changes Co-authored-by: Miyuru Dayarathna --- main.cpp | 11 ++--- src/backend/JasmineGraphBackend.cpp | 45 +++++++------------ src/frontend/JasmineGraphFrontEnd.cpp | 24 +++++----- .../core/executor/impl/PageRankExecutor.cpp | 25 +++++------ .../executor/impl/TriangleCountExecutor.cpp | 12 ++--- src/frontend/core/scheduler/JobScheduler.cpp | 7 ++- src/nativestore/DataPublisher.cpp | 2 +- src/nativestore/NodeManager.cpp | 2 +- src/nativestore/PropertyLink.cpp | 2 +- .../PerformanceSQLiteDBInterface.cpp | 7 ++- src/query/algorithms/triangles/Triangles.cpp | 4 +- src/server/JasmineGraphInstance.cpp | 2 +- ...asmineGraphInstanceFileTransferService.cpp | 6 +-- src/server/JasmineGraphInstanceService.cpp | 2 +- src/util/Utils.cpp | 6 +-- src/util/scheduler/SchedulerService.cpp | 2 +- 16 files changed, 68 insertions(+), 91 deletions(-) diff --git a/main.cpp b/main.cpp index d1fc55e09..5cdd468b1 100644 --- a/main.cpp +++ b/main.cpp @@ -34,18 +34,15 @@ Logger main_logger; int jasminegraph_profile = PROFILE_DOCKER; #endif -void fnExit3(void) { - delete (server); -} +void fnExit3(void) { delete (server); } int main(int argc, char *argv[]) { atexit(fnExit3); if (argc <= 1) { - main_logger.log( + main_logger.error( "\"Use argument 1 to start JasmineGraph in Master mode. Use 2 " - " to start as worker", - "error"); + " to start as worker"); return -1; } std::cout << argc << std::endl; @@ -78,7 +75,7 @@ int main(int argc, char *argv[]) { schedulerThread.join(); delete server; } else if (mode == Conts::JASMINEGRAPH_RUNTIME_PROFILE_WORKER) { - main_logger.log(to_string(argc), "info"); + main_logger.info(to_string(argc)); if (argc < 8) { main_logger.info( diff --git a/src/backend/JasmineGraphBackend.cpp b/src/backend/JasmineGraphBackend.cpp index 19e1e3c08..4e526b73d 100644 --- a/src/backend/JasmineGraphBackend.cpp +++ b/src/backend/JasmineGraphBackend.cpp @@ -27,17 +27,12 @@ void *backendservicesesion(void *dummyPt) { int connFd = sessionargs->connFd; SQLiteDBInterface *sqLiteDbInterface = sessionargs->sqlite; delete sessionargs; - backend_logger.log("Thread No: " + to_string(pthread_self()), "info"); - char data[301]; - bzero(data, 301); + backend_logger.info("Thread No: " + to_string(pthread_self())); + char data[BACKEND_DATA_LENGTH + 1]; bool loop = false; while (!loop) { - bzero(data, 301); - read(connFd, data, 300); - - string line(data); - backend_logger.log("Command received: " + line, "info"); - line = Utils::trim_copy(line); + string line = Utils::read_str_trim_wrapper(connFd, data, BACKEND_DATA_LENGTH); + backend_logger.info("Command received: " + line); if (line.compare(EXIT_BACKEND) == 0) { write(connFd, EXIT_ACK.c_str(), EXIT_ACK.size()); @@ -47,32 +42,22 @@ void *backendservicesesion(void *dummyPt) { write(connFd, HANDSHAKE_OK.c_str(), HANDSHAKE_OK.size()); write(connFd, "\r\n", 2); - char host[301]; - bzero(host, 301); - read(connFd, host, 300); - string hostname(host); - hostname = Utils::trim_copy(hostname); + string hostname = Utils::read_str_trim_wrapper(connFd, data, BACKEND_DATA_LENGTH); write(connFd, HOST_OK.c_str(), HOST_OK.size()); - backend_logger.log("Hostname of the worker: " + hostname, "info"); + backend_logger.info("Hostname of the worker: " + hostname); } else if (line.compare(ACKNOWLEGE_MASTER) == 0) { int result_wr = write(connFd, WORKER_INFO_SEND.c_str(), WORKER_INFO_SEND.size()); if (result_wr < 0) { - backend_logger.log("Error writing to socket", "error"); + backend_logger.error("Error writing to socket"); } result_wr = write(connFd, "\r\n", 2); if (result_wr < 0) { - backend_logger.log("Error writing to socket", "error"); + backend_logger.error("Error writing to socket"); } // We get the name and the path to graph as a pair separated by |. - char worker_info_data[301]; - bzero(worker_info_data, 301); - string name = ""; - - read(connFd, worker_info_data, 300); - - string worker_info(worker_info_data); + string worker_info = Utils::read_str_trim_wrapper(connFd, data, BACKEND_DATA_LENGTH); worker_info.erase(std::remove(worker_info.begin(), worker_info.end(), '\n'), worker_info.end()); worker_info.erase(std::remove(worker_info.begin(), worker_info.end(), '\r'), worker_info.end()); @@ -89,7 +74,7 @@ void *backendservicesesion(void *dummyPt) { sleep(1); } } - backend_logger.log("Closing thread " + to_string(pthread_self()) + " and connection", "info"); + backend_logger.info("Closing thread " + to_string(pthread_self()) + " and connection"); close(connFd); return NULL; } @@ -112,7 +97,7 @@ int JasmineGraphBackend::run() { listenFd = socket(AF_INET, SOCK_STREAM, 0); if (listenFd < 0) { - backend_logger.log("Cannot open socket", "error"); + backend_logger.error("Cannot open socket"); return 0; } @@ -129,7 +114,7 @@ int JasmineGraphBackend::run() { // bind socket if (bind(listenFd, (struct sockaddr *)&svrAdd, sizeof(svrAdd)) < 0) { - backend_logger.log("Cannot bind on port " + portNo, "error"); + backend_logger.error("Cannot bind on port " + portNo); return 0; } @@ -138,16 +123,16 @@ int JasmineGraphBackend::run() { len = sizeof(clntAdd); while (true) { - backend_logger.log("Backend Listening", "info"); + backend_logger.info("Backend Listening"); // this is where client connects. svr will hang in this mode until client conn int connFd = accept(listenFd, (struct sockaddr *)&clntAdd, &len); if (connFd < 0) { - backend_logger.log("Cannot accept connection", "error"); + backend_logger.error("Cannot accept connection"); continue; } - backend_logger.log("Connection successful", "info"); + backend_logger.info("Connection successful"); backendservicesessionargs *sessionargs = new backendservicesessionargs; sessionargs->sqlite = this->sqlite; sessionargs->connFd = connFd; diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index 3cb70e807..af25daa90 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -55,7 +55,7 @@ using namespace std::chrono; std::atomic highPriorityTaskCount; static int connFd; -static int currentFESession; +static volatile int currentFESession; static bool canCalibrate = true; Logger frontend_logger; std::set processData; @@ -98,7 +98,6 @@ static vector getWorkerClients(SQLiteDBInterface *sqlite) { for (int i = 0; i < workerList.size(); i++) { Utils::worker currentWorker = workerList.at(i); string workerHost = currentWorker.hostname; - string workerID = currentWorker.workerID; int workerPort = atoi(string(currentWorker.port).c_str()); DataPublisher *workerClient = new DataPublisher(workerPort, workerHost); workerClients.push_back(workerClient); @@ -114,6 +113,15 @@ void *frontendservicesesion(void *dummyPt) { PerformanceSQLiteDBInterface *perfSqlite = sessionargs->perfSqlite; JobScheduler *jobScheduler = sessionargs->jobScheduler; delete sessionargs; + if (currentFESession++ > Conts::MAX_FE_SESSIONS) { + if (!Utils::send_str_wrapper(connFd, "JasmineGraph server is busy. Please try again later.")) { + frontend_logger.error("Error writing to socket"); + } + close(connFd); + currentFESession--; + return NULL; + } + frontend_logger.info("Thread No: " + to_string(pthread_self())); frontend_logger.info("Master IP: " + masterIP); char data[FRONTEND_DATA_LENGTH + 1]; @@ -134,13 +142,6 @@ void *frontendservicesesion(void *dummyPt) { bool loop_exit = false; int failCnt = 0; while (!loop_exit) { - if (currentFESession > Conts::MAX_FE_SESSIONS) { - if (!Utils::send_str_wrapper(connFd, "JasmineGraph server is busy. Please try again later.")) { - frontend_logger.error("Error writing to socket"); - } - break; - } - string line = Utils::read_str_wrapper(connFd, data, FRONTEND_DATA_LENGTH, true); if (line.empty()) { failCnt++; @@ -958,10 +959,9 @@ static void add_graph_cust_command(std::string masterIP, int connFd, SQLiteDBInt string reformattedFilePath = partitioner.reformatDataSet(edgeListPath, newGraphID); partitioner.loadDataSet(reformattedFilePath, newGraphID); partitioner.constructMetisFormat(Conts::GRAPH_TYPE_NORMAL_REFORMATTED); - fullFileList = partitioner.partitioneWithGPMetis(""); - } else { - fullFileList = partitioner.partitioneWithGPMetis(""); } + fullFileList = partitioner.partitioneWithGPMetis(""); + // Graph type should be changed to identify graphs with attributes // because this graph type has additional attribute files to be uploaded JasmineGraphServer *server = JasmineGraphServer::getInstance(); diff --git a/src/frontend/core/executor/impl/PageRankExecutor.cpp b/src/frontend/core/executor/impl/PageRankExecutor.cpp index 1cef28222..3e1524d14 100644 --- a/src/frontend/core/executor/impl/PageRankExecutor.cpp +++ b/src/frontend/core/executor/impl/PageRankExecutor.cpp @@ -71,8 +71,7 @@ void PageRankExecutor::execute() { processStatusMutex.unlock(); } - pageRank_logger.log("###PAGERANK-EXECUTOR### Started with graph ID : " + graphId + " Master IP : " + masterIP, - "info"); + pageRank_logger.info("###PAGERANK-EXECUTOR### Started with graph ID : " + graphId + " Master IP : " + masterIP); int partitionCount = 0; std::vector> intermRes; @@ -87,16 +86,15 @@ void PageRankExecutor::execute() { int dataPort; std::string workerList; - std::map::iterator workerIter; - for (workerIter = graphPartitionedHosts.begin(); workerIter != graphPartitionedHosts.end(); workerIter++) { + for (auto workerIter = graphPartitionedHosts.begin(); workerIter != graphPartitionedHosts.end(); workerIter++) { JasmineGraphServer::workerPartitions workerPartition = workerIter->second; host = workerIter->first; port = workerPartition.port; dataPort = workerPartition.dataPort; - for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); - partitionit != workerPartition.partitionID.end(); partitionit++) { - std::string partition = *partitionit; + for (auto partitionIterator = workerPartition.partitionID.begin(); + partitionIterator != workerPartition.partitionID.end(); partitionIterator++) { + std::string partition = *partitionIterator; workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } } @@ -104,15 +102,15 @@ void PageRankExecutor::execute() { workerList.pop_back(); pageRank_logger.info("Worker list " + workerList); - for (workerIter = graphPartitionedHosts.begin(); workerIter != graphPartitionedHosts.end(); workerIter++) { + for (auto workerIter = graphPartitionedHosts.begin(); workerIter != graphPartitionedHosts.end(); workerIter++) { JasmineGraphServer::workerPartitions workerPartition = workerIter->second; host = workerIter->first; port = workerPartition.port; dataPort = workerPartition.dataPort; - for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); - partitionit != workerPartition.partitionID.end(); partitionit++) { - std::string partition = *partitionit; + for (auto partitionIterator = workerPartition.partitionID.begin(); + partitionIterator != workerPartition.partitionID.end(); partitionIterator++) { + std::string partition = *partitionIterator; intermRes.push_back(std::async(std::launch::async, PageRankExecutor::doPageRank, graphId, alpha, iterations, partition, host, port, dataPort, workerList)); } @@ -137,7 +135,7 @@ void PageRankExecutor::execute() { canCalibrate = false; } } else { - pageRank_logger.log("###PAGERANK-EXECUTOR### Inserting initial record for SLA ", "info"); + pageRank_logger.info("###PAGERANK-EXECUTOR### Inserting initial record for SLA "); Utils::updateSLAInformation(perfDB, graphId, partitionCount, 0, PAGE_RANK, Conts::SLA_CATEGORY::LATENCY); statResponse.push_back(std::async(std::launch::async, AbstractExecutor::collectPerformaceData, perfDB, graphId.c_str(), PAGE_RANK, Conts::SLA_CATEGORY::LATENCY, partitionCount, @@ -173,8 +171,7 @@ void PageRankExecutor::execute() { } processStatusMutex.lock(); - std::set::iterator processCompleteIterator; - for (processCompleteIterator = processData.begin(); processCompleteIterator != processData.end(); + for (auto processCompleteIterator = processData.begin(); processCompleteIterator != processData.end(); ++processCompleteIterator) { ProcessInfo processInformation = *processCompleteIterator; diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index dcac3356e..cf51c22f2 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -69,13 +69,13 @@ static void allocate(int p, string w, std::map &alloc, std::set &remain, std::map> &p_avail) { int p0 = *remain.begin(); - size_t m = 1000000000; + size_t minimum = 1000000000; for (auto it = remain.begin(); it != remain.end(); it++) { - int p = *it; - auto &ws = p_avail[p]; - if (ws.size() > 0 && ws.size() < m) { - m = ws.size(); - p0 = p; + int partition = *it; + auto &workers = p_avail[partition]; + if (workers.size() > 0 && workers.size() < minimum) { + minimum = workers.size(); + p0 = partition; } } return p0; diff --git a/src/frontend/core/scheduler/JobScheduler.cpp b/src/frontend/core/scheduler/JobScheduler.cpp index 23a91fc02..cfbe21faa 100644 --- a/src/frontend/core/scheduler/JobScheduler.cpp +++ b/src/frontend/core/scheduler/JobScheduler.cpp @@ -38,7 +38,7 @@ void *startScheduler(void *dummyPt) { PerformanceUtil::init(); while (true) { if (jobQueue.size() > 0) { - jobScheduler_Logger.log("##JOB SCHEDULER## Jobs Available for Scheduling", "info"); + jobScheduler_Logger.info("##JOB SCHEDULER## Jobs Available for Scheduling"); std::vector pendingHPJobList; std::vector highPriorityGraphList; @@ -57,9 +57,8 @@ void *startScheduler(void *dummyPt) { } if (pendingHPJobList.size() > 0) { - jobScheduler_Logger.log( - "##JOB SCHEDULER## High Priority Jobs in Queue: " + std::to_string(pendingHPJobList.size()), - "info"); + jobScheduler_Logger.info("##JOB SCHEDULER## High Priority Jobs in Queue: " + + std::to_string(pendingHPJobList.size())); std::string masterIP = pendingHPJobList[0].getMasterIP(); std::string jobType = pendingHPJobList[0].getJobType(); std::string category = pendingHPJobList[0].getParameter(Conts::PARAM_KEYS::CATEGORY); diff --git a/src/nativestore/DataPublisher.cpp b/src/nativestore/DataPublisher.cpp index 27ec77a87..2fafa2add 100644 --- a/src/nativestore/DataPublisher.cpp +++ b/src/nativestore/DataPublisher.cpp @@ -28,7 +28,7 @@ DataPublisher::DataPublisher(int worker_port, std::string worker_address) { server = gethostbyname(worker_address.c_str()); if (server == NULL) { - data_publisher_logger.log("ERROR, no host named " + worker_address, "error"); + data_publisher_logger.error("ERROR, no host named " + worker_address); pthread_exit(NULL); } diff --git a/src/nativestore/NodeManager.cpp b/src/nativestore/NodeManager.cpp index f2565007a..ad404e525 100644 --- a/src/nativestore/NodeManager.cpp +++ b/src/nativestore/NodeManager.cpp @@ -83,7 +83,7 @@ NodeManager::NodeManager(GraphConfig gConfig) { // std::ios::binary); // TODO (tmkasun): set PropertyLink nextPropertyIndex after validating by modulus check from file number of bytes - node_manager_logger.log("NodesDB, PropertiesDB, and RelationsDB files opened (or created) successfully.", "info"); + node_manager_logger.info("NodesDB, PropertiesDB, and RelationsDB files opened (or created) successfully."); // unsigned int nextAddress; // unsigned int propertyBlockAddress = 0; // PropertyLink::propertiesDB->seekg(propertyBlockAddress * PropertyLink::PROPERTY_BLOCK_SIZE); diff --git a/src/nativestore/PropertyLink.cpp b/src/nativestore/PropertyLink.cpp index 4c450b5f3..1972b9793 100644 --- a/src/nativestore/PropertyLink.cpp +++ b/src/nativestore/PropertyLink.cpp @@ -126,7 +126,7 @@ unsigned int PropertyLink::insert(std::string name, const char* value) { property_link_logger.warn("Property key/name already exist key = " + std::string(name)); return this->blockAddress; } else if (this->nextPropAddress) { // Traverse to the edge/end of the link list - property_link_logger.log("Next Called", "info"); + property_link_logger.info("Next Called"); return this->next()->insert(name, value); } else { // No next link means end of the link, Now add the new link // property_link_logger.debug("Next prop index = " + std::to_string(PropertyLink::nextPropertyIndex)); diff --git a/src/performancedb/PerformanceSQLiteDBInterface.cpp b/src/performancedb/PerformanceSQLiteDBInterface.cpp index 384a38895..a61aa6f19 100644 --- a/src/performancedb/PerformanceSQLiteDBInterface.cpp +++ b/src/performancedb/PerformanceSQLiteDBInterface.cpp @@ -28,13 +28,12 @@ int PerformanceSQLiteDBInterface::init() { } } - int rc = - sqlite3_open(this->databaseLocation.c_str(), &database); + int rc = sqlite3_open(this->databaseLocation.c_str(), &database); if (rc) { - perfdb_logger.log("Cannot open database: " + string(sqlite3_errmsg(database)), "error"); + perfdb_logger.error("Cannot open database: " + string(sqlite3_errmsg(database))); return -1; } - perfdb_logger.log("Database opened successfully", "info"); + perfdb_logger.info("Database opened successfully"); return 0; } diff --git a/src/query/algorithms/triangles/Triangles.cpp b/src/query/algorithms/triangles/Triangles.cpp index c3ff7d6c4..00b08dbd8 100644 --- a/src/query/algorithms/triangles/Triangles.cpp +++ b/src/query/algorithms/triangles/Triangles.cpp @@ -32,7 +32,7 @@ long Triangles::run(JasmineGraphHashMapLocalStore &graphDB, JasmineGraphHashMapC long Triangles::run(JasmineGraphHashMapLocalStore &graphDB, JasmineGraphHashMapCentralStore ¢ralStore, JasmineGraphHashMapDuplicateCentralStore &duplicateCentralStore, std::string graphId, std::string partitionId, int threadPriority) { - triangle_logger.log("###TRIANGLE### Triangle Counting: Started", "info"); + triangle_logger.info("###TRIANGLE### Triangle Counting: Started"); map> localSubGraphMap = graphDB.getUnderlyingHashMap(); map> centralDBSubGraphMap = centralStore.getUnderlyingHashMap(); map> duplicateCentralDBSubGraphMap = duplicateCentralStore.getUnderlyingHashMap(); @@ -77,7 +77,7 @@ long Triangles::run(JasmineGraphHashMapLocalStore &graphDB, JasmineGraphHashMapC auto mergeDur = mergeEnd - mergeBbegin; auto mergeMsDuration = std::chrono::duration_cast(mergeDur).count(); - triangle_logger.log(" Merge time Taken: " + std::to_string(mergeMsDuration) + " milliseconds", "info"); + triangle_logger.info(" Merge time Taken: " + std::to_string(mergeMsDuration) + " milliseconds"); const TriangleResult &triangleResult = countTriangles(localSubGraphMap, degreeDistribution, false); return triangleResult.count; diff --git a/src/server/JasmineGraphInstance.cpp b/src/server/JasmineGraphInstance.cpp index dfea42b72..0c585eeef 100644 --- a/src/server/JasmineGraphInstance.cpp +++ b/src/server/JasmineGraphInstance.cpp @@ -89,7 +89,7 @@ void JasmineGraphInstance::startNmonAnalyzer(string enableNmon, int serverPort) } } if (!result.empty()) { - graphInstance_logger.log("Error in performance database backup process", "error"); + graphInstance_logger.error("Error in performance database backup process"); } pclose(input); diff --git a/src/server/JasmineGraphInstanceFileTransferService.cpp b/src/server/JasmineGraphInstanceFileTransferService.cpp index 10dd6abac..77ad885a5 100644 --- a/src/server/JasmineGraphInstanceFileTransferService.cpp +++ b/src/server/JasmineGraphInstanceFileTransferService.cpp @@ -85,7 +85,7 @@ void JasmineGraphInstanceFileTransferService::run(int dataPort) { } int connFd; listen(listenFd, 10); - file_service_logger.log("Worker FileTransfer Service listening on port " + to_string(dataPort), "info"); + file_service_logger.info("Worker FileTransfer Service listening on port " + to_string(dataPort)); len = sizeof(clntAdd); @@ -93,10 +93,10 @@ void JasmineGraphInstanceFileTransferService::run(int dataPort) { connFd = accept(listenFd, (struct sockaddr *)&clntAdd, &len); if (connFd < 0) { - file_service_logger.log("Cannot accept connection to port " + to_string(dataPort), "error"); + file_service_logger.info("Cannot accept connection to port " + to_string(dataPort)); continue; } - file_service_logger.log("Connection successful to port " + to_string(dataPort), "info"); + file_service_logger.info("Connection successful to port " + to_string(dataPort)); pid_t pid = fork(); if (pid == 0) { diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index d1de9d4bd..0c706f719 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -2570,7 +2570,7 @@ static void page_rank_command(int connFd, int serverPort, *loop_exit_p = true; return; } - instance_logger.log("Sent : " + JasmineGraphInstanceProtocol::OK, "info"); + instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); instance_logger.info("Finish : Calculate Local PageRank."); } diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 954d59ac7..8b7eb34c5 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -282,7 +282,7 @@ std::string Utils::getJasmineGraphHome() { jasminegraph_home = std::string(temp); } if (jasminegraph_home.empty()) { - util_logger.log("Returning empty value for " + Conts::JASMINEGRAPH_HOME, "warn"); + util_logger.warn("Returning empty value for " + Conts::JASMINEGRAPH_HOME); } return jasminegraph_home; } @@ -304,7 +304,7 @@ std::string Utils::getHomeDir() { * @param filePath */ int Utils::copyFile(const std::string sourceFilePath, const std::string destinationFilePath) { - util_logger.log("Starting file copy source: " + sourceFilePath + " destination: " + destinationFilePath, "info"); + util_logger.info("Starting file copy source: " + sourceFilePath + " destination: " + destinationFilePath); std::string command = "cp " + sourceFilePath + " " + destinationFilePath; int status = system(command.c_str()); if (status != 0) util_logger.error("Copy failed with exit code " + std::to_string(status)); @@ -495,7 +495,7 @@ void Utils::updateSLAInformation(PerformanceSQLiteDBInterface *perfSqlite, std:: perfSqlite->runInsert(insertQuery); } } else { - util_logger.log("Invalid SLA " + category + " for " + command + " command", "error"); + util_logger.error("Invalid SLA " + category + " for " + command + " command"); } } diff --git a/src/util/scheduler/SchedulerService.cpp b/src/util/scheduler/SchedulerService.cpp index 5860ba07b..70645d5fc 100644 --- a/src/util/scheduler/SchedulerService.cpp +++ b/src/util/scheduler/SchedulerService.cpp @@ -24,7 +24,7 @@ void SchedulerService::startScheduler() { if (schedulerEnabled == "true") { sleep(3); - schedulerservice_logger.log("#######SCHEDULER ENABLED#####", "info"); + schedulerservice_logger.info("#######SCHEDULER ENABLED#####"); startPerformanceScheduler(); } From ab028108e146c86340605c52ec36ecbb0ac64780 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 17 Apr 2024 18:12:02 +0530 Subject: [PATCH 140/143] Remove redundant logs --- src/frontend/JasmineGraphFrontEnd.cpp | 3 -- .../executor/impl/TriangleCountExecutor.cpp | 42 ++----------------- 2 files changed, 4 insertions(+), 41 deletions(-) diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index af25daa90..a398b25a7 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -122,10 +122,7 @@ void *frontendservicesesion(void *dummyPt) { return NULL; } - frontend_logger.info("Thread No: " + to_string(pthread_self())); - frontend_logger.info("Master IP: " + masterIP); char data[FRONTEND_DATA_LENGTH + 1]; - // Initiate Thread thread input_stream_handler; // Initiate kafka consumer parameters diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index cf51c22f2..fb00f14c8 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -105,7 +105,7 @@ static int alloc_plan(std::map &alloc, std::set &remain, std:: if (ws.empty()) return (int)remain.size(); sort(ws.begin(), ws.end(), [&loads](string &w1, string &w2) { return LOAD_PREFERENCE[loads[w1]] > LOAD_PREFERENCE[loads[w2]]; - }); // load=1 goes first and load=3 goes last + }); // load=1 goes first and load=3 goes last. The order is 1,0,2,3 for 4 cores. struct best_alloc { std::map alloc; std::set remain; @@ -249,11 +249,9 @@ static int alloc_net_plan(std::map &alloc, std::vector &parts, sort(wts.begin(), wts.end(), [&loads](string &w1, string &w2) { int l1 = loads[w1]; int l2 = loads[w2]; - // TODO: temporarily commented for scale up only - // if (l1 < 3 && l2 < 3) return LOAD_PREFERENCE[l1] > LOAD_PREFERENCE[loads[w2]]; if (l1 < 3 || l2 < 3) return l1 < l2; return l1 <= l2; - }); // load=1 goes first and load=3 goes last + }); // load=1 goes first and load=3 goes last. The order is 1,0,2,3 for 4 cores. const auto &ws = p_avail.find(p)->second; int minLoad = 100000000; for (auto it = loads.begin(); it != loads.end(); it++) { @@ -356,15 +354,12 @@ static void filter_partitions(std::map> &partitionMa } std::map alloc; - cout << "calling alloc_plan" << endl; int unallocated = alloc_plan(alloc, remain, p_avail, loads); - cout << "alloc_plan returned " << unallocated << endl; if (unallocated > 0) { - cout << "calling reallocate_parts" << endl; + triangleCount_logger.info(to_string(unallocated) + " partitions remaining after alloc_plan"); auto copying = reallocate_parts(alloc, remain, P_AVAIL); - cout << "reallocate_parts completed" << endl; scale_up(loads, workers, copying.size()); - cout << "scale_up completed" << endl; + triangleCount_logger.info("Scale up completed"); map net_loads; for (auto it = loads.begin(); it != loads.end(); it++) { @@ -378,13 +373,10 @@ static void filter_partitions(std::map> &partitionMa std::map> transfer; int net_load = alloc_net_plan(alloc, copying, transfer, net_loads, loads, P_AVAIL, 100000000); - cout << "alloc_net_plan completed with net_load=" << net_load << endl; for (auto it = transfer.begin(); it != transfer.end(); it++) { auto p = it->first; auto w_to = it->second.second; alloc[p] = w_to; - cout << "planned sending partition " << p << " from " << it->second.first << " to " << it->second.second - << endl; } if (!transfer.empty()) { @@ -396,7 +388,6 @@ static void filter_partitions(std::map> &partitionMa string port = workerData[i][1].second; string dport = workerData[i][2].second; dataPortMap[ip + ":" + port] = dport; - cout << "dataPortMap[" << ip + ":" + port << "] = " << dport << endl; } map workers_r; // "ip:port" => id for (auto it = workers.begin(); it != workers.end(); it++) { @@ -521,35 +512,11 @@ void TriangleCountExecutor::execute() { isCompositeAggregation = true; } - cout << "initial partitionMap = {" << endl; - for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { - cout << " " << it->first << ": ["; - auto &pts = it->second; - for (auto it2 = pts.begin(); it2 != pts.end(); it2++) { - cout << *it2 << ", "; - } - cout << "]" << endl; - } - cout << "}" << endl; - cout << endl; - std::unique_ptr k8sInterface(new K8sInterface()); if (jasminegraph_profile == PROFILE_K8S && k8sInterface->getJasmineGraphConfig("auto_scaling_enabled") == "true") { filter_partitions(partitionMap, sqlite, graphId); } - cout << "final partitionMap = {" << endl; - for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { - cout << " " << it->first << ": ["; - auto &pts = it->second; - for (auto it2 = pts.begin(); it2 != pts.end(); it2++) { - cout << *it2 << ", "; - } - cout << "]" << endl; - } - cout << "}" << endl; - cout << endl; - std::vector> fileCombinations; if (isCompositeAggregation) { std::string aggregatorFilePath = @@ -576,7 +543,6 @@ void TriangleCountExecutor::execute() { int partitionCount = 0; vector workerList = Utils::getWorkerList(sqlite); int workerListSize = workerList.size(); - cout << "workerListSize = " << workerListSize << endl; for (int i = 0; i < workerListSize; i++) { Utils::worker currentWorker = workerList.at(i); string host = currentWorker.hostname; From 280e3e7133bdd69148aade7b796e8fdf34f0f75c Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Fri, 19 Apr 2024 02:43:28 +0530 Subject: [PATCH 141/143] Fix issues Co-authored-by: Miyuru Dayarathna --- .../executor/impl/TriangleCountExecutor.cpp | 252 +++++------------- src/k8s/K8sWorkerController.cpp | 38 +-- src/nativestore/PropertyLink.cpp | 1 - src/performance/metrics/PerformanceUtil.cpp | 9 +- .../metrics/StatisticCollector.cpp | 32 +-- src/server/JasmineGraphServer.cpp | 11 +- src/util/Utils.cpp | 17 +- src/util/logger/Logger.cpp | 1 - 8 files changed, 108 insertions(+), 253 deletions(-) diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index fb00f14c8..2c182d54b 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -654,7 +654,7 @@ long TriangleCountExecutor::getTriangleCount( std::unordered_map>> *triangleTree_p, std::mutex *triangleTreeMutex_p) { int sockfd; - char data[301]; + char data[INSTANCE_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -664,7 +664,7 @@ long TriangleCountExecutor::getTriangleCount( sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; + triangleCount_logger.error("Cannot create socket"); return 0; } @@ -685,7 +685,7 @@ long TriangleCountExecutor::getTriangleCount( bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; + triangleCount_logger.error("ERROR connecting"); return 0; } @@ -697,7 +697,7 @@ long TriangleCountExecutor::getTriangleCount( } triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - string response = Utils::read_str_trim_wrapper(sockfd, data, 300); + string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); @@ -708,7 +708,7 @@ long TriangleCountExecutor::getTriangleCount( } triangleCount_logger.log("Sent : " + masterIP, "info"); - response = Utils::read_str_trim_wrapper(sockfd, data, 300); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -722,7 +722,7 @@ long TriangleCountExecutor::getTriangleCount( } triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::TRIANGLES, "info"); - response = Utils::read_str_trim_wrapper(sockfd, data, 300); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); result_wr = write(sockfd, std::to_string(graphId).c_str(), std::to_string(graphId).size()); @@ -732,7 +732,7 @@ long TriangleCountExecutor::getTriangleCount( } triangleCount_logger.log("Sent : Graph ID " + std::to_string(graphId), "info"); - response = Utils::read_str_trim_wrapper(sockfd, data, 300); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -745,7 +745,7 @@ long TriangleCountExecutor::getTriangleCount( triangleCount_logger.log("Sent : Partition ID " + std::to_string(partitionId), "info"); - response = Utils::read_str_trim_wrapper(sockfd, data, 300); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -757,7 +757,7 @@ long TriangleCountExecutor::getTriangleCount( } triangleCount_logger.log("Sent : Thread Priority " + std::to_string(threadPriority), "info"); - response = Utils::read_str_trim_wrapper(sockfd, data, 300); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); triangleCount_logger.log("Got response : |" + response + "|", "info"); triangleCount = atol(response.c_str()); } @@ -1091,7 +1091,7 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio std::string aggregatorPort, std::string masterIP, std::string fileType, std::string fileName) { int sockfd; - char data[301]; + char data[INSTANCE_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -1101,7 +1101,7 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; + triangleCount_logger.error("Cannot create socket"); return 0; } @@ -1116,25 +1116,19 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; + triangleCount_logger.error("ERROR connecting"); return 0; } - bzero(data, 301); int result_wr = write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); + string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); result_wr = write(sockfd, masterIP.c_str(), masterIP.size()); @@ -1142,12 +1136,9 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + masterIP, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -1160,11 +1151,7 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_TYPE) == 0) { result_wr = write(sockfd, fileType.c_str(), fileType.size()); @@ -1172,11 +1159,7 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (fileType.compare(JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_AGGREGATE) == 0) { if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { result_wr = write(sockfd, graphId.c_str(), graphId.size()); @@ -1185,11 +1168,7 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { result_wr = write(sockfd, partitionId.c_str(), partitionId.size()); @@ -1197,10 +1176,7 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - isFileAccessible = Utils::trim_copy(response); + isFileAccessible = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } } } else if (fileType.compare(JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_COMPOSITE) == 0) { @@ -1213,10 +1189,7 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - isFileAccessible = Utils::trim_copy(response); + isFileAccessible = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } } } @@ -1231,7 +1204,7 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st std::string aggregatorDataPort, std::string fileName, std::string masterIP) { int sockfd; - char data[301]; + char data[INSTANCE_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -1245,7 +1218,7 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; + triangleCount_logger.error("Cannot create socket"); return 0; } @@ -1264,25 +1237,19 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; + triangleCount_logger.error("ERROR connecting"); return 0; } - bzero(data, 301); int result_wr = write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); + string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); result_wr = write(sockfd, masterIP.c_str(), masterIP.size()); @@ -1290,12 +1257,9 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + masterIP, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -1307,14 +1271,10 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::SEND_COMPOSITE_CENTRALSTORE_TO_AGGREGATOR, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_NAME) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_NAME, "info"); result_wr = write(sockfd, fileName.c_str(), fileName.size()); @@ -1322,14 +1282,9 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : File Name " + fileName, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_LEN) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_LEN, "info"); result_wr = write(sockfd, fileLength.c_str(), fileLength.size()); @@ -1337,14 +1292,9 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : File Length: " + fileLength, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_CONT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_CONT, "info"); triangleCount_logger.log("Going to send file through service", "info"); @@ -1367,7 +1317,7 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::FILE_RECV_CHK, "info"); triangleCount_logger.log("Checking if file is received", "info"); - response = Utils::read_str_trim_wrapper(sockfd, data, 300); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::FILE_RECV_WAIT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::FILE_RECV_WAIT, "info"); triangleCount_logger.log("Checking file status : " + to_string(count), "info"); @@ -1391,12 +1341,9 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT, "info"); sleep(1); @@ -1420,7 +1367,7 @@ std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( std::string aggregatorHostName, std::string aggregatorPort, std::string compositeCentralStoreFileList, std::string masterIP, std::string availableFileList, int threadPriority) { int sockfd; - char data[301]; + char data[INSTANCE_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -1429,7 +1376,7 @@ std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; + triangleCount_logger.error("Cannot create socket"); return {}; } @@ -1444,25 +1391,19 @@ std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; + triangleCount_logger.error("ERROR connecting"); return {}; } - bzero(data, 301); int result_wr = write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); + string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); result_wr = write(sockfd, masterIP.c_str(), masterIP.size()); @@ -1470,13 +1411,10 @@ std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + masterIP, "info"); triangleCount_logger.log("Port : " + aggregatorPort, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -1488,14 +1426,10 @@ std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::AGGREGATE_COMPOSITE_CENTRALSTORE_TRIANGLES, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); result_wr = write(sockfd, availableFileList.c_str(), availableFileList.size()); @@ -1506,10 +1440,7 @@ std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( triangleCount_logger.log("Sent : Available File List " + availableFileList, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -1532,18 +1463,13 @@ std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( std::string chunk = chunksVector.at(loopCount); write(sockfd, chunk.c_str(), chunk.size()); } else { - bzero(data, 301); - read(sockfd, data, 300); - string chunkStatus = (data); + string chunkStatus = Utils::read_str_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); std::string chunk = chunksVector.at(loopCount); write(sockfd, chunk.c_str(), chunk.size()); } } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -1556,10 +1482,7 @@ std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( triangleCount_logger.log("Sent : Thread Priority " + std::to_string(threadPriority), "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); string status = response.substr(response.size() - 5); std::basic_ostringstream resultStream; resultStream << response.substr(0, response.size() - 5); @@ -1569,10 +1492,7 @@ std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); status = response.substr(response.size() - 5); resultStream << response.substr(0, response.size() - 5); } @@ -1595,7 +1515,7 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr std::string aggregatorDataPort, int graphId, int partitionId, std::string masterIP) { int sockfd; - char data[301]; + char data[INSTANCE_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -1610,7 +1530,7 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; + triangleCount_logger.error("Cannot create socket"); return ""; } @@ -1625,7 +1545,7 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; + triangleCount_logger.error("ERROR connecting"); return ""; } @@ -1635,14 +1555,9 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); + string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); result_wr = write(sockfd, masterIP.c_str(), masterIP.size()); @@ -1650,12 +1565,9 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + masterIP, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -1667,13 +1579,9 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::SEND_CENTRALSTORE_TO_AGGREGATOR, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_NAME) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_NAME, "info"); result_wr = write(sockfd, fileName.c_str(), fileName.size()); @@ -1681,14 +1589,9 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : File Name " + fileName, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_LEN) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_LEN, "info"); result_wr = write(sockfd, fileLength.c_str(), fileLength.size()); @@ -1696,14 +1599,9 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : File Length: " + fileLength, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_CONT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_CONT, "info"); triangleCount_logger.log("Going to send file through service", "info"); @@ -1726,7 +1624,7 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::FILE_RECV_CHK, "info"); triangleCount_logger.log("Checking if file is received", "info"); - response = Utils::read_str_trim_wrapper(sockfd, data, 300); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::FILE_RECV_WAIT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::FILE_RECV_WAIT, "info"); triangleCount_logger.log("Checking file status : " + to_string(count), "info"); @@ -1750,12 +1648,9 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT, "info"); sleep(1); @@ -1780,7 +1675,7 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP std::string graphId, std::string masterIP, int threadPriority) { int sockfd; - char data[301]; + char data[INSTANCE_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -1789,7 +1684,7 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; + triangleCount_logger.error("Cannot create socket"); return 0; } @@ -1804,25 +1699,19 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; + triangleCount_logger.error("ERROR connecting"); return 0; } - bzero(data, 301); int result_wr = write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); + string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); result_wr = write(sockfd, masterIP.c_str(), masterIP.size()); @@ -1830,12 +1719,9 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + masterIP, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -1847,13 +1733,9 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::AGGREGATE_CENTRALSTORE_TRIANGLES, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); result_wr = write(sockfd, graphId.c_str(), graphId.size()); @@ -1861,13 +1743,9 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : Graph ID " + graphId, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -1877,13 +1755,9 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : Partition ID " + partitionId, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -1893,13 +1767,9 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : Partition ID List : " + partitionIdList, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -1912,7 +1782,7 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP triangleCount_logger.log("Sent : Thread Priority " + std::to_string(threadPriority), "info"); - response = Utils::read_str_trim_wrapper(sockfd, data, 300); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); string status = response.substr(response.size() - 5); std::basic_ostringstream resultStream; resultStream << response.substr(0, response.size() - 5); @@ -1923,7 +1793,7 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorP if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - response = Utils::read_str_trim_wrapper(sockfd, data, 300); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); status = response.substr(response.size() - 5); resultStream << response.substr(0, response.size() - 5); } diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 2cfa852dc..6f17b95a3 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -51,31 +51,34 @@ static K8sWorkerController *instance = nullptr; K8sWorkerController *K8sWorkerController::getInstance() { if (instance == nullptr) { + controller_logger.error("K8sWorkerController is not instantiated"); throw std::runtime_error("K8sWorkerController is not instantiated"); } return instance; } +static std::mutex instanceMutex; K8sWorkerController *K8sWorkerController::getInstance(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb) { - // TODO(thevindu-w): synchronize if (instance == nullptr) { - instance = new K8sWorkerController(masterIp, numberOfWorkers, metadb); - try { - instance->maxWorkers = stoi(instance->interface->getJasmineGraphConfig("max_worker_count")); - } catch (std::invalid_argument &e) { - controller_logger.error("Invalid max_worker_count value. Defaulted to 4"); - instance->maxWorkers = 4; - } + instanceMutex.lock(); + if (instance == nullptr) { // double-checking lock + instance = new K8sWorkerController(masterIp, numberOfWorkers, metadb); + try { + instance->maxWorkers = stoi(instance->interface->getJasmineGraphConfig("max_worker_count")); + } catch (std::invalid_argument &e) { + controller_logger.error("Invalid max_worker_count value. Defaulted to 4"); + instance->maxWorkers = 4; + } - // Delete all the workers from the database - metadb->runUpdate("DELETE FROM worker"); - int workersAttached = instance->attachExistingWorkers(); - if (numberOfWorkers - workersAttached > 0) { - instance->scaleUp(numberOfWorkers - workersAttached); + // Delete all the workers from the database + metadb->runUpdate("DELETE FROM worker"); + int workersAttached = instance->attachExistingWorkers(); + if (numberOfWorkers - workersAttached > 0) { + instance->scaleUp(numberOfWorkers - workersAttached); + } } - } else { - controller_logger.warn("Not initializing again"); + instanceMutex.unlock(); } return instance; } @@ -87,6 +90,7 @@ std::string K8sWorkerController::spawnWorker(int workerId) { if (volume != nullptr && volume->metadata != nullptr && volume->metadata->name != nullptr) { controller_logger.info("Worker " + std::to_string(workerId) + " persistent volume created successfully"); } else { + controller_logger.error("Worker " + std::to_string(workerId) + " persistent volume creation failed"); throw std::runtime_error("Worker " + std::to_string(workerId) + " persistent volume creation failed"); } @@ -94,6 +98,7 @@ std::string K8sWorkerController::spawnWorker(int workerId) { if (claim != nullptr && claim->metadata != nullptr && claim->metadata->name != nullptr) { controller_logger.info("Worker " + std::to_string(workerId) + " persistent volume claim created successfully"); } else { + controller_logger.error("Worker " + std::to_string(workerId) + " persistent volume claim creation failed"); throw std::runtime_error("Worker " + std::to_string(workerId) + " persistent volume claim creation failed"); } @@ -101,6 +106,7 @@ std::string K8sWorkerController::spawnWorker(int workerId) { if (service != nullptr && service->metadata != nullptr && service->metadata->name != nullptr) { controller_logger.info("Worker " + std::to_string(workerId) + " service created successfully"); } else { + controller_logger.error("Worker " + std::to_string(workerId) + " service creation failed"); throw std::runtime_error("Worker " + std::to_string(workerId) + " service creation failed"); } @@ -109,8 +115,8 @@ std::string K8sWorkerController::spawnWorker(int workerId) { v1_deployment_t *deployment = this->interface->createJasmineGraphWorkerDeployment(workerId, ip, this->masterIp); if (deployment != nullptr && deployment->metadata != nullptr && deployment->metadata->name != nullptr) { controller_logger.info("Worker " + std::to_string(workerId) + " deployment created successfully"); - } else { + controller_logger.error("Worker " + std::to_string(workerId) + " deployment creation failed"); throw std::runtime_error("Worker " + std::to_string(workerId) + " deployment creation failed"); } k8sSpawnMutex.unlock(); diff --git a/src/nativestore/PropertyLink.cpp b/src/nativestore/PropertyLink.cpp index 1972b9793..287bde6cf 100644 --- a/src/nativestore/PropertyLink.cpp +++ b/src/nativestore/PropertyLink.cpp @@ -126,7 +126,6 @@ unsigned int PropertyLink::insert(std::string name, const char* value) { property_link_logger.warn("Property key/name already exist key = " + std::string(name)); return this->blockAddress; } else if (this->nextPropAddress) { // Traverse to the edge/end of the link list - property_link_logger.info("Next Called"); return this->next()->insert(name, value); } else { // No next link means end of the link, Now add the new link // property_link_logger.debug("Next prop index = " + std::to_string(PropertyLink::nextPropertyIndex)); diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index c7c293478..fe17a8ec9 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -54,13 +54,10 @@ int PerformanceUtil::collectPerformanceStatistics() { double currentLoadAverage = StatisticCollector::getLoadAverage(); Utils::send_job("", "load_average", std::to_string(currentLoadAverage)); - /* long totalSwapSpace = StatisticCollector::getTotalSwapSpace(); Utils::send_job("", "total_swap_space", std::to_string(totalSwapSpace)); - */ // Per process - /* long memoryUsage = StatisticCollector::getMemoryUsageByProcess(); Utils::send_job("", "memory_usage", std::to_string(memoryUsage)); @@ -69,7 +66,7 @@ int PerformanceUtil::collectPerformanceStatistics() { int socketCount = StatisticCollector::getSocketCount(); Utils::send_job("", "socket_count", std::to_string(socketCount)); - */ + scheduler_logger.info("Pushed performance metrics"); return 0; } @@ -441,8 +438,6 @@ void PerformanceUtil::adjustAggregateLoadMap(std::map '9') p++; long long total = 0; @@ -190,7 +186,7 @@ static void getCpuCycles(long long *totalp, long long *idlep) { long long value = strtoll(p, &end_ptr, 10); p = end_ptr; if (value < 0) { - std::cerr << "Value is " << value << " for line " << line << std::endl; + stat_logger.error("Value is " + to_string(value) + " for line " + string(line)); } if (field == 4) { idle += value; diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 3485796f2..7f6f87515 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -94,11 +94,15 @@ JasmineGraphServer::JasmineGraphServer() { this->jobScheduler->init(); } +static std::mutex instanceMutex; JasmineGraphServer *JasmineGraphServer::getInstance() { static JasmineGraphServer *instance = nullptr; - // TODO(thevindu-w): synchronize if (instance == nullptr) { - instance = new JasmineGraphServer(); + instanceMutex.lock(); + if (instance == nullptr) { // double-checking lock + instance = new JasmineGraphServer(); + } + instanceMutex.unlock(); } return instance; } @@ -844,7 +848,6 @@ static std::vector getWorkers(size_t npart) { } } size_t len = workerListAll->size(); - std::cout << "workerListAll len = " << len << std::endl; std::vector workerList; for (int i = 0; i < npart; i++) { JasmineGraphServer::worker worker_min; @@ -861,7 +864,6 @@ static std::vector getWorkers(size_t npart) { string workerHostPort = worker_min.hostname + ":" + to_string(worker_min.port); cpu_loads[workerHostPort] += 0.25; // 0.25 = 1/nproc workerList.push_back(worker_min); - std::cout << "worker = " << worker_min.hostname << ":" << worker_min.port << std::endl; } return workerList; } @@ -1532,7 +1534,6 @@ static void degreeDistributionCommon(std::string graphID, std::string command) { } Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); - server_logger.info("Sent: " + workerList); } } } diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 8b7eb34c5..7cddf831d 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -746,16 +746,6 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, curl = curl_easy_init(); if (curl) { std::string hostPGAddr; - /* - const char *envWorkerID = getenv("WORKER_ID"); - std::string workerID; - if (envWorkerID) { - workerID = std::string(getenv("WORKER_ID")); - } else { - workerID = "-1"; - } - */ - const char *hostAddress = getenv("HOST_NAME"); const char *port = getenv("PORT"); @@ -765,7 +755,6 @@ std::string Utils::send_job(std::string job_group_name, std::string metric_name, } else { uniqueWorkerID = "Master"; } - // hostPGAddr = pushGatewayJobAddr + job_group_name + "_" + workerID; hostPGAddr = pushGatewayJobAddr + uniqueWorkerID; curl_easy_setopt(curl, CURLOPT_URL, hostPGAddr.c_str()); @@ -823,9 +812,9 @@ std::map Utils::getMetricMap(std::string metricName) { res = curl_easy_perform(curl); if (res != CURLE_OK) { - std::cerr << "cURL failed: " << curl_easy_strerror(res) << std::endl; + util_logger.error("cURL failed: " + string(curl_easy_strerror(res))); } else { - std::cout << response_cpu_usages << std::endl; + util_logger.info(response_cpu_usages); Json::Value root; Json::Reader reader; reader.parse(response_cpu_usages, root); @@ -925,7 +914,7 @@ bool Utils::uploadFileToWorker(std::string host, int port, int dataPort, int gra return false; } - util_logger.info("Going to send file through file transfer service to worker"); + util_logger.info("Going to send file" + filePath + "/" + fileName + "through file transfer service to worker"); Utils::sendFileThroughService(host, dataPort, fileName, filePath); string response; diff --git a/src/util/logger/Logger.cpp b/src/util/logger/Logger.cpp index f7a8949b1..b8bfe97dc 100644 --- a/src/util/logger/Logger.cpp +++ b/src/util/logger/Logger.cpp @@ -49,7 +49,6 @@ void Logger::log(std::string message, const std::string log_type) { cout << " [" << millis << "] [" << log_type << "] [" << worker_name << " : " << getpid() << ":" << tid << "] " << message << endl; return; - // message = "[" + worker_name + " : " + to_string(tid) + "] " + message; if (log_type.compare("info") == 0) { daily_logger->info(message); From d32d610e2a71f065ae7e5308c78efe7fd19007c1 Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Wed, 24 Apr 2024 21:44:12 +0530 Subject: [PATCH 142/143] Improve comment in logger --- src/util/logger/Logger.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/util/logger/Logger.cpp b/src/util/logger/Logger.cpp index b8bfe97dc..bc1a6dba0 100644 --- a/src/util/logger/Logger.cpp +++ b/src/util/logger/Logger.cpp @@ -45,7 +45,8 @@ void Logger::log(std::string message, const std::string log_type) { struct timeval tv; gettimeofday(&tv, NULL); long millis = tv.tv_sec * 1000 + tv.tv_usec / 1000; - // TODO: temporary fix only. + // TODO: This temporarily fixes spdlog hanging after forking. This will prevent using the actual logger and simulate + // the behavior using cout instead. But it will not write logs to the log file. cout << " [" << millis << "] [" << log_type << "] [" << worker_name << " : " << getpid() << ":" << tid << "] " << message << endl; return; From a0c9222b0dbe23c45277ae160528603ae8922c08 Mon Sep 17 00:00:00 2001 From: Mohamed Ishad Date: Fri, 26 Apr 2024 21:08:26 +0530 Subject: [PATCH 143/143] Define enums to represent command line arguments --- main.cpp | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/main.cpp b/main.cpp index 5cdd468b1..44a00d3c8 100644 --- a/main.cpp +++ b/main.cpp @@ -34,6 +34,26 @@ Logger main_logger; int jasminegraph_profile = PROFILE_DOCKER; #endif +enum args { + PROFILE = 1, + MODE = 2 +}; + +enum master_mode_args { + MASTER_IP = 3, + NUMBER_OF_WORKERS = 4, + WORKER_IPS = 5, + ENABLE_NMON = 6 +}; + +enum worker_mode_args { + HOST_NAME = 3, + MASTER_HOST = 4, + SERVER_PORT = 5, + SERVER_DATA_PORT = 6, + WORKER_ENABLE_NMON = 7 +}; + void fnExit3(void) { delete (server); } int main(int argc, char *argv[]) { @@ -47,9 +67,9 @@ int main(int argc, char *argv[]) { } std::cout << argc << std::endl; - int mode = atoi(argv[2]); + int mode = atoi(argv[args::MODE]); std::string JASMINEGRAPH_HOME = Utils::getJasmineGraphHome(); - jasminegraph_profile = strcmp(argv[1], "docker") == 0 ? PROFILE_DOCKER : PROFILE_K8S; + jasminegraph_profile = strcmp(argv[args::PROFILE], "docker") == 0 ? PROFILE_DOCKER : PROFILE_K8S; std::string enableNmon = "false"; main_logger.info("Using JASMINE_GRAPH_HOME=" + JASMINEGRAPH_HOME); @@ -57,10 +77,10 @@ int main(int argc, char *argv[]) { thread schedulerThread(SchedulerService::startScheduler); if (mode == Conts::JASMINEGRAPH_RUNTIME_PROFILE_MASTER) { - std::string masterIp = argv[3]; - int numberOfWorkers = atoi(argv[4]); - std::string workerIps = argv[5]; - enableNmon = argv[6]; + std::string masterIp = argv[master_mode_args::MASTER_IP]; + int numberOfWorkers = atoi(argv[master_mode_args::NUMBER_OF_WORKERS]); + std::string workerIps = argv[master_mode_args::WORKER_IPS]; + enableNmon = argv[master_mode_args::ENABLE_NMON]; server = JasmineGraphServer::getInstance(); if (jasminegraph_profile == PROFILE_K8S) { @@ -85,13 +105,13 @@ int main(int argc, char *argv[]) { } string hostName; - hostName = argv[3]; - setenv("HOST_NAME", argv[3], 1); - std::string masterHost = argv[4]; - int serverPort = atoi(argv[5]); - setenv("PORT", argv[5], 1); - int serverDataPort = atoi(argv[6]); - enableNmon = argv[7]; + hostName = argv[worker_mode_args::HOST_NAME]; + setenv("HOST_NAME", argv[worker_mode_args::HOST_NAME], 1); + std::string masterHost = argv[worker_mode_args::MASTER_HOST]; + int serverPort = atoi(argv[worker_mode_args::SERVER_PORT]); + setenv("PORT", argv[worker_mode_args::SERVER_PORT], 1); + int serverDataPort = atoi(argv[worker_mode_args::SERVER_DATA_PORT]); + enableNmon = argv[worker_mode_args::WORKER_ENABLE_NMON]; std::cout << "In worker mode" << std::endl; instance = new JasmineGraphInstance();