diff --git a/CMakeLists.txt b/CMakeLists.txt index 17ddffdb0..6d272cc53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ set(HEADERS globals.h src/frontend/core/executor/AbstractExecutor.h src/frontend/core/executor/impl/TriangleCountExecutor.h src/frontend/core/executor/impl/StreamingTriangleCountExecutor.h + src/frontend/core/executor/impl/SheepTriangleCountExecutor.h src/frontend/core/factory/ExecutorFactory.h src/frontend/core/scheduler/JobScheduler.h src/knowledgegraph/construction/Pipeline.h @@ -39,6 +40,7 @@ set(HEADERS globals.h src/ml/trainer/JasmineGraphTrainingSchedular.h src/partitioner/local/JSONParser.h src/partitioner/local/MetisPartitioner.h + src/partitioner/local/SheepPartitioner.h src/partitioner/local/RDFParser.h src/partitioner/local/RDFPartitioner.h src/partitioner/stream/Partition.h @@ -140,6 +142,7 @@ set(SOURCES src/backend/JasmineGraphBackend.cpp src/frontend/core/executor/AbstractExecutor.cpp src/frontend/core/executor/impl/TriangleCountExecutor.cpp src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp + src/frontend/core/executor/impl/SheepTriangleCountExecutor.cpp src/frontend/core/factory/ExecutorFactory.cpp src/frontend/core/scheduler/JobScheduler.cpp src/knowledgegraph/construction/Pipeline.cpp @@ -151,6 +154,7 @@ set(SOURCES src/backend/JasmineGraphBackend.cpp src/ml/trainer/JasmineGraphTrainingSchedular.cpp src/partitioner/local/JSONParser.cpp src/partitioner/local/MetisPartitioner.cpp + src/partitioner/local/SheepPartitioner.cpp src/partitioner/local/RDFParser.cpp src/partitioner/local/RDFPartitioner.cpp src/partitioner/stream/Partition.cpp @@ -161,6 +165,7 @@ set(SOURCES src/backend/JasmineGraphBackend.cpp src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp src/query/algorithms/triangles/Triangles.cpp src/query/algorithms/triangles/StreamingTriangles.cpp + src/query/algorithms/triangles/SheepTriangles.cpp src/query/processor/cypher/astbuilder/ASTBuilder.cpp src/query/processor/cypher/astbuilder/ASTInternalNode.cpp src/query/processor/cypher/astbuilder/ASTLeafNoValue.cpp diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index 0d91fb187..ea2672ede 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -38,6 +38,7 @@ limitations under the License. #include "../nativestore/RelationBlock.h" #include "../partitioner/local/JSONParser.h" #include "../partitioner/local/MetisPartitioner.h" +#include "../partitioner/local/SheepPartitioner.h" #include "../partitioner/local/RDFParser.h" #include "../partitioner/local/RDFPartitioner.h" #include "../partitioner/stream/Partitioner.h" @@ -161,6 +162,10 @@ static void predict_command(std::string masterIP, int connFd, SQLiteDBInterface static void start_remote_worker_command(int connFd, bool *loop_exit_p); static void sla_command(int connFd, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfSqlite, bool *loop_exit_p); +static void sheep_command(const std::string& masterIP, int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p); +static void sheep_triangles_command(const std::string& masterIP, int conn_fd, SQLiteDBInterface *sqlite, + PerformanceSQLiteDBInterface *perfSqlite, + JobScheduler *jobScheduler, bool *loop_exit_p); std::map> JasmineGraphFrontEnd::kgConstructionRates = {}; static vector getWorkerClients(SQLiteDBInterface* sqlite) { const vector& workerList = Utils::getWorkerList(sqlite); @@ -284,7 +289,7 @@ void* frontendservicesesion(void* dummyPt) { JasmineGraphFrontEnd::constructKGStreamLocalTXTCommand(masterIP, connFd, numberOfPartitions, sqlite, &loop_exit); } else if (line.compare(STOP_CONSTRUCT_KG) == 0) { - JasmineGraphFrontEnd::stop_graph_streaming(connFd, sqlite, &loop_exit); + JasmineGraphFrontEnd::stop_graph_streaming(connFd, &loop_exit); } else if (line.compare(0, STOP_STREAM_KAFKA.length(), STOP_STREAM_KAFKA) == 0 && (line.length() == STOP_STREAM_KAFKA.length() || std::isspace(static_cast(line[STOP_STREAM_KAFKA.length()])))) { @@ -349,6 +354,10 @@ void* frontendservicesesion(void* dummyPt) { start_remote_worker_command(connFd, &loop_exit); } else if (line.compare(SLA) == 0) { sla_command(connFd, sqlite, perfSqlite, &loop_exit); + } else if (line.compare(SHEEP) == 0) { + sheep_command(masterIP, connFd, sqlite, &loop_exit); + } else if (line.compare(SHTRIAN) == 0) { + sheep_triangles_command(masterIP, connFd, sqlite, perfSqlite, jobScheduler, &loop_exit); } else { frontend_logger.error("Message format not recognized " + line); int result_wr = write(connFd, INVALID_FORMAT.c_str(), INVALID_FORMAT.size()); @@ -4656,7 +4665,7 @@ static void kafka_topics_command(int connFd, SQLiteDBInterface *sqlite, bool *lo writeSocketResultOrEmpty(connFd, result, loop_exit_p); } - void JasmineGraphFrontEnd::stop_graph_streaming(int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p) { + void JasmineGraphFrontEnd::stop_graph_streaming(int connFd, bool *loop_exit_p) { std::string message1 = "Graph ID?"; int resultWr = write(connFd, message1.c_str(), message1.length()); if (resultWr < 0) { @@ -4698,10 +4707,204 @@ static void kafka_topics_command(int connFd, SQLiteDBInterface *sqlite, bool *lo std::string sqlStatement = "UPDATE graph SET kg_construction_status = 'paused' WHERE idgraph = " + userResS; - sqlite->runUpdate(sqlStatement); + } else { + std::string message2 = "Graph Id not Found"; + int resultWr = write(connFd, message2.c_str(), message2.length()); + } +} + +static void sheep_command(const std::string& masterIP, int conn_fd, SQLiteDBInterface *sqlite, bool *loop_exit_p) { + frontend_logger.info("Starting sheep partitioning command"); + if (!writeSocketLine(conn_fd, "send graph name", loop_exit_p)) { + return; + } + string graphName = readTrimmedSocketInput(conn_fd); + frontend_logger.info("Graph name received: " + graphName); + + if (!writeSocketLine(conn_fd, "send graph path", loop_exit_p)) { + return; + } + string graphPath = readTrimmedSocketInput(conn_fd); + frontend_logger.info("Graph path received: " + graphPath); + + if (!writeSocketLine(conn_fd, "send number of partitions", loop_exit_p)) { + return; + } + string partitionCount = readTrimmedSocketInput(conn_fd); + int numPartitions = 0; + try { + numPartitions = std::stoi(partitionCount); + } catch (const std::invalid_argument&) { + frontend_logger.error("Invalid partition count received: " + partitionCount); + *loop_exit_p = true; + writeSocketLine(conn_fd, "error: invalid partition count", loop_exit_p); + return; + } + frontend_logger.info("Number of partitions: " + to_string(numPartitions)); + + // Check if graph file exists + if (!Utils::fileExists(graphPath)) { + frontend_logger.error("Graph file does not exist: " + graphPath); + writeSocketLine(conn_fd, "error: graph file not found", loop_exit_p); + *loop_exit_p = true; + return; + } + + // Insert graph record into metadb + std::time_t time = chrono::system_clock::to_time_t(chrono::system_clock::now()); + string uploadStartTime(26, '\0'); + ctime_r(&time, &uploadStartTime[0]); + uploadStartTime = Utils::trim_copy(uploadStartTime); + + string sqlStatement = + "INSERT INTO graph (name,upload_path,upload_start_time,upload_end_time,graph_status_idgraph_status," + "vertexcount,centralpartitioncount,edgecount) VALUES(\"" + + graphName + R"(", ")" + graphPath + R"(", ")" + uploadStartTime + R"(", ""," )" + + to_string(Conts::GRAPH_STATUS::LOADING) + R"(", "", "", "")"; + int graphID = sqlite->runInsert(sqlStatement); + + if (graphID < 0) { + frontend_logger.error("Failed to insert graph into database"); + writeSocketLine(conn_fd, "error: database insertion failed", loop_exit_p); + *loop_exit_p = true; + return; + } + + frontend_logger.info("Graph record created with ID: " + to_string(graphID)); + + // Prepare output path in format: datafolder/graphID_ + // The partitioner will append partition number and create central/local store files + string outputPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + + "/" + to_string(graphID) + "_"; + + // Create SheepPartitioner instance and partition the graph + SheepPartitioner sheepPartitioner(sqlite); + vector> fullFileList = + sheepPartitioner.partitionGraph(graphID, graphPath, outputPath, numPartitions); + + if (!fullFileList.empty()) { + frontend_logger.info("Sheep partitioning completed successfully for graph ID: " + to_string(graphID)); + + // Upload partition files to workers + JasmineGraphServer *server = JasmineGraphServer::getInstance(); + server->uploadGraphLocally(graphID, Conts::GRAPH_TYPE_NORMAL, fullFileList, masterIP); + + // Clean up temporary directory if it exists + if (string tempDir = Utils::getHomeDir() + "/.jasminegraph/tmp/" + to_string(graphID); + Utils::fileExists(tempDir)) { + Utils::deleteDirectory(tempDir); + } + + JasmineGraphFrontEndCommon::getAndUpdateUploadTime(to_string(graphID), sqlite); + + string message = "sheep partitioning completed for graph ID: " + to_string(graphID); + writeSocketLine(conn_fd, message, loop_exit_p); + if (*loop_exit_p) { + return; + } + } else { + frontend_logger.error("Sheep partitioning failed"); + writeSocketLine(conn_fd, "error: sheep partitioning failed", loop_exit_p); + *loop_exit_p = true; + } +} + +static void sheep_triangles_command(const std::string& masterIP, int conn_fd, SQLiteDBInterface *sqlite, + PerformanceSQLiteDBInterface *perfSqlite, JobScheduler *jobScheduler, + bool *loop_exit_p) { + frontend_logger.info("Starting sheep triangle counting command"); + + int uniqueId = JasmineGraphFrontEndCommon::getUid(); + if (!writeSocketLine(conn_fd, GRAPHID_SEND, loop_exit_p)) { + return; + } + string graph_id = readTrimmedSocketInput(conn_fd); + + if (!JasmineGraphFrontEndCommon::graphExistsByID(graph_id, sqlite)) { + string error_message = "The specified graph id does not exist"; + writeSocketLine(conn_fd, error_message, loop_exit_p); + *loop_exit_p = true; + return; + } + + if (!writeSocketLine(conn_fd, PRIORITY, loop_exit_p)) { + return; + } + string priority = readTrimmedSocketInput(conn_fd); + + if (!(std::find_if(priority.begin(), priority.end(), [](unsigned char c) { return !std::isdigit(c); }) == + priority.end())) { + *loop_exit_p = true; + string error_message = "Priority should be numeric and > 1 or empty"; + writeSocketLine(conn_fd, error_message, loop_exit_p); + return; + } + + int threadPriority = std::atoi(priority.c_str()); + + static std::atomic reqCounter = 0; + string reqId = to_string(reqCounter++); + frontend_logger.info("Started processing sheep triangle counting request " + reqId); + auto begin = chrono::high_resolution_clock::now(); + JobRequest jobDetails; + jobDetails.setJobId(std::to_string(uniqueId)); + jobDetails.setJobType(SHEEP_TRIANGLES); + + long graphSLA = -1; // This prevents auto calibration for priority=1 (=default priority) + if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { + // All high priority threads will be set the same high priority level + threadPriority = Conts::HIGH_PRIORITY_DEFAULT_VALUE; + graphSLA = JasmineGraphFrontEndCommon::getSLAForGraphId(sqlite, perfSqlite, graph_id, SHEEP_TRIANGLES, + Conts::SLA_CATEGORY::LATENCY); + jobDetails.addParameter(Conts::PARAM_KEYS::GRAPH_SLA, std::to_string(graphSLA)); + } + + if (graphSLA == 0) { + if (JasmineGraphFrontEnd::areRunningJobsForSameGraph()) { + if (canCalibrate) { + // initial calibration + jobDetails.addParameter(Conts::PARAM_KEYS::AUTO_CALIBRATION, "false"); } else { - std::string message2 = "Graph Id not Found"; - int resultWr = write(connFd, message2.c_str(), message2.length()); + // auto calibration + jobDetails.addParameter(Conts::PARAM_KEYS::AUTO_CALIBRATION, "true"); } + } else { + frontend_logger.error("Can't calibrate the graph now"); } + } + + jobDetails.setPriority(threadPriority); + jobDetails.setMasterIP(masterIP); + jobDetails.addParameter(Conts::PARAM_KEYS::GRAPH_ID, graph_id); + jobDetails.addParameter(Conts::PARAM_KEYS::CATEGORY, Conts::SLA_CATEGORY::LATENCY); + if (canCalibrate) { + jobDetails.addParameter(Conts::PARAM_KEYS::CAN_CALIBRATE, "true"); + } else { + jobDetails.addParameter(Conts::PARAM_KEYS::CAN_CALIBRATE, "false"); + } + + jobScheduler->pushJob(jobDetails); + JobResponse jobResponse = jobScheduler->getResult(jobDetails); + if (std::string errorMessage = jobResponse.getParameter(Conts::PARAM_KEYS::ERROR_MESSAGE); !errorMessage.empty()) { + *loop_exit_p = true; + writeSocketLine(conn_fd, errorMessage, loop_exit_p); + return; + } + std::string sheepTriangleCount = jobResponse.getParameter(Conts::PARAM_KEYS::TRIANGLE_COUNT); + + if (threadPriority == Conts::HIGH_PRIORITY_DEFAULT_VALUE) { + highPriorityTaskCount--; + } + + auto end = chrono::high_resolution_clock::now(); + auto dur = end - begin; + auto msDuration = std::chrono::duration_cast(dur).count(); + frontend_logger.info("Req: " + reqId + " Sheep-Partitioned Triangle Count: " + sheepTriangleCount + + " Time Taken: " + to_string(msDuration) + " milliseconds"); + + writeSocketLine(conn_fd, sheepTriangleCount, loop_exit_p); + if (*loop_exit_p) { + return; + } +} diff --git a/src/frontend/JasmineGraphFrontEnd.h b/src/frontend/JasmineGraphFrontEnd.h index 4711a2d0e..6d0cfe389 100644 --- a/src/frontend/JasmineGraphFrontEnd.h +++ b/src/frontend/JasmineGraphFrontEnd.h @@ -62,7 +62,7 @@ class JasmineGraphFrontEnd { static bool areRunningJobsForSameGraph(); static bool constructKGStreamHDFSCommand(std::string masterIP, int connFd, int numberOfPartitions, SQLiteDBInterface *sqlite, bool *loop_exit_p); - static void stop_graph_streaming(int connFd, SQLiteDBInterface*sqlite, bool *loop_exit_p); + static void stop_graph_streaming(int connFd, bool *loop_exit_p); static bool constructKGStreamLocalTXTCommand(std::string masterIP, int connFd, int numberOfPartitions, SQLiteDBInterface *sqlite, bool *loop_exit_p); static bool strian_exit; diff --git a/src/frontend/JasmineGraphFrontEndProtocol.cpp b/src/frontend/JasmineGraphFrontEndProtocol.cpp index 35f91a88c..a2da76ae3 100644 --- a/src/frontend/JasmineGraphFrontEndProtocol.cpp +++ b/src/frontend/JasmineGraphFrontEndProtocol.cpp @@ -68,3 +68,6 @@ const string AGENT_PLAN = "graphrag"; const string PROPERTIES = "prp"; const string UPDATE_PARTITION_META = "update-partition-meta"; const string META = "meta"; +const string SHEEP = "sheep"; +const string SHTRIAN = "shtrian"; +const string SHEEP_TRIANGLES = "sheep-triangles"; diff --git a/src/frontend/JasmineGraphFrontEndProtocol.h b/src/frontend/JasmineGraphFrontEndProtocol.h index 6b4e658a8..7a5c59e3c 100644 --- a/src/frontend/JasmineGraphFrontEndProtocol.h +++ b/src/frontend/JasmineGraphFrontEndProtocol.h @@ -100,6 +100,9 @@ extern const string AGENT_PLAN; extern const string PROPERTIES; extern const string UPDATE_PARTITION_META; extern const string META; +extern const string SHEEP; +extern const string SHTRIAN; +extern const string SHEEP_TRIANGLES; class JasminGraphFrontEndProtocol { // Note that this protocol do not need a handshake session since the communication in most of the time is conducted diff --git a/src/frontend/core/domain/JobRequest.cpp b/src/frontend/core/domain/JobRequest.cpp index 032bf3a12..61b4f6fd3 100644 --- a/src/frontend/core/domain/JobRequest.cpp +++ b/src/frontend/core/domain/JobRequest.cpp @@ -18,27 +18,32 @@ std::string JobRequest::getJobId() const { return jobId; } void JobRequest::setJobId(std::string inputJobId) { jobId = inputJobId; } -std::string JobRequest::getJobType() { return jobType; } +std::string JobRequest::getJobType() const { return jobType; } void JobRequest::setJobType(std::string inputJobType) { jobType = inputJobType; } void JobRequest::addParameter(std::string key, std::string value) { requestParams[key] = value; } -std::string JobRequest::getParameter(std::string key) { return requestParams[key]; } +std::string JobRequest::getParameter(std::string key) const { + if (requestParams.find(key) != requestParams.end()) { + return requestParams.at(key); + } + return ""; +} void JobRequest::setPriority(int priority) { this->priority = priority; } -int JobRequest::getPriority() { return priority; } +int JobRequest::getPriority() const { return priority; } void JobRequest::setMasterIP(std::string masterip) { this->masterIP = masterip; } -std::string JobRequest::getMasterIP() { return masterIP; } +std::string JobRequest::getMasterIP() const { return masterIP; } void JobRequest::setBeginTime(std::chrono::time_point begin) { this->begin = begin; } -std::chrono::time_point JobRequest::getBegin() { +std::chrono::time_point JobRequest::getBegin() const { return begin; } diff --git a/src/frontend/core/domain/JobRequest.h b/src/frontend/core/domain/JobRequest.h index afa8e63c0..b01d36775 100644 --- a/src/frontend/core/domain/JobRequest.h +++ b/src/frontend/core/domain/JobRequest.h @@ -32,15 +32,15 @@ class JobRequest { std::string getJobId() const; void setJobId(std::string inputJobId); - std::string getJobType(); + std::string getJobType() const; void setJobType(std::string inputJobType); void addParameter(std::string key, std::string value); - std::string getParameter(std::string key); + std::string getParameter(std::string key) const; void setPriority(int priority); - int getPriority(); + int getPriority() const; void setMasterIP(std::string masterip); - std::string getMasterIP(); - std::chrono::time_point getBegin(); + std::string getMasterIP() const; + std::chrono::time_point getBegin() const; void setBeginTime(std::chrono::time_point begin); }; diff --git a/src/frontend/core/executor/AbstractExecutor.h b/src/frontend/core/executor/AbstractExecutor.h index cbfc7aafa..85d9d9569 100644 --- a/src/frontend/core/executor/AbstractExecutor.h +++ b/src/frontend/core/executor/AbstractExecutor.h @@ -32,6 +32,7 @@ class AbstractExecutor { AbstractExecutor(); AbstractExecutor(JobRequest jobRequest); static std::vector> getCombinations(std::vector inputVector); + virtual ~AbstractExecutor() = default; virtual void execute() = 0; static int collectPerformaceData(PerformanceSQLiteDBInterface *perDB, std::string graphId, std::string command, std::string category, int partitionCount, diff --git a/src/frontend/core/executor/impl/CypherQueryExecutor.cpp b/src/frontend/core/executor/impl/CypherQueryExecutor.cpp index 8fe8ceceb..aa6d8f852 100644 --- a/src/frontend/core/executor/impl/CypherQueryExecutor.cpp +++ b/src/frontend/core/executor/impl/CypherQueryExecutor.cpp @@ -244,6 +244,9 @@ void CypherQueryExecutor::execute() { cypher_logger.error("Missing key in val2 for comparison: " + Operator::aggregateKey); return false; } + + if (*val1 == *val2) return false; + bool result; if (val1->is_number_integer() && val2->is_number_integer()) { result = val1->get() > val2->get(); @@ -357,9 +360,9 @@ void CypherQueryExecutor::execute() { workerResponded = true; JobResponse jobResponse; jobResponse.setJobId(request.getJobId()); - responseVector.push_back(jobResponse); responseVectorMutex.lock(); + responseVector.push_back(jobResponse); responseMap[request.getJobId()] = jobResponse; responseVectorMutex.unlock(); diff --git a/src/frontend/core/executor/impl/PageRankExecutor.cpp b/src/frontend/core/executor/impl/PageRankExecutor.cpp index ceefa2f08..c87c9835b 100644 --- a/src/frontend/core/executor/impl/PageRankExecutor.cpp +++ b/src/frontend/core/executor/impl/PageRankExecutor.cpp @@ -154,9 +154,9 @@ void PageRankExecutor::execute() { workerResponded = true; JobResponse jobResponse; jobResponse.setJobId(request.getJobId()); - responseVector.push_back(jobResponse); responseVectorMutex.lock(); + responseVector.push_back(jobResponse); responseMap[request.getJobId()] = jobResponse; responseVectorMutex.unlock(); diff --git a/src/frontend/core/executor/impl/SheepTriangleCountExecutor.cpp b/src/frontend/core/executor/impl/SheepTriangleCountExecutor.cpp new file mode 100644 index 000000000..53bb013b2 --- /dev/null +++ b/src/frontend/core/executor/impl/SheepTriangleCountExecutor.cpp @@ -0,0 +1,33 @@ +/** +Copyright 2026 JasmineGraph Team +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +#include "SheepTriangleCountExecutor.h" +#include "TriangleCountExecutor.h" + +Logger sheepTriangleCount_logger; + +SheepTriangleCountExecutor::SheepTriangleCountExecutor(SQLiteDBInterface *db, PerformanceSQLiteDBInterface *perfDb, + const JobRequest& jobRequest) + : AbstractExecutor(jobRequest), sqlite(db), perfDB(perfDb) { +} + +int SheepTriangleCountExecutor::getUid() const { + static int counter = 0; + return counter++; +} + +void SheepTriangleCountExecutor::execute() { + TriangleCountExecutor::executeTriangleCount(sqlite, perfDB, request, + TriangleCountCommandType::SHEEP_TRIANGLES, + ThreadingStrategy::ASYNC_BASED, sheepTriangleCount_logger); +} diff --git a/src/frontend/core/executor/impl/SheepTriangleCountExecutor.h b/src/frontend/core/executor/impl/SheepTriangleCountExecutor.h new file mode 100644 index 000000000..7b36ded5a --- /dev/null +++ b/src/frontend/core/executor/impl/SheepTriangleCountExecutor.h @@ -0,0 +1,51 @@ +/** +Copyright 2026 JasmineGraph Team +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +#ifndef JASMINEGRAPH_SHEEPTRIANGLECOUNTEXECUTOR_H +#define JASMINEGRAPH_SHEEPTRIANGLECOUNTEXECUTOR_H + +#include +#include +#include + +#include "../../../../metadb/SQLiteDBInterface.h" +#include "../../../../performance/metrics/PerformanceUtil.h" +#include "../../../../performancedb/PerformanceSQLiteDBInterface.h" +#include "../../../../server/JasmineGraphInstanceProtocol.h" +#include "../../../../server/JasmineGraphServer.h" +#include "../../../JasmineGraphFrontEndProtocol.h" +#include "../../CoreConstants.h" +#include "../AbstractExecutor.h" + +/** + * Dedicated executor for sheep-partitioned graph triangle counting. + * This executor is specialized for graphs partitioned using the sheep algorithm + * and uses optimized triangle counting methods. + * + * Uses async-based threading strategy and delegates to TriangleCountExecutor's shared logic. + */ +class SheepTriangleCountExecutor : public AbstractExecutor { + public: + SheepTriangleCountExecutor(SQLiteDBInterface *db, PerformanceSQLiteDBInterface *perfDb, + const JobRequest& jobRequest); + + void execute() override; + + int getUid() const; + + private: + SQLiteDBInterface *sqlite; + PerformanceSQLiteDBInterface *perfDB; +}; + +#endif // JASMINEGRAPH_SHEEPTRIANGLECOUNTEXECUTOR_H diff --git a/src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp b/src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp index 5d48ac815..a5fff80ae 100644 --- a/src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp @@ -109,8 +109,9 @@ void StreamingTriangleCountExecutor::execute() { jobResponse.setJobId(request.getJobId()); jobResponse.addParameter(Conts::PARAM_KEYS::STREAMING_TRIANGLE_COUNT, std::to_string(result)); jobResponse.setEndTime(chrono::high_resolution_clock::now()); - responseVector.push_back(jobResponse); + std::scoped_lock lock(responseVectorMutex); + responseVector.push_back(jobResponse); responseMap[request.getJobId()] = jobResponse; } diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index ee0e9e53e..fb99b74d6 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -37,8 +37,8 @@ static string isFileAccessibleToWorker(std::string graphId, std::string partitio std::string aggregatorPort, std::string masterIP, std::string fileType, std::string fileName); static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, - int threadPriority, - const std::map> &partitionMap); + int threadPriority, const std::map, + std::less<>> &partitionMap); static int updateTriangleTreeAndGetTriangleCount( const std::vector &triangles, std::unordered_map>> *triangleTree_p, @@ -77,8 +77,8 @@ TriangleCountExecutor::TriangleCountExecutor(SQLiteDBInterface *db, PerformanceS this->request = jobRequest; } -static void allocate(int p, string w, std::map &alloc, std::set &remain, - std::map> &p_avail, std::map &loads) { +void allocate(int p, string w, std::map &alloc, std::set &remain, + std::map> &p_avail, std::map> &loads) { alloc[p] = w; remain.erase(p); p_avail.erase(p); @@ -110,8 +110,8 @@ static int get_min_partition(std::set &remain, std::map LOAD_PREFERENCE = {2, 3, 1, 0}; -static int alloc_plan(std::map &alloc, std::set &remain, std::map> &p_avail, - std::map &loads) { +int alloc_plan(std::map &alloc, std::set &remain, std::map> &p_avail, + std::map> &loads) { for (bool done = false; !done;) { string w = ""; done = true; @@ -137,7 +137,7 @@ static int alloc_plan(std::map &alloc, std::set &remain, std:: std::map alloc; std::set remain; std::map> p_avail; - std::map loads; + std::map> loads; }; int best_rem = remain.size(); struct best_alloc best = {.alloc = alloc, .remain = remain, .p_avail = p_avail, .loads = loads}; @@ -171,7 +171,7 @@ static int alloc_plan(std::map &alloc, std::set &remain, std:: return best_rem; } -static std::vector reallocate_parts(std::map &alloc, std::set &remain, +std::vector reallocate_parts(std::map &alloc, std::set &remain, const std::map> &P_AVAIL) { map P_COUNT; for (auto it = P_AVAIL.begin(); it != P_AVAIL.end(); it++) { @@ -222,7 +222,7 @@ static std::vector reallocate_parts(std::map &alloc, std::set< return copying; } -static void scale_up(std::map &loads, map &workers, int copy_cnt) { +void scale_up(std::map> &loads, map> &workers, int copy_cnt) { int curr_load = 0; for (auto it = loads.begin(); it != loads.end(); it++) { curr_load += it->second; @@ -244,13 +244,14 @@ static void scale_up(std::map &loads, map &workers, } } -static int alloc_net_plan(std::map &alloc, std::vector &parts, - std::map> &transfer, std::map &net_loads, - std::map &loads, const std::map> &p_avail, - int curr_best) { +int alloc_net_plan(std::map &alloc, std::vector &parts, + std::map> &transfer, + std::map> &net_loads, + std::map> &loads, + const std::map> &p_avail, + int curr_best) { int curr_load = std::max_element(net_loads.begin(), net_loads.end(), - [](const std::map::value_type &p1, - const std::map::value_type &p2) { return p1.second < p2.second; }) + [](const auto &p1, const auto &p2) { return p1.second < p2.second; }) ->second; if (curr_load >= curr_best) { return curr_load; @@ -262,8 +263,8 @@ static int alloc_net_plan(std::map &alloc, std::vector &parts, struct best_net_alloc { std::map alloc; std::map> transfer; - std::map net_loads; - std::map loads; + std::map> net_loads; + std::map> loads; }; int best = curr_best; struct best_net_alloc best_plan = {.transfer = transfer, .net_loads = net_loads, .loads = loads}; @@ -325,19 +326,21 @@ static int alloc_net_plan(std::map &alloc, std::vector &parts, return best; } -static void filter_partitions(std::map> &partitionMap, SQLiteDBInterface *sqlite, - string graphId) { - map workers; // id => "ip:port" +static map> get_workers(SQLiteDBInterface *sqlite) { + map> workers; const std::vector>> &results = sqlite->runSelect("SELECT DISTINCT idworker,ip,server_port FROM worker;"); - for (int i = 0; i < results.size(); i++) { + for (size_t i = 0; i < results.size(); i++) { string workerId = results[i][0].second; string ip = results[i][1].second; string port = results[i][2].second; workers[workerId] = ip + ":" + port; } + return workers; +} - map loads; +static map> get_loads(const map> &workers) { + map> loads; const map &cpu_map = Utils::getMetricMap("cpu_usage"); for (auto it = workers.begin(); it != workers.end(); it++) { auto workerId = it->first; @@ -351,9 +354,11 @@ static void filter_partitions(std::map> &partitionMa loads[workerId] = 0; } } + return loads; +} - std::map> p_avail; - std::set remain; +static void build_avail_partitions(const std::map, std::less<>> &partitionMap, + std::map> &p_avail, std::set &remain) { for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { auto worker = it->first; auto &partitions = it->second; @@ -363,8 +368,10 @@ static void filter_partitions(std::map> &partitionMa remain.insert(partition); } } - const std::map> P_AVAIL = p_avail; // get a copy and make it const +} +static void filter_overloaded_workers(const map> &loads, + std::map> &p_avail) { for (auto loadIt = loads.begin(); loadIt != loads.end(); loadIt++) { if (loadIt->second < 3) continue; auto w = loadIt->first; @@ -379,6 +386,56 @@ static void filter_partitions(std::map> &partitionMa } } } +} + +static void execute_partition_transfers(SQLiteDBInterface *sqlite, + const std::map> &transfer, + const map> &workers, + const string &graphId) { + if (transfer.empty()) return; + const std::vector>> &workerData = + sqlite->runSelect("SELECT DISTINCT ip,server_port,server_data_port FROM worker;"); + map dataPortMap; // "ip:port" => data_port + for (const auto &row : workerData) { + string ip = row[0].second; + string port = row[1].second; + string dport = row[2].second; + dataPortMap[ip + ":" + port] = dport; + } + std::vector transferThreads; + transferThreads.reserve(transfer.size()); + for (auto it = transfer.begin(); it != transfer.end(); it++) { + auto partition = it->first; + auto from_worker = it->second.first; + auto to_worker = it->second.second; + auto w_from = workers.find(from_worker)->second; + auto w_to = workers.find(to_worker)->second; + const auto &ip_port_from = Utils::split(w_from, ':'); + auto ip_from = ip_port_from[0]; + auto port_from = stoi(ip_port_from[1]); + const auto &ip_port_to = Utils::split(w_to, ':'); + auto ip_to = ip_port_to[0]; + auto dport_to = stoi(dataPortMap[w_to]); + transferThreads.emplace_back(&Utils::transferPartition, ip_from, port_from, ip_to, + dport_to, graphId, to_string(partition), to_worker, sqlite); + } + for (auto &t : transferThreads) { + t.join(); + } +} + +void filter_partitions(std::map, std::less<>> &partitionMap, SQLiteDBInterface *sqlite, + const string &graphId) { + auto workers = get_workers(sqlite); + auto loads = get_loads(workers); + + std::map> p_avail; + std::set remain; + build_avail_partitions(partitionMap, p_avail, remain); + + const std::map> P_AVAIL = p_avail; // get a copy and make it const + + filter_overloaded_workers(loads, p_avail); std::map alloc; int unallocated = alloc_plan(alloc, remain, p_avail, loads); @@ -388,7 +445,7 @@ static void filter_partitions(std::map> &partitionMa scale_up(loads, workers, copying.size()); triangleCount_logger.info("Scale up completed"); - map net_loads; + map> net_loads; for (auto it = loads.begin(); it != loads.end(); it++) { net_loads[it->first] = 0; } @@ -406,41 +463,7 @@ static void filter_partitions(std::map> &partitionMa alloc[p] = w_to; } - if (!transfer.empty()) { - const std::vector>> &workerData = - sqlite->runSelect("SELECT DISTINCT ip,server_port,server_data_port FROM worker;"); - map dataPortMap; // "ip:port" => data_port - for (int i = 0; i < workerData.size(); i++) { - string ip = workerData[i][0].second; - string port = workerData[i][1].second; - string dport = workerData[i][2].second; - dataPortMap[ip + ":" + port] = dport; - } - map workers_r; // "ip:port" => id - for (auto it = workers.begin(); it != workers.end(); it++) { - workers_r[it->second] = it->first; - } - thread transferThreads[transfer.size()]; - int threadCnt = 0; - for (auto it = transfer.begin(); it != transfer.end(); it++) { - auto partition = it->first; - auto from_worker = it->second.first; - auto to_worker = it->second.second; - auto w_from = workers[from_worker]; - auto w_to = workers[to_worker]; - const auto &ip_port_from = Utils::split(w_from, ':'); - auto ip_from = ip_port_from[0]; - auto port_from = stoi(ip_port_from[1]); - const auto &ip_port_to = Utils::split(w_to, ':'); - auto ip_to = ip_port_to[0]; - auto dport_to = stoi(dataPortMap[w_to]); - transferThreads[threadCnt++] = std::thread(&Utils::transferPartition, ip_from, port_from, ip_to, - dport_to, graphId, to_string(partition), to_worker, sqlite); - } - for (int i = 0; i < threadCnt; i++) { - transferThreads[i].join(); - } - } + execute_partition_transfers(sqlite, transfer, workers, graphId); } partitionMap.clear(); for (auto it = alloc.begin(); it != alloc.end(); it++) { @@ -450,319 +473,16 @@ static void filter_partitions(std::map> &partitionMa } } -void TriangleCountExecutor::execute() { - // Start automatic OpenTelemetry tracing for triangle count execution - std::string graphId = request.getParameter(Conts::PARAM_KEYS::GRAPH_ID); - OTEL_TRACE_FUNCTION(); - - schedulerMutex.lock(); - time_t curr_time = time(NULL); - // 8 seconds = upper bound to the time to send performance metrics after allocating trian task to a worker - if (curr_time < last_exec_time + 8) { - sleep(last_exec_time + 9 - curr_time); // 9 = 8+1 to ensure it waits more than 8 seconds - } - int uniqueId = getUid(); - std::string masterIP = request.getMasterIP(); - std::string canCalibrateString = request.getParameter(Conts::PARAM_KEYS::CAN_CALIBRATE); - std::string queueTime = request.getParameter(Conts::PARAM_KEYS::QUEUE_TIME); - - bool canCalibrate = Utils::parseBoolean(canCalibrateString); - int threadPriority = request.getPriority(); - - std::string autoCalibrateString = request.getParameter(Conts::PARAM_KEYS::AUTO_CALIBRATION); - bool autoCalibrate = Utils::parseBoolean(autoCalibrateString); - - if (threadPriority == Conts::HIGH_PRIORITY_DEFAULT_VALUE) { - highPriorityGraphList.push_back(graphId); - } - - // Below code is used to update the process details - std::chrono::milliseconds startTime = duration_cast(system_clock::now().time_since_epoch()); - - struct ProcessInfo processInformation; - processInformation.id = uniqueId; - processInformation.graphId = graphId; - processInformation.processName = TRIANGLES; - processInformation.priority = threadPriority; - processInformation.startTimestamp = startTime.count(); - - if (!queueTime.empty()) { - long sleepTime = atol(queueTime.c_str()); - processInformation.sleepTime = sleepTime; - insertProcessInfo(processInformation); - std::this_thread::sleep_for(std::chrono::milliseconds(sleepTime)); - } else { - insertProcessInfo(processInformation); - } - - triangleCount_logger.log( - "###TRIANGLE-COUNT-EXECUTOR### Started with graph ID : " + graphId + " Master IP : " + masterIP, "info"); - long result = 0; - bool isCompositeAggregation = false; - Utils::worker aggregatorWorker; - std::vector intermThreads; - std::vector intermRes; - std::vector statThreads; - std::vector statResponse; - std::vector compositeCentralStoreFiles; - - auto begin = chrono::high_resolution_clock::now(); - - string sqlStatement = - "SELECT DISTINCT worker_idworker,partition_idpartition " - "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " - "WHERE partition_graph_idgraph=" + - graphId + ";"; - - const std::vector>> &results = sqlite->runSelect(sqlStatement); - - std::map> partitionMap; - - for (auto i = results.begin(); i != results.end(); ++i) { - const std::vector> &rowData = *i; - - string workerID = rowData.at(0).second; - string partitionId = rowData.at(1).second; - - if (partitionMap.find(workerID) == partitionMap.end()) { - std::vector partitionVec; - partitionVec.push_back(partitionId); - partitionMap[workerID] = partitionVec; - } else { - partitionMap[workerID].push_back(partitionId); - } - - triangleCount_logger.info("###TRIANGLE-COUNT-EXECUTOR### Getting Triangle Count : PartitionId " + partitionId); - } - - if (results.size() > Conts::COMPOSITE_CENTRAL_STORE_WORKER_THRESHOLD) { - isCompositeAggregation = true; - } - - if (jasminegraph_profile == PROFILE_K8S) { - std::unique_ptr k8sInterface(new K8sInterface()); - if (k8sInterface->getJasmineGraphConfig("auto_scaling_enabled") == "true") { - filter_partitions(partitionMap, sqlite, graphId); - } - } - - std::vector> fileCombinations; - if (isCompositeAggregation) { - std::string aggregatorFilePath = - Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); - std::vector graphFiles = Utils::getListOfFilesInDirectory(aggregatorFilePath); - - std::string compositeFileNameFormat = graphId + "_compositecentralstore_"; - - for (auto graphFilesIterator = graphFiles.begin(); graphFilesIterator != graphFiles.end(); - ++graphFilesIterator) { - std::string graphFileName = *graphFilesIterator; - - if ((graphFileName.find(compositeFileNameFormat) == 0) && - (graphFileName.find(".gz") != std::string::npos)) { - compositeCentralStoreFiles.push_back(graphFileName); - } - } - fileCombinations = AbstractExecutor::getCombinations(compositeCentralStoreFiles); - } - - for (const auto &[worker, partitions] : partitionMap) { - if (used_workers.find(worker) != used_workers.end()) { - used_workers[worker]++; - } else { - used_workers[worker] = 1; - } - } - - std::map combinationWorkerMap; - std::unordered_map>> triangleTree; - std::mutex triangleTreeMutex; - int partitionCount = 0; - for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { - partitionCount += it->second.size(); - } - intermRes.resize(partitionCount, 0); - int currentPartitionIndex = 0; - - // Track worker information for better tracing - std::vector> workerTaskInfo; // {workerID, partitionId, host} - - // Capture the master trace context for all workers - std::string masterTraceContext = OpenTelemetryUtil::getCurrentTraceContext(); - - vector workerList = Utils::getWorkerList(sqlite); - int workerListSize = workerList.size(); - for (int i = 0; i < workerListSize; i++) { - Utils::worker currentWorker = workerList.at(i); - string host = currentWorker.hostname; - string workerID = currentWorker.workerID; - string partitionId; - int workerPort = atoi(string(currentWorker.port).c_str()); - int workerDataPort = atoi(string(currentWorker.dataPort).c_str()); - triangleCount_logger.info("worker_" + workerID + " host=" + host + ":" + to_string(workerPort) + ":" + - to_string(workerDataPort)); - const std::vector &partitionList = partitionMap[workerID]; - for (auto partitionIterator = partitionList.begin(); partitionIterator != partitionList.end(); - ++partitionIterator) { - partitionId = *partitionIterator; - triangleCount_logger.info("> partition" + partitionId); - - // Store worker task information for tracing - workerTaskInfo.push_back(std::make_tuple(workerID, partitionId, host)); - - { - OTEL_TRACE_OPERATION("distribute_to_worker_" + workerID + "_partition_" + partitionId); - - int graphIdInt = atoi(graphId.c_str()); - int partitionIdInt = atoi(partitionId.c_str()); - intermThreads.push_back(std::thread([&intermRes, currentPartitionIndex, graphIdInt, host, workerPort, - workerDataPort, partitionIdInt, masterIP, uniqueId, isCompositeAggregation, threadPriority, - fileCombinations, &combinationWorkerMap, &triangleTree, &triangleTreeMutex, masterTraceContext]() { - intermRes[currentPartitionIndex] = TriangleCountExecutor::getTriangleCount(graphIdInt, host, - workerPort, workerDataPort, partitionIdInt, masterIP, uniqueId, - isCompositeAggregation, threadPriority, fileCombinations, &combinationWorkerMap, - &triangleTree, &triangleTreeMutex, masterTraceContext); - })); - currentPartitionIndex++; - } - } - } - - PerformanceUtil::init(); - - std::string query = - "SELECT attempt from graph_sla INNER JOIN sla_category where graph_sla.id_sla_category=sla_category.id and " - "graph_sla.graph_id='" + - graphId + "' and graph_sla.partition_count='" + std::to_string(partitionCount) + - "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + "' and sla_category.command='" + TRIANGLES + - "';"; - - const std::vector>> &queryResults = perfDB->runSelect(query); - - if (queryResults.size() > 0) { - std::string attemptString = queryResults[0][0].second; - int calibratedAttempts = atoi(attemptString.c_str()); - - if (calibratedAttempts >= Conts::MAX_SLA_CALIBRATE_ATTEMPTS) { - canCalibrate = false; - } - } else { - triangleCount_logger.log("###TRIANGLE-COUNT-EXECUTOR### Inserting initial record for SLA ", "info"); - Utils::updateSLAInformation(perfDB, graphId, partitionCount, 0, TRIANGLES, Conts::SLA_CATEGORY::LATENCY); - statResponse.push_back(0); - statThreads.push_back(std::thread([&statResponse, this, graphId, partitionCount, masterIP, autoCalibrate]() { - statResponse[0] = AbstractExecutor::collectPerformaceData(this->perfDB, - graphId, TRIANGLES, Conts::SLA_CATEGORY::LATENCY, partitionCount, masterIP, autoCalibrate); - })); - isStatCollect = true; - } - - last_exec_time = time(NULL); - schedulerMutex.unlock(); - - // Collect worker results with automatic tracing - { - OTEL_TRACE_OPERATION("collect_worker_results"); - - int taskIndex = 0; - for (auto &intermThread : intermThreads) { - // Get worker information for this task - const auto& taskInfo = workerTaskInfo[taskIndex]; - string workerID = std::get<0>(taskInfo); - string partitionId = std::get<1>(taskInfo); - string host = std::get<2>(taskInfo); - - { - OTEL_TRACE_OPERATION("wait_for_worker_" + workerID + "_partition_" + partitionId + "_on_" + host); - triangleCount_logger.info("Waiting for result from worker_" + workerID + - " partition_" + partitionId + - " host_" + host + - " uuid=" + to_string(uniqueId)); - if (intermThread.joinable()) { - intermThread.join(); - } - long worker_result = intermRes[taskIndex]; - triangleCount_logger.info("Received result " + std::to_string(worker_result) + - " from worker_" + workerID + - " partition_" + partitionId); - - { - OTEL_TRACE_OPERATION("aggregate_result_worker_" + workerID + "_partition_" + partitionId); - result += worker_result; - } - } - taskIndex++; - } - - // Cleanup data structures - { - OTEL_TRACE_OPERATION("cleanup_worker_data_structures"); - triangleTree.clear(); - combinationWorkerMap.clear(); - } - } - - if (!isCompositeAggregation) { - // Restore the master trace context before aggregation - OpenTelemetryUtil::receiveAndSetTraceContext(masterTraceContext, "central store aggregation"); - - OTEL_TRACE_OPERATION("central_store_aggregation"); - - long aggregatedTriangleCount = - aggregateCentralStoreTriangles(sqlite, graphId, masterIP, threadPriority, partitionMap); - result += aggregatedTriangleCount; - - workerResponded = true; - triangleCount_logger.log( - "###TRIANGLE-COUNT-EXECUTOR### Getting Triangle Count : Completed: Triangles " + to_string(result), "info"); - } - - schedulerMutex.lock(); - for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { - string worker = it->first; - used_workers[worker]--; - } - for (auto it = used_workers.cbegin(); it != used_workers.cend();) { - if (it->second <= 0) { - used_workers.erase(it++); - } else { - it++; - } - } - schedulerMutex.unlock(); - - workerResponded = true; - - JobResponse jobResponse; - jobResponse.setJobId(request.getJobId()); - jobResponse.addParameter(Conts::PARAM_KEYS::TRIANGLE_COUNT, std::to_string(result)); - responseVector.push_back(jobResponse); - - responseVectorMutex.lock(); - responseMap[request.getJobId()] = jobResponse; - responseVectorMutex.unlock(); - - auto end = chrono::high_resolution_clock::now(); - auto dur = end - begin; - auto msDuration = std::chrono::duration_cast(dur).count(); - - std::string durationString = std::to_string(msDuration); - - if (canCalibrate || autoCalibrate) { - Utils::updateSLAInformation(perfDB, graphId, partitionCount, msDuration, TRIANGLES, - Conts::SLA_CATEGORY::LATENCY); - isStatCollect = false; - } - - removeProcessInfoById(uniqueId); - joinAllThreads(statThreads); +void TriangleCountExecutor::execute() { + executeTriangleCount(sqlite, perfDB, request, TriangleCountCommandType::TRIANGLES, + ThreadingStrategy::THREAD_BASED, triangleCount_logger); } long TriangleCountExecutor::getTriangleCount( int graphId, std::string host, int port, int dataPort, int partitionId, std::string masterIP, int uniqueId, bool isCompositeAggregation, int threadPriority, std::vector> fileCombinations, - std::map *combinationWorkerMap_p, + std::map> *combinationWorkerMap_p, std::unordered_map>> *triangleTree_p, std::mutex *triangleTreeMutex_p, const std::string& masterTraceContext) { @@ -972,7 +692,7 @@ long TriangleCountExecutor::getTriangleCount( std::string adjustedTransferredFile = transferredFiles.substr(0, transferredFiles.size() - 1); fileCombinationMutex.lock(); - std::map &combinationWorkerMap = *combinationWorkerMap_p; + std::map> &combinationWorkerMap = *combinationWorkerMap_p; if (combinationWorkerMap.find(combinationKey) == combinationWorkerMap.end()) { if (partitionIdSet.find(std::to_string(partitionId)) != partitionIdSet.end()) { combinationWorkerMap[combinationKey] = std::to_string(partitionId); @@ -1110,13 +830,10 @@ static int updateTriangleTreeAndGetTriangleCount( return aggregateCount; } -static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, - int threadPriority, - const std::map> &partitionMap) { - OTEL_TRACE_FUNCTION(); - - vector partitionsVector; - std::map partitionWorkerMap; // partition_id => worker_id +static std::pair, std::map>> buildPartitionInfo( + const std::map, std::less<>> &partitionMap) { + std::vector partitionsVector; + std::map> partitionWorkerMap; // partition_id => worker_id for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { const auto &parts = it->second; string worker = it->first; @@ -1126,17 +843,13 @@ static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::strin partitionsVector.push_back(partition); } } + return {partitionsVector, partitionWorkerMap}; +} - const std::vector> &partitionCombinations = AbstractExecutor::getCombinations(partitionsVector); - std::map workerWeightMap; - std::vector triangleCountThreads; - std::vector triangleCountResponse(partitionCombinations.size(), ""); - std::string result = ""; - long aggregatedTriangleCount = 0; - +static std::map, std::less<>> fetchWorkerDataMap(SQLiteDBInterface *sqlite) { const std::vector>> &workerDataResult = sqlite->runSelect("SELECT DISTINCT idworker,ip,server_port,server_data_port FROM worker;"); - map> workerDataMap; // worker_id => [ip,port,data_port] + std::map, std::less<>> workerDataMap; // worker_id => [ip,port,data_port] for (auto it = workerDataResult.begin(); it != workerDataResult.end(); it++) { const auto &ipPortDport = *it; string id = ipPortDport[0].second; @@ -1145,36 +858,131 @@ static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::strin string dport = ipPortDport[3].second; workerDataMap[id] = {ip, port, dport}; } + return workerDataMap; +} - int comboIndex = 0; - for (auto partitonCombinationsIterator = partitionCombinations.begin(); - partitonCombinationsIterator != partitionCombinations.end(); partitonCombinationsIterator++) { - const std::vector &partitionCombination = *partitonCombinationsIterator; - std::vector remoteGraphCopyThreads; - int minimumWeight = 0; - std::string minWeightWorker; - std::string minWeightWorkerPartition; - - for (auto partCombinationIterator = partitionCombination.begin(); - partCombinationIterator != partitionCombination.end(); partCombinationIterator++) { - string part = *partCombinationIterator; - string workerId = partitionWorkerMap[part]; - auto workerWeightMapIterator = workerWeightMap.find(workerId); - if (workerWeightMapIterator != workerWeightMap.end()) { - int weight = workerWeightMapIterator->second; - - if (minimumWeight == 0 || minimumWeight > weight) { - minimumWeight = weight + 1; - minWeightWorker = workerId; - minWeightWorkerPartition = part; - } - } else { - minimumWeight = 1; +static std::pair findMinWeightWorker( + const std::vector &partitionCombination, + const std::map> &partitionWorkerMap, + std::map> &workerWeightMap) { + + int minimumWeight = 0; + std::string minWeightWorker; + std::string minWeightWorkerPartition; + + for (auto partCombinationIterator = partitionCombination.begin(); + partCombinationIterator != partitionCombination.end(); partCombinationIterator++) { + string part = *partCombinationIterator; + auto iter = partitionWorkerMap.find(part); + if (iter == partitionWorkerMap.end()) { + continue; + } + string workerId = iter->second; + auto workerWeightMapIterator = workerWeightMap.find(workerId); + if (workerWeightMapIterator != workerWeightMap.end()) { + int weight = workerWeightMapIterator->second; + + if (minimumWeight == 0 || minimumWeight > weight) { + minimumWeight = weight + 1; minWeightWorker = workerId; minWeightWorkerPartition = part; } + } else { + minimumWeight = 1; + minWeightWorker = workerId; + minWeightWorkerPartition = part; + } + } + workerWeightMap[minWeightWorker] = minimumWeight; + return {minWeightWorker, minWeightWorkerPartition}; +} + +struct CopyContext { + const std::string &graphId; + const std::string &masterIP; + const std::string &minWeightWorker; + const std::string &minWeightWorkerPartition; + const std::string &aggregatorIp; + const std::string &aggregatorPort; + const std::string &aggregatorDataPort; +}; + +static std::string buildPartitionIdListAndCopyFiles( + const std::vector &partitionCombination, + const std::map> &partitionWorkerMap, + const CopyContext &ctx) { + + std::string partitionIdList = ""; + std::vector remoteGraphCopyThreads; + + for (auto partitionCombinationIterator = partitionCombination.begin(); + partitionCombinationIterator != partitionCombination.end(); ++partitionCombinationIterator) { + string part = *partitionCombinationIterator; + auto iter = partitionWorkerMap.find(part); + if (iter == partitionWorkerMap.end()) { + continue; + } + string workerId = iter->second; + + if (part != ctx.minWeightWorkerPartition) { + partitionIdList += part + ","; + } + if (workerId != ctx.minWeightWorker) { + std::string centralStoreAvailable = isFileAccessibleToWorker( + ctx.graphId, part, ctx.aggregatorIp, ctx.aggregatorPort, ctx.masterIP, + JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_AGGREGATE, std::string()); + + if (centralStoreAvailable.compare("false") == 0) { + int graphIdInt = atoi(ctx.graphId.c_str()); + int partInt = atoi(part.c_str()); + std::string aggregatorIp = ctx.aggregatorIp; + std::string aggregatorPort = ctx.aggregatorPort; + std::string aggregatorDataPort = ctx.aggregatorDataPort; + std::string masterIP = ctx.masterIP; + remoteGraphCopyThreads.push_back(std::thread([aggregatorIp, aggregatorPort, aggregatorDataPort, + graphIdInt, partInt, masterIP]() { + TriangleCountExecutor::copyCentralStoreToAggregator(aggregatorIp, + aggregatorPort, aggregatorDataPort, graphIdInt, partInt, masterIP); + })); + } + } + } + + for (auto &thread : remoteGraphCopyThreads) { + if (thread.joinable()) { + thread.join(); } - workerWeightMap[minWeightWorker] = minimumWeight; + } + + if (partitionIdList.empty()) { + return ""; + } + return partitionIdList.substr(0, partitionIdList.size() - 1); +} + +long TriangleCountExecutor::aggregateCentralStoreTriangles( + SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, int threadPriority, + const std::map, std::less<>> &partitionMap) { + OTEL_TRACE_FUNCTION(); + + auto [partitionsVector, partitionWorkerMap] = buildPartitionInfo(partitionMap); + + const std::vector> &partitionCombinations = AbstractExecutor::getCombinations(partitionsVector); + std::map> workerWeightMap; + std::vector triangleCountThreads; + std::vector triangleCountResponse(partitionCombinations.size(), ""); + std::string result = ""; + long aggregatedTriangleCount = 0; + + std::map, std::less<>> workerDataMap = fetchWorkerDataMap(sqlite); + + int comboIndex = 0; + for (auto partitonCombinationsIterator = partitionCombinations.begin(); + partitonCombinationsIterator != partitionCombinations.end(); partitonCombinationsIterator++) { + const std::vector &partitionCombination = *partitonCombinationsIterator; + + auto [minWeightWorker, minWeightWorkerPartition] = findMinWeightWorker( + partitionCombination, partitionWorkerMap, workerWeightMap); const auto &workerData = workerDataMap[minWeightWorker]; std::string aggregatorIp = workerData[0]; @@ -1183,39 +991,18 @@ static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::strin std::string aggregatorPartitionId = minWeightWorkerPartition; - std::string partitionIdList = ""; - for (auto partitionCombinationIterator = partitionCombination.begin(); - partitionCombinationIterator != partitionCombination.end(); ++partitionCombinationIterator) { - string part = *partitionCombinationIterator; - string workerId = partitionWorkerMap[part]; + CopyContext copyCtx{ + graphId, + masterIP, + minWeightWorker, + minWeightWorkerPartition, + aggregatorIp, + aggregatorPort, + aggregatorDataPort + }; - if (part != minWeightWorkerPartition) { - partitionIdList += part + ","; - } - if (workerId != minWeightWorker) { - std::string centralStoreAvailable = isFileAccessibleToWorker( - graphId, part, aggregatorIp, aggregatorPort, masterIP, - JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_AGGREGATE, std::string()); - - if (centralStoreAvailable.compare("false") == 0) { - int graphIdInt = atoi(graphId.c_str()); - int partInt = atoi(part.c_str()); - remoteGraphCopyThreads.push_back(std::thread([aggregatorIp, aggregatorPort, aggregatorDataPort, - graphIdInt, partInt, masterIP]() { - TriangleCountExecutor::copyCentralStoreToAggregator(aggregatorIp, - aggregatorPort, aggregatorDataPort, graphIdInt, partInt, masterIP); - })); - } - } - } - - for (auto &thread : remoteGraphCopyThreads) { - if (thread.joinable()) { - thread.join(); - } - } - - std::string adjustedPartitionIdList = partitionIdList.substr(0, partitionIdList.size() - 1); + std::string adjustedPartitionIdList = buildPartitionIdListAndCopyFiles( + partitionCombination, partitionWorkerMap, copyCtx); // Capture current trace context before async call std::string currentTraceContext = OpenTelemetryUtil::getCurrentTraceContext(); @@ -1240,7 +1027,7 @@ static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::strin } const std::vector &triangles = Utils::split(result, ':'); - std::set uniqueTriangleSet; + std::set> uniqueTriangleSet; for (auto triangleIterator = triangles.begin(); triangleIterator != triangles.end(); ++triangleIterator) { std::string triangle = *triangleIterator; @@ -2002,3 +1789,426 @@ int TriangleCountExecutor::getUid() { static std::atomic uid{0}; return ++uid; } + +static std::map, std::less<>> buildPartitionMap( + SQLiteDBInterface *sqlite, const std::string &graphId, const std::string &logPrefix, Logger &logger) { + string sqlStatement = + "SELECT DISTINCT worker_idworker,partition_idpartition " + "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " + "WHERE partition_graph_idgraph=" + graphId + ";"; + + const std::vector>> &results = sqlite->runSelect(sqlStatement); + std::map, std::less<>> partitionMap; + + for (auto i = results.begin(); i != results.end(); ++i) { + const std::vector> &rowData = *i; + string workerID = rowData.at(0).second; + string partitionId = rowData.at(1).second; + if (partitionMap.find(workerID) == partitionMap.end()) { + std::vector partitionVec; + partitionVec.push_back(partitionId); + partitionMap[workerID] = partitionVec; + } else { + partitionMap[workerID].push_back(partitionId); + } + logger.info(logPrefix + " Getting Triangle Count : PartitionId " + partitionId); + } + + if (jasminegraph_profile == PROFILE_K8S) { + std::unique_ptr k8sInterface(new K8sInterface()); + if (k8sInterface->getJasmineGraphConfig("auto_scaling_enabled") == "true") { + filter_partitions(partitionMap, sqlite, graphId); + } + } + return partitionMap; +} + +static std::vector> getCompositeFileCombinations(const std::string &graphId) { + std::vector compositeCentralStoreFiles; + std::string aggregatorFilePath = Utils::getJasmineGraphProperty( + "org.jasminegraph.server.instance.aggregatefolder"); + std::vector graphFiles = Utils::getListOfFilesInDirectory(aggregatorFilePath); + std::string compositeFileNameFormat = graphId + "_compositecentralstore_"; + for (auto graphFilesIterator = graphFiles.begin(); graphFilesIterator != graphFiles.end(); + ++graphFilesIterator) { + std::string graphFileName = *graphFilesIterator; + if ((graphFileName.find(compositeFileNameFormat) == 0) && + (graphFileName.find(".gz") != std::string::npos)) { + compositeCentralStoreFiles.push_back(graphFileName); + } + } + return AbstractExecutor::getCombinations(compositeCentralStoreFiles); +} + +struct WorkerTaskContext { + int graphIdInt; + std::string masterIP; + int uniqueId; + bool isCompositeAggregation; + int threadPriority; + const std::vector> &fileCombinations; + std::map> &combinationWorkerMap; + std::unordered_map>> &triangleTree; + std::mutex &triangleTreeMutex; + std::string masterTraceContext; +}; + +struct TaskCollector { + std::vector> &workerTaskInfo; + std::vector &intermThreads; + std::vector &intermResThread; + std::vector> &intermResFuture; + ThreadingStrategy strategy; +}; + +static int distributeTasksToWorkers( + SQLiteDBInterface *sqlite, + const std::map, std::less<>> &partitionMap, + const WorkerTaskContext &taskCtx, + TaskCollector &collector) { + + vector workerList = Utils::getWorkerList(sqlite); + int workerListSize = workerList.size(); + int partitionCount = 0; + + for (int i = 0; i < workerListSize; i++) { + Utils::worker currentWorker = workerList.at(i); + string host = currentWorker.hostname; + string workerID = currentWorker.workerID; + int workerPort = atoi(string(currentWorker.port).c_str()); + int workerDataPort = atoi(string(currentWorker.dataPort).c_str()); + triangleCount_logger.info("worker_" + workerID + " host=" + host + ":" + to_string(workerPort) + ":" + + to_string(workerDataPort)); + + auto partitionIter = partitionMap.find(workerID); + if (partitionIter == partitionMap.end()) { + continue; + } + const std::vector &partitionList = partitionIter->second; + for (auto partitionIterator = partitionList.begin(); partitionIterator != partitionList.end(); + ++partitionIterator) { + string partitionId = *partitionIterator; + triangleCount_logger.info("> partition" + partitionId); + collector.workerTaskInfo.emplace_back(workerID, partitionId, host); + OTEL_TRACE_OPERATION("distribute_to_worker_" + workerID + "_partition_" + partitionId); + int partitionIdInt = atoi(partitionId.c_str()); + + if (collector.strategy == ThreadingStrategy::THREAD_BASED) { + int currentIndex = partitionCount; + collector.intermResThread.push_back(0); + + std::map> *combWorkerMapPtr = &taskCtx.combinationWorkerMap; + std::unordered_map>> *triangleTreePtr = + &taskCtx.triangleTree; + std::mutex *triangleTreeMutexPtr = &taskCtx.triangleTreeMutex; + std::vector> fileCombs = taskCtx.fileCombinations; + std::string mTraceCtx = taskCtx.masterTraceContext; + std::vector &intermResThreadRef = collector.intermResThread; + std::string mIP = taskCtx.masterIP; + int gId = taskCtx.graphIdInt; + int uId = taskCtx.uniqueId; + bool isCompAgg = taskCtx.isCompositeAggregation; + int tPriority = taskCtx.threadPriority; + + collector.intermThreads.push_back(std::thread([&intermResThreadRef, currentIndex, gId, host, workerPort, + workerDataPort, partitionIdInt, mIP, uId, isCompAgg, tPriority, + fileCombs, combWorkerMapPtr, triangleTreePtr, triangleTreeMutexPtr, mTraceCtx]() { + intermResThreadRef[currentIndex] = TriangleCountExecutor::getTriangleCount(gId, host, + workerPort, workerDataPort, partitionIdInt, mIP, uId, + isCompAgg, tPriority, fileCombs, combWorkerMapPtr, + triangleTreePtr, triangleTreeMutexPtr, mTraceCtx); + })); + } else { + std::map> *combWorkerMapPtr = &taskCtx.combinationWorkerMap; + std::unordered_map>> *triangleTreePtr = + &taskCtx.triangleTree; + std::mutex *triangleTreeMutexPtr = &taskCtx.triangleTreeMutex; + std::vector> fileCombs = taskCtx.fileCombinations; + std::string mTraceCtx = taskCtx.masterTraceContext; + std::string mIP = taskCtx.masterIP; + int gId = taskCtx.graphIdInt; + int uId = taskCtx.uniqueId; + bool isCompAgg = taskCtx.isCompositeAggregation; + int tPriority = taskCtx.threadPriority; + + std::packaged_task task([gId, host, workerPort, workerDataPort, + partitionIdInt, mIP, uId, isCompAgg, tPriority, fileCombs, + combWorkerMapPtr, triangleTreePtr, triangleTreeMutexPtr, mTraceCtx]() { + return TriangleCountExecutor::getTriangleCount(gId, host, + workerPort, workerDataPort, partitionIdInt, mIP, uId, + isCompAgg, tPriority, fileCombs, combWorkerMapPtr, + triangleTreePtr, triangleTreeMutexPtr, mTraceCtx); + }); + collector.intermResFuture.push_back(task.get_future()); + collector.intermThreads.push_back(std::thread(std::move(task))); + } + partitionCount++; + } + } + return partitionCount; +} + +static std::vector handleSLACalibration( + PerformanceSQLiteDBInterface *perfDB, + const std::string &graphId, + int partitionCount, + const std::string &commandName, + const std::string &masterIP, + bool autoCalibrate, + bool &canCalibrate) { + + PerformanceUtil::init(); + std::string logPrefix = (commandName == TRIANGLES) ? "###TRIANGLE-COUNT-EXECUTOR###" + : "###SHEEP-TRIANGLE-COUNT-EXECUTOR###"; + std::string query = "SELECT attempt from graph_sla INNER JOIN sla_category where " + "graph_sla.id_sla_category=sla_category.id and graph_sla.graph_id='" + graphId + + "' and graph_sla.partition_count='" + std::to_string(partitionCount) + + "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + + "' and sla_category.command='" + commandName + "';"; + const std::vector>> &queryResults = perfDB->runSelect(query); + std::vector statThreads; + + if (queryResults.size() > 0) { + std::string attemptString = queryResults[0][0].second; + int calibratedAttempts = atoi(attemptString.c_str()); + if (calibratedAttempts >= Conts::MAX_SLA_CALIBRATE_ATTEMPTS) { + canCalibrate = false; + } + } else { + triangleCount_logger.log(logPrefix + " Inserting initial record for SLA ", "info"); + Utils::updateSLAInformation(perfDB, graphId, partitionCount, 0, commandName, Conts::SLA_CATEGORY::LATENCY); + statThreads.push_back(std::thread([perfDB, graphId, commandName, partitionCount, masterIP, autoCalibrate]() { + AbstractExecutor::collectPerformaceData(perfDB, graphId, commandName, Conts::SLA_CATEGORY::LATENCY, + partitionCount, masterIP, autoCalibrate); + })); + isStatCollect = true; + } + return statThreads; +} + +static long gatherWorkerResults( + ThreadingStrategy strategy, + int uniqueId, + const std::vector> &workerTaskInfo, + std::vector &intermThreads, + const std::vector &intermResThread, + std::vector> &intermResFuture, + Logger &logger) { + + long totalResult = 0; + int taskIndex = 0; + if (strategy == ThreadingStrategy::THREAD_BASED) { + for (auto &intermThread : intermThreads) { + const auto& [workerID, partitionId, host] = workerTaskInfo[taskIndex]; + OTEL_TRACE_OPERATION("wait_for_worker_" + workerID + "_partition_" + partitionId + "_on_" + host); + logger.info("Waiting for result from worker_" + workerID + " partition_" + partitionId + + " host_" + host + " uuid=" + to_string(uniqueId)); + if (intermThread.joinable()) { intermThread.join(); } + long worker_result = intermResThread[taskIndex]; + logger.info("Received result " + std::to_string(worker_result) + " from worker_" + + workerID + " partition_" + partitionId); + OTEL_TRACE_OPERATION("aggregate_result_worker_" + workerID + "_partition_" + partitionId); + totalResult += worker_result; + taskIndex++; + } + } else { + for (auto &&futureCall : intermResFuture) { + const auto& [workerID, partitionId, host] = workerTaskInfo[taskIndex]; + OTEL_TRACE_OPERATION("wait_for_worker_" + workerID + "_partition_" + partitionId + "_on_" + host); + logger.info("Waiting for result from worker_" + workerID + " partition_" + partitionId + + " host_" + host + " uuid=" + to_string(uniqueId)); + long worker_result = futureCall.get(); + logger.info("Received result " + std::to_string(worker_result) + " from worker_" + + workerID + " partition_" + partitionId); + OTEL_TRACE_OPERATION("aggregate_result_worker_" + workerID + "_partition_" + partitionId); + totalResult += worker_result; + taskIndex++; + } + for (auto &intermThread : intermThreads) { + if (intermThread.joinable()) { + intermThread.join(); + } + } + } + return totalResult; +} + +void TriangleCountExecutor::executeTriangleCount(SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfDB, + const JobRequest &request, TriangleCountCommandType commandType, + ThreadingStrategy strategy, Logger &logger) { + std::string graphId = request.getParameter(Conts::PARAM_KEYS::GRAPH_ID); + OTEL_TRACE_FUNCTION(); + + std::unique_lock lock(schedulerMutex, std::defer_lock); + lock.lock(); + if (time_t curr_time = time(nullptr); curr_time < last_exec_time + 8) { + time_t sleep_duration = last_exec_time + 9 - curr_time; + last_exec_time = curr_time + sleep_duration; + lock.unlock(); + std::this_thread::sleep_for(std::chrono::seconds(sleep_duration)); + lock.lock(); + } else { + last_exec_time = curr_time; + } + + int uniqueId = getUid(); + std::string masterIP = request.getMasterIP(); + std::string canCalibrateString = request.getParameter(Conts::PARAM_KEYS::CAN_CALIBRATE); + std::string queueTime = request.getParameter(Conts::PARAM_KEYS::QUEUE_TIME); + bool canCalibrate = Utils::parseBoolean(canCalibrateString); + int threadPriority = request.getPriority(); + std::string autoCalibrateString = request.getParameter(Conts::PARAM_KEYS::AUTO_CALIBRATION); + bool autoCalibrate = Utils::parseBoolean(autoCalibrateString); + + if (threadPriority == Conts::HIGH_PRIORITY_DEFAULT_VALUE) { + highPriorityGraphList.push_back(graphId); + } + + std::string commandName = (commandType == TriangleCountCommandType::TRIANGLES) ? TRIANGLES : SHEEP_TRIANGLES; + std::string logPrefix = (commandType == TriangleCountCommandType::TRIANGLES) + ? "###TRIANGLE-COUNT-EXECUTOR###" + : "###SHEEP-TRIANGLE-COUNT-EXECUTOR###"; + + std::chrono::milliseconds startTime = duration_cast(system_clock::now().time_since_epoch()); + struct ProcessInfo processInformation; + processInformation.id = uniqueId; + processInformation.graphId = graphId; + processInformation.processName = commandName; + processInformation.priority = threadPriority; + processInformation.startTimestamp = startTime.count(); + + if (!queueTime.empty()) { + long sleepTime = atol(queueTime.c_str()); + processInformation.sleepTime = sleepTime; + insertProcessInfo(processInformation); + std::this_thread::sleep_for(std::chrono::milliseconds(sleepTime)); + } else { + insertProcessInfo(processInformation); + } + + logger.log(logPrefix + " Started with graph ID : " + graphId + " Master IP : " + masterIP, "info"); + + long result = 0; + bool isCompositeAggregation = false; + auto begin = chrono::high_resolution_clock::now(); + + std::map, std::less<>> partitionMap = + buildPartitionMap(sqlite, graphId, logPrefix, logger); + + size_t totalPartitions = 0; + for (const auto &[worker, partitions] : partitionMap) { + totalPartitions += partitions.size(); + } + if (totalPartitions > Conts::COMPOSITE_CENTRAL_STORE_WORKER_THRESHOLD) { + isCompositeAggregation = true; + } + + std::vector> fileCombinations; + if (isCompositeAggregation) { + fileCombinations = getCompositeFileCombinations(graphId); + } + + for (const auto &[worker, partitions] : partitionMap) { + if (used_workers.find(worker) != used_workers.end()) { + used_workers[worker]++; + } else { + used_workers[worker] = 1; + } + } + + std::map> combinationWorkerMap; + std::unordered_map>> triangleTree; + std::mutex triangleTreeMutex; + std::vector> workerTaskInfo; + std::string masterTraceContext = OpenTelemetryUtil::getCurrentTraceContext(); + + std::vector intermThreads; + std::vector intermResThread; + std::vector> intermResFuture; + + WorkerTaskContext taskCtx{ + atoi(graphId.c_str()), + masterIP, + uniqueId, + isCompositeAggregation, + threadPriority, + fileCombinations, + combinationWorkerMap, + triangleTree, + triangleTreeMutex, + masterTraceContext + }; + + TaskCollector collector{ + workerTaskInfo, + intermThreads, + intermResThread, + intermResFuture, + strategy + }; + + int partitionCount = distributeTasksToWorkers(sqlite, partitionMap, taskCtx, collector); + + std::vector statThreads = handleSLACalibration( + perfDB, graphId, partitionCount, commandName, masterIP, autoCalibrate, canCalibrate); + + if (time_t actual_time = time(nullptr); actual_time > last_exec_time) { + last_exec_time = actual_time; + } + lock.unlock(); + + { + OTEL_TRACE_OPERATION("collect_worker_results"); + result += gatherWorkerResults(strategy, uniqueId, workerTaskInfo, intermThreads, intermResThread, + intermResFuture, logger); + + OTEL_TRACE_OPERATION("cleanup_worker_data_structures"); + triangleTree.clear(); + combinationWorkerMap.clear(); + } + + if (!isCompositeAggregation) { + OpenTelemetryUtil::receiveAndSetTraceContext(masterTraceContext, "central store aggregation"); + OTEL_TRACE_OPERATION("central_store_aggregation"); + long aggregatedTriangleCount = aggregateCentralStoreTriangles(sqlite, graphId, masterIP, + threadPriority, partitionMap); + result += aggregatedTriangleCount; + workerResponded = true; + logger.log(logPrefix + " Getting Triangle Count : Completed: Triangles " + to_string(result), "info"); + } + + { + std::scoped_lock scheduler_lock(schedulerMutex); + for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { + string worker = it->first; + used_workers[worker]--; + } + for (auto it = used_workers.cbegin(); it != used_workers.cend();) { + if (it->second <= 0) { + used_workers.erase(it++); + } else { + it++; + } + } + } + workerResponded = true; + + JobResponse jobResponse; + jobResponse.setJobId(request.getJobId()); + jobResponse.addParameter(Conts::PARAM_KEYS::TRIANGLE_COUNT, std::to_string(result)); + responseVectorMutex.lock(); + responseVector.push_back(jobResponse); + responseMap[request.getJobId()] = jobResponse; + responseVectorMutex.unlock(); + + auto end = chrono::high_resolution_clock::now(); + auto dur = end - begin; + auto msDuration = std::chrono::duration_cast(dur).count(); + if (canCalibrate || autoCalibrate) { + Utils::updateSLAInformation(perfDB, graphId, partitionCount, msDuration, commandName, + Conts::SLA_CATEGORY::LATENCY); + isStatCollect = false; + } + removeProcessInfoById(uniqueId); + joinAllThreads(statThreads); +} diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.h b/src/frontend/core/executor/impl/TriangleCountExecutor.h index e8bb1df43..e1a7ab7b2 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.h +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.h @@ -27,6 +27,18 @@ limitations under the License. #include "../../CoreConstants.h" #include "../AbstractExecutor.h" +// Enum for command types (used in logging and SLA tracking) +enum class TriangleCountCommandType { + TRIANGLES, + SHEEP_TRIANGLES +}; + +// Enum for threading strategy +enum class ThreadingStrategy { + THREAD_BASED, // Uses std::thread + ASYNC_BASED // Uses std::async +}; + class TriangleCountExecutor : public AbstractExecutor { public: TriangleCountExecutor(); @@ -35,12 +47,17 @@ class TriangleCountExecutor : public AbstractExecutor { void execute(); - int getUid(); + static int getUid(); + + // Shared execute method for both regular and sheep executors + static void executeTriangleCount(SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfDB, + const JobRequest &request, TriangleCountCommandType commandType, + ThreadingStrategy strategy, Logger &logger); static long getTriangleCount( int graphId, std::string host, int port, int dataPort, int partitionId, std::string masterIP, int uniqueId, bool isCompositeAggregation, int threadPriority, std::vector> fileCombinations, - std::map *combinationWorkerMap_p, + std::map> *combinationWorkerMap_p, std::unordered_map>> *triangleTree_p, std::mutex *triangleTreeMutex_p, const std::string& masterTraceContext); @@ -58,6 +75,10 @@ class TriangleCountExecutor : public AbstractExecutor { std::string aggregatorDataPort, int graphId, int partitionId, std::string masterIP); + static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, + int threadPriority, + const std::map, std::less<>> &partitionMap); + static string countCentralStoreTriangles(std::string aggregatorPort, std::string host, std::string partitionId, std::string partitionIdList, std::string graphId, std::string masterIP, int threadPriority, std::string traceContext); @@ -71,4 +92,27 @@ class TriangleCountExecutor : public AbstractExecutor { PerformanceSQLiteDBInterface *perfDB; }; +// Partition filtering and allocation helper functions (shared between executors) +void allocate(int p, std::string w, std::map &alloc, std::set &remain, + std::map> &p_avail, std::map> &loads); + +int alloc_plan(std::map &alloc, std::set &remain, + std::map> &p_avail, std::map> &loads); + +std::vector reallocate_parts(std::map &alloc, std::set &remain, + const std::map> &P_AVAIL); + +void scale_up(std::map> &loads, + std::map> &workers, int copy_cnt); + +int alloc_net_plan(std::map &alloc, std::vector &parts, + std::map> &transfer, + std::map> &net_loads, std::map> &loads, + const std::map> &P_AVAIL, int C); + +void filter_partitions(std::map, std::less<>> &partitionMap, + SQLiteDBInterface *sqlite, const std::string &graphId); + +// Shared synchronization primitives and state + #endif // JASMINEGRAPH_TRIANGLECOUNTEXECUTOR_H diff --git a/src/frontend/core/factory/ExecutorFactory.cpp b/src/frontend/core/factory/ExecutorFactory.cpp index 3f1e1cad6..fd80d6ccd 100644 --- a/src/frontend/core/factory/ExecutorFactory.cpp +++ b/src/frontend/core/factory/ExecutorFactory.cpp @@ -14,6 +14,7 @@ limitations under the License. #include "ExecutorFactory.h" #include "../executor/impl/TriangleCountExecutor.h" +#include "../executor/impl/SheepTriangleCountExecutor.h" #include "../executor/impl/StreamingTriangleCountExecutor.h" #include "../executor/impl/PageRankExecutor.h" #include "../executor/impl/CypherQueryExecutor.h" @@ -25,19 +26,21 @@ ExecutorFactory::ExecutorFactory(SQLiteDBInterface *db, PerformanceSQLiteDBInter this->perfDB = perfDb; } -AbstractExecutor* ExecutorFactory::getExecutor(JobRequest jobRequest) { +std::unique_ptr ExecutorFactory::getExecutor(JobRequest jobRequest) { if (TRIANGLES == jobRequest.getJobType()) { - return new TriangleCountExecutor(this->sqliteDB, this->perfDB, jobRequest); + return std::make_unique(this->sqliteDB, this->perfDB, jobRequest); + } else if (SHEEP_TRIANGLES == jobRequest.getJobType()) { + return std::make_unique(this->sqliteDB, this->perfDB, jobRequest); } else if (STREAMING_TRIANGLES == jobRequest.getJobType()) { - return new StreamingTriangleCountExecutor(this->sqliteDB, jobRequest); + return std::make_unique(this->sqliteDB, jobRequest); } else if (PAGE_RANK == jobRequest.getJobType()) { - return new PageRankExecutor(this->sqliteDB, this->perfDB, jobRequest); + return std::make_unique(this->sqliteDB, this->perfDB, jobRequest); } else if (CYPHER == jobRequest.getJobType()) { - return new CypherQueryExecutor(this->sqliteDB, this->perfDB, jobRequest); + return std::make_unique(this->sqliteDB, this->perfDB, jobRequest); } else if (SEMANTIC_BEAM_SEARCH == jobRequest.getJobType()) { - return new SemanticBeamSearchExecutor(this->sqliteDB, this->perfDB, jobRequest); + return std::make_unique(this->sqliteDB, this->perfDB, jobRequest); } else if (AGENT_PLAN == jobRequest.getJobType()) { - return new AgentPlanExecutor(this->sqliteDB, this->perfDB, jobRequest); + return std::make_unique(this->sqliteDB, this->perfDB, jobRequest); } return nullptr; } diff --git a/src/frontend/core/factory/ExecutorFactory.h b/src/frontend/core/factory/ExecutorFactory.h index c441c3f5b..1a3296c94 100644 --- a/src/frontend/core/factory/ExecutorFactory.h +++ b/src/frontend/core/factory/ExecutorFactory.h @@ -14,6 +14,7 @@ limitations under the License. #ifndef JASMINEGRAPH_EXECUTORFACTORY_H #define JASMINEGRAPH_EXECUTORFACTORY_H +#include #include #include "../../../metadb/SQLiteDBInterface.h" @@ -25,7 +26,7 @@ limitations under the License. class ExecutorFactory { public: ExecutorFactory(SQLiteDBInterface *db, PerformanceSQLiteDBInterface *perfDb); - AbstractExecutor* getExecutor(JobRequest jobRequest); + std::unique_ptr getExecutor(JobRequest jobRequest); private: SQLiteDBInterface *sqliteDB; diff --git a/src/frontend/core/scheduler/JobScheduler.cpp b/src/frontend/core/scheduler/JobScheduler.cpp index 7d88abc24..305af7fcd 100644 --- a/src/frontend/core/scheduler/JobScheduler.cpp +++ b/src/frontend/core/scheduler/JobScheduler.cpp @@ -12,6 +12,7 @@ limitations under the License. */ #include "JobScheduler.h" +#include #include "../../../util/Conts.h" #include "../../../util/logger/Logger.h" @@ -37,67 +38,71 @@ void *startScheduler(void *dummyPt) { JobScheduler *refToScheduler = (JobScheduler *)dummyPt; PerformanceUtil::init(); while (true) { - if (jobQueue.size() > 0) { - jobScheduler_Logger.info("##JOB SCHEDULER## Jobs Available for Scheduling"); - - std::vector pendingHPJobList; - std::vector highPriorityGraphList; - - while (!jobQueue.empty()) { - JobRequest request = jobQueue.top(); - - if (request.getPriority() == Conts::HIGH_PRIORITY_DEFAULT_VALUE && request.getJobType() == TRIANGLES) { - pendingHPJobList.push_back(request); - highPriorityGraphList.push_back(request.getParameter(Conts::PARAM_KEYS::GRAPH_ID)); - jobQueue.pop(); - } else if (request.getPriority() == Conts::HIGH_PRIORITY_DEFAULT_VALUE - && request.getJobType() == CYPHER) { - pendingHPJobList.push_back(request); - highPriorityGraphList.push_back(request.getParameter(Conts::PARAM_KEYS::GRAPH_ID)); - jobQueue.pop(); - } else if (request.getPriority() == Conts::HIGH_PRIORITY_DEFAULT_VALUE - && request.getJobType() == SEMANTIC_BEAM_SEARCH) { - pendingHPJobList.push_back(request); - highPriorityGraphList.push_back(request.getParameter(Conts::PARAM_KEYS::GRAPH_ID)); - jobQueue.pop(); - } else { - jobQueue.pop(); - JobScheduler::processJob(request, refToScheduler->sqlite, refToScheduler->perfSqlite); - } + if (jobQueue.empty()) { + std::this_thread::sleep_for(std::chrono::seconds(Conts::SCHEDULER_SLEEP_TIME)); + continue; + } + + jobScheduler_Logger.info("##JOB SCHEDULER## Jobs Available for Scheduling"); + + std::vector pendingHPJobList; + std::vector highPriorityGraphList; + + while (!jobQueue.empty()) { + JobRequest request = jobQueue.top(); + + if (request.getPriority() == Conts::HIGH_PRIORITY_DEFAULT_VALUE && + (request.getJobType() == TRIANGLES || request.getJobType() == SHEEP_TRIANGLES)) { + pendingHPJobList.push_back(request); + highPriorityGraphList.push_back(request.getParameter(Conts::PARAM_KEYS::GRAPH_ID)); + jobQueue.pop(); + } else if (request.getPriority() == Conts::HIGH_PRIORITY_DEFAULT_VALUE + && request.getJobType() == CYPHER) { + pendingHPJobList.push_back(request); + highPriorityGraphList.push_back(request.getParameter(Conts::PARAM_KEYS::GRAPH_ID)); + jobQueue.pop(); + } else if (request.getPriority() == Conts::HIGH_PRIORITY_DEFAULT_VALUE + && request.getJobType() == SEMANTIC_BEAM_SEARCH) { + pendingHPJobList.push_back(request); + highPriorityGraphList.push_back(request.getParameter(Conts::PARAM_KEYS::GRAPH_ID)); + jobQueue.pop(); + } else { + jobQueue.pop(); + JobScheduler::processJob(request, refToScheduler->sqlite, refToScheduler->perfSqlite); } + } - if (pendingHPJobList.size() > 0) { - jobScheduler_Logger.info("##JOB SCHEDULER## High Priority Jobs in Queue: " + - std::to_string(pendingHPJobList.size())); - std::string masterIP = pendingHPJobList[0].getMasterIP(); - std::string jobType = pendingHPJobList[0].getJobType(); - std::string category = pendingHPJobList[0].getParameter(Conts::PARAM_KEYS::CATEGORY); - - std::vector scheduleTimeVector = PerformanceUtil::getResourceAvailableTime( - highPriorityGraphList, jobType, category, masterIP, pendingHPJobList); - - for (int index = 0; index != pendingHPJobList.size(); ++index) { - JobRequest hpRequest = pendingHPJobList[index]; - long queueTime = scheduleTimeVector[index]; - - if (queueTime < 0) { - JobResponse failedJobResponse; - failedJobResponse.setJobId(hpRequest.getJobId()); - failedJobResponse.addParameter(Conts::PARAM_KEYS::ERROR_MESSAGE, - "Rejecting the job request because " - "SLA cannot be maintained"); - responseVectorMutex.lock(); - responseMap[hpRequest.getJobId()] = failedJobResponse; - responseVectorMutex.unlock(); - continue; - } - - hpRequest.addParameter(Conts::PARAM_KEYS::QUEUE_TIME, std::to_string(queueTime)); - JobScheduler::processJob(hpRequest, refToScheduler->sqlite, refToScheduler->perfSqlite); - } + if (pendingHPJobList.empty()) { + continue; + } + + jobScheduler_Logger.info("##JOB SCHEDULER## High Priority Jobs in Queue: " + + std::to_string(pendingHPJobList.size())); + std::string masterIP = pendingHPJobList[0].getMasterIP(); + std::string jobType = pendingHPJobList[0].getJobType(); + std::string category = pendingHPJobList[0].getParameter(Conts::PARAM_KEYS::CATEGORY); + + std::vector scheduleTimeVector = PerformanceUtil::getResourceAvailableTime( + highPriorityGraphList, jobType, category, masterIP, pendingHPJobList); + + for (int index = 0; index != pendingHPJobList.size(); ++index) { + JobRequest hpRequest = pendingHPJobList[index]; + long queueTime = scheduleTimeVector[index]; + + if (queueTime < 0) { + JobResponse failedJobResponse; + failedJobResponse.setJobId(hpRequest.getJobId()); + failedJobResponse.addParameter(Conts::PARAM_KEYS::ERROR_MESSAGE, + "Rejecting the job request because " + "SLA cannot be maintained"); + responseVectorMutex.lock(); + responseMap[hpRequest.getJobId()] = failedJobResponse; + responseVectorMutex.unlock(); + continue; } - } else { - std::this_thread::sleep_for(std::chrono::seconds(Conts::SCHEDULER_SLEEP_TIME)); + + hpRequest.addParameter(Conts::PARAM_KEYS::QUEUE_TIME, std::to_string(queueTime)); + JobScheduler::processJob(hpRequest, refToScheduler->sqlite, refToScheduler->perfSqlite); } } return NULL; @@ -114,15 +119,13 @@ void JobScheduler::processJob(JobRequest request, SQLiteDBInterface *sqlite, Per } void JobScheduler::executeJob(JobRequest request, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfDB) { - ExecutorFactory *executorFactory = new ExecutorFactory(sqlite, perfDB); - AbstractExecutor *abstractExecutor = executorFactory->getExecutor(request); - delete executorFactory; + ExecutorFactory executorFactory(sqlite, perfDB); + std::unique_ptr abstractExecutor = executorFactory.getExecutor(request); if (abstractExecutor == nullptr) { jobScheduler_Logger.error("abstractExecutor is null"); return; } abstractExecutor->execute(); - delete abstractExecutor; } void JobScheduler::pushJob(JobRequest jobDetails) { jobQueue.push(jobDetails); } diff --git a/src/frontend/ui/JasmineGraphFrontEndUI.cpp b/src/frontend/ui/JasmineGraphFrontEndUI.cpp index 8bf28f5c3..92593abed 100644 --- a/src/frontend/ui/JasmineGraphFrontEndUI.cpp +++ b/src/frontend/ui/JasmineGraphFrontEndUI.cpp @@ -80,14 +80,22 @@ static void remove_graph_command(std::string masterIP, int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p, std::string command); static void remove_all_graphs_command(std::string masterIP, int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p); -static void triangles_command(std::string masterIP, - int connFd, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfSqlite, +static void triangles_command(const std::string &masterIP, + int conn_fd, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfSqlite, + JobScheduler *jobScheduler, bool *loop_exit_p, const std::string &command); +static void sheep_triangles_command(const std::string &masterIP, + int conn_fd, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfSqlite, JobScheduler *jobScheduler, bool *loop_exit_p, std::string command); static void get_degree_command(int connFd, std::string command, int numberOfPartition, std::string type, bool *loop_exit_p); static void cypher_ast_command(int connFd, vector &workerClients, int numberOfPartitions, bool *loop_exit, std::string command); +static bool sendResponse(int conn_fd, const std::string &response, bool *loop_exit_p, size_t length = 0); +static void processTriangleCount(const std::string &masterIP, int conn_fd, SQLiteDBInterface *sqlite, + PerformanceSQLiteDBInterface *perfSqlite, JobScheduler *jobScheduler, + bool *loop_exit_p, const std::string &command); + static void semantic_beam_search_command(int connFd, std::string command, int numberOfPartitions, bool *loop_exit_p , JobScheduler *jobScheduler); static void get_properties_command(int connFd, bool *loop_exit_p); @@ -165,6 +173,8 @@ void *uifrontendservicesesion(void *dummyPt) { add_graph_command(masterIP, connFd, sqlite, &loop_exit, line); } else if (token.compare(TRIANGLES) == 0) { triangles_command(masterIP, connFd, sqlite, perfSqlite, jobScheduler, &loop_exit, line); + } else if (token.compare(SHTRIAN) == 0) { + sheep_triangles_command(masterIP, connFd, sqlite, perfSqlite, jobScheduler, &loop_exit, line); } else if (token.compare(RMGR) == 0) { remove_graph_command(masterIP, connFd, sqlite, &loop_exit, line); } else if (token.compare(TRUNCATE) == 0) { @@ -190,7 +200,7 @@ void *uifrontendservicesesion(void *dummyPt) { JasmineGraphFrontEnd::constructKGStreamLocalTXTCommand(masterIP, connFd, numberOfPartitions, sqlite, &loop_exit); } else if (line.compare(STOP_CONSTRUCT_KG) == 0) { - JasmineGraphFrontEnd::stop_graph_streaming(connFd, sqlite, &loop_exit); + JasmineGraphFrontEnd::stop_graph_streaming(connFd, &loop_exit); } else if (token.compare("UPBYTES") == 0) { send_uploaded_bytes(connFd, sqlite, &loop_exit, line); } else { @@ -282,6 +292,118 @@ int JasmineGraphFrontEndUI::run() { } } +static bool sendResponse(int conn_fd, const std::string &response, bool *loop_exit_p, size_t length) { + size_t write_len = (length > 0) ? length : response.length(); + int result_wr = write(conn_fd, response.c_str(), write_len); + if (result_wr < 0) { + ui_frontend_logger.error("Error writing to socket"); + *loop_exit_p = true; + return false; + } + result_wr = write(conn_fd, Conts::CARRIAGE_RETURN_NEW_LINE.c_str(), Conts::CARRIAGE_RETURN_NEW_LINE.size()); + if (result_wr < 0) { + ui_frontend_logger.error("Error writing to socket"); + *loop_exit_p = true; + return false; + } + return true; +} + +static void processTriangleCount(const std::string &masterIP, int conn_fd, SQLiteDBInterface *sqlite, + PerformanceSQLiteDBInterface *perfSqlite, JobScheduler *jobScheduler, + bool *loop_exit_p, const std::string &command) { + char delimiter = '|'; + std::stringstream ss(command); + std::string token; + std::string graph_id; + std::string priority; + + std::getline(ss, token, delimiter); + std::getline(ss, graph_id, delimiter); + std::string jobType = (token == SHTRIAN) ? SHEEP_TRIANGLES : TRIANGLES; + std::getline(ss, priority, delimiter); + + ui_frontend_logger.info("recieved graph id: " + graph_id); + ui_frontend_logger.info("Priority: " + priority); + + int uniqueId = JasmineGraphFrontEndCommon::getUid(); + + if (!JasmineGraphFrontEndCommon::graphExistsByID(graph_id, sqlite)) { + sendResponse(conn_fd, "The specified graph id does not exist", loop_exit_p, FRONTEND_COMMAND_LENGTH); + return; + } + + if (!(std::find_if(priority.begin(), priority.end(), [](unsigned char c) { return !std::isdigit(c); }) == + priority.end())) { + *loop_exit_p = true; + sendResponse(conn_fd, "Priority should be numeric and > 1 or empty", loop_exit_p); + return; + } + + int threadPriority = std::atoi(priority.c_str()); + + static volatile int reqCounter = 0; + string reqId = to_string(reqCounter++); + ui_frontend_logger.info("Started processing triangle counting request " + reqId); + auto begin = chrono::high_resolution_clock::now(); + JobRequest jobDetails; + jobDetails.setJobId(std::to_string(uniqueId)); + jobDetails.setJobType(jobType); + + long graphSLA = -1; + if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { + threadPriority = Conts::HIGH_PRIORITY_DEFAULT_VALUE; + graphSLA = JasmineGraphFrontEndCommon::getSLAForGraphId(sqlite, perfSqlite, graph_id, jobType, + Conts::SLA_CATEGORY::LATENCY); + jobDetails.addParameter(Conts::PARAM_KEYS::GRAPH_SLA, std::to_string(graphSLA)); + } + + if (graphSLA == 0) { + if (JasmineGraphFrontEnd::areRunningJobsForSameGraph()) { + if (canCalibrate) { + jobDetails.addParameter(Conts::PARAM_KEYS::AUTO_CALIBRATION, "false"); + } else { + jobDetails.addParameter(Conts::PARAM_KEYS::AUTO_CALIBRATION, "true"); + } + } else { + ui_frontend_logger.error("Can't calibrate the graph now"); + } + } + + jobDetails.setPriority(threadPriority); + jobDetails.setMasterIP(masterIP); + jobDetails.addParameter(Conts::PARAM_KEYS::GRAPH_ID, graph_id); + jobDetails.addParameter(Conts::PARAM_KEYS::CATEGORY, Conts::SLA_CATEGORY::LATENCY); + if (canCalibrate) { + jobDetails.addParameter(Conts::PARAM_KEYS::CAN_CALIBRATE, "true"); + } else { + jobDetails.addParameter(Conts::PARAM_KEYS::CAN_CALIBRATE, "false"); + } + + jobScheduler->pushJob(jobDetails); + JobResponse jobResponse = jobScheduler->getResult(jobDetails); + std::string errorMessage = jobResponse.getParameter(Conts::PARAM_KEYS::ERROR_MESSAGE); + + if (!errorMessage.empty()) { + *loop_exit_p = true; + sendResponse(conn_fd, errorMessage, loop_exit_p); + return; + } + + std::string triangleCount = jobResponse.getParameter(Conts::PARAM_KEYS::TRIANGLE_COUNT); + + if (threadPriority == Conts::HIGH_PRIORITY_DEFAULT_VALUE) { + highPriorityTaskCount--; + } + + auto end = chrono::high_resolution_clock::now(); + auto dur = end - begin; + auto msDuration = std::chrono::duration_cast(dur).count(); + ui_frontend_logger.info("Req: " + reqId + " Triangle Count: " + triangleCount + + " Time Taken: " + to_string(msDuration) + " milliseconds"); + sendResponse(conn_fd, triangleCount, loop_exit_p); +} + static void list_command(int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p) { ui_frontend_logger.debug("list_command: started"); @@ -776,145 +898,16 @@ static void remove_all_graphs_command(std::string masterIP, } } -static void triangles_command(std::string masterIP, int connFd, +static void triangles_command(const std::string &masterIP, + int conn_fd, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfSqlite, + JobScheduler *jobScheduler, bool *loop_exit_p, const std::string &command) { + processTriangleCount(masterIP, conn_fd, sqlite, perfSqlite, jobScheduler, loop_exit_p, command); +} + +static void sheep_triangles_command(const std::string &masterIP, int conn_fd, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfSqlite, JobScheduler *jobScheduler, bool *loop_exit_p, std::string command) { - char delimiter = '|'; - std::stringstream ss(command); - std::string token; - std::string graph_id; - std::string priority; - - std::getline(ss, token, delimiter); - std::getline(ss, graph_id, delimiter); - std::getline(ss, priority, delimiter); - - ui_frontend_logger.info("recieved graph id: " + graph_id); - ui_frontend_logger.info("Priority: " + priority); - - // add RDF graph - int uniqueId = JasmineGraphFrontEndCommon::getUid(); - - string name = ""; - - if (!JasmineGraphFrontEndCommon::graphExistsByID(graph_id, sqlite)) { - string error_message = "The specified graph id does not exist"; - int result_wr = write(connFd, error_message.c_str(), FRONTEND_COMMAND_LENGTH); - if (result_wr < 0) { - ui_frontend_logger.error("Error writing to socket"); - *loop_exit_p = true; - return; - } - - result_wr = write(connFd, Conts::CARRIAGE_RETURN_NEW_LINE.c_str(), Conts::CARRIAGE_RETURN_NEW_LINE.size()); - if (result_wr < 0) { - ui_frontend_logger.error("Error writing to socket"); - *loop_exit_p = true; - } - } else { - if (!(std::find_if(priority.begin(), priority.end(), [](unsigned char c) { return !std::isdigit(c); }) == - priority.end())) { - *loop_exit_p = true; - string error_message = "Priority should be numeric and > 1 or empty"; - int result_wr = write(connFd, error_message.c_str(), error_message.length()); - if (result_wr < 0) { - ui_frontend_logger.error("Error writing to socket"); - return; - } - - result_wr = write(connFd, Conts::CARRIAGE_RETURN_NEW_LINE.c_str(), Conts::CARRIAGE_RETURN_NEW_LINE.size()); - if (result_wr < 0) { - ui_frontend_logger.error("Error writing to socket"); - } - return; - } - - int threadPriority = std::atoi(priority.c_str()); - - static volatile int reqCounter = 0; - string reqId = to_string(reqCounter++); - ui_frontend_logger.info("Started processing request " + reqId); - auto begin = chrono::high_resolution_clock::now(); - JobRequest jobDetails; - jobDetails.setJobId(std::to_string(uniqueId)); - jobDetails.setJobType(TRIANGLES); - - long graphSLA = -1; // This prevents auto calibration for priority=1 (=default priority) - if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { - // All high priority threads will be set the same high priority level - threadPriority = Conts::HIGH_PRIORITY_DEFAULT_VALUE; - graphSLA = JasmineGraphFrontEndCommon::getSLAForGraphId(sqlite, perfSqlite, graph_id, TRIANGLES, - Conts::SLA_CATEGORY::LATENCY); - jobDetails.addParameter(Conts::PARAM_KEYS::GRAPH_SLA, std::to_string(graphSLA)); - } - - if (graphSLA == 0) { - if (JasmineGraphFrontEnd::areRunningJobsForSameGraph()) { - if (canCalibrate) { - // initial calibration - jobDetails.addParameter(Conts::PARAM_KEYS::AUTO_CALIBRATION, "false"); - } else { - // auto calibration - jobDetails.addParameter(Conts::PARAM_KEYS::AUTO_CALIBRATION, "true"); - } - } else { - // TODO(ASHOK12011234): Need to investigate for multiple graphs - ui_frontend_logger.error("Can't calibrate the graph now"); - } - } - - jobDetails.setPriority(threadPriority); - jobDetails.setMasterIP(masterIP); - jobDetails.addParameter(Conts::PARAM_KEYS::GRAPH_ID, graph_id); - jobDetails.addParameter(Conts::PARAM_KEYS::CATEGORY, Conts::SLA_CATEGORY::LATENCY); - if (canCalibrate) { - jobDetails.addParameter(Conts::PARAM_KEYS::CAN_CALIBRATE, "true"); - } else { - jobDetails.addParameter(Conts::PARAM_KEYS::CAN_CALIBRATE, "false"); - } - - jobScheduler->pushJob(jobDetails); - JobResponse jobResponse = jobScheduler->getResult(jobDetails); - std::string errorMessage = jobResponse.getParameter(Conts::PARAM_KEYS::ERROR_MESSAGE); - - if (!errorMessage.empty()) { - *loop_exit_p = true; - int result_wr = write(connFd, errorMessage.c_str(), errorMessage.length()); - - if (result_wr < 0) { - ui_frontend_logger.error("Error writing to socket"); - return; - } - result_wr = write(connFd, Conts::CARRIAGE_RETURN_NEW_LINE.c_str(), Conts::CARRIAGE_RETURN_NEW_LINE.size()); - if (result_wr < 0) { - ui_frontend_logger.error("Error writing to socket"); - } - return; - } - - std::string triangleCount = jobResponse.getParameter(Conts::PARAM_KEYS::TRIANGLE_COUNT); - - if (threadPriority == Conts::HIGH_PRIORITY_DEFAULT_VALUE) { - highPriorityTaskCount--; - } - - auto end = chrono::high_resolution_clock::now(); - auto dur = end - begin; - auto msDuration = std::chrono::duration_cast(dur).count(); - ui_frontend_logger.info("Req: " + reqId + " Triangle Count: " + triangleCount + - " Time Taken: " + to_string(msDuration) + " milliseconds"); - int result_wr = write(connFd, triangleCount.c_str(), triangleCount.length()); - if (result_wr < 0) { - ui_frontend_logger.error("Error writing to socket"); - *loop_exit_p = true; - return; - } - result_wr = write(connFd, Conts::CARRIAGE_RETURN_NEW_LINE.c_str(), Conts::CARRIAGE_RETURN_NEW_LINE.size()); - if (result_wr < 0) { - ui_frontend_logger.error("Error writing to socket"); - *loop_exit_p = true; - } - } + processTriangleCount(masterIP, conn_fd, sqlite, perfSqlite, jobScheduler, loop_exit_p, command); } static void get_degree_command(int connFd, std::string command, int numberOfPartition, diff --git a/src/partitioner/local/SheepPartitioner.cpp b/src/partitioner/local/SheepPartitioner.cpp new file mode 100644 index 000000000..dfbda294d --- /dev/null +++ b/src/partitioner/local/SheepPartitioner.cpp @@ -0,0 +1,454 @@ +/** +Copyright 2026 JasminGraph Team +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +#include "SheepPartitioner.h" +#include "../../util/logger/Logger.h" +#include +#include +#include +#include +#include + +using namespace std; + +Logger sheep_partitioner_logger; + +SheepPartitioner::SheepPartitioner(SQLiteDBInterface *sqlite) + : sqlite(sqlite) {} + +bool SheepPartitioner::loadGraph(const string &graphPath) { + sheep_partitioner_logger.info("Loading graph from: " + graphPath); + + ifstream inputFile(graphPath); + if (!inputFile.is_open()) { + sheep_partitioner_logger.error("Failed to open graph file: " + graphPath); + return false; + } + + string line; + size_t lineCount = 0; + + while (getline(inputFile, line)) { + lineCount++; + + // Skip empty lines and comments + if (line.empty() || line[0] == '#') { + continue; + } + + vertex_id source; + vertex_id target; + std::istringstream iss(line); + + if (iss >> source >> target) { + // Add edge to adjacency list + adjacencyList[source].push_back(target); + adjacencyList[target].push_back(source); + totalEdges++; + } + } + + inputFile.close(); + + sheep_partitioner_logger.info("Loaded " + to_string(adjacencyList.size()) + + " vertices and " + to_string(totalEdges) + " edges"); + return true; +} + +std::vector SheepPartitioner::getDegreeSequence() const { + sheep_partitioner_logger.info("Computing degree sequence"); + + // Create vector of (vertex, degree) pairs + std::vector> vertexDegrees; + vertexDegrees.reserve(adjacencyList.size()); + + for (const auto &[vertex, neighbors] : adjacencyList) { + vertexDegrees.emplace_back(vertex, neighbors.size()); + } + + // Sort by degree (descending) + std::sort(vertexDegrees.begin(), vertexDegrees.end(), + [](const auto &a, const auto &b) { return a.second > b.second; }); + + // Extract vertex sequence + std::vector sequence; + sequence.reserve(vertexDegrees.size()); + + for (const auto &[vertex, degree] : vertexDegrees) { + sequence.push_back(vertex); + } + + sheep_partitioner_logger.info("Degree sequence computed with " + + to_string(sequence.size()) + " vertices"); + return sequence; +} + +double SheepPartitioner::calculatePartitionScore(vertex_id vertex, partition_id partition, + size_t maxPartitionSize) { + // Count neighbors in this partition + size_t neighborsInPartition = 0; + + const auto &neighbors = adjacencyList[vertex]; + for (vertex_id neighbor : neighbors) { + auto it = vertexToPartition.find(neighbor); + if (it != vertexToPartition.end() && it->second == partition) { + neighborsInPartition++; + } + } + + // Calculate balance penalty + double currentSize = partitionSizes[partition]; + double balancePenalty = 0.0; + + if (currentSize >= maxPartitionSize) { + // Heavily penalize over-capacity partitions + balancePenalty = -1000.0 * (currentSize - maxPartitionSize + 1); + } else { + // Slight penalty for imbalance + double loadFactor = currentSize / (double)maxPartitionSize; + balancePenalty = -10.0 * loadFactor; + } + + // Score = neighbors in partition + balance penalty + // Higher score is better + return neighborsInPartition + balancePenalty; +} + +void SheepPartitioner::assignPartitions() { + sheep_partitioner_logger.info("Assigning vertices to partitions"); + + // Get degree-ordered sequence + std::vector sequence = getDegreeSequence(); + + // Initialize partition sizes + partitionSizes.resize(numPartitions, 0); + + // Calculate maximum partition size (with 3% balance tolerance) + size_t totalVertices = adjacencyList.size(); + size_t maxPartitionSize = (totalVertices / numPartitions) * 1.03; + + sheep_partitioner_logger.info("Max partition size: " + to_string(maxPartitionSize)); + + // Process vertices in degree order + for (vertex_id vertex : sequence) { + partition_id bestPartition = 0; + double bestScore = -std::numeric_limits::infinity(); + + // Evaluate each partition + for (partition_id p = 0; p < numPartitions; p++) { + double score = calculatePartitionScore(vertex, p, maxPartitionSize); + + if (score > bestScore) { + bestScore = score; + bestPartition = p; + } + } + + // Assign vertex to best partition + vertexToPartition[vertex] = bestPartition; + partitionSizes[bestPartition]++; + } + + sheep_partitioner_logger.info("Vertex assignment complete"); + + // Log partition sizes + for (size_t i = 0; i < numPartitions; i++) { + sheep_partitioner_logger.info("Partition " + to_string(i) + " size: " + + to_string(partitionSizes[i])); + } +} + +void SheepPartitioner::calculateEdgeCuts() { + edgeCuts = 0; + + for (const auto &[source, targets] : adjacencyList) { + partition_id sourcePart = vertexToPartition[source]; + + for (vertex_id target : targets) { + // Count each cross-partition edge once + if (source < target && sourcePart != vertexToPartition[target]) { + edgeCuts++; + } + } + } + + sheep_partitioner_logger.info("Edge cuts: " + to_string(edgeCuts) + + " (" + to_string((edgeCuts * 100.0) / totalEdges) + "%)"); +} + +string SheepPartitioner::extractGraphID(string_view outputPath) const { + string graphID = "0"; + if (size_t lastSlash = outputPath.find_last_of("/\\"); lastSlash != string::npos) { + string_view filename = outputPath.substr(lastSlash + 1); + if (size_t firstUnderscore = filename.find('_'); firstUnderscore != string::npos) { + graphID = string(filename.substr(0, firstUnderscore)); + } + } + return graphID; +} + +void SheepPartitioner::buildEdgeMaps( + std::vector> &partitionVertices, + std::vector &localEdgeCounts, + std::vector ¢ralEdgeCounts, + std::vector>> &localStoreSets, + std::vector>> ¢ralStoreSets, + std::vector>> &duplicateCentralStoreSets) { + + // Build edge maps from adjacency list + // Key insight: adjacencyList has both A->B and B->A for each edge + // We process ALL edges where source vertex is in each partition (like Metis) + // This ensures each partition gets all edges involving its vertices + for (const auto &[source, targets] : adjacencyList) { + partition_id sourcePart = vertexToPartition[source]; + + // Track vertex in its partition + partitionVertices[sourcePart].insert(source); + + for (vertex_id target : targets) { + partition_id targetPart = vertexToPartition[target]; + partitionVertices[targetPart].insert(target); + + if (sourcePart == targetPart) { + // Local edge - both vertices in same partition + // Using set ensures no duplicates even if input has them + localStoreSets[sourcePart][source].insert(target); + // Count edges only once (when processing from lower vertex ID) + localEdgeCounts[sourcePart] += (source < target ? 1 : 0); + } else { + // Cross-partition edge + // Add to source partition's central store + centralStoreSets[sourcePart][source].insert(target); + // Add to target partition's duplicate central store + duplicateCentralStoreSets[targetPart][source].insert(target); + // Count edges only once + centralEdgeCounts[sourcePart] += (source < target ? 1 : 0); + } + } + } +} + +void SheepPartitioner::convertStoreSetsToMaps( + const std::vector>> &sets, + std::vector>> &maps) const { + + for (size_t i = 0; i < numPartitions; i++) { + for (const auto &[key, val] : sets[i]) { + maps[i][key] = std::vector(val.begin(), val.end()); + } + } +} + +bool SheepPartitioner::serializeSinglePartition( + size_t partitionId, + const string &outputPath, + const string &graphID, + const std::map> &localMap, + const std::map> ¢ralMap, + const std::map> &duplicateCentralMap) { + + string localFilename = outputPath + to_string(partitionId); + string dirPath = outputPath.substr(0, outputPath.find_last_of("/\\") + 1); + string centralFilename = dirPath + graphID + "_centralstore_" + to_string(partitionId); + string duplicateCentralFilename = dirPath + graphID + "_centralstore_dp_" + to_string(partitionId); + + // Store local store using FlatBuffers + if (!JasmineGraphHashMapLocalStore::storePartEdgeMap(localMap, localFilename)) { + sheep_partitioner_logger.error("Failed to serialize local store for partition " + to_string(partitionId)); + return false; + } + + // Compress local store file + Utils::compressFile(localFilename); + partitionFileMap[partitionId] = localFilename + ".gz"; + + // Store central store using FlatBuffers + if (!JasmineGraphHashMapCentralStore::storePartEdgeMap(centralMap, centralFilename)) { + sheep_partitioner_logger.error("Failed to serialize central store for partition " + to_string(partitionId)); + return false; + } + + // Compress central store file + Utils::compressFile(centralFilename); + centralStoreFileList[partitionId] = centralFilename + ".gz"; + + // Store duplicate central store using FlatBuffers + if (!JasmineGraphHashMapCentralStore::storePartEdgeMap(duplicateCentralMap, duplicateCentralFilename)) { + sheep_partitioner_logger.error( + "Failed to serialize duplicate central store for partition " + to_string(partitionId)); + return false; + } + + // Compress duplicate central store file + Utils::compressFile(duplicateCentralFilename); + centralStoreDuplicateFileList[partitionId] = duplicateCentralFilename + ".gz"; + + sheep_partitioner_logger.info("Serialized and compressed partition " + to_string(partitionId)); + return true; +} + +bool SheepPartitioner::writePartitions(const string &outputPath) { + sheep_partitioner_logger.info("Writing partitions with central/local store format to: " + outputPath); + + try { + // Create output directory if needed + std::filesystem::path basePath(outputPath); + if (std::filesystem::path parentDir = basePath.parent_path(); + !parentDir.empty() && !std::filesystem::exists(parentDir)) { + std::filesystem::create_directories(parentDir); + } + + // Track vertices and edges per partition + std::vector> partitionVertices(numPartitions); + std::vector localEdgeCounts(numPartitions, 0); + std::vector centralEdgeCounts(numPartitions, 0); + + // Use sets for automatic deduplication during construction, then convert to vectors + std::vector>> localStoreSets(numPartitions); + std::vector>> centralStoreSets(numPartitions); + std::vector>> duplicateCentralStoreSets(numPartitions); + + // Extract graphID from outputPath + string graphID = extractGraphID(outputPath); + + // Build edge maps from adjacency list + buildEdgeMaps(partitionVertices, localEdgeCounts, centralEdgeCounts, + localStoreSets, centralStoreSets, duplicateCentralStoreSets); + + // Convert sets to vectors for serialization + std::vector>> localStoreMaps(numPartitions); + std::vector>> centralStoreMaps(numPartitions); + std::vector>> duplicateCentralStoreMaps(numPartitions); + + convertStoreSetsToMaps(localStoreSets, localStoreMaps); + convertStoreSetsToMaps(centralStoreSets, centralStoreMaps); + convertStoreSetsToMaps(duplicateCentralStoreSets, duplicateCentralStoreMaps); + + // Serialize and compress files using FlatBuffers format + for (size_t i = 0; i < numPartitions; i++) { + if (!serializeSinglePartition(i, outputPath, graphID, + localStoreMaps[i], centralStoreMaps[i], duplicateCentralStoreMaps[i])) { + return false; + } + } + + // Store partition statistics for later metadb update + partitionVertexCounts.clear(); + partitionEdgeCountsVec.clear(); + for (size_t i = 0; i < numPartitions; i++) { + partitionVertexCounts.push_back(partitionVertices[i].size()); + // Total edges = local edges + central edges + partitionEdgeCountsVec.push_back(localEdgeCounts[i] + centralEdgeCounts[i]); + } + + sheep_partitioner_logger.info("Successfully wrote " + to_string(numPartitions) + + " partition files (local store, central store, and duplicate central store)"); + + // Log statistics for each partition + for (size_t i = 0; i < numPartitions; i++) { + sheep_partitioner_logger.info("Partition " + to_string(i) + + ": local edges=" + to_string(localEdgeCounts[i]) + + ", central edges=" + to_string(centralEdgeCounts[i])); + } + + return true; + } catch (const std::invalid_argument &e) { + sheep_partitioner_logger.error("Invalid argument error writing partitions: " + string(e.what())); + return false; + } +} + +std::vector> SheepPartitioner::partitionGraph(int graphID, const string &graphPath, + const string &outputPath, int numberOfPartitions) { + sheep_partitioner_logger.info("Starting sheep partitioning for graph ID: " + to_string(graphID)); + sheep_partitioner_logger.info("Input graph: " + graphPath); + sheep_partitioner_logger.info("Output path: " + outputPath); + sheep_partitioner_logger.info("Number of partitions: " + to_string(numberOfPartitions)); + + this->numPartitions = numberOfPartitions; + + // Check if input graph exists + if (!std::filesystem::exists(graphPath)) { + sheep_partitioner_logger.error("Input graph file does not exist: " + graphPath); + return fullFileList; + } + + // Load the graph + if (!loadGraph(graphPath)) { + return fullFileList; + } + + // Assign vertices to partitions + assignPartitions(); + + // Calculate edge cuts + calculateEdgeCuts(); + + // Write partitions to files + if (!writePartitions(outputPath)) { + return fullFileList; + } + + // Update metadata database + try { + // Update graph table with statistics + string sqlStatement = "UPDATE graph SET vertexcount = '" + to_string(adjacencyList.size()) + + "', centralpartitioncount = '" + to_string(numPartitions) + + "', edgecount = '" + to_string(totalEdges) + + "', id_algorithm = 'sheep' WHERE idgraph = '" + to_string(graphID) + "'"; + sqlite->runUpdate(sqlStatement); + + sheep_partitioner_logger.info("Updated graph table with statistics"); + + // Insert partition records + for (size_t i = 0; i < numPartitions; i++) { + string partitionInsert = + "INSERT INTO partition (idpartition, graph_idgraph, vertexcount, central_vertexcount, edgecount) " + "VALUES('" + to_string(i) + "', '" + to_string(graphID) + "', '" + + to_string(partitionVertexCounts[i]) + "', '0', '" + + to_string(partitionEdgeCountsVec[i]) + "')"; + sqlite->runUpdate(partitionInsert); + } + + sheep_partitioner_logger.info("Successfully partitioned graph " + to_string(graphID) + + " using sheep algorithm with " + to_string(numPartitions) + " partitions"); + } catch (const std::out_of_range &e) { + sheep_partitioner_logger.error("Out of range error during database update: " + string(e.what())); + return fullFileList; + } + + // Build and return file lists for worker distribution + fullFileList.push_back(partitionFileMap); + fullFileList.push_back(centralStoreFileList); + fullFileList.push_back(centralStoreDuplicateFileList); + fullFileList.emplace_back(); // Empty attribute files + fullFileList.emplace_back(); // Empty central attribute files + fullFileList.emplace_back(); // Empty composite central files + + return fullFileList; +} + +void SheepPartitioner::getPartitioningStats() { + sheep_partitioner_logger.info("=== Partitioning Statistics ==="); + sheep_partitioner_logger.info("Total vertices: " + to_string(adjacencyList.size())); + sheep_partitioner_logger.info("Total edges: " + to_string(totalEdges)); + sheep_partitioner_logger.info("Number of partitions: " + to_string(numPartitions)); + sheep_partitioner_logger.info("Edge cuts: " + to_string(edgeCuts)); + sheep_partitioner_logger.info("Edge cut ratio: " + + to_string((edgeCuts * 100.0) / totalEdges) + "%"); + + for (size_t i = 0; i < numPartitions; i++) { + sheep_partitioner_logger.info("Partition " + to_string(i) + " size: " + + to_string(partitionSizes[i])); + } +} diff --git a/src/partitioner/local/SheepPartitioner.h b/src/partitioner/local/SheepPartitioner.h new file mode 100644 index 000000000..3d02c9b5f --- /dev/null +++ b/src/partitioner/local/SheepPartitioner.h @@ -0,0 +1,152 @@ +/** +Copyright 2026 JasminGraph Team +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +#ifndef JASMINEGRAPH_SHEEPPARTITIONER_H +#define JASMINEGRAPH_SHEEPPARTITIONER_H + +#include +#include +#include +#include +#include +#include +#include +#include "../../metadb/SQLiteDBInterface.h" +#include "../../util/Utils.h" +#include "../../localstore/JasmineGraphHashMapLocalStore.h" +#include "../../centralstore/JasmineGraphHashMapCentralStore.h" + +using std::string; +using std::string_view; + +using vertex_id = unsigned long; +using partition_id = short; + +/** + * SheepPartitioner implements a streaming graph partitioning algorithm + * based on elimination trees and balanced bin-packing. + * + * The algorithm processes vertices in degree-sorted order and assigns + * them to partitions to minimize edge cuts while maintaining balance. + */ +class SheepPartitioner { + public: + explicit SheepPartitioner(SQLiteDBInterface *sqlite); + + /** + * Partition a graph using the sheep streaming partitioning algorithm + * @param graphID Graph identifier + * @param graphPath Path to the input graph file (edge list format) + * @param outputPath Base path for output partition files + * @param numPartitions Number of partitions to create + * @return vector of file maps for distribution to workers + */ + std::vector> partitionGraph(int graphID, const string &graphPath, + const string &outputPath, int numberOfPartitions); + + /** + * Get partitioning statistics after partitioning + */ + void getPartitioningStats(); + + private: + SQLiteDBInterface *sqlite = nullptr; + + // Graph structure + std::unordered_map> adjacencyList; + std::unordered_map vertexToPartition; + std::vector partitionSizes; + + // Statistics + size_t totalEdges = 0; + size_t edgeCuts = 0; + size_t numPartitions = 0; + std::vector partitionVertexCounts; + std::vector partitionEdgeCountsVec; + + // File lists for worker distribution + std::map partitionFileMap; + std::map centralStoreFileList; + std::map centralStoreDuplicateFileList; + std::vector> fullFileList; + + /** + * Load graph from edge list file + * Expected format: source_vertex target_vertex (per line) + */ + bool loadGraph(const string &graphPath); + + /** + * Sort vertices by degree (descending order) + */ + std::vector getDegreeSequence() const; + + /** + * Assign vertices to partitions using streaming algorithm + * Based on elimination tree partitioning with balance constraints + */ + void assignPartitions(); + + /** + * Calculate partition score for a vertex + * Score favors partitions with more neighbors and less load + */ + double calculatePartitionScore(vertex_id vertex, partition_id partition, + size_t maxPartitionSize); + + /** + * Write partitions to output files + */ + bool writePartitions(const string &outputPath); + + /** + * Extract graphID from outputPath (format: path/graphID_) + */ + string extractGraphID(string_view outputPath) const; + + /** + * Build local and central edge maps and counts from adjacency list + */ + void buildEdgeMaps( + std::vector> &partitionVertices, + std::vector &localEdgeCounts, + std::vector ¢ralEdgeCounts, + std::vector>> &localStoreSets, + std::vector>> ¢ralStoreSets, + std::vector>> &duplicateCentralStoreSets); + + /** + * Convert sets to vectors for serialization + */ + void convertStoreSetsToMaps( + const std::vector>> &sets, + std::vector>> &maps) const; + + /** + * Serialize and compress a single partition using FlatBuffers + */ + bool serializeSinglePartition( + size_t partitionId, + const string &outputPath, + const string &graphID, + const std::map> &localMap, + const std::map> ¢ralMap, + const std::map> &duplicateCentralMap); + + /** + * Calculate edge cut statistics + */ + void calculateEdgeCuts(); +}; + +#endif // JASMINEGRAPH_SHEEPPARTITIONER_H diff --git a/src/query/algorithms/triangles/SheepTriangles.cpp b/src/query/algorithms/triangles/SheepTriangles.cpp new file mode 100644 index 000000000..2def659af --- /dev/null +++ b/src/query/algorithms/triangles/SheepTriangles.cpp @@ -0,0 +1,141 @@ +/** +Copyright 2026 JasminGraph Team +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +#include "SheepTriangles.h" +#include "../../../util/logger/Logger.h" +#include "../../../util/telemetry/OpenTelemetryUtil.h" +#include +#include + +Logger sheep_triangle_logger; + +long SheepTriangles::run(JasmineGraphHashMapLocalStore &graphDB, + JasmineGraphHashMapCentralStore ¢ralStore, + JasmineGraphHashMapDuplicateCentralStore &duplicateCentralStore, + const std::string &graphId, + const std::string &partitionId) { + OTEL_TRACE_FUNCTION(); + + std::string workerInfo = "worker_" + graphId + "_partition_" + partitionId; + sheep_triangle_logger.info("###SHEEP-TRIANGLE### " + workerInfo + " Starting optimized sheep triangle counting"); + + auto startTime = std::chrono::high_resolution_clock::now(); + + // Extract data from stores + std::map> localMap; + std::map> centralMap; + std::map> duplicateMap; + + { + ScopedTracer data_extract("sheep_extract_data"); + localMap = graphDB.getUnderlyingHashMap(); + centralMap = centralStore.getUnderlyingHashMap(); + duplicateMap = duplicateCentralStore.getUnderlyingHashMap(); + } + + // Merge stores efficiently + { + ScopedTracer merge_phase("sheep_merge_stores"); + mergeStores(localMap, centralMap, duplicateMap); + } + + auto mergeEnd = std::chrono::high_resolution_clock::now(); + auto mergeDuration = std::chrono::duration_cast(mergeEnd - startTime).count(); + sheep_triangle_logger.info("Merge time: " + std::to_string(mergeDuration) + " ms"); + + // Count triangles with optimized algorithm + SheepTriangleResult result; + { + ScopedTracer count_phase("sheep_count_triangles"); + result = countTriangles(localMap, false); + } + + auto endTime = std::chrono::high_resolution_clock::now(); + auto totalDuration = std::chrono::duration_cast(endTime - startTime).count(); + sheep_triangle_logger.info("###SHEEP-TRIANGLE### " + workerInfo + " Complete. Count: " + + std::to_string(result.count) + " Time: " + std::to_string(totalDuration) + " ms"); + + return result.count; +} + +void SheepTriangles::mergeStores( + std::map> &localMap, + std::map> ¢ralMap, + const std::map> &duplicateMap) { + + // Merge duplicate central into central (avoiding duplicates) + for (const auto &[vertex, edges] : duplicateMap) { + centralMap[vertex].insert(edges.begin(), edges.end()); + } + + // Merge central into local + for (const auto &[vertex, edges] : centralMap) { + localMap[vertex].insert(edges.begin(), edges.end()); + } +} + +static long countTrianglesForPair(long u, long v, const std::unordered_set &u_neighbors, + const std::unordered_set &v_neighbors, bool returnTriangles, + std::ostringstream &triangleStream) { + long count = 0; + for (long w : u_neighbors) { + if (w <= v) continue; // Enforce v < w + + // Check if v is connected to w (completing the triangle) + if (v_neighbors.find(w) != v_neighbors.end()) { + count++; + + if (returnTriangles) { + triangleStream << u << "," << v << "," << w << ":"; + } + } + } + return count; +} + +SheepTriangleResult SheepTriangles::countTriangles( + std::map> &edgeMap, + bool returnTriangles) { + + SheepTriangleResult result; + result.count = 0; + + std::ostringstream triangleStream; + + // For each vertex u + for (const auto &[u, u_neighbors] : edgeMap) { + // For each neighbor v of u where v > u (ordering prevents duplicates) + for (long v : u_neighbors) { + if (v <= u) continue; // Enforce u < v + + auto v_it = edgeMap.find(v); + if (v_it == edgeMap.end()) continue; + const auto &v_neighbors = v_it->second; + + result.count += countTrianglesForPair(u, v, u_neighbors, v_neighbors, + returnTriangles, triangleStream); + } + } + + if (returnTriangles) { + std::string triangles = triangleStream.str(); + if (triangles.empty()) { + result.triangles = "NILL"; + } else { + triangles.pop_back(); // Remove trailing ':' + result.triangles = std::move(triangles); + } + } + + return result; +} diff --git a/src/query/algorithms/triangles/SheepTriangles.h b/src/query/algorithms/triangles/SheepTriangles.h new file mode 100644 index 000000000..fecae108a --- /dev/null +++ b/src/query/algorithms/triangles/SheepTriangles.h @@ -0,0 +1,72 @@ +/** +Copyright 2026 JasminGraph Team +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +#ifndef JASMINEGRAPH_SHEEPTRIANGLES_H +#define JASMINEGRAPH_SHEEPTRIANGLES_H + +#include +#include +#include +#include "../../../centralstore/JasmineGraphHashMapCentralStore.h" +#include "../../../centralstore/JasmineGraphHashMapDuplicateCentralStore.h" +#include "../../../localstore/JasmineGraphHashMapLocalStore.h" + +struct SheepTriangleResult { + long count; + std::string triangles; +}; + +/** + * Optimized triangle counting for sheep-partitioned graphs. + * + * Key differences from standard algorithm: + * - Handles bidirectional edge representation efficiently + * - Optimized for sheep partitioner's edge distribution pattern + * - Uses vertex-ordered triangle enumeration to prevent double counting + */ +class SheepTriangles { + public: + /** + * Count triangles in sheep-partitioned graph + * @param graphDB Local partition data + * @param centralStore Central store with cross-partition edges + * @param duplicateCentralStore Duplicate central store + * @param graphId Graph identifier + * @param partitionId Partition identifier + * @return Triangle count + */ + static long run(JasmineGraphHashMapLocalStore &graphDB, + JasmineGraphHashMapCentralStore ¢ralStore, + JasmineGraphHashMapDuplicateCentralStore &duplicateCentralStore, + const std::string &graphId, + const std::string &partitionId); + + private: + /** + * Core triangle counting with ordered enumeration + * Uses vertex ordering (u < v < w) to ensure each triangle counted exactly once + */ + static SheepTriangleResult countTriangles( + std::map> &edgeMap, + bool returnTriangles = false); + + /** + * Merge stores efficiently for sheep format + */ + static void mergeStores( + std::map> &localMap, + std::map> ¢ralMap, + const std::map> &duplicateMap); +}; + +#endif // JASMINEGRAPH_SHEEPTRIANGLES_H diff --git a/src/query/processor/cypher/runtime/OperatorExecutor.cpp b/src/query/processor/cypher/runtime/OperatorExecutor.cpp index f38395ed5..6a64a8001 100644 --- a/src/query/processor/cypher/runtime/OperatorExecutor.cpp +++ b/src/query/processor/cypher/runtime/OperatorExecutor.cpp @@ -1432,6 +1432,8 @@ struct Row { json val1 = getNestedValue(data, sortKey); json val2 = getNestedValue(other.data, sortKey); + if (val1 == val2) return false; + bool result; if (val1.is_number_integer() && val2.is_number_integer()) { result = val1.get() > val2.get(); @@ -1544,6 +1546,8 @@ void OperatorExecutor::OrderBy(SharedBuffer &buffer, std::string jsonPlan, Graph json val1 = a.getNestedValue(a.data, sortKey); json val2 = b.getNestedValue(b.data, sortKey); + if (val1 == val2) return false; + bool result; if (val1.is_number_integer() && val2.is_number_integer()) { result = val1.get() > val2.get(); diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index eec46064e..c41b52b89 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -28,6 +28,8 @@ limitations under the License. #include "../knowledgegraph/construction/Pipeline.h" #include "../knowledgegraph/construction/VLLMTupleStreamer.h" #include "../query/algorithms/triangles/StreamingTriangles.h" +#include "../query/algorithms/triangles/SheepTriangles.h" +#include "../query/algorithms/triangles/SheepTriangles.h" #include "../query/processor/cypher/runtime/InstanceHandler.h" #include "../query/processor/cypher/util/SharedBuffer.h" #include "../query/processor/nlp/semanticbeamsearch/SemanticBeamSearch.h" @@ -75,33 +77,39 @@ static void delete_graph_command(int connFd, bool* loop_exit_p); static void delete_graph_fragment_command(int connFd, bool* loop_exit_p); static void duplicate_centralstore_command(int connFd, int serverPort, bool* loop_exit_p); static void worker_in_degree_distribution_command( - int connFd, std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, bool* loop_exit_p); + int connFd, std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p); static void in_degree_distribution_command( - int connFd, int serverPort, std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, bool* loop_exit_p); + int connFd, int serverPort, + std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p); static void worker_out_degree_distribution_command( - int connFd, std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, bool* loop_exit_p); + int connFd, std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p); static void out_degree_distribution_command( - int connFd, int serverPort, std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, bool* loop_exit_p); + int connFd, int serverPort, + std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p); static void page_rank_command(int connFd, int serverPort, - std::map& graphDBMapCentralStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p); static void worker_page_rank_distribution_command( - int connFd, int serverPort, std::map& graphDBMapCentralStores, + int connFd, int serverPort, + std::map>& graphDBMapCentralStores, bool* loop_exit_p); static void egonet_command(int connFd, int serverPort, - std::map& graphDBMapCentralStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p); static void worker_egonet_command(int connFd, int serverPort, - std::map& graphDBMapCentralStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p); static void triangles_command( - int connFd, int serverPort, std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, - std::map& graphDBMapDuplicateCentralStores, + int connFd, int serverPort, + std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, + std::map>& graphDBMapDuplicateCentralStores, bool* loop_exit_p); static void streaming_triangles_command( int connFd, int serverPort, std::map& incrementalLocalStoreMap, @@ -140,8 +148,10 @@ static void send_priority_command(int connFd, bool* loop_exit_p); static std::string initiate_command_common(int connFd, bool* loop_exit_p); static void batch_upload_common(int connFd, bool* loop_exit_p, bool batch_upload); static void degree_distribution_common(int connFd, int serverPort, - std::map &graphDBMapLocalStores, - std::map &graphDBMapCentralStores, + std::map> &graphDBMapLocalStores, + std::map> &graphDBMapCentralStores, bool *loop_exit_p, bool in); static void push_partition_command(int connFd, bool *loop_exit_p); static void push_file_command(int connFd, bool *loop_exit_p); @@ -166,9 +176,9 @@ static void hdfs_start_stream_command(int connFd, bool *loop_exit_p, bool isLoca InstanceStreamHandler &instanceStreamHandler); long countLocalTriangles( std::string graphId, std::string partitionId, - std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, - std::map& graphDBMapDuplicateCentralStores, + std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, + std::map>& graphDBMapDuplicateCentralStores, int threadPriority); static void processFile(string basicString, bool isLocal, InstanceStreamHandler& handler, bool isEmbedGraph); @@ -190,10 +200,11 @@ void* instanceservicesession(void* dummyPt) { instanceservicesessionargs sessionargs = *sessionargs_p; int connFd = sessionargs.connFd; string cmd = sessionargs.cmd; - std::map *graphDBMapLocalStores = sessionargs.graphDBMapLocalStores; - std::map *graphDBMapCentralStores = + std::map> *graphDBMapLocalStores = sessionargs.graphDBMapLocalStores; + std::map> *graphDBMapCentralStores = sessionargs.graphDBMapCentralStores; - std::map* graphDBMapDuplicateCentralStores = + std::map>* graphDBMapDuplicateCentralStores = sessionargs.graphDBMapDuplicateCentralStores; std::map& incrementalLocalStoreMap = *(sessionargs.incrementalLocalStore); @@ -385,9 +396,9 @@ void JasmineGraphInstanceService::run(string masterHost, string host, int server len = sizeof(clntAdd); pthread_mutex_init(&file_lock, NULL); - std::map graphDBMapLocalStores; - std::map graphDBMapCentralStores; - std::map graphDBMapDuplicateCentralStores; + std::map> graphDBMapLocalStores; + std::map> graphDBMapCentralStores; + std::map> graphDBMapDuplicateCentralStores; std::map incrementalLocalStore; std::thread perfThread = std::thread(&PerformanceUtil::collectPerformanceStatistics); perfThread.detach(); @@ -511,16 +522,14 @@ void writeCatalogRecord(string record) { outfile.close(); } -long countLocalTriangles( +static void loadTriangleStores( std::string graphId, std::string partitionId, - std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, - std::map& graphDBMapDuplicateCentralStores, + std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, + std::map>& graphDBMapDuplicateCentralStores, int threadPriority) { OTEL_TRACE_FUNCTION(); - long result; - instance_logger.info("###INSTANCE### Local Triangle Count : Started: Graph ID " + graphId + " Partition " + partitionId); @@ -528,31 +537,87 @@ long countLocalTriangles( std::string centralGraphIdentifier = graphId + "_centralstore_" + partitionId; std::string duplicateCentralGraphIdentifier = graphId + "_centralstore_dp_" + partitionId; - auto localMapIterator = graphDBMapLocalStores.find(graphIdentifier); - auto centralStoreIterator = graphDBMapCentralStores.find(graphIdentifier); - auto duplicateCentralStoreIterator = graphDBMapDuplicateCentralStores.find(graphIdentifier); - - if (localMapIterator == graphDBMapLocalStores.end() && + if (graphDBMapLocalStores.find(graphIdentifier) == graphDBMapLocalStores.end() && JasmineGraphInstanceService::isGraphDBExists(graphId, partitionId)) { JasmineGraphInstanceService::loadLocalStore(graphId, partitionId, graphDBMapLocalStores); } - JasmineGraphHashMapLocalStore graphDB = graphDBMapLocalStores[graphIdentifier]; - if (centralStoreIterator == graphDBMapCentralStores.end() && + if (graphDBMapCentralStores.find(centralGraphIdentifier) == graphDBMapCentralStores.end() && JasmineGraphInstanceService::isInstanceCentralStoreExists(graphId, partitionId)) { JasmineGraphInstanceService::loadInstanceCentralStore(graphId, partitionId, graphDBMapCentralStores); } - JasmineGraphHashMapCentralStore centralGraphDB = graphDBMapCentralStores[centralGraphIdentifier]; - if (duplicateCentralStoreIterator == graphDBMapDuplicateCentralStores.end() && + if (graphDBMapDuplicateCentralStores.find(duplicateCentralGraphIdentifier) == + graphDBMapDuplicateCentralStores.end() && JasmineGraphInstanceService::isInstanceDuplicateCentralStoreExists(graphId, partitionId)) { JasmineGraphInstanceService::loadInstanceDuplicateCentralStore(graphId, partitionId, graphDBMapDuplicateCentralStores); } - JasmineGraphHashMapDuplicateCentralStore duplicateCentralGraphDB = - graphDBMapDuplicateCentralStores[duplicateCentralGraphIdentifier]; +} + +struct TriangleStores { + JasmineGraphHashMapLocalStore localStore; + JasmineGraphHashMapCentralStore centralStore; + JasmineGraphHashMapDuplicateCentralStore duplicateCentralStore; +}; + +static TriangleStores getTriangleStores( + std::string graphId, std::string partitionId, + std::map> &graphDBMapLocalStores, + std::map> &graphDBMapCentralStores, + std::map> &graphDBMapDuplicateCentralStores, + int threadPriority) { + instance_logger.info("###INSTANCE### Local Triangle Count : Started: Graph ID " + graphId + + " Partition " + partitionId); + + loadTriangleStores(graphId, partitionId, graphDBMapLocalStores, graphDBMapCentralStores, + graphDBMapDuplicateCentralStores, threadPriority); + + std::string graphIdentifier = graphId + "_" + partitionId; + std::string centralGraphIdentifier = graphId + "_centralstore_" + partitionId; + std::string duplicateCentralGraphIdentifier = graphId + "_centralstore_dp_" + partitionId; + + return {graphDBMapLocalStores[graphIdentifier], + graphDBMapCentralStores[centralGraphIdentifier], + graphDBMapDuplicateCentralStores[duplicateCentralGraphIdentifier]}; +} + +long countLocalTriangles( + std::string graphId, std::string partitionId, + std::map> &graphDBMapLocalStores, + std::map> &graphDBMapCentralStores, + std::map> &graphDBMapDuplicateCentralStores, + int threadPriority) { + OTEL_TRACE_FUNCTION(); + + TriangleStores stores = getTriangleStores(graphId, partitionId, graphDBMapLocalStores, + graphDBMapCentralStores, graphDBMapDuplicateCentralStores, + threadPriority); - result = Triangles::run(graphDB, centralGraphDB, duplicateCentralGraphDB, graphId, partitionId, threadPriority); + instance_logger.info("###INSTANCE### Using standard Triangles algorithm"); + long result = Triangles::run(stores.localStore, stores.centralStore, stores.duplicateCentralStore, + graphId, partitionId, threadPriority); + + instance_logger.info("###INSTANCE### Local Triangle Count : Completed: Triangles: " + to_string(result)); + + return result; +} + +long countLocalSheepTriangles( + std::string graphId, std::string partitionId, + std::map> &graphDBMapLocalStores, + std::map> &graphDBMapCentralStores, + std::map> &graphDBMapDuplicateCentralStores, + int threadPriority) { + OTEL_TRACE_FUNCTION(); + + TriangleStores stores = getTriangleStores(graphId, partitionId, graphDBMapLocalStores, + graphDBMapCentralStores, graphDBMapDuplicateCentralStores, + threadPriority); + + instance_logger.info("###INSTANCE### Using SheepTriangles algorithm"); + long result = SheepTriangles::run(stores.localStore, stores.centralStore, stores.duplicateCentralStore, + graphId, partitionId); instance_logger.info("###INSTANCE### Local Triangle Count : Completed: Triangles: " + to_string(result)); @@ -604,8 +669,8 @@ JasmineGraphIncrementalLocalStore* JasmineGraphInstanceService::loadStreamingSto } void JasmineGraphInstanceService::loadLocalStore( - std::string graphId, std::string partitionId, - std::map& graphDBMapLocalStores) { + const std::string& graphId, const std::string& partitionId, + std::map> &graphDBMapLocalStores) { instance_logger.info("###INSTANCE### Loading Local Store : Started"); std::string graphIdentifier = graphId + "_" + partitionId; std::string folderLocation = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); @@ -616,8 +681,8 @@ void JasmineGraphInstanceService::loadLocalStore( } void JasmineGraphInstanceService::loadInstanceCentralStore( - std::string graphId, std::string partitionId, - std::map& graphDBMapCentralStores) { + const std::string &graphId, const std::string &partitionId, + std::map> &graphDBMapCentralStores) { instance_logger.info("###INSTANCE### Loading central Store : Started"); std::string graphIdentifier = graphId + "_centralstore_" + partitionId; JasmineGraphHashMapCentralStore jasmineGraphHashMapCentralStore(stoi(graphId), stoi(partitionId)); @@ -627,8 +692,8 @@ void JasmineGraphInstanceService::loadInstanceCentralStore( } void JasmineGraphInstanceService::loadInstanceDuplicateCentralStore( - std::string graphId, std::string partitionId, - std::map& graphDBMapDuplicateCentralStores) { + const std::string &graphId, const std::string &partitionId, + std::map> &graphDBMapDuplicateCentralStores) { std::string graphIdentifier = graphId + "_centralstore_dp_" + partitionId; JasmineGraphHashMapDuplicateCentralStore jasmineGraphHashMapCentralStore(stoi(graphId), stoi(partitionId)); jasmineGraphHashMapCentralStore.loadGraph(); @@ -705,7 +770,7 @@ static string aggregateCentralStoreTriangles(std::string graphId, std::string pa string JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(std::string compositeFileList, std::string availableFileList, - int threadPriority) { + int) { instance_logger.info("###INSTANCE### Started Aggregating Composite Central Store Triangles"); std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); std::string dataFolder = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); @@ -1283,9 +1348,11 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int return true; } -map calculateOutDegreeDist(string graphID, string partitionID, int serverPort, - std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, +map calculateOutDegreeDist(const string& graphID, const string& partitionID, int serverPort, + std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, std::vector& workerSockets) { map degreeDistribution = calculateLocalOutDegreeDist(graphID, partitionID, graphDBMapLocalStores, graphDBMapCentralStores); @@ -1307,14 +1374,15 @@ map calculateOutDegreeDist(string graphID, string partitionID, int s } map calculateLocalOutDegreeDist( - string graphID, string partitionID, std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores) { + const string& graphID, const string& partitionID, + std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores) { auto t_start = std::chrono::high_resolution_clock::now(); JasmineGraphHashMapLocalStore graphDB; JasmineGraphHashMapCentralStore centralDB; - std::map::iterator it; - std::map::iterator itcen; + std::map>::iterator it; + std::map>::iterator itcen; if (JasmineGraphInstanceService::isGraphDBExists(graphID, partitionID)) { JasmineGraphInstanceService::loadLocalStore(graphID, partitionID, graphDBMapLocalStores); @@ -1354,11 +1422,12 @@ map calculateLocalOutDegreeDist( } map calculateLocalInDegreeDist( - string graphID, string partitionID, std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores) { + const string& graphID, const string& partitionID, + std::map>& graphDBMapLocalStores, + const std::map>& graphDBMapCentralStores) { JasmineGraphHashMapLocalStore graphDB; - std::map::iterator it; + std::map>::iterator it; if (JasmineGraphInstanceService::isGraphDBExists(graphID, partitionID)) { JasmineGraphInstanceService::loadLocalStore(graphID, partitionID, graphDBMapLocalStores); @@ -1372,9 +1441,11 @@ map calculateLocalInDegreeDist( return degreeDistribution; } -map calculateInDegreeDist(string graphID, string partitionID, int serverPort, - std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, +map calculateInDegreeDist(const string& graphID, const string& partitionID, int serverPort, + std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, std::vector& workerSockets, string workerList) { auto t_start = std::chrono::high_resolution_clock::now(); @@ -1394,7 +1465,7 @@ map calculateInDegreeDist(string graphID, string partitionID, int se JasmineGraphHashMapCentralStore centralDB; - std::map::iterator itcen; + std::map>::iterator itcen; if (JasmineGraphInstanceService::isInstanceCentralStoreExists(graphID, workerPartitionID)) { JasmineGraphInstanceService::loadInstanceCentralStore(graphID, workerPartitionID, graphDBMapCentralStores); @@ -2347,8 +2418,8 @@ static void duplicate_centralstore_command(int connFd, int serverPort, bool* loo } static void worker_in_degree_distribution_command( - int connFd, std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, bool* loop_exit_p) { + int connFd, std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; return; @@ -2404,7 +2475,7 @@ static void worker_in_degree_distribution_command( JasmineGraphHashMapCentralStore centralDB; - std::map::iterator itcen; + std::map>::iterator itcen; if (JasmineGraphInstanceService::isInstanceCentralStoreExists(graphID, workerPartitionID)) { JasmineGraphInstanceService::loadInstanceCentralStore(graphID, workerPartitionID, graphDBMapCentralStores); @@ -2445,8 +2516,10 @@ static void worker_in_degree_distribution_command( } static void degree_distribution_common(int connFd, int serverPort, - std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, + std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p, bool in) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2497,14 +2570,15 @@ static void degree_distribution_common(int connFd, int serverPort, } static void in_degree_distribution_command( - int connFd, int serverPort, std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, bool* loop_exit_p) { + int connFd, int serverPort, + std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p) { degree_distribution_common(connFd, serverPort, graphDBMapLocalStores, graphDBMapCentralStores, loop_exit_p, true); } static void worker_out_degree_distribution_command( - int connFd, std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, bool* loop_exit_p) { + int connFd, std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; return; @@ -2539,14 +2613,16 @@ static void worker_out_degree_distribution_command( } static void out_degree_distribution_command( - int connFd, int serverPort, std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, bool* loop_exit_p) { + int connFd, int serverPort, + std::map> &graphDBMapLocalStores, + std::map> &graphDBMapCentralStores, + bool *loop_exit_p) { degree_distribution_common(connFd, serverPort, graphDBMapLocalStores, graphDBMapCentralStores, loop_exit_p, false); } static void page_rank_command(int connFd, int serverPort, - std::map& graphDBMapCentralStores, - bool* loop_exit_p) { + std::map> &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; return; @@ -2622,7 +2698,7 @@ static void page_rank_command(int connFd, int serverPort, JasmineGraphHashMapLocalStore graphDB; JasmineGraphHashMapCentralStore centralDB; - std::map graphDBMapLocalStoresPgrnk; + std::map> graphDBMapLocalStoresPgrnk; if (JasmineGraphInstanceService::isGraphDBExists(graphID, partitionID)) { JasmineGraphInstanceService::loadLocalStore(graphID, partitionID, graphDBMapLocalStoresPgrnk); } @@ -2698,7 +2774,8 @@ static void page_rank_command(int connFd, int serverPort, } static void worker_page_rank_distribution_command( - int connFd, int serverPort, std::map& graphDBMapCentralStores, + int connFd, int serverPort, + std::map>& graphDBMapCentralStores, bool* loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2775,7 +2852,7 @@ static void worker_page_rank_distribution_command( JasmineGraphHashMapLocalStore graphDB; JasmineGraphHashMapCentralStore centralDB; - std::map graphDBMapLocalStoresPgrnk; + std::map> graphDBMapLocalStoresPgrnk; if (JasmineGraphInstanceService::isGraphDBExists(graphID, partitionID)) { JasmineGraphInstanceService::loadLocalStore(graphID, partitionID, graphDBMapLocalStoresPgrnk); } @@ -2826,7 +2903,7 @@ static void worker_page_rank_distribution_command( } static void egonet_command(int connFd, int serverPort, - std::map& graphDBMapCentralStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2859,7 +2936,7 @@ static void egonet_command(int connFd, int serverPort, JasmineGraphHashMapLocalStore graphDB; JasmineGraphHashMapCentralStore centralDB; - std::map graphDBMapLocalStoresPgrnk; + std::map> graphDBMapLocalStoresPgrnk; if (JasmineGraphInstanceService::isGraphDBExists(graphID, partitionID)) { JasmineGraphInstanceService::loadLocalStore(graphID, partitionID, graphDBMapLocalStoresPgrnk); } @@ -2875,7 +2952,8 @@ static void egonet_command(int connFd, int serverPort, } static void worker_egonet_command(int connFd, int serverPort, - std::map& graphDBMapCentralStores, + std::map>& graphDBMapCentralStores, bool* loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2921,7 +2999,7 @@ static void worker_egonet_command(int connFd, int serverPort, JasmineGraphHashMapLocalStore graphDB; JasmineGraphHashMapCentralStore centralDB; - std::map graphDBMapLocalStoresPgrnk; + std::map> graphDBMapLocalStoresPgrnk; if (JasmineGraphInstanceService::isGraphDBExists(graphID, partitionID)) { JasmineGraphInstanceService::loadLocalStore(graphID, partitionID, graphDBMapLocalStoresPgrnk); } @@ -2957,9 +3035,10 @@ static void worker_egonet_command(int connFd, int serverPort, } static void triangles_command( - int connFd, int serverPort, std::map& graphDBMapLocalStores, - std::map& graphDBMapCentralStores, - std::map& graphDBMapDuplicateCentralStores, + int connFd, int serverPort, + std::map>& graphDBMapLocalStores, + std::map>& graphDBMapCentralStores, + std::map>& graphDBMapDuplicateCentralStores, bool* loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2992,16 +3071,10 @@ static void triangles_command( return; } - // Receive trace context from master string traceContext = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); - - // Use utility function to validate and set trace context OpenTelemetryUtil::receiveAndSetTraceContext(traceContext, "triangle counting"); - - // Start tracing AFTER trace context is set to ensure proper parent-child relationship OTEL_TRACE_FUNCTION(); - // Add worker identification attributes to distinguish workers in traces OpenTelemetryUtil::addSpanAttribute("worker.id", "worker_" + std::to_string(serverPort)); OpenTelemetryUtil::addSpanAttribute("worker.port", std::to_string(serverPort)); OpenTelemetryUtil::addSpanAttribute("partition.id", partitionId); @@ -3021,7 +3094,7 @@ static void triangles_command( perfThread.detach(); long localCount = countLocalTriangles(graphID, partitionId, graphDBMapLocalStores, graphDBMapCentralStores, - graphDBMapDuplicateCentralStores, threadPriority); + graphDBMapDuplicateCentralStores, threadPriority); if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { threadPriorityMutex.lock(); @@ -3039,6 +3112,7 @@ static void triangles_command( } } + static void streaming_triangles_command( int connFd, int serverPort, std::map& incrementalLocalStoreMap, bool* loop_exit_p) { diff --git a/src/server/JasmineGraphInstanceService.h b/src/server/JasmineGraphInstanceService.h index 8685ea644..11e2e4040 100644 --- a/src/server/JasmineGraphInstanceService.h +++ b/src/server/JasmineGraphInstanceService.h @@ -54,23 +54,29 @@ int deleteGraphPartition(std::string graphID, std::string partitionID); int deleteStreamingGraphPartition(std::string graphID, std::string partitionID); void removeGraphFragments(std::string graphID); -map calculateOutDegreeDist(string graphID, string partitionID, int serverPort, - std::map &graphDBMapLocalStores, - std::map &graphDBMapCentralStores, +map calculateOutDegreeDist(const string &graphID, const string &partitionID, int serverPort, + std::map> &graphDBMapLocalStores, + std::map> &graphDBMapCentralStores, std::vector &workerSockets); map calculateLocalOutDegreeDist( - string graphID, string partitionID, std::map &graphDBMapLocalStores, - std::map &graphDBMapCentralStores); - -map calculateInDegreeDist(string graphID, string partitionID, int serverPort, - std::map &graphDBMapLocalStores, - std::map &graphDBMapCentralStores, + const string &graphID, const string &partitionID, + std::map> &graphDBMapLocalStores, + std::map> &graphDBMapCentralStores); + +map calculateInDegreeDist(const string &graphID, const string &partitionID, int serverPort, + std::map> &graphDBMapLocalStores, + std::map> &graphDBMapCentralStores, std::vector &workerSockets, string workerList); map calculateLocalInDegreeDist( - string graphID, string partitionID, std::map &graphDBMapLocalStores, - std::map &graphDBMapCentralStores); + const string &graphID, const string &partitionID, + std::map> &graphDBMapLocalStores, + const std::map> &graphDBMapCentralStores); map>> calculateLocalEgoNet(string graphID, string partitionID, int serverPort, JasmineGraphHashMapLocalStore localDB, @@ -105,9 +111,9 @@ struct instanceservicesessionargs { int port; int dataPort; string cmd; - std::map *graphDBMapLocalStores; - std::map *graphDBMapCentralStores; - std::map *graphDBMapDuplicateCentralStores; + std::map> *graphDBMapLocalStores; + std::map> *graphDBMapCentralStores; + std::map> *graphDBMapDuplicateCentralStores; std::map *incrementalLocalStore; }; @@ -122,18 +128,20 @@ class JasmineGraphInstanceService { static bool isGraphDBExists(std::string graphId, std::string partitionId); static bool isInstanceCentralStoreExists(std::string graphId, std::string partitionId); static bool isInstanceDuplicateCentralStoreExists(std::string graphId, std::string partitionId); - static void loadLocalStore(std::string graphId, std::string partitionId, - std::map &graphDBMapLocalStores); + static void loadLocalStore( + const std::string& graphId, const std::string& partitionId, + std::map> &graphDBMapLocalStores); static JasmineGraphIncrementalLocalStore *loadStreamingStore( std::string graphId, std::string partitionId, std::map &graphDBMapStreamingStores, std::string openMode , bool isEmbed); static void loadInstanceCentralStore( - std::string graphId, std::string partitionId, - std::map &graphDBMapCentralStores); + const std::string &graphId, const std::string &partitionId, + std::map> &graphDBMapCentralStores); static void loadInstanceDuplicateCentralStore( - std::string graphId, std::string partitionId, - std::map &graphDBMapDuplicateCentralStores); + const std::string &graphId, const std::string &partitionId, + std::map> + &graphDBMapDuplicateCentralStores); static JasmineGraphHashMapCentralStore *loadCentralStore(std::string centralStoreFileName); static string aggregateCentralStoreTriangles(std::string graphId, std::string partitionId, std::string partitionIdList, int threadPriority); diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index be182634b..33e433089 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -17,10 +17,12 @@ limitations under the License. #include #include +#include #include #include #include #include +#include #include "../../globals.h" #include "../k8s/K8sWorkerController.h" @@ -75,6 +77,7 @@ static void degreeDistributionCommon(std::string graphID, std::string command); static int getPortByHost(const std::string &host); static int getDataPortByHost(const std::string &host); static size_t getWorkerCount(); +static void joinCompletedThreads(std::thread *workerThreads, int file_count, std::string_view graphType, int count); static std::vector hostWorkerList; @@ -1000,30 +1003,50 @@ void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, } int count = 0; int file_count = 0; + + // Increase concurrent thread limit for I/O-bound operations + // File uploads are I/O-bound, not CPU-bound, so we can handle more concurrent operations + const int MAX_CONCURRENT_THREADS = std::min(total_threads, 200); std::thread *workerThreads = new std::thread[total_threads]; + std::vector> partitionWorkerAssignments; + partitionWorkerAssignments.reserve(partitionFileMap.size()); + while (count < total_threads) { const auto &workerList = getWorkers(partitionFileMap.size()); - while (true) { - if (count >= total_threads) { - break; + while (count < total_threads) { + // Wait if we've hit the concurrent thread limit - join completed threads + if (count >= MAX_CONCURRENT_THREADS && count - file_count >= MAX_CONCURRENT_THREADS) { + joinCompletedThreads(workerThreads, file_count, graphType, count); } + + worker worker = workerList[graphUploadWorkerTracker]; std::string partitionFileName = partitionFileMap[file_count]; + std::string centralFile = centralStoreFileMap[file_count]; + std::string centralDuplFile = centralStoreDuplFileMap[file_count]; + + // Launch partition file upload thread workerThreads[count++] = std::thread(batchUploadFile, worker.hostname, worker.port, worker.dataPort, graphID, partitionFileName, masterHost); - copyCentralStoreToAggregateLocation(centralStoreFileMap[file_count]); - workerThreads[count++] = std::thread(batchUploadCentralStore, worker.hostname, worker.port, worker.dataPort, - graphID, centralStoreFileMap[file_count], masterHost); + + // Launch central store upload thread (includes aggregator copy) + workerThreads[count++] = std::thread([this, centralFile, worker, graphID]() { + copyCentralStoreToAggregateLocation(centralFile); + batchUploadCentralStore(worker.hostname, worker.port, worker.dataPort, + graphID, centralFile, masterHost); + }); if (compositeCentralStoreFileMap.find(file_count) != compositeCentralStoreFileMap.end()) { - copyCentralStoreToAggregateLocation(compositeCentralStoreFileMap[file_count]); - workerThreads[count++] = - std::thread(batchUploadCompositeCentralstoreFile, worker.hostname, worker.port, worker.dataPort, - graphID, compositeCentralStoreFileMap[file_count], masterHost); + std::string compositeFile = compositeCentralStoreFileMap[file_count]; + workerThreads[count++] = std::thread([this, compositeFile, worker, graphID]() { + copyCentralStoreToAggregateLocation(compositeFile); + batchUploadCompositeCentralstoreFile(worker.hostname, worker.port, worker.dataPort, + graphID, compositeFile, masterHost); + }); } workerThreads[count++] = std::thread(batchUploadCentralStore, worker.hostname, worker.port, worker.dataPort, - graphID, centralStoreDuplFileMap[file_count], masterHost); + graphID, centralDuplFile, masterHost); if (graphType == Conts::GRAPH_WITH_ATTRIBUTES) { workerThreads[count++] = std::thread(batchUploadAttributeFile, worker.hostname, worker.port, worker.dataPort, graphID, @@ -1032,7 +1055,9 @@ void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, std::thread(batchUploadCentralAttributeFile, worker.hostname, worker.port, worker.dataPort, graphID, centralStoreAttributeFileMap[file_count], masterHost); } - assignPartitionToWorker(partitionFileName, graphID, worker.hostname, worker.port); + + // Defer database operations to reduce blocking + partitionWorkerAssignments.emplace_back(partitionFileName, graphID, worker.hostname, worker.port); file_count++; graphUploadWorkerTracker++; @@ -1040,11 +1065,21 @@ void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, } } - server_logger.info("Total number of threads to join : " + to_string(count)); + server_logger.info("Waiting for all " + to_string(count) + " threads to complete..."); + // Join all remaining threads for (int threadCount = 0; threadCount < count; threadCount++) { - workerThreads[threadCount].join(); - server_logger.info("Thread " + to_string(threadCount) + " joined"); + if (workerThreads[threadCount].joinable()) { + workerThreads[threadCount].join(); + } + } + server_logger.info("All upload threads completed"); + + // Now perform database operations in batch after all uploads complete + server_logger.info("Assigning partitions to workers in database..."); + for (const auto& [partitionFileName, graphId, host, port] : partitionWorkerAssignments) { + assignPartitionToWorker(partitionFileName, graphId, host, port); } + server_logger.info("Database assignments completed"); std::time_t time = chrono::system_clock::to_time_t(chrono::system_clock::now()); string uploadEndTime = ctime(&time); @@ -1111,17 +1146,32 @@ static bool batchUploadCentralStore(std::string host, int port, int dataPort, in JasmineGraphInstanceProtocol::BATCH_UPLOAD_CENTRAL); } +static void joinCompletedThreads(std::thread *workerThreads, int file_count, std::string_view graphType, int count) { + int batch_start = file_count * (graphType == Conts::GRAPH_WITH_ATTRIBUTES ? 6 : 4); + int batch_end = std::min(batch_start + 10, count); + for (int i = batch_start; i < batch_end; i++) { + if (workerThreads[i].joinable()) { + workerThreads[i].join(); + } + } +} + void JasmineGraphServer::copyCentralStoreToAggregateLocation(std::string filePath) { - std::string result = "SUCCESS"; std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); if (access(aggregatorDirPath.c_str(), F_OK)) { Utils::createDirectory(aggregatorDirPath); } - std::string copyCommand = "cp " + filePath + " " + aggregatorDirPath; - if (system(copyCommand.c_str())) { - server_logger.error("Copying " + filePath + " into " + aggregatorDirPath + " failed"); + // Use C++ filesystem for better performance (no shell process spawn) + try { + std::filesystem::path source(filePath); + std::filesystem::path dest = std::filesystem::path(aggregatorDirPath) / source.filename(); + + // Use copy_options::overwrite_existing for idempotency + std::filesystem::copy_file(source, dest, std::filesystem::copy_options::overwrite_existing); + } catch (const std::filesystem::filesystem_error& e) { + server_logger.error("Copying " + filePath + " into " + aggregatorDirPath + " failed: " + e.what()); } } diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index c663e11b3..7fc4a3c1e 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -17,6 +17,7 @@ limitations under the License. #include #include #include +#include #include #include #include @@ -26,6 +27,7 @@ limitations under the License. #include #include #include +#include #include #include @@ -1164,6 +1166,71 @@ std::fstream* Utils::openFile(const string& path, std::ios_base::openmode mode) return new std::fstream(path, mode | std::ios::binary); } +static bool waitForFileReception(int sockfd, char* data, const std::string& filePath, + const std::string& fileName, int maxRetries) { + int count = 0; + int sleepMs = 50; + while (count < maxRetries) { + if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::FILE_RECV_CHK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + util_logger.debug("Sent: " + JasmineGraphInstanceProtocol::FILE_RECV_CHK); + + util_logger.debug("Checking if file is received"); + if (std::string response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); + response.compare(JasmineGraphInstanceProtocol::FILE_RECV_WAIT) == 0) { + util_logger.debug("Received: " + JasmineGraphInstanceProtocol::FILE_RECV_WAIT); + util_logger.debug("Checking file status : " + std::to_string(count)); + count++; + std::this_thread::sleep_for(std::chrono::milliseconds(sleepMs)); + // Exponential backoff up to 1 second + if (sleepMs < 1000) { + sleepMs = std::min(sleepMs * 2, 1000); + } + continue; + } else if (response.compare(JasmineGraphInstanceProtocol::FILE_ACK) == 0) { + util_logger.debug("Received: " + JasmineGraphInstanceProtocol::FILE_ACK); + util_logger.debug("File transfer completed for file : " + filePath); + return true; + } + count++; // Prevent infinite loop if unexpected response + } + util_logger.error("File reception timeout for: " + fileName); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; +} + +static bool waitForBatchUpload(int sockfd, char* data, const std::string& fileName, int maxRetries) { + int count = 0; + while (count < maxRetries) { + if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + util_logger.debug("Sent: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); + + if (std::string response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); + response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT) == 0) { + util_logger.debug("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT); + sleep(1); + continue; + } else if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK) == 0) { + util_logger.debug("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK); + util_logger.debug("Batch upload completed: " + fileName); + return true; + } + count++; // Prevent infinite loop if unexpected response + } + util_logger.error("Batch upload timeout for: " + fileName); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; +} + bool Utils::uploadFileToWorker(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP, std::string uploadType) { util_logger.debug("Host:" + host + " Port:" + to_string(port) + " DPort:" + to_string(dataPort)); @@ -1204,6 +1271,19 @@ bool Utils::uploadFileToWorker(std::string host, int port, int dataPort, int gra return false; } + // Optimize TCP socket for file uploads + int flag = 1; + setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(int)); + + // Increase send/receive buffer sizes for better throughput (1MB) + int bufsize = 1024 * 1024; + setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, (char *)&bufsize, sizeof(bufsize)); + setsockopt(sockfd, SOL_SOCKET, SO_RCVBUF, (char *)&bufsize, sizeof(bufsize)); + + // Enable address reuse + int reuse = 1; + setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse)); + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, uploadType, JasmineGraphInstanceProtocol::OK)) { Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); @@ -1238,50 +1318,16 @@ bool Utils::uploadFileToWorker(std::string host, int port, int dataPort, int gra util_logger.debug("Going to send file" + filePath + "/" + fileName + " through file transfer service to worker"); Utils::sendFileThroughService(host, dataPort, fileName, filePath); - string response; - int count = 0; - while (true) { - if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::FILE_RECV_CHK)) { - Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); - close(sockfd); - return false; - } - util_logger.debug("Sent: " + JasmineGraphInstanceProtocol::FILE_RECV_CHK); + int maxRetries = 300; // 5 minutes max with exponential backoff - util_logger.debug("Checking if file is received"); - response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::FILE_RECV_WAIT) == 0) { - util_logger.debug("Received: " + JasmineGraphInstanceProtocol::FILE_RECV_WAIT); - util_logger.debug("Checking file status : " + to_string(count)); - count++; - sleep(1); - continue; - } else if (response.compare(JasmineGraphInstanceProtocol::FILE_ACK) == 0) { - util_logger.debug("Received: " + JasmineGraphInstanceProtocol::FILE_ACK); - util_logger.debug("File transfer completed for file : " + filePath); - break; - } + if (!waitForFileReception(sockfd, data, filePath, fileName, maxRetries)) { + return false; } - // Next we wait till the batch upload completes - while (true) { - if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK)) { - Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); - close(sockfd); - return false; - } - util_logger.debug("Sent: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); - response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT) == 0) { - util_logger.debug("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT); - sleep(1); - continue; - } else if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK) == 0) { - util_logger.debug("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK); - util_logger.debug("Batch upload completed: " + fileName); - break; - } + if (!waitForBatchUpload(sockfd, data, fileName, maxRetries)) { + return false; } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; @@ -1450,6 +1496,19 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f return false; } + // Optimize TCP socket for high-throughput file transfers + int flag = 1; + setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(int)); + + // Increase send/receive buffer sizes for better throughput (1MB) + int bufsize = 1024 * 1024; + setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, (char *)&bufsize, sizeof(bufsize)); + setsockopt(sockfd, SOL_SOCKET, SO_RCVBUF, (char *)&bufsize, sizeof(bufsize)); + + // Enable address reuse + int reuse = 1; + setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse)); + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, JasmineGraphInstanceProtocol::SEND_FILE_LEN)) { close(sockfd); @@ -1464,13 +1523,16 @@ bool Utils::sendFileThroughService(std::string host, int dataPort, std::string f } bool status = true; + // Use larger buffer for better throughput (64KB) + const int BUFFER_SIZE = 65536; + std::vector buff(BUFFER_SIZE); + while (true) { - unsigned char buff[1024]; - int nread = fread(buff, 1, sizeof(buff), fp); + int nread = fread(buff.data(), 1, BUFFER_SIZE, fp); /* If read was success, send data. */ if (nread > 0) { - write(sockfd, buff, nread); + write(sockfd, buff.data(), nread); } else { if (feof(fp)) util_logger.debug("End of file"); if (ferror(fp)) { @@ -1599,9 +1661,8 @@ void Utils::assignPartitionToWorker(int graphId, int partitionIndex, string host } sqlite->finalize(); - sqliteMutex.unlock(); - delete sqlite; + sqliteMutex.unlock(); } diff --git a/src/util/telemetry/OpenTelemetryUtil.h b/src/util/telemetry/OpenTelemetryUtil.h index f02e17d45..eb6d41892 100644 --- a/src/util/telemetry/OpenTelemetryUtil.h +++ b/src/util/telemetry/OpenTelemetryUtil.h @@ -296,6 +296,10 @@ class ScopedTracer { std::string operation_name_; }; +#define CONCAT_INNER(a, b) a ## b +#define CONCAT(a, b) CONCAT_INNER(a, b) +#define UNIQUE_VAR(prefix) CONCAT(prefix, __LINE__) + /** * Convenience macro for automatic function tracing * Creates a scoped tracer that automatically traces the entire function @@ -309,7 +313,7 @@ class ScopedTracer { * @param op_name Name of the operation to trace */ #define OTEL_TRACE_OPERATION(op_name) \ - ScopedTracer __operation_tracer(op_name) + ScopedTracer UNIQUE_VAR(__operation_tracer_)(op_name) /** * Convenience macro for tracing with attributes @@ -317,4 +321,4 @@ class ScopedTracer { * @param attrs Map of attributes */ #define OTEL_TRACE_WITH_ATTRS(op_name, attrs) \ - ScopedTracer __operation_tracer(op_name, attrs) + ScopedTracer UNIQUE_VAR(__operation_tracer_)(op_name, attrs) diff --git a/test-k8s.sh b/test-k8s.sh index 60d94f2da..1e8345813 100755 --- a/test-k8s.sh +++ b/test-k8s.sh @@ -111,7 +111,7 @@ ready_hdfs() { cat $HDFS_CONF_FILE echo "Waiting for JasmineGraph Master pod to be running..." - kubectl wait --for=condition=Ready pod -l type=master,service=jasminegraph --timeout=300s + kubectl wait --for=condition=Ready pod -l type=master,service=jasminegraph --timeout=400s MASTER_POD=$(kubectl get pods | grep jasminegraph-master | awk '{print $1}') kubectl cp "${TEST_ROOT}/env_init/config/hdfs/hdfs_config.txt" ${MASTER_POD}:/var/tmp/config/hdfs_config.txt