Skip to content

feat: Add batch job submission for task graph #82

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sitaowang1998 merged 2 commits into from
Mar 25, 2025
Merged

feat: Add batch job submission for task graph #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a6fc40f
Add batch task submisstion for task graph
sitaowang1998 Mar 24, 2025
7e2bbdb
Fix clang tidy by split up client test
sitaowang1998 Mar 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions src/spider/client/Driver.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,10 +270,23 @@ class Driver {
graph.m_impl->reset_ids();
boost::uuids::random_generator gen;
boost::uuids::uuid const job_id = gen();
core::StorageErr const err
= m_metadata_storage->add_job(*m_conn, job_id, m_id, graph.m_impl->get_graph());
if (!err.success()) {
throw ConnectionException(fmt::format("Failed to start job: {}", err.description));
if (nullptr != m_batch) {
core::StorageErr const err = m_metadata_storage->add_job_batch(
*m_conn,
*m_batch,
job_id,
m_id,
graph.m_impl->get_graph()
);
if (!err.success()) {
throw ConnectionException(fmt::format("Failed to start job: {}", err.description));
}
} else {
core::StorageErr const err
= m_metadata_storage->add_job(*m_conn, job_id, m_id, graph.m_impl->get_graph());
if (!err.success()) {
throw ConnectionException(fmt::format("Failed to start job: {}", err.description));
}
}

return Job<ReturnType>{
Expand Down
177 changes: 119 additions & 58 deletions tests/client/client-test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,36 +42,7 @@ constexpr int cJobFailed = 2;

constexpr int cBatchSize = 10;

} // namespace

// NOLINTNEXTLINE(bugprone-exception-escape)
auto main(int argc, char** argv) -> int {
// NOLINTNEXTLINE(misc-include-cleaner)
spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%^%l%$] [spider.scheduler] %v");
#ifndef NDEBUG
spdlog::set_level(spdlog::level::trace);
#endif

boost::program_options::variables_map const args = parse_args(argc, argv);

std::string storage_url;
try {
if (!args.contains("storage_url")) {
spdlog::error("storage_url is required");
return cCmdArgParseErr;
}
storage_url = args["storage_url"].as<std::string>();
} catch (boost::bad_any_cast& e) {
return cCmdArgParseErr;
} catch (boost::program_options::error& e) {
return cCmdArgParseErr;
}

// Create driver
spider::Driver driver{storage_url};
spdlog::debug("Driver created");

// Run a complicated graph that should succeed
auto test_graph(spider::Driver& driver) -> int {
spider::TaskGraph const left = driver.bind(&sum_test, &data_test, &data_test);
spider::TaskGraph const graph = driver.bind(&sum_test, left, &sum_test);
spdlog::debug("Graph created");
Expand All @@ -91,16 +62,18 @@ auto main(int argc, char** argv) -> int {
spdlog::error("Wrong job result. Get {}. Expect 10", graph_job.get_result());
return cJobFailed;
}
return 0;
}

// Run task with multiple results should succeed
spider::Job<std::tuple<int, int>> swap_job = driver.start(&swap_test, 1, 2);
auto test_multi_result(spider::Driver& driver) -> int {
spider::Job<std::tuple<int, int>> job = driver.start(&swap_test, 1, 2);
spdlog::debug("Multiple result job started");
swap_job.wait_complete();
if (swap_job.get_status() != spider::JobStatus::Succeeded) {
job.wait_complete();
if (job.get_status() != spider::JobStatus::Succeeded) {
spdlog::error("Multiple result job failed");
return cJobFailed;
}
std::tuple<int, int> swap_result = swap_job.get_result();
std::tuple<int, int> swap_result = job.get_result();
if (std::get<0>(swap_result) != 2 || std::get<1>(swap_result) != 1) {
spdlog::error(
"Wrong multiple result job result. Get ({}, {}). Expect (2, 1)",
Expand All @@ -109,8 +82,10 @@ auto main(int argc, char** argv) -> int {
);
return cJobFailed;
}
return 0;
}

// Run fail job
auto test_fail_job(spider::Driver& driver) -> int {
spider::Job fail_job = driver.start(&error_test, 1);
spdlog::debug("Fail job started");
fail_job.wait_complete();
Expand All @@ -119,48 +94,43 @@ auto main(int argc, char** argv) -> int {
spdlog::error("Job should fail");
return cJobFailed;
}
return 0;
}

// Run random fail job
constexpr int cFailRate = 5;
spider::Job random_fail_job = driver.start(&random_fail_test, cFailRate);
spdlog::debug("Random fail job started");
random_fail_job.wait_complete();
spdlog::debug("Random fail job completed");
if (random_fail_job.get_status() != spider::JobStatus::Succeeded) {
spdlog::error("Random fail job failed");
return cJobFailed;
}

// Run task that creates data
spider::Job create_data_job = driver.start(&create_data_test, 1);
auto test_task_create_data(spider::Driver& driver) -> int {
spider::Job job = driver.start(&create_data_test, 1);
spdlog::debug("Create data job started");
create_data_job.wait_complete();
job.wait_complete();
spdlog::debug("Create data job completed");
if (create_data_job.get_status() != spider::JobStatus::Succeeded) {
if (job.get_status() != spider::JobStatus::Succeeded) {
spdlog::error("Create data job failed");
return cJobFailed;
}
spider::Data<int> data_result = create_data_job.get_result();
spider::Data<int> data_result = job.get_result();
if (data_result.get() != 1) {
spdlog::error("Create data job failed");
return cJobFailed;
}
return 0;
}

// Run task that creates task
spider::Job create_task_job = driver.start(&create_task_test, 1, 2);
auto test_task_create_task(spider::Driver& driver) -> int {
spider::Job job = driver.start(&create_task_test, 1, 2);
spdlog::debug("Create task job started");
create_task_job.wait_complete();
job.wait_complete();
spdlog::debug("Create task job completed");
if (create_task_job.get_status() != spider::JobStatus::Succeeded) {
if (job.get_status() != spider::JobStatus::Succeeded) {
spdlog::error("Create task job failed");
return cJobFailed;
}
if (create_task_job.get_result() != 3) {
if (job.get_result() != 3) {
spdlog::error("Create task job failed");
return cJobFailed;
}
return 0;
}

// Run batch submission
auto test_function_batch_submission(spider::Driver& driver) -> int {
std::vector<spider::Job<int>> jobs;
jobs.reserve(cBatchSize);
driver.begin_batch_start();
Expand All @@ -181,6 +151,97 @@ auto main(int argc, char** argv) -> int {
return cJobFailed;
}
}
return 0;
}

auto test_graph_batch_submission(spider::Driver& driver) -> int {
std::vector<spider::Job<int>> jobs;
jobs.reserve(cBatchSize);
driver.begin_batch_start();
spider::TaskGraph const graph = driver.bind(&sum_test, &sum_test, &sum_test);
for (int i = 0; i < cBatchSize; ++i) {
jobs.emplace_back(driver.start(graph, i, i, i, i));
}
driver.end_batch_start();
for (int i = 0; i < cBatchSize; ++i) {
spider::Job<int>& job = jobs[i];
job.wait_complete();
if (job.get_status() != spider::JobStatus::Succeeded) {
spdlog::error("Batch graph job failed");
return cJobFailed;
}
int const result = job.get_result();
if (result != i * 4) {
spdlog::error("Batch job wrong result. Expect {}. Get {}.", i * 4, result);
return cJobFailed;
}
}
return 0;
}

} // namespace

// NOLINTNEXTLINE(bugprone-exception-escape)
auto main(int argc, char** argv) -> int {
// NOLINTNEXTLINE(misc-include-cleaner)
spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%^%l%$] [spider.scheduler] %v");
#ifndef NDEBUG
spdlog::set_level(spdlog::level::trace);
#endif

boost::program_options::variables_map const args = parse_args(argc, argv);

std::string storage_url;
try {
if (!args.contains("storage_url")) {
spdlog::error("storage_url is required");
return cCmdArgParseErr;
}
storage_url = args["storage_url"].as<std::string>();
} catch (boost::bad_any_cast& e) {
return cCmdArgParseErr;
} catch (boost::program_options::error& e) {
return cCmdArgParseErr;
}

// Create driver
spider::Driver driver{storage_url};
spdlog::debug("Driver created");

int result = test_graph(driver);
if (0 != result) {
return result;
}

result = test_multi_result(driver);
if (0 != result) {
return result;
}

result = test_fail_job(driver);
if (0 != result) {
return result;
}

result = test_task_create_data(driver);
if (0 != result) {
return result;
}

result = test_task_create_task(driver);
if (0 != result) {
return result;
}

result = test_function_batch_submission(driver);
if (0 != result) {
return result;
}

result = test_graph_batch_submission(driver);
if (0 != result) {
return result;
}

return 0;
}