Skip to content

Commit c1429c7

Browse files
committed
[fix](executor) prevent BE crash when split process throws unexpectedly
Catch exceptions around split->process() in TimeSharingTaskExecutor and convert them to split failure status. This avoids worker thread termination and BE crash for cases : ``` erminate called after throwing an instance of 'doris::Exception' what(): [E-7412] assert cast err:[E-7412] Bad cast from ... doris::vectorized::ScannerSplitRunner::process_for(std::chrono::duration<long, std::ratio<1l, 1000000000l> >) at /home/zcp/repo_center/doris_release/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:420 10# doris::vectorized::PrioritizedSplitRunner::process() at /home/zcp/repo_center/doris_release/doris/be/src/vec/exec/executor/time_sharing/prioritized_split_runner.cpp:104 11# doris::vectorized::TimeSharingTaskExecutor::_dispatch_thread() at /home/zcp/repo_center/doris_release/doris/be/src/vec/exec/executor/time_sharing/time_sharing_task_executor.cpp:568 12# ``` w , while keeping MEM_ALLOC_FAILED mapped to MemoryLimitExceeded.
1 parent 24cde64 commit c1429c7

File tree

1 file changed

+24
-1
lines changed

1 file changed

+24
-1
lines changed

be/src/exec/scan/task_executor/time_sharing/time_sharing_task_executor.cpp

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -553,7 +553,30 @@ void TimeSharingTaskExecutor::_dispatch_thread() {
553553
_running_splits.insert(split);
554554
}
555555

556-
Result<SharedListenableFuture<Void>> blocked_future_result = split->process();
556+
auto blocked_future_result = [&]() -> Result<SharedListenableFuture<Void>> {
557+
try {
558+
doris::enable_thread_catch_bad_alloc++;
559+
Defer defer {[&]() { doris::enable_thread_catch_bad_alloc--; }};
560+
return split->process();
561+
} catch (const doris::Exception& e) {
562+
if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) {
563+
return unexpected(Status::MemoryLimitExceeded(
564+
"PreCatch error code:{}, {}, __FILE__:{}, __LINE__:{}, "
565+
"__FUNCTION__:{}",
566+
e.code(), e.to_string(), __FILE__, __LINE__, __PRETTY_FUNCTION__));
567+
}
568+
return unexpected(e.to_status());
569+
} catch (const std::exception& e) {
570+
return unexpected(Status::InternalError(
571+
"split process threw std::exception, split_id: {}, worker_id: {}, "
572+
"reason: {}",
573+
split->split_id(), split->worker_id(), e.what()));
574+
} catch (...) {
575+
return unexpected(Status::InternalError(
576+
"split process threw unknown exception, split_id: {}, worker_id: {}",
577+
split->split_id(), split->worker_id()));
578+
}
579+
}();
557580

558581
if (!blocked_future_result.has_value()) {
559582
LOG(WARNING) << "split process failed, split_id: " << split->split_id()

0 commit comments

Comments
 (0)