Skip to content

Commit fad087c

Browse files
committed
Fix timeout logic
1 parent ad3cb60 commit fad087c

File tree

1 file changed

+4
-4
lines changed

1 file changed

+4
-4
lines changed

csrc/nv_internal/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,7 @@ __global__ void moeA2ADispatchKernel(
403403
#pragma unroll 1 // No unroll
404404
for (int peer_rank = lane_id; peer_rank < ep_size; peer_rank += warpSize) {
405405
bool flag_set = false;
406-
[[maybe_unused]] clock_t s = clock64();
406+
[[maybe_unused]] auto s = clock64();
407407
do {
408408
uint32_t* flag_ptr = &ptrs.completion_flags[rank_id][peer_rank];
409409
uint32_t flag_value;
@@ -417,7 +417,7 @@ __global__ void moeA2ADispatchKernel(
417417
rank_id, peer_rank, flag_value, expected_value, flag_ptr);
418418
#endif
419419
flag_set = flag_value == expected_value;
420-
} while (!flag_set || check_timeout(s));
420+
} while (!flag_set && !check_timeout(s));
421421

422422
if (__builtin_expect(!flag_set, 0)) {
423423
printf("dispatch: ---Rank %d timed out waiting for completion flag from rank %d\n",
@@ -708,7 +708,7 @@ __global__ void moeA2ACombineKernel(
708708
#pragma unroll 1 // No unroll
709709
for (int peer_rank = lane_id; peer_rank < ep_size; peer_rank += warpSize) {
710710
bool flag_set = false;
711-
[[maybe_unused]] clock_t s = clock64();
711+
[[maybe_unused]] auto s = clock64();
712712
do {
713713
uint32_t* flag_ptr = &ptrs.completion_flags[rank_id][peer_rank];
714714
uint32_t flag_value;
@@ -722,7 +722,7 @@ __global__ void moeA2ACombineKernel(
722722
rank_id, peer_rank, flag_value, expected_value, flag_ptr);
723723
#endif
724724
flag_set = flag_value == expected_value;
725-
} while (!flag_set || check_timeout(s));
725+
} while (!flag_set && !check_timeout(s));
726726

727727
if (__builtin_expect(!flag_set, 0)) {
728728
printf("combine: ---Rank %d timed out waiting for completion flag from rank %d\n", rank_id,

0 commit comments

Comments
 (0)