fix(alerts): correct p-queue usage (#1355)

Avoid awating on the call to `add()`. Doing so causes the call to await not only for the function to be enqueued, but also finish execution.

This section of the [documentation](https://www.npmjs.com/package/p-queue) is key:
> [!IMPORTANT] If you await this promise, you will wait for the task to finish running, which may defeat the purpose of using a queue for concurrency. See the [Usage](https://www.npmjs.com/package/p-queue#usage) section for examples.

Author: dhable

.changeset/odd-countries-judge.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+Fix check alert to actually honor concurrent evaluation.

packages/api/src/tasks/checkAlerts/index.ts

@@ -793,7 +793,7 @@ export default class CheckAlertTask implements HdxTask<CheckAlertsTaskArgs> {
         );
 
         for (const alert of alerts) {
-          await this.task_queue.add(() =>
+          this.task_queue.add(async () =>
             processAlert(
               alertTask.now,
               alert,
@@ -855,7 +855,7 @@ export default class CheckAlertTask implements HdxTask<CheckAlertsTaskArgs> {
     for (const task of alertTasks) {
       const teamWebhooksById =
         teamToWebhooks.get(task.conn.team.toString()) ?? new Map();
-      await this.task_queue.add(() =>
+      this.task_queue.add(async () =>
         this.processAlertTask(task, teamWebhooksById),
       );
     }
@@ -866,6 +866,9 @@ export default class CheckAlertTask implements HdxTask<CheckAlertsTaskArgs> {
       'finished scheduling alert tasks on the task_queue',
     );
 
+    // make sure to await here to drain the work queue and allow
+    // functions to execute. if not, execute will terminate without
+    // executing all checks
     await this.task_queue.onIdle();
     logger.info(
       {