fix: return all SQS messages as batch failures on non-ScaleError exceptions; enable throttle retries

lambdas/functions/control-plane/src/github/auth.test.ts

@@ -62,6 +62,7 @@ describe('Test createOctoClient', () => {
     expect(result.request.endpoint.DEFAULTS.baseUrl).toBe(enterpriseServer);
     expect(result.request.endpoint.DEFAULTS.mediaType.previews).toStrictEqual(['antiope']);
   });
+
 });
 
 describe('Test createGithubAppAuth', () => {

lambdas/functions/control-plane/src/github/auth.ts

@@ -54,11 +54,15 @@ export async function createOctokitClient(token: string, ghesApiUrl = ''): Promi
     throttle: {
       onRateLimit: (retryAfter: number, options: Required<EndpointDefaults>) => {
         logger.warn(
-          `GitHub rate limit: Request quota exhausted for request ${options.method} ${options.url}. Requested `,
+          `GitHub rate limit: Request quota exhausted for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds.`,
         );
+        return true;
       },
       onSecondaryRateLimit: (retryAfter: number, options: Required<EndpointDefaults>) => {
-        logger.warn(`GitHub rate limit: SecondaryRateLimit detected for request ${options.method} ${options.url}`);
+        logger.warn(
+          `GitHub rate limit: SecondaryRateLimit detected for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds.`,
+        );
+        return true;
       },
     },
   });

lambdas/functions/control-plane/src/lambda.test.ts

@@ -97,10 +97,11 @@ describe('Test scale up lambda wrapper.', () => {
     await expect(scaleUpHandler(sqsEvent, context)).resolves.not.toThrow();
   });
 
-  it('Non scale should resolve.', async () => {
+  it('Non scale error should return message as batch failure for retry.', async () => {
     const error = new Error('Non scale should resolve.');
     vi.mocked(scaleUp).mockRejectedValue(error);
-    await expect(scaleUpHandler(sqsEvent, context)).resolves.not.toThrow();
+    const result = await scaleUpHandler(sqsEvent, context);
+    expect(result).toEqual({ batchItemFailures: [{ itemIdentifier: sqsRecord.messageId }] });
   });
 
   it('Scale should create a batch failure message', async () => {
@@ -208,13 +209,22 @@ describe('Test scale up lambda wrapper.', () => {
       await scaleUpHandler(multiRecordEvent, context);
     });
 
-    it('Should return all failed messages when scaleUp throws non-ScaleError', async () => {
+    it('Should return all messages as failures when scaleUp throws non-ScaleError', async () => {
       const records = createMultipleRecords(2);
       const multiRecordEvent: SQSEvent = { Records: records };
 
       vi.mocked(scaleUp).mockRejectedValue(new Error('Generic error'));
 
       const result = await scaleUpHandler(multiRecordEvent, context);
+      expect(result).toEqual({
+        batchItemFailures: [{ itemIdentifier: 'message-0' }, { itemIdentifier: 'message-1' }],
+      });
+    });
+
+    it('Should return empty failures when batch is empty and scaleUp throws non-ScaleError', async () => {
+      const emptyEvent: SQSEvent = { Records: [] };
+      vi.mocked(scaleUp).mockRejectedValue(new Error('Generic error'));
+      const result = await scaleUpHandler(emptyEvent, context);
       expect(result).toEqual({ batchItemFailures: [] });
     });
 

lambdas/functions/control-plane/src/lambda.ts

@@ -55,9 +55,11 @@ export async function scaleUpHandler(event: SQSEvent, context: Context): Promise
       batchItemFailures.push(...e.toBatchItemFailures(sqsMessages));
       logger.warn(`${e.detailedMessage} A retry will be attempted via SQS.`, { error: e });
     } else {
-      logger.error(`Error processing batch (size: ${sqsMessages.length}): ${(e as Error).message}, ignoring batch`, {
-        error: e,
-      });
+      logger.error(
+        `Error processing batch (size: ${sqsMessages.length}): ${(e as Error).message}, batch will be retried via SQS.`,
+        { error: e },
+      );
+      batchItemFailures.push(...sqsMessages.map(({ messageId }) => ({ itemIdentifier: messageId })));
     }
 
     return { batchItemFailures };