feat: separate unique violation retry from deadlock retry (#1674)

* feat: separate unique violation retry from deadlock retry

* fix: error import

Author: boris-w

apps/nestjs-backend/src/features/record/open-api/record-open-api.service.ts

@@ -20,7 +20,7 @@ import { IThresholdConfig, ThresholdConfig } from '../../../configs/threshold.co
 import { EventEmitterService } from '../../../event-emitter/event-emitter.service';
 import { Events } from '../../../event-emitter/events';
 import type { IClsStore } from '../../../types/cls';
-import { retryOnDeadlock } from '../../../utils/retry-on-dead-lock';
+import { retryOnDeadlock, retryOnUniqueViolation } from '../../../utils/retry-decorator';
 import { AttachmentsStorageService } from '../../attachments/attachments-storage.service';
 import { AttachmentsService } from '../../attachments/attachments.service';
 import { getPublicFullStorageUrl } from '../../attachments/plugins/utils';
@@ -54,6 +54,7 @@ export class RecordOpenApiService {
   ) {}
 
   @retryOnDeadlock()
+  @retryOnUniqueViolation()
   async multipleCreateRecords(
     tableId: string,
     createRecordsRo: ICreateRecordsRo,
@@ -263,6 +264,7 @@ export class RecordOpenApiService {
   }
 
   @retryOnDeadlock()
+  @retryOnUniqueViolation()
   async updateRecords(
     tableId: string,
     updateRecordsRo: IUpdateRecordsRo & {

…end/src/utils/retry-on-dead-lock.spec.ts …ackend/src/utils/retry-decorator.spec.tsapps/nestjs-backend/src/utils/retry-on-dead-lock.spec.ts renamed to apps/nestjs-backend/src/utils/retry-decorator.spec.ts apps/nestjs-backend/src/utils/retry-on-dead-lock.spec.ts renamed to apps/nestjs-backend/src/utils/retry-decorator.spec.ts

@@ -1,6 +1,6 @@
 /* eslint-disable sonarjs/no-identical-functions */
 import { Prisma } from '@prisma/client';
-import { retryOnDeadlock } from './retry-on-dead-lock';
+import { retryOnDeadlock } from './retry-decorator';
 
 class TestService {
   @retryOnDeadlock()
@@ -37,7 +37,7 @@ describe('RetryOnDeadlock Decorator', () => {
       thresholdConfig: () => ({
         dbDeadlock: {
           maxRetries: 3,
-          initialBackoff: 300,
+          initialBackoff: 200,
           jitter: 1,
         },
       }),

apps/nestjs-backend/src/utils/retry-decorator.ts

@@ -0,0 +1,78 @@
+import { Logger } from '@nestjs/common';
+import { HttpErrorCode } from '@teable/core';
+import { thresholdConfig } from '../configs/threshold.config';
+import { CustomHttpException } from '../custom.exception';
+
+interface IRetryOptions {
+  maxRetries?: number;
+  initialBackoff?: number;
+  jitter?: number;
+}
+
+interface IRetryConfig {
+  errorCodes: string[];
+  errorMessage: string;
+  errorCode: HttpErrorCode;
+  loggerName: string;
+}
+
+function createRetryDecorator(config: IRetryConfig) {
+  const logger = new Logger(config.loggerName);
+
+  return function (opt?: IRetryOptions) {
+    const { dbDeadlock } = thresholdConfig();
+    const {
+      maxRetries = dbDeadlock.maxRetries,
+      initialBackoff = dbDeadlock.initialBackoff,
+      jitter = dbDeadlock.jitter,
+    } = opt ?? {};
+
+    return function (_target: unknown, _propertyKey: string, descriptor: PropertyDescriptor) {
+      const originalMethod = descriptor.value;
+
+      descriptor.value = async function (...args: unknown[]) {
+        let retries = 0;
+        let backoff = initialBackoff + Math.random() * jitter;
+
+        while (retries <= maxRetries) {
+          try {
+            return await originalMethod.apply(this, args);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          } catch (error: any) {
+            const { errorCodes, errorMessage, errorCode } = config;
+            if (
+              errorCodes.includes(error.code) ||
+              (error.meta?.code && errorCodes.includes(error.meta.code as string))
+            ) {
+              if (retries === maxRetries) {
+                logger.error(`${errorMessage} after ${retries} retries`, error.stack);
+                throw new CustomHttpException(errorMessage, errorCode);
+              }
+              await new Promise((resolve) => setTimeout(resolve, backoff));
+              backoff *= 1.5 + Math.random() * jitter;
+            } else {
+              throw error;
+            }
+          }
+          retries++;
+        }
+      };
+
+      return descriptor;
+    };
+  };
+}
+
+export const retryOnDeadlock = createRetryDecorator({
+  errorCodes: ['40P01', 'P2034'],
+  errorMessage: 'Database deadlock detected',
+  errorCode: HttpErrorCode.DATABASE_CONNECTION_UNAVAILABLE,
+  loggerName: 'DeadlockRetryDecorator',
+});
+
+export const retryOnUniqueViolation = createRetryDecorator({
+  errorCodes: ['23505'],
+  errorMessage: 'Database unique violation detected',
+  errorCode: HttpErrorCode.DATABASE_CONNECTION_UNAVAILABLE,
+  loggerName: 'UniqueViolationRetryDecorator',
+});

apps/nestjs-backend/src/utils/retry-on-dead-lock.ts

@@ -2,7 +2,7 @@
 import type { INestApplication } from '@nestjs/common';
 import { DriverClient } from '@teable/core';
 import { Prisma, PrismaService } from '@teable/db-main-prisma';
-import { retryOnDeadlock } from '../src/utils/retry-on-dead-lock';
+import { retryOnDeadlock } from '../src/utils/retry-decorator';
 import { initApp } from './utils/init-app';
 
 const deadLockTableA = 'dead_lock_a';

apps/nestjs-backend/test/dead-lock.e2e-spec.ts

@@ -1,6 +1,6 @@
 /* eslint-disable sonarjs/no-duplicate-string */
 import type { INestApplication } from '@nestjs/common';
-import type { IFieldRo, ISelectFieldOptions } from '@teable/core';
+import type { IFieldRo, IFieldVo, ISelectFieldOptions } from '@teable/core';
 import { CellFormat, DriverClient, FieldKeyType, FieldType, Relationship } from '@teable/core';
 import { updateRecords, type ITableFullVo } from '@teable/openapi';
 import {
@@ -910,4 +910,118 @@ describe('OpenAPI RecordController (e2e)', () => {
       expect(records).toBeDefined();
     });
   });
+
+  describe('ops index conflict', () => {
+    let table: ITableFullVo;
+    let tableLinkField: IFieldVo;
+    let linkTable: ITableFullVo;
+    beforeAll(async () => {
+      table = await createTable(baseId, {
+        name: 'table1',
+        fields: [
+          {
+            type: FieldType.SingleLineText,
+            name: 'field1',
+          },
+        ],
+      });
+      linkTable = await createTable(baseId, {
+        name: 'linkTable',
+        fields: [
+          {
+            type: FieldType.SingleLineText,
+            name: 'field1',
+          },
+        ],
+        records: [
+          {
+            fields: {
+              field1: 'test1',
+            },
+          },
+          {
+            fields: {
+              field1: 'test2',
+            },
+          },
+          {
+            fields: {
+              field1: 'test3',
+            },
+          },
+          {
+            fields: {
+              field1: 'test4',
+            },
+          },
+        ],
+      });
+      tableLinkField = await createField(table.id, {
+        name: 'linkField',
+        type: FieldType.Link,
+        options: {
+          relationship: Relationship.ManyMany,
+          foreignTableId: linkTable.id,
+        },
+      });
+    });
+
+    afterAll(async () => {
+      await permanentDeleteTable(baseId, table.id);
+      await permanentDeleteTable(baseId, linkTable.id);
+    });
+
+    it('should create a record with link field', async () => {
+      await Promise.all([
+        createRecords(table.id, {
+          records: [
+            {
+              fields: {
+                [tableLinkField.id]: [{ id: linkTable.records[0].id }],
+              },
+            },
+            {
+              fields: {
+                [tableLinkField.id]: [{ id: linkTable.records[1].id }],
+              },
+            },
+            {
+              fields: {
+                [tableLinkField.id]: [{ id: linkTable.records[2].id }],
+              },
+            },
+            {
+              fields: {
+                [tableLinkField.id]: [{ id: linkTable.records[3].id }],
+              },
+            },
+          ],
+        }),
+        createRecords(table.id, {
+          records: [
+            {
+              fields: {
+                [tableLinkField.id]: [{ id: linkTable.records[0].id }],
+              },
+            },
+            {
+              fields: {
+                [tableLinkField.id]: [{ id: linkTable.records[1].id }],
+              },
+            },
+            {
+              fields: {
+                [tableLinkField.id]: [{ id: linkTable.records[2].id }],
+              },
+            },
+            {
+              fields: {
+                [tableLinkField.id]: [{ id: linkTable.records[3].id }],
+              },
+            },
+          ],
+        }),
+      ]);
+    });
+  });
 });