FAI-17536 | Add GitLab epics stream with tms_Epic integration

destinations/airbyte-faros-destination/src/converters/gitlab/faros_epics.ts

@@ -0,0 +1,53 @@
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+import {FarosEpicOutput} from 'faros-airbyte-common/gitlab';
+import {Utils} from 'faros-js-client';
+
+import {DestinationModel, DestinationRecord} from '../converter';
+import {GitlabCommon, GitlabConverter} from './common';
+
+export class FarosEpics extends GitlabConverter {
+  readonly destinationModels: ReadonlyArray<DestinationModel> = ['tms_Epic'];
+
+  id(record: AirbyteRecord): string {
+    const epic = record?.record?.data as FarosEpicOutput;
+    return String(epic?.id);
+  }
+
+  async convert(
+    record: AirbyteRecord
+  ): Promise<ReadonlyArray<DestinationRecord>> {
+    const epic = record.record.data as FarosEpicOutput;
+    const res: DestinationRecord[] = [];
+
+    if (!epic?.id) {
+      return [];
+    }
+
+    const uid = String(epic.id);
+
+    // Convert epic state to status category
+    const statusCategory = epic.state === 'opened' ? 'Todo' : 'Done';
+
+    // Create the epic record
+    res.push({
+      model: 'tms_Epic',
+      record: {
+        uid,
+        name: epic.title,
+        description: Utils.cleanAndTruncate(
+          epic.description,
+          GitlabCommon.MAX_DESCRIPTION_LENGTH
+        ),
+        status: {
+          category: statusCategory,
+          detail: epic.state,
+        },
+        source: this.streamName.source,
+        createdAt: Utils.toDate(epic.created_at),
+        updatedAt: Utils.toDate(epic.updated_at),
+      },
+    });
+
+    return res;
+  }
+}

destinations/airbyte-faros-destination/src/converters/gitlab/faros_issues.ts

@@ -34,7 +34,6 @@ export class FarosIssues extends GitlabConverter {
     const uid = String(issue.id);
     const taskKey = {uid, source: this.streamName.source};
 
-
     // Create project key from project_path and group_id
     const projectKey = {
       uid: `${toLower(issue.group_id)}/${toLower(issue.project_path)}`,
@@ -86,6 +85,9 @@ export class FarosIssues extends GitlabConverter {
         creator: issue.author_username
           ? {uid: issue.author_username, source: this.streamName.source}
           : null,
+        epic: issue.epic_id
+          ? {uid: String(issue.epic_id), source: this.streamName.source}
+          : undefined,
         createdAt: Utils.toDate(issue.created_at),
         updatedAt: Utils.toDate(issue.updated_at),
         source: this.streamName.source,

destinations/airbyte-faros-destination/test/converters/__snapshots__/faros_gitlab.test.ts.snap

@@ -2,10 +2,10 @@
 
 exports[`faros_gitlab process records from all streams 1`] = `
 Array [
-  "Processed 13 records",
-  "Processed records by stream: {\\\\\\"mytestsource__gitlab__faros_commits\\\\\\":2,\\\\\\"mytestsource__gitlab__faros_deployments\\\\\\":3,\\\\\\"mytestsource__gitlab__faros_groups\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_issues\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_merge_request_reviews\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_merge_requests\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_projects\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_releases\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_tags\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_users\\\\\\":1}\\"},\\"type\\":\\"LOG\\"}",
-  "Would write 48 records",
-  "Would write records by model: {\\\\\\"cicd_Artifact\\\\\\":2,\\\\\\"cicd_ArtifactCommitAssociation\\\\\\":2,\\\\\\"cicd_ArtifactDeployment\\\\\\":2,\\\\\\"cicd_Build\\\\\\":2,\\\\\\"cicd_BuildCommitAssociation\\\\\\":2,\\\\\\"cicd_Deployment\\\\\\":3,\\\\\\"cicd_Organization\\\\\\":1,\\\\\\"cicd_Pipeline\\\\\\":1,\\\\\\"cicd_Release\\\\\\":1,\\\\\\"cicd_ReleaseTagAssociation\\\\\\":1,\\\\\\"compute_Application\\\\\\":4,\\\\\\"faros_VcsRepositoryOptions\\\\\\":1,\\\\\\"tms_Label\\\\\\":1,\\\\\\"tms_Project\\\\\\":1,\\\\\\"tms_Task\\\\\\":1,\\\\\\"tms_TaskAssignment\\\\\\":1,\\\\\\"tms_TaskBoard\\\\\\":1,\\\\\\"tms_TaskBoardProjectRelationship\\\\\\":1,\\\\\\"tms_TaskBoardRelationship\\\\\\":1,\\\\\\"tms_TaskProjectRelationship\\\\\\":1,\\\\\\"tms_TaskTag\\\\\\":1,\\\\\\"vcs_Branch\\\\\\":2,\\\\\\"vcs_Commit\\\\\\":2,\\\\\\"vcs_Label\\\\\\":1,\\\\\\"vcs_Membership\\\\\\":1,\\\\\\"vcs_Organization\\\\\\":1,\\\\\\"vcs_PullRequest\\\\\\":1,\\\\\\"vcs_PullRequestComment\\\\\\":2,\\\\\\"vcs_PullRequestLabel\\\\\\":1,\\\\\\"vcs_PullRequestReview\\\\\\":3,\\\\\\"vcs_Repository\\\\\\":1,\\\\\\"vcs_Tag\\\\\\":1,\\\\\\"vcs_User\\\\\\":1}\\"},\\"type\\":\\"LOG\\"}",
+  "Processed 15 records",
+  "Processed records by stream: {\\\\\\"mytestsource__gitlab__faros_commits\\\\\\":2,\\\\\\"mytestsource__gitlab__faros_deployments\\\\\\":3,\\\\\\"mytestsource__gitlab__faros_epics\\\\\\":2,\\\\\\"mytestsource__gitlab__faros_groups\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_issues\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_merge_request_reviews\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_merge_requests\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_projects\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_releases\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_tags\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_users\\\\\\":1}\\"},\\"type\\":\\"LOG\\"}",
+  "Would write 50 records",
+  "Would write records by model: {\\\\\\"cicd_Artifact\\\\\\":2,\\\\\\"cicd_ArtifactCommitAssociation\\\\\\":2,\\\\\\"cicd_ArtifactDeployment\\\\\\":2,\\\\\\"cicd_Build\\\\\\":2,\\\\\\"cicd_BuildCommitAssociation\\\\\\":2,\\\\\\"cicd_Deployment\\\\\\":3,\\\\\\"cicd_Organization\\\\\\":1,\\\\\\"cicd_Pipeline\\\\\\":1,\\\\\\"cicd_Release\\\\\\":1,\\\\\\"cicd_ReleaseTagAssociation\\\\\\":1,\\\\\\"compute_Application\\\\\\":4,\\\\\\"faros_VcsRepositoryOptions\\\\\\":1,\\\\\\"tms_Epic\\\\\\":2,\\\\\\"tms_Label\\\\\\":1,\\\\\\"tms_Project\\\\\\":1,\\\\\\"tms_Task\\\\\\":1,\\\\\\"tms_TaskAssignment\\\\\\":1,\\\\\\"tms_TaskBoard\\\\\\":1,\\\\\\"tms_TaskBoardProjectRelationship\\\\\\":1,\\\\\\"tms_TaskBoardRelationship\\\\\\":1,\\\\\\"tms_TaskProjectRelationship\\\\\\":1,\\\\\\"tms_TaskTag\\\\\\":1,\\\\\\"vcs_Branch\\\\\\":2,\\\\\\"vcs_Commit\\\\\\":2,\\\\\\"vcs_Label\\\\\\":1,\\\\\\"vcs_Membership\\\\\\":1,\\\\\\"vcs_Organization\\\\\\":1,\\\\\\"vcs_PullRequest\\\\\\":1,\\\\\\"vcs_PullRequestComment\\\\\\":2,\\\\\\"vcs_PullRequestLabel\\\\\\":1,\\\\\\"vcs_PullRequestReview\\\\\\":3,\\\\\\"vcs_Repository\\\\\\":1,\\\\\\"vcs_Tag\\\\\\":1,\\\\\\"vcs_User\\\\\\":1}\\"},\\"type\\":\\"LOG\\"}",
   "Skipped 0 records",
   "Errored 0 records",
 ]
@@ -452,6 +452,10 @@ Array [
         "uid": "ypc-faros",
       },
       "description": "There's something clearly broken",
+      "epic": Object {
+        "source": "GitLab",
+        "uid": "1001",
+      },
       "name": "Test issue",
       "source": "GitLab",
       "status": Object {
@@ -849,6 +853,34 @@ Array [
       "url": null,
     },
   },
+  Object {
+    "tms_Epic": Object {
+      "createdAt": "2024-01-01T10:00:00.000Z",
+      "description": "Epic for implementing comprehensive user authentication system",
+      "name": "User Authentication Epic",
+      "source": "GitLab",
+      "status": Object {
+        "category": "Todo",
+        "detail": "opened",
+      },
+      "uid": "1001",
+      "updatedAt": "2024-02-15T14:30:00.000Z",
+    },
+  },
+  Object {
+    "tms_Epic": Object {
+      "createdAt": "2024-01-15T09:00:00.000Z",
+      "description": "Sub-epic for OAuth provider integrations",
+      "name": "OAuth Integration",
+      "source": "GitLab",
+      "status": Object {
+        "category": "Done",
+        "detail": "closed",
+      },
+      "uid": "1002",
+      "updatedAt": "2024-02-28T16:00:00.000Z",
+    },
+  },
   Object {
     "vcs_Branch": Object {
       "name": "feature/add-new-feature",

destinations/airbyte-faros-destination/test/resources/faros_gitlab/all-streams.json

@@ -210,6 +210,7 @@
         "assignee_usernames": [
           "test-assignee"
         ],
+        "epic_id": 1001,
         "group_id": "12372707",
         "project_path": "testy"
       }
@@ -503,5 +504,39 @@
       }
     },
     "type": "RECORD"
+  },
+  {
+    "record": {
+      "stream": "mytestsource__gitlab__faros_epics",
+      "emitted_at": 1751581500000,
+      "data": {
+        "__brand": "FarosEpic",
+        "id": 1001,
+        "group_id": "1001",
+        "title": "User Authentication Epic",
+        "description": "Epic for implementing comprehensive user authentication system",
+        "state": "opened",
+        "created_at": "2024-01-01T10:00:00.000Z",
+        "updated_at": "2024-02-15T14:30:00.000Z"
+      }
+    },
+    "type": "RECORD"
+  },
+  {
+    "record": {
+      "stream": "mytestsource__gitlab__faros_epics",
+      "emitted_at": 1751581500000,
+      "data": {
+        "__brand": "FarosEpic",
+        "id": 1002,
+        "group_id": "1001",
+        "title": "OAuth Integration",
+        "description": "Sub-epic for OAuth provider integrations",
+        "state": "closed",
+        "created_at": "2024-01-15T09:00:00.000Z",
+        "updated_at": "2024-02-28T16:00:00.000Z"
+      }
+    },
+    "type": "RECORD"
   }
 ]

destinations/airbyte-faros-destination/test/resources/faros_gitlab/catalog.json

@@ -59,6 +59,12 @@
         "name": "mytestsource__gitlab__faros_deployments"
       },
       "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "mytestsource__gitlab__faros_epics"
+      },
+      "destination_sync_mode": "overwrite"
     }
   ]
 }

faros-airbyte-common/src/gitlab/types.ts

@@ -2,6 +2,7 @@ import {
   Camelize,
   CommitSchema,
   DeploymentSchema,
+  EpicSchema,
   EventSchema,
   GroupSchema,
   IssueSchema,
@@ -128,6 +129,7 @@ export type FarosIssueOutput = {
   project_path: string;
   author_username: string;
   assignee_usernames: string[];
+  epic_id?: number | null;
 } & Pick<
   IssueSchema,
   | 'id'
@@ -146,12 +148,7 @@ export type FarosReleaseOutput = {
   project_path: string;
 } & Pick<
   ReleaseSchema,
-  | 'tag_name'
-  | 'name'
-  | 'description'
-  | 'created_at'
-  | 'released_at'
-  | '_links'
+  'tag_name' | 'name' | 'description' | 'created_at' | 'released_at' | '_links'
 >;
 
 export type FarosDeploymentOutput = {
@@ -171,3 +168,11 @@ export type FarosDeploymentOutput = {
   | 'deployable'
   | 'environment'
 >;
+
+export type FarosEpicOutput = {
+  readonly __brand: 'FarosEpic';
+  group_id: string;
+} & Pick<
+  EpicSchema,
+  'id' | 'title' | 'description' | 'state' | 'created_at' | 'updated_at'
+>;

sources/gitlab-source/resources/schemas/farosEpics.json

@@ -0,0 +1,107 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "additionalProperties": true,
+  "properties": {
+    "__brand": {
+      "type": "string"
+    },
+    "id": {
+      "type": "integer"
+    },
+    "iid": {
+      "type": "integer"
+    },
+    "group_id": {
+      "type": "string"
+    },
+    "parent_id": {
+      "type": ["integer", "null"]
+    },
+    "parent_iid": {
+      "type": ["integer", "null"]
+    },
+    "title": {
+      "type": "string"
+    },
+    "description": {
+      "type": ["string", "null"]
+    },
+    "state": {
+      "type": "string",
+      "enum": ["opened", "closed"]
+    },
+    "confidential": {
+      "type": "boolean"
+    },
+    "start_date": {
+      "type": ["string", "null"],
+      "format": "date"
+    },
+    "due_date": {
+      "type": ["string", "null"],
+      "format": "date"
+    },
+    "start_date_is_fixed": {
+      "type": ["boolean", "null"]
+    },
+    "start_date_fixed": {
+      "type": ["string", "null"],
+      "format": "date"
+    },
+    "start_date_from_inherited_source": {
+      "type": ["string", "null"],
+      "format": "date"
+    },
+    "due_date_is_fixed": {
+      "type": ["boolean", "null"]
+    },
+    "due_date_fixed": {
+      "type": ["string", "null"],
+      "format": "date"
+    },
+    "due_date_from_inherited_source": {
+      "type": ["string", "null"],
+      "format": "date"
+    },
+    "created_at": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "updated_at": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "closed_at": {
+      "type": ["string", "null"],
+      "format": "date-time"
+    },
+    "labels": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "author_username": {
+      "type": "string"
+    },
+    "web_url": {
+      "type": "string",
+      "format": "uri"
+    }
+  },
+  "required": [
+    "__brand",
+    "id",
+    "iid",
+    "group_id",
+    "title",
+    "state",
+    "confidential",
+    "created_at",
+    "updated_at",
+    "labels",
+    "author_username",
+    "web_url"
+  ]
+}