diff --git a/destinations/airbyte-faros-destination/src/converters/gitlab/faros_epics.ts b/destinations/airbyte-faros-destination/src/converters/gitlab/faros_epics.ts new file mode 100644 index 000000000..e9150b2c3 --- /dev/null +++ b/destinations/airbyte-faros-destination/src/converters/gitlab/faros_epics.ts @@ -0,0 +1,53 @@ +import {AirbyteRecord} from 'faros-airbyte-cdk'; +import {FarosEpicOutput} from 'faros-airbyte-common/gitlab'; +import {Utils} from 'faros-js-client'; + +import {DestinationModel, DestinationRecord} from '../converter'; +import {GitlabCommon, GitlabConverter} from './common'; + +export class FarosEpics extends GitlabConverter { + readonly destinationModels: ReadonlyArray = ['tms_Epic']; + + id(record: AirbyteRecord): string { + const epic = record?.record?.data as FarosEpicOutput; + return String(epic?.id); + } + + async convert( + record: AirbyteRecord + ): Promise> { + const epic = record.record.data as FarosEpicOutput; + const res: DestinationRecord[] = []; + + if (!epic?.id) { + return []; + } + + const uid = String(epic.id); + + // Convert epic state to status category + const statusCategory = epic.state === 'opened' ? 'Todo' : 'Done'; + + // Create the epic record + res.push({ + model: 'tms_Epic', + record: { + uid, + name: epic.title, + description: Utils.cleanAndTruncate( + epic.description, + GitlabCommon.MAX_DESCRIPTION_LENGTH + ), + status: { + category: statusCategory, + detail: epic.state, + }, + source: this.streamName.source, + createdAt: Utils.toDate(epic.created_at), + updatedAt: Utils.toDate(epic.updated_at), + }, + }); + + return res; + } +} diff --git a/destinations/airbyte-faros-destination/src/converters/gitlab/faros_issues.ts b/destinations/airbyte-faros-destination/src/converters/gitlab/faros_issues.ts index 26bd7fdc2..eba171537 100644 --- a/destinations/airbyte-faros-destination/src/converters/gitlab/faros_issues.ts +++ b/destinations/airbyte-faros-destination/src/converters/gitlab/faros_issues.ts @@ -34,7 +34,6 @@ export class FarosIssues extends GitlabConverter { const uid = String(issue.id); const taskKey = {uid, source: this.streamName.source}; - // Create project key from project_path and group_id const projectKey = { uid: `${toLower(issue.group_id)}/${toLower(issue.project_path)}`, @@ -86,6 +85,9 @@ export class FarosIssues extends GitlabConverter { creator: issue.author_username ? {uid: issue.author_username, source: this.streamName.source} : null, + epic: issue.epic_id + ? {uid: String(issue.epic_id), source: this.streamName.source} + : undefined, createdAt: Utils.toDate(issue.created_at), updatedAt: Utils.toDate(issue.updated_at), source: this.streamName.source, diff --git a/destinations/airbyte-faros-destination/test/converters/__snapshots__/faros_gitlab.test.ts.snap b/destinations/airbyte-faros-destination/test/converters/__snapshots__/faros_gitlab.test.ts.snap index 72e785285..bc6a11269 100644 --- a/destinations/airbyte-faros-destination/test/converters/__snapshots__/faros_gitlab.test.ts.snap +++ b/destinations/airbyte-faros-destination/test/converters/__snapshots__/faros_gitlab.test.ts.snap @@ -2,10 +2,10 @@ exports[`faros_gitlab process records from all streams 1`] = ` Array [ - "Processed 13 records", - "Processed records by stream: {\\\\\\"mytestsource__gitlab__faros_commits\\\\\\":2,\\\\\\"mytestsource__gitlab__faros_deployments\\\\\\":3,\\\\\\"mytestsource__gitlab__faros_groups\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_issues\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_merge_request_reviews\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_merge_requests\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_projects\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_releases\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_tags\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_users\\\\\\":1}\\"},\\"type\\":\\"LOG\\"}", - "Would write 48 records", - "Would write records by model: {\\\\\\"cicd_Artifact\\\\\\":2,\\\\\\"cicd_ArtifactCommitAssociation\\\\\\":2,\\\\\\"cicd_ArtifactDeployment\\\\\\":2,\\\\\\"cicd_Build\\\\\\":2,\\\\\\"cicd_BuildCommitAssociation\\\\\\":2,\\\\\\"cicd_Deployment\\\\\\":3,\\\\\\"cicd_Organization\\\\\\":1,\\\\\\"cicd_Pipeline\\\\\\":1,\\\\\\"cicd_Release\\\\\\":1,\\\\\\"cicd_ReleaseTagAssociation\\\\\\":1,\\\\\\"compute_Application\\\\\\":4,\\\\\\"faros_VcsRepositoryOptions\\\\\\":1,\\\\\\"tms_Label\\\\\\":1,\\\\\\"tms_Project\\\\\\":1,\\\\\\"tms_Task\\\\\\":1,\\\\\\"tms_TaskAssignment\\\\\\":1,\\\\\\"tms_TaskBoard\\\\\\":1,\\\\\\"tms_TaskBoardProjectRelationship\\\\\\":1,\\\\\\"tms_TaskBoardRelationship\\\\\\":1,\\\\\\"tms_TaskProjectRelationship\\\\\\":1,\\\\\\"tms_TaskTag\\\\\\":1,\\\\\\"vcs_Branch\\\\\\":2,\\\\\\"vcs_Commit\\\\\\":2,\\\\\\"vcs_Label\\\\\\":1,\\\\\\"vcs_Membership\\\\\\":1,\\\\\\"vcs_Organization\\\\\\":1,\\\\\\"vcs_PullRequest\\\\\\":1,\\\\\\"vcs_PullRequestComment\\\\\\":2,\\\\\\"vcs_PullRequestLabel\\\\\\":1,\\\\\\"vcs_PullRequestReview\\\\\\":3,\\\\\\"vcs_Repository\\\\\\":1,\\\\\\"vcs_Tag\\\\\\":1,\\\\\\"vcs_User\\\\\\":1}\\"},\\"type\\":\\"LOG\\"}", + "Processed 15 records", + "Processed records by stream: {\\\\\\"mytestsource__gitlab__faros_commits\\\\\\":2,\\\\\\"mytestsource__gitlab__faros_deployments\\\\\\":3,\\\\\\"mytestsource__gitlab__faros_epics\\\\\\":2,\\\\\\"mytestsource__gitlab__faros_groups\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_issues\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_merge_request_reviews\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_merge_requests\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_projects\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_releases\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_tags\\\\\\":1,\\\\\\"mytestsource__gitlab__faros_users\\\\\\":1}\\"},\\"type\\":\\"LOG\\"}", + "Would write 50 records", + "Would write records by model: {\\\\\\"cicd_Artifact\\\\\\":2,\\\\\\"cicd_ArtifactCommitAssociation\\\\\\":2,\\\\\\"cicd_ArtifactDeployment\\\\\\":2,\\\\\\"cicd_Build\\\\\\":2,\\\\\\"cicd_BuildCommitAssociation\\\\\\":2,\\\\\\"cicd_Deployment\\\\\\":3,\\\\\\"cicd_Organization\\\\\\":1,\\\\\\"cicd_Pipeline\\\\\\":1,\\\\\\"cicd_Release\\\\\\":1,\\\\\\"cicd_ReleaseTagAssociation\\\\\\":1,\\\\\\"compute_Application\\\\\\":4,\\\\\\"faros_VcsRepositoryOptions\\\\\\":1,\\\\\\"tms_Epic\\\\\\":2,\\\\\\"tms_Label\\\\\\":1,\\\\\\"tms_Project\\\\\\":1,\\\\\\"tms_Task\\\\\\":1,\\\\\\"tms_TaskAssignment\\\\\\":1,\\\\\\"tms_TaskBoard\\\\\\":1,\\\\\\"tms_TaskBoardProjectRelationship\\\\\\":1,\\\\\\"tms_TaskBoardRelationship\\\\\\":1,\\\\\\"tms_TaskProjectRelationship\\\\\\":1,\\\\\\"tms_TaskTag\\\\\\":1,\\\\\\"vcs_Branch\\\\\\":2,\\\\\\"vcs_Commit\\\\\\":2,\\\\\\"vcs_Label\\\\\\":1,\\\\\\"vcs_Membership\\\\\\":1,\\\\\\"vcs_Organization\\\\\\":1,\\\\\\"vcs_PullRequest\\\\\\":1,\\\\\\"vcs_PullRequestComment\\\\\\":2,\\\\\\"vcs_PullRequestLabel\\\\\\":1,\\\\\\"vcs_PullRequestReview\\\\\\":3,\\\\\\"vcs_Repository\\\\\\":1,\\\\\\"vcs_Tag\\\\\\":1,\\\\\\"vcs_User\\\\\\":1}\\"},\\"type\\":\\"LOG\\"}", "Skipped 0 records", "Errored 0 records", ] @@ -452,6 +452,10 @@ Array [ "uid": "ypc-faros", }, "description": "There's something clearly broken", + "epic": Object { + "source": "GitLab", + "uid": "1001", + }, "name": "Test issue", "source": "GitLab", "status": Object { @@ -849,6 +853,34 @@ Array [ "url": null, }, }, + Object { + "tms_Epic": Object { + "createdAt": "2024-01-01T10:00:00.000Z", + "description": "Epic for implementing comprehensive user authentication system", + "name": "User Authentication Epic", + "source": "GitLab", + "status": Object { + "category": "Todo", + "detail": "opened", + }, + "uid": "1001", + "updatedAt": "2024-02-15T14:30:00.000Z", + }, + }, + Object { + "tms_Epic": Object { + "createdAt": "2024-01-15T09:00:00.000Z", + "description": "Sub-epic for OAuth provider integrations", + "name": "OAuth Integration", + "source": "GitLab", + "status": Object { + "category": "Done", + "detail": "closed", + }, + "uid": "1002", + "updatedAt": "2024-02-28T16:00:00.000Z", + }, + }, Object { "vcs_Branch": Object { "name": "feature/add-new-feature", diff --git a/destinations/airbyte-faros-destination/test/resources/faros_gitlab/all-streams.json b/destinations/airbyte-faros-destination/test/resources/faros_gitlab/all-streams.json index 26b5a872b..3173e3080 100644 --- a/destinations/airbyte-faros-destination/test/resources/faros_gitlab/all-streams.json +++ b/destinations/airbyte-faros-destination/test/resources/faros_gitlab/all-streams.json @@ -210,6 +210,7 @@ "assignee_usernames": [ "test-assignee" ], + "epic_id": 1001, "group_id": "12372707", "project_path": "testy" } @@ -503,5 +504,39 @@ } }, "type": "RECORD" + }, + { + "record": { + "stream": "mytestsource__gitlab__faros_epics", + "emitted_at": 1751581500000, + "data": { + "__brand": "FarosEpic", + "id": 1001, + "group_id": "1001", + "title": "User Authentication Epic", + "description": "Epic for implementing comprehensive user authentication system", + "state": "opened", + "created_at": "2024-01-01T10:00:00.000Z", + "updated_at": "2024-02-15T14:30:00.000Z" + } + }, + "type": "RECORD" + }, + { + "record": { + "stream": "mytestsource__gitlab__faros_epics", + "emitted_at": 1751581500000, + "data": { + "__brand": "FarosEpic", + "id": 1002, + "group_id": "1001", + "title": "OAuth Integration", + "description": "Sub-epic for OAuth provider integrations", + "state": "closed", + "created_at": "2024-01-15T09:00:00.000Z", + "updated_at": "2024-02-28T16:00:00.000Z" + } + }, + "type": "RECORD" } ] diff --git a/destinations/airbyte-faros-destination/test/resources/faros_gitlab/catalog.json b/destinations/airbyte-faros-destination/test/resources/faros_gitlab/catalog.json index bd7f39c33..23b3c82e6 100644 --- a/destinations/airbyte-faros-destination/test/resources/faros_gitlab/catalog.json +++ b/destinations/airbyte-faros-destination/test/resources/faros_gitlab/catalog.json @@ -59,6 +59,12 @@ "name": "mytestsource__gitlab__faros_deployments" }, "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "mytestsource__gitlab__faros_epics" + }, + "destination_sync_mode": "overwrite" } ] } diff --git a/faros-airbyte-common/src/gitlab/types.ts b/faros-airbyte-common/src/gitlab/types.ts index 47ba87c6a..53c9ce554 100644 --- a/faros-airbyte-common/src/gitlab/types.ts +++ b/faros-airbyte-common/src/gitlab/types.ts @@ -2,6 +2,7 @@ import { Camelize, CommitSchema, DeploymentSchema, + EpicSchema, EventSchema, GroupSchema, IssueSchema, @@ -128,6 +129,7 @@ export type FarosIssueOutput = { project_path: string; author_username: string; assignee_usernames: string[]; + epic_id?: number | null; } & Pick< IssueSchema, | 'id' @@ -146,12 +148,7 @@ export type FarosReleaseOutput = { project_path: string; } & Pick< ReleaseSchema, - | 'tag_name' - | 'name' - | 'description' - | 'created_at' - | 'released_at' - | '_links' + 'tag_name' | 'name' | 'description' | 'created_at' | 'released_at' | '_links' >; export type FarosDeploymentOutput = { @@ -171,3 +168,11 @@ export type FarosDeploymentOutput = { | 'deployable' | 'environment' >; + +export type FarosEpicOutput = { + readonly __brand: 'FarosEpic'; + group_id: string; +} & Pick< + EpicSchema, + 'id' | 'title' | 'description' | 'state' | 'created_at' | 'updated_at' +>; diff --git a/sources/gitlab-source/resources/schemas/farosEpics.json b/sources/gitlab-source/resources/schemas/farosEpics.json new file mode 100644 index 000000000..fb51a166f --- /dev/null +++ b/sources/gitlab-source/resources/schemas/farosEpics.json @@ -0,0 +1,107 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "__brand": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "iid": { + "type": "integer" + }, + "group_id": { + "type": "string" + }, + "parent_id": { + "type": ["integer", "null"] + }, + "parent_iid": { + "type": ["integer", "null"] + }, + "title": { + "type": "string" + }, + "description": { + "type": ["string", "null"] + }, + "state": { + "type": "string", + "enum": ["opened", "closed"] + }, + "confidential": { + "type": "boolean" + }, + "start_date": { + "type": ["string", "null"], + "format": "date" + }, + "due_date": { + "type": ["string", "null"], + "format": "date" + }, + "start_date_is_fixed": { + "type": ["boolean", "null"] + }, + "start_date_fixed": { + "type": ["string", "null"], + "format": "date" + }, + "start_date_from_inherited_source": { + "type": ["string", "null"], + "format": "date" + }, + "due_date_is_fixed": { + "type": ["boolean", "null"] + }, + "due_date_fixed": { + "type": ["string", "null"], + "format": "date" + }, + "due_date_from_inherited_source": { + "type": ["string", "null"], + "format": "date" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "closed_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "author_username": { + "type": "string" + }, + "web_url": { + "type": "string", + "format": "uri" + } + }, + "required": [ + "__brand", + "id", + "iid", + "group_id", + "title", + "state", + "confidential", + "created_at", + "updated_at", + "labels", + "author_username", + "web_url" + ] +} diff --git a/sources/gitlab-source/src/gitlab.ts b/sources/gitlab-source/src/gitlab.ts index 61a5f4dc0..79d79979e 100644 --- a/sources/gitlab-source/src/gitlab.ts +++ b/sources/gitlab-source/src/gitlab.ts @@ -2,7 +2,9 @@ import { Camelize, CommitSchema, DeploymentSchema, + EpicSchemaWithExpandedLabels, EventSchema, + GitbeakerRequestError, Gitlab as GitlabClient, GroupSchema, IssueSchema, @@ -18,6 +20,7 @@ import {validateBucketingConfig} from 'faros-airbyte-common/common'; import { FarosCommitOutput, FarosDeploymentOutput, + FarosEpicOutput, FarosGroupOutput, FarosIssueOutput, FarosMergeRequestOutput, @@ -103,9 +106,9 @@ export class GitLab { this.logger.debug('Verifying GitLab credentials by fetching metadata'); const metadata = await this.client.Metadata.show(); if (metadata?.version) { - this.logger.debug( + this.logger.info( 'GitLab credentials verified.', - `Connected to GitLab version ${metadata.version}` + `Connected to GitLab version ${metadata.version} (enterprise: ${metadata.enterprise})` ); } else { this.logger.error( @@ -614,6 +617,7 @@ export class GitLab { author_username: typedIssue.author.username, assignee_usernames: typedIssue.assignees?.map((assignee: any) => assignee.username) ?? [], + epic_id: typedIssue.epic?.id ?? null, }; } } @@ -728,4 +732,58 @@ export class GitLab { }; } } + + async *getEpics( + groupId: string, + since?: Date, + until?: Date + ): AsyncGenerator> { + const options: any = { + perPage: this.pageSize, + orderBy: 'updated_at', + sort: 'desc', + }; + + if (since) { + options.updatedAfter = since.toISOString(); + } + + if (until) { + options.updatedBefore = until.toISOString(); + } + + try { + for await (const epic of this.offsetPagination((paginationOptions) => + this.client.Epics.all(groupId, {...options, ...paginationOptions}) + )) { + const typedEpic = epic as EpicSchemaWithExpandedLabels; + this.userCollector.collectUser(typedEpic.author); + + yield { + __brand: 'FarosEpic', + ...pick(typedEpic, [ + 'id', + 'title', + 'description', + 'state', + 'created_at', + 'updated_at', + ]), + }; + } + } catch (err: any) { + if (err instanceof GitbeakerRequestError) { + if (err.cause?.description?.includes('403')) { + this.logger.info( + `Epics feature is not available for group ${groupId}. This is expected for groups without Premium/Ultimate license.` + ); + return; + } + } + this.logger.error( + `Failed to fetch epics for group ${groupId}: ${err.message}` + ); + throw new VError(err, `Error fetching epics for group ${groupId}`); + } + } } diff --git a/sources/gitlab-source/src/index.ts b/sources/gitlab-source/src/index.ts index 3480c6922..4c0815db1 100644 --- a/sources/gitlab-source/src/index.ts +++ b/sources/gitlab-source/src/index.ts @@ -24,6 +24,7 @@ import { import {RunMode, RunModeStreams} from './streams/common'; import {FarosCommits} from './streams/faros_commits'; import {FarosDeployments} from './streams/faros_deployments'; +import {FarosEpics} from './streams/faros_epics'; import {FarosGroups} from './streams/faros_groups'; import {FarosIssues} from './streams/faros_issues'; import {FarosMergeRequestReviews} from './streams/faros_merge_request_reviews'; @@ -75,6 +76,7 @@ export class GitLabSource extends AirbyteSourceBase { return [ new FarosCommits(config, this.logger, farosClient), new FarosDeployments(config, this.logger, farosClient), + new FarosEpics(config, this.logger, farosClient), new FarosGroups(config, this.logger, farosClient), new FarosIssues(config, this.logger, farosClient), new FarosMergeRequests(config, this.logger, farosClient), diff --git a/sources/gitlab-source/src/streams/common.ts b/sources/gitlab-source/src/streams/common.ts index da3124d3b..f83450438 100644 --- a/sources/gitlab-source/src/streams/common.ts +++ b/sources/gitlab-source/src/streams/common.ts @@ -33,6 +33,7 @@ export enum RunMode { export const MinimumStreamNames = [ 'faros_commits', + 'faros_epics', 'faros_groups', 'faros_issues', 'faros_merge_requests', @@ -44,6 +45,7 @@ export const MinimumStreamNames = [ export const FullStreamNames = [ 'faros_commits', 'faros_deployments', + 'faros_epics', 'faros_groups', 'faros_issues', 'faros_merge_requests', @@ -58,6 +60,7 @@ export const FullStreamNames = [ export const CustomStreamNames = [ 'faros_commits', 'faros_deployments', + 'faros_epics', 'faros_groups', 'faros_issues', 'faros_merge_requests', diff --git a/sources/gitlab-source/src/streams/faros_epics.ts b/sources/gitlab-source/src/streams/faros_epics.ts new file mode 100644 index 000000000..73eb1b707 --- /dev/null +++ b/sources/gitlab-source/src/streams/faros_epics.ts @@ -0,0 +1,60 @@ +import {StreamKey, SyncMode} from 'faros-airbyte-cdk'; +import {FarosEpicOutput} from 'faros-airbyte-common/gitlab'; +import {Utils} from 'faros-js-client'; +import {Dictionary} from 'ts-essentials'; + +import {GitLab} from '../gitlab'; +import { + GroupStreamSlice, + StreamBase, + StreamState, + StreamWithGroupSlices, +} from './common'; + +export class FarosEpics extends StreamWithGroupSlices { + getJsonSchema(): Dictionary { + return require('../../resources/schemas/farosEpics.json'); + } + + get primaryKey(): StreamKey { + return 'id'; + } + + get cursorField(): string { + return 'updated_at'; + } + + async *readRecords( + syncMode: SyncMode, + cursorField?: string[], + streamSlice?: GroupStreamSlice, + streamState?: StreamState + ): AsyncGenerator { + const gitlab = await GitLab.instance(this.config, this.logger); + const groupKey = StreamBase.groupKey(streamSlice.group_id); + const [since, until] = this.getUpdateRange(streamState?.[groupKey]?.cutoff); + + for await (const epic of gitlab.getEpics( + streamSlice.group_id, + since, + until + )) { + yield { + ...epic, + group_id: streamSlice.group_id, + }; + } + } + + getUpdatedState( + currentStreamState: StreamState, + latestRecord: FarosEpicOutput + ): StreamState { + const groupKey = StreamBase.groupKey(latestRecord.group_id); + return this.getUpdatedStreamState( + Utils.toDate(latestRecord.updated_at), + currentStreamState, + groupKey + ); + } +} diff --git a/sources/gitlab-source/test/__snapshots__/index.test.ts.snap b/sources/gitlab-source/test/__snapshots__/index.test.ts.snap index ebdeb693d..2d281d0e9 100644 --- a/sources/gitlab-source/test/__snapshots__/index.test.ts.snap +++ b/sources/gitlab-source/test/__snapshots__/index.test.ts.snap @@ -350,6 +350,35 @@ Implements requested functionality", ] `; +exports[`index streams - faros epics 1`] = ` +[ + { + "author": { + "username": "hkmshb", + }, + "created_at": "2024-01-01T10:00:00.000Z", + "description": "Epic for implementing comprehensive user authentication system", + "group_id": "test-group", + "id": 1001, + "state": "opened", + "title": "User Authentication Epic", + "updated_at": "2024-02-15T14:30:00.000Z", + }, + { + "author": { + "username": "testuser", + }, + "created_at": "2024-01-15T09:00:00.000Z", + "description": "Sub-epic for OAuth provider integrations", + "group_id": "test-group", + "id": 1002, + "state": "closed", + "title": "OAuth Integration", + "updated_at": "2024-02-28T16:00:00.000Z", + }, +] +`; + exports[`index streams - faros groups 1`] = ` [ { @@ -379,6 +408,7 @@ exports[`index streams - faros issues 1`] = ` }, "created_at": "2021-02-27T15:50:47.923Z", "description": "The repository needs to be set up.", + "epic_id": 1001, "group_id": "1", "id": 79927767, "labels": [ diff --git a/sources/gitlab-source/test/index.test.ts b/sources/gitlab-source/test/index.test.ts index 5d82ddf4c..7ed930d9d 100644 --- a/sources/gitlab-source/test/index.test.ts +++ b/sources/gitlab-source/test/index.test.ts @@ -126,6 +126,7 @@ function setupBasicMocks(): any { getIssues: jest.fn().mockReturnValue(createAsyncGeneratorMock([])), getReleases: jest.fn().mockReturnValue(createAsyncGeneratorMock([])), getDeployments: jest.fn().mockReturnValue(createAsyncGeneratorMock([])), + getEpics: jest.fn().mockReturnValue(createAsyncGeneratorMock([])), userCollector: mockUserCollector, }; @@ -516,9 +517,13 @@ describe('index', () => { }); test('streams - faros deployments', async () => { - const deployments = readTestResourceAsJSON('faros_deployments/deployments.json'); + const deployments = readTestResourceAsJSON( + 'faros_deployments/deployments.json' + ); const {gitlab} = setupBasicMocks(); - gitlab.getDeployments.mockReturnValue(createAsyncGeneratorMock(deployments)); + gitlab.getDeployments.mockReturnValue( + createAsyncGeneratorMock(deployments) + ); await sourceReadTest({ source, @@ -536,6 +541,27 @@ describe('index', () => { ); }); + test('streams - faros epics', async () => { + const epics = readTestResourceAsJSON('faros_epics/epics.json'); + const {gitlab} = setupBasicMocks(); + gitlab.getEpics.mockReturnValue(createAsyncGeneratorMock(epics)); + + await sourceReadTest({ + source, + configOrPath: 'config.json', + catalogOrPath: 'faros_epics/catalog.json', + checkRecordsData: (records) => { + expect(records).toMatchSnapshot(); + }, + }); + + expect(gitlab.getEpics).toHaveBeenCalledWith( + 'test-group', + expect.any(Date), + expect.any(Date) + ); + }); + test('round robin bucket execution', async () => { jest.spyOn(GitLab, 'instance').mockResolvedValue({ checkConnection: jest.fn().mockResolvedValue(undefined), diff --git a/sources/gitlab-source/test/resources/faros_epics/catalog.json b/sources/gitlab-source/test/resources/faros_epics/catalog.json new file mode 100644 index 000000000..4b8f3f04e --- /dev/null +++ b/sources/gitlab-source/test/resources/faros_epics/catalog.json @@ -0,0 +1,9 @@ +{ + "streams": [ + { + "stream": { + "name": "faros_epics" + } + } + ] +} \ No newline at end of file diff --git a/sources/gitlab-source/test/resources/faros_epics/epics.json b/sources/gitlab-source/test/resources/faros_epics/epics.json new file mode 100644 index 000000000..329c6fc78 --- /dev/null +++ b/sources/gitlab-source/test/resources/faros_epics/epics.json @@ -0,0 +1,24 @@ +[ + { + "id": 1001, + "title": "User Authentication Epic", + "description": "Epic for implementing comprehensive user authentication system", + "state": "opened", + "created_at": "2024-01-01T10:00:00.000Z", + "updated_at": "2024-02-15T14:30:00.000Z", + "author": { + "username": "hkmshb" + } + }, + { + "id": 1002, + "title": "OAuth Integration", + "description": "Sub-epic for OAuth provider integrations", + "state": "closed", + "created_at": "2024-01-15T09:00:00.000Z", + "updated_at": "2024-02-28T16:00:00.000Z", + "author": { + "username": "testuser" + } + } +] diff --git a/sources/gitlab-source/test/resources/faros_issues/issues.json b/sources/gitlab-source/test/resources/faros_issues/issues.json index 930f0aebf..7df834289 100644 --- a/sources/gitlab-source/test/resources/faros_issues/issues.json +++ b/sources/gitlab-source/test/resources/faros_issues/issues.json @@ -15,6 +15,7 @@ "author": { "username": "hkmshb" }, + "epic_id": 1001, "group_id": "13083", "project_path": "faros-test" }, diff --git a/sources/gitlab-source/test_files/catalog.json b/sources/gitlab-source/test_files/catalog.json new file mode 100644 index 000000000..7d5cea64e --- /dev/null +++ b/sources/gitlab-source/test_files/catalog.json @@ -0,0 +1,52 @@ +{ + "streams": [ + { + "stream": { + "name": "faros_epics", + "jsonSchema": { + "properties": {} + }, + "supportedSyncModes": [ + "full_refresh", + "incremental" + ], + "sourceDefinedCursor": true, + "defaultCursorField": [ + "updated_at" + ], + "sourceDefinedPrimaryKey": [ + ["id"] + ] + }, + "config": { + "syncMode": "incremental", + "destinationSyncMode": "append", + "selected": true + } + }, + { + "stream": { + "name": "faros_issues", + "jsonSchema": { + "properties": {} + }, + "supportedSyncModes": [ + "full_refresh", + "incremental" + ], + "sourceDefinedCursor": true, + "defaultCursorField": [ + "updated_at" + ], + "sourceDefinedPrimaryKey": [ + ["id"] + ] + }, + "config": { + "syncMode": "incremental", + "destinationSyncMode": "append", + "selected": true + } + } + ] +} diff --git a/sources/gitlab-source/test_files/config.json b/sources/gitlab-source/test_files/config.json new file mode 100644 index 000000000..da0f9d244 --- /dev/null +++ b/sources/gitlab-source/test_files/config.json @@ -0,0 +1,6 @@ +{ + "authentication": { + "type": "token", + "personal_access_token": "glpat-behSonUx2kxP3qYuioH4w286MQp1OmhneXlpCw.01.120xdshrt" + } +}