From 41d1c1968cf52274b85ca6e7bbaae18efa665473 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 17 Mar 2021 14:34:04 +0000 Subject: [PATCH 01/31] data-quality: Add rule for scheduledTimezone in partialSchedule --- .../timezone-in-partial-schedule-spec.js | 74 +++++++++++++++++++ .../timezone-in-partial-schedule.js | 55 ++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 src/rules/data-quality/timezone-in-partial-schedule-spec.js create mode 100644 src/rules/data-quality/timezone-in-partial-schedule.js diff --git a/src/rules/data-quality/timezone-in-partial-schedule-spec.js b/src/rules/data-quality/timezone-in-partial-schedule-spec.js new file mode 100644 index 00000000..2f56b5c5 --- /dev/null +++ b/src/rules/data-quality/timezone-in-partial-schedule-spec.js @@ -0,0 +1,74 @@ +const TimezoneInPartialSchedule = require('./timezone-in-partial-schedule'); +const Model = require('../../classes/model'); +const ModelNode = require('../../classes/model-node'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +describe('TimezoneInPartialSchedule', () => { + const rule = new TimezoneInPartialSchedule(); + const model = new Model({ + type: 'PartialSchedule', + fields: { + startTime: { + fieldName: 'startTime', + requiredType: 'https://schema.org/Time', + }, + endTime: { + fieldName: 'endTime', + requiredType: 'https://schema.org/Time', + }, + scheduleTimezone: { + fieldName: 'scheduleTimezone', + requiredType: 'https://schema.org/Text', + }, + }, + }, 'latest'); + + it('should target PartialSchedule models', () => { + const isTargeted = rule.isModelTargeted(model); + expect(isTargeted).toBe(true); + }); + + it('should not return errors when startTime, endTime, and scheduleTimezone are present', async () => { + const data = { + '@type': 'PartialSchedule', + startTime: '08:30', + endTime: '09:30', + scheduleTimezone: 'Europe/London', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(0); + }); + + it('should return errors when startTime and endTime are present, but scheduleTimezone is not', async () => { + const data = { + '@type': 'PartialSchedule', + startTime: '08:30', + endTime: '09:30', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(1); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.MISSING_REQUIRED_FIELD); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); +}); diff --git a/src/rules/data-quality/timezone-in-partial-schedule.js b/src/rules/data-quality/timezone-in-partial-schedule.js new file mode 100644 index 00000000..6a146185 --- /dev/null +++ b/src/rules/data-quality/timezone-in-partial-schedule.js @@ -0,0 +1,55 @@ +const Rule = require('../rule'); +// const PropertyHelper = require('../../helpers/property'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorCategory = require('../../errors/validation-error-category'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +module.exports = class TimezoneInPartialSchedule extends Rule { + constructor(options) { + super(options); + this.targetModels = ['PartialSchedule']; + this.meta = { + name: 'TimezoneInPartialSchedule', + description: 'Validates that scheduleTimezone is present when startTime or endTime are present.', + tests: { + default: { + message: 'scheduleTimezone must be present when startTime or endTime are present', + sampleValues: { + field: 'scheduleTimezone', + allowedValues: 'Europe/London', + }, + category: ValidationErrorCategory.CONFORMANCE, + severity: ValidationErrorSeverity.FAILURE, + type: ValidationErrorType.MISSING_REQUIRED_FIELD, + }, + }, + }; + } + + validateModel(node) { + const startTime = node.getValue('startTime'); + const endTime = node.getValue('endTime'); + const scheduleTimezone = node.getValue('scheduleTimezone'); + + if (typeof startTime === 'undefined' + && typeof endTime === 'undefined' + ) { + return []; + } + const errors = []; + + if (typeof scheduleTimezone === 'undefined') { + errors.push( + this.createError( + 'default', + { + scheduleTimezone, + path: node.getPath('scheduleTimezone'), + }, + ), + ); + } + + return errors; + } +}; From b24a1168e15f3877f9515af5a7d83f7c18a8538a Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 17 Mar 2021 15:01:57 +0000 Subject: [PATCH 02/31] data-quality: Remove dangling comment --- src/rules/data-quality/timezone-in-partial-schedule.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/rules/data-quality/timezone-in-partial-schedule.js b/src/rules/data-quality/timezone-in-partial-schedule.js index 6a146185..24eea3e8 100644 --- a/src/rules/data-quality/timezone-in-partial-schedule.js +++ b/src/rules/data-quality/timezone-in-partial-schedule.js @@ -1,5 +1,4 @@ const Rule = require('../rule'); -// const PropertyHelper = require('../../helpers/property'); const ValidationErrorType = require('../../errors/validation-error-type'); const ValidationErrorCategory = require('../../errors/validation-error-category'); const ValidationErrorSeverity = require('../../errors/validation-error-severity'); From cedb66c8111be22ee83d36429eaf3b40ded65361 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 17 Mar 2021 15:02:28 +0000 Subject: [PATCH 03/31] data-quality: Add rule for repeatCount is positive integer --- .../repeatcount-is-positive-integer-spec.js | 85 +++++++++++++++++++ .../repeatcount-is-positive-integer.js | 52 ++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 src/rules/data-quality/repeatcount-is-positive-integer-spec.js create mode 100644 src/rules/data-quality/repeatcount-is-positive-integer.js diff --git a/src/rules/data-quality/repeatcount-is-positive-integer-spec.js b/src/rules/data-quality/repeatcount-is-positive-integer-spec.js new file mode 100644 index 00000000..6382826f --- /dev/null +++ b/src/rules/data-quality/repeatcount-is-positive-integer-spec.js @@ -0,0 +1,85 @@ +const RepeatCountIsPositiveInteger = require('./repeatcount-is-positive-integer'); +const Model = require('../../classes/model'); +const ModelNode = require('../../classes/model-node'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +describe('RepeatCountIsPositiveInteger', () => { + const rule = new RepeatCountIsPositiveInteger(); + const model = new Model({ + type: 'Schedule', + fields: { + repeatCount: { + fieldName: 'repeatCount', + requiredType: 'https://schema.org/Integer', + }, + }, + }, 'latest'); + + it('should target Schedule model', () => { + const isTargeted = rule.isModelTargeted(model); + expect(isTargeted).toBe(true); + }); + + it('should not return an error when repeatCount is a positive integer', async () => { + const data = { + '@type': 'Schedule', + repeatCount: 10, + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(0); + }); + + it('should return an error when repeatCount is a negative number', async () => { + const data = { + '@type': 'Schedule', + repeatCount: -10, + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(1); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.UNSUPPORTED_VALUE); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); + + it('should return an error when repeatCount is not an integer', async () => { + const data = { + '@type': 'Schedule', + repeatCount: 10.10, + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(1); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.UNSUPPORTED_VALUE); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); +}); diff --git a/src/rules/data-quality/repeatcount-is-positive-integer.js b/src/rules/data-quality/repeatcount-is-positive-integer.js new file mode 100644 index 00000000..b32f575a --- /dev/null +++ b/src/rules/data-quality/repeatcount-is-positive-integer.js @@ -0,0 +1,52 @@ +const Rule = require('../rule'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorCategory = require('../../errors/validation-error-category'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +module.exports = class RepeatCountIsPositiveInteger extends Rule { + constructor(options) { + super(options); + this.targetModels = ['Schedule', 'PartialSchedule']; + this.meta = { + name: 'RepeatCountIsPositiveInteger', + description: 'Validates that repeatCount is a positive integer', + tests: { + default: { + message: 'repeatCount must be a positive integer', + sampleValues: { + field: 'repeatCount', + allowedValues: 10, + }, + category: ValidationErrorCategory.CONFORMANCE, + severity: ValidationErrorSeverity.FAILURE, + type: ValidationErrorType.UNSUPPORTED_VALUE, + }, + }, + }; + } + + validateModel(node) { + const repeatCount = node.getValue('repeatCount'); + + if (typeof repeatCount === 'undefined' + ) { + return []; + } + const errors = []; + + if (repeatCount < 1 + || repeatCount % 1 !== 0) { + errors.push( + this.createError( + 'default', + { + repeatCount, + path: node.getPath('repeatCount'), + }, + ), + ); + } + + return errors; + } +}; From cbe7f3a9c57dec1a5b7eed838bdb37e90fbd5e72 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 17 Mar 2021 16:14:15 +0000 Subject: [PATCH 04/31] package: Install moment-timezone --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 167cc2a3..e67758b8 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "html-entities": "^1.3.1", "jsonpath": "^1.0.2", "moment": "^2.24.0", + "moment-timezone": "^0.5.33", "rrule": "^2.6.2", "striptags": "^3.1.1", "uritemplate": "^0.3.4", From 7f0725df574daba9222ad38ac976ac05dd8e470c Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 17 Mar 2021 16:14:45 +0000 Subject: [PATCH 05/31] data-quality: Add rule to ensure timezone matches IANA list scheduleTimezone rule: update message with link to IANA --- ...etimezone-in-partialschedule-rule-spec.js} | 2 +- ...heduletimezone-in-partialschedule-rule.js} | 0 .../repeatcount-format-rule-spec.js} | 2 +- .../repeatcount-format-rule.js} | 0 .../scheduletimezone-format-rule-spec.js | 63 +++++++++++++++++++ .../format/scheduletimezone-format-rule.js | 52 +++++++++++++++ 6 files changed, 117 insertions(+), 2 deletions(-) rename src/rules/data-quality/{timezone-in-partial-schedule-spec.js => scheduletimezone-in-partialschedule-rule-spec.js} (95%) rename src/rules/data-quality/{timezone-in-partial-schedule.js => scheduletimezone-in-partialschedule-rule.js} (100%) rename src/rules/{data-quality/repeatcount-is-positive-integer-spec.js => format/repeatcount-format-rule-spec.js} (96%) rename src/rules/{data-quality/repeatcount-is-positive-integer.js => format/repeatcount-format-rule.js} (100%) create mode 100644 src/rules/format/scheduletimezone-format-rule-spec.js create mode 100644 src/rules/format/scheduletimezone-format-rule.js diff --git a/src/rules/data-quality/timezone-in-partial-schedule-spec.js b/src/rules/data-quality/scheduletimezone-in-partialschedule-rule-spec.js similarity index 95% rename from src/rules/data-quality/timezone-in-partial-schedule-spec.js rename to src/rules/data-quality/scheduletimezone-in-partialschedule-rule-spec.js index 2f56b5c5..a1cec352 100644 --- a/src/rules/data-quality/timezone-in-partial-schedule-spec.js +++ b/src/rules/data-quality/scheduletimezone-in-partialschedule-rule-spec.js @@ -1,4 +1,4 @@ -const TimezoneInPartialSchedule = require('./timezone-in-partial-schedule'); +const TimezoneInPartialSchedule = require('./scheduletimezone-in-partial-schedule-rule'); const Model = require('../../classes/model'); const ModelNode = require('../../classes/model-node'); const ValidationErrorType = require('../../errors/validation-error-type'); diff --git a/src/rules/data-quality/timezone-in-partial-schedule.js b/src/rules/data-quality/scheduletimezone-in-partialschedule-rule.js similarity index 100% rename from src/rules/data-quality/timezone-in-partial-schedule.js rename to src/rules/data-quality/scheduletimezone-in-partialschedule-rule.js diff --git a/src/rules/data-quality/repeatcount-is-positive-integer-spec.js b/src/rules/format/repeatcount-format-rule-spec.js similarity index 96% rename from src/rules/data-quality/repeatcount-is-positive-integer-spec.js rename to src/rules/format/repeatcount-format-rule-spec.js index 6382826f..1d5afb1b 100644 --- a/src/rules/data-quality/repeatcount-is-positive-integer-spec.js +++ b/src/rules/format/repeatcount-format-rule-spec.js @@ -1,4 +1,4 @@ -const RepeatCountIsPositiveInteger = require('./repeatcount-is-positive-integer'); +const RepeatCountIsPositiveInteger = require('./repeatcount-format-rule'); const Model = require('../../classes/model'); const ModelNode = require('../../classes/model-node'); const ValidationErrorType = require('../../errors/validation-error-type'); diff --git a/src/rules/data-quality/repeatcount-is-positive-integer.js b/src/rules/format/repeatcount-format-rule.js similarity index 100% rename from src/rules/data-quality/repeatcount-is-positive-integer.js rename to src/rules/format/repeatcount-format-rule.js diff --git a/src/rules/format/scheduletimezone-format-rule-spec.js b/src/rules/format/scheduletimezone-format-rule-spec.js new file mode 100644 index 00000000..e378f996 --- /dev/null +++ b/src/rules/format/scheduletimezone-format-rule-spec.js @@ -0,0 +1,63 @@ +const ScheduleTimezoneMatchesIANAList = require('./scheduletimezone-format-rule'); +const Model = require('../../classes/model'); +const ModelNode = require('../../classes/model-node'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +describe('ScheduleTimezoneMatchesIANAList', () => { + const rule = new ScheduleTimezoneMatchesIANAList(); + const model = new Model({ + type: 'PartialSchedule', + fields: { + scheduleTimezone: { + fieldName: 'scheduleTimezone', + requiredType: 'https://schema.org/Text', + }, + }, + }, 'latest'); + + it('should target PartialSchedule models', () => { + const isTargeted = rule.isModelTargeted(model); + expect(isTargeted).toBe(true); + }); + + it('should not return errors when scheduleTimezone is in the IANA database list', async () => { + const data = { + '@type': 'PartialSchedule', + scheduleTimezone: 'Europe/London', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(0); + }); + + it('should return errors when scheduleTimezone is not in the IANA database list', async () => { + const data = { + '@type': 'PartialSchedule', + scheduleTimezone: 'Europe/Slough', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(1); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.MISSING_REQUIRED_FIELD); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); +}); diff --git a/src/rules/format/scheduletimezone-format-rule.js b/src/rules/format/scheduletimezone-format-rule.js new file mode 100644 index 00000000..2eb412bc --- /dev/null +++ b/src/rules/format/scheduletimezone-format-rule.js @@ -0,0 +1,52 @@ +const momentTZ = require('moment-timezone'); +const Rule = require('../rule'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorCategory = require('../../errors/validation-error-category'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +module.exports = class ScheduleTimezoneMatchesIANAList extends Rule { + constructor(options) { + super(options); + this.targetModels = ['Schedule', 'PartialSchedule']; + this.meta = { + name: 'ScheduleTimezoneMatchesIANAList', + description: 'Validates that scheduleTimezone matches an IANA Timezone.', + tests: { + default: { + message: 'scheduleTimezone must be one of the timezones contained in the [IANA Timezone database](https://www.iana.org/time-zones)', + sampleValues: { + field: 'scheduleTimezone', + allowedValues: 'Europe/London', + }, + category: ValidationErrorCategory.CONFORMANCE, + severity: ValidationErrorSeverity.FAILURE, + type: ValidationErrorType.MISSING_REQUIRED_FIELD, + }, + }, + }; + } + + validateModel(node) { + const timezoneList = momentTZ.tz.names(); + const scheduleTimezone = node.getValue('scheduleTimezone'); + + if (typeof scheduleTimezone === 'undefined') { + return []; + } + const errors = []; + + if (!timezoneList.includes(scheduleTimezone)) { + errors.push( + this.createError( + 'default', + { + scheduleTimezone, + path: node.getPath('scheduleTimezone'), + }, + ), + ); + } + + return errors; + } +}; From a9dc16affa2c545ea4ee6c1a3651a7ca320685f2 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 17 Mar 2021 16:57:30 +0000 Subject: [PATCH 06/31] tests: rename import --- .../scheduletimezone-in-partialschedule-rule-spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rules/data-quality/scheduletimezone-in-partialschedule-rule-spec.js b/src/rules/data-quality/scheduletimezone-in-partialschedule-rule-spec.js index a1cec352..9b09a96c 100644 --- a/src/rules/data-quality/scheduletimezone-in-partialschedule-rule-spec.js +++ b/src/rules/data-quality/scheduletimezone-in-partialschedule-rule-spec.js @@ -1,4 +1,4 @@ -const TimezoneInPartialSchedule = require('./scheduletimezone-in-partial-schedule-rule'); +const TimezoneInPartialSchedule = require('./scheduletimezone-in-partialschedule-rule'); const Model = require('../../classes/model'); const ModelNode = require('../../classes/model-node'); const ValidationErrorType = require('../../errors/validation-error-type'); From 26d0ffa08a88f31eb0473a8a0a9106284a9f3d1c Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Fri, 19 Mar 2021 14:18:27 +0000 Subject: [PATCH 07/31] rules: Refactor repeatCount rule to use minValueInclusive --- src/classes/field.js | 4 ++++ .../format/repeatcount-format-rule-spec.js | 4 +++- src/rules/format/repeatcount-format-rule.js | 19 ++++++++++--------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/classes/field.js b/src/classes/field.js index 75092b2e..532ea4b8 100644 --- a/src/classes/field.js +++ b/src/classes/field.js @@ -63,6 +63,10 @@ const Field = class { return this.data.allowReferencing; } + get minValueInclusive() { + return this.data.minValueInclusive; + } + get standard() { return this.data.standard; } diff --git a/src/rules/format/repeatcount-format-rule-spec.js b/src/rules/format/repeatcount-format-rule-spec.js index 1d5afb1b..718c862b 100644 --- a/src/rules/format/repeatcount-format-rule-spec.js +++ b/src/rules/format/repeatcount-format-rule-spec.js @@ -6,18 +6,20 @@ const ValidationErrorSeverity = require('../../errors/validation-error-severity' describe('RepeatCountIsPositiveInteger', () => { const rule = new RepeatCountIsPositiveInteger(); + const model = new Model({ type: 'Schedule', fields: { repeatCount: { fieldName: 'repeatCount', + minValueInclusive: 1, requiredType: 'https://schema.org/Integer', }, }, }, 'latest'); it('should target Schedule model', () => { - const isTargeted = rule.isModelTargeted(model); + const isTargeted = rule.isFieldTargeted(model, 'repeatCount'); expect(isTargeted).toBe(true); }); diff --git a/src/rules/format/repeatcount-format-rule.js b/src/rules/format/repeatcount-format-rule.js index b32f575a..4574e8fc 100644 --- a/src/rules/format/repeatcount-format-rule.js +++ b/src/rules/format/repeatcount-format-rule.js @@ -6,7 +6,7 @@ const ValidationErrorSeverity = require('../../errors/validation-error-severity' module.exports = class RepeatCountIsPositiveInteger extends Rule { constructor(options) { super(options); - this.targetModels = ['Schedule', 'PartialSchedule']; + this.targetFields = { Schedule: 'repeatCount', PartialSchedule: 'repeatCount' }; this.meta = { name: 'RepeatCountIsPositiveInteger', description: 'Validates that repeatCount is a positive integer', @@ -25,23 +25,24 @@ module.exports = class RepeatCountIsPositiveInteger extends Rule { }; } - validateModel(node) { - const repeatCount = node.getValue('repeatCount'); + validateField(node, field) { + const fieldObj = node.model.getField(field); + const fieldValue = node.getValue(field); - if (typeof repeatCount === 'undefined' - ) { + if (typeof fieldValue !== 'number') { return []; } + const errors = []; - if (repeatCount < 1 - || repeatCount % 1 !== 0) { + if (typeof fieldObj.minValueInclusive !== 'undefined' + && (fieldValue < fieldObj.minValueInclusive || fieldValue % 1 !== 0)) { errors.push( this.createError( 'default', { - repeatCount, - path: node.getPath('repeatCount'), + fieldValue, + path: node.getPath(field), }, ), ); From b1f4a3ed883f40eb7dd5427e87d3fb34820acecf Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 24 Mar 2021 10:56:23 +0000 Subject: [PATCH 08/31] data-quality: Ensure Schedule templates (id & url) are valid UriTemplates --- src/classes/field.js | 4 + .../schedule-templates-are-valid-rule-spec.js | 127 ++++++++++++++++++ .../schedule-templates-are-valid-rule.js | 52 +++++++ 3 files changed, 183 insertions(+) create mode 100644 src/rules/data-quality/schedule-templates-are-valid-rule-spec.js create mode 100644 src/rules/data-quality/schedule-templates-are-valid-rule.js diff --git a/src/classes/field.js b/src/classes/field.js index 532ea4b8..dd7a0a82 100644 --- a/src/classes/field.js +++ b/src/classes/field.js @@ -67,6 +67,10 @@ const Field = class { return this.data.minValueInclusive; } + get valueConstraint() { + return this.data.valueConstraint; + } + get standard() { return this.data.standard; } diff --git a/src/rules/data-quality/schedule-templates-are-valid-rule-spec.js b/src/rules/data-quality/schedule-templates-are-valid-rule-spec.js new file mode 100644 index 00000000..afe15056 --- /dev/null +++ b/src/rules/data-quality/schedule-templates-are-valid-rule-spec.js @@ -0,0 +1,127 @@ +const ScheduleTemplatesValid = require('./schedule-templates-are-valid-rule'); +const Model = require('../../classes/model'); +const ModelNode = require('../../classes/model-node'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +describe('ScheduleTemplatesValid', () => { + let model; + let rule; + + beforeEach(() => { + model = new Model({ + type: 'Schedule', + fields: { + idTemplate: { + fieldName: 'idTemplate', + sameAs: 'https://openactive.io/idTemplate', + requiredType: 'https://schema.org/Text', + example: 'https://api.example.org/session-series/123/{startDate}', + description: [ + 'An RFC6570 compliant URI template that can be used to generate a unique identifier (`@id`) for every event described by the schedule. This property is required if the data provider is supporting third-party booking via the Open Booking API, or providing complimentary individual `subEvent`s.', + ], + valueConstraint: 'UriTemplate', + }, + urlTemplate: { + fieldName: 'urlTemplate', + sameAs: 'https://schema.org/urlTemplate', + requiredType: 'https://schema.org/Text', + example: 'https://example.org/session-series/123/{startDate}', + description: [ + 'An RFC6570 compliant URI template that can be used to generate a unique `url` for every event described by the schedule. This property is required if the data provider wants to provide participants with a unique URL to book to attend an event.', + ], + valueConstraint: 'UriTemplate', + }, + }, + }, 'latest'); + rule = new ScheduleTemplatesValid(); + }); + + it('should target idTemplate and urlTemplate in Schedule model', () => { + let isTargeted = rule.isFieldTargeted(model, 'idTemplate'); + expect(isTargeted).toBe(true); + + isTargeted = rule.isFieldTargeted(model, 'urlTemplate'); + expect(isTargeted).toBe(true); + }); + + it('should return no errors if the urlTemplate is valid', async () => { + const data = { + '@type': 'Schedule', + urlTemplate: 'https://api.example.org/session-series/123/{startDate}', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(0); + }); + + it('should return no errors if the idTemplate is valid', async () => { + const data = { + '@type': 'Schedule', + idTemplate: 'https://api.example.org/session-series/123/{startDate}', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(0); + }); + + it('should return errors if the urlTemplate is not valid', async () => { + const data = { + '@type': 'Schedule', + urlTemplate: 'htts://api.example.org/session-series/123/', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(1); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.INVALID_FORMAT); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); + + it('should return errors if the idTemplate is not valid', async () => { + const data = { + '@type': 'Schedule', + idTemplate: 'htts://api.example.org/session-series/123/', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(1); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.INVALID_FORMAT); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); +}); diff --git a/src/rules/data-quality/schedule-templates-are-valid-rule.js b/src/rules/data-quality/schedule-templates-are-valid-rule.js new file mode 100644 index 00000000..17f7e786 --- /dev/null +++ b/src/rules/data-quality/schedule-templates-are-valid-rule.js @@ -0,0 +1,52 @@ +const Rule = require('../rule'); +const PropertyHelper = require('../../helpers/property'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorCategory = require('../../errors/validation-error-category'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +module.exports = class ScheduleTemplatesValid extends Rule { + constructor(options) { + super(options); + this.targetFields = { Schedule: ['urlTemplate', 'idTemplate'] }; + this.meta = { + name: 'ScheduleTemplatesValid', + description: 'Validates that Schedule urlTemplate and idTemplate fields are the correct format', + tests: { + default: { + description: 'Validates that Schedule idTemplate or urlTemplate fields are in the correct format.', + message: 'The field must contain a valid resource identifier. For example `{startDate}` in `https://api.example.org/session-series/123/{startDate}`', + category: ValidationErrorCategory.CONFORMANCE, + severity: ValidationErrorSeverity.FAILURE, + type: ValidationErrorType.INVALID_FORMAT, + }, + }, + }; + } + + validateField(node, field) { + const fieldObj = node.model.getField(field); + const fieldValue = node.getValue(field); + + if (typeof fieldValue !== 'string') { + return []; + } + + const errors = []; + + if (typeof fieldObj.valueConstraint !== 'undefined' + && (fieldObj.valueConstraint === 'UriTemplate' + && !PropertyHelper.isUrlTemplate(fieldValue))) { + errors.push( + this.createError( + 'default', + { + fieldValue, + path: node.getPath(field), + }, + ), + ); + } + + return errors; + } +}; From be6d53dbd76457a507ba0e4e3869133785c96d90 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 24 Mar 2021 17:11:18 +0000 Subject: [PATCH 09/31] data-quality: schedule-templates: Add ensure {startDate} rule --- src/errors/validation-error-type.js | 1 + ...e-templates-contain-startdate-rule-spec.js | 127 ++++++++++++++++++ ...hedule-templates-contain-startdate-rule.js | 48 +++++++ 3 files changed, 176 insertions(+) create mode 100644 src/rules/data-quality/schedule-templates-contain-startdate-rule-spec.js create mode 100644 src/rules/data-quality/schedule-templates-contain-startdate-rule.js diff --git a/src/errors/validation-error-type.js b/src/errors/validation-error-type.js index 44b20b40..6d769cdb 100644 --- a/src/errors/validation-error-type.js +++ b/src/errors/validation-error-type.js @@ -41,6 +41,7 @@ const ValidationErrorType = { WRONG_BASE_TYPE: 'wrong_base_type', FIELD_NOT_ALLOWED: 'field_not_allowed', BELOW_MIN_VALUE_INCLUSIVE: 'BELOW_MIN_VALUE_INCLUSIVE', + URI_TEMPLATE_MISSING_PLACEHOLDER: 'uri_template_missing_placeholder', }; module.exports = Object.freeze(ValidationErrorType); diff --git a/src/rules/data-quality/schedule-templates-contain-startdate-rule-spec.js b/src/rules/data-quality/schedule-templates-contain-startdate-rule-spec.js new file mode 100644 index 00000000..d7b54b16 --- /dev/null +++ b/src/rules/data-quality/schedule-templates-contain-startdate-rule-spec.js @@ -0,0 +1,127 @@ +const ScheduleTemplatesContainStartDate = require('./schedule-templates-contain-startdate-rule'); +const Model = require('../../classes/model'); +const ModelNode = require('../../classes/model-node'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +describe('ScheduleTemplatesContainStartDate', () => { + let model; + let rule; + + beforeEach(() => { + model = new Model({ + type: 'Schedule', + fields: { + idTemplate: { + fieldName: 'idTemplate', + sameAs: 'https://openactive.io/idTemplate', + requiredType: 'https://schema.org/Text', + example: 'https://api.example.org/session-series/123/{startDate}', + description: [ + 'An RFC6570 compliant URI template that can be used to generate a unique identifier (`@id`) for every event described by the schedule. This property is required if the data provider is supporting third-party booking via the Open Booking API, or providing complimentary individual `subEvent`s.', + ], + valueConstraint: 'UriTemplate', + }, + urlTemplate: { + fieldName: 'urlTemplate', + sameAs: 'https://schema.org/urlTemplate', + requiredType: 'https://schema.org/Text', + example: 'https://example.org/session-series/123/{startDate}', + description: [ + 'An RFC6570 compliant URI template that can be used to generate a unique `url` for every event described by the schedule. This property is required if the data provider wants to provide participants with a unique URL to book to attend an event.', + ], + valueConstraint: 'UriTemplate', + }, + }, + }, 'latest'); + rule = new ScheduleTemplatesContainStartDate(); + }); + + it('should target idTemplate and urlTemplate in Schedule model', () => { + let isTargeted = rule.isFieldTargeted(model, 'idTemplate'); + expect(isTargeted).toBe(true); + + isTargeted = rule.isFieldTargeted(model, 'urlTemplate'); + expect(isTargeted).toBe(true); + }); + + it('should return no errors if the urlTemplate contains {startDate}', async () => { + const data = { + '@type': 'Schedule', + urlTemplate: 'https://api.example.org/session-series/123/{startDate}', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(0); + }); + + it('should return no errors if the idTemplate contains {startDate}', async () => { + const data = { + '@type': 'Schedule', + idTemplate: 'https://api.example.org/session-series/123/{startDate}', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(0); + }); + + it('should return errors if the urlTemplate does not contain {startDate}', async () => { + const data = { + '@type': 'Schedule', + urlTemplate: 'htts://api.example.org/session-series/123/', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(1); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.URI_TEMPLATE_MISSING_PLACEHOLDER); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); + + it('should return errors if the idTemplate does not contain {startDate}', async () => { + const data = { + '@type': 'Schedule', + idTemplate: 'htts://api.example.org/session-series/123/', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(1); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.URI_TEMPLATE_MISSING_PLACEHOLDER); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); +}); diff --git a/src/rules/data-quality/schedule-templates-contain-startdate-rule.js b/src/rules/data-quality/schedule-templates-contain-startdate-rule.js new file mode 100644 index 00000000..3439e1f5 --- /dev/null +++ b/src/rules/data-quality/schedule-templates-contain-startdate-rule.js @@ -0,0 +1,48 @@ +const Rule = require('../rule'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorCategory = require('../../errors/validation-error-category'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +module.exports = class ScheduleTemplatesContainStartDate extends Rule { + constructor(options) { + super(options); + this.targetFields = { Schedule: ['urlTemplate', 'idTemplate'] }; + this.meta = { + name: 'ScheduleTemplatesContainStartDate', + description: 'Validates that the urlTemplate or idTemplate fields in a Schedule model contain the {startDate} placeholder.', + tests: { + default: { + description: 'Validates that the urlTemplate or idTemplate fields in a Schedule model contain the {startDate} placeholder.', + message: 'The urlTemplate or idTemplate field in a Schedule model must contain the {startDate} placeholder, e.g. `https://api.example.org/session-series/123/{startDate}`', + category: ValidationErrorCategory.CONFORMANCE, + severity: ValidationErrorSeverity.FAILURE, + type: ValidationErrorType.URI_TEMPLATE_MISSING_PLACEHOLDER, + }, + }, + }; + } + + validateField(node, field) { + const fieldValue = node.getValue(field); + + if (typeof fieldValue !== 'string') { + return []; + } + + const errors = []; + + if (!fieldValue.includes('{startDate}')) { + errors.push( + this.createError( + 'default', + { + fieldValue, + path: node.getPath(field), + }, + ), + ); + } + + return errors; + } +}; From b0ff0fb5e0130f9ef1319a865d4821dd5e21c4e1 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Fri, 19 Mar 2021 15:30:34 +0000 Subject: [PATCH 10/31] data-quality: Add rule to check for recurrence data in schedule --- package.json | 3 +- src/helpers/datetime-helper.js | 11 ++ src/helpers/frequency-converter.js | 27 ++++ ...dule-contains-recurrence-data-rule-spec.js | 131 ++++++++++++++++++ .../schedule-contains-recurrence-data-rule.js | 122 ++++++++++++++++ 5 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 src/helpers/datetime-helper.js create mode 100644 src/helpers/frequency-converter.js create mode 100644 src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js create mode 100644 src/rules/data-quality/schedule-contains-recurrence-data-rule.js diff --git a/package.json b/package.json index e67758b8..057dd347 100644 --- a/package.json +++ b/package.json @@ -27,10 +27,11 @@ "axios": "^0.19.2", "currency-codes": "^1.5.1", "html-entities": "^1.3.1", + "iso8601-duration": "^1.3.0", "jsonpath": "^1.0.2", "moment": "^2.24.0", "moment-timezone": "^0.5.33", - "rrule": "^2.6.2", + "rrule": "^2.6.4", "striptags": "^3.1.1", "uritemplate": "^0.3.4", "validator": "^10.11.0", diff --git a/src/helpers/datetime-helper.js b/src/helpers/datetime-helper.js new file mode 100644 index 00000000..e02375b8 --- /dev/null +++ b/src/helpers/datetime-helper.js @@ -0,0 +1,11 @@ +function getDateTime(dateString, timeString) { + if (typeof timeString !== 'undefined') { + return new Date(`${dateString}T${timeString}`); + } + if (typeof dateString !== 'undefined') { + return new Date(dateString); + } + return undefined; +} + +module.exports = getDateTime; diff --git a/src/helpers/frequency-converter.js b/src/helpers/frequency-converter.js new file mode 100644 index 00000000..fb463e66 --- /dev/null +++ b/src/helpers/frequency-converter.js @@ -0,0 +1,27 @@ +const { parse } = require('iso8601-duration'); +const { RRule } = require('rrule'); + +function getFrequency(repeatFrequency) { + if (typeof repeatFrequency !== 'undefined') { + const frequency = parse(repeatFrequency); + + if (frequency.hours !== 0) { + return { freq: RRule.HOURLY, interval: frequency.hours }; + } + if (frequency.days !== 0) { + return { freq: RRule.DAILY, interval: frequency.days }; + } + if (frequency.weeks !== 0) { + return { freq: RRule.WEEKLY, interval: frequency.weeks }; + } + if (frequency.months !== 0) { + return { freq: RRule.MONTHLY, interval: frequency.months }; + } + if (frequency.years !== 0) { + return { freq: RRule.YEARLY, interval: frequency.years }; + } + } + return { freq: undefined, interval: 0 }; +} + +module.exports = getFrequency; diff --git a/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js b/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js new file mode 100644 index 00000000..c4f78391 --- /dev/null +++ b/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js @@ -0,0 +1,131 @@ +const ValidRecurrenceRule = require('./schedule-contains-recurrence-data-rule'); +const Model = require('../../classes/model'); +const ModelNode = require('../../classes/model-node'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +describe('ValidRecurrenceRule', () => { + const rule = new ValidRecurrenceRule(); + const model = new Model({ + type: 'Schedule', + fields: { + repeatFrequency: { + fieldname: 'byDay', + requiredType: 'https://schema.org/Duration', + }, + byDay: { + fieldname: 'byDay', + requiredType: 'ArrayOf#https://schema.org/DayOfWeek', + alternativeTypes: ['ArrayOf#https://schema.org/Text'], + }, + byMonth: { + fieldname: 'byMonth', + requiredType: 'https://schema.org/Integer', + }, + byMonthDay: { + fieldname: 'byMonthDay', + requiredType: 'https://schema.org/Integer', + }, + startDate: { + fieldname: 'startDate', + requiredType: 'https://schema.org/Date', + }, + EndDate: { + fieldname: 'EndDate', + requiredType: 'https://schema.org/Date', + }, + startTime: { + fieldname: 'startTime', + requiredType: 'https://schema.org/Time', + }, + EndTime: { + fieldname: 'EndTime', + requiredType: 'https://schema.org/Time', + }, + count: { + fieldname: 'count', + requiredType: 'https://schema.org/Integer', + }, + scheduleTimezone: { + fieldName: 'scheduleTimezone', + requiredType: 'https://schema.org/Text', + }, + }, + }, 'latest'); + + it('should target Schedule models', () => { + const isTargeted = rule.isModelTargeted(model); + expect(isTargeted).toBe(true); + }); + + it('should return errors when startDate is missing', async () => { + const data = { + '@type': 'Schedule', + startTime: '08:30', + endTime: '09:30', + scheduleTimezone: 'Europe/London', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(2); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.MISSING_REQUIRED_FIELD); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); + + it('should return errors when startTime is missing', async () => { + const data = { + '@type': 'Schedule', + startDate: '2021-03-19', + repeatFrequency: 'P1W', + count: 10000, + scheduleTimezone: 'Europe/London', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(2); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.MISSING_REQUIRED_FIELD); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); + + it('should not return errors when there are sufficent properties to build a valid recurrence rule', async () => { + const data = { + '@type': 'Schedule', + startDate: '2021-03-19', + startTime: '08:30', + repeatFrequency: 'P1W', + count: 1, + scheduleTimezone: 'Europe/London', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(0); + }); +}); diff --git a/src/rules/data-quality/schedule-contains-recurrence-data-rule.js b/src/rules/data-quality/schedule-contains-recurrence-data-rule.js new file mode 100644 index 00000000..ea00bb2b --- /dev/null +++ b/src/rules/data-quality/schedule-contains-recurrence-data-rule.js @@ -0,0 +1,122 @@ +const { RRule } = require('rrule'); +const Rule = require('../rule'); +const getFrequency = require('../../helpers/frequency-converter'); +const getDateTime = require('../../helpers/datetime-helper'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorCategory = require('../../errors/validation-error-category'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +module.exports = class ValidRecurrenceRule extends Rule { + constructor(options) { + super(options); + this.targetModels = ['Schedule']; + this.meta = { + name: 'ValidRecurrenceRule', + description: + 'Validates that the Schedule contains the correct information to generate a valid iCal recurrence rule.', + tests: { + default: { + message: + 'Schedule must contains the correct information to generate a valid iCal recurrence rule.', + sampleValues: { + startTime: '08:30', + endTime: '09:30', + startDate: '2021-03-19', + repeatFrequency: 'P1W', + count: 10, + scheduleTimezone: 'Europe/London', + }, + category: ValidationErrorCategory.CONFORMANCE, + severity: ValidationErrorSeverity.FAILURE, + type: ValidationErrorType.MISSING_REQUIRED_FIELD, + }, + dtStart: { + message: + 'The recurrence rule must contain a startDate, startTime, and scheduledTimezone to generate the schedule.', + sampleValues: { + startTime: '08:30', + startDate: '2021-03-19', + scheduleTimezone: 'Europe/London', + }, + category: ValidationErrorCategory.CONFORMANCE, + severity: ValidationErrorSeverity.FAILURE, + type: ValidationErrorType.MISSING_REQUIRED_FIELD, + }, + }, + }; + } + + validateModel(node) { + const errors = []; + + const { freq, interval } = getFrequency(node.getValue('repeatFrequency')); + const byDay = node.getValue('byDay'); + const byMonth = node.getValue('byMonth'); + const byMonthDay = node.getValue('byMonthDay'); + const startDate = node.getValue('startDate'); + const startTime = node.getValue('startTime'); + const endDate = node.getValue('endDate'); + const endTime = node.getValue('endTime'); + const count = node.getValue('count'); + const scheduleTimezone = node.getValue('scheduleTimezone'); + + const dtStart = getDateTime(startDate, startTime); + const dtEnd = getDateTime(endDate, endTime); + + const rruleOptions = { freq, interval }; // this is the only required one + + if (typeof startDate === 'undefined' + || typeof startTime === 'undefined' + || typeof scheduleTimezone === 'undefined') { + errors.push( + this.createError('dtStart', { + value: undefined, + path: node, + }), + ); + } else { + rruleOptions.dtstart = dtStart; + } + + if (typeof byDay !== 'undefined') { + rruleOptions.byweekday = byDay; + } + if (typeof byMonth !== 'undefined') { + rruleOptions.bymonth = byMonth; + } + if (typeof byMonthDay !== 'undefined') { + rruleOptions.bymonthday = byMonthDay; + } + if (typeof dtEnd !== 'undefined') { + rruleOptions.until = dtEnd; + } + if (typeof count !== 'undefined') { + rruleOptions.count = count; + } + if (typeof scheduleTimezone !== 'undefined') { + rruleOptions.tzid = scheduleTimezone; + } + + try { + const rule = new RRule(rruleOptions); // eslint-disable-line no-new + const firstEvent = rule.all()[0]; + if (firstEvent.getTime() !== dtStart.getTime()) { + errors.push( + this.createError('default', { + error: 'The first event in the generated by RRule does not match the Schedule data.', + path: node, + }), + ); + } + } catch (error) { + errors.push( + this.createError('default', { + error, + path: node, + }), + ); + } + + return errors; + } +}; From 8d6707a6eef9987aa90270970279a89fb3e8628e Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Fri, 26 Mar 2021 10:52:03 +0000 Subject: [PATCH 11/31] data-quality: schedule-exception-dates: Add rule that checks if exception dates are outside the recurrence rule --- .../schedule-contains-recurrence-data-rule.js | 2 +- ...exceptdates-match-recurrence-dates-spec.js | 116 ++++++++++++++++++ ...dule-exceptdates-match-recurrence-dates.js | 105 ++++++++++++++++ 3 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 src/rules/data-quality/schedule-exceptdates-match-recurrence-dates-spec.js create mode 100644 src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js diff --git a/src/rules/data-quality/schedule-contains-recurrence-data-rule.js b/src/rules/data-quality/schedule-contains-recurrence-data-rule.js index ea00bb2b..3359de6c 100644 --- a/src/rules/data-quality/schedule-contains-recurrence-data-rule.js +++ b/src/rules/data-quality/schedule-contains-recurrence-data-rule.js @@ -98,7 +98,7 @@ module.exports = class ValidRecurrenceRule extends Rule { } try { - const rule = new RRule(rruleOptions); // eslint-disable-line no-new + const rule = new RRule(rruleOptions); const firstEvent = rule.all()[0]; if (firstEvent.getTime() !== dtStart.getTime()) { errors.push( diff --git a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates-spec.js b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates-spec.js new file mode 100644 index 00000000..d4beccf3 --- /dev/null +++ b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates-spec.js @@ -0,0 +1,116 @@ +const ExceptDatesAreInSchedule = require('./schedule-exceptdates-match-recurrence-dates'); +const Model = require('../../classes/model'); +const ModelNode = require('../../classes/model-node'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +describe('ExceptDatesAreInSchedule', () => { + const rule = new ExceptDatesAreInSchedule(); + const model = new Model({ + type: 'Schedule', + fields: { + repeatFrequency: { + fieldname: 'byDay', + requiredType: 'https://schema.org/Duration', + }, + byDay: { + fieldname: 'byDay', + requiredType: 'ArrayOf#https://schema.org/DayOfWeek', + alternativeTypes: ['ArrayOf#https://schema.org/Text'], + }, + byMonth: { + fieldname: 'byMonth', + requiredType: 'https://schema.org/Integer', + }, + byMonthDay: { + fieldname: 'byMonthDay', + requiredType: 'https://schema.org/Integer', + }, + startDate: { + fieldname: 'startDate', + requiredType: 'https://schema.org/Date', + }, + EndDate: { + fieldname: 'EndDate', + requiredType: 'https://schema.org/Date', + }, + startTime: { + fieldname: 'startTime', + requiredType: 'https://schema.org/Time', + }, + EndTime: { + fieldname: 'EndTime', + requiredType: 'https://schema.org/Time', + }, + count: { + fieldname: 'count', + requiredType: 'https://schema.org/Integer', + }, + scheduleTimezone: { + fieldName: 'scheduleTimezone', + requiredType: 'https://schema.org/Text', + }, + exceptDate: { + fieldName: 'exceptDate', + requiredType: 'ArrayOf#https://schema.org/DateTime', + alternativeTypes: ['ArrayOf#https://schema.org/Date'], + + }, + }, + }, 'latest'); + + it('should target Schedule models', () => { + const isTargeted = rule.isModelTargeted(model); + expect(isTargeted).toBe(true); + }); + + it('should return errors when exceptDate values are outside the recurrence rule series', async () => { + const data = { + '@type': 'Schedule', + startDate: '2021-03-19', + startTime: '08:30', + repeatFrequency: 'P1W', + count: 10, + exceptDate: ['2021-03-27T08:30:00Z'], + scheduleTimezone: 'Europe/London', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(1); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.MISSING_REQUIRED_FIELD); + expect(error.severity).toBe(ValidationErrorSeverity.WARNING); + } + }); + + it('should not return errors when the exceptDate is within the recurrence rule schedule', async () => { + const data = { + '@type': 'Schedule', + startDate: '2021-03-19', + startTime: '08:30', + repeatFrequency: 'P1W', + count: 10, + exceptDate: ['2021-03-26T08:30:00Z'], + scheduleTimezone: 'Europe/London', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(0); + }); +}); diff --git a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js new file mode 100644 index 00000000..3afc966c --- /dev/null +++ b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js @@ -0,0 +1,105 @@ +const { RRule } = require('rrule'); +const Rule = require('../rule'); +const getFrequency = require('../../helpers/frequency-converter'); +const getDateTime = require('../../helpers/datetime-helper'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorCategory = require('../../errors/validation-error-category'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +module.exports = class ExceptDatesAreInSchedule extends Rule { + constructor(options) { + super(options); + this.targetModels = ['Schedule']; + this.meta = { + name: 'ExceptDatesAreInSchedule', + description: + 'Validates that the Schedule contains exceptDates that are part of the recurrence rule.', + tests: { + exDate: { + message: + '{{value}} must be one of the events generated by the recurrence rule.', + sampleValues: { + startTime: '08:30', + endTime: '09:30', + startDate: '2021-03-19', + repeatFrequency: 'P1W', + count: 10, + exceptDates: ['2021-03-26T08:30:00Z'], + scheduleTimezone: 'Europe/London', + }, + category: ValidationErrorCategory.CONFORMANCE, + severity: ValidationErrorSeverity.WARNING, + type: ValidationErrorType.MISSING_REQUIRED_FIELD, + }, + }, + }; + } + + validateModel(node) { + const errors = []; + + const { freq, interval } = getFrequency(node.getValue('repeatFrequency')); + const byDay = node.getValue('byDay'); + const byMonth = node.getValue('byMonth'); + const byMonthDay = node.getValue('byMonthDay'); + const startDate = node.getValue('startDate'); + const startTime = node.getValue('startTime'); + const endDate = node.getValue('endDate'); + const endTime = node.getValue('endTime'); + const count = node.getValue('count'); + const scheduleTimezone = node.getValue('scheduleTimezone'); + const exceptDate = node.getValue('exceptDate'); + + if (typeof exceptDate === 'undefined') { + return []; + } + + const dtStart = getDateTime(startDate, startTime); + const dtEnd = getDateTime(endDate, endTime); + + const rruleOptions = { freq, interval }; // this is the only required one + + if (typeof dtStart !== 'undefined') { + rruleOptions.dtstart = dtStart; + } + if (typeof byDay !== 'undefined') { + rruleOptions.byweekday = byDay; + } + if (typeof byMonth !== 'undefined') { + rruleOptions.bymonth = byMonth; + } + if (typeof byMonthDay !== 'undefined') { + rruleOptions.bymonthday = byMonthDay; + } + if (typeof dtEnd !== 'undefined') { + rruleOptions.until = dtEnd; + } + if (typeof count !== 'undefined') { + rruleOptions.count = count; + } + if (typeof scheduleTimezone !== 'undefined') { + rruleOptions.tzid = scheduleTimezone; + } + + try { + const rule = new RRule(rruleOptions); + const allEvents = rule.all(); + const simplifiedAllEvents = allEvents.map(event => event.getTime()); + for (const date of exceptDate) { + const simplifiedDate = new Date(date).getTime(); + if (!simplifiedAllEvents.includes(simplifiedDate)) { + errors.push( + this.createError('exDate', { + value: date, + path: node, + }), + ); + } + } + } catch (error) { + return []; + } + + return errors; + } +}; From c397c08a679be768a98ae0010a3a7c25eb75e2db Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Fri, 26 Mar 2021 11:42:34 +0000 Subject: [PATCH 12/31] data-helpers: refactor tests to use generateRRuleOptions --- src/helpers/datetime-helper.js | 2 +- src/helpers/rrule-options.js | 52 +++++++++++++++++++ ...dule-contains-recurrence-data-rule-spec.js | 2 +- .../schedule-contains-recurrence-data-rule.js | 48 +++-------------- ...dule-exceptdates-match-recurrence-dates.js | 46 ++-------------- 5 files changed, 64 insertions(+), 86 deletions(-) create mode 100644 src/helpers/rrule-options.js diff --git a/src/helpers/datetime-helper.js b/src/helpers/datetime-helper.js index e02375b8..218c797b 100644 --- a/src/helpers/datetime-helper.js +++ b/src/helpers/datetime-helper.js @@ -1,5 +1,5 @@ function getDateTime(dateString, timeString) { - if (typeof timeString !== 'undefined') { + if (typeof dateString !== 'undefined' && typeof timeString !== 'undefined') { return new Date(`${dateString}T${timeString}`); } if (typeof dateString !== 'undefined') { diff --git a/src/helpers/rrule-options.js b/src/helpers/rrule-options.js new file mode 100644 index 00000000..4f8ba4d2 --- /dev/null +++ b/src/helpers/rrule-options.js @@ -0,0 +1,52 @@ +const getFrequency = require('./frequency-converter'); +const getDateTime = require('./datetime-helper'); + +function generateRRuleOptions(node) { + const { freq, interval } = getFrequency(node.getValue('repeatFrequency')); + const properties = { + freq, + interval, + byDay: node.getValue('byDay'), + byMonth: node.getValue('byMonth'), + byMonthDay: node.getValue('byMonthDay'), + startDate: node.getValue('startDate'), + startTime: node.getValue('startTime'), + endDate: node.getValue('endDate'), + endTime: node.getValue('endTime'), + count: node.getValue('count'), + scheduleTimezone: node.getValue('scheduleTimezone'), + exceptDate: node.getValue('exceptDate'), + }; + + console.info(properties.startDate, properties.startTime); + + const dtStart = getDateTime(properties.startDate, properties.startTime); + const dtEnd = getDateTime(properties.endDate, properties.endTime); + + const rruleOptions = { freq, interval }; // this is the only required one + + if (typeof dtStart !== 'undefined') { + rruleOptions.dtstart = dtStart; + } + if (typeof dtEnd !== 'undefined') { + rruleOptions.until = dtEnd; + } + if (typeof properties.byDay !== 'undefined') { + rruleOptions.byweekday = properties.byDay; + } + if (typeof properties.byMonth !== 'undefined') { + rruleOptions.bymonth = properties.byMonth; + } + if (typeof properties.byMonthDay !== 'undefined') { + rruleOptions.bymonthday = properties.byMonthDay; + } + if (typeof properties.count !== 'undefined') { + rruleOptions.count = properties.count; + } + if (typeof properties.scheduleTimezone !== 'undefined') { + rruleOptions.tzid = properties.scheduleTimezone; + } + return { rruleOptions, properties }; +} + +module.exports = generateRRuleOptions; diff --git a/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js b/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js index c4f78391..72aa7984 100644 --- a/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js +++ b/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js @@ -100,7 +100,7 @@ describe('ValidRecurrenceRule', () => { const errors = await rule.validate(nodeToTest); - expect(errors.length).toBe(2); + expect(errors.length).toBe(1); for (const error of errors) { expect(error.type).toBe(ValidationErrorType.MISSING_REQUIRED_FIELD); expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); diff --git a/src/rules/data-quality/schedule-contains-recurrence-data-rule.js b/src/rules/data-quality/schedule-contains-recurrence-data-rule.js index 3359de6c..a72500f1 100644 --- a/src/rules/data-quality/schedule-contains-recurrence-data-rule.js +++ b/src/rules/data-quality/schedule-contains-recurrence-data-rule.js @@ -1,7 +1,6 @@ const { RRule } = require('rrule'); const Rule = require('../rule'); -const getFrequency = require('../../helpers/frequency-converter'); -const getDateTime = require('../../helpers/datetime-helper'); +const generateRRuleOptions = require('../../helpers/rrule-options'); const ValidationErrorType = require('../../errors/validation-error-type'); const ValidationErrorCategory = require('../../errors/validation-error-category'); const ValidationErrorSeverity = require('../../errors/validation-error-severity'); @@ -49,58 +48,23 @@ module.exports = class ValidRecurrenceRule extends Rule { validateModel(node) { const errors = []; - const { freq, interval } = getFrequency(node.getValue('repeatFrequency')); - const byDay = node.getValue('byDay'); - const byMonth = node.getValue('byMonth'); - const byMonthDay = node.getValue('byMonthDay'); - const startDate = node.getValue('startDate'); - const startTime = node.getValue('startTime'); - const endDate = node.getValue('endDate'); - const endTime = node.getValue('endTime'); - const count = node.getValue('count'); - const scheduleTimezone = node.getValue('scheduleTimezone'); + const { rruleOptions, properties } = generateRRuleOptions(node); - const dtStart = getDateTime(startDate, startTime); - const dtEnd = getDateTime(endDate, endTime); - - const rruleOptions = { freq, interval }; // this is the only required one - - if (typeof startDate === 'undefined' - || typeof startTime === 'undefined' - || typeof scheduleTimezone === 'undefined') { + if (typeof properties.startDate === 'undefined' + || typeof properties.startTime === 'undefined' + || typeof properties.scheduleTimezone === 'undefined') { errors.push( this.createError('dtStart', { value: undefined, path: node, }), ); - } else { - rruleOptions.dtstart = dtStart; - } - - if (typeof byDay !== 'undefined') { - rruleOptions.byweekday = byDay; - } - if (typeof byMonth !== 'undefined') { - rruleOptions.bymonth = byMonth; - } - if (typeof byMonthDay !== 'undefined') { - rruleOptions.bymonthday = byMonthDay; - } - if (typeof dtEnd !== 'undefined') { - rruleOptions.until = dtEnd; - } - if (typeof count !== 'undefined') { - rruleOptions.count = count; - } - if (typeof scheduleTimezone !== 'undefined') { - rruleOptions.tzid = scheduleTimezone; } try { const rule = new RRule(rruleOptions); const firstEvent = rule.all()[0]; - if (firstEvent.getTime() !== dtStart.getTime()) { + if (firstEvent.getTime() !== rruleOptions.dtstart.getTime()) { errors.push( this.createError('default', { error: 'The first event in the generated by RRule does not match the Schedule data.', diff --git a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js index 3afc966c..eedff02f 100644 --- a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js +++ b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js @@ -1,7 +1,6 @@ const { RRule } = require('rrule'); const Rule = require('../rule'); -const getFrequency = require('../../helpers/frequency-converter'); -const getDateTime = require('../../helpers/datetime-helper'); +const generateRRuleOptions = require('../../helpers/rrule-options'); const ValidationErrorType = require('../../errors/validation-error-type'); const ValidationErrorCategory = require('../../errors/validation-error-category'); const ValidationErrorSeverity = require('../../errors/validation-error-severity'); @@ -38,54 +37,17 @@ module.exports = class ExceptDatesAreInSchedule extends Rule { validateModel(node) { const errors = []; - const { freq, interval } = getFrequency(node.getValue('repeatFrequency')); - const byDay = node.getValue('byDay'); - const byMonth = node.getValue('byMonth'); - const byMonthDay = node.getValue('byMonthDay'); - const startDate = node.getValue('startDate'); - const startTime = node.getValue('startTime'); - const endDate = node.getValue('endDate'); - const endTime = node.getValue('endTime'); - const count = node.getValue('count'); - const scheduleTimezone = node.getValue('scheduleTimezone'); - const exceptDate = node.getValue('exceptDate'); + const { rruleOptions, properties } = generateRRuleOptions(node); - if (typeof exceptDate === 'undefined') { + if (typeof properties.exceptDate === 'undefined') { return []; } - const dtStart = getDateTime(startDate, startTime); - const dtEnd = getDateTime(endDate, endTime); - - const rruleOptions = { freq, interval }; // this is the only required one - - if (typeof dtStart !== 'undefined') { - rruleOptions.dtstart = dtStart; - } - if (typeof byDay !== 'undefined') { - rruleOptions.byweekday = byDay; - } - if (typeof byMonth !== 'undefined') { - rruleOptions.bymonth = byMonth; - } - if (typeof byMonthDay !== 'undefined') { - rruleOptions.bymonthday = byMonthDay; - } - if (typeof dtEnd !== 'undefined') { - rruleOptions.until = dtEnd; - } - if (typeof count !== 'undefined') { - rruleOptions.count = count; - } - if (typeof scheduleTimezone !== 'undefined') { - rruleOptions.tzid = scheduleTimezone; - } - try { const rule = new RRule(rruleOptions); const allEvents = rule.all(); const simplifiedAllEvents = allEvents.map(event => event.getTime()); - for (const date of exceptDate) { + for (const date of properties.exceptDate) { const simplifiedDate = new Date(date).getTime(); if (!simplifiedAllEvents.includes(simplifiedDate)) { errors.push( From 876e0f03adf564068b892762caef714b619a0e34 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Fri, 26 Mar 2021 11:43:22 +0000 Subject: [PATCH 13/31] data-helpers: Remove old logging --- src/helpers/rrule-options.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/helpers/rrule-options.js b/src/helpers/rrule-options.js index 4f8ba4d2..ae11427e 100644 --- a/src/helpers/rrule-options.js +++ b/src/helpers/rrule-options.js @@ -18,8 +18,6 @@ function generateRRuleOptions(node) { exceptDate: node.getValue('exceptDate'), }; - console.info(properties.startDate, properties.startTime); - const dtStart = getDateTime(properties.startDate, properties.startTime); const dtEnd = getDateTime(properties.endDate, properties.endTime); From 5bc8b22c0aaee45954695b1d9d7ee9fb687b516b Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Fri, 26 Mar 2021 12:04:44 +0000 Subject: [PATCH 14/31] helpers: Separate concerns between helper files --- src/helpers/rrule-options.js | 29 ++++++------------- src/helpers/schedule-properties.js | 22 ++++++++++++++ .../schedule-contains-recurrence-data-rule.js | 4 ++- ...dule-exceptdates-match-recurrence-dates.js | 4 ++- 4 files changed, 37 insertions(+), 22 deletions(-) create mode 100644 src/helpers/schedule-properties.js diff --git a/src/helpers/rrule-options.js b/src/helpers/rrule-options.js index ae11427e..d41052d7 100644 --- a/src/helpers/rrule-options.js +++ b/src/helpers/rrule-options.js @@ -1,28 +1,17 @@ -const getFrequency = require('./frequency-converter'); const getDateTime = require('./datetime-helper'); -function generateRRuleOptions(node) { - const { freq, interval } = getFrequency(node.getValue('repeatFrequency')); - const properties = { - freq, - interval, - byDay: node.getValue('byDay'), - byMonth: node.getValue('byMonth'), - byMonthDay: node.getValue('byMonthDay'), - startDate: node.getValue('startDate'), - startTime: node.getValue('startTime'), - endDate: node.getValue('endDate'), - endTime: node.getValue('endTime'), - count: node.getValue('count'), - scheduleTimezone: node.getValue('scheduleTimezone'), - exceptDate: node.getValue('exceptDate'), - }; - +function generateRRuleOptions(properties) { const dtStart = getDateTime(properties.startDate, properties.startTime); const dtEnd = getDateTime(properties.endDate, properties.endTime); - const rruleOptions = { freq, interval }; // this is the only required one + const rruleOptions = {}; + if (typeof properties.freq !== 'undefined') { + rruleOptions.freq = properties.freq; + } + if (typeof properties.interval !== 'undefined') { + rruleOptions.interval = properties.interval; + } if (typeof dtStart !== 'undefined') { rruleOptions.dtstart = dtStart; } @@ -44,7 +33,7 @@ function generateRRuleOptions(node) { if (typeof properties.scheduleTimezone !== 'undefined') { rruleOptions.tzid = properties.scheduleTimezone; } - return { rruleOptions, properties }; + return rruleOptions; } module.exports = generateRRuleOptions; diff --git a/src/helpers/schedule-properties.js b/src/helpers/schedule-properties.js new file mode 100644 index 00000000..f732e2e7 --- /dev/null +++ b/src/helpers/schedule-properties.js @@ -0,0 +1,22 @@ +const getFrequency = require('./frequency-converter'); + +function getScheduleProperties(node) { + const { freq, interval } = getFrequency(node.getValue('repeatFrequency')); + const properties = { + freq, + interval, + byDay: node.getValue('byDay'), + byMonth: node.getValue('byMonth'), + byMonthDay: node.getValue('byMonthDay'), + startDate: node.getValue('startDate'), + startTime: node.getValue('startTime'), + endDate: node.getValue('endDate'), + endTime: node.getValue('endTime'), + count: node.getValue('count'), + scheduleTimezone: node.getValue('scheduleTimezone'), + exceptDate: node.getValue('exceptDate'), + }; + return properties; +} + +module.exports = getScheduleProperties; diff --git a/src/rules/data-quality/schedule-contains-recurrence-data-rule.js b/src/rules/data-quality/schedule-contains-recurrence-data-rule.js index a72500f1..8e1a4314 100644 --- a/src/rules/data-quality/schedule-contains-recurrence-data-rule.js +++ b/src/rules/data-quality/schedule-contains-recurrence-data-rule.js @@ -1,6 +1,7 @@ const { RRule } = require('rrule'); const Rule = require('../rule'); const generateRRuleOptions = require('../../helpers/rrule-options'); +const getScheduleProperties = require('../../helpers/schedule-properties'); const ValidationErrorType = require('../../errors/validation-error-type'); const ValidationErrorCategory = require('../../errors/validation-error-category'); const ValidationErrorSeverity = require('../../errors/validation-error-severity'); @@ -48,7 +49,8 @@ module.exports = class ValidRecurrenceRule extends Rule { validateModel(node) { const errors = []; - const { rruleOptions, properties } = generateRRuleOptions(node); + const properties = getScheduleProperties(node); + const rruleOptions = generateRRuleOptions(properties); if (typeof properties.startDate === 'undefined' || typeof properties.startTime === 'undefined' diff --git a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js index eedff02f..0d08742e 100644 --- a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js +++ b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js @@ -4,6 +4,7 @@ const generateRRuleOptions = require('../../helpers/rrule-options'); const ValidationErrorType = require('../../errors/validation-error-type'); const ValidationErrorCategory = require('../../errors/validation-error-category'); const ValidationErrorSeverity = require('../../errors/validation-error-severity'); +const getScheduleProperties = require('../../helpers/schedule-properties'); module.exports = class ExceptDatesAreInSchedule extends Rule { constructor(options) { @@ -37,7 +38,8 @@ module.exports = class ExceptDatesAreInSchedule extends Rule { validateModel(node) { const errors = []; - const { rruleOptions, properties } = generateRRuleOptions(node); + const properties = getScheduleProperties(node); + const rruleOptions = generateRRuleOptions(properties); if (typeof properties.exceptDate === 'undefined') { return []; From 0402176853e53547c82c1c8f014b127f30199455 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 31 Mar 2021 10:45:06 +0100 Subject: [PATCH 15/31] data-quality: repeatCount rule now superseded by minValueInclusive rule (and data type rule) --- .../format/repeatcount-format-rule-spec.js | 87 ------------------- src/rules/format/repeatcount-format-rule.js | 53 ----------- 2 files changed, 140 deletions(-) delete mode 100644 src/rules/format/repeatcount-format-rule-spec.js delete mode 100644 src/rules/format/repeatcount-format-rule.js diff --git a/src/rules/format/repeatcount-format-rule-spec.js b/src/rules/format/repeatcount-format-rule-spec.js deleted file mode 100644 index 718c862b..00000000 --- a/src/rules/format/repeatcount-format-rule-spec.js +++ /dev/null @@ -1,87 +0,0 @@ -const RepeatCountIsPositiveInteger = require('./repeatcount-format-rule'); -const Model = require('../../classes/model'); -const ModelNode = require('../../classes/model-node'); -const ValidationErrorType = require('../../errors/validation-error-type'); -const ValidationErrorSeverity = require('../../errors/validation-error-severity'); - -describe('RepeatCountIsPositiveInteger', () => { - const rule = new RepeatCountIsPositiveInteger(); - - const model = new Model({ - type: 'Schedule', - fields: { - repeatCount: { - fieldName: 'repeatCount', - minValueInclusive: 1, - requiredType: 'https://schema.org/Integer', - }, - }, - }, 'latest'); - - it('should target Schedule model', () => { - const isTargeted = rule.isFieldTargeted(model, 'repeatCount'); - expect(isTargeted).toBe(true); - }); - - it('should not return an error when repeatCount is a positive integer', async () => { - const data = { - '@type': 'Schedule', - repeatCount: 10, - }; - - const nodeToTest = new ModelNode( - '$', - data, - null, - model, - ); - - const errors = await rule.validate(nodeToTest); - - expect(errors.length).toBe(0); - }); - - it('should return an error when repeatCount is a negative number', async () => { - const data = { - '@type': 'Schedule', - repeatCount: -10, - }; - - const nodeToTest = new ModelNode( - '$', - data, - null, - model, - ); - - const errors = await rule.validate(nodeToTest); - - expect(errors.length).toBe(1); - for (const error of errors) { - expect(error.type).toBe(ValidationErrorType.UNSUPPORTED_VALUE); - expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); - } - }); - - it('should return an error when repeatCount is not an integer', async () => { - const data = { - '@type': 'Schedule', - repeatCount: 10.10, - }; - - const nodeToTest = new ModelNode( - '$', - data, - null, - model, - ); - - const errors = await rule.validate(nodeToTest); - - expect(errors.length).toBe(1); - for (const error of errors) { - expect(error.type).toBe(ValidationErrorType.UNSUPPORTED_VALUE); - expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); - } - }); -}); diff --git a/src/rules/format/repeatcount-format-rule.js b/src/rules/format/repeatcount-format-rule.js deleted file mode 100644 index 4574e8fc..00000000 --- a/src/rules/format/repeatcount-format-rule.js +++ /dev/null @@ -1,53 +0,0 @@ -const Rule = require('../rule'); -const ValidationErrorType = require('../../errors/validation-error-type'); -const ValidationErrorCategory = require('../../errors/validation-error-category'); -const ValidationErrorSeverity = require('../../errors/validation-error-severity'); - -module.exports = class RepeatCountIsPositiveInteger extends Rule { - constructor(options) { - super(options); - this.targetFields = { Schedule: 'repeatCount', PartialSchedule: 'repeatCount' }; - this.meta = { - name: 'RepeatCountIsPositiveInteger', - description: 'Validates that repeatCount is a positive integer', - tests: { - default: { - message: 'repeatCount must be a positive integer', - sampleValues: { - field: 'repeatCount', - allowedValues: 10, - }, - category: ValidationErrorCategory.CONFORMANCE, - severity: ValidationErrorSeverity.FAILURE, - type: ValidationErrorType.UNSUPPORTED_VALUE, - }, - }, - }; - } - - validateField(node, field) { - const fieldObj = node.model.getField(field); - const fieldValue = node.getValue(field); - - if (typeof fieldValue !== 'number') { - return []; - } - - const errors = []; - - if (typeof fieldObj.minValueInclusive !== 'undefined' - && (fieldValue < fieldObj.minValueInclusive || fieldValue % 1 !== 0)) { - errors.push( - this.createError( - 'default', - { - fieldValue, - path: node.getPath(field), - }, - ), - ); - } - - return errors; - } -}; From 76ea071617a9b216cbdded5589bd79ffbf2bb994 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 31 Mar 2021 11:02:47 +0100 Subject: [PATCH 16/31] data-quality: schedules: Update messages and rule tests --- .../schedule-contains-recurrence-data-rule.js | 13 ++----------- .../schedule-exceptdates-match-recurrence-dates.js | 8 +------- .../scheduletimezone-in-partialschedule-rule.js | 4 ++-- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/src/rules/data-quality/schedule-contains-recurrence-data-rule.js b/src/rules/data-quality/schedule-contains-recurrence-data-rule.js index 8e1a4314..80ab0c29 100644 --- a/src/rules/data-quality/schedule-contains-recurrence-data-rule.js +++ b/src/rules/data-quality/schedule-contains-recurrence-data-rule.js @@ -17,22 +17,14 @@ module.exports = class ValidRecurrenceRule extends Rule { tests: { default: { message: - 'Schedule must contains the correct information to generate a valid iCal recurrence rule.', - sampleValues: { - startTime: '08:30', - endTime: '09:30', - startDate: '2021-03-19', - repeatFrequency: 'P1W', - count: 10, - scheduleTimezone: 'Europe/London', - }, + 'The first event that is generated by the `Schedule` does not match the `startDate` and `startTime`.', category: ValidationErrorCategory.CONFORMANCE, severity: ValidationErrorSeverity.FAILURE, type: ValidationErrorType.MISSING_REQUIRED_FIELD, }, dtStart: { message: - 'The recurrence rule must contain a startDate, startTime, and scheduledTimezone to generate the schedule.', + 'The recurrence rule must contain a `startDate`, `startTime`, and `scheduledTimezone` to generate the schedule.', sampleValues: { startTime: '08:30', startDate: '2021-03-19', @@ -69,7 +61,6 @@ module.exports = class ValidRecurrenceRule extends Rule { if (firstEvent.getTime() !== rruleOptions.dtstart.getTime()) { errors.push( this.createError('default', { - error: 'The first event in the generated by RRule does not match the Schedule data.', path: node, }), ); diff --git a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js index 0d08742e..a1771416 100644 --- a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js +++ b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js @@ -19,13 +19,7 @@ module.exports = class ExceptDatesAreInSchedule extends Rule { message: '{{value}} must be one of the events generated by the recurrence rule.', sampleValues: { - startTime: '08:30', - endTime: '09:30', - startDate: '2021-03-19', - repeatFrequency: 'P1W', - count: 10, - exceptDates: ['2021-03-26T08:30:00Z'], - scheduleTimezone: 'Europe/London', + value: '2020-03-16T12:30:00Z', }, category: ValidationErrorCategory.CONFORMANCE, severity: ValidationErrorSeverity.WARNING, diff --git a/src/rules/data-quality/scheduletimezone-in-partialschedule-rule.js b/src/rules/data-quality/scheduletimezone-in-partialschedule-rule.js index 24eea3e8..66124b3f 100644 --- a/src/rules/data-quality/scheduletimezone-in-partialschedule-rule.js +++ b/src/rules/data-quality/scheduletimezone-in-partialschedule-rule.js @@ -9,10 +9,10 @@ module.exports = class TimezoneInPartialSchedule extends Rule { this.targetModels = ['PartialSchedule']; this.meta = { name: 'TimezoneInPartialSchedule', - description: 'Validates that scheduleTimezone is present when startTime or endTime are present.', + description: 'Validates that `scheduleTimezone` is present when `startTime` or `endTime` are present.', tests: { default: { - message: 'scheduleTimezone must be present when startTime or endTime are present', + message: '`scheduleTimezone` must be present when `startTime` or `endTime` are present', sampleValues: { field: 'scheduleTimezone', allowedValues: 'Europe/London', From e1312f8ae78a6f49350e76a42066575a8dd4c961 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 31 Mar 2021 11:05:40 +0100 Subject: [PATCH 17/31] rebase: Resolve duplicate --- src/classes/field.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/classes/field.js b/src/classes/field.js index dd7a0a82..571f4899 100644 --- a/src/classes/field.js +++ b/src/classes/field.js @@ -63,10 +63,6 @@ const Field = class { return this.data.allowReferencing; } - get minValueInclusive() { - return this.data.minValueInclusive; - } - get valueConstraint() { return this.data.valueConstraint; } From 7d4b78483fb6a1ba173499640469bbe17cee401b Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 31 Mar 2021 11:40:47 +0100 Subject: [PATCH 18/31] data-quality: recurrence rule: Update tests to include more information --- .../schedule-contains-recurrence-data-rule.js | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/rules/data-quality/schedule-contains-recurrence-data-rule.js b/src/rules/data-quality/schedule-contains-recurrence-data-rule.js index 80ab0c29..de4de825 100644 --- a/src/rules/data-quality/schedule-contains-recurrence-data-rule.js +++ b/src/rules/data-quality/schedule-contains-recurrence-data-rule.js @@ -15,9 +15,21 @@ module.exports = class ValidRecurrenceRule extends Rule { description: 'Validates that the Schedule contains the correct information to generate a valid iCal recurrence rule.', tests: { - default: { + matchingFirstEvent: { message: - 'The first event that is generated by the `Schedule` does not match the `startDate` and `startTime`.', + 'The first event that is generated by the `Schedule` ({{firstEvent}}) does not match the `startDate` ({{startDate}}) and `startTime` ({{startTime}}).', + sampleValues: { + startTime: '08:30', + startDate: '2021-03-19', + firstEvent: '2021-03-20T09:40:00Z', + }, + category: ValidationErrorCategory.CONFORMANCE, + severity: ValidationErrorSeverity.FAILURE, + type: ValidationErrorType.MISSING_REQUIRED_FIELD, + }, + rruleCreation: { + message: + 'There was an error generating the RRule from the data provided. Error: {{error}}', category: ValidationErrorCategory.CONFORMANCE, severity: ValidationErrorSeverity.FAILURE, type: ValidationErrorType.MISSING_REQUIRED_FIELD, @@ -60,14 +72,17 @@ module.exports = class ValidRecurrenceRule extends Rule { const firstEvent = rule.all()[0]; if (firstEvent.getTime() !== rruleOptions.dtstart.getTime()) { errors.push( - this.createError('default', { + this.createError('matchingFirstEvent', { + startDate: properties.startDate, + startTime: properties.startTime, + firstEvent, path: node, }), ); } } catch (error) { errors.push( - this.createError('default', { + this.createError('rruleCreation', { error, path: node, }), From d729577174c2d9aeacb1b2e51b08f74d89a03025 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 31 Mar 2021 11:44:48 +0100 Subject: [PATCH 19/31] data-quality: Update exception date rule test message --- .../schedule-exceptdates-match-recurrence-dates.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js index a1771416..9e3a8b9a 100644 --- a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js +++ b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js @@ -17,9 +17,10 @@ module.exports = class ExceptDatesAreInSchedule extends Rule { tests: { exDate: { message: - '{{value}} must be one of the events generated by the recurrence rule.', + '{{date}} must be one of the events generated by the recurrence rule - {{allEvents}}', sampleValues: { - value: '2020-03-16T12:30:00Z', + value: '2020-03-23T12:30:00Z', + allEvents: ['202-03-16T12:30:00Z', '202-03-23T12:30:00Z', '202-03-30T12:30:00Z'], }, category: ValidationErrorCategory.CONFORMANCE, severity: ValidationErrorSeverity.WARNING, @@ -48,7 +49,8 @@ module.exports = class ExceptDatesAreInSchedule extends Rule { if (!simplifiedAllEvents.includes(simplifiedDate)) { errors.push( this.createError('exDate', { - value: date, + date, + allEvents, path: node, }), ); From d9ec6f0dba883096b779bb9affddcd34cc3a0426 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 31 Mar 2021 11:53:58 +0100 Subject: [PATCH 20/31] validation types: Correct case for consistency --- src/errors/validation-error-type.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/errors/validation-error-type.js b/src/errors/validation-error-type.js index 6d769cdb..486c2fcc 100644 --- a/src/errors/validation-error-type.js +++ b/src/errors/validation-error-type.js @@ -40,7 +40,7 @@ const ValidationErrorType = { TYPE_LIMITS_USE: 'type_limits_use', WRONG_BASE_TYPE: 'wrong_base_type', FIELD_NOT_ALLOWED: 'field_not_allowed', - BELOW_MIN_VALUE_INCLUSIVE: 'BELOW_MIN_VALUE_INCLUSIVE', + BELOW_MIN_VALUE_INCLUSIVE: 'below_min_value_inclusive', URI_TEMPLATE_MISSING_PLACEHOLDER: 'uri_template_missing_placeholder', }; From 4c7f8a9aeeacd05d9a6e6caa5828586a710d6c3e Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Wed, 31 Mar 2021 11:55:41 +0100 Subject: [PATCH 21/31] data-quality: schedule: Exception dates error type update --- src/errors/validation-error-type.js | 1 + .../schedule-exceptdates-match-recurrence-dates-spec.js | 2 +- .../data-quality/schedule-exceptdates-match-recurrence-dates.js | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/errors/validation-error-type.js b/src/errors/validation-error-type.js index 486c2fcc..456179aa 100644 --- a/src/errors/validation-error-type.js +++ b/src/errors/validation-error-type.js @@ -42,6 +42,7 @@ const ValidationErrorType = { FIELD_NOT_ALLOWED: 'field_not_allowed', BELOW_MIN_VALUE_INCLUSIVE: 'below_min_value_inclusive', URI_TEMPLATE_MISSING_PLACEHOLDER: 'uri_template_missing_placeholder', + EXCEPTION_DATES_NOT_IN_SCHEDULE: 'exception_dates_not_in_schedule', }; module.exports = Object.freeze(ValidationErrorType); diff --git a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates-spec.js b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates-spec.js index d4beccf3..97786358 100644 --- a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates-spec.js +++ b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates-spec.js @@ -86,7 +86,7 @@ describe('ExceptDatesAreInSchedule', () => { expect(errors.length).toBe(1); for (const error of errors) { - expect(error.type).toBe(ValidationErrorType.MISSING_REQUIRED_FIELD); + expect(error.type).toBe(ValidationErrorType.EXCEPTION_DATES_NOT_IN_SCHEDULE); expect(error.severity).toBe(ValidationErrorSeverity.WARNING); } }); diff --git a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js index 9e3a8b9a..9c7eca3c 100644 --- a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js +++ b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates.js @@ -24,7 +24,7 @@ module.exports = class ExceptDatesAreInSchedule extends Rule { }, category: ValidationErrorCategory.CONFORMANCE, severity: ValidationErrorSeverity.WARNING, - type: ValidationErrorType.MISSING_REQUIRED_FIELD, + type: ValidationErrorType.EXCEPTION_DATES_NOT_IN_SCHEDULE, }, }, }; From 4bb1e75fa8fe8579117f6e5ef84b333a566f4c6b Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Thu, 1 Apr 2021 10:38:36 +0100 Subject: [PATCH 22/31] data-quality: schedule: Check scheduleEventType is a valid event subClass --- src/errors/validation-error-type.js | 1 + ...ntype-is-valid-event-subclass-rule-spec.js | 118 ++++++++++++++++++ ...leeventype-is-valid-event-subclass-rule.js | 82 ++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule-spec.js create mode 100644 src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule.js diff --git a/src/errors/validation-error-type.js b/src/errors/validation-error-type.js index 456179aa..b46eabb1 100644 --- a/src/errors/validation-error-type.js +++ b/src/errors/validation-error-type.js @@ -43,6 +43,7 @@ const ValidationErrorType = { BELOW_MIN_VALUE_INCLUSIVE: 'below_min_value_inclusive', URI_TEMPLATE_MISSING_PLACEHOLDER: 'uri_template_missing_placeholder', EXCEPTION_DATES_NOT_IN_SCHEDULE: 'exception_dates_not_in_schedule', + INVALID_SCHEDULE_EVENT_TYPE: 'invalid_schedule_event_type', }; module.exports = Object.freeze(ValidationErrorType); diff --git a/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule-spec.js b/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule-spec.js new file mode 100644 index 00000000..cb8e49ee --- /dev/null +++ b/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule-spec.js @@ -0,0 +1,118 @@ +const ScheduleEventTypeIsEventSubclass = require('./scheduleeventype-is-valid-event-subclass-rule'); +const Model = require('../../classes/model'); +const ModelNode = require('../../classes/model-node'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +describe('ScheduleEventTypeIsEventSubclass', () => { + let rule; + + beforeEach(() => { + rule = new ScheduleEventTypeIsEventSubclass(); + }); + + it('should target models of any type', () => { + const model = new Model({ + type: 'Schedule', + }, 'latest'); + + const isTargeted = rule.isModelTargeted(model); + expect(isTargeted).toBe(true); + }); + + it('should return no errors if scheduleEventType is a subClass of Event', async () => { + const model = new Model({ + type: 'Schedule', + subClassGraph: ['#Event'], + }, 'latest'); + + const data = { + scheduledEventType: 'ScheduledSession', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(0); + }); + + it('should return errors if scheduleEventType is not a subClass of Event', async () => { + const model = new Model({ + type: 'Schedule', + subClassGraph: ['#Event'], + }, 'latest'); + + const data = { + scheduledEventType: 'Place', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(1); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.INVALID_SCHEDULE_EVENT_TYPE); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); + + it('should return errors if scheduleEventType does not have a valid model', async () => { + const model = new Model({ + type: 'Schedule', + subClassGraph: ['#Event'], + }, 'latest'); + + const data = { + scheduledEventType: 'Banana', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(1); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.INVALID_SCHEDULE_EVENT_TYPE); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); + + it('should return errors if scheduleEventType does not have a subClassGraph', async () => { + const model = new Model({ + type: 'Schedule', + subClassGraph: ['#Event'], + }, 'latest'); + + const data = { + scheduledEventType: 'DownloadData', + }; + + const nodeToTest = new ModelNode( + '$', + data, + null, + model, + ); + const errors = await rule.validate(nodeToTest); + + expect(errors.length).toBe(1); + for (const error of errors) { + expect(error.type).toBe(ValidationErrorType.INVALID_SCHEDULE_EVENT_TYPE); + expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); + } + }); +}); diff --git a/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule.js b/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule.js new file mode 100644 index 00000000..e9c2e94f --- /dev/null +++ b/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule.js @@ -0,0 +1,82 @@ +const Rule = require('../rule'); +const DataModelHelper = require('../../helpers/data-model'); +const ValidationErrorType = require('../../errors/validation-error-type'); +const ValidationErrorCategory = require('../../errors/validation-error-category'); +const ValidationErrorSeverity = require('../../errors/validation-error-severity'); + +module.exports = class ScheduleEventTypeIsEventSubclass extends Rule { + constructor(options) { + super(options); + this.targetModels = 'Schedule'; + this.meta = { + name: 'ScheduleEventTypeIsEventSubclass', + description: 'The `scheduleEventType` in the `Schedule` is not a subclass of `Event`.', + tests: { + modelNotRecognised: { + message: 'The model described by `scheduleEventType` ({{value}}) is not a valid model type. See the [Inheritence Overview](https://developer.openactive.io/publishing-data/data-feeds/types-of-feed#schema-org-type-inheritance-overview) for more information.', + sampleValues: { + value: 'BigSwimEvent', + }, + category: ValidationErrorCategory.CONFORMANCE, + severity: ValidationErrorSeverity.FAILURE, + type: ValidationErrorType.INVALID_SCHEDULE_EVENT_TYPE, + }, + modelHasNoSubClassGraph: { + message: 'The model described by `scheduleEventType` ({{value}}) does not list any subclass types. See the [Inheritence Overview](https://developer.openactive.io/publishing-data/data-feeds/types-of-feed#schema-org-type-inheritance-overview) for more information.', + sampleValues: { + value: 'DownloadData', + }, + category: ValidationErrorCategory.CONFORMANCE, + severity: ValidationErrorSeverity.FAILURE, + type: ValidationErrorType.INVALID_SCHEDULE_EVENT_TYPE, + }, + modelIsNotEventSubClass: { + message: 'The `scheduleEventType` ({{value}}) in `Schedule` does not inherit from `Event`. See the [Inheritence Overview](https://developer.openactive.io/publishing-data/data-feeds/types-of-feed#schema-org-type-inheritance-overview) for more information.', + sampleValues: { + value: 'ScheduledSession', + }, + category: ValidationErrorCategory.CONFORMANCE, + severity: ValidationErrorSeverity.FAILURE, + type: ValidationErrorType.INVALID_SCHEDULE_EVENT_TYPE, + }, + }, + }; + } + + validateModel(node) { + const errors = []; + let model; + let errorCondition; + const scheduledEventType = node.getValue('scheduledEventType'); + + try { + model = DataModelHelper.loadModel(scheduledEventType, 'latest'); + } catch (error) { + model = undefined; + } + + if (typeof model === 'undefined') { + errorCondition = 'modelNotRecognised'; + } else if (typeof model.subClassGraph === 'undefined') { + errorCondition = 'modelHasNoSubClassGraph'; + } else if (model.subClassGraph.indexOf('#Event') === -1) { + errorCondition = 'modelIsNotEventSubClass'; + } + + if (errorCondition === 'modelIsNotEventSubClass') { + errors.push( + this.createError( + errorCondition, + { + value: scheduledEventType, + path: node.getPath('scheduledEventType'), + }, + ), + ); + } else { + return []; + } + + return errors; + } +}; From 95c782f97baceb090711b05e3d79530e7f81d5d1 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Thu, 1 Apr 2021 10:42:10 +0100 Subject: [PATCH 23/31] data-quality: schedule: Catch errors when the model or subclassgraph is unknown --- .../scheduleeventype-is-valid-event-subclass-rule-spec.js | 2 +- .../scheduleeventype-is-valid-event-subclass-rule.js | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule-spec.js b/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule-spec.js index cb8e49ee..e894d3cd 100644 --- a/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule-spec.js +++ b/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule-spec.js @@ -91,7 +91,7 @@ describe('ScheduleEventTypeIsEventSubclass', () => { } }); - it('should return errors if scheduleEventType does not have a subClassGraph', async () => { + it('should not errors if scheduleEventType does not have a subClassGraph', async () => { const model = new Model({ type: 'Schedule', subClassGraph: ['#Event'], diff --git a/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule.js b/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule.js index e9c2e94f..7951b066 100644 --- a/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule.js +++ b/src/rules/data-quality/scheduleeventype-is-valid-event-subclass-rule.js @@ -63,7 +63,7 @@ module.exports = class ScheduleEventTypeIsEventSubclass extends Rule { errorCondition = 'modelIsNotEventSubClass'; } - if (errorCondition === 'modelIsNotEventSubClass') { + if (errorCondition) { errors.push( this.createError( errorCondition, @@ -73,8 +73,6 @@ module.exports = class ScheduleEventTypeIsEventSubclass extends Rule { }, ), ); - } else { - return []; } return errors; From aff4abc25cf2874b3c6308d0ec2c724ac1e33084 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Mon, 5 Apr 2021 16:58:47 +0100 Subject: [PATCH 24/31] data-quality: schedules: Convert to UTC for RRule --- package.json | 1 + src/helpers/datetime-helper.js | 9 ++++----- src/helpers/rrule-options.js | 6 ++++-- .../schedule-contains-recurrence-data-rule-spec.js | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 057dd347..262a9b46 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "html-entities": "^1.3.1", "iso8601-duration": "^1.3.0", "jsonpath": "^1.0.2", + "luxon": "^1.26.0", "moment": "^2.24.0", "moment-timezone": "^0.5.33", "rrule": "^2.6.4", diff --git a/src/helpers/datetime-helper.js b/src/helpers/datetime-helper.js index 218c797b..7badcb23 100644 --- a/src/helpers/datetime-helper.js +++ b/src/helpers/datetime-helper.js @@ -1,9 +1,8 @@ -function getDateTime(dateString, timeString) { +const { DateTime } = require('luxon'); + +function getDateTime(ianaTimezone, dateString, timeString) { if (typeof dateString !== 'undefined' && typeof timeString !== 'undefined') { - return new Date(`${dateString}T${timeString}`); - } - if (typeof dateString !== 'undefined') { - return new Date(dateString); + return DateTime.fromISO(`${dateString}T${timeString}`, { zone: ianaTimezone }).toJSDate(); } return undefined; } diff --git a/src/helpers/rrule-options.js b/src/helpers/rrule-options.js index d41052d7..299af18d 100644 --- a/src/helpers/rrule-options.js +++ b/src/helpers/rrule-options.js @@ -1,8 +1,10 @@ const getDateTime = require('./datetime-helper'); function generateRRuleOptions(properties) { - const dtStart = getDateTime(properties.startDate, properties.startTime); - const dtEnd = getDateTime(properties.endDate, properties.endTime); + const toRruleDate = (date, time) => getDateTime('UTC', date, time); + + const dtStart = toRruleDate(properties.startDate, properties.startTime); + const dtEnd = toRruleDate(properties.endDate, properties.endTime); const rruleOptions = {}; diff --git a/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js b/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js index 72aa7984..c4f78391 100644 --- a/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js +++ b/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js @@ -100,7 +100,7 @@ describe('ValidRecurrenceRule', () => { const errors = await rule.validate(nodeToTest); - expect(errors.length).toBe(1); + expect(errors.length).toBe(2); for (const error of errors) { expect(error.type).toBe(ValidationErrorType.MISSING_REQUIRED_FIELD); expect(error.severity).toBe(ValidationErrorSeverity.FAILURE); From a75821082b95c91d0c04b08a97f0531ba752956b Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Mon, 5 Apr 2021 17:01:27 +0100 Subject: [PATCH 25/31] helpers: Refactor date function calls --- src/helpers/rrule-options.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/helpers/rrule-options.js b/src/helpers/rrule-options.js index 299af18d..068ab6d8 100644 --- a/src/helpers/rrule-options.js +++ b/src/helpers/rrule-options.js @@ -1,10 +1,8 @@ const getDateTime = require('./datetime-helper'); function generateRRuleOptions(properties) { - const toRruleDate = (date, time) => getDateTime('UTC', date, time); - - const dtStart = toRruleDate(properties.startDate, properties.startTime); - const dtEnd = toRruleDate(properties.endDate, properties.endTime); + const dtStart = getDateTime('UTC', properties.startDate, properties.startTime); + const dtEnd = getDateTime('UTC', properties.endDate, properties.endTime); const rruleOptions = {}; From f8678300cf3659952aa7f3e3f5d604ad8d9087bb Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Mon, 5 Apr 2021 16:27:41 +0000 Subject: [PATCH 26/31] helpers: Ensure node uses UTC envvar for datetime functions --- src/helpers/datetime-helper.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/helpers/datetime-helper.js b/src/helpers/datetime-helper.js index 7badcb23..9347e291 100644 --- a/src/helpers/datetime-helper.js +++ b/src/helpers/datetime-helper.js @@ -1,6 +1,9 @@ const { DateTime } = require('luxon'); function getDateTime(ianaTimezone, dateString, timeString) { + if (process.env.TZ !== 'UTC') { + throw new Error(`Schedule generation logic relies on 'TZ' env var being set to 'UTC'. It is currently: ${process.env.TZ}`); + } if (typeof dateString !== 'undefined' && typeof timeString !== 'undefined') { return DateTime.fromISO(`${dateString}T${timeString}`, { zone: ianaTimezone }).toJSDate(); } From 6823bf8aaa677d343270f70091a886231053927e Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Thu, 8 Apr 2021 16:46:28 +0100 Subject: [PATCH 27/31] helpers: Set node process timezone Set process.env.TZ before doing any timezone calculations --- src/helpers/datetime-helper.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/helpers/datetime-helper.js b/src/helpers/datetime-helper.js index 9347e291..c8fc50ab 100644 --- a/src/helpers/datetime-helper.js +++ b/src/helpers/datetime-helper.js @@ -1,6 +1,7 @@ const { DateTime } = require('luxon'); function getDateTime(ianaTimezone, dateString, timeString) { + process.env.TZ = 'UTC'; if (process.env.TZ !== 'UTC') { throw new Error(`Schedule generation logic relies on 'TZ' env var being set to 'UTC'. It is currently: ${process.env.TZ}`); } From df598b89f9ccc50921404ab677b7afa9939af230 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Thu, 8 Apr 2021 16:47:00 +0100 Subject: [PATCH 28/31] data-quality: schedule: Use daylight savings shift for test --- ...exceptdates-match-recurrence-dates-spec.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates-spec.js b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates-spec.js index 97786358..06dca867 100644 --- a/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates-spec.js +++ b/src/rules/data-quality/schedule-exceptdates-match-recurrence-dates-spec.js @@ -94,11 +94,22 @@ describe('ExceptDatesAreInSchedule', () => { it('should not return errors when the exceptDate is within the recurrence rule schedule', async () => { const data = { '@type': 'Schedule', - startDate: '2021-03-19', - startTime: '08:30', + startDate: '2021-02-18', + startTime: '07:30', + count: 20, repeatFrequency: 'P1W', - count: 10, - exceptDate: ['2021-03-26T08:30:00Z'], + exceptDate: [ + '2021-02-18T07:30:00.000Z', + '2021-02-25T07:30:00.000Z', + '2021-03-04T07:30:00.000Z', + '2021-03-11T07:30:00.000Z', + '2021-03-18T07:30:00.000Z', + '2021-03-25T07:30:00.000Z', + '2021-04-01T06:30:00.000Z', + '2021-04-08T06:30:00.000Z', + '2021-04-15T06:30:00.000Z', + '2021-04-22T06:30:00.000Z', + ], scheduleTimezone: 'Europe/London', }; From ac3566edd08adf119d9792c2bf1c0e8fef1abde1 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Thu, 8 Apr 2021 17:00:32 +0100 Subject: [PATCH 29/31] package.json: Fix rrule version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 262a9b46..c8c968d2 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "luxon": "^1.26.0", "moment": "^2.24.0", "moment-timezone": "^0.5.33", - "rrule": "^2.6.4", + "rrule": "2.6.4", "striptags": "^3.1.1", "uritemplate": "^0.3.4", "validator": "^10.11.0", From dc3ee6139fbe39a657eccf0fd0aaa413c153fff7 Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Thu, 8 Apr 2021 17:13:59 +0100 Subject: [PATCH 30/31] helpers: Update datetime function Remove the process.env.TZ check - we explicitly set this Add a comment about setting this value --- src/helpers/datetime-helper.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/helpers/datetime-helper.js b/src/helpers/datetime-helper.js index c8fc50ab..206aa995 100644 --- a/src/helpers/datetime-helper.js +++ b/src/helpers/datetime-helper.js @@ -1,10 +1,9 @@ const { DateTime } = require('luxon'); function getDateTime(ianaTimezone, dateString, timeString) { + // Node pulls the timezone from the system on initialisation using the TZ environment variable. + // We can change process.env.TZ to UTC. This will update the current Node process. process.env.TZ = 'UTC'; - if (process.env.TZ !== 'UTC') { - throw new Error(`Schedule generation logic relies on 'TZ' env var being set to 'UTC'. It is currently: ${process.env.TZ}`); - } if (typeof dateString !== 'undefined' && typeof timeString !== 'undefined') { return DateTime.fromISO(`${dateString}T${timeString}`, { zone: ianaTimezone }).toJSDate(); } From aba2103bc0ae311e57ff15be60c3fa8a8fe7488a Mon Sep 17 00:00:00 2001 From: Jared Parnell Date: Fri, 9 Apr 2021 17:04:36 +0100 Subject: [PATCH 31/31] schedule rule: spec: Simplify count --- .../data-quality/schedule-contains-recurrence-data-rule-spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js b/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js index c4f78391..b6a3b509 100644 --- a/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js +++ b/src/rules/data-quality/schedule-contains-recurrence-data-rule-spec.js @@ -87,7 +87,7 @@ describe('ValidRecurrenceRule', () => { '@type': 'Schedule', startDate: '2021-03-19', repeatFrequency: 'P1W', - count: 10000, + count: 1, scheduleTimezone: 'Europe/London', };