Add support for NGSI-LD valueType and more NGSI-LD sub-property types.

CHANGES_NEXT_RELEASE

@@ -1 +1,2 @@
 - Add: Support NGSI-LD QueryEntities endpoint for lazy attributes (#1722)
+- Add: Support for NGSI-LD valueType and more NGSI-LD sub-property types (new config server.ldSupport.datatype and associated env var IOTA_LD_SUPPORT_DATA_TYPE) (#1723)

doc/admin.md

@@ -145,15 +145,17 @@ used by default. E.g.:
 When connected to an **NGSI-LD** context broker, an IoT Agent is able to indicate whether it is willing to accept `null`
 values and also whether it is able to process the **NGSI-LD** `datasetId` metadata element. Setting these values to
 `false` will cause the IoT Agent to return a 400 **Bad Request** HTTP status code explaining that the IoT Agent does not
-support nulls or multi-attribute requests if they are encountered.
+support nulls or multi-attribute requests if they are encountered. It is also possible to pass on attribute datatypes
+using `@type` or `valueType` if desired.
 
 ```javascript
 {
     baseRoot: '/',
     port: 4041,
     ldSupport : {
       null: true,
-      datasetId: true
+      datasetId: true,
+      datatype: 'valueType'
     }
 }
 ```
@@ -446,8 +448,9 @@ For example in a device document stored in MongoDB will be extended with a subdo
 
 #### `useCBflowControl`
 
-If this flag is activated, when iotAgent invokes Context Broker will use [flowControl option](https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/admin/perf_tuning.md#updates-flow-control-mechanism). This flag is overwritten by
-`useCBflowControl` flag in group or device. This flag is disabled by default.
+If this flag is activated, when iotAgent invokes Context Broker will use
+[flowControl option](https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/admin/perf_tuning.md#updates-flow-control-mechanism).
+This flag is overwritten by `useCBflowControl` flag in group or device. This flag is disabled by default.
 
 ### Configuration using environment variables
 
@@ -465,6 +468,7 @@ overrides.
 | IOTA_CB_NGSI_VERSION                 | `contextBroker.ngsiVersion`     |
 | IOTA_NORTH_HOST                      | `server.host`                   |
 | IOTA_NORTH_PORT                      | `server.port`                   |
+| IOTA_LD_SUPPORT_DATA_TYPE            | `server.ldSupport.datatype`     |
 | IOTA_LD_SUPPORT_NULL                 | `server.ldSupport.null`         |
 | IOTA_LD_SUPPORT_DATASET_ID           | `server.ldSupport.datasetId`    |
 | IOTA_PROVIDER_URL                    | `providerUrl`                   |

lib/commonConfig.js

@@ -157,6 +157,7 @@ function processEnvironmentVariables() {
         'IOTA_FALLBACK_PATH',
         'IOTA_LD_SUPPORT_NULL',
         'IOTA_LD_SUPPORT_DATASET_ID',
+        'IOTA_LD_SUPPORT_DATA_TYPE',
         'IOTA_EXPRESS_LIMIT',
         'IOTA_USE_CB_FLOW_CONTROL',
         'IOTA_STORE_LAST_MEASURE'
@@ -264,7 +265,12 @@ function processEnvironmentVariables() {
         config.server.port = process.env.IOTA_NORTH_PORT;
     }
 
-    config.server.ldSupport = config.server.ldSupport || { null: true, datasetId: true, merge: false };
+    config.server.ldSupport = config.server.ldSupport || {
+        null: true,
+        datasetId: true,
+        merge: false,
+        dataType: 'none'
+    };
 
     if (process.env.IOTA_LD_SUPPORT_NULL) {
         config.server.ldSupport.null = process.env.IOTA_LD_SUPPORT_NULL === 'true';
@@ -275,6 +281,9 @@ function processEnvironmentVariables() {
     if (process.env.IOTA_LD_SUPPORT_MERGE) {
         config.server.ldSupport.datasetId = process.env.IOTA_LD_SUPPORT_MERGE === 'true';
     }
+    if (process.env.IOTA_LD_SUPPORT_DATA_TYPE) {
+        config.server.ldSupport.dataType = process.env.IOTA_LD_SUPPORT_DATA_TYPE;
+    }
 
     if (process.env.IOTA_PROVIDER_URL) {
         config.providerUrl = process.env.IOTA_PROVIDER_URL;
@@ -511,7 +520,7 @@ function getConfig() {
 function getConfigForTypeInformation() {
     // Just return relevant configuration flags
     // avoid to include server, authentication, mongodb, orion and iotamanger info
-    let conf = {
+    const conf = {
         timestamp: config.timestamp,
         defaultResource: config.defaultResource,
         explicitAttrs: config.explicitAttrs,

lib/services/devices/devices-NGSI-LD.js

@@ -33,7 +33,6 @@ const logger = require('logops');
 const config = require('../../commonConfig');
 const ngsiLD = require('../ngsi/entities-NGSI-LD');
 const utils = require('../northBound/restUtils');
-const moment = require('moment');
 const _ = require('underscore');
 const registrationUtils = require('./registrationUtils');
 const NGSIv2 = require('./devices-NGSI-v2');
@@ -153,7 +152,7 @@ function updateEntityNgsiLD(deviceData, updatedDevice, callback) {
     ) {
         options.json[constants.TIMESTAMP_ATTRIBUTE] = {
             type: constants.TIMESTAMP_TYPE_NGSI2,
-            value: moment()
+            value: new Date().toISOString()
         };
     }
 

lib/services/devices/devices-NGSI-v2.js

@@ -41,7 +41,6 @@ const registrationUtils = require('./registrationUtils');
 const _ = require('underscore');
 const utils = require('../northBound/restUtils');
 const NGSIv2 = require('../ngsi/entities-NGSI-v2');
-const moment = require('moment');
 const context = {
     op: 'IoTAgentNGSI.Devices-v2'
 };
@@ -222,7 +221,7 @@ function updateEntityNgsi2(deviceData, updatedDevice, callback) {
     ) {
         options.json[constants.TIMESTAMP_ATTRIBUTE] = {
             type: constants.TIMESTAMP_TYPE_NGSI2,
-            value: moment()
+            value: new Date().toISOString()
         };
     }
 

lib/services/ngsi/entities-NGSI-LD.js

@@ -59,10 +59,13 @@ function convertAttrNGSILD(attr) {
         return undefined;
     }
     let obj = { type: 'Property', value: attr.value };
+    let hasValueType = true;
 
     switch (attr.type.toLowerCase()) {
         // Properties
         case 'property':
+            hasValueType = false;
+            break;
         case 'string':
         case 'text':
         case 'textunrestricted':
@@ -96,76 +99,129 @@ function convertAttrNGSILD(attr) {
             }
             break;
 
+        case 'object':
+        case 'array':
+            try {
+                obj.value = JSON.parse(attr.value);
+            } catch (e) {
+                // Do nothing
+            }
+            break;
+
         // Temporal Properties
         case 'datetime':
-            obj.value = {
-                '@type': 'DateTime',
-                '@value': moment.tz(attr.value, 'Etc/UTC').toISOString()
-            };
+            obj.value = moment.tz(attr.value, 'Etc/UTC').toISOString();
             break;
         case 'date':
-            obj.value = {
-                '@type': 'Date',
-                '@value': moment.tz(attr.value, 'Etc/UTC').format(moment.HTML5_FMT.DATE)
-            };
+            obj.value = moment.tz(attr.value, 'Etc/UTC').format(moment.HTML5_FMT.DATE);
             break;
         case 'time':
-            obj.value = {
-                '@type': 'Time',
-                '@value': moment.tz(attr.value, 'Etc/UTC').format(moment.HTML5_FMT.TIME_SECONDS)
-            };
+            obj.value = moment
+                .tz(new Date('0000-01-01 ' + attr.value), 'Etc/UTC')
+                .format(moment.HTML5_FMT.TIME_SECONDS);
             break;
 
         // GeoProperties
         case 'geoproperty':
         case 'point':
         case 'geo:point':
         case 'geo:json':
+            hasValueType = false;
             obj.type = 'GeoProperty';
             obj.value = NGSIUtils.getLngLats('Point', attr.value);
             break;
         case 'linestring':
         case 'geo:linestring':
+            hasValueType = false;
             obj.type = 'GeoProperty';
             obj.value = NGSIUtils.getLngLats('LineString', attr.value);
             break;
         case 'polygon':
         case 'geo:polygon':
+            hasValueType = false;
             obj.type = 'GeoProperty';
             obj.value = NGSIUtils.getLngLats('Polygon', attr.value);
             break;
         case 'multipoint':
         case 'geo:multipoint':
+            hasValueType = false;
             obj.type = 'GeoProperty';
             obj.value = NGSIUtils.getLngLats('MultiPoint', attr.value);
             break;
         case 'multilinestring':
         case 'geo:multilinestring':
+            hasValueType = false;
             obj.type = 'GeoProperty';
             obj.value = NGSIUtils.getLngLats('MultiLineString', attr.value);
             break;
         case 'multipolygon':
         case 'geo:multipolygon':
+            hasValueType = false;
             obj.type = 'GeoProperty';
             obj.value = NGSIUtils.getLngLats('MultiPolygon', attr.value);
             break;
 
         // Relationships
         case 'relationship':
+            hasValueType = false;
             obj.type = 'Relationship';
             obj.object = attr.value;
             delete obj.value;
             break;
 
         // LanguageProperties
         case 'languageproperty':
+            hasValueType = false;
             obj.type = 'LanguageProperty';
             obj.languageMap = attr.value;
             delete obj.value;
             break;
 
+        // VocabProperties
+        case 'vocabproperty':
+            hasValueType = false;
+            obj.type = 'VocabProperty';
+            obj.vocab = attr.value;
+            delete obj.value;
+            break;
+        // JsonProperties
+        case 'jsonproperty':
+            hasValueType = false;
+            obj.type = 'JsonProperty';
+            obj.json = attr.value;
+            delete obj.value;
+            break;
+        // ListProperties
+        case 'listproperty':
+            hasValueType = false;
+            obj.type = 'ListProperty';
+            obj.listValue = attr.value;
+            delete obj.value;
+            break;
+        // ListRelationship
+        case 'listrelationship':
+            hasValueType = false;
+            obj.type = 'ListRelationship';
+            obj.listObject = attr.value;
+            delete obj.value;
+            break;
+
         default:
-            obj.value = { '@type': attr.type, '@value': attr.value };
+            obj.value = attr.value;
+    }
+
+    if (hasValueType) {
+        switch (config.getConfig().server.ldSupport.dataType) {
+            case '@type':
+                obj.value = {
+                    '@type': attr.type,
+                    '@value': obj.value
+                };
+                break;
+            case 'valueType':
+                obj.valueType = attr.type;
+                break;
+        }
     }
 
     if (!!obj && attr.metadata) {

lib/services/northBound/restUtils.js

@@ -30,7 +30,6 @@ const errors = require('../../errors');
 const constants = require('../../constants');
 const intoTrans = require('../common/domain').intoTrans;
 const revalidator = require('revalidator');
-const moment = require('moment');
 const context = {
     op: 'IoTAgentNGSI.RestUtils'
 };
@@ -121,6 +120,11 @@ function checkBody(template) {
     };
 }
 
+function isISOString(val) {
+    const d = new Date(val);
+    return !Number.isNaN(d.valueOf()) && d.toISOString() === val;
+}
+
 /**
  * Checks if the timestamp properties of NGSIv2 entities are valid ISO8601 dates.
  *
@@ -131,7 +135,7 @@ function IsValidTimestampedNgsi2(payload) {
     function isValidTimestampedNgsi2Entity(entity) {
         for (const i in entity) {
             if (entity.hasOwnProperty(i)) {
-                if (i === constants.TIMESTAMP_ATTRIBUTE && !moment(entity[i].value, moment.ISO_8601).isValid()) {
+                if (i === constants.TIMESTAMP_ATTRIBUTE && !isISOString(entity[i].value)) {
                     return false;
                 }
             }

test/unit/ngsi-ld/examples/contextRequests/createDatetimeProvisionedDevice.json

@@ -4,10 +4,7 @@
         "id": "urn:ngsi-ld:MicroLights:FirstMicroLight",
         "timestamp": {
             "type": "Property",
-            "value": {
-                "@type": "DateTime",
-                "@value": "1970-01-01T00:00:00.000Z"
-            }
+            "value": "1970-01-01T00:00:00.000Z"
         },
         "type": "MicroLights"
     }

test/unit/ngsi-ld/examples/contextRequests/createProvisionedDevice.json

@@ -3,24 +3,15 @@
         "@context": "http://context.json-ld",
         "commandAttr_info": {
             "type": "Property",
-            "value": {
-                "@type": "commandResult",
-                "@value": " "
-            }
+            "value": " "
         },
         "commandAttr_status": {
             "type": "Property",
-            "value": {
-                "@type": "commandStatus",
-                "@value": "UNKNOWN"
-            }
+            "value":  "UNKNOWN"
         },
         "hardcodedAttr": {
             "type": "Property",
-            "value": {
-                "@type": "hardcodedType",
-                "@value": "hardcodedValue"
-            }
+            "value": "hardcodedValue"
         },
         "id": "urn:ngsi-ld:TheLightType:TheFirstLight",
         "type": "TheLightType"

test/unit/ngsi-ld/examples/contextRequests/createProvisionedDeviceMultientity.json

@@ -3,24 +3,15 @@
         "@context": "http://context.json-ld",
         "commandAttr_info": {
             "type": "Property",
-            "value": {
-                "@type": "commandResult",
-                "@value": " "
-            }
+            "value":  " "
         },
         "commandAttr_status": {
             "type": "Property",
-            "value": {
-                "@type": "commandStatus",
-                "@value": "UNKNOWN"
-            }
+            "value": "UNKNOWN"
         },
         "hardcodedAttr": {
             "type": "Property",
-            "value": {
-                "@type": "hardcodedType",
-                "@value": "hardcodedValue"
-            }
+            "value":  "hardcodedType"
         },
         "id": "urn:ngsi-ld:TheLightType:TheFirstLight",
         "type": "TheLightType"