Updating local metadata on checkin fails and then updating the local metadata can no longer happen #5674
Description
It is possible that updating the local metadata when an Elastic Agent checks in can fail with:
{
"error": "Eat bulk checkin error; Keep on truckin'",
"error.message": "elastic fail 400: document_parsing_exception: [1:208] object mapping for [local_metadata] tried to parse field [local_metadata] as object, but found a concrete value"
}
This was believed to be a mapping issue, but that doesn't seem like the case as we are still seeing this error with the latest mappings applied:
Mappings
{
".fleet-agents-7": {
"mappings": {
"dynamic": "false",
"_meta": {
"version": "9.1.4",
"managed_index_mappings_version": 4
},
"properties": {
"access_api_key_id": {
"type": "keyword"
},
"action_seq_no": {
"type": "integer",
"index": false
},
"active": {
"type": "boolean"
},
"agent": {
"properties": {
"id": {
"type": "keyword"
},
"version": {
"type": "keyword"
}
}
},
"audit_unenrolled_reason": {
"type": "keyword"
},
"audit_unenrolled_time": {
"type": "date"
},
"components": {
"type": "object",
"enabled": false
},
"default_api_key": {
"type": "keyword"
},
"default_api_key_history": {
"type": "object",
"enabled": false
},
"default_api_key_id": {
"type": "keyword"
},
"enrolled_at": {
"type": "date"
},
"enrollment_id": {
"type": "keyword"
},
"last_checkin": {
"type": "date"
},
"last_checkin_message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 1024
}
}
},
"last_checkin_status": {
"type": "keyword"
},
"last_updated": {
"type": "date"
},
"local_metadata": {
"properties": {
"elastic": {
"properties": {
"agent": {
"properties": {
"build": {
"properties": {
"original": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"complete": {
"type": "boolean"
},
"fips": {
"type": "boolean"
},
"id": {
"type": "keyword"
},
"log_level": {
"type": "keyword"
},
"snapshot": {
"type": "boolean"
},
"unprivileged": {
"type": "boolean"
},
"upgradeable": {
"type": "boolean"
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 16
}
}
}
}
}
}
},
"host": {
"properties": {
"architecture": {
"type": "keyword"
},
"hostname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "keyword"
},
"ip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 64
}
}
},
"mac": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 17
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"os": {
"properties": {
"family": {
"type": "keyword"
},
"full": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 128
}
}
},
"kernel": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 128
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"platform": {
"type": "keyword"
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 32
}
}
}
}
}
}
},
"namespaces": {
"type": "keyword"
},
"packages": {
"type": "keyword"
},
"policy_coordinator_idx": {
"type": "integer"
},
"policy_id": {
"type": "keyword"
},
"policy_output_permissions_hash": {
"type": "keyword"
},
"policy_revision_idx": {
"type": "integer"
},
"shared_id": {
"type": "keyword"
},
"tags": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"unenrolled_at": {
"type": "date"
},
"unenrolled_reason": {
"type": "keyword"
},
"unenrollment_started_at": {
"type": "date"
},
"unhealthy_reason": {
"type": "keyword"
},
"updated_at": {
"type": "date"
},
"upgrade_attempts": {
"type": "date"
},
"upgrade_details": {
"properties": {
"action_id": {
"type": "keyword"
},
"metadata": {
"properties": {
"download_percent": {
"type": "double"
},
"download_rate": {
"type": "double"
},
"error_msg": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 1024
}
}
},
"failed_state": {
"type": "keyword"
},
"reason": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 1024
}
}
},
"retry_error_msg": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 1024
}
}
},
"retry_until": {
"type": "date"
},
"scheduled_at": {
"type": "date"
}
}
},
"state": {
"type": "keyword"
},
"target_version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 16
}
}
}
}
},
"upgrade_started_at": {
"type": "date"
},
"upgrade_status": {
"type": "keyword"
},
"upgraded_at": {
"type": "date"
},
"user_provided_metadata": {
"type": "object",
"enabled": false
}
}
}
}
}
It is possible to see the actual structure of the local metadata used by the Elastic Agent when it checks in and it looks like the following:
"req.LocalMeta": {
"os": {
"kernel": "10.0.26100.6584 (WinBuild.160101.0800)",
"name": "Windows 11 Enterprise",
"family": "windows",
"version": "10.0",
"platform": "windows",
"full": "Windows 11 Enterprise(10.0)"
},
"elastic": {
"agent": {
"build.original": "9.1.5 (build: 7bee013e95ad3bdca704daf1924df27d880efbf1 at 2025-10-03 07:31:41 +0000 UTC)",
"unprivileged": false,
"upgradeable": true,
"log_level": "info",
"fips": false,
"id": "<REDACTED>",
"complete": false,
"version": "9.1.5",
"snapshot": false
}
},
"host": {
"hostname": "<REDACTED>",
"ip": [
"<REDACTED>",
"<REDACTED>",
"<REDACTED>"
],
"name": "<REDACTED>",
"id": "<REDACTED>",
"mac": [
"<REDACTED>",
"<REDACTED>",
"<REDACTED>"
],
"architecture": "x86_64"
}
},
All values in the local metadata maps to the mapping correctly. It is not clear if the issue is with what is being written or if its an issue with what is already in the document and the mappings are trying to read it before doing the replacement.
For confirmed bugs, please report:
- Version: All