better logging for config state; release 4.10.0 (#4386)

This cherry-picks an improvement from the 3.x branch:

> This adds a periodic (once per minute) dump of effective config changes
> from the default, once per minute at 'trace'-level.
>
> This also fixes an issue where a central-config update of `log_level`
> did not update the *child* logger on the APM client. Only the agent's
> own logger's level was updated.
>
> This adds trans.sampled to some debug logging output.

and prepares for a v4.10.0 release.

Refs: #4291 (equivalent on 3.x branch)

Author: trentm

CHANGELOG.asciidoc

@@ -33,6 +33,16 @@ Notes:
 
 See the <<upgrade-to-v4>> guide.
 
+[[release-notes-4.10.0]]
+==== 4.10.0 - 2024/12/24
+
+[float]
+===== Features
+
+* Improve trace-level logging to better support debugging central config
+  and transaction sampling issues. ({issues}4291[#4291])
+
+
 [[release-notes-4.9.0]]
 ==== 4.9.0 - 2024/12/09
 
@@ -50,8 +60,6 @@ See the <<upgrade-to-v4>> guide.
   resulting in the APM agent no longer sending tracing data.
   ({pull}4359[#4359])
 
-[float]
-===== Chores
 
 
 [[release-notes-4.8.1]]

lib/agent.js

@@ -24,6 +24,7 @@ const errors = require('./errors');
 const { InflightEventSet } = require('./InflightEventSet');
 const { Instrumentation } = require('./instrumentation');
 const { elasticApmAwsLambda } = require('./lambda');
+const logging = require('./logging');
 const Metrics = require('./metrics');
 const parsers = require('./parsers');
 const symbols = require('./symbols');
@@ -317,6 +318,23 @@ Agent.prototype.start = function (opts) {
 
   this.logger.info(preambleData, 'Elastic APM Node.js Agent v%s', version);
 
+  if (!logging.isLoggerCustom(this.logger)) {
+    // Periodically dump the current config (delta from defaults) when logging
+    // at "trace"-level. This allows getting the effective config from a running
+    // agent by setting trace-level logging and getting 1 minute of logs.
+    // (Sometimes getting logs from application *start* is no possible.)
+    setInterval(() => {
+      if (this.logger.isLevelEnabled('trace')) {
+        try {
+          const currConfig = this._conf.getCurrConfig();
+          this.logger.trace({ currConfig }, 'currConfig');
+        } catch (err) {
+          this.logger.trace({ err }, 'error calculating currConfig');
+        }
+      }
+    }, 60 * 1000).unref();
+  }
+
   if (isPreviewVersion) {
     this.logger.warn(
       'Version %s is a pre-release and not intended for use in production environments',

lib/apm-client/apm-client.js

@@ -70,6 +70,8 @@ function createApmClient(config, agent) {
             !logging.isLoggerCustom(agent.logger)
           ) {
             logging.setLogLevel(agent.logger, value);
+            // Hackily also set the HttpApmClient._log level.
+            logging.setLogLevel(client._log, value);
             agent.logger.info(
               `Central config success: updated logger with new logLevel: ${value}`,
             );

lib/config/config.js

@@ -34,6 +34,7 @@ const {
   setStartOptions,
   getPreambleData,
   readEnvOptions,
+  CENTRAL_CONFIG_OPTS,
 } = require('./schema');
 
 const {
@@ -244,6 +245,27 @@ class Config {
     }
     return loggable;
   }
+
+  // Returns an object showing the current config, excluding default values.
+  getCurrConfig() {
+    const currConfig = {};
+
+    // Start with the values from the logging preamble. This selected keys
+    // that were specified, and handles redaction.
+    for (let [k, v] of Object.entries(this.loggingPreambleData.config)) {
+      currConfig[k] = v.value;
+    }
+
+    // Then add the current value of any var possibly set by central config.
+    currConfig.centralConfig = this.centralConfig;
+    if (this.centralConfig) {
+      for (let k of Object.values(CENTRAL_CONFIG_OPTS)) {
+        currConfig[k] = this[k];
+      }
+    }
+
+    return currConfig;
+  }
 }
 
 function validateServiceName(s) {

lib/instrumentation/index.js

@@ -745,6 +745,10 @@ Instrumentation.prototype.addEndedTransaction = function (transaction) {
     !transaction.sampled &&
     !agent._apmClient.supportsKeepingUnsampledTransaction()
   ) {
+    agent.logger.debug(
+      { trans: transaction.id, trace: transaction.traceId },
+      'dropping unsampled transaction',
+    );
     return;
   }
 

lib/instrumentation/transaction.js

@@ -88,6 +88,7 @@ function Transaction(agent, name, ...args) {
     trans: this.id,
     parent: this.parentId,
     trace: this.traceId,
+    sampled: this.sampled,
     name: this.name,
     type: this.type,
   });

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "elastic-apm-node",
-  "version": "4.9.0",
+  "version": "4.10.0",
   "description": "The official Elastic APM agent for Node.js",
   "type": "commonjs",
   "main": "index.js",