Conti/all tooling changes

.gitignore

@@ -134,3 +134,10 @@ packages/datadog-plugin-azure-functions/test/integration-test/fixtures/node_modu
 __azurite_db_queue__.json
 __azurite_db_queue_extent__.json
 __queuestorage__/AzuriteConfig
+
+
+# ignore apm-instrumentation-toolkit analysis results dirs
+.analysis/
+
+# ignore claude settings
+.claude/

LICENSE-3rdparty.csv

@@ -6,15 +6,18 @@ require,@datadog/native-iast-taint-tracking,Apache license 2.0,Copyright 2018 Da
 require,@datadog/openfeature-node-server,Apache license 2.0,Copyright 2024 Datadog Inc.
 require,@datadog/pprof,Apache license 2.0,Copyright 2019 Google Inc.
 require,@datadog/sketches-js,Apache license 2.0,Copyright 2020 Datadog Inc.
+require,@datadog/source-map,BSD-3-Clause,Copyright (c) 2009-2011, Mozilla Foundation and contributors
 require,@datadog/wasm-js-rewriter,Apache license 2.0,Copyright 2018 Datadog Inc.
 require,@opentelemetry/api,Apache license 2.0,Copyright OpenTelemetry Authors
 require,@opentelemetry/api-logs,Apache license 2.0,Copyright OpenTelemetry Authors
 require,@opentelemetry/core,Apache license 2.0,Copyright OpenTelemetry Authors
 require,@opentelemetry/resources,Apache license 2.0,Copyright OpenTelemetry Authors
 require,@isaacs/ttlcache,Blue Oak,Copyright Isaac Z. Schlueter and Contributors
+require,astring,MIT,Copyright 2015 David Bonnet
 require,crypto-randomuuid,MIT,Copyright 2021 Node.js Foundation and contributors
 require,dc-polyfill,MIT,Copyright 2023 Datadog Inc.
 require,escape-string-regexp,MIT,Copyright Sindre Sorhus
+require,esquery,MIT,Copyright 2013 Joel Feenstra
 require,ignore,MIT,Copyright 2013 Kael Zhang and contributors
 require,import-in-the-middle,Apache license 2.0,Copyright 2021 Datadog Inc.
 require,istanbul-lib-coverage,BSD-3-Clause,Copyright 2012-2015 Yahoo! Inc.
@@ -23,6 +26,7 @@ require,jsonpath-plus,MIT,Copyright (c) 2011-2019 Stefan Goessner, Subbu Allamar
 require,limiter,MIT,Copyright 2011 John Hurliman
 require,lodash.sortby,MIT,Copyright JS Foundation and other contributors
 require,lru-cache,ISC,Copyright (c) 2010-2022 Isaac Z. Schlueter and Contributors
+require,meriyah,ISC,Copyright 2019 KFlash and others
 require,module-details-from-path,MIT,Copyright 2016 Thomas Watson Steen
 require,mutexify,MIT,Copyright (c) 2014 Mathias Buus
 require,opentracing,MIT,Copyright 2016 Resonance Labs Inc

eslint.config.mjs

@@ -45,6 +45,7 @@ export default [
       '**/versions', // This is effectively a node_modules tree.
       '**/acmeair-nodejs', // We don't own this.
       '**/vendor', // Generally, we didn't author this code.
+      '**/.analysis', // Ignore apm-instrumentation-toolkit analysis results
       'integration-tests/code-origin/typescript.js', // Generated
       'integration-tests/debugger/target-app/source-map-support/bundle.js', // Generated
       'integration-tests/debugger/target-app/source-map-support/hello/world.js', // Generated

integration-tests/helpers/index.js

@@ -22,6 +22,11 @@ const { DEBUG } = process.env
 // This is set by the setShouldKill function
 let shouldKill
 
+// Symbol constants for dynamic value matching in assertObjectContains
+const ANY_STRING = Symbol.for('test.ANY_STRING')
+const ANY_NUMBER = Symbol.for('test.ANY_NUMBER')
+const ANY_VALUE = Symbol.for('test.ANY_VALUE')
+
 /**
  * @param {string} filename
  * @param {string} cwd
@@ -57,7 +62,7 @@ async function runAndCheckOutput (filename, cwd, expectedOut, expectedSource) {
 
   if (expectedSource) {
     assert.match(out, new RegExp(`instrumentation source: ${expectedSource}`),
-    `Expected the process to output "${expectedSource}", but logs only contain: "${out}"`)
+      `Expected the process to output "${expectedSource}", but logs only contain: "${out}"`)
   }
   return pid
 }
@@ -80,9 +85,9 @@ async function runAndCheckWithTelemetry (filename, expectedOut, expectedTelemetr
   const msgs = await cleanup()
   if (expectedTelemetryPoints.length === 0) {
     // assert no telemetry sent
-    assert.strictEqual(msgs.length, 0, `Expected no telemetry, but got:\n${
-      msgs.map(msg => JSON.stringify(msg[1].points)).join('\n')
-    }`)
+    assert.strictEqual(
+      msgs.length, 0, `Expected no telemetry, but got:\n${msgs.map(msg => JSON.stringify(msg[1].points)).join('\n')
+      }`)
   } else {
     assertTelemetryPoints(pid, msgs, expectedTelemetryPoints)
   }
@@ -632,9 +637,8 @@ function setShouldKill (value) {
   })
 }
 
-// @ts-expect-error assert.partialDeepStrictEqual does not exist on older Node.js versions
-// eslint-disable-next-line n/no-unsupported-features/node-builtins
-const assertObjectContains = assert.partialDeepStrictEqual || function assertObjectContains (actual, expected) {
+// we use our own assertObjectContains, to account for any types
+const assertObjectContains = function assertObjectContains (actual, expected) {
   if (Array.isArray(expected)) {
     assert.ok(Array.isArray(actual), `Expected array but got ${typeof actual}`)
     let startIndex = 0
@@ -661,7 +665,13 @@ const assertObjectContains = assert.partialDeepStrictEqual || function assertObj
   }
 
   for (const [key, val] of Object.entries(expected)) {
-    if (val !== null && typeof val === 'object') {
+    if (val === ANY_STRING) {
+      assert.strictEqual(typeof actual[key], 'string', `Expected ${key} to be a string but got ${typeof actual[key]}`)
+    } else if (val === ANY_NUMBER) {
+      assert.strictEqual(typeof actual[key], 'number', `Expected ${key} to be a number but got ${typeof actual[key]}`)
+    } else if (val === ANY_VALUE) {
+      assert.ok(actual[key] !== undefined, `Expected ${key} to be present but it was undefined`)
+    } else if (val !== null && typeof val === 'object') {
       assert.ok(Object.hasOwn(actual, key))
       assert.notStrictEqual(actual[key], null)
       assert.strictEqual(typeof actual[key], 'object')

loader-hook.mjs

@@ -2,6 +2,7 @@ import regexpEscape from 'escape-string-regexp'
 import * as iitm from 'import-in-the-middle/hook.mjs'
 import hooks from './packages/datadog-instrumentations/src/helpers/hooks.js'
 import configHelper from './packages/dd-trace/src/config-helper.js'
+import * as rewriterLoader from './packages/datadog-instrumentations/src/helpers/rewriter/loader.mjs'
 
 // For some reason `getEnvironmentVariable` is not otherwise available to ESM.
 const env = configHelper.getEnvironmentVariable
@@ -17,6 +18,10 @@ function initialize (data = {}) {
   return iitm.initialize(data)
 }
 
+function load (url, context, nextLoad) {
+  return rewriterLoader.load(url, context, (url, context) => iitm.load(url, context, nextLoad))
+}
+
 function addInstrumentations (data) {
   const instrumentations = Object.keys(hooks)
 
@@ -48,5 +53,5 @@ function addExclusions (data) {
   )
 }
 
-export { initialize }
-export { load, getFormat, resolve, getSource } from 'import-in-the-middle/hook.mjs'
+export { initialize, load }
+export { getFormat, resolve, getSource } from 'import-in-the-middle/hook.mjs'

package.json

@@ -130,15 +130,18 @@
     "@datadog/openfeature-node-server": "^0.2.0",
     "@datadog/pprof": "5.12.0",
     "@datadog/sketches-js": "2.1.1",
+    "@datadog/source-map": "npm:source-map@^0.6.0",
     "@datadog/wasm-js-rewriter": "5.0.1",
     "@isaacs/ttlcache": "^2.0.1",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "@opentelemetry/api-logs": "<1.0.0",
     "@opentelemetry/core": ">=1.14.0 <1.31.0",
     "@opentelemetry/resources": ">=1.0.0 <1.31.0",
+    "astring": "^1.9.0",
     "crypto-randomuuid": "^1.0.0",
     "dc-polyfill": "^0.1.10",
     "escape-string-regexp": "^5.0.0",
+    "esquery": "^1.6.0",
     "ignore": "^7.0.5",
     "import-in-the-middle": "^1.14.2",
     "istanbul-lib-coverage": "^3.2.2",
@@ -147,6 +150,7 @@
     "limiter": "^1.1.5",
     "lodash.sortby": "^4.7.0",
     "lru-cache": "^10.4.3",
+    "meriyah": "^6.1.4",
     "module-details-from-path": "^1.0.4",
     "mutexify": "^1.4.0",
     "opentracing": ">=0.14.7",

packages/datadog-instrumentations/index.js

@@ -2,3 +2,4 @@
 
 require('./src/helpers/bundler-register')
 require('./src/helpers/register')
+require('./src/helpers/rewriter/loader')

packages/datadog-instrumentations/src/helpers/instrument.js

@@ -2,6 +2,7 @@
 
 const dc = require('dc-polyfill')
 const instrumentations = require('./instrumentations')
+const rewriterInstrumentations = require('./rewriter/instrumentations')
 const { AsyncResource } = require('async_hooks')
 
 const channelMap = {}
@@ -22,6 +23,15 @@ exports.tracingChannel = function (name) {
   return tc
 }
 
+exports.getHooks = function getHooks (names) {
+  names = [names].flat()
+
+  return rewriterInstrumentations
+    .map(inst => inst.module)
+    .filter(({ name }) => names.includes(name))
+    .map(({ name, versionRange, filePath }) => ({ name, versions: [versionRange], file: filePath }))
+}
+
 /**
  * @param {object} args
  * @param {string|string[]} args.name module name

packages/datadog-instrumentations/src/helpers/register.js

@@ -10,6 +10,7 @@ const checkRequireCache = require('./check-require-cache')
 const telemetry = require('../../../dd-trace/src/guardrails/telemetry')
 const { isInServerlessEnvironment } = require('../../../dd-trace/src/serverless')
 const { getEnvironmentVariables } = require('../../../dd-trace/src/config-helper')
+const rewriter = require('./rewriter')
 
 const envs = getEnvironmentVariables()
 
@@ -48,6 +49,10 @@ if (DD_TRACE_DEBUG && DD_TRACE_DEBUG.toLowerCase() !== 'false') {
 const seenCombo = new Set()
 const allInstrumentations = {}
 
+for (const inst of disabledInstrumentations) {
+  rewriter.disable(inst)
+}
+
 // TODO: make this more efficient
 for (const packageName of names) {
   if (disabledInstrumentations.has(packageName)) continue

packages/datadog-instrumentations/src/helpers/rewriter/index.js

@@ -0,0 +1,132 @@
+'use strict'
+
+// The rewriter works effectively the same as Orchestrion with some additions:
+// - Supports an `astQuery` field to filter AST nodes with an esquery query.
+// - Supports replacing methods of child class instance in the base constructor.
+
+const { readFileSync } = require('fs')
+const { join } = require('path')
+const semifies = require('semifies')
+const transforms = require('./transforms')
+const log = require('../../../../dd-trace/src/log')
+const instrumentations = require('./instrumentations')
+const { getEnvironmentVariable } = require('../../../../dd-trace/src/config-helper')
+
+const NODE_OPTIONS = getEnvironmentVariable('NODE_OPTIONS')
+
+const supported = {}
+const disabled = new Set()
+
+// TODO: Source maps without `--enable-source-maps`.
+const enableSourceMaps = NODE_OPTIONS?.includes('--enable-source-maps') ||
+  process.execArgv?.some(arg => arg.includes('--enable-source-maps'))
+
+let parse
+let generate
+let esquery
+let SourceMapGenerator
+
+function rewrite (content, filename, format) {
+  if (!content) return content
+
+  try {
+    let ast
+
+    filename = filename.replace('file://', '')
+
+    for (const inst of instrumentations) {
+      const { astQuery, functionQuery = {}, module: { name, versionRange, filePath } } = inst
+      const { kind } = functionQuery
+      const operator = kind === 'Async' ? 'tracePromise' : 'traceSync' // TODO: traceCallback
+      const transform = transforms[operator]
+
+      if (disabled.has(name)) continue
+      if (!filename.endsWith(`${name}/${filePath}`)) continue
+      if (!transform) continue
+      if (!satisfies(filename, filePath, versionRange)) continue
+
+      parse ??= require('meriyah').parse
+      generate ??= require('astring').generate
+      esquery ??= require('esquery')
+
+      ast ??= parse(content.toString(), { loc: true, ranges: true, module: format === 'module' })
+
+      const query = astQuery || fromFunctionQuery(functionQuery)
+      const selector = esquery.parse(query)
+      const state = { ...inst, format, operator }
+
+      esquery.traverse(ast, selector, (...args) => transform(state, ...args))
+    }
+
+    if (ast) {
+      if (!enableSourceMaps || SourceMapGenerator) return generate(ast)
+
+      // TODO: Can we use the same version of `source-map` that DI uses?
+      SourceMapGenerator ??= require('@datadog/source-map/lib/source-map-generator').SourceMapGenerator
+
+      const sourceMap = new SourceMapGenerator({ file: filename })
+      const code = generate(ast, { sourceMap })
+      const map = Buffer.from(sourceMap.toString()).toString('base64')
+
+      return code + '\n' + `//# sourceMappingURL=data:application/json;base64,${map}`
+    }
+  } catch (e) {
+    log.error(e)
+  }
+
+  return content
+}
+
+function disable (instrumentation) {
+  disabled.add(instrumentation)
+}
+
+function satisfies (filename, filePath, versions) {
+  const [basename] = filename.split(filePath)
+
+  if (supported[basename] === undefined) {
+    try {
+      const pkg = JSON.parse(readFileSync(
+        join(basename, 'package.json'), 'utf8'
+      ))
+
+      supported[basename] = semifies(pkg.version, versions)
+    } catch {
+      supported[basename] = false
+    }
+  }
+
+  return supported[basename]
+}
+
+// TODO: Support index
+function fromFunctionQuery (functionQuery) {
+  const { methodName, functionName, expressionName, className } = functionQuery
+  const queries = []
+
+  if (className) {
+    queries.push(
+      `[id.name="${className}"]`,
+      `[id.name="${className}"] > ClassBody > [key.name="${methodName}"] > [async]`,
+      `[id.name="${className}"] > ClassExpression > ClassBody > [key.name="${methodName}"] > [async]`
+    )
+  } else if (methodName) {
+    queries.push(
+      `ClassBody > [key.name="${methodName}"] > [async]`,
+      `Property[key.name="${methodName}"] > [async]`
+    )
+  }
+
+  if (functionName) {
+    queries.push(`FunctionDeclaration[id.name="${functionName}"][async]`)
+  } else if (expressionName) {
+    queries.push(
+      `FunctionExpression[id.name="${expressionName}"][async]`,
+      `ArrowFunctionExpression[id.name="${expressionName}"][async]`
+    )
+  }
+
+  return queries.join(', ')
+}
+
+module.exports = { rewrite, disable }

packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js

@@ -0,0 +1,25 @@
+'use strict'
+
+const { readdirSync, readFileSync } = require('fs')
+const { join } = require('path')
+
+const instrumentations = []
+
+const files = readdirSync(__dirname).filter(f => f.endsWith('.json'))
+
+// load all JSON instrumentations
+for (const file of files) {
+  try {
+    const content = JSON.parse(readFileSync(join(__dirname, file), 'utf8'))
+    if (Array.isArray(content)) {
+      instrumentations.push(...content)
+    } else {
+      instrumentations.push(content)
+    }
+  } catch (e) {
+    // eslint-disable-next-line no-console
+    console.error(`Failed to load instrumentation config: ${file}`, e.message)
+  }
+}
+
+module.exports = instrumentations