Skip to content

Commit 4b4c83e

Browse files
authored
Compute and set http endpoint when route is not available (#6861)
1 parent d680a50 commit 4b4c83e

File tree

19 files changed

+833
-128
lines changed

19 files changed

+833
-128
lines changed

benchmark/core.js

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ const Histogram = require('../packages/dd-trace/src/histogram')
2828
const histogram = new Histogram()
2929
const runtimeMetrics = require('../packages/dd-trace/src/runtime_metrics')
3030
const log = require('../packages/dd-trace/src/log')
31+
const { calculateHttpEndpoint } = require('../packages/dd-trace/src/plugins/util/url')
3132

3233
const encoder04 = new Agent04Encoder({ flush: () => encoder04.makePayload() })
3334
const encoder05 = new Agent05Encoder({ flush: () => encoder05.makePayload() })
@@ -202,5 +203,35 @@ suite
202203
log.debug(() => (new Error('boom')).message)
203204
}
204205
})
206+
.add('calculateHttpEndpoint (simple)', {
207+
fn () {
208+
calculateHttpEndpoint('/api/users')
209+
}
210+
})
211+
.add('calculateHttpEndpoint (with integers)', {
212+
fn () {
213+
calculateHttpEndpoint('/api/users/12345/posts/67890')
214+
}
215+
})
216+
.add('calculateHttpEndpoint (with hex)', {
217+
fn () {
218+
calculateHttpEndpoint('/api/sessions/a1b2c3d4e5f6/data')
219+
}
220+
})
221+
.add('calculateHttpEndpoint (mixed patterns)', {
222+
fn () {
223+
calculateHttpEndpoint('/v1/users/123/sessions/a1b2c3/orders/456-789')
224+
}
225+
})
226+
.add('calculateHttpEndpoint (deep path)', {
227+
fn () {
228+
calculateHttpEndpoint('/a/b/c/d/e/f/g/h/i/j/k/l/m')
229+
}
230+
})
231+
.add('calculateHttpEndpoint (full URL)', {
232+
fn () {
233+
calculateHttpEndpoint('https://api.example.com:8080/v2/products/98765/reviews')
234+
}
235+
})
205236

206237
suite.run()

ext/tags.d.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ declare const tags: {
1515
HTTP_METHOD: 'http.method'
1616
HTTP_STATUS_CODE: 'http.status_code'
1717
HTTP_ROUTE: 'http.route'
18+
HTTP_ENDPOINT: 'http.endpoint'
1819
HTTP_REQUEST_HEADERS: 'http.request.headers'
1920
HTTP_RESPONSE_HEADERS: 'http.response.headers'
2021
HTTP_USERAGENT: 'http.useragent',

ext/tags.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ const tags = {
2020
HTTP_METHOD: 'http.method',
2121
HTTP_STATUS_CODE: 'http.status_code',
2222
HTTP_ROUTE: 'http.route',
23+
HTTP_ENDPOINT: 'http.endpoint',
2324
HTTP_REQUEST_HEADERS: 'http.request.headers',
2425
HTTP_RESPONSE_HEADERS: 'http.response.headers',
2526
HTTP_USERAGENT: 'http.useragent',
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
'use strict'
2+
3+
const axios = require('axios')
4+
const { expect } = require('chai')
5+
const { describe, it, beforeEach, afterEach, before } = require('mocha')
6+
7+
const agent = require('../../dd-trace/test/plugins/agent')
8+
9+
describe('Plugin', () => {
10+
let http
11+
let listener
12+
let appListener
13+
let port
14+
let app
15+
16+
['http', 'node:http'].forEach(pluginToBeLoaded => {
17+
describe(`${pluginToBeLoaded}/server`, () => {
18+
describe('http.endpoint', () => {
19+
before(() => {
20+
// Needed when this spec file run together with other spec files, in which case the agent config is not
21+
// re-loaded unless the existing agent is wiped first.
22+
// And we need the agent config to be re-loaded in order to enable appsec.
23+
agent.wipe()
24+
})
25+
26+
beforeEach(async () => {
27+
return agent.load('http', {}, { appsec: { enabled: true } })
28+
.then(() => {
29+
http = require(pluginToBeLoaded)
30+
})
31+
})
32+
33+
afterEach(() => {
34+
appListener && appListener.close()
35+
return agent.close({ ritmReset: false })
36+
})
37+
38+
beforeEach(() => {
39+
app = null
40+
listener = (req, res) => {
41+
app && app(req, res)
42+
res.writeHead(200)
43+
res.end()
44+
}
45+
})
46+
47+
beforeEach(done => {
48+
const server = new http.Server(listener)
49+
appListener = server
50+
.listen(0, 'localhost', () => {
51+
port = appListener.address().port
52+
done()
53+
})
54+
})
55+
56+
it('should set http.endpoint with int when no route is available', done => {
57+
agent
58+
.assertSomeTraces(traces => {
59+
expect(traces[0][0]).to.have.property('name', 'web.request')
60+
expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/users/123`)
61+
expect(traces[0][0].meta).to.not.have.property('http.route')
62+
expect(traces[0][0].meta).to.have.property('http.endpoint', '/users/{param:int}')
63+
})
64+
.then(done)
65+
.catch(done)
66+
67+
axios.get(`http://localhost:${port}/users/123`).catch(done)
68+
})
69+
70+
it('should set http.endpoint with int_id when no route is available', done => {
71+
agent
72+
.assertSomeTraces(traces => {
73+
expect(traces[0][0]).to.have.property('name', 'web.request')
74+
expect(traces[0][0].meta).to.not.have.property('http.route')
75+
expect(traces[0][0].meta).to.have.property('http.endpoint', '/resources/{param:int_id}')
76+
})
77+
.then(done)
78+
.catch(done)
79+
80+
axios.get(`http://localhost:${port}/resources/123-456`).catch(done)
81+
})
82+
83+
it('should set http.endpoint with hex when no route is available', done => {
84+
agent
85+
.assertSomeTraces(traces => {
86+
expect(traces[0][0]).to.have.property('name', 'web.request')
87+
expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/orders/abc123`)
88+
expect(traces[0][0].meta).to.not.have.property('http.route')
89+
expect(traces[0][0].meta).to.have.property('http.endpoint', '/orders/{param:hex}')
90+
})
91+
.then(done)
92+
.catch(done)
93+
94+
axios.get(`http://localhost:${port}/orders/abc123`).catch(done)
95+
})
96+
97+
it('should set http.endpoint with hex_id when no route is available', done => {
98+
agent
99+
.assertSomeTraces(traces => {
100+
expect(traces[0][0]).to.have.property('name', 'web.request')
101+
expect(traces[0][0].meta).to.not.have.property('http.route')
102+
expect(traces[0][0].meta).to.have.property('http.endpoint', '/resources/{param:hex_id}')
103+
})
104+
.then(done)
105+
.catch(done)
106+
107+
axios.get(`http://localhost:${port}/resources/abc-123`).catch(done)
108+
})
109+
})
110+
})
111+
})
112+
})

packages/dd-trace/src/appsec/reporter.js

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -546,10 +546,6 @@ function finishRequest (req, res, storedResponseHeaders, requestBody) {
546546
reportRequestBody(rootSpan, requestBody)
547547
}
548548

549-
if (tags['appsec.event'] === 'true' && typeof req.route?.path === 'string') {
550-
newTags['http.endpoint'] = req.route.path
551-
}
552-
553549
rootSpan.addTags(newTags)
554550
}
555551

packages/dd-trace/src/config.js

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -545,6 +545,7 @@ class Config {
545545
DD_TRACE_RATE_LIMIT,
546546
DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED,
547547
DD_TRACE_REPORT_HOSTNAME,
548+
DD_TRACE_RESOURCE_RENAMING_ENABLED,
548549
DD_TRACE_SAMPLE_RATE,
549550
DD_TRACE_SAMPLING_RULES,
550551
DD_TRACE_SCOPE,
@@ -829,6 +830,9 @@ class Config {
829830
target['remoteConfig.pollInterval'] = maybeFloat(DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS)
830831
unprocessedTarget['remoteConfig.pollInterval'] = DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS
831832
this.#setBoolean(target, 'reportHostname', DD_TRACE_REPORT_HOSTNAME)
833+
if (DD_TRACE_RESOURCE_RENAMING_ENABLED !== undefined) {
834+
this.#setBoolean(target, 'resourceRenamingEnabled', DD_TRACE_RESOURCE_RENAMING_ENABLED)
835+
}
832836
// only used to explicitly set runtimeMetrics to false
833837
const otelSetRuntimeMetrics = String(OTEL_METRICS_EXPORTER).toLowerCase() === 'none'
834838
? false
@@ -1254,6 +1258,15 @@ class Config {
12541258
this.#setBoolean(calc, 'isGitUploadEnabled',
12551259
calc.isIntelligentTestRunnerEnabled && !isFalse(getEnv('DD_CIVISIBILITY_GIT_UPLOAD_ENABLED')))
12561260

1261+
// Enable resourceRenamingEnabled when appsec is enabled and only
1262+
// if DD_TRACE_RESOURCE_RENAMING_ENABLED is not explicitly set
1263+
if (this.#env.resourceRenamingEnabled === undefined) {
1264+
const appsecEnabled = this.#options['appsec.enabled'] ?? this.#env['appsec.enabled']
1265+
if (appsecEnabled) {
1266+
this.#setBoolean(calc, 'resourceRenamingEnabled', true)
1267+
}
1268+
}
1269+
12571270
this.#setBoolean(calc, 'spanComputePeerService', this.#getSpanComputePeerService())
12581271
this.#setBoolean(calc, 'stats.enabled', this.#isTraceStatsComputationEnabled())
12591272
const defaultPropagationStyle = this.#getDefaultPropagationStyle(this.#optionsArg)

packages/dd-trace/src/config_defaults.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,7 @@ module.exports = {
169169
'remoteConfig.enabled': true,
170170
'remoteConfig.pollInterval': 5, // seconds
171171
reportHostname: false,
172+
resourceRenamingEnabled: false,
172173
'runtimeMetrics.enabled': false,
173174
'runtimeMetrics.eventLoop': true,
174175
'runtimeMetrics.gc': true,

packages/dd-trace/src/encode/span-stats.js

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ class SpanStatsEncoder extends AgentEncoder {
3131
}
3232

3333
_encodeStat (bytes, stat) {
34-
this._encodeMapPrefix(bytes, 12)
34+
this._encodeMapPrefix(bytes, 14)
3535

3636
this._encodeString(bytes, 'Service')
3737
const service = stat.Service || DEFAULT_SERVICE_NAME
@@ -70,6 +70,12 @@ class SpanStatsEncoder extends AgentEncoder {
7070

7171
this._encodeString(bytes, 'TopLevelHits')
7272
this._encodeLong(bytes, stat.TopLevelHits)
73+
74+
this._encodeString(bytes, 'HTTPMethod')
75+
this._encodeString(bytes, stat.HTTPMethod)
76+
77+
this._encodeString(bytes, 'HTTPEndpoint')
78+
this._encodeString(bytes, stat.HTTPEndpoint)
7379
}
7480

7581
_encodeBucket (bytes, bucket) {

packages/dd-trace/src/plugin_manager.js

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,8 @@ module.exports = class PluginManager {
165165
traceWebsocketMessagesEnabled,
166166
traceWebsocketMessagesInheritSampling,
167167
traceWebsocketMessagesSeparateTraces,
168-
experimental
168+
experimental,
169+
resourceRenamingEnabled
169170
} = this._tracerConfig
170171

171172
const sharedConfig = {
@@ -184,7 +185,8 @@ module.exports = class PluginManager {
184185
traceWebsocketMessagesEnabled,
185186
traceWebsocketMessagesInheritSampling,
186187
traceWebsocketMessagesSeparateTraces,
187-
experimental
188+
experimental,
189+
resourceRenamingEnabled
188190
}
189191

190192
if (logInjection !== undefined) {

packages/dd-trace/src/plugins/util/url.js

Lines changed: 119 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,118 @@
22

33
const { URL } = require('url')
44

5+
const HTTP2_HEADER_AUTHORITY = ':authority'
6+
const HTTP2_HEADER_SCHEME = ':scheme'
7+
const HTTP2_HEADER_PATH = ':path'
8+
9+
const PATH_REGEX = /^(?:[a-z]+:\/\/(?:[^?/]+))?(?<path>\/[^?]*)(?:(\?).*)?$/
10+
11+
const INT_SEGMENT = /^[1-9][0-9]+$/ // Integer of size at least 2 (>=10)
12+
const INT_ID_SEGMENT = /^(?=.*[0-9].*)[0-9._-]{3,}$/ // Mixed string with digits and delimiters
13+
const HEX_SEGMENT = /^(?=.*[0-9].*)[A-Fa-f0-9]{6,}$/ // Hexadecimal digits of size at least 6 with at least one decimal digit
14+
const HEX_ID_SEGMENT = /^(?=.*[0-9].*)[A-Fa-f0-9._-]{6,}$/ // Mixed string with hex digits and delimiters
15+
const STRING_SEGMENT = /^.{20,}|.*[%&'()*+,:=@].*$/ // Long string or a string containing special characters
16+
17+
/**
18+
* Extract full URL from HTTP request
19+
* @param {import('http').IncomingMessage} req
20+
* @returns {string} Full URL
21+
*/
22+
function extractURL (req) {
23+
const headers = req.headers
24+
25+
if (req.stream) {
26+
return `${headers[HTTP2_HEADER_SCHEME]}://${headers[HTTP2_HEADER_AUTHORITY]}${headers[HTTP2_HEADER_PATH]}`
27+
}
28+
29+
const protocol = getProtocol(req)
30+
return `${protocol}://${req.headers.host}${req.originalUrl || req.url}`
31+
}
32+
33+
function getProtocol (req) {
34+
return (req.socket?.encrypted || req.connection?.encrypted) ? 'https' : 'http'
35+
}
36+
37+
/**
38+
* Obfuscate query string
39+
*
40+
* @param {object} config
41+
* @param {string} url
42+
* @returns {string} obfuscated URL
43+
*/
44+
function obfuscateQs (config, url) {
45+
const { queryStringObfuscation } = config
46+
47+
if (queryStringObfuscation === false) return url
48+
49+
const i = url.indexOf('?')
50+
if (i === -1) return url
51+
52+
const path = url.slice(0, i)
53+
if (queryStringObfuscation === true) return path
54+
55+
let qs = url.slice(i + 1)
56+
57+
qs = qs.replace(queryStringObfuscation, '<redacted>')
58+
59+
return `${path}?${qs}`
60+
}
61+
62+
/**
63+
* Extract URL path from URL using regex pattern instead of Node.js URL API because:
64+
*
65+
* - Handles edge cases like malformed URLs
66+
* - Works with relative paths
67+
* - Cross tracers compatibility
68+
*
69+
* @param {string} url
70+
* @returns {string} Url path
71+
*/
72+
function extractPathFromUrl (url) {
73+
if (!url) return '/'
74+
const match = url.match(PATH_REGEX)
75+
76+
return match?.groups?.path || '/'
77+
}
78+
79+
/**
80+
* Calculate http.endpoint from URL path
81+
*
82+
* @param {string} url
83+
* @returns {string} The normalized endpoint
84+
*/
85+
function calculateHttpEndpoint (url) {
86+
const path = extractPathFromUrl(url)
87+
88+
// Split path by '/' and filter empty elements
89+
const elements = path.split('/').filter(Boolean)
90+
91+
// Keep only first 8 non-empty elements
92+
const limitedElements = elements.slice(0, 8)
93+
94+
// Apply regex replacements to each element respecting this order
95+
const normalizedElements = limitedElements.map(element => {
96+
if (INT_SEGMENT.test(element)) return '{param:int}'
97+
98+
if (INT_ID_SEGMENT.test(element)) return '{param:int_id}'
99+
100+
if (HEX_SEGMENT.test(element)) return '{param:hex}'
101+
102+
if (HEX_ID_SEGMENT.test(element)) return '{param:hex_id}'
103+
104+
if (STRING_SEGMENT.test(element)) return '{param:str}'
105+
106+
// No match
107+
return element
108+
})
109+
110+
const endpoint = normalizedElements.length > 0
111+
? '/' + normalizedElements.join('/')
112+
: '/'
113+
114+
return endpoint
115+
}
116+
5117
function filterSensitiveInfoFromRepository (repositoryUrl) {
6118
if (!repositoryUrl) {
7119
return ''
@@ -25,4 +137,10 @@ function filterSensitiveInfoFromRepository (repositoryUrl) {
25137
}
26138
}
27139

28-
module.exports = { filterSensitiveInfoFromRepository }
140+
module.exports = {
141+
extractURL,
142+
obfuscateQs,
143+
calculateHttpEndpoint,
144+
filterSensitiveInfoFromRepository,
145+
extractPathFromUrl // test only
146+
}

0 commit comments

Comments
 (0)