Skip to content

Commit 98cd1c8

Browse files
dhedgecockdhedgecock
authored
Add experimental logs query support (#34)
1 parent bdd5c6b commit 98cd1c8

6 files changed

Lines changed: 346 additions & 5 deletions

File tree

‎src/plugin.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"type": "datasource",
44
"name": "ServiceNow Cloud Observability",
55
"id": "servicenow-cloudobservability-datasource",
6+
"logs": true,
67
"metrics": true,
78
"info": {
89
"description": "Instantly visualize ServiceNow Cloud Observability (formerly known as Lightstep) data in Grafana",

‎src/preprocessors/__snapshots__/index.test.js.snap‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Jest Snapshot v1, https://goo.gl/fbAQLP
2+
3+
exports[`preprocesses logs successfully 1`] = `
4+
Object {
5+
"fields": Array [
6+
Object {
7+
"config": Object {},
8+
"labels": undefined,
9+
"name": "time",
10+
"type": "time",
11+
"values": Array [
12+
1691409972788,
13+
1691409971908,
14+
],
15+
},
16+
Object {
17+
"config": Object {},
18+
"labels": undefined,
19+
"name": "content",
20+
"type": "string",
21+
"values": Array [
22+
"one",
23+
"two",
24+
],
25+
},
26+
Object {
27+
"config": Object {},
28+
"labels": undefined,
29+
"name": "level",
30+
"type": "string",
31+
"values": Array [
32+
"error",
33+
"info",
34+
],
35+
},
36+
Object {
37+
"config": Object {},
38+
"labels": undefined,
39+
"name": "severity",
40+
"type": "string",
41+
"values": Array [
42+
"ErrorSeverity",
43+
"InfoSeverity",
44+
],
45+
},
46+
Object {
47+
"config": Object {},
48+
"labels": undefined,
49+
"name": "http.status_code",
50+
"type": "number",
51+
"values": Array [
52+
200,
53+
undefined,
54+
],
55+
},
56+
Object {
57+
"config": Object {},
58+
"labels": undefined,
59+
"name": "large_batch",
60+
"type": "boolean",
61+
"values": Array [
62+
true,
63+
false,
64+
],
65+
},
66+
Object {
67+
"config": Object {},
68+
"labels": undefined,
69+
"name": "trace_id",
70+
"type": "string",
71+
"values": Array [
72+
"d29a3fa8fb446ec65eb691a3259a541e",
73+
"d0fa420269652931236c94bc54d2233e",
74+
],
75+
},
76+
Object {
77+
"config": Object {},
78+
"labels": undefined,
79+
"name": "customer",
80+
"type": "string",
81+
"values": Array [
82+
undefined,
83+
"hipcore",
84+
],
85+
},
86+
],
87+
"meta": Object {
88+
"preferredVisualisationType": "logs",
89+
},
90+
"name": undefined,
91+
"refId": "a",
92+
}
93+
`;

‎src/preprocessors/index.test.js‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import { preprocessData } from './index';
2+
3+
test('preprocesses logs successfully', () => {
4+
const logsDataFrame = preprocessData(
5+
{
6+
data: {
7+
attributes: {
8+
logs: [
9+
[
10+
1691409972788,
11+
{
12+
event: 'one',
13+
severity: 'ErrorSeverity',
14+
tags: {
15+
'http.status_code': 200,
16+
large_batch: true,
17+
trace_id: 'd29a3fa8fb446ec65eb691a3259a541e',
18+
},
19+
},
20+
],
21+
[
22+
1691409971908,
23+
{
24+
event: 'two',
25+
severity: 'InfoSeverity',
26+
tags: {
27+
customer: 'hipcore',
28+
large_batch: false,
29+
trace_id: 'd0fa420269652931236c94bc54d2233e',
30+
},
31+
},
32+
],
33+
],
34+
},
35+
},
36+
},
37+
{ refId: 'a' },
38+
''
39+
);
40+
41+
expect(logsDataFrame.toJSON()).toMatchSnapshot();
42+
});

‎src/preprocessors/index.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,19 @@
11
import { DataFrame } from '@grafana/data';
2-
import { LightstepQuery, QueryRes } from '../types';
2+
import { LightstepQuery, QueryLogsRes, QueryRes } from '../types';
3+
import { preprocessLogs } from './logs';
34
import { preprocessTimeseries } from './timeseries';
45

56
/**
67
* Preprocessor entry point routes responses to the correct preprocessor
78
*/
89
export function preprocessData(res: QueryRes, query: LightstepQuery, notebookURL: string): DataFrame {
9-
return preprocessTimeseries(res, query, notebookURL);
10+
if (isLogsRes(res)) {
11+
return preprocessLogs(res, query);
12+
} else {
13+
return preprocessTimeseries(res, query, notebookURL);
14+
}
15+
}
16+
17+
function isLogsRes(res: QueryRes): res is QueryLogsRes {
18+
return 'logs' in res.data.attributes;
1019
}

‎src/preprocessors/logs.ts‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
import { MutableDataFrame, FieldType, LogLevel } from '@grafana/data';
2+
import { LightstepQuery, QueryLogsRes } from 'types';
3+
4+
/**
5+
* Response pre-processor that converts the LS response data into Grafana wide
6+
* data frames, eg:
7+
*
8+
* **Lightstep API response shape**
9+
* ```json
10+
* {
11+
* "query": {
12+
* "data": {
13+
* "attributes": {
14+
* "logs": [
15+
* [1691409972788, { event: "one", severity: "ErrorSeverity", tags: { ... } }],
16+
* [1691409971908, { event: "two", severity: "InfoSeverity", tags: { ... } }]
17+
* ]
18+
* }
19+
* }
20+
* }
21+
* }
22+
* ```
23+
*
24+
* **Grafana DataFrame shape**
25+
* ```js
26+
* [
27+
* { name: 'time', type: FieldType.time, values: [0, 1, 2] },
28+
* { name: 'content', type: FieldType.string, values: ["one", "two"] },
29+
* { name: 'level', type: FieldType.string, values: ["error", "info"] }
30+
* ]
31+
* ```
32+
*/
33+
export function preprocessLogs(res: QueryLogsRes, query: LightstepQuery) {
34+
/** Map of "detected fields" aka log tags that have been added to the data frame */
35+
const detectedFields = new Map<string, boolean>();
36+
37+
const frame = new MutableDataFrame({
38+
refId: query.refId,
39+
meta: {
40+
preferredVisualisationType: 'logs',
41+
},
42+
fields: [
43+
{ name: 'time', type: FieldType.time },
44+
{ name: 'content', type: FieldType.string },
45+
{ name: 'level', type: FieldType.string },
46+
{ name: 'severity', type: FieldType.string },
47+
],
48+
});
49+
50+
res.data.attributes.logs.forEach(([timestamp, log]) => {
51+
let tags = {};
52+
if ('tags' in log && log.tags !== null && typeof log.tags === 'object') {
53+
tags = log.tags;
54+
Object.entries(log.tags).forEach(([key, value]) => {
55+
// Add every log tag to the set of detected fields once
56+
if (!detectedFields.has(key)) {
57+
detectedFields.set(key, true);
58+
frame.addField({
59+
name: key,
60+
type: getFieldTypeForValue(value),
61+
});
62+
}
63+
});
64+
}
65+
66+
frame.add({
67+
time: timestamp,
68+
content: log.event,
69+
level: getLevel(log),
70+
severity: log.severity,
71+
...tags,
72+
});
73+
});
74+
75+
return frame;
76+
}
77+
78+
// --------------------------------------------------------
79+
// INTERNAL
80+
81+
/** Best effort mapping fn to detect the correct "level" for a log. This
82+
* determines the color coding of the log line in the panel. */
83+
function getLevel(log: Record<string, unknown>): LogLevel {
84+
let logLevel: unknown = log.severityNumber ?? log.severityText ?? log.level ?? log.severity ?? 0;
85+
86+
if (typeof logLevel === 'string') {
87+
logLevel = logLevel.toLowerCase();
88+
}
89+
90+
// --- Grafana natively support levels
91+
if (String(logLevel) in LogLevel) {
92+
return LogLevel[logLevel as LogLevel];
93+
}
94+
95+
switch (logLevel) {
96+
// --- OTel SeverityNumber
97+
case 1:
98+
case 2:
99+
case 3:
100+
case 4:
101+
return LogLevel.trace;
102+
case 5:
103+
case 6:
104+
case 7:
105+
case 8:
106+
return LogLevel.debug;
107+
case 9:
108+
case 10:
109+
case 11:
110+
case 12:
111+
return LogLevel.info;
112+
case 13:
113+
case 14:
114+
case 15:
115+
case 16:
116+
return LogLevel.warning;
117+
case 17:
118+
case 18:
119+
case 19:
120+
case 20:
121+
return LogLevel.error;
122+
case 21:
123+
case 22:
124+
case 23:
125+
case 24:
126+
return LogLevel.critical;
127+
128+
// --- OTel SeverityText
129+
case 'trace':
130+
return LogLevel.trace;
131+
case 'debug':
132+
return LogLevel.debug;
133+
case 'info':
134+
return LogLevel.info;
135+
case 'warn':
136+
return LogLevel.warning;
137+
case 'error':
138+
return LogLevel.error;
139+
case 'fatal':
140+
return LogLevel.critical;
141+
142+
// --- Misc common levels
143+
case 'verboseseverity':
144+
return LogLevel.debug;
145+
case 'infoseverity':
146+
return LogLevel.info;
147+
case 'warningseverity':
148+
return LogLevel.warning;
149+
case 'errorseverity':
150+
return LogLevel.error;
151+
case 'immediateseverity':
152+
return LogLevel.critical;
153+
case 'fatalseverity':
154+
return LogLevel.critical;
155+
default:
156+
return LogLevel.unknown;
157+
}
158+
}
159+
160+
/** Detects the correct FieldType enum for log fields */
161+
function getFieldTypeForValue(value: unknown): FieldType {
162+
switch (typeof value) {
163+
case 'string':
164+
return FieldType.string;
165+
case 'number':
166+
return FieldType.number;
167+
case 'boolean':
168+
return FieldType.boolean;
169+
default:
170+
return FieldType.other;
171+
}
172+
}

‎src/types.ts‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,17 +33,41 @@ export interface LightstepSecureJsonData {
3333
// --------------------------------------------------------
3434
// DATA SHAPES
3535

36-
export type QueryRes = QueryTimeseriesRes;
36+
export type QueryRes = QueryLogsRes | QueryTimeseriesRes;
3737

38+
/**
39+
* Response shape for a timeseries query
40+
* @example metric requests | rate | group_by [customer], sum
41+
*/
3842
export interface QueryTimeseriesRes {
3943
data: {
4044
attributes: {
4145
series: Array<{
4246
'group-labels': string[];
43-
points: Point[];
47+
/** Array of timestamp, value tuples */
48+
points: Array<[timestamp: number, value: number]>;
4449
}>;
4550
};
4651
};
4752
}
4853

49-
type Point = [timestamp: number, value: number];
54+
/**
55+
* Response shape for a logs query.
56+
* @example logs | filter tags.customer == "name"
57+
*/
58+
export interface QueryLogsRes {
59+
data: {
60+
attributes: {
61+
/** Array of timestamp, event tuples */
62+
logs: Array<
63+
[
64+
timestamp: number,
65+
/** nb: log fields do not have a formal schema */
66+
log: {
67+
[key: string]: unknown;
68+
}
69+
]
70+
>;
71+
};
72+
};
73+
}

0 commit comments

Comments
 (0)