Add experimental logs query support

src/plugin.json

@@ -3,6 +3,7 @@
   "type": "datasource",
   "name": "ServiceNow Cloud Observability",
   "id": "servicenow-cloudobservability-datasource",
+  "logs": true,
   "metrics": true,
   "info": {
     "description": "Instantly visualize ServiceNow Cloud Observability (formerly known as Lightstep) data in Grafana",

src/preprocessors/__snapshots__/index.test.js.snap

@@ -0,0 +1,93 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`preprocesses logs successfully 1`] = `
+Object {
+  "fields": Array [
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "time",
+      "type": "time",
+      "values": Array [
+        1691409972788,
+        1691409971908,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "content",
+      "type": "string",
+      "values": Array [
+        "one",
+        "two",
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "level",
+      "type": "string",
+      "values": Array [
+        "error",
+        "info",
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "severity",
+      "type": "string",
+      "values": Array [
+        "ErrorSeverity",
+        "InfoSeverity",
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "http.status_code",
+      "type": "number",
+      "values": Array [
+        200,
+        undefined,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "large_batch",
+      "type": "boolean",
+      "values": Array [
+        true,
+        false,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "trace_id",
+      "type": "string",
+      "values": Array [
+        "d29a3fa8fb446ec65eb691a3259a541e",
+        "d0fa420269652931236c94bc54d2233e",
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "customer",
+      "type": "string",
+      "values": Array [
+        undefined,
+        "hipcore",
+      ],
+    },
+  ],
+  "meta": Object {
+    "preferredVisualisationType": "logs",
+  },
+  "name": undefined,
+  "refId": "a",
+}
+`;

src/preprocessors/index.test.js

@@ -0,0 +1,42 @@
+import { preprocessData } from './index';
+
+test('preprocesses logs successfully', () => {
+  const logsDataFrame = preprocessData(
+    {
+      data: {
+        attributes: {
+          logs: [
+            [
+              1691409972788,
+              {
+                event: 'one',
+                severity: 'ErrorSeverity',
+                tags: {
+                  'http.status_code': 200,
+                  large_batch: true,
+                  trace_id: 'd29a3fa8fb446ec65eb691a3259a541e',
+                },
+              },
+            ],
+            [
+              1691409971908,
+              {
+                event: 'two',
+                severity: 'InfoSeverity',
+                tags: {
+                  customer: 'hipcore',
+                  large_batch: false,
+                  trace_id: 'd0fa420269652931236c94bc54d2233e',
+                },
+              },
+            ],
+          ],
+        },
+      },
+    },
+    { refId: 'a' },
+    ''
+  );
+
+  expect(logsDataFrame.toJSON()).toMatchSnapshot();
+});

src/preprocessors/index.ts

@@ -1,10 +1,19 @@
 import { DataFrame } from '@grafana/data';
-import { LightstepQuery, QueryRes } from '../types';
+import { LightstepQuery, QueryLogsRes, QueryRes } from '../types';
+import { preprocessLogs } from './logs';
 import { preprocessTimeseries } from './timeseries';
 
 /**
  * Preprocessor entry point routes responses to the correct preprocessor
  */
 export function preprocessData(res: QueryRes, query: LightstepQuery, notebookURL: string): DataFrame {
-  return preprocessTimeseries(res, query, notebookURL);
+  if (isLogsRes(res)) {
+    return preprocessLogs(res, query);
+  } else {
+    return preprocessTimeseries(res, query, notebookURL);
+  }
+}
+
+function isLogsRes(res: QueryRes): res is QueryLogsRes {
+  return 'logs' in res.data.attributes;
 }

src/preprocessors/logs.ts

@@ -0,0 +1,172 @@
+import { MutableDataFrame, FieldType, LogLevel } from '@grafana/data';
+import { LightstepQuery, QueryLogsRes } from 'types';
+
+/**
+ * Response pre-processor that converts the LS response data into Grafana wide
+ * data frames, eg:
+ *
+ * **Lightstep API response shape**
+ * ```json
+ * {
+ *   "query": {
+ *     "data": {
+ *       "attributes": {
+ *         "logs": [
+ *           [1691409972788, { event: "one", severity: "ErrorSeverity", tags: { ... } }],
+ *           [1691409971908, { event: "two", severity: "InfoSeverity", tags: { ... } }]
+ *         ]
+ *       }
+ *     }
+ *   }
+ * }
+ * ```
+ *
+ * **Grafana DataFrame shape**
+ * ```js
+ * [
+ *   { name: 'time', type: FieldType.time, values: [0, 1, 2] },
+ *   { name: 'content', type: FieldType.string, values: ["one", "two"] },
+ *   { name: 'level', type: FieldType.string, values: ["error", "info"] }
+ * ]
+ * ```
+ */
+export function preprocessLogs(res: QueryLogsRes, query: LightstepQuery) {
+  /** Map of "detected fields" aka log tags that have been added to the data frame */
+  const detectedFields = new Map<string, boolean>();
+
+  const frame = new MutableDataFrame({
+    refId: query.refId,
+    meta: {
+      preferredVisualisationType: 'logs',
+    },
+    fields: [
+      { name: 'time', type: FieldType.time },
+      { name: 'content', type: FieldType.string },
+      { name: 'level', type: FieldType.string },
+      { name: 'severity', type: FieldType.string },
+    ],
+  });
+
+  res.data.attributes.logs.forEach(([timestamp, log]) => {
+    let tags = {};
+    if ('tags' in log && log.tags !== null && typeof log.tags === 'object') {
+      tags = log.tags;
+      Object.entries(log.tags).forEach(([key, value]) => {
+        // Add every log tag to the set of detected fields once
+        if (!detectedFields.has(key)) {
+          detectedFields.set(key, true);
+          frame.addField({
+            name: key,
+            type: getFieldTypeForValue(value),
+          });
+        }
+      });
+    }
+
+    frame.add({
+      time: timestamp,
+      content: log.event,
+      level: getLevel(log),
+      severity: log.severity,
+      ...tags,
+    });
+  });
+
+  return frame;
+}
+
+// --------------------------------------------------------
+// INTERNAL
+
+/** Best effort mapping fn to detect the correct "level" for a log. This
+ * determines the color coding of the log line in the panel. */
+function getLevel(log: Record<string, unknown>): LogLevel {
+  let logLevel: unknown = log.severityNumber ?? log.severityText ?? log.level ?? log.severity ?? 0;
+
+  if (typeof logLevel === 'string') {
+    logLevel = logLevel.toLowerCase();
+  }
+
+  // --- Grafana natively support levels
+  if (String(logLevel) in LogLevel) {
+    return LogLevel[logLevel as LogLevel];
+  }
+
+  switch (logLevel) {
+    // --- OTel SeverityNumber
+    case 1:
+    case 2:
+    case 3:
+    case 4:
+      return LogLevel.trace;
+    case 5:
+    case 6:
+    case 7:
+    case 8:
+      return LogLevel.debug;
+    case 9:
+    case 10:
+    case 11:
+    case 12:
+      return LogLevel.info;
+    case 13:
+    case 14:
+    case 15:
+    case 16:
+      return LogLevel.warning;
+    case 17:
+    case 18:
+    case 19:
+    case 20:
+      return LogLevel.error;
+    case 21:
+    case 22:
+    case 23:
+    case 24:
+      return LogLevel.critical;
+
+    // --- OTel SeverityText
+    case 'trace':
+      return LogLevel.trace;
+    case 'debug':
+      return LogLevel.debug;
+    case 'info':
+      return LogLevel.info;
+    case 'warn':
+      return LogLevel.warning;
+    case 'error':
+      return LogLevel.error;
+    case 'fatal':
+      return LogLevel.critical;
+
+    // --- Misc common levels
+    case 'verboseseverity':
+      return LogLevel.debug;
+    case 'infoseverity':
+      return LogLevel.info;
+    case 'warningseverity':
+      return LogLevel.warning;
+    case 'errorseverity':
+      return LogLevel.error;
+    case 'immediateseverity':
+      return LogLevel.critical;
+    case 'fatalseverity':
+      return LogLevel.critical;
+    default:
+      return LogLevel.unknown;
+  }
+}
+
+/** Detects the correct FieldType enum for log fields */
+function getFieldTypeForValue(value: unknown): FieldType {
+  switch (typeof value) {
+    case 'string':
+      return FieldType.string;
+    case 'number':
+      return FieldType.number;
+    case 'boolean':
+      return FieldType.boolean;
+    default:
+      return FieldType.other;
+  }
+}

src/types.ts

@@ -33,17 +33,41 @@ export interface LightstepSecureJsonData {
 // --------------------------------------------------------
 // DATA SHAPES
 
-export type QueryRes = QueryTimeseriesRes;
+export type QueryRes = QueryLogsRes | QueryTimeseriesRes;
 
+/**
+ * Response shape for a timeseries query
+ * @example metric requests | rate | group_by [customer], sum
+ */
 export interface QueryTimeseriesRes {
   data: {
     attributes: {
       series: Array<{
         'group-labels': string[];
-        points: Point[];
+        /** Array of timestamp, value tuples */
+        points: Array<[timestamp: number, value: number]>;
       }>;
     };
   };
 }
 
-type Point = [timestamp: number, value: number];
+/**
+ * Response shape for a logs query.
+ * @example logs | filter tags.customer == "name"
+ */
+export interface QueryLogsRes {
+  data: {
+    attributes: {
+      /** Array of timestamp, event tuples */
+      logs: Array<
+        [
+          timestamp: number,
+          /** nb: log fields do not have a formal schema */
+          log: {
+            [key: string]: unknown;
+          }
+        ]
+      >;
+    };
+  };
+}