Better detection of log fields (#38)

Author: dhedgecock

src/preprocessors/__snapshots__/index.test.js.snap

@@ -46,41 +46,41 @@ Object {
     Object {
       "config": Object {},
       "labels": undefined,
-      "name": "http.status_code",
-      "type": "number",
+      "name": "tags",
+      "type": "other",
       "values": Array [
-        200,
-        undefined,
+        "{\\"http.status_code\\":200,\\"large_batch\\":true,\\"trace_id\\":\\"d29a3fa8fb446ec65eb691a3259a541e\\"}",
+        "{\\"customer\\":\\"hipcore\\",\\"large_batch\\":false,\\"trace_id\\":\\"d0fa420269652931236c94bc54d2233e\\"}",
       ],
     },
     Object {
       "config": Object {},
       "labels": undefined,
-      "name": "large_batch",
-      "type": "boolean",
+      "name": "k8s_environment",
+      "type": "string",
       "values": Array [
-        true,
-        false,
+        undefined,
+        "production",
       ],
     },
     Object {
       "config": Object {},
       "labels": undefined,
-      "name": "trace_id",
+      "name": "k8s_namespace",
       "type": "string",
       "values": Array [
-        "d29a3fa8fb446ec65eb691a3259a541e",
-        "d0fa420269652931236c94bc54d2233e",
+        undefined,
+        "default",
       ],
     },
     Object {
       "config": Object {},
       "labels": undefined,
-      "name": "customer",
+      "name": "k8s_pod",
       "type": "string",
       "values": Array [
         undefined,
-        "hipcore",
+        "hipcore-pod",
       ],
     },
   ],

src/preprocessors/index.test.js

@@ -28,6 +28,9 @@ test('preprocesses logs successfully', () => {
                   large_batch: false,
                   trace_id: 'd0fa420269652931236c94bc54d2233e',
                 },
+                k8s_environment: 'production',
+                k8s_namespace: 'default',
+                k8s_pod: 'hipcore-pod',
               },
             ],
           ],

src/preprocessors/logs.ts

@@ -40,35 +40,38 @@ export function preprocessLogs(res: QueryLogsRes, query: LightstepQuery) {
       preferredVisualisationType: 'logs',
     },
     fields: [
+      // time, content, and level are required by the Grafana panel, all other
+      // fields are considered custom fields
       { name: 'time', type: FieldType.time },
       { name: 'content', type: FieldType.string },
       { name: 'level', type: FieldType.string },
-      { name: 'severity', type: FieldType.string },
     ],
   });
 
   res.data.attributes.logs.forEach(([timestamp, log]) => {
-    let tags = {};
-    if ('tags' in log && log.tags !== null && typeof log.tags === 'object') {
-      tags = log.tags;
-      Object.entries(log.tags).forEach(([key, value]) => {
-        // Add every log tag to the set of detected fields once
-        if (!detectedFields.has(key)) {
-          detectedFields.set(key, true);
-          frame.addField({
-            name: key,
-            type: getFieldTypeForValue(value),
-          });
-        }
-      });
-    }
+    const { body, Body, event, ...customFields } = log;
+
+    const transformedFields: Record<string, unknown> = {};
+    Object.entries(customFields).forEach(([key, value]) => {
+      // Add every field to the set of detected fields once
+      if (!detectedFields.has(key)) {
+        detectedFields.set(key, true);
+        frame.addField({
+          name: key,
+          type: getFieldTypeForValue(value),
+        });
+      }
+
+      // Complex fields (objects, arrays) aren't supported by Grafana, so we
+      // need to stringify them
+      transformedFields[key] = typeof value === 'object' ? JSON.stringify(value) : value;
+    });
 
     frame.add({
       time: timestamp,
-      content: log.event,
+      content: body ?? Body ?? event ?? '',
       level: getLevel(log),
-      severity: log.severity,
-      ...tags,
+      ...transformedFields,
     });
   });