discovery addDocument JSON support

Fixed addDocument to allow custom filename
added addJsonDocument method to make things easier
refactoring and lots of tests
Fixes #474

Author: nfriedly

discovery/v1.js

@@ -20,6 +20,7 @@ const util = require('util');
 const requestFactory = require('../lib/requestwrapper');
 const BaseService = require('../lib/base_service');
 const pick = require('object.pick');
+const isStream = require('isstream');
 
 /**
  *
@@ -443,15 +444,46 @@ DiscoveryV1.prototype.getCollectionFields = function(params, callback) {
   return requestFactory(parameters, callback);
 };
 
+/**
+ * Ensures a filename is always set because discovery ignores documents that do not have a filename set.
+ * @param file
+ * @private
+ */
+DiscoveryV1._ensureFilename = function(file) {
+  // no changes needed for streams created by fs.ReadStream (or similar looking streams)
+  if (isStream.isReadable(file) && file.path) {
+    return file;
+  }
+
+  // next handle request-style value/options objects
+  if (file && file.hasOwnProperty('value') && file.hasOwnProperty('options')) {
+    if (file.options.filename) {
+      return file;
+    }
+    return {
+      value: file.value,
+      options: Object.assign({ filename: '_' }, file.options)
+    };
+  }
+
+  // finally, handle all other cases by wrapping them in a request-style value/options object
+  return {
+    value: file,
+    options: {
+      filename: '_'
+    }
+  };
+};
+
 /**
  * Add a document to a collection
- * @param params
+ * @param {object} params
  * @param {String} params.environment_id environment guid for the collection
  * @param {string} params.collection_id the guid of the collection to delete
- * @param {Buffer|ReadableStream|Object} params.file a file to post (smaller than 50mb)
+ * @param {Buffer|ReadableStream|Object} params.file a file to post (smaller than 50mb). In some cases you will need to supply an object with {value: data, options: {contentType: /...'}}
  * @param {string} [params.configuration_id] config guid
- * @param {object|string} [params.metadata] JSON object with file metadata including content-type (will infer if missing)
- * @param callback
+ * @param {object} [params.metadata] JSON object with file metadata
+ * @param {function} callback
  * @return {ReadableStream|undefined}
  */
 DiscoveryV1.prototype.addDocument = function(params, callback) {
@@ -460,18 +492,9 @@ DiscoveryV1.prototype.addDocument = function(params, callback) {
   const queryParams = pick(params, ['configuration_id']);
   const formDataParams = pick(params, ['file', 'metadata']);
 
-  // if we get a buffer or object, we need to include stuff about filename for the service
+  // Discovery only accepts files that have a filename specified
   if (formDataParams.file) {
-    if (
-      typeof formDataParams.file.filename !== 'string' &&
-      !(formDataParams.file.options && typeof formDataParams.file.options.filename !== 'string') &&
-      !(formDataParams.file.path && typeof formDataParams.file.path !== 'string') &&
-      !(formDataParams.file.name && typeof formDataParams.file.name !== 'string')
-    ) {
-      const filedat = formDataParams.file;
-      // the filename used below is because the name must exist
-      formDataParams.file = { value: filedat, options: { filename: '_' } };
-    }
+    formDataParams.file = DiscoveryV1._ensureFilename(formDataParams.file);
   }
 
   if (formDataParams.metadata && typeof formDataParams.metadata === 'object') {
@@ -487,12 +510,35 @@ DiscoveryV1.prototype.addDocument = function(params, callback) {
       formData: formDataParams,
       json: true
     },
-    requiredParams: ['environment_id', 'collection_id', 'file'],
+    requiredParams: ['environment_id', 'collection_id'],
     defaultOptions: this._options
   };
   return requestFactory(parameters, callback);
 };
 
+/**
+ * Helper method, similar to addDocument except that the file param expects a JSON object
+ * @param {object} params
+ * @param {String} params.environment_id environment guid for the collection
+ * @param {string} params.collection_id the guid of the collection to delete
+ * @param {Object} params.file non-stringified JSON object
+ * @param {string} [params.configuration_id] config guid
+ * @param {object} [params.metadata] JSON object with file metadata
+ * @param {function} callback
+ * @return {ReadableStream|undefined}
+ */
+DiscoveryV1.prototype.addJsonDocument = function(params, callback) {
+  params = Object.assign({}, params, {
+    file: {
+      value: JSON.stringify(params.file),
+      options: {
+        filename: '_.json'
+      }
+    }
+  });
+  return this.addDocument(params, callback);
+};
+
 /**
  * Update or partially update a document to create or replace an existing document
  * @param params
@@ -501,7 +547,7 @@ DiscoveryV1.prototype.addDocument = function(params, callback) {
  * @param {string} params.document_id the guid of the document to update
  * @param {Buffer|ReadableStream|Object} params.file a file to post (smaller than 50mb)
  * @param {string} [params.configuration_id] config guid
- * @param {string} [params.metadata] file metadata, including content-type (will infer if missing)
+ * @param {object} [params.metadata] file metadata, including content-type (will infer if missing)
  * @param callback
  * @return {ReadableStream|undefined}
  */
@@ -511,18 +557,13 @@ DiscoveryV1.prototype.updateDocument = function(params, callback) {
   const queryParams = pick(params, ['configuration_id']);
   const formDataParams = pick(params, ['file', 'metadata']);
 
-  // if we get a buffer or object, we need to include stuff about filename for the service
+  // Discovery only accepts files that have a filename specified
   if (formDataParams.file) {
-    if (
-      typeof formDataParams.file.filename !== 'string' &&
-      !(formDataParams.file.options && typeof formDataParams.file.options.filename !== 'string') &&
-      !(formDataParams.file.path && typeof formDataParams.file.path !== 'string') &&
-      !(formDataParams.file.name && typeof formDataParams.file.name !== 'string')
-    ) {
-      const filedat = formDataParams.file;
-      // the filename used below is because the name must exist
-      formDataParams.file = { value: filedat, options: { filename: '_' } };
-    }
+    formDataParams.file = DiscoveryV1._ensureFilename(formDataParams.file);
+  }
+
+  if (formDataParams.metadata && typeof formDataParams.metadata === 'object') {
+    formDataParams.metadata = JSON.stringify(formDataParams.metadata);
   }
 
   const parameters = {

test/integration/test.discovery.js

@@ -6,6 +6,8 @@ const authHelper = require('./auth_helper.js');
 const auth = authHelper.auth;
 const describe = authHelper.describe; // this runs describe.skip if there is no auth.js file :)
 const assert = require('assert');
+const async = require('async');
+const fs = require('fs');
 
 const THIRTY_SECONDS = 30000;
 const TWO_SECONDS = 2000;
@@ -18,11 +20,18 @@ describe.skip('discovery_integration', function() {
   let discovery;
   let environment_id; // = auth.discovery.environment_id;
   let configuration_id; // = auth.discovery.configuration_id;
-  // var collection_id;
+  let collection_id;
 
   before(function() {
     nock.enableNetConnect();
-    discovery = new DiscoveryV1(auth.discovery);
+    discovery = new DiscoveryV1(
+      Object.assign({}, auth.discovery, {
+        version_date: DiscoveryV1.VERSION_DATE_2017_04_27
+      })
+    );
+    environment_id = auth.discovery.environment_id;
+    configuration_id = auth.discovery.configuration_id;
+    collection_id = auth.discovery.collection_id;
   });
 
   after(function() {
@@ -75,6 +84,7 @@ describe.skip('discovery_integration', function() {
     );
   });
 
+  // todo: delete the collection after the testing is complete
   it.skip('should createCollection()', function(done) {
     discovery.createCollection(
       {
@@ -92,15 +102,102 @@ describe.skip('discovery_integration', function() {
       }
     );
   });
-  // it('should getCollections()', function(done) {
-  //
-  // });
-  //
-  // it('should getCollection()', function(done) {
-  //
-  // });
-  //
-  // it('should deleteCollection()', function(done) {
-  //
-  // });
+
+  it('getCollections()', function(done) {
+    discovery.getCollections(
+      {
+        environment_id: environment_id,
+        configuration_id: configuration_id
+      },
+      function(err, res) {
+        assert.ifError(err);
+        assert(res);
+        // console.log(res);
+        assert(Array.isArray(res.collections));
+        done(err);
+      }
+    );
+  });
+
+  describe('add-query-delete', function() {
+    it('addDocument()', function(done) {
+      const document_obj = {
+        environment_id: environment_id,
+        collection_id: collection_id,
+        file: fs.createReadStream(__dirname + '../resources/sample-docx.docx')
+      };
+
+      discovery.addDocument(document_obj, function(err, response) {
+        assert.ifError(err);
+        assert(response.document_id);
+        done(err);
+      });
+    });
+
+    it('addJsonDocument()', function(done) {
+      const document_obj = {
+        environment_id: environment_id,
+        collection_id: collection_id,
+        file: {
+          foo: 'bar',
+          from: 'node-sdk integration test',
+          test_date: new Date().toString()
+        }
+      };
+
+      discovery.addJsonDocument(document_obj, function(err, response) {
+        assert.ifError(err);
+        assert(response.document_id);
+        done(err);
+      });
+    });
+
+    it('query()', function(done) {
+      discovery.query(
+        {
+          environment_id: environment_id,
+          collection_id: collection_id,
+          query: ''
+        },
+        function(err, res) {
+          assert.ifError(err);
+          assert(res);
+          assert.equal(typeof res.matching_results, 'number');
+          assert(Array.isArray(res.results));
+          done(err);
+        }
+      );
+    });
+
+    it('delete all documents', function(done) {
+      discovery.query(
+        {
+          environment_id: environment_id,
+          collection_id: collection_id,
+          query: ''
+        },
+        function(err, res) {
+          assert.ifError(err);
+          async.eachSeries(
+            res.results,
+            function(doc, next) {
+              // console.log('deleting ', doc);
+              discovery.deleteDocument(
+                {
+                  environment_id: environment_id,
+                  collection_id: collection_id,
+                  document_id: doc.id
+                },
+                function(err, res) {
+                  // console.log('deleted', err, res);
+                  next(err);
+                }
+              );
+            },
+            done
+          );
+        }
+      );
+    });
+  });
 });

test/unit/test.discovery.v1.js

@@ -4,6 +4,7 @@ const assert = require('assert');
 const DiscoveryV1 = require('../../discovery/v1');
 const fs = require('fs');
 const path = require('path');
+const stream = require('stream');
 
 const nock = require('nock');
 
@@ -364,6 +365,78 @@ describe('discovery-v1', function() {
 
           return result;
         }
+
+        describe('_ensureFilename()', function() {
+          it('should pass through ReadStreams unmodified', function() {
+            const src = fs.createReadStream(__dirname + '../resources/sample-docx.docx');
+            assert.equal(DiscoveryV1._ensureFilename(src), src);
+          });
+
+          it('should pass through value/options objects with a filename', function() {
+            const src = {
+              value: 'foo',
+              options: {
+                filename: 'foo.bar'
+              }
+            };
+            const actual = DiscoveryV1._ensureFilename(src);
+            assert.equal(actual, src);
+            assert.deepEqual(actual, {
+              value: 'foo',
+              options: {
+                filename: 'foo.bar'
+              }
+            });
+          });
+
+          it('should create new object/values with a filename when missing', function() {
+            const src = {
+              value: '{"foo": "bar"}',
+              options: {
+                contentType: 'application/json'
+              }
+            };
+            const actual = DiscoveryV1._ensureFilename(src);
+            assert.deepEqual(actual, {
+              value: '{"foo": "bar"}',
+              options: {
+                contentType: 'application/json',
+                filename: '_'
+              }
+            });
+            assert.notEqual(actual, src, 'it should be a new object, not a modification of the existing one');
+          });
+
+          it('should wrap buffers', function() {
+            const src = Buffer.from([1, 2, 3, 4]);
+            assert.deepEqual(DiscoveryV1._ensureFilename(src), {
+              value: src,
+              options: {
+                filename: '_'
+              }
+            });
+          });
+
+          it('should wrap strings', function() {
+            const src = 'foo';
+            assert.deepEqual(DiscoveryV1._ensureFilename(src), {
+              value: src,
+              options: {
+                filename: '_'
+              }
+            });
+          });
+
+          it('should wrap streams', function() {
+            const src = new stream.Readable();
+            assert.deepEqual(DiscoveryV1._ensureFilename(src), {
+              value: src,
+              options: {
+                filename: '_'
+              }
+            });
+          });
+        }); // end of _ensureFilename()
       });
     });
   });