From f70cded634b6faa9d4df09b5ec5c39f4fb7529a2 Mon Sep 17 00:00:00 2001
From: Benjamin Gamard <b.gamard@sismics.com>
Date: Thu, 1 Mar 2018 12:28:29 +0100
Subject: [PATCH] file importer

---
 .gitignore              |   1 +
 README.md               |   1 +
 docs-importer/README.md |  22 ++++++++
 docs-importer/main.js   | 108 +++++++++++++++++++++++++++++++++++++---
 4 files changed, 124 insertions(+), 8 deletions(-)
 create mode 100644 docs-importer/README.md

diff --git a/.gitignore b/.gitignore
index 5eb68f06..4f52e0fa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,4 @@
 /.project
 *.iml
 node_modules
+import_test
diff --git a/README.md b/README.md
index 5e6e25d5..d3502047 100644
--- a/README.md
+++ b/README.md
@@ -47,6 +47,7 @@ Features
 - Document sharing by URL
 - RESTful Web API
 - Fully featured Android client
+- [Mass files importer](https://github.com/sismics/docs/tree/master/docs-importer) (single or scan mode) ![New!](https://www.sismics.com/public/img/new.png)
 - Tested to 100k documents
 
 Download
diff --git a/docs-importer/README.md b/docs-importer/README.md
new file mode 100644
index 00000000..ff91b6f9
--- /dev/null
+++ b/docs-importer/README.md
@@ -0,0 +1,22 @@
+File Importer
+=============
+
+This tool can be used to do a single import of files or to periodically scan for files in an input folder.
+
+Requirements
+------------
+- NodeJS 9
+- NPM
+
+Usage
+-----
+```console
+npm install
+node main.js
+```
+
+A wizard will ask you for the import configuration and write it in `~/.config/preferences/com.sismics.docs.importer.pref`
+
+Daemon mode
+-----------
+The daemon mode scan the input directory every 30 seconds for new files. Once a file is found and imported, it is **deleted**.
\ No newline at end of file
diff --git a/docs-importer/main.js b/docs-importer/main.js
index 3d8c8f22..1a358f98 100644
--- a/docs-importer/main.js
+++ b/docs-importer/main.js
@@ -8,11 +8,12 @@ const fs = require('fs');
 const request = require('request').defaults({
   jar: true
 });
-// request.debug = true;
 
 // Load preferences
 const prefs = new preferences('com.sismics.docs.importer',{
-  importer: {}
+  importer: {
+    daemon: false
+  }
 }, {
   encrypt: false,
   format: 'yaml'
@@ -91,7 +92,7 @@ const askCredentials = () => {
         remember: true
       }
     }, function (error, response) {
-      if (!response || response.statusCode !== 200) {
+      if (error || !response || response.statusCode !== 200) {
         spinner.fail('Username or password incorrect');
         askCredentials();
         return;
@@ -123,16 +124,107 @@ const askPath = () => {
       text: 'Checking import path',
       spinner: 'flips'
     }).start();
-    fs.lstat(answers.path, (err, stats) => {
-      if (err || !stats.isDirectory()) {
+    fs.lstat(answers.path, (error, stats) => {
+      if (error || !stats.isDirectory()) {
         spinner.fail('Please enter a valid directory path');
+        askPath();
         return;
       }
 
-      recursive(answers.path, function (err, files) {
-        spinner.succeed(files.length + ' files in this directory');
-        // TODO Then?
+      fs.access(answers.path, fs.W_OK | fs.R_OK, (error) => {
+        if (error) {
+          spinner.fail('This directory is not writable');
+          askPath();
+          return;
+        }
+
+        recursive(answers.path, function (error, files) {
+          spinner.succeed(files.length + ' files in this directory');
+          askDaemon();
+        });
       });
     });
   });
 };
+
+// Ask for daemon mode
+const askDaemon = () => {
+  console.log('');
+
+  inquirer.prompt([
+    {
+      type: 'confirm',
+      name: 'daemon',
+      message: 'Do you want to run the importer in daemon mode (it will poll the input directory for new files, import and delete them)?',
+      default: prefs.importer.daemon === true
+    }
+  ]).then(answers => {
+    // Save daemon
+    prefs.importer.daemon = answers.daemon;
+
+    start();
+  });
+};
+
+// Start the import
+const start = () => {
+  if (prefs.importer.daemon) {
+    console.log('\nPolling the input folder for new files...');
+
+    let resolve = () => {
+      importFiles(true, () => {
+        setTimeout(resolve, 30000);
+      });
+    };
+    resolve();
+  } else {
+    importFiles(false, () => {});
+  }
+};
+
+// Import the files
+const importFiles = (remove, filesImported) => {
+  recursive(prefs.importer.path, function (error, files) {
+    if (files.length === 0) {
+      filesImported();
+      return;
+    }
+
+    let index = 0;
+    let resolve = () => {
+      const file = files[index++];
+      if (file) {
+        importFile(file, remove, resolve);
+      } else {
+        filesImported();
+      }
+    };
+    resolve();
+  });
+};
+
+// Import a file
+const importFile = (file, remove, resolve) => {
+  const spinner = ora({
+    text: 'Importing: ' + file,
+    spinner: 'flips'
+  }).start();
+
+  request.put({
+    url: prefs.importer.baseUrl + '/api/file',
+    formData: {
+      file: fs.createReadStream(file)
+    }
+  }, function (error, response) {
+    if (error || !response || response.statusCode !== 200) {
+      spinner.fail('Upload failed for ' + file + ': ' + error);
+      resolve();
+      return;
+    }
+    spinner.succeed('Upload successful for ' + file);
+    if (remove) {
+      fs.unlinkSync(file);
+    }
+    resolve();
+  });
+};
\ No newline at end of file