'use strict'; const recursive = require('recursive-readdir'); const minimatch = require("minimatch"); const ora = require('ora'); const inquirer = require('inquirer'); const preferences = require('preferences'); const fs = require('fs'); const argv = require('minimist')(process.argv); const _ = require('underscore'); const request = require('request').defaults({ jar: true }); const qs = require('querystring'); // Load preferences const prefs = new preferences('com.sismics.docs.importer',{ importer: { daemon: false } }, { encrypt: false, format: 'yaml' }); // Welcome message console.log('Teedy Importer 1.9, https://teedy.io' + '\n\n' + 'This program let you import files from your system to Teedy' + '\n'); // Ask for the base URL const askBaseUrl = () => { inquirer.prompt([ { type: 'input', name: 'baseUrl', message: 'What is the base URL of your Teedy? (eg. https://teedy.mycompany.com)', default: prefs.importer.baseUrl } ]).then(answers => { // Save base URL prefs.importer.baseUrl = answers.baseUrl; // Test base URL const spinner = ora({ text: 'Checking connection to Teedy', spinner: 'flips' }).start(); request(answers.baseUrl + '/api/app', function (error, response) { if (!response || response.statusCode !== 200) { spinner.fail('Connection to Teedy failed: ' + error); askBaseUrl(); return; } spinner.succeed('Connection OK'); askCredentials(); }); }); }; // Ask for credentials const askCredentials = () => { console.log(''); inquirer.prompt([ { type: 'input', name: 'username', message: 'Account\'s username?', default: prefs.importer.username }, { type: 'password', name: 'password', message: 'Account\'s password?', default: prefs.importer.password } ]).then(answers => { // Save credentials prefs.importer.username = answers.username; prefs.importer.password = answers.password; // Test credentials const spinner = ora({ text: 'Checking connection to Teedy', spinner: 'flips' }).start(); request.post({ url: prefs.importer.baseUrl + '/api/user/login', form: { username: answers.username, password: answers.password, remember: true } }, function (error, response) { if (error || !response || response.statusCode !== 200) { spinner.fail('Username or password incorrect'); askCredentials(); return; } spinner.succeed('Authentication OK'); askPath(); }); }); }; // Ask for the path const askPath = () => { console.log(''); inquirer.prompt([ { type: 'input', name: 'path', message: 'What is the folder path you want to import?', default: prefs.importer.path } ]).then(answers => { // Save path prefs.importer.path = answers.path; // Test path const spinner = ora({ text: 'Checking import path', spinner: 'flips' }).start(); fs.lstat(answers.path, (error, stats) => { if (error || !stats.isDirectory()) { spinner.fail('Please enter a valid directory path'); askPath(); return; } fs.access(answers.path, fs.W_OK | fs.R_OK, (error) => { if (error) { spinner.fail('This directory is not writable'); askPath(); return; } recursive(answers.path, function (error, files) { spinner.succeed(files.length + ' files in this directory'); askFileFilter(); }); }); }); }); }; // Ask for the file filter const askFileFilter = () => { console.log(''); inquirer.prompt([ { type: 'input', name: 'fileFilter', message: 'What pattern do you want to use to match files? (eg. *.+(pdf|txt|jpg))', default: prefs.importer.fileFilter || "*" } ]).then(answers => { // Save fileFilter prefs.importer.fileFilter = answers.fileFilter; askTag(); }); }; // Ask for the tag to add const askTag = () => { console.log(''); // Load tags const spinner = ora({ text: 'Loading tags', spinner: 'flips' }).start(); request.get({ url: prefs.importer.baseUrl + '/api/tag/list', }, function (error, response, body) { if (error || !response || response.statusCode !== 200) { spinner.fail('Error loading tags'); askTag(); return; } spinner.succeed('Tags loaded'); const tags = JSON.parse(body).tags; const defaultTag = _.findWhere(tags, { id: prefs.importer.tag }); const defaultTagName = defaultTag ? defaultTag.name : 'No tag'; inquirer.prompt([ { type: 'list', name: 'tag', message: 'Which tag to add to all imported documents?', default: defaultTagName, choices: [ 'No tag' ].concat(_.pluck(tags, 'name')) } ]).then(answers => { // Save tag prefs.importer.tag = answers.tag === 'No tag' ? '' : _.findWhere(tags, { name: answers.tag }).id; askAddTag(); }); }); }; const askAddTag = () => { console.log(''); inquirer.prompt([ { type: 'confirm', name: 'addtags', message: 'Do you want to add tags from the filename given with # ?', default: prefs.importer.addtags === true } ]).then(answers => { // Save daemon prefs.importer.addtags = answers.addtags; // Save all preferences in case the program is sig-killed askLang(); }); } const askLang = () => { console.log(''); // Load tags const spinner = ora({ text: 'Loading default language', spinner: 'flips' }).start(); request.get({ url: prefs.importer.baseUrl + '/api/app', }, function (error, response, body) { if (error || !response || response.statusCode !== 200) { spinner.fail('Connection to Teedy failed: ' + error); askLang(); return; } spinner.succeed('Language loaded'); const defaultLang = prefs.importer.lang ? prefs.importer.lang : JSON.parse(body).default_language; inquirer.prompt([ { type: 'input', name: 'lang', message: 'Which should be the default language of the document?', default: defaultLang } ]).then(answers => { // Save tag prefs.importer.lang = answers.lang askCopyFolder(); }); }); }; const askCopyFolder = () => { console.log(''); inquirer.prompt([ { type: 'input', name: 'copyFolder', message: 'Enter a path to copy files before they are deleted or leave empty to disable. The path must end with a \'/\' on MacOS and Linux or with a \'\\\' on Windows. Entering \'undefined\' will disable this again after setting the folder.', default: prefs.importer.copyFolder } ]).then(answers => { // Save path prefs.importer.copyFolder = answers.copyFolder=='undefined' ? '' : answers.copyFolder; if (prefs.importer.copyFolder) { // Test path const spinner = ora({ text: 'Checking copy folder path', spinner: 'flips' }).start(); fs.lstat(answers.copyFolder, (error, stats) => { if (error || !stats.isDirectory()) { spinner.fail('Please enter a valid directory path'); askCopyFolder(); return; } fs.access(answers.copyFolder, fs.W_OK | fs.R_OK, (error) => { if (error) { spinner.fail('This directory is not writable'); askCopyFolder(); return; } spinner.succeed('Copy folder set!'); askDaemon(); }); }); } else {askDaemon();} }); }; // Ask for daemon mode const askDaemon = () => { console.log(''); inquirer.prompt([ { type: 'confirm', name: 'daemon', message: 'Do you want to run the importer in daemon mode (it will poll the input directory for new files, import and delete them)?', default: prefs.importer.daemon === true } ]).then(answers => { // Save daemon prefs.importer.daemon = answers.daemon; // Save all preferences in case the program is sig-killed prefs.save(); start(); }); }; // Start the importer const start = () => { request.post({ url: prefs.importer.baseUrl + '/api/user/login', form: { username: prefs.importer.username, password: prefs.importer.password, remember: true } }, function (error, response) { if (error || !response || response.statusCode !== 200) { console.error('\nUsername or password incorrect'); return; } // Start the actual import if (prefs.importer.daemon) { console.log('\nPolling the input folder for new files...'); let resolve = () => { importFiles(true, () => { setTimeout(resolve, 30000); }); }; resolve(); } else { importFiles(false, () => {}); } }); }; // Import the files const importFiles = (remove, filesImported) => { recursive(prefs.importer.path, function (error, files) { files = files.filter(minimatch.filter(prefs.importer.fileFilter || '*', { matchBase: true })); if (files.length === 0) { filesImported(); return; } let index = 0; let resolve = () => { const file = files[index++]; if (file) { importFile(file, remove, resolve); } else { filesImported(); } }; resolve(); }); }; // Import a file const importFile = (file, remove, resolve) => { const spinner = ora({ text: 'Importing: ' + file, spinner: 'flips' }).start(); // Remove path of file let filename = file.replace(/^.*[\\\/]/, ''); // Get Tags given as hashtags from filename let taglist = filename.match(/#[^\s:#]+/mg); taglist = taglist ? taglist.map(s => s.substr(1)) : []; // Get available tags and UUIDs from server request.get({ url: prefs.importer.baseUrl + '/api/tag/list', }, function (error, response, body) { if (error || !response || response.statusCode !== 200) { spinner.fail('Error loading tags'); return; } let tagsarray = {}; for (let l of JSON.parse(body).tags) { tagsarray[l.name] = l.id; } // Intersect tags from filename with existing tags on server let foundtags = []; for (let j of taglist) { // If the tag is last in the filename it could include a file extension and would not be recognized if (j.includes('.') && !tagsarray.hasOwnProperty(j) && !foundtags.includes(tagsarray[j])) { while (j.includes('.') && !tagsarray.hasOwnProperty(j)) { j = j.replace(/\.[^.]*$/,''); } } if (tagsarray.hasOwnProperty(j) && !foundtags.includes(tagsarray[j])) { foundtags.push(tagsarray[j]); filename = filename.split('#'+j).join(''); } } if (prefs.importer.tag !== '' && !foundtags.includes(prefs.importer.tag)){ foundtags.push(prefs.importer.tag); } let data = {} if (prefs.importer.addtags) { data = { title: prefs.importer.addtags ? filename : file.replace(/^.*[\\\/]/, '').substring(0, 100), language: prefs.importer.lang || 'eng', tags: foundtags } } else { data = { title: prefs.importer.addtags ? filename : file.replace(/^.*[\\\/]/, '').substring(0, 100), language: prefs.importer.lang || 'eng', tags: prefs.importer.tag === '' ? undefined : prefs.importer.tag } } // Create document request.put({ url: prefs.importer.baseUrl + '/api/document', form: qs.stringify(data) }, function (error, response, body) { if (error || !response || response.statusCode !== 200) { spinner.fail('Upload failed for ' + file + ': ' + error); resolve(); return; } // Upload file request.put({ url: prefs.importer.baseUrl + '/api/file', formData: { id: JSON.parse(body).id, file: fs.createReadStream(file) } }, function (error, response) { if (error || !response || response.statusCode !== 200) { spinner.fail('Upload failed for ' + file + ': ' + error); resolve(); return; } spinner.succeed('Upload successful for ' + file); if (remove) { if (prefs.importer.copyFolder) { fs.copyFileSync(file, prefs.importer.copyFolder + file.replace(/^.*[\\\/]/, '')); fs.unlinkSync(file); } else {fs.unlinkSync(file);} } resolve(); }); }); }); }; // Entrypoint: daemon mode or wizard if (argv.hasOwnProperty('d')) { console.log('Starting in quiet mode with the following configuration:\n' + 'Base URL: ' + prefs.importer.baseUrl + '\n' + 'Username: ' + prefs.importer.username + '\n' + 'Password: ***********\n' + 'Tag: ' + prefs.importer.tag + '\n' + 'Add tags given #: ' + prefs.importer.addtags + '\n' + 'Language: ' + prefs.importer.lang + '\n' + 'Daemon mode: ' + prefs.importer.daemon + '\n' + 'Copy folder: ' + prefs.importer.copyFolder + '\n' + 'File filter: ' + prefs.importer.fileFilter ); start(); } else { askBaseUrl(); }