docs/docs-importer/main.js

502 lines
13 KiB
JavaScript
Raw Normal View History

2018-02-28 22:53:06 +01:00
'use strict';
const recursive = require('recursive-readdir');
2020-06-23 22:31:49 +02:00
const minimatch = require("minimatch");
2018-02-28 22:53:06 +01:00
const ora = require('ora');
const inquirer = require('inquirer');
const preferences = require('preferences');
const fs = require('fs');
const argv = require('minimist')(process.argv);
const _ = require('underscore');
2018-02-28 22:53:06 +01:00
const request = require('request').defaults({
jar: true
});
const qs = require('querystring');
2018-02-28 22:53:06 +01:00
// Load preferences
const prefs = new preferences('com.sismics.docs.importer',{
2018-03-01 12:28:29 +01:00
importer: {
daemon: false
}
2018-02-28 22:53:06 +01:00
}, {
encrypt: false,
format: 'yaml'
});
// Welcome message
2020-04-21 21:13:20 +02:00
console.log('Teedy Importer 1.8, https://teedy.io' +
2018-02-28 22:53:06 +01:00
'\n\n' +
2019-02-12 13:57:54 +01:00
'This program let you import files from your system to Teedy' +
2018-02-28 22:53:06 +01:00
'\n');
// Ask for the base URL
const askBaseUrl = () => {
inquirer.prompt([
{
type: 'input',
name: 'baseUrl',
2019-02-12 13:57:54 +01:00
message: 'What is the base URL of your Teedy? (eg. https://teedy.mycompany.com)',
2018-02-28 22:53:06 +01:00
default: prefs.importer.baseUrl
}
]).then(answers => {
// Save base URL
prefs.importer.baseUrl = answers.baseUrl;
// Test base URL
const spinner = ora({
2019-02-12 13:57:54 +01:00
text: 'Checking connection to Teedy',
2018-02-28 22:53:06 +01:00
spinner: 'flips'
}).start();
request(answers.baseUrl + '/api/app', function (error, response) {
if (!response || response.statusCode !== 200) {
2019-02-12 13:57:54 +01:00
spinner.fail('Connection to Teedy failed: ' + error);
2018-02-28 22:53:06 +01:00
askBaseUrl();
return;
}
spinner.succeed('Connection OK');
askCredentials();
});
});
};
// Ask for credentials
const askCredentials = () => {
console.log('');
inquirer.prompt([
{
type: 'input',
name: 'username',
message: 'Account\'s username?',
default: prefs.importer.username
},
{
type: 'password',
name: 'password',
message: 'Account\'s password?',
default: prefs.importer.password
}
]).then(answers => {
// Save credentials
prefs.importer.username = answers.username;
prefs.importer.password = answers.password;
// Test credentials
const spinner = ora({
2019-02-12 13:57:54 +01:00
text: 'Checking connection to Teedy',
2018-02-28 22:53:06 +01:00
spinner: 'flips'
}).start();
request.post({
url: prefs.importer.baseUrl + '/api/user/login',
form: {
username: answers.username,
password: answers.password,
remember: true
}
}, function (error, response) {
2018-03-01 12:28:29 +01:00
if (error || !response || response.statusCode !== 200) {
2018-02-28 22:53:06 +01:00
spinner.fail('Username or password incorrect');
askCredentials();
return;
}
spinner.succeed('Authentication OK');
askPath();
});
});
};
// Ask for the path
const askPath = () => {
console.log('');
inquirer.prompt([
{
type: 'input',
name: 'path',
message: 'What is the folder path you want to import?',
default: prefs.importer.path
}
]).then(answers => {
// Save path
prefs.importer.path = answers.path;
// Test path
const spinner = ora({
text: 'Checking import path',
spinner: 'flips'
}).start();
2018-03-01 12:28:29 +01:00
fs.lstat(answers.path, (error, stats) => {
if (error || !stats.isDirectory()) {
2018-02-28 22:53:06 +01:00
spinner.fail('Please enter a valid directory path');
2018-03-01 12:28:29 +01:00
askPath();
2018-02-28 22:53:06 +01:00
return;
}
2018-03-01 12:28:29 +01:00
fs.access(answers.path, fs.W_OK | fs.R_OK, (error) => {
if (error) {
spinner.fail('This directory is not writable');
askPath();
return;
}
recursive(answers.path, function (error, files) {
spinner.succeed(files.length + ' files in this directory');
2020-06-23 22:31:49 +02:00
askFileFilter();
2018-03-01 12:28:29 +01:00
});
2018-02-28 22:53:06 +01:00
});
});
});
};
2018-03-01 12:28:29 +01:00
2020-06-23 22:31:49 +02:00
// Ask for the file filter
const askFileFilter = () => {
console.log('');
inquirer.prompt([
{
type: 'input',
name: 'fileFilter',
message: 'What pattern do you want to use to match files? (eg. *.+(pdf|txt|jpg))',
default: prefs.importer.fileFilter || "*"
}
]).then(answers => {
// Save fileFilter
prefs.importer.fileFilter = answers.fileFilter;
askTag();
});
};
// Ask for the tag to add
const askTag = () => {
console.log('');
// Load tags
const spinner = ora({
text: 'Loading tags',
spinner: 'flips'
}).start();
request.get({
url: prefs.importer.baseUrl + '/api/tag/list',
}, function (error, response, body) {
if (error || !response || response.statusCode !== 200) {
spinner.fail('Error loading tags');
askTag();
return;
}
spinner.succeed('Tags loaded');
const tags = JSON.parse(body).tags;
const defaultTag = _.findWhere(tags, { id: prefs.importer.tag });
const defaultTagName = defaultTag ? defaultTag.name : 'No tag';
inquirer.prompt([
{
type: 'list',
name: 'tag',
message: 'Which tag to add to all imported documents?',
default: defaultTagName,
choices: [ 'No tag' ].concat(_.pluck(tags, 'name'))
}
]).then(answers => {
// Save tag
prefs.importer.tag = answers.tag === 'No tag' ?
'' : _.findWhere(tags, { name: answers.tag }).id;
askAddTag();
});
});
};
const askAddTag = () => {
console.log('');
inquirer.prompt([
{
type: 'confirm',
name: 'addtags',
message: 'Do you want to add tags from the filename given with # ?',
default: prefs.importer.addtags === true
}
]).then(answers => {
// Save daemon
prefs.importer.addtags = answers.addtags;
// Save all preferences in case the program is sig-killed
askLang();
});
}
const askLang = () => {
console.log('');
// Load tags
const spinner = ora({
text: 'Loading default language',
spinner: 'flips'
}).start();
request.get({
url: prefs.importer.baseUrl + '/api/app',
}, function (error, response, body) {
if (error || !response || response.statusCode !== 200) {
spinner.fail('Connection to Teedy failed: ' + error);
askLang();
return;
}
spinner.succeed('Language loaded');
const defaultLang = prefs.importer.lang ? prefs.importer.lang : JSON.parse(body).default_language;
inquirer.prompt([
{
type: 'input',
name: 'lang',
message: 'Which should be the default language of the document?',
default: defaultLang
}
]).then(answers => {
// Save tag
prefs.importer.lang = answers.lang
askCopyFolder();
});
});
};
const askCopyFolder = () => {
console.log('');
inquirer.prompt([
{
type: 'input',
name: 'copyFolder',
message: 'Enter a path to copy files before they are deleted or leave empty to disable. The path must end with a \'/\' on MacOS and Linux or with a \'\\\' on Windows. Entering \'undefined\' will disable this again after setting the folder.',
default: prefs.importer.copyFolder
}
]).then(answers => {
// Save path
prefs.importer.copyFolder = answers.copyFolder=='undefined' ? '' : answers.copyFolder;
if (prefs.importer.copyFolder) {
// Test path
const spinner = ora({
text: 'Checking copy folder path',
spinner: 'flips'
}).start();
fs.lstat(answers.copyFolder, (error, stats) => {
if (error || !stats.isDirectory()) {
spinner.fail('Please enter a valid directory path');
askCopyFolder();
return;
}
fs.access(answers.copyFolder, fs.W_OK | fs.R_OK, (error) => {
if (error) {
spinner.fail('This directory is not writable');
askCopyFolder();
return;
}
spinner.succeed('Copy folder set!');
askDaemon();
});
});
}
else {askDaemon();}
});
};
2018-03-01 12:28:29 +01:00
// Ask for daemon mode
const askDaemon = () => {
console.log('');
inquirer.prompt([
{
type: 'confirm',
name: 'daemon',
message: 'Do you want to run the importer in daemon mode (it will poll the input directory for new files, import and delete them)?',
default: prefs.importer.daemon === true
}
]).then(answers => {
// Save daemon
prefs.importer.daemon = answers.daemon;
// Save all preferences in case the program is sig-killed
prefs.save();
2018-03-01 12:28:29 +01:00
start();
});
};
// Start the importer
2018-03-01 12:28:29 +01:00
const start = () => {
request.post({
url: prefs.importer.baseUrl + '/api/user/login',
form: {
username: prefs.importer.username,
password: prefs.importer.password,
remember: true
}
}, function (error, response) {
if (error || !response || response.statusCode !== 200) {
console.error('\nUsername or password incorrect');
return;
}
2018-03-01 12:28:29 +01:00
// Start the actual import
if (prefs.importer.daemon) {
console.log('\nPolling the input folder for new files...');
let resolve = () => {
importFiles(true, () => {
setTimeout(resolve, 30000);
});
};
resolve();
} else {
importFiles(false, () => {});
}
});
2018-03-01 12:28:29 +01:00
};
// Import the files
const importFiles = (remove, filesImported) => {
recursive(prefs.importer.path, function (error, files) {
2020-06-23 22:31:49 +02:00
files = files.filter(minimatch.filter(prefs.importer.fileFilter ?? "*", {matchBase: true}));
2018-03-01 12:28:29 +01:00
if (files.length === 0) {
filesImported();
return;
}
let index = 0;
let resolve = () => {
const file = files[index++];
if (file) {
importFile(file, remove, resolve);
} else {
filesImported();
}
};
resolve();
});
};
// Import a file
const importFile = (file, remove, resolve) => {
const spinner = ora({
text: 'Importing: ' + file,
spinner: 'flips'
}).start();
// Remove path of file
let filename = file.replace(/^.*[\\\/]/, '');
// Get Tags given as hashtags from filename
let taglist = filename.match(/#[^\s:#]+/mg);
taglist = taglist ? taglist.map(s => s.substr(1)) : [];
// Get available tags and UUIDs from server
request.get({
url: prefs.importer.baseUrl + '/api/tag/list',
}, function (error, response, body) {
2018-03-01 12:28:29 +01:00
if (error || !response || response.statusCode !== 200) {
spinner.fail('Error loading tags');
2018-03-01 12:28:29 +01:00
return;
}
let tagsarray = {};
for (let l of JSON.parse(body).tags) {
tagsarray[l.name] = l.id;
}
// Intersect tags from filename with existing tags on server
let foundtags = [];
for (let j of taglist) {
// If the tag is last in the filename it could include a file extension and would not be recognized
if (j.includes('.') && !tagsarray.hasOwnProperty(j) && !foundtags.includes(tagsarray[j])) {
while (j.includes('.') && !tagsarray.hasOwnProperty(j)) {
j = j.replace(/\.[^.]*$/,'');
}
}
if (tagsarray.hasOwnProperty(j) && !foundtags.includes(tagsarray[j])) {
foundtags.push(tagsarray[j]);
filename = filename.split('#'+j).join('');
}
}
if (prefs.importer.tag !== '' && !foundtags.includes(prefs.importer.tag)){
foundtags.push(prefs.importer.tag);
}
let data = {}
if (prefs.importer.addtags) {
data = {
title: prefs.importer.addtags ? filename : file.replace(/^.*[\\\/]/, '').substring(0, 100),
language: prefs.importer.lang || 'eng',
tags: foundtags
}
}
else {
data = {
title: prefs.importer.addtags ? filename : file.replace(/^.*[\\\/]/, '').substring(0, 100),
language: prefs.importer.lang || 'eng',
tags: prefs.importer.tag === '' ? undefined : prefs.importer.tag
}
}
// Create document
request.put({
url: prefs.importer.baseUrl + '/api/document',
form: qs.stringify(data)
}, function (error, response, body) {
if (error || !response || response.statusCode !== 200) {
spinner.fail('Upload failed for ' + file + ': ' + error);
resolve();
return;
}
// Upload file
request.put({
url: prefs.importer.baseUrl + '/api/file',
formData: {
id: JSON.parse(body).id,
file: fs.createReadStream(file)
}
}, function (error, response) {
if (error || !response || response.statusCode !== 200) {
spinner.fail('Upload failed for ' + file + ': ' + error);
resolve();
return;
}
spinner.succeed('Upload successful for ' + file);
if (remove) {
if (prefs.importer.copyFolder) {
fs.copyFileSync(file, prefs.importer.copyFolder + file.replace(/^.*[\\\/]/, ''));
fs.unlinkSync(file);
}
else {fs.unlinkSync(file);}
}
resolve();
});
});
2018-03-01 12:28:29 +01:00
});
};
// Entrypoint: daemon mode or wizard
if (argv.hasOwnProperty('d')) {
console.log('Starting in quiet mode with the following configuration:\n' +
'Base URL: ' + prefs.importer.baseUrl + '\n' +
'Username: ' + prefs.importer.username + '\n' +
'Password: ***********\n' +
'Tag: ' + prefs.importer.tag + '\n' +
'Add tags given #: ' + prefs.importer.addtags + '\n' +
'Language: ' + prefs.importer.lang + '\n' +
'Daemon mode: ' + prefs.importer.daemon + '\n' +
2020-06-23 22:31:49 +02:00
'Copy folder: ' + prefs.importer.copyFolder + '\n' +
'File filter: ' + prefs.importer.fileFilter
);
start();
} else {
askBaseUrl();
}