diff --git a/docs-importer/Dockerfile b/docs-importer/Dockerfile new file mode 100644 index 00000000..73f240c0 --- /dev/null +++ b/docs-importer/Dockerfile @@ -0,0 +1,17 @@ +FROM node:14.2-alpine AS builder +WORKDIR /build +COPY node_modules/ ./node_modules/ +COPY main.js package-lock.json package.json ./ +RUN npm install && npm install -g pkg +RUN pkg -t node14-alpine-x64 . + +FROM alpine +ENV TEEDY_TAG= TEEDY_ADDTAGS=false TEEDY_LANG=eng TEEDY_URL='http://localhost:8080' TEEDY_USERNAME=username TEEDY_PASSWORD=password +RUN apk add --no-cache \ + libc6-compat \ + libstdc++ +ADD pref /root/.config/preferences/com.sismics.docs.importer.pref +ADD env.sh / +COPY --from=builder /build/teedy-importer ./ + +CMD ["/bin/ash","-c","/env.sh && /teedy-importer -d"] diff --git a/docs-importer/README.md b/docs-importer/README.md index 68afac71..b123a1a0 100644 --- a/docs-importer/README.md +++ b/docs-importer/README.md @@ -1,35 +1,55 @@ -File Importer -============= +# File Importer This tool can be used to do a single import of files or to periodically scan for files in an input folder. -Downloads ---------- +## Downloads + Built binaries for Windows/Linux/MacOSX can be found at -Usage ------ +## Usage + ```console ./docs-importer-macos (for MacOSX) ./docs-importer-linux (for Linux) docs-importer-win.exe (for Windows) ``` -A wizard will ask you for the import configuration and write it in `~/.config/preferences/com.sismics.docs.importer.pref` +A wizard will ask you for the import configuration and write it in `~/.config/preferences/com.sismics.docs.importer.pref`. +Words following a `#` in the filename will be added as tags to the document, if there is a tag with the same name on the Server. For the next start, pass the `-d` argument to skip the wizard: + ```console ./docs-importer-linux -d ``` -Daemon mode ------------ +## Daemon mode + The daemon mode scan the input directory every 30 seconds for new files. Once a file is found and imported, it is **deleted**. -Build from sources ------------------- +## Docker + +The docker image needs a volume mount of a previously generated preference file to `/root/.config/preferences/com.sismics.docs.importer.pref`. The container will start the importer in daemon mode. It will look for files in `/import`. +Example usage: + +``` +docker build -t teedy-import . +docker run --name teedy-import -d -v /path/to/preferencefile:/root/.config/preferences/com.sismics.docs.importer.pref -v /path/to/import/folder:/import teedy-import +``` +### Environment variables +Instead of mounting the preferences file, the options can also be set by setting the environment variables `TEEDY_TAG`, `TEEDY_ADDTAGS`, `TEEDY_LANG`, `TEEDY_URL`, `TEEDY_USERNAME` and `TEEDY_PASSWORD`. +The latter three have to be set for the importer to work. The value of `TEEDY_TAG` has to be set to the UUID of the tag, not the name (The UUID can be found by visiting `baseUrl/api/tag/list` in your browser). +Example usage: + +``` +docker build -t teedy-import . +docker run --name teedy-import -d -e TEEDY_TAG=2071fdf7-0e26-409d-b53d-f25823a5eb9e -e TEEDY_ADDTAGS=false -e TEEDY_LANG=eng -e TEEDY_URL='http://teedy.example.com:port' -e TEEDY_USERNAME=username -e TEEDY_PASSWORD=superSecretPassword -v /path/to/import/folder:/import teedy-import +``` + +## Build from sources + ```console npm install npm install -g pkg pkg . -``` \ No newline at end of file +``` diff --git a/docs-importer/env.sh b/docs-importer/env.sh new file mode 100755 index 00000000..fddd2d6a --- /dev/null +++ b/docs-importer/env.sh @@ -0,0 +1,9 @@ +#!/bin/ash +file=/root/.config/preferences/com.sismics.docs.importer.pref +sed -i "s/env1/$TEEDY_TAG/g" $file +sed -i "s/env2/$TEEDY_ADDTAGS/g" $file +sed -i "s/env3/$TEEDY_LANG/g" $file +sed -i "s,env4,$TEEDY_URL,g" $file +sed -i "s/env5/$TEEDY_USERNAME/g" $file +sed -i "s/env6/$TEEDY_PASSWORD/g" $file +echo "Environment variables replaced" \ No newline at end of file diff --git a/docs-importer/main.js b/docs-importer/main.js index 818f8573..f2ae749b 100644 --- a/docs-importer/main.js +++ b/docs-importer/main.js @@ -10,6 +10,7 @@ const _ = require('underscore'); const request = require('request').defaults({ jar: true }); +const qs = require('querystring'); // Load preferences const prefs = new preferences('com.sismics.docs.importer',{ @@ -176,7 +177,7 @@ const askTag = () => { { type: 'list', name: 'tag', - message: 'Which tag to add on imported documents?', + message: 'Which tag to add to all imported documents?', default: defaultTagName, choices: [ 'No tag' ].concat(_.pluck(tags, 'name')) } @@ -184,6 +185,62 @@ const askTag = () => { // Save tag prefs.importer.tag = answers.tag === 'No tag' ? '' : _.findWhere(tags, { name: answers.tag }).id; + askAddTag(); + }); + }); +}; + + +const askAddTag = () => { + console.log(''); + + inquirer.prompt([ + { + type: 'confirm', + name: 'addtags', + message: 'Do you want to add tags from the filename given with # ?', + default: prefs.importer.addtags === true + } + ]).then(answers => { + // Save daemon + prefs.importer.addtags = answers.addtags; + + // Save all preferences in case the program is sig-killed + askLang(); + }); +} + + +const askLang = () => { + console.log(''); + + // Load tags + const spinner = ora({ + text: 'Loading default language', + spinner: 'flips' + }).start(); + + request.get({ + url: prefs.importer.baseUrl + '/api/app', + }, function (error, response, body) { + if (error || !response || response.statusCode !== 200) { + spinner.fail('Connection to Teedy failed: ' + error); + askLang(); + return; + } + spinner.succeed('Language loaded'); + const defaultLang = prefs.importer.lang ? prefs.importer.lang : JSON.parse(body).default_language; + + inquirer.prompt([ + { + type: 'input', + name: 'lang', + message: 'Which should be the default language of the document?', + default: defaultLang + } + ]).then(answers => { + // Save tag + prefs.importer.lang = answers.lang askDaemon(); }); }); @@ -270,37 +327,83 @@ const importFile = (file, remove, resolve) => { spinner: 'flips' }).start(); - request.put({ - url: prefs.importer.baseUrl + '/api/document', - form: { - title: file.replace(/^.*[\\\/]/, '').substring(0, 100), - language: 'eng', - tags: prefs.importer.tag === '' ? undefined : prefs.importer.tag - } - }, function (error, response, body) { + // Remove path of file + let filename = file.replace(/^.*[\\\/]/, ''); + + // Get Tags given as hashtags from filename + let taglist = filename.match(/#[^\s:#]+/mg); + taglist = taglist ? taglist.map(s => s.substr(1)) : []; + + // Get available tags and UUIDs from server + request.get({ + url: prefs.importer.baseUrl + '/api/tag/list', + }, function (error, response, body) { if (error || !response || response.statusCode !== 200) { - spinner.fail('Upload failed for ' + file + ': ' + error); - resolve(); + spinner.fail('Error loading tags'); return; } + + let tagsarray = {}; + for (let l of JSON.parse(body).tags) { + tagsarray[l.name] = l.id; + } - request.put({ - url: prefs.importer.baseUrl + '/api/file', - formData: { - id: JSON.parse(body).id, - file: fs.createReadStream(file) + // Intersect tags from filename with existing tags on server + let foundtags = []; + for (let j of taglist) { + if (tagsarray.hasOwnProperty(j) && !foundtags.includes(tagsarray[j])) { + foundtags.push(tagsarray[j]); + filename = filename.split('#'+j).join(''); } - }, function (error, response) { + } + if (prefs.importer.tag !== '' && !foundtags.includes(prefs.importer.tag)){ + foundtags.push(prefs.importer.tag); + } + + let data = {} + if (prefs.importer.addtags) { + data = { + title: prefs.importer.addtags ? filename : file.replace(/^.*[\\\/]/, '').substring(0, 100), + language: prefs.importer.lang || 'eng', + tags: foundtags + } + } + else { + data = { + title: prefs.importer.addtags ? filename : file.replace(/^.*[\\\/]/, '').substring(0, 100), + language: prefs.importer.lang || 'eng' + } + } + // Create document + request.put({ + url: prefs.importer.baseUrl + '/api/document', + form: qs.stringify(data) + }, function (error, response, body) { if (error || !response || response.statusCode !== 200) { spinner.fail('Upload failed for ' + file + ': ' + error); resolve(); return; } - spinner.succeed('Upload successful for ' + file); - if (remove) { - fs.unlinkSync(file); - } - resolve(); + + // Upload file + request.put({ + url: prefs.importer.baseUrl + '/api/file', + formData: { + id: JSON.parse(body).id, + file: fs.createReadStream(file) + } + }, function (error, response) { + if (error || !response || response.statusCode !== 200) { + spinner.fail('Upload failed for ' + file + ': ' + error); + resolve(); + return; + } + spinner.succeed('Upload successful for ' + file); + if (remove) { + fs.unlinkSync(file); + } + resolve(); + }); }); }); }; @@ -312,7 +415,10 @@ if (argv.hasOwnProperty('d')) { 'Username: ' + prefs.importer.username + '\n' + 'Password: ***********\n' + 'Tag: ' + prefs.importer.tag + '\n' + - 'Daemon mode: ' + prefs.importer.daemon); + 'Add tags given #: ' + prefs.importer.addtags + '\n' + + 'Language: ' + prefs.importer.lang + '\n' + + 'Daemon mode: ' + prefs.importer.daemon + ); start(); } else { askBaseUrl(); diff --git a/docs-importer/package-lock.json b/docs-importer/package-lock.json index ca20d6c0..bd30e36e 100644 --- a/docs-importer/package-lock.json +++ b/docs-importer/package-lock.json @@ -1,6 +1,6 @@ { "name": "teedy-importer", - "version": "1.8", + "version": "1.8.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -36,7 +36,7 @@ "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=", "requires": { "sprintf-js": "~1.0.2" } @@ -75,7 +75,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=", - "optional": true, "requires": { "tweetnacl": "^0.14.3" } @@ -91,7 +90,7 @@ "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=", "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -219,7 +218,6 @@ "version": "0.1.1", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=", - "optional": true, "requires": { "jsbn": "~0.1.0" } @@ -396,9 +394,9 @@ "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, "js-yaml": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.10.0.tgz", - "integrity": "sha512-O2v52ffjLa9VeM43J4XocZE//WT9N0IiwDa3KSHH7Tu8CtH+1qM8SIZvnsTh6v+4yFy5KUY3BHUVwjpfAWsjIA==", + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", "requires": { "argparse": "^1.0.7", "esprima": "^4.0.0" @@ -407,8 +405,7 @@ "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", - "optional": true + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" }, "json-schema": { "version": "0.2.3", @@ -437,14 +434,14 @@ } }, "lodash": { - "version": "4.17.5", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz", - "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==" + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" }, "log-symbols": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz", - "integrity": "sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==", + "integrity": "sha1-V0Dhxdbw39pK2TI7UzIQfva0xAo=", "requires": { "chalk": "^2.0.1" } @@ -465,34 +462,27 @@ "mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" + "integrity": "sha1-ggyGo5M0ZA6ZUWkovQP8qIBX0CI=" }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=", "requires": { "brace-expansion": "^1.1.7" } }, "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "requires": { - "minimist": "0.0.8" - }, - "dependencies": { - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" - } + "minimist": "^1.2.5" } }, "mute-stream": { @@ -544,7 +534,7 @@ "preferences": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/preferences/-/preferences-1.0.2.tgz", - "integrity": "sha512-cRjA8Galk1HDDBOKjx6DhTwfy5+FVZtH7ogg6rgTLX8Ak4wi55RaS4uRztJuVPd+md1jZo99bH/h1Q9bQQK8bg==", + "integrity": "sha1-UDaZN8ZpBIoEPKF+REaEQ278WUU=", "requires": { "graceful-fs": "^4.1.2", "js-yaml": "^3.10.0", @@ -559,14 +549,14 @@ "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" }, "qs": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", - "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" + "version": "6.9.4", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz", + "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==" }, "recursive-readdir": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.2.tgz", - "integrity": "sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg==", + "integrity": "sha1-mUb7MnThYo3m42svZxSVO0hFCU8=", "requires": { "minimatch": "3.0.4" } @@ -598,6 +588,13 @@ "tough-cookie": "~2.3.3", "tunnel-agent": "^0.6.0", "uuid": "^3.1.0" + }, + "dependencies": { + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + } } }, "restore-cursor": { @@ -630,6 +627,11 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, "signal-exit": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", @@ -654,9 +656,9 @@ "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, "sshpk": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz", - "integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=", + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", "requires": { "asn1": "~0.2.3", "assert-plus": "^1.0.0", @@ -665,13 +667,14 @@ "ecc-jsbn": "~0.1.1", "getpass": "^0.1.1", "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", "tweetnacl": "~0.14.0" } }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "integrity": "sha1-q5Pyeo3BPSjKyBXEYhQ6bZASrp4=", "requires": { "is-fullwidth-code-point": "^2.0.0", "strip-ansi": "^4.0.0" @@ -711,7 +714,7 @@ "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "integrity": "sha1-bTQzWIl2jSGyvNoKonfO07G/rfk=", "requires": { "os-tmpdir": "~1.0.2" } @@ -735,8 +738,7 @@ "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", - "optional": true + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, "underscore": { "version": "1.8.3", diff --git a/docs-importer/package.json b/docs-importer/package.json index ec9fab45..1234c034 100644 --- a/docs-importer/package.json +++ b/docs-importer/package.json @@ -1,6 +1,6 @@ { "name": "teedy-importer", - "version": "1.8", + "version": "1.8.0", "description": "Import files to Teedy", "bin": "main.js", "scripts": { @@ -11,6 +11,9 @@ "url": "git+https://github.com/sismics/docs.git" }, "author": "Benjamin Gamard", + "contributors": [ + "Cornelius Hoffmann " + ], "license": "GPL-2.0", "bugs": { "url": "https://github.com/sismics/docs/issues" @@ -18,9 +21,10 @@ "homepage": "https://github.com/sismics/docs#readme", "dependencies": { "inquirer": "^5.1.0", - "minimist": "^1.2.0", + "minimist": "^1.2.5", "ora": "^2.0.0", "preferences": "^1.0.2", + "qs": "^6.9.4", "recursive-readdir": "^2.2.2", "request": "^2.83.0", "underscore": "^1.8.3" diff --git a/docs-importer/pref b/docs-importer/pref new file mode 100644 index 00000000..7a7a3b2d --- /dev/null +++ b/docs-importer/pref @@ -0,0 +1,9 @@ +importer: + daemon: true + path: import + tag: 'env1' + addtags: 'env2' + lang: 'env3' + baseUrl: 'env4' + username: 'env5' + password: 'env6'