vmario/docs
1
0
Fork 0
mirror of https://github.com/sismics/docs.git synced 2024-12-22 11:23:48 +01:00
Code Issues Packages Projects Releases Wiki Activity

Improve the file importer (#415)

Browse Source 
Improve the bulk importer (tags by filename, document language, Docker container)
This commit is contained in:
Cornelicorn 2020-05-22 15:18:19 +02:00 committed by GitHub
parent 3f67bd471b
commit 612fab2aef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 246 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs-importer/Dockerfile Normal file
View File

@ -0,0 +1,17 @@
FROM node:14.2-alpine AS builder
WORKDIR /build
COPY node_modules/ ./node_modules/
COPY main.js package-lock.json package.json ./
RUN npm install && npm install -g pkg
RUN pkg -t node14-alpine-x64 .
FROM alpine
ENV TEEDY_TAG= TEEDY_ADDTAGS=false TEEDY_LANG=eng TEEDY_URL='http://localhost:8080' TEEDY_USERNAME=username TEEDY_PASSWORD=password
RUN apk add --no-cache \
libc6-compat \
libstdc++
ADD pref /root/.config/preferences/com.sismics.docs.importer.pref
ADD env.sh /
COPY --from=builder /build/teedy-importer ./
CMD ["/bin/ash","-c","/env.sh && /teedy-importer -d"]

44
docs-importer/README.md
View File

@ -1,35 +1,55 @@
File Importer
=============
# File Importer
This tool can be used to do a single import of files or to periodically scan for files in an input folder.
Downloads
---------
## Downloads
Built binaries for Windows/Linux/MacOSX can be found at <https://github.com/sismics/docs/releases>
Usage
-----
## Usage
```console
./docs-importer-macos (for MacOSX)
./docs-importer-linux (for Linux)
docs-importer-win.exe (for Windows)
```
A wizard will ask you for the import configuration and write it in `~/.config/preferences/com.sismics.docs.importer.pref`
A wizard will ask you for the import configuration and write it in `~/.config/preferences/com.sismics.docs.importer.pref`.
Words following a `#` in the filename will be added as tags to the document, if there is a tag with the same name on the Server.
For the next start, pass the `-d` argument to skip the wizard:
```console
./docs-importer-linux -d
```
Daemon mode
-----------
## Daemon mode
The daemon mode scan the input directory every 30 seconds for new files. Once a file is found and imported, it is **deleted**.
Build from sources
------------------
## Docker
The docker image needs a volume mount of a previously generated preference file to `/root/.config/preferences/com.sismics.docs.importer.pref`. The container will start the importer in daemon mode. It will look for files in `/import`.
Example usage:
```
docker build -t teedy-import .
docker run --name teedy-import -d -v /path/to/preferencefile:/root/.config/preferences/com.sismics.docs.importer.pref -v /path/to/import/folder:/import teedy-import
```
### Environment variables
Instead of mounting the preferences file, the options can also be set by setting the environment variables `TEEDY_TAG`, `TEEDY_ADDTAGS`, `TEEDY_LANG`, `TEEDY_URL`, `TEEDY_USERNAME` and `TEEDY_PASSWORD`.
The latter three have to be set for the importer to work. The value of `TEEDY_TAG` has to be set to the UUID of the tag, not the name (The UUID can be found by visiting `baseUrl/api/tag/list` in your browser).
Example usage:
```
docker build -t teedy-import .
docker run --name teedy-import -d -e TEEDY_TAG=2071fdf7-0e26-409d-b53d-f25823a5eb9e -e TEEDY_ADDTAGS=false -e TEEDY_LANG=eng -e TEEDY_URL='http://teedy.example.com:port' -e TEEDY_USERNAME=username -e TEEDY_PASSWORD=superSecretPassword -v /path/to/import/folder:/import teedy-import
```
## Build from sources
```console
npm install
npm install -g pkg
pkg .
```
```

9
docs-importer/env.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/ash
file=/root/.config/preferences/com.sismics.docs.importer.pref
sed -i "s/env1/$TEEDY_TAG/g" $file
sed -i "s/env2/$TEEDY_ADDTAGS/g" $file
sed -i "s/env3/$TEEDY_LANG/g" $file
sed -i "s,env4,$TEEDY_URL,g" $file
sed -i "s/env5/$TEEDY_USERNAME/g" $file
sed -i "s/env6/$TEEDY_PASSWORD/g" $file
echo "Environment variables replaced"

152
docs-importer/main.js
View File

@ -10,6 +10,7 @@ const _ = require('underscore');
const request = require('request').defaults({
jar: true
});
const qs = require('querystring');
// Load preferences
const prefs = new preferences('com.sismics.docs.importer',{
@ -176,7 +177,7 @@ const askTag = () => {
{
type: 'list',
name: 'tag',
message: 'Which tag to add on imported documents?',
message: 'Which tag to add to all imported documents?',
default: defaultTagName,
choices: [ 'No tag' ].concat(_.pluck(tags, 'name'))
}
@ -184,6 +185,62 @@ const askTag = () => {
// Save tag
prefs.importer.tag = answers.tag === 'No tag' ?
'' : _.findWhere(tags, { name: answers.tag }).id;
askAddTag();
});
});
};
const askAddTag = () => {
console.log('');
inquirer.prompt([
{
type: 'confirm',
name: 'addtags',
message: 'Do you want to add tags from the filename given with # ?',
default: prefs.importer.addtags === true
}
]).then(answers => {
// Save daemon
prefs.importer.addtags = answers.addtags;
// Save all preferences in case the program is sig-killed
askLang();
});
}
const askLang = () => {
console.log('');
// Load tags
const spinner = ora({
text: 'Loading default language',
spinner: 'flips'
}).start();
request.get({
url: prefs.importer.baseUrl + '/api/app',
}, function (error, response, body) {
if (error || !response || response.statusCode !== 200) {
spinner.fail('Connection to Teedy failed: ' + error);
askLang();
return;
}
spinner.succeed('Language loaded');
const defaultLang = prefs.importer.lang ? prefs.importer.lang : JSON.parse(body).default_language;
inquirer.prompt([
{
type: 'input',
name: 'lang',
message: 'Which should be the default language of the document?',
default: defaultLang
}
]).then(answers => {
// Save tag
prefs.importer.lang = answers.lang
askDaemon();
});
});
@ -270,37 +327,83 @@ const importFile = (file, remove, resolve) => {
spinner: 'flips'
}).start();
request.put({
url: prefs.importer.baseUrl + '/api/document',
form: {
title: file.replace(/^.*[\\\/]/, '').substring(0, 100),
language: 'eng',
tags: prefs.importer.tag === '' ? undefined : prefs.importer.tag
}
}, function (error, response, body) {
// Remove path of file
let filename = file.replace(/^.*[\\\/]/, '');
// Get Tags given as hashtags from filename
let taglist = filename.match(/#[^\s:#]+/mg);
taglist = taglist ? taglist.map(s => s.substr(1)) : [];
// Get available tags and UUIDs from server
request.get({
url: prefs.importer.baseUrl + '/api/tag/list',
}, function (error, response, body) {
if (error || !response || response.statusCode !== 200) {
spinner.fail('Upload failed for ' + file + ': ' + error);
resolve();
spinner.fail('Error loading tags');
return;
}
let tagsarray = {};
for (let l of JSON.parse(body).tags) {
tagsarray[l.name] = l.id;
}
request.put({
url: prefs.importer.baseUrl + '/api/file',
formData: {
id: JSON.parse(body).id,
file: fs.createReadStream(file)
// Intersect tags from filename with existing tags on server
let foundtags = [];
for (let j of taglist) {
if (tagsarray.hasOwnProperty(j) && !foundtags.includes(tagsarray[j])) {
foundtags.push(tagsarray[j]);
filename = filename.split('#'+j).join('');
}
}, function (error, response) {
}
if (prefs.importer.tag !== '' && !foundtags.includes(prefs.importer.tag)){
foundtags.push(prefs.importer.tag);
}
let data = {}
if (prefs.importer.addtags) {
data = {
title: prefs.importer.addtags ? filename : file.replace(/^.*[\\\/]/, '').substring(0, 100),
language: prefs.importer.lang || 'eng',
tags: foundtags
}
}
else {
data = {
title: prefs.importer.addtags ? filename : file.replace(/^.*[\\\/]/, '').substring(0, 100),
language: prefs.importer.lang || 'eng'
}
}
// Create document
request.put({
url: prefs.importer.baseUrl + '/api/document',
form: qs.stringify(data)
}, function (error, response, body) {
if (error || !response || response.statusCode !== 200) {
spinner.fail('Upload failed for ' + file + ': ' + error);
resolve();
return;
}
spinner.succeed('Upload successful for ' + file);
if (remove) {
fs.unlinkSync(file);
}
resolve();
// Upload file
request.put({
url: prefs.importer.baseUrl + '/api/file',
formData: {
id: JSON.parse(body).id,
file: fs.createReadStream(file)
}
}, function (error, response) {
if (error || !response || response.statusCode !== 200) {
spinner.fail('Upload failed for ' + file + ': ' + error);
resolve();
return;
}
spinner.succeed('Upload successful for ' + file);
if (remove) {
fs.unlinkSync(file);
}
resolve();
});
});
});
};
@ -312,7 +415,10 @@ if (argv.hasOwnProperty('d')) {
'Username: ' + prefs.importer.username + '\n' +
'Password: ***********\n' +
'Tag: ' + prefs.importer.tag + '\n' +
'Daemon mode: ' + prefs.importer.daemon);
'Add tags given #: ' + prefs.importer.addtags + '\n' +
'Language: ' + prefs.importer.lang + '\n' +
'Daemon mode: ' + prefs.importer.daemon
);
start();
} else {
askBaseUrl();

86
docs-importer/package-lock.json generated
View File

@ -1,6 +1,6 @@
{
"name": "teedy-importer",
"version": "1.8",
"version": "1.8.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -36,7 +36,7 @@
"argparse": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
"integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
"integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=",
"requires": {
"sprintf-js": "~1.0.2"
}
@ -75,7 +75,6 @@
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz",
"integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=",
"optional": true,
"requires": {
"tweetnacl": "^0.14.3"
}
@ -91,7 +90,7 @@
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=",
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@ -219,7 +218,6 @@
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz",
"integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=",
"optional": true,
"requires": {
"jsbn": "~0.1.0"
}
@ -396,9 +394,9 @@
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
},
"js-yaml": {
"version": "3.10.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.10.0.tgz",
"integrity": "sha512-O2v52ffjLa9VeM43J4XocZE//WT9N0IiwDa3KSHH7Tu8CtH+1qM8SIZvnsTh6v+4yFy5KUY3BHUVwjpfAWsjIA==",
"version": "3.13.1",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz",
"integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==",
"requires": {
"argparse": "^1.0.7",
"esprima": "^4.0.0"
@ -407,8 +405,7 @@
"jsbn": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=",
"optional": true
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
},
"json-schema": {
"version": "0.2.3",
@ -437,14 +434,14 @@
}
},
"lodash": {
"version": "4.17.5",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz",
"integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw=="
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
},
"log-symbols": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz",
"integrity": "sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==",
"integrity": "sha1-V0Dhxdbw39pK2TI7UzIQfva0xAo=",
"requires": {
"chalk": "^2.0.1"
}
@ -465,34 +462,27 @@
"mimic-fn": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz",
"integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ=="
"integrity": "sha1-ggyGo5M0ZA6ZUWkovQP8qIBX0CI="
},
"minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=",
"requires": {
"brace-expansion": "^1.1.7"
}
},
"minimist": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
"integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
},
"mkdirp": {
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
"integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
"version": "0.5.5",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
"integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
"requires": {
"minimist": "0.0.8"
},
"dependencies": {
"minimist": {
"version": "0.0.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
}
"minimist": "^1.2.5"
}
},
"mute-stream": {
@ -544,7 +534,7 @@
"preferences": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/preferences/-/preferences-1.0.2.tgz",
"integrity": "sha512-cRjA8Galk1HDDBOKjx6DhTwfy5+FVZtH7ogg6rgTLX8Ak4wi55RaS4uRztJuVPd+md1jZo99bH/h1Q9bQQK8bg==",
"integrity": "sha1-UDaZN8ZpBIoEPKF+REaEQ278WUU=",
"requires": {
"graceful-fs": "^4.1.2",
"js-yaml": "^3.10.0",
@ -559,14 +549,14 @@
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
},
"qs": {
"version": "6.5.1",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz",
"integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A=="
"version": "6.9.4",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz",
"integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ=="
},
"recursive-readdir": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.2.tgz",
"integrity": "sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg==",
"integrity": "sha1-mUb7MnThYo3m42svZxSVO0hFCU8=",
"requires": {
"minimatch": "3.0.4"
}
@ -598,6 +588,13 @@
"tough-cookie": "~2.3.3",
"tunnel-agent": "^0.6.0",
"uuid": "^3.1.0"
},
"dependencies": {
"qs": {
"version": "6.5.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
}
}
},
"restore-cursor": {
@ -630,6 +627,11 @@
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
"integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
},
"safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"signal-exit": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
@ -654,9 +656,9 @@
"integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw="
},
"sshpk": {
"version": "1.13.1",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz",
"integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=",
"version": "1.16.1",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
"integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==",
"requires": {
"asn1": "~0.2.3",
"assert-plus": "^1.0.0",
@ -665,13 +667,14 @@
"ecc-jsbn": "~0.1.1",
"getpass": "^0.1.1",
"jsbn": "~0.1.0",
"safer-buffer": "^2.0.2",
"tweetnacl": "~0.14.0"
}
},
"string-width": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
"integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
"integrity": "sha1-q5Pyeo3BPSjKyBXEYhQ6bZASrp4=",
"requires": {
"is-fullwidth-code-point": "^2.0.0",
"strip-ansi": "^4.0.0"
@ -711,7 +714,7 @@
"tmp": {
"version": "0.0.33",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
"integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==",
"integrity": "sha1-bTQzWIl2jSGyvNoKonfO07G/rfk=",
"requires": {
"os-tmpdir": "~1.0.2"
}
@ -735,8 +738,7 @@
"tweetnacl": {
"version": "0.14.5",
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
"optional": true
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
},
"underscore": {
"version": "1.8.3",

8
docs-importer/package.json
View File

@ -1,6 +1,6 @@
{
"name": "teedy-importer",
"version": "1.8",
"version": "1.8.0",
"description": "Import files to Teedy",
"bin": "main.js",
"scripts": {
@ -11,6 +11,9 @@
"url": "git+https://github.com/sismics/docs.git"
},
"author": "Benjamin Gamard",
"contributors": [
"Cornelius Hoffmann <coding@hoffmn.de>"
],
"license": "GPL-2.0",
"bugs": {
"url": "https://github.com/sismics/docs/issues"
@ -18,9 +21,10 @@
"homepage": "https://github.com/sismics/docs#readme",
"dependencies": {
"inquirer": "^5.1.0",
"minimist": "^1.2.0",
"minimist": "^1.2.5",
"ora": "^2.0.0",
"preferences": "^1.0.2",
"qs": "^6.9.4",
"recursive-readdir": "^2.2.2",
"request": "^2.83.0",
"underscore": "^1.8.3"

9
docs-importer/pref Normal file
View File

@ -0,0 +1,9 @@
importer:
daemon: true
path: import
tag: 'env1'
addtags: 'env2'
lang: 'env3'
baseUrl: 'env4'
username: 'env5'
password: 'env6'