Merge remote-tracking branch 'origin/master'

This commit is contained in:
bgamard 2020-08-28 17:34:03 +02:00
commit e1fa17691d
12 changed files with 58 additions and 18 deletions

View File

@ -4,7 +4,7 @@ language: java
before_install: before_install:
- sudo add-apt-repository -y ppa:mc3man/trusty-media - sudo add-apt-repository -y ppa:mc3man/trusty-media
- sudo apt-get -qq update - sudo apt-get -qq update
- sudo apt-get -y -q install ffmpeg mediainfo tesseract-ocr tesseract-ocr-fra tesseract-ocr-ita tesseract-ocr-kor tesseract-ocr-rus tesseract-ocr-ukr tesseract-ocr-spa tesseract-ocr-ara tesseract-ocr-hin tesseract-ocr-deu tesseract-ocr-pol tesseract-ocr-jpn tesseract-ocr-por tesseract-ocr-tha tesseract-ocr-jpn tesseract-ocr-chi-sim tesseract-ocr-chi-tra tesseract-ocr-nld tesseract-ocr-tur tesseract-ocr-heb tesseract-ocr-hun tesseract-ocr-fin tesseract-ocr-swe tesseract-ocr-lav - sudo apt-get -y -q install ffmpeg mediainfo tesseract-ocr tesseract-ocr-fra tesseract-ocr-ita tesseract-ocr-kor tesseract-ocr-rus tesseract-ocr-ukr tesseract-ocr-spa tesseract-ocr-ara tesseract-ocr-hin tesseract-ocr-deu tesseract-ocr-pol tesseract-ocr-jpn tesseract-ocr-por tesseract-ocr-tha tesseract-ocr-jpn tesseract-ocr-chi-sim tesseract-ocr-chi-tra tesseract-ocr-nld tesseract-ocr-tur tesseract-ocr-heb tesseract-ocr-hun tesseract-ocr-fin tesseract-ocr-swe tesseract-ocr-lav tesseract-ocr-dan
- sudo apt-get -y -q install haveged && sudo service haveged start - sudo apt-get -y -q install haveged && sudo service haveged start
after_success: after_success:
- | - |
@ -17,6 +17,13 @@ after_success:
docker tag $REPO:$COMMIT $REPO:$TAG docker tag $REPO:$COMMIT $REPO:$TAG
docker tag $REPO:$COMMIT $REPO:travis-$TRAVIS_BUILD_NUMBER docker tag $REPO:$COMMIT $REPO:travis-$TRAVIS_BUILD_NUMBER
docker push $REPO docker push $REPO
cd docs-importer
export REPO=sismics/docs-importer
export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi`
docker build -f Dockerfile -t $REPO:$COMMIT .
docker tag $REPO:$COMMIT $REPO:$TAG
docker tag $REPO:$COMMIT $REPO:travis-$TRAVIS_BUILD_NUMBER
docker push $REPO
fi fi
env: env:
global: global:

View File

@ -1,7 +1,7 @@
FROM sismics/ubuntu-jetty:9.4.12 FROM sismics/ubuntu-jetty:9.4.12
MAINTAINER b.gamard@sismics.com MAINTAINER b.gamard@sismics.com
RUN apt-get update && apt-get -y -q install ffmpeg mediainfo tesseract-ocr tesseract-ocr-fra tesseract-ocr-ita tesseract-ocr-kor tesseract-ocr-rus tesseract-ocr-ukr tesseract-ocr-spa tesseract-ocr-ara tesseract-ocr-hin tesseract-ocr-deu tesseract-ocr-pol tesseract-ocr-jpn tesseract-ocr-por tesseract-ocr-tha tesseract-ocr-jpn tesseract-ocr-chi-sim tesseract-ocr-chi-tra tesseract-ocr-nld tesseract-ocr-tur tesseract-ocr-heb tesseract-ocr-hun tesseract-ocr-fin tesseract-ocr-swe tesseract-ocr-lav && \ RUN apt-get update && apt-get -y -q install ffmpeg mediainfo tesseract-ocr tesseract-ocr-fra tesseract-ocr-ita tesseract-ocr-kor tesseract-ocr-rus tesseract-ocr-ukr tesseract-ocr-spa tesseract-ocr-ara tesseract-ocr-hin tesseract-ocr-deu tesseract-ocr-pol tesseract-ocr-jpn tesseract-ocr-por tesseract-ocr-tha tesseract-ocr-jpn tesseract-ocr-chi-sim tesseract-ocr-chi-tra tesseract-ocr-nld tesseract-ocr-tur tesseract-ocr-heb tesseract-ocr-hun tesseract-ocr-fin tesseract-ocr-swe tesseract-ocr-lav tesseract-ocr-dan && \
apt-get clean && rm -rf /var/lib/apt/lists/* apt-get clean && rm -rf /var/lib/apt/lists/*
# Remove the embedded javax.mail jar from Jetty # Remove the embedded javax.mail jar from Jetty

View File

@ -38,7 +38,7 @@ public class Constants {
/** /**
* Supported document languages. * Supported document languages.
*/ */
public static final List<String> SUPPORTED_LANGUAGES = Lists.newArrayList("eng", "fra", "ita", "deu", "spa", "por", "pol", "rus", "ukr", "ara", "hin", "chi_sim", "chi_tra", "jpn", "tha", "kor", "nld", "tur", "heb", "hun", "fin", "swe", "lav"); public static final List<String> SUPPORTED_LANGUAGES = Lists.newArrayList("eng", "fra", "ita", "deu", "spa", "por", "pol", "rus", "ukr", "ara", "hin", "chi_sim", "chi_tra", "jpn", "tha", "kor", "nld", "tur", "heb", "hun", "fin", "swe", "lav", "dan");
/** /**
* Base URL environment variable. * Base URL environment variable.

View File

@ -1,12 +1,11 @@
FROM node:14.2-alpine AS builder FROM node:14.2-alpine AS builder
WORKDIR /build WORKDIR /build
COPY node_modules/ ./node_modules/
COPY main.js package-lock.json package.json ./ COPY main.js package-lock.json package.json ./
RUN npm install && npm install -g pkg RUN npm install && npm install -g pkg
RUN pkg -t node14-alpine-x64 . RUN pkg -t node14-alpine-x64 .
FROM alpine FROM alpine
ENV TEEDY_TAG= TEEDY_ADDTAGS=false TEEDY_LANG=eng TEEDY_URL='http://localhost:8080' TEEDY_USERNAME=username TEEDY_PASSWORD=password TEEDY_COPYFOLDER= ENV TEEDY_TAG= TEEDY_ADDTAGS=false TEEDY_LANG=eng TEEDY_URL='http://localhost:8080' TEEDY_USERNAME=username TEEDY_PASSWORD=password TEEDY_COPYFOLDER= TEEDY_FILEFILTER=
RUN apk add --no-cache \ RUN apk add --no-cache \
libc6-compat \ libc6-compat \
libstdc++ libstdc++

View File

@ -29,21 +29,19 @@ The daemon mode scan the input directory every 30 seconds for new files. Once a
## Docker ## Docker
The docker image needs a volume mount of a previously generated preference file to `/root/.config/preferences/com.sismics.docs.importer.pref`. The container will start the importer in daemon mode. It will look for files in `/import`. The docker image needs a volume mounted from a previously generated preference file at `/root/.config/preferences/com.sismics.docs.importer.pref`. The container will start the importer in daemon mode. It will look for files in `/import`.
Example usage: Example usage:
``` ```
docker build -t teedy-import . docker run --name teedy-import -d -v /path/to/preferencefile:/root/.config/preferences/com.sismics.docs.importer.pref -v /path/to/import/folder:/import sismics/docs-importer:latest
docker run --name teedy-import -d -v /path/to/preferencefile:/root/.config/preferences/com.sismics.docs.importer.pref -v /path/to/import/folder:/import teedy-import
``` ```
### Environment variables ### Environment variables
Instead of mounting the preferences file, the options can also be set by setting the environment variables `TEEDY_TAG`, `TEEDY_ADDTAGS`, `TEEDY_LANG`, `TEEDY_COPYFOLDER`, `TEEDY_URL`, `TEEDY_USERNAME` and `TEEDY_PASSWORD`. Instead of mounting the preferences file, the options can also be set by setting the environment variables `TEEDY_TAG`, `TEEDY_ADDTAGS`, `TEEDY_LANG`, `TEEDY_COPYFOLDER`, `TEEDY_FILEFILTER`, `TEEDY_URL`, `TEEDY_USERNAME` and `TEEDY_PASSWORD`.
The latter three have to be set for the importer to work. The value of `TEEDY_TAG` has to be set to the UUID of the tag, not the name (The UUID can be found by visiting `baseUrl/api/tag/list` in your browser). The latter three have to be set for the importer to work. The value of `TEEDY_TAG` has to be set to the UUID of the tag, not the name (The UUID can be found by visiting `baseUrl/api/tag/list` in your browser).
Example usage: Example usage:
``` ```
docker build -t teedy-import . docker run --name teedy-import -d -e TEEDY_TAG=2071fdf7-0e26-409d-b53d-f25823a5eb9e -e TEEDY_ADDTAGS=false -e TEEDY_LANG=eng -e TEEDY_URL='http://teedy.example.com:port' -e TEEDY_USERNAME=username -e TEEDY_PASSWORD=superSecretPassword -v /path/to/import/folder:/import sismics/docs-importer:latest
docker run --name teedy-import -d -e TEEDY_TAG=2071fdf7-0e26-409d-b53d-f25823a5eb9e -e TEEDY_ADDTAGS=false -e TEEDY_LANG=eng -e TEEDY_URL='http://teedy.example.com:port' -e TEEDY_USERNAME=username -e TEEDY_PASSWORD=superSecretPassword -v /path/to/import/folder:/import teedy-import
``` ```
## Build from sources ## Build from sources

View File

@ -7,4 +7,5 @@ sed -i "s,env4,$TEEDY_URL,g" $file
sed -i "s/env5/$TEEDY_USERNAME/g" $file sed -i "s/env5/$TEEDY_USERNAME/g" $file
sed -i "s/env6/$TEEDY_PASSWORD/g" $file sed -i "s/env6/$TEEDY_PASSWORD/g" $file
sed -i "s,env7,$TEEDY_COPYFOLDER,g" $file sed -i "s,env7,$TEEDY_COPYFOLDER,g" $file
sed -i "s,env8,$TEEDY_FILEFILTER,g" $file
echo "Environment variables replaced" echo "Environment variables replaced"

View File

@ -1,6 +1,7 @@
'use strict'; 'use strict';
const recursive = require('recursive-readdir'); const recursive = require('recursive-readdir');
const minimatch = require("minimatch");
const ora = require('ora'); const ora = require('ora');
const inquirer = require('inquirer'); const inquirer = require('inquirer');
const preferences = require('preferences'); const preferences = require('preferences');
@ -142,13 +143,32 @@ const askPath = () => {
recursive(answers.path, function (error, files) { recursive(answers.path, function (error, files) {
spinner.succeed(files.length + ' files in this directory'); spinner.succeed(files.length + ' files in this directory');
askTag(); askFileFilter();
}); });
}); });
}); });
}); });
}; };
// Ask for the file filter
const askFileFilter = () => {
console.log('');
inquirer.prompt([
{
type: 'input',
name: 'fileFilter',
message: 'What pattern do you want to use to match files? (eg. *.+(pdf|txt|jpg))',
default: prefs.importer.fileFilter || "*"
}
]).then(answers => {
// Save fileFilter
prefs.importer.fileFilter = answers.fileFilter;
askTag();
});
};
// Ask for the tag to add // Ask for the tag to add
const askTag = () => { const askTag = () => {
console.log(''); console.log('');
@ -344,6 +364,8 @@ const start = () => {
// Import the files // Import the files
const importFiles = (remove, filesImported) => { const importFiles = (remove, filesImported) => {
recursive(prefs.importer.path, function (error, files) { recursive(prefs.importer.path, function (error, files) {
files = files.filter(minimatch.filter(prefs.importer.fileFilter ?? "*", {matchBase: true}));
if (files.length === 0) { if (files.length === 0) {
filesImported(); filesImported();
return; return;
@ -393,6 +415,12 @@ const importFile = (file, remove, resolve) => {
// Intersect tags from filename with existing tags on server // Intersect tags from filename with existing tags on server
let foundtags = []; let foundtags = [];
for (let j of taglist) { for (let j of taglist) {
// If the tag is last in the filename it could include a file extension and would not be recognized
if (j.includes('.') && !tagsarray.hasOwnProperty(j) && !foundtags.includes(tagsarray[j])) {
while (j.includes('.') && !tagsarray.hasOwnProperty(j)) {
j = j.replace(/\.[^.]*$/,'');
}
}
if (tagsarray.hasOwnProperty(j) && !foundtags.includes(tagsarray[j])) { if (tagsarray.hasOwnProperty(j) && !foundtags.includes(tagsarray[j])) {
foundtags.push(tagsarray[j]); foundtags.push(tagsarray[j]);
filename = filename.split('#'+j).join(''); filename = filename.split('#'+j).join('');
@ -413,7 +441,8 @@ const importFile = (file, remove, resolve) => {
else { else {
data = { data = {
title: prefs.importer.addtags ? filename : file.replace(/^.*[\\\/]/, '').substring(0, 100), title: prefs.importer.addtags ? filename : file.replace(/^.*[\\\/]/, '').substring(0, 100),
language: prefs.importer.lang || 'eng' language: prefs.importer.lang || 'eng',
tags: prefs.importer.tag === '' ? undefined : prefs.importer.tag
} }
} }
// Create document // Create document
@ -464,7 +493,8 @@ if (argv.hasOwnProperty('d')) {
'Add tags given #: ' + prefs.importer.addtags + '\n' + 'Add tags given #: ' + prefs.importer.addtags + '\n' +
'Language: ' + prefs.importer.lang + '\n' + 'Language: ' + prefs.importer.lang + '\n' +
'Daemon mode: ' + prefs.importer.daemon + '\n' + 'Daemon mode: ' + prefs.importer.daemon + '\n' +
'Copy folder: ' + prefs.importer.copyFolder 'Copy folder: ' + prefs.importer.copyFolder + '\n' +
'File filter: ' + prefs.importer.fileFilter
); );
start(); start();
} else { } else {

View File

@ -26,6 +26,7 @@
"preferences": "^1.0.2", "preferences": "^1.0.2",
"qs": "^6.9.4", "qs": "^6.9.4",
"recursive-readdir": "^2.2.2", "recursive-readdir": "^2.2.2",
"minimatch": "^3.0.4",
"request": "^2.83.0", "request": "^2.83.0",
"underscore": "^1.8.3" "underscore": "^1.8.3"
} }

View File

@ -7,4 +7,5 @@ importer:
baseUrl: 'env4' baseUrl: 'env4'
username: 'env5' username: 'env5'
password: 'env6' password: 'env6'
copyFolder: 'env7' copyFolder: 'env7'
fileFilter: 'env8'

View File

@ -209,7 +209,9 @@ public class RouteResource extends BaseResource {
routeStepDao.endRouteStep(routeStepDto.getId(), routeStepTransition, comment, principal.getId()); routeStepDao.endRouteStep(routeStepDto.getId(), routeStepTransition, comment, principal.getId());
RouteStepDto newRouteStep = routeStepDao.getCurrentStep(documentId); RouteStepDto newRouteStep = routeStepDao.getCurrentStep(documentId);
RoutingUtil.updateAcl(documentId, newRouteStep, routeStepDto, principal.getId()); RoutingUtil.updateAcl(documentId, newRouteStep, routeStepDto, principal.getId());
RoutingUtil.sendRouteStepEmail(documentId, routeStepDto); if (newRouteStep != null) {
RoutingUtil.sendRouteStepEmail(documentId, newRouteStep);
}
JsonObjectBuilder response = Json.createObjectBuilder() JsonObjectBuilder response = Json.createObjectBuilder()
.add("readable", aclDao.checkPermission(documentId, PermType.READ, getTargetIdList(null))); .add("readable", aclDao.checkPermission(documentId, PermType.READ, getTargetIdList(null)));

View File

@ -527,7 +527,8 @@ angular.module('docs',
{ key: 'hun', label: 'Magyar' }, { key: 'hun', label: 'Magyar' },
{ key: 'fin', label: 'Suomi' }, { key: 'fin', label: 'Suomi' },
{ key: 'swe', label: 'Svenska' }, { key: 'swe', label: 'Svenska' },
{ key: 'lav', label: 'Latviešu' } { key: 'lav', label: 'Latviešu' },
{ key: 'dan', label: 'Dansk' }
]; ];
}) })
/** /**

View File

@ -226,7 +226,7 @@ public class TestRouteResource extends BaseJerseyTest {
.param("transition", "APPROVED")), JsonObject.class); .param("transition", "APPROVED")), JsonObject.class);
Assert.assertFalse(json.containsKey("route_step")); Assert.assertFalse(json.containsKey("route_step"));
Assert.assertTrue(json.getBoolean("readable")); // Admin can read everything Assert.assertTrue(json.getBoolean("readable")); // Admin can read everything
Assert.assertTrue(popEmail().contains("workflow step")); Assert.assertNull(popEmail()); // Last step does not send any email
// Get the route on document 1 // Get the route on document 1
json = target().path("/route") json = target().path("/route")