Merge remote-tracking branch 'origin/master'

.travis.yml

@@ -4,7 +4,7 @@ language: java
 before_install:
   - sudo add-apt-repository -y ppa:mc3man/trusty-media
   - sudo apt-get -qq update
-  - sudo apt-get -y -q install ffmpeg mediainfo tesseract-ocr tesseract-ocr-fra tesseract-ocr-ita tesseract-ocr-kor tesseract-ocr-rus tesseract-ocr-ukr tesseract-ocr-spa tesseract-ocr-ara tesseract-ocr-hin tesseract-ocr-deu tesseract-ocr-pol tesseract-ocr-jpn tesseract-ocr-por tesseract-ocr-tha tesseract-ocr-jpn tesseract-ocr-chi-sim tesseract-ocr-chi-tra tesseract-ocr-nld tesseract-ocr-tur tesseract-ocr-heb tesseract-ocr-hun tesseract-ocr-fin tesseract-ocr-swe tesseract-ocr-lav
+  - sudo apt-get -y -q install ffmpeg mediainfo tesseract-ocr tesseract-ocr-fra tesseract-ocr-ita tesseract-ocr-kor tesseract-ocr-rus tesseract-ocr-ukr tesseract-ocr-spa tesseract-ocr-ara tesseract-ocr-hin tesseract-ocr-deu tesseract-ocr-pol tesseract-ocr-jpn tesseract-ocr-por tesseract-ocr-tha tesseract-ocr-jpn tesseract-ocr-chi-sim tesseract-ocr-chi-tra tesseract-ocr-nld tesseract-ocr-tur tesseract-ocr-heb tesseract-ocr-hun tesseract-ocr-fin tesseract-ocr-swe tesseract-ocr-lav tesseract-ocr-dan
   - sudo apt-get -y -q install haveged && sudo service haveged start
 after_success:
   - |
@@ -17,6 +17,13 @@ after_success:
       docker tag $REPO:$COMMIT $REPO:$TAG
       docker tag $REPO:$COMMIT $REPO:travis-$TRAVIS_BUILD_NUMBER
       docker push $REPO
+      cd docs-importer
+      export REPO=sismics/docs-importer
+      export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi`
+      docker build -f Dockerfile -t $REPO:$COMMIT .
+      docker tag $REPO:$COMMIT $REPO:$TAG
+      docker tag $REPO:$COMMIT $REPO:travis-$TRAVIS_BUILD_NUMBER
+      docker push $REPO
     fi
 env:
   global:

Dockerfile

@@ -1,7 +1,7 @@
 FROM sismics/ubuntu-jetty:9.4.12
 MAINTAINER b.gamard@sismics.com
 
-RUN apt-get update && apt-get -y -q install ffmpeg mediainfo tesseract-ocr tesseract-ocr-fra tesseract-ocr-ita tesseract-ocr-kor tesseract-ocr-rus tesseract-ocr-ukr tesseract-ocr-spa tesseract-ocr-ara tesseract-ocr-hin tesseract-ocr-deu tesseract-ocr-pol tesseract-ocr-jpn tesseract-ocr-por tesseract-ocr-tha tesseract-ocr-jpn tesseract-ocr-chi-sim tesseract-ocr-chi-tra tesseract-ocr-nld tesseract-ocr-tur tesseract-ocr-heb tesseract-ocr-hun tesseract-ocr-fin tesseract-ocr-swe tesseract-ocr-lav && \
+RUN apt-get update && apt-get -y -q install ffmpeg mediainfo tesseract-ocr tesseract-ocr-fra tesseract-ocr-ita tesseract-ocr-kor tesseract-ocr-rus tesseract-ocr-ukr tesseract-ocr-spa tesseract-ocr-ara tesseract-ocr-hin tesseract-ocr-deu tesseract-ocr-pol tesseract-ocr-jpn tesseract-ocr-por tesseract-ocr-tha tesseract-ocr-jpn tesseract-ocr-chi-sim tesseract-ocr-chi-tra tesseract-ocr-nld tesseract-ocr-tur tesseract-ocr-heb tesseract-ocr-hun tesseract-ocr-fin tesseract-ocr-swe tesseract-ocr-lav tesseract-ocr-dan && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Remove the embedded javax.mail jar from Jetty

docs-core/src/main/java/com/sismics/docs/core/constant/Constants.java

@@ -38,7 +38,7 @@ public class Constants {
     /**
      * Supported document languages.
      */
-    public static final List<String> SUPPORTED_LANGUAGES = Lists.newArrayList("eng", "fra", "ita", "deu", "spa", "por", "pol", "rus", "ukr", "ara", "hin", "chi_sim", "chi_tra", "jpn", "tha", "kor", "nld", "tur", "heb", "hun", "fin", "swe", "lav");
+    public static final List<String> SUPPORTED_LANGUAGES = Lists.newArrayList("eng", "fra", "ita", "deu", "spa", "por", "pol", "rus", "ukr", "ara", "hin", "chi_sim", "chi_tra", "jpn", "tha", "kor", "nld", "tur", "heb", "hun", "fin", "swe", "lav", "dan");
 
     /**
      * Base URL environment variable.

docs-importer/Dockerfile

@@ -1,12 +1,11 @@
 FROM node:14.2-alpine AS builder
 WORKDIR /build
-COPY node_modules/ ./node_modules/
 COPY main.js package-lock.json package.json ./
 RUN npm install && npm install -g pkg
 RUN pkg -t node14-alpine-x64 .
 
 FROM alpine
-ENV TEEDY_TAG= TEEDY_ADDTAGS=false TEEDY_LANG=eng TEEDY_URL='http://localhost:8080' TEEDY_USERNAME=username TEEDY_PASSWORD=password TEEDY_COPYFOLDER=
+ENV TEEDY_TAG= TEEDY_ADDTAGS=false TEEDY_LANG=eng TEEDY_URL='http://localhost:8080' TEEDY_USERNAME=username TEEDY_PASSWORD=password TEEDY_COPYFOLDER= TEEDY_FILEFILTER=
 RUN apk add --no-cache \
     libc6-compat \
     libstdc++ 

docs-importer/README.md

@@ -29,21 +29,19 @@ The daemon mode scan the input directory every 30 seconds for new files. Once a
 
 ## Docker
 
-The docker image needs a volume mount of a previously generated preference file to `/root/.config/preferences/com.sismics.docs.importer.pref`. The container will start the importer in daemon mode. It will look for files in `/import`.
+The docker image needs a volume mounted from a previously generated preference file at `/root/.config/preferences/com.sismics.docs.importer.pref`. The container will start the importer in daemon mode. It will look for files in `/import`.
 Example usage:
 
 ```
-docker build -t teedy-import .
+docker run --name teedy-import -d -v /path/to/preferencefile:/root/.config/preferences/com.sismics.docs.importer.pref -v /path/to/import/folder:/import sismics/docs-importer:latest
-docker run --name teedy-import -d -v /path/to/preferencefile:/root/.config/preferences/com.sismics.docs.importer.pref -v /path/to/import/folder:/import teedy-import
 ```
 ### Environment variables
-Instead of mounting the preferences file, the options can also be set by setting the environment variables `TEEDY_TAG`, `TEEDY_ADDTAGS`, `TEEDY_LANG`, `TEEDY_COPYFOLDER`, `TEEDY_URL`, `TEEDY_USERNAME` and `TEEDY_PASSWORD`.
+Instead of mounting the preferences file, the options can also be set by setting the environment variables `TEEDY_TAG`, `TEEDY_ADDTAGS`, `TEEDY_LANG`, `TEEDY_COPYFOLDER`, `TEEDY_FILEFILTER`, `TEEDY_URL`, `TEEDY_USERNAME` and `TEEDY_PASSWORD`.
 The latter three have to be set for the importer to work. The value of `TEEDY_TAG` has to be set to the UUID of the tag, not the name (The UUID can be found by visiting `baseUrl/api/tag/list` in your browser).
 Example usage:
 
 ```
-docker build -t teedy-import .
+docker run --name teedy-import -d -e TEEDY_TAG=2071fdf7-0e26-409d-b53d-f25823a5eb9e -e TEEDY_ADDTAGS=false -e TEEDY_LANG=eng -e TEEDY_URL='http://teedy.example.com:port' -e TEEDY_USERNAME=username -e TEEDY_PASSWORD=superSecretPassword -v /path/to/import/folder:/import sismics/docs-importer:latest
-docker run --name teedy-import -d -e TEEDY_TAG=2071fdf7-0e26-409d-b53d-f25823a5eb9e -e TEEDY_ADDTAGS=false -e TEEDY_LANG=eng -e TEEDY_URL='http://teedy.example.com:port' -e TEEDY_USERNAME=username -e TEEDY_PASSWORD=superSecretPassword -v /path/to/import/folder:/import teedy-import
 ```
 
 ## Build from sources

docs-importer/env.sh

@@ -7,4 +7,5 @@ sed -i "s,env4,$TEEDY_URL,g" $file
 sed -i "s/env5/$TEEDY_USERNAME/g" $file
 sed -i "s/env6/$TEEDY_PASSWORD/g" $file
 sed -i "s,env7,$TEEDY_COPYFOLDER,g" $file
+sed -i "s,env8,$TEEDY_FILEFILTER,g" $file
 echo "Environment variables replaced"

docs-importer/main.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const recursive = require('recursive-readdir');
+const minimatch = require("minimatch");
 const ora = require('ora');
 const inquirer = require('inquirer');
 const preferences = require('preferences');
@@ -142,13 +143,32 @@ const askPath = () => {
 
         recursive(answers.path, function (error, files) {
           spinner.succeed(files.length + ' files in this directory');
-          askTag();
+          askFileFilter();
         });
       });
     });
   });
 };
 
+// Ask for the file filter
+const askFileFilter = () => {
+  console.log('');
+
+  inquirer.prompt([
+    {
+      type: 'input',
+      name: 'fileFilter',
+      message: 'What pattern do you want to use to match files? (eg. *.+(pdf|txt|jpg))',
+      default: prefs.importer.fileFilter || "*"
+    }
+  ]).then(answers => {
+    // Save fileFilter
+    prefs.importer.fileFilter = answers.fileFilter;
+
+    askTag();
+  });
+};
+
 // Ask for the tag to add
 const askTag = () => {
   console.log('');
@@ -344,6 +364,8 @@ const start = () => {
 // Import the files
 const importFiles = (remove, filesImported) => {
   recursive(prefs.importer.path, function (error, files) {
+
+    files = files.filter(minimatch.filter(prefs.importer.fileFilter ?? "*", {matchBase: true}));
     if (files.length === 0) {
       filesImported();
       return;
@@ -393,6 +415,12 @@ const importFile = (file, remove, resolve) => {
     // Intersect tags from filename with existing tags on server
     let foundtags = [];
     for (let j of taglist) {
+      // If the tag is last in the filename it could include a file extension and would not be recognized
+      if (j.includes('.') && !tagsarray.hasOwnProperty(j) && !foundtags.includes(tagsarray[j])) {
+        while (j.includes('.') && !tagsarray.hasOwnProperty(j)) {
+          j = j.replace(/\.[^.]*$/,'');
+        }
+      }
       if (tagsarray.hasOwnProperty(j) && !foundtags.includes(tagsarray[j])) {
         foundtags.push(tagsarray[j]);
         filename = filename.split('#'+j).join('');
@@ -413,7 +441,8 @@ const importFile = (file, remove, resolve) => {
     else {
       data = {
         title: prefs.importer.addtags ? filename : file.replace(/^.*[\\\/]/, '').substring(0, 100),
-        language: prefs.importer.lang || 'eng'
+        language: prefs.importer.lang || 'eng',
+        tags: prefs.importer.tag === '' ? undefined : prefs.importer.tag
       }
     }
     // Create document
@@ -464,7 +493,8 @@ if (argv.hasOwnProperty('d')) {
     'Add tags given #: ' + prefs.importer.addtags + '\n' +
     'Language: ' + prefs.importer.lang + '\n' +
     'Daemon mode: ' + prefs.importer.daemon + '\n' +
-    'Copy folder: ' + prefs.importer.copyFolder
+    'Copy folder: ' + prefs.importer.copyFolder + '\n' +
+    'File filter: ' + prefs.importer.fileFilter
     );
   start();
 } else {

docs-importer/package.json

@@ -26,6 +26,7 @@
     "preferences": "^1.0.2",
     "qs": "^6.9.4",
     "recursive-readdir": "^2.2.2",
+    "minimatch": "^3.0.4",
     "request": "^2.83.0",
     "underscore": "^1.8.3"
   }

docs-importer/pref

@@ -7,4 +7,5 @@ importer:
   baseUrl: 'env4'
   username: 'env5'
   password: 'env6'
-  copyFolder: 'env7'
+  copyFolder: 'env7'
+  fileFilter: 'env8'

docs-web/src/main/java/com/sismics/docs/rest/resource/RouteResource.java

@@ -209,7 +209,9 @@ public class RouteResource extends BaseResource {
         routeStepDao.endRouteStep(routeStepDto.getId(), routeStepTransition, comment, principal.getId());
         RouteStepDto newRouteStep = routeStepDao.getCurrentStep(documentId);
         RoutingUtil.updateAcl(documentId, newRouteStep, routeStepDto, principal.getId());
-        RoutingUtil.sendRouteStepEmail(documentId, routeStepDto);
+        if (newRouteStep != null) {
+            RoutingUtil.sendRouteStepEmail(documentId, newRouteStep);
+        }
 
         JsonObjectBuilder response = Json.createObjectBuilder()
                 .add("readable", aclDao.checkPermission(documentId, PermType.READ, getTargetIdList(null)));

docs-web/src/main/webapp/src/app/docs/app.js

@@ -527,7 +527,8 @@ angular.module('docs',
     { key: 'hun', label: 'Magyar' },
     { key: 'fin', label: 'Suomi' },
     { key: 'swe', label: 'Svenska' },
-    { key: 'lav', label: 'Latviešu' }  
+    { key: 'lav', label: 'Latviešu' },
+    { key: 'dan', label: 'Dansk' }
   ];
 })
 /**

docs-web/src/test/java/com/sismics/docs/rest/TestRouteResource.java

@@ -226,7 +226,7 @@ public class TestRouteResource extends BaseJerseyTest {
                         .param("transition", "APPROVED")), JsonObject.class);
         Assert.assertFalse(json.containsKey("route_step"));
         Assert.assertTrue(json.getBoolean("readable")); // Admin can read everything
-        Assert.assertTrue(popEmail().contains("workflow step"));
+        Assert.assertNull(popEmail()); // Last step does not send any email
 
         // Get the route on document 1
         json = target().path("/route")