Improve Inbox Scanning (#407)

closes #386: delete emails after import + closes #405: auto tag documents imported by email

docs-core/src/main/java/com/sismics/docs/core/constant/ConfigType.java

@@ -42,5 +42,7 @@ public enum ConfigType {
     INBOX_PORT,
     INBOX_USERNAME,
     INBOX_PASSWORD,
-    INBOX_TAG
+    INBOX_TAG,
+    INBOX_AUTOMATIC_TAGS,
+    INBOX_DELETE_IMPORTED
 }

docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java

@@ -1,16 +1,21 @@
 package com.sismics.docs.core.service;
 
+import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.AbstractScheduledService;
 import com.sismics.docs.core.constant.ConfigType;
 import com.sismics.docs.core.dao.TagDao;
+import com.sismics.docs.core.dao.criteria.TagCriteria;
+import com.sismics.docs.core.dao.dto.TagDto;
 import com.sismics.docs.core.event.DocumentCreatedAsyncEvent;
+import com.sismics.docs.core.model.jpa.Config;
 import com.sismics.docs.core.model.jpa.Document;
 import com.sismics.docs.core.model.jpa.Tag;
 import com.sismics.docs.core.util.ConfigUtil;
 import com.sismics.docs.core.util.DocumentUtil;
 import com.sismics.docs.core.util.FileUtil;
 import com.sismics.docs.core.util.TransactionUtil;
+import com.sismics.docs.core.util.jpa.SortCriteria;
 import com.sismics.util.EmailUtil;
 import com.sismics.util.context.ThreadLocalContext;
 import org.apache.commons.lang.StringUtils;
@@ -19,9 +24,10 @@ import org.slf4j.LoggerFactory;
 
 import javax.mail.*;
 import javax.mail.search.FlagTerm;
-import java.util.Date;
+import java.util.*;
-import java.util.Properties;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Inbox scanning service.
@@ -79,11 +85,13 @@ public class InboxService extends AbstractScheduledService {
             lastSyncDate = new Date();
             lastSyncMessageCount = 0;
             try {
+                HashMap<String, String> tagsNameToId = getAllTags();
+
                 inbox = openInbox();
                 Message[] messages = inbox.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false));
                 log.info(messages.length + " messages found");
                 for (Message message : messages) {
-                    importMessage(message);
+                    importMessage(message, tagsNameToId);
                     lastSyncMessageCount++;
                 }
             } catch (FolderClosedException e) {
@@ -94,7 +102,8 @@ public class InboxService extends AbstractScheduledService {
             } finally {
                 try {
                     if (inbox != null) {
-                        inbox.close(false);
+                        // The parameter controls if the messages flagged to be deleted, should actually get deleted.
+                        inbox.close(ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_DELETE_IMPORTED));
                         inbox.getStore().close();
                     }
                 } catch (Exception e) {
@@ -183,7 +192,7 @@ public class InboxService extends AbstractScheduledService {
      * @param message Message
      * @throws Exception e
      */
-    private void importMessage(Message message) throws Exception {
+    private void importMessage(Message message, HashMap<String, String> tags) throws Exception {
         log.info("Importing message: " + message.getSubject());
 
         // Parse the mail
@@ -194,12 +203,27 @@ public class InboxService extends AbstractScheduledService {
 
         // Create the document
         Document document = new Document();
-        document.setUserId("admin");
+        String subject = mailContent.getSubject();
-        if (mailContent.getSubject() == null) {
+        if (subject == null) {
-            document.setTitle("Imported email from EML file");
+            subject = "Imported email from EML file";
-        } else {
-            document.setTitle(StringUtils.abbreviate(mailContent.getSubject(), 100));
         }
+
+        HashSet<String> tagsFound = new HashSet<>();
+        if (tags != null) {
+            Pattern pattern = Pattern.compile("#([^\\s:#]+)");
+            Matcher matcher = pattern.matcher(subject);
+            while (matcher.find()) {
+                if (tags.containsKey(matcher.group(1)) && tags.get(matcher.group(1)) != null) {
+                    tagsFound.add(tags.get(matcher.group(1)));
+                    subject = subject.replaceFirst("#" + matcher.group(1), "");
+                }
+            }
+            log.debug("Tags found: " + String.join(", ", tagsFound));
+            subject = subject.trim().replaceAll(" +", " ");
+        }
+
+        document.setUserId("admin");
+        document.setTitle(StringUtils.abbreviate(subject, 100));
         document.setDescription(StringUtils.abbreviate(mailContent.getMessage(), 4000));
         document.setSubject(StringUtils.abbreviate(mailContent.getSubject(), 500));
         document.setFormat("EML");
@@ -220,10 +244,15 @@ public class InboxService extends AbstractScheduledService {
             TagDao tagDao = new TagDao();
             Tag tag = tagDao.getById(tagId);
             if (tag != null) {
-                tagDao.updateTagList(document.getId(), Sets.newHashSet(tagId));
+                tagsFound.add(tagId);
             }
         }
 
+        // Update tags
+        if (!tagsFound.isEmpty()) {
+            new TagDao().updateTagList(document.getId(), tagsFound);
+        }
+
         // Raise a document created event
         DocumentCreatedAsyncEvent documentCreatedAsyncEvent = new DocumentCreatedAsyncEvent();
         documentCreatedAsyncEvent.setUserId("admin");
@@ -235,6 +264,29 @@ public class InboxService extends AbstractScheduledService {
             FileUtil.createFile(fileContent.getName(), null, fileContent.getFile(), fileContent.getSize(),
                     document.getLanguage(), "admin", document.getId());
         }
+
+        if (ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_DELETE_IMPORTED)) {
+            message.setFlag(Flags.Flag.DELETED, true);
+        }
+    }
+
+    /**
+     * Fetches a HashMap with all tag names as keys and their respective ids as values.
+     *
+     * @return HashMap with all tags or null if not enabled
+     */
+    private HashMap<String, String> getAllTags() {
+        if (!ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_AUTOMATIC_TAGS)) {
+            return null;
+        }
+        TagDao tagDao = new TagDao();
+        List<TagDto> tags = tagDao.findByCriteria(new TagCriteria().setTargetIdList(null), new SortCriteria(1, true));
+
+        HashMap<String, String> tagsNameToId = new HashMap<>();
+        for (TagDto tagDto : tags) {
+            tagsNameToId.put(tagDto.getName(), tagDto.getId());
+        }
+        return tagsNameToId;
     }
 
     public Date getLastSyncDate() {

docs-core/src/main/resources/config.properties

@@ -1 +1 @@
-db.version=24
+db.version=25

docs-core/src/main/resources/db/update/dbupdate-025-0.sql

@@ -0,0 +1,3 @@
+insert into T_CONFIG(CFG_ID_C, CFG_VALUE_C) values('INBOX_AUTOMATIC_TAGS', 'false');
+insert into T_CONFIG(CFG_ID_C, CFG_VALUE_C) values('INBOX_DELETE_IMPORTED', 'false');
+update T_CONFIG set CFG_VALUE_C = '25' where CFG_ID_C = 'DB_VERSION';

docs-web/src/dev/resources/config.properties

@@ -1,3 +1,3 @@
 api.current_version=${project.version}
 api.min_version=1.0
-db.version=24
+db.version=25

docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java

@@ -328,6 +328,8 @@ public class AppResource extends BaseResource {
 
         ConfigDao configDao = new ConfigDao();
         Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED);
+        Boolean autoTags = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_AUTOMATIC_TAGS);
+        Boolean deleteImported = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_DELETE_IMPORTED);
         Config hostnameConfig = configDao.getById(ConfigType.INBOX_HOSTNAME);
         Config portConfig = configDao.getById(ConfigType.INBOX_PORT);
         Config usernameConfig = configDao.getById(ConfigType.INBOX_USERNAME);
@@ -336,6 +338,8 @@ public class AppResource extends BaseResource {
         JsonObjectBuilder response = Json.createObjectBuilder();
 
         response.add("enabled", enabled);
+        response.add("autoTagsEnabled", autoTags);
+        response.add("deleteImported", deleteImported);
         if (hostnameConfig == null) {
             response.addNull("hostname");
         } else {
@@ -405,6 +409,8 @@ public class AppResource extends BaseResource {
     @POST
     @Path("config_inbox")
     public Response configInbox(@FormParam("enabled") Boolean enabled,
+                                @FormParam("autoTagsEnabled") Boolean autoTagsEnabled,
+                                @FormParam("deleteImported") Boolean deleteImported,
                                 @FormParam("hostname") String hostname,
                                 @FormParam("port") String portStr,
                                 @FormParam("username") String username,
@@ -422,6 +428,8 @@ public class AppResource extends BaseResource {
         // Just update the changed configuration
         ConfigDao configDao = new ConfigDao();
         configDao.update(ConfigType.INBOX_ENABLED, enabled.toString());
+        configDao.update(ConfigType.INBOX_AUTOMATIC_TAGS, autoTagsEnabled.toString());
+        configDao.update(ConfigType.INBOX_DELETE_IMPORTED, deleteImported.toString());
         if (!Strings.isNullOrEmpty(hostname)) {
             configDao.update(ConfigType.INBOX_HOSTNAME, hostname);
         }

docs-web/src/main/webapp/src/locale/en.json

@@ -432,7 +432,9 @@
       "last_sync": "Last synchronization: {{ data.date | date: 'medium' }}, {{ data.count }} message{{ data.count > 1 ? 's' : '' }} imported",
       "test_success": "The connection to the inbox is successful ({{ count }} <strong>unread</strong> message{{ count > 1 ? 's' : '' }})",
       "test_fail": "An error occurred while connecting to the inbox, please check the parameters",
-      "saved": "IMAP configuration saved successfully"
+      "saved": "IMAP configuration saved successfully",
+      "autoTagsEnabled": "Automatically add tags from subject line marked with #",
+      "deleteImported": "Delete message from mailbox after import"
     },
     "monitoring": {
       "background_tasks": "Background tasks",

docs-web/src/main/webapp/src/partial/docs/settings.inbox.html

@@ -17,6 +17,20 @@
     </div>
   </div>
 
+  <div class="form-group">
+    <label class="col-sm-2 control-label" for="inboxAutoTagsEnabled">{{ 'settings.inbox.autoTagsEnabled' | translate }}</label>
+    <div class="col-sm-7">
+      <input name="autoTagsEnabled" type="checkbox" id="inboxAutoTagsEnabled" ng-model="inbox.autoTagsEnabled" />
+    </div>
+  </div>
+
+  <div class="form-group">
+    <label class="col-sm-2 control-label" for="inboxDeleteImported">{{ 'settings.inbox.deleteImported' | translate }}</label>
+    <div class="col-sm-7">
+      <input name="deleteImported" type="checkbox" id="inboxDeleteImported" ng-model="inbox.deleteImported" />
+    </div>
+  </div>
+
   <div class="form-group" ng-class="{ 'has-error': !inboxForm.hostname.$valid && inboxForm.$dirty }">
     <label class="col-sm-2 control-label" for="inboxHostname">{{ 'settings.inbox.hostname' | translate }}</label>
     <div class="col-sm-7">

docs-web/src/prod/resources/config.properties

@@ -1,3 +1,3 @@
 api.current_version=${project.version}
 api.min_version=1.0
-db.version=24
+db.version=25

docs-web/src/stress/resources/config.properties

@@ -1,3 +1,3 @@
 api.current_version=${project.version}
 api.min_version=1.0
-db.version=24
+db.version=25

docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java

@@ -254,6 +254,8 @@ public class TestAppResource extends BaseJerseyTest {
                 .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
                 .post(Entity.form(new Form()
                         .param("enabled", "true")
+                        .param("autoTagsEnabled", "false")
+                        .param("deleteImported", "false")
                         .param("hostname", "localhost")
                         .param("port", "9755")
                         .param("username", "test@sismics.com")