From 95c37a03f8ce31841f43fdfd5e75927b53b6b0b8 Mon Sep 17 00:00:00 2001 From: cadast <37808546+cadast@users.noreply.github.com> Date: Thu, 14 May 2020 13:59:11 +0200 Subject: [PATCH] Improve Inbox Scanning (#407) closes #386: delete emails after import + closes #405: auto tag documents imported by email --- .../docs/core/constant/ConfigType.java | 4 +- .../docs/core/service/InboxService.java | 74 ++++++++++++++++--- .../src/main/resources/config.properties | 2 +- .../resources/db/update/dbupdate-025-0.sql | 3 + docs-web/src/dev/resources/config.properties | 2 +- .../docs/rest/resource/AppResource.java | 8 ++ docs-web/src/main/webapp/src/locale/en.json | 4 +- .../src/partial/docs/settings.inbox.html | 14 ++++ docs-web/src/prod/resources/config.properties | 2 +- .../src/stress/resources/config.properties | 2 +- .../sismics/docs/rest/TestAppResource.java | 2 + 11 files changed, 100 insertions(+), 17 deletions(-) create mode 100644 docs-core/src/main/resources/db/update/dbupdate-025-0.sql diff --git a/docs-core/src/main/java/com/sismics/docs/core/constant/ConfigType.java b/docs-core/src/main/java/com/sismics/docs/core/constant/ConfigType.java index e4e41db5..5c7869d3 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/constant/ConfigType.java +++ b/docs-core/src/main/java/com/sismics/docs/core/constant/ConfigType.java @@ -42,5 +42,7 @@ public enum ConfigType { INBOX_PORT, INBOX_USERNAME, INBOX_PASSWORD, - INBOX_TAG + INBOX_TAG, + INBOX_AUTOMATIC_TAGS, + INBOX_DELETE_IMPORTED } diff --git a/docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java b/docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java index ae0b7500..0669dd37 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java +++ b/docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java @@ -1,16 +1,21 @@ package com.sismics.docs.core.service; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.AbstractScheduledService; import com.sismics.docs.core.constant.ConfigType; import com.sismics.docs.core.dao.TagDao; +import com.sismics.docs.core.dao.criteria.TagCriteria; +import com.sismics.docs.core.dao.dto.TagDto; import com.sismics.docs.core.event.DocumentCreatedAsyncEvent; +import com.sismics.docs.core.model.jpa.Config; import com.sismics.docs.core.model.jpa.Document; import com.sismics.docs.core.model.jpa.Tag; import com.sismics.docs.core.util.ConfigUtil; import com.sismics.docs.core.util.DocumentUtil; import com.sismics.docs.core.util.FileUtil; import com.sismics.docs.core.util.TransactionUtil; +import com.sismics.docs.core.util.jpa.SortCriteria; import com.sismics.util.EmailUtil; import com.sismics.util.context.ThreadLocalContext; import org.apache.commons.lang.StringUtils; @@ -19,9 +24,10 @@ import org.slf4j.LoggerFactory; import javax.mail.*; import javax.mail.search.FlagTerm; -import java.util.Date; -import java.util.Properties; +import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Inbox scanning service. @@ -79,11 +85,13 @@ public class InboxService extends AbstractScheduledService { lastSyncDate = new Date(); lastSyncMessageCount = 0; try { + HashMap tagsNameToId = getAllTags(); + inbox = openInbox(); Message[] messages = inbox.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false)); log.info(messages.length + " messages found"); for (Message message : messages) { - importMessage(message); + importMessage(message, tagsNameToId); lastSyncMessageCount++; } } catch (FolderClosedException e) { @@ -94,7 +102,8 @@ public class InboxService extends AbstractScheduledService { } finally { try { if (inbox != null) { - inbox.close(false); + // The parameter controls if the messages flagged to be deleted, should actually get deleted. + inbox.close(ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_DELETE_IMPORTED)); inbox.getStore().close(); } } catch (Exception e) { @@ -183,7 +192,7 @@ public class InboxService extends AbstractScheduledService { * @param message Message * @throws Exception e */ - private void importMessage(Message message) throws Exception { + private void importMessage(Message message, HashMap tags) throws Exception { log.info("Importing message: " + message.getSubject()); // Parse the mail @@ -194,12 +203,27 @@ public class InboxService extends AbstractScheduledService { // Create the document Document document = new Document(); - document.setUserId("admin"); - if (mailContent.getSubject() == null) { - document.setTitle("Imported email from EML file"); - } else { - document.setTitle(StringUtils.abbreviate(mailContent.getSubject(), 100)); + String subject = mailContent.getSubject(); + if (subject == null) { + subject = "Imported email from EML file"; } + + HashSet tagsFound = new HashSet<>(); + if (tags != null) { + Pattern pattern = Pattern.compile("#([^\\s:#]+)"); + Matcher matcher = pattern.matcher(subject); + while (matcher.find()) { + if (tags.containsKey(matcher.group(1)) && tags.get(matcher.group(1)) != null) { + tagsFound.add(tags.get(matcher.group(1))); + subject = subject.replaceFirst("#" + matcher.group(1), ""); + } + } + log.debug("Tags found: " + String.join(", ", tagsFound)); + subject = subject.trim().replaceAll(" +", " "); + } + + document.setUserId("admin"); + document.setTitle(StringUtils.abbreviate(subject, 100)); document.setDescription(StringUtils.abbreviate(mailContent.getMessage(), 4000)); document.setSubject(StringUtils.abbreviate(mailContent.getSubject(), 500)); document.setFormat("EML"); @@ -220,10 +244,15 @@ public class InboxService extends AbstractScheduledService { TagDao tagDao = new TagDao(); Tag tag = tagDao.getById(tagId); if (tag != null) { - tagDao.updateTagList(document.getId(), Sets.newHashSet(tagId)); + tagsFound.add(tagId); } } + // Update tags + if (!tagsFound.isEmpty()) { + new TagDao().updateTagList(document.getId(), tagsFound); + } + // Raise a document created event DocumentCreatedAsyncEvent documentCreatedAsyncEvent = new DocumentCreatedAsyncEvent(); documentCreatedAsyncEvent.setUserId("admin"); @@ -235,6 +264,29 @@ public class InboxService extends AbstractScheduledService { FileUtil.createFile(fileContent.getName(), null, fileContent.getFile(), fileContent.getSize(), document.getLanguage(), "admin", document.getId()); } + + if (ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_DELETE_IMPORTED)) { + message.setFlag(Flags.Flag.DELETED, true); + } + } + + /** + * Fetches a HashMap with all tag names as keys and their respective ids as values. + * + * @return HashMap with all tags or null if not enabled + */ + private HashMap getAllTags() { + if (!ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_AUTOMATIC_TAGS)) { + return null; + } + TagDao tagDao = new TagDao(); + List tags = tagDao.findByCriteria(new TagCriteria().setTargetIdList(null), new SortCriteria(1, true)); + + HashMap tagsNameToId = new HashMap<>(); + for (TagDto tagDto : tags) { + tagsNameToId.put(tagDto.getName(), tagDto.getId()); + } + return tagsNameToId; } public Date getLastSyncDate() { diff --git a/docs-core/src/main/resources/config.properties b/docs-core/src/main/resources/config.properties index a2ce72d5..9bbc002e 100644 --- a/docs-core/src/main/resources/config.properties +++ b/docs-core/src/main/resources/config.properties @@ -1 +1 @@ -db.version=24 \ No newline at end of file +db.version=25 \ No newline at end of file diff --git a/docs-core/src/main/resources/db/update/dbupdate-025-0.sql b/docs-core/src/main/resources/db/update/dbupdate-025-0.sql new file mode 100644 index 00000000..bde9adf3 --- /dev/null +++ b/docs-core/src/main/resources/db/update/dbupdate-025-0.sql @@ -0,0 +1,3 @@ +insert into T_CONFIG(CFG_ID_C, CFG_VALUE_C) values('INBOX_AUTOMATIC_TAGS', 'false'); +insert into T_CONFIG(CFG_ID_C, CFG_VALUE_C) values('INBOX_DELETE_IMPORTED', 'false'); +update T_CONFIG set CFG_VALUE_C = '25' where CFG_ID_C = 'DB_VERSION'; diff --git a/docs-web/src/dev/resources/config.properties b/docs-web/src/dev/resources/config.properties index 9dd002b9..2319007b 100644 --- a/docs-web/src/dev/resources/config.properties +++ b/docs-web/src/dev/resources/config.properties @@ -1,3 +1,3 @@ api.current_version=${project.version} api.min_version=1.0 -db.version=24 \ No newline at end of file +db.version=25 \ No newline at end of file diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java index ff871d9e..60cde6bb 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java @@ -328,6 +328,8 @@ public class AppResource extends BaseResource { ConfigDao configDao = new ConfigDao(); Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED); + Boolean autoTags = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_AUTOMATIC_TAGS); + Boolean deleteImported = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_DELETE_IMPORTED); Config hostnameConfig = configDao.getById(ConfigType.INBOX_HOSTNAME); Config portConfig = configDao.getById(ConfigType.INBOX_PORT); Config usernameConfig = configDao.getById(ConfigType.INBOX_USERNAME); @@ -336,6 +338,8 @@ public class AppResource extends BaseResource { JsonObjectBuilder response = Json.createObjectBuilder(); response.add("enabled", enabled); + response.add("autoTagsEnabled", autoTags); + response.add("deleteImported", deleteImported); if (hostnameConfig == null) { response.addNull("hostname"); } else { @@ -405,6 +409,8 @@ public class AppResource extends BaseResource { @POST @Path("config_inbox") public Response configInbox(@FormParam("enabled") Boolean enabled, + @FormParam("autoTagsEnabled") Boolean autoTagsEnabled, + @FormParam("deleteImported") Boolean deleteImported, @FormParam("hostname") String hostname, @FormParam("port") String portStr, @FormParam("username") String username, @@ -422,6 +428,8 @@ public class AppResource extends BaseResource { // Just update the changed configuration ConfigDao configDao = new ConfigDao(); configDao.update(ConfigType.INBOX_ENABLED, enabled.toString()); + configDao.update(ConfigType.INBOX_AUTOMATIC_TAGS, autoTagsEnabled.toString()); + configDao.update(ConfigType.INBOX_DELETE_IMPORTED, deleteImported.toString()); if (!Strings.isNullOrEmpty(hostname)) { configDao.update(ConfigType.INBOX_HOSTNAME, hostname); } diff --git a/docs-web/src/main/webapp/src/locale/en.json b/docs-web/src/main/webapp/src/locale/en.json index ce04b367..4b34de29 100644 --- a/docs-web/src/main/webapp/src/locale/en.json +++ b/docs-web/src/main/webapp/src/locale/en.json @@ -432,7 +432,9 @@ "last_sync": "Last synchronization: {{ data.date | date: 'medium' }}, {{ data.count }} message{{ data.count > 1 ? 's' : '' }} imported", "test_success": "The connection to the inbox is successful ({{ count }} unread message{{ count > 1 ? 's' : '' }})", "test_fail": "An error occurred while connecting to the inbox, please check the parameters", - "saved": "IMAP configuration saved successfully" + "saved": "IMAP configuration saved successfully", + "autoTagsEnabled": "Automatically add tags from subject line marked with #", + "deleteImported": "Delete message from mailbox after import" }, "monitoring": { "background_tasks": "Background tasks", diff --git a/docs-web/src/main/webapp/src/partial/docs/settings.inbox.html b/docs-web/src/main/webapp/src/partial/docs/settings.inbox.html index 6e27a7c6..c4fff525 100644 --- a/docs-web/src/main/webapp/src/partial/docs/settings.inbox.html +++ b/docs-web/src/main/webapp/src/partial/docs/settings.inbox.html @@ -17,6 +17,20 @@ +
+ +
+ +
+
+ +
+ +
+ +
+
+
diff --git a/docs-web/src/prod/resources/config.properties b/docs-web/src/prod/resources/config.properties index 9dd002b9..2319007b 100644 --- a/docs-web/src/prod/resources/config.properties +++ b/docs-web/src/prod/resources/config.properties @@ -1,3 +1,3 @@ api.current_version=${project.version} api.min_version=1.0 -db.version=24 \ No newline at end of file +db.version=25 \ No newline at end of file diff --git a/docs-web/src/stress/resources/config.properties b/docs-web/src/stress/resources/config.properties index 9dd002b9..2319007b 100644 --- a/docs-web/src/stress/resources/config.properties +++ b/docs-web/src/stress/resources/config.properties @@ -1,3 +1,3 @@ api.current_version=${project.version} api.min_version=1.0 -db.version=24 \ No newline at end of file +db.version=25 \ No newline at end of file diff --git a/docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java b/docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java index b4c72fa6..68ce4f4f 100644 --- a/docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java +++ b/docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java @@ -254,6 +254,8 @@ public class TestAppResource extends BaseJerseyTest { .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken) .post(Entity.form(new Form() .param("enabled", "true") + .param("autoTagsEnabled", "false") + .param("deleteImported", "false") .param("hostname", "localhost") .param("port", "9755") .param("username", "test@sismics.com")