vmario/docs
1
0
Fork 0
mirror of https://github.com/sismics/docs.git synced 2024-11-21 21:47:57 +01:00
Code Issues Packages Projects Releases Wiki Activity

Improve Inbox Scanning (#407)

Browse Source 
closes #386: delete emails after import + closes #405: auto tag documents imported by email
This commit is contained in:
cadast 2020-05-14 13:59:11 +02:00 committed by GitHub
parent 0d058b9c9c
commit 95c37a03f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 100 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs-core/src/main/java/com/sismics/docs/core/constant/ConfigType.java
View File

@ -42,5 +42,7 @@ public enum ConfigType {
INBOX_PORT,
INBOX_USERNAME,
INBOX_PASSWORD,
INBOX_TAG
INBOX_TAG,
INBOX_AUTOMATIC_TAGS,
INBOX_DELETE_IMPORTED
}

74
docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java
View File

@ -1,16 +1,21 @@
package com.sismics.docs.core.service;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AbstractScheduledService;
import com.sismics.docs.core.constant.ConfigType;
import com.sismics.docs.core.dao.TagDao;
import com.sismics.docs.core.dao.criteria.TagCriteria;
import com.sismics.docs.core.dao.dto.TagDto;
import com.sismics.docs.core.event.DocumentCreatedAsyncEvent;
import com.sismics.docs.core.model.jpa.Config;
import com.sismics.docs.core.model.jpa.Document;
import com.sismics.docs.core.model.jpa.Tag;
import com.sismics.docs.core.util.ConfigUtil;
import com.sismics.docs.core.util.DocumentUtil;
import com.sismics.docs.core.util.FileUtil;
import com.sismics.docs.core.util.TransactionUtil;
import com.sismics.docs.core.util.jpa.SortCriteria;
import com.sismics.util.EmailUtil;
import com.sismics.util.context.ThreadLocalContext;
import org.apache.commons.lang.StringUtils;
@ -19,9 +24,10 @@ import org.slf4j.LoggerFactory;
import javax.mail.*;
import javax.mail.search.FlagTerm;
import java.util.Date;
import java.util.Properties;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Inbox scanning service.
@ -79,11 +85,13 @@ public class InboxService extends AbstractScheduledService {
lastSyncDate = new Date();
lastSyncMessageCount = 0;
try {
HashMap<String, String> tagsNameToId = getAllTags();
inbox = openInbox();
Message[] messages = inbox.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false));
log.info(messages.length + " messages found");
for (Message message : messages) {
importMessage(message);
importMessage(message, tagsNameToId);
lastSyncMessageCount++;
}
} catch (FolderClosedException e) {
@ -94,7 +102,8 @@ public class InboxService extends AbstractScheduledService {
} finally {
try {
if (inbox != null) {
inbox.close(false);
// The parameter controls if the messages flagged to be deleted, should actually get deleted.
inbox.close(ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_DELETE_IMPORTED));
inbox.getStore().close();
}
} catch (Exception e) {
@ -183,7 +192,7 @@ public class InboxService extends AbstractScheduledService {
* @param message Message
* @throws Exception e
*/
private void importMessage(Message message) throws Exception {
private void importMessage(Message message, HashMap<String, String> tags) throws Exception {
log.info("Importing message: " + message.getSubject());
// Parse the mail
@ -194,12 +203,27 @@ public class InboxService extends AbstractScheduledService {
// Create the document
Document document = new Document();
document.setUserId("admin");
if (mailContent.getSubject() == null) {
document.setTitle("Imported email from EML file");
} else {
document.setTitle(StringUtils.abbreviate(mailContent.getSubject(), 100));
String subject = mailContent.getSubject();
if (subject == null) {
subject = "Imported email from EML file";
}
HashSet<String> tagsFound = new HashSet<>();
if (tags != null) {
Pattern pattern = Pattern.compile("#([^\\s:#]+)");
Matcher matcher = pattern.matcher(subject);
while (matcher.find()) {
if (tags.containsKey(matcher.group(1)) && tags.get(matcher.group(1)) != null) {
tagsFound.add(tags.get(matcher.group(1)));
subject = subject.replaceFirst("#" + matcher.group(1), "");
}
}
log.debug("Tags found: " + String.join(", ", tagsFound));
subject = subject.trim().replaceAll(" +", " ");
}
document.setUserId("admin");
document.setTitle(StringUtils.abbreviate(subject, 100));
document.setDescription(StringUtils.abbreviate(mailContent.getMessage(), 4000));
document.setSubject(StringUtils.abbreviate(mailContent.getSubject(), 500));
document.setFormat("EML");
@ -220,10 +244,15 @@ public class InboxService extends AbstractScheduledService {
TagDao tagDao = new TagDao();
Tag tag = tagDao.getById(tagId);
if (tag != null) {
tagDao.updateTagList(document.getId(), Sets.newHashSet(tagId));
tagsFound.add(tagId);
}
}
// Update tags
if (!tagsFound.isEmpty()) {
new TagDao().updateTagList(document.getId(), tagsFound);
}
// Raise a document created event
DocumentCreatedAsyncEvent documentCreatedAsyncEvent = new DocumentCreatedAsyncEvent();
documentCreatedAsyncEvent.setUserId("admin");
@ -235,6 +264,29 @@ public class InboxService extends AbstractScheduledService {
FileUtil.createFile(fileContent.getName(), null, fileContent.getFile(), fileContent.getSize(),
document.getLanguage(), "admin", document.getId());
}
if (ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_DELETE_IMPORTED)) {
message.setFlag(Flags.Flag.DELETED, true);
}
}
/**
* Fetches a HashMap with all tag names as keys and their respective ids as values.
*
* @return HashMap with all tags or null if not enabled
*/
private HashMap<String, String> getAllTags() {
if (!ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_AUTOMATIC_TAGS)) {
return null;
}
TagDao tagDao = new TagDao();
List<TagDto> tags = tagDao.findByCriteria(new TagCriteria().setTargetIdList(null), new SortCriteria(1, true));
HashMap<String, String> tagsNameToId = new HashMap<>();
for (TagDto tagDto : tags) {
tagsNameToId.put(tagDto.getName(), tagDto.getId());
}
return tagsNameToId;
}
public Date getLastSyncDate() {

2
docs-core/src/main/resources/config.properties
View File

@ -1 +1 @@
db.version=24
db.version=25

3
docs-core/src/main/resources/db/update/dbupdate-025-0.sql Normal file
View File

@ -0,0 +1,3 @@
insert into T_CONFIG(CFG_ID_C, CFG_VALUE_C) values('INBOX_AUTOMATIC_TAGS', 'false');
insert into T_CONFIG(CFG_ID_C, CFG_VALUE_C) values('INBOX_DELETE_IMPORTED', 'false');
update T_CONFIG set CFG_VALUE_C = '25' where CFG_ID_C = 'DB_VERSION';

2
docs-web/src/dev/resources/config.properties
View File

@ -1,3 +1,3 @@
api.current_version=${project.version}
api.min_version=1.0
db.version=24
db.version=25

8
docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java
View File

@ -328,6 +328,8 @@ public class AppResource extends BaseResource {
ConfigDao configDao = new ConfigDao();
Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED);
Boolean autoTags = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_AUTOMATIC_TAGS);
Boolean deleteImported = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_DELETE_IMPORTED);
Config hostnameConfig = configDao.getById(ConfigType.INBOX_HOSTNAME);
Config portConfig = configDao.getById(ConfigType.INBOX_PORT);
Config usernameConfig = configDao.getById(ConfigType.INBOX_USERNAME);
@ -336,6 +338,8 @@ public class AppResource extends BaseResource {
JsonObjectBuilder response = Json.createObjectBuilder();
response.add("enabled", enabled);
response.add("autoTagsEnabled", autoTags);
response.add("deleteImported", deleteImported);
if (hostnameConfig == null) {
response.addNull("hostname");
} else {
@ -405,6 +409,8 @@ public class AppResource extends BaseResource {
@POST
@Path("config_inbox")
public Response configInbox(@FormParam("enabled") Boolean enabled,
@FormParam("autoTagsEnabled") Boolean autoTagsEnabled,
@FormParam("deleteImported") Boolean deleteImported,
@FormParam("hostname") String hostname,
@FormParam("port") String portStr,
@FormParam("username") String username,
@ -422,6 +428,8 @@ public class AppResource extends BaseResource {
// Just update the changed configuration
ConfigDao configDao = new ConfigDao();
configDao.update(ConfigType.INBOX_ENABLED, enabled.toString());
configDao.update(ConfigType.INBOX_AUTOMATIC_TAGS, autoTagsEnabled.toString());
configDao.update(ConfigType.INBOX_DELETE_IMPORTED, deleteImported.toString());
if (!Strings.isNullOrEmpty(hostname)) {
configDao.update(ConfigType.INBOX_HOSTNAME, hostname);
}

4
docs-web/src/main/webapp/src/locale/en.json
View File

@ -432,7 +432,9 @@
"last_sync": "Last synchronization: {{ data.date | date: 'medium' }}, {{ data.count }} message{{ data.count > 1 ? 's' : '' }} imported",
"test_success": "The connection to the inbox is successful ({{ count }} <strong>unread</strong> message{{ count > 1 ? 's' : '' }})",
"test_fail": "An error occurred while connecting to the inbox, please check the parameters",
"saved": "IMAP configuration saved successfully"
"saved": "IMAP configuration saved successfully",
"autoTagsEnabled": "Automatically add tags from subject line marked with #",
"deleteImported": "Delete message from mailbox after import"
},
"monitoring": {
"background_tasks": "Background tasks",

14
docs-web/src/main/webapp/src/partial/docs/settings.inbox.html
View File

@ -17,6 +17,20 @@
</div>
</div>
<div class="form-group">
<label class="col-sm-2 control-label" for="inboxAutoTagsEnabled">{{ 'settings.inbox.autoTagsEnabled' | translate }}</label>
<div class="col-sm-7">
<input name="autoTagsEnabled" type="checkbox" id="inboxAutoTagsEnabled" ng-model="inbox.autoTagsEnabled" />
</div>
</div>
<div class="form-group">
<label class="col-sm-2 control-label" for="inboxDeleteImported">{{ 'settings.inbox.deleteImported' | translate }}</label>
<div class="col-sm-7">
<input name="deleteImported" type="checkbox" id="inboxDeleteImported" ng-model="inbox.deleteImported" />
</div>
</div>
<div class="form-group" ng-class="{ 'has-error': !inboxForm.hostname.$valid && inboxForm.$dirty }">
<label class="col-sm-2 control-label" for="inboxHostname">{{ 'settings.inbox.hostname' | translate }}</label>
<div class="col-sm-7">

2
docs-web/src/prod/resources/config.properties
View File

@ -1,3 +1,3 @@
api.current_version=${project.version}
api.min_version=1.0
db.version=24
db.version=25

2
docs-web/src/stress/resources/config.properties
View File

@ -1,3 +1,3 @@
api.current_version=${project.version}
api.min_version=1.0
db.version=24
db.version=25

2
docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java
View File

@ -254,6 +254,8 @@ public class TestAppResource extends BaseJerseyTest {
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
.post(Entity.form(new Form()
.param("enabled", "true")
.param("autoTagsEnabled", "false")
.param("deleteImported", "false")
.param("hostname", "localhost")
.param("port", "9755")
.param("username", "test@sismics.com")