diff --git a/docs-core/pom.xml b/docs-core/pom.xml index 8976d700..38372ed5 100644 --- a/docs-core/pom.xml +++ b/docs-core/pom.xml @@ -111,6 +111,11 @@ org.apache.lucene lucene-queryparser + + + com.sun.mail + javax.mail + diff --git a/docs-core/src/main/java/com/sismics/docs/core/constant/ConfigType.java b/docs-core/src/main/java/com/sismics/docs/core/constant/ConfigType.java index 990e95ba..e4e41db5 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/constant/ConfigType.java +++ b/docs-core/src/main/java/com/sismics/docs/core/constant/ConfigType.java @@ -32,5 +32,15 @@ public enum ConfigType { SMTP_PORT, SMTP_FROM, SMTP_USERNAME, - SMTP_PASSWORD + SMTP_PASSWORD, + + /** + * Inbox scanning configuration. + */ + INBOX_ENABLED, + INBOX_HOSTNAME, + INBOX_PORT, + INBOX_USERNAME, + INBOX_PASSWORD, + INBOX_TAG } diff --git a/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java b/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java index 7da91f14..11df7dc4 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java +++ b/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java @@ -10,6 +10,7 @@ import com.sismics.docs.core.listener.async.*; import com.sismics.docs.core.listener.sync.DeadEventListener; import com.sismics.docs.core.model.jpa.Config; import com.sismics.docs.core.model.jpa.User; +import com.sismics.docs.core.service.InboxService; import com.sismics.docs.core.service.IndexingService; import com.sismics.docs.core.util.PdfUtil; import com.sismics.util.EnvironmentUtil; @@ -52,6 +53,11 @@ public class AppContext { */ private IndexingService indexingService; + /** + * Inbox scanning service. + */ + private InboxService inboxService; + /** * Asynchronous executors. */ @@ -69,6 +75,10 @@ public class AppContext { indexingService = new IndexingService(luceneStorageConfig != null ? luceneStorageConfig.getValue() : null); indexingService.startAsync(); + // Start inbox service + inboxService = new InboxService(); + inboxService.startAsync(); + // Register fonts PdfUtil.registerFonts(); @@ -175,4 +185,8 @@ public class AppContext { public IndexingService getIndexingService() { return indexingService; } + + public InboxService getInboxService() { + return inboxService; + } } diff --git a/docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java b/docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java new file mode 100644 index 00000000..d568b690 --- /dev/null +++ b/docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java @@ -0,0 +1,210 @@ +package com.sismics.docs.core.service; + +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.AbstractScheduledService; +import com.sismics.docs.core.constant.ConfigType; +import com.sismics.docs.core.dao.jpa.TagDao; +import com.sismics.docs.core.event.DocumentCreatedAsyncEvent; +import com.sismics.docs.core.model.jpa.Document; +import com.sismics.docs.core.model.jpa.Tag; +import com.sismics.docs.core.util.ConfigUtil; +import com.sismics.docs.core.util.DocumentUtil; +import com.sismics.docs.core.util.FileUtil; +import com.sismics.docs.core.util.TransactionUtil; +import com.sismics.util.EmailUtil; +import com.sismics.util.context.ThreadLocalContext; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.mail.*; +import java.util.Date; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Inbox scanning service. + * + * @author bgamard + */ +public class InboxService extends AbstractScheduledService { + /** + * Logger. + */ + private static final Logger log = LoggerFactory.getLogger(InboxService.class); + + public InboxService() { + } + + @Override + protected void startUp() { + } + + @Override + protected void shutDown() { + } + + @Override + protected void runOneIteration() { + syncInbox(); + } + + public void syncInbox() { + TransactionUtil.handle(new Runnable() { + @Override + public void run() { + Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED); + if (!enabled) { + return; + } + + Folder inbox = null; + try { + inbox = openInbox(); + + int count = inbox.getMessageCount(); + Message[] messages = inbox.getMessages(1, count); + for (Message message : messages) { + if (!message.getFlags().contains(Flags.Flag.SEEN)) { + importMessage(message); + } + } + } catch (Exception e) { + log.error("Error synching the inbox", e); + } finally { + try { + if (inbox != null) { + inbox.close(false); + inbox.getStore().close(); + } + } catch (Exception e) { + // NOP + } + } + } + }); + } + + public int testInbox() { + final AtomicInteger count = new AtomicInteger(-1); + TransactionUtil.handle(new Runnable() { + @Override + public void run() { + Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED); + if (!enabled) { + return; + } + + Folder inbox = null; + try { + inbox = openInbox(); + count.set(inbox.getMessageCount()); + } catch (Exception e) { + log.error("Error testing inbox", e); + } finally { + try { + if (inbox != null) { + inbox.close(false); + inbox.getStore().close(); + } + } catch (Exception e) { + // NOP + } + } + } + }); + + return count.get(); + } + + @Override + protected Scheduler scheduler() { + return Scheduler.newFixedDelaySchedule(0, 15, TimeUnit.MINUTES); + } + + /** + * Open the remote inbox. + * + * @return Opened inbox folder + */ + private Folder openInbox() throws Exception { + Properties properties = new Properties(); + String port = ConfigUtil.getConfigStringValue(ConfigType.INBOX_PORT); + properties.put("mail.imap.host", ConfigUtil.getConfigStringValue(ConfigType.INBOX_HOSTNAME)); + properties.put("mail.imap.port", port); + boolean isSsl = "993".equals(port); + properties.put("mail.imap.ssl.enable", String.valueOf(isSsl)); + properties.setProperty("mail.imap.socketFactory.class", + isSsl ? "javax.net.ssl.SSLSocketFactory" : "javax.net.DefaultSocketFactory"); + properties.setProperty("mail.imap.socketFactory.fallback", "true"); + properties.setProperty("mail.imap.socketFactory.port", port); + + Session session = Session.getDefaultInstance(properties); + + Store store = session.getStore("imap"); + store.connect(ConfigUtil.getConfigStringValue(ConfigType.INBOX_USERNAME), + ConfigUtil.getConfigStringValue(ConfigType.INBOX_PASSWORD)); + + Folder inbox = store.getFolder("INBOX"); + inbox.open(Folder.READ_WRITE); + return inbox; + } + + /** + * Import an email. + * + * @param message Message + * @throws Exception + */ + private void importMessage(Message message) throws Exception { + // Parse the mail + EmailUtil.MailContent mailContent = new EmailUtil.MailContent(); + mailContent.setSubject(message.getSubject()); + mailContent.setDate(message.getSentDate()); + EmailUtil.parseMailContent(message, mailContent); + + // Create the document + Document document = new Document(); + document.setUserId("admin"); + if (mailContent.getSubject() == null) { + document.setTitle("Imported email from EML file"); + } else { + document.setTitle(StringUtils.abbreviate(mailContent.getSubject(), 100)); + } + document.setDescription(StringUtils.abbreviate(mailContent.getMessage(), 4000)); + document.setSubject(StringUtils.abbreviate(mailContent.getSubject(), 500)); + document.setFormat("EML"); + document.setSource("Inbox"); + document.setLanguage(ConfigUtil.getConfigStringValue(ConfigType.DEFAULT_LANGUAGE)); + if (mailContent.getDate() == null) { + document.setCreateDate(new Date()); + } else { + document.setCreateDate(mailContent.getDate()); + } + + // Save the document, create the base ACLs + document = DocumentUtil.createDocument(document, "admin"); + + // Add the tag + String tagId = ConfigUtil.getConfigStringValue(ConfigType.INBOX_TAG); + if (tagId != null) { + TagDao tagDao = new TagDao(); + Tag tag = tagDao.getById(tagId); + if (tag != null) { + tagDao.updateTagList(document.getId(), Sets.newHashSet(tagId)); + } + } + + // Raise a document created event + DocumentCreatedAsyncEvent documentCreatedAsyncEvent = new DocumentCreatedAsyncEvent(); + documentCreatedAsyncEvent.setUserId("admin"); + documentCreatedAsyncEvent.setDocument(document); + ThreadLocalContext.get().addAsyncEvent(documentCreatedAsyncEvent); + + // Add files to the document + for (EmailUtil.FileContent fileContent : mailContent.getFileContentList()) { + FileUtil.createFile(fileContent.getName(), fileContent.getFile(), fileContent.getSize(), "eng", "admin", document.getId()); + } + } +} diff --git a/docs-core/src/main/java/com/sismics/docs/core/service/IndexingService.java b/docs-core/src/main/java/com/sismics/docs/core/service/IndexingService.java index 5502af52..5800464b 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/service/IndexingService.java +++ b/docs-core/src/main/java/com/sismics/docs/core/service/IndexingService.java @@ -114,7 +114,7 @@ public class IndexingService extends AbstractScheduledService { } @Override - protected void runOneIteration() throws Exception { + protected void runOneIteration() { TransactionUtil.handle(new Runnable() { @Override public void run() { diff --git a/docs-core/src/main/resources/config.properties b/docs-core/src/main/resources/config.properties index 0f5bd5e5..3be02720 100644 --- a/docs-core/src/main/resources/config.properties +++ b/docs-core/src/main/resources/config.properties @@ -1 +1 @@ -db.version=16 \ No newline at end of file +db.version=17 \ No newline at end of file diff --git a/docs-core/src/main/resources/db/update/dbupdate-017-0.sql b/docs-core/src/main/resources/db/update/dbupdate-017-0.sql new file mode 100644 index 00000000..abd20826 --- /dev/null +++ b/docs-core/src/main/resources/db/update/dbupdate-017-0.sql @@ -0,0 +1,2 @@ +insert into T_CONFIG(CFG_ID_C, CFG_VALUE_C) values('INBOX_ENABLED', 'false'); +update T_CONFIG set CFG_VALUE_C = '17' where CFG_ID_C = 'DB_VERSION'; diff --git a/docs-web/pom.xml b/docs-web/pom.xml index 4d239548..95fa0114 100644 --- a/docs-web/pom.xml +++ b/docs-web/pom.xml @@ -127,6 +127,12 @@ subethasmtp-wiser test + + + com.icegreen + greenmail + test + diff --git a/docs-web/src/dev/resources/config.properties b/docs-web/src/dev/resources/config.properties index 548243cf..1c41d115 100644 --- a/docs-web/src/dev/resources/config.properties +++ b/docs-web/src/dev/resources/config.properties @@ -1,3 +1,3 @@ api.current_version=${project.version} api.min_version=1.0 -db.version=16 \ No newline at end of file +db.version=17 \ No newline at end of file diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java index 11e52c20..f5001795 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java @@ -8,6 +8,7 @@ import com.sismics.docs.core.dao.jpa.DocumentDao; import com.sismics.docs.core.dao.jpa.FileDao; import com.sismics.docs.core.dao.jpa.UserDao; import com.sismics.docs.core.event.RebuildIndexAsyncEvent; +import com.sismics.docs.core.model.context.AppContext; import com.sismics.docs.core.model.jpa.Config; import com.sismics.docs.core.model.jpa.File; import com.sismics.docs.core.model.jpa.User; @@ -174,10 +175,10 @@ public class AppResource extends BaseResource { * @apiName GetAppConfigSmtp * @apiGroup App * @apiSuccess {String} hostname SMTP hostname - * @apiSuccess {String} port - * @apiSuccess {String} username - * @apiSuccess {String} password - * @apiSuccess {String} from + * @apiSuccess {String} port SMTP port + * @apiSuccess {String} username SMTP username + * @apiSuccess {String} password SMTP password + * @apiSuccess {String} from From address * @apiError (client) ForbiddenError Access denied * @apiPermission admin * @apiVersion 1.5.0 @@ -295,6 +296,161 @@ public class AppResource extends BaseResource { return Response.ok().build(); } + /** + * Get the inbox configuration. + * + * @api {get} /app/config_inbox Get the inbox scanning configuration + * @apiName GetAppConfigInbox + * @apiGroup App + * @apiSuccess {Boolean} enabled True if the inbox scanning is enabled + * @apiSuccess {String} hostname IMAP hostname + * @apiSuccess {String} port IMAP port + * @apiSuccess {String} username IMAP username + * @apiSuccess {String} password IMAP password + * @apiSuccess {String} tag Tag for created documents + * @apiError (client) ForbiddenError Access denied + * @apiPermission admin + * @apiVersion 1.5.0 + * + * @return Response + */ + @GET + @Path("config_inbox") + public Response getConfigInbox() { + if (!authenticate()) { + throw new ForbiddenClientException(); + } + checkBaseFunction(BaseFunction.ADMIN); + + ConfigDao configDao = new ConfigDao(); + Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED); + Config hostnameConfig = configDao.getById(ConfigType.INBOX_HOSTNAME); + Config portConfig = configDao.getById(ConfigType.INBOX_PORT); + Config usernameConfig = configDao.getById(ConfigType.INBOX_USERNAME); + Config passwordConfig = configDao.getById(ConfigType.INBOX_PASSWORD); + Config tagConfig = configDao.getById(ConfigType.INBOX_TAG); + JsonObjectBuilder response = Json.createObjectBuilder(); + + response.add("enabled", enabled); + if (hostnameConfig == null) { + response.addNull("hostname"); + } else { + response.add("hostname", hostnameConfig.getValue()); + } + if (portConfig == null) { + response.addNull("port"); + } else { + response.add("port", Integer.valueOf(portConfig.getValue())); + } + if (usernameConfig == null) { + response.addNull("username"); + } else { + response.add("username", usernameConfig.getValue()); + } + if (passwordConfig == null) { + response.addNull("password"); + } else { + response.add("password", passwordConfig.getValue()); + } + if (tagConfig == null) { + response.addNull("tag"); + } else { + response.add("tag", tagConfig.getValue()); + } + + return Response.ok().entity(response.build()).build(); + } + + /** + * Configure the inbox. + * + * @api {post} /app/config_inbox Configure the inbox scanning + * @apiName PostAppConfigInbox + * @apiGroup App + * @apiParam {Boolean} enabled True if the inbox scanning is enabled + * @apiParam {String} hostname IMAP hostname + * @apiParam {Integer} port IMAP port + * @apiParam {String} username IMAP username + * @apiParam {String} password IMAP password + * @apiParam {String} tag Tag for created documents + * @apiError (client) ForbiddenError Access denied + * @apiError (client) ValidationError Validation error + * @apiPermission admin + * @apiVersion 1.5.0 + * + * @param enabled True if the inbox scanning is enabled + * @param hostname IMAP hostname + * @param portStr IMAP port + * @param username IMAP username + * @param password IMAP password + * @param tag Tag for created documents + * @return Response + */ + @POST + @Path("config_inbox") + public Response configInbox(@FormParam("enabled") Boolean enabled, + @FormParam("hostname") String hostname, + @FormParam("port") String portStr, + @FormParam("username") String username, + @FormParam("password") String password, + @FormParam("tag") String tag) { + if (!authenticate()) { + throw new ForbiddenClientException(); + } + checkBaseFunction(BaseFunction.ADMIN); + ValidationUtil.validateRequired(enabled, "enabled"); + if (!Strings.isNullOrEmpty(portStr)) { + ValidationUtil.validateInteger(portStr, "port"); + } + + // Just update the changed configuration + ConfigDao configDao = new ConfigDao(); + configDao.update(ConfigType.INBOX_ENABLED, enabled.toString()); + if (!Strings.isNullOrEmpty(hostname)) { + configDao.update(ConfigType.INBOX_HOSTNAME, hostname); + } + if (!Strings.isNullOrEmpty(portStr)) { + configDao.update(ConfigType.INBOX_PORT, portStr); + } + if (!Strings.isNullOrEmpty(username)) { + configDao.update(ConfigType.INBOX_USERNAME, username); + } + if (!Strings.isNullOrEmpty(password)) { + configDao.update(ConfigType.INBOX_PASSWORD, password); + } + if (!Strings.isNullOrEmpty(tag)) { + configDao.update(ConfigType.INBOX_TAG, tag); + } + + return Response.ok().build(); + } + + /** + * Test the inbox. + * + * @api {post} /app/test_inbox Test the inbox scanning + * @apiName PostAppTestInbox + * @apiGroup App + * @apiSuccess {Number} Number of unread emails in the inbox + * @apiError (client) ForbiddenError Access denied + * @apiPermission admin + * @apiVersion 1.5.0 + * + * @return Response + */ + @POST + @Path("test_inbox") + public Response testInbox() { + if (!authenticate()) { + throw new ForbiddenClientException(); + } + checkBaseFunction(BaseFunction.ADMIN); + + return Response.ok().entity(Json.createObjectBuilder() + .add("count", AppContext.getInstance().getInboxService().testInbox()) + .build()).build(); + } + /** * Retrieve the application logs. * diff --git a/docs-web/src/prod/resources/config.properties b/docs-web/src/prod/resources/config.properties index 548243cf..1c41d115 100644 --- a/docs-web/src/prod/resources/config.properties +++ b/docs-web/src/prod/resources/config.properties @@ -1,3 +1,3 @@ api.current_version=${project.version} api.min_version=1.0 -db.version=16 \ No newline at end of file +db.version=17 \ No newline at end of file diff --git a/docs-web/src/stress/resources/config.properties b/docs-web/src/stress/resources/config.properties index 548243cf..1c41d115 100644 --- a/docs-web/src/stress/resources/config.properties +++ b/docs-web/src/stress/resources/config.properties @@ -1,3 +1,3 @@ api.current_version=${project.version} api.min_version=1.0 -db.version=16 \ No newline at end of file +db.version=17 \ No newline at end of file diff --git a/docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java b/docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java index d595c4dc..4e86e8fb 100644 --- a/docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java +++ b/docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java @@ -1,5 +1,9 @@ package com.sismics.docs.rest; +import com.icegreen.greenmail.util.GreenMail; +import com.icegreen.greenmail.util.GreenMailUtil; +import com.icegreen.greenmail.util.ServerSetup; +import com.sismics.docs.core.model.context.AppContext; import com.sismics.util.filter.TokenBasedSecurityFilter; import org.junit.Assert; import org.junit.Test; @@ -218,4 +222,92 @@ public class TestAppResource extends BaseJerseyTest { Assert.assertTrue(json.isNull("password")); Assert.assertEquals("contact@sismics.com", json.getString("from")); } + + /** + * Test inbox scanning. + */ + @Test + public void testInbox() { + // Login admin + String adminToken = clientUtil.login("admin", "admin", false); + + // Create a tag + JsonObject json = target().path("/tag").request() + .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken) + .put(Entity.form(new Form() + .param("name", "Inbox") + .param("color", "#ff0000")), JsonObject.class); + String tagInboxId = json.getString("id"); + + // Get inbox configuration + json = target().path("/app/config_inbox").request() + .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken) + .get(JsonObject.class); + Assert.assertFalse(json.getBoolean("enabled")); + Assert.assertTrue(json.isNull("hostname")); + Assert.assertTrue(json.isNull("port")); + Assert.assertTrue(json.isNull("username")); + Assert.assertTrue(json.isNull("password")); + Assert.assertTrue(json.isNull("tag")); + + // Change inbox configuration + target().path("/app/config_inbox").request() + .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken) + .post(Entity.form(new Form() + .param("enabled", "true") + .param("hostname", "localhost") + .param("port", "143") + .param("username", "test@sismics.com") + .param("password", "12345678") + .param("tag", tagInboxId) + ), JsonObject.class); + + // Get inbox configuration + json = target().path("/app/config_inbox").request() + .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken) + .get(JsonObject.class); + Assert.assertTrue(json.getBoolean("enabled")); + Assert.assertEquals("localhost", json.getString("hostname")); + Assert.assertEquals(143, json.getInt("port")); + Assert.assertEquals("test@sismics.com", json.getString("username")); + Assert.assertEquals("12345678", json.getString("password")); + Assert.assertEquals(tagInboxId, json.getString("tag")); + + GreenMail greenMail = new GreenMail(new ServerSetup[] { ServerSetup.SMTP, ServerSetup.IMAP }); + greenMail.setUser("test@sismics.com", "12345678"); + greenMail.start(); + + // Test the inbox + json = target().path("/app/test_inbox").request() + .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken) + .post(Entity.form(new Form()), JsonObject.class); + Assert.assertEquals(0, json.getJsonNumber("count").intValue()); + + // Send an email + GreenMailUtil.sendTextEmail("test@sismics.com", "test@sismicsdocs.com", "Test email 1", "Test content 1", ServerSetup.SMTP); + + // Trigger an inbox sync + AppContext.getInstance().getInboxService().syncInbox(); + + // Search for added documents + json = target().path("/document/list") + .queryParam("search", "tag:Inbox") + .request() + .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken) + .get(JsonObject.class); + Assert.assertEquals(1, json.getJsonArray("documents").size()); + + // Trigger an inbox sync + AppContext.getInstance().getInboxService().syncInbox(); + + // Search for added documents + json = target().path("/document/list") + .queryParam("search", "tag:Inbox") + .request() + .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken) + .get(JsonObject.class); + Assert.assertEquals(1, json.getJsonArray("documents").size()); + + greenMail.stop(); + } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 7014008d..c9f688ce 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,8 @@ 1.6.5 1.3.1 1.2 + 1.5.7 + 1.5.6 1.11.2 9.2.13.v20150730 @@ -412,6 +414,24 @@ org.subethamail subethasmtp-wiser ${org.subethamail.subethasmtp-wiser.version} + + + javax.mail + mail + + + + + + com.icegreen + greenmail + ${com.icegreen.greenmail.version} + + + + com.sun.mail + javax.mail + ${com.sun.mail.javax.mail.version}