#180: IMAP inbox synching (api)

docs-core/pom.xml

@@ -112,6 +112,11 @@
       <artifactId>lucene-queryparser</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.sun.mail</groupId>
+      <artifactId>javax.mail</artifactId>
+    </dependency>
+    
     <!-- Only there to read old index and rebuild them -->
     <dependency>
       <groupId>org.apache.lucene</groupId>

docs-core/src/main/java/com/sismics/docs/core/constant/ConfigType.java

@@ -32,5 +32,15 @@ public enum ConfigType {
     SMTP_PORT,
     SMTP_FROM,
     SMTP_USERNAME,
-    SMTP_PASSWORD
+    SMTP_PASSWORD,
+
+    /**
+     * Inbox scanning configuration.
+     */
+    INBOX_ENABLED,
+    INBOX_HOSTNAME,
+    INBOX_PORT,
+    INBOX_USERNAME,
+    INBOX_PASSWORD,
+    INBOX_TAG
 }

docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java

@@ -10,6 +10,7 @@ import com.sismics.docs.core.listener.async.*;
 import com.sismics.docs.core.listener.sync.DeadEventListener;
 import com.sismics.docs.core.model.jpa.Config;
 import com.sismics.docs.core.model.jpa.User;
+import com.sismics.docs.core.service.InboxService;
 import com.sismics.docs.core.service.IndexingService;
 import com.sismics.docs.core.util.PdfUtil;
 import com.sismics.util.EnvironmentUtil;
@@ -52,6 +53,11 @@ public class AppContext {
      */
     private IndexingService indexingService;
 
+    /**
+     * Inbox scanning service.
+     */
+    private InboxService inboxService;
+
     /**
      * Asynchronous executors.
      */
@@ -69,6 +75,10 @@ public class AppContext {
         indexingService = new IndexingService(luceneStorageConfig != null ? luceneStorageConfig.getValue() : null);
         indexingService.startAsync();
 
+        // Start inbox service
+        inboxService = new InboxService();
+        inboxService.startAsync();
+
         // Register fonts
         PdfUtil.registerFonts();
 
@@ -175,4 +185,8 @@ public class AppContext {
     public IndexingService getIndexingService() {
         return indexingService;
     }
+
+    public InboxService getInboxService() {
+        return inboxService;
+    }
 }

docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java

@@ -0,0 +1,210 @@
+package com.sismics.docs.core.service;
+
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.AbstractScheduledService;
+import com.sismics.docs.core.constant.ConfigType;
+import com.sismics.docs.core.dao.jpa.TagDao;
+import com.sismics.docs.core.event.DocumentCreatedAsyncEvent;
+import com.sismics.docs.core.model.jpa.Document;
+import com.sismics.docs.core.model.jpa.Tag;
+import com.sismics.docs.core.util.ConfigUtil;
+import com.sismics.docs.core.util.DocumentUtil;
+import com.sismics.docs.core.util.FileUtil;
+import com.sismics.docs.core.util.TransactionUtil;
+import com.sismics.util.EmailUtil;
+import com.sismics.util.context.ThreadLocalContext;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.mail.*;
+import java.util.Date;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Inbox scanning service.
+ *
+ * @author bgamard
+ */
+public class InboxService extends AbstractScheduledService {
+    /**
+     * Logger.
+     */
+    private static final Logger log = LoggerFactory.getLogger(InboxService.class);
+
+    public InboxService() {
+    }
+
+    @Override
+    protected void startUp() {
+    }
+
+    @Override
+    protected void shutDown() {
+    }
+    
+    @Override
+    protected void runOneIteration() {
+        syncInbox();
+    }
+
+    public void syncInbox() {
+        TransactionUtil.handle(new Runnable() {
+            @Override
+            public void run() {
+                Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED);
+                if (!enabled) {
+                    return;
+                }
+
+                Folder inbox = null;
+                try {
+                    inbox = openInbox();
+
+                    int count = inbox.getMessageCount();
+                    Message[] messages = inbox.getMessages(1, count);
+                    for (Message message : messages) {
+                        if (!message.getFlags().contains(Flags.Flag.SEEN)) {
+                            importMessage(message);
+                        }
+                    }
+                } catch (Exception e) {
+                    log.error("Error synching the inbox", e);
+                } finally {
+                    try {
+                        if (inbox != null) {
+                            inbox.close(false);
+                            inbox.getStore().close();
+                        }
+                    } catch (Exception e) {
+                        // NOP
+                    }
+                }
+            }
+        });
+    }
+
+    public int testInbox() {
+        final AtomicInteger count = new AtomicInteger(-1);
+        TransactionUtil.handle(new Runnable() {
+            @Override
+            public void run() {
+                Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED);
+                if (!enabled) {
+                    return;
+                }
+
+                Folder inbox = null;
+                try {
+                    inbox = openInbox();
+                    count.set(inbox.getMessageCount());
+                } catch (Exception e) {
+                    log.error("Error testing inbox", e);
+                } finally {
+                    try {
+                        if (inbox != null) {
+                            inbox.close(false);
+                            inbox.getStore().close();
+                        }
+                    } catch (Exception e) {
+                        // NOP
+                    }
+                }
+            }
+        });
+
+        return count.get();
+    }
+
+    @Override
+    protected Scheduler scheduler() {
+        return Scheduler.newFixedDelaySchedule(0, 15, TimeUnit.MINUTES);
+    }
+
+    /**
+     * Open the remote inbox.
+     *
+     * @return Opened inbox folder
+     */
+    private Folder openInbox() throws Exception {
+        Properties properties = new Properties();
+        String port = ConfigUtil.getConfigStringValue(ConfigType.INBOX_PORT);
+        properties.put("mail.imap.host", ConfigUtil.getConfigStringValue(ConfigType.INBOX_HOSTNAME));
+        properties.put("mail.imap.port", port);
+        boolean isSsl = "993".equals(port);
+        properties.put("mail.imap.ssl.enable", String.valueOf(isSsl));
+        properties.setProperty("mail.imap.socketFactory.class",
+                isSsl ? "javax.net.ssl.SSLSocketFactory" : "javax.net.DefaultSocketFactory");
+        properties.setProperty("mail.imap.socketFactory.fallback", "true");
+        properties.setProperty("mail.imap.socketFactory.port", port);
+
+        Session session = Session.getDefaultInstance(properties);
+
+        Store store = session.getStore("imap");
+        store.connect(ConfigUtil.getConfigStringValue(ConfigType.INBOX_USERNAME),
+                ConfigUtil.getConfigStringValue(ConfigType.INBOX_PASSWORD));
+
+        Folder inbox = store.getFolder("INBOX");
+        inbox.open(Folder.READ_WRITE);
+        return inbox;
+    }
+
+    /**
+     * Import an email.
+     *
+     * @param message Message
+     * @throws Exception
+     */
+    private void importMessage(Message message) throws Exception {
+        // Parse the mail
+        EmailUtil.MailContent mailContent = new EmailUtil.MailContent();
+        mailContent.setSubject(message.getSubject());
+        mailContent.setDate(message.getSentDate());
+        EmailUtil.parseMailContent(message, mailContent);
+
+        // Create the document
+        Document document = new Document();
+        document.setUserId("admin");
+        if (mailContent.getSubject() == null) {
+            document.setTitle("Imported email from EML file");
+        } else {
+            document.setTitle(StringUtils.abbreviate(mailContent.getSubject(), 100));
+        }
+        document.setDescription(StringUtils.abbreviate(mailContent.getMessage(), 4000));
+        document.setSubject(StringUtils.abbreviate(mailContent.getSubject(), 500));
+        document.setFormat("EML");
+        document.setSource("Inbox");
+        document.setLanguage(ConfigUtil.getConfigStringValue(ConfigType.DEFAULT_LANGUAGE));
+        if (mailContent.getDate() == null) {
+            document.setCreateDate(new Date());
+        } else {
+            document.setCreateDate(mailContent.getDate());
+        }
+
+        // Save the document, create the base ACLs
+        document = DocumentUtil.createDocument(document, "admin");
+
+        // Add the tag
+        String tagId = ConfigUtil.getConfigStringValue(ConfigType.INBOX_TAG);
+        if (tagId != null) {
+            TagDao tagDao = new TagDao();
+            Tag tag = tagDao.getById(tagId);
+            if (tag != null) {
+                tagDao.updateTagList(document.getId(), Sets.newHashSet(tagId));
+            }
+        }
+
+        // Raise a document created event
+        DocumentCreatedAsyncEvent documentCreatedAsyncEvent = new DocumentCreatedAsyncEvent();
+        documentCreatedAsyncEvent.setUserId("admin");
+        documentCreatedAsyncEvent.setDocument(document);
+        ThreadLocalContext.get().addAsyncEvent(documentCreatedAsyncEvent);
+
+        // Add files to the document
+        for (EmailUtil.FileContent fileContent : mailContent.getFileContentList()) {
+            FileUtil.createFile(fileContent.getName(), fileContent.getFile(), fileContent.getSize(), "eng", "admin", document.getId());
+        }
+    }
+}

docs-core/src/main/java/com/sismics/docs/core/service/IndexingService.java

@@ -114,7 +114,7 @@ public class IndexingService extends AbstractScheduledService {
     }
     
     @Override
-    protected void runOneIteration() throws Exception {
+    protected void runOneIteration() {
         TransactionUtil.handle(new Runnable() {
             @Override
             public void run() {

docs-core/src/main/resources/config.properties

@@ -1 +1 @@
-db.version=16
+db.version=17

docs-core/src/main/resources/db/update/dbupdate-017-0.sql

@@ -0,0 +1,2 @@
+insert into T_CONFIG(CFG_ID_C, CFG_VALUE_C) values('INBOX_ENABLED', 'false');
+update T_CONFIG set CFG_VALUE_C = '17' where CFG_ID_C = 'DB_VERSION';

docs-web/pom.xml

@@ -128,6 +128,12 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>com.icegreen</groupId>
+      <artifactId>greenmail</artifactId>
+      <scope>test</scope>
+    </dependency>
+    
   </dependencies>
 
   <build>

docs-web/src/dev/resources/config.properties

@@ -1,3 +1,3 @@
 api.current_version=${project.version}
 api.min_version=1.0
-db.version=16
+db.version=17

docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java

@@ -8,6 +8,7 @@ import com.sismics.docs.core.dao.jpa.DocumentDao;
 import com.sismics.docs.core.dao.jpa.FileDao;
 import com.sismics.docs.core.dao.jpa.UserDao;
 import com.sismics.docs.core.event.RebuildIndexAsyncEvent;
+import com.sismics.docs.core.model.context.AppContext;
 import com.sismics.docs.core.model.jpa.Config;
 import com.sismics.docs.core.model.jpa.File;
 import com.sismics.docs.core.model.jpa.User;
@@ -174,10 +175,10 @@ public class AppResource extends BaseResource {
      * @apiName GetAppConfigSmtp
      * @apiGroup App
      * @apiSuccess {String} hostname SMTP hostname
-     * @apiSuccess {String} port
+     * @apiSuccess {String} port SMTP port
-     * @apiSuccess {String} username
+     * @apiSuccess {String} username SMTP username
-     * @apiSuccess {String} password
+     * @apiSuccess {String} password SMTP password
-     * @apiSuccess {String} from
+     * @apiSuccess {String} from From address
      * @apiError (client) ForbiddenError Access denied
      * @apiPermission admin
      * @apiVersion 1.5.0
@@ -295,6 +296,161 @@ public class AppResource extends BaseResource {
         return Response.ok().build();
     }
 
+    /**
+     * Get the inbox configuration.
+     *
+     * @api {get} /app/config_inbox Get the inbox scanning configuration
+     * @apiName GetAppConfigInbox
+     * @apiGroup App
+     * @apiSuccess {Boolean} enabled True if the inbox scanning is enabled
+     * @apiSuccess {String} hostname IMAP hostname
+     * @apiSuccess {String} port IMAP port
+     * @apiSuccess {String} username IMAP username
+     * @apiSuccess {String} password IMAP password
+     * @apiSuccess {String} tag Tag for created documents
+     * @apiError (client) ForbiddenError Access denied
+     * @apiPermission admin
+     * @apiVersion 1.5.0
+     *
+     * @return Response
+     */
+    @GET
+    @Path("config_inbox")
+    public Response getConfigInbox() {
+        if (!authenticate()) {
+            throw new ForbiddenClientException();
+        }
+        checkBaseFunction(BaseFunction.ADMIN);
+
+        ConfigDao configDao = new ConfigDao();
+        Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED);
+        Config hostnameConfig = configDao.getById(ConfigType.INBOX_HOSTNAME);
+        Config portConfig = configDao.getById(ConfigType.INBOX_PORT);
+        Config usernameConfig = configDao.getById(ConfigType.INBOX_USERNAME);
+        Config passwordConfig = configDao.getById(ConfigType.INBOX_PASSWORD);
+        Config tagConfig = configDao.getById(ConfigType.INBOX_TAG);
+        JsonObjectBuilder response = Json.createObjectBuilder();
+
+        response.add("enabled", enabled);
+        if (hostnameConfig == null) {
+            response.addNull("hostname");
+        } else {
+            response.add("hostname", hostnameConfig.getValue());
+        }
+        if (portConfig == null) {
+            response.addNull("port");
+        } else {
+            response.add("port", Integer.valueOf(portConfig.getValue()));
+        }
+        if (usernameConfig == null) {
+            response.addNull("username");
+        } else {
+            response.add("username", usernameConfig.getValue());
+        }
+        if (passwordConfig == null) {
+            response.addNull("password");
+        } else {
+            response.add("password", passwordConfig.getValue());
+        }
+        if (tagConfig == null) {
+            response.addNull("tag");
+        } else {
+            response.add("tag", tagConfig.getValue());
+        }
+
+        return Response.ok().entity(response.build()).build();
+    }
+
+    /**
+     * Configure the inbox.
+     *
+     * @api {post} /app/config_inbox Configure the inbox scanning
+     * @apiName PostAppConfigInbox
+     * @apiGroup App
+     * @apiParam {Boolean} enabled True if the inbox scanning is enabled
+     * @apiParam {String} hostname IMAP hostname
+     * @apiParam {Integer} port IMAP port
+     * @apiParam {String} username IMAP username
+     * @apiParam {String} password IMAP password
+     * @apiParam {String} tag Tag for created documents
+     * @apiError (client) ForbiddenError Access denied
+     * @apiError (client) ValidationError Validation error
+     * @apiPermission admin
+     * @apiVersion 1.5.0
+     *
+     * @param enabled True if the inbox scanning is enabled
+     * @param hostname IMAP hostname
+     * @param portStr IMAP port
+     * @param username IMAP username
+     * @param password IMAP password
+     * @param tag Tag for created documents
+     * @return Response
+     */
+    @POST
+    @Path("config_inbox")
+    public Response configInbox(@FormParam("enabled") Boolean enabled,
+                                @FormParam("hostname") String hostname,
+                                @FormParam("port") String portStr,
+                                @FormParam("username") String username,
+                                @FormParam("password") String password,
+                                @FormParam("tag") String tag) {
+        if (!authenticate()) {
+            throw new ForbiddenClientException();
+        }
+        checkBaseFunction(BaseFunction.ADMIN);
+        ValidationUtil.validateRequired(enabled, "enabled");
+        if (!Strings.isNullOrEmpty(portStr)) {
+            ValidationUtil.validateInteger(portStr, "port");
+        }
+
+        // Just update the changed configuration
+        ConfigDao configDao = new ConfigDao();
+        configDao.update(ConfigType.INBOX_ENABLED, enabled.toString());
+        if (!Strings.isNullOrEmpty(hostname)) {
+            configDao.update(ConfigType.INBOX_HOSTNAME, hostname);
+        }
+        if (!Strings.isNullOrEmpty(portStr)) {
+            configDao.update(ConfigType.INBOX_PORT, portStr);
+        }
+        if (!Strings.isNullOrEmpty(username)) {
+            configDao.update(ConfigType.INBOX_USERNAME, username);
+        }
+        if (!Strings.isNullOrEmpty(password)) {
+            configDao.update(ConfigType.INBOX_PASSWORD, password);
+        }
+        if (!Strings.isNullOrEmpty(tag)) {
+            configDao.update(ConfigType.INBOX_TAG, tag);
+        }
+
+        return Response.ok().build();
+    }
+
+    /**
+     * Test the inbox.
+     *
+     * @api {post} /app/test_inbox Test the inbox scanning
+     * @apiName PostAppTestInbox
+     * @apiGroup App
+     * @apiSuccess {Number} Number of unread emails in the inbox
+     * @apiError (client) ForbiddenError Access denied
+     * @apiPermission admin
+     * @apiVersion 1.5.0
+     *
+     * @return Response
+     */
+    @POST
+    @Path("test_inbox")
+    public Response testInbox() {
+        if (!authenticate()) {
+            throw new ForbiddenClientException();
+        }
+        checkBaseFunction(BaseFunction.ADMIN);
+
+        return Response.ok().entity(Json.createObjectBuilder()
+                .add("count", AppContext.getInstance().getInboxService().testInbox())
+                .build()).build();
+    }
+
     /**
      * Retrieve the application logs.
      *

docs-web/src/prod/resources/config.properties

@@ -1,3 +1,3 @@
 api.current_version=${project.version}
 api.min_version=1.0
-db.version=16
+db.version=17

docs-web/src/stress/resources/config.properties

@@ -1,3 +1,3 @@
 api.current_version=${project.version}
 api.min_version=1.0
-db.version=16
+db.version=17

docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java

@@ -1,5 +1,9 @@
 package com.sismics.docs.rest;
 
+import com.icegreen.greenmail.util.GreenMail;
+import com.icegreen.greenmail.util.GreenMailUtil;
+import com.icegreen.greenmail.util.ServerSetup;
+import com.sismics.docs.core.model.context.AppContext;
 import com.sismics.util.filter.TokenBasedSecurityFilter;
 import org.junit.Assert;
 import org.junit.Test;
@@ -218,4 +222,92 @@ public class TestAppResource extends BaseJerseyTest {
         Assert.assertTrue(json.isNull("password"));
         Assert.assertEquals("contact@sismics.com", json.getString("from"));
     }
+
+    /**
+     * Test inbox scanning.
+     */
+    @Test
+    public void testInbox() {
+        // Login admin
+        String adminToken = clientUtil.login("admin", "admin", false);
+
+        // Create a tag
+        JsonObject json = target().path("/tag").request()
+                .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
+                .put(Entity.form(new Form()
+                        .param("name", "Inbox")
+                        .param("color", "#ff0000")), JsonObject.class);
+        String tagInboxId = json.getString("id");
+
+        // Get inbox configuration
+        json = target().path("/app/config_inbox").request()
+                .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
+                .get(JsonObject.class);
+        Assert.assertFalse(json.getBoolean("enabled"));
+        Assert.assertTrue(json.isNull("hostname"));
+        Assert.assertTrue(json.isNull("port"));
+        Assert.assertTrue(json.isNull("username"));
+        Assert.assertTrue(json.isNull("password"));
+        Assert.assertTrue(json.isNull("tag"));
+
+        // Change inbox configuration
+        target().path("/app/config_inbox").request()
+                .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
+                .post(Entity.form(new Form()
+                        .param("enabled", "true")
+                        .param("hostname", "localhost")
+                        .param("port", "143")
+                        .param("username", "test@sismics.com")
+                        .param("password", "12345678")
+                        .param("tag", tagInboxId)
+                ), JsonObject.class);
+
+        // Get inbox configuration
+        json = target().path("/app/config_inbox").request()
+                .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
+                .get(JsonObject.class);
+        Assert.assertTrue(json.getBoolean("enabled"));
+        Assert.assertEquals("localhost", json.getString("hostname"));
+        Assert.assertEquals(143, json.getInt("port"));
+        Assert.assertEquals("test@sismics.com", json.getString("username"));
+        Assert.assertEquals("12345678", json.getString("password"));
+        Assert.assertEquals(tagInboxId, json.getString("tag"));
+
+        GreenMail greenMail = new GreenMail(new ServerSetup[] { ServerSetup.SMTP, ServerSetup.IMAP });
+        greenMail.setUser("test@sismics.com", "12345678");
+        greenMail.start();
+
+        // Test the inbox
+        json = target().path("/app/test_inbox").request()
+                .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
+                .post(Entity.form(new Form()), JsonObject.class);
+        Assert.assertEquals(0, json.getJsonNumber("count").intValue());
+
+        // Send an email
+        GreenMailUtil.sendTextEmail("test@sismics.com", "test@sismicsdocs.com", "Test email 1", "Test content 1", ServerSetup.SMTP);
+
+        // Trigger an inbox sync
+        AppContext.getInstance().getInboxService().syncInbox();
+
+        // Search for added documents
+        json = target().path("/document/list")
+                .queryParam("search", "tag:Inbox")
+                .request()
+                .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
+                .get(JsonObject.class);
+        Assert.assertEquals(1, json.getJsonArray("documents").size());
+
+        // Trigger an inbox sync
+        AppContext.getInstance().getInboxService().syncInbox();
+
+        // Search for added documents
+        json = target().path("/document/list")
+                .queryParam("search", "tag:Inbox")
+                .request()
+                .cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
+                .get(JsonObject.class);
+        Assert.assertEquals(1, json.getJsonArray("documents").size());
+
+        greenMail.stop();
+    }
 }

pom.xml

@@ -44,6 +44,8 @@
     <com.levigo.jbig2.levigo-jbig2-imageio.version>1.6.5</com.levigo.jbig2.levigo-jbig2-imageio.version>
     <com.github.jai-imageio.jai-imageio-core.version>1.3.1</com.github.jai-imageio.jai-imageio-core.version>
     <org.subethamail.subethasmtp-wiser.version>1.2</org.subethamail.subethasmtp-wiser.version>
+    <com.icegreen.greenmail.version>1.5.7</com.icegreen.greenmail.version>
+    <com.sun.mail.javax.mail.version>1.5.6</com.sun.mail.javax.mail.version>
     <org.jsoup.jsoup.version>1.11.2</org.jsoup.jsoup.version>
 
     <org.eclipse.jetty.jetty-server.version>9.2.13.v20150730</org.eclipse.jetty.jetty-server.version>
@@ -412,6 +414,24 @@
         <groupId>org.subethamail</groupId>
         <artifactId>subethasmtp-wiser</artifactId>
         <version>${org.subethamail.subethasmtp-wiser.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>javax.mail</groupId>
+            <artifactId>mail</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+
+      <dependency>
+        <groupId>com.icegreen</groupId>
+        <artifactId>greenmail</artifactId>
+        <version>${com.icegreen.greenmail.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>com.sun.mail</groupId>
+        <artifactId>javax.mail</artifactId>
+        <version>${com.sun.mail.javax.mail.version}</version>
       </dependency>
 
       <!-- Servlet listener to register SPI ImageIO plugins -->