vmario
/
docs
mirror of https://github.com/sismics/docs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#180: IMAP inbox synching (api)

This commit is contained in:
Benjamin Gamard 2018-02-27 14:58:37 +01:00
parent f379b4e5ab
commit 3720a881a4
14 changed files with 525 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs-core/pom.xml
View File

@ -111,6 +111,11 @@
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
</dependency>
<dependency>
<groupId>com.sun.mail</groupId>
<artifactId>javax.mail</artifactId>
</dependency>
<!-- Only there to read old index and rebuild them -->
<dependency>

12
docs-core/src/main/java/com/sismics/docs/core/constant/ConfigType.java
View File

@ -32,5 +32,15 @@ public enum ConfigType {
SMTP_PORT,
SMTP_FROM,
SMTP_USERNAME,
SMTP_PASSWORD
SMTP_PASSWORD,
/**
* Inbox scanning configuration.
*/
INBOX_ENABLED,
INBOX_HOSTNAME,
INBOX_PORT,
INBOX_USERNAME,
INBOX_PASSWORD,
INBOX_TAG
}

14
docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java
View File

@ -10,6 +10,7 @@ import com.sismics.docs.core.listener.async.*;
import com.sismics.docs.core.listener.sync.DeadEventListener;
import com.sismics.docs.core.model.jpa.Config;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.service.InboxService;
import com.sismics.docs.core.service.IndexingService;
import com.sismics.docs.core.util.PdfUtil;
import com.sismics.util.EnvironmentUtil;
@ -52,6 +53,11 @@ public class AppContext {
*/
private IndexingService indexingService;
/**
* Inbox scanning service.
*/
private InboxService inboxService;
/**
* Asynchronous executors.
*/
@ -69,6 +75,10 @@ public class AppContext {
indexingService = new IndexingService(luceneStorageConfig != null ? luceneStorageConfig.getValue() : null);
indexingService.startAsync();
// Start inbox service
inboxService = new InboxService();
inboxService.startAsync();
// Register fonts
PdfUtil.registerFonts();
@ -175,4 +185,8 @@ public class AppContext {
public IndexingService getIndexingService() {
return indexingService;
}
public InboxService getInboxService() {
return inboxService;
}
}

210
docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java Normal file
View File

@ -0,0 +1,210 @@
package com.sismics.docs.core.service;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AbstractScheduledService;
import com.sismics.docs.core.constant.ConfigType;
import com.sismics.docs.core.dao.jpa.TagDao;
import com.sismics.docs.core.event.DocumentCreatedAsyncEvent;
import com.sismics.docs.core.model.jpa.Document;
import com.sismics.docs.core.model.jpa.Tag;
import com.sismics.docs.core.util.ConfigUtil;
import com.sismics.docs.core.util.DocumentUtil;
import com.sismics.docs.core.util.FileUtil;
import com.sismics.docs.core.util.TransactionUtil;
import com.sismics.util.EmailUtil;
import com.sismics.util.context.ThreadLocalContext;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.mail.*;
import java.util.Date;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Inbox scanning service.
*
* @author bgamard
*/
public class InboxService extends AbstractScheduledService {
/**
* Logger.
*/
private static final Logger log = LoggerFactory.getLogger(InboxService.class);
public InboxService() {
}
@Override
protected void startUp() {
}
@Override
protected void shutDown() {
}
@Override
protected void runOneIteration() {
syncInbox();
}
public void syncInbox() {
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED);
if (!enabled) {
return;
}
Folder inbox = null;
try {
inbox = openInbox();
int count = inbox.getMessageCount();
Message[] messages = inbox.getMessages(1, count);
for (Message message : messages) {
if (!message.getFlags().contains(Flags.Flag.SEEN)) {
importMessage(message);
}
}
} catch (Exception e) {
log.error("Error synching the inbox", e);
} finally {
try {
if (inbox != null) {
inbox.close(false);
inbox.getStore().close();
}
} catch (Exception e) {
// NOP
}
}
}
});
}
public int testInbox() {
final AtomicInteger count = new AtomicInteger(-1);
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED);
if (!enabled) {
return;
}
Folder inbox = null;
try {
inbox = openInbox();
count.set(inbox.getMessageCount());
} catch (Exception e) {
log.error("Error testing inbox", e);
} finally {
try {
if (inbox != null) {
inbox.close(false);
inbox.getStore().close();
}
} catch (Exception e) {
// NOP
}
}
}
});
return count.get();
}
@Override
protected Scheduler scheduler() {
return Scheduler.newFixedDelaySchedule(0, 15, TimeUnit.MINUTES);
}
/**
* Open the remote inbox.
*
* @return Opened inbox folder
*/
private Folder openInbox() throws Exception {
Properties properties = new Properties();
String port = ConfigUtil.getConfigStringValue(ConfigType.INBOX_PORT);
properties.put("mail.imap.host", ConfigUtil.getConfigStringValue(ConfigType.INBOX_HOSTNAME));
properties.put("mail.imap.port", port);
boolean isSsl = "993".equals(port);
properties.put("mail.imap.ssl.enable", String.valueOf(isSsl));
properties.setProperty("mail.imap.socketFactory.class",
isSsl ? "javax.net.ssl.SSLSocketFactory" : "javax.net.DefaultSocketFactory");
properties.setProperty("mail.imap.socketFactory.fallback", "true");
properties.setProperty("mail.imap.socketFactory.port", port);
Session session = Session.getDefaultInstance(properties);
Store store = session.getStore("imap");
store.connect(ConfigUtil.getConfigStringValue(ConfigType.INBOX_USERNAME),
ConfigUtil.getConfigStringValue(ConfigType.INBOX_PASSWORD));
Folder inbox = store.getFolder("INBOX");
inbox.open(Folder.READ_WRITE);
return inbox;
}
/**
* Import an email.
*
* @param message Message
* @throws Exception
*/
private void importMessage(Message message) throws Exception {
// Parse the mail
EmailUtil.MailContent mailContent = new EmailUtil.MailContent();
mailContent.setSubject(message.getSubject());
mailContent.setDate(message.getSentDate());
EmailUtil.parseMailContent(message, mailContent);
// Create the document
Document document = new Document();
document.setUserId("admin");
if (mailContent.getSubject() == null) {
document.setTitle("Imported email from EML file");
} else {
document.setTitle(StringUtils.abbreviate(mailContent.getSubject(), 100));
}
document.setDescription(StringUtils.abbreviate(mailContent.getMessage(), 4000));
document.setSubject(StringUtils.abbreviate(mailContent.getSubject(), 500));
document.setFormat("EML");
document.setSource("Inbox");
document.setLanguage(ConfigUtil.getConfigStringValue(ConfigType.DEFAULT_LANGUAGE));
if (mailContent.getDate() == null) {
document.setCreateDate(new Date());
} else {
document.setCreateDate(mailContent.getDate());
}
// Save the document, create the base ACLs
document = DocumentUtil.createDocument(document, "admin");
// Add the tag
String tagId = ConfigUtil.getConfigStringValue(ConfigType.INBOX_TAG);
if (tagId != null) {
TagDao tagDao = new TagDao();
Tag tag = tagDao.getById(tagId);
if (tag != null) {
tagDao.updateTagList(document.getId(), Sets.newHashSet(tagId));
}
}
// Raise a document created event
DocumentCreatedAsyncEvent documentCreatedAsyncEvent = new DocumentCreatedAsyncEvent();
documentCreatedAsyncEvent.setUserId("admin");
documentCreatedAsyncEvent.setDocument(document);
ThreadLocalContext.get().addAsyncEvent(documentCreatedAsyncEvent);
// Add files to the document
for (EmailUtil.FileContent fileContent : mailContent.getFileContentList()) {
FileUtil.createFile(fileContent.getName(), fileContent.getFile(), fileContent.getSize(), "eng", "admin", document.getId());
}
}
}

2
docs-core/src/main/java/com/sismics/docs/core/service/IndexingService.java
View File

@ -114,7 +114,7 @@ public class IndexingService extends AbstractScheduledService {
}
@Override
protected void runOneIteration() throws Exception {
protected void runOneIteration() {
TransactionUtil.handle(new Runnable() {
@Override
public void run() {

2
docs-core/src/main/resources/config.properties
View File

@ -1 +1 @@
db.version=16
db.version=17

2
docs-core/src/main/resources/db/update/dbupdate-017-0.sql Normal file
View File

@ -0,0 +1,2 @@
insert into T_CONFIG(CFG_ID_C, CFG_VALUE_C) values('INBOX_ENABLED', 'false');
update T_CONFIG set CFG_VALUE_C = '17' where CFG_ID_C = 'DB_VERSION';

6
docs-web/pom.xml
View File

@ -127,6 +127,12 @@
<artifactId>subethasmtp-wiser</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.icegreen</groupId>
<artifactId>greenmail</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

2
docs-web/src/dev/resources/config.properties
View File

@ -1,3 +1,3 @@
api.current_version=${project.version}
api.min_version=1.0
db.version=16
db.version=17

164
docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java
View File

@ -8,6 +8,7 @@ import com.sismics.docs.core.dao.jpa.DocumentDao;
import com.sismics.docs.core.dao.jpa.FileDao;
import com.sismics.docs.core.dao.jpa.UserDao;
import com.sismics.docs.core.event.RebuildIndexAsyncEvent;
import com.sismics.docs.core.model.context.AppContext;
import com.sismics.docs.core.model.jpa.Config;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
@ -174,10 +175,10 @@ public class AppResource extends BaseResource {
* @apiName GetAppConfigSmtp
* @apiGroup App
* @apiSuccess {String} hostname SMTP hostname
* @apiSuccess {String} port
* @apiSuccess {String} username
* @apiSuccess {String} password
* @apiSuccess {String} from
* @apiSuccess {String} port SMTP port
* @apiSuccess {String} username SMTP username
* @apiSuccess {String} password SMTP password
* @apiSuccess {String} from From address
* @apiError (client) ForbiddenError Access denied
* @apiPermission admin
* @apiVersion 1.5.0
@ -295,6 +296,161 @@ public class AppResource extends BaseResource {
return Response.ok().build();
}
/**
* Get the inbox configuration.
*
* @api {get} /app/config_inbox Get the inbox scanning configuration
* @apiName GetAppConfigInbox
* @apiGroup App
* @apiSuccess {Boolean} enabled True if the inbox scanning is enabled
* @apiSuccess {String} hostname IMAP hostname
* @apiSuccess {String} port IMAP port
* @apiSuccess {String} username IMAP username
* @apiSuccess {String} password IMAP password
* @apiSuccess {String} tag Tag for created documents
* @apiError (client) ForbiddenError Access denied
* @apiPermission admin
* @apiVersion 1.5.0
*
* @return Response
*/
@GET
@Path("config_inbox")
public Response getConfigInbox() {
if (!authenticate()) {
throw new ForbiddenClientException();
}
checkBaseFunction(BaseFunction.ADMIN);
ConfigDao configDao = new ConfigDao();
Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED);
Config hostnameConfig = configDao.getById(ConfigType.INBOX_HOSTNAME);
Config portConfig = configDao.getById(ConfigType.INBOX_PORT);
Config usernameConfig = configDao.getById(ConfigType.INBOX_USERNAME);
Config passwordConfig = configDao.getById(ConfigType.INBOX_PASSWORD);
Config tagConfig = configDao.getById(ConfigType.INBOX_TAG);
JsonObjectBuilder response = Json.createObjectBuilder();
response.add("enabled", enabled);
if (hostnameConfig == null) {
response.addNull("hostname");
} else {
response.add("hostname", hostnameConfig.getValue());
}
if (portConfig == null) {
response.addNull("port");
} else {
response.add("port", Integer.valueOf(portConfig.getValue()));
}
if (usernameConfig == null) {
response.addNull("username");
} else {
response.add("username", usernameConfig.getValue());
}
if (passwordConfig == null) {
response.addNull("password");
} else {
response.add("password", passwordConfig.getValue());
}
if (tagConfig == null) {
response.addNull("tag");
} else {
response.add("tag", tagConfig.getValue());
}
return Response.ok().entity(response.build()).build();
}
/**
* Configure the inbox.
*
* @api {post} /app/config_inbox Configure the inbox scanning
* @apiName PostAppConfigInbox
* @apiGroup App
* @apiParam {Boolean} enabled True if the inbox scanning is enabled
* @apiParam {String} hostname IMAP hostname
* @apiParam {Integer} port IMAP port
* @apiParam {String} username IMAP username
* @apiParam {String} password IMAP password
* @apiParam {String} tag Tag for created documents
* @apiError (client) ForbiddenError Access denied
* @apiError (client) ValidationError Validation error
* @apiPermission admin
* @apiVersion 1.5.0
*
* @param enabled True if the inbox scanning is enabled
* @param hostname IMAP hostname
* @param portStr IMAP port
* @param username IMAP username
* @param password IMAP password
* @param tag Tag for created documents
* @return Response
*/
@POST
@Path("config_inbox")
public Response configInbox(@FormParam("enabled") Boolean enabled,
@FormParam("hostname") String hostname,
@FormParam("port") String portStr,
@FormParam("username") String username,
@FormParam("password") String password,
@FormParam("tag") String tag) {
if (!authenticate()) {
throw new ForbiddenClientException();
}
checkBaseFunction(BaseFunction.ADMIN);
ValidationUtil.validateRequired(enabled, "enabled");
if (!Strings.isNullOrEmpty(portStr)) {
ValidationUtil.validateInteger(portStr, "port");
}
// Just update the changed configuration
ConfigDao configDao = new ConfigDao();
configDao.update(ConfigType.INBOX_ENABLED, enabled.toString());
if (!Strings.isNullOrEmpty(hostname)) {
configDao.update(ConfigType.INBOX_HOSTNAME, hostname);
}
if (!Strings.isNullOrEmpty(portStr)) {
configDao.update(ConfigType.INBOX_PORT, portStr);
}
if (!Strings.isNullOrEmpty(username)) {
configDao.update(ConfigType.INBOX_USERNAME, username);
}
if (!Strings.isNullOrEmpty(password)) {
configDao.update(ConfigType.INBOX_PASSWORD, password);
}
if (!Strings.isNullOrEmpty(tag)) {
configDao.update(ConfigType.INBOX_TAG, tag);
}
return Response.ok().build();
}
/**
* Test the inbox.
*
* @api {post} /app/test_inbox Test the inbox scanning
* @apiName PostAppTestInbox
* @apiGroup App
* @apiSuccess {Number} Number of unread emails in the inbox
* @apiError (client) ForbiddenError Access denied
* @apiPermission admin
* @apiVersion 1.5.0
*
* @return Response
*/
@POST
@Path("test_inbox")
public Response testInbox() {
if (!authenticate()) {
throw new ForbiddenClientException();
}
checkBaseFunction(BaseFunction.ADMIN);
return Response.ok().entity(Json.createObjectBuilder()
.add("count", AppContext.getInstance().getInboxService().testInbox())
.build()).build();
}
/**
* Retrieve the application logs.
*

2
docs-web/src/prod/resources/config.properties
View File

@ -1,3 +1,3 @@
api.current_version=${project.version}
api.min_version=1.0
db.version=16
db.version=17

2
docs-web/src/stress/resources/config.properties
View File

@ -1,3 +1,3 @@
api.current_version=${project.version}
api.min_version=1.0
db.version=16
db.version=17

92
docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java
View File

@ -1,5 +1,9 @@
package com.sismics.docs.rest;
import com.icegreen.greenmail.util.GreenMail;
import com.icegreen.greenmail.util.GreenMailUtil;
import com.icegreen.greenmail.util.ServerSetup;
import com.sismics.docs.core.model.context.AppContext;
import com.sismics.util.filter.TokenBasedSecurityFilter;
import org.junit.Assert;
import org.junit.Test;
@ -218,4 +222,92 @@ public class TestAppResource extends BaseJerseyTest {
Assert.assertTrue(json.isNull("password"));
Assert.assertEquals("contact@sismics.com", json.getString("from"));
}
/**
* Test inbox scanning.
*/
@Test
public void testInbox() {
// Login admin
String adminToken = clientUtil.login("admin", "admin", false);
// Create a tag
JsonObject json = target().path("/tag").request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
.put(Entity.form(new Form()
.param("name", "Inbox")
.param("color", "#ff0000")), JsonObject.class);
String tagInboxId = json.getString("id");
// Get inbox configuration
json = target().path("/app/config_inbox").request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
.get(JsonObject.class);
Assert.assertFalse(json.getBoolean("enabled"));
Assert.assertTrue(json.isNull("hostname"));
Assert.assertTrue(json.isNull("port"));
Assert.assertTrue(json.isNull("username"));
Assert.assertTrue(json.isNull("password"));
Assert.assertTrue(json.isNull("tag"));
// Change inbox configuration
target().path("/app/config_inbox").request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
.post(Entity.form(new Form()
.param("enabled", "true")
.param("hostname", "localhost")
.param("port", "143")
.param("username", "test@sismics.com")
.param("password", "12345678")
.param("tag", tagInboxId)
), JsonObject.class);
// Get inbox configuration
json = target().path("/app/config_inbox").request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
.get(JsonObject.class);
Assert.assertTrue(json.getBoolean("enabled"));
Assert.assertEquals("localhost", json.getString("hostname"));
Assert.assertEquals(143, json.getInt("port"));
Assert.assertEquals("test@sismics.com", json.getString("username"));
Assert.assertEquals("12345678", json.getString("password"));
Assert.assertEquals(tagInboxId, json.getString("tag"));
GreenMail greenMail = new GreenMail(new ServerSetup[] { ServerSetup.SMTP, ServerSetup.IMAP });
greenMail.setUser("test@sismics.com", "12345678");
greenMail.start();
// Test the inbox
json = target().path("/app/test_inbox").request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
.post(Entity.form(new Form()), JsonObject.class);
Assert.assertEquals(0, json.getJsonNumber("count").intValue());
// Send an email
GreenMailUtil.sendTextEmail("test@sismics.com", "test@sismicsdocs.com", "Test email 1", "Test content 1", ServerSetup.SMTP);
// Trigger an inbox sync
AppContext.getInstance().getInboxService().syncInbox();
// Search for added documents
json = target().path("/document/list")
.queryParam("search", "tag:Inbox")
.request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
.get(JsonObject.class);
Assert.assertEquals(1, json.getJsonArray("documents").size());
// Trigger an inbox sync
AppContext.getInstance().getInboxService().syncInbox();
// Search for added documents
json = target().path("/document/list")
.queryParam("search", "tag:Inbox")
.request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, adminToken)
.get(JsonObject.class);
Assert.assertEquals(1, json.getJsonArray("documents").size());
greenMail.stop();
}
}

20
pom.xml
View File

@ -44,6 +44,8 @@
<com.levigo.jbig2.levigo-jbig2-imageio.version>1.6.5</com.levigo.jbig2.levigo-jbig2-imageio.version>
<com.github.jai-imageio.jai-imageio-core.version>1.3.1</com.github.jai-imageio.jai-imageio-core.version>
<org.subethamail.subethasmtp-wiser.version>1.2</org.subethamail.subethasmtp-wiser.version>
<com.icegreen.greenmail.version>1.5.7</com.icegreen.greenmail.version>
<com.sun.mail.javax.mail.version>1.5.6</com.sun.mail.javax.mail.version>
<org.jsoup.jsoup.version>1.11.2</org.jsoup.jsoup.version>
<org.eclipse.jetty.jetty-server.version>9.2.13.v20150730</org.eclipse.jetty.jetty-server.version>
@ -412,6 +414,24 @@
<groupId>org.subethamail</groupId>
<artifactId>subethasmtp-wiser</artifactId>
<version>${org.subethamail.subethasmtp-wiser.version}</version>
<exclusions>
<exclusion>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.icegreen</groupId>
<artifactId>greenmail</artifactId>
<version>${com.icegreen.greenmail.version}</version>
</dependency>
<dependency>
<groupId>com.sun.mail</groupId>
<artifactId>javax.mail</artifactId>
<version>${com.sun.mail.javax.mail.version}</version>
</dependency>
<!-- Servlet listener to register SPI ImageIO plugins -->