#177: import document from EML file (api done)

docs-core/src/main/java/com/sismics/docs/core/util/DocumentUtil.java

@@ -13,6 +13,13 @@ import com.sismics.docs.core.model.jpa.Document;
  * @author bgamard
  */
 public class DocumentUtil {
+    /**
+     * Create a document and add the base ACLs.
+     *
+     * @param document Document
+     * @param userId User creating the document
+     * @return Created document
+     */
     public static Document createDocument(Document document, String userId) {
         DocumentDao documentDao = new DocumentDao();
         String documentId = documentDao.create(document, userId);

docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java

@@ -1,10 +1,19 @@
 package com.sismics.docs.core.util;
 
+import com.google.common.base.Strings;
+import com.sismics.docs.core.constant.Constants;
+import com.sismics.docs.core.dao.jpa.FileDao;
+import com.sismics.docs.core.dao.jpa.UserDao;
+import com.sismics.docs.core.event.DocumentUpdatedAsyncEvent;
+import com.sismics.docs.core.event.FileCreatedAsyncEvent;
 import com.sismics.docs.core.model.jpa.File;
+import com.sismics.docs.core.model.jpa.User;
 import com.sismics.tess4j.Tesseract;
 import com.sismics.util.ImageDeskew;
 import com.sismics.util.ImageUtil;
 import com.sismics.util.Scalr;
+import com.sismics.util.context.ThreadLocalContext;
+import com.sismics.util.mime.MimeTypeUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -167,4 +176,92 @@ public class FileUtil {
             Files.delete(thumbnailFile);
         }
     }
+
+    /**
+     * Create a new file.
+     *
+     * @param name File name, can be null
+     * @param unencryptedFile Path to the unencrypted file
+     * @param fileSize File size
+     * @param language File language, can be null if associated to no document
+     * @param userId User ID creating the file
+     * @param documentId Associated document ID or null if no document
+     * @return File ID
+     * @throws Exception e
+     */
+    public static String createFile(String name, Path unencryptedFile, long fileSize, String language, String userId, String documentId) throws Exception {
+        // Validate mime type
+        String mimeType;
+        try {
+            mimeType = MimeTypeUtil.guessMimeType(unencryptedFile, name);
+        } catch (IOException e) {
+            throw new IOException("ErrorGuessMime", e);
+        }
+
+        // Validate user quota
+        UserDao userDao = new UserDao();
+        User user = userDao.getById(userId);
+        if (user.getStorageCurrent() + fileSize > user.getStorageQuota()) {
+            throw new IOException("QuotaReached");
+        }
+
+        // Validate global quota
+        String globalStorageQuotaStr = System.getenv(Constants.GLOBAL_QUOTA_ENV);
+        if (!Strings.isNullOrEmpty(globalStorageQuotaStr)) {
+            long globalStorageQuota = Long.valueOf(globalStorageQuotaStr);
+            long globalStorageCurrent = userDao.getGlobalStorageCurrent();
+            if (globalStorageCurrent + fileSize > globalStorageQuota) {
+                throw new IOException("QuotaReached");
+            }
+        }
+
+        // Get files of this document
+        FileDao fileDao = new FileDao();
+        int order = 0;
+        if (documentId != null) {
+            for (File file : fileDao.getByDocumentId(userId, documentId)) {
+                file.setOrder(order++);
+            }
+        }
+
+        // Create the file
+        File file = new File();
+        file.setOrder(order);
+        file.setDocumentId(documentId);
+        file.setName(name);
+        file.setMimeType(mimeType);
+        file.setUserId(userId);
+        String fileId = fileDao.create(file, userId);
+
+        // Guess the mime type a second time, for open document format (first detected as simple ZIP file)
+        file.setMimeType(MimeTypeUtil.guessOpenDocumentFormat(file, unencryptedFile));
+
+        // Convert to PDF if necessary (for thumbnail and text extraction)
+        java.nio.file.Path unencryptedPdfFile = PdfUtil.convertToPdf(file, unencryptedFile);
+
+        // Save the file
+        FileUtil.save(unencryptedFile, unencryptedPdfFile, file, user.getPrivateKey());
+
+        // Update the user quota
+        user.setStorageCurrent(user.getStorageCurrent() + fileSize);
+        userDao.updateQuota(user);
+
+        // Raise a new file created event and document updated event if we have a document
+        if (documentId != null) {
+            FileCreatedAsyncEvent fileCreatedAsyncEvent = new FileCreatedAsyncEvent();
+            fileCreatedAsyncEvent.setUserId(userId);
+            fileCreatedAsyncEvent.setLanguage(language);
+            fileCreatedAsyncEvent.setFile(file);
+            fileCreatedAsyncEvent.setUnencryptedFile(unencryptedFile);
+            fileCreatedAsyncEvent.setUnencryptedPdfFile(unencryptedPdfFile);
+            ThreadLocalContext.get().addAsyncEvent(fileCreatedAsyncEvent);
+
+            DocumentUpdatedAsyncEvent documentUpdatedAsyncEvent = new DocumentUpdatedAsyncEvent();
+            documentUpdatedAsyncEvent.setUserId(userId);
+            documentUpdatedAsyncEvent.setDocumentId(documentId);
+            ThreadLocalContext.get().addAsyncEvent(documentUpdatedAsyncEvent);
+        }
+
+        return fileId;
+    }
 }

docs-core/src/main/java/com/sismics/util/EmailUtil.java

@@ -79,7 +79,7 @@ public class EmailUtil {
      * @param subject Email subject
      * @param paramMap Email parameters
      */
-    public static void sendEmail(String templateName, UserDto recipientUser, String subject, Map<String, Object> paramMap) {
+    private static void sendEmail(String templateName, UserDto recipientUser, String subject, Map<String, Object> paramMap) {
         if (log.isInfoEnabled()) {
             log.info("Sending email from template=" + templateName + " to user " + recipientUser);
         }
@@ -175,6 +175,15 @@ public class EmailUtil {
         sendEmail(templateName, recipientUser, subject, paramMap);
     }
 
+    /**
+     * Parse an email content to be imported.
+     *
+     * @param part Email part
+     * @param mailContent Mail content modified by side-effect
+     *
+     * @throws MessagingException e
+     * @throws IOException e
+     */
     public static void parseMailContent(Part part, MailContent mailContent) throws MessagingException, IOException {
         Object content = part.getContent();
         if (content instanceof Multipart) {
@@ -261,5 +270,17 @@ public class EmailUtil {
         private String name;
         private Path file;
         private long size;
+
+        public String getName() {
+            return name;
+        }
+
+        public Path getFile() {
+            return file;
+        }
+
+        public long getSize() {
+            return size;
+        }
     }
 }

docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java

@@ -17,6 +17,7 @@ import com.sismics.docs.core.model.jpa.Document;
 import com.sismics.docs.core.model.jpa.File;
 import com.sismics.docs.core.model.jpa.User;
 import com.sismics.docs.core.util.DocumentUtil;
+import com.sismics.docs.core.util.FileUtil;
 import com.sismics.docs.core.util.PdfUtil;
 import com.sismics.docs.core.util.jpa.PaginatedList;
 import com.sismics.docs.core.util.jpa.PaginatedLists;
@@ -405,7 +406,7 @@ public class DocumentResource extends BaseResource {
     /**
      * Parse a query according to the specified syntax, eg.:
      * tag:assurance tag:other before:2012 after:2011-09 shared:yes lang:fra thing
-     * 
+     *
      * @param search Search query
      * @return DocumentCriteria
      */
@@ -414,10 +415,10 @@ public class DocumentResource extends BaseResource {
         if (Strings.isNullOrEmpty(search)) {
             return documentCriteria;
         }
-        
+
         TagDao tagDao = new TagDao();
         UserDao userDao = new UserDao();
-        DateTimeParser[] parsers = { 
+        DateTimeParser[] parsers = {
                 DateTimeFormat.forPattern("yyyy").getParser(),
                 DateTimeFormat.forPattern("yyyy-MM").getParser(),
                 DateTimeFormat.forPattern("yyyy-MM-dd").getParser() };
@@ -425,7 +426,7 @@ public class DocumentResource extends BaseResource {
         DateTimeFormatter monthFormatter = new DateTimeFormatter(null, parsers[1]);
         DateTimeFormatter dayFormatter = new DateTimeFormatter(null, parsers[2]);
         DateTimeFormatter formatter = new DateTimeFormatterBuilder().append( null, parsers ).toFormatter();
-        
+
         String[] criteriaList = search.split("  *");
         List<String> query = new ArrayList<>();
         List<String> fullQuery = new ArrayList<>();
@@ -519,7 +520,7 @@ public class DocumentResource extends BaseResource {
                     break;
             }
         }
-        
+
         documentCriteria.setSearch(Joiner.on(" ").join(query));
         documentCriteria.setFullSearch(Joiner.on(" ").join(fullQuery));
         return documentCriteria;
@@ -845,7 +846,16 @@ public class DocumentResource extends BaseResource {
         documentCreatedAsyncEvent.setDocument(document);
         ThreadLocalContext.get().addAsyncEvent(documentCreatedAsyncEvent);
 
-        // TODO Add files to the document
+        // Add files to the document
+        try {
+            for (EmailUtil.FileContent fileContent : mailContent.getFileContentList()) {
+                FileUtil.createFile(fileContent.getName(), fileContent.getFile(), fileContent.getSize(), "eng", principal.getId(), document.getId());
+            }
+        } catch (IOException e) {
+            throw new ClientException(e.getMessage(), e.getMessage(), e);
+        } catch (Exception e) {
+            throw new ServerException("FileError", "Error adding a file", e);
+        }
 
         JsonObjectBuilder response = Json.createObjectBuilder()
                 .add("id", document.getId());

docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java

@@ -3,7 +3,6 @@ package com.sismics.docs.rest.resource;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import com.google.common.io.ByteStreams;
-import com.sismics.docs.core.constant.Constants;
 import com.sismics.docs.core.constant.PermType;
 import com.sismics.docs.core.dao.jpa.AclDao;
 import com.sismics.docs.core.dao.jpa.DocumentDao;
@@ -27,7 +26,6 @@ import com.sismics.util.HttpUtil;
 import com.sismics.util.JsonUtil;
 import com.sismics.util.context.ThreadLocalContext;
 import com.sismics.util.mime.MimeType;
-import com.sismics.util.mime.MimeTypeUtil;
 import org.glassfish.jersey.media.multipart.FormDataBodyPart;
 import org.glassfish.jersey.media.multipart.FormDataParam;
 
@@ -98,10 +96,6 @@ public class FileResource extends BaseResource {
         // Validate input data
         ValidationUtil.validateRequired(fileBodyPart, "file");
 
-        // Get the current user
-        UserDao userDao = new UserDao();
-        User user = userDao.getById(principal.getId());
-        
         // Get the document
         DocumentDto documentDto = null;
         if (Strings.isNullOrEmpty(documentId)) {
@@ -114,7 +108,7 @@ public class FileResource extends BaseResource {
             }
         }
         
-        // Keep unencrypted data temporary on disk, because we will need it two times
+        // Keep unencrypted data temporary on disk
         java.nio.file.Path unencryptedFile;
         long fileSize;
         try {
@@ -125,78 +119,11 @@ public class FileResource extends BaseResource {
             throw new ServerException("StreamError", "Error reading the input file", e);
         }
 
-        // Validate mime type
-        String name = fileBodyPart.getContentDisposition() != null ?
-                fileBodyPart.getContentDisposition().getFileName() : null;
-        String mimeType;
         try {
-            mimeType = MimeTypeUtil.guessMimeType(unencryptedFile, name);
+            String name = fileBodyPart.getContentDisposition() != null ?
-        } catch (IOException e) {
+                    fileBodyPart.getContentDisposition().getFileName() : null;
-            throw new ServerException("ErrorGuessMime", "Error guessing mime type", e);
+            String fileId = FileUtil.createFile(name, unencryptedFile, fileSize, documentDto == null ?
-        }
+                    null : documentDto.getLanguage(), principal.getId(), documentId);
-
-        // Validate user quota
-        if (user.getStorageCurrent() + fileSize > user.getStorageQuota()) {
-            throw new ClientException("QuotaReached", "Quota limit reached");
-        }
-
-        // Validate global quota
-        String globalStorageQuotaStr = System.getenv(Constants.GLOBAL_QUOTA_ENV);
-        if (!Strings.isNullOrEmpty(globalStorageQuotaStr)) {
-            long globalStorageQuota = Long.valueOf(globalStorageQuotaStr);
-            long globalStorageCurrent = userDao.getGlobalStorageCurrent();
-            if (globalStorageCurrent + fileSize > globalStorageQuota) {
-                throw new ClientException("QuotaReached", "Global quota limit reached");
-            }
-        }
-
-        try {
-            // Get files of this document
-            FileDao fileDao = new FileDao();
-            int order = 0;
-            if (documentId != null) {
-                for (File file : fileDao.getByDocumentId(principal.getId(), documentId)) {
-                    file.setOrder(order++);
-                }
-            }
-            
-            // Create the file
-            File file = new File();
-            file.setOrder(order);
-            file.setDocumentId(documentId);
-            file.setName(name);
-            file.setMimeType(mimeType);
-            file.setUserId(principal.getId());
-            String fileId = fileDao.create(file, principal.getId());
-            
-            // Guess the mime type a second time, for open document format (first detected as simple ZIP file)
-            file.setMimeType(MimeTypeUtil.guessOpenDocumentFormat(file, unencryptedFile));
-
-            // Convert to PDF if necessary (for thumbnail and text extraction)
-            java.nio.file.Path unencryptedPdfFile = PdfUtil.convertToPdf(file, unencryptedFile);
-
-            // Save the file
-            FileUtil.save(unencryptedFile, unencryptedPdfFile, file, user.getPrivateKey());
-            
-            // Update the user quota
-            user.setStorageCurrent(user.getStorageCurrent() + fileSize);
-            userDao.updateQuota(user);
-            
-            // Raise a new file created event and document updated event if we have a document
-            if (documentId != null) {
-                FileCreatedAsyncEvent fileCreatedAsyncEvent = new FileCreatedAsyncEvent();
-                fileCreatedAsyncEvent.setUserId(principal.getId());
-                fileCreatedAsyncEvent.setLanguage(documentDto.getLanguage());
-                fileCreatedAsyncEvent.setFile(file);
-                fileCreatedAsyncEvent.setUnencryptedFile(unencryptedFile);
-                fileCreatedAsyncEvent.setUnencryptedPdfFile(unencryptedPdfFile);
-                ThreadLocalContext.get().addAsyncEvent(fileCreatedAsyncEvent);
-                
-                DocumentUpdatedAsyncEvent documentUpdatedAsyncEvent = new DocumentUpdatedAsyncEvent();
-                documentUpdatedAsyncEvent.setUserId(principal.getId());
-                documentUpdatedAsyncEvent.setDocumentId(documentId);
-                ThreadLocalContext.get().addAsyncEvent(documentUpdatedAsyncEvent);
-            }
 
             // Always return OK
             JsonObjectBuilder response = Json.createObjectBuilder()
@@ -204,6 +131,8 @@ public class FileResource extends BaseResource {
                     .add("id", fileId)
                     .add("size", fileSize);
             return Response.ok().entity(response.build()).build();
+        } catch (IOException e) {
+            throw new ClientException(e.getMessage(), e.getMessage(), e);
         } catch (Exception e) {
             throw new ServerException("FileError", "Error adding a file", e);
         }

docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java

@@ -615,10 +615,11 @@ public class TestDocumentResource extends BaseJerseyTest {
         String documentEmlToken = clientUtil.login("document_eml");
 
         // Import a document as EML
+        JsonObject json;
         try (InputStream is = Resources.getResource("file/test_mail.eml").openStream()) {
             StreamDataBodyPart streamDataBodyPart = new StreamDataBodyPart("file", is, "test_mail.eml");
             try (FormDataMultiPart multiPart = new FormDataMultiPart()) {
-                target()
+                json = target()
                         .register(MultiPartFeature.class)
                         .path("/document/eml").request()
                         .cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentEmlToken)
@@ -626,5 +627,35 @@ public class TestDocumentResource extends BaseJerseyTest {
                                 MediaType.MULTIPART_FORM_DATA_TYPE), JsonObject.class);
             }
         }
+
+        String documentId = json.getString("id");
+        Assert.assertNotNull(documentId);
+
+        // Get the document
+        json = target().path("/document/" + documentId).request()
+                .cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentEmlToken)
+                .get(JsonObject.class);
+        Assert.assertEquals("subject here", json.getString("title"));
+        Assert.assertTrue(json.getString("description").contains("content here"));
+        Assert.assertEquals("subject here", json.getString("subject"));
+        Assert.assertEquals("EML", json.getString("format"));
+        Assert.assertEquals("Email", json.getString("source"));
+        Assert.assertEquals("eng", json.getString("language"));
+        Assert.assertEquals(1519222261000L, json.getJsonNumber("create_date").longValue());
+
+        // Get all files from a document
+        json = target().path("/file/list")
+                .queryParam("id", documentId)
+                .request()
+                .cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentEmlToken)
+                .get(JsonObject.class);
+        JsonArray files = json.getJsonArray("files");
+        Assert.assertEquals(2, files.size());
+        Assert.assertEquals("14_UNHCR_nd.pdf", files.getJsonObject(0).getString("name"));
+        Assert.assertEquals(251216L, files.getJsonObject(0).getJsonNumber("size").longValue());
+        Assert.assertEquals("application/pdf", files.getJsonObject(0).getString("mimetype"));
+        Assert.assertEquals("refugee status determination.pdf", files.getJsonObject(1).getString("name"));
+        Assert.assertEquals(279276L, files.getJsonObject(1).getJsonNumber("size").longValue());
+        Assert.assertEquals("application/pdf", files.getJsonObject(1).getString("mimetype"));
     }
 }