From b95ec019de2bc046cb9d43ca1c4e3e4db4476507 Mon Sep 17 00:00:00 2001 From: Benjamin Gamard Date: Thu, 22 Feb 2018 10:46:32 +0100 Subject: [PATCH] #177: import document from EML file (api done) --- .../sismics/docs/core/util/DocumentUtil.java | 7 ++ .../com/sismics/docs/core/util/FileUtil.java | 97 +++++++++++++++++++ .../main/java/com/sismics/util/EmailUtil.java | 23 ++++- .../docs/rest/resource/DocumentResource.java | 22 +++-- .../docs/rest/resource/FileResource.java | 85 ++-------------- .../docs/rest/TestDocumentResource.java | 33 ++++++- 6 files changed, 181 insertions(+), 86 deletions(-) diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/DocumentUtil.java b/docs-core/src/main/java/com/sismics/docs/core/util/DocumentUtil.java index 3814d026..c2464333 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/util/DocumentUtil.java +++ b/docs-core/src/main/java/com/sismics/docs/core/util/DocumentUtil.java @@ -13,6 +13,13 @@ import com.sismics.docs.core.model.jpa.Document; * @author bgamard */ public class DocumentUtil { + /** + * Create a document and add the base ACLs. + * + * @param document Document + * @param userId User creating the document + * @return Created document + */ public static Document createDocument(Document document, String userId) { DocumentDao documentDao = new DocumentDao(); String documentId = documentDao.create(document, userId); diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java index 3d67a6a7..df7b799b 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java +++ b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java @@ -1,10 +1,19 @@ package com.sismics.docs.core.util; +import com.google.common.base.Strings; +import com.sismics.docs.core.constant.Constants; +import com.sismics.docs.core.dao.jpa.FileDao; +import com.sismics.docs.core.dao.jpa.UserDao; +import com.sismics.docs.core.event.DocumentUpdatedAsyncEvent; +import com.sismics.docs.core.event.FileCreatedAsyncEvent; import com.sismics.docs.core.model.jpa.File; +import com.sismics.docs.core.model.jpa.User; import com.sismics.tess4j.Tesseract; import com.sismics.util.ImageDeskew; import com.sismics.util.ImageUtil; import com.sismics.util.Scalr; +import com.sismics.util.context.ThreadLocalContext; +import com.sismics.util.mime.MimeTypeUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -167,4 +176,92 @@ public class FileUtil { Files.delete(thumbnailFile); } } + + /** + * Create a new file. + * + * @param name File name, can be null + * @param unencryptedFile Path to the unencrypted file + * @param fileSize File size + * @param language File language, can be null if associated to no document + * @param userId User ID creating the file + * @param documentId Associated document ID or null if no document + * @return File ID + * @throws Exception e + */ + public static String createFile(String name, Path unencryptedFile, long fileSize, String language, String userId, String documentId) throws Exception { + // Validate mime type + String mimeType; + try { + mimeType = MimeTypeUtil.guessMimeType(unencryptedFile, name); + } catch (IOException e) { + throw new IOException("ErrorGuessMime", e); + } + + // Validate user quota + UserDao userDao = new UserDao(); + User user = userDao.getById(userId); + if (user.getStorageCurrent() + fileSize > user.getStorageQuota()) { + throw new IOException("QuotaReached"); + } + + // Validate global quota + String globalStorageQuotaStr = System.getenv(Constants.GLOBAL_QUOTA_ENV); + if (!Strings.isNullOrEmpty(globalStorageQuotaStr)) { + long globalStorageQuota = Long.valueOf(globalStorageQuotaStr); + long globalStorageCurrent = userDao.getGlobalStorageCurrent(); + if (globalStorageCurrent + fileSize > globalStorageQuota) { + throw new IOException("QuotaReached"); + } + } + + // Get files of this document + FileDao fileDao = new FileDao(); + int order = 0; + if (documentId != null) { + for (File file : fileDao.getByDocumentId(userId, documentId)) { + file.setOrder(order++); + } + } + + // Create the file + File file = new File(); + file.setOrder(order); + file.setDocumentId(documentId); + file.setName(name); + file.setMimeType(mimeType); + file.setUserId(userId); + String fileId = fileDao.create(file, userId); + + // Guess the mime type a second time, for open document format (first detected as simple ZIP file) + file.setMimeType(MimeTypeUtil.guessOpenDocumentFormat(file, unencryptedFile)); + + // Convert to PDF if necessary (for thumbnail and text extraction) + java.nio.file.Path unencryptedPdfFile = PdfUtil.convertToPdf(file, unencryptedFile); + + // Save the file + FileUtil.save(unencryptedFile, unencryptedPdfFile, file, user.getPrivateKey()); + + // Update the user quota + user.setStorageCurrent(user.getStorageCurrent() + fileSize); + userDao.updateQuota(user); + + // Raise a new file created event and document updated event if we have a document + if (documentId != null) { + FileCreatedAsyncEvent fileCreatedAsyncEvent = new FileCreatedAsyncEvent(); + fileCreatedAsyncEvent.setUserId(userId); + fileCreatedAsyncEvent.setLanguage(language); + fileCreatedAsyncEvent.setFile(file); + fileCreatedAsyncEvent.setUnencryptedFile(unencryptedFile); + fileCreatedAsyncEvent.setUnencryptedPdfFile(unencryptedPdfFile); + ThreadLocalContext.get().addAsyncEvent(fileCreatedAsyncEvent); + + DocumentUpdatedAsyncEvent documentUpdatedAsyncEvent = new DocumentUpdatedAsyncEvent(); + documentUpdatedAsyncEvent.setUserId(userId); + documentUpdatedAsyncEvent.setDocumentId(documentId); + ThreadLocalContext.get().addAsyncEvent(documentUpdatedAsyncEvent); + } + + return fileId; + } } diff --git a/docs-core/src/main/java/com/sismics/util/EmailUtil.java b/docs-core/src/main/java/com/sismics/util/EmailUtil.java index e93f5813..58988036 100644 --- a/docs-core/src/main/java/com/sismics/util/EmailUtil.java +++ b/docs-core/src/main/java/com/sismics/util/EmailUtil.java @@ -79,7 +79,7 @@ public class EmailUtil { * @param subject Email subject * @param paramMap Email parameters */ - public static void sendEmail(String templateName, UserDto recipientUser, String subject, Map paramMap) { + private static void sendEmail(String templateName, UserDto recipientUser, String subject, Map paramMap) { if (log.isInfoEnabled()) { log.info("Sending email from template=" + templateName + " to user " + recipientUser); } @@ -175,6 +175,15 @@ public class EmailUtil { sendEmail(templateName, recipientUser, subject, paramMap); } + /** + * Parse an email content to be imported. + * + * @param part Email part + * @param mailContent Mail content modified by side-effect + * + * @throws MessagingException e + * @throws IOException e + */ public static void parseMailContent(Part part, MailContent mailContent) throws MessagingException, IOException { Object content = part.getContent(); if (content instanceof Multipart) { @@ -261,5 +270,17 @@ public class EmailUtil { private String name; private Path file; private long size; + + public String getName() { + return name; + } + + public Path getFile() { + return file; + } + + public long getSize() { + return size; + } } } diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java index 2effd82f..e7f08aa6 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java @@ -17,6 +17,7 @@ import com.sismics.docs.core.model.jpa.Document; import com.sismics.docs.core.model.jpa.File; import com.sismics.docs.core.model.jpa.User; import com.sismics.docs.core.util.DocumentUtil; +import com.sismics.docs.core.util.FileUtil; import com.sismics.docs.core.util.PdfUtil; import com.sismics.docs.core.util.jpa.PaginatedList; import com.sismics.docs.core.util.jpa.PaginatedLists; @@ -405,7 +406,7 @@ public class DocumentResource extends BaseResource { /** * Parse a query according to the specified syntax, eg.: * tag:assurance tag:other before:2012 after:2011-09 shared:yes lang:fra thing - * + * * @param search Search query * @return DocumentCriteria */ @@ -414,10 +415,10 @@ public class DocumentResource extends BaseResource { if (Strings.isNullOrEmpty(search)) { return documentCriteria; } - + TagDao tagDao = new TagDao(); UserDao userDao = new UserDao(); - DateTimeParser[] parsers = { + DateTimeParser[] parsers = { DateTimeFormat.forPattern("yyyy").getParser(), DateTimeFormat.forPattern("yyyy-MM").getParser(), DateTimeFormat.forPattern("yyyy-MM-dd").getParser() }; @@ -425,7 +426,7 @@ public class DocumentResource extends BaseResource { DateTimeFormatter monthFormatter = new DateTimeFormatter(null, parsers[1]); DateTimeFormatter dayFormatter = new DateTimeFormatter(null, parsers[2]); DateTimeFormatter formatter = new DateTimeFormatterBuilder().append( null, parsers ).toFormatter(); - + String[] criteriaList = search.split(" *"); List query = new ArrayList<>(); List fullQuery = new ArrayList<>(); @@ -519,7 +520,7 @@ public class DocumentResource extends BaseResource { break; } } - + documentCriteria.setSearch(Joiner.on(" ").join(query)); documentCriteria.setFullSearch(Joiner.on(" ").join(fullQuery)); return documentCriteria; @@ -845,7 +846,16 @@ public class DocumentResource extends BaseResource { documentCreatedAsyncEvent.setDocument(document); ThreadLocalContext.get().addAsyncEvent(documentCreatedAsyncEvent); - // TODO Add files to the document + // Add files to the document + try { + for (EmailUtil.FileContent fileContent : mailContent.getFileContentList()) { + FileUtil.createFile(fileContent.getName(), fileContent.getFile(), fileContent.getSize(), "eng", principal.getId(), document.getId()); + } + } catch (IOException e) { + throw new ClientException(e.getMessage(), e.getMessage(), e); + } catch (Exception e) { + throw new ServerException("FileError", "Error adding a file", e); + } JsonObjectBuilder response = Json.createObjectBuilder() .add("id", document.getId()); diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java index 9c53154f..e283e584 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java @@ -3,7 +3,6 @@ package com.sismics.docs.rest.resource; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.io.ByteStreams; -import com.sismics.docs.core.constant.Constants; import com.sismics.docs.core.constant.PermType; import com.sismics.docs.core.dao.jpa.AclDao; import com.sismics.docs.core.dao.jpa.DocumentDao; @@ -27,7 +26,6 @@ import com.sismics.util.HttpUtil; import com.sismics.util.JsonUtil; import com.sismics.util.context.ThreadLocalContext; import com.sismics.util.mime.MimeType; -import com.sismics.util.mime.MimeTypeUtil; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataParam; @@ -98,10 +96,6 @@ public class FileResource extends BaseResource { // Validate input data ValidationUtil.validateRequired(fileBodyPart, "file"); - // Get the current user - UserDao userDao = new UserDao(); - User user = userDao.getById(principal.getId()); - // Get the document DocumentDto documentDto = null; if (Strings.isNullOrEmpty(documentId)) { @@ -114,7 +108,7 @@ public class FileResource extends BaseResource { } } - // Keep unencrypted data temporary on disk, because we will need it two times + // Keep unencrypted data temporary on disk java.nio.file.Path unencryptedFile; long fileSize; try { @@ -125,78 +119,11 @@ public class FileResource extends BaseResource { throw new ServerException("StreamError", "Error reading the input file", e); } - // Validate mime type - String name = fileBodyPart.getContentDisposition() != null ? - fileBodyPart.getContentDisposition().getFileName() : null; - String mimeType; try { - mimeType = MimeTypeUtil.guessMimeType(unencryptedFile, name); - } catch (IOException e) { - throw new ServerException("ErrorGuessMime", "Error guessing mime type", e); - } - - // Validate user quota - if (user.getStorageCurrent() + fileSize > user.getStorageQuota()) { - throw new ClientException("QuotaReached", "Quota limit reached"); - } - - // Validate global quota - String globalStorageQuotaStr = System.getenv(Constants.GLOBAL_QUOTA_ENV); - if (!Strings.isNullOrEmpty(globalStorageQuotaStr)) { - long globalStorageQuota = Long.valueOf(globalStorageQuotaStr); - long globalStorageCurrent = userDao.getGlobalStorageCurrent(); - if (globalStorageCurrent + fileSize > globalStorageQuota) { - throw new ClientException("QuotaReached", "Global quota limit reached"); - } - } - - try { - // Get files of this document - FileDao fileDao = new FileDao(); - int order = 0; - if (documentId != null) { - for (File file : fileDao.getByDocumentId(principal.getId(), documentId)) { - file.setOrder(order++); - } - } - - // Create the file - File file = new File(); - file.setOrder(order); - file.setDocumentId(documentId); - file.setName(name); - file.setMimeType(mimeType); - file.setUserId(principal.getId()); - String fileId = fileDao.create(file, principal.getId()); - - // Guess the mime type a second time, for open document format (first detected as simple ZIP file) - file.setMimeType(MimeTypeUtil.guessOpenDocumentFormat(file, unencryptedFile)); - - // Convert to PDF if necessary (for thumbnail and text extraction) - java.nio.file.Path unencryptedPdfFile = PdfUtil.convertToPdf(file, unencryptedFile); - - // Save the file - FileUtil.save(unencryptedFile, unencryptedPdfFile, file, user.getPrivateKey()); - - // Update the user quota - user.setStorageCurrent(user.getStorageCurrent() + fileSize); - userDao.updateQuota(user); - - // Raise a new file created event and document updated event if we have a document - if (documentId != null) { - FileCreatedAsyncEvent fileCreatedAsyncEvent = new FileCreatedAsyncEvent(); - fileCreatedAsyncEvent.setUserId(principal.getId()); - fileCreatedAsyncEvent.setLanguage(documentDto.getLanguage()); - fileCreatedAsyncEvent.setFile(file); - fileCreatedAsyncEvent.setUnencryptedFile(unencryptedFile); - fileCreatedAsyncEvent.setUnencryptedPdfFile(unencryptedPdfFile); - ThreadLocalContext.get().addAsyncEvent(fileCreatedAsyncEvent); - - DocumentUpdatedAsyncEvent documentUpdatedAsyncEvent = new DocumentUpdatedAsyncEvent(); - documentUpdatedAsyncEvent.setUserId(principal.getId()); - documentUpdatedAsyncEvent.setDocumentId(documentId); - ThreadLocalContext.get().addAsyncEvent(documentUpdatedAsyncEvent); - } + String name = fileBodyPart.getContentDisposition() != null ? + fileBodyPart.getContentDisposition().getFileName() : null; + String fileId = FileUtil.createFile(name, unencryptedFile, fileSize, documentDto == null ? + null : documentDto.getLanguage(), principal.getId(), documentId); // Always return OK JsonObjectBuilder response = Json.createObjectBuilder() @@ -204,6 +131,8 @@ public class FileResource extends BaseResource { .add("id", fileId) .add("size", fileSize); return Response.ok().entity(response.build()).build(); + } catch (IOException e) { + throw new ClientException(e.getMessage(), e.getMessage(), e); } catch (Exception e) { throw new ServerException("FileError", "Error adding a file", e); } diff --git a/docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java b/docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java index 79785032..a38cf032 100644 --- a/docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java +++ b/docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java @@ -615,10 +615,11 @@ public class TestDocumentResource extends BaseJerseyTest { String documentEmlToken = clientUtil.login("document_eml"); // Import a document as EML + JsonObject json; try (InputStream is = Resources.getResource("file/test_mail.eml").openStream()) { StreamDataBodyPart streamDataBodyPart = new StreamDataBodyPart("file", is, "test_mail.eml"); try (FormDataMultiPart multiPart = new FormDataMultiPart()) { - target() + json = target() .register(MultiPartFeature.class) .path("/document/eml").request() .cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentEmlToken) @@ -626,5 +627,35 @@ public class TestDocumentResource extends BaseJerseyTest { MediaType.MULTIPART_FORM_DATA_TYPE), JsonObject.class); } } + + String documentId = json.getString("id"); + Assert.assertNotNull(documentId); + + // Get the document + json = target().path("/document/" + documentId).request() + .cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentEmlToken) + .get(JsonObject.class); + Assert.assertEquals("subject here", json.getString("title")); + Assert.assertTrue(json.getString("description").contains("content here")); + Assert.assertEquals("subject here", json.getString("subject")); + Assert.assertEquals("EML", json.getString("format")); + Assert.assertEquals("Email", json.getString("source")); + Assert.assertEquals("eng", json.getString("language")); + Assert.assertEquals(1519222261000L, json.getJsonNumber("create_date").longValue()); + + // Get all files from a document + json = target().path("/file/list") + .queryParam("id", documentId) + .request() + .cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentEmlToken) + .get(JsonObject.class); + JsonArray files = json.getJsonArray("files"); + Assert.assertEquals(2, files.size()); + Assert.assertEquals("14_UNHCR_nd.pdf", files.getJsonObject(0).getString("name")); + Assert.assertEquals(251216L, files.getJsonObject(0).getJsonNumber("size").longValue()); + Assert.assertEquals("application/pdf", files.getJsonObject(0).getString("mimetype")); + Assert.assertEquals("refugee status determination.pdf", files.getJsonObject(1).getString("name")); + Assert.assertEquals(279276L, files.getJsonObject(1).getJsonNumber("size").longValue()); + Assert.assertEquals("application/pdf", files.getJsonObject(1).getString("mimetype")); } } \ No newline at end of file