vmario
/
docs
mirror of https://github.com/sismics/docs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#177: import document from EML file (api done)

This commit is contained in:
Benjamin Gamard 2018-02-22 10:46:32 +01:00
parent d3a40ebca8
commit b95ec019de
6 changed files with 181 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs-core/src/main/java/com/sismics/docs/core/util/DocumentUtil.java
View File

@ -13,6 +13,13 @@ import com.sismics.docs.core.model.jpa.Document;
* @author bgamard
*/
public class DocumentUtil {
/**
* Create a document and add the base ACLs.
*
* @param document Document
* @param userId User creating the document
* @return Created document
*/
public static Document createDocument(Document document, String userId) {
DocumentDao documentDao = new DocumentDao();
String documentId = documentDao.create(document, userId);

97
docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java
View File

@ -1,10 +1,19 @@
package com.sismics.docs.core.util;
import com.google.common.base.Strings;
import com.sismics.docs.core.constant.Constants;
import com.sismics.docs.core.dao.jpa.FileDao;
import com.sismics.docs.core.dao.jpa.UserDao;
import com.sismics.docs.core.event.DocumentUpdatedAsyncEvent;
import com.sismics.docs.core.event.FileCreatedAsyncEvent;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.tess4j.Tesseract;
import com.sismics.util.ImageDeskew;
import com.sismics.util.ImageUtil;
import com.sismics.util.Scalr;
import com.sismics.util.context.ThreadLocalContext;
import com.sismics.util.mime.MimeTypeUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -167,4 +176,92 @@ public class FileUtil {
Files.delete(thumbnailFile);
}
}
/**
* Create a new file.
*
* @param name File name, can be null
* @param unencryptedFile Path to the unencrypted file
* @param fileSize File size
* @param language File language, can be null if associated to no document
* @param userId User ID creating the file
* @param documentId Associated document ID or null if no document
* @return File ID
* @throws Exception e
*/
public static String createFile(String name, Path unencryptedFile, long fileSize, String language, String userId, String documentId) throws Exception {
// Validate mime type
String mimeType;
try {
mimeType = MimeTypeUtil.guessMimeType(unencryptedFile, name);
} catch (IOException e) {
throw new IOException("ErrorGuessMime", e);
}
// Validate user quota
UserDao userDao = new UserDao();
User user = userDao.getById(userId);
if (user.getStorageCurrent() + fileSize > user.getStorageQuota()) {
throw new IOException("QuotaReached");
}
// Validate global quota
String globalStorageQuotaStr = System.getenv(Constants.GLOBAL_QUOTA_ENV);
if (!Strings.isNullOrEmpty(globalStorageQuotaStr)) {
long globalStorageQuota = Long.valueOf(globalStorageQuotaStr);
long globalStorageCurrent = userDao.getGlobalStorageCurrent();
if (globalStorageCurrent + fileSize > globalStorageQuota) {
throw new IOException("QuotaReached");
}
}
// Get files of this document
FileDao fileDao = new FileDao();
int order = 0;
if (documentId != null) {
for (File file : fileDao.getByDocumentId(userId, documentId)) {
file.setOrder(order++);
}
}
// Create the file
File file = new File();
file.setOrder(order);
file.setDocumentId(documentId);
file.setName(name);
file.setMimeType(mimeType);
file.setUserId(userId);
String fileId = fileDao.create(file, userId);
// Guess the mime type a second time, for open document format (first detected as simple ZIP file)
file.setMimeType(MimeTypeUtil.guessOpenDocumentFormat(file, unencryptedFile));
// Convert to PDF if necessary (for thumbnail and text extraction)
java.nio.file.Path unencryptedPdfFile = PdfUtil.convertToPdf(file, unencryptedFile);
// Save the file
FileUtil.save(unencryptedFile, unencryptedPdfFile, file, user.getPrivateKey());
// Update the user quota
user.setStorageCurrent(user.getStorageCurrent() + fileSize);
userDao.updateQuota(user);
// Raise a new file created event and document updated event if we have a document
if (documentId != null) {
FileCreatedAsyncEvent fileCreatedAsyncEvent = new FileCreatedAsyncEvent();
fileCreatedAsyncEvent.setUserId(userId);
fileCreatedAsyncEvent.setLanguage(language);
fileCreatedAsyncEvent.setFile(file);
fileCreatedAsyncEvent.setUnencryptedFile(unencryptedFile);
fileCreatedAsyncEvent.setUnencryptedPdfFile(unencryptedPdfFile);
ThreadLocalContext.get().addAsyncEvent(fileCreatedAsyncEvent);
DocumentUpdatedAsyncEvent documentUpdatedAsyncEvent = new DocumentUpdatedAsyncEvent();
documentUpdatedAsyncEvent.setUserId(userId);
documentUpdatedAsyncEvent.setDocumentId(documentId);
ThreadLocalContext.get().addAsyncEvent(documentUpdatedAsyncEvent);
}
return fileId;
}
}

23
docs-core/src/main/java/com/sismics/util/EmailUtil.java
View File

@ -79,7 +79,7 @@ public class EmailUtil {
* @param subject Email subject
* @param paramMap Email parameters
*/
public static void sendEmail(String templateName, UserDto recipientUser, String subject, Map<String, Object> paramMap) {
private static void sendEmail(String templateName, UserDto recipientUser, String subject, Map<String, Object> paramMap) {
if (log.isInfoEnabled()) {
log.info("Sending email from template=" + templateName + " to user " + recipientUser);
}
@ -175,6 +175,15 @@ public class EmailUtil {
sendEmail(templateName, recipientUser, subject, paramMap);
}
/**
* Parse an email content to be imported.
*
* @param part Email part
* @param mailContent Mail content modified by side-effect
*
* @throws MessagingException e
* @throws IOException e
*/
public static void parseMailContent(Part part, MailContent mailContent) throws MessagingException, IOException {
Object content = part.getContent();
if (content instanceof Multipart) {
@ -261,5 +270,17 @@ public class EmailUtil {
private String name;
private Path file;
private long size;
public String getName() {
return name;
}
public Path getFile() {
return file;
}
public long getSize() {
return size;
}
}
}

22
docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java
View File

@ -17,6 +17,7 @@ import com.sismics.docs.core.model.jpa.Document;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.util.DocumentUtil;
import com.sismics.docs.core.util.FileUtil;
import com.sismics.docs.core.util.PdfUtil;
import com.sismics.docs.core.util.jpa.PaginatedList;
import com.sismics.docs.core.util.jpa.PaginatedLists;
@ -405,7 +406,7 @@ public class DocumentResource extends BaseResource {
/**
* Parse a query according to the specified syntax, eg.:
* tag:assurance tag:other before:2012 after:2011-09 shared:yes lang:fra thing
*
*
* @param search Search query
* @return DocumentCriteria
*/
@ -414,10 +415,10 @@ public class DocumentResource extends BaseResource {
if (Strings.isNullOrEmpty(search)) {
return documentCriteria;
}
TagDao tagDao = new TagDao();
UserDao userDao = new UserDao();
DateTimeParser[] parsers = {
DateTimeParser[] parsers = {
DateTimeFormat.forPattern("yyyy").getParser(),
DateTimeFormat.forPattern("yyyy-MM").getParser(),
DateTimeFormat.forPattern("yyyy-MM-dd").getParser() };
@ -425,7 +426,7 @@ public class DocumentResource extends BaseResource {
DateTimeFormatter monthFormatter = new DateTimeFormatter(null, parsers[1]);
DateTimeFormatter dayFormatter = new DateTimeFormatter(null, parsers[2]);
DateTimeFormatter formatter = new DateTimeFormatterBuilder().append( null, parsers ).toFormatter();
String[] criteriaList = search.split(" *");
List<String> query = new ArrayList<>();
List<String> fullQuery = new ArrayList<>();
@ -519,7 +520,7 @@ public class DocumentResource extends BaseResource {
break;
}
}
documentCriteria.setSearch(Joiner.on(" ").join(query));
documentCriteria.setFullSearch(Joiner.on(" ").join(fullQuery));
return documentCriteria;
@ -845,7 +846,16 @@ public class DocumentResource extends BaseResource {
documentCreatedAsyncEvent.setDocument(document);
ThreadLocalContext.get().addAsyncEvent(documentCreatedAsyncEvent);
// TODO Add files to the document
// Add files to the document
try {
for (EmailUtil.FileContent fileContent : mailContent.getFileContentList()) {
FileUtil.createFile(fileContent.getName(), fileContent.getFile(), fileContent.getSize(), "eng", principal.getId(), document.getId());
}
} catch (IOException e) {
throw new ClientException(e.getMessage(), e.getMessage(), e);
} catch (Exception e) {
throw new ServerException("FileError", "Error adding a file", e);
}
JsonObjectBuilder response = Json.createObjectBuilder()
.add("id", document.getId());

85
docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java
View File

@ -3,7 +3,6 @@ package com.sismics.docs.rest.resource;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.io.ByteStreams;
import com.sismics.docs.core.constant.Constants;
import com.sismics.docs.core.constant.PermType;
import com.sismics.docs.core.dao.jpa.AclDao;
import com.sismics.docs.core.dao.jpa.DocumentDao;
@ -27,7 +26,6 @@ import com.sismics.util.HttpUtil;
import com.sismics.util.JsonUtil;
import com.sismics.util.context.ThreadLocalContext;
import com.sismics.util.mime.MimeType;
import com.sismics.util.mime.MimeTypeUtil;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataParam;
@ -98,10 +96,6 @@ public class FileResource extends BaseResource {
// Validate input data
ValidationUtil.validateRequired(fileBodyPart, "file");
// Get the current user
UserDao userDao = new UserDao();
User user = userDao.getById(principal.getId());
// Get the document
DocumentDto documentDto = null;
if (Strings.isNullOrEmpty(documentId)) {
@ -114,7 +108,7 @@ public class FileResource extends BaseResource {
}
}
// Keep unencrypted data temporary on disk, because we will need it two times
// Keep unencrypted data temporary on disk
java.nio.file.Path unencryptedFile;
long fileSize;
try {
@ -125,78 +119,11 @@ public class FileResource extends BaseResource {
throw new ServerException("StreamError", "Error reading the input file", e);
}
// Validate mime type
String name = fileBodyPart.getContentDisposition() != null ?
fileBodyPart.getContentDisposition().getFileName() : null;
String mimeType;
try {
mimeType = MimeTypeUtil.guessMimeType(unencryptedFile, name);
} catch (IOException e) {
throw new ServerException("ErrorGuessMime", "Error guessing mime type", e);
}
// Validate user quota
if (user.getStorageCurrent() + fileSize > user.getStorageQuota()) {
throw new ClientException("QuotaReached", "Quota limit reached");
}
// Validate global quota
String globalStorageQuotaStr = System.getenv(Constants.GLOBAL_QUOTA_ENV);
if (!Strings.isNullOrEmpty(globalStorageQuotaStr)) {
long globalStorageQuota = Long.valueOf(globalStorageQuotaStr);
long globalStorageCurrent = userDao.getGlobalStorageCurrent();
if (globalStorageCurrent + fileSize > globalStorageQuota) {
throw new ClientException("QuotaReached", "Global quota limit reached");
}
}
try {
// Get files of this document
FileDao fileDao = new FileDao();
int order = 0;
if (documentId != null) {
for (File file : fileDao.getByDocumentId(principal.getId(), documentId)) {
file.setOrder(order++);
}
}
// Create the file
File file = new File();
file.setOrder(order);
file.setDocumentId(documentId);
file.setName(name);
file.setMimeType(mimeType);
file.setUserId(principal.getId());
String fileId = fileDao.create(file, principal.getId());
// Guess the mime type a second time, for open document format (first detected as simple ZIP file)
file.setMimeType(MimeTypeUtil.guessOpenDocumentFormat(file, unencryptedFile));
// Convert to PDF if necessary (for thumbnail and text extraction)
java.nio.file.Path unencryptedPdfFile = PdfUtil.convertToPdf(file, unencryptedFile);
// Save the file
FileUtil.save(unencryptedFile, unencryptedPdfFile, file, user.getPrivateKey());
// Update the user quota
user.setStorageCurrent(user.getStorageCurrent() + fileSize);
userDao.updateQuota(user);
// Raise a new file created event and document updated event if we have a document
if (documentId != null) {
FileCreatedAsyncEvent fileCreatedAsyncEvent = new FileCreatedAsyncEvent();
fileCreatedAsyncEvent.setUserId(principal.getId());
fileCreatedAsyncEvent.setLanguage(documentDto.getLanguage());
fileCreatedAsyncEvent.setFile(file);
fileCreatedAsyncEvent.setUnencryptedFile(unencryptedFile);
fileCreatedAsyncEvent.setUnencryptedPdfFile(unencryptedPdfFile);
ThreadLocalContext.get().addAsyncEvent(fileCreatedAsyncEvent);
DocumentUpdatedAsyncEvent documentUpdatedAsyncEvent = new DocumentUpdatedAsyncEvent();
documentUpdatedAsyncEvent.setUserId(principal.getId());
documentUpdatedAsyncEvent.setDocumentId(documentId);
ThreadLocalContext.get().addAsyncEvent(documentUpdatedAsyncEvent);
}
String name = fileBodyPart.getContentDisposition() != null ?
fileBodyPart.getContentDisposition().getFileName() : null;
String fileId = FileUtil.createFile(name, unencryptedFile, fileSize, documentDto == null ?
null : documentDto.getLanguage(), principal.getId(), documentId);
// Always return OK
JsonObjectBuilder response = Json.createObjectBuilder()
@ -204,6 +131,8 @@ public class FileResource extends BaseResource {
.add("id", fileId)
.add("size", fileSize);
return Response.ok().entity(response.build()).build();
} catch (IOException e) {
throw new ClientException(e.getMessage(), e.getMessage(), e);
} catch (Exception e) {
throw new ServerException("FileError", "Error adding a file", e);
}

33
docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java
View File

@ -615,10 +615,11 @@ public class TestDocumentResource extends BaseJerseyTest {
String documentEmlToken = clientUtil.login("document_eml");
// Import a document as EML
JsonObject json;
try (InputStream is = Resources.getResource("file/test_mail.eml").openStream()) {
StreamDataBodyPart streamDataBodyPart = new StreamDataBodyPart("file", is, "test_mail.eml");
try (FormDataMultiPart multiPart = new FormDataMultiPart()) {
target()
json = target()
.register(MultiPartFeature.class)
.path("/document/eml").request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentEmlToken)
@ -626,5 +627,35 @@ public class TestDocumentResource extends BaseJerseyTest {
MediaType.MULTIPART_FORM_DATA_TYPE), JsonObject.class);
}
}
String documentId = json.getString("id");
Assert.assertNotNull(documentId);
// Get the document
json = target().path("/document/" + documentId).request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentEmlToken)
.get(JsonObject.class);
Assert.assertEquals("subject here", json.getString("title"));
Assert.assertTrue(json.getString("description").contains("content here"));
Assert.assertEquals("subject here", json.getString("subject"));
Assert.assertEquals("EML", json.getString("format"));
Assert.assertEquals("Email", json.getString("source"));
Assert.assertEquals("eng", json.getString("language"));
Assert.assertEquals(1519222261000L, json.getJsonNumber("create_date").longValue());
// Get all files from a document
json = target().path("/file/list")
.queryParam("id", documentId)
.request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentEmlToken)
.get(JsonObject.class);
JsonArray files = json.getJsonArray("files");
Assert.assertEquals(2, files.size());
Assert.assertEquals("14_UNHCR_nd.pdf", files.getJsonObject(0).getString("name"));
Assert.assertEquals(251216L, files.getJsonObject(0).getJsonNumber("size").longValue());
Assert.assertEquals("application/pdf", files.getJsonObject(0).getString("mimetype"));
Assert.assertEquals("refugee status determination.pdf", files.getJsonObject(1).getString("name"));
Assert.assertEquals(279276L, files.getJsonObject(1).getJsonNumber("size").longValue());
Assert.assertEquals("application/pdf", files.getJsonObject(1).getString("mimetype"));
}
}