getFileContentList() {
+ return fileContentList;
+ }
+
+ public MailContent setSubject(String subject) {
+ this.subject = subject;
+ return this;
+ }
+
+ public Date getDate() {
+ return date;
+ }
+
+ public MailContent setDate(Date date) {
+ this.date = date;
+ return this;
+ }
+ }
+
+ /**
+ * Structure defining a file from an email to be imported.
+ */
+ public static class FileContent {
+ private String name;
+ private Path file;
+ private long size;
+ }
}
diff --git a/docs-core/src/main/java/com/sismics/util/HtmlToPlainText.java b/docs-core/src/main/java/com/sismics/util/HtmlToPlainText.java
new file mode 100644
index 00000000..62fcd212
--- /dev/null
+++ b/docs-core/src/main/java/com/sismics/util/HtmlToPlainText.java
@@ -0,0 +1,130 @@
+package com.sismics.util;
+
+import org.jsoup.Jsoup;
+import org.jsoup.helper.StringUtil;
+import org.jsoup.helper.Validate;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.nodes.Node;
+import org.jsoup.nodes.TextNode;
+import org.jsoup.select.Elements;
+import org.jsoup.select.NodeTraversor;
+import org.jsoup.select.NodeVisitor;
+
+import java.io.IOException;
+
+/**
+ * HTML to plain-text. This example program demonstrates the use of jsoup to convert HTML input to lightly-formatted
+ * plain-text. That is divergent from the general goal of jsoup's .text() methods, which is to get clean data from a
+ * scrape.
+ *
+ * Note that this is a fairly simplistic formatter -- for real world use you'll want to embrace and extend.
+ *
+ *
+ * To invoke from the command line, assuming you've downloaded the jsoup jar to your current directory:
+ * java -cp jsoup.jar org.jsoup.examples.HtmlToPlainText url [selector]
+ * where url is the URL to fetch, and selector is an optional CSS selector.
+ *
+ * @author Jonathan Hedley, jonathan@hedley.net
+ */
+public class HtmlToPlainText {
+ private static final String userAgent = "Mozilla/5.0 (jsoup)";
+ private static final int timeout = 5 * 1000;
+
+ public static void main(String... args) throws IOException {
+ Validate.isTrue(args.length == 1 || args.length == 2, "usage: java -cp jsoup.jar org.jsoup.examples.HtmlToPlainText url [selector]");
+ final String url = args[0];
+ final String selector = args.length == 2 ? args[1] : null;
+
+ // fetch the specified URL and parse to a HTML DOM
+ Document doc = Jsoup.connect(url).userAgent(userAgent).timeout(timeout).get();
+
+ HtmlToPlainText formatter = new HtmlToPlainText();
+
+ if (selector != null) {
+ Elements elements = doc.select(selector); // get each element that matches the CSS selector
+ for (Element element : elements) {
+ String plainText = formatter.getPlainText(element); // format that element to plain text
+ System.out.println(plainText);
+ }
+ } else { // format the whole doc
+ String plainText = formatter.getPlainText(doc);
+ System.out.println(plainText);
+ }
+ }
+
+ /**
+ * Format an Element to plain-text
+ * @param element the root element to format
+ * @return formatted text
+ */
+ public String getPlainText(Element element) {
+ FormattingVisitor formatter = new FormattingVisitor();
+ NodeTraversor.traverse(formatter, element); // walk the DOM, and call .head() and .tail() for each node
+
+ return formatter.toString();
+ }
+
+ // the formatting rules, implemented in a breadth-first DOM traverse
+ private class FormattingVisitor implements NodeVisitor {
+ private static final int maxWidth = 80;
+ private int width = 0;
+ private StringBuilder accum = new StringBuilder(); // holds the accumulated text
+
+ // hit when the node is first seen
+ public void head(Node node, int depth) {
+ String name = node.nodeName();
+ if (node instanceof TextNode)
+ append(((TextNode) node).text()); // TextNodes carry all user-readable text in the DOM.
+ else if (name.equals("li"))
+ append("\n * ");
+ else if (name.equals("dt"))
+ append(" ");
+ else if (StringUtil.in(name, "p", "h1", "h2", "h3", "h4", "h5", "tr"))
+ append("\n");
+ }
+
+ // hit when all of the node's children (if any) have been visited
+ public void tail(Node node, int depth) {
+ String name = node.nodeName();
+ if (StringUtil.in(name, "br", "dd", "dt", "p", "h1", "h2", "h3", "h4", "h5"))
+ append("\n");
+ else if (name.equals("a"))
+ append(String.format(" <%s>", node.absUrl("href")));
+ }
+
+ // appends text to the string builder with a simple word wrap method
+ private void append(String text) {
+ if (text.startsWith("\n"))
+ width = 0; // reset counter if starts with a newline. only from formats above, not in natural text
+ if (text.equals(" ") &&
+ (accum.length() == 0 || StringUtil.in(accum.substring(accum.length() - 1), " ", "\n")))
+ return; // don't accumulate long runs of empty spaces
+
+ if (text.length() + width > maxWidth) { // won't fit, needs to wrap
+ String words[] = text.split("\\s+");
+ for (int i = 0; i < words.length; i++) {
+ String word = words[i];
+ boolean last = i == words.length - 1;
+ if (!last) // insert a space if not the last word
+ word = word + " ";
+ if (word.length() + width > maxWidth) { // wrap and reset counter
+ accum.append("\n").append(word);
+ width = word.length();
+ } else {
+ accum.append(word);
+ width += word.length();
+ }
+ }
+ } else { // fits as is, without need to wrap text
+ accum.append(text);
+ width += text.length();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return accum.toString();
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs-core/src/test/java/com/sismics/util/TestDateUtil.java b/docs-core/src/test/java/com/sismics/util/TestDateUtil.java
deleted file mode 100644
index 4c587881..00000000
--- a/docs-core/src/test/java/com/sismics/util/TestDateUtil.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package com.sismics.util;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test of the date utilities.
- *
- * @author jtremeaux
- */
-public class TestDateUtil {
-
- @Test
- public void guessTimezoneCodeTest() throws Exception {
- Assert.assertEquals("Thu, 04 APR 2013 20:37:27 +10", DateUtil.guessTimezoneOffset("Thu, 04 APR 2013 20:37:27 AEST"));
- }
-}
diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java
index 480b2786..2effd82f 100644
--- a/docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java
+++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java
@@ -13,10 +13,10 @@ import com.sismics.docs.core.event.DocumentCreatedAsyncEvent;
import com.sismics.docs.core.event.DocumentDeletedAsyncEvent;
import com.sismics.docs.core.event.DocumentUpdatedAsyncEvent;
import com.sismics.docs.core.event.FileDeletedAsyncEvent;
-import com.sismics.docs.core.model.jpa.Acl;
import com.sismics.docs.core.model.jpa.Document;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
+import com.sismics.docs.core.util.DocumentUtil;
import com.sismics.docs.core.util.PdfUtil;
import com.sismics.docs.core.util.jpa.PaginatedList;
import com.sismics.docs.core.util.jpa.PaginatedLists;
@@ -26,9 +26,12 @@ import com.sismics.rest.exception.ForbiddenClientException;
import com.sismics.rest.exception.ServerException;
import com.sismics.rest.util.AclUtil;
import com.sismics.rest.util.ValidationUtil;
+import com.sismics.util.EmailUtil;
import com.sismics.util.JsonUtil;
import com.sismics.util.context.ThreadLocalContext;
import com.sismics.util.mime.MimeType;
+import org.glassfish.jersey.media.multipart.FormDataBodyPart;
+import org.glassfish.jersey.media.multipart.FormDataParam;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
@@ -38,11 +41,18 @@ import org.joda.time.format.DateTimeParser;
import javax.json.Json;
import javax.json.JsonArrayBuilder;
import javax.json.JsonObjectBuilder;
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import javax.mail.Session;
+import javax.mail.internet.MimeMessage;
import javax.ws.rs.*;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
import java.text.MessageFormat;
import java.util.*;
@@ -595,7 +605,6 @@ public class DocumentResource extends BaseResource {
}
// Create the document
- DocumentDao documentDao = new DocumentDao();
Document document = new Document();
document.setUserId(principal.getId());
document.setTitle(title);
@@ -614,31 +623,16 @@ public class DocumentResource extends BaseResource {
} else {
document.setCreateDate(createDate);
}
- String documentId = documentDao.create(document, principal.getId());
-
- // Create read ACL
- AclDao aclDao = new AclDao();
- Acl acl = new Acl();
- acl.setPerm(PermType.READ);
- acl.setType(AclType.USER);
- acl.setSourceId(documentId);
- acl.setTargetId(principal.getId());
- aclDao.create(acl, principal.getId());
-
- // Create write ACL
- acl = new Acl();
- acl.setPerm(PermType.WRITE);
- acl.setType(AclType.USER);
- acl.setSourceId(documentId);
- acl.setTargetId(principal.getId());
- aclDao.create(acl, principal.getId());
-
+
+ // Save the document, create the base ACLs
+ document = DocumentUtil.createDocument(document, principal.getId());
+
// Update tags
- updateTagList(documentId, tagList);
-
+ updateTagList(document.getId(), tagList);
+
// Update relations
- updateRelationList(documentId, relationList);
-
+ updateRelationList(document.getId(), relationList);
+
// Raise a document created event
DocumentCreatedAsyncEvent documentCreatedAsyncEvent = new DocumentCreatedAsyncEvent();
documentCreatedAsyncEvent.setUserId(principal.getId());
@@ -646,7 +640,7 @@ public class DocumentResource extends BaseResource {
ThreadLocalContext.get().addAsyncEvent(documentCreatedAsyncEvent);
JsonObjectBuilder response = Json.createObjectBuilder()
- .add("id", documentId);
+ .add("id", document.getId());
return Response.ok().entity(response.build()).build();
}
@@ -772,52 +766,92 @@ public class DocumentResource extends BaseResource {
}
/**
- * Update tags list on a document.
- *
- * @param documentId Document ID
- * @param tagList Tag ID list
+ * Import a new document from an EML file.
+ *
+ * @api {put} /document/eml Import a new document from an EML file
+ * @apiName PutDocumentEml
+ * @apiGroup Document
+ * @apiParam {String} file File data
+ * @apiError (client) ForbiddenError Access denied
+ * @apiError (client) ValidationError Validation error
+ * @apiError (server) StreamError Error reading the input file
+ * @apiError (server) ErrorGuessMime Error guessing mime type
+ * @apiError (client) QuotaReached Quota limit reached
+ * @apiError (server) FileError Error adding a file
+ * @apiPermission user
+ * @apiVersion 1.5.0
+ *
+ * @param fileBodyPart File to import
+ * @return Response
*/
- private void updateTagList(String documentId, List tagList) {
- if (tagList != null) {
- TagDao tagDao = new TagDao();
- Set tagSet = new HashSet<>();
- Set tagIdSet = new HashSet<>();
- List tagDtoList = tagDao.findByCriteria(new TagCriteria().setTargetIdList(getTargetIdList(null)), null);
- for (TagDto tagDto : tagDtoList) {
- tagIdSet.add(tagDto.getId());
- }
- for (String tagId : tagList) {
- if (!tagIdSet.contains(tagId)) {
- throw new ClientException("TagNotFound", MessageFormat.format("Tag not found: {0}", tagId));
- }
- tagSet.add(tagId);
- }
- tagDao.updateTagList(documentId, tagSet);
+ @PUT
+ @Path("eml")
+ @Consumes("multipart/form-data")
+ public Response importEml(@FormDataParam("file") FormDataBodyPart fileBodyPart) {
+ if (!authenticate()) {
+ throw new ForbiddenClientException();
}
- }
-
- /**
- * Update relations list on a document.
- *
- * @param documentId Document ID
- * @param relationList Relation ID list
- */
- private void updateRelationList(String documentId, List relationList) {
- if (relationList != null) {
- DocumentDao documentDao = new DocumentDao();
- RelationDao relationDao = new RelationDao();
- Set documentIdSet = new HashSet<>();
- for (String targetDocId : relationList) {
- // ACL are not checked, because the editing user is not forced to view the target document
- Document document = documentDao.getById(targetDocId);
- if (document != null && !documentId.equals(targetDocId)) {
- documentIdSet.add(targetDocId);
- }
- }
- relationDao.updateRelationList(documentId, documentIdSet);
+
+ // Validate input data
+ ValidationUtil.validateRequired(fileBodyPart, "file");
+
+ // Save the file to a temporary file
+ java.nio.file.Path unencryptedFile;
+ try {
+ unencryptedFile = ThreadLocalContext.get().createTemporaryFile();
+ Files.copy(fileBodyPart.getValueAs(InputStream.class), unencryptedFile, StandardCopyOption.REPLACE_EXISTING);
+ } catch (IOException e) {
+ throw new ServerException("StreamError", "Error reading the input file", e);
}
+
+ // Read the EML file
+ Properties props = new Properties();
+ Session mailSession = Session.getDefaultInstance(props, null);
+ EmailUtil.MailContent mailContent = new EmailUtil.MailContent();
+ try (InputStream inputStream = Files.newInputStream(unencryptedFile)) {
+ Message message = new MimeMessage(mailSession, inputStream);
+ mailContent.setSubject(message.getSubject());
+ mailContent.setDate(message.getSentDate());
+ EmailUtil.parseMailContent(message, mailContent);
+ } catch (IOException | MessagingException e) {
+ throw new ServerException("StreamError", "Error reading the temporary file", e);
+ }
+
+ // Create the document
+ Document document = new Document();
+ document.setUserId(principal.getId());
+ if (mailContent.getSubject() == null) {
+ document.setTitle("Imported email from EML file");
+ } else {
+ document.setTitle(mailContent.getSubject());
+ }
+ document.setDescription(mailContent.getMessage());
+ document.setSubject(mailContent.getSubject());
+ document.setFormat("EML");
+ document.setSource("Email");
+ document.setLanguage("eng");
+ if (mailContent.getDate() == null) {
+ document.setCreateDate(new Date());
+ } else {
+ document.setCreateDate(mailContent.getDate());
+ }
+
+ // Save the document, create the base ACLs
+ document = DocumentUtil.createDocument(document, principal.getId());
+
+ // Raise a document created event
+ DocumentCreatedAsyncEvent documentCreatedAsyncEvent = new DocumentCreatedAsyncEvent();
+ documentCreatedAsyncEvent.setUserId(principal.getId());
+ documentCreatedAsyncEvent.setDocument(document);
+ ThreadLocalContext.get().addAsyncEvent(documentCreatedAsyncEvent);
+
+ // TODO Add files to the document
+
+ JsonObjectBuilder response = Json.createObjectBuilder()
+ .add("id", document.getId());
+ return Response.ok().entity(response.build()).build();
}
-
+
/**
* Deletes a document.
*
@@ -873,4 +907,51 @@ public class DocumentResource extends BaseResource {
.add("status", "ok");
return Response.ok().entity(response.build()).build();
}
+
+ /**
+ * Update tags list on a document.
+ *
+ * @param documentId Document ID
+ * @param tagList Tag ID list
+ */
+ private void updateTagList(String documentId, List tagList) {
+ if (tagList != null) {
+ TagDao tagDao = new TagDao();
+ Set tagSet = new HashSet<>();
+ Set tagIdSet = new HashSet<>();
+ List tagDtoList = tagDao.findByCriteria(new TagCriteria().setTargetIdList(getTargetIdList(null)), null);
+ for (TagDto tagDto : tagDtoList) {
+ tagIdSet.add(tagDto.getId());
+ }
+ for (String tagId : tagList) {
+ if (!tagIdSet.contains(tagId)) {
+ throw new ClientException("TagNotFound", MessageFormat.format("Tag not found: {0}", tagId));
+ }
+ tagSet.add(tagId);
+ }
+ tagDao.updateTagList(documentId, tagSet);
+ }
+ }
+
+ /**
+ * Update relations list on a document.
+ *
+ * @param documentId Document ID
+ * @param relationList Relation ID list
+ */
+ private void updateRelationList(String documentId, List relationList) {
+ if (relationList != null) {
+ DocumentDao documentDao = new DocumentDao();
+ RelationDao relationDao = new RelationDao();
+ Set documentIdSet = new HashSet<>();
+ for (String targetDocId : relationList) {
+ // ACL are not checked, because the editing user is not forced to view the target document
+ Document document = documentDao.getById(targetDocId);
+ if (document != null && !documentId.equals(targetDocId)) {
+ documentIdSet.add(targetDocId);
+ }
+ }
+ relationDao.updateRelationList(documentId, documentIdSet);
+ }
+ }
}
diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java
index 9ea3b51e..9c53154f 100644
--- a/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java
+++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java
@@ -77,7 +77,6 @@ public class FileResource extends BaseResource {
* @apiError (client) NotFound Document not found
* @apiError (server) StreamError Error reading the input file
* @apiError (server) ErrorGuessMime Error guessing mime type
- * @apiError (client) InvalidFileType File type not recognized
* @apiError (client) QuotaReached Quota limit reached
* @apiError (server) FileError Error adding a file
* @apiPermission user
diff --git a/docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java b/docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java
index 3ece751a..79785032 100644
--- a/docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java
+++ b/docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java
@@ -602,4 +602,29 @@ public class TestDocumentResource extends BaseJerseyTest {
Assert.assertTrue(fileBytes.length > 0); // Images rendered from PDF differ in size from OS to OS due to font issues
Assert.assertEquals(MimeType.IMAGE_JPEG, MimeTypeUtil.guessMimeType(fileBytes, null));
}
+
+ /**
+ * Test EML import.
+ *
+ * @throws Exception e
+ */
+ @Test
+ public void testEmlImport() throws Exception {
+ // Login document_eml
+ clientUtil.createUser("document_eml");
+ String documentEmlToken = clientUtil.login("document_eml");
+
+ // Import a document as EML
+ try (InputStream is = Resources.getResource("file/test_mail.eml").openStream()) {
+ StreamDataBodyPart streamDataBodyPart = new StreamDataBodyPart("file", is, "test_mail.eml");
+ try (FormDataMultiPart multiPart = new FormDataMultiPart()) {
+ target()
+ .register(MultiPartFeature.class)
+ .path("/document/eml").request()
+ .cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentEmlToken)
+ .put(Entity.entity(multiPart.bodyPart(streamDataBodyPart),
+ MediaType.MULTIPART_FORM_DATA_TYPE), JsonObject.class);
+ }
+ }
+ }
}
\ No newline at end of file
diff --git a/docs-web/src/test/resources/file/test_mail.eml b/docs-web/src/test/resources/file/test_mail.eml
new file mode 100644
index 00000000..4bcde4c5
--- /dev/null
+++ b/docs-web/src/test/resources/file/test_mail.eml
@@ -0,0 +1,9346 @@
+MIME-Version: 1.0
+Received: by 10.176.77.84 with HTTP; Wed, 21 Feb 2018 06:11:01 -0800 (PST)
+Date: Wed, 21 Feb 2018 15:11:01 +0100
+Delivered-To: benjamin.gam@gmail.com
+Message-ID:
+Subject: subject here
+From: Benjamin
+To: Benjamin
+Content-Type: multipart/mixed; boundary="f403045ee78c42bb580565b97f70"
+
+--f403045ee78c42bb580565b97f70
+Content-Type: multipart/alternative; boundary="f403045ee78c42bb500565b97f6e"
+
+--f403045ee78c42bb500565b97f6e
+Content-Type: text/plain; charset="UTF-8"
+
+content here
+even *html* *content*
+
+--f403045ee78c42bb500565b97f6e
+Content-Type: text/html; charset="UTF-8"
+
+content here
even html content