diff --git a/docs-core/src/main/java/com/sismics/docs/core/event/ExtractFileAsyncEvent.java b/docs-core/src/main/java/com/sismics/docs/core/event/ExtractFileAsyncEvent.java deleted file mode 100644 index 28dd6faa..00000000 --- a/docs-core/src/main/java/com/sismics/docs/core/event/ExtractFileAsyncEvent.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.sismics.docs.core.event; - -import com.google.common.base.Objects; - -/** - * Extract file content event. - * - * @author bgamard - */ -public class ExtractFileAsyncEvent { - @Override - public String toString() { - return Objects.toStringHelper(this) - .toString(); - } -} diff --git a/docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java b/docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java index cdbc2976..2823781d 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java +++ b/docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java @@ -1,5 +1,7 @@ package com.sismics.docs.core.event; +import java.io.InputStream; + import com.google.common.base.Objects; import com.sismics.docs.core.model.jpa.Document; import com.sismics.docs.core.model.jpa.File; @@ -20,6 +22,11 @@ public class FileCreatedAsyncEvent { */ private Document document; + /** + * Unencrypted input stream containing the file. + */ + private InputStream inputStream; + /** * Getter of file. * @@ -55,6 +62,24 @@ public class FileCreatedAsyncEvent { public void setDocument(Document document) { this.document = document; } + + /** + * Getter of inputStream. + * + * @return the inputStream + */ + public InputStream getInputStream() { + return inputStream; + } + + /** + * Setter de inputStream. + * + * @param inputStream inputStream + */ + public void setInputStream(InputStream inputStream) { + this.inputStream = inputStream; + } @Override public String toString() { diff --git a/docs-core/src/main/java/com/sismics/docs/core/listener/async/ExtractFileAsyncListener.java b/docs-core/src/main/java/com/sismics/docs/core/listener/async/ExtractFileAsyncListener.java deleted file mode 100644 index 8b532560..00000000 --- a/docs-core/src/main/java/com/sismics/docs/core/listener/async/ExtractFileAsyncListener.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.sismics.docs.core.listener.async; - -import java.text.MessageFormat; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.eventbus.Subscribe; -import com.sismics.docs.core.dao.jpa.DocumentDao; -import com.sismics.docs.core.dao.jpa.FileDao; -import com.sismics.docs.core.event.ExtractFileAsyncEvent; -import com.sismics.docs.core.model.jpa.Document; -import com.sismics.docs.core.model.jpa.File; -import com.sismics.docs.core.util.FileUtil; -import com.sismics.docs.core.util.TransactionUtil; - -/** - * Listener on extract content from all files. - * - * @author bgamard - */ -public class ExtractFileAsyncListener { - /** - * Logger. - */ - private static final Logger log = LoggerFactory.getLogger(ExtractFileAsyncListener.class); - - /** - * Extract content from all files. - * - * @param extractFileAsyncEvent Extract file content event - * @throws Exception - */ - @Subscribe - public void on(final ExtractFileAsyncEvent extractFileAsyncEvent) throws Exception { - if (log.isInfoEnabled()) { - log.info("Extract file content event: " + extractFileAsyncEvent.toString()); - } - - TransactionUtil.handle(new Runnable() { - @Override - public void run() { - FileDao fileDao = new FileDao(); - DocumentDao documentDao = new DocumentDao(); - List fileList = fileDao.findAll(); - for (File file : fileList) { - long startTime = System.currentTimeMillis(); - Document document = documentDao.getById(file.getDocumentId()); - file.setContent(FileUtil.extractContent(document, file)); - TransactionUtil.commit(); - log.info(MessageFormat.format("File content extracted in {0}ms", System.currentTimeMillis() - startTime)); - } - } - }); - } -} diff --git a/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java b/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java index ffb2af22..57154ac0 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java +++ b/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java @@ -39,7 +39,7 @@ public class FileCreatedAsyncListener { // OCR the file final File file = fileCreatedAsyncEvent.getFile(); long startTime = System.currentTimeMillis(); - final String content = FileUtil.extractContent(fileCreatedAsyncEvent.getDocument(), file); + final String content = FileUtil.extractContent(fileCreatedAsyncEvent.getDocument(), file, fileCreatedAsyncEvent.getInputStream()); log.info(MessageFormat.format("File content extracted in {0}ms", System.currentTimeMillis() - startTime)); // Store the OCR-ization result in the database diff --git a/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java b/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java index cf997a4b..266901d1 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java +++ b/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java @@ -16,7 +16,6 @@ import com.sismics.docs.core.listener.async.DocumentDeletedAsyncListener; import com.sismics.docs.core.listener.async.DocumentUpdatedAsyncListener; import com.sismics.docs.core.listener.async.FileCreatedAsyncListener; import com.sismics.docs.core.listener.async.FileDeletedAsyncListener; -import com.sismics.docs.core.listener.async.ExtractFileAsyncListener; import com.sismics.docs.core.listener.async.RebuildIndexAsyncListener; import com.sismics.docs.core.listener.sync.DeadEventListener; import com.sismics.docs.core.model.jpa.Config; @@ -82,7 +81,6 @@ public class AppContext { asyncEventBus.register(new DocumentUpdatedAsyncListener()); asyncEventBus.register(new DocumentDeletedAsyncListener()); asyncEventBus.register(new RebuildIndexAsyncListener()); - asyncEventBus.register(new ExtractFileAsyncListener()); } /** diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/EncryptionUtil.java b/docs-core/src/main/java/com/sismics/docs/core/util/EncryptionUtil.java index 50963fe4..d86d9759 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/util/EncryptionUtil.java +++ b/docs-core/src/main/java/com/sismics/docs/core/util/EncryptionUtil.java @@ -28,6 +28,11 @@ public class EncryptionUtil { */ private static final String SALT = "LEpxZmm2SMu2PeKzPNrar2rhVAS6LrrgvXKeL9uyXC4vgKHg"; + static { + // Initialize Bouncy Castle provider + Security.insertProviderAt(new BouncyCastleProvider(), 1); + } + /** * Generate a private key. * @@ -39,22 +44,6 @@ public class EncryptionUtil { return new BigInteger(176, random).toString(32); } - /** - * Encrypt an InputStream using the specified private key. - * - * @param is InputStream to encrypt - * @param privateKey Private key - * @return Encrypted stream - * @throws Exception - */ - public static InputStream encryptStream(InputStream is, String privateKey) throws Exception { - checkBouncyCastleProvider(); - if (Strings.isNullOrEmpty(privateKey)) { - throw new IllegalArgumentException("The private key is null or empty"); - } - return new CipherInputStream(is, getCipher(privateKey, Cipher.ENCRYPT_MODE)); - } - /** * Decrypt an InputStream using the specified private key. * @@ -63,11 +52,24 @@ public class EncryptionUtil { * @return Encrypted stream * @throws Exception */ - public static InputStream decryptStream(InputStream is, String privateKey) throws Exception { - checkBouncyCastleProvider(); + public static InputStream decryptInputStream(InputStream is, String privateKey) throws Exception { return new CipherInputStream(is, getCipher(privateKey, Cipher.DECRYPT_MODE)); } + /** + * Return an encryption cipher. + * + * @param privateKey Private key + * @return Encryption cipher + * @throws Exception + */ + public static Cipher getEncryptionCipher(String privateKey) throws Exception { + if (Strings.isNullOrEmpty(privateKey)) { + throw new IllegalArgumentException("The private key is null or empty"); + } + return getCipher(privateKey, Cipher.ENCRYPT_MODE); + } + /** * Initialize a Cipher. * @@ -84,13 +86,4 @@ public class EncryptionUtil { cipher.init(mode, desKey); return cipher; } - - /** - * Initialize the Bouncy Castle provider if necessary. - */ - private static void checkBouncyCastleProvider() { - if (Security.getProvider("BouncyCastleProvider") == null) { - Security.insertProviderAt(new BouncyCastleProvider(), 1); - } - } } diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java index 7fc7c78a..3bdad624 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java +++ b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java @@ -1,13 +1,18 @@ package com.sismics.docs.core.util; import java.awt.image.BufferedImage; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.CipherOutputStream; import javax.imageio.ImageIO; import net.sourceforge.tess4j.Tesseract; @@ -42,36 +47,36 @@ public class FileUtil { * * @param document Document linked to the file * @param file File to extract + * @param inputStream Unencrypted input stream * @return Content extract */ - public static String extractContent(Document document, File file) { + public static String extractContent(Document document, File file, InputStream inputStream) { String content = null; if (ImageUtil.isImage(file.getMimeType())) { - content = ocrFile(document, file); + content = ocrFile(inputStream, document); } else if (file.getMimeType().equals(MimeType.APPLICATION_PDF)) { - content = extractPdf(file); + content = extractPdf(inputStream); } return content; } /** - * Optical character recognition on a file. + * Optical character recognition on a stream. * + * @param inputStream Unencrypted input stream * @param document Document linked to the file - * @param file File to OCR * @return Content extracted */ - private static String ocrFile(Document document, File file) { + private static String ocrFile(InputStream inputStream, Document document) { Tesseract instance = Tesseract.getInstance(); - java.io.File storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile(); String content = null; BufferedImage image = null; try { - image = ImageIO.read(storedfile); + image = ImageIO.read(inputStream); } catch (IOException e) { - log.error("Error reading the image " + storedfile, e); + log.error("Error reading the image", e); } // Upscale and grayscale the image @@ -85,7 +90,7 @@ public class FileUtil { instance.setLanguage(document.getLanguage()); content = instance.doOCR(image); } catch (Exception e) { - log.error("Error while OCR-izing the file " + storedfile, e); + log.error("Error while OCR-izing the image", e); } return content; @@ -94,19 +99,18 @@ public class FileUtil { /** * Extract text from a PDF. * - * @param file File to extract + * @param inputStream Unencrypted input stream * @return Content extracted */ - private static String extractPdf(File file) { + private static String extractPdf(InputStream inputStream) { String content = null; PDDocument pdfDocument = null; - java.io.File storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile(); try { PDFTextStripper stripper = new PDFTextStripper(); - pdfDocument = PDDocument.load(storedfile.getAbsolutePath(), true); + pdfDocument = PDDocument.load(inputStream, true); content = stripper.getText(pdfDocument); } catch (IOException e) { - log.error("Error while extracting text from the PDF " + storedfile, e); + log.error("Error while extracting text from the PDF", e); } finally { if (pdfDocument != null) { try { @@ -123,41 +127,39 @@ public class FileUtil { /** * Save a file on the storage filesystem. * - * @param is InputStream + * @param inputStream Unencrypted input stream * @param file File to save - * @throws IOException + * @param privateKey Private key used for encryption + * @throws Exception */ - public static void save(InputStream is, File file) throws IOException { - // TODO Encrypt file and variations - + public static void save(InputStream inputStream, File file, String privateKey) throws Exception { + Cipher cipher = EncryptionUtil.getEncryptionCipher(privateKey); Path path = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()); - Files.copy(is, path); + Files.copy(new CipherInputStream(inputStream, cipher), path); // Generate file variations - try { - saveVariations(file, path.toFile()); - } catch (IOException e) { - // Don't rethrow Exception from file variations generation - log.error("Error creating file variations", e); - } + inputStream.reset(); + saveVariations(file, inputStream, cipher); + inputStream.reset(); } /** * Generate file variations. * * @param file File from database - * @param originalFile Original file - * @throws IOException + * @param inputStream Unencrypted input stream + * @param cipher Cipher to use for encryption + * @throws Exception */ - public static void saveVariations(File file, java.io.File originalFile) throws IOException { + public static void saveVariations(File file, InputStream inputStream, Cipher cipher) throws Exception { BufferedImage image = null; if (ImageUtil.isImage(file.getMimeType())) { - image = ImageIO.read(originalFile); + image = ImageIO.read(inputStream); } else if(file.getMimeType().equals(MimeType.APPLICATION_PDF)) { // Generate preview from the first page of the PDF PDDocument pdfDocument = null; try { - pdfDocument = PDDocument.load(originalFile.getAbsolutePath(), true); + pdfDocument = PDDocument.load(inputStream, true); @SuppressWarnings("unchecked") List pageList = pdfDocument.getDocumentCatalog().getAllPages(); if (pageList.size() > 0) { @@ -174,8 +176,24 @@ public class FileUtil { BufferedImage web = Scalr.resize(image, Scalr.Method.AUTOMATIC, Scalr.Mode.AUTOMATIC, 1280, Scalr.OP_ANTIALIAS); BufferedImage thumbnail = Scalr.resize(image, Scalr.Method.AUTOMATIC, Scalr.Mode.AUTOMATIC, 256, Scalr.OP_ANTIALIAS); image.flush(); - ImageUtil.writeJpeg(web, Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_web").toFile()); - ImageUtil.writeJpeg(thumbnail, Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_thumb").toFile()); + + // Write "web" encrypted image + java.io.File outputFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_web").toFile(); + OutputStream outputStream = new CipherOutputStream(new FileOutputStream(outputFile), cipher); + try { + ImageUtil.writeJpeg(web, outputStream); + } finally { + outputStream.close(); + } + + // Write "thumb" encrypted image + outputFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_thumb").toFile(); + outputStream = new CipherOutputStream(new FileOutputStream(outputFile), cipher); + try { + ImageUtil.writeJpeg(thumbnail, outputStream); + } finally { + outputStream.close(); + } } } diff --git a/docs-core/src/main/java/com/sismics/util/ImageUtil.java b/docs-core/src/main/java/com/sismics/util/ImageUtil.java index 70ab1fb2..001ba76b 100644 --- a/docs-core/src/main/java/com/sismics/util/ImageUtil.java +++ b/docs-core/src/main/java/com/sismics/util/ImageUtil.java @@ -1,16 +1,17 @@ package com.sismics.util; -import com.sismics.util.mime.MimeType; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Iterator; import javax.imageio.IIOImage; import javax.imageio.ImageIO; import javax.imageio.ImageWriteParam; import javax.imageio.ImageWriter; -import javax.imageio.stream.FileImageOutputStream; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.util.Iterator; +import javax.imageio.stream.ImageOutputStream; + +import com.sismics.util.mime.MimeType; /** * Image processing utilities. @@ -23,26 +24,26 @@ public class ImageUtil { * Write a high quality JPEG. * * @param image - * @param file + * @param outputStream Output stream * @throws IOException */ - public static void writeJpeg(BufferedImage image, File file) throws IOException { + public static void writeJpeg(BufferedImage image, OutputStream outputStream) throws IOException { Iterator iter = ImageIO.getImageWritersByFormatName("jpeg"); ImageWriter writer = null; - FileImageOutputStream output = null; + ImageOutputStream imageOutputStream = null; try { writer = (ImageWriter) iter.next(); ImageWriteParam iwp = writer.getDefaultWriteParam(); iwp.setCompressionMode(ImageWriteParam.MODE_EXPLICIT); iwp.setCompressionQuality(1.f); - output = new FileImageOutputStream(file); - writer.setOutput(output); + imageOutputStream = ImageIO.createImageOutputStream(outputStream); + writer.setOutput(imageOutputStream); IIOImage iioImage = new IIOImage(image, null, null); writer.write(null, iioImage, iwp); } finally { - if (output != null) { + if (imageOutputStream != null) { try { - output.close(); + imageOutputStream.close(); } catch (Exception inner) { // NOP } diff --git a/docs-core/src/main/java/com/sismics/util/mime/MimeTypeUtil.java b/docs-core/src/main/java/com/sismics/util/mime/MimeTypeUtil.java index 82fa5faf..e9e8cd00 100644 --- a/docs-core/src/main/java/com/sismics/util/mime/MimeTypeUtil.java +++ b/docs-core/src/main/java/com/sismics/util/mime/MimeTypeUtil.java @@ -1,6 +1,7 @@ package com.sismics.util.mime; import java.io.InputStream; +import java.io.UnsupportedEncodingException; /** * Utility to check MIME types. @@ -24,6 +25,18 @@ public class MimeTypeUtil { if (readCount <= 0) { throw new Exception("Cannot read input file"); } + + return guessMimeType(headerBytes); + } + + /** + * Try to guess the MIME type of a file by its magic number (header). + * + * @param headerBytes File header (first bytes) + * @return MIME type + * @throws UnsupportedEncodingException + */ + public static String guessMimeType(byte[] headerBytes) throws UnsupportedEncodingException { String header = new String(headerBytes, "US-ASCII"); if (header.startsWith("PK")) { diff --git a/docs-core/src/main/resources/db/update/dbupdate-006-0.sql b/docs-core/src/main/resources/db/update/dbupdate-006-0.sql index 1ce7f827..1377ff03 100644 --- a/docs-core/src/main/resources/db/update/dbupdate-006-0.sql +++ b/docs-core/src/main/resources/db/update/dbupdate-006-0.sql @@ -1,2 +1,3 @@ alter table T_USER add column USE_PRIVATEKEY_C varchar(100) default '' not null; +update T_USER set USE_PRIVATEKEY_C = 'AdminPk' where USE_ID_C = 'admin'; update T_CONFIG set CFG_VALUE_C='6' where CFG_ID_C='DB_VERSION'; diff --git a/docs-core/src/test/java/com/sismics/docs/core/util/TestEncryptUtil.java b/docs-core/src/test/java/com/sismics/docs/core/util/TestEncryptUtil.java index bf4e53d7..6dac1fb0 100644 --- a/docs-core/src/test/java/com/sismics/docs/core/util/TestEncryptUtil.java +++ b/docs-core/src/test/java/com/sismics/docs/core/util/TestEncryptUtil.java @@ -2,9 +2,11 @@ package com.sismics.docs.core.util; import java.io.InputStream; +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; + import junit.framework.Assert; -import org.bouncycastle.util.io.Streams; import org.junit.Test; import com.google.common.base.Strings; @@ -32,14 +34,15 @@ public class TestEncryptUtil { @Test public void encryptStreamTest() throws Exception { try { - EncryptionUtil.encryptStream(this.getClass().getResourceAsStream("/file/udhr.pdf"), ""); + EncryptionUtil.getEncryptionCipher(""); Assert.fail(); } catch (IllegalArgumentException e) { // NOP } - InputStream inputStream = EncryptionUtil.encryptStream(this.getClass().getResourceAsStream("/file/udhr.pdf"), pk); - byte[] encryptedData = Streams.readAll(inputStream); - byte[] assertData = Streams.readAll(this.getClass().getResourceAsStream("/file/udhr_encrypted.pdf")); + Cipher cipher = EncryptionUtil.getEncryptionCipher(pk); + InputStream inputStream = new CipherInputStream(this.getClass().getResourceAsStream("/file/udhr.pdf"), cipher); + byte[] encryptedData = ByteStreams.toByteArray(inputStream); + byte[] assertData = ByteStreams.toByteArray(this.getClass().getResourceAsStream("/file/udhr_encrypted.pdf")); Assert.assertTrue(ByteStreams.equal( ByteStreams.newInputStreamSupplier(encryptedData), ByteStreams.newInputStreamSupplier(assertData))); @@ -47,9 +50,9 @@ public class TestEncryptUtil { @Test public void decryptStreamTest() throws Exception { - InputStream inputStream = EncryptionUtil.decryptStream(this.getClass().getResourceAsStream("/file/udhr_encrypted.pdf"), pk); - byte[] encryptedData = Streams.readAll(inputStream); - byte[] assertData = Streams.readAll(this.getClass().getResourceAsStream("/file/udhr.pdf")); + InputStream inputStream = EncryptionUtil.decryptInputStream(this.getClass().getResourceAsStream("/file/udhr_encrypted.pdf"), pk); + byte[] encryptedData = ByteStreams.toByteArray(inputStream); + byte[] assertData = ByteStreams.toByteArray(this.getClass().getResourceAsStream("/file/udhr.pdf")); Assert.assertTrue(ByteStreams.equal( ByteStreams.newInputStreamSupplier(encryptedData), ByteStreams.newInputStreamSupplier(assertData))); diff --git a/docs-parent/TODO b/docs-parent/TODO index ec061751..e69de29b 100644 --- a/docs-parent/TODO +++ b/docs-parent/TODO @@ -1 +0,0 @@ -- Encrypt files stored on FS (server) diff --git a/docs-web/src/dev/main/webapp/web-override.xml b/docs-web/src/dev/main/webapp/web-override.xml index ea56b8c9..ce2949ce 100644 --- a/docs-web/src/dev/main/webapp/web-override.xml +++ b/docs-web/src/dev/main/webapp/web-override.xml @@ -3,7 +3,7 @@ xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd" - version="3.0"> + version="3.0" metadata-complete="true"> diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java index 06402533..2a624e01 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java @@ -1,7 +1,5 @@ package com.sismics.docs.rest.resource; -import java.io.IOException; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -26,12 +24,10 @@ import com.sismics.docs.core.dao.jpa.DocumentDao; import com.sismics.docs.core.dao.jpa.FileDao; import com.sismics.docs.core.dao.jpa.criteria.DocumentCriteria; import com.sismics.docs.core.dao.jpa.dto.DocumentDto; -import com.sismics.docs.core.event.ExtractFileAsyncEvent; import com.sismics.docs.core.model.context.AppContext; import com.sismics.docs.core.model.jpa.File; import com.sismics.docs.core.util.ConfigUtil; import com.sismics.docs.core.util.DirectoryUtil; -import com.sismics.docs.core.util.FileUtil; import com.sismics.docs.core.util.jpa.PaginatedList; import com.sismics.docs.core.util.jpa.PaginatedLists; import com.sismics.docs.core.util.jpa.SortCriteria; @@ -147,29 +143,6 @@ public class AppResource extends BaseResource { return Response.ok().entity(response).build(); } - /** - * Extract content from all files again. - * - * @return Response - * @throws JSONException - */ - @POST - @Path("batch/extract") - @Produces(MediaType.APPLICATION_JSON) - public Response batchExtract() throws JSONException { - if (!authenticate()) { - throw new ForbiddenClientException(); - } - checkBaseFunction(BaseFunction.ADMIN); - - // Raise an extract file content event - AppContext.getInstance().getAsyncEventBus().post(new ExtractFileAsyncEvent()); - - JSONObject response = new JSONObject(); - response.put("status", "ok"); - return Response.ok().entity(response).build(); - } - /** * Destroy and rebuild Lucene index. * @@ -197,7 +170,7 @@ public class AppResource extends BaseResource { } /** - * Destroy and rebuild Lucene index. + * Clean storage. * * @return Response * @throws JSONException @@ -233,38 +206,4 @@ public class AppResource extends BaseResource { response.put("status", "ok"); return Response.ok().entity(response).build(); } - - /** - * Regenerate file variations. - * - * @return Response - * @throws JSONException - */ - @POST - @Path("batch/file_variations") - @Produces(MediaType.APPLICATION_JSON) - public Response batchFileVariations() throws JSONException { - if (!authenticate()) { - throw new ForbiddenClientException(); - } - checkBaseFunction(BaseFunction.ADMIN); - - // Get all files - FileDao fileDao = new FileDao(); - List fileList = fileDao.findAll(); - - // Generate variations for each file - for (File file : fileList) { - java.io.File originalFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile(); - try { - FileUtil.saveVariations(file, originalFile); - } catch (IOException e) { - throw new ServerException("FileError", "Error generating file variations", e); - } - } - - JSONObject response = new JSONObject(); - response.put("status", "ok"); - return Response.ok().entity(response).build(); - } } diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java index 6f14e925..43e08aff 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java @@ -1,7 +1,10 @@ package com.sismics.docs.rest.resource; -import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Paths; import java.text.MessageFormat; import java.text.SimpleDateFormat; @@ -20,22 +23,28 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import com.google.common.collect.Lists; +import com.google.common.io.ByteStreams; import com.sismics.docs.core.dao.jpa.DocumentDao; import com.sismics.docs.core.dao.jpa.FileDao; import com.sismics.docs.core.dao.jpa.ShareDao; +import com.sismics.docs.core.dao.jpa.UserDao; import com.sismics.docs.core.event.FileCreatedAsyncEvent; import com.sismics.docs.core.event.FileDeletedAsyncEvent; import com.sismics.docs.core.model.context.AppContext; import com.sismics.docs.core.model.jpa.Document; import com.sismics.docs.core.model.jpa.File; +import com.sismics.docs.core.model.jpa.User; import com.sismics.docs.core.util.DirectoryUtil; +import com.sismics.docs.core.util.EncryptionUtil; import com.sismics.docs.core.util.FileUtil; import com.sismics.rest.exception.ClientException; import com.sismics.rest.exception.ForbiddenClientException; @@ -77,20 +86,30 @@ public class FileResource extends BaseResource { // Get the document DocumentDao documentDao = new DocumentDao(); + FileDao fileDao = new FileDao(); + UserDao userDao = new UserDao(); Document document; + User user; try { document = documentDao.getDocument(documentId, principal.getId()); + user = userDao.getById(principal.getId()); } catch (NoResultException e) { throw new ClientException("DocumentNotFound", MessageFormat.format("Document not found: {0}", documentId)); } - FileDao fileDao = new FileDao(); + // Keep unencrypted data in memory, because we will need it two times + byte[] fileData; + try { + fileData = ByteStreams.toByteArray(fileBodyPart.getValueAs(InputStream.class)); + } catch (IOException e) { + throw new ServerException("StreamError", "Error reading the input file", e); + } + InputStream fileInputStream = new ByteArrayInputStream(fileData); // Validate mime type - InputStream is = new BufferedInputStream(fileBodyPart.getValueAs(InputStream.class)); String mimeType; try { - mimeType = MimeTypeUtil.guessMimeType(is); + mimeType = MimeTypeUtil.guessMimeType(fileInputStream); } catch (Exception e) { throw new ServerException("ErrorGuessMime", "Error guessing mime type", e); } @@ -113,12 +132,13 @@ public class FileResource extends BaseResource { String fileId = fileDao.create(file); // Save the file - FileUtil.save(is, file); + FileUtil.save(fileInputStream, file, user.getPrivateKey()); // Raise a new file created event FileCreatedAsyncEvent fileCreatedAsyncEvent = new FileCreatedAsyncEvent(); fileCreatedAsyncEvent.setDocument(document); fileCreatedAsyncEvent.setFile(file); + fileCreatedAsyncEvent.setInputStream(fileInputStream); AppContext.getInstance().getAsyncEventBus().post(fileCreatedAsyncEvent); // Always return ok @@ -288,10 +308,12 @@ public class FileResource extends BaseResource { // Get the file FileDao fileDao = new FileDao(); DocumentDao documentDao = new DocumentDao(); + UserDao userDao = new UserDao(); File file; + Document document; try { file = fileDao.getFile(fileId); - Document document = documentDao.getDocument(file.getDocumentId()); + document = documentDao.getDocument(file.getDocumentId()); // Check document visibility ShareDao shareDao = new ShareDao(); @@ -304,12 +326,13 @@ public class FileResource extends BaseResource { // Get the stored file - // TODO Decrypt file java.io.File storedfile; String mimeType; + boolean decrypt = false; if (size != null) { storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), fileId + "_" + size).toFile(); mimeType = MimeType.IMAGE_JPEG; // Thumbnails are JPEG + decrypt = true; // Thumbnails are encrypted if (!storedfile.exists()) { storedfile = new java.io.File(getClass().getResource("/image/file.png").getFile()); mimeType = MimeType.IMAGE_PNG; @@ -317,11 +340,35 @@ public class FileResource extends BaseResource { } else { storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), fileId).toFile(); mimeType = file.getMimeType(); + decrypt = true; // Original files are encrypted + } + + // Stream the output and decrypt it if necessary + StreamingOutput stream; + User user = userDao.getById(document.getUserId()); + try { + InputStream fileInputStream = new FileInputStream(storedfile); + final InputStream responseInputStream = decrypt ? + EncryptionUtil.decryptInputStream(fileInputStream, user.getPrivateKey()) : fileInputStream; + + stream = new StreamingOutput() { + @Override + public void write(OutputStream outputStream) throws IOException, WebApplicationException { + try { + ByteStreams.copy(responseInputStream, outputStream); + } finally { + responseInputStream.close(); + outputStream.close(); + } + } + }; + } catch (Exception e) { + throw new ServerException("FileError", "Error while reading the file", e); } - return Response.ok(storedfile) + return Response.ok(stream) .header("Content-Type", mimeType) - .header("Expires", new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z").format(new Date().getTime() + 3600000 * 24 * 7)) + .header("Expires", new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z").format(new Date().getTime() + 3600000 * 24)) .build(); } } diff --git a/docs-web/src/main/webapp/WEB-INF/web.xml b/docs-web/src/main/webapp/WEB-INF/web.xml index e71ef87f..031bab53 100644 --- a/docs-web/src/main/webapp/WEB-INF/web.xml +++ b/docs-web/src/main/webapp/WEB-INF/web.xml @@ -3,9 +3,11 @@ xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd" - version="3.0"> + version="3.0" metadata-complete="true"> Docs - + + + requestContextFilter diff --git a/docs-web/src/main/webapp/partial/docs/document.edit.html b/docs-web/src/main/webapp/partial/docs/document.edit.html index 120fbcd1..3f09d2e5 100644 --- a/docs-web/src/main/webapp/partial/docs/document.edit.html +++ b/docs-web/src/main/webapp/partial/docs/document.edit.html @@ -1,6 +1,6 @@ - + -
+
diff --git a/docs-web/src/main/webapp/partial/docs/document.view.html b/docs-web/src/main/webapp/partial/docs/document.view.html index df5f8716..6627316f 100644 --- a/docs-web/src/main/webapp/partial/docs/document.view.html +++ b/docs-web/src/main/webapp/partial/docs/document.view.html @@ -1,6 +1,6 @@ - + -
+
@@ -9,7 +9,7 @@
-

{{ document.title }} {{ document.create_date | date: 'yyyy-MM-dd' }}

+

{{ document.title }} {{ document.create_date | date: 'yyyy-MM-dd' }}

diff --git a/docs-web/src/main/webapp/partial/docs/settings.user.edit.html b/docs-web/src/main/webapp/partial/docs/settings.user.edit.html index 034357ae..1cbebe12 100644 --- a/docs-web/src/main/webapp/partial/docs/settings.user.edit.html +++ b/docs-web/src/main/webapp/partial/docs/settings.user.edit.html @@ -1,6 +1,6 @@ - + -

+

Edit "{{ user.username }}"

diff --git a/docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java b/docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java index 3cfc0040..878790d4 100644 --- a/docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java +++ b/docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java @@ -43,13 +43,6 @@ public class TestAppResource extends BaseJerseyTest { Assert.assertTrue(totalMemory > 0 && totalMemory > freeMemory); Assert.assertEquals(0, json.getInt("document_count")); - // OCR-ize all files - appResource = resource().path("/app/batch/extract"); - appResource.addFilter(new CookieAuthenticationFilter(adminAuthenticationToken)); - response = appResource.post(ClientResponse.class); - Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); - json = response.getEntity(JSONObject.class); - // Rebuild Lucene index appResource = resource().path("/app/batch/reindex"); appResource.addFilter(new CookieAuthenticationFilter(adminAuthenticationToken)); diff --git a/docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java b/docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java index 8fa059cd..a02a929a 100644 --- a/docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java +++ b/docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java @@ -1,6 +1,7 @@ package com.sismics.docs.rest; import java.io.BufferedInputStream; +import java.io.FileInputStream; import java.io.InputStream; import java.nio.file.Paths; @@ -15,6 +16,8 @@ import org.junit.Test; import com.google.common.io.ByteStreams; import com.sismics.docs.core.util.DirectoryUtil; import com.sismics.docs.rest.filter.CookieAuthenticationFilter; +import com.sismics.util.mime.MimeType; +import com.sismics.util.mime.MimeTypeUtil; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse.Status; import com.sun.jersey.api.client.WebResource; @@ -88,6 +91,7 @@ public class TestFileResource extends BaseJerseyTest { Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); InputStream is = response.getEntityInputStream(); byte[] fileBytes = ByteStreams.toByteArray(is); + Assert.assertEquals(MimeType.IMAGE_JPEG, MimeTypeUtil.guessMimeType(fileBytes)); Assert.assertEquals(163510, fileBytes.length); // Get the thumbnail data @@ -99,6 +103,7 @@ public class TestFileResource extends BaseJerseyTest { Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); is = response.getEntityInputStream(); fileBytes = ByteStreams.toByteArray(is); + Assert.assertEquals(MimeType.IMAGE_JPEG, MimeTypeUtil.guessMimeType(fileBytes)); Assert.assertEquals(41935, fileBytes.length); // Get the web data @@ -110,45 +115,14 @@ public class TestFileResource extends BaseJerseyTest { Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); is = response.getEntityInputStream(); fileBytes = ByteStreams.toByteArray(is); + Assert.assertEquals(MimeType.IMAGE_JPEG, MimeTypeUtil.guessMimeType(fileBytes)); Assert.assertEquals(551084, fileBytes.length); - // Regenerate file variations - String adminAuthenticationToken = clientUtil.login("admin", "admin", false); - WebResource appResource = resource().path("/app/batch/file_variations"); - appResource.addFilter(new CookieAuthenticationFilter(adminAuthenticationToken)); - response = appResource.post(ClientResponse.class); - Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); - - // Get the file data - fileResource = resource().path("/file/" + file1Id + "/data"); - fileResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken)); - response = fileResource.get(ClientResponse.class); - Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); - is = response.getEntityInputStream(); - fileBytes = ByteStreams.toByteArray(is); - Assert.assertEquals(163510, fileBytes.length); - - // Get the thumbnail data - fileResource = resource().path("/file/" + file1Id + "/data"); - fileResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken)); - getParams = new MultivaluedMapImpl(); - getParams.putSingle("size", "thumb"); - response = fileResource.queryParams(getParams).get(ClientResponse.class); - Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); - is = response.getEntityInputStream(); - fileBytes = ByteStreams.toByteArray(is); - Assert.assertEquals(41935, fileBytes.length); - - // Get the web data - fileResource = resource().path("/file/" + file1Id + "/data"); - fileResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken)); - getParams = new MultivaluedMapImpl(); - getParams.putSingle("size", "web"); - response = fileResource.queryParams(getParams).get(ClientResponse.class); - Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); - is = response.getEntityInputStream(); - fileBytes = ByteStreams.toByteArray(is); - Assert.assertEquals(551084, fileBytes.length); + // Check that the files are not readable directly from FS + java.io.File storedFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file1Id).toFile(); + InputStream storedFileInputStream = new BufferedInputStream(new FileInputStream(storedFile)); + Assert.assertNull(MimeTypeUtil.guessMimeType(storedFileInputStream)); + storedFileInputStream.close(); // Get all files from a document fileResource = resource().path("/file/list"); @@ -195,7 +169,7 @@ public class TestFileResource extends BaseJerseyTest { Assert.assertEquals("ok", json.getString("status")); // Check that files are deleted from FS - java.io.File storedFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file1Id).toFile(); + storedFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file1Id).toFile(); java.io.File webFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file1Id + "_web").toFile(); java.io.File thumbnailFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file1Id + "_thumb").toFile(); Assert.assertFalse(storedFile.exists());