Encrypt stored files in SHA 256

docs-core/src/main/java/com/sismics/docs/core/event/ExtractFileAsyncEvent.java

@@ -1,16 +0,0 @@
-package com.sismics.docs.core.event;
-
-import com.google.common.base.Objects;
-
-/**
- * Extract file content event.
- *
- * @author bgamard
- */
-public class ExtractFileAsyncEvent {
-    @Override
-    public String toString() {
-        return Objects.toStringHelper(this)
-                .toString();
-    }
-}

docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java

@@ -1,5 +1,7 @@
 package com.sismics.docs.core.event;
 
+import java.io.InputStream;
+
 import com.google.common.base.Objects;
 import com.sismics.docs.core.model.jpa.Document;
 import com.sismics.docs.core.model.jpa.File;
@@ -20,6 +22,11 @@ public class FileCreatedAsyncEvent {
      */
     private Document document;
     
+    /**
+     * Unencrypted input stream containing the file.
+     */
+    private InputStream inputStream;
+    
     /**
      * Getter of file.
      *
@@ -55,6 +62,24 @@ public class FileCreatedAsyncEvent {
     public void setDocument(Document document) {
         this.document = document;
     }
+    
+    /**
+     * Getter of inputStream.
+     *
+     * @return the inputStream
+     */
+    public InputStream getInputStream() {
+        return inputStream;
+    }
+
+    /**
+     * Setter de inputStream.
+     *
+     * @param inputStream inputStream
+     */
+    public void setInputStream(InputStream inputStream) {
+        this.inputStream = inputStream;
+    }
 
     @Override
     public String toString() {

docs-core/src/main/java/com/sismics/docs/core/listener/async/ExtractFileAsyncListener.java

@@ -1,57 +0,0 @@
-package com.sismics.docs.core.listener.async;
-
-import java.text.MessageFormat;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.eventbus.Subscribe;
-import com.sismics.docs.core.dao.jpa.DocumentDao;
-import com.sismics.docs.core.dao.jpa.FileDao;
-import com.sismics.docs.core.event.ExtractFileAsyncEvent;
-import com.sismics.docs.core.model.jpa.Document;
-import com.sismics.docs.core.model.jpa.File;
-import com.sismics.docs.core.util.FileUtil;
-import com.sismics.docs.core.util.TransactionUtil;
-
-/**
- * Listener on extract content from all files.
- * 
- * @author bgamard
- */
-public class ExtractFileAsyncListener {
-    /**
-     * Logger.
-     */
-    private static final Logger log = LoggerFactory.getLogger(ExtractFileAsyncListener.class);
-
-    /**
-     * Extract content from all files.
-     * 
-     * @param extractFileAsyncEvent Extract file content event
-     * @throws Exception
-     */
-    @Subscribe
-    public void on(final ExtractFileAsyncEvent extractFileAsyncEvent) throws Exception {
-        if (log.isInfoEnabled()) {
-            log.info("Extract file content event: " + extractFileAsyncEvent.toString());
-        }
-
-        TransactionUtil.handle(new Runnable() {
-            @Override
-            public void run() {
-                FileDao fileDao = new FileDao();
-                DocumentDao documentDao = new DocumentDao();
-                List<File> fileList = fileDao.findAll();
-                for (File file : fileList) {
-                    long startTime = System.currentTimeMillis();
-                    Document document = documentDao.getById(file.getDocumentId());
-                    file.setContent(FileUtil.extractContent(document, file));
-                    TransactionUtil.commit();
-                    log.info(MessageFormat.format("File content extracted in {0}ms", System.currentTimeMillis() - startTime));
-                }
-            }
-        });
-    }
-}

docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java

@@ -39,7 +39,7 @@ public class FileCreatedAsyncListener {
         // OCR the file
         final File file = fileCreatedAsyncEvent.getFile();
         long startTime = System.currentTimeMillis();
-        final String content = FileUtil.extractContent(fileCreatedAsyncEvent.getDocument(), file);
+        final String content = FileUtil.extractContent(fileCreatedAsyncEvent.getDocument(), file, fileCreatedAsyncEvent.getInputStream());
         log.info(MessageFormat.format("File content extracted in {0}ms", System.currentTimeMillis() - startTime));
         
         // Store the OCR-ization result in the database

docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java

@@ -16,7 +16,6 @@ import com.sismics.docs.core.listener.async.DocumentDeletedAsyncListener;
 import com.sismics.docs.core.listener.async.DocumentUpdatedAsyncListener;
 import com.sismics.docs.core.listener.async.FileCreatedAsyncListener;
 import com.sismics.docs.core.listener.async.FileDeletedAsyncListener;
-import com.sismics.docs.core.listener.async.ExtractFileAsyncListener;
 import com.sismics.docs.core.listener.async.RebuildIndexAsyncListener;
 import com.sismics.docs.core.listener.sync.DeadEventListener;
 import com.sismics.docs.core.model.jpa.Config;
@@ -82,7 +81,6 @@ public class AppContext {
         asyncEventBus.register(new DocumentUpdatedAsyncListener());
         asyncEventBus.register(new DocumentDeletedAsyncListener());
         asyncEventBus.register(new RebuildIndexAsyncListener());
-        asyncEventBus.register(new ExtractFileAsyncListener());
     }
 
     /**

docs-core/src/main/java/com/sismics/docs/core/util/EncryptionUtil.java

@@ -28,6 +28,11 @@ public class EncryptionUtil {
      */
     private static final String SALT = "LEpxZmm2SMu2PeKzPNrar2rhVAS6LrrgvXKeL9uyXC4vgKHg";
     
+    static {
+        // Initialize Bouncy Castle provider
+        Security.insertProviderAt(new BouncyCastleProvider(), 1);
+    }
+    
     /**
      * Generate a private key.
      * 
@@ -39,22 +44,6 @@ public class EncryptionUtil {
         return new BigInteger(176, random).toString(32);
     }
     
-    /**
-     * Encrypt an InputStream using the specified private key.
-     * 
-     * @param is InputStream to encrypt
-     * @param privateKey Private key
-     * @return Encrypted stream
-     * @throws Exception 
-     */
-    public static InputStream encryptStream(InputStream is, String privateKey) throws Exception {
-        checkBouncyCastleProvider();
-        if (Strings.isNullOrEmpty(privateKey)) {
-            throw new IllegalArgumentException("The private key is null or empty");
-        }
-        return new CipherInputStream(is, getCipher(privateKey, Cipher.ENCRYPT_MODE));
-    }
-    
     /**
      * Decrypt an InputStream using the specified private key.
      * 
@@ -63,11 +52,24 @@ public class EncryptionUtil {
      * @return Encrypted stream
      * @throws Exception 
      */
-    public static InputStream decryptStream(InputStream is, String privateKey) throws Exception {
+    public static InputStream decryptInputStream(InputStream is, String privateKey) throws Exception {
-        checkBouncyCastleProvider();
         return new CipherInputStream(is, getCipher(privateKey, Cipher.DECRYPT_MODE));
     }
     
+    /**
+     * Return an encryption cipher.
+     * 
+     * @param privateKey Private key
+     * @return Encryption cipher
+     * @throws Exception
+     */
+    public static Cipher getEncryptionCipher(String privateKey) throws Exception {
+        if (Strings.isNullOrEmpty(privateKey)) {
+            throw new IllegalArgumentException("The private key is null or empty");
+        }
+        return getCipher(privateKey, Cipher.ENCRYPT_MODE);
+    }
+    
     /**
      * Initialize a Cipher.
      * 
@@ -84,13 +86,4 @@ public class EncryptionUtil {
         cipher.init(mode, desKey);
         return cipher;
     }
-    
-    /**
-     * Initialize the Bouncy Castle provider if necessary.
-     */
-    private static void checkBouncyCastleProvider() {
-        if (Security.getProvider("BouncyCastleProvider") == null) {
-            Security.insertProviderAt(new BouncyCastleProvider(), 1);
-        }
-    }
 }

docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java

@@ -1,13 +1,18 @@
 package com.sismics.docs.core.util;
 
 import java.awt.image.BufferedImage;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
 
+import javax.crypto.Cipher;
+import javax.crypto.CipherInputStream;
+import javax.crypto.CipherOutputStream;
 import javax.imageio.ImageIO;
 
 import net.sourceforge.tess4j.Tesseract;
@@ -42,36 +47,36 @@ public class FileUtil {
      * 
      * @param document Document linked to the file
      * @param file File to extract
+     * @param inputStream Unencrypted input stream
      * @return Content extract
      */
-    public static String extractContent(Document document, File file) {
+    public static String extractContent(Document document, File file, InputStream inputStream) {
         String content = null;
         
         if (ImageUtil.isImage(file.getMimeType())) {
-            content = ocrFile(document, file);
+            content = ocrFile(inputStream, document);
         } else if (file.getMimeType().equals(MimeType.APPLICATION_PDF)) {
-            content = extractPdf(file);
+            content = extractPdf(inputStream);
         }
         
         return content;
     }
     
     /**
-     * Optical character recognition on a file.
+     * Optical character recognition on a stream.
      * 
+     * @param inputStream Unencrypted input stream
      * @param document Document linked to the file
-     * @param file File to OCR
      * @return Content extracted
      */
-    private static String ocrFile(Document document, File file) {
+    private static String ocrFile(InputStream inputStream, Document document) {
         Tesseract instance = Tesseract.getInstance();
-        java.io.File storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile();
         String content = null;
         BufferedImage image = null;
         try {
-            image = ImageIO.read(storedfile);
+            image = ImageIO.read(inputStream);
         } catch (IOException e) {
-            log.error("Error reading the image " + storedfile, e);
+            log.error("Error reading the image", e);
         }
         
         // Upscale and grayscale the image
@@ -85,7 +90,7 @@ public class FileUtil {
             instance.setLanguage(document.getLanguage());
             content = instance.doOCR(image);
         } catch (Exception e) {
-            log.error("Error while OCR-izing the file " + storedfile, e);
+            log.error("Error while OCR-izing the image", e);
         }
         
         return content;
@@ -94,19 +99,18 @@ public class FileUtil {
     /**
      * Extract text from a PDF.
      * 
-     * @param file File to extract
+     * @param inputStream Unencrypted input stream
      * @return Content extracted
      */
-    private static String extractPdf(File file) {
+    private static String extractPdf(InputStream inputStream) {
         String content = null;
         PDDocument pdfDocument = null;
-        java.io.File storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile();
         try {
             PDFTextStripper stripper = new PDFTextStripper();
-            pdfDocument = PDDocument.load(storedfile.getAbsolutePath(), true);
+            pdfDocument = PDDocument.load(inputStream, true);
             content = stripper.getText(pdfDocument);
         } catch (IOException e) {
-            log.error("Error while extracting text from the PDF " + storedfile, e);
+            log.error("Error while extracting text from the PDF", e);
         } finally {
             if (pdfDocument != null) {
                 try {
@@ -123,41 +127,39 @@ public class FileUtil {
     /**
      * Save a file on the storage filesystem.
      * 
-     * @param is InputStream
+     * @param inputStream Unencrypted input stream
      * @param file File to save
-     * @throws IOException
+     * @param privateKey Private key used for encryption
+     * @throws Exception
      */
-    public static void save(InputStream is, File file) throws IOException {
+    public static void save(InputStream inputStream, File file, String privateKey) throws Exception {
-        // TODO Encrypt file and variations
+        Cipher cipher = EncryptionUtil.getEncryptionCipher(privateKey);
-        
         Path path = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId());
-        Files.copy(is, path);
+        Files.copy(new CipherInputStream(inputStream, cipher), path);
         
         // Generate file variations
-        try {
+        inputStream.reset();
-            saveVariations(file, path.toFile());
+        saveVariations(file, inputStream, cipher);
-        } catch (IOException e) {
+        inputStream.reset();
-            // Don't rethrow Exception from file variations generation
-            log.error("Error creating file variations", e);
-        }
     }
 
     /**
      * Generate file variations.
      * 
      * @param file File from database
-     * @param originalFile Original file
+     * @param inputStream Unencrypted input stream
-     * @throws IOException
+     * @param cipher Cipher to use for encryption
+     * @throws Exception
      */
-    public static void saveVariations(File file, java.io.File originalFile) throws IOException {
+    public static void saveVariations(File file, InputStream inputStream, Cipher cipher) throws Exception {
         BufferedImage image = null;
         if (ImageUtil.isImage(file.getMimeType())) {
-            image = ImageIO.read(originalFile);
+            image = ImageIO.read(inputStream);
         } else if(file.getMimeType().equals(MimeType.APPLICATION_PDF)) {
             // Generate preview from the first page of the PDF
             PDDocument pdfDocument = null;
             try {
-                pdfDocument = PDDocument.load(originalFile.getAbsolutePath(), true);
+                pdfDocument = PDDocument.load(inputStream, true);
                 @SuppressWarnings("unchecked")
                 List<PDPage> pageList = pdfDocument.getDocumentCatalog().getAllPages();
                 if (pageList.size() > 0) {
@@ -174,8 +176,24 @@ public class FileUtil {
             BufferedImage web = Scalr.resize(image, Scalr.Method.AUTOMATIC, Scalr.Mode.AUTOMATIC, 1280, Scalr.OP_ANTIALIAS);
             BufferedImage thumbnail = Scalr.resize(image, Scalr.Method.AUTOMATIC, Scalr.Mode.AUTOMATIC, 256, Scalr.OP_ANTIALIAS);
             image.flush();
-            ImageUtil.writeJpeg(web, Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_web").toFile());
+            
-            ImageUtil.writeJpeg(thumbnail, Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_thumb").toFile());
+            // Write "web" encrypted image
+            java.io.File outputFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_web").toFile();
+            OutputStream outputStream = new CipherOutputStream(new FileOutputStream(outputFile), cipher);
+            try {
+                ImageUtil.writeJpeg(web, outputStream);
+            } finally {
+                outputStream.close();
+            }
+            
+            // Write "thumb" encrypted image
+            outputFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_thumb").toFile();
+            outputStream = new CipherOutputStream(new FileOutputStream(outputFile), cipher);
+            try {
+                ImageUtil.writeJpeg(thumbnail, outputStream);
+            } finally {
+                outputStream.close();
+            }
         }
     }
 

docs-core/src/main/java/com/sismics/util/ImageUtil.java

@@ -1,16 +1,17 @@
 package com.sismics.util;
 
-import com.sismics.util.mime.MimeType;
+import java.awt.image.BufferedImage;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Iterator;
 
 import javax.imageio.IIOImage;
 import javax.imageio.ImageIO;
 import javax.imageio.ImageWriteParam;
 import javax.imageio.ImageWriter;
-import javax.imageio.stream.FileImageOutputStream;
+import javax.imageio.stream.ImageOutputStream;
-import java.awt.image.BufferedImage;
+
-import java.io.File;
+import com.sismics.util.mime.MimeType;
-import java.io.IOException;
-import java.util.Iterator;
 
 /**
  * Image processing utilities.
@@ -23,26 +24,26 @@ public class ImageUtil {
      * Write a high quality JPEG.
      * 
      * @param image
-     * @param file
+     * @param outputStream Output stream
      * @throws IOException
      */
-    public static void writeJpeg(BufferedImage image, File file) throws IOException {
+    public static void writeJpeg(BufferedImage image, OutputStream outputStream) throws IOException {
         Iterator<ImageWriter> iter = ImageIO.getImageWritersByFormatName("jpeg");
         ImageWriter writer = null;
-        FileImageOutputStream output = null;
+        ImageOutputStream imageOutputStream = null;
         try {
             writer = (ImageWriter) iter.next();
             ImageWriteParam iwp = writer.getDefaultWriteParam();
             iwp.setCompressionMode(ImageWriteParam.MODE_EXPLICIT);
             iwp.setCompressionQuality(1.f);
-            output = new FileImageOutputStream(file);
+            imageOutputStream = ImageIO.createImageOutputStream(outputStream);
-            writer.setOutput(output);
+            writer.setOutput(imageOutputStream);
             IIOImage iioImage = new IIOImage(image, null, null);
             writer.write(null, iioImage, iwp);
         } finally {
-            if (output != null) {
+            if (imageOutputStream != null) {
                 try {
-                    output.close();
+                    imageOutputStream.close();
                 } catch (Exception inner) {
                     // NOP
                 }

docs-core/src/main/java/com/sismics/util/mime/MimeTypeUtil.java

@@ -1,6 +1,7 @@
 package com.sismics.util.mime;
 
 import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
 
 /**
  * Utility to check MIME types.
@@ -24,6 +25,18 @@ public class MimeTypeUtil {
         if (readCount <= 0) {
             throw new Exception("Cannot read input file");
         }
+        
+        return guessMimeType(headerBytes);
+    }
+
+    /**
+     * Try to guess the MIME type of a file by its magic number (header).
+     * 
+     * @param headerBytes File header (first bytes)
+     * @return MIME type
+     * @throws UnsupportedEncodingException
+     */
+    public static String guessMimeType(byte[] headerBytes) throws UnsupportedEncodingException {
         String header = new String(headerBytes, "US-ASCII");
         
         if (header.startsWith("PK")) {

docs-core/src/main/resources/db/update/dbupdate-006-0.sql

@@ -1,2 +1,3 @@
 alter table T_USER add column USE_PRIVATEKEY_C varchar(100) default '' not null;
+update T_USER set USE_PRIVATEKEY_C = 'AdminPk' where USE_ID_C = 'admin';
 update T_CONFIG set CFG_VALUE_C='6' where CFG_ID_C='DB_VERSION';

docs-core/src/test/java/com/sismics/docs/core/util/TestEncryptUtil.java

@@ -2,9 +2,11 @@ package com.sismics.docs.core.util;
 
 import java.io.InputStream;
 
+import javax.crypto.Cipher;
+import javax.crypto.CipherInputStream;
+
 import junit.framework.Assert;
 
-import org.bouncycastle.util.io.Streams;
 import org.junit.Test;
 
 import com.google.common.base.Strings;
@@ -32,14 +34,15 @@ public class TestEncryptUtil {
     @Test
     public void encryptStreamTest() throws Exception {
         try {
-            EncryptionUtil.encryptStream(this.getClass().getResourceAsStream("/file/udhr.pdf"), "");
+            EncryptionUtil.getEncryptionCipher("");
             Assert.fail();
         } catch (IllegalArgumentException e) {
             // NOP
         }
-        InputStream inputStream = EncryptionUtil.encryptStream(this.getClass().getResourceAsStream("/file/udhr.pdf"), pk);
+        Cipher cipher = EncryptionUtil.getEncryptionCipher(pk);
-        byte[] encryptedData = Streams.readAll(inputStream);
+        InputStream inputStream = new CipherInputStream(this.getClass().getResourceAsStream("/file/udhr.pdf"), cipher);
-        byte[] assertData = Streams.readAll(this.getClass().getResourceAsStream("/file/udhr_encrypted.pdf"));
+        byte[] encryptedData = ByteStreams.toByteArray(inputStream);
+        byte[] assertData = ByteStreams.toByteArray(this.getClass().getResourceAsStream("/file/udhr_encrypted.pdf"));
         Assert.assertTrue(ByteStreams.equal(
                 ByteStreams.newInputStreamSupplier(encryptedData),
                 ByteStreams.newInputStreamSupplier(assertData)));
@@ -47,9 +50,9 @@ public class TestEncryptUtil {
     
     @Test
     public void decryptStreamTest() throws Exception {
-        InputStream inputStream = EncryptionUtil.decryptStream(this.getClass().getResourceAsStream("/file/udhr_encrypted.pdf"), pk);
+        InputStream inputStream = EncryptionUtil.decryptInputStream(this.getClass().getResourceAsStream("/file/udhr_encrypted.pdf"), pk);
-        byte[] encryptedData = Streams.readAll(inputStream);
+        byte[] encryptedData = ByteStreams.toByteArray(inputStream);
-        byte[] assertData = Streams.readAll(this.getClass().getResourceAsStream("/file/udhr.pdf"));
+        byte[] assertData = ByteStreams.toByteArray(this.getClass().getResourceAsStream("/file/udhr.pdf"));
         Assert.assertTrue(ByteStreams.equal(
                 ByteStreams.newInputStreamSupplier(encryptedData),
                 ByteStreams.newInputStreamSupplier(assertData)));

docs-parent/TODO

@@ -1 +0,0 @@
-- Encrypt files stored on FS (server)

docs-web/src/dev/main/webapp/web-override.xml

@@ -3,7 +3,7 @@
   xmlns="http://java.sun.com/xml/ns/javaee"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
-  version="3.0"> 
+  version="3.0" metadata-complete="true"> 
 
   <!-- Override init parameter to avoid nasty file locking issue on windows. -->
   <servlet>

docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java

@@ -1,7 +1,5 @@
 package com.sismics.docs.rest.resource;
 
-import java.io.IOException;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -26,12 +24,10 @@ import com.sismics.docs.core.dao.jpa.DocumentDao;
 import com.sismics.docs.core.dao.jpa.FileDao;
 import com.sismics.docs.core.dao.jpa.criteria.DocumentCriteria;
 import com.sismics.docs.core.dao.jpa.dto.DocumentDto;
-import com.sismics.docs.core.event.ExtractFileAsyncEvent;
 import com.sismics.docs.core.model.context.AppContext;
 import com.sismics.docs.core.model.jpa.File;
 import com.sismics.docs.core.util.ConfigUtil;
 import com.sismics.docs.core.util.DirectoryUtil;
-import com.sismics.docs.core.util.FileUtil;
 import com.sismics.docs.core.util.jpa.PaginatedList;
 import com.sismics.docs.core.util.jpa.PaginatedLists;
 import com.sismics.docs.core.util.jpa.SortCriteria;
@@ -147,29 +143,6 @@ public class AppResource extends BaseResource {
         return Response.ok().entity(response).build();
     }
     
-    /**
-     * Extract content from all files again.
-     * 
-     * @return Response
-     * @throws JSONException
-     */
-    @POST
-    @Path("batch/extract")
-    @Produces(MediaType.APPLICATION_JSON)
-    public Response batchExtract() throws JSONException {
-        if (!authenticate()) {
-            throw new ForbiddenClientException();
-        }
-        checkBaseFunction(BaseFunction.ADMIN);
-        
-        // Raise an extract file content event
-        AppContext.getInstance().getAsyncEventBus().post(new ExtractFileAsyncEvent());
-        
-        JSONObject response = new JSONObject();
-        response.put("status", "ok");
-        return Response.ok().entity(response).build();
-    }
-    
     /**
      * Destroy and rebuild Lucene index.
      * 
@@ -197,7 +170,7 @@ public class AppResource extends BaseResource {
     }
     
     /**
-     * Destroy and rebuild Lucene index.
+     * Clean storage.
      * 
      * @return Response
      * @throws JSONException
@@ -233,38 +206,4 @@ public class AppResource extends BaseResource {
         response.put("status", "ok");
         return Response.ok().entity(response).build();
     }
-    
-    /**
-     * Regenerate file variations.
-     * 
-     * @return Response
-     * @throws JSONException
-     */
-    @POST
-    @Path("batch/file_variations")
-    @Produces(MediaType.APPLICATION_JSON)
-    public Response batchFileVariations() throws JSONException {
-        if (!authenticate()) {
-            throw new ForbiddenClientException();
-        }
-        checkBaseFunction(BaseFunction.ADMIN);
-        
-        // Get all files
-        FileDao fileDao = new FileDao();
-        List<File> fileList = fileDao.findAll();
-        
-        // Generate variations for each file
-        for (File file : fileList) {
-            java.io.File originalFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile();
-            try {
-                FileUtil.saveVariations(file, originalFile);
-            } catch (IOException e) {
-                throw new ServerException("FileError", "Error generating file variations", e);
-            }
-        }
-        
-        JSONObject response = new JSONObject();
-        response.put("status", "ok");
-        return Response.ok().entity(response).build();
-    }
 }

docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java

@@ -1,7 +1,10 @@
 package com.sismics.docs.rest.resource;
 
-import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.file.Paths;
 import java.text.MessageFormat;
 import java.text.SimpleDateFormat;
@@ -20,22 +23,28 @@ import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
 
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 
 import com.google.common.collect.Lists;
+import com.google.common.io.ByteStreams;
 import com.sismics.docs.core.dao.jpa.DocumentDao;
 import com.sismics.docs.core.dao.jpa.FileDao;
 import com.sismics.docs.core.dao.jpa.ShareDao;
+import com.sismics.docs.core.dao.jpa.UserDao;
 import com.sismics.docs.core.event.FileCreatedAsyncEvent;
 import com.sismics.docs.core.event.FileDeletedAsyncEvent;
 import com.sismics.docs.core.model.context.AppContext;
 import com.sismics.docs.core.model.jpa.Document;
 import com.sismics.docs.core.model.jpa.File;
+import com.sismics.docs.core.model.jpa.User;
 import com.sismics.docs.core.util.DirectoryUtil;
+import com.sismics.docs.core.util.EncryptionUtil;
 import com.sismics.docs.core.util.FileUtil;
 import com.sismics.rest.exception.ClientException;
 import com.sismics.rest.exception.ForbiddenClientException;
@@ -77,20 +86,30 @@ public class FileResource extends BaseResource {
 
         // Get the document
         DocumentDao documentDao = new DocumentDao();
+        FileDao fileDao = new FileDao();
+        UserDao userDao = new UserDao();
         Document document;
+        User user;
         try {
             document = documentDao.getDocument(documentId, principal.getId());
+            user = userDao.getById(principal.getId());
         } catch (NoResultException e) {
             throw new ClientException("DocumentNotFound", MessageFormat.format("Document not found: {0}", documentId));
         }
         
-        FileDao fileDao = new FileDao();
+        // Keep unencrypted data in memory, because we will need it two times
+        byte[] fileData;
+        try {
+            fileData = ByteStreams.toByteArray(fileBodyPart.getValueAs(InputStream.class));
+        } catch (IOException e) {
+            throw new ServerException("StreamError", "Error reading the input file", e);
+        }
+        InputStream fileInputStream = new ByteArrayInputStream(fileData);
         
         // Validate mime type
-        InputStream is = new BufferedInputStream(fileBodyPart.getValueAs(InputStream.class));
         String mimeType;
         try {
-            mimeType = MimeTypeUtil.guessMimeType(is);
+            mimeType = MimeTypeUtil.guessMimeType(fileInputStream);
         } catch (Exception e) {
             throw new ServerException("ErrorGuessMime", "Error guessing mime type", e);
         }
@@ -113,12 +132,13 @@ public class FileResource extends BaseResource {
             String fileId = fileDao.create(file);
             
             // Save the file
-            FileUtil.save(is, file);
+            FileUtil.save(fileInputStream, file, user.getPrivateKey());
             
             // Raise a new file created event
             FileCreatedAsyncEvent fileCreatedAsyncEvent = new FileCreatedAsyncEvent();
             fileCreatedAsyncEvent.setDocument(document);
             fileCreatedAsyncEvent.setFile(file);
+            fileCreatedAsyncEvent.setInputStream(fileInputStream);
             AppContext.getInstance().getAsyncEventBus().post(fileCreatedAsyncEvent);
 
             // Always return ok
@@ -288,10 +308,12 @@ public class FileResource extends BaseResource {
         // Get the file
         FileDao fileDao = new FileDao();
         DocumentDao documentDao = new DocumentDao();
+        UserDao userDao = new UserDao();
         File file;
+        Document document;
         try {
             file = fileDao.getFile(fileId);
-            Document document = documentDao.getDocument(file.getDocumentId());
+            document = documentDao.getDocument(file.getDocumentId());
             
             // Check document visibility
             ShareDao shareDao = new ShareDao();
@@ -304,12 +326,13 @@ public class FileResource extends BaseResource {
 
         
         // Get the stored file
-        // TODO Decrypt file
         java.io.File storedfile;
         String mimeType;
+        boolean decrypt = false;
         if (size != null) {
             storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), fileId + "_" + size).toFile();
             mimeType = MimeType.IMAGE_JPEG; // Thumbnails are JPEG
+            decrypt = true; // Thumbnails are encrypted
             if (!storedfile.exists()) {
                 storedfile = new java.io.File(getClass().getResource("/image/file.png").getFile());
                 mimeType = MimeType.IMAGE_PNG;
@@ -317,11 +340,35 @@ public class FileResource extends BaseResource {
         } else {
             storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), fileId).toFile();
             mimeType = file.getMimeType();
+            decrypt = true; // Original files are encrypted
+        }
+        
+        // Stream the output and decrypt it if necessary
+        StreamingOutput stream;
+        User user = userDao.getById(document.getUserId());
+        try {
+            InputStream fileInputStream = new FileInputStream(storedfile);
+            final InputStream responseInputStream = decrypt ?
+                    EncryptionUtil.decryptInputStream(fileInputStream, user.getPrivateKey()) : fileInputStream;
+                    
+            stream = new StreamingOutput() {
+                @Override
+                public void write(OutputStream outputStream) throws IOException, WebApplicationException {
+                    try {
+                        ByteStreams.copy(responseInputStream, outputStream);
+                    } finally {
+                        responseInputStream.close();
+                        outputStream.close();
+                    }
+                }
+            };
+        } catch (Exception e) {
+            throw new ServerException("FileError", "Error while reading the file", e);
         }
 
-        return Response.ok(storedfile)
+        return Response.ok(stream)
                 .header("Content-Type", mimeType)
-                .header("Expires", new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z").format(new Date().getTime() + 3600000 * 24 * 7))
+                .header("Expires", new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z").format(new Date().getTime() + 3600000 * 24))
                 .build();
     }
 }

docs-web/src/main/webapp/WEB-INF/web.xml

@@ -3,9 +3,11 @@
   xmlns="http://java.sun.com/xml/ns/javaee"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
-  version="3.0"> 
+  version="3.0" metadata-complete="true"> 
   <display-name>Docs</display-name>
-
+  
+  <absolute-ordering></absolute-ordering>
+  
   <!-- This filter is used to secure URLs -->
   <filter>
     <filter-name>requestContextFilter</filter-name>

docs-web/src/main/webapp/partial/docs/document.edit.html

@@ -1,6 +1,6 @@
-<img src="img/loader.gif" ng-if="!document && isEdit()" />
+<img src="img/loader.gif" ng-show="!document && isEdit()" />
 
-<div ng-if="document || !isEdit()">
+<div ng-show="document || !isEdit()">
   <form class="form-horizontal" name="documentForm">
     <div class="control-group" ng-class="{ error: !documentForm.title.$valid }">
       <label class="control-label" for="inputTitle">Title</label>

docs-web/src/main/webapp/partial/docs/document.view.html

@@ -1,6 +1,6 @@
-<img src="img/loader.gif" ng-if="!document" />
+<img src="img/loader.gif" ng-show="!document" />
 
-<div ng-if="document">
+<div ng-show="document">
 	<div class="text-right">
 	  <div class="btn-group">
 	    <button class="btn btn-danger" ng-click="deleteDocument(document)"><span class="icon-trash icon-white"></span> Delete</button>
@@ -9,7 +9,7 @@
 	</div>
 	
 	<div class="page-header">
-	  <h1>{{ document.title }} <small>{{ document.create_date | date: 'yyyy-MM-dd' }}</small> <img ng-src="img/flag/{{ document.language }}.png" title="{{ document.language }}" /></h1>
+	  <h1>{{ document.title }} <small>{{ document.create_date | date: 'yyyy-MM-dd' }}</small> <img ng-if="document" ng-src="img/flag/{{ document.language }}.png" title="{{ document.language }}" /></h1>
 	  <p>
 	    <button class="btn btn-small btn-inverse" ng-click="share()"><span class="icon-share icon-white"></span> Share</button>
 	    <button class="btn btn-small" ng-repeat="share in document.shares" ng-click="showShare(share)"><span class="icon-ok"></span> {{ share.name ? share.name : 'shared' }}</button>

docs-web/src/main/webapp/partial/docs/settings.user.edit.html

@@ -1,6 +1,6 @@
-<img src="img/loader.gif" ng-if="!user && isEdit()" />
+<img src="img/loader.gif" ng-show="!user && isEdit()" />
 
-<div ng-if="user || !isEdit()">
+<div ng-show="user || !isEdit()">
   <h2 ng-show="isEdit()">Edit
     <small>"{{ user.username }}"</small>
   </h2>

docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java

@@ -43,13 +43,6 @@ public class TestAppResource extends BaseJerseyTest {
         Assert.assertTrue(totalMemory > 0 && totalMemory > freeMemory);
         Assert.assertEquals(0, json.getInt("document_count"));
         
-        // OCR-ize all files
-        appResource = resource().path("/app/batch/extract");
-        appResource.addFilter(new CookieAuthenticationFilter(adminAuthenticationToken));
-        response = appResource.post(ClientResponse.class);
-        Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
-        json = response.getEntity(JSONObject.class);
-        
         // Rebuild Lucene index
         appResource = resource().path("/app/batch/reindex");
         appResource.addFilter(new CookieAuthenticationFilter(adminAuthenticationToken));

docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java

@@ -1,6 +1,7 @@
 package com.sismics.docs.rest;
 
 import java.io.BufferedInputStream;
+import java.io.FileInputStream;
 import java.io.InputStream;
 import java.nio.file.Paths;
 
@@ -15,6 +16,8 @@ import org.junit.Test;
 import com.google.common.io.ByteStreams;
 import com.sismics.docs.core.util.DirectoryUtil;
 import com.sismics.docs.rest.filter.CookieAuthenticationFilter;
+import com.sismics.util.mime.MimeType;
+import com.sismics.util.mime.MimeTypeUtil;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.ClientResponse.Status;
 import com.sun.jersey.api.client.WebResource;
@@ -88,6 +91,7 @@ public class TestFileResource extends BaseJerseyTest {
         Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
         InputStream is = response.getEntityInputStream();
         byte[] fileBytes = ByteStreams.toByteArray(is);
+        Assert.assertEquals(MimeType.IMAGE_JPEG, MimeTypeUtil.guessMimeType(fileBytes));
         Assert.assertEquals(163510, fileBytes.length);
         
         // Get the thumbnail data
@@ -99,6 +103,7 @@ public class TestFileResource extends BaseJerseyTest {
         Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
         is = response.getEntityInputStream();
         fileBytes = ByteStreams.toByteArray(is);
+        Assert.assertEquals(MimeType.IMAGE_JPEG, MimeTypeUtil.guessMimeType(fileBytes));
         Assert.assertEquals(41935, fileBytes.length);
         
         // Get the web data
@@ -110,45 +115,14 @@ public class TestFileResource extends BaseJerseyTest {
         Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
         is = response.getEntityInputStream();
         fileBytes = ByteStreams.toByteArray(is);
+        Assert.assertEquals(MimeType.IMAGE_JPEG, MimeTypeUtil.guessMimeType(fileBytes));
         Assert.assertEquals(551084, fileBytes.length);
         
-        // Regenerate file variations
+        // Check that the files are not readable directly from FS
-        String adminAuthenticationToken = clientUtil.login("admin", "admin", false);
+        java.io.File storedFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file1Id).toFile();
-        WebResource appResource = resource().path("/app/batch/file_variations");
+        InputStream storedFileInputStream = new BufferedInputStream(new FileInputStream(storedFile));
-        appResource.addFilter(new CookieAuthenticationFilter(adminAuthenticationToken));
+        Assert.assertNull(MimeTypeUtil.guessMimeType(storedFileInputStream));
-        response = appResource.post(ClientResponse.class);
+        storedFileInputStream.close();
-        Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
-        
-        // Get the file data
-        fileResource = resource().path("/file/" + file1Id + "/data");
-        fileResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken));
-        response = fileResource.get(ClientResponse.class);
-        Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
-        is = response.getEntityInputStream();
-        fileBytes = ByteStreams.toByteArray(is);
-        Assert.assertEquals(163510, fileBytes.length);
-        
-        // Get the thumbnail data
-        fileResource = resource().path("/file/" + file1Id + "/data");
-        fileResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken));
-        getParams = new MultivaluedMapImpl();
-        getParams.putSingle("size", "thumb");
-        response = fileResource.queryParams(getParams).get(ClientResponse.class);
-        Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
-        is = response.getEntityInputStream();
-        fileBytes = ByteStreams.toByteArray(is);
-        Assert.assertEquals(41935, fileBytes.length);
-        
-        // Get the web data
-        fileResource = resource().path("/file/" + file1Id + "/data");
-        fileResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken));
-        getParams = new MultivaluedMapImpl();
-        getParams.putSingle("size", "web");
-        response = fileResource.queryParams(getParams).get(ClientResponse.class);
-        Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
-        is = response.getEntityInputStream();
-        fileBytes = ByteStreams.toByteArray(is);
-        Assert.assertEquals(551084, fileBytes.length);
         
         // Get all files from a document
         fileResource = resource().path("/file/list");
@@ -195,7 +169,7 @@ public class TestFileResource extends BaseJerseyTest {
         Assert.assertEquals("ok", json.getString("status"));
         
         // Check that files are deleted from FS
-        java.io.File storedFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file1Id).toFile();
+        storedFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file1Id).toFile();
         java.io.File webFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file1Id + "_web").toFile();
         java.io.File thumbnailFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file1Id + "_thumb").toFile();
         Assert.assertFalse(storedFile.exists());