From 2a619849f4fd74ef5d732b055877c23a457db948 Mon Sep 17 00:00:00 2001 From: Benjamin Gamard Date: Wed, 14 Mar 2018 15:13:09 +0100 Subject: [PATCH] #182: thumbnail generation asynchronous --- .../core/event/FileCreatedAsyncEvent.java | 16 ----- .../async/FileCreatedAsyncListener.java | 59 ++++++++++++++++--- .../com/sismics/docs/core/util/FileUtil.java | 38 ++---------- .../docs/rest/resource/FileResource.java | 2 - 4 files changed, 56 insertions(+), 59 deletions(-) diff --git a/docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java b/docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java index 5dca827e..d83b5de1 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java +++ b/docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java @@ -26,13 +26,6 @@ public class FileCreatedAsyncEvent extends UserEvent { */ private Path unencryptedFile; - /** - * Unencrypted file containing PDF representation - * of the original file. May be null if the PDF conversion is not - * necessary or not possible. - */ - private Path unencryptedPdfFile; - public File getFile() { return file; } @@ -58,15 +51,6 @@ public class FileCreatedAsyncEvent extends UserEvent { return this; } - public Path getUnencryptedPdfFile() { - return unencryptedPdfFile; - } - - public FileCreatedAsyncEvent setUnencryptedPdfFile(Path unencryptedPdfFile) { - this.unencryptedPdfFile = unencryptedPdfFile; - return this; - } - @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java b/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java index 59250bf4..330f117e 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java +++ b/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java @@ -2,15 +2,23 @@ package com.sismics.docs.core.listener.async; import com.google.common.eventbus.Subscribe; import com.sismics.docs.core.dao.jpa.FileDao; +import com.sismics.docs.core.dao.jpa.UserDao; import com.sismics.docs.core.dao.lucene.LuceneDao; import com.sismics.docs.core.event.FileCreatedAsyncEvent; import com.sismics.docs.core.model.jpa.File; +import com.sismics.docs.core.model.jpa.User; +import com.sismics.docs.core.util.EncryptionUtil; import com.sismics.docs.core.util.FileUtil; +import com.sismics.docs.core.util.PdfUtil; import com.sismics.docs.core.util.TransactionUtil; +import com.sismics.util.mime.MimeTypeUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.crypto.Cipher; +import java.nio.file.Path; import java.text.MessageFormat; +import java.util.concurrent.atomic.AtomicReference; /** * Listener on file created. @@ -26,22 +34,55 @@ public class FileCreatedAsyncListener { /** * File created. * - * @param fileCreatedAsyncEvent File created event + * @param event File created event */ @Subscribe - public void on(final FileCreatedAsyncEvent fileCreatedAsyncEvent) { + public void on(final FileCreatedAsyncEvent event) { if (log.isInfoEnabled()) { - log.info("File created event: " + fileCreatedAsyncEvent.toString()); + log.info("File created event: " + event.toString()); + } + + // Guess the mime type a second time, for open document format (first detected as simple ZIP file) + final File file = event.getFile(); + file.setMimeType(MimeTypeUtil.guessOpenDocumentFormat(file, event.getUnencryptedFile())); + + // Convert to PDF if necessary (for thumbnail and text extraction) + Path unencryptedPdfFile = null; + try { + unencryptedPdfFile = PdfUtil.convertToPdf(file, event.getUnencryptedFile()); + } catch (Exception e) { + log.error("Unable to convert to PDF", e); + } + + // Get the user from the database + final AtomicReference user = new AtomicReference<>(); + TransactionUtil.handle(new Runnable() { + @Override + public void run() { + UserDao userDao = new UserDao(); + user.set(userDao.getById(event.getUserId())); + } + }); + if (user.get() == null) { + // The user has been deleted meanwhile + return; + } + + // Generate file variations + try { + Cipher cipher = EncryptionUtil.getEncryptionCipher(user.get().getPrivateKey()); + FileUtil.saveVariations(file, event.getUnencryptedFile(), unencryptedPdfFile, cipher); + } catch (Exception e) { + log.error("Unable to generate thumbnails", e); } // Extract text content from the file - final File file = fileCreatedAsyncEvent.getFile(); long startTime = System.currentTimeMillis(); - final String content = FileUtil.extractContent(fileCreatedAsyncEvent.getLanguage(), file, - fileCreatedAsyncEvent.getUnencryptedFile(), fileCreatedAsyncEvent.getUnencryptedPdfFile()); + final String content = FileUtil.extractContent(event.getLanguage(), file, + event.getUnencryptedFile(), unencryptedPdfFile); log.info(MessageFormat.format("File content extracted in {0}ms", System.currentTimeMillis() - startTime)); - - // Store the text content in the database + + // Save the file to database TransactionUtil.handle(new Runnable() { @Override public void run() { @@ -58,7 +99,7 @@ public class FileCreatedAsyncListener { // Update Lucene index LuceneDao luceneDao = new LuceneDao(); - luceneDao.createFile(fileCreatedAsyncEvent.getFile()); + luceneDao.createFile(event.getFile()); FileUtil.endProcessingFile(file.getId()); } diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java index 9002ecbc..3c77bede 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java +++ b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java @@ -121,29 +121,6 @@ public class FileUtil { return ocrFile(image, language); } - /** - * Save a file on the storage filesystem. - * - * @param unencryptedFile Unencrypted file - * @param unencryptedPdfFile Unencrypted PDF file - * @param file File to save - * @param privateKey Private key used for encryption - */ - public static void save(Path unencryptedFile, Path unencryptedPdfFile, File file, String privateKey) throws Exception { - Cipher cipher = EncryptionUtil.getEncryptionCipher(privateKey); - Path path = DirectoryUtil.getStorageDirectory().resolve(file.getId()); - try (InputStream inputStream = Files.newInputStream(unencryptedFile)) { - Files.copy(new CipherInputStream(inputStream, cipher), path); - } - - // Generate file variations (errors non-blocking) - try { - saveVariations(file, unencryptedFile, unencryptedPdfFile, cipher); - } catch (Exception e) { - log.error("Unable to generate thumbnails", e); - } - } - /** * Generate file variations. * @@ -152,7 +129,7 @@ public class FileUtil { * @param unencryptedPdfFile Unencrypted PDF file * @param cipher Cipher to use for encryption */ - private static void saveVariations(File file, Path unencryptedFile, Path unencryptedPdfFile, Cipher cipher) throws Exception { + public static void saveVariations(File file, Path unencryptedFile, Path unencryptedPdfFile, Cipher cipher) throws Exception { BufferedImage image = null; if (ImageUtil.isImage(file.getMimeType())) { try (InputStream inputStream = Files.newInputStream(unencryptedFile)) { @@ -262,14 +239,12 @@ public class FileUtil { file.setUserId(userId); String fileId = fileDao.create(file, userId); - // Guess the mime type a second time, for open document format (first detected as simple ZIP file) - file.setMimeType(MimeTypeUtil.guessOpenDocumentFormat(file, unencryptedFile)); - - // Convert to PDF if necessary (for thumbnail and text extraction) - java.nio.file.Path unencryptedPdfFile = PdfUtil.convertToPdf(file, unencryptedFile); - // Save the file - FileUtil.save(unencryptedFile, unencryptedPdfFile, file, user.getPrivateKey()); + Cipher cipher = EncryptionUtil.getEncryptionCipher(user.getPrivateKey()); + Path path = DirectoryUtil.getStorageDirectory().resolve(file.getId()); + try (InputStream inputStream = Files.newInputStream(unencryptedFile)) { + Files.copy(new CipherInputStream(inputStream, cipher), path); + } // Update the user quota user.setStorageCurrent(user.getStorageCurrent() + fileSize); @@ -283,7 +258,6 @@ public class FileUtil { fileCreatedAsyncEvent.setLanguage(language); fileCreatedAsyncEvent.setFile(file); fileCreatedAsyncEvent.setUnencryptedFile(unencryptedFile); - fileCreatedAsyncEvent.setUnencryptedPdfFile(unencryptedPdfFile); ThreadLocalContext.get().addAsyncEvent(fileCreatedAsyncEvent); DocumentUpdatedAsyncEvent documentUpdatedAsyncEvent = new DocumentUpdatedAsyncEvent(); diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java index 21e04536..85d24074 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java @@ -17,7 +17,6 @@ import com.sismics.docs.core.model.jpa.User; import com.sismics.docs.core.util.DirectoryUtil; import com.sismics.docs.core.util.EncryptionUtil; import com.sismics.docs.core.util.FileUtil; -import com.sismics.docs.core.util.PdfUtil; import com.sismics.rest.exception.ClientException; import com.sismics.rest.exception.ForbiddenClientException; import com.sismics.rest.exception.ServerException; @@ -202,7 +201,6 @@ public class FileResource extends BaseResource { fileCreatedAsyncEvent.setLanguage(documentDto.getLanguage()); fileCreatedAsyncEvent.setFile(file); fileCreatedAsyncEvent.setUnencryptedFile(unencryptedFile); - fileCreatedAsyncEvent.setUnencryptedPdfFile(PdfUtil.convertToPdf(file, unencryptedFile)); ThreadLocalContext.get().addAsyncEvent(fileCreatedAsyncEvent); DocumentUpdatedAsyncEvent documentUpdatedAsyncEvent = new DocumentUpdatedAsyncEvent();