vmario/docs
1
0
Fork 0
mirror of https://github.com/sismics/docs.git synced 2024-11-25 15:17:57 +01:00
Code Issues Packages Projects Releases Wiki Activity

#182: thumbnail generation asynchronous

Browse Source
This commit is contained in:
Benjamin Gamard 2018-03-14 15:13:09 +01:00
parent 1e57ee5fb3
commit 2a619849f4
4 changed files with 56 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java
View File

@ -26,13 +26,6 @@ public class FileCreatedAsyncEvent extends UserEvent {
*/
private Path unencryptedFile;
/**
* Unencrypted file containing PDF representation
* of the original file. May be null if the PDF conversion is not
* necessary or not possible.
*/
private Path unencryptedPdfFile;
public File getFile() {
return file;
}
@ -58,15 +51,6 @@ public class FileCreatedAsyncEvent extends UserEvent {
return this;
}
public Path getUnencryptedPdfFile() {
return unencryptedPdfFile;
}
public FileCreatedAsyncEvent setUnencryptedPdfFile(Path unencryptedPdfFile) {
this.unencryptedPdfFile = unencryptedPdfFile;
return this;
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this)

57
docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java
View File

@ -2,15 +2,23 @@ package com.sismics.docs.core.listener.async;
import com.google.common.eventbus.Subscribe;
import com.sismics.docs.core.dao.jpa.FileDao;
import com.sismics.docs.core.dao.jpa.UserDao;
import com.sismics.docs.core.dao.lucene.LuceneDao;
import com.sismics.docs.core.event.FileCreatedAsyncEvent;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.util.EncryptionUtil;
import com.sismics.docs.core.util.FileUtil;
import com.sismics.docs.core.util.PdfUtil;
import com.sismics.docs.core.util.TransactionUtil;
import com.sismics.util.mime.MimeTypeUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.crypto.Cipher;
import java.nio.file.Path;
import java.text.MessageFormat;
import java.util.concurrent.atomic.AtomicReference;
/**
* Listener on file created.
@ -26,22 +34,55 @@ public class FileCreatedAsyncListener {
/**
* File created.
*
* @param fileCreatedAsyncEvent File created event
* @param event File created event
*/
@Subscribe
public void on(final FileCreatedAsyncEvent fileCreatedAsyncEvent) {
public void on(final FileCreatedAsyncEvent event) {
if (log.isInfoEnabled()) {
log.info("File created event: " + fileCreatedAsyncEvent.toString());
log.info("File created event: " + event.toString());
}
// Guess the mime type a second time, for open document format (first detected as simple ZIP file)
final File file = event.getFile();
file.setMimeType(MimeTypeUtil.guessOpenDocumentFormat(file, event.getUnencryptedFile()));
// Convert to PDF if necessary (for thumbnail and text extraction)
Path unencryptedPdfFile = null;
try {
unencryptedPdfFile = PdfUtil.convertToPdf(file, event.getUnencryptedFile());
} catch (Exception e) {
log.error("Unable to convert to PDF", e);
}
// Get the user from the database
final AtomicReference<User> user = new AtomicReference<>();
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
UserDao userDao = new UserDao();
user.set(userDao.getById(event.getUserId()));
}
});
if (user.get() == null) {
// The user has been deleted meanwhile
return;
}
// Generate file variations
try {
Cipher cipher = EncryptionUtil.getEncryptionCipher(user.get().getPrivateKey());
FileUtil.saveVariations(file, event.getUnencryptedFile(), unencryptedPdfFile, cipher);
} catch (Exception e) {
log.error("Unable to generate thumbnails", e);
}
// Extract text content from the file
final File file = fileCreatedAsyncEvent.getFile();
long startTime = System.currentTimeMillis();
final String content = FileUtil.extractContent(fileCreatedAsyncEvent.getLanguage(), file,
fileCreatedAsyncEvent.getUnencryptedFile(), fileCreatedAsyncEvent.getUnencryptedPdfFile());
final String content = FileUtil.extractContent(event.getLanguage(), file,
event.getUnencryptedFile(), unencryptedPdfFile);
log.info(MessageFormat.format("File content extracted in {0}ms", System.currentTimeMillis() - startTime));
// Store the text content in the database
// Save the file to database
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
@ -58,7 +99,7 @@ public class FileCreatedAsyncListener {
// Update Lucene index
LuceneDao luceneDao = new LuceneDao();
luceneDao.createFile(fileCreatedAsyncEvent.getFile());
luceneDao.createFile(event.getFile());
FileUtil.endProcessingFile(file.getId());
}

38
docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java
View File

@ -121,29 +121,6 @@ public class FileUtil {
return ocrFile(image, language);
}
/**
* Save a file on the storage filesystem.
*
* @param unencryptedFile Unencrypted file
* @param unencryptedPdfFile Unencrypted PDF file
* @param file File to save
* @param privateKey Private key used for encryption
*/
public static void save(Path unencryptedFile, Path unencryptedPdfFile, File file, String privateKey) throws Exception {
Cipher cipher = EncryptionUtil.getEncryptionCipher(privateKey);
Path path = DirectoryUtil.getStorageDirectory().resolve(file.getId());
try (InputStream inputStream = Files.newInputStream(unencryptedFile)) {
Files.copy(new CipherInputStream(inputStream, cipher), path);
}
// Generate file variations (errors non-blocking)
try {
saveVariations(file, unencryptedFile, unencryptedPdfFile, cipher);
} catch (Exception e) {
log.error("Unable to generate thumbnails", e);
}
}
/**
* Generate file variations.
*
@ -152,7 +129,7 @@ public class FileUtil {
* @param unencryptedPdfFile Unencrypted PDF file
* @param cipher Cipher to use for encryption
*/
private static void saveVariations(File file, Path unencryptedFile, Path unencryptedPdfFile, Cipher cipher) throws Exception {
public static void saveVariations(File file, Path unencryptedFile, Path unencryptedPdfFile, Cipher cipher) throws Exception {
BufferedImage image = null;
if (ImageUtil.isImage(file.getMimeType())) {
try (InputStream inputStream = Files.newInputStream(unencryptedFile)) {
@ -262,14 +239,12 @@ public class FileUtil {
file.setUserId(userId);
String fileId = fileDao.create(file, userId);
// Guess the mime type a second time, for open document format (first detected as simple ZIP file)
file.setMimeType(MimeTypeUtil.guessOpenDocumentFormat(file, unencryptedFile));
// Convert to PDF if necessary (for thumbnail and text extraction)
java.nio.file.Path unencryptedPdfFile = PdfUtil.convertToPdf(file, unencryptedFile);
// Save the file
FileUtil.save(unencryptedFile, unencryptedPdfFile, file, user.getPrivateKey());
Cipher cipher = EncryptionUtil.getEncryptionCipher(user.getPrivateKey());
Path path = DirectoryUtil.getStorageDirectory().resolve(file.getId());
try (InputStream inputStream = Files.newInputStream(unencryptedFile)) {
Files.copy(new CipherInputStream(inputStream, cipher), path);
}
// Update the user quota
user.setStorageCurrent(user.getStorageCurrent() + fileSize);
@ -283,7 +258,6 @@ public class FileUtil {
fileCreatedAsyncEvent.setLanguage(language);
fileCreatedAsyncEvent.setFile(file);
fileCreatedAsyncEvent.setUnencryptedFile(unencryptedFile);
fileCreatedAsyncEvent.setUnencryptedPdfFile(unencryptedPdfFile);
ThreadLocalContext.get().addAsyncEvent(fileCreatedAsyncEvent);
DocumentUpdatedAsyncEvent documentUpdatedAsyncEvent = new DocumentUpdatedAsyncEvent();

2
docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java
View File

@ -17,7 +17,6 @@ import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.util.DirectoryUtil;
import com.sismics.docs.core.util.EncryptionUtil;
import com.sismics.docs.core.util.FileUtil;
import com.sismics.docs.core.util.PdfUtil;
import com.sismics.rest.exception.ClientException;
import com.sismics.rest.exception.ForbiddenClientException;
import com.sismics.rest.exception.ServerException;
@ -202,7 +201,6 @@ public class FileResource extends BaseResource {
fileCreatedAsyncEvent.setLanguage(documentDto.getLanguage());
fileCreatedAsyncEvent.setFile(file);
fileCreatedAsyncEvent.setUnencryptedFile(unencryptedFile);
fileCreatedAsyncEvent.setUnencryptedPdfFile(PdfUtil.convertToPdf(file, unencryptedFile));
ThreadLocalContext.get().addAsyncEvent(fileCreatedAsyncEvent);
DocumentUpdatedAsyncEvent documentUpdatedAsyncEvent = new DocumentUpdatedAsyncEvent();