From d428e89c305bc7f0605c94262a40fe680dddfda7 Mon Sep 17 00:00:00 2001 From: bgamard Date: Tue, 19 May 2020 14:44:41 +0200 Subject: [PATCH] #412: process files outside of a transaction --- .../async/FileProcessingAsyncListener.java | 111 ++++++++++-------- .../util/io/InputStreamReaderThread.java | 4 +- 2 files changed, 60 insertions(+), 55 deletions(-) diff --git a/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileProcessingAsyncListener.java b/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileProcessingAsyncListener.java index 780ee84b..bf18a9d9 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileProcessingAsyncListener.java +++ b/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileProcessingAsyncListener.java @@ -28,6 +28,7 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.text.MessageFormat; +import java.util.concurrent.atomic.AtomicReference; /** * Listener on file processing. @@ -52,20 +53,7 @@ public class FileProcessingAsyncListener { log.info("File created event: " + event.toString()); } - TransactionUtil.handle(() -> { - // Generate thumbnail, extract content - File file = new FileDao().getActiveById(event.getFileId()); - if (file == null) { - // The file has been deleted since - return; - } - processFile(event, file); - - // Update index with the file updated by side effect - AppContext.getInstance().getIndexingHandler().createFile(file); - }); - - FileUtil.endProcessingFile(event.getFileId()); + processFile(event, true); } /** @@ -78,54 +66,84 @@ public class FileProcessingAsyncListener { public void on(final FileUpdatedAsyncEvent event) { log.info("File updated event: " + event.toString()); - TransactionUtil.handle(() -> { - log.info("File processing phase 0: " + event.getFileId()); + processFile(event, false); + } + /** + * Process a file : + * Generate thumbnails + * Extract and save text content + * + * @param event File event + * @param isFileCreated True if the file was just created + */ + private void processFile(FileEvent event, boolean isFileCreated) { + AtomicReference file = new AtomicReference<>(); + AtomicReference user = new AtomicReference<>(); + + // Open a first transaction to get what we need to start the processing + TransactionUtil.handle(() -> { // Generate thumbnail, extract content - File file = new FileDao().getActiveById(event.getFileId()); - if (file == null) { + file.set(new FileDao().getActiveById(event.getFileId())); + if (file.get() == null) { // The file has been deleted since return; } - processFile(event, file); - // Update index with the file updated by side effect - AppContext.getInstance().getIndexingHandler().updateFile(file); + // Get the creating user from the database for its private key + UserDao userDao = new UserDao(); + user.set(userDao.getById(file.get().getUserId())); + }); - log.info("File processing phase 6: " + file.getId()); + // Process the file outside of a transaction + if (user.get() == null || file.get() == null) { + // The user or file has been deleted + FileUtil.endProcessingFile(event.getFileId()); + return; + } + String content = extractContent(event, user.get(), file.get()); + + // Open a new transaction to save the file content + TransactionUtil.handle(() -> { + // Save the file to database + FileDao fileDao = new FileDao(); + File freshFile = fileDao.getActiveById(event.getFileId()); + if (freshFile == null) { + // The file has been deleted since the text extraction started, ignore the result + return; + } + + freshFile.setContent(content); + fileDao.updateContent(freshFile); + + // Update index with the updated file + if (isFileCreated) { + AppContext.getInstance().getIndexingHandler().createFile(freshFile); + } else { + AppContext.getInstance().getIndexingHandler().updateFile(freshFile); + } }); FileUtil.endProcessingFile(event.getFileId()); } /** - * Process the file (create/update). + * Extract text content from a file. + * This is executed outside of a transaction. * * @param event File event + * @param user User whom created the file * @param file Fresh file + * @return Text content */ - private void processFile(FileEvent event, File file) { - log.info("File processing phase 1: " + file.getId()); - + private String extractContent(FileEvent event, User user, File file) { // Find a format handler FormatHandler formatHandler = FormatHandlerUtil.find(file.getMimeType()); if (formatHandler == null) { log.info("Format unhandled: " + file.getMimeType()); - return; + return null; } - log.info("File processing phase 2: " + file.getId()); - - // Get the creating user from the database for its private key - UserDao userDao = new UserDao(); - User user = userDao.getById(file.getUserId()); - if (user == null) { - // The user has been deleted meanwhile - return; - } - - log.info("File processing phase 3: " + file.getId()); - // Generate file variations try { Cipher cipher = EncryptionUtil.getEncryptionCipher(user.getPrivateKey()); @@ -152,11 +170,10 @@ public class FileProcessingAsyncListener { log.error("Unable to generate thumbnails for: " + file, e); } - log.info("File processing phase 4: " + file.getId()); - // Extract text content from the file long startTime = System.currentTimeMillis(); String content = null; + log.error("Start extracting content from: " + file); try { content = formatHandler.extractContent(event.getLanguage(), event.getUnencryptedFile()); } catch (Exception e) { @@ -164,16 +181,6 @@ public class FileProcessingAsyncListener { } log.info(MessageFormat.format("File content extracted in {0}ms: " + file.getId(), System.currentTimeMillis() - startTime)); - // Save the file to database - FileDao fileDao = new FileDao(); - if (fileDao.getActiveById(file.getId()) == null) { - // The file has been deleted since the text extraction started, ignore the result - return; - } - - file.setContent(content); - fileDao.updateContent(file); - - log.info("File processing phase 5: " + file.getId()); + return content; } } diff --git a/docs-core/src/main/java/com/sismics/util/io/InputStreamReaderThread.java b/docs-core/src/main/java/com/sismics/util/io/InputStreamReaderThread.java index ff2da29d..8556fc88 100644 --- a/docs-core/src/main/java/com/sismics/util/io/InputStreamReaderThread.java +++ b/docs-core/src/main/java/com/sismics/util/io/InputStreamReaderThread.java @@ -34,9 +34,7 @@ public class InputStreamReaderThread extends Thread { try { BufferedReader reader = closer.register(new BufferedReader(new InputStreamReader(is))); for (String line = reader.readLine(); line != null; line = reader.readLine()) { - if (logger.isDebugEnabled()) { - logger.debug(String.format(name + ": %s", line)); - } + logger.error(String.format(name + ": %s", line)); } } catch (IOException x) { // NOP