#412: process files outside of a transaction

docs-core/src/main/java/com/sismics/docs/core/listener/async/FileProcessingAsyncListener.java

@@ -28,6 +28,7 @@ import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.text.MessageFormat;
+import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * Listener on file processing.
@@ -52,20 +53,7 @@ public class FileProcessingAsyncListener {
             log.info("File created event: " + event.toString());
         }
 
-        TransactionUtil.handle(() -> {
+        processFile(event, true);
-            // Generate thumbnail, extract content
-            File file = new FileDao().getActiveById(event.getFileId());
-            if (file == null) {
-                // The file has been deleted since
-                return;
-            }
-            processFile(event, file);
-
-            // Update index with the file updated by side effect
-            AppContext.getInstance().getIndexingHandler().createFile(file);
-        });
-
-        FileUtil.endProcessingFile(event.getFileId());
     }
 
     /**
@@ -78,54 +66,84 @@ public class FileProcessingAsyncListener {
     public void on(final FileUpdatedAsyncEvent event) {
         log.info("File updated event: " + event.toString());
 
-        TransactionUtil.handle(() -> {
+        processFile(event, false);
-            log.info("File processing phase 0: " + event.getFileId());
+    }
 
+    /**
+     * Process a file :
+     * Generate thumbnails
+     * Extract and save text content
+     *
+     * @param event File event
+     * @param isFileCreated True if the file was just created
+     */
+    private void processFile(FileEvent event, boolean isFileCreated) {
+        AtomicReference<File> file = new AtomicReference<>();
+        AtomicReference<User> user = new AtomicReference<>();
+
+        // Open a first transaction to get what we need to start the processing
+        TransactionUtil.handle(() -> {
             // Generate thumbnail, extract content
-            File file = new FileDao().getActiveById(event.getFileId());
+            file.set(new FileDao().getActiveById(event.getFileId()));
-            if (file == null) {
+            if (file.get() == null) {
                 // The file has been deleted since
                 return;
             }
-            processFile(event, file);
 
-            // Update index with the file updated by side effect
+            // Get the creating user from the database for its private key
-            AppContext.getInstance().getIndexingHandler().updateFile(file);
+            UserDao userDao = new UserDao();
+            user.set(userDao.getById(file.get().getUserId()));
+        });
 
-            log.info("File processing phase 6: " + file.getId());
+        // Process the file outside of a transaction
+        if (user.get() == null || file.get() == null) {
+            // The user or file has been deleted
+            FileUtil.endProcessingFile(event.getFileId());
+            return;
+        }
+        String content = extractContent(event, user.get(), file.get());
+
+        // Open a new transaction to save the file content
+        TransactionUtil.handle(() -> {
+            // Save the file to database
+            FileDao fileDao = new FileDao();
+            File freshFile = fileDao.getActiveById(event.getFileId());
+            if (freshFile == null) {
+                // The file has been deleted since the text extraction started, ignore the result
+                return;
+            }
+
+            freshFile.setContent(content);
+            fileDao.updateContent(freshFile);
+
+            // Update index with the updated file
+            if (isFileCreated) {
+                AppContext.getInstance().getIndexingHandler().createFile(freshFile);
+            } else {
+                AppContext.getInstance().getIndexingHandler().updateFile(freshFile);
+            }
         });
 
         FileUtil.endProcessingFile(event.getFileId());
     }
 
     /**
-     * Process the file (create/update).
+     * Extract text content from a file.
+     * This is executed outside of a transaction.
      *
      * @param event File event
+     * @param user User whom created the file
      * @param file Fresh file
+     * @return Text content
      */
-    private void processFile(FileEvent event, File file) {
+    private String extractContent(FileEvent event, User user, File file) {
-        log.info("File processing phase 1: " + file.getId());
-
         // Find a format handler
         FormatHandler formatHandler = FormatHandlerUtil.find(file.getMimeType());
         if (formatHandler == null) {
             log.info("Format unhandled: " + file.getMimeType());
-            return;
+            return null;
         }
 
-        log.info("File processing phase 2: " + file.getId());
-
-        // Get the creating user from the database for its private key
-        UserDao userDao = new UserDao();
-        User user = userDao.getById(file.getUserId());
-        if (user == null) {
-            // The user has been deleted meanwhile
-            return;
-        }
-
-        log.info("File processing phase 3: " + file.getId());
-
         // Generate file variations
         try {
             Cipher cipher = EncryptionUtil.getEncryptionCipher(user.getPrivateKey());
@@ -152,11 +170,10 @@ public class FileProcessingAsyncListener {
             log.error("Unable to generate thumbnails for: " + file, e);
         }
 
-        log.info("File processing phase 4: " + file.getId());
-
         // Extract text content from the file
         long startTime = System.currentTimeMillis();
         String content = null;
+        log.error("Start extracting content from: " + file);
         try {
             content = formatHandler.extractContent(event.getLanguage(), event.getUnencryptedFile());
         } catch (Exception e) {
@@ -164,16 +181,6 @@ public class FileProcessingAsyncListener {
         }
         log.info(MessageFormat.format("File content extracted in {0}ms: " + file.getId(), System.currentTimeMillis() - startTime));
 
-        // Save the file to database
+        return content;
-        FileDao fileDao = new FileDao();
-        if (fileDao.getActiveById(file.getId()) == null) {
-            // The file has been deleted since the text extraction started, ignore the result
-            return;
-        }
-
-        file.setContent(content);
-        fileDao.updateContent(file);
-
-        log.info("File processing phase 5: " + file.getId());
     }
 }

docs-core/src/main/java/com/sismics/util/io/InputStreamReaderThread.java

@@ -34,9 +34,7 @@ public class InputStreamReaderThread extends Thread {
         try {
             BufferedReader reader = closer.register(new BufferedReader(new InputStreamReader(is)));
             for (String line = reader.readLine(); line != null; line = reader.readLine()) {
-                if (logger.isDebugEnabled()) {
+                logger.error(String.format(name + ": %s", line));
-                    logger.debug(String.format(name + ": %s", line));
-                }
             }
         } catch (IOException x) {
             // NOP