#412: process files outside of a transaction

This commit is contained in:
bgamard 2020-05-19 14:44:41 +02:00
parent 9b2aeb7480
commit d428e89c30
2 changed files with 60 additions and 55 deletions

View File

@ -28,6 +28,7 @@ import java.io.OutputStream;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.concurrent.atomic.AtomicReference;
/** /**
* Listener on file processing. * Listener on file processing.
@ -52,20 +53,7 @@ public class FileProcessingAsyncListener {
log.info("File created event: " + event.toString()); log.info("File created event: " + event.toString());
} }
TransactionUtil.handle(() -> { processFile(event, true);
// Generate thumbnail, extract content
File file = new FileDao().getActiveById(event.getFileId());
if (file == null) {
// The file has been deleted since
return;
}
processFile(event, file);
// Update index with the file updated by side effect
AppContext.getInstance().getIndexingHandler().createFile(file);
});
FileUtil.endProcessingFile(event.getFileId());
} }
/** /**
@ -78,54 +66,84 @@ public class FileProcessingAsyncListener {
public void on(final FileUpdatedAsyncEvent event) { public void on(final FileUpdatedAsyncEvent event) {
log.info("File updated event: " + event.toString()); log.info("File updated event: " + event.toString());
TransactionUtil.handle(() -> { processFile(event, false);
log.info("File processing phase 0: " + event.getFileId()); }
/**
* Process a file :
* Generate thumbnails
* Extract and save text content
*
* @param event File event
* @param isFileCreated True if the file was just created
*/
private void processFile(FileEvent event, boolean isFileCreated) {
AtomicReference<File> file = new AtomicReference<>();
AtomicReference<User> user = new AtomicReference<>();
// Open a first transaction to get what we need to start the processing
TransactionUtil.handle(() -> {
// Generate thumbnail, extract content // Generate thumbnail, extract content
File file = new FileDao().getActiveById(event.getFileId()); file.set(new FileDao().getActiveById(event.getFileId()));
if (file == null) { if (file.get() == null) {
// The file has been deleted since // The file has been deleted since
return; return;
} }
processFile(event, file);
// Update index with the file updated by side effect // Get the creating user from the database for its private key
AppContext.getInstance().getIndexingHandler().updateFile(file); UserDao userDao = new UserDao();
user.set(userDao.getById(file.get().getUserId()));
});
log.info("File processing phase 6: " + file.getId()); // Process the file outside of a transaction
if (user.get() == null || file.get() == null) {
// The user or file has been deleted
FileUtil.endProcessingFile(event.getFileId());
return;
}
String content = extractContent(event, user.get(), file.get());
// Open a new transaction to save the file content
TransactionUtil.handle(() -> {
// Save the file to database
FileDao fileDao = new FileDao();
File freshFile = fileDao.getActiveById(event.getFileId());
if (freshFile == null) {
// The file has been deleted since the text extraction started, ignore the result
return;
}
freshFile.setContent(content);
fileDao.updateContent(freshFile);
// Update index with the updated file
if (isFileCreated) {
AppContext.getInstance().getIndexingHandler().createFile(freshFile);
} else {
AppContext.getInstance().getIndexingHandler().updateFile(freshFile);
}
}); });
FileUtil.endProcessingFile(event.getFileId()); FileUtil.endProcessingFile(event.getFileId());
} }
/** /**
* Process the file (create/update). * Extract text content from a file.
* This is executed outside of a transaction.
* *
* @param event File event * @param event File event
* @param user User whom created the file
* @param file Fresh file * @param file Fresh file
* @return Text content
*/ */
private void processFile(FileEvent event, File file) { private String extractContent(FileEvent event, User user, File file) {
log.info("File processing phase 1: " + file.getId());
// Find a format handler // Find a format handler
FormatHandler formatHandler = FormatHandlerUtil.find(file.getMimeType()); FormatHandler formatHandler = FormatHandlerUtil.find(file.getMimeType());
if (formatHandler == null) { if (formatHandler == null) {
log.info("Format unhandled: " + file.getMimeType()); log.info("Format unhandled: " + file.getMimeType());
return; return null;
} }
log.info("File processing phase 2: " + file.getId());
// Get the creating user from the database for its private key
UserDao userDao = new UserDao();
User user = userDao.getById(file.getUserId());
if (user == null) {
// The user has been deleted meanwhile
return;
}
log.info("File processing phase 3: " + file.getId());
// Generate file variations // Generate file variations
try { try {
Cipher cipher = EncryptionUtil.getEncryptionCipher(user.getPrivateKey()); Cipher cipher = EncryptionUtil.getEncryptionCipher(user.getPrivateKey());
@ -152,11 +170,10 @@ public class FileProcessingAsyncListener {
log.error("Unable to generate thumbnails for: " + file, e); log.error("Unable to generate thumbnails for: " + file, e);
} }
log.info("File processing phase 4: " + file.getId());
// Extract text content from the file // Extract text content from the file
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
String content = null; String content = null;
log.error("Start extracting content from: " + file);
try { try {
content = formatHandler.extractContent(event.getLanguage(), event.getUnencryptedFile()); content = formatHandler.extractContent(event.getLanguage(), event.getUnencryptedFile());
} catch (Exception e) { } catch (Exception e) {
@ -164,16 +181,6 @@ public class FileProcessingAsyncListener {
} }
log.info(MessageFormat.format("File content extracted in {0}ms: " + file.getId(), System.currentTimeMillis() - startTime)); log.info(MessageFormat.format("File content extracted in {0}ms: " + file.getId(), System.currentTimeMillis() - startTime));
// Save the file to database return content;
FileDao fileDao = new FileDao();
if (fileDao.getActiveById(file.getId()) == null) {
// The file has been deleted since the text extraction started, ignore the result
return;
}
file.setContent(content);
fileDao.updateContent(file);
log.info("File processing phase 5: " + file.getId());
} }
} }

View File

@ -34,9 +34,7 @@ public class InputStreamReaderThread extends Thread {
try { try {
BufferedReader reader = closer.register(new BufferedReader(new InputStreamReader(is))); BufferedReader reader = closer.register(new BufferedReader(new InputStreamReader(is)));
for (String line = reader.readLine(); line != null; line = reader.readLine()) { for (String line = reader.readLine(); line != null; line = reader.readLine()) {
if (logger.isDebugEnabled()) { logger.error(String.format(name + ": %s", line));
logger.debug(String.format(name + ": %s", line));
}
} }
} catch (IOException x) { } catch (IOException x) {
// NOP // NOP