vmario/docs
1
0
Fork 0
mirror of https://github.com/sismics/docs.git synced 2024-06-26 11:31:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

#412: process files outside of a transaction

Browse Source
This commit is contained in:
bgamard 2020-05-19 14:44:41 +02:00
parent 9b2aeb7480
commit d428e89c30
2 changed files with 60 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
docs-core/src/main/java/com/sismics/docs/core/listener/async/FileProcessingAsyncListener.java
View File

@ -28,6 +28,7 @@ import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.MessageFormat;
import java.util.concurrent.atomic.AtomicReference;
/**
* Listener on file processing.
@ -52,20 +53,7 @@ public class FileProcessingAsyncListener {
log.info("File created event: " + event.toString());
}
TransactionUtil.handle(() -> {
// Generate thumbnail, extract content
File file = new FileDao().getActiveById(event.getFileId());
if (file == null) {
// The file has been deleted since
return;
}
processFile(event, file);
// Update index with the file updated by side effect
AppContext.getInstance().getIndexingHandler().createFile(file);
});
FileUtil.endProcessingFile(event.getFileId());
processFile(event, true);
}
/**
@ -78,54 +66,84 @@ public class FileProcessingAsyncListener {
public void on(final FileUpdatedAsyncEvent event) {
log.info("File updated event: " + event.toString());
TransactionUtil.handle(() -> {
log.info("File processing phase 0: " + event.getFileId());
processFile(event, false);
}
/**
* Process a file :
* Generate thumbnails
* Extract and save text content
*
* @param event File event
* @param isFileCreated True if the file was just created
*/
private void processFile(FileEvent event, boolean isFileCreated) {
AtomicReference<File> file = new AtomicReference<>();
AtomicReference<User> user = new AtomicReference<>();
// Open a first transaction to get what we need to start the processing
TransactionUtil.handle(() -> {
// Generate thumbnail, extract content
File file = new FileDao().getActiveById(event.getFileId());
if (file == null) {
file.set(new FileDao().getActiveById(event.getFileId()));
if (file.get() == null) {
// The file has been deleted since
return;
}
processFile(event, file);
// Update index with the file updated by side effect
AppContext.getInstance().getIndexingHandler().updateFile(file);
// Get the creating user from the database for its private key
UserDao userDao = new UserDao();
user.set(userDao.getById(file.get().getUserId()));
});
log.info("File processing phase 6: " + file.getId());
// Process the file outside of a transaction
if (user.get() == null || file.get() == null) {
// The user or file has been deleted
FileUtil.endProcessingFile(event.getFileId());
return;
}
String content = extractContent(event, user.get(), file.get());
// Open a new transaction to save the file content
TransactionUtil.handle(() -> {
// Save the file to database
FileDao fileDao = new FileDao();
File freshFile = fileDao.getActiveById(event.getFileId());
if (freshFile == null) {
// The file has been deleted since the text extraction started, ignore the result
return;
}
freshFile.setContent(content);
fileDao.updateContent(freshFile);
// Update index with the updated file
if (isFileCreated) {
AppContext.getInstance().getIndexingHandler().createFile(freshFile);
} else {
AppContext.getInstance().getIndexingHandler().updateFile(freshFile);
}
});
FileUtil.endProcessingFile(event.getFileId());
}
/**
* Process the file (create/update).
* Extract text content from a file.
* This is executed outside of a transaction.
*
* @param event File event
* @param user User whom created the file
* @param file Fresh file
* @return Text content
*/
private void processFile(FileEvent event, File file) {
log.info("File processing phase 1: " + file.getId());
private String extractContent(FileEvent event, User user, File file) {
// Find a format handler
FormatHandler formatHandler = FormatHandlerUtil.find(file.getMimeType());
if (formatHandler == null) {
log.info("Format unhandled: " + file.getMimeType());
return;
return null;
}
log.info("File processing phase 2: " + file.getId());
// Get the creating user from the database for its private key
UserDao userDao = new UserDao();
User user = userDao.getById(file.getUserId());
if (user == null) {
// The user has been deleted meanwhile
return;
}
log.info("File processing phase 3: " + file.getId());
// Generate file variations
try {
Cipher cipher = EncryptionUtil.getEncryptionCipher(user.getPrivateKey());
@ -152,11 +170,10 @@ public class FileProcessingAsyncListener {
log.error("Unable to generate thumbnails for: " + file, e);
}
log.info("File processing phase 4: " + file.getId());
// Extract text content from the file
long startTime = System.currentTimeMillis();
String content = null;
log.error("Start extracting content from: " + file);
try {
content = formatHandler.extractContent(event.getLanguage(), event.getUnencryptedFile());
} catch (Exception e) {
@ -164,16 +181,6 @@ public class FileProcessingAsyncListener {
}
log.info(MessageFormat.format("File content extracted in {0}ms: " + file.getId(), System.currentTimeMillis() - startTime));
// Save the file to database
FileDao fileDao = new FileDao();
if (fileDao.getActiveById(file.getId()) == null) {
// The file has been deleted since the text extraction started, ignore the result
return;
}
file.setContent(content);
fileDao.updateContent(file);
log.info("File processing phase 5: " + file.getId());
return content;
}
}

4
docs-core/src/main/java/com/sismics/util/io/InputStreamReaderThread.java
View File

@ -34,9 +34,7 @@ public class InputStreamReaderThread extends Thread {
try {
BufferedReader reader = closer.register(new BufferedReader(new InputStreamReader(is)));
for (String line = reader.readLine(); line != null; line = reader.readLine()) {
if (logger.isDebugEnabled()) {
logger.debug(String.format(name + ": %s", line));
}
logger.error(String.format(name + ": %s", line));
}
} catch (IOException x) {
// NOP