From d0335b6b161058250ec8cc44eeb2357f96176f54 Mon Sep 17 00:00:00 2001 From: Benjamin Gamard Date: Mon, 9 Apr 2018 13:02:39 +0200 Subject: [PATCH] Closes #206: action to process all files --- .../docs/core/constant/ActionType.java | 7 +- .../sismics/docs/core/util/ActionUtil.java | 4 ++ .../com/sismics/docs/core/util/FileUtil.java | 2 +- .../core/util/action/ProcessFilesAction.java | 64 ++++++++++++++++++ .../docs/rest/resource/AppResource.java | 65 +------------------ .../docs/rest/resource/FileResource.java | 20 +++--- docs-web/src/main/webapp/src/locale/en.json | 3 +- .../partial/docs/settings.workflow.edit.html | 1 + 8 files changed, 90 insertions(+), 76 deletions(-) create mode 100644 docs-core/src/main/java/com/sismics/docs/core/util/action/ProcessFilesAction.java diff --git a/docs-core/src/main/java/com/sismics/docs/core/constant/ActionType.java b/docs-core/src/main/java/com/sismics/docs/core/constant/ActionType.java index c58222f5..65bd5470 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/constant/ActionType.java +++ b/docs-core/src/main/java/com/sismics/docs/core/constant/ActionType.java @@ -14,5 +14,10 @@ public enum ActionType { /** * Remove a tag. */ - REMOVE_TAG + REMOVE_TAG, + + /** + * Process files. + */ + PROCESS_FILES } diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/ActionUtil.java b/docs-core/src/main/java/com/sismics/docs/core/util/ActionUtil.java index f365fcfc..9d00b4b7 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/util/ActionUtil.java +++ b/docs-core/src/main/java/com/sismics/docs/core/util/ActionUtil.java @@ -4,6 +4,7 @@ import com.sismics.docs.core.constant.ActionType; import com.sismics.docs.core.dao.dto.DocumentDto; import com.sismics.docs.core.util.action.Action; import com.sismics.docs.core.util.action.AddTagAction; +import com.sismics.docs.core.util.action.ProcessFilesAction; import com.sismics.docs.core.util.action.RemoveTagAction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +37,9 @@ public class ActionUtil { case REMOVE_TAG: action = new RemoveTagAction(); break; + case PROCESS_FILES: + action = new ProcessFilesAction(); + break; default: log.error("Action type not handled: " + actionType); break; diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java index 53ee5b7b..a4604038 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java +++ b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java @@ -39,7 +39,7 @@ public class FileUtil { /** * File ID of files currently being processed. */ - private static Set processingFileSet = Collections.synchronizedSet(new HashSet()); + private static Set processingFileSet = Collections.synchronizedSet(new HashSet<>()); /** * Optical character recognition on an image. diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/action/ProcessFilesAction.java b/docs-core/src/main/java/com/sismics/docs/core/util/action/ProcessFilesAction.java new file mode 100644 index 00000000..920e228f --- /dev/null +++ b/docs-core/src/main/java/com/sismics/docs/core/util/action/ProcessFilesAction.java @@ -0,0 +1,64 @@ +package com.sismics.docs.core.util.action; + +import com.sismics.docs.core.dao.FileDao; +import com.sismics.docs.core.dao.UserDao; +import com.sismics.docs.core.dao.dto.DocumentDto; +import com.sismics.docs.core.event.FileUpdatedAsyncEvent; +import com.sismics.docs.core.model.jpa.File; +import com.sismics.docs.core.model.jpa.User; +import com.sismics.docs.core.util.DirectoryUtil; +import com.sismics.docs.core.util.EncryptionUtil; +import com.sismics.docs.core.util.FileUtil; +import com.sismics.util.context.ThreadLocalContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.json.JsonObject; +import java.nio.file.Path; +import java.util.List; + +/** + * Action to process all files. + * + * @author bgamard + */ +public class ProcessFilesAction implements Action { + /** + * Logger. + */ + private static final Logger log = LoggerFactory.getLogger(ProcessFilesAction.class); + + @Override + public void execute(DocumentDto documentDto, JsonObject action) { + FileDao fileDao = new FileDao(); + List fileList = fileDao.getByDocumentId(null, documentDto.getId()); + + try { + for (File file : fileList) { + // Get the creating user + UserDao userDao = new UserDao(); + User user = userDao.getById(file.getUserId()); + + // Decrypt the file + Path storedFile = DirectoryUtil.getStorageDirectory().resolve(file.getId()); + Path unencryptedFile = EncryptionUtil.decryptFile(storedFile, user.getPrivateKey()); + + // Start the asynchronous processing + FileUtil.startProcessingFile(file.getId()); + FileUpdatedAsyncEvent event = new FileUpdatedAsyncEvent(); + event.setUserId("admin"); + event.setLanguage(documentDto.getLanguage()); + event.setFile(file); + event.setUnencryptedFile(unencryptedFile); + ThreadLocalContext.get().addAsyncEvent(event); + } + } catch (Exception e) { + log.error("Error processing a file", e); + } + } + + @Override + public void validate(JsonObject action) { + // No parameter, so always OK + } +} diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java index 97533de8..5e8568ca 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java @@ -11,7 +11,6 @@ import com.sismics.docs.core.event.RebuildIndexAsyncEvent; import com.sismics.docs.core.model.context.AppContext; import com.sismics.docs.core.model.jpa.Config; import com.sismics.docs.core.model.jpa.File; -import com.sismics.docs.core.model.jpa.User; import com.sismics.docs.core.service.InboxService; import com.sismics.docs.core.util.ConfigUtil; import com.sismics.docs.core.util.DirectoryUtil; @@ -509,7 +508,7 @@ public class AppResource extends BaseResource { // Get the memory appender org.apache.log4j.Logger logger = org.apache.log4j.Logger.getRootLogger(); Appender appender = logger.getAppender("MEMORY"); - if (appender == null || !(appender instanceof MemoryAppender)) { + if (!(appender instanceof MemoryAppender)) { throw new ServerException("ServerError", "MEMORY appender not configured"); } MemoryAppender memoryAppender = (MemoryAppender) appender; @@ -539,7 +538,7 @@ public class AppResource extends BaseResource { } /** - * Destroy and rebuild Lucene index. + * Destroy and rebuild the search index. * * @api {post} /app/batch/reindex Rebuild the search index * @apiName PostAppBatchReindex @@ -686,64 +685,4 @@ public class AppResource extends BaseResource { .add("status", "ok"); return Response.ok().entity(response.build()).build(); } - - /** - * Recompute the quota for each user. - * - * @api {post} /app/batch/recompute_quota Recompute user quotas - * @apiName PostAppBatchRecomputeQuota - * @apiGroup App - * @apiSuccess {String} status Status OK - * @apiError (client) ForbiddenError Access denied - * @apiError (server) MissingFile File does not exist - * @apiPermission admin - * @apiVersion 1.5.0 - * - * @return Response - */ - @POST - @Path("batch/recompute_quota") - public Response batchRecomputeQuota() { - if (!authenticate()) { - throw new ForbiddenClientException(); - } - checkBaseFunction(BaseFunction.ADMIN); - - // Get all files - FileDao fileDao = new FileDao(); - List fileList = fileDao.findAll(); - - // Count each file for the corresponding user quota - UserDao userDao = new UserDao(); - Map userMap = new HashMap<>(); - for (File file : fileList) { - java.nio.file.Path storedFile = DirectoryUtil.getStorageDirectory().resolve(file.getId()); - User user; - if (userMap.containsKey(file.getUserId())) { - user = userMap.get(file.getUserId()); - } else { - user = userDao.getById(file.getUserId()); - user.setStorageCurrent(0L); - userMap.put(user.getId(), user); - } - - try { - user.setStorageCurrent(user.getStorageCurrent() + Files.size(storedFile)); - } catch (IOException e) { - throw new ServerException("MissingFile", "File does not exist", e); - } - } - - // Save all users - for (User user : userMap.values()) { - if (user.getDeleteDate() == null) { - userDao.updateQuota(user); - } - } - - // Always return OK - JsonObjectBuilder response = Json.createObjectBuilder() - .add("status", "ok"); - return Response.ok().entity(response.build()).build(); - } } diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java index de466f51..53ffc18c 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java @@ -284,10 +284,6 @@ public class FileResource extends BaseResource { throw new ForbiddenClientException(); } - // Get the current user - UserDao userDao = new UserDao(); - User user = userDao.getById(principal.getId()); - // Get the document and the file DocumentDao documentDao = new DocumentDao(); FileDao fileDao = new FileDao(); @@ -300,17 +296,21 @@ public class FileResource extends BaseResource { throw new NotFoundException(); } + // Get the creating user + UserDao userDao = new UserDao(); + User user = userDao.getById(file.getUserId()); + // Start the processing asynchronously try { java.nio.file.Path storedFile = DirectoryUtil.getStorageDirectory().resolve(id); java.nio.file.Path unencryptedFile = EncryptionUtil.decryptFile(storedFile, user.getPrivateKey()); FileUtil.startProcessingFile(id); - FileUpdatedAsyncEvent fileUpdatedAsyncEvent = new FileUpdatedAsyncEvent(); - fileUpdatedAsyncEvent.setUserId(principal.getId()); - fileUpdatedAsyncEvent.setLanguage(documentDto.getLanguage()); - fileUpdatedAsyncEvent.setFile(file); - fileUpdatedAsyncEvent.setUnencryptedFile(unencryptedFile); - ThreadLocalContext.get().addAsyncEvent(fileUpdatedAsyncEvent); + FileUpdatedAsyncEvent event = new FileUpdatedAsyncEvent(); + event.setUserId(principal.getId()); + event.setLanguage(documentDto.getLanguage()); + event.setFile(file); + event.setUnencryptedFile(unencryptedFile); + ThreadLocalContext.get().addAsyncEvent(event); } catch (Exception e) { throw new ServerException("ProcessingError", "Error processing this file", e); } diff --git a/docs-web/src/main/webapp/src/locale/en.json b/docs-web/src/main/webapp/src/locale/en.json index 6b0ce223..fdae3d28 100644 --- a/docs-web/src/main/webapp/src/locale/en.json +++ b/docs-web/src/main/webapp/src/locale/en.json @@ -515,7 +515,8 @@ }, "action_type": { "ADD_TAG": "Add a tag", - "REMOVE_TAG": "Remove a tag" + "REMOVE_TAG": "Remove a tag", + "PROCESS_FILES": "Process files" }, "pagination": { "previous": "Previous", diff --git a/docs-web/src/main/webapp/src/partial/docs/settings.workflow.edit.html b/docs-web/src/main/webapp/src/partial/docs/settings.workflow.edit.html index 950da70c..dfdfeeef 100644 --- a/docs-web/src/main/webapp/src/partial/docs/settings.workflow.edit.html +++ b/docs-web/src/main/webapp/src/partial/docs/settings.workflow.edit.html @@ -116,6 +116,7 @@