Closes #206: action to process all files

This commit is contained in:
Benjamin Gamard 2018-04-09 13:02:39 +02:00
parent 6d35020840
commit d0335b6b16
8 changed files with 90 additions and 76 deletions

View File

@ -14,5 +14,10 @@ public enum ActionType {
/** /**
* Remove a tag. * Remove a tag.
*/ */
REMOVE_TAG REMOVE_TAG,
/**
* Process files.
*/
PROCESS_FILES
} }

View File

@ -4,6 +4,7 @@ import com.sismics.docs.core.constant.ActionType;
import com.sismics.docs.core.dao.dto.DocumentDto; import com.sismics.docs.core.dao.dto.DocumentDto;
import com.sismics.docs.core.util.action.Action; import com.sismics.docs.core.util.action.Action;
import com.sismics.docs.core.util.action.AddTagAction; import com.sismics.docs.core.util.action.AddTagAction;
import com.sismics.docs.core.util.action.ProcessFilesAction;
import com.sismics.docs.core.util.action.RemoveTagAction; import com.sismics.docs.core.util.action.RemoveTagAction;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -36,6 +37,9 @@ public class ActionUtil {
case REMOVE_TAG: case REMOVE_TAG:
action = new RemoveTagAction(); action = new RemoveTagAction();
break; break;
case PROCESS_FILES:
action = new ProcessFilesAction();
break;
default: default:
log.error("Action type not handled: " + actionType); log.error("Action type not handled: " + actionType);
break; break;

View File

@ -39,7 +39,7 @@ public class FileUtil {
/** /**
* File ID of files currently being processed. * File ID of files currently being processed.
*/ */
private static Set<String> processingFileSet = Collections.synchronizedSet(new HashSet<String>()); private static Set<String> processingFileSet = Collections.synchronizedSet(new HashSet<>());
/** /**
* Optical character recognition on an image. * Optical character recognition on an image.

View File

@ -0,0 +1,64 @@
package com.sismics.docs.core.util.action;
import com.sismics.docs.core.dao.FileDao;
import com.sismics.docs.core.dao.UserDao;
import com.sismics.docs.core.dao.dto.DocumentDto;
import com.sismics.docs.core.event.FileUpdatedAsyncEvent;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.util.DirectoryUtil;
import com.sismics.docs.core.util.EncryptionUtil;
import com.sismics.docs.core.util.FileUtil;
import com.sismics.util.context.ThreadLocalContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.json.JsonObject;
import java.nio.file.Path;
import java.util.List;
/**
* Action to process all files.
*
* @author bgamard
*/
public class ProcessFilesAction implements Action {
/**
* Logger.
*/
private static final Logger log = LoggerFactory.getLogger(ProcessFilesAction.class);
@Override
public void execute(DocumentDto documentDto, JsonObject action) {
FileDao fileDao = new FileDao();
List<File> fileList = fileDao.getByDocumentId(null, documentDto.getId());
try {
for (File file : fileList) {
// Get the creating user
UserDao userDao = new UserDao();
User user = userDao.getById(file.getUserId());
// Decrypt the file
Path storedFile = DirectoryUtil.getStorageDirectory().resolve(file.getId());
Path unencryptedFile = EncryptionUtil.decryptFile(storedFile, user.getPrivateKey());
// Start the asynchronous processing
FileUtil.startProcessingFile(file.getId());
FileUpdatedAsyncEvent event = new FileUpdatedAsyncEvent();
event.setUserId("admin");
event.setLanguage(documentDto.getLanguage());
event.setFile(file);
event.setUnencryptedFile(unencryptedFile);
ThreadLocalContext.get().addAsyncEvent(event);
}
} catch (Exception e) {
log.error("Error processing a file", e);
}
}
@Override
public void validate(JsonObject action) {
// No parameter, so always OK
}
}

View File

@ -11,7 +11,6 @@ import com.sismics.docs.core.event.RebuildIndexAsyncEvent;
import com.sismics.docs.core.model.context.AppContext; import com.sismics.docs.core.model.context.AppContext;
import com.sismics.docs.core.model.jpa.Config; import com.sismics.docs.core.model.jpa.Config;
import com.sismics.docs.core.model.jpa.File; import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.service.InboxService; import com.sismics.docs.core.service.InboxService;
import com.sismics.docs.core.util.ConfigUtil; import com.sismics.docs.core.util.ConfigUtil;
import com.sismics.docs.core.util.DirectoryUtil; import com.sismics.docs.core.util.DirectoryUtil;
@ -509,7 +508,7 @@ public class AppResource extends BaseResource {
// Get the memory appender // Get the memory appender
org.apache.log4j.Logger logger = org.apache.log4j.Logger.getRootLogger(); org.apache.log4j.Logger logger = org.apache.log4j.Logger.getRootLogger();
Appender appender = logger.getAppender("MEMORY"); Appender appender = logger.getAppender("MEMORY");
if (appender == null || !(appender instanceof MemoryAppender)) { if (!(appender instanceof MemoryAppender)) {
throw new ServerException("ServerError", "MEMORY appender not configured"); throw new ServerException("ServerError", "MEMORY appender not configured");
} }
MemoryAppender memoryAppender = (MemoryAppender) appender; MemoryAppender memoryAppender = (MemoryAppender) appender;
@ -539,7 +538,7 @@ public class AppResource extends BaseResource {
} }
/** /**
* Destroy and rebuild Lucene index. * Destroy and rebuild the search index.
* *
* @api {post} /app/batch/reindex Rebuild the search index * @api {post} /app/batch/reindex Rebuild the search index
* @apiName PostAppBatchReindex * @apiName PostAppBatchReindex
@ -686,64 +685,4 @@ public class AppResource extends BaseResource {
.add("status", "ok"); .add("status", "ok");
return Response.ok().entity(response.build()).build(); return Response.ok().entity(response.build()).build();
} }
/**
* Recompute the quota for each user.
*
* @api {post} /app/batch/recompute_quota Recompute user quotas
* @apiName PostAppBatchRecomputeQuota
* @apiGroup App
* @apiSuccess {String} status Status OK
* @apiError (client) ForbiddenError Access denied
* @apiError (server) MissingFile File does not exist
* @apiPermission admin
* @apiVersion 1.5.0
*
* @return Response
*/
@POST
@Path("batch/recompute_quota")
public Response batchRecomputeQuota() {
if (!authenticate()) {
throw new ForbiddenClientException();
}
checkBaseFunction(BaseFunction.ADMIN);
// Get all files
FileDao fileDao = new FileDao();
List<File> fileList = fileDao.findAll();
// Count each file for the corresponding user quota
UserDao userDao = new UserDao();
Map<String, User> userMap = new HashMap<>();
for (File file : fileList) {
java.nio.file.Path storedFile = DirectoryUtil.getStorageDirectory().resolve(file.getId());
User user;
if (userMap.containsKey(file.getUserId())) {
user = userMap.get(file.getUserId());
} else {
user = userDao.getById(file.getUserId());
user.setStorageCurrent(0L);
userMap.put(user.getId(), user);
}
try {
user.setStorageCurrent(user.getStorageCurrent() + Files.size(storedFile));
} catch (IOException e) {
throw new ServerException("MissingFile", "File does not exist", e);
}
}
// Save all users
for (User user : userMap.values()) {
if (user.getDeleteDate() == null) {
userDao.updateQuota(user);
}
}
// Always return OK
JsonObjectBuilder response = Json.createObjectBuilder()
.add("status", "ok");
return Response.ok().entity(response.build()).build();
}
} }

View File

@ -284,10 +284,6 @@ public class FileResource extends BaseResource {
throw new ForbiddenClientException(); throw new ForbiddenClientException();
} }
// Get the current user
UserDao userDao = new UserDao();
User user = userDao.getById(principal.getId());
// Get the document and the file // Get the document and the file
DocumentDao documentDao = new DocumentDao(); DocumentDao documentDao = new DocumentDao();
FileDao fileDao = new FileDao(); FileDao fileDao = new FileDao();
@ -300,17 +296,21 @@ public class FileResource extends BaseResource {
throw new NotFoundException(); throw new NotFoundException();
} }
// Get the creating user
UserDao userDao = new UserDao();
User user = userDao.getById(file.getUserId());
// Start the processing asynchronously // Start the processing asynchronously
try { try {
java.nio.file.Path storedFile = DirectoryUtil.getStorageDirectory().resolve(id); java.nio.file.Path storedFile = DirectoryUtil.getStorageDirectory().resolve(id);
java.nio.file.Path unencryptedFile = EncryptionUtil.decryptFile(storedFile, user.getPrivateKey()); java.nio.file.Path unencryptedFile = EncryptionUtil.decryptFile(storedFile, user.getPrivateKey());
FileUtil.startProcessingFile(id); FileUtil.startProcessingFile(id);
FileUpdatedAsyncEvent fileUpdatedAsyncEvent = new FileUpdatedAsyncEvent(); FileUpdatedAsyncEvent event = new FileUpdatedAsyncEvent();
fileUpdatedAsyncEvent.setUserId(principal.getId()); event.setUserId(principal.getId());
fileUpdatedAsyncEvent.setLanguage(documentDto.getLanguage()); event.setLanguage(documentDto.getLanguage());
fileUpdatedAsyncEvent.setFile(file); event.setFile(file);
fileUpdatedAsyncEvent.setUnencryptedFile(unencryptedFile); event.setUnencryptedFile(unencryptedFile);
ThreadLocalContext.get().addAsyncEvent(fileUpdatedAsyncEvent); ThreadLocalContext.get().addAsyncEvent(event);
} catch (Exception e) { } catch (Exception e) {
throw new ServerException("ProcessingError", "Error processing this file", e); throw new ServerException("ProcessingError", "Error processing this file", e);
} }

View File

@ -515,7 +515,8 @@
}, },
"action_type": { "action_type": {
"ADD_TAG": "Add a tag", "ADD_TAG": "Add a tag",
"REMOVE_TAG": "Remove a tag" "REMOVE_TAG": "Remove a tag",
"PROCESS_FILES": "Process files"
}, },
"pagination": { "pagination": {
"previous": "Previous", "previous": "Previous",

View File

@ -116,6 +116,7 @@
<select title="Action type" class="form-control" ng-model="transition.actionType"> <select title="Action type" class="form-control" ng-model="transition.actionType">
<option value="ADD_TAG">{{ 'action_type.ADD_TAG' | translate }}</option> <option value="ADD_TAG">{{ 'action_type.ADD_TAG' | translate }}</option>
<option value="REMOVE_TAG">{{ 'action_type.REMOVE_TAG' | translate }}</option> <option value="REMOVE_TAG">{{ 'action_type.REMOVE_TAG' | translate }}</option>
<option value="PROCESS_FILES">{{ 'action_type.PROCESS_FILES' | translate }}</option>
</select> </select>
<span class="input-group-addon btn" ng-click="addAction(transition)"> <span class="input-group-addon btn" ng-click="addAction(transition)">
<span class="fas fa-plus-circle"></span> <span class="fas fa-plus-circle"></span>