From fd95ecc4cb04a69b25ce92b7212e9fb09804e5f6 Mon Sep 17 00:00:00 2001 From: jendib Date: Sun, 18 Aug 2013 02:12:48 +0200 Subject: [PATCH] Batch to regenerate all file variations --- .../async/FileDeletedAsyncListener.java | 18 +------- .../com/sismics/docs/core/util/FileUtil.java | 40 ++++++++++++++-- docs-parent/TODO | 5 +- .../docs/rest/resource/AppResource.java | 46 ++++++++++++++++--- .../sismics/docs/rest/TestFileResource.java | 38 +++++++++++++++ 5 files changed, 117 insertions(+), 30 deletions(-) diff --git a/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileDeletedAsyncListener.java b/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileDeletedAsyncListener.java index 639b6e46..43b027b7 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileDeletedAsyncListener.java +++ b/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileDeletedAsyncListener.java @@ -1,7 +1,5 @@ package com.sismics.docs.core.listener.async; -import java.nio.file.Paths; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -9,7 +7,7 @@ import com.google.common.eventbus.Subscribe; import com.sismics.docs.core.dao.lucene.LuceneDao; import com.sismics.docs.core.event.FileDeletedAsyncEvent; import com.sismics.docs.core.model.jpa.File; -import com.sismics.docs.core.util.DirectoryUtil; +import com.sismics.docs.core.util.FileUtil; /** * Listener on file deleted. @@ -36,19 +34,7 @@ public class FileDeletedAsyncListener { // Delete the file from storage File file = fileDeletedAsyncEvent.getFile(); - java.io.File storedFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile(); - java.io.File webFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_web").toFile(); - java.io.File thumbnailFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_thumb").toFile(); - - if (storedFile.exists()) { - storedFile.delete(); - } - if (webFile.exists()) { - webFile.delete(); - } - if (thumbnailFile.exists()) { - thumbnailFile.delete(); - } + FileUtil.delete(file); // Update Lucene index LuceneDao luceneDao = new LuceneDao(); diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java index 6cf7e99c..fc0fc2ca 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java +++ b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java @@ -77,15 +77,26 @@ public class FileUtil { * * @param is InputStream * @param file File to save - * @throws Exception + * @throws IOException */ - public static void save(InputStream is, File file) throws Exception { + public static void save(InputStream is, File file) throws IOException { Path path = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()); Files.copy(is, path); - // In case of image, save thumbnails + // Generate file variations + saveVariations(file, path.toFile()); + } + + /** + * Generate file variations. + * + * @param file File from database + * @param originalFile Original file + * @throws IOException + */ + public static void saveVariations(File file, java.io.File originalFile) throws IOException { if (ImageUtil.isImage(file.getMimeType())) { - BufferedImage image = ImageIO.read(path.toFile()); + BufferedImage image = ImageIO.read(originalFile); BufferedImage web = Scalr.resize(image, Scalr.Method.AUTOMATIC, Scalr.Mode.AUTOMATIC, 1280, Scalr.OP_ANTIALIAS); BufferedImage thumbnail = Scalr.resize(image, Scalr.Method.AUTOMATIC, Scalr.Mode.AUTOMATIC, 256, Scalr.OP_ANTIALIAS); image.flush(); @@ -93,4 +104,25 @@ public class FileUtil { ImageUtil.writeJpeg(thumbnail, Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_thumb").toFile()); } } + + /** + * Remove a file from the storage filesystem. + * + * @param file File to delete + */ + public static void delete(File file) { + java.io.File storedFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile(); + java.io.File webFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_web").toFile(); + java.io.File thumbnailFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_thumb").toFile(); + + if (storedFile.exists()) { + storedFile.delete(); + } + if (webFile.exists()) { + webFile.delete(); + } + if (thumbnailFile.exists()) { + thumbnailFile.delete(); + } + } } diff --git a/docs-parent/TODO b/docs-parent/TODO index a1c1191a..d57cb696 100644 --- a/docs-parent/TODO +++ b/docs-parent/TODO @@ -1,7 +1,6 @@ - New image rescale between thumbnail and original (client) -- Batch to regenerate all thumbnails (server) -- Special criteria to search inside OCR-ed content (eg. full:uranium) (server) -- Special criteria to search on a specific time span (eg. at:2013-06) (server) +- Criteria to search inside OCR-ed content (eg. full:uranium) (server) +- Criteria to search on a specific time span (eg. at:2013-06) (server) - Show help on special criterias (client) - Disable Add/Edit button while uploading (client) diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java index 65cd0cd0..b93c694c 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java @@ -1,5 +1,7 @@ package com.sismics.docs.rest.resource; +import java.io.IOException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -29,6 +31,7 @@ import com.sismics.docs.core.model.context.AppContext; import com.sismics.docs.core.model.jpa.File; import com.sismics.docs.core.util.ConfigUtil; import com.sismics.docs.core.util.DirectoryUtil; +import com.sismics.docs.core.util.FileUtil; import com.sismics.docs.core.util.jpa.PaginatedList; import com.sismics.docs.core.util.jpa.PaginatedLists; import com.sismics.docs.core.util.jpa.SortCriteria; @@ -220,13 +223,8 @@ public class AppResource extends BaseResource { java.io.File[] storedFileList = DirectoryUtil.getStorageDirectory().listFiles(); for (java.io.File storedFile : storedFileList) { String fileName = storedFile.getName(); - if (fileName.endsWith("_web")) { - fileName = fileName.replace("_web", ""); - } - if (fileName.endsWith("_thumb")) { - fileName = fileName.replace("_thumb", ""); - } - if (!fileMap.containsKey(fileName)) { + String[] fileNameArray = fileName.split("_"); + if (!fileMap.containsKey(fileNameArray[0])) { storedFile.delete(); } } @@ -235,4 +233,38 @@ public class AppResource extends BaseResource { response.put("status", "ok"); return Response.ok().entity(response).build(); } + + /** + * Regenerate file variations. + * + * @return Response + * @throws JSONException + */ + @POST + @Path("batch/file_variations") + @Produces(MediaType.APPLICATION_JSON) + public Response batchFileVariations() throws JSONException { + if (!authenticate()) { + throw new ForbiddenClientException(); + } + checkBaseFunction(BaseFunction.ADMIN); + + // Get all files + FileDao fileDao = new FileDao(); + List fileList = fileDao.findAll(); + + // Generate variations for each file + for (File file : fileList) { + java.io.File originalFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile(); + try { + FileUtil.saveVariations(file, originalFile); + } catch (IOException e) { + throw new ServerException("FileError", "Error generating file variations", e); + } + } + + JSONObject response = new JSONObject(); + response.put("status", "ok"); + return Response.ok().entity(response).build(); + } } diff --git a/docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java b/docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java index b2d759f6..8fa059cd 100644 --- a/docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java +++ b/docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java @@ -112,6 +112,44 @@ public class TestFileResource extends BaseJerseyTest { fileBytes = ByteStreams.toByteArray(is); Assert.assertEquals(551084, fileBytes.length); + // Regenerate file variations + String adminAuthenticationToken = clientUtil.login("admin", "admin", false); + WebResource appResource = resource().path("/app/batch/file_variations"); + appResource.addFilter(new CookieAuthenticationFilter(adminAuthenticationToken)); + response = appResource.post(ClientResponse.class); + Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); + + // Get the file data + fileResource = resource().path("/file/" + file1Id + "/data"); + fileResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken)); + response = fileResource.get(ClientResponse.class); + Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); + is = response.getEntityInputStream(); + fileBytes = ByteStreams.toByteArray(is); + Assert.assertEquals(163510, fileBytes.length); + + // Get the thumbnail data + fileResource = resource().path("/file/" + file1Id + "/data"); + fileResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken)); + getParams = new MultivaluedMapImpl(); + getParams.putSingle("size", "thumb"); + response = fileResource.queryParams(getParams).get(ClientResponse.class); + Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); + is = response.getEntityInputStream(); + fileBytes = ByteStreams.toByteArray(is); + Assert.assertEquals(41935, fileBytes.length); + + // Get the web data + fileResource = resource().path("/file/" + file1Id + "/data"); + fileResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken)); + getParams = new MultivaluedMapImpl(); + getParams.putSingle("size", "web"); + response = fileResource.queryParams(getParams).get(ClientResponse.class); + Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); + is = response.getEntityInputStream(); + fileBytes = ByteStreams.toByteArray(is); + Assert.assertEquals(551084, fileBytes.length); + // Get all files from a document fileResource = resource().path("/file/list"); fileResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken));