vmario
/
docs
mirror of https://github.com/sismics/docs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Index and generate thumbnails from PDF

This commit is contained in:
jendib 2013-08-18 13:48:08 +02:00
parent 4f8076427b
commit d3f59554f8
11 changed files with 245 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs-core/pom.xml
View File

@ -117,6 +117,11 @@
<artifactId>imgscalr-lib</artifactId>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
</dependency>
<!-- OCR dependencies -->
<dependency>
<groupId>jna</groupId>

4
docs-core/src/main/java/com/sismics/docs/core/event/OcrFileAsyncEvent.java → docs-core/src/main/java/com/sismics/docs/core/event/ExtractFileAsyncEvent.java
View File

@ -3,11 +3,11 @@ package com.sismics.docs.core.event;
import com.google.common.base.Objects;
/**
* OCR all files in database event.
* Extract file content event.
*
* @author bgamard
*/
public class OcrFileAsyncEvent {
public class ExtractFileAsyncEvent {
@Override
public String toString() {
return Objects.toStringHelper(this)

21
docs-core/src/main/java/com/sismics/docs/core/listener/async/OcrFileAsyncListener.java → docs-core/src/main/java/com/sismics/docs/core/listener/async/ExtractFileAsyncListener.java
View File

@ -9,33 +9,33 @@ import org.slf4j.LoggerFactory;
import com.google.common.eventbus.Subscribe;
import com.sismics.docs.core.dao.jpa.DocumentDao;
import com.sismics.docs.core.dao.jpa.FileDao;
import com.sismics.docs.core.event.OcrFileAsyncEvent;
import com.sismics.docs.core.event.ExtractFileAsyncEvent;
import com.sismics.docs.core.model.jpa.Document;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.util.FileUtil;
import com.sismics.docs.core.util.TransactionUtil;
/**
* Listener on OCR all files in database.
* Listener on extract content from all files.
*
* @author bgamard
*/
public class OcrFileAsyncListener {
public class ExtractFileAsyncListener {
/**
* Logger.
*/
private static final Logger log = LoggerFactory.getLogger(OcrFileAsyncListener.class);
private static final Logger log = LoggerFactory.getLogger(ExtractFileAsyncListener.class);
/**
* OCR all files.
* Extract content from all files.
*
* @param ocrFileAsyncEvent OCR all files in database event
* @param extractFileAsyncEvent Extract file content event
* @throws Exception
*/
@Subscribe
public void on(final OcrFileAsyncEvent ocrFileAsyncEvent) throws Exception {
public void on(final ExtractFileAsyncEvent extractFileAsyncEvent) throws Exception {
if (log.isInfoEnabled()) {
log.info("OCR all files in database event: " + ocrFileAsyncEvent.toString());
log.info("Extract file content event: " + extractFileAsyncEvent.toString());
}
TransactionUtil.handle(new Runnable() {
@ -47,10 +47,9 @@ public class OcrFileAsyncListener {
for (File file : fileList) {
long startTime = System.currentTimeMillis();
Document document = documentDao.getById(file.getDocumentId());
String content = FileUtil.ocrFile(document, file);
file.setContent(content);
file.setContent(FileUtil.extractContent(document, file));
TransactionUtil.commit();
log.info(MessageFormat.format("File OCR-ized in {0}ms", System.currentTimeMillis() - startTime));
log.info(MessageFormat.format("File content extracted in {0}ms", System.currentTimeMillis() - startTime));
}
}
});

2
docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java
View File

@ -39,7 +39,7 @@ public class FileCreatedAsyncListener {
// OCR the file
final File file = fileCreatedAsyncEvent.getFile();
long startTime = System.currentTimeMillis();
final String content = FileUtil.ocrFile(fileCreatedAsyncEvent.getDocument(), file);
final String content = FileUtil.extractContent(fileCreatedAsyncEvent.getDocument(), file);
log.info(MessageFormat.format("File OCR-ized in {0}ms", System.currentTimeMillis() - startTime));
// Store the OCR-ization result in the database

4
docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java
View File

@ -16,7 +16,7 @@ import com.sismics.docs.core.listener.async.DocumentDeletedAsyncListener;
import com.sismics.docs.core.listener.async.DocumentUpdatedAsyncListener;
import com.sismics.docs.core.listener.async.FileCreatedAsyncListener;
import com.sismics.docs.core.listener.async.FileDeletedAsyncListener;
import com.sismics.docs.core.listener.async.OcrFileAsyncListener;
import com.sismics.docs.core.listener.async.ExtractFileAsyncListener;
import com.sismics.docs.core.listener.async.RebuildIndexAsyncListener;
import com.sismics.docs.core.listener.sync.DeadEventListener;
import com.sismics.docs.core.model.jpa.Config;
@ -82,7 +82,7 @@ public class AppContext {
asyncEventBus.register(new DocumentUpdatedAsyncListener());
asyncEventBus.register(new DocumentDeletedAsyncListener());
asyncEventBus.register(new RebuildIndexAsyncListener());
asyncEventBus.register(new OcrFileAsyncListener());
asyncEventBus.register(new ExtractFileAsyncListener());
}
/**

87
docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java
View File

@ -6,11 +6,15 @@ import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import javax.imageio.ImageIO;
import net.sourceforge.tess4j.Tesseract;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.util.PDFTextStripper;
import org.imgscalr.Scalr;
import org.imgscalr.Scalr.Method;
import org.imgscalr.Scalr.Mode;
@ -20,6 +24,7 @@ import org.slf4j.LoggerFactory;
import com.sismics.docs.core.model.jpa.Document;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.util.ImageUtil;
import com.sismics.util.mime.MimeType;
/**
* File entity utilities.
@ -33,18 +38,32 @@ public class FileUtil {
private static final Logger log = LoggerFactory.getLogger(FileUtil.class);
/**
* OCR a file.
* Extract content from a file.
*
* @param document Document linked to the file
* @param file File to extract
* @return Content extract
*/
public static String extractContent(Document document, File file) {
String content = null;
if (ImageUtil.isImage(file.getMimeType())) {
content = ocrFile(document, file);
} else if (file.getMimeType().equals(MimeType.APPLICATION_PDF)) {
content = extractPdf(file);
}
return content;
}
/**
* Optical character recognition on a file.
*
* @param document Document linked to the file
* @param file File to OCR
* @return OCR-ized content
* @return Content extracted
*/
public static String ocrFile(Document document, final File file) {
if (!ImageUtil.isImage(file.getMimeType())) {
// The file is not OCR-izable
return null;
}
private static String ocrFile(Document document, File file) {
Tesseract instance = Tesseract.getInstance();
java.io.File storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile();
String content = null;
@ -72,6 +91,35 @@ public class FileUtil {
return content;
}
/**
* Extract text from a PDF.
*
* @param file File to extract
* @return Content extracted
*/
private static String extractPdf(File file) {
String content = null;
PDDocument pdfDocument = null;
java.io.File storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile();
try {
PDFTextStripper stripper = new PDFTextStripper();
pdfDocument = PDDocument.load(storedfile);
content = stripper.getText(pdfDocument);
} catch (IOException e) {
log.error("Error while extracting text from the PDF " + storedfile, e);
} finally {
if (pdfDocument != null) {
try {
pdfDocument.close();
} catch (IOException e) {
// NOP
}
}
}
return content;
}
/**
* Save a file on the storage filesystem.
*
@ -84,7 +132,12 @@ public class FileUtil {
Files.copy(is, path);
// Generate file variations
saveVariations(file, path.toFile());
try {
saveVariations(file, path.toFile());
} catch (IOException e) {
// Don't rethrow Exception from file variations generation
log.error("Error creating file variations", e);
}
}
/**
@ -95,8 +148,22 @@ public class FileUtil {
* @throws IOException
*/
public static void saveVariations(File file, java.io.File originalFile) throws IOException {
BufferedImage image = null;
if (ImageUtil.isImage(file.getMimeType())) {
BufferedImage image = ImageIO.read(originalFile);
image = ImageIO.read(originalFile);
} else if(file.getMimeType().equals(MimeType.APPLICATION_PDF)) {
// Generate preview from the first page of the PDF
PDDocument pdfDocument = PDDocument.load(originalFile);
@SuppressWarnings("unchecked")
List<PDPage> pageList = pdfDocument.getDocumentCatalog().getAllPages();
if (pageList.size() > 0) {
PDPage page = pageList.get(0);
image = page.convertToImage();
}
}
if (image != null) {
// Generate thumbnails from image
BufferedImage web = Scalr.resize(image, Scalr.Method.AUTOMATIC, Scalr.Mode.AUTOMATIC, 1280, Scalr.OP_ANTIALIAS);
BufferedImage thumbnail = Scalr.resize(image, Scalr.Method.AUTOMATIC, Scalr.Mode.AUTOMATIC, 256, Scalr.OP_ANTIALIAS);
image.flush();

2
docs-parent/TODO
View File

@ -1,2 +0,0 @@
- Extract text from PDF for indexing, see PDFBox (server)
- Make thumbnail of the first page of PDF, see PDFBox (server)

157
docs-parent/pom.xml
View File

@ -15,7 +15,7 @@
<maven.compiler.target>1.7</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!-- Dependencies version (external) -->
<!-- Dependencies version (external) -->
<org.apache.commons.commons-compress.version>1.5</org.apache.commons.commons-compress.version>
<commons-lang.commons-lang.version>2.6</commons-lang.commons-lang.version>
<commons-io.commons-io.version>2.1</commons-io.commons-io.version>
@ -62,6 +62,7 @@
<org.mortbay.jetty.jetty-maven-plugin.version>8.1.2.v20120308</org.mortbay.jetty.jetty-maven-plugin.version>
<org.vafer.jdeb.version>1.0.1</org.vafer.jdeb.version>
<com.samaxes.maven.minify-maven-plugin.version>1.7</com.samaxes.maven.minify-maven-plugin.version>
<org.apache.pdfbox.pdfbox.version>1.8.2</org.apache.pdfbox.pdfbox.version>
</properties>
<scm>
@ -436,24 +437,30 @@
<version>${org.imgscalr.imgscalr-lib.version}</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>${org.apache.pdfbox.pdfbox.version}</version>
</dependency>
<!-- OCR dependencies -->
<dependency>
<groupId>jna</groupId>
<artifactId>jna</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>jai</groupId>
<artifactId>imageio</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>tess4j</groupId>
<artifactId>tess4j</artifactId>
<version>1.0</version>
</dependency>
<groupId>jna</groupId>
<artifactId>jna</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>jai</groupId>
<artifactId>imageio</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>tess4j</groupId>
<artifactId>tess4j</artifactId>
<version>1.0</version>
</dependency>
</dependencies>
</dependencyManagement>
@ -478,64 +485,64 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<version>2.3.1</version>
<executions>
<execution>
<id>install-jna</id>
<phase>validate</phase>
<configuration>
<file>${project.basedir}/lib/jna.jar</file>
<repositoryLayout>default</repositoryLayout>
<groupId>jna</groupId>
<artifactId>jna</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<generatePom>true</generatePom>
</configuration>
<goals>
<goal>install-file</goal>
</goals>
</execution>
<execution>
<id>install-jai-imageio</id>
<phase>validate</phase>
<configuration>
<file>${project.basedir}/lib/jai_imageio.jar</file>
<repositoryLayout>default</repositoryLayout>
<groupId>jai</groupId>
<artifactId>imageio</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<generatePom>true</generatePom>
</configuration>
<goals>
<goal>install-file</goal>
</goals>
</execution>
<execution>
<id>install-tess4j</id>
<phase>validate</phase>
<configuration>
<file>${project.basedir}/lib/tess4j.jar</file>
<repositoryLayout>default</repositoryLayout>
<groupId>tess4j</groupId>
<artifactId>tess4j</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<generatePom>true</generatePom>
</configuration>
<goals>
<goal>install-file</goal>
</goals>
</execution>
</executions>
</plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<version>2.3.1</version>
<executions>
<execution>
<id>install-jna</id>
<phase>validate</phase>
<configuration>
<file>${project.basedir}/lib/jna.jar</file>
<repositoryLayout>default</repositoryLayout>
<groupId>jna</groupId>
<artifactId>jna</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<generatePom>true</generatePom>
</configuration>
<goals>
<goal>install-file</goal>
</goals>
</execution>
<execution>
<id>install-jai-imageio</id>
<phase>validate</phase>
<configuration>
<file>${project.basedir}/lib/jai_imageio.jar</file>
<repositoryLayout>default</repositoryLayout>
<groupId>jai</groupId>
<artifactId>imageio</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<generatePom>true</generatePom>
</configuration>
<goals>
<goal>install-file</goal>
</goals>
</execution>
<execution>
<id>install-tess4j</id>
<phase>validate</phase>
<configuration>
<file>${project.basedir}/lib/tess4j.jar</file>
<repositoryLayout>default</repositoryLayout>
<groupId>tess4j</groupId>
<artifactId>tess4j</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<generatePom>true</generatePom>
</configuration>
<goals>
<goal>install-file</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>

4
docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java
View File

@ -26,7 +26,7 @@ import com.sismics.docs.core.dao.jpa.DocumentDao;
import com.sismics.docs.core.dao.jpa.FileDao;
import com.sismics.docs.core.dao.jpa.criteria.DocumentCriteria;
import com.sismics.docs.core.dao.jpa.dto.DocumentDto;
import com.sismics.docs.core.event.OcrFileAsyncEvent;
import com.sismics.docs.core.event.ExtractFileAsyncEvent;
import com.sismics.docs.core.model.context.AppContext;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.util.ConfigUtil;
@ -163,7 +163,7 @@ public class AppResource extends BaseResource {
checkBaseFunction(BaseFunction.ADMIN);
// Raise a OCR file event
AppContext.getInstance().getAsyncEventBus().post(new OcrFileAsyncEvent());
AppContext.getInstance().getAsyncEventBus().post(new ExtractFileAsyncEvent());
JSONObject response = new JSONObject();
response.put("status", "ok");

64
docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java
View File

@ -15,6 +15,7 @@ import org.codehaus.jettison.json.JSONObject;
import org.joda.time.format.DateTimeFormat;
import org.junit.Test;
import com.google.common.io.ByteStreams;
import com.sismics.docs.rest.filter.CookieAuthenticationFilter;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
@ -354,4 +355,67 @@ public class TestDocumentResource extends BaseJerseyTest {
json = response.getEntity(JSONObject.class);
Assert.assertEquals(Status.BAD_REQUEST, Status.fromStatusCode(response.getStatus()));
}
/**
* Test PDF extraction.
*
* @throws Exception
*/
@Test
public void testPdfExtraction() throws Exception {
// Login document2
clientUtil.createUser("document2");
String document2Token = clientUtil.login("document2");
// Create a document
WebResource documentResource = resource().path("/document");
documentResource.addFilter(new CookieAuthenticationFilter(document2Token));
MultivaluedMapImpl postParams = new MultivaluedMapImpl();
postParams.add("title", "My super title document 1");
postParams.add("description", "My super description for document 1");
postParams.add("language", "eng");
long create1Date = new Date().getTime();
postParams.add("create_date", create1Date);
ClientResponse response = documentResource.put(ClientResponse.class, postParams);
Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
JSONObject json = response.getEntity(JSONObject.class);
String document1Id = json.optString("id");
Assert.assertNotNull(document1Id);
// Add a PDF file
WebResource fileResource = resource().path("/file");
fileResource.addFilter(new CookieAuthenticationFilter(document2Token));
FormDataMultiPart form = new FormDataMultiPart();
InputStream file = this.getClass().getResourceAsStream("/file/wikipedia.pdf");
FormDataBodyPart fdp = new FormDataBodyPart("file",
new BufferedInputStream(file),
MediaType.APPLICATION_OCTET_STREAM_TYPE);
form.bodyPart(fdp);
form.field("id", document1Id);
response = fileResource.type(MediaType.MULTIPART_FORM_DATA).put(ClientResponse.class, form);
Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
json = response.getEntity(JSONObject.class);
String file1Id = json.getString("id");
// Search documents by query in full content
documentResource = resource().path("/document/list");
documentResource.addFilter(new CookieAuthenticationFilter(document2Token));
MultivaluedMapImpl getParams = new MultivaluedMapImpl();
getParams.putSingle("search", "full:vrandecic");
response = documentResource.queryParams(getParams).get(ClientResponse.class);
json = response.getEntity(JSONObject.class);
Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
Assert.assertTrue(json.getJSONArray("documents").length() == 1);
// Get the file thumbnail data
fileResource = resource().path("/file/" + file1Id + "/data");
fileResource.addFilter(new CookieAuthenticationFilter(document2Token));
getParams = new MultivaluedMapImpl();
getParams.putSingle("size", "thumb");
response = fileResource.queryParams(getParams).get(ClientResponse.class);
Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
InputStream is = response.getEntityInputStream();
byte[] fileBytes = ByteStreams.toByteArray(is);
Assert.assertEquals(3457, fileBytes.length);
}
}

BIN
docs-web/src/test/resources/file/wikipedia.pdf Normal file
View File

Binary file not shown.