Index and generate thumbnails from PDF

docs-core/pom.xml

@@ -117,6 +117,11 @@
       <artifactId>imgscalr-lib</artifactId>
     </dependency>
     
+    <dependency>
+      <groupId>org.apache.pdfbox</groupId>
+      <artifactId>pdfbox</artifactId>
+    </dependency>
+    
     <!-- OCR dependencies -->
     <dependency>
       <groupId>jna</groupId>

docs-core/src/main/java/com/sismics/docs/core/event/ExtractFileAsyncEvent.java

@@ -3,11 +3,11 @@ package com.sismics.docs.core.event;
 import com.google.common.base.Objects;
 
 /**
- * OCR all files in database event.
+ * Extract file content event.
  *
  * @author bgamard
  */
-public class OcrFileAsyncEvent {
+public class ExtractFileAsyncEvent {
     @Override
     public String toString() {
         return Objects.toStringHelper(this)

docs-core/src/main/java/com/sismics/docs/core/listener/async/ExtractFileAsyncListener.java

@@ -9,33 +9,33 @@ import org.slf4j.LoggerFactory;
 import com.google.common.eventbus.Subscribe;
 import com.sismics.docs.core.dao.jpa.DocumentDao;
 import com.sismics.docs.core.dao.jpa.FileDao;
-import com.sismics.docs.core.event.OcrFileAsyncEvent;
+import com.sismics.docs.core.event.ExtractFileAsyncEvent;
 import com.sismics.docs.core.model.jpa.Document;
 import com.sismics.docs.core.model.jpa.File;
 import com.sismics.docs.core.util.FileUtil;
 import com.sismics.docs.core.util.TransactionUtil;
 
 /**
- * Listener on OCR all files in database.
+ * Listener on extract content from all files.
  * 
  * @author bgamard
  */
-public class OcrFileAsyncListener {
+public class ExtractFileAsyncListener {
     /**
      * Logger.
      */
-    private static final Logger log = LoggerFactory.getLogger(OcrFileAsyncListener.class);
+    private static final Logger log = LoggerFactory.getLogger(ExtractFileAsyncListener.class);
 
     /**
-     * OCR all files.
+     * Extract content from all files.
      * 
-     * @param ocrFileAsyncEvent OCR all files in database event
+     * @param extractFileAsyncEvent Extract file content event
      * @throws Exception
      */
     @Subscribe
-    public void on(final OcrFileAsyncEvent ocrFileAsyncEvent) throws Exception {
+    public void on(final ExtractFileAsyncEvent extractFileAsyncEvent) throws Exception {
         if (log.isInfoEnabled()) {
-            log.info("OCR all files in database event: " + ocrFileAsyncEvent.toString());
+            log.info("Extract file content event: " + extractFileAsyncEvent.toString());
         }
 
         TransactionUtil.handle(new Runnable() {
@@ -47,10 +47,9 @@ public class OcrFileAsyncListener {
                 for (File file : fileList) {
                     long startTime = System.currentTimeMillis();
                     Document document = documentDao.getById(file.getDocumentId());
-                    String content = FileUtil.ocrFile(document, file);
-                    file.setContent(content);
+                    file.setContent(FileUtil.extractContent(document, file));
                     TransactionUtil.commit();
-                    log.info(MessageFormat.format("File OCR-ized in {0}ms", System.currentTimeMillis() - startTime));
+                    log.info(MessageFormat.format("File content extracted in {0}ms", System.currentTimeMillis() - startTime));
                 }
             }
         });

docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java

@@ -39,7 +39,7 @@ public class FileCreatedAsyncListener {
         // OCR the file
         final File file = fileCreatedAsyncEvent.getFile();
         long startTime = System.currentTimeMillis();
-        final String content = FileUtil.ocrFile(fileCreatedAsyncEvent.getDocument(), file);
+        final String content = FileUtil.extractContent(fileCreatedAsyncEvent.getDocument(), file);
         log.info(MessageFormat.format("File OCR-ized in {0}ms", System.currentTimeMillis() - startTime));
         
         // Store the OCR-ization result in the database

docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java

@@ -16,7 +16,7 @@ import com.sismics.docs.core.listener.async.DocumentDeletedAsyncListener;
 import com.sismics.docs.core.listener.async.DocumentUpdatedAsyncListener;
 import com.sismics.docs.core.listener.async.FileCreatedAsyncListener;
 import com.sismics.docs.core.listener.async.FileDeletedAsyncListener;
-import com.sismics.docs.core.listener.async.OcrFileAsyncListener;
+import com.sismics.docs.core.listener.async.ExtractFileAsyncListener;
 import com.sismics.docs.core.listener.async.RebuildIndexAsyncListener;
 import com.sismics.docs.core.listener.sync.DeadEventListener;
 import com.sismics.docs.core.model.jpa.Config;
@@ -82,7 +82,7 @@ public class AppContext {
         asyncEventBus.register(new DocumentUpdatedAsyncListener());
         asyncEventBus.register(new DocumentDeletedAsyncListener());
         asyncEventBus.register(new RebuildIndexAsyncListener());
-        asyncEventBus.register(new OcrFileAsyncListener());
+        asyncEventBus.register(new ExtractFileAsyncListener());
     }
 
     /**

docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java

@@ -6,11 +6,15 @@ import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.List;
 
 import javax.imageio.ImageIO;
 
 import net.sourceforge.tess4j.Tesseract;
 
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.util.PDFTextStripper;
 import org.imgscalr.Scalr;
 import org.imgscalr.Scalr.Method;
 import org.imgscalr.Scalr.Mode;
@@ -20,6 +24,7 @@ import org.slf4j.LoggerFactory;
 import com.sismics.docs.core.model.jpa.Document;
 import com.sismics.docs.core.model.jpa.File;
 import com.sismics.util.ImageUtil;
+import com.sismics.util.mime.MimeType;
 
 /**
  * File entity utilities.
@@ -33,18 +38,32 @@ public class FileUtil {
     private static final Logger log = LoggerFactory.getLogger(FileUtil.class);
     
     /**
-     * OCR a file.
+     * Extract content from a file.
+     * 
+     * @param document Document linked to the file
+     * @param file File to extract
+     * @return Content extract
+     */
+    public static String extractContent(Document document, File file) {
+        String content = null;
+        
+        if (ImageUtil.isImage(file.getMimeType())) {
+            content = ocrFile(document, file);
+        } else if (file.getMimeType().equals(MimeType.APPLICATION_PDF)) {
+            content = extractPdf(file);
+        }
+        
+        return content;
+    }
+    
+    /**
+     * Optical character recognition on a file.
      * 
      * @param document Document linked to the file
      * @param file File to OCR
-     * @return OCR-ized content
+     * @return Content extracted
      */
-    public static String ocrFile(Document document, final File file) {
-        if (!ImageUtil.isImage(file.getMimeType())) {
-            // The file is not OCR-izable
-            return null;
-        }
-        
+    private static String ocrFile(Document document, File file) {
         Tesseract instance = Tesseract.getInstance();
         java.io.File storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile();
         String content = null;
@@ -72,6 +91,35 @@ public class FileUtil {
         return content;
     }
     
+    /**
+     * Extract text from a PDF.
+     * 
+     * @param file File to extract
+     * @return Content extracted
+     */
+    private static String extractPdf(File file) {
+        String content = null;
+        PDDocument pdfDocument = null;
+        java.io.File storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile();
+        try {
+            PDFTextStripper stripper = new PDFTextStripper();
+            pdfDocument = PDDocument.load(storedfile);
+            content = stripper.getText(pdfDocument);
+        } catch (IOException e) {
+            log.error("Error while extracting text from the PDF " + storedfile, e);
+        } finally {
+            if (pdfDocument != null) {
+                try {
+                    pdfDocument.close();
+                } catch (IOException e) {
+                    // NOP
+                }
+            }
+        }
+        
+        return content;
+    }
+    
     /**
      * Save a file on the storage filesystem.
      * 
@@ -84,7 +132,12 @@ public class FileUtil {
         Files.copy(is, path);
         
         // Generate file variations
-        saveVariations(file, path.toFile());
+        try {
+            saveVariations(file, path.toFile());
+        } catch (IOException e) {
+            // Don't rethrow Exception from file variations generation
+            log.error("Error creating file variations", e);
+        }
     }
 
     /**
@@ -95,8 +148,22 @@ public class FileUtil {
      * @throws IOException
      */
     public static void saveVariations(File file, java.io.File originalFile) throws IOException {
+        BufferedImage image = null;
         if (ImageUtil.isImage(file.getMimeType())) {
-            BufferedImage image = ImageIO.read(originalFile);
+            image = ImageIO.read(originalFile);
+        } else if(file.getMimeType().equals(MimeType.APPLICATION_PDF)) {
+            // Generate preview from the first page of the PDF
+            PDDocument pdfDocument = PDDocument.load(originalFile);
+            @SuppressWarnings("unchecked")
+            List<PDPage> pageList = pdfDocument.getDocumentCatalog().getAllPages();
+            if (pageList.size() > 0) {
+                PDPage page = pageList.get(0);
+                image = page.convertToImage();
+            }
+        }
+        
+        if (image != null) {
+            // Generate thumbnails from image
             BufferedImage web = Scalr.resize(image, Scalr.Method.AUTOMATIC, Scalr.Mode.AUTOMATIC, 1280, Scalr.OP_ANTIALIAS);
             BufferedImage thumbnail = Scalr.resize(image, Scalr.Method.AUTOMATIC, Scalr.Mode.AUTOMATIC, 256, Scalr.OP_ANTIALIAS);
             image.flush();

docs-parent/TODO

@@ -1,2 +0,0 @@
-- Extract text from PDF for indexing, see PDFBox (server)
-- Make thumbnail of the first page of PDF, see PDFBox (server)

docs-parent/pom.xml

@@ -15,7 +15,7 @@
     <maven.compiler.target>1.7</maven.compiler.target>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 
-  	<!-- Dependencies version (external) -->
+    <!-- Dependencies version (external) -->
     <org.apache.commons.commons-compress.version>1.5</org.apache.commons.commons-compress.version>
     <commons-lang.commons-lang.version>2.6</commons-lang.commons-lang.version>
     <commons-io.commons-io.version>2.1</commons-io.commons-io.version>
@@ -62,6 +62,7 @@
     <org.mortbay.jetty.jetty-maven-plugin.version>8.1.2.v20120308</org.mortbay.jetty.jetty-maven-plugin.version>
     <org.vafer.jdeb.version>1.0.1</org.vafer.jdeb.version>
     <com.samaxes.maven.minify-maven-plugin.version>1.7</com.samaxes.maven.minify-maven-plugin.version>
+    <org.apache.pdfbox.pdfbox.version>1.8.2</org.apache.pdfbox.pdfbox.version>
   </properties>
   
   <scm>
@@ -436,24 +437,30 @@
         <version>${org.imgscalr.imgscalr-lib.version}</version>
       </dependency>
       
+      <dependency>
+        <groupId>org.apache.pdfbox</groupId>
+        <artifactId>pdfbox</artifactId>
+        <version>${org.apache.pdfbox.pdfbox.version}</version>
+      </dependency>
+      
       <!-- OCR dependencies -->
       <dependency>
-	      <groupId>jna</groupId>
-	      <artifactId>jna</artifactId>
-	      <version>1.0</version>
-	    </dependency>
+        <groupId>jna</groupId>
+        <artifactId>jna</artifactId>
+        <version>1.0</version>
+      </dependency>
       
-	    <dependency>
-	      <groupId>jai</groupId>
-	      <artifactId>imageio</artifactId>
-	      <version>1.0</version>
-	    </dependency>
+      <dependency>
+        <groupId>jai</groupId>
+        <artifactId>imageio</artifactId>
+        <version>1.0</version>
+      </dependency>
       
-	    <dependency>
-	      <groupId>tess4j</groupId>
-	      <artifactId>tess4j</artifactId>
-	      <version>1.0</version>
-	    </dependency>
+      <dependency>
+        <groupId>tess4j</groupId>
+        <artifactId>tess4j</artifactId>
+        <version>1.0</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
   
@@ -478,64 +485,64 @@
       <build>
         <plugins>
           <plugin>
-		        <groupId>org.apache.maven.plugins</groupId>
-		        <artifactId>maven-install-plugin</artifactId>
-		        <version>2.3.1</version>
-		        <executions>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-install-plugin</artifactId>
+            <version>2.3.1</version>
+            <executions>
             
-		          <execution>
-		            <id>install-jna</id>
-		            <phase>validate</phase>
-		            <configuration>
-		              <file>${project.basedir}/lib/jna.jar</file>
-		              <repositoryLayout>default</repositoryLayout>
-		              <groupId>jna</groupId>
-		              <artifactId>jna</artifactId>
-		              <version>1.0</version>
-		              <packaging>jar</packaging>
-		              <generatePom>true</generatePom>
-		            </configuration>
-		            <goals>
-		              <goal>install-file</goal>
-		            </goals>
-		          </execution>
+              <execution>
+                <id>install-jna</id>
+                <phase>validate</phase>
+                <configuration>
+                  <file>${project.basedir}/lib/jna.jar</file>
+                  <repositoryLayout>default</repositoryLayout>
+                  <groupId>jna</groupId>
+                  <artifactId>jna</artifactId>
+                  <version>1.0</version>
+                  <packaging>jar</packaging>
+                  <generatePom>true</generatePom>
+                </configuration>
+                <goals>
+                  <goal>install-file</goal>
+                </goals>
+              </execution>
               
-		          <execution>
-		            <id>install-jai-imageio</id>
-		            <phase>validate</phase>
-		            <configuration>
-		              <file>${project.basedir}/lib/jai_imageio.jar</file>
-		              <repositoryLayout>default</repositoryLayout>
-		              <groupId>jai</groupId>
-		              <artifactId>imageio</artifactId>
-		              <version>1.0</version>
-		              <packaging>jar</packaging>
-		              <generatePom>true</generatePom>
-		            </configuration>
-		            <goals>
-		              <goal>install-file</goal>
-		            </goals>
-		          </execution>
+              <execution>
+                <id>install-jai-imageio</id>
+                <phase>validate</phase>
+                <configuration>
+                  <file>${project.basedir}/lib/jai_imageio.jar</file>
+                  <repositoryLayout>default</repositoryLayout>
+                  <groupId>jai</groupId>
+                  <artifactId>imageio</artifactId>
+                  <version>1.0</version>
+                  <packaging>jar</packaging>
+                  <generatePom>true</generatePom>
+                </configuration>
+                <goals>
+                  <goal>install-file</goal>
+                </goals>
+              </execution>
               
-		          <execution>
-		            <id>install-tess4j</id>
-		            <phase>validate</phase>
-		            <configuration>
-		              <file>${project.basedir}/lib/tess4j.jar</file>
-		              <repositoryLayout>default</repositoryLayout>
-		              <groupId>tess4j</groupId>
-		              <artifactId>tess4j</artifactId>
-		              <version>1.0</version>
-		              <packaging>jar</packaging>
-		              <generatePom>true</generatePom>
-		            </configuration>
-		            <goals>
-		              <goal>install-file</goal>
-		            </goals>
-		          </execution>
+              <execution>
+                <id>install-tess4j</id>
+                <phase>validate</phase>
+                <configuration>
+                  <file>${project.basedir}/lib/tess4j.jar</file>
+                  <repositoryLayout>default</repositoryLayout>
+                  <groupId>tess4j</groupId>
+                  <artifactId>tess4j</artifactId>
+                  <version>1.0</version>
+                  <packaging>jar</packaging>
+                  <generatePom>true</generatePom>
+                </configuration>
+                <goals>
+                  <goal>install-file</goal>
+                </goals>
+              </execution>
               
-		        </executions>
-		      </plugin>
+            </executions>
+          </plugin>
         </plugins>
       </build>
     </profile>

docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java

@@ -26,7 +26,7 @@ import com.sismics.docs.core.dao.jpa.DocumentDao;
 import com.sismics.docs.core.dao.jpa.FileDao;
 import com.sismics.docs.core.dao.jpa.criteria.DocumentCriteria;
 import com.sismics.docs.core.dao.jpa.dto.DocumentDto;
-import com.sismics.docs.core.event.OcrFileAsyncEvent;
+import com.sismics.docs.core.event.ExtractFileAsyncEvent;
 import com.sismics.docs.core.model.context.AppContext;
 import com.sismics.docs.core.model.jpa.File;
 import com.sismics.docs.core.util.ConfigUtil;
@@ -163,7 +163,7 @@ public class AppResource extends BaseResource {
         checkBaseFunction(BaseFunction.ADMIN);
         
         // Raise a OCR file event
-        AppContext.getInstance().getAsyncEventBus().post(new OcrFileAsyncEvent());
+        AppContext.getInstance().getAsyncEventBus().post(new ExtractFileAsyncEvent());
         
         JSONObject response = new JSONObject();
         response.put("status", "ok");

docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java

@@ -15,6 +15,7 @@ import org.codehaus.jettison.json.JSONObject;
 import org.joda.time.format.DateTimeFormat;
 import org.junit.Test;
 
+import com.google.common.io.ByteStreams;
 import com.sismics.docs.rest.filter.CookieAuthenticationFilter;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.ClientResponse.Status;
@@ -354,4 +355,67 @@ public class TestDocumentResource extends BaseJerseyTest {
         json = response.getEntity(JSONObject.class);
         Assert.assertEquals(Status.BAD_REQUEST, Status.fromStatusCode(response.getStatus()));
     }
+    
+    /**
+     * Test PDF extraction.
+     * 
+     * @throws Exception 
+     */
+    @Test
+    public void testPdfExtraction() throws Exception {
+        // Login document2
+        clientUtil.createUser("document2");
+        String document2Token = clientUtil.login("document2");
+
+        // Create a document
+        WebResource documentResource = resource().path("/document");
+        documentResource.addFilter(new CookieAuthenticationFilter(document2Token));
+        MultivaluedMapImpl postParams = new MultivaluedMapImpl();
+        postParams.add("title", "My super title document 1");
+        postParams.add("description", "My super description for document 1");
+        postParams.add("language", "eng");
+        long create1Date = new Date().getTime();
+        postParams.add("create_date", create1Date);
+        ClientResponse response = documentResource.put(ClientResponse.class, postParams);
+        Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
+        JSONObject json = response.getEntity(JSONObject.class);
+        String document1Id = json.optString("id");
+        Assert.assertNotNull(document1Id);
+        
+        // Add a PDF file
+        WebResource fileResource = resource().path("/file");
+        fileResource.addFilter(new CookieAuthenticationFilter(document2Token));
+        FormDataMultiPart form = new FormDataMultiPart();
+        InputStream file = this.getClass().getResourceAsStream("/file/wikipedia.pdf");
+        FormDataBodyPart fdp = new FormDataBodyPart("file",
+                new BufferedInputStream(file),
+                MediaType.APPLICATION_OCTET_STREAM_TYPE);
+        form.bodyPart(fdp);
+        form.field("id", document1Id);
+        response = fileResource.type(MediaType.MULTIPART_FORM_DATA).put(ClientResponse.class, form);
+        Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
+        json = response.getEntity(JSONObject.class);
+        String file1Id = json.getString("id");
+        
+        // Search documents by query in full content
+        documentResource = resource().path("/document/list");
+        documentResource.addFilter(new CookieAuthenticationFilter(document2Token));
+        MultivaluedMapImpl getParams = new MultivaluedMapImpl();
+        getParams.putSingle("search", "full:vrandecic");
+        response = documentResource.queryParams(getParams).get(ClientResponse.class);
+        json = response.getEntity(JSONObject.class);
+        Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
+        Assert.assertTrue(json.getJSONArray("documents").length() == 1);
+        
+        // Get the file thumbnail data
+        fileResource = resource().path("/file/" + file1Id + "/data");
+        fileResource.addFilter(new CookieAuthenticationFilter(document2Token));
+        getParams = new MultivaluedMapImpl();
+        getParams.putSingle("size", "thumb");
+        response = fileResource.queryParams(getParams).get(ClientResponse.class);
+        Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
+        InputStream is = response.getEntityInputStream();
+        byte[] fileBytes = ByteStreams.toByteArray(is);
+        Assert.assertEquals(3457, fileBytes.length);
+    }
 }