vmario
/
docs
mirror of https://github.com/sismics/docs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Java 8, Tesseract 4, Ubuntu (#214) 

Java 8, Tesseract 4, Ubuntu 18.04
This commit is contained in:
Benjamin Gamard 2018-03-18 22:21:31 +01:00 committed by GitHub
parent 7ea8d0c0f7
commit 5220b13e7d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 178 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Dockerfile
View File

@ -1,11 +1,10 @@
FROM sismics/jetty:9.2.20-jdk7
FROM sismics/ubuntu-jetty:9.3.11
MAINTAINER b.gamard@sismics.com
RUN echo "deb http://http.debian.net/debian jessie-backports main contrib non-free" | tee /etc/apt/sources.list.d/backports.list
RUN apt-get update && apt-get -y -q install ffmpeg mediainfo tesseract-ocr tesseract-ocr-fra tesseract-ocr-ita tesseract-ocr-kor tesseract-ocr-rus tesseract-ocr-ukr tesseract-ocr-spa tesseract-ocr-ara tesseract-ocr-hin tesseract-ocr-deu tesseract-ocr-pol tesseract-ocr-jpn tesseract-ocr-por tesseract-ocr-tha tesseract-ocr-jpn tesseract-ocr-chi-sim tesseract-ocr-chi-tra && \
apt-get clean && rm -rf /var/lib/apt/lists/*
ENV TESSDATA_PREFIX /usr/share/tesseract-ocr
ENV TESSDATA_PREFIX /usr/share/tesseract-ocr/4.00/
ENV LC_NUMERIC C
# Remove the embedded javax.mail jar from Jetty

30
docs-core/src/main/java/com/sismics/docs/core/dao/jpa/dto/TagStatDto.java
View File

@ -1,30 +0,0 @@
package com.sismics.docs.core.dao.jpa.dto;
/**
* Tag stat DTO.
*
* @author bgamard
*/
public class TagStatDto extends TagDto {
private int count;
/**
* Getter of count.
*
* @return the count
*/
public int getCount() {
return count;
}
/**
* Setter of count.
*
* @param count count
*/
public void setCount(int count) {
this.count = count;
}
}

17
docs-core/src/main/java/com/sismics/docs/core/listener/async/DocumentCreatedAsyncListener.java
View File

@ -31,16 +31,13 @@ public class DocumentCreatedAsyncListener {
log.info("Document created event: " + event.toString());
}
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
// Add the first contributor (the creator of the document)
ContributorDao contributorDao = new ContributorDao();
Contributor contributor = new Contributor();
contributor.setDocumentId(event.getDocument().getId());
contributor.setUserId(event.getUserId());
contributorDao.create(contributor);
}
TransactionUtil.handle(() -> {
// Add the first contributor (the creator of the document)
ContributorDao contributorDao = new ContributorDao();
Contributor contributor = new Contributor();
contributor.setDocumentId(event.getDocument().getId());
contributor.setUserId(event.getUserId());
contributorDao.create(contributor);
});
// Update Lucene index

45
docs-core/src/main/java/com/sismics/docs/core/listener/async/DocumentUpdatedAsyncListener.java
View File

@ -34,32 +34,29 @@ public class DocumentUpdatedAsyncListener {
log.info("Document updated event: " + event.toString());
}
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
// Update Lucene index
DocumentDao documentDao = new DocumentDao();
LuceneDao luceneDao = new LuceneDao();
luceneDao.updateDocument(documentDao.getById(event.getDocumentId()));
// Update contributors list
ContributorDao contributorDao = new ContributorDao();
List<Contributor> contributorList = contributorDao.findByDocumentId(event.getDocumentId());
// Check if the user firing this event is not already a contributor
for (Contributor contributor : contributorList) {
if (contributor.getUserId().equals(event.getUserId())) {
// The current user is already a contributor on this document, don't do anything
return;
}
TransactionUtil.handle(() -> {
// Update Lucene index
DocumentDao documentDao = new DocumentDao();
LuceneDao luceneDao = new LuceneDao();
luceneDao.updateDocument(documentDao.getById(event.getDocumentId()));
// Update contributors list
ContributorDao contributorDao = new ContributorDao();
List<Contributor> contributorList = contributorDao.findByDocumentId(event.getDocumentId());
// Check if the user firing this event is not already a contributor
for (Contributor contributor : contributorList) {
if (contributor.getUserId().equals(event.getUserId())) {
// The current user is already a contributor on this document, don't do anything
return;
}
// Add a new contributor
Contributor contributor = new Contributor();
contributor.setDocumentId(event.getDocumentId());
contributor.setUserId(event.getUserId());
contributorDao.create(contributor);
}
// Add a new contributor
Contributor contributor = new Contributor();
contributor.setDocumentId(event.getDocumentId());
contributor.setUserId(event.getUserId());
contributorDao.create(contributor);
});
}
}

28
docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java
View File

@ -59,12 +59,9 @@ public class FileCreatedAsyncListener {
// Get the user from the database
final AtomicReference<User> user = new AtomicReference<>();
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
UserDao userDao = new UserDao();
user.set(userDao.getById(event.getUserId()));
}
TransactionUtil.handle(() -> {
UserDao userDao = new UserDao();
user.set(userDao.getById(event.getUserId()));
});
if (user.get() == null) {
// The user has been deleted meanwhile
@ -108,18 +105,15 @@ public class FileCreatedAsyncListener {
log.info(MessageFormat.format("File content extracted in {0}ms", System.currentTimeMillis() - startTime));
// Save the file to database
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
FileDao fileDao = new FileDao();
if (fileDao.getActiveById(file.getId()) == null) {
// The file has been deleted since the text extraction started, ignore the result
return;
}
file.setContent(content.get());
fileDao.update(file);
TransactionUtil.handle(() -> {
FileDao fileDao = new FileDao();
if (fileDao.getActiveById(file.getId()) == null) {
// The file has been deleted since the text extraction started, ignore the result
return;
}
file.setContent(content.get());
fileDao.update(file);
});
if (file.getDocumentId() != null) {

23
docs-core/src/main/java/com/sismics/docs/core/listener/async/PasswordLostAsyncListener.java
View File

@ -35,19 +35,16 @@ public class PasswordLostAsyncListener {
log.info("Password lost event: " + passwordLostEvent.toString());
}
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
final UserDto user = passwordLostEvent.getUser();
final PasswordRecovery passwordRecovery = passwordLostEvent.getPasswordRecovery();
// Send the password recovery email
Map<String, Object> paramRootMap = new HashMap<>();
paramRootMap.put("user_name", user.getUsername());
paramRootMap.put("password_recovery_key", passwordRecovery.getId());
EmailUtil.sendEmail(Constants.EMAIL_TEMPLATE_PASSWORD_RECOVERY, user, paramRootMap);
}
TransactionUtil.handle(() -> {
final UserDto user = passwordLostEvent.getUser();
final PasswordRecovery passwordRecovery = passwordLostEvent.getPasswordRecovery();
// Send the password recovery email
Map<String, Object> paramRootMap = new HashMap<>();
paramRootMap.put("user_name", user.getUsername());
paramRootMap.put("password_recovery_key", passwordRecovery.getId());
EmailUtil.sendEmail(Constants.EMAIL_TEMPLATE_PASSWORD_RECOVERY, user, paramRootMap);
});
}
}

39
docs-core/src/main/java/com/sismics/docs/core/listener/async/RebuildIndexAsyncListener.java
View File

@ -1,10 +1,5 @@
package com.sismics.docs.core.listener.async;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.eventbus.Subscribe;
import com.sismics.docs.core.dao.jpa.DocumentDao;
import com.sismics.docs.core.dao.jpa.FileDao;
@ -13,6 +8,10 @@ import com.sismics.docs.core.event.RebuildIndexAsyncEvent;
import com.sismics.docs.core.model.jpa.Document;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.util.TransactionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
/**
* Listener on rebuild index.
@ -29,30 +28,26 @@ public class RebuildIndexAsyncListener {
* Rebuild Lucene index.
*
* @param rebuildIndexAsyncEvent Index rebuild event
* @throws Exception
*/
@Subscribe
public void on(final RebuildIndexAsyncEvent rebuildIndexAsyncEvent) throws Exception {
public void on(final RebuildIndexAsyncEvent rebuildIndexAsyncEvent) {
if (log.isInfoEnabled()) {
log.info("Rebuild index event: " + rebuildIndexAsyncEvent.toString());
}
// Fetch all documents and files
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
// Fetch all documents
DocumentDao documentDao = new DocumentDao();
List<Document> documentList = documentDao.findAll();
// Fetch all files
FileDao fileDao = new FileDao();
List<File> fileList = fileDao.findAll();
// Rebuild index
LuceneDao luceneDao = new LuceneDao();
luceneDao.rebuildIndex(documentList, fileList);
}
TransactionUtil.handle(() -> {
// Fetch all documents
DocumentDao documentDao = new DocumentDao();
List<Document> documentList = documentDao.findAll();
// Fetch all files
FileDao fileDao = new FileDao();
List<File> fileList = fileDao.findAll();
// Rebuild index
LuceneDao luceneDao = new LuceneDao();
luceneDao.rebuildIndex(documentList, fileList);
});
}
}

19
docs-core/src/main/java/com/sismics/docs/core/listener/async/RouteStepValidateAsyncListener.java
View File

@ -34,19 +34,16 @@ public class RouteStepValidateAsyncListener {
log.info("Route step validate event: " + routeStepValidateEvent.toString());
}
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
final UserDto user = routeStepValidateEvent.getUser();
TransactionUtil.handle(() -> {
final UserDto user = routeStepValidateEvent.getUser();
// Send route step validated email
Map<String, Object> paramRootMap = new HashMap<>();
paramRootMap.put("user_name", user.getUsername());
paramRootMap.put("document_id", routeStepValidateEvent.getDocument().getId());
paramRootMap.put("document_title", routeStepValidateEvent.getDocument().getTitle());
// Send route step validated email
Map<String, Object> paramRootMap = new HashMap<>();
paramRootMap.put("user_name", user.getUsername());
paramRootMap.put("document_id", routeStepValidateEvent.getDocument().getId());
paramRootMap.put("document_title", routeStepValidateEvent.getDocument().getTitle());
EmailUtil.sendEmail(Constants.EMAIL_TEMPLATE_ROUTE_STEP_VALIDATE, user, paramRootMap);
}
EmailUtil.sendEmail(Constants.EMAIL_TEMPLATE_ROUTE_STEP_VALIDATE, user, paramRootMap);
});
}
}

59
docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java
View File

@ -67,41 +67,38 @@ public class InboxService extends AbstractScheduledService {
* Synchronize the inbox.
*/
public void syncInbox() {
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED);
if (!enabled) {
return;
}
TransactionUtil.handle(() -> {
Boolean enabled = ConfigUtil.getConfigBooleanValue(ConfigType.INBOX_ENABLED);
if (!enabled) {
return;
}
log.info("Synchronizing IMAP inbox...");
Folder inbox = null;
lastSyncError = null;
lastSyncDate = new Date();
lastSyncMessageCount = 0;
log.info("Synchronizing IMAP inbox...");
Folder inbox = null;
lastSyncError = null;
lastSyncDate = new Date();
lastSyncMessageCount = 0;
try {
inbox = openInbox();
Message[] messages = inbox.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false));
log.info(messages.length + " messages found");
for (Message message : messages) {
importMessage(message);
lastSyncMessageCount++;
}
} catch (FolderClosedException e) {
// Ignore this, we will just continue importing on the next cycle
} catch (Exception e) {
log.error("Error synching the inbox", e);
lastSyncError = e.getMessage();
} finally {
try {
inbox = openInbox();
Message[] messages = inbox.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false));
log.info(messages.length + " messages found");
for (Message message : messages) {
importMessage(message);
lastSyncMessageCount++;
if (inbox != null) {
inbox.close(false);
inbox.getStore().close();
}
} catch (FolderClosedException e) {
// Ignore this, we will just continue importing on the next cycle
} catch (Exception e) {
log.error("Error synching the inbox", e);
lastSyncError = e.getMessage();
} finally {
try {
if (inbox != null) {
inbox.close(false);
inbox.getStore().close();
}
} catch (Exception e) {
// NOP
}
// NOP
}
}
});

7
docs-core/src/main/java/com/sismics/docs/core/service/IndexingService.java
View File

@ -115,11 +115,8 @@ public class IndexingService extends AbstractScheduledService {
@Override
protected void runOneIteration() {
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
// NOP
}
TransactionUtil.handle(() -> {
// NOP
});
}

7
docs-web-common/src/main/java/com/sismics/util/filter/RequestContextFilter.java
View File

@ -59,12 +59,7 @@ public class RequestContextFilter implements Filter {
org.apache.log4j.Logger.getRootLogger().addAppender(fileAppender);
// Initialize the application context
TransactionUtil.handle(new Runnable() {
@Override
public void run() {
AppContext.getInstance();
}
});
TransactionUtil.handle(AppContext::getInstance);
}
@Override

71
docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java
View File

@ -51,7 +51,6 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.text.MessageFormat;
@ -296,14 +295,11 @@ public class DocumentResource extends BaseResource {
}
// Convert to PDF
StreamingOutput stream = new StreamingOutput() {
@Override
public void write(OutputStream outputStream) throws IOException, WebApplicationException {
try {
PdfUtil.convertToPdf(documentDto, fileList, fitImageToPage, metadata, margin, outputStream);
} catch (Exception e) {
throw new IOException(e);
}
StreamingOutput stream = outputStream -> {
try {
PdfUtil.convertToPdf(documentDto, fileList, fitImageToPage, metadata, margin, outputStream);
} catch (Exception e) {
throw new IOException(e);
}
};
@ -452,7 +448,7 @@ public class DocumentResource extends BaseResource {
// New tag criteria
List<TagDto> tagDtoList = TagUtil.findByName(params[1], allTagDtoList);
if (documentCriteria.getTagIdList() == null) {
documentCriteria.setTagIdList(new ArrayList<String>());
documentCriteria.setTagIdList(new ArrayList<>());
}
if (tagDtoList.isEmpty()) {
// No tag found, the request must returns nothing
@ -492,32 +488,39 @@ public class DocumentResource extends BaseResource {
// New specific date criteria
try {
boolean isUpdated = params[0].startsWith("u");
if (params[1].length() == 10) {
DateTime date = dayFormatter.parseDateTime(params[1]);
if (isUpdated) {
documentCriteria.setUpdateDateMin(date.toDate());
documentCriteria.setUpdateDateMax(date.plusDays(1).minusSeconds(1).toDate());
} else {
documentCriteria.setCreateDateMin(date.toDate());
documentCriteria.setCreateDateMax(date.plusDays(1).minusSeconds(1).toDate());
switch (params[1].length()) {
case 10: {
DateTime date = dayFormatter.parseDateTime(params[1]);
if (isUpdated) {
documentCriteria.setUpdateDateMin(date.toDate());
documentCriteria.setUpdateDateMax(date.plusDays(1).minusSeconds(1).toDate());
} else {
documentCriteria.setCreateDateMin(date.toDate());
documentCriteria.setCreateDateMax(date.plusDays(1).minusSeconds(1).toDate());
}
break;
}
} else if (params[1].length() == 7) {
DateTime date = monthFormatter.parseDateTime(params[1]);
if (isUpdated) {
documentCriteria.setUpdateDateMin(date.toDate());
documentCriteria.setUpdateDateMax(date.plusMonths(1).minusSeconds(1).toDate());
} else {
documentCriteria.setCreateDateMin(date.toDate());
documentCriteria.setCreateDateMax(date.plusMonths(1).minusSeconds(1).toDate());
case 7: {
DateTime date = monthFormatter.parseDateTime(params[1]);
if (isUpdated) {
documentCriteria.setUpdateDateMin(date.toDate());
documentCriteria.setUpdateDateMax(date.plusMonths(1).minusSeconds(1).toDate());
} else {
documentCriteria.setCreateDateMin(date.toDate());
documentCriteria.setCreateDateMax(date.plusMonths(1).minusSeconds(1).toDate());
}
break;
}
} else if (params[1].length() == 4) {
DateTime date = yearFormatter.parseDateTime(params[1]);
if (isUpdated) {
documentCriteria.setUpdateDateMin(date.toDate());
documentCriteria.setUpdateDateMax(date.plusYears(1).minusSeconds(1).toDate());
} else {
documentCriteria.setCreateDateMin(date.toDate());
documentCriteria.setCreateDateMax(date.plusYears(1).minusSeconds(1).toDate());
case 4: {
DateTime date = yearFormatter.parseDateTime(params[1]);
if (isUpdated) {
documentCriteria.setUpdateDateMin(date.toDate());
documentCriteria.setUpdateDateMax(date.plusYears(1).minusSeconds(1).toDate());
} else {
documentCriteria.setCreateDateMin(date.toDate());
documentCriteria.setCreateDateMax(date.plusYears(1).minusSeconds(1).toDate());
}
break;
}
}
} catch (IllegalArgumentException e) {

63
docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java
View File

@ -39,7 +39,6 @@ import javax.ws.rs.core.Response.Status;
import javax.ws.rs.core.StreamingOutput;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Paths;
@ -506,18 +505,15 @@ public class FileResource extends BaseResource {
final InputStream responseInputStream = decrypt ?
EncryptionUtil.decryptInputStream(fileInputStream, user.getPrivateKey()) : fileInputStream;
stream = new StreamingOutput() {
@Override
public void write(OutputStream outputStream) throws IOException, WebApplicationException {
stream = outputStream -> {
try {
ByteStreams.copy(responseInputStream, outputStream);
} finally {
try {
ByteStreams.copy(responseInputStream, outputStream);
} finally {
try {
responseInputStream.close();
outputStream.close();
} catch (IOException e) {
// Ignore
}
responseInputStream.close();
outputStream.close();
} catch (IOException e) {
// Ignore
}
}
};
@ -578,32 +574,29 @@ public class FileResource extends BaseResource {
final List<File> fileList = fileDao.getByDocumentId(principal.getId(), documentId);
// Create the ZIP stream
StreamingOutput stream = new StreamingOutput() {
@Override
public void write(OutputStream outputStream) throws IOException, WebApplicationException {
try (ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream)) {
// Add each file to the ZIP stream
int index = 0;
for (File file : fileList) {
java.nio.file.Path storedfile = DirectoryUtil.getStorageDirectory().resolve(file.getId());
InputStream fileInputStream = Files.newInputStream(storedfile);
// Add the decrypted file to the ZIP stream
// Files are encrypted by the creator of them
User user = userDao.getById(file.getUserId());
try (InputStream decryptedStream = EncryptionUtil.decryptInputStream(fileInputStream, user.getPrivateKey())) {
ZipEntry zipEntry = new ZipEntry(index + "-" + file.getFullName(Integer.toString(index)));
zipOutputStream.putNextEntry(zipEntry);
ByteStreams.copy(decryptedStream, zipOutputStream);
zipOutputStream.closeEntry();
} catch (Exception e) {
throw new WebApplicationException(e);
}
index++;
StreamingOutput stream = outputStream -> {
try (ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream)) {
// Add each file to the ZIP stream
int index = 0;
for (File file : fileList) {
java.nio.file.Path storedfile = DirectoryUtil.getStorageDirectory().resolve(file.getId());
InputStream fileInputStream = Files.newInputStream(storedfile);
// Add the decrypted file to the ZIP stream
// Files are encrypted by the creator of them
User user = userDao.getById(file.getUserId());
try (InputStream decryptedStream = EncryptionUtil.decryptInputStream(fileInputStream, user.getPrivateKey())) {
ZipEntry zipEntry = new ZipEntry(index + "-" + file.getFullName(Integer.toString(index)));
zipOutputStream.putNextEntry(zipEntry);
ByteStreams.copy(decryptedStream, zipOutputStream);
zipOutputStream.closeEntry();
} catch (Exception e) {
throw new WebApplicationException(e);
}
index++;
}
outputStream.close();
}
outputStream.close();
};
// Write to the output

13
pom.xml
View File

@ -11,8 +11,8 @@
<name>Docs Parent</name>
<properties>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!-- Dependencies version (external) -->
@ -49,10 +49,9 @@
<org.jsoup.jsoup.version>1.11.2</org.jsoup.jsoup.version>
<com.github.jai-imageio.jai-imageio-jpeg2000.version>1.3.0</com.github.jai-imageio.jai-imageio-jpeg2000.version>
<!-- Migrate to Java 8 before upgrading this -->
<org.eclipse.jetty.jetty-server.version>9.2.13.v20150730</org.eclipse.jetty.jetty-server.version>
<org.eclipse.jetty.jetty-webapp.version>9.2.13.v20150730</org.eclipse.jetty.jetty-webapp.version>
<org.eclipse.jetty.jetty-servlet.version>9.2.13.v20150730</org.eclipse.jetty.jetty-servlet.version>
<org.eclipse.jetty.jetty-server.version>9.3.11.v20160721</org.eclipse.jetty.jetty-server.version>
<org.eclipse.jetty.jetty-webapp.version>9.3.11.v20160721</org.eclipse.jetty.jetty-webapp.version>
<org.eclipse.jetty.jetty-servlet.version>9.3.11.v20160721</org.eclipse.jetty.jetty-servlet.version>
<!-- Plugins version -->
<org.apache.maven.plugins.maven-antrun-plugin.version>1.8</org.apache.maven.plugins.maven-antrun-plugin.version>
@ -61,7 +60,7 @@
<org.apache.maven.plugins.maven-resources-plugin.version>2.7</org.apache.maven.plugins.maven-resources-plugin.version>
<org.apache.maven.plugins.maven-war-plugin.version>2.6</org.apache.maven.plugins.maven-war-plugin.version>
<org.apache.maven.plugins.maven-surefire-plugin.version>2.19.1</org.apache.maven.plugins.maven-surefire-plugin.version>
<org.eclipse.jetty.jetty-maven-plugin.version>9.2.13.v20150730</org.eclipse.jetty.jetty-maven-plugin.version>
<org.eclipse.jetty.jetty-maven-plugin.version>9.3.11.v20160721</org.eclipse.jetty.jetty-maven-plugin.version>
</properties>
<scm>