Merge branch 'master' into sismics_prod

This commit is contained in:
Benjamin Gamard 2017-07-31 14:48:59 +02:00
commit 43084d9d86
14 changed files with 247 additions and 98 deletions

View File

@ -5,7 +5,20 @@ before_install:
- sudo apt-get -qq update
- sudo apt-get -y -q install tesseract-ocr tesseract-ocr-fra tesseract-ocr-jpn
- sudo apt-get -y -q install haveged && sudo service haveged start
after_success:
- mvn -Pprod -DskipTests clean install
- docker login -u $DOCKER_USER -p $DOCKER_PASS
- export REPO=sismics/docs
- export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi`
- docker build -f Dockerfile -t $REPO:$COMMIT .
- docker tag $REPO:$COMMIT $REPO:$TAG
- docker tag $REPO:$COMMIT $REPO:travis-$TRAVIS_BUILD_NUMBER
- docker push $REPO
env:
global:
- TESSDATA_PREFIX=/usr/share/tesseract-ocr
- LC_NUMERIC=C
- secure: LRGpjWORb0qy6VuypZjTAfA8uRHlFUMTwb77cenS9PPRBxuSnctC531asS9Xg3DqC5nsRxBBprgfCKotn5S8nBSD1ceHh84NASyzLSBft3xSMbg7f/2i7MQ+pGVwLncusBU6E/drnMFwZBleo+9M8Tf96axY5zuUp90MUTpSgt0=
- secure: bCDDR6+I7PmSkuTYZv1HF/z98ANX/SFEESUCqxVmV5Gs0zFC0vQXaPJQ2xaJNRop1HZBFMZLeMMPleb0iOs985smpvK2F6Rbop9Tu+Vyo0uKqv9tbZ7F8Nfgnv9suHKZlL84FNeUQZJX6vsFIYPEJ/r7K5P/M0PdUy++fEwxEhU=
- secure: ewXnzbkgCIHpDWtaWGMa1OYZJ/ki99zcIl4jcDPIC0eB3njX/WgfcC6i0Ke9mLqDqwXarWJ6helm22sNh+xtQiz6isfBtBX+novfRt9AANrBe3koCMUemMDy7oh5VflBaFNP0DVb8LSCnwf6dx6ZB5E9EB8knvk40quc/cXpGjY=
- COMMIT=${TRAVIS_COMMIT::8}

View File

@ -41,7 +41,16 @@ Download
--------
The latest release is downloadable here: <https://github.com/sismics/docs/releases> in WAR format.
You will need a Java webapp server to run it, like [Jetty](http://eclipse.org/jetty/) or [Tomcat](http://tomcat.apache.org/)
You will need a Java webapp server to run it, like [Jetty](http://eclipse.org/jetty/) or [Tomcat](http://tomcat.apache.org/).
The default admin password is "admin". Don't forget to change it before going to production.
Install with Docker
-------------------
From a Docker host, run this command to download and install Sismics Docs. The server will run on <http://[your-docker-host-ip]:8100>.
The default admin password is "admin". Don't forget to change it before going to production.
docker run --rm --name sismics_docs_latest -d -p 8100:8080 -v sismics_docs_latest:/data sismics/docs:latest
How to build Docs from the sources
----------------------------------

View File

@ -9,6 +9,7 @@ import java.util.concurrent.TimeUnit;
import com.google.common.eventbus.AsyncEventBus;
import com.google.common.eventbus.EventBus;
import com.lowagie.text.FontFactory;
import com.sismics.docs.core.constant.ConfigType;
import com.sismics.docs.core.dao.jpa.ConfigDao;
import com.sismics.docs.core.listener.async.DocumentCreatedAsyncListener;
@ -20,6 +21,7 @@ import com.sismics.docs.core.listener.async.RebuildIndexAsyncListener;
import com.sismics.docs.core.listener.sync.DeadEventListener;
import com.sismics.docs.core.model.jpa.Config;
import com.sismics.docs.core.service.IndexingService;
import com.sismics.docs.core.util.PdfUtil;
import com.sismics.util.EnvironmentUtil;
/**
@ -59,10 +61,13 @@ public class AppContext {
private AppContext() {
resetEventBus();
// Start indexing service
ConfigDao configDao = new ConfigDao();
Config luceneStorageConfig = configDao.getById(ConfigType.LUCENE_DIRECTORY_STORAGE);
indexingService = new IndexingService(luceneStorageConfig != null ? luceneStorageConfig.getValue() : null);
indexingService.startAsync();
PdfUtil.registerFonts();
}
/**

View File

@ -1,26 +1,24 @@
package com.sismics.docs.core.util;
import java.awt.image.BufferedImage;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import javax.crypto.CipherOutputStream;
import javax.imageio.ImageIO;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.tess4j.Tesseract;
import com.sismics.util.ImageUtil;
import org.imgscalr.Scalr;
import org.imgscalr.Scalr.Method;
import org.imgscalr.Scalr.Mode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.tess4j.Tesseract;
import com.sismics.util.ImageUtil;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import javax.crypto.CipherOutputStream;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
/**
* File entity utilities.
@ -64,11 +62,12 @@ public class FileUtil {
private static String ocrFile(InputStream inputStream, String language) {
Tesseract instance = Tesseract.getInstance();
String content = null;
BufferedImage image = null;
BufferedImage image;
try {
image = ImageIO.read(inputStream);
} catch (IOException e) {
log.error("Error reading the image", e);
return null;
}
// Upscale and grayscale the image
@ -92,10 +91,9 @@ public class FileUtil {
* Save a file on the storage filesystem.
*
* @param inputStream Unencrypted input stream
* @param pdf
* @param pdfInputStream PDF input stream
* @param file File to save
* @param privateKey Private key used for encryption
* @throws Exception
*/
public static void save(InputStream inputStream, InputStream pdfInputStream, File file, String privateKey) throws Exception {
Cipher cipher = EncryptionUtil.getEncryptionCipher(privateKey);
@ -114,9 +112,8 @@ public class FileUtil {
* @param inputStream Unencrypted input stream
* @param pdfInputStream Unencrypted PDF input stream
* @param cipher Cipher to use for encryption
* @throws Exception
*/
public static void saveVariations(File file, InputStream inputStream, InputStream pdfInputStream, Cipher cipher) throws Exception {
private static void saveVariations(File file, InputStream inputStream, InputStream pdfInputStream, Cipher cipher) throws Exception {
BufferedImage image = null;
if (ImageUtil.isImage(file.getMimeType())) {
image = ImageIO.read(inputStream);
@ -151,7 +148,6 @@ public class FileUtil {
* Remove a file from the storage filesystem.
*
* @param file File to delete
* @throws IOException
*/
public static void delete(File file) throws IOException {
Path storedFile = DirectoryUtil.getStorageDirectory().resolve(file.getId());

View File

@ -1,18 +1,18 @@
package com.sismics.docs.core.util;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import javax.imageio.ImageIO;
import com.google.common.base.Charsets;
import com.google.common.base.Strings;
import com.google.common.io.ByteStreams;
import com.google.common.io.CharStreams;
import com.google.common.io.Closer;
import com.google.common.io.Resources;
import com.lowagie.text.*;
import com.lowagie.text.pdf.PdfWriter;
import com.sismics.docs.core.dao.jpa.dto.DocumentDto;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.util.pdf.PdfPage;
import com.sismics.util.ImageUtil;
import com.sismics.util.mime.MimeType;
import org.apache.pdfbox.io.MemoryUsageSetting;
import org.apache.pdfbox.multipdf.PDFMergerUtility;
import org.apache.pdfbox.pdmodel.PDDocument;
@ -32,13 +32,15 @@ import org.odftoolkit.odfdom.doc.OdfTextDocument;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Strings;
import com.google.common.io.Closer;
import com.sismics.docs.core.dao.jpa.dto.DocumentDto;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.util.pdf.PdfPage;
import com.sismics.util.ImageUtil;
import com.sismics.util.mime.MimeType;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
/**
* PDF utilities.
@ -86,7 +88,6 @@ public class PdfUtil {
* @param inputStream InputStream
* @param reset Reset the stream after usage
* @return PDF input stream
* @throws Exception
*/
public static InputStream convertToPdf(File file, InputStream inputStream, boolean reset) throws Exception {
if (file.getMimeType().equals(MimeType.APPLICATION_PDF)) {
@ -102,17 +103,47 @@ public class PdfUtil {
return convertOpenDocumentText(inputStream, reset);
}
if (file.getMimeType().equals(MimeType.TEXT_PLAIN) || file.getMimeType().equals(MimeType.TEXT_CSV)) {
return convertTextPlain(inputStream, reset);
}
// PDF conversion not necessary/possible
return null;
}
/**
* Convert a text plain document to PDF.
*
* @param inputStream Unecnrypted input stream
* @param reset Reset the stream after usage
* @return PDF input stream
*/
private static InputStream convertTextPlain(InputStream inputStream, boolean reset) throws Exception {
Document output = new Document(PageSize.A4, 40, 40, 40, 40);
ByteArrayOutputStream pdfOutputStream = new ByteArrayOutputStream();
PdfWriter.getInstance(output, pdfOutputStream);
output.open();
String content = CharStreams.toString(new InputStreamReader(inputStream, Charsets.UTF_8));
Font font = FontFactory.getFont("LiberationMono-Regular");
Paragraph paragraph = new Paragraph(content, font);
paragraph.setAlignment(Element.ALIGN_LEFT);
output.add(paragraph);
output.close();
if (reset) {
inputStream.reset();
}
return new ByteArrayInputStream(pdfOutputStream.toByteArray());
}
/**
* Convert an open document text file to PDF.
*
* @param inputStream Unencrypted input stream
* @param reset Reset the stream after usage
* @return PDF input stream
* @throws Exception
*/
private static InputStream convertOpenDocumentText(InputStream inputStream, boolean reset) throws Exception {
ByteArrayOutputStream pdfOutputStream = new ByteArrayOutputStream();
@ -131,7 +162,6 @@ public class PdfUtil {
* @param inputStream Unencrypted input stream
* @param reset Reset the stream after usage
* @return PDF input stream
* @throws Exception
*/
private static InputStream convertOfficeDocument(InputStream inputStream, boolean reset) throws Exception {
ByteArrayOutputStream pdfOutputStream = new ByteArrayOutputStream();
@ -153,7 +183,6 @@ public class PdfUtil {
* @param metadata Add a page with metadata
* @param margin Margins in millimeters
* @return PDF input stream
* @throws IOException
*/
public static InputStream convertToPdf(DocumentDto documentDto, List<File> fileList,
boolean fitImageToPage, boolean metadata, int margin) throws Exception {
@ -282,7 +311,6 @@ public class PdfUtil {
*
* @param inputStream PDF document
* @return Render of the first page
* @throws IOException
*/
public static BufferedImage renderFirstPage(InputStream inputStream) throws IOException {
try (PDDocument pdfDocument = PDDocument.load(inputStream)) {
@ -290,4 +318,20 @@ public class PdfUtil {
return renderer.renderImage(0);
}
}
/**
* Register fonts.
*/
public static void registerFonts() {
URL url = Resources.getResource("fonts/LiberationMono-Regular.ttf");
try (InputStream is = url.openStream()) {
Path file = Files.createTempFile("sismics_docs_font_mono", ".ttf");
try (OutputStream os = Files.newOutputStream(file)) {
ByteStreams.copy(is, os);
}
FontFactory.register(file.toAbsolutePath().toString(), "LiberationMono-Regular");
} catch (IOException e) {
log.error("Error loading font", e);
}
}
}

View File

@ -92,6 +92,10 @@ public class MimeTypeUtil {
return "odt";
case MimeType.OFFICE_DOCUMENT:
return "docx";
case MimeType.TEXT_PLAIN:
return "txt";
case MimeType.TEXT_CSV:
return "csv";
default:
return "bin";
}

View File

@ -13,6 +13,7 @@ import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletResponse;
import com.lowagie.text.FontFactory;
import org.apache.log4j.Level;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.RollingFileAppender;
@ -71,6 +72,9 @@ public class RequestContextFilter implements Filter {
AppContext.getInstance();
}
});
// Register fonts
FontFactory.registerDirectories();
}
@Override

View File

@ -23,7 +23,7 @@ angular.module('docs').controller('DocumentView', function ($scope, $state, $sta
*/
$scope.comment = '';
$scope.addComment = function() {
if ($scope.comment.length == 0) {
if ($scope.comment.length === 0) {
return;
}
@ -40,9 +40,20 @@ angular.module('docs').controller('DocumentView', function ($scope, $state, $sta
* Delete a comment.
*/
$scope.deleteComment = function(comment) {
var title = 'Delete comment';
var msg = 'Do you really want to delete this comment?';
var btns = [
{result: 'cancel', label: 'Cancel'},
{result: 'ok', label: 'OK', cssClass: 'btn-primary'}
];
$dialog.messageBox(title, msg, btns, function (result) {
if (result === 'ok') {
Restangular.one('comment', comment.id).remove().then(function() {
$scope.comments.splice($scope.comments.indexOf(comment), 1);
});
}
});
};
/**
@ -57,7 +68,7 @@ angular.module('docs').controller('DocumentView', function ($scope, $state, $sta
];
$dialog.messageBox(title, msg, btns, function (result) {
if (result == 'ok') {
if (result === 'ok') {
Restangular.one('document', document.id).remove().then(function() {
$scope.loadDocuments();
$state.go('document.default');
@ -74,7 +85,7 @@ angular.module('docs').controller('DocumentView', function ($scope, $state, $sta
templateUrl: 'partial/docs/document.share.html',
controller: 'DocumentModalShare'
}).result.then(function (name) {
if (name == null) {
if (name === null) {
return;
}
@ -100,18 +111,19 @@ angular.module('docs').controller('DocumentView', function ($scope, $state, $sta
var title = 'Shared document';
var msg = 'You can share this document by giving this link. ' +
'Note that everyone having this link can see the document.<br/>' +
'<input class="form-control share-link" type="text" readonly="readonly" value="' + link + '" />';
'<input class="form-control share-link" type="text" readonly="readonly" value="' + link + '"' +
' onclick="this.select(); document.execCommand(\'copy\');" />';
var btns = [
{result: 'unshare', label: 'Unshare', cssClass: 'btn-danger'},
{result: 'close', label: 'Close'}
];
$dialog.messageBox(title, msg, btns, function (result) {
if (result == 'unshare') {
if (result === 'unshare') {
// Unshare this document and update the local shares
Restangular.one('share', share.id).remove().then(function () {
$scope.document.acls = _.reject($scope.document.acls, function(s) {
return share.id == s.id;
return share.id === s.id;
});
});
}

View File

@ -10,9 +10,9 @@
<a ng-click="openFile(file)">
<img class="thumbnail-file" ng-src="../api/file/{{ file.id }}/data?size=thumb" tooltip="{{ file.mimetype }} | {{ file.size | filesize }}" tooltip-placement="top" />
</a>
<div class="caption pointer" ng-click="file.checked = !file.checked">
<div class="caption">
<div class="pull-left">
<input type="checkbox" ng-model="file.checked" />
<input type="checkbox" ng-model="file.checked" style="width: 26px; height: 26px" />
</div>
<div class="pull-right">
<button class="btn btn-danger" ng-click="deleteFile($event, file)"><span class="glyphicon glyphicon-trash"></span></button>

View File

@ -129,7 +129,7 @@
<div class="form-group">
<div class="col-sm-offset-2 col-sm-10">
<button type="submit" class="btn btn-primary" ng-disabled="!documentForm.$valid || fileIsUploading" ng-click="edit()">{{ isEdit() ? 'Edit' : 'Add' }}</button>
<button type="submit" class="btn btn-primary" ng-disabled="!documentForm.$valid || fileIsUploading" ng-click="edit()">Save</button>
<button type="submit" class="btn btn-default" ng-click="cancel()" ng-disabled="fileIsUploading">Cancel</button>
</div>
</div>

View File

@ -40,9 +40,10 @@
<div class="page-header">
<h1>
{{ document.title }} <small>{{ document.create_date | date: 'yyyy-MM-dd' }}
{{ document.title }}
<img ng-if="document" ng-src="img/flag/{{ document.language }}.png" title="Document language: {{ document.language }}" />
<small>{{ document.create_date | date: 'yyyy-MM-dd' }}
by <a href="#/user/{{ document.creator }}">{{ document.creator }}</a></small>
<img ng-if="document" ng-src="img/flag/{{ document.language }}.png" title="{{ document.language }}" />
</h1>
<p ng-show="document.writable">
@ -107,7 +108,7 @@
<p>
{{ comment.content }}<br />
<span class="text-muted">{{ comment.create_date | date: 'yyyy-MM-dd' }}</span>
<span class="text-muted pull-right btn-link"
<span class="text-muted pull-right btn-link pointer"
ng-show="document.writable || userInfo.username == comment.creator"
ng-click="deleteComment(comment)">Delete</span>
</p>

View File

@ -1,7 +1,17 @@
package com.sismics.docs.rest;
import java.io.InputStream;
import java.util.Date;
import com.google.common.io.ByteStreams;
import com.google.common.io.Resources;
import com.sismics.docs.core.util.DirectoryUtil;
import com.sismics.util.filter.TokenBasedSecurityFilter;
import com.sismics.util.mime.MimeType;
import com.sismics.util.mime.MimeTypeUtil;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
import org.glassfish.jersey.media.multipart.file.StreamDataBodyPart;
import org.joda.time.format.DateTimeFormat;
import org.junit.Assert;
import org.junit.Test;
import javax.json.JsonArray;
import javax.json.JsonObject;
@ -10,20 +20,8 @@ import javax.ws.rs.core.Form;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
import org.glassfish.jersey.media.multipart.file.StreamDataBodyPart;
import org.joda.time.format.DateTimeFormat;
import org.junit.Assert;
import org.junit.Test;
import com.google.common.io.ByteStreams;
import com.google.common.io.Resources;
import com.sismics.docs.core.util.DirectoryUtil;
import com.sismics.util.filter.TokenBasedSecurityFilter;
import com.sismics.util.mime.MimeType;
import com.sismics.util.mime.MimeTypeUtil;
import java.io.InputStream;
import java.util.Date;
/**
* Exhaustive test of the document resource.
@ -545,4 +543,63 @@ public class TestDocumentResource extends BaseJerseyTest {
Assert.assertTrue(fileBytes.length > 0); // Images rendered from PDF differ in size from OS to OS due to font issues
Assert.assertEquals(MimeType.IMAGE_JPEG, MimeTypeUtil.guessMimeType(fileBytes, null));
}
/**
* Test plain text extraction.
*
* @throws Exception e
*/
@Test
public void testPlainTextExtraction() throws Exception {
// Login document_plain
clientUtil.createUser("document_plain");
String documentPlainToken = clientUtil.login("document_plain");
// Create a document
long create1Date = new Date().getTime();
JsonObject json = target().path("/document").request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentPlainToken)
.put(Entity.form(new Form()
.param("title", "My super title document 1")
.param("description", "My super description for document 1")
.param("language", "eng")
.param("create_date", Long.toString(create1Date))), JsonObject.class);
String document1Id = json.getString("id");
Assert.assertNotNull(document1Id);
// Add a plain text file
String file1Id;
try (InputStream is = Resources.getResource("file/document.txt").openStream()) {
StreamDataBodyPart streamDataBodyPart = new StreamDataBodyPart("file", is, "document.txt");
try (FormDataMultiPart multiPart = new FormDataMultiPart()) {
json = target()
.register(MultiPartFeature.class)
.path("/file").request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentPlainToken)
.put(Entity.entity(multiPart.field("id", document1Id).bodyPart(streamDataBodyPart),
MediaType.MULTIPART_FORM_DATA_TYPE), JsonObject.class);
file1Id = json.getString("id");
Assert.assertNotNull(file1Id);
}
}
// Search documents by query in full content
json = target().path("/document/list")
.queryParam("search", "full:love")
.request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentPlainToken)
.get(JsonObject.class);
Assert.assertTrue(json.getJsonArray("documents").size() == 1);
// Get the file thumbnail data
Response response = target().path("/file/" + file1Id + "/data")
.queryParam("size", "thumb")
.request()
.cookie(TokenBasedSecurityFilter.COOKIE_NAME, documentPlainToken)
.get();
InputStream is = (InputStream) response.getEntity();
byte[] fileBytes = ByteStreams.toByteArray(is);
Assert.assertTrue(fileBytes.length > 0); // Images rendered from PDF differ in size from OS to OS due to font issues
Assert.assertEquals(MimeType.IMAGE_JPEG, MimeTypeUtil.guessMimeType(fileBytes, null));
}
}

View File

@ -0,0 +1,4 @@
This is a test document
Please love me
&é"'(-è_çà)=$^ù*
조선글