diff --git a/docs-core/src/main/java/com/sismics/docs/core/dao/DocumentDao.java b/docs-core/src/main/java/com/sismics/docs/core/dao/DocumentDao.java index fea0d822..f3bb555c 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/dao/DocumentDao.java +++ b/docs-core/src/main/java/com/sismics/docs/core/dao/DocumentDao.java @@ -45,16 +45,20 @@ public class DocumentDao { /** * Returns the list of all active documents. - * + * + * @param offset Offset + * @param limit Limit * @return List of documents */ @SuppressWarnings("unchecked") - public List findAll() { + public List findAll(int offset, int limit) { EntityManager em = ThreadLocalContext.get().getEntityManager(); Query q = em.createQuery("select d from Document d where d.deleteDate is null"); + q.setFirstResult(offset); + q.setMaxResults(limit); return q.getResultList(); } - + /** * Returns the list of all active documents from a user. * diff --git a/docs-core/src/main/java/com/sismics/docs/core/dao/FileDao.java b/docs-core/src/main/java/com/sismics/docs/core/dao/FileDao.java index 1a15c7ad..4610621e 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/dao/FileDao.java +++ b/docs-core/src/main/java/com/sismics/docs/core/dao/FileDao.java @@ -42,13 +42,17 @@ public class FileDao { /** * Returns the list of all files. - * + * + * @param offset Offset + * @param limit Limit * @return List of files */ @SuppressWarnings("unchecked") - public List findAll() { + public List findAll(int offset, int limit) { EntityManager em = ThreadLocalContext.get().getEntityManager(); Query q = em.createQuery("select f from File f where f.deleteDate is null"); + q.setFirstResult(offset); + q.setMaxResults(limit); return q.getResultList(); } diff --git a/docs-core/src/main/java/com/sismics/docs/core/listener/async/RebuildIndexAsyncListener.java b/docs-core/src/main/java/com/sismics/docs/core/listener/async/RebuildIndexAsyncListener.java index 7bf3bff6..07f481a4 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/listener/async/RebuildIndexAsyncListener.java +++ b/docs-core/src/main/java/com/sismics/docs/core/listener/async/RebuildIndexAsyncListener.java @@ -36,19 +36,32 @@ public class RebuildIndexAsyncListener { if (log.isInfoEnabled()) { log.info("Rebuild index event: " + event.toString()); } - - // Fetch all documents and files + + // Clear the index + AppContext.getInstance().getIndexingHandler().clearIndex(); + + // Index all documents TransactionUtil.handle(() -> { - // Fetch all documents + int offset = 0; DocumentDao documentDao = new DocumentDao(); - List documentList = documentDao.findAll(); + List documentList; + do { + documentList = documentDao.findAll(offset, 100); + AppContext.getInstance().getIndexingHandler().createDocuments(documentList); + offset += 100; + } while (documentList.size() > 0); + }); - // Fetch all files + // Index all files + TransactionUtil.handle(() -> { + int offset = 0; FileDao fileDao = new FileDao(); - List fileList = fileDao.findAll(); - - // Rebuild index - AppContext.getInstance().getIndexingHandler().rebuildIndex(documentList, fileList); + List fileList; + do { + fileList = fileDao.findAll(offset, 100); + AppContext.getInstance().getIndexingHandler().createFiles(fileList); + offset += 100; + } while (fileList.size() > 0); }); } } diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/indexing/IndexingHandler.java b/docs-core/src/main/java/com/sismics/docs/core/util/indexing/IndexingHandler.java index d1a94f57..490651c6 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/util/indexing/IndexingHandler.java +++ b/docs-core/src/main/java/com/sismics/docs/core/util/indexing/IndexingHandler.java @@ -36,12 +36,23 @@ public interface IndexingHandler { void shutDown(); /** - * Fully rebuild the index. + * Clear the index. + */ + void clearIndex(); + + /** + * Index a list of documents. * * @param documentList All documents + */ + void createDocuments(List documentList); + + /** + * Index a list of files. + * * @param fileList All files */ - void rebuildIndex(List documentList, List fileList); + void createFiles(List fileList); /** * Index a new document. diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/indexing/LuceneIndexingHandler.java b/docs-core/src/main/java/com/sismics/docs/core/util/indexing/LuceneIndexingHandler.java index 1bb50227..6062557c 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/util/indexing/LuceneIndexingHandler.java +++ b/docs-core/src/main/java/com/sismics/docs/core/util/indexing/LuceneIndexingHandler.java @@ -101,7 +101,7 @@ public class LuceneIndexingHandler implements IndexingHandler { // Create an index writer IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); config.setCommitOnClose(true); - config.setMergeScheduler(new SerialMergeScheduler()); + config.setMergeScheduler(new ConcurrentMergeScheduler()); indexWriter = new IndexWriter(directory, config); // Check index version and rebuild it if necessary @@ -142,18 +142,23 @@ public class LuceneIndexingHandler implements IndexingHandler { } @Override - public void rebuildIndex(final List documentList, final List fileList) { - handle(indexWriter -> { - // Empty index - indexWriter.deleteAll(); + public void clearIndex() { + handle(IndexWriter::deleteAll); + } - // Add all documents + @Override + public void createDocuments(List documentList) { + handle(indexWriter -> { for (Document document : documentList) { org.apache.lucene.document.Document luceneDocument = getDocumentFromDocument(document); indexWriter.addDocument(luceneDocument); } + }); + } - // Add all files + @Override + public void createFiles(List fileList) { + handle(indexWriter -> { for (File file : fileList) { org.apache.lucene.document.Document luceneDocument = getDocumentFromFile(file); indexWriter.addDocument(luceneDocument); diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/jpa/PaginatedLists.java b/docs-core/src/main/java/com/sismics/docs/core/util/jpa/PaginatedLists.java index d415a3a0..15b51ad7 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/util/jpa/PaginatedLists.java +++ b/docs-core/src/main/java/com/sismics/docs/core/util/jpa/PaginatedLists.java @@ -36,7 +36,7 @@ public class PaginatedLists { if (pageSize > MAX_PAGE_SIZE) { pageSize = MAX_PAGE_SIZE; } - return new PaginatedList(pageSize, offset); + return new PaginatedList<>(pageSize, offset); } /** @@ -54,11 +54,11 @@ public class PaginatedLists { * @param paginatedList Paginated list object containing parameters, and into which results are added by side effects * @param queryParam Query parameters */ - public static void executeCountQuery(PaginatedList paginatedList, QueryParam queryParam) { + private static void executeCountQuery(PaginatedList paginatedList, QueryParam queryParam) { StringBuilder sb = new StringBuilder("select count(*) as result_count from ("); sb.append(queryParam.getQueryString()); sb.append(") as t1"); - + QueryParam countQueryParam = new QueryParam(sb.toString(), queryParam.getParameterMap()); Query q = QueryUtil.getNativeQuery(countQueryParam); @@ -70,7 +70,6 @@ public class PaginatedLists { /** * Executes a query and returns the data of the currunt page. * - * @param em EntityManager * @param paginatedList Paginated list object containing parameters, and into which results are added by side effects * @param queryParam Query parameters * @return List of results diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java index fd0f1491..ff871d9e 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java @@ -595,7 +595,7 @@ public class AppResource extends BaseResource { // Get all files FileDao fileDao = new FileDao(); - List fileList = fileDao.findAll(); + List fileList = fileDao.findAll(0, Integer.MAX_VALUE); Map fileMap = new HashMap<>(); for (File file : fileList) { fileMap.put(file.getId(), file);