vmario/docs
1
0
Fork 0
mirror of https://github.com/sismics/docs.git synced 2024-11-22 22:07:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

#62: Migration to Lucene 5 (without rebuilding old index)

Browse Source
This commit is contained in:
jendib 2016-03-01 01:01:10 +01:00
parent 943465a390
commit 7f19f8c112
6 changed files with 29 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs-core/pom.xml
View File

@ -93,11 +93,6 @@
<artifactId>lucene-queryparser</artifactId> <artifactId>lucene-queryparser</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-highlighter</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.imgscalr</groupId> <groupId>org.imgscalr</groupId>
<artifactId>imgscalr-lib</artifactId> <artifactId>imgscalr-lib</artifactId>

135
docs-core/src/main/java/com/sismics/docs/core/dao/lucene/DocsStandardAnalyzer.java
View File

@ -1,135 +0,0 @@
package com.sismics.docs.core.dao.lucene;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.standard.ClassicAnalyzer;
import org.apache.lucene.analysis.standard.ClassicTokenizer;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
import java.io.IOException;
import java.io.Reader;
/**
* Filters {@link StandardTokenizer} with {@link StandardFilter}, {@link
* LowerCaseFilter} and {@link StopFilter}, using a list of
* English stop words.
*
* <a name="version"/>
* <p>You must specify the required {@link Version}
* compatibility when creating StandardAnalyzer:
* <ul>
* <li> As of 3.4, Hiragana and Han characters are no longer wrongly split
* from their combining characters. If you use a previous version number,
* you get the exact broken behavior for backwards compatibility.
* <li> As of 3.1, StandardTokenizer implements Unicode text segmentation,
* and StopFilter correctly handles Unicode 4.0 supplementary characters
* in stopwords. {@link ClassicTokenizer} and {@link ClassicAnalyzer}
* are the pre-3.1 implementations of StandardTokenizer and
* StandardAnalyzer.
* <li> As of 2.9, StopFilter preserves position increments
* <li> As of 2.4, Tokens incorrectly identified as acronyms
* are corrected (see <a href="https://issues.apache.org/jira/browse/LUCENE-1068">LUCENE-1068</a>)
* </ul>
*/
public final class DocsStandardAnalyzer extends StopwordAnalyzerBase {
/** Default maximum allowed token length */
public static final int DEFAULT_MAX_TOKEN_LENGTH = 255;
private int maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH;
/** An unmodifiable set containing some common English words that are usually not
useful for searching. */
public static final CharArraySet STOP_WORDS_SET = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
/** Builds an analyzer with the given stop words.
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
* @param stopWords stop words */
public DocsStandardAnalyzer(Version matchVersion, CharArraySet stopWords) {
super(matchVersion, stopWords);
}
/** Builds an analyzer with the default stop words ({@link
* #STOP_WORDS_SET}).
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
*/
public DocsStandardAnalyzer(Version matchVersion) {
this(matchVersion, STOP_WORDS_SET);
}
/** Builds an analyzer with the stop words from the given reader.
* @see WordlistLoader#getWordSet(Reader, Version)
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
* @param stopwords Reader to read stop words from */
public DocsStandardAnalyzer(Version matchVersion, Reader stopwords) throws IOException {
this(matchVersion, loadStopwordSet(stopwords, matchVersion));
}
/**
* Set maximum allowed token length. If a token is seen
* that exceeds this length then it is discarded. This
* setting only takes effect the next time tokenStream or
* tokenStream is called.
*/
public void setMaxTokenLength(int length) {
maxTokenLength = length;
}
/**
* @see #setMaxTokenLength
*/
public int getMaxTokenLength() {
return maxTokenLength;
}
@Override
protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
final StandardTokenizer src = new StandardTokenizer(matchVersion, reader);
src.setMaxTokenLength(maxTokenLength);
TokenStream tok = new StandardFilter(matchVersion, src);
tok = new LowerCaseFilter(matchVersion, tok);
tok = new StopFilter(matchVersion, tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
protected void setReader(final Reader reader) throws IOException {
src.setMaxTokenLength(DocsStandardAnalyzer.this.maxTokenLength);
super.setReader(reader);
}
};
}
@Override
protected Reader initReader(String fieldName, Reader reader) {
if (fieldName.equals("title") || fieldName.equals("description")) {
return new HTMLStripCharFilter(super.initReader(fieldName, reader));
}
return super.initReader(fieldName, reader);
}
}

35
docs-core/src/main/java/com/sismics/docs/core/dao/lucene/LuceneDao.java
View File

@ -6,6 +6,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField; import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField; import org.apache.lucene.document.TextField;
@ -19,7 +20,6 @@ import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.Version;
import com.sismics.docs.core.model.context.AppContext; import com.sismics.docs.core.model.context.AppContext;
import com.sismics.docs.core.model.jpa.Document; import com.sismics.docs.core.model.jpa.Document;
@ -152,22 +152,23 @@ public class LuceneDao {
fullSearchQuery = "\"" + QueryParserUtil.escape(fullSearchQuery) + "\""; fullSearchQuery = "\"" + QueryParserUtil.escape(fullSearchQuery) + "\"";
// Build search query // Build search query
StandardQueryParser qpHelper = new StandardQueryParser(new DocsStandardAnalyzer(Version.LUCENE_42)); StandardQueryParser qpHelper = new StandardQueryParser(new StandardAnalyzer());
qpHelper.setPhraseSlop(100000); // PhraseQuery add terms qpHelper.setPhraseSlop(100000); // PhraseQuery add terms
// Search on documents and files // Search on documents and files
BooleanQuery query = new BooleanQuery(); BooleanQuery query = new BooleanQuery.Builder()
query.add(qpHelper.parse(searchQuery, "title"), Occur.SHOULD); .add(qpHelper.parse(searchQuery, "title"), Occur.SHOULD)
query.add(qpHelper.parse(searchQuery, "description"), Occur.SHOULD); .add(qpHelper.parse(searchQuery, "description"), Occur.SHOULD)
query.add(qpHelper.parse(searchQuery, "subject"), Occur.SHOULD); .add(qpHelper.parse(searchQuery, "subject"), Occur.SHOULD)
query.add(qpHelper.parse(searchQuery, "identifier"), Occur.SHOULD); .add(qpHelper.parse(searchQuery, "identifier"), Occur.SHOULD)
query.add(qpHelper.parse(searchQuery, "publisher"), Occur.SHOULD); .add(qpHelper.parse(searchQuery, "publisher"), Occur.SHOULD)
query.add(qpHelper.parse(searchQuery, "format"), Occur.SHOULD); .add(qpHelper.parse(searchQuery, "format"), Occur.SHOULD)
query.add(qpHelper.parse(searchQuery, "source"), Occur.SHOULD); .add(qpHelper.parse(searchQuery, "source"), Occur.SHOULD)
query.add(qpHelper.parse(searchQuery, "type"), Occur.SHOULD); .add(qpHelper.parse(searchQuery, "type"), Occur.SHOULD)
query.add(qpHelper.parse(searchQuery, "coverage"), Occur.SHOULD); .add(qpHelper.parse(searchQuery, "coverage"), Occur.SHOULD)
query.add(qpHelper.parse(searchQuery, "rights"), Occur.SHOULD); .add(qpHelper.parse(searchQuery, "rights"), Occur.SHOULD)
query.add(qpHelper.parse(fullSearchQuery, "content"), Occur.SHOULD); .add(qpHelper.parse(fullSearchQuery, "content"), Occur.SHOULD)
.build();
// Search // Search
DirectoryReader directoryReader = AppContext.getInstance().getIndexingService().getDirectoryReader(); DirectoryReader directoryReader = AppContext.getInstance().getIndexingService().getDirectoryReader();
@ -183,7 +184,7 @@ public class LuceneDao {
// Extract document IDs // Extract document IDs
for (int i = 0; i < docs.length; i++) { for (int i = 0; i < docs.length; i++) {
org.apache.lucene.document.Document document = searcher.doc(docs[i].doc); org.apache.lucene.document.Document document = searcher.doc(docs[i].doc);
String type = document.get("type"); String type = document.get("doctype");
String documentId = null; String documentId = null;
if (type.equals("document")) { if (type.equals("document")) {
documentId = document.get("id"); documentId = document.get("id");
@ -205,7 +206,7 @@ public class LuceneDao {
private org.apache.lucene.document.Document getDocumentFromDocument(Document document) { private org.apache.lucene.document.Document getDocumentFromDocument(Document document) {
org.apache.lucene.document.Document luceneDocument = new org.apache.lucene.document.Document(); org.apache.lucene.document.Document luceneDocument = new org.apache.lucene.document.Document();
luceneDocument.add(new StringField("id", document.getId(), Field.Store.YES)); luceneDocument.add(new StringField("id", document.getId(), Field.Store.YES));
luceneDocument.add(new StringField("type", "document", Field.Store.YES)); luceneDocument.add(new StringField("doctype", "document", Field.Store.YES));
luceneDocument.add(new TextField("title", document.getTitle(), Field.Store.NO)); luceneDocument.add(new TextField("title", document.getTitle(), Field.Store.NO));
if (document.getDescription() != null) { if (document.getDescription() != null) {
luceneDocument.add(new TextField("description", document.getDescription(), Field.Store.NO)); luceneDocument.add(new TextField("description", document.getDescription(), Field.Store.NO));
@ -248,7 +249,7 @@ public class LuceneDao {
private org.apache.lucene.document.Document getDocumentFromFile(File file, Document document) { private org.apache.lucene.document.Document getDocumentFromFile(File file, Document document) {
org.apache.lucene.document.Document luceneDocument = new org.apache.lucene.document.Document(); org.apache.lucene.document.Document luceneDocument = new org.apache.lucene.document.Document();
luceneDocument.add(new StringField("id", file.getId(), Field.Store.YES)); luceneDocument.add(new StringField("id", file.getId(), Field.Store.YES));
luceneDocument.add(new StringField("type", "file", Field.Store.YES)); luceneDocument.add(new StringField("doctype", "file", Field.Store.YES));
luceneDocument.add(new StringField("document_id", file.getDocumentId(), Field.Store.YES)); luceneDocument.add(new StringField("document_id", file.getDocumentId(), Field.Store.YES));
if (file.getContent() != null) { if (file.getContent() != null) {
luceneDocument.add(new TextField("content", file.getContent(), Field.Store.NO)); luceneDocument.add(new TextField("content", file.getContent(), Field.Store.NO));

6
docs-core/src/main/java/com/sismics/docs/core/service/IndexingService.java
View File

@ -8,7 +8,7 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SimpleFSDirectory; import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.store.SimpleFSLockFactory; import org.apache.lucene.store.SingleInstanceLockFactory;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -59,7 +59,7 @@ public class IndexingService extends AbstractScheduledService {
Path luceneDirectory = DirectoryUtil.getLuceneDirectory(); Path luceneDirectory = DirectoryUtil.getLuceneDirectory();
log.info("Using file Lucene storage: {}", luceneDirectory); log.info("Using file Lucene storage: {}", luceneDirectory);
try { try {
directory = new SimpleFSDirectory(luceneDirectory.toFile(), new SimpleFSLockFactory()); directory = new SimpleFSDirectory(luceneDirectory, new SingleInstanceLockFactory());
} catch (IOException e) { } catch (IOException e) {
log.error("Error initializing Lucene index", e); log.error("Error initializing Lucene index", e);
} }
@ -127,10 +127,10 @@ public class IndexingService extends AbstractScheduledService {
*/ */
public DirectoryReader getDirectoryReader() { public DirectoryReader getDirectoryReader() {
if (directoryReader == null) { if (directoryReader == null) {
try {
if (!DirectoryReader.indexExists(directory)) { if (!DirectoryReader.indexExists(directory)) {
return null; return null;
} }
try {
directoryReader = DirectoryReader.open(directory); directoryReader = DirectoryReader.open(directory);
} catch (IOException e) { } catch (IOException e) {
log.error("Error creating the directory reader", e); log.error("Error creating the directory reader", e);

19
docs-core/src/main/java/com/sismics/docs/core/util/LuceneUtil.java
View File

@ -1,16 +1,16 @@
package com.sismics.docs.core.util; package com.sismics.docs.core.util;
import com.sismics.docs.core.dao.lucene.DocsStandardAnalyzer; import java.io.IOException;
import com.sismics.docs.core.model.context.AppContext;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.SerialMergeScheduler; import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Version;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.IOException; import com.sismics.docs.core.model.context.AppContext;
/** /**
* Lucene utils. * Lucene utils.
@ -31,7 +31,7 @@ public class LuceneUtil {
*/ */
public static void handle(LuceneRunnable runnable) { public static void handle(LuceneRunnable runnable) {
// Standard analyzer // Standard analyzer
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_42, new DocsStandardAnalyzer(Version.LUCENE_42)); IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
// Merge sequentially, because Lucene writing is already done asynchronously // Merge sequentially, because Lucene writing is already done asynchronously
config.setMergeScheduler(new SerialMergeScheduler()); config.setMergeScheduler(new SerialMergeScheduler());
@ -45,15 +45,6 @@ public class LuceneUtil {
log.error("Cannot create IndexWriter", e); log.error("Cannot create IndexWriter", e);
} }
// Unlock index if needed
try {
if (IndexWriter.isLocked(directory)) {
IndexWriter.unlock(directory);
}
} catch (IOException e) {
log.error("Cannot unlock Lucene directory", e);
}
try { try {
runnable.run(indexWriter); runnable.run(indexWriter);
} catch (Exception e) { } catch (Exception e) {

8
docs-parent/pom.xml
View File

@ -28,7 +28,7 @@
<com.h2database.h2.version>1.4.191</com.h2database.h2.version> <com.h2database.h2.version>1.4.191</com.h2database.h2.version>
<org.glassfish.jersey.version>2.22.1</org.glassfish.jersey.version> <org.glassfish.jersey.version>2.22.1</org.glassfish.jersey.version>
<org.mindrot.jbcrypt>0.3m</org.mindrot.jbcrypt> <org.mindrot.jbcrypt>0.3m</org.mindrot.jbcrypt>
<org.apache.lucene.version>4.2.0</org.apache.lucene.version> <org.apache.lucene.version>5.5.0</org.apache.lucene.version>
<org.imgscalr.imgscalr-lib.version>4.2</org.imgscalr.imgscalr-lib.version> <org.imgscalr.imgscalr-lib.version>4.2</org.imgscalr.imgscalr-lib.version>
<org.apache.pdfbox.pdfbox.version>2.0.0-RC3</org.apache.pdfbox.pdfbox.version> <org.apache.pdfbox.pdfbox.version>2.0.0-RC3</org.apache.pdfbox.pdfbox.version>
<org.bouncycastle.bcprov-jdk15on.version>1.54</org.bouncycastle.bcprov-jdk15on.version> <org.bouncycastle.bcprov-jdk15on.version>1.54</org.bouncycastle.bcprov-jdk15on.version>
@ -339,12 +339,6 @@
<version>${org.apache.lucene.version}</version> <version>${org.apache.lucene.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-highlighter</artifactId>
<version>${org.apache.lucene.version}</version>
</dependency>
<dependency> <dependency>
<groupId>org.imgscalr</groupId> <groupId>org.imgscalr</groupId>
<artifactId>imgscalr-lib</artifactId> <artifactId>imgscalr-lib</artifactId>