Search criterias on full content and specific date

This commit is contained in:
jendib 2013-08-18 13:02:50 +02:00
parent fd95ecc4cb
commit 4f8076427b
12 changed files with 133 additions and 44 deletions

View File

@ -154,9 +154,9 @@ public class DocumentDao {
criteriaList.add("d.DOC_IDUSER_C = :userId"); criteriaList.add("d.DOC_IDUSER_C = :userId");
parameterMap.put("userId", criteria.getUserId()); parameterMap.put("userId", criteria.getUserId());
} }
if (!Strings.isNullOrEmpty(criteria.getSearch())) { if (!Strings.isNullOrEmpty(criteria.getSearch()) || !Strings.isNullOrEmpty(criteria.getFullSearch())) {
LuceneDao luceneDao = new LuceneDao(); LuceneDao luceneDao = new LuceneDao();
Set<String> documentIdList = luceneDao.search(criteria.getUserId(), criteria.getSearch()); Set<String> documentIdList = luceneDao.search(criteria.getUserId(), criteria.getSearch(), criteria.getFullSearch());
if (documentIdList.size() == 0) { if (documentIdList.size() == 0) {
// If the search doesn't find any document, the request should return nothing // If the search doesn't find any document, the request should return nothing
documentIdList.add(UUID.randomUUID().toString()); documentIdList.add(UUID.randomUUID().toString());

View File

@ -20,6 +20,11 @@ public class DocumentCriteria {
*/ */
private String search; private String search;
/**
* Full content search query.
*/
private String fullSearch;
/** /**
* Minimum creation date. * Minimum creation date.
*/ */
@ -81,6 +86,24 @@ public class DocumentCriteria {
this.search = search; this.search = search;
} }
/**
* Getter of fullSearch.
*
* @return the fullSearch
*/
public String getFullSearch() {
return fullSearch;
}
/**
* Setter of fullSearch.
*
* @param fullSearch fullSearch
*/
public void setFullSearch(String fullSearch) {
this.fullSearch = fullSearch;
}
/** /**
* Getter of createDateMin. * Getter of createDateMin.
* *

View File

@ -143,13 +143,15 @@ public class LuceneDao {
* Search files. * Search files.
* *
* @param userId User ID to filter on * @param userId User ID to filter on
* @param searchQuery Search query * @param searchQuery Search query on title and description
* @param fullSearchQuery Search query on all fields
* @return List of document IDs * @return List of document IDs
* @throws Exception * @throws Exception
*/ */
public Set<String> search(String userId, String searchQuery) throws Exception { public Set<String> search(String userId, String searchQuery, String fullSearchQuery) throws Exception {
// Escape query and add quotes so QueryParser generate a PhraseQuery // Escape query and add quotes so QueryParser generate a PhraseQuery
searchQuery = "\"" + QueryParserUtil.escape(searchQuery) + "\""; searchQuery = "\"" + QueryParserUtil.escape(searchQuery + " " + fullSearchQuery) + "\"";
fullSearchQuery = "\"" + QueryParserUtil.escape(fullSearchQuery) + "\"";
// Build search query // Build search query
StandardQueryParser qpHelper = new StandardQueryParser(new DocsStandardAnalyzer(Version.LUCENE_42)); StandardQueryParser qpHelper = new StandardQueryParser(new DocsStandardAnalyzer(Version.LUCENE_42));
@ -157,9 +159,9 @@ public class LuceneDao {
// Search on documents and files // Search on documents and files
BooleanQuery query = new BooleanQuery(); BooleanQuery query = new BooleanQuery();
query.add(qpHelper.parse(searchQuery, "content"), Occur.SHOULD);
query.add(qpHelper.parse(searchQuery, "title"), Occur.SHOULD); query.add(qpHelper.parse(searchQuery, "title"), Occur.SHOULD);
query.add(qpHelper.parse(searchQuery, "description"), Occur.SHOULD); query.add(qpHelper.parse(searchQuery, "description"), Occur.SHOULD);
query.add(qpHelper.parse(fullSearchQuery, "content"), Occur.SHOULD);
// Filter on provided user ID // Filter on provided user ID
List<Term> terms = new ArrayList<Term>(); List<Term> terms = new ArrayList<Term>();

View File

@ -1,8 +1,2 @@
- New image rescale between thumbnail and original (client)
- Criteria to search inside OCR-ed content (eg. full:uranium) (server)
- Criteria to search on a specific time span (eg. at:2013-06) (server)
- Show help on special criterias (client)
- Disable Add/Edit button while uploading (client)
- Extract text from PDF for indexing, see PDFBox (server) - Extract text from PDF for indexing, see PDFBox (server)
- Make thumbnail of the first page of PDF, see PDFBox (server) - Make thumbnail of the first page of PDF, see PDFBox (server)

View File

@ -207,10 +207,14 @@ public class DocumentResource extends BaseResource {
DateTimeFormat.forPattern("yyyy").getParser(), DateTimeFormat.forPattern("yyyy").getParser(),
DateTimeFormat.forPattern("yyyy-MM").getParser(), DateTimeFormat.forPattern("yyyy-MM").getParser(),
DateTimeFormat.forPattern("yyyy-MM-dd").getParser() }; DateTimeFormat.forPattern("yyyy-MM-dd").getParser() };
DateTimeFormatter yearFormatter = new DateTimeFormatter(null, parsers[0]);
DateTimeFormatter monthFormatter = new DateTimeFormatter(null, parsers[1]);
DateTimeFormatter dayFormatter = new DateTimeFormatter(null, parsers[2]);
DateTimeFormatter formatter = new DateTimeFormatterBuilder().append( null, parsers ).toFormatter(); DateTimeFormatter formatter = new DateTimeFormatterBuilder().append( null, parsers ).toFormatter();
String[] criteriaList = search.split(" *"); String[] criteriaList = search.split(" *");
List<String> query = new ArrayList<>(); List<String> query = new ArrayList<>();
List<String> fullQuery = new ArrayList<>();
for (String criteria : criteriaList) { for (String criteria : criteriaList) {
String[] params = criteria.split(":"); String[] params = criteria.split(":");
if (params.length != 2 || Strings.isNullOrEmpty(params[0]) || Strings.isNullOrEmpty(params[1])) { if (params.length != 2 || Strings.isNullOrEmpty(params[0]) || Strings.isNullOrEmpty(params[1])) {
@ -233,7 +237,7 @@ public class DocumentResource extends BaseResource {
documentCriteria.getTagIdList().add(tag.getId()); documentCriteria.getTagIdList().add(tag.getId());
} }
} else if (params[0].equals("after") || params[0].equals("before")) { } else if (params[0].equals("after") || params[0].equals("before")) {
// New date criteria // New date span criteria
try { try {
DateTime date = formatter.parseDateTime(params[1]); DateTime date = formatter.parseDateTime(params[1]);
if (params[0].equals("before")) documentCriteria.setCreateDateMax(date.toDate()); if (params[0].equals("before")) documentCriteria.setCreateDateMax(date.toDate());
@ -241,22 +245,45 @@ public class DocumentResource extends BaseResource {
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
// NOP // NOP
} }
} else if (params[0].equals("at")) {
// New specific date criteria
try {
if (params[1].length() == 10) {
DateTime date = dayFormatter.parseDateTime(params[1]);
documentCriteria.setCreateDateMin(date.toDate());
documentCriteria.setCreateDateMax(date.plusDays(1).toDate());
} else if (params[1].length() == 7) {
DateTime date = monthFormatter.parseDateTime(params[1]);
documentCriteria.setCreateDateMin(date.toDate());
documentCriteria.setCreateDateMax(date.plusMonths(1).toDate());
} else if (params[1].length() == 4) {
DateTime date = yearFormatter.parseDateTime(params[1]);
documentCriteria.setCreateDateMin(date.toDate());
documentCriteria.setCreateDateMax(date.plusYears(1).toDate());
}
} catch (IllegalArgumentException e) {
// NOP
}
} else if (params[0].equals("shared")) { } else if (params[0].equals("shared")) {
// New shared state criteria // New shared state criteria
if (params[1].equals("yes")) { if (params[1].equals("yes")) {
documentCriteria.setShared(true); documentCriteria.setShared(true);
} }
} else if (params[0].equals("lang")) { } else if (params[0].equals("lang")) {
// New shared state criteria // New language criteria
if (Constants.SUPPORTED_LANGUAGES.contains(params[1])) { if (Constants.SUPPORTED_LANGUAGES.contains(params[1])) {
documentCriteria.setLanguage(params[1]); documentCriteria.setLanguage(params[1]);
} }
} else if (params[0].equals("full")) {
// New full content search criteria
fullQuery.add(params[1]);
} else { } else {
query.add(criteria); query.add(criteria);
} }
} }
documentCriteria.setSearch(Joiner.on(" ").join(query)); documentCriteria.setSearch(Joiner.on(" ").join(query));
documentCriteria.setFullSearch(Joiner.on(" ").join(fullQuery));
return documentCriteria; return documentCriteria;
} }

View File

@ -43,7 +43,7 @@
</div> </div>
</div> </div>
<div class="form-actions"> <div class="form-actions">
<button type="submit" class="btn btn-primary" ng-disabled="!documentForm.$valid" ng-click="edit()">{{ isEdit() ? 'Edit' : 'Add' }}</button> <button type="submit" class="btn btn-primary" ng-disabled="!documentForm.$valid || fileIsUploading" ng-click="edit()">{{ isEdit() ? 'Edit' : 'Add' }}</button>
<button type="submit" class="btn" ng-click="cancel()">Cancel</button> <button type="submit" class="btn" ng-click="cancel()">Cancel</button>
</div> </div>
</form> </form>

View File

@ -7,7 +7,16 @@
</p> </p>
<p class="input-prepend text-center input-block-level"> <p class="input-prepend text-center input-block-level">
<span class="add-on"><span class="icon-search"></span></span> <span class="add-on"><span class="icon-search"
tooltip-placement="bottom"
tooltip-html-unsafe="before:2012-05<br/>
after:2012-05<br/>
at:2012-05<br/>
tag:car<br/>
full:led<br/>
shared:yes<br/>
lang:fra"
></span></span>
<input type="text" placeholder="Search" ng-model="search" class="span10" /> <input type="text" placeholder="Search" ng-model="search" class="span10" />
</p> </p>

View File

@ -25,7 +25,7 @@
<li class="span2 text-center thumbnail-container" ng-repeat="file in files"> <li class="span2 text-center thumbnail-container" ng-repeat="file in files">
<div class="thumbnail"> <div class="thumbnail">
<a ng-click="openFile(file)"> <a ng-click="openFile(file)">
<img class="thumbnail-file" ng-src="api/file/{{ file.id }}/data?thumbnail=true" tooltip="{{ file.mimetype }}" tooltip-placement="top" /> <img class="thumbnail-file" ng-src="api/file/{{ file.id }}/data?size=thumb" tooltip="{{ file.mimetype }}" tooltip-placement="top" />
</a> </a>
<div class="caption"> <div class="caption">
<div class="pull-left"> <div class="pull-left">

View File

@ -9,13 +9,13 @@
</div> </div>
<div class="btn-group pull-right"> <div class="btn-group pull-right">
<button type="button" class="btn" ng-click="openFile()"><span class="icon-share"></span></button> <button type="button" class="btn" ng-click="openFile()"><span class="icon-download-alt"></span></button>
</div> </div>
</div> </div>
<img ng-if="$stateParams.fileId && (file.mimetype == 'image/png' || file.mimetype == 'image/jpeg' || file.mimetype == 'image/gif')" ng-src="api/file/{{ $stateParams.fileId }}/data" /> <img ng-if="$stateParams.fileId && (file.mimetype == 'image/png' || file.mimetype == 'image/jpeg' || file.mimetype == 'image/gif')" ng-src="api/file/{{ $stateParams.fileId }}/data?size=web" />
<div class="text-center" ng-if="$stateParams.fileId && file.mimetype == 'application/pdf'"> <div class="text-center" ng-if="$stateParams.fileId && file.mimetype == 'application/pdf'">
<img ng-src="api/file/{{ $stateParams.fileId }}/data?thumbnail=true" /> <img ng-src="api/file/{{ $stateParams.fileId }}/data?size=web" />
</div> </div>

View File

@ -9,13 +9,13 @@
</div> </div>
<div class="btn-group pull-right"> <div class="btn-group pull-right">
<button type="button" class="btn" ng-click="openFile()"><span class="icon-share"></span></button> <button type="button" class="btn" ng-click="openFile()"><span class="icon-download-alt"></span></button>
</div> </div>
</div> </div>
<img ng-if="$stateParams.fileId && (file.mimetype == 'image/png' || file.mimetype == 'image/jpeg' || file.mimetype == 'image/gif')" ng-src="api/file/{{ $stateParams.fileId }}/data?share={{ $stateParams.shareId }}" /> <img ng-if="$stateParams.fileId && (file.mimetype == 'image/png' || file.mimetype == 'image/jpeg' || file.mimetype == 'image/gif')" ng-src="api/file/{{ $stateParams.fileId }}/data?size=web&share={{ $stateParams.shareId }}" />
<div class="text-center" ng-if="$stateParams.fileId && file.mimetype == 'application/pdf'"> <div class="text-center" ng-if="$stateParams.fileId && file.mimetype == 'application/pdf'">
<img ng-src="api/file/{{ $stateParams.fileId }}/data?thumbnail=true&share={{ $stateParams.shareId }}" /> <img ng-src="api/file/{{ $stateParams.fileId }}/data?size=web&share={{ $stateParams.shareId }}" />
</div> </div>

View File

@ -13,7 +13,7 @@
<li class="span2 text-center thumbnail-container" ng-repeat="file in files"> <li class="span2 text-center thumbnail-container" ng-repeat="file in files">
<div class="thumbnail"> <div class="thumbnail">
<a ng-click="openFile(file)"> <a ng-click="openFile(file)">
<img class="thumbnail-file" ng-src="api/file/{{ file.id }}/data?thumbnail=true&share={{ $stateParams.shareId }}" tooltip="{{ file.mimetype }}" tooltip-placement="top" /> <img class="thumbnail-file" ng-src="api/file/{{ file.id }}/data?size=thumb&share={{ $stateParams.shareId }}" tooltip="{{ file.mimetype }}" tooltip-placement="top" />
</a> </a>
</div> </div>
</li> </li>

View File

@ -1,6 +1,20 @@
package com.sismics.docs.rest; package com.sismics.docs.rest;
import java.io.BufferedInputStream;
import java.io.InputStream;
import java.util.Date;
import javax.ws.rs.core.MediaType;
import junit.framework.Assert;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.joda.time.format.DateTimeFormat;
import org.junit.Test;
import com.sismics.docs.rest.filter.CookieAuthenticationFilter; import com.sismics.docs.rest.filter.CookieAuthenticationFilter;
import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status; import com.sun.jersey.api.client.ClientResponse.Status;
@ -9,18 +23,6 @@ import com.sun.jersey.core.util.MultivaluedMapImpl;
import com.sun.jersey.multipart.FormDataBodyPart; import com.sun.jersey.multipart.FormDataBodyPart;
import com.sun.jersey.multipart.FormDataMultiPart; import com.sun.jersey.multipart.FormDataMultiPart;
import junit.framework.Assert;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Test;
import java.io.BufferedInputStream;
import java.io.InputStream;
import java.util.Date;
import javax.ws.rs.core.MediaType;
/** /**
* Exhaustive test of the document resource. * Exhaustive test of the document resource.
* *
@ -108,20 +110,29 @@ public class TestDocumentResource extends BaseJerseyTest {
Assert.assertEquals("SuperTag", tags.getJSONObject(0).getString("name")); Assert.assertEquals("SuperTag", tags.getJSONObject(0).getString("name"));
Assert.assertEquals("#ffff00", tags.getJSONObject(0).getString("color")); Assert.assertEquals("#ffff00", tags.getJSONObject(0).getString("color"));
// Search documents by query // Search documents by query in full content
documentResource = resource().path("/document/list"); documentResource = resource().path("/document/list");
documentResource.addFilter(new CookieAuthenticationFilter(document1Token)); documentResource.addFilter(new CookieAuthenticationFilter(document1Token));
getParams = new MultivaluedMapImpl(); getParams = new MultivaluedMapImpl();
getParams.putSingle("search", "uranium"); getParams.putSingle("search", "full:uranium");
response = documentResource.queryParams(getParams).get(ClientResponse.class); response = documentResource.queryParams(getParams).get(ClientResponse.class);
json = response.getEntity(JSONObject.class); json = response.getEntity(JSONObject.class);
Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
documents = json.getJSONArray("documents"); documents = json.getJSONArray("documents");
Assert.assertTrue(documents.length() == 1); Assert.assertTrue(documents.length() == 1);
Assert.assertEquals(document1Id, documents.getJSONObject(0).getString("id"));
Assert.assertEquals(create1Date, documents.getJSONObject(0).getLong("create_date"));
// Search documents by query // Search documents by query in full content
documentResource = resource().path("/document/list");
documentResource.addFilter(new CookieAuthenticationFilter(document1Token));
getParams = new MultivaluedMapImpl();
getParams.putSingle("search", "full:title");
response = documentResource.queryParams(getParams).get(ClientResponse.class);
json = response.getEntity(JSONObject.class);
Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
documents = json.getJSONArray("documents");
Assert.assertTrue(documents.length() == 1);
// Search documents by query in title/description
documentResource = resource().path("/document/list"); documentResource = resource().path("/document/list");
documentResource.addFilter(new CookieAuthenticationFilter(document1Token)); documentResource.addFilter(new CookieAuthenticationFilter(document1Token));
getParams = new MultivaluedMapImpl(); getParams = new MultivaluedMapImpl();
@ -132,7 +143,7 @@ public class TestDocumentResource extends BaseJerseyTest {
documents = json.getJSONArray("documents"); documents = json.getJSONArray("documents");
Assert.assertTrue(documents.length() == 1); Assert.assertTrue(documents.length() == 1);
// Search documents by query // Search documents by query in title/description
documentResource = resource().path("/document/list"); documentResource = resource().path("/document/list");
documentResource.addFilter(new CookieAuthenticationFilter(document1Token)); documentResource.addFilter(new CookieAuthenticationFilter(document1Token));
getParams = new MultivaluedMapImpl(); getParams = new MultivaluedMapImpl();
@ -143,7 +154,19 @@ public class TestDocumentResource extends BaseJerseyTest {
documents = json.getJSONArray("documents"); documents = json.getJSONArray("documents");
Assert.assertTrue(documents.length() == 1); Assert.assertTrue(documents.length() == 1);
// Search documents by date // Search documents by specific date
documentResource = resource().path("/document/list");
documentResource.addFilter(new CookieAuthenticationFilter(document1Token));
getParams = new MultivaluedMapImpl();
getParams.putSingle("search", "at:" + DateTimeFormat.forPattern("yyyy-MM").print(new Date().getTime()));
response = documentResource.queryParams(getParams).get(ClientResponse.class);
json = response.getEntity(JSONObject.class);
Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
documents = json.getJSONArray("documents");
Assert.assertTrue(documents.length() == 1);
Assert.assertEquals(document1Id, documents.getJSONObject(0).getString("id"));
// Search documents by date span
documentResource = resource().path("/document/list"); documentResource = resource().path("/document/list");
documentResource.addFilter(new CookieAuthenticationFilter(document1Token)); documentResource.addFilter(new CookieAuthenticationFilter(document1Token));
getParams = new MultivaluedMapImpl(); getParams = new MultivaluedMapImpl();
@ -197,7 +220,7 @@ public class TestDocumentResource extends BaseJerseyTest {
documentResource = resource().path("/document/list"); documentResource = resource().path("/document/list");
documentResource.addFilter(new CookieAuthenticationFilter(document1Token)); documentResource.addFilter(new CookieAuthenticationFilter(document1Token));
getParams = new MultivaluedMapImpl(); getParams = new MultivaluedMapImpl();
getParams.putSingle("search", "after:2010 before:2040-08 tag:super shared:yes lang:eng uranium"); getParams.putSingle("search", "after:2010 before:2040-08 tag:super shared:yes lang:eng title description full:uranium");
response = documentResource.queryParams(getParams).get(ClientResponse.class); response = documentResource.queryParams(getParams).get(ClientResponse.class);
json = response.getEntity(JSONObject.class); json = response.getEntity(JSONObject.class);
Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
@ -216,6 +239,17 @@ public class TestDocumentResource extends BaseJerseyTest {
documents = json.getJSONArray("documents"); documents = json.getJSONArray("documents");
Assert.assertTrue(documents.length() == 0); Assert.assertTrue(documents.length() == 0);
// Search documents (nothing)
documentResource = resource().path("/document/list");
documentResource.addFilter(new CookieAuthenticationFilter(document1Token));
getParams = new MultivaluedMapImpl();
getParams.putSingle("search", "full:random");
response = documentResource.queryParams(getParams).get(ClientResponse.class);
json = response.getEntity(JSONObject.class);
Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
documents = json.getJSONArray("documents");
Assert.assertTrue(documents.length() == 0);
// Search documents (nothing) // Search documents (nothing)
documentResource = resource().path("/document/list"); documentResource = resource().path("/document/list");
documentResource.addFilter(new CookieAuthenticationFilter(document1Token)); documentResource.addFilter(new CookieAuthenticationFilter(document1Token));