Closes #48: Delete linked data properly + batch to clean orphan data

docs-core/src/main/java/com/sismics/docs/core/dao/jpa/DocumentDao.java

@@ -67,6 +67,20 @@ public class DocumentDao {
         return q.getResultList();
     }
     
+    /**
+     * Returns the list of all documents from a user.
+     * 
+     * @param userId User ID
+     * @return List of documents
+     */
+    @SuppressWarnings("unchecked")
+    public List<Document> findByUserId(String userId) {
+        EntityManager em = ThreadLocalContext.get().getEntityManager();
+        Query q = em.createQuery("select d from Document d where d.userId = :userId and d.deleteDate is null");
+        q.setParameter("userId", userId);
+        return q.getResultList();
+    }
+    
     /**
      * Returns an active document.
      * 
@@ -149,13 +163,12 @@ public class DocumentDao {
         q.setParameter("dateNow", dateNow);
         q.executeUpdate();
         
-        // TODO Delete share from deleted ACLs
+        q = em.createQuery("update Acl a set a.deleteDate = :dateNow where a.sourceId = :documentId and a.deleteDate is null");
-//        q = em.createQuery("update Share s set s.deleteDate = :dateNow where s.documentId = :documentId and s.deleteDate is null");
+        q.setParameter("documentId", id);
-//        q.setParameter("documentId", id);
+        q.setParameter("dateNow", dateNow);
-//        q.setParameter("dateNow", dateNow);
+        q.executeUpdate();
-//        q.executeUpdate();
         
-        q = em.createQuery("update Acl a set a.deleteDate = :dateNow where a.sourceId = :documentId");
+        q = em.createQuery("update DocumentTag dt set dt.deleteDate = :dateNow where dt.documentId = :documentId and dt.deleteDate is not null");
         q.setParameter("documentId", id);
         q.setParameter("dateNow", dateNow);
         q.executeUpdate();

docs-core/src/main/java/com/sismics/docs/core/dao/jpa/FileDao.java

@@ -53,6 +53,20 @@ public class FileDao {
         return q.getResultList();
     }
     
+    /**
+     * Returns the list of all files from a user.
+     * 
+     * @param userId User ID
+     * @return List of files
+     */
+    @SuppressWarnings("unchecked")
+    public List<File> findByUserId(String userId) {
+        EntityManager em = ThreadLocalContext.get().getEntityManager();
+        Query q = em.createQuery("select f from File f where f.userId = :userId and f.deleteDate is null");
+        q.setParameter("userId", userId);
+        return q.getResultList();
+    }
+    
     /**
      * Returns an active file.
      * 

docs-core/src/main/java/com/sismics/docs/core/dao/jpa/TagDao.java

@@ -240,10 +240,12 @@ public class TagDao {
         Tag tagDb = (Tag) q.getSingleResult();
         
         // Delete the tag
-        tagDb.setDeleteDate(new Date());
+        Date dateNow = new Date();
+        tagDb.setDeleteDate(dateNow);
 
         // Delete linked data
-        q = em.createQuery("delete DocumentTag dt where dt.tagId = :tagId");
+        q = em.createQuery("update DocumentTag dt set dt.deleteDate = :dateNow where dt.tagId = :tagId and dt.deleteDate is not null");
+        q.setParameter("dateNow", dateNow);
         q.setParameter("tagId", tagId);
         q.executeUpdate();
         
@@ -291,3 +293,4 @@ public class TagDao {
         return tagFromDb;
     }
 }
+

docs-core/src/main/java/com/sismics/docs/core/dao/jpa/UserDao.java

@@ -204,6 +204,26 @@ public class UserDao {
         q.setParameter("userId", userFromDb.getId());
         q.executeUpdate();
         
+        q = em.createQuery("update Document d set d.deleteDate = :dateNow where d.userId = :userId and d.deleteDate is null");
+        q.setParameter("userId", userFromDb.getId());
+        q.setParameter("dateNow", dateNow);
+        q.executeUpdate();
+        
+        q = em.createQuery("update File f set f.deleteDate = :dateNow where f.userId = :userId and f.deleteDate is null");
+        q.setParameter("userId", userFromDb.getId());
+        q.setParameter("dateNow", dateNow);
+        q.executeUpdate();
+        
+        q = em.createQuery("update Acl a set a.deleteDate = :dateNow where a.targetId = :userId and a.deleteDate is null");
+        q.setParameter("userId", userFromDb.getId());
+        q.setParameter("dateNow", dateNow);
+        q.executeUpdate();
+        
+        q = em.createQuery("update Comment c set c.deleteDate = :dateNow where c.userId = :userId and c.deleteDate is null");
+        q.setParameter("userId", userFromDb.getId());
+        q.setParameter("dateNow", dateNow);
+        q.executeUpdate();
+        
         // Create audit log
         AuditLogUtil.create(userFromDb, AuditLogType.DELETE);
     }

docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java

@@ -11,6 +11,8 @@ import java.util.ResourceBundle;
 import javax.json.Json;
 import javax.json.JsonArrayBuilder;
 import javax.json.JsonObjectBuilder;
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
@@ -19,7 +21,8 @@ import javax.ws.rs.core.Response;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Appender;
-import org.apache.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.sismics.docs.core.dao.jpa.FileDao;
 import com.sismics.docs.core.dao.jpa.UserDao;
@@ -33,6 +36,7 @@ import com.sismics.docs.core.util.jpa.PaginatedLists;
 import com.sismics.docs.rest.constant.BaseFunction;
 import com.sismics.rest.exception.ForbiddenClientException;
 import com.sismics.rest.exception.ServerException;
+import com.sismics.util.context.ThreadLocalContext;
 import com.sismics.util.log4j.LogCriteria;
 import com.sismics.util.log4j.LogEntry;
 import com.sismics.util.log4j.MemoryAppender;
@@ -44,6 +48,11 @@ import com.sismics.util.log4j.MemoryAppender;
  */
 @Path("/app")
 public class AppResource extends BaseResource {
+    /**
+     * Logger.
+     */
+    private static final Logger log = LoggerFactory.getLogger(AppResource.class);
+    
     /**
      * Return the information about the application.
      * 
@@ -92,7 +101,7 @@ public class AppResource extends BaseResource {
         // TODO Change level by minLevel (returns all logs above)
 
         // Get the memory appender
-        Logger logger = Logger.getRootLogger();
+        org.apache.log4j.Logger logger = org.apache.log4j.Logger.getRootLogger();
         Appender appender = logger.getAppender("MEMORY");
         if (appender == null || !(appender instanceof MemoryAppender)) {
             throw new ServerException("ServerError", "MEMORY appender not configured");
@@ -168,6 +177,7 @@ public class AppResource extends BaseResource {
         for (File file : fileList) {
             fileMap.put(file.getId(), file);
         }
+        log.info("Checking {} files", fileMap.size());
         
         // Check if each stored file is valid
         try (DirectoryStream<java.nio.file.Path> storedFileList = Files.newDirectoryStream(DirectoryUtil.getStorageDirectory())) {
@@ -175,6 +185,7 @@ public class AppResource extends BaseResource {
                 String fileName = storedFile.getFileName().toString();
                 String[] fileNameArray = fileName.split("_");
                 if (!fileMap.containsKey(fileNameArray[0])) {
+                    log.info("Deleting orphan files at this location: {}", storedFile);
                     Files.delete(storedFile);
                 }
             }
@@ -182,6 +193,54 @@ public class AppResource extends BaseResource {
             throw new ServerException("FileError", "Error deleting orphan files", e);
         }
         
+        // Hard delete orphan audit logs
+        EntityManager em = ThreadLocalContext.get().getEntityManager();
+        StringBuilder sb = new StringBuilder("delete from T_AUDIT_LOG al where al.LOG_ID_C in (select al.LOG_ID_C from T_AUDIT_LOG al ");
+        sb.append(" left join T_DOCUMENT d on d.DOC_ID_C = al.LOG_IDENTITY_C and d.DOC_DELETEDATE_D is null ");
+        sb.append(" left join T_ACL a on a.ACL_ID_C = al.LOG_IDENTITY_C and a.ACL_DELETEDATE_D is null ");
+        sb.append(" left join T_COMMENT c on c.COM_ID_C = al.LOG_IDENTITY_C and c.COM_DELETEDATE_D is null ");
+        sb.append(" left join T_FILE f on f.FIL_ID_C = al.LOG_IDENTITY_C and f.FIL_DELETEDATE_D is null ");
+        sb.append(" left join T_TAG t on t.TAG_ID_C = al.LOG_IDENTITY_C and t.TAG_DELETEDATE_D is null ");
+        sb.append(" left join T_USER u on u.USE_ID_C = al.LOG_IDENTITY_C and u.USE_DELETEDATE_D is null ");
+        sb.append(" where d.DOC_ID_C is null and a.ACL_ID_C is null and c.COM_ID_C is null and f.FIL_ID_C is null and t.TAG_ID_C is null and u.USE_ID_C is null)");
+        Query q = em.createNativeQuery(sb.toString());
+        log.info("Deleting {} orphan audit logs", q.executeUpdate());
+        
+        // Hard delete orphan ACLs
+        sb = new StringBuilder("delete from T_ACL a where a.ACL_ID_C in (select a.ACL_ID_C from T_ACL a ");
+        sb.append(" left join T_SHARE s on s.SHA_ID_C = a.ACL_TARGETID_C ");
+        sb.append(" left join T_USER u on u.USE_ID_C = a.ACL_TARGETID_C ");
+        sb.append(" left join T_DOCUMENT d on d.DOC_ID_C = a.ACL_SOURCEID_C ");
+        sb.append(" where s.SHA_ID_C is null and u.USE_ID_C is null or d.DOC_ID_C is null)");
+        q = em.createNativeQuery(sb.toString());
+        log.info("Deleting {} orphan ACLs", q.executeUpdate());
+        
+        // Hard delete orphan comments
+        q = em.createNativeQuery("delete from T_COMMENT c where c.COM_ID_C in (select c.COM_ID_C from T_COMMENT c left join T_DOCUMENT d on d.DOC_ID_C = c.COM_IDDOC_C and d.DOC_DELETEDATE_D is null where d.DOC_ID_C is null)");
+        log.info("Deleting {} orphan comments", q.executeUpdate());
+        
+        // Hard delete orphan document tag links
+        q = em.createNativeQuery("delete from T_DOCUMENT_TAG dt where dt.DOT_ID_C in (select dt.DOT_ID_C from T_DOCUMENT_TAG dt left join T_DOCUMENT d on dt.DOT_IDDOCUMENT_C = d.DOC_ID_C and d.DOC_DELETEDATE_D is null left join T_TAG t on t.TAG_ID_C = dt.DOT_IDTAG_C and t.TAG_DELETEDATE_D is null where d.DOC_ID_C is null or t.TAG_ID_C is null)");
+        log.info("Deleting {} orphan document tag links", q.executeUpdate());
+        
+        // Hard delete orphan shares
+        q = em.createNativeQuery("delete from T_SHARE s where s.SHA_ID_C in (select s.SHA_ID_C from T_SHARE s left join T_ACL a on a.ACL_TARGETID_C = s.SHA_ID_C and a.ACL_DELETEDATE_D is null where a.ACL_ID_C is null)");
+        log.info("Deleting {} orphan shares", q.executeUpdate());
+        
+        // Hard delete orphan tags
+        q = em.createNativeQuery("delete from T_TAG t where t.TAG_ID_C in (select t.TAG_ID_C from T_TAG t left join T_USER u on u.USE_ID_C = t.TAG_IDUSER_C and u.USE_DELETEDATE_D is null where u.USE_ID_C is null)");
+        log.info("Deleting {} orphan tags", q.executeUpdate());
+        
+        // Hard delete softly deleted data
+        log.info("Deleting {} soft deleted document tag links", em.createQuery("delete DocumentTag dt where dt.deleteDate is not null").executeUpdate());
+        log.info("Deleting {} soft deleted ACLs", em.createQuery("delete Acl a where a.deleteDate is not null").executeUpdate());
+        log.info("Deleting {} soft deleted shares", em.createQuery("delete Share s where s.deleteDate is not null").executeUpdate());
+        log.info("Deleting {} soft deleted tags", em.createQuery("delete Tag t where t.deleteDate is not null").executeUpdate());
+        log.info("Deleting {} soft deleted comments", em.createQuery("delete Comment c where c.deleteDate is not null").executeUpdate());
+        log.info("Deleting {} soft deleted files", em.createQuery("delete File f where f.deleteDate is not null").executeUpdate());
+        log.info("Deleting {} soft deleted documents", em.createQuery("delete Document d where d.deleteDate is not null").executeUpdate());
+        log.info("Deleting {} soft deleted users", em.createQuery("delete User u where u.deleteDate is not null").executeUpdate());
+        
         // Always return OK
         JsonObjectBuilder response = Json.createObjectBuilder()
                 .add("status", "ok");

docs-web/src/main/java/com/sismics/docs/rest/resource/UserResource.java

@@ -2,6 +2,7 @@ package com.sismics.docs.rest.resource;
 
 import java.security.NoSuchAlgorithmException;
 import java.util.Date;
+import java.util.List;
 import java.util.Set;
 
 import javax.json.Json;
@@ -26,11 +27,18 @@ import org.apache.commons.lang.StringUtils;
 import com.google.common.base.Strings;
 import com.sismics.docs.core.constant.Constants;
 import com.sismics.docs.core.dao.jpa.AuthenticationTokenDao;
+import com.sismics.docs.core.dao.jpa.DocumentDao;
+import com.sismics.docs.core.dao.jpa.FileDao;
 import com.sismics.docs.core.dao.jpa.RoleBaseFunctionDao;
 import com.sismics.docs.core.dao.jpa.UserDao;
 import com.sismics.docs.core.dao.jpa.criteria.UserCriteria;
 import com.sismics.docs.core.dao.jpa.dto.UserDto;
+import com.sismics.docs.core.event.DocumentDeletedAsyncEvent;
+import com.sismics.docs.core.event.FileDeletedAsyncEvent;
+import com.sismics.docs.core.model.context.AppContext;
 import com.sismics.docs.core.model.jpa.AuthenticationToken;
+import com.sismics.docs.core.model.jpa.Document;
+import com.sismics.docs.core.model.jpa.File;
 import com.sismics.docs.core.model.jpa.User;
 import com.sismics.docs.core.util.EncryptionUtil;
 import com.sismics.docs.core.util.jpa.PaginatedList;
@@ -345,10 +353,30 @@ public class UserResource extends BaseResource {
             throw new ClientException("ForbiddenError", "The admin user cannot be deleted");
         }
         
+        // Find linked data
+        DocumentDao documentDao = new DocumentDao();
+        List<Document> documentList = documentDao.findByUserId(principal.getId());
+        FileDao fileDao = new FileDao();
+        List<File> fileList = fileDao.findByUserId(principal.getId());
+        
         // Delete the user
         UserDao userDao = new UserDao();
         userDao.delete(principal.getName());
         
+        // Raise deleted events for documents
+        for (Document document : documentList) {
+            DocumentDeletedAsyncEvent documentDeletedAsyncEvent = new DocumentDeletedAsyncEvent();
+            documentDeletedAsyncEvent.setDocument(document);
+            AppContext.getInstance().getAsyncEventBus().post(documentDeletedAsyncEvent);
+        }
+        
+        // Raise deleted events for files
+        for (File file : fileList) {
+            FileDeletedAsyncEvent fileDeletedAsyncEvent = new FileDeletedAsyncEvent();
+            fileDeletedAsyncEvent.setFile(file);
+            AppContext.getInstance().getAsyncEventBus().post(fileDeletedAsyncEvent);
+        }
+        
         // Always return OK
         JsonObjectBuilder response = Json.createObjectBuilder()
                 .add("status", "ok");
@@ -383,9 +411,29 @@ public class UserResource extends BaseResource {
             throw new ClientException("ForbiddenError", "The admin user cannot be deleted");
         }
         
+        // Find linked data
+        DocumentDao documentDao = new DocumentDao();
+        List<Document> documentList = documentDao.findByUserId(user.getId());
+        FileDao fileDao = new FileDao();
+        List<File> fileList = fileDao.findByUserId(user.getId());
+        
         // Delete the user
         userDao.delete(user.getUsername());
         
+        // Raise deleted events for documents
+        for (Document document : documentList) {
+            DocumentDeletedAsyncEvent documentDeletedAsyncEvent = new DocumentDeletedAsyncEvent();
+            documentDeletedAsyncEvent.setDocument(document);
+            AppContext.getInstance().getAsyncEventBus().post(documentDeletedAsyncEvent);
+        }
+        
+        // Raise deleted events for files
+        for (File file : fileList) {
+            FileDeletedAsyncEvent fileDeletedAsyncEvent = new FileDeletedAsyncEvent();
+            fileDeletedAsyncEvent.setFile(file);
+            AppContext.getInstance().getAsyncEventBus().post(fileDeletedAsyncEvent);
+        }
+        
         // Always return OK
         JsonObjectBuilder response = Json.createObjectBuilder()
                 .add("status", "ok");