001    /**
002     * Copyright (c) 2000-2012 Liferay, Inc. All rights reserved.
003     *
004     * This library is free software; you can redistribute it and/or modify it under
005     * the terms of the GNU Lesser General Public License as published by the Free
006     * Software Foundation; either version 2.1 of the License, or (at your option)
007     * any later version.
008     *
009     * This library is distributed in the hope that it will be useful, but WITHOUT
010     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
011     * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
012     * details.
013     */
014    
015    package com.liferay.portlet.documentlibrary.util;
016    
017    import com.liferay.portal.kernel.exception.PortalException;
018    import com.liferay.portal.kernel.exception.SystemException;
019    import com.liferay.portal.kernel.log.Log;
020    import com.liferay.portal.kernel.log.LogFactoryUtil;
021    import com.liferay.portal.kernel.portlet.LiferayPortletURL;
022    import com.liferay.portal.kernel.portlet.LiferayWindowState;
023    import com.liferay.portal.kernel.search.BaseIndexer;
024    import com.liferay.portal.kernel.search.BooleanClauseOccur;
025    import com.liferay.portal.kernel.search.BooleanQuery;
026    import com.liferay.portal.kernel.search.BooleanQueryFactoryUtil;
027    import com.liferay.portal.kernel.search.Document;
028    import com.liferay.portal.kernel.search.DocumentImpl;
029    import com.liferay.portal.kernel.search.Field;
030    import com.liferay.portal.kernel.search.Indexer;
031    import com.liferay.portal.kernel.search.SearchContext;
032    import com.liferay.portal.kernel.search.SearchEngineUtil;
033    import com.liferay.portal.kernel.search.SearchException;
034    import com.liferay.portal.kernel.search.Summary;
035    import com.liferay.portal.kernel.util.ArrayUtil;
036    import com.liferay.portal.kernel.util.GetterUtil;
037    import com.liferay.portal.kernel.util.ListUtil;
038    import com.liferay.portal.kernel.util.PropsKeys;
039    import com.liferay.portal.kernel.util.StringPool;
040    import com.liferay.portal.kernel.util.StringUtil;
041    import com.liferay.portal.kernel.util.Validator;
042    import com.liferay.portal.kernel.workflow.WorkflowConstants;
043    import com.liferay.portal.model.Group;
044    import com.liferay.portal.security.permission.ActionKeys;
045    import com.liferay.portal.security.permission.PermissionChecker;
046    import com.liferay.portal.service.GroupLocalServiceUtil;
047    import com.liferay.portal.util.PortalUtil;
048    import com.liferay.portal.util.PortletKeys;
049    import com.liferay.portal.util.PrefsPropsUtil;
050    import com.liferay.portal.util.PropsValues;
051    import com.liferay.portlet.asset.model.AssetCategory;
052    import com.liferay.portlet.asset.service.AssetCategoryLocalServiceUtil;
053    import com.liferay.portlet.asset.service.AssetTagLocalServiceUtil;
054    import com.liferay.portlet.documentlibrary.model.DLFileEntry;
055    import com.liferay.portlet.documentlibrary.model.DLFileEntryMetadata;
056    import com.liferay.portlet.documentlibrary.model.DLFileEntryType;
057    import com.liferay.portlet.documentlibrary.model.DLFileVersion;
058    import com.liferay.portlet.documentlibrary.model.DLFolder;
059    import com.liferay.portlet.documentlibrary.model.DLFolderConstants;
060    import com.liferay.portlet.documentlibrary.service.DLFileEntryLocalServiceUtil;
061    import com.liferay.portlet.documentlibrary.service.DLFileEntryMetadataLocalServiceUtil;
062    import com.liferay.portlet.documentlibrary.service.DLFileEntryTypeLocalServiceUtil;
063    import com.liferay.portlet.documentlibrary.service.DLFolderLocalServiceUtil;
064    import com.liferay.portlet.documentlibrary.service.DLFolderServiceUtil;
065    import com.liferay.portlet.documentlibrary.service.permission.DLFileEntryPermission;
066    import com.liferay.portlet.dynamicdatamapping.model.DDMStructure;
067    import com.liferay.portlet.dynamicdatamapping.service.DDMStructureLocalServiceUtil;
068    import com.liferay.portlet.dynamicdatamapping.storage.Fields;
069    import com.liferay.portlet.dynamicdatamapping.storage.StorageEngineUtil;
070    import com.liferay.portlet.dynamicdatamapping.util.DDMIndexerUtil;
071    import com.liferay.portlet.expando.model.ExpandoBridge;
072    import com.liferay.portlet.expando.util.ExpandoBridgeFactoryUtil;
073    import com.liferay.portlet.expando.util.ExpandoBridgeIndexerUtil;
074    
075    import java.io.IOException;
076    import java.io.InputStream;
077    
078    import java.util.ArrayList;
079    import java.util.Collection;
080    import java.util.LinkedHashMap;
081    import java.util.List;
082    import java.util.Locale;
083    import java.util.Set;
084    import java.util.TreeSet;
085    
086    import javax.portlet.PortletRequest;
087    import javax.portlet.PortletURL;
088    import javax.portlet.WindowStateException;
089    
090    /**
091     * @author Brian Wing Shun Chan
092     * @author Raymond Augé
093     * @author Alexander Chow
094     */
095    public class DLIndexer extends BaseIndexer {
096    
097            public static final String[] CLASS_NAMES = {DLFileEntry.class.getName()};
098    
099            public static final String PORTLET_ID = PortletKeys.DOCUMENT_LIBRARY;
100    
101            public String[] getClassNames() {
102                    return CLASS_NAMES;
103            }
104    
105            public String getPortletId() {
106                    return PORTLET_ID;
107            }
108    
109            @Override
110            public boolean hasPermission(
111                            PermissionChecker permissionChecker, long entryClassPK,
112                            String actionId)
113                    throws Exception {
114    
115                    return DLFileEntryPermission.contains(
116                            permissionChecker, entryClassPK, ActionKeys.VIEW);
117            }
118    
119            @Override
120            public boolean isFilterSearch() {
121                    return _FILTER_SEARCH;
122            }
123    
124            @Override
125            public boolean isPermissionAware() {
126                    return _PERMISSION_AWARE;
127            }
128    
129            @Override
130            public void postProcessContextQuery(
131                            BooleanQuery contextQuery, SearchContext searchContext)
132                    throws Exception {
133    
134                    int status = GetterUtil.getInteger(
135                            searchContext.getAttribute(Field.STATUS),
136                            WorkflowConstants.STATUS_APPROVED);
137    
138                    if (status != WorkflowConstants.STATUS_ANY) {
139                            contextQuery.addRequiredTerm(Field.STATUS, status);
140                    }
141    
142                    long[] folderIds = searchContext.getFolderIds();
143    
144                    if ((folderIds != null) && (folderIds.length > 0)) {
145                            if (folderIds[0] == DLFolderConstants.DEFAULT_PARENT_FOLDER_ID) {
146                                    return;
147                            }
148    
149                            BooleanQuery folderIdsQuery = BooleanQueryFactoryUtil.create(
150                                    searchContext);
151    
152                            for (long folderId : folderIds) {
153                                    try {
154                                            DLFolderServiceUtil.getFolder(folderId);
155                                    }
156                                    catch (Exception e) {
157                                            continue;
158                                    }
159    
160                                    folderIdsQuery.addTerm(Field.FOLDER_ID, folderId);
161                            }
162    
163                            contextQuery.add(folderIdsQuery, BooleanClauseOccur.MUST);
164                    }
165            }
166    
167            @Override
168            public void postProcessSearchQuery(
169                            BooleanQuery searchQuery, SearchContext searchContext)
170                    throws Exception {
171    
172                    Set<DDMStructure> ddmStructuresSet = new TreeSet<DDMStructure>();
173    
174                    long[] groupIds = searchContext.getGroupIds();
175    
176                    if ((groupIds != null) && (groupIds.length > 0)) {
177                            List<DLFileEntryType> dlFileEntryTypes =
178                                    DLFileEntryTypeLocalServiceUtil.getFileEntryTypes(groupIds);
179    
180                            for (DLFileEntryType dlFileEntryType : dlFileEntryTypes) {
181                                    ddmStructuresSet.addAll(dlFileEntryType.getDDMStructures());
182                            }
183                    }
184    
185                    Group group = GroupLocalServiceUtil.getCompanyGroup(
186                            searchContext.getCompanyId());
187    
188                    DDMStructure tikaRawMetadataStructure =
189                            DDMStructureLocalServiceUtil.fetchStructure(
190                                    group.getGroupId(), "TikaRawMetadata");
191    
192                    if (tikaRawMetadataStructure != null) {
193                            ddmStructuresSet.add(tikaRawMetadataStructure);
194                    }
195    
196                    for (DDMStructure ddmStructure : ddmStructuresSet) {
197                            addSearchDDMStruture(searchQuery, searchContext, ddmStructure);
198                    }
199    
200                    addSearchTerm(searchQuery, searchContext, Field.USER_NAME, false);
201    
202                    addSearchTerm(searchQuery, searchContext, "extension", false);
203                    addSearchTerm(searchQuery, searchContext, "fileEntryTypeId", false);
204                    addSearchTerm(searchQuery, searchContext, "path", false);
205    
206                    LinkedHashMap<String, Object> params =
207                            (LinkedHashMap<String, Object>)searchContext.getAttribute("params");
208    
209                    if (params != null) {
210                            String expandoAttributes = (String)params.get("expandoAttributes");
211    
212                            if (Validator.isNotNull(expandoAttributes)) {
213                                    addSearchExpando(searchQuery, searchContext, expandoAttributes);
214                            }
215                    }
216            }
217    
218            protected void addFileEntryTypeAttributes(
219                            Document document, DLFileVersion dlFileVersion)
220                    throws PortalException, SystemException {
221    
222                    DLFileEntryType dlFileEntryType =
223                            DLFileEntryTypeLocalServiceUtil.getFileEntryType(
224                                    dlFileVersion.getFileEntryTypeId());
225    
226                    List<DDMStructure> ddmStructures = dlFileEntryType.getDDMStructures();
227    
228                    Group group = GroupLocalServiceUtil.getCompanyGroup(
229                            dlFileVersion.getCompanyId());
230    
231                    DDMStructure tikaRawMetadataStructure =
232                            DDMStructureLocalServiceUtil.fetchStructure(
233                                    group.getGroupId(), "TikaRawMetadata");
234    
235                    if (tikaRawMetadataStructure != null) {
236                            ddmStructures = ListUtil.copy(ddmStructures);
237    
238                            ddmStructures.add(tikaRawMetadataStructure);
239                    }
240    
241                    for (DDMStructure ddmStructure : ddmStructures) {
242                            Fields fields = null;
243    
244                            try {
245                                    DLFileEntryMetadata fileEntryMetadata =
246                                            DLFileEntryMetadataLocalServiceUtil.getFileEntryMetadata(
247                                                    ddmStructure.getStructureId(),
248                                                    dlFileVersion.getFileVersionId());
249    
250                                    fields = StorageEngineUtil.getFields(
251                                            fileEntryMetadata.getDDMStorageId());
252                            }
253                            catch (Exception e) {
254                            }
255    
256                            if (fields != null) {
257                                    DDMIndexerUtil.addAttributes(document, ddmStructure, fields);
258                            }
259                    }
260            }
261    
262            @Override
263            protected void doDelete(Object obj) throws Exception {
264                    DLFileEntry dlFileEntry = (DLFileEntry)obj;
265    
266                    Document document = new DocumentImpl();
267    
268                    document.addUID(PORTLET_ID, dlFileEntry.getFileEntryId());
269    
270                    SearchEngineUtil.deleteDocument(
271                            dlFileEntry.getCompanyId(), document.get(Field.UID));
272            }
273    
274            @Override
275            protected Document doGetDocument(Object obj) throws Exception {
276                    DLFileEntry dlFileEntry = (DLFileEntry)obj;
277    
278                    if (_log.isDebugEnabled()) {
279                            _log.debug("Indexing document " + dlFileEntry);
280                    }
281    
282                    boolean indexContent = true;
283    
284                    InputStream is = null;
285    
286                    try {
287                            if (PropsValues.DL_FILE_INDEXING_MAX_SIZE == 0) {
288                                    indexContent = false;
289                            }
290                            else if (PropsValues.DL_FILE_INDEXING_MAX_SIZE != -1) {
291                                    if (dlFileEntry.getSize() >
292                                                    PropsValues.DL_FILE_INDEXING_MAX_SIZE) {
293    
294                                            indexContent = false;
295                                    }
296                            }
297    
298                            if (indexContent) {
299                                    String[] ignoreExtensions = PrefsPropsUtil.getStringArray(
300                                            PropsKeys.DL_FILE_INDEXING_IGNORE_EXTENSIONS,
301                                            StringPool.COMMA);
302    
303                                    if (ArrayUtil.contains(
304                                                    ignoreExtensions,
305                                                    StringPool.PERIOD + dlFileEntry.getExtension())) {
306    
307                                            indexContent = false;
308                                    }
309                            }
310    
311                            if (indexContent) {
312                                    is = dlFileEntry.getFileVersion().getContentStream(false);
313                            }
314                    }
315                    catch (Exception e) {
316                    }
317    
318                    if (indexContent && (is == null)) {
319                            if (_log.isDebugEnabled()) {
320                                    _log.debug(
321                                            "Document " + dlFileEntry + " does not have any content");
322                            }
323    
324                            return null;
325                    }
326    
327                    try {
328                            Document document = new DocumentImpl();
329    
330                            long fileEntryId = dlFileEntry.getFileEntryId();
331    
332                            document.addUID(PORTLET_ID, fileEntryId);
333    
334                            List<AssetCategory> assetCategories =
335                                    AssetCategoryLocalServiceUtil.getCategories(
336                                            DLFileEntry.class.getName(), fileEntryId);
337    
338                            long[] assetCategoryIds = StringUtil.split(
339                                    ListUtil.toString(
340                                            assetCategories, AssetCategory.CATEGORY_ID_ACCESSOR),
341                                    0L);
342    
343                            document.addKeyword(Field.ASSET_CATEGORY_IDS, assetCategoryIds);
344    
345                            String[] assetCategoryNames = StringUtil.split(
346                                    ListUtil.toString(
347                                            assetCategories, AssetCategory.NAME_ACCESSOR));
348    
349                            document.addKeyword(Field.ASSET_CATEGORY_NAMES, assetCategoryNames);
350    
351                            String[] assetTagNames = AssetTagLocalServiceUtil.getTagNames(
352                                    DLFileEntry.class.getName(), fileEntryId);
353    
354                            document.addKeyword(Field.ASSET_TAG_NAMES, assetTagNames);
355    
356                            document.addKeyword(Field.COMPANY_ID, dlFileEntry.getCompanyId());
357    
358                            if (indexContent) {
359                                    try {
360                                            document.addFile(Field.CONTENT, is, dlFileEntry.getTitle());
361                                    }
362                                    catch (IOException ioe) {
363                                            throw new SearchException(
364                                                    "Cannot extract text from file" + dlFileEntry);
365                                    }
366                            }
367    
368                            document.addText(Field.DESCRIPTION, dlFileEntry.getDescription());
369                            document.addKeyword(
370                                    Field.ENTRY_CLASS_NAME, DLFileEntry.class.getName());
371                            document.addKeyword(Field.ENTRY_CLASS_PK, fileEntryId);
372                            document.addKeyword(Field.FOLDER_ID, dlFileEntry.getFolderId());
373                            document.addKeyword(
374                                    Field.GROUP_ID, getParentGroupId(dlFileEntry.getGroupId()));
375                            document.addDate(
376                                    Field.MODIFIED_DATE, dlFileEntry.getModifiedDate());
377                            document.addKeyword(Field.PORTLET_ID, PORTLET_ID);
378                            document.addText(
379                                    Field.PROPERTIES, dlFileEntry.getLuceneProperties());
380                            document.addKeyword(Field.SCOPE_GROUP_ID, dlFileEntry.getGroupId());
381    
382                            DLFileVersion dlFileVersion = dlFileEntry.getFileVersion();
383    
384                            document.addKeyword(Field.STATUS, dlFileVersion.getStatus());
385                            document.addText(Field.TITLE, dlFileEntry.getTitle());
386    
387                            long userId = dlFileEntry.getUserId();
388    
389                            document.addKeyword(Field.USER_ID, userId);
390                            document.addKeyword(
391                                    Field.USER_NAME, PortalUtil.getUserName(
392                                            userId, dlFileEntry.getUserName()),
393                                    true);
394    
395                            document.addKeyword(
396                                    "dataRepositoryId", dlFileEntry.getDataRepositoryId());
397                            document.addKeyword("extension", dlFileEntry.getExtension());
398                            document.addKeyword(
399                                    "fileEntryTypeId", dlFileEntry.getFileEntryTypeId());
400                            document.addKeyword("path", dlFileEntry.getTitle());
401    
402                            ExpandoBridge expandoBridge =
403                                    ExpandoBridgeFactoryUtil.getExpandoBridge(
404                                            dlFileEntry.getCompanyId(), DLFileEntry.class.getName(),
405                                            dlFileVersion.getFileVersionId());
406    
407                            ExpandoBridgeIndexerUtil.addAttributes(document, expandoBridge);
408    
409                            addFileEntryTypeAttributes(document, dlFileVersion);
410    
411                            if (_log.isDebugEnabled()) {
412                                    _log.debug("Document " + dlFileEntry + " indexed successfully");
413                            }
414    
415                            return document;
416                    }
417                    finally {
418                            if (is != null) {
419                                    try {
420                                            is.close();
421                                    }
422                                    catch (IOException ioe) {
423                                    }
424                            }
425                    }
426            }
427    
428            @Override
429            protected Summary doGetSummary(
430                    Document document, Locale locale, String snippet,
431                    PortletURL portletURL) {
432    
433                    LiferayPortletURL liferayPortletURL = (LiferayPortletURL)portletURL;
434    
435                    liferayPortletURL.setLifecycle(PortletRequest.ACTION_PHASE);
436    
437                    try {
438                            liferayPortletURL.setWindowState(LiferayWindowState.EXCLUSIVE);
439                    }
440                    catch (WindowStateException wse) {
441                    }
442    
443                    String title = document.get(Field.TITLE);
444    
445                    String content = snippet;
446    
447                    if (Validator.isNull(snippet)) {
448                            content = StringUtil.shorten(document.get(Field.CONTENT), 200);
449                    }
450    
451                    String fileEntryId = document.get(Field.ENTRY_CLASS_PK);
452    
453                    portletURL.setParameter("struts_action", "/document_library/get_file");
454                    portletURL.setParameter("fileEntryId", fileEntryId);
455    
456                    return new Summary(title, content, portletURL);
457            }
458    
459            @Override
460            protected void doReindex(Object obj) throws Exception {
461                    DLFileEntry dlFileEntry = (DLFileEntry)obj;
462    
463                    DLFileVersion dlFileVersion = dlFileEntry.getLatestFileVersion(true);
464    
465                    if (!dlFileVersion.isApproved()) {
466                            return;
467                    }
468    
469                    Document document = getDocument(dlFileEntry);
470    
471                    if (document != null) {
472                            SearchEngineUtil.updateDocument(
473                                    dlFileEntry.getCompanyId(), document);
474                    }
475            }
476    
477            @Override
478            protected void doReindex(String className, long classPK) throws Exception {
479                    DLFileEntry dlFileEntry = DLFileEntryLocalServiceUtil.getFileEntry(
480                            classPK);
481    
482                    doReindex(dlFileEntry);
483            }
484    
485            @Override
486            protected void doReindex(String[] ids) throws Exception {
487                    if (ids.length == 1) {
488                            long companyId = GetterUtil.getLong(ids[0]);
489    
490                            reindexFolders(companyId);
491                            reindexRoot(companyId);
492                    }
493                    else {
494                            long companyId = GetterUtil.getLong(ids[0]);
495                            long groupId = GetterUtil.getLong(ids[2]);
496                            long dataRepositoryId = GetterUtil.getLong(ids[3]);
497    
498                            reindexFileEntries(companyId, groupId, dataRepositoryId);
499                    }
500            }
501    
502            @Override
503            protected String getPortletId(SearchContext searchContext) {
504                    return PORTLET_ID;
505            }
506    
507            protected void reindexFileEntries(
508                            long companyId, long groupId, long dataRepositoryId)
509                    throws Exception {
510    
511                    long folderId = DLFolderConstants.getFolderId(
512                            groupId, dataRepositoryId);
513    
514                    int fileEntriesCount = DLFileEntryLocalServiceUtil.getFileEntriesCount(
515                            companyId, folderId);
516    
517                    int fileEntriesPages = fileEntriesCount / Indexer.DEFAULT_INTERVAL;
518    
519                    for (int i = 0; i <= fileEntriesPages; i++) {
520                            int fileEntriesStart = (i * Indexer.DEFAULT_INTERVAL);
521                            int fileEntriesEnd = fileEntriesStart + Indexer.DEFAULT_INTERVAL;
522    
523                            reindexFileEntries(
524                                    companyId, groupId, folderId, fileEntriesStart, fileEntriesEnd);
525                    }
526            }
527    
528            protected void reindexFileEntries(
529                            long companyId, long groupId, long folderId, int fileEntriesStart,
530                            int fileEntriesEnd)
531                    throws Exception {
532    
533                    Collection<Document> documents = new ArrayList<Document>();
534    
535                    List<DLFileEntry> dlFileEntries =
536                            DLFileEntryLocalServiceUtil.getFileEntries(
537                                    groupId, folderId, fileEntriesStart, fileEntriesEnd, null);
538    
539                    for (DLFileEntry dlFileEntry : dlFileEntries) {
540                            Document document = getDocument(dlFileEntry);
541    
542                            if (document != null) {
543                                    documents.add(document);
544                            }
545                    }
546    
547                    SearchEngineUtil.updateDocuments(companyId, documents);
548            }
549    
550            protected void reindexFolders(long companyId) throws Exception {
551                    int folderCount = DLFolderLocalServiceUtil.getCompanyFoldersCount(
552                            companyId);
553    
554                    int folderPages = folderCount / Indexer.DEFAULT_INTERVAL;
555    
556                    for (int i = 0; i <= folderPages; i++) {
557                            int folderStart = (i * Indexer.DEFAULT_INTERVAL);
558                            int folderEnd = folderStart + Indexer.DEFAULT_INTERVAL;
559    
560                            reindexFolders(companyId, folderStart, folderEnd);
561                    }
562            }
563    
564            protected void reindexFolders(
565                            long companyId, int folderStart, int folderEnd)
566                    throws Exception {
567    
568                    List<DLFolder> dlFolders = DLFolderLocalServiceUtil.getCompanyFolders(
569                            companyId, folderStart, folderEnd);
570    
571                    for (DLFolder dlFolder : dlFolders) {
572                            String portletId = PortletKeys.DOCUMENT_LIBRARY;
573                            long groupId = dlFolder.getGroupId();
574                            long folderId = dlFolder.getFolderId();
575    
576                            String[] newIds = {
577                                    String.valueOf(companyId), portletId, String.valueOf(groupId),
578                                    String.valueOf(folderId)
579                            };
580    
581                            reindex(newIds);
582                    }
583            }
584    
585            protected void reindexRoot(long companyId) throws Exception {
586                    int groupCount = GroupLocalServiceUtil.getCompanyGroupsCount(companyId);
587    
588                    int groupPages = groupCount / Indexer.DEFAULT_INTERVAL;
589    
590                    for (int i = 0; i <= groupPages; i++) {
591                            int groupStart = (i * Indexer.DEFAULT_INTERVAL);
592                            int groupEnd = groupStart + Indexer.DEFAULT_INTERVAL;
593    
594                            reindexRoot(companyId, groupStart, groupEnd);
595                    }
596            }
597    
598            protected void reindexRoot(long companyId, int groupStart, int groupEnd)
599                    throws Exception {
600    
601                    List<Group> groups = GroupLocalServiceUtil.getCompanyGroups(
602                            companyId, groupStart, groupEnd);
603    
604                    for (Group group : groups) {
605                            String portletId = PortletKeys.DOCUMENT_LIBRARY;
606                            long groupId = group.getGroupId();
607                            long folderId = groupId;
608    
609                            String[] newIds = {
610                                    String.valueOf(companyId), portletId, String.valueOf(groupId),
611                                    String.valueOf(folderId)
612                            };
613    
614                            reindex(newIds);
615                    }
616            }
617    
618            private static final boolean _FILTER_SEARCH = true;
619    
620            private static final boolean _PERMISSION_AWARE = true;
621    
622            private static Log _log = LogFactoryUtil.getLog(DLIndexer.class);
623    
624    }