Serving non-Sitecore data in Sitecore. Part 1: Configuring Solr cores for external data and cross-core search.

Recently I had a chance to work on an interesting project where we needed not only to consume data from a non-Sitecore application through API, but also make it look like the data lives in Sitecore which means that urls had to be consistent with the rest of the Sitecore solution and search would bring both Sitecore and non-Sitecore results.

The solution that I came up with includes custom Solr (4.3.1) index for non-Sitecore data, two Solr cores that serve aggregate results from non-Sitecore core with master one and non-Sitecore core with web core, custom Sitecore Multilist with Search field that accepts _indexname parameter, custom crawler and a couple of other components.

Part 1: Configuring Solr core for external data

First of all I would like to thank Cameron Palmer for his post on indexing of external data (http://www.awareweb.com/awareblog/9-30-14-indexingsitecore7). This post was instrumental for this solution.

The external data that needed to be indexed lived in a system called netFORUM. netFORUM has a very nice API, but the focus of this post series is on Sitecore solution, so I'm going to omit everything related to the actual data retrieval. Lets just say, there is a layer in Sitecore solution that does all the work of pulling in the data. There is class in the solution that holds all external object properties and is called ExternalProduct.

Here are the classes that were created to enable external data indexing:

IndexableProductField.cs

    public class IndexableProductField : IIndexableDataField
    {
        private object _concreteObject;
        private PropertyInfo _fieldInfo;

        public IndexableProductField(object concreteObject, PropertyInfo fieldInfo)
        {
            this._concreteObject = concreteObject;
            this._fieldInfo = fieldInfo;
        }

        public Type FieldType
        {
            get { return _fieldInfo.PropertyType; }
        }

        public object Id
        {
            get { return _fieldInfo.Name.ToLower(); }
        }

        public string Name
        {
            get { return _fieldInfo.Name; }
        }

        public string TypeKey
        {
            get { return "string"; }
        }

        public object Value
        {
            get
            {
                return _fieldInfo.GetValue(_concreteObject);
            }
        }
    }
       

IndexableProductEntity.cs

    public class IndexableProductEntity : IIndexable, IIndexableBuiltinFields
    {
        private ExtendedProduct _entity;
        private UrlProductType _productType;
        private IEnumerable _fields;
        private const string _databaseName = "netforum";
        public virtual IIndexFieldStorageValueFormatter IndexFieldStorageValueFormatter { get; set; }

        public IndexableProductEntity(ExtendedProduct entity, UrlProductType productType)
        {
            _entity = entity;
            _fields = _entity.GetType()
                   .GetProperties(BindingFlags.Public
                    | BindingFlags.Instance
                    | BindingFlags.IgnoreCase)
                   .Select(fi => new IndexableProductField(_entity, fi));
            _productType = productType;       
        }

        public virtual string AbsolutePath
        {
            get { return "/"; }
        }

        public CultureInfo Culture
        {
            get { return CultureInfo.CurrentCulture; }
        }

        public virtual IEnumerable Fields
        {
            get { return _fields; }
        }

        public IIndexableDataField GetFieldById(object fieldId)
        {
            return _fields.FirstOrDefault(f => f.Id == fieldId);
        }

        public IIndexableDataField GetFieldByName(string fieldName)
        {
            return _fields.FirstOrDefault(f => f.Name.ToLower() == fieldName.ToLower());
        }

        public IIndexableId Id
        {
            get { return new IndexableId(_entity.Id); }
        }

        public void LoadAllFields()
        {
            _fields = _entity.GetType()
                       .GetProperties(BindingFlags.Public
                            | BindingFlags.Instance
                            | BindingFlags.IgnoreCase)
                   .Select(fi => new IndexableProductField(_entity, fi));
        }

        public IIndexableUniqueId UniqueId
        {
            get
            {
                var uri = new ItemUri(ID.Parse(_entity.Id).ToString(), Sitecore.Context.Language, Sitecore.Data.Version.First, _databaseName);
                return new IndexableUniqueId(new SitecoreItemUniqueId(uri));
            }
        }

        public virtual string DataSource
        {
            get { return "NetForumEntity"; }
        }

        public virtual Guid ProductTypeId
        {
            get
            {
                return _entity.ProductTypeId;
            }
        }
        public virtual string ProductCode
        {
            get
            {
                return _entity.ProductCode;
            }
        }
        public virtual string Name
        {
            get
            {
                return _entity.ProductName;
            }
        }

        object IIndexableBuiltinFields.Group
        {
            get
            {
                return (object)_entity.Id;
            }
        }
        bool IIndexableBuiltinFields.IsClone
        {
            get
            {
                return false;
            }
        }
        int IIndexableBuiltinFields.Version
        {
            get
            {
                return 1;
            }
        }

        bool IIndexableBuiltinFields.IsLatestVersion
        {
            get
            {
                return true;
            }
            set { }
        }

        object IIndexableBuiltinFields.TemplateId
        {
            get
            {
                return (object)this.ProductTypeId;
            }
        }
        string IIndexableBuiltinFields.Language
        {
            get
            {
                return Sitecore.Context.Language.ToString();
            }
        }
        string IIndexableBuiltinFields.Database
        {
            get
            {
                return _databaseName;
            }
        }
        string IIndexableBuiltinFields.ID
        {
            get
            {
                return ShortID.Parse(_entity.Id.ToString()).ToString().ToLower();
            }
        }

        public string CreatedBy
        {
            get { return "netForum"; }
        }

        public DateTime CreatedDate
        {
            get { return _entity.CatalogDate; }
        }

        public string DisplayName
        {
            get { return _entity.ProductName; }
        }

        public string FullPath
        {
            get
            {
                var urlFormat = Settings.GetSetting("ProductUrlFormat", "{0}/{1}/{2}/{3}");
                return string.Format(urlFormat, _productType.ParentSiteItem.Paths.FullPath, _productType.Name, _entity.ProductCode, ItemUtil.ProposeValidItemName(_entity.ProductName));
            }
        }

        public object Parent
        {
            get { return _productType.ParentSiteItem.ID; }
        }

        public IEnumerable Paths
        {
            get { return new List(); }
        }

        public string TemplateName
        {
            get { return _entity.ProductTypeCode; }
        }

        public string UpdatedBy
        {
            get { return "netForum"; }
        }

        public DateTime UpdatedDate
        {
            get { return _entity.CatalogDate; }
        }
       

NetForumEntityCrawler.cs

    public class NetForumEntityCrawler : FlatDataCrawler
    {
        private readonly IExtendedProductService _productService;

        public NetForumEntityCrawler()
        {
            _productService = DependencyResolver.Current.GetService();
        }

        public NetForumEntityCrawler(IExtendedProductService productService)
        {
            _productService = productService;
        }

        protected override IEnumerable GetItemsToIndex()
        {
            var productTypes = UrlConfiguration.Instance(BusinessConstants.NetForumProducts.DatabaseName).ProductTypes;
            if (productTypes != null && productTypes.Any())
            {
                var productTypeIds = productTypes.Select(pt=>pt.ProductTypeId);
                return _productService.GetProductsByProductTypeIds(productTypeIds).Select(p => new IndexableProductEntity(p, productTypes.FirstOrDefault(pt => pt.ProductTypeId == p.ProductTypeId)));
            }
            return new List();
        }        

        protected override IndexableProductEntity GetIndexable(IIndexableUniqueId indexableUniqueId)
        {
            var productType = UrlConfiguration.Instance(BusinessConstants.NetForumProducts.DatabaseName).ProductTypes;
            if (productType != null && productType.Any())
            {
                var productTypeIds = productType.Select(pt => pt.ProductTypeId);
                var product = _productService.GetProduct(((Guid)indexableUniqueId.Value));
                if (product != null && productTypeIds.Contains(product.ProductTypeId))
                {
                    return new IndexableProductEntity(product, productType.FirstOrDefault(pt => pt.ProductTypeId == product.ProductTypeId));
                }
            }
            return null;
        }

        protected override IndexableProductEntity GetIndexableAndCheckDeletes(IIndexableUniqueId indexableUniqueId)
        {
            if (CustomLinkConfigurationUtilities.ProductTypes != null && CustomLinkConfigurationUtilities.ProductTypes.Any())
            {
                var product = _productService.GetProduct(((Guid)indexableUniqueId.Value));
                if (product != null && CustomLinkConfigurationUtilities.ProductTypeIds.Contains(product.ProductTypeId))
                {
                    return new IndexableProductEntity(product, CustomLinkConfigurationUtilities.ProductTypes.FirstOrDefault(pt => pt.ProductTypeId == product.ProductTypeId));
                }
            }
            return null;
        }

        protected override IEnumerable GetIndexablesToUpdateOnDelete(IIndexableUniqueId indexableUniqueId)
        {
            if (CustomLinkConfigurationUtilities.ProductTypeIds != null && CustomLinkConfigurationUtilities.ProductTypeIds.Any())
            {
                var product = _productService.GetProduct(((Guid)indexableUniqueId.Value));
                if (product != null && CustomLinkConfigurationUtilities.ProductTypeIds.Contains(product.ProductTypeId))
                {
                    return new List() { indexableUniqueId };
                }
            }
            return null;
        }

        protected override bool IndexUpdateNeedDelete(IndexableProductEntity indexable)
        {
            //Set to false in SitecoreItemCrawler
            return true;
        }
        
    }
       

Two config patch files were added to add sitecore_netforum_entities, sitecore_netforum_entities_rebuild, sitecore_globalsearch_master_index and sitecore_globalsearch_web_index.

Sitecore.ContentSearch.Solr.Index.NetForumEntities.config

<?xml version="1.0" encoding="utf-8" ?>

<configuration xmlns:patch="http://www.sitecore.net/xmlconfig/">

  <sitecore>

    <contentSearch>

      <configuration type="Sitecore.ContentSearch.ContentSearchConfiguration, Sitecore.ContentSearch">

        <indexes hint="list:AddIndex">

          <index id="sitecore_netforum_entities" type="Sitecore.ContentSearch.SolrProvider.SolrSearchIndex, Sitecore.ContentSearch.SolrProvider">

            <param desc="name">$(id)</param>

            <param desc="core">$(id)</param>

            <param desc="propertyStore" ref="contentSearch/indexConfigurations/databasePropertyStore" param1="$(id)" />

            <configuration ref="contentSearch/indexConfigurations/defaultSolrIndexConfiguration" />

            <strategies hint="list:AddStrategy">

              <strategy ref="contentSearch/indexConfigurations/indexUpdateStrategies/intervalAsyncNetForum" />

            </strategies>

            <locations hint="list:AddCrawler">

              <crawler type="Custom.Business.ContentSearch.Indexing.Crawlers.NetForumEntityCrawler, Custom.Business">

              </crawler>

            </locations>

          </index>

        </indexes>

      </configuration>

      <indexConfigurations>

        <defaultSolrIndexConfiguration type="Sitecore.ContentSearch.SolrProvider.SolrIndexConfiguration, Sitecore.ContentSearch.SolrProvider">

          <fieldMap type="Sitecore.ContentSearch.SolrProvider.SolrFieldMap, Sitecore.ContentSearch.SolrProvider">

            <typeMatches hint="raw:AddTypeMatch">

                <typeMatch patch:after="*[@typeName='double']"  typeName="decimal" type="System.Decimal" fieldNameFormat="{0}_s" settingType="Sitecore.ContentSearch.SolrProvider.SolrSearchFieldConfiguration, Sitecore.ContentSearch.SolrProvider" />

                <typeMatch patch:after="*[@typeName='double']"  typeName="productType" type="Custom.DomainObjects.NetForum.Commerce.ProductType,Custom.DomainObjects" fieldNameFormat="{0}_s" settingType="Sitecore.ContentSearch.SolrProvider.SolrSearchFieldConfiguration, Sitecore.ContentSearch.SolrProvider" />

                <typeMatch patch:after="*[@typeName='double']"  typeName="priceCollection" type="System.Collections.Generic.List`1[Custom.DomainObjects.NetForum.Commerce.Price,Custom.DomainObjects]" fieldNameFormat="{0}_sm" multiValued="true" settingType="Sitecore.ContentSearch.SolrProvider.SolrSearchFieldConfiguration, Sitecore.ContentSearch.SolrProvider" />

                <typeMatch patch:after="*[@typeName='double']"  typeName="price" type="Custom.DomainObjects.NetForum.Commerce.Price,Custom.DomainObjects" fieldNameFormat="{0}_s" settingType="Sitecore.ContentSearch.SolrProvider.SolrSearchFieldConfiguration, Sitecore.ContentSearch.SolrProvider" />

            </typeMatches>

            <fieldNames hint="raw:AddFieldByFieldName">

              <field fieldName="ProductType" returnType="string" />

              <field fieldName="ThumbnailUrl" returnType="string" />

              <field fieldName="YearPublished" returnType="string" />

            </fieldNames>

          </fieldMap>

        </defaultSolrIndexConfiguration>

      </indexConfigurations>

    </contentSearch>

  </sitecore>

</configuration>

Sitecore.ContentSearch.Solr.Index.GlobalSearch.config

<?xml version="1.0" encoding="utf-8" ?>

<configuration xmlns:patch="http://www.sitecore.net/xmlconfig/">

  <sitecore>

    <contentSearch>

      <configuration type="Sitecore.ContentSearch.ContentSearchConfiguration, Sitecore.ContentSearch">

        <indexes hint="list:AddIndex">

          <index id="sitecore_globalsearch_master_index" type="Sitecore.ContentSearch.SolrProvider.SolrSearchIndex, Sitecore.ContentSearch.SolrProvider">

            <param desc="name">$(id)</param>

            <param desc="core">$(id)</param>

            <param desc="propertyStore" ref="contentSearch/indexConfigurations/databasePropertyStore" param1="$(id)" />

            <configuration ref="contentSearch/indexConfigurations/defaultSolrIndexConfiguration" />

            <strategies hint="list:AddStrategy">

              <strategy ref="contentSearch/indexConfigurations/indexUpdateStrategies/manual" />

            </strategies>

          </index>

        </indexes>

        <indexes hint="list:AddIndex">

          <index id="sitecore_globalsearch_web_index" type="Sitecore.ContentSearch.SolrProvider.SolrSearchIndex, Sitecore.ContentSearch.SolrProvider">

            <param desc="name">$(id)</param>

            <param desc="core">$(id)</param>

            <param desc="propertyStore" ref="contentSearch/indexConfigurations/databasePropertyStore" param1="$(id)" />

            <configuration ref="contentSearch/indexConfigurations/defaultSolrIndexConfiguration" />

            <strategies hint="list:AddStrategy">

              <strategy ref="contentSearch/indexConfigurations/indexUpdateStrategies/manual" />

            </strategies>

          </index>

        </indexes>

      </configuration>

    </contentSearch>

  </sitecore>

</configuration>

Solr Configurations

On the Solr side netFORUM schema.xml and solrconfig.xml files were the same as the rest of the cores. However Global Search core had a couple of differences in solrconfig.xml.

In the select handler I added shard configuration that tricks Solr to think that it searches across shards, while it merely combines two cores and produces aggregate results.

<requestHandler name="/select" class="solr.SearchHandler">
    <!-- default values for query parameters can be specified, these
         will be overridden by parameters in the request
      -->
    <lst name="defaults">
      <str name="echoParams">explicit</str>
      <int name="rows">100</int>
      <str name="df">text</str>
      <bool name="terms">true</bool>
      <str name="spellcheck">true</str>
      <str name="spellcheck.collate">true</str>
      <str name="spellcheck.extendedResults">true</str>
      <str name="shards">localhost:8983/solr/sitecore_master_index,localhost:8983/solr/sitecore_netforum_entities</str>
      <str name="shards.qt">select</str>
    </lst>
    <arr name="last-components">
      <str>terms</str>
      <str>spellcheck</str>
    </arr>  
  </requestHandler>

Next: Part 2: Configuring urls for netFORUM entities.

Serving non-Sitecore data in Sitecore. Part 3: Cross-core search and custom multilist with search field.

Making _indexname parameter optional

Once Solr and Sitecore configurations are in place and external data is indexed, we can start querying data. By default each query that Sitecore makes to Solr includes _indexname filtering parameter, which makes cross-core querying a bit tricky. I had to extend Sitecore's SolrCompositeQuery to implement Solr query operator, grouping and highlighting which I've blogged about (here), so it only made sense to add ability to omit _indexname parameter from the query in the extension class. All I did I added a parameter to all my custom methods that indicates whether the index name should be included or not.

public class ExtendedCompositeQuery : SolrCompositeQuery
{
    public ExtendedCompositeQuery(AbstractSolrQuery query, AbstractSolrQuery filterQuery, IEnumerable methods,
    IEnumerable virtualFieldProcessors, IEnumerable facetQueries, QueryOptions options,
            LocalParams localParams = null, bool filterByIndexName = true)
            : base(query, filterQuery, methods, virtualFieldProcessors, facetQueries)
    {
        QueryOptions = options;
        LocalParams = localParams;
        FilterByIndexName = filterByIndexName;         
     }

     public QueryOptions QueryOptions { get; set; }
     public LocalParams LocalParams { get; set; }

     public bool FilterByIndexName { get; set; }

}

In extended LinqToSolrIndex class I added a check for this parameter in Execute method.

if (compositeQuery.FilterByIndexName)
{
     options.AddFilterQueries(new SolrQueryByField("_indexname", context.Index.Name));
}

That provided an ability to run cross-core queries if I needed to.

Custom Index Multilist with Search field

Another featured that had to implemented is handpicking of netFORUM entities and linking to them. Since these entities only live in Solr index, I needed to provide a user friendly mechanism to pick and choose entities from index. To solve this problem I created a new Sitecore field that is almost exact duplicate of default "Multilist with Search", cleared the Control field and entered values into Assembly and Class fields that match with field extension class I created.

CustomIndexUISearchResultItem.cs

 To make sure that queries from this new field take a different route than SitecoreUISearchResultItem and UISearchResult ones, I've implemented a new class called CustomIndexUISearchResultItem.

    [DebuggerDisplay("Name={Name}, TemplateName={TemplateName}, Version={Version}, Language={Language}")]
    public class CustomIndexUISearchResultItem : UISearchResult
    {
        private string group;
        
        [XmlIgnore]
        [IndexField("_uniqueid")]
        public ItemUri Uri { get; set; }

        public string Version
        {
            get
            {
                if (this.Uri == (ItemUri)null)
                    this.Uri = new ItemUri(this["_uniqueid"]);
                return this.Uri.Version.Number.ToString((IFormatProvider)CultureInfo.InvariantCulture);
            }
            set
            {
            }
        }

        public string Bucket { get; set; }

        [IndexField("_group")]
        public string Id
        {
            get
            {
                return ShortID.Decode(this.group);
            }
            set
            {
                this.group = value;
            }
        }

        [IndexField("__smallcreateddate")]
        [DateTimeFormatter(DateTimeFormatterAttribute.DateTimeKind.ServerTime)]
        public DateTime CreatedDate { get; set; }

        [IndexField("parsedcreatedby")]
        public string CreatedBy { get; set; }

        [IndexField("_isclone")]
        public bool IsClone { get; set; }

        public override string ItemId
        {
            get
            {
                if (this.Uri == (ItemUri)null)
                {
                    if (this["_uniqueid"] == null)
                        return string.Empty;
                    this.Uri = new ItemUri(this["_uniqueid"]);
                }
                return this.Uri.ItemID.ToString();
            }
        }

        [IndexField("_path")]
        public List Paths { get; set; }

        [IndexField("_parent")]
        public ID Parent { get; set; }

        public List Languages { get; set; }

        public List QuickActions { get; set; }

        public List DynamicQuickActions { get; set; }

        [IndexField("_displayname")]
        public string DisplayName { get; set; }

        [IndexField("_fullpath")]
        public string Path { get; set; }

        [IndexField("haschildren")]
        public bool HasChildren { get; set; }

        [IndexField("_templatename")]
        public string TemplateName { get; set; }

        [IndexField("_template")]
        public string TemplateId { get; set; }

        [DateTimeFormatter(DateTimeFormatterAttribute.DateTimeKind.ServerTime)]
        [IndexField("__smallupdateddate")]
        public DateTime Updated { get; set; }

        public List> Fields
        {
            get
            {
                return this.fields;
            }
        }

        public new string this[string key]
        {
            get
            {
                return base[key];
            }
            set
            {
                base[key] = value;
            }
        }

        public new object this[ObjectIndexerKey key]
        {
            get
            {
                return base[key];
            }
            set
            {
                base[key] = value;
            }
        }

        public override string ToString()
        {
            return Enumerable.Aggregate<string, string>(Enumerable.Select<KeyValuePair<string, object>, string>((IEnumerable<KeyValuePair<string, object>>)this.fields, (Func<KeyValuePair<string, object>, string>)(pair => pair.Key)), string.Format("{0}, {1}, {2}", (object)this.Uri.ItemID, (object)this.Uri.Language, (object)this.Uri.Version), (Func<string, string, string>)((current, key) => current + (object)", " + (string)this.fields.Find((Predicate<KeyValuePair<string, object>>)(pair => pair.Key == key)).Value));
        }

    }

Field Search Service

The service that performs the search when user pages through results in Multilist with Search field had to be extended as well to make it go through the CustomLinkToSolrIndex extension.

    [UsedImplicitly]
    public class CustomIndexSearch : SearchHttpTaskAsyncHandler, IRequiresSessionState
    {
        private static readonly object ThisLock = new object();
        private static volatile Hashtable cacheHashtable;

        public override bool IsReusable
        {
            get
            {
                return false;
            }
        }

        private static Hashtable CacheHashTable
        {
            get
            {
                if (CustomIndexSearch.cacheHashtable == null)
                {
                    lock (CustomIndexSearch.ThisLock)
                    {
                        if (CustomIndexSearch.cacheHashtable == null)
                            CustomIndexSearch.cacheHashtable = new Hashtable();
                    }
                }
                return CustomIndexSearch.cacheHashtable;
            }
        }

        public override void ProcessRequest(HttpContext context)
        {
        }

        public override async Task ProcessRequestAsync(HttpContext context)
        {
            if (ContentSearchManager.Locator.GetInstance<IContentSearchConfigurationSettings>().ItemBucketsEnabled())
            {
                context.Response.ContentType = "application/json";
                context.Response.ContentEncoding = Encoding.UTF8;
                this.Stopwatch = new Stopwatch();
                this.ItemsPerPage = BucketConfigurationSettings.DefaultNumberOfResultsPerPage;
                this.ExtractSearchQuery(context.Request.QueryString);
                this.CheckSecurity();
                if (!this.AbortSearch)
                {
                    bool @bool = MainUtil.GetBool(SearchHelper.GetDebug(this.SearchQuery), false);
                    if (@bool)
                    {
                        this.SearchQuery.RemoveAll((Predicate<SearchStringModel>)(x => x.Type == "debug"));
                        if (!BucketConfigurationSettings.EnableBucketDebug)
                            Sitecore.Buckets.Util.Constants.EnableTemporaryBucketDebug = true;
                    }
                    Database database = Sitecore.Buckets.Extensions.StringExtensions.IsNullOrEmpty(this.Database) ? Context.ContentDatabase : Factory.GetDatabase(this.Database);
                    if (!this.RunFacet)
                    {
                        this.StoreUserContextSearches();
                        this.ItemsPerPage = SitecoreExtensions.IsNumeric(SearchHelper.GetPageSize(this.SearchQuery)) ? int.Parse(SearchHelper.GetPageSize(this.SearchQuery)) : BucketConfigurationSettings.DefaultNumberOfResultsPerPage;
                        SitecoreIndexableItem sitecoreIndexableItem = (SitecoreIndexableItem)(database.GetItem(this.LocationFilter) ?? database.GetRootItem());
                        ISearchIndex searchIndex = SitecoreExtensions.IsEmpty(this.IndexName) ? ContentSearchManager.GetIndex((IIndexable)sitecoreIndexableItem) : ContentSearchManager.GetIndex(this.IndexName);
                        using (IProviderSearchContext searchContext1 = searchIndex.CreateSearchContext(SearchSecurityOptions.Default))
                        {
                            this.Stopwatch.Start();
                            var noIndexNameSearchModel = this.SearchQuery.Where(q => q.Type != "indexName");
                            var source1 = CustomLinqHelper.CreateQuery<CustomIndexUISearchResultItem>(searchContext1, noIndexNameSearchModel);
                            int num2 = int.Parse(this.PageNumber);
                            source1 = source1.Page(num2, this.ItemsPerPage);
                            var results = source1.GetResults<CustomIndexUISearchResultItem>();
                            IEnumerable<UISearchResult> source2 = Enumerable.Select<SearchHit<CustomIndexUISearchResultItem>, CustomIndexUISearchResultItem>(results.Hits, (Func<SearchHit<CustomIndexUISearchResultItem>, CustomIndexUISearchResultItem>)(h => h.Document));
                            if (BucketConfigurationSettings.EnableBucketDebug || Sitecore.Buckets.Util.Constants.EnableTemporaryBucketDebug)
                            {
                                SearchLog.Log.Info(string.Format("Search Query : {0}", ((IHasNativeQuery)source1).Query), (Exception)null);
                                SearchLog.Log.Info(string.Format("Search Index : {0}", (object)searchIndex.Name), (Exception)null);
                            }
                            List<UISearchResult> resultItems = Enumerable.ToList<UISearchResult>(source2);
                            int totalSearchResults = results.TotalSearchResults;
                            int num1 = totalSearchResults % this.ItemsPerPage == 0 ? totalSearchResults / this.ItemsPerPage : totalSearchResults / this.ItemsPerPage + 1;

                            if ((num2 - 1) * this.ItemsPerPage >= totalSearchResults)
                                num2 = 1;
                            List<TemplateFieldItem> templateFields = new List<TemplateFieldItem>();
                            if (source2 != null && Context.ContentDatabase != null)
                            {
                                using (IProviderSearchContext searchContext2 = searchIndex.CreateSearchContext(SearchSecurityOptions.Default))
                                {
                                    var enumerable = CustomIndexSearch.ProcessCachedDisplayedSearch(sitecoreIndexableItem, searchContext2);
                                    var itemCache = CacheManager.GetItemCache(Context.ContentDatabase);
                                    foreach (Tuple<string, string, string> tuple in enumerable)
                                    {
                                        Language result;
                                        Sitecore.Globalization.Language.TryParse(tuple.Item2, out result);
                                        Item ownerItem = itemCache.GetItem(new ID(tuple.Item1), result, new Sitecore.Data.Version(tuple.Item3));
                                        if (ownerItem == null)
                                        {
                                            ownerItem = Context.ContentDatabase.GetItem(new ID(tuple.Item1), result, new Sitecore.Data.Version(tuple.Item3));
                                            if (ownerItem != null)
                                                CacheManager.GetItemCache(Context.ContentDatabase).AddItem(ownerItem.ID, result, ownerItem.Version, ownerItem);
                                        }
                                        if (ownerItem != null && !templateFields.Contains(FieldTypeManager.GetTemplateFieldItem(new Field(ownerItem.ID, ownerItem))))
                                            templateFields.Add(FieldTypeManager.GetTemplateFieldItem(new Field(ownerItem.ID, ownerItem)));
                                    }
                                    resultItems = FillItemPipeline.Run(new FillItemArgs(templateFields, resultItems, this.Language));
                                }
                            }
                            if (this.IndexName == string.Empty)
                                resultItems = Enumerable.ToList<UISearchResult>(Sitecore.Buckets.Extensions.EnumerableExtensions.RemoveWhere<UISearchResult>((IEnumerable<UISearchResult>)resultItems, (Predicate<UISearchResult>)(item =>
                                {
                                    if (item.Name != null)
                                        return item.Content == null;
                                    return true;
                                })));
                            if (!BucketConfigurationSettings.SecuredItems.Equals("hide", StringComparison.InvariantCultureIgnoreCase))
                            {
                                if (totalSearchResults > BucketConfigurationSettings.DefaultNumberOfResultsPerPage && resultItems.Count < BucketConfigurationSettings.DefaultNumberOfResultsPerPage && num2 <= num1)
                                {
                                    while (resultItems.Count < BucketConfigurationSettings.DefaultNumberOfResultsPerPage)
                                        resultItems.Add(new UISearchResult()
                                        {
                                            ItemId = Guid.NewGuid().ToString()
                                        });
                                }
                                else if (resultItems.Count < totalSearchResults && num2 == 1)
                                {
                                    while (resultItems.Count < totalSearchResults && totalSearchResults < BucketConfigurationSettings.DefaultNumberOfResultsPerPage)
                                        resultItems.Add(new UISearchResult()
                                        {
                                            ItemId = Guid.NewGuid().ToString()
                                        });
                                }
                            }
                            this.Stopwatch.Stop();
                            IEnumerable<Tuple<View, object>> enumerable1 = FetchContextDataPipeline.Run(new FetchContextDataArgs((IEnumerable<SearchStringModel>)this.SearchQuery, searchContext1, (IIndexable)sitecoreIndexableItem));
                            IEnumerable<Tuple<int, View, string, IEnumerable<UISearchResult>>> enumerable2 = FetchContextViewPipeline.Run(new FetchContextViewArgs((IEnumerable<SearchStringModel>)this.SearchQuery, searchContext1, (IIndexable)sitecoreIndexableItem, (IEnumerable<TemplateFieldItem>)templateFields));
                            string callback = this.Callback;
                            string str1 = "(";
                            string str2 = JsonConvert.SerializeObject((object)new FullSearch()
                            {
                                PageNumbers = num1,
                                items = (IEnumerable<UISearchResult>)resultItems,
                                launchType = SearchHttpTaskAsyncHandler.GetEditorLaunchType(),
                                SearchTime = this.SearchTime,
                                SearchCount = totalSearchResults.ToString(),
                                ContextData = enumerable1,
                                ContextDataView = enumerable2,
                                CurrentPage = num2,
                                Location = (Context.ContentDatabase.GetItem(this.LocationFilter) != null ? Context.ContentDatabase.GetItem(this.LocationFilter).Name : Translate.Text("current item"))
                            });
                            string str3 = ")";
                            context.Response.Write(callback + str1 + str2 + str3);
                            if (!BucketConfigurationSettings.EnableBucketDebug)
                            {
                                if (!Sitecore.Buckets.Util.Constants.EnableTemporaryBucketDebug)
                                    if (@bool)
                                        Sitecore.Buckets.Util.Constants.EnableTemporaryBucketDebug = false;
                            }
                            SearchLog.Log.Info("Search Took : " + (object)this.Stopwatch.ElapsedMilliseconds + "ms", (Exception)null);
                        }
                    }
                    else
                    {
                        string callback = this.Callback;
                        string str1 = "(";
                        string str2 = JsonConvert.SerializeObject((object)new FullSearch()
                        {
                            PageNumbers = 1,
                            facets = GetFacetsPipeline.Run(new GetFacetsArgs(this.SearchQuery, this.LocationFilter)),
                            SearchCount = "1",
                            CurrentPage = 1
                        });
                        string str3 = ")";
                        context.Response.Write(callback + str1 + str2 + str3);
                    }
                }
            }
        }

        private static IEnumerable<Tuple<string, string, string>> ProcessCachedDisplayedSearch(SitecoreIndexableItem startLocationItem, IProviderSearchContext searchContext)
        {
            string cacheName = "IsDisplayedInSearchResults" + "[" + Context.ContentDatabase.Name + "]";
            Cache cache = (Cache)CustomIndexSearch.CacheHashTable[(object)cacheName];
            IEnumerable<Tuple<string, string, string>> enumerable = cache != null ? cache.GetValue((object)"cachedIsDisplayedSearch") as IEnumerable<Tuple<string, string, string>> : (IEnumerable<Tuple<string, string, string>>)null;
            if (enumerable == null)
            {
                CultureInfo culture = startLocationItem != null ? startLocationItem.Culture : new CultureInfo(Settings.DefaultLanguage);
                enumerable = (IEnumerable<Tuple<string, string, string>>)Enumerable.ToList<SitecoreUISearchResultItem>((IEnumerable<SitecoreUISearchResultItem>)Queryable.Where<SitecoreUISearchResultItem>(searchContext.GetQueryable<SitecoreUISearchResultItem>((IExecutionContext)new CultureExecutionContext(culture)), (Expression<Func<SitecoreUISearchResultItem, bool>>)(templateField => templateField["Is Displayed in Search Results".ToLowerInvariant()] == "1"))).ConvertAll<Tuple<string, string, string>>((Converter<SitecoreUISearchResultItem, Tuple<string, string, string>>)(d => new Tuple<string, string, string>(d.GetItem().ID.ToString(), d.Language, d.Version)));
                if (CustomIndexSearch.CacheHashTable[(object)cacheName] == null)
                {
                    lock (CustomIndexSearch.CacheHashTable.SyncRoot)
                    {
                        if (CustomIndexSearch.CacheHashTable[(object)cacheName] == null)
                        {
                            cache = (Cache)new DisplayedInSearchResultsCache(cacheName, new List<ID>()
              {
                new ID(Sitecore.Buckets.Util.Constants.IsDisplayedInSearchResults)
              });
                            CustomIndexSearch.cacheHashtable[(object)cacheName] = (object)cache;
                        }
                    }
                }
                cache.Add("cachedIsDisplayedSearch", (object)enumerable, Settings.Caching.DefaultFilteredItemsCacheSize);
            }
            return enumerable;
        }
    }

CustomIndexBucketList.cs

CustomIndexBucketList extends Sitecore SearchList.

public class CustomIndexBucketList : SearchList
    {
        private IEnumerable<CustomIndexUISearchResultItem> _searchResults;
        protected override string ScriptParameters
        {
            get
            {
                return string.Format("'{0}'", string.Join("', '", this.ID, this.ClientID, this.PageNumber, "/sitecore/shell/Applications/Buckets/Services/CustomIndexSearch.ashx", this.Filter, SearchHelper.GetDatabaseUrlParameter("&"), this.TypeHereToSearch, this.Of, (this.EnableSetNewStartLocation ? true : false)));
            }
        }

        protected override void RenderStartLocationInput(HtmlTextWriter output)
        {
            if (!this.EnableSetNewStartLocation)
                return;
            base.RenderStartLocationInput(output);
        }

        protected override Item[] GetItems(Item current)
        {
            Assert.ArgumentNotNull((object)current, "current");
            NameValueCollection nameValues1 = StringUtil.GetNameValues(this.Source, '=', '&');
            foreach (string index in nameValues1.AllKeys)
            {
                nameValues1[index] = HttpUtility.JavaScriptStringEncode(nameValues1[index]);
            }

            string str3 = nameValues1["Filter"];
            if (str3 != null)
            {
                NameValueCollection nameValues2 = StringUtil.GetNameValues(str3, ':', '|');
                if (nameValues2.Count == 0 && str3 != string.Empty)
                    this.Filter = this.Filter + "&_content=" + str3;
                foreach (string index in nameValues2.Keys)
                    this.Filter = this.Filter + "&" + index + "=" + nameValues2[index];
            }

            List<SearchStringModel> searchStringModel = Sitecore.Buckets.Extensions.StringExtensions.IsNullOrEmpty(str3) ? new List<SearchStringModel>() : Enumerable.ToList<SearchStringModel>(SearchStringModel.ParseQueryString(str3));

            this.ExtractQueryStringAndPopulateModel(nameValues1, searchStringModel, "FullTextQuery", "_content", "_content", false);
            this.ExtractQueryStringAndPopulateModel(nameValues1, searchStringModel, "Language", "language", "parsedlanguage", false);
            this.ExtractQueryStringAndPopulateModel(nameValues1, searchStringModel, "SortField", "sort", "sort", false);
            this.ExtractQueryStringAndPopulateModel(nameValues1, searchStringModel, "TemplateFilter", "template", "template", true);

            string sourceString = nameValues1["PageSize"];
            string str4 = Sitecore.Buckets.Extensions.StringExtensions.IsNullOrEmpty(sourceString) ? "20" : sourceString;

            int result;
            if (!int.TryParse(str4, out result))
            {
                result = 20;
                this.LogSourceQueryError(current, "PageSize", str4, result.ToString());
            }
            int pageSize = Sitecore.Buckets.Extensions.StringExtensions.IsNullOrEmpty(str4) ? 20 : result;

            ISearchIndex searchIndex;
            IndexName = nameValues1["IndexName"];
            if (IndexName != null)
            {
                this.Filter = this.Filter + "&indexName=" + IndexName;
                searchIndex = ContentSearchManager.GetIndex(IndexName);
                using (IProviderSearchContext searchContext = searchIndex.CreateSearchContext(SearchSecurityOptions.Default))
                {
                    var query = LinqHelper.CreateQuery<CustomIndexUISearchResultItem>(searchContext, (IEnumerable<SearchStringModel>)searchStringModel);
                    query = query.Take(pageSize);
                    var results = query.GetResults<CustomIndexUISearchResultItem>();
                    int num = results.TotalSearchResults;
                    this.PageNumber = num % pageSize == 0 ? num / pageSize : num / pageSize + 1;
                    _searchResults = results.Select(s => s.Document);
                    return results.Select(sitecoreItem => new Item(Sitecore.Data.ID.Parse(sitecoreItem.Document.Id), new Sitecore.Data.ItemData(new ItemDefinition(Sitecore.Data.ID.Parse(sitecoreItem.Document.Id), sitecoreItem.Document.Name, TemplateIDs.StandardTemplate, TemplateIDs.StandardTemplate), current.Language, Sitecore.Data.Version.Latest, GetFieldList(sitecoreItem.Document)), current.Database)).ToArray();
                }
            }
            return base.GetItems(current);
        }

        /// <summary>
        /// Outputs the string.
        /// </summary>
        /// <param name="item4">The item4.</param>
        /// <returns></returns>
        public override string OutputString(Item item4)
        {
            Item bucketItemOrParent = ItemExtensions.GetParentBucketItemOrParent(item4);
            string str = bucketItemOrParent != null ? "- " + bucketItemOrParent.Name : string.Empty;
            if (_searchResults != null && _searchResults.Any())
            {
                var currentItem = _searchResults.FirstOrDefault(i => Sitecore.Data.ID.Parse(i.Id) == item4.ID);
                if (currentItem != null)
                {
                    return string.Format("{0} ({1})", item4.DisplayName, currentItem.TemplateName);
                }
            }
            return string.Format("{0} ({1} {2} - {3} - {4})", item4.DisplayName, item4.TemplateName, str, item4.Version.Number, item4.Language.Name);
        }

        protected override void DoRender(HtmlTextWriter output)
        {
            if (!ContentSearchManager.Locator.GetInstance<IContentSearchConfigurationSettings>().ItemBucketsEnabled())
            {
                output.Write(Translate.Text("The field cannot be displayed because the Item Buckets feature is disabled."));
            }
            else
            {
                ArrayList selected;
                OrderedDictionary unselected;
                this.GetSelectedItems(this.GetItems(Sitecore.Context.ContentDatabase.GetItem(this.ItemID)), out selected, out unselected);
                StringBuilder stringBuilder = new StringBuilder();
                foreach (DictionaryEntry dictionaryEntry in unselected)
                {
                    Item obj = dictionaryEntry.Value as Item;
                    if (obj != null)
                    {
                        stringBuilder.Append(obj.DisplayName + ",");
                        stringBuilder.Append(this.GetItemValue(obj) + ",");
                    }
                }
                this.RenderStartLocationInput(output);
                output.Write("<input type='hidden' width='100%' id='multilistValues" + (object)this.ClientID + "' value='" + stringBuilder.ToString() + "' style='width: 200px;margin-left:3px;'>");
                this.ServerProperties["ID"] = (object)this.ID;
                string str1 = string.Empty;
                if (this.ReadOnly)
                    str1 = " disabled='disabled'";
                output.Write("<input id='" + this.ID + "_Value' type='hidden' value='" + StringUtil.EscapeQuote(this.Value) + "' />");
                output.Write("<div class='scContentControlSearchListContainer'>");
                output.Write("<table" + this.GetControlAttributes() + ">");
                output.Write("<tr>");
                output.Write("<td class='scContentControlMultilistCaption' width='50%'>" + Translate.Text("All") + "</td>");
                output.Write("<td width='20'>" + Images.GetSpacer(20, 1) + "</td>");
                output.Write("<td class='scContentControlMultilistCaption' width='50%'>" + Translate.Text("Selected") + "</td>");
                output.Write("<td width='20'>" + Images.GetSpacer(20, 1) + "</td>");
                output.Write("</tr>");
                output.Write("<tr>");
                output.Write("<td valign='top' height='100%'>");
                output.Write("<div class='scMultilistNav'><input type='text' class='scIgnoreModified bucketSearch inactive' value='" + this.TypeHereToSearch + "' id='filterBox" + this.ClientID + "' " + (Sitecore.Context.ContentDatabase.GetItem(this.ItemID).Access.CanWrite() ? string.Empty : "disabled") + ">");
                output.Write("<a id='prev" + this.ClientID + "' class='hovertext'>" + Images.GetImage("Office/16x16/arrow_left.png", 16, 16, "absmiddle") + Translate.Text("Prev") + "</a>");
                output.Write("<a id='next" + this.ClientID + "' class='hovertext'> " + Translate.Text("Next") + Images.GetImage("Office/16x16/arrow_right.png", 16, 16, "absmiddle") + "</a>");
                output.Write("<a id='refresh" + this.ClientID + "' class='hovertext'> " + Images.GetImage("Office/16x16/refresh.png", 16, 16, "absmiddle") + Translate.Text("Refresh") + "</a>");
                output.Write("<span><span><strong>" + Translate.Text("Page number") + ": </strong></span><span id='pageNumber" + this.ClientID + "'></span></span></div>");
                string str2 = !UIUtil.IsIE() || UIUtil.GetBrowserMajorVersion() != 9 ? "10" : "11";
                output.Write("<select id=\"" + this.ID + "_unselected\" class='scBucketListBox' multiple=\"multiple\" size=\"" + str2 + "\"" + str1 + " >");
                foreach (DictionaryEntry dictionaryEntry in unselected)
                {
                    Item obj = dictionaryEntry.Value as Item;
                    if (obj != null)
                    {
                        string str3 = this.OutputString(obj);
                        output.Write("<option value='" + this.GetItemValue(obj) + "'>" + str3 + "</option>");
                    }
                }
                output.Write("</select>");
                output.Write("</td>");
                output.Write("<td valign='top'>");
                output.Write("<img class='' height='16' width='16' border='0' alt='' style='margin: 15px;' src='/sitecore/shell/themes/standard/Images/blank.png'/>");
                output.Write("<br />");
                this.RenderButton(output, "Office/16x16/navigate_right.png", string.Empty, "btnRight" + this.ClientID);
                output.Write("<br />");
                this.RenderButton(output, "Office/16x16/navigate_left.png", string.Empty, "btnLeft" + this.ClientID);
                output.Write("</td>");
                output.Write("<td valign='top' height='100%'>");
                output.Write("<select id='" + this.ID + "_selected' class='scBucketListSelectedBox' multiple='multiple' size='10'" + str1 + ">");
                var selectedItems = this.GetSelectedIndexItems(selected);
                for (int index = 0; index < selected.Count; ++index)
                {
                    Item obj1 = selected[index] as Item;
                    if (obj1 != null)
                    {
                        string str3 = this.OutputString(obj1);
                        output.Write("<option value='" + this.GetItemValue(obj1) + "'>" + str3 + "</option>");
                    }
                    else
                    {
                        string path = selected[index] as string;
                        if (path != null)
                        {
                            var item = selectedItems.FirstOrDefault(i => i.ItemId == path);
                            if (item != null)
                            {
                                output.Write("<option value='" + path + "'>" + item.Name + " (" + item.TemplateName + ")</option>");
                            }
                            else
                            {
                                output.Write("<option value='" + path + "'>" + path + "</option>");
                            }
                        }
                    }
                }
                output.Write("</select>");
                output.Write("</td>");
                output.Write("<td valign='top'>");
                output.Write("<img class='' height='16' width='16' border='0' alt='' style='margin: 15px 0;' src='/sitecore/shell/themes/standard/Images/blank.png'/>");
                output.Write("<br />");
                this.RenderButton(output, "Office/16x16/navigate_up.png", "javascript:scContent.multilistMoveUp('" + this.ID + "')", "btnUp" + this.ClientID);
                output.Write("<br />");
                this.RenderButton(output, "Office/16x16/navigate_down.png", "javascript:scContent.multilistMoveDown('" + this.ID + "')", "btnDown" + this.ClientID);
                output.Write("</td>");
                output.Write("</tr>");
                output.Write("<div style='border:1px solid #999999;font:8pt tahoma;display:none;padding:2px;margin:4px 0px 4px 0px;height:14px' id='" + this.ID + "_all_help'></div>");
                output.Write("<div style='border:1px solid #999999;font:8pt tahoma;display:none;padding:2px;margin:4px 0px 4px 0px;height:14px' id='" + this.ID + "_selected_help'></div>");
                output.Write("</table>");
                output.Write("</div>");
                this.RenderScript(output);
            }
        }

        private IEnumerable<CustomIndexUISearchResultItem> GetSelectedIndexItems(ArrayList selected)
        {
            if (IndexName != null && selected != null && selected.Count > 0)
            {
                try
                {
                    var searchIndex = ContentSearchManager.GetIndex(IndexName);
                    var selectedIds = selected.ToArray().Where(i => !(i is Item));
                    if (selectedIds.Any())
                    {
                        using (IProviderSearchContext searchContext = searchIndex.CreateSearchContext(SearchSecurityOptions.Default))
                        {
                            var query = LinqHelper.CreateQuery<CustomIndexUISearchResultItem>(searchContext, new List<SearchStringModel>());
                            var filterPredicate = PredicateBuilder.True<CustomIndexUISearchResultItem>();
                            foreach (var id in selectedIds)
                            {
                                filterPredicate = filterPredicate.Or(i => i[(ObjectIndexerKey)"_group"] == (ID)Sitecore.Data.ID.Parse(id));
                            }
                            query = query.Where(filterPredicate);
                            var results = query.GetResults<CustomIndexUISearchResultItem>();
                            return results.Select(s => s.Document);
                        }
                    }
                }
                catch (Exception ex)
                {
                    Log.Error(ex.Message, ex, this);
                }
            }
            return new List<CustomIndexUISearchResultItem>();
        }

        /// <summary>
        /// Gets the field list.
        /// </summary>
        /// <param name="sitecoreItem">The sitecore item.</param>
        /// <returns></returns>
        private static FieldList GetFieldList(CustomIndexUISearchResultItem sitecoreItem)
        {
            var list = new FieldList();
            list.Add(TemplateIDs.TemplateField, sitecoreItem.TemplateName);
            return list;
        }

        private string IndexName { get; set; }
    }

This concludes this series of posts about cross-core Solr search implementation and indexing of external data. I hope you find it useful.