RangeStringFormatter

[nightowl888]

RangeStringFormatter

[dtosato]

@NightOwl888
Here below there is another test on DateTime data. In this case it is not clear how to apply the Lucene.Net.Documents.DateTools.StringToDate() method and choose a specific culture for the dates stored into the index.

[TestMethod]
public void CustomDateTimeFormattingTest()
{
    // Create index.
    string indexPath = @"d:\temp\BoboBrowseNet\CustomDateTimeFormattingTest";
    {
        using (Lucene.Net.Store.Directory directory = FSDirectory.Open(new System.IO.DirectoryInfo(indexPath)))
        {
            DateTime date;
            using (IndexWriter modifier = new IndexWriter(directory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), true, Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED))
            {
                {
                    date = new DateTime(2011, 6, 10);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2011, 7, 10);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2010, 2, 12);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2010, 7, 3);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(1998, 1, 2);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2014, 1, 1);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2014, 2, 1);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2014, 2, 1);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2014, 2, 16);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2014, 5, 11);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }
            }
        }
    }

    // Field.
    string field = "date";

    // Facet.
    DateTime lv1 = new DateTime(1990, 01, 01);
    DateTime hv1 = new DateTime(2012, 01, 01);
    DateTime lv2 = new DateTime(2012, 02, 01);
    DateTime hv2 = new DateTime(2015, 01, 01);
    string lv1Format = DateTools.DateToString(lv1, DateTools.Resolution.DAY);
    string hv1Format = DateTools.DateToString(hv1, DateTools.Resolution.DAY);
    string lv2Format = DateTools.DateToString(lv2, DateTools.Resolution.DAY);
    string hv2Format = DateTools.DateToString(hv2, DateTools.Resolution.DAY);
    string facetRange1 = string.Format("[{0} TO {1}]", lv1Format, hv1Format);
    string facetRange2 = string.Format("[{0} TO {1}]", lv2Format, hv2Format);
    IFacetHandler dateFacetHandler = new RangeFacetHandler(field, new List<string> { facetRange1, facetRange2 });

    ICollection<IFacetHandler> handlerList = new IFacetHandler[] { dateFacetHandler };

    // Lucene index.
    using (Lucene.Net.Store.Directory idx = FSDirectory.Open(new System.IO.DirectoryInfo(indexPath)))
    {
        using (IndexReader reader = IndexReader.Open(idx, true))
        {
            // Bobo reader.
            using (BoboIndexReader boboReader = BoboIndexReader.GetInstance(reader, handlerList))
            {
                // Request.
                BrowseRequest browseRequest = new BrowseRequest();
                browseRequest.Count = 10;
                browseRequest.Offset = 0;
                browseRequest.FetchStoredFields = true;
                browseRequest.Query = new MatchAllDocsQuery();

                // Selection.
                BrowseSelection sel = new BrowseSelection(field);
                browseRequest.AddSelection(sel);

                // Output.
                FacetSpec spec = new FacetSpec();
                spec.OrderBy = FacetSpec.FacetSortSpec.OrderValueAsc;
                spec.MaxCount = 10;
                browseRequest.SetFacetSpec(field, spec);

                // Browse.
                IBrowsable browser = new BoboBrowser(boboReader);
                BrowseResult result = browser.Browse(browseRequest);

                // Results.
                int totalHits = result.NumHits;
                BrowseHit[] hits = result.Hits;
                IDictionary<String, IFacetAccessible> facetMap = result.FacetMap;
                IFacetAccessible facets = facetMap[field];
                List<BrowseFacet> facetVals = facets.GetFacets().ToList();

                // Create a range string formatterly.
                RangeStringFormatter<string> formatter = new RangeStringFormatter<string>("[{0:d} TO {1:d}]", " [{0:d} ->", " {1:d}]");

                // Check.
                Assert.AreEqual(2, facetVals.Count());

                string formattedForDisplay1 = formatter.Format(facetVals[0].Value);
                string formattedForDisplay2 = formatter.Format(facetVals[1].Value);

                Assert.AreEqual(new BrowseFacet("[01/01/1990 -> 01/01/2012]", 5), formattedForDisplay1);
                Assert.AreEqual(new BrowseFacet("[02/01/2012 -> 01/01/2015]", 5), formattedForDisplay2);
            }
        }
    }
}

[NightOwl888]

The type parameter of RangeStringFormatter<T> is for converting the strings back to their native type. Ideally, we wouldn't have to specify it (again) but the underlying type information of the FacetHandler is lost when getting to the response. I couldn't find a way to pass this information through, so you need to specify this as (usually) the same datatype as what you specify for the RangeFacetHandler.

However, you have another issue in your last message (actually, there is more than one, but we will get to that later). You are clearly expecting the strings to be dates, but you have specified to use them as strings by not specifying a PredefinedTermListFactory<DateTime> in the RangeFacetHandler.

IFacetHandler dateFacetHandler = new RangeFacetHandler(field, new List<string> { facetRange1, facetRange2 });

// Should be

IFacetHandler dateFacetHandler = new RangeFacetHandler(field, new PredefinedTermListFactory<DateTime>(), new List<string> { facetRange1, facetRange2 });

Then, in your output, you have to once again specify that it is a <DateTime> - this is to ensure that the output data (which is type string) is converted back to its native type DateTime for formatting. This is a requirement of .NET string formatting - you must have a native data type in order to make formatting changes to it. Also, it only works on dates and numbers. String values will essentially just be a concatenation because there is no such thing as formatting a string type in .NET.

RangeStringFormatter<string> formatter = new RangeStringFormatter<string>("[{0:d} TO {1:d}]", " [{0:d} ->", " {1:d}]");

// Should be

RangeStringFormatter<DateTime> formatter = new RangeStringFormatter<DateTime>("[{0:d} TO {1:d}]", " [{0:d} ->", " {1:d}]");

Now you will get range strings specified in the DateTime format you prefer, using the culture of the current thread.

Finally, these strings will not necessarily compare against the output BrowseFacet instance - they are not expected to. Formatting range strings is for output for benefit of the end user.

Assert.AreEqual(new BrowseFacet("[01/01/1990 -> 01/01/2012]", 5), formattedForDisplay1);
Assert.AreEqual(new BrowseFacet("[02/01/2012 -> 01/01/2015]", 5), formattedForDisplay2);

// Should be

Assert.AreEqual("[1/1/1990 TO 1/1/2012]", formattedForDisplay1);
Assert.AreEqual("[2/1/2012 TO 1/1/2015]", formattedForDisplay2);

Also, if you want the range strings to be predictable (for unit testing), you should set the culture of the thread explicitly. You may get different results than what is above depending on the culture of your development machine, so you should set Thread.CurrentThread.CurrentCulture explicitly to ensure every machine will run the unit test the in the same culture. You might want to set this in your unit test initializer. You probably wouldn't do this in production, it is just for unit testing.

Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US");

If, for example, you wanted a custom format for display on an HTML UI with check boxes, you would put both the original range string into the input element and the formatted range string into its associated label element.

<input data-range-string="[19900101 TO 20120101]" id="19900101_TO_20120101" name="19900101_TO_20120101" type="checkbox" value="false" />
<label for="19900101_TO_20120101">[1/1/1990 TO 1/1/2012]</label>

Note that the id and name fields are a cleaned up version of the range string in this case. This is done to make them easy to use in JavaScript.

When the box is checked, you might want to automatically submit the selections (of all of the check boxes on the UI) back to the server immediately via AJAX so you can update the list of results. You would do this by submitting the value of data-range-string back to the server and adding it as a selection value (along with any other selected values) for the browse request.

// Selection.
BrowseSelection sel = new BrowseSelection(field);
sel.AddValue("[19900101 TO 20120101]");
browseRequest.AddSelection(sel);

Note that you would typically submit several selections, possibly multiple values each, spanning several different fields to narrow down the results.

As for the formatted value (the value in the <label> element above), there is no need to submit it back to the server. It is essentially read-only for benefit of the user and BoboBrowse doesn't have any other use for it. The value in the data-range-string field is the only value it has any use for as a selection.

Here is a sample page so you can envision the workflow. I don't know if that site uses BoboBrowse, but the workflow is the same. The first group of price ranges would be a RangeFacetHandler, and it would have to use a RangeStringFormatter<decimal> in order to put them into a format similar to what is on that site. However, each check box will also need the original format string to submit as selections.

Nearly all of the other facets on the page would just use a SimpleFacetHandler. In those cases, the string submitted back to the server as a selection value would most likely match exactly the string that is displayed to the user.

Note that in either case, it would be more practical to put the count field into a separate <span> from the facet value so it can be updated dynamically via JavaScript. Calling ToString() on the BrowseFacet might work for debugging, but it is not always useful in a real world application. That is why the RangeStringFormatter doesn't include it.

[dtosato]

I have modified the code as you told me, but the result is still not as expected. In fact the test output is:

Assert.AreEqual failed. Expected:<[1/1/1990 -> 1/1/2012]>. Actual:<[1/1/1990 TO 1/1/2012]>.     

Maybe to change TO to ->, I have to use the following constructor

public RangeStringFormatter(string format, IFormatProvider provider);

Is this right? Which is the way to use it?

Here the modified test.

[TestMethod]
public void CustomDateTimeFormattingTest()
{
    Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US");

    // Create index.
    string indexPath = @"d:\temp\CustomDateTimeFormattingTest";
    {
        using (Lucene.Net.Store.Directory directory = FSDirectory.Open(new System.IO.DirectoryInfo(indexPath)))
        {
            DateTime date;
            using (IndexWriter modifier = new IndexWriter(directory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), true, Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED))
            {
                {
                    date = new DateTime(2011, 6, 10);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2011, 7, 10);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2010, 2, 12);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2010, 7, 3);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(1998, 1, 2);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2014, 1, 1);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2014, 2, 1);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2014, 2, 1);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2014, 2, 16);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }

                {
                    date = new DateTime(2014, 5, 11);
                    Document doc = new Document();
                    doc.Add(new Field("date", DateTools.DateToString(date, DateTools.Resolution.DAY), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    modifier.AddDocument(doc);
                }
            }
        }
    }

    // Field.
    string field = "date";

    // Facet.
    DateTime lv1 = new DateTime(1990, 01, 01);
    DateTime hv1 = new DateTime(2012, 01, 01);
    DateTime lv2 = new DateTime(2012, 02, 01);
    DateTime hv2 = new DateTime(2015, 01, 01);
    string lv1Format = DateTools.DateToString(lv1, DateTools.Resolution.DAY);
    string hv1Format = DateTools.DateToString(hv1, DateTools.Resolution.DAY);
    string lv2Format = DateTools.DateToString(lv2, DateTools.Resolution.DAY);
    string hv2Format = DateTools.DateToString(hv2, DateTools.Resolution.DAY);
    string facetRange1 = string.Format("[{0} TO {1}]", lv1Format, hv1Format);
    string facetRange2 = string.Format("[{0} TO {1}]", lv2Format, hv2Format);
    IFacetHandler dateFacetHandler = new RangeFacetHandler(
        field,
        new PredefinedTermListFactory<DateTime>(), 
        new List<string> { facetRange1, facetRange2 });

    ICollection<IFacetHandler> handlerList = new IFacetHandler[] { dateFacetHandler };

    // Lucene index.
    using (Lucene.Net.Store.Directory idx = FSDirectory.Open(new System.IO.DirectoryInfo(indexPath)))
    {
        using (IndexReader reader = IndexReader.Open(idx, true))
        {
            // Bobo reader.
            using (BoboIndexReader boboReader = BoboIndexReader.GetInstance(reader, handlerList))
            {
                // Request.
                BrowseRequest browseRequest = new BrowseRequest();
                browseRequest.Count = 10;
                browseRequest.Offset = 0;
                browseRequest.FetchStoredFields = true;
                browseRequest.Query = new MatchAllDocsQuery();

                // Selection.
                BrowseSelection sel = new BrowseSelection(field);
                browseRequest.AddSelection(sel);

                // Output.
                FacetSpec spec = new FacetSpec();
                spec.OrderBy = FacetSpec.FacetSortSpec.OrderValueAsc;
                spec.MaxCount = 10;
                browseRequest.SetFacetSpec(field, spec);

                // Browse.
                IBrowsable browser = new BoboBrowser(boboReader);
                BrowseResult result = browser.Browse(browseRequest);

                // Results.
                int totalHits = result.NumHits;
                BrowseHit[] hits = result.Hits;
                IDictionary<String, IFacetAccessible> facetMap = result.FacetMap;
                IFacetAccessible facets = facetMap[field];
                List<BrowseFacet> facetVals = facets.GetFacets().ToList();

                // Create a range string formatterly.
                RangeStringFormatter<DateTime> formatter = new RangeStringFormatter<DateTime>("[{0:d} TO {1:d}]", " [{0:d} ->", " {1:d}]");

                // Check.
                Assert.AreEqual(2, facetVals.Count());

                string formattedForDisplay1 = formatter.Format(facetVals[0].Value);
                string formattedForDisplay2 = formatter.Format(facetVals[1].Value);

                Assert.AreEqual("[1/1/1990 -> 1/1/2012]", formattedForDisplay1);
                Assert.AreEqual("[2/1/2012 -> 1/1/2015]", formattedForDisplay2);
            }
        }
    }
}

[NightOwl888]

No, that is not it. The range string that you have specified to create in this line:

// Create a range string formatterly.
RangeStringFormatter<DateTime> formatter = new RangeStringFormatter<DateTime>("[{0:d} TO {1:d}]", " [{0:d} ->", " {1:d}]");

is [{0:d} TO {1:d}], so that is what is created in the output. If you want the range strings you expect, you will need to change the format to [{0:d} -> {1:d}].

Also, the openLowerBoundFormat (which formats a range like [* TO 2001/12/31]) and openUpperBoundFormat (which formats a range like [2005/06/30 TO *]) appear to be reversed in your example.

[dtosato]

OK, now it is clear.