Search completion; complete rework of UX.

This is a major reworking of how the search-completion systems works.

Previously, the goal had been to emulate roughly how chrome itself
handles completions.  This has turned out to be a bad idea:

   - It creates issues around what it means to hit <Enter> in the
     vomnibar.
   - And the contents of the vomnibar change asynchronously (because we
     fetch completions asynchronously), so it creates the possibility
     that the effect of <Enter> changes depending on how long the user
     waits before typing <Enter>.

All of that is bad.

This commit changes things:

   - In normal omni mode, the vomnibar looks and behaves pretty much
     like it always has, just with some extra completion suggestions
     thrown in.

   - And in custom-search-engine mode it also behaves mostly as it has
     previously, but (again, possibly) with some extra completion
     suggestions thrown in, and with many useless suggestions excluded.

This is all far more Vimium-like than Chrome-like.

2 files changed, 32 insertions, 47 deletions

diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee
index ebf56dde..68edad99 100644
--- a/background_scripts/completion.coffee
+++ b/background_scripts/completion.coffee
@@ -10,7 +10,7 @@
 #  - refresh(): (optional) refreshes the completer's data source (e.g. refetches the list of bookmarks).
 #  - cancel(): (optional) cancels any pending, cancelable action.
 class Suggestion
-  showRelevancy: false # Set this to true to render relevancy when debugging the ranking scores.
+  showRelevancy: true # Set this to true to render relevancy when debugging the ranking scores.
 
   constructor: (@options) ->
     # Required options.
@@ -397,20 +397,19 @@ class SearchEngineCompleter
       callback engines
 
       # Let the front-end vomnibar know the search-engine keywords.  It needs to know them so that, when the
-      # query goes from "w" to "w ", the vomnibar synchronously launches the next filter() request (all of which avoids
-      # an ugly delay).
+      # query goes from "w" to "w ", the vomnibar can synchronously launch the next filter() request (which
+      # avoids an ugly delay/flicker).
       port.postMessage
         handler: "keywords"
         keywords: key for own key of engines
 
   filter: (request, onComplete) ->
     { queryTerms, query, engine } = request
-    [ primarySuggestion, removePrimarySuggestion ] = [ null, false ]
 
     { custom, searchUrl, description } =
       if engine
         { keyword, searchUrl, description } = engine
-        extend request, { searchUrl, suppressLeadingKeyword: keyword }
+        extend request, { searchUrl, customSearchMode: true }
         custom: true
         searchUrl: searchUrl
         description: description
@@ -421,35 +420,32 @@ class SearchEngineCompleter
 
     return onComplete [] unless custom or 0 < queryTerms.length
 
-    factor = Math.max 0, Math.min 1, Settings.get "omniSearchWeight"
+    factor = Math.max 0.0, Math.min 1.0, Settings.get "omniSearchWeight"
     haveCompletionEngine = (0.0 < factor or custom) and CompletionSearch.haveCompletionEngine searchUrl
 
     # Relevancy:
     #   - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word
     #     relevancy, say).  We assume that the completion engine has already factored that in.  Also,
-    #     completion engines often handle spelling mistakes, in which case we wouldn't find the query terms
-    #     in the suggestion anyway.
+    #     completion engines sometimes handle spelling mistakes, in which case we wouldn't find the query
+    #     terms in the suggestion anyway.
     #   - Scores are weighted such that they retain the order provided by the completion engine.
     #   - The relavancy is higher if the query term is longer.  The idea is that search suggestions are more
     #     likely to be relevant if, after typing some number of characters, the user hasn't yet found
     #     a useful suggestion from another completer.
     #
     characterCount = query.length - queryTerms.length + 1
-    relevancy = (if custom then 0.9 else factor) * (Math.min(characterCount, 12.0)/12.0)
+    relevancy = (if custom then 0.5 else factor) * 12.0 / Math.max 12.0, characterCount
+    console.log factor, relevancy
 
-    # This filter is applied to all of the suggestions from all of the completers.
+    # This filter is applied to all of the suggestions from all of the completers, after they have been
+    # aggregated by the MultiCompleter.
     filter = (suggestions) ->
       return suggestions unless custom and haveCompletionEngine
 
-      # The primary suggestion was just a guess.  If we've managed fetch actual completions (asynchronously),
-      # then we now remove it.
-      if removePrimarySuggestion
-        suggestions = suggestions.filter (suggestion) -> suggestion != primarySuggestion
-
       # We only accept suggestions:
       #   - from this completer, or
-      #   - from other completers, but then only if their URL matches this search engine and this query
-      #     (that is only if their URL could have been generated by this search engine).
+      #   - from other completers, but then only if their URL matches this search engine and matches this
+      #   query (that is only if their URL could have been generated by this search engine).
       suggestions.filter (suggestion) ->
         suggestion.type == description or
           # This is a suggestion for the same search engine.
@@ -466,7 +462,6 @@ class SearchEngineCompleter
       autoSelect: custom
       forceAutoSelect: custom
       highlightTerms: not haveCompletionEngine
-      searchSuggestionType: "primary"
 
     mkSuggestion = (suggestion) ->
       new Suggestion
@@ -477,43 +472,27 @@ class SearchEngineCompleter
         relevancy: relevancy *= 0.9
         insertText: suggestion
         highlightTerms: false
-        searchSuggestionType: "completion"
-
-    deliverCompletions = (onComplete, completions, args...) ->
-      # Make the first suggestion float to the top of the vomnibar (except if we would be competing with the
-      # domain completer, which also assigns a relevancy of 1).
-      if 0 < completions.length
-        if custom # or (1 < queryTerms.length or /\S\s/.test query)
-          extend completions[0],
-            relevancy: 1
-            autoSelect: custom
-            forceAutoSelect: custom
-            insertText: null
-      onComplete completions, args...
 
     cachedSuggestions =
       if haveCompletionEngine then CompletionSearch.complete searchUrl, queryTerms else null
 
-    suggestions =
-      if cachedSuggestions? and 0 < cachedSuggestions.length
-        cachedSuggestions.map mkSuggestion
-      else if custom
-        [ primarySuggestion ]
-      else
-        []
+    suggestions = []
+    suggestions.push primarySuggestion if custom
+    suggestions.push cachedSuggestions.map(mkSuggestion)... if custom and cachedSuggestions?
 
     if queryTerms.length == 0 or cachedSuggestions? or not haveCompletionEngine
       # There is no prospect of adding further completions.
-      deliverCompletions onComplete, suggestions, { filter, continuation: null }
+      suggestions.push cachedSuggestions.map(mkSuggestion)... if cachedSuggestions?
+      onComplete suggestions, { filter, continuation: null }
     else
-      # Post the initial suggestions, then deliver further completions asynchronously, as a continuation.
-      deliverCompletions onComplete, suggestions,
+      # Post the initial suggestions, but then deliver any further completions asynchronously, as a
+      # continuation.
+      onComplete suggestions,
         filter: filter
         continuation: (onComplete) =>
           CompletionSearch.complete searchUrl, queryTerms, (suggestions = []) =>
             console.log "fetched suggestions:", suggestions.length, query if SearchEngineCompleter.debug
-            removePrimarySuggestion = 0 < suggestions.length
-            deliverCompletions onComplete, suggestions.map mkSuggestion
+            onComplete suggestions.map mkSuggestion
 
 # A completer which calls filter() on many completers, aggregates the results, ranks them, and returns the top
 # 10. All queries from the vomnibar come through a multi completer.
diff --git a/background_scripts/completion_search.coffee b/background_scripts/completion_search.coffee
index a9521a3d..c6824594 100644
--- a/background_scripts/completion_search.coffee
+++ b/background_scripts/completion_search.coffee
@@ -1,6 +1,6 @@
 
 CompletionSearch =
-  debug: false
+  debug: true
   inTransit: {}
   completionCache: new SimpleCache 2 * 60 * 60 * 1000, 5000 # Two hours, 5000 entries.
   engineCache:new SimpleCache 1000 * 60 * 60 * 1000 # 1000 hours.
@@ -75,13 +75,16 @@ CompletionSearch =
         # Verify that the previous query is a prefix of the current query.
         return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase()
         # Verify that every previous suggestion contains the text of the new query.
-        for suggestion in (@mostRecentSuggestions.map (s) -> s.toLowerCase())
+        # Note: @mostRecentSuggestions may also be empty, in which case we drop though. The effect is that
+        # previous queries with no suggestions suppress subsequent no-hope HTTP requests as the user continues
+        # to type.
+        for suggestion in @mostRecentSuggestions
           return false unless 0 <= suggestion.indexOf query
         # Ok. Re-use the suggestion.
         true
 
       if reusePreviousSuggestions
-        console.log "reuse previous query:", @mostRecentQuery if @debug
+        console.log "reuse previous query:", @mostRecentQuery, @mostRecentSuggestions.length if @debug
         return callback @completionCache.set completionCacheKey, @mostRecentSuggestions
 
     # That's all of the caches we can try.  Bail if the caller is only requesting synchronous results.  We
@@ -104,8 +107,11 @@ CompletionSearch =
             # incorrect or out-of-date completion engines.
             try
               suggestions = engine.parse xhr
+              # Make all suggestions lower case.  It looks odd when suggestions from one completion engine are
+              # upper case, and those from another are lower case.
+              suggestions = (suggestion.toLowerCase() for suggestion in suggestions)
               # Filter out the query itself. It's not adding anything.
-              suggestions = (suggestion for suggestion in suggestions when suggestion.toLowerCase() != query)
+              suggestions = (suggestion for suggestion in suggestions when suggestion != query)
               console.log "GET", url if @debug
             catch
               suggestions = []