Rework some aspects of search engine completion.

- We arrange that:
   - for custom search engines, or
   - for other searches where there is at least one match
  that match is at the top of the list (relevancy 1).

This top suggestion will the suggestion which will be used to popuplated
the promted text in the vomnibar.  If we arrange that it's at the top of
the list, then it is also the text that we get in the vomnibar if we hit
<Tab>.

This leaves us in the position where:

   - If the user wants just the text they've typed, then they hit
     <Enter>.

   - If they want all of the text in the input (including the prompted
     text), then they hit <Tab>, <Enter>.  This is a very natural UX.
     It feels like <Tab> is filling in the prompted text, although all
     it's really doing is choosing a suggestion (the first one), as it
     always has done.

There is one small catch. We need to avoid a clash with the domain
completer, which also likes to force its suggestion to the top of the
list.  There's a test for that case.  Basically, we can apply this trick
if it's a custom search, or if the user has finished typing the first
query term.

2 files changed, 64 insertions, 45 deletions

diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee
index 28650711..02030444 100644
--- a/background_scripts/completion.coffee
+++ b/background_scripts/completion.coffee
@@ -403,7 +403,7 @@ class SearchEngineCompleter
         keywords: key for own key of engines
 
   filter: ({ queryTerms, query, engine }, onComplete) ->
-    suggestions = []
+    [ primarySuggestion, removePrimarySuggestion ] = [ null, false ]
 
     { custom, searchUrl, description } =
       if engine
@@ -418,7 +418,6 @@ class SearchEngineCompleter
 
     return onComplete [] unless custom or 0 < queryTerms.length
 
-    query = queryTerms.join " "
     factor = Settings.get "omniSearchWeight"
     haveCompletionEngine = CompletionSearch.haveCompletionEngine searchUrl
     haveCompletionEngine = false if factor == 0.0 and not custom
@@ -445,66 +444,85 @@ class SearchEngineCompleter
 
     filter = null
     if useExclusiveVomnibar
-      # We accept suggestions from this completer; and we also accept suggestions from other completers, but
-      # only if their URL matches this search engine and the query (ie. they could have been generated by this
-      # search engine previously).
-      filter = (suggestion) ->
-        suggestion.type == description or
-          # This is a suggestion for the same search engine.
-          (suggestion.url.startsWith(engine.searchUrlPrefix) and
-            # And the URL suffix (which must contain the query part) matches the current query.
-            RankingUtils.matches queryTerms, suggestion.url[engine.searchUrlPrefix.length..])
+      filter = (suggestions) ->
+        # We accept suggestions from this completer; and we also accept suggestions from other completers, but
+        # only if their URL matches this search engine and this query (ie. only if they could have been
+        # generated by this search engine previously).
+        suggestions = suggestions.filter (suggestion) ->
+          suggestion.type == description or
+            # This is a suggestion for the same search engine.
+            (suggestion.url.startsWith(engine.searchUrlPrefix) and
+              # And the URL suffix (which must contain the query part) matches the current query.
+              RankingUtils.matches queryTerms, suggestion.url[engine.searchUrlPrefix.length..])
+
+        # If we've delivered suggestions from a completion engine, then we can strip out the primary
+        # suggestion.
+        if removePrimarySuggestion
+          suggestions = suggestions.filter (suggestion) -> suggestion != primarySuggestion
+
+        suggestions
 
     # For custom search engines, we add a single, top-ranked entry for the unmodified query.  This
-    # suggestion always appears at the top of the list.
+    # suggestion appears at the top of the list.  This is the primary suggestion.
     if custom
-      suggestions.push new Suggestion
+      primarySuggestion = new Suggestion
         queryTerms: queryTerms
         type: description
         url: Utils.createSearchUrl queryTerms, searchUrl
-        title: query
+        title: queryTerms.join " "
         relevancy: 1
-        insertText: if useExclusiveVomnibar then query else null
+        insertText: query
         # We suppress the leading keyword, for example "w something" becomes "something" in the vomnibar.
         suppressLeadingKeyword: true
-        customSearchEnginePrimarySuggestion: true
         # Toggles for the legacy behaviour.
         autoSelect: not useExclusiveVomnibar
         forceAutoSelect: not useExclusiveVomnibar
         highlightTerms: not useExclusiveVomnibar
 
-    mkSuggestion = do ->
-      (suggestion) ->
-        new Suggestion
-          queryTerms: queryTerms
-          type: description
-          url: Utils.createSearchUrl suggestion, searchUrl
-          title: suggestion
-          relevancy: relevancy *= 0.9
-          insertText: suggestion
-          highlightTerms: false
-          customSearchEngineCompletionSuggestion: true
+    mkSuggestion = (suggestion) ->
+      new Suggestion
+        queryTerms: queryTerms
+        type: description
+        url: Utils.createSearchUrl suggestion, searchUrl
+        title: suggestion
+        relevancy: relevancy *= 0.9
+        insertText: suggestion
+        highlightTerms: false
+        searchEngineCompletionSuggestion: true
+
+    deliverCompletions = (onComplete, completions, args...) ->
+      # Make the first suggestion float to the top of the vomnibar (except if we would be competing with the
+      # domain completer).
+      if 0 < completions.length
+        if custom or (1 < queryTerms.length or /\S\s/.test query)
+          completions[0].relevancy = 1
+      onComplete completions, args...
 
     # If we have cached suggestions, then we can bundle them immediately (otherwise we'll have to fetch them
     # asynchronously).
-    cachedSuggestions = CompletionSearch.complete searchUrl, queryTerms
+    cachedSuggestions = null
+    cachedSuggestions = CompletionSearch.complete searchUrl, queryTerms if haveCompletionEngine
+
+    suggestions =
+      if haveCompletionEngine and cachedSuggestions? and 0 < cachedSuggestions.length
+        cachedSuggestions.map mkSuggestion
+      else if custom
+        [ primarySuggestion ]
+      else
+        []
 
-    # Post suggestions and bail if we already have all of the suggestions, or if there is no prospect of
-    # adding further suggestions.
     if queryTerms.length == 0 or cachedSuggestions? or not haveCompletionEngine
-      if haveCompletionEngine and cachedSuggestions?
-        console.log "cached suggestions:", cachedSuggestions.length, query if SearchEngineCompleter.debug
-        suggestions.push cachedSuggestions.map(mkSuggestion)...
-      return onComplete suggestions, { filter, continuation: null }
-
-    # Post any initial suggestion, and then deliver the rest of the suggestions as a continuation (so,
-    # asynchronously).
-    onComplete suggestions,
-      filter: filter
-      continuation: (onComplete) =>
-        CompletionSearch.complete searchUrl, queryTerms, (suggestions = []) =>
-          console.log "fetched suggestions:", suggestions.length, query if SearchEngineCompleter.debug
-          onComplete suggestions.map mkSuggestion
+      # There is no prospect of adding further completions.
+      deliverCompletions onComplete, suggestions, { filter, continuation: null }
+    else
+      # Post initial suggestions, then deliver further completions asynchronously, as a continuation.
+      deliverCompletions onComplete, suggestions,
+        filter: filter
+        continuation: (onComplete) =>
+          CompletionSearch.complete searchUrl, queryTerms, (suggestions = []) =>
+            console.log "fetched suggestions:", suggestions.length, query if SearchEngineCompleter.debug
+            removePrimarySuggestion = primarySuggestion? and 0 < suggestions.length
+            deliverCompletions onComplete, suggestions.map mkSuggestion
 
 # A completer which calls filter() on many completers, aggregates the results, ranks them, and returns the top
 # 10. All queries from the vomnibar come through a multi completer.
@@ -543,7 +561,7 @@ class MultiCompleter
     # Once all completers have finished, process the results and post them, and run any continuations or a
     # pending query.
     jobs.onReady =>
-      suggestions = suggestions.filter filter for filter in filters
+      suggestions = filter suggestions for filter in filters
       shouldRunContinuations = 0 < continuations.length and not @mostRecentQuery?
 
       # Post results, unless there are none and we will be running a continuation.  This avoids
@@ -565,6 +583,7 @@ class MultiCompleter
               callback()
 
         jobs.onReady =>
+          suggestions = filter suggestions for filter in filters
           suggestions = @prepareSuggestions queryTerms, suggestions
           # We post these results even if a new query has started.  The vomnibar will not display them
           # (because they're arriving too late), but it will cache them.
diff --git a/pages/vomnibar.coffee b/pages/vomnibar.coffee
index 6bfa2474..b301bec3 100644
--- a/pages/vomnibar.coffee
+++ b/pages/vomnibar.coffee
@@ -132,7 +132,7 @@ class VomnibarUI
     # Bail if there's an update pending (because then @input and the completion state are out of sync).
     return if @updateTimer?
 
-    completions = @completions.filter (completion) -> completion.customSearchEngineCompletionSuggestion
+    completions = @completions.filter (completion) -> completion.searchEngineCompletionSuggestion
     return unless 0 < completions.length
 
     query = value.ltrim().split(/\s+/).join(" ").toLowerCase()