Re-work completions: extend engine wrapper to handle prefixes.

This commit contains the bulk og the material changes for which the
previous commits established the basis.

1) Add a general framework for detecting query prefixes in search URLs,
adding them to query sent to the completion engine, and stripping them
from the resulting suggestions.  This allows the user to have a
search engine...

   j: http://www.google.com/search?q=javascript+%s Javascript

and have the prefix "javascript" included (automatically) in queries
sent to completion engines, which results in substantially better
suggestions.

2) Re-work completion for Google Maps in a simpler form.

4 files changed, 86 insertions, 55 deletions

diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee
index c83066a6..6b58f4ea 100644
--- a/background_scripts/completion.coffee
+++ b/background_scripts/completion.coffee
@@ -450,7 +450,7 @@ class SearchEngineCompleter
     { keyword, searchUrl, description } = engine
     extend request, searchUrl, customSearchMode: true
 
-    factor = 0.5
+    @previousSuggestions[searchUrl] ?= []
     haveCompletionEngine = CompletionSearch.haveCompletionEngine searchUrl
 
     # This filter is applied to all of the suggestions from all of the completers, after they have been
@@ -466,13 +466,14 @@ class SearchEngineCompleter
           )
 
     # If a previous suggestion still matches the query, then we keep it (even if the completion engine may not
-    # return it for the current query).  This allows the user to pick suggestions by typing fragments of their
-    # text, without regard to whether the completion engine can complete the actual text of the query.
+    # return it for the current query).  This allows the user to pick suggestions they've previously seen by
+    # typing fragments of their text, without regard to whether the completion engine can continue to complete
+    # the actual text of the query.
     previousSuggestions =
       if queryTerms.length == 0
         []
       else
-        for url, suggestion of @previousSuggestions
+        for url, suggestion of @previousSuggestions[searchUrl]
           continue unless RankingUtils.matches queryTerms, suggestion.title
           # Reset various fields, they may not be correct wrt. the current query.
           extend suggestion, relevancy: null, html: null, queryTerms: queryTerms
@@ -484,6 +485,7 @@ class SearchEngineCompleter
       type: description
       url: Utils.createSearchUrl queryTerms, searchUrl
       title: queryTerms.join " "
+      searchUrl: searchUrl
       relevancy: 2.0
       autoSelect: true
       highlightTerms: false
@@ -496,11 +498,12 @@ class SearchEngineCompleter
       count = 0
       (suggestion) =>
         url = Utils.createSearchUrl suggestion, searchUrl
-        @previousSuggestions[url] = new Suggestion
+        @previousSuggestions[searchUrl][url] = new Suggestion
           queryTerms: queryTerms
           type: description
           url: url
           title: suggestion
+          searchUrl: searchUrl
           insertText: suggestion
           highlightTerms: false
           highlightTermsExcludeUrl: true
diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee
index afbb2040..b572375d 100644
--- a/background_scripts/completion_engines.coffee
+++ b/background_scripts/completion_engines.coffee
@@ -15,9 +15,9 @@
 #      successfully), and returns a list of suggestions (a list of strings).  This method is always executed
 #      within the context of a try/catch block, so errors do not propagate.
 #
-#   4. For documentation only, each completion engine *must* and example custom search engine.  The example
-#      must include an example "keyword" and and example "searchUrl", and may include and example
-#      "description".
+#   4. Each completion engine *must* include an example custom search engine.  The example must include an
+#      example "keyword" and an example "searchUrl", and may include an example "description" and an
+#      "explanation".
 #
 # Each new completion engine must be added to the list "CompletionEngines" at the bottom of this file.
 #
@@ -45,37 +45,34 @@ class GoogleXMLBaseEngine extends BaseEngine
       suggestion
 
 class Google extends GoogleXMLBaseEngine
-  constructor: (regexps = null) ->
+  constructor: () ->
     super
       engineUrl: "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=%s"
-      regexps: regexps ? "^https?://[a-z]+\\.google\\.(com|ie|co\\.uk|ca|com\\.au)/"
+      regexps: "^https?://[a-z]+\\.google\\.(com|ie|co\\.uk|ca|com\\.au)/"
       example:
         searchUrl: "http://www.google.com/search?q=%s"
         keyword: "g"
 
-## # A wrapper class for Google completions.  This adds prefix terms to the query, and strips those terms from
-## # the resulting suggestions.  For example, for Google Maps, we add "map of" as a prefix, then strip "map of"
-## # from the resulting suggestions.
-## class GoogleWithPrefix extends Google
-##   constructor: (prefix, args...) ->
-##     super args...
-##     prefix = prefix.trim()
-##     @prefix = "#{prefix} "
-##     @queryTerms = prefix.split /\s+/
-##   getUrl: (queryTerms) -> super [ @queryTerms..., queryTerms... ]
-##   parse: (xhr) ->
-##     super(xhr)
-##       .filter (suggestion) => suggestion.startsWith @prefix
-##       .map (suggestion) => suggestion[@prefix.length..].ltrim()
-##
-## # For Google Maps, we add the prefix "map of" to the query, and send it to Google's general search engine,
-## # then strip "map of" from the resulting suggestions.
-## class GoogleMaps extends GoogleWithPrefix
-##   constructor: ->
-##     super "map of", "^https?://[a-z]+\\.google\\.(com|ie|co\\.uk|ca|com\\.au)/maps"
-##     @exampleSearchUrl = "https://www.google.com/maps?q=%s"
-##     @exampleKeyword = "m"
-##     @exampleDescription = "Google maps"
+class GoogleMaps extends GoogleXMLBaseEngine
+  prefix: "map of "
+  constructor: () ->
+    super
+      engineUrl: "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{@prefix.split(' ').join '+'}%s"
+      regexps: "^https?://[a-z]+\\.google\\.(com|ie|co\\.uk|ca|com\\.au)/maps"
+      example:
+        searchUrl: "https://www.google.com/maps?q=%s"
+        keyword: "m"
+        explanation:
+          """
+          This uses regular Google completion, but prepends the text "<tt>map of</tt>" to the query.  It works
+          well for places, countries, states, geographical regions and the like, but will not perform address
+          search.
+          """
+
+  parse: (xhr) ->
+    for suggestion in super xhr
+      continue unless suggestion.startsWith @prefix
+      suggestion[@prefix.length..]
 
 class Youtube extends GoogleXMLBaseEngine
   constructor: ->
@@ -154,7 +151,7 @@ class DummyCompletionEngine extends BaseEngine
 # Note: Order matters here.
 CompletionEngines = [
   Youtube
-  # GoogleMaps
+  GoogleMaps
   Google
   DuckDuckGo
   Wikipedia
diff --git a/background_scripts/completion_search.coffee b/background_scripts/completion_search.coffee
index b3ae88d4..09261ff6 100644
--- a/background_scripts/completion_search.coffee
+++ b/background_scripts/completion_search.coffee
@@ -1,12 +1,38 @@
 
-class EngineWrapper
+# This is a wrapper class for completion engines.  It handles the case where a custom search engine includes a
+# prefix query term (or terms).  For example:
+#
+#   http://www.google.com/search?q=javascript+%s
+#
+# In this case, we get better suggestions if we include the term "javascript" in queries sent to the
+# completion engine.  This wrapper handles adding such prefixes to completion-engine queries and removing them
+# from the resulting suggestions.
+class EnginePrefixWrapper
   constructor: (@searchUrl, @engine) ->
 
   getUrl: (queryTerms) ->
+    # This tests whether @searchUrl contains something of the form "...=abc+def+%s...", from which we extract
+    # a prefix of the form "abc def ".
+    if /\=.+\+%s/.test @searchUrl
+      terms = @searchUrl.replace /\+%s.*/, ""
+      terms = terms.replace /.*=/, ""
+      terms = terms.replace /\+/g, " "
+
+      queryTerms = [ terms.split(" ")..., queryTerms... ]
+      prefix = "#{terms} "
+
+      @postprocessSuggestions =
+        (suggestions) ->
+          for suggestion in suggestions
+            continue unless suggestion.startsWith prefix
+            suggestion[prefix.length..]
+
     @engine.getUrl queryTerms
 
   parse: (xhr) ->
-    @engine.parse xhr
+    @postprocessSuggestions @engine.parse xhr
+
+  postprocessSuggestions: (suggestions) -> suggestions
 
 CompletionSearch =
   debug: false
@@ -74,22 +100,23 @@ CompletionSearch =
 
     # If the user appears to be typing a continuation of the characters of the most recent query, then we can
     # sometimes re-use the previous suggestions.
-    if @mostRecentQuery? and @mostRecentSuggestions?
-      reusePreviousSuggestions = do =>
-        # Verify that the previous query is a prefix of the current query.
-        return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase()
-        # Verify that every previous suggestion contains the text of the new query.
-        # Note: @mostRecentSuggestions may also be empty, in which case we drop though. The effect is that
-        # previous queries with no suggestions suppress subsequent no-hope HTTP requests as the user continues
-        # to type.
-        for suggestion in @mostRecentSuggestions
-          return false unless 0 <= suggestion.indexOf query
-        # Ok. Re-use the suggestion.
-        true
-
-      if reusePreviousSuggestions
-        console.log "reuse previous query:", @mostRecentQuery, @mostRecentSuggestions.length if @debug
-        return callback @completionCache.set completionCacheKey, @mostRecentSuggestions
+    if @mostRecentQuery? and @mostRecentSuggestions? and @mostRecentSearchUrl?
+      if searchUrl == @mostRecentSearchUrl
+        reusePreviousSuggestions = do =>
+          # Verify that the previous query is a prefix of the current query.
+          return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase()
+          # Verify that every previous suggestion contains the text of the new query.
+          # Note: @mostRecentSuggestions may also be empty, in which case we drop though. The effect is that
+          # previous queries with no suggestions suppress subsequent no-hope HTTP requests as the user continues
+          # to type.
+          for suggestion in @mostRecentSuggestions
+            return false unless 0 <= suggestion.indexOf query
+          # Ok. Re-use the suggestion.
+          true
+
+        if reusePreviousSuggestions
+          console.log "reuse previous query:", @mostRecentQuery, @mostRecentSuggestions.length if @debug
+          return callback @completionCache.set completionCacheKey, @mostRecentSuggestions
 
     # That's all of the caches we can try.  Bail if the caller is only requesting synchronous results.  We
     # signal that we haven't found a match by returning null.
@@ -102,7 +129,7 @@ CompletionSearch =
 
         # Elide duplicate requests. First fetch the suggestions...
         @inTransit[completionCacheKey] ?= new AsyncDataFetcher (callback) =>
-          engine = new EngineWrapper searchUrl, @lookupEngine searchUrl
+          engine = new EnginePrefixWrapper searchUrl, @lookupEngine searchUrl
           url = engine.getUrl queryTerms
 
           @get searchUrl, url, (xhr = null) =>
@@ -128,6 +155,7 @@ CompletionSearch =
 
         # ... then use the suggestions.
         @inTransit[completionCacheKey].use (suggestions) =>
+          @mostRecentSearchUrl = searchUrl
           @mostRecentQuery = query
           @mostRecentSuggestions = suggestions
           callback @completionCache.set completionCacheKey, suggestions
diff --git a/pages/completion_engines.coffee b/pages/completion_engines.coffee
index d744b3b3..790f2968 100644
--- a/pages/completion_engines.coffee
+++ b/pages/completion_engines.coffee
@@ -11,12 +11,15 @@ DomUtils.documentReady ->
     engine = new engine
     html.push "<h4>#{engine.constructor.name}</h4>\n"
     html.push "<div class=\"engine\">"
+    if engine.example.explanation
+      html.push "<p>#{engine.example.explanation}</p>"
     if engine.regexps
+      html.push "<p>"
+      html.push "Regular expression#{if 1 < engine.regexps.length then 's' else ''}:"
       html.push "<pre>"
       html.push "#{cleanUpRegexp re}\n" for re in engine.regexps
       html.push "</pre>"
-    if engine.prefix
-      html.push "<p>This uses the general Google completion engine, but adds the prefix \"<tt>#{engine.prefix.trim()}</tt>\" to the query.</p>"
+      html.push "</p>"
     if engine.example.searchUrl and engine.example.keyword
       engine.example.description ||= engine.constructor.name
       html.push "<p>"