diff options
Diffstat (limited to 'background_scripts')
| -rw-r--r-- | background_scripts/completion.coffee | 29 | ||||
| -rw-r--r-- | background_scripts/completion_engines.coffee | 192 | ||||
| -rw-r--r-- | background_scripts/completion_search.coffee | 78 |
3 files changed, 182 insertions, 117 deletions
diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index cbc5e698..189929b4 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -454,29 +454,32 @@ class SearchEngineCompleter { keyword, searchUrl, description } = engine extend request, searchUrl, customSearchMode: true - factor = 0.5 + @previousSuggestions[searchUrl] ?= [] haveCompletionEngine = CompletionSearch.haveCompletionEngine searchUrl # This filter is applied to all of the suggestions from all of the completers, after they have been # aggregated by the MultiCompleter. filter = (suggestions) -> - suggestions.filter (suggestion) -> - # We only keep suggestions which either *were* generated by this search engine, or *could have - # been* generated by this search engine (and match the current query). - suggestion.searchUrl == searchUrl or - ( - terms = Utils.extractQuery searchUrl, suggestion.url - terms and RankingUtils.matches queryTerms, terms - ) + # We only keep suggestions which either *were* generated by this search engine, or *could have + # been* generated by this search engine (and match the current query). + for suggestion in suggestions + if suggestion.isSearchSuggestion or suggestion.isCustomSearch + suggestion + else + terms = Utils.extractQuery searchUrl, suggestion.url + continue unless terms and RankingUtils.matches queryTerms, terms + suggestion.url = Utils.createSearchUrl terms, searchUrl + suggestion # If a previous suggestion still matches the query, then we keep it (even if the completion engine may not - # return it for the current query). This allows the user to pick suggestions by typing fragments of their - # text, without regard to whether the completion engine can complete the actual text of the query. + # return it for the current query). This allows the user to pick suggestions that they've previously seen + # by typing fragments of their text, without regard to whether the completion engine can continue to + # complete the actual text of the query. previousSuggestions = if queryTerms.length == 0 [] else - for url, suggestion of @previousSuggestions + for _, suggestion of @previousSuggestions[searchUrl] continue unless RankingUtils.matches queryTerms, suggestion.title # Reset various fields, they may not be correct wrt. the current query. extend suggestion, relevancy: null, html: null, queryTerms: queryTerms @@ -501,7 +504,7 @@ class SearchEngineCompleter count = 0 (suggestion) => url = Utils.createSearchUrl suggestion, searchUrl - @previousSuggestions[url] = new Suggestion + @previousSuggestions[searchUrl][url] = new Suggestion queryTerms: queryTerms type: description url: url diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index dcbf99c6..b572375d 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -1,122 +1,152 @@ -# A completion engine provides search suggestions for a search engine. A search engine is identified by a -# "searchUrl", e.g. Settings.get("searchUrl"), or a custom search engine URL. +# A completion engine provides search suggestions for a custom search engine. A custom search engine is +# identified by a "searchUrl". An "engineUrl" is used for fetching suggestions, whereas a "searchUrl" is used +# for the actual search itself. # -# Each completion engine defines three functions: +# Each completion engine defines: # -# 1. "match" - This takes a searchUrl and returns a boolean indicating whether this completion engine can -# perform completion for the given search engine. +# 1. An "engineUrl". This is the URL to use for search completions and is passed as the option "engineUrl" +# to the "BaseEngine" constructor. # -# 2. "getUrl" - This takes a list of query terms (queryTerms) and generates a completion URL, that is, a URL -# which will provide completions for this completion engine. +# 2. One or more regular expressions which define the custom search engine URLs for which the completion +# engine will be used. This is passed as the "regexps" option to the "BaseEngine" constructor. # -# 3. "parse" - This takes a successful XMLHttpRequest object (the request has completed successfully), and -# returns a list of suggestions (a list of strings). This method is always executed within the context -# of a try/catch block, so errors do not propagate. +# 3. A "parse" function. This takes a successful XMLHttpRequest object (the request has completed +# successfully), and returns a list of suggestions (a list of strings). This method is always executed +# within the context of a try/catch block, so errors do not propagate. +# +# 4. Each completion engine *must* include an example custom search engine. The example must include an +# example "keyword" and an example "searchUrl", and may include an example "description" and an +# "explanation". # # Each new completion engine must be added to the list "CompletionEngines" at the bottom of this file. # # The lookup logic which uses these completion engines is in "./completion_search.coffee". # -# A base class for common regexp-based matching engines. -class RegexpEngine - constructor: (args...) -> @regexps = args.map (regexp) -> new RegExp regexp +# A base class for common regexp-based matching engines. "options" must define: +# options.engineUrl: the URL to use for the completion engine. This must be a string. +# options.regexps: one or regular expressions. This may either a single string or a list of strings. +# options.example: an example object containing at least "keyword" and "searchUrl", and optional "description". +class BaseEngine + constructor: (options) -> + extend this, options + @regexps = [ @regexps ] if "string" == typeof @regexps + @regexps = @regexps.map (regexp) -> new RegExp regexp + match: (searchUrl) -> Utils.matchesAnyRegexp @regexps, searchUrl + getUrl: (queryTerms) -> Utils.createSearchUrl queryTerms, @engineUrl -# Several Google completion engines package XML responses in this way. -class GoogleXMLRegexpEngine extends RegexpEngine +# Several Google completion engines package responses as XML. This parses such XML. +class GoogleXMLBaseEngine extends BaseEngine parse: (xhr) -> for suggestion in xhr.responseXML.getElementsByTagName "suggestion" continue unless suggestion = suggestion.getAttribute "data" suggestion -class Google extends GoogleXMLRegexpEngine - # Example search URL: http://www.google.com/search?q=%s - constructor: (regexps = null) -> - super regexps ? "^https?://[a-z]+\\.google\\.(com|ie|co\\.uk|ca|com\\.au)/" - - getUrl: (queryTerms) -> - Utils.createSearchUrl queryTerms, - "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=%s" - -# A wrapper class for Google completions. This adds prefix terms to the query, and strips those terms from -# the resulting suggestions. For example, for Google Maps, we add "map of" as a prefix, then strip "map of" -# from the resulting suggestions. -class GoogleWithPrefix - constructor: (prefix, args...) -> - @engine = new Google args... - @prefix = "#{prefix} " - @queryTerms = prefix.split /\s+/ - match: (args...) -> @engine.match args... - getUrl: (queryTerms) -> @engine.getUrl [ @queryTerms..., queryTerms... ] +class Google extends GoogleXMLBaseEngine + constructor: () -> + super + engineUrl: "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=%s" + regexps: "^https?://[a-z]+\\.google\\.(com|ie|co\\.uk|ca|com\\.au)/" + example: + searchUrl: "http://www.google.com/search?q=%s" + keyword: "g" + +class GoogleMaps extends GoogleXMLBaseEngine + prefix: "map of " + constructor: () -> + super + engineUrl: "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{@prefix.split(' ').join '+'}%s" + regexps: "^https?://[a-z]+\\.google\\.(com|ie|co\\.uk|ca|com\\.au)/maps" + example: + searchUrl: "https://www.google.com/maps?q=%s" + keyword: "m" + explanation: + """ + This uses regular Google completion, but prepends the text "<tt>map of</tt>" to the query. It works + well for places, countries, states, geographical regions and the like, but will not perform address + search. + """ + parse: (xhr) -> - for suggestion in @engine.parse xhr + for suggestion in super xhr continue unless suggestion.startsWith @prefix - suggestion[@prefix.length..].ltrim() - -# For Google Maps, we add the prefix "map of" to the query, and send it to Google's general search engine, -# then strip "map of" from the resulting suggestions. -class GoogleMaps extends GoogleWithPrefix - # Example search URL: https://www.google.com/maps?q=%s - constructor: -> super "map of", "^https?://[a-z]+\\.google\\.(com|ie|co\\.uk|ca|com\\.au)/maps" + suggestion[@prefix.length..] -class Youtube extends GoogleXMLRegexpEngine - # Example search URL: http://www.youtube.com/results?search_query=%s +class Youtube extends GoogleXMLBaseEngine constructor: -> - super "^https?://[a-z]+\\.youtube\\.com/results" + super + engineUrl: "http://suggestqueries.google.com/complete/search?client=youtube&ds=yt&xml=t&q=%s" + regexps: "^https?://[a-z]+\\.youtube\\.com/results" + example: + searchUrl: "http://www.youtube.com/results?search_query=%s" + keyword: "y" + +class Wikipedia extends BaseEngine + constructor: -> + super + engineUrl: "https://en.wikipedia.org/w/api.php?action=opensearch&format=json&search=%s" + regexps: "^https?://[a-z]+\\.wikipedia\\.org/" + example: + searchUrl: "http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s" + keyword: "w" - getUrl: (queryTerms) -> - Utils.createSearchUrl queryTerms, - "http://suggestqueries.google.com/complete/search?client=youtube&ds=yt&xml=t&q=%s" + parse: (xhr) -> JSON.parse(xhr.responseText)[1] -class Wikipedia extends RegexpEngine - # Example search URL: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s +class Bing extends BaseEngine constructor: -> - super "^https?://[a-z]+\\.wikipedia\\.org/" + super + engineUrl: "http://api.bing.com/osjson.aspx?query=%s" + regexps: "^https?://www\\.bing\\.com/search" + example: + searchUrl: "https://www.bing.com/search?q=%s" + keyword: "b" - getUrl: (queryTerms) -> - Utils.createSearchUrl queryTerms, - "https://en.wikipedia.org/w/api.php?action=opensearch&format=json&search=%s" + parse: (xhr) -> JSON.parse(xhr.responseText)[1] - parse: (xhr) -> - JSON.parse(xhr.responseText)[1] +class Amazon extends BaseEngine + constructor: -> + super + engineUrl: "https://completion.amazon.com/search/complete?method=completion&search-alias=aps&client=amazon-search-ui&mkt=1&q=%s" + regexps: "^https?://www\\.amazon\\.(com|co\\.uk|ca|com\\.au)/s/" + example: + searchUrl: "http://www.amazon.com/s/?field-keywords=%s" + keyword: "a" -class Bing extends RegexpEngine - # Example search URL: https://www.bing.com/search?q=%s - constructor: -> super "^https?://www\\.bing\\.com/search" - getUrl: (queryTerms) -> Utils.createSearchUrl queryTerms, "http://api.bing.com/osjson.aspx?query=%s" parse: (xhr) -> JSON.parse(xhr.responseText)[1] -class Amazon extends RegexpEngine - # Example search URL: http://www.amazon.com/s/?field-keywords=%s - constructor: -> super "^https?://www\\.amazon\\.(com|co\\.uk|ca|com\\.au)/s/" - getUrl: (queryTerms) -> - Utils.createSearchUrl queryTerms, - "https://completion.amazon.com/search/complete?method=completion&search-alias=aps&client=amazon-search-ui&mkt=1&q=%s" - parse: (xhr) -> JSON.parse(xhr.responseText)[1] +class DuckDuckGo extends BaseEngine + constructor: -> + super + engineUrl: "https://duckduckgo.com/ac/?q=%s" + regexps: "^https?://([a-z]+\\.)?duckduckgo\\.com/" + example: + searchUrl: "https://duckduckgo.com/?q=%s" + keyword: "d" -class DuckDuckGo extends RegexpEngine - # Example search URL: https://duckduckgo.com/?q=%s - constructor: -> super "^https?://([a-z]+\\.)?duckduckgo\\.com/" - getUrl: (queryTerms) -> Utils.createSearchUrl queryTerms, "https://duckduckgo.com/ac/?q=%s" parse: (xhr) -> suggestion.phrase for suggestion in JSON.parse xhr.responseText -class Webster extends RegexpEngine - # Example search URL: http://www.merriam-webster.com/dictionary/%s - constructor: -> super "^https?://www.merriam-webster.com/dictionary/" - getUrl: (queryTerms) -> Utils.createSearchUrl queryTerms, "http://www.merriam-webster.com/autocomplete?query=%s" +class Webster extends BaseEngine + constructor: -> + super + engineUrl: "http://www.merriam-webster.com/autocomplete?query=%s" + regexps: "^https?://www.merriam-webster.com/dictionary/" + example: + searchUrl: "http://www.merriam-webster.com/dictionary/%s" + keyword: "dw" + description: "Dictionary" + parse: (xhr) -> JSON.parse(xhr.responseText).suggestions # A dummy search engine which is guaranteed to match any search URL, but never produces completions. This # allows the rest of the logic to be written knowing that there will always be a completion engine match. -class DummyCompletionEngine - dummy: true - match: -> true - # We return a useless URL which we know will succeed, but which won't generate any network traffic. - getUrl: -> chrome.runtime.getURL "content_scripts/vimium.css" - parse: -> [] +class DummyCompletionEngine extends BaseEngine + constructor: -> + super + regexps: "." + dummy: true # Note: Order matters here. CompletionEngines = [ diff --git a/background_scripts/completion_search.coffee b/background_scripts/completion_search.coffee index b26194e6..7926b45b 100644 --- a/background_scripts/completion_search.coffee +++ b/background_scripts/completion_search.coffee @@ -1,4 +1,39 @@ +# This is a wrapper class for completion engines. It handles the case where a custom search engine includes a +# prefix query term (or terms). For example: +# +# http://www.google.com/search?q=javascript+%s +# +# In this case, we get better suggestions if we include the term "javascript" in queries sent to the +# completion engine. This wrapper handles adding such prefixes to completion-engine queries and removing them +# from the resulting suggestions. +class EnginePrefixWrapper + constructor: (@searchUrl, @engine) -> + + getUrl: (queryTerms) -> + # This tests whether @searchUrl contains something of the form "...=abc+def+%s...", from which we extract + # a prefix of the form "abc def ". + if /\=.+\+%s/.test @searchUrl + terms = @searchUrl.replace /\+%s.*/, "" + terms = terms.replace /.*=/, "" + terms = terms.replace /\+/g, " " + + queryTerms = [ terms.split(" ")..., queryTerms... ] + prefix = "#{terms} " + + @postprocessSuggestions = + (suggestions) -> + for suggestion in suggestions + continue unless suggestion.startsWith prefix + suggestion[prefix.length..] + + @engine.getUrl queryTerms + + parse: (xhr) -> + @postprocessSuggestions @engine.parse xhr + + postprocessSuggestions: (suggestions) -> suggestions + CompletionSearch = debug: false inTransit: {} @@ -58,34 +93,30 @@ CompletionSearch = return callback [] if 1 == queryTerms.length and Utils.isUrl query return callback [] if Utils.hasJavascriptPrefix query - # Cache completions. However, completions depend upon both the searchUrl and the query terms. So we need - # to generate a key. We mix in some junk generated by pwgen. A key clash might be possible, but - # is vanishingly unlikely. - junk = "//Zi?ei5;o//" - completionCacheKey = searchUrl + junk + queryTerms.map((s) -> s.toLowerCase()).join junk - + completionCacheKey = JSON.stringify [ searchUrl, queryTerms ] if @completionCache.has completionCacheKey console.log "hit", completionCacheKey if @debug return callback @completionCache.get completionCacheKey # If the user appears to be typing a continuation of the characters of the most recent query, then we can # sometimes re-use the previous suggestions. - if @mostRecentQuery? and @mostRecentSuggestions? - reusePreviousSuggestions = do => - # Verify that the previous query is a prefix of the current query. - return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase() - # Verify that every previous suggestion contains the text of the new query. - # Note: @mostRecentSuggestions may also be empty, in which case we drop though. The effect is that - # previous queries with no suggestions suppress subsequent no-hope HTTP requests as the user continues - # to type. - for suggestion in @mostRecentSuggestions - return false unless 0 <= suggestion.indexOf query - # Ok. Re-use the suggestion. - true - - if reusePreviousSuggestions - console.log "reuse previous query:", @mostRecentQuery, @mostRecentSuggestions.length if @debug - return callback @completionCache.set completionCacheKey, @mostRecentSuggestions + if @mostRecentQuery? and @mostRecentSuggestions? and @mostRecentSearchUrl? + if searchUrl == @mostRecentSearchUrl + reusePreviousSuggestions = do => + # Verify that the previous query is a prefix of the current query. + return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase() + # Verify that every previous suggestion contains the text of the new query. + # Note: @mostRecentSuggestions may also be empty, in which case we drop though. The effect is that + # previous queries with no suggestions suppress subsequent no-hope HTTP requests as the user continues + # to type. + for suggestion in @mostRecentSuggestions + return false unless 0 <= suggestion.indexOf query + # Ok. Re-use the suggestion. + true + + if reusePreviousSuggestions + console.log "reuse previous query:", @mostRecentQuery, @mostRecentSuggestions.length if @debug + return callback @completionCache.set completionCacheKey, @mostRecentSuggestions # That's all of the caches we can try. Bail if the caller is only requesting synchronous results. We # signal that we haven't found a match by returning null. @@ -98,7 +129,7 @@ CompletionSearch = # Elide duplicate requests. First fetch the suggestions... @inTransit[completionCacheKey] ?= new AsyncDataFetcher (callback) => - engine = @lookupEngine searchUrl + engine = new EnginePrefixWrapper searchUrl, @lookupEngine searchUrl url = engine.getUrl queryTerms @get searchUrl, url, (xhr = null) => @@ -124,6 +155,7 @@ CompletionSearch = # ... then use the suggestions. @inTransit[completionCacheKey].use (suggestions) => + @mostRecentSearchUrl = searchUrl @mostRecentQuery = query @mostRecentSuggestions = suggestions callback @completionCache.set completionCacheKey, suggestions |