From 1debac63fcc71c88427da9b1ae450067c15cd2b2 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 1 May 2015 12:20:19 +0100 Subject: Rename SearchEngineCompleter to CustomSearchEngineCompleter. The name "SearchEngineCompleter" is more appropriate for completions actually delivered by a search engine. Also, what was SearchEngineCompleter is usually referred to as "custom search engines" elsewhere. --- background_scripts/completion.coffee | 16 ++++++++-------- background_scripts/main.coffee | 2 +- background_scripts/settings.coffee | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 6a1c0d30..21730c4e 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -322,7 +322,7 @@ class TabCompleter tabRecency.recencyScore(suggestion.tabId) # A completer which will return your search engines -class SearchEngineCompleter +class CustomSearchEngineCompleter searchEngines: {} filter: (queryTerms, onComplete) -> @@ -344,7 +344,7 @@ class SearchEngineCompleter computeRelevancy: -> 1 refresh: -> - @searchEngines = SearchEngineCompleter.getSearchEngines() + @searchEngines = CustomSearchEngineCompleter.getSearchEngines() getSearchEngineMatches: (queryTerms) -> (1 < queryTerms.length and @searchEngines[queryTerms[0]]) or {} @@ -353,9 +353,9 @@ class SearchEngineCompleter # mapping in @searchEnginesMap. @searchEnginesMap: null - # Parse the custom search engines setting and cache it in SearchEngineCompleter.searchEnginesMap. + # Parse the custom search engines setting and cache it in CustomSearchEngineCompleter.searchEnginesMap. @parseSearchEngines: (searchEnginesText) -> - searchEnginesMap = SearchEngineCompleter.searchEnginesMap = {} + searchEnginesMap = CustomSearchEngineCompleter.searchEnginesMap = {} for line in searchEnginesText.split /\n/ tokens = line.trim().split /\s+/ continue if tokens.length < 2 or tokens[0].startsWith('"') or tokens[0].startsWith("#") @@ -367,9 +367,9 @@ class SearchEngineCompleter # Fetch the search-engine map, building it if necessary. @getSearchEngines: -> - unless SearchEngineCompleter.searchEnginesMap? - SearchEngineCompleter.parseSearchEngines Settings.get "searchEngines" - SearchEngineCompleter.searchEnginesMap + unless CustomSearchEngineCompleter.searchEnginesMap? + CustomSearchEngineCompleter.parseSearchEngines Settings.get "searchEngines" + CustomSearchEngineCompleter.searchEnginesMap # A completer which calls filter() on many completers, aggregates the results, ranks them, and returns the top # 10. Queries from the vomnibar frontend script come through a multi completer. @@ -617,7 +617,7 @@ root.MultiCompleter = MultiCompleter root.HistoryCompleter = HistoryCompleter root.DomainCompleter = DomainCompleter root.TabCompleter = TabCompleter -root.SearchEngineCompleter = SearchEngineCompleter +root.CustomSearchEngineCompleter = CustomSearchEngineCompleter root.HistoryCache = HistoryCache root.RankingUtils = RankingUtils root.RegexpCache = RegexpCache diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index 37c65592..353f0e53 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -47,7 +47,7 @@ completionSources = history: new HistoryCompleter() domains: new DomainCompleter() tabs: new TabCompleter() - seachEngines: new SearchEngineCompleter() + seachEngines: new CustomSearchEngineCompleter() completers = omni: new MultiCompleter([ diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index a4d95c81..a73a9d5c 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -33,7 +33,7 @@ root.Settings = Settings = root.refreshCompletionKeysAfterMappingSave() searchEngines: (value) -> - root.SearchEngineCompleter.parseSearchEngines value + root.CustomSearchEngineCompleter.parseSearchEngines value exclusionRules: (value) -> root.Exclusions.postUpdateHook value -- cgit v1.2.3 From 3616e0d01ffb38e9b7d344b99ad6ce7bc6aa071e Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 1 May 2015 13:19:23 +0100 Subject: Search completion; initial working version. --- background_scripts/completion.coffee | 62 +++++++++++++++++++++++++++++++++++- background_scripts/main.coffee | 6 ++-- 2 files changed, 65 insertions(+), 3 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 21730c4e..4f94f9e9 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -13,7 +13,7 @@ # It also has an attached "computeRelevancyFunction" which determines how well this item matches the given # query terms. class Suggestion - showRelevancy: false # Set this to true to render relevancy when debugging the ranking scores. + showRelevancy: true # Set this to true to render relevancy when debugging the ranking scores. # - type: one of [bookmark, history, tab]. # - computeRelevancyFunction: a function which takes a Suggestion and returns a relevancy score @@ -321,6 +321,65 @@ class TabCompleter else tabRecency.recencyScore(suggestion.tabId) +# searchUrl is the URL that will be used for the search, either the default search URL, or a custom +# search-engine URL. The other arguments area obvious. +# If we know the search-suggestion URL for searchUrl, then use it to pass a list of suggestions to callback. +# Otherwise, just call callback. +# +# Note: That's all TBD. For now, we just assume Google and use it. +# +getOnlineSuggestions = do -> + xhrs = {} # Maps searchUrl to outstanding HTTP request. + (searchUrl, queryTerms, callback) -> + # Cancel any outstanding requests. + xhrs?[searchUrl]?.abort() + xhrs[searchUrl] = null + + sendNoSuggestions = -> xhrs[searchUrl] = null; callback [] + return sendNoSuggestions() if queryTerms.length == 0 + + url = "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{Utils.createSearchQuery queryTerms}" + xhrs[searchUrl] = xhr = new XMLHttpRequest() + xhr.open "GET", url, true + xhr.timeout = 500 + xhr.ontimeout = sendNoSuggestions + xhr.onerror = sendNoSuggestions + xhr.send() + + xhr.onreadystatechange = (response) => + if xhr.readyState == 4 + suggestions = xhr.responseXML?.getElementsByTagName "suggestion" + return sendNoSuggestions() unless xhr.status == 200 and suggestions + xhr[searchUrl] = null + suggestions = + for suggestion in suggestions + continue unless suggestion = suggestion.getAttribute "data" + suggestion + callback suggestions + +class SearchEngineCompleter + refresh: -> + filter: (queryTerms, onComplete) -> + return onComplete([]) if queryTerms.length == 0 + + getOnlineSuggestions Settings.get("searchUrl"), queryTerms, (suggestions) => + completions = + for suggestion in suggestions + url = Utils.createSearchUrl suggestion.split /\s+/ + new Suggestion queryTerms, "suggestion", url, suggestion, @computeRelevancy + characterCount = queryTerms.join(" ").length + completion.characterCount = characterCount for completion in completions + onComplete completions + + computeRelevancy: (suggestion) -> + # We score search-engine completions by word relevancy, but weight increasingly as the number of + # characters in the query terms increases. The idea is that, the more the user has had to type, the less + # likely it is that one of the other suggestion types has found what they're looking for, so the more + # likely it is that a search suggestion will be useful. + # (1.0 - (1.0 / suggestion.characterCount)) * + (Math.min(suggestion.characterCount, 12)/12) * + RankingUtils.wordRelevancy suggestion.queryTerms, suggestion.title, suggestion.title + # A completer which will return your search engines class CustomSearchEngineCompleter searchEngines: {} @@ -617,6 +676,7 @@ root.MultiCompleter = MultiCompleter root.HistoryCompleter = HistoryCompleter root.DomainCompleter = DomainCompleter root.TabCompleter = TabCompleter +root.SearchEngineCompleter = SearchEngineCompleter root.CustomSearchEngineCompleter = CustomSearchEngineCompleter root.HistoryCache = HistoryCache root.RankingUtils = RankingUtils diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index 353f0e53..6f7db05c 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -47,11 +47,13 @@ completionSources = history: new HistoryCompleter() domains: new DomainCompleter() tabs: new TabCompleter() - seachEngines: new CustomSearchEngineCompleter() + customSearchEngines: new CustomSearchEngineCompleter() + searchEngines: new SearchEngineCompleter() completers = omni: new MultiCompleter([ - completionSources.seachEngines, + completionSources.searchEngines, + completionSources.customSearchEngines, completionSources.bookmarks, completionSources.history, completionSources.domains]) -- cgit v1.2.3 From d848f50fb1c199de581bf63e18495b7f4d0c4faf Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 2 May 2015 08:30:15 +0100 Subject: Search completion; refactor to separate file. --- background_scripts/completion.coffee | 59 ++++--------------- background_scripts/search_engines.coffee | 98 ++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 48 deletions(-) create mode 100644 background_scripts/search_engines.coffee (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 4f94f9e9..40c0d119 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -321,63 +321,26 @@ class TabCompleter else tabRecency.recencyScore(suggestion.tabId) -# searchUrl is the URL that will be used for the search, either the default search URL, or a custom -# search-engine URL. The other arguments area obvious. -# If we know the search-suggestion URL for searchUrl, then use it to pass a list of suggestions to callback. -# Otherwise, just call callback. -# -# Note: That's all TBD. For now, we just assume Google and use it. -# -getOnlineSuggestions = do -> - xhrs = {} # Maps searchUrl to outstanding HTTP request. - (searchUrl, queryTerms, callback) -> - # Cancel any outstanding requests. - xhrs?[searchUrl]?.abort() - xhrs[searchUrl] = null - - sendNoSuggestions = -> xhrs[searchUrl] = null; callback [] - return sendNoSuggestions() if queryTerms.length == 0 - - url = "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{Utils.createSearchQuery queryTerms}" - xhrs[searchUrl] = xhr = new XMLHttpRequest() - xhr.open "GET", url, true - xhr.timeout = 500 - xhr.ontimeout = sendNoSuggestions - xhr.onerror = sendNoSuggestions - xhr.send() - - xhr.onreadystatechange = (response) => - if xhr.readyState == 4 - suggestions = xhr.responseXML?.getElementsByTagName "suggestion" - return sendNoSuggestions() unless xhr.status == 200 and suggestions - xhr[searchUrl] = null - suggestions = - for suggestion in suggestions - continue unless suggestion = suggestion.getAttribute "data" - suggestion - callback suggestions - class SearchEngineCompleter refresh: -> - filter: (queryTerms, onComplete) -> - return onComplete([]) if queryTerms.length == 0 - getOnlineSuggestions Settings.get("searchUrl"), queryTerms, (suggestions) => - completions = - for suggestion in suggestions - url = Utils.createSearchUrl suggestion.split /\s+/ - new Suggestion queryTerms, "suggestion", url, suggestion, @computeRelevancy - characterCount = queryTerms.join(" ").length - completion.characterCount = characterCount for completion in completions - onComplete completions + filter: (queryTerms, onComplete) -> + SearchEngines.complete Settings.get("searchUrl"), queryTerms, (suggestions = []) => + console.log suggestions.length + characterCount = queryTerms.join("").length + completions = + for suggestion in suggestions + url = Utils.createSearchUrl suggestion.split /\s+/ + new Suggestion queryTerms, "search", url, suggestion, @computeRelevancy, characterCount + onComplete completions computeRelevancy: (suggestion) -> # We score search-engine completions by word relevancy, but weight increasingly as the number of # characters in the query terms increases. The idea is that, the more the user has had to type, the less # likely it is that one of the other suggestion types has found what they're looking for, so the more - # likely it is that a search suggestion will be useful. + # likely it is that this suggestion will be useful. # (1.0 - (1.0 / suggestion.characterCount)) * - (Math.min(suggestion.characterCount, 12)/12) * + (Math.min(suggestion.extraRelevancyData, 12)/12) * RankingUtils.wordRelevancy suggestion.queryTerms, suggestion.title, suggestion.title # A completer which will return your search engines diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee new file mode 100644 index 00000000..485accc1 --- /dev/null +++ b/background_scripts/search_engines.coffee @@ -0,0 +1,98 @@ + +# Each completer implements three functions: +# +# match: can this completer be used for this search URL? +# getUrl: map these query terms to a completion URL. +# parse: extract suggestions from the resulting (successful) XMLHttpRequest. +# +Google = + name: "Google" + match: (searchUrl) -> + true # TBD. + + getUrl: (queryTerms) -> + "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{Utils.createSearchQuery queryTerms}" + + parse: (xhr, callback) -> + if suggestions = xhr?.responseXML?.getElementsByTagName "suggestion" + suggestions = + for suggestion in suggestions + continue unless suggestion = suggestion.getAttribute "data" + suggestion + callback suggestions + else + callback [] + +# A dummy search engine which is guaranteed to match any search URL, but produces no completions. This allows +# the rest of the logic to be written knowing that there will be a search engine match. +DummySearchEngine = + name: "Dummy" + match: -> true + # We return a useless URL which we know will succeed, but which won't generate any network traffic. + getUrl: -> chrome.runtime.getURL "content_scripts/vimium.css" + parse: (_, callback) -> callback [] + +CompletionEngines = [ Google, DummySearchEngine ] + +SearchEngines = + cancel: (searchUrl, callback = null) -> + @requests[searchUrl]?.abort() + delete @requests[searchUrl] + callback? null + + # Perform and HTTP GET. + # searchUrl is the search engine's URL, e.g. Settings.get("searchUrl") + # url is the URL to fetch + # callback will be called a successful XMLHttpRequest object, or null. + get: (searchUrl, url, callback) -> + @requests ?= {} # Maps searchUrls to any outstanding HTTP request for that search engine. + @cancel searchUrl + + # We cache the results of recent requests (with a two-hour expiry). + @requestCache ?= new SimpleCache 2 * 60 * 60 * 1000 + + if @requestCache.has url + callback @requestCache.get url + return + + @requests[searchUrl] = xhr = new XMLHttpRequest() + xhr.open "GET", url, true + xhr.timeout = 500 + xhr.ontimeout = => @cancel searchUrl, callback + xhr.onerror = => @cancel searchUrl, callback + xhr.send() + + xhr.onreadystatechange = => + if xhr.readyState == 4 + if xhr.status == 200 + @requests[searchUrl] = null + callback @requestCache.set url, xhr + else + callback null + + # Look up the search engine for this search URL. Because of DummySearchEngine, above, we know there will + # always be a match. Imagining that there may be many search engines, and knowing that this is called for + # every character entered, we cache the result. + lookupEngine: (searchUrl) -> + @engineCache ?= new SimpleCache 24 * 60 * 60 * 1000 + if @engineCache.has searchUrl + @engineCache.get searchUrl + else + for engine in CompletionEngines + return @engineCache.set searchUrl, engine if engine.match searchUrl + + # This is the main (actually, the only) entry point. + # searchUrl is the search engine's URL, e.g. Settings.get("searchUrl") + # queryTerms are the queryTerms + # callback will be applied to a list of suggestion strings (which will be an empty list, if anything goes + # wrong). + complete: (searchUrl, queryTerms, callback) -> + return callback [] unless 0 < queryTerms.length + + engine = @lookupEngine searchUrl + url = engine.getUrl queryTerms + @get searchUrl, url, (xhr = null) -> + if xhr? then engine.parse xhr, callback else callback [] + +root = exports ? window +root.SearchEngines = SearchEngines -- cgit v1.2.3 From 1257fc7a4fe7b9d0bfe1ad7ab7255f8dba4b988d Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 2 May 2015 11:34:18 +0100 Subject: Search completion; do not complete URLs. --- background_scripts/search_engines.coffee | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'background_scripts') diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index 485accc1..5d69d087 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -89,6 +89,12 @@ SearchEngines = complete: (searchUrl, queryTerms, callback) -> return callback [] unless 0 < queryTerms.length + # Don't try to complete general URLs. + return callback [] if 1 == queryTerms.length and Utils.isUrl queryTerms[0] + + # Don't try to complete Javascrip URLs. + return callback [] if 0 < queryTerms.length and Utils.hasJavascriptPrefix queryTerms[0] + engine = @lookupEngine searchUrl url = engine.getUrl queryTerms @get searchUrl, url, (xhr = null) -> -- cgit v1.2.3 From 41495d11e6608767dde299223f10c8a606d4a8fb Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 2 May 2015 12:14:16 +0100 Subject: Search completion; minor tweaks. Including: - Make completers classes. That way, we may be able to get better code reuse. --- background_scripts/completion.coffee | 9 +++---- background_scripts/search_engines.coffee | 46 +++++++++++++++++++------------- 2 files changed, 31 insertions(+), 24 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 40c0d119..d8cf1667 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -326,7 +326,6 @@ class SearchEngineCompleter filter: (queryTerms, onComplete) -> SearchEngines.complete Settings.get("searchUrl"), queryTerms, (suggestions = []) => - console.log suggestions.length characterCount = queryTerms.join("").length completions = for suggestion in suggestions @@ -335,11 +334,11 @@ class SearchEngineCompleter onComplete completions computeRelevancy: (suggestion) -> - # We score search-engine completions by word relevancy, but weight increasingly as the number of + # We score search-engine completions by word relevancy, but weight the score increasingly as the number of # characters in the query terms increases. The idea is that, the more the user has had to type, the less - # likely it is that one of the other suggestion types has found what they're looking for, so the more - # likely it is that this suggestion will be useful. - # (1.0 - (1.0 / suggestion.characterCount)) * + # likely it is that one of the other suggestion types has proven useful, so the more likely it is that + # this suggestion will be useful. + # NOTE(smblott) This will require tweaking. (Math.min(suggestion.extraRelevancyData, 12)/12) * RankingUtils.wordRelevancy suggestion.queryTerms, suggestion.title, suggestion.title diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index 5d69d087..3dfea180 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -5,10 +5,14 @@ # getUrl: map these query terms to a completion URL. # parse: extract suggestions from the resulting (successful) XMLHttpRequest. # -Google = +class Google + constructor: -> name: "Google" match: (searchUrl) -> - true # TBD. + return true if /^https?:\/\/[a-z]+.google.com\//.test searchUrl + # NOTE(smblott). A temporary hack, just for me, and just for development. Will be removed. + return true if /localhost\/.*\/booky/.test searchUrl + false getUrl: (queryTerms) -> "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{Utils.createSearchQuery queryTerms}" @@ -23,16 +27,17 @@ Google = else callback [] -# A dummy search engine which is guaranteed to match any search URL, but produces no completions. This allows -# the rest of the logic to be written knowing that there will be a search engine match. -DummySearchEngine = +# A dummy search engine which is guaranteed to match any search URL, but never produces completions. This +# allows the rest of the logic to be written knowing that there will be a search engine match. +class DummySearchEngine + constructor: -> name: "Dummy" match: -> true # We return a useless URL which we know will succeed, but which won't generate any network traffic. getUrl: -> chrome.runtime.getURL "content_scripts/vimium.css" parse: (_, callback) -> callback [] -CompletionEngines = [ Google, DummySearchEngine ] +completionEngines = [ Google, DummySearchEngine ] SearchEngines = cancel: (searchUrl, callback = null) -> @@ -40,16 +45,18 @@ SearchEngines = delete @requests[searchUrl] callback? null - # Perform and HTTP GET. - # searchUrl is the search engine's URL, e.g. Settings.get("searchUrl") - # url is the URL to fetch - # callback will be called a successful XMLHttpRequest object, or null. + # Perform an HTTP GET. + # searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"). + # url is the URL to fetch. + # callback will be called with a successful XMLHttpRequest object, or null. get: (searchUrl, url, callback) -> @requests ?= {} # Maps searchUrls to any outstanding HTTP request for that search engine. @cancel searchUrl - # We cache the results of recent requests (with a two-hour expiry). - @requestCache ?= new SimpleCache 2 * 60 * 60 * 1000 + # We cache the results of the most-recent 1000 requests (with a two-hour expiry). + # FIXME(smblott) Currently we're caching XMLHttpRequest objects, which is wasteful of memory. It would be + # better to handle caching at a higher level. + @requestCache ?= new SimpleCache 2 * 60 * 60 * 1000, 1000 if @requestCache.has url callback @requestCache.get url @@ -70,21 +77,22 @@ SearchEngines = else callback null - # Look up the search engine for this search URL. Because of DummySearchEngine, above, we know there will - # always be a match. Imagining that there may be many search engines, and knowing that this is called for - # every character entered, we cache the result. + # Look up the search-completion engine for this search URL. Because of DummySearchEngine, above, we know + # there will always be a match. Imagining that there may be many search engines, and knowing that this is + # called for every character entered, we cache the result. lookupEngine: (searchUrl) -> @engineCache ?= new SimpleCache 24 * 60 * 60 * 1000 if @engineCache.has searchUrl @engineCache.get searchUrl else - for engine in CompletionEngines + for engine in completionEngines + engine = new engine() return @engineCache.set searchUrl, engine if engine.match searchUrl # This is the main (actually, the only) entry point. - # searchUrl is the search engine's URL, e.g. Settings.get("searchUrl") - # queryTerms are the queryTerms - # callback will be applied to a list of suggestion strings (which will be an empty list, if anything goes + # searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"). + # queryTerms are the queryTerms. + # callback will be applied to a list of suggestion strings (which may be an empty list, if anything goes # wrong). complete: (searchUrl, queryTerms, callback) -> return callback [] unless 0 < queryTerms.length -- cgit v1.2.3 From a855cc15393fbf6296ac1ecf278c5f1b736c81b9 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 2 May 2015 12:39:50 +0100 Subject: Search completion; cache at a higher level. ... and tweak caching constants. --- background_scripts/search_engines.coffee | 42 +++++++++++++++++++------------- 1 file changed, 25 insertions(+), 17 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index 3dfea180..f07a9df4 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -17,15 +17,13 @@ class Google getUrl: (queryTerms) -> "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{Utils.createSearchQuery queryTerms}" - parse: (xhr, callback) -> - if suggestions = xhr?.responseXML?.getElementsByTagName "suggestion" - suggestions = - for suggestion in suggestions - continue unless suggestion = suggestion.getAttribute "data" - suggestion - callback suggestions - else - callback [] + # Returns a list of suggestions (strings). + parse: (xhr) -> + suggestions = xhr?.responseXML?.getElementsByTagName "suggestion" + return [] unless suggestions + for suggestion in suggestions + continue unless suggestion = suggestion.getAttribute "data" + suggestion # A dummy search engine which is guaranteed to match any search URL, but never produces completions. This # allows the rest of the logic to be written knowing that there will be a search engine match. @@ -35,7 +33,7 @@ class DummySearchEngine match: -> true # We return a useless URL which we know will succeed, but which won't generate any network traffic. getUrl: -> chrome.runtime.getURL "content_scripts/vimium.css" - parse: (_, callback) -> callback [] + parse: -> [] completionEngines = [ Google, DummySearchEngine ] @@ -53,10 +51,8 @@ SearchEngines = @requests ?= {} # Maps searchUrls to any outstanding HTTP request for that search engine. @cancel searchUrl - # We cache the results of the most-recent 1000 requests (with a two-hour expiry). - # FIXME(smblott) Currently we're caching XMLHttpRequest objects, which is wasteful of memory. It would be - # better to handle caching at a higher level. - @requestCache ?= new SimpleCache 2 * 60 * 60 * 1000, 1000 + # We cache the results of the most-recent 1000 requests with a one-minute expiry. + @requestCache ?= new SimpleCache 1 * 60 * 1000, 1000 if @requestCache.has url callback @requestCache.get url @@ -81,7 +77,7 @@ SearchEngines = # there will always be a match. Imagining that there may be many search engines, and knowing that this is # called for every character entered, we cache the result. lookupEngine: (searchUrl) -> - @engineCache ?= new SimpleCache 24 * 60 * 60 * 1000 + @engineCache ?= new SimpleCache 30 * 60 * 60 * 1000 # 30 hours (these are small, we can keep them longer). if @engineCache.has searchUrl @engineCache.get searchUrl else @@ -103,10 +99,22 @@ SearchEngines = # Don't try to complete Javascrip URLs. return callback [] if 0 < queryTerms.length and Utils.hasJavascriptPrefix queryTerms[0] + # Cache completions. However, completions depend upon both the searchUrl and the query terms. So we need + # to generate a key. We mix in some nonsense generated by pwgen. There is the possibility of a key clash, + # but it's vanishingly small. + junk = "//Zi?ei5;o//" + completionCacheKey = searchUrl + junk + queryTerms.join junk + @completionCache ?= new SimpleCache 6 * 60 * 60 * 1000, 2000 # Six hours, 2000 entries. + if @completionCache.has completionCacheKey + return callback @completionCache.get completionCacheKey + engine = @lookupEngine searchUrl url = engine.getUrl queryTerms - @get searchUrl, url, (xhr = null) -> - if xhr? then engine.parse xhr, callback else callback [] + @get searchUrl, url, (xhr = null) => + if xhr? + callback @completionCache.set completionCacheKey, engine.parse xhr + else + callback [] root = exports ? window root.SearchEngines = SearchEngines -- cgit v1.2.3 From 77d4e3b81b965eecb078db82bc92e747de1c371a Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 2 May 2015 15:00:05 +0100 Subject: Search completion; vomnibar integration. --- background_scripts/completion.coffee | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index d8cf1667..c13d0625 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -330,7 +330,9 @@ class SearchEngineCompleter completions = for suggestion in suggestions url = Utils.createSearchUrl suggestion.split /\s+/ - new Suggestion queryTerms, "search", url, suggestion, @computeRelevancy, characterCount + suggestion = new Suggestion queryTerms, "search", url, suggestion, @computeRelevancy, characterCount + suggestion.insertText = true + suggestion onComplete completions computeRelevancy: (suggestion) -> -- cgit v1.2.3 From 6073d820ff25f824b575a797f160b3204e0e9863 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 2 May 2015 15:19:20 +0100 Subject: Search completion; generalize Google regexp. --- background_scripts/search_engines.coffee | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index f07a9df4..e96e2c76 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -7,11 +7,17 @@ # class Google constructor: -> + @regexps = [ + # We include matches for the major English-speaking TLDs. + new RegExp "^https?://[a-z]+\.google\.(com|ie|co.uk|ca|com.au)/" + # NOTE(smblott). A temporary hack, just for me, and just for development. Will be removed. + new RegExp "localhost/.*/booky" + ] + name: "Google" match: (searchUrl) -> - return true if /^https?:\/\/[a-z]+.google.com\//.test searchUrl - # NOTE(smblott). A temporary hack, just for me, and just for development. Will be removed. - return true if /localhost\/.*\/booky/.test searchUrl + for re in @regexps + return true if re.test searchUrl false getUrl: (queryTerms) -> -- cgit v1.2.3 From 8329f3cbe95e6a39e500aa15e54c6c44fad9cb7e Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 2 May 2015 16:29:42 +0100 Subject: Search completion; refactor, add Youtube. Major refactoring. Unified tratment of custom search engines and general searches. --- background_scripts/completion.coffee | 97 +++++++++++++++++--------------- background_scripts/main.coffee | 2 - background_scripts/search_engines.coffee | 32 +++++++++-- background_scripts/settings.coffee | 2 +- 4 files changed, 81 insertions(+), 52 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index c13d0625..cb5f64b0 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -322,52 +322,61 @@ class TabCompleter tabRecency.recencyScore(suggestion.tabId) class SearchEngineCompleter - refresh: -> - - filter: (queryTerms, onComplete) -> - SearchEngines.complete Settings.get("searchUrl"), queryTerms, (suggestions = []) => - characterCount = queryTerms.join("").length - completions = - for suggestion in suggestions - url = Utils.createSearchUrl suggestion.split /\s+/ - suggestion = new Suggestion queryTerms, "search", url, suggestion, @computeRelevancy, characterCount - suggestion.insertText = true - suggestion - onComplete completions - - computeRelevancy: (suggestion) -> - # We score search-engine completions by word relevancy, but weight the score increasingly as the number of - # characters in the query terms increases. The idea is that, the more the user has had to type, the less - # likely it is that one of the other suggestion types has proven useful, so the more likely it is that - # this suggestion will be useful. - # NOTE(smblott) This will require tweaking. - (Math.min(suggestion.extraRelevancyData, 12)/12) * - RankingUtils.wordRelevancy suggestion.queryTerms, suggestion.title, suggestion.title - -# A completer which will return your search engines -class CustomSearchEngineCompleter searchEngines: {} filter: (queryTerms, onComplete) -> - {url: url, description: description} = @getSearchEngineMatches queryTerms + { keyword: keyword, url: url, description: description } = @getSearchEngineMatches queryTerms + custom = url? suggestions = [] - if url - url = url.replace(/%s/g, Utils.createSearchQuery queryTerms[1..]) - if description - type = description - query = queryTerms[1..].join " " + + mkUrl = + if custom + (string) -> url.replace /%s/g, Utils.createSearchQuery string.split /\s+/ else - type = "search" - query = queryTerms[0] + ": " + queryTerms[1..].join(" ") - suggestion = new Suggestion(queryTerms, type, url, query, @computeRelevancy) - suggestion.autoSelect = true - suggestions.push(suggestion) - onComplete(suggestions) + (string) -> Utils.createSearchUrl string.split /\s+/ + + type = if description? then description else "search" + searchUrl = if custom then url else Settings.get "searchUrl" + query = queryTerms[1..].join " " + + # For custom search engines, we add an auto-selected suggestion. + if custom + title = if description? then query else queryTerms[0] + ": " + query + suggestions.push @mkSuggestion false, queryTerms, type, mkUrl(query), description, @computeRelevancy + suggestions[0].autoSelect = true + suggestions[0].relevancyScore = 1 + queryTerms = queryTerms[1..] + + # For custom search-engine queries, this adds suggestions only if we have a completer. For other queries, + # this adds suggestions for the default search engine (if we have a completer for that). + SearchEngines.complete searchUrl, queryTerms, (newSuggestions = []) => + characterCount = query.length - queryTerms.length + 1 + for suggestion in newSuggestions + suggestions.push @mkSuggestion true, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, characterCount + + if custom + for suggestion in suggestions + suggestion.reinsertPrefix = "#{keyword} " if suggestion.insertText - computeRelevancy: -> 1 + onComplete suggestions + + mkSuggestion: (insertText, args...) -> + suggestion = new Suggestion args... + suggestion.insertText = insertText + suggestion + + computeRelevancy: (suggestion) -> + suggestion.relevancyScore ? + # We score search-engine completions by word relevancy, but weight the score increasingly as the number of + # characters in the query terms increases. The idea is that, the more the user has had to type, the less + # likely it is that one of the other suggestion types has proven useful, so the more likely it is that + # this suggestion will be useful. + # NOTE(smblott) This will require tweaking. + (Math.min(suggestion.extraRelevancyData, 12)/12) * + RankingUtils.wordRelevancy suggestion.queryTerms, suggestion.title, suggestion.title refresh: -> - @searchEngines = CustomSearchEngineCompleter.getSearchEngines() + @searchEngines = SearchEngineCompleter.getSearchEngines() getSearchEngineMatches: (queryTerms) -> (1 < queryTerms.length and @searchEngines[queryTerms[0]]) or {} @@ -376,23 +385,24 @@ class CustomSearchEngineCompleter # mapping in @searchEnginesMap. @searchEnginesMap: null - # Parse the custom search engines setting and cache it in CustomSearchEngineCompleter.searchEnginesMap. + # Parse the custom search engines setting and cache it in SearchEngineCompleter.searchEnginesMap. @parseSearchEngines: (searchEnginesText) -> - searchEnginesMap = CustomSearchEngineCompleter.searchEnginesMap = {} + searchEnginesMap = SearchEngineCompleter.searchEnginesMap = {} for line in searchEnginesText.split /\n/ tokens = line.trim().split /\s+/ continue if tokens.length < 2 or tokens[0].startsWith('"') or tokens[0].startsWith("#") keywords = tokens[0].split ":" continue unless keywords.length == 2 and not keywords[1] # So, like: [ "w", "" ]. searchEnginesMap[keywords[0]] = + keyword: keywords[0] url: tokens[1] description: tokens[2..].join(" ") # Fetch the search-engine map, building it if necessary. @getSearchEngines: -> - unless CustomSearchEngineCompleter.searchEnginesMap? - CustomSearchEngineCompleter.parseSearchEngines Settings.get "searchEngines" - CustomSearchEngineCompleter.searchEnginesMap + unless SearchEngineCompleter.searchEnginesMap? + SearchEngineCompleter.parseSearchEngines Settings.get "searchEngines" + SearchEngineCompleter.searchEnginesMap # A completer which calls filter() on many completers, aggregates the results, ranks them, and returns the top # 10. Queries from the vomnibar frontend script come through a multi completer. @@ -641,7 +651,6 @@ root.HistoryCompleter = HistoryCompleter root.DomainCompleter = DomainCompleter root.TabCompleter = TabCompleter root.SearchEngineCompleter = SearchEngineCompleter -root.CustomSearchEngineCompleter = CustomSearchEngineCompleter root.HistoryCache = HistoryCache root.RankingUtils = RankingUtils root.RegexpCache = RegexpCache diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index 6f7db05c..4d2546fc 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -47,13 +47,11 @@ completionSources = history: new HistoryCompleter() domains: new DomainCompleter() tabs: new TabCompleter() - customSearchEngines: new CustomSearchEngineCompleter() searchEngines: new SearchEngineCompleter() completers = omni: new MultiCompleter([ completionSources.searchEngines, - completionSources.customSearchEngines, completionSources.bookmarks, completionSources.history, completionSources.domains]) diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index e96e2c76..e68cf85d 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -1,4 +1,9 @@ +matchesAnyRegexp = (regexps, string) -> + for re in regexps + return true if re.test string + false + # Each completer implements three functions: # # match: can this completer be used for this search URL? @@ -15,10 +20,7 @@ class Google ] name: "Google" - match: (searchUrl) -> - for re in @regexps - return true if re.test searchUrl - false + match: (searchUrl) -> matchesAnyRegexp @regexps, searchUrl getUrl: (queryTerms) -> "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{Utils.createSearchQuery queryTerms}" @@ -31,6 +33,26 @@ class Google continue unless suggestion = suggestion.getAttribute "data" suggestion +class Youtube + constructor: -> + @regexps = [ new RegExp "https?://[a-z]+\.youtube\.com/results" ] + + name: "YouTube" + match: (searchUrl) -> matchesAnyRegexp @regexps, searchUrl + + getUrl: (queryTerms) -> + "http://suggestqueries.google.com/complete/search?client=youtube&ds=yt&q=#{Utils.createSearchQuery queryTerms}" + + # Returns a list of suggestions (strings). + parse: (xhr) -> + try + text = xhr.responseText + text = text.replace /^[^(]*\(/, "" + text = text.replace /\)[^\)]*$/, "" + suggestion[0] for suggestion in JSON.parse(text)[1] + catch + [] + # A dummy search engine which is guaranteed to match any search URL, but never produces completions. This # allows the rest of the logic to be written knowing that there will be a search engine match. class DummySearchEngine @@ -41,7 +63,7 @@ class DummySearchEngine getUrl: -> chrome.runtime.getURL "content_scripts/vimium.css" parse: -> [] -completionEngines = [ Google, DummySearchEngine ] +completionEngines = [ Google, Youtube, DummySearchEngine ] SearchEngines = cancel: (searchUrl, callback = null) -> diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index a73a9d5c..a4d95c81 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -33,7 +33,7 @@ root.Settings = Settings = root.refreshCompletionKeysAfterMappingSave() searchEngines: (value) -> - root.CustomSearchEngineCompleter.parseSearchEngines value + root.SearchEngineCompleter.parseSearchEngines value exclusionRules: (value) -> root.Exclusions.postUpdateHook value -- cgit v1.2.3 From ba4e8018e3d8cd80e0fa9ac541e37e7eee37028f Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 2 May 2015 17:32:28 +0100 Subject: Search completion; tweaks and refactoring. --- background_scripts/completion.coffee | 33 +++++---- background_scripts/search_engines.coffee | 115 +++++++++++++++++-------------- 2 files changed, 80 insertions(+), 68 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index cb5f64b0..5b5dc191 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -337,22 +337,32 @@ class SearchEngineCompleter type = if description? then description else "search" searchUrl = if custom then url else Settings.get "searchUrl" - query = queryTerms[1..].join " " + query = queryTerms[(if custom then 1 else 0)..].join " " # For custom search engines, we add an auto-selected suggestion. if custom title = if description? then query else queryTerms[0] + ": " + query - suggestions.push @mkSuggestion false, queryTerms, type, mkUrl(query), description, @computeRelevancy + suggestions.push @mkSuggestion false, queryTerms, type, mkUrl(query), description, @computeRelevancy, 1 suggestions[0].autoSelect = true - suggestions[0].relevancyScore = 1 queryTerms = queryTerms[1..] # For custom search-engine queries, this adds suggestions only if we have a completer. For other queries, # this adds suggestions for the default search engine (if we have a completer for that). - SearchEngines.complete searchUrl, queryTerms, (newSuggestions = []) => + SearchEngines.complete searchUrl, queryTerms, (searchSuggestions = []) => + + # Scoring: + # - The score does not depend upon the actual suggestion (so, it does not depend upon word relevancy). + # We assume that the completion engine has already factored that in. + # - The score is higher if the query is longer. The idea is that search suggestions are more likely + # to be relevant if, after typing quite some number of characters, the user hasn't yet found a + # useful suggestion from another completer. + # - Scores are weighted such that they retain the ordering provided by the completion engine. characterCount = query.length - queryTerms.length + 1 - for suggestion in newSuggestions - suggestions.push @mkSuggestion true, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, characterCount + score = 0.8 * (Math.min(characterCount, 12.0)/12.0) + + for suggestion in searchSuggestions + suggestions.push @mkSuggestion true, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score + score *= 0.9 if custom for suggestion in suggestions @@ -365,15 +375,8 @@ class SearchEngineCompleter suggestion.insertText = insertText suggestion - computeRelevancy: (suggestion) -> - suggestion.relevancyScore ? - # We score search-engine completions by word relevancy, but weight the score increasingly as the number of - # characters in the query terms increases. The idea is that, the more the user has had to type, the less - # likely it is that one of the other suggestion types has proven useful, so the more likely it is that - # this suggestion will be useful. - # NOTE(smblott) This will require tweaking. - (Math.min(suggestion.extraRelevancyData, 12)/12) * - RankingUtils.wordRelevancy suggestion.queryTerms, suggestion.title, suggestion.title + # The score is computed in filter() and provided here via suggestion.extraRelevancyData. + computeRelevancy: (suggestion) -> suggestion.extraRelevancyData refresh: -> @searchEngines = SearchEngineCompleter.getSearchEngines() diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index e68cf85d..3c6654e3 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -1,49 +1,54 @@ -matchesAnyRegexp = (regexps, string) -> - for re in regexps - return true if re.test string - false - -# Each completer implements three functions: +# A completion engine provides search suggestions for a search engine. A search engine is identified by a +# "searchUrl", e.g. Settings.get("searchUrl"), or a custom search engine. +# +# Each completion engine defines three functions: +# +# 1. "match" - This takes a searchUrl, and returns a boolean indicating whether this completion engine can +# perform completion for the given search engine. # -# match: can this completer be used for this search URL? -# getUrl: map these query terms to a completion URL. -# parse: extract suggestions from the resulting (successful) XMLHttpRequest. +# 2. "getUrl" - This takes a list of query terms (queryTerms) and generates a completion URL, that is, a URL +# which will provide completions for this completion engine. # -class Google +# 3. "parse" - This takes a successful XMLHttpRequest object (the request has completed successfully), and +# returns a list of suggestions (a list of strings). +# +# The main (only) completion entry point is SearchEngines.complete(). This implements all lookup and caching +# logic. It is possible to add new completion engines without changing the SearchEngines infrastructure +# itself. + +# A base class for common regexp-based matching engines. +class RegexpEngine + constructor: (@regexps) -> + match: (searchUrl) -> Utils.matchesAnyRegexp @regexps, searchUrl + +# Completion engine for English-language Google search. +class Google extends RegexpEngine constructor: -> - @regexps = [ - # We include matches for the major English-speaking TLDs. + super [ + # We match the major English-speaking TLDs. new RegExp "^https?://[a-z]+\.google\.(com|ie|co.uk|ca|com.au)/" - # NOTE(smblott). A temporary hack, just for me, and just for development. Will be removed. - new RegExp "localhost/.*/booky" + new RegExp "localhost/cgi-bin/booky" # Only for smblott. ] - name: "Google" - match: (searchUrl) -> matchesAnyRegexp @regexps, searchUrl - getUrl: (queryTerms) -> "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{Utils.createSearchQuery queryTerms}" - # Returns a list of suggestions (strings). parse: (xhr) -> - suggestions = xhr?.responseXML?.getElementsByTagName "suggestion" - return [] unless suggestions - for suggestion in suggestions - continue unless suggestion = suggestion.getAttribute "data" - suggestion + try + for suggestion in xhr.responseXML.getElementsByTagName "suggestion" + continue unless suggestion = suggestion.getAttribute "data" + suggestion + catch + [] -class Youtube +class Youtube extends RegexpEngine constructor: -> - @regexps = [ new RegExp "https?://[a-z]+\.youtube\.com/results" ] - - name: "YouTube" - match: (searchUrl) -> matchesAnyRegexp @regexps, searchUrl + super [ new RegExp "https?://[a-z]+\.youtube\.com/results" ] getUrl: (queryTerms) -> "http://suggestqueries.google.com/complete/search?client=youtube&ds=yt&q=#{Utils.createSearchQuery queryTerms}" - # Returns a list of suggestions (strings). parse: (xhr) -> try text = xhr.responseText @@ -56,8 +61,6 @@ class Youtube # A dummy search engine which is guaranteed to match any search URL, but never produces completions. This # allows the rest of the logic to be written knowing that there will be a search engine match. class DummySearchEngine - constructor: -> - name: "Dummy" match: -> true # We return a useless URL which we know will succeed, but which won't generate any network traffic. getUrl: -> chrome.runtime.getURL "content_scripts/vimium.css" @@ -72,15 +75,13 @@ SearchEngines = callback? null # Perform an HTTP GET. - # searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"). - # url is the URL to fetch. - # callback will be called with a successful XMLHttpRequest object, or null. get: (searchUrl, url, callback) -> - @requests ?= {} # Maps searchUrls to any outstanding HTTP request for that search engine. + @requests ?= {} # Maps a searchUrl to any outstanding HTTP request for that search engine. @cancel searchUrl - # We cache the results of the most-recent 1000 requests with a one-minute expiry. - @requestCache ?= new SimpleCache 1 * 60 * 1000, 1000 + # We cache the results of the most-recent 100 successfully XMLHttpRequests with a ten-second (ie. very + # short) expiry. + @requestCache ?= new SimpleCache 10 * 1000, 100 if @requestCache.has url callback @requestCache.get url @@ -88,22 +89,23 @@ SearchEngines = @requests[searchUrl] = xhr = new XMLHttpRequest() xhr.open "GET", url, true - xhr.timeout = 500 + # We set a fairly short timeout. If we block for too long, then we block *all* completers. + xhr.timeout = 300 xhr.ontimeout = => @cancel searchUrl, callback xhr.onerror = => @cancel searchUrl, callback xhr.send() xhr.onreadystatechange = => if xhr.readyState == 4 + @requests[searchUrl] = null if xhr.status == 200 - @requests[searchUrl] = null callback @requestCache.set url, xhr else callback null - # Look up the search-completion engine for this search URL. Because of DummySearchEngine, above, we know - # there will always be a match. Imagining that there may be many search engines, and knowing that this is - # called for every character entered, we cache the result. + # Look up the search-completion engine for this searchUrl. Because of DummySearchEngine, above, we know + # there will always be a match. Imagining that there may be many completion engines, and knowing that this + # is called for every input event in the vomnibar, we cache the result. lookupEngine: (searchUrl) -> @engineCache ?= new SimpleCache 30 * 60 * 60 * 1000 # 30 hours (these are small, we can keep them longer). if @engineCache.has searchUrl @@ -114,22 +116,24 @@ SearchEngines = return @engineCache.set searchUrl, engine if engine.match searchUrl # This is the main (actually, the only) entry point. - # searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"). - # queryTerms are the queryTerms. - # callback will be applied to a list of suggestion strings (which may be an empty list, if anything goes - # wrong). + # - searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"), or a custome search engine's URL. + # This is only used as a key for determining the relevant completion engine. + # - queryTerms are the queryTerms. + # - callback will be applied to a list of suggestion strings (which may be an empty list, if anything goes + # wrong). complete: (searchUrl, queryTerms, callback) -> + # We can't complete empty queries. return callback [] unless 0 < queryTerms.length - # Don't try to complete general URLs. + # We don't complete URLs. return callback [] if 1 == queryTerms.length and Utils.isUrl queryTerms[0] - # Don't try to complete Javascrip URLs. - return callback [] if 0 < queryTerms.length and Utils.hasJavascriptPrefix queryTerms[0] + # We don't complete Javascript URLs. + return callback [] if Utils.hasJavascriptPrefix queryTerms[0] # Cache completions. However, completions depend upon both the searchUrl and the query terms. So we need - # to generate a key. We mix in some nonsense generated by pwgen. There is the possibility of a key clash, - # but it's vanishingly small. + # to generate a key. We mix in some nonsense generated by pwgen. A key clash is possible, but vanishingly + # unlikely. junk = "//Zi?ei5;o//" completionCacheKey = searchUrl + junk + queryTerms.join junk @completionCache ?= new SimpleCache 6 * 60 * 60 * 1000, 2000 # Six hours, 2000 entries. @@ -140,9 +144,14 @@ SearchEngines = url = engine.getUrl queryTerms @get searchUrl, url, (xhr = null) => if xhr? - callback @completionCache.set completionCacheKey, engine.parse xhr + # We keep at most three suggestions, the top three. These are most likely to be useful. + callback @completionCache.set completionCacheKey, engine.parse(xhr)[...3] else - callback [] + callback @completionCache.set completionCacheKey, callback [] + # We cache failures, but remove them after just ten minutes. This (it is hoped) avoids repeated + # XMLHttpRequest failures over a short period of time. + removeCompletionCacheKey = => @completionCache.set completionCacheKey, null + setTimeout removeCompletionCacheKey, 10 * 60 * 1000 # Ten minutes. root = exports ? window root.SearchEngines = SearchEngines -- cgit v1.2.3 From 6fecc4e24392650c540354918af641b29aa6b2b4 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 3 May 2015 10:12:23 +0100 Subject: Search completion; Wikipedia. --- background_scripts/search_engines.coffee | 53 +++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 18 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index 3c6654e3..c61baf17 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -35,12 +35,9 @@ class Google extends RegexpEngine "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{Utils.createSearchQuery queryTerms}" parse: (xhr) -> - try - for suggestion in xhr.responseXML.getElementsByTagName "suggestion" - continue unless suggestion = suggestion.getAttribute "data" - suggestion - catch - [] + for suggestion in xhr.responseXML.getElementsByTagName "suggestion" + continue unless suggestion = suggestion.getAttribute "data" + suggestion class Youtube extends RegexpEngine constructor: -> @@ -50,13 +47,20 @@ class Youtube extends RegexpEngine "http://suggestqueries.google.com/complete/search?client=youtube&ds=yt&q=#{Utils.createSearchQuery queryTerms}" parse: (xhr) -> - try - text = xhr.responseText - text = text.replace /^[^(]*\(/, "" - text = text.replace /\)[^\)]*$/, "" - suggestion[0] for suggestion in JSON.parse(text)[1] - catch - [] + text = xhr.responseText + text = text.replace /^[^(]*\(/, "" + text = text.replace /\)[^\)]*$/, "" + suggestion[0] for suggestion in JSON.parse(text)[1] + +class Wikipedia extends RegexpEngine + constructor: -> + super [ new RegExp "https?://[a-z]+\.wikipedia\.org/" ] + + getUrl: (queryTerms) -> + "https://en.wikipedia.org/w/api.php?action=opensearch&format=json&search=#{Utils.createSearchQuery queryTerms}" + + parse: (xhr) -> + JSON.parse(xhr.responseText)[1] # A dummy search engine which is guaranteed to match any search URL, but never produces completions. This # allows the rest of the logic to be written knowing that there will be a search engine match. @@ -66,7 +70,12 @@ class DummySearchEngine getUrl: -> chrome.runtime.getURL "content_scripts/vimium.css" parse: -> [] -completionEngines = [ Google, Youtube, DummySearchEngine ] +completionEngines = [ + Google + Youtube + Wikipedia + DummySearchEngine +] SearchEngines = cancel: (searchUrl, callback = null) -> @@ -142,11 +151,19 @@ SearchEngines = engine = @lookupEngine searchUrl url = engine.getUrl queryTerms + query = queryTerms.join(" ").toLowerCase() @get searchUrl, url, (xhr = null) => - if xhr? - # We keep at most three suggestions, the top three. These are most likely to be useful. - callback @completionCache.set completionCacheKey, engine.parse(xhr)[...3] - else + # Parsing the response may fail if we receive an unexpected or an unexpectedly-formatted response. In + # all cases, we fall back to the catch clause, below. + try + suggestions = engine.parse xhr + # Make sure we really do have an iterable of strings. + suggestions = (suggestion for suggestion in suggestions when "string" == typeof suggestion) + # Filter out the query itself. It's not adding anything. + suggestions = (suggestion for suggestion in suggestions when suggestion.toLowerCase() != query) + # We keep at most three suggestions, the top three. + callback @completionCache.set completionCacheKey, suggestions[...3] + catch callback @completionCache.set completionCacheKey, callback [] # We cache failures, but remove them after just ten minutes. This (it is hoped) avoids repeated # XMLHttpRequest failures over a short period of time. -- cgit v1.2.3 From 05431a5041913d78e44058e9e7f42ef9eee29ce9 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 3 May 2015 10:24:37 +0100 Subject: Search completion; Google maps. --- background_scripts/search_engines.coffee | 17 +++++++++++++++-- background_scripts/settings.coffee | 6 +++++- 2 files changed, 20 insertions(+), 3 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index c61baf17..90fcb99f 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -41,7 +41,7 @@ class Google extends RegexpEngine class Youtube extends RegexpEngine constructor: -> - super [ new RegExp "https?://[a-z]+\.youtube\.com/results" ] + super [ new RegExp "^https?://[a-z]+\.youtube\.com/results" ] getUrl: (queryTerms) -> "http://suggestqueries.google.com/complete/search?client=youtube&ds=yt&q=#{Utils.createSearchQuery queryTerms}" @@ -53,8 +53,9 @@ class Youtube extends RegexpEngine suggestion[0] for suggestion in JSON.parse(text)[1] class Wikipedia extends RegexpEngine + # Example search URL: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s constructor: -> - super [ new RegExp "https?://[a-z]+\.wikipedia\.org/" ] + super [ new RegExp "^https?://[a-z]+\.wikipedia\.org/" ] getUrl: (queryTerms) -> "https://en.wikipedia.org/w/api.php?action=opensearch&format=json&search=#{Utils.createSearchQuery queryTerms}" @@ -62,6 +63,18 @@ class Wikipedia extends RegexpEngine parse: (xhr) -> JSON.parse(xhr.responseText)[1] +class GoogleMaps extends RegexpEngine + constructor: -> + super [ new RegExp "^https?://www\.google\.com/maps/search/" ] + + getUrl: (queryTerms) -> + "https://www.google.com/s?tbm=map&fp=1&gs_ri=maps&source=hp&suggest=p&authuser=0&hl=en&pf=p&tch=1&ech=2&q=#{Utils.createSearchQuery queryTerms}" + + parse: (xhr) -> + console.log xhr + [] + + 'google-maps': 'https://www.google.com/maps/search/', # A dummy search engine which is guaranteed to match any search URL, but never produces completions. This # allows the rest of the logic to be written knowing that there will be a search engine match. class DummySearchEngine diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index a4d95c81..01277741 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -90,7 +90,11 @@ root.Settings = Settings = # default/fall back search engine searchUrl: "http://www.google.com/search?q=" # put in an example search engine - searchEngines: "w: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s wikipedia" + searchEngines: [ + "w: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedia", "" + "t: http://www.youtube.com/results?search_query=%s Youtube", "" + "m: https://www.google.com/maps/search/%s Google Maps", "" + ].join "\n" newTabUrl: "chrome://newtab" grabBackFocus: false -- cgit v1.2.3 From 7ecc8805053cfe32549136412d16d5c62d6949c7 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 3 May 2015 11:21:06 +0100 Subject: Search completion; Google maps out, Amazon, Bing in. --- background_scripts/main.coffee | 9 ++-- background_scripts/search_engines.coffee | 71 +++++++++++++++++++------------- background_scripts/settings.coffee | 12 ++++-- 3 files changed, 56 insertions(+), 36 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index 4d2546fc..e678b2f7 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -51,10 +51,11 @@ completionSources = completers = omni: new MultiCompleter([ - completionSources.searchEngines, - completionSources.bookmarks, - completionSources.history, - completionSources.domains]) + completionSources.searchEngines + # completionSources.bookmarks, + # completionSources.history, + # completionSources.domains + ]) bookmarks: new MultiCompleter([completionSources.bookmarks]) tabs: new MultiCompleter([completionSources.tabs]) diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index 90fcb99f..362de3cc 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -22,8 +22,15 @@ class RegexpEngine constructor: (@regexps) -> match: (searchUrl) -> Utils.matchesAnyRegexp @regexps, searchUrl -# Completion engine for English-language Google search. -class Google extends RegexpEngine +# Several Google completion engines package responses in this way. +class GoogleXMLRegexpEngine extends RegexpEngine + parse: (xhr) -> + for suggestion in xhr.responseXML.getElementsByTagName "suggestion" + continue unless suggestion = suggestion.getAttribute "data" + suggestion + +class Google extends GoogleXMLRegexpEngine + # Example search URL: http://www.google.com/search?q=%s constructor: -> super [ # We match the major English-speaking TLDs. @@ -34,23 +41,13 @@ class Google extends RegexpEngine getUrl: (queryTerms) -> "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{Utils.createSearchQuery queryTerms}" - parse: (xhr) -> - for suggestion in xhr.responseXML.getElementsByTagName "suggestion" - continue unless suggestion = suggestion.getAttribute "data" - suggestion - -class Youtube extends RegexpEngine +class Youtube extends GoogleXMLRegexpEngine + # Example search URL: http://www.youtube.com/results?search_query=%s constructor: -> super [ new RegExp "^https?://[a-z]+\.youtube\.com/results" ] getUrl: (queryTerms) -> - "http://suggestqueries.google.com/complete/search?client=youtube&ds=yt&q=#{Utils.createSearchQuery queryTerms}" - - parse: (xhr) -> - text = xhr.responseText - text = text.replace /^[^(]*\(/, "" - text = text.replace /\)[^\)]*$/, "" - suggestion[0] for suggestion in JSON.parse(text)[1] + "http://suggestqueries.google.com/complete/search?client=youtube&ds=yt&xml=t&q=#{Utils.createSearchQuery queryTerms}" class Wikipedia extends RegexpEngine # Example search URL: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s @@ -63,18 +60,34 @@ class Wikipedia extends RegexpEngine parse: (xhr) -> JSON.parse(xhr.responseText)[1] -class GoogleMaps extends RegexpEngine - constructor: -> - super [ new RegExp "^https?://www\.google\.com/maps/search/" ] - - getUrl: (queryTerms) -> - "https://www.google.com/s?tbm=map&fp=1&gs_ri=maps&source=hp&suggest=p&authuser=0&hl=en&pf=p&tch=1&ech=2&q=#{Utils.createSearchQuery queryTerms}" +## class GoogleMaps extends RegexpEngine +## # Example search URL: https://www.google.com/maps/search/%s +## constructor: -> +## super [ new RegExp "^https?://www\.google\.com/maps/search/" ] +## +## getUrl: (queryTerms) -> +## console.log "xxxxxxxxxxxxxxxxxxxxx" +## "https://www.google.com/s?tbm=map&fp=1&gs_ri=maps&source=hp&suggest=p&authuser=0&hl=en&pf=p&tch=1&ech=2&q=#{Utils.createSearchQuery queryTerms}" +## +## parse: (xhr) -> +## console.log "yyy", xhr.responseText +## data = JSON.parse xhr.responseText +## console.log "zzz" +## console.log data +## [] + +class Bing extends RegexpEngine + # Example search URL: https://www.bing.com/search?q=%s + constructor: -> super [ new RegExp "^https?://www\.bing\.com/search" ] + getUrl: (queryTerms) -> "http://api.bing.com/osjson.aspx?query=#{Utils.createSearchQuery queryTerms}" + parse: (xhr) -> JSON.parse(xhr.responseText)[1] + +class Amazon extends RegexpEngine + # Example search URL: http://www.amazon.com/s/?field-keywords=%s + constructor: -> super [ new RegExp "^https?://www\.amazon\.com/s/" ] + getUrl: (queryTerms) -> "https://completion.amazon.com/search/complete?method=completion&search-alias=aps&client=amazon-search-ui&mkt=1&q=#{Utils.createSearchQuery queryTerms}" + parse: (xhr) -> JSON.parse(xhr.responseText)[1] - parse: (xhr) -> - console.log xhr - [] - - 'google-maps': 'https://www.google.com/maps/search/', # A dummy search engine which is guaranteed to match any search URL, but never produces completions. This # allows the rest of the logic to be written knowing that there will be a search engine match. class DummySearchEngine @@ -84,9 +97,11 @@ class DummySearchEngine parse: -> [] completionEngines = [ - Google Youtube + Google Wikipedia + Bing + Amazon DummySearchEngine ] @@ -112,7 +127,7 @@ SearchEngines = @requests[searchUrl] = xhr = new XMLHttpRequest() xhr.open "GET", url, true # We set a fairly short timeout. If we block for too long, then we block *all* completers. - xhr.timeout = 300 + xhr.timeout = 500 xhr.ontimeout = => @cancel searchUrl, callback xhr.onerror = => @cancel searchUrl, callback xhr.send() diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index 01277741..06b59af8 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -91,10 +91,14 @@ root.Settings = Settings = searchUrl: "http://www.google.com/search?q=" # put in an example search engine searchEngines: [ - "w: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedia", "" - "t: http://www.youtube.com/results?search_query=%s Youtube", "" - "m: https://www.google.com/maps/search/%s Google Maps", "" - ].join "\n" + # FIXME(smblott) Comment out these before merge. + "w: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedia" + "t: http://www.youtube.com/results?search_query=%s Youtube" + "m: https://www.google.com/maps/search/%s Google Maps" + "b: https://www.bing.com/search?q=%s Bing" + "y: http://www.youtube.com/results?search_query=%s Youtube" + "az: http://www.amazon.com/s/?field-keywords=%s Amazon" + ].join "\n\n" newTabUrl: "chrome://newtab" grabBackFocus: false -- cgit v1.2.3 From 07a62680008b6c86b0cf8a12969334bd7c1e389c Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 3 May 2015 11:29:43 +0100 Subject: Search completion; DuckDuckGo. --- background_scripts/main.coffee | 9 ++++----- background_scripts/search_engines.coffee | 8 ++++++++ background_scripts/settings.coffee | 3 ++- 3 files changed, 14 insertions(+), 6 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index e678b2f7..4d2546fc 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -51,11 +51,10 @@ completionSources = completers = omni: new MultiCompleter([ - completionSources.searchEngines - # completionSources.bookmarks, - # completionSources.history, - # completionSources.domains - ]) + completionSources.searchEngines, + completionSources.bookmarks, + completionSources.history, + completionSources.domains]) bookmarks: new MultiCompleter([completionSources.bookmarks]) tabs: new MultiCompleter([completionSources.tabs]) diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index 362de3cc..043e3268 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -88,6 +88,13 @@ class Amazon extends RegexpEngine getUrl: (queryTerms) -> "https://completion.amazon.com/search/complete?method=completion&search-alias=aps&client=amazon-search-ui&mkt=1&q=#{Utils.createSearchQuery queryTerms}" parse: (xhr) -> JSON.parse(xhr.responseText)[1] +class DuckDuckGo extends RegexpEngine + # Example search URL: https://duckduckgo.com/?q=%s + constructor: -> super [ new RegExp "^https?://([a-z]+\.)?duckduckgo\.com/" ] + getUrl: (queryTerms) -> "https://duckduckgo.com/ac/?q=#{Utils.createSearchQuery queryTerms}" + parse: (xhr) -> + suggestion.phrase for suggestion in JSON.parse xhr.responseText + # A dummy search engine which is guaranteed to match any search URL, but never produces completions. This # allows the rest of the logic to be written knowing that there will be a search engine match. class DummySearchEngine @@ -99,6 +106,7 @@ class DummySearchEngine completionEngines = [ Youtube Google + DuckDuckGo Wikipedia Bing Amazon diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index 06b59af8..07db3a89 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -91,11 +91,12 @@ root.Settings = Settings = searchUrl: "http://www.google.com/search?q=" # put in an example search engine searchEngines: [ - # FIXME(smblott) Comment out these before merge. + # FIXME(smblott) Comment these out before merge. "w: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedia" "t: http://www.youtube.com/results?search_query=%s Youtube" "m: https://www.google.com/maps/search/%s Google Maps" "b: https://www.bing.com/search?q=%s Bing" + "d: https://duckduckgo.com/?q=%s DuckDuckGo" "y: http://www.youtube.com/results?search_query=%s Youtube" "az: http://www.amazon.com/s/?field-keywords=%s Amazon" ].join "\n\n" -- cgit v1.2.3 From 98636859e0944b82fa6e3988e0ca943b5a27b145 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 3 May 2015 11:34:19 +0100 Subject: Search completion; tweaks and fixes. --- background_scripts/completion.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 5b5dc191..c8650f45 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -342,7 +342,7 @@ class SearchEngineCompleter # For custom search engines, we add an auto-selected suggestion. if custom title = if description? then query else queryTerms[0] + ": " + query - suggestions.push @mkSuggestion false, queryTerms, type, mkUrl(query), description, @computeRelevancy, 1 + suggestions.push @mkSuggestion false, queryTerms, type, mkUrl(query), title, @computeRelevancy, 1 suggestions[0].autoSelect = true queryTerms = queryTerms[1..] -- cgit v1.2.3 From db59e05216ff0ce6bdd5af5fb6a6f7f3b1860a8e Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 3 May 2015 11:48:06 +0100 Subject: Search completion; generalize Amazon. --- background_scripts/search_engines.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index 043e3268..a80c218c 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -84,7 +84,7 @@ class Bing extends RegexpEngine class Amazon extends RegexpEngine # Example search URL: http://www.amazon.com/s/?field-keywords=%s - constructor: -> super [ new RegExp "^https?://www\.amazon\.com/s/" ] + constructor: -> super [ new RegExp "^https?://www\.amazon\.(com|co.uk|ca|com.au)/s/" ] getUrl: (queryTerms) -> "https://completion.amazon.com/search/complete?method=completion&search-alias=aps&client=amazon-search-ui&mkt=1&q=#{Utils.createSearchQuery queryTerms}" parse: (xhr) -> JSON.parse(xhr.responseText)[1] -- cgit v1.2.3 From 776f617ece5d333fe70df903982a18d65fc2776a Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 3 May 2015 17:22:20 +0100 Subject: Search completion; make completion lookup asynchronous. --- background_scripts/completion.coffee | 97 ++++++++++++++++++++------------ background_scripts/main.coffee | 15 +++-- background_scripts/search_engines.coffee | 3 +- 3 files changed, 70 insertions(+), 45 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index c8650f45..fee4778a 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -346,29 +346,31 @@ class SearchEngineCompleter suggestions[0].autoSelect = true queryTerms = queryTerms[1..] - # For custom search-engine queries, this adds suggestions only if we have a completer. For other queries, - # this adds suggestions for the default search engine (if we have a completer for that). - SearchEngines.complete searchUrl, queryTerms, (searchSuggestions = []) => - - # Scoring: - # - The score does not depend upon the actual suggestion (so, it does not depend upon word relevancy). - # We assume that the completion engine has already factored that in. - # - The score is higher if the query is longer. The idea is that search suggestions are more likely - # to be relevant if, after typing quite some number of characters, the user hasn't yet found a - # useful suggestion from another completer. - # - Scores are weighted such that they retain the ordering provided by the completion engine. - characterCount = query.length - queryTerms.length + 1 - score = 0.8 * (Math.min(characterCount, 12.0)/12.0) - - for suggestion in searchSuggestions - suggestions.push @mkSuggestion true, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score - score *= 0.9 - - if custom - for suggestion in suggestions - suggestion.reinsertPrefix = "#{keyword} " if suggestion.insertText - - onComplete suggestions + onComplete suggestions, (onComplete) => + suggestions = [] + # For custom search-engine queries, this adds suggestions only if we have a completer. For other queries, + # this adds suggestions for the default search engine (if we have a completer for that). + SearchEngines.complete searchUrl, queryTerms, (searchSuggestions = []) => + + # Scoring: + # - The score does not depend upon the actual suggestion (so, it does not depend upon word relevancy). + # We assume that the completion engine has already factored that in. + # - The score is higher if the query is longer. The idea is that search suggestions are more likely + # to be relevant if, after typing quite some number of characters, the user hasn't yet found a + # useful suggestion from another completer. + # - Scores are weighted such that they retain the ordering provided by the completion engine. + characterCount = query.length - queryTerms.length + 1 + score = 0.8 * (Math.min(characterCount, 12.0)/12.0) + + for suggestion in searchSuggestions + suggestions.push @mkSuggestion true, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score + score *= 0.9 + + if custom + for suggestion in suggestions + suggestion.reinsertPrefix = "#{keyword} " if suggestion.insertText + + onComplete suggestions mkSuggestion: (insertText, args...) -> suggestion = new Suggestion args... @@ -410,9 +412,11 @@ class SearchEngineCompleter # A completer which calls filter() on many completers, aggregates the results, ranks them, and returns the top # 10. Queries from the vomnibar frontend script come through a multi completer. class MultiCompleter - constructor: (@completers) -> @maxResults = 10 + constructor: (@completers) -> + @maxResults = 10 - refresh: -> completer.refresh() for completer in @completers when completer.refresh + refresh: -> + completer.refresh?() for completer in @completers filter: (queryTerms, onComplete) -> # Allow only one query to run at a time. @@ -424,21 +428,40 @@ class MultiCompleter @filterInProgress = true suggestions = [] completersFinished = 0 + continuation = null for completer in @completers # Call filter() on every source completer and wait for them all to finish before returning results. - completer.filter queryTerms, (newSuggestions) => - suggestions = suggestions.concat(newSuggestions) - completersFinished += 1 - if completersFinished >= @completers.length - results = @sortSuggestions(suggestions)[0...@maxResults] - result.generateHtml() for result in results - onComplete(results) - @filterInProgress = false - @filter(@mostRecentQuery.queryTerms, @mostRecentQuery.onComplete) if @mostRecentQuery - - sortSuggestions: (suggestions) -> - suggestion.computeRelevancy(@queryTerms) for suggestion in suggestions + # At most one of the completers (SearchEngineCompleter) may pass a continuation function, which will be + # called asynchronously after the results of all of the other completers have been posted. Any + # additional results from this continuation will be added to the existing results and posted. We don't + # call the continuation if another query is already waiting. + completer.filter queryTerms, (newSuggestions, cont = null) => + # Allow completers to execute concurrently. + Utils.nextTick => + suggestions = suggestions.concat newSuggestions + continuation = cont if cont? + completersFinished += 1 + if completersFinished >= @completers.length + onComplete @prepareSuggestions(suggestions), keepAlive: continuation? + onDone = => + @filterInProgress = false + @filter @mostRecentQuery.queryTerms, @mostRecentQuery.onComplete if @mostRecentQuery + # We add a very short delay. It is possible for all of this processing to have been handled + # pretty-much synchronously, which would have prevented any newly-arriving queries from + # registering. + Utils.setTimeout 10, => + if continuation? and not @mostRecentQuery + continuation (newSuggestions) => + onComplete @prepareSuggestions suggestions.concat(newSuggestions) + onDone() + else + onDone() + + prepareSuggestions: (suggestions) -> + suggestion.computeRelevancy @queryTerms for suggestion in suggestions suggestions.sort (a, b) -> b.relevancy - a.relevancy + suggestions = suggestions[0...@maxResults] + suggestion.generateHtml() for suggestion in suggestions suggestions # Utilities which help us compute a relevancy score for a given item. diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index 4d2546fc..45619023 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -50,11 +50,13 @@ completionSources = searchEngines: new SearchEngineCompleter() completers = - omni: new MultiCompleter([ - completionSources.searchEngines, - completionSources.bookmarks, - completionSources.history, - completionSources.domains]) + omni: new MultiCompleter [ + completionSources.bookmarks + completionSources.history + completionSources.domains + # This comes last, because it delivers additional, asynchronous results. + completionSources.searchEngines + ] bookmarks: new MultiCompleter([completionSources.bookmarks]) tabs: new MultiCompleter([completionSources.tabs]) @@ -220,7 +222,8 @@ refreshCompleter = (request) -> completers[request.name].refresh() whitespaceRegexp = /\s+/ filterCompleter = (args, port) -> queryTerms = if (args.query == "") then [] else args.query.split(whitespaceRegexp) - completers[args.name].filter(queryTerms, (results) -> port.postMessage({ id: args.id, results: results })) + completers[args.name].filter queryTerms, (results, extra = {}) -> + port.postMessage extend extra, id: args.id, results: results chrome.tabs.onSelectionChanged.addListener (tabId, selectionInfo) -> if (selectionChangedHandlers.length > 0) diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index a80c218c..abf8c86e 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -134,8 +134,7 @@ SearchEngines = @requests[searchUrl] = xhr = new XMLHttpRequest() xhr.open "GET", url, true - # We set a fairly short timeout. If we block for too long, then we block *all* completers. - xhr.timeout = 500 + xhr.timeout = 750 xhr.ontimeout = => @cancel searchUrl, callback xhr.onerror = => @cancel searchUrl, callback xhr.send() -- cgit v1.2.3 From 136b132c81ae4f7a3b296109427fd757ca05dacf Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 4 May 2015 06:35:22 +0100 Subject: Search completion; tweak scoring and synchronization. --- background_scripts/completion.coffee | 121 ++++++++++++++++++------------- background_scripts/search_engines.coffee | 85 ++++++++++------------ 2 files changed, 108 insertions(+), 98 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index fee4778a..37e9ea6b 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -335,42 +335,67 @@ class SearchEngineCompleter else (string) -> Utils.createSearchUrl string.split /\s+/ - type = if description? then description else "search" + haveDescription = description? and 0 < description.trim().length + type = if haveDescription then description else "search" searchUrl = if custom then url else Settings.get "searchUrl" - query = queryTerms[(if custom then 1 else 0)..].join " " # For custom search engines, we add an auto-selected suggestion. if custom - title = if description? then query else queryTerms[0] + ": " + query + query = queryTerms[1..].join " " + title = if haveDescription then query else keyword + ": " + query suggestions.push @mkSuggestion false, queryTerms, type, mkUrl(query), title, @computeRelevancy, 1 suggestions[0].autoSelect = true queryTerms = queryTerms[1..] - onComplete suggestions, (onComplete) => + if queryTerms.length == 0 + return onComplete suggestions + + onComplete suggestions, (existingSuggestions, onComplete) => suggestions = [] # For custom search-engine queries, this adds suggestions only if we have a completer. For other queries, # this adds suggestions for the default search engine (if we have a completer for that). - SearchEngines.complete searchUrl, queryTerms, (searchSuggestions = []) => - - # Scoring: - # - The score does not depend upon the actual suggestion (so, it does not depend upon word relevancy). - # We assume that the completion engine has already factored that in. - # - The score is higher if the query is longer. The idea is that search suggestions are more likely - # to be relevant if, after typing quite some number of characters, the user hasn't yet found a - # useful suggestion from another completer. - # - Scores are weighted such that they retain the ordering provided by the completion engine. - characterCount = query.length - queryTerms.length + 1 - score = 0.8 * (Math.min(characterCount, 12.0)/12.0) - - for suggestion in searchSuggestions - suggestions.push @mkSuggestion true, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score - score *= 0.9 - if custom - for suggestion in suggestions - suggestion.reinsertPrefix = "#{keyword} " if suggestion.insertText - - onComplete suggestions + # Scoring: + # - The score does not depend upon the actual suggestion (so, it does not depend upon word + # relevancy). We assume that the completion engine has already factored that in. Also, completion + # engines often handle spelling mistakes, in which case we wouldn't find the query terms in the + # suggestion anyway. + # - The score is based on the length of the last query term. The idea is that the user is already + # happy with the earlier terms. + # - The score is higher if the last query term is longer. The idea is that search suggestions are more + # likely to be relevant if, after typing some number of characters, the user hasn't yet found + # a useful suggestion from another completer. + # - Scores are weighted such that they retain the order provided by the completion engine. + characterCount = queryTerms[queryTerms.length - 1].length + score = 0.6 * (Math.min(characterCount, 10.0)/10.0) + + if 0 < existingSuggestions.length + existingSuggestionMinScore = existingSuggestions[existingSuggestions.length-1].relevancy + if score < existingSuggestionMinScore and MultiCompleter.maxResults <= existingSuggestions.length + # No suggestion we propose will have a high enough score to beat the existing suggestions, so bail + # immediately. + return onComplete [] + + # We pause in case the user is still typing. + Utils.setTimeout 250, handler = @mostRecentHandler = => + return onComplete [] if handler != @mostRecentHandler # Bail if another completion has begun. + + SearchEngines.complete searchUrl, queryTerms, (searchSuggestions = []) => + for suggestion in searchSuggestions + suggestions.push @mkSuggestion true, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score + score *= 0.9 + + if custom + # For custom search engines, we need to tell the front end to insert the search engine's keyword + # when copying a suggestion into the vomnibar. + suggestion.reinsertPrefix = "#{keyword} " for suggestion in suggestions + + # We keep at least three suggestions (if possible) and at most six. We keep more than three only if + # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions + # from other completers. That would potentially be a problem because there is no relationship + # between the relevancy scores produced here and those produced by other completers. + count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length + onComplete suggestions[...count] mkSuggestion: (insertText, args...) -> suggestion = new Suggestion args... @@ -412,8 +437,10 @@ class SearchEngineCompleter # A completer which calls filter() on many completers, aggregates the results, ranks them, and returns the top # 10. Queries from the vomnibar frontend script come through a multi completer. class MultiCompleter + @maxResults: 10 + constructor: (@completers) -> - @maxResults = 10 + @maxResults = MultiCompleter.maxResults refresh: -> completer.refresh?() for completer in @completers @@ -429,36 +456,30 @@ class MultiCompleter suggestions = [] completersFinished = 0 continuation = null + # Call filter() on every source completer and wait for them all to finish before returning results. + # At most one of the completers (SearchEngineCompleter) may pass a continuation function, which will be + # called after the results of all of the other completers have been posted. Any additional results + # from this continuation will be added to the existing results and posted later. We don't call the + # continuation if another query is already waiting. for completer in @completers - # Call filter() on every source completer and wait for them all to finish before returning results. - # At most one of the completers (SearchEngineCompleter) may pass a continuation function, which will be - # called asynchronously after the results of all of the other completers have been posted. Any - # additional results from this continuation will be added to the existing results and posted. We don't - # call the continuation if another query is already waiting. - completer.filter queryTerms, (newSuggestions, cont = null) => - # Allow completers to execute concurrently. + do (completer) => Utils.nextTick => - suggestions = suggestions.concat newSuggestions - continuation = cont if cont? - completersFinished += 1 - if completersFinished >= @completers.length - onComplete @prepareSuggestions(suggestions), keepAlive: continuation? - onDone = => + completer.filter queryTerms, (newSuggestions, cont = null) => + suggestions = suggestions.concat newSuggestions + continuation = cont if cont? + if @completers.length <= ++completersFinished + shouldRunContinuation = continuation? and not @mostRecentQuery + onComplete @prepareSuggestions(queryTerms, suggestions), keepAlive: shouldRunContinuation + # Allow subsequent queries to begin. @filterInProgress = false - @filter @mostRecentQuery.queryTerms, @mostRecentQuery.onComplete if @mostRecentQuery - # We add a very short delay. It is possible for all of this processing to have been handled - # pretty-much synchronously, which would have prevented any newly-arriving queries from - # registering. - Utils.setTimeout 10, => - if continuation? and not @mostRecentQuery - continuation (newSuggestions) => - onComplete @prepareSuggestions suggestions.concat(newSuggestions) - onDone() + if shouldRunContinuation + continuation suggestions, (newSuggestions) => + onComplete @prepareSuggestions queryTerms, suggestions.concat(newSuggestions) else - onDone() + @filter @mostRecentQuery.queryTerms, @mostRecentQuery.onComplete if @mostRecentQuery - prepareSuggestions: (suggestions) -> - suggestion.computeRelevancy @queryTerms for suggestion in suggestions + prepareSuggestions: (queryTerms, suggestions) -> + suggestion.computeRelevancy queryTerms for suggestion in suggestions suggestions.sort (a, b) -> b.relevancy - a.relevancy suggestions = suggestions[0...@maxResults] suggestion.generateHtml() for suggestion in suggestions diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index abf8c86e..3ddbe742 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -114,38 +114,16 @@ completionEngines = [ ] SearchEngines = - cancel: (searchUrl, callback = null) -> - @requests[searchUrl]?.abort() - delete @requests[searchUrl] - callback? null - - # Perform an HTTP GET. get: (searchUrl, url, callback) -> - @requests ?= {} # Maps a searchUrl to any outstanding HTTP request for that search engine. - @cancel searchUrl - - # We cache the results of the most-recent 100 successfully XMLHttpRequests with a ten-second (ie. very - # short) expiry. - @requestCache ?= new SimpleCache 10 * 1000, 100 - - if @requestCache.has url - callback @requestCache.get url - return - - @requests[searchUrl] = xhr = new XMLHttpRequest() + xhr = new XMLHttpRequest() xhr.open "GET", url, true - xhr.timeout = 750 - xhr.ontimeout = => @cancel searchUrl, callback - xhr.onerror = => @cancel searchUrl, callback + xhr.timeout = 1000 + xhr.ontimeout = xhr.onerror = -> callback null xhr.send() - xhr.onreadystatechange = => + xhr.onreadystatechange = -> if xhr.readyState == 4 - @requests[searchUrl] = null - if xhr.status == 200 - callback @requestCache.set url, xhr - else - callback null + callback(if xhr.status == 200 then xhr else null) # Look up the search-completion engine for this searchUrl. Because of DummySearchEngine, above, we know # there will always be a match. Imagining that there may be many completion engines, and knowing that this @@ -176,7 +154,7 @@ SearchEngines = return callback [] if Utils.hasJavascriptPrefix queryTerms[0] # Cache completions. However, completions depend upon both the searchUrl and the query terms. So we need - # to generate a key. We mix in some nonsense generated by pwgen. A key clash is possible, but vanishingly + # to generate a key. We mix in some junk generated by pwgen. A key clash is possible, but vanishingly # unlikely. junk = "//Zi?ei5;o//" completionCacheKey = searchUrl + junk + queryTerms.join junk @@ -184,26 +162,37 @@ SearchEngines = if @completionCache.has completionCacheKey return callback @completionCache.get completionCacheKey - engine = @lookupEngine searchUrl - url = engine.getUrl queryTerms - query = queryTerms.join(" ").toLowerCase() - @get searchUrl, url, (xhr = null) => - # Parsing the response may fail if we receive an unexpected or an unexpectedly-formatted response. In - # all cases, we fall back to the catch clause, below. - try - suggestions = engine.parse xhr - # Make sure we really do have an iterable of strings. - suggestions = (suggestion for suggestion in suggestions when "string" == typeof suggestion) - # Filter out the query itself. It's not adding anything. - suggestions = (suggestion for suggestion in suggestions when suggestion.toLowerCase() != query) - # We keep at most three suggestions, the top three. - callback @completionCache.set completionCacheKey, suggestions[...3] - catch - callback @completionCache.set completionCacheKey, callback [] - # We cache failures, but remove them after just ten minutes. This (it is hoped) avoids repeated - # XMLHttpRequest failures over a short period of time. - removeCompletionCacheKey = => @completionCache.set completionCacheKey, null - setTimeout removeCompletionCacheKey, 10 * 60 * 1000 # Ten minutes. + fetchSuggestions = (callback) => + engine = @lookupEngine searchUrl + url = engine.getUrl queryTerms + query = queryTerms.join(" ").toLowerCase() + @get searchUrl, url, (xhr = null) => + # Parsing the response may fail if we receive an unexpected or an unexpectedly-formatted response. In + # all cases, we fall back to the catch clause, below. + try + suggestions = engine.parse xhr + # Make sure we really do have an iterable of strings. + suggestions = (suggestion for suggestion in suggestions when "string" == typeof suggestion) + # Filter out the query itself. It's not adding anything. + suggestions = (suggestion for suggestion in suggestions when suggestion.toLowerCase() != query) + catch + suggestions = [] + # We cache failures, but remove them after just ten minutes. This (it is hoped) avoids repeated + # XMLHttpRequest failures over a short period of time. + removeCompletionCacheKey = => @completionCache.set completionCacheKey, null + setTimeout removeCompletionCacheKey, 10 * 60 * 1000 # Ten minutes. + + callback suggestions + + # Don't allow duplicate identical active requests. This can happen, for example, when the user enters or + # removes a space, or when they enter a character and immediately delete it. + @inTransit ?= {} + unless @inTransit[completionCacheKey]?.push callback + queue = @inTransit[completionCacheKey] = [] + fetchSuggestions (suggestions) => + callback @completionCache.set completionCacheKey, suggestions + delete @inTransit[completionCacheKey] + callback suggestions for callback in queue root = exports ? window root.SearchEngines = SearchEngines -- cgit v1.2.3 From 85aa77ce1daec7bd1452cd7813a0d5d408729408 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 4 May 2015 09:54:36 +0100 Subject: Search completion; prevent vomnibar flicker. --- background_scripts/completion.coffee | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 37e9ea6b..a75cf6b5 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -469,7 +469,11 @@ class MultiCompleter continuation = cont if cont? if @completers.length <= ++completersFinished shouldRunContinuation = continuation? and not @mostRecentQuery - onComplete @prepareSuggestions(queryTerms, suggestions), keepAlive: shouldRunContinuation + # We don't post results immediately if there are none, and we're going to run a continuation + # (ie. a SearchEngineCompleter). This prevents hiding the vomnibar briefly before showing it + # again, which looks ugly. + unless shouldRunContinuation and suggestions.length == 0 + onComplete @prepareSuggestions(queryTerms, suggestions), keepAlive: shouldRunContinuation # Allow subsequent queries to begin. @filterInProgress = false if shouldRunContinuation -- cgit v1.2.3 From 9de495ededee7f98504d6fe308ea73526956a7f3 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 4 May 2015 10:17:45 +0100 Subject: Search completion; hardwire search engines. These should be removed later. --- background_scripts/settings.coffee | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'background_scripts') diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index 07db3a89..b802937e 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -5,6 +5,8 @@ root = exports ? window root.Settings = Settings = get: (key) -> + # FIXME(smblott). Remove this line. + return @defaults.searchEngines if key == "searchEngines" if (key of localStorage) then JSON.parse(localStorage[key]) else @defaults[key] set: (key, value) -> @@ -92,6 +94,9 @@ root.Settings = Settings = # put in an example search engine searchEngines: [ # FIXME(smblott) Comment these out before merge. + "# THESE ARE HARD WIRED.\n# YOU CANNOT CHANGE THEM IN THIS VERSION.\n# FOR DEVELOPMENT ONLY." + "g: http://www.google.com/search?q=%s Google" + "l: http://www.google.com/search?q=%s&btnI I'm feeling lucky..." "w: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedia" "t: http://www.youtube.com/results?search_query=%s Youtube" "m: https://www.google.com/maps/search/%s Google Maps" -- cgit v1.2.3 From b2ae01c8604ed87f52afa3b5e769fb340dcb1b93 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 4 May 2015 10:27:08 +0100 Subject: Search completion; force the first match to the top of the list. This may or may not be a good idea. It means that the first completion is just a tab away. --- background_scripts/completion.coffee | 3 +++ 1 file changed, 3 insertions(+) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index a75cf6b5..795b4658 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -385,6 +385,9 @@ class SearchEngineCompleter suggestions.push @mkSuggestion true, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score score *= 0.9 + # Experimental. Force the best match to the top of the list. + suggestions[0].extraRelevancyData = 0.9999999 if 0 < suggestions.length + if custom # For custom search engines, we need to tell the front end to insert the search engine's keyword # when copying a suggestion into the vomnibar. -- cgit v1.2.3 From 4e86053f9464c04b9e422625478f8e827f37e533 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 4 May 2015 11:42:59 +0100 Subject: Search completion; do not highlight search terms. Highlighting the search terms suggests they are in some way contributing to the match. They are not, so don't highlight them. This gets particularly ugly when you have short, single-letter costom search engines (eg. w), and have all of the "w"s highlighted -- for not useful reason. --- background_scripts/completion.coffee | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 795b4658..b503a452 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -29,15 +29,17 @@ class Suggestion generateHtml: -> return @html if @html relevancyHtml = if @showRelevancy then "#{@computeRelevancy()}" else "" + highlightTerms = + if @noHighlightTerms then ((s) -> Utils.escapeHtml s) else ((s) => @highlightTerms Utils.escapeHtml s) # NOTE(philc): We're using these vimium-specific class names so we don't collide with the page's CSS. @html = """
#{@type} - #{@highlightTerms(Utils.escapeHtml(@title))} + #{highlightTerms @title}
- #{@shortenUrl(@highlightTerms(Utils.escapeHtml(@url)))} + #{@shortenUrl highlightTerms @url} #{relevancyHtml}
""" @@ -402,8 +404,7 @@ class SearchEngineCompleter mkSuggestion: (insertText, args...) -> suggestion = new Suggestion args... - suggestion.insertText = insertText - suggestion + extend suggestion, insertText: insertText, noHighlightTerms: true # The score is computed in filter() and provided here via suggestion.extraRelevancyData. computeRelevancy: (suggestion) -> suggestion.extraRelevancyData -- cgit v1.2.3 From f9d35083ad511080ee5c01c5c97ef29503f3c0ba Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 4 May 2015 11:55:37 +0100 Subject: Search completion; simplify/document suggestion options. --- background_scripts/completion.coffee | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index b503a452..f09a079c 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -23,6 +23,11 @@ class Suggestion @title ||= "" # When @autoSelect is truthy, the suggestion is automatically pre-selected in the vomnibar. @autoSelect = false + # If @noHighlightTerms is falsy, then we don't highlight matched terms in the title and URL. + @autoSelect = true + # If @insertText is a string, then the indicated text is inserted into the vomnibar input when the + # suggestion is selected. + @insertText = null computeRelevancy: -> @relevancy = @computeRelevancyFunction(this) @@ -345,7 +350,7 @@ class SearchEngineCompleter if custom query = queryTerms[1..].join " " title = if haveDescription then query else keyword + ": " + query - suggestions.push @mkSuggestion false, queryTerms, type, mkUrl(query), title, @computeRelevancy, 1 + suggestions.push @mkSuggestion null, queryTerms, type, mkUrl(query), title, @computeRelevancy, 1 suggestions[0].autoSelect = true queryTerms = queryTerms[1..] @@ -384,17 +389,13 @@ class SearchEngineCompleter SearchEngines.complete searchUrl, queryTerms, (searchSuggestions = []) => for suggestion in searchSuggestions - suggestions.push @mkSuggestion true, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score + insertText = if custom then "#{keyword} #{suggestion}" else suggestion + suggestions.push @mkSuggestion insertText, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score score *= 0.9 # Experimental. Force the best match to the top of the list. suggestions[0].extraRelevancyData = 0.9999999 if 0 < suggestions.length - if custom - # For custom search engines, we need to tell the front end to insert the search engine's keyword - # when copying a suggestion into the vomnibar. - suggestion.reinsertPrefix = "#{keyword} " for suggestion in suggestions - # We keep at least three suggestions (if possible) and at most six. We keep more than three only if # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions # from other completers. That would potentially be a problem because there is no relationship -- cgit v1.2.3 From 7f99155824edd7424bd28feae401459da6a2ba8f Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 4 May 2015 12:38:00 +0100 Subject: Search completion; instrument for debugging. --- background_scripts/completion.coffee | 35 +++++++++++++-------------- background_scripts/search_engines.coffee | 41 ++++++++++++++++++++++---------- 2 files changed, 45 insertions(+), 31 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index f09a079c..1c07a71a 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -383,25 +383,21 @@ class SearchEngineCompleter # immediately. return onComplete [] - # We pause in case the user is still typing. - Utils.setTimeout 250, handler = @mostRecentHandler = => - return onComplete [] if handler != @mostRecentHandler # Bail if another completion has begun. - - SearchEngines.complete searchUrl, queryTerms, (searchSuggestions = []) => - for suggestion in searchSuggestions - insertText = if custom then "#{keyword} #{suggestion}" else suggestion - suggestions.push @mkSuggestion insertText, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score - score *= 0.9 - - # Experimental. Force the best match to the top of the list. - suggestions[0].extraRelevancyData = 0.9999999 if 0 < suggestions.length - - # We keep at least three suggestions (if possible) and at most six. We keep more than three only if - # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions - # from other completers. That would potentially be a problem because there is no relationship - # between the relevancy scores produced here and those produced by other completers. - count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length - onComplete suggestions[...count] + SearchEngines.complete searchUrl, queryTerms, (searchSuggestions = []) => + for suggestion in searchSuggestions + insertText = if custom then "#{keyword} #{suggestion}" else suggestion + suggestions.push @mkSuggestion insertText, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score + score *= 0.9 + + # Experimental. Force the best match to the top of the list. + suggestions[0].extraRelevancyData = 0.9999999 if 0 < suggestions.length + + # We keep at least three suggestions (if possible) and at most six. We keep more than three only if + # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions + # from other completers. That would potentially be a problem because there is no relationship + # between the relevancy scores produced here and those produced by other completers. + count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length + onComplete suggestions[...count] mkSuggestion: (insertText, args...) -> suggestion = new Suggestion args... @@ -474,6 +470,7 @@ class MultiCompleter continuation = cont if cont? if @completers.length <= ++completersFinished shouldRunContinuation = continuation? and not @mostRecentQuery + console.log "skip continuation" if continuation? and not shouldRunContinuation # We don't post results immediately if there are none, and we're going to run a continuation # (ie. a SearchEngineCompleter). This prevents hiding the vomnibar briefly before showing it # again, which looks ugly. diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index 3ddbe742..608115f3 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -114,6 +114,8 @@ completionEngines = [ ] SearchEngines = + debug: true + get: (searchUrl, url, callback) -> xhr = new XMLHttpRequest() xhr.open "GET", url, true @@ -144,11 +146,18 @@ SearchEngines = # - callback will be applied to a list of suggestion strings (which may be an empty list, if anything goes # wrong). complete: (searchUrl, queryTerms, callback) -> + @mostRecentHandler = null + # We can't complete empty queries. return callback [] unless 0 < queryTerms.length - # We don't complete URLs. - return callback [] if 1 == queryTerms.length and Utils.isUrl queryTerms[0] + if 1 == queryTerms.length + # We don't complete URLs. + return callback [] if Utils.isUrl queryTerms[0] + # We don't complete less then three characters: the results are usually useless. This also prevents + # one- and two-character custom search engine keywords from being sent to the default completer (e.g. + # the initial "w" before typing "w something"). + return callback [] unless 2 < queryTerms[0].length # We don't complete Javascript URLs. return callback [] if Utils.hasJavascriptPrefix queryTerms[0] @@ -158,13 +167,15 @@ SearchEngines = # unlikely. junk = "//Zi?ei5;o//" completionCacheKey = searchUrl + junk + queryTerms.join junk - @completionCache ?= new SimpleCache 6 * 60 * 60 * 1000, 2000 # Six hours, 2000 entries. + @completionCache ?= new SimpleCache 60 * 60 * 1000, 2000 # One hour, 2000 entries. if @completionCache.has completionCacheKey + console.log "hit", completionCacheKey if @debug return callback @completionCache.get completionCacheKey fetchSuggestions = (callback) => engine = @lookupEngine searchUrl url = engine.getUrl queryTerms + console.log "get", url if @debug query = queryTerms.join(" ").toLowerCase() @get searchUrl, url, (xhr = null) => # Parsing the response may fail if we receive an unexpected or an unexpectedly-formatted response. In @@ -184,15 +195,21 @@ SearchEngines = callback suggestions - # Don't allow duplicate identical active requests. This can happen, for example, when the user enters or - # removes a space, or when they enter a character and immediately delete it. - @inTransit ?= {} - unless @inTransit[completionCacheKey]?.push callback - queue = @inTransit[completionCacheKey] = [] - fetchSuggestions (suggestions) => - callback @completionCache.set completionCacheKey, suggestions - delete @inTransit[completionCacheKey] - callback suggestions for callback in queue + # We pause in case the user is still typing. + Utils.setTimeout 200, handler = @mostRecentHandler = => + if handler != @mostRecentHandler # Bail if another completion has begun. + console.log "bail", completionCacheKey if @debug + return callback [] + # Don't allow duplicate identical active requests. This can happen, for example, when the user enters or + # removes a space, or when they enter a character and immediately delete it. + @inTransit ?= {} + unless @inTransit[completionCacheKey]?.push callback + queue = @inTransit[completionCacheKey] = [] + fetchSuggestions (suggestions) => + callback @completionCache.set completionCacheKey, suggestions + delete @inTransit[completionCacheKey] + console.log "callbacks", queue.length, completionCacheKey if @debug and 0 < queue.length + callback suggestions for callback in queue root = exports ? window root.SearchEngines = SearchEngines -- cgit v1.2.3 From 44b24f43a30e2f2ebe47e70e74413488ad3a676f Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 4 May 2015 13:29:48 +0100 Subject: Search completion; fix mistake with autoSelect. --- background_scripts/completion.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 1c07a71a..d206a285 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -24,7 +24,7 @@ class Suggestion # When @autoSelect is truthy, the suggestion is automatically pre-selected in the vomnibar. @autoSelect = false # If @noHighlightTerms is falsy, then we don't highlight matched terms in the title and URL. - @autoSelect = true + @noHighlightTerms = false # If @insertText is a string, then the indicated text is inserted into the vomnibar input when the # suggestion is selected. @insertText = null -- cgit v1.2.3 From b8e018df407fdca6e2418a90e8ad6cf6b3659425 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 4 May 2015 14:21:37 +0100 Subject: Search completion; disable experimental feature. --- background_scripts/completion.coffee | 3 --- 1 file changed, 3 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index d206a285..e76139c2 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -389,9 +389,6 @@ class SearchEngineCompleter suggestions.push @mkSuggestion insertText, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score score *= 0.9 - # Experimental. Force the best match to the top of the list. - suggestions[0].extraRelevancyData = 0.9999999 if 0 < suggestions.length - # We keep at least three suggestions (if possible) and at most six. We keep more than three only if # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions # from other completers. That would potentially be a problem because there is no relationship -- cgit v1.2.3 From 1a9ea85f683c892592d7092c361e835c8b0f4f3e Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Tue, 5 May 2015 06:14:12 +0100 Subject: Search completion; tweak scoring. --- background_scripts/completion.coffee | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index e76139c2..33d8a563 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -353,6 +353,8 @@ class SearchEngineCompleter suggestions.push @mkSuggestion null, queryTerms, type, mkUrl(query), title, @computeRelevancy, 1 suggestions[0].autoSelect = true queryTerms = queryTerms[1..] + else + query = queryTerms.join " " if queryTerms.length == 0 return onComplete suggestions @@ -367,13 +369,11 @@ class SearchEngineCompleter # relevancy). We assume that the completion engine has already factored that in. Also, completion # engines often handle spelling mistakes, in which case we wouldn't find the query terms in the # suggestion anyway. - # - The score is based on the length of the last query term. The idea is that the user is already - # happy with the earlier terms. - # - The score is higher if the last query term is longer. The idea is that search suggestions are more + # - The score is higher if the query term is longer. The idea is that search suggestions are more # likely to be relevant if, after typing some number of characters, the user hasn't yet found # a useful suggestion from another completer. # - Scores are weighted such that they retain the order provided by the completion engine. - characterCount = queryTerms[queryTerms.length - 1].length + characterCount = query.length - queryTerms.length + 1 score = 0.6 * (Math.min(characterCount, 10.0)/10.0) if 0 < existingSuggestions.length -- cgit v1.2.3 From 7c855af38cfdf32a05b90b2da4711720ebac8865 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Tue, 5 May 2015 14:47:02 +0100 Subject: Search completion; user is typing. Add plumbing to allow the front end to directly inform completers when the user is typing. --- background_scripts/completion.coffee | 6 ++++++ background_scripts/main.coffee | 10 +++++++--- background_scripts/search_engines.coffee | 8 ++++++-- 3 files changed, 19 insertions(+), 5 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 33d8a563..92936098 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -331,6 +331,9 @@ class TabCompleter class SearchEngineCompleter searchEngines: {} + userIsTyping: -> + SearchEngines.userIsTyping() + filter: (queryTerms, onComplete) -> { keyword: keyword, url: url, description: description } = @getSearchEngineMatches queryTerms custom = url? @@ -443,6 +446,9 @@ class MultiCompleter refresh: -> completer.refresh?() for completer in @completers + userIsTyping: -> + completer.userIsTyping?() for completer in @completers + filter: (queryTerms, onComplete) -> # Allow only one query to run at a time. if @filterInProgress diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index 45619023..44644769 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -221,9 +221,13 @@ refreshCompleter = (request) -> completers[request.name].refresh() whitespaceRegexp = /\s+/ filterCompleter = (args, port) -> - queryTerms = if (args.query == "") then [] else args.query.split(whitespaceRegexp) - completers[args.name].filter queryTerms, (results, extra = {}) -> - port.postMessage extend extra, id: args.id, results: results + if args.name? and args.userIsTyping + completers[args.name].userIsTyping?() + + if args.id? and args.name? and args.query? + queryTerms = if (args.query == "") then [] else args.query.split(whitespaceRegexp) + completers[args.name].filter queryTerms, (results, extra = {}) -> + port.postMessage extend extra, id: args.id, results: results chrome.tabs.onSelectionChanged.addListener (tabId, selectionInfo) -> if (selectionChangedHandlers.length > 0) diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index 608115f3..74e752e3 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -13,7 +13,7 @@ # 3. "parse" - This takes a successful XMLHttpRequest object (the request has completed successfully), and # returns a list of suggestions (a list of strings). # -# The main (only) completion entry point is SearchEngines.complete(). This implements all lookup and caching +# The main completion entry point is SearchEngines.complete(). This implements all lookup and caching # logic. It is possible to add new completion engines without changing the SearchEngines infrastructure # itself. @@ -197,7 +197,7 @@ SearchEngines = # We pause in case the user is still typing. Utils.setTimeout 200, handler = @mostRecentHandler = => - if handler != @mostRecentHandler # Bail if another completion has begun. + if handler != @mostRecentHandler # Bail if another completion has begun, or the user is typing. console.log "bail", completionCacheKey if @debug return callback [] # Don't allow duplicate identical active requests. This can happen, for example, when the user enters or @@ -211,5 +211,9 @@ SearchEngines = console.log "callbacks", queue.length, completionCacheKey if @debug and 0 < queue.length callback suggestions for callback in queue + userIsTyping: -> + console.log "reset (typing)" if @debug and @mostRecentHandler? + @mostRecentHandler = null + root = exports ? window root.SearchEngines = SearchEngines -- cgit v1.2.3 From 32895a96b3da225bd1b46d4b96ab98fd71755281 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Tue, 5 May 2015 16:16:59 +0100 Subject: Search completion; changes in response to @mrmr1993. --- background_scripts/completion.coffee | 8 ++++---- background_scripts/search_engines.coffee | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 92936098..39f8a140 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -34,17 +34,15 @@ class Suggestion generateHtml: -> return @html if @html relevancyHtml = if @showRelevancy then "#{@computeRelevancy()}" else "" - highlightTerms = - if @noHighlightTerms then ((s) -> Utils.escapeHtml s) else ((s) => @highlightTerms Utils.escapeHtml s) # NOTE(philc): We're using these vimium-specific class names so we don't collide with the page's CSS. @html = """
#{@type} - #{highlightTerms @title} + #{@highlightTerms Utils.escapeHtml @title}
- #{@shortenUrl highlightTerms @url} + #{@shortenUrl @highlightTerms Utils.escapeHtml @url} #{relevancyHtml}
""" @@ -85,6 +83,7 @@ class Suggestion # Wraps each occurence of the query terms in the given string in a . highlightTerms: (string) -> + return string if @noHighlightTerms ranges = [] escapedTerms = @queryTerms.map (term) -> Utils.escapeHtml(term) for term in escapedTerms @@ -399,6 +398,7 @@ class SearchEngineCompleter count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length onComplete suggestions[...count] + # FIXME(smblott) Refactor Suggestion constructor as per @mrmr1993's comment in #1635. mkSuggestion: (insertText, args...) -> suggestion = new Suggestion args... extend suggestion, insertText: insertText, noHighlightTerms: true diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee index 74e752e3..63c61a47 100644 --- a/background_scripts/search_engines.coffee +++ b/background_scripts/search_engines.coffee @@ -34,7 +34,7 @@ class Google extends GoogleXMLRegexpEngine constructor: -> super [ # We match the major English-speaking TLDs. - new RegExp "^https?://[a-z]+\.google\.(com|ie|co.uk|ca|com.au)/" + new RegExp "^https?://[a-z]+\.google\.(com|ie|co\.uk|ca|com\.au)/" new RegExp "localhost/cgi-bin/booky" # Only for smblott. ] -- cgit v1.2.3 From 28807bd25b27e5404228a638f2ab5e6c00f606cc Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Tue, 5 May 2015 16:43:48 +0100 Subject: Search completion; misc. --- background_scripts/completion.coffee | 2 +- background_scripts/completion_engines.coffee | 219 +++++++++++++++++++++++++++ background_scripts/search_engines.coffee | 219 --------------------------- 3 files changed, 220 insertions(+), 220 deletions(-) create mode 100644 background_scripts/completion_engines.coffee delete mode 100644 background_scripts/search_engines.coffee (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 39f8a140..729e86ab 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -385,7 +385,7 @@ class SearchEngineCompleter # immediately. return onComplete [] - SearchEngines.complete searchUrl, queryTerms, (searchSuggestions = []) => + CompletionEngines.complete searchUrl, queryTerms, (searchSuggestions = []) => for suggestion in searchSuggestions insertText = if custom then "#{keyword} #{suggestion}" else suggestion suggestions.push @mkSuggestion insertText, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee new file mode 100644 index 00000000..0177806a --- /dev/null +++ b/background_scripts/completion_engines.coffee @@ -0,0 +1,219 @@ + +# A completion engine provides search suggestions for a search engine. A search engine is identified by a +# "searchUrl", e.g. Settings.get("searchUrl"), or a custom search engine. +# +# Each completion engine defines three functions: +# +# 1. "match" - This takes a searchUrl, and returns a boolean indicating whether this completion engine can +# perform completion for the given search engine. +# +# 2. "getUrl" - This takes a list of query terms (queryTerms) and generates a completion URL, that is, a URL +# which will provide completions for this completion engine. +# +# 3. "parse" - This takes a successful XMLHttpRequest object (the request has completed successfully), and +# returns a list of suggestions (a list of strings). +# +# The main completion entry point is CompletionEngines.complete(). This implements all lookup and caching +# logic. It is possible to add new completion engines without changing the CompletionEngines infrastructure +# itself. + +# A base class for common regexp-based matching engines. +class RegexpEngine + constructor: (@regexps) -> + match: (searchUrl) -> Utils.matchesAnyRegexp @regexps, searchUrl + +# Several Google completion engines package responses in this way. +class GoogleXMLRegexpEngine extends RegexpEngine + parse: (xhr) -> + for suggestion in xhr.responseXML.getElementsByTagName "suggestion" + continue unless suggestion = suggestion.getAttribute "data" + suggestion + +class Google extends GoogleXMLRegexpEngine + # Example search URL: http://www.google.com/search?q=%s + constructor: -> + super [ + # We match the major English-speaking TLDs. + new RegExp "^https?://[a-z]+\.google\.(com|ie|co\.uk|ca|com\.au)/" + new RegExp "localhost/cgi-bin/booky" # Only for smblott. + ] + + getUrl: (queryTerms) -> + "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{Utils.createSearchQuery queryTerms}" + +class Youtube extends GoogleXMLRegexpEngine + # Example search URL: http://www.youtube.com/results?search_query=%s + constructor: -> + super [ new RegExp "^https?://[a-z]+\.youtube\.com/results" ] + + getUrl: (queryTerms) -> + "http://suggestqueries.google.com/complete/search?client=youtube&ds=yt&xml=t&q=#{Utils.createSearchQuery queryTerms}" + +class Wikipedia extends RegexpEngine + # Example search URL: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s + constructor: -> + super [ new RegExp "^https?://[a-z]+\.wikipedia\.org/" ] + + getUrl: (queryTerms) -> + "https://en.wikipedia.org/w/api.php?action=opensearch&format=json&search=#{Utils.createSearchQuery queryTerms}" + + parse: (xhr) -> + JSON.parse(xhr.responseText)[1] + +## class GoogleMaps extends RegexpEngine +## # Example search URL: https://www.google.com/maps/search/%s +## constructor: -> +## super [ new RegExp "^https?://www\.google\.com/maps/search/" ] +## +## getUrl: (queryTerms) -> +## console.log "xxxxxxxxxxxxxxxxxxxxx" +## "https://www.google.com/s?tbm=map&fp=1&gs_ri=maps&source=hp&suggest=p&authuser=0&hl=en&pf=p&tch=1&ech=2&q=#{Utils.createSearchQuery queryTerms}" +## +## parse: (xhr) -> +## console.log "yyy", xhr.responseText +## data = JSON.parse xhr.responseText +## console.log "zzz" +## console.log data +## [] + +class Bing extends RegexpEngine + # Example search URL: https://www.bing.com/search?q=%s + constructor: -> super [ new RegExp "^https?://www\.bing\.com/search" ] + getUrl: (queryTerms) -> "http://api.bing.com/osjson.aspx?query=#{Utils.createSearchQuery queryTerms}" + parse: (xhr) -> JSON.parse(xhr.responseText)[1] + +class Amazon extends RegexpEngine + # Example search URL: http://www.amazon.com/s/?field-keywords=%s + constructor: -> super [ new RegExp "^https?://www\.amazon\.(com|co.uk|ca|com.au)/s/" ] + getUrl: (queryTerms) -> "https://completion.amazon.com/search/complete?method=completion&search-alias=aps&client=amazon-search-ui&mkt=1&q=#{Utils.createSearchQuery queryTerms}" + parse: (xhr) -> JSON.parse(xhr.responseText)[1] + +class DuckDuckGo extends RegexpEngine + # Example search URL: https://duckduckgo.com/?q=%s + constructor: -> super [ new RegExp "^https?://([a-z]+\.)?duckduckgo\.com/" ] + getUrl: (queryTerms) -> "https://duckduckgo.com/ac/?q=#{Utils.createSearchQuery queryTerms}" + parse: (xhr) -> + suggestion.phrase for suggestion in JSON.parse xhr.responseText + +# A dummy search engine which is guaranteed to match any search URL, but never produces completions. This +# allows the rest of the logic to be written knowing that there will be a search engine match. +class DummySearchEngine + match: -> true + # We return a useless URL which we know will succeed, but which won't generate any network traffic. + getUrl: -> chrome.runtime.getURL "content_scripts/vimium.css" + parse: -> [] + +completionEngines = [ + Youtube + Google + DuckDuckGo + Wikipedia + Bing + Amazon + DummySearchEngine +] + +CompletionEngines = + debug: true + + get: (searchUrl, url, callback) -> + xhr = new XMLHttpRequest() + xhr.open "GET", url, true + xhr.timeout = 1000 + xhr.ontimeout = xhr.onerror = -> callback null + xhr.send() + + xhr.onreadystatechange = -> + if xhr.readyState == 4 + callback(if xhr.status == 200 then xhr else null) + + # Look up the search-completion engine for this searchUrl. Because of DummySearchEngine, above, we know + # there will always be a match. Imagining that there may be many completion engines, and knowing that this + # is called for every input event in the vomnibar, we cache the result. + lookupEngine: (searchUrl) -> + @engineCache ?= new SimpleCache 30 * 60 * 60 * 1000 # 30 hours (these are small, we can keep them longer). + if @engineCache.has searchUrl + @engineCache.get searchUrl + else + for engine in completionEngines + engine = new engine() + return @engineCache.set searchUrl, engine if engine.match searchUrl + + # This is the main (actually, the only) entry point. + # - searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"), or a custome search engine's URL. + # This is only used as a key for determining the relevant completion engine. + # - queryTerms are the queryTerms. + # - callback will be applied to a list of suggestion strings (which may be an empty list, if anything goes + # wrong). + complete: (searchUrl, queryTerms, callback) -> + @mostRecentHandler = null + + # We can't complete empty queries. + return callback [] unless 0 < queryTerms.length + + if 1 == queryTerms.length + # We don't complete URLs. + return callback [] if Utils.isUrl queryTerms[0] + # We don't complete less then three characters: the results are usually useless. This also prevents + # one- and two-character custom search engine keywords from being sent to the default completer (e.g. + # the initial "w" before typing "w something"). + return callback [] unless 2 < queryTerms[0].length + + # We don't complete Javascript URLs. + return callback [] if Utils.hasJavascriptPrefix queryTerms[0] + + # Cache completions. However, completions depend upon both the searchUrl and the query terms. So we need + # to generate a key. We mix in some junk generated by pwgen. A key clash is possible, but vanishingly + # unlikely. + junk = "//Zi?ei5;o//" + completionCacheKey = searchUrl + junk + queryTerms.join junk + @completionCache ?= new SimpleCache 60 * 60 * 1000, 2000 # One hour, 2000 entries. + if @completionCache.has completionCacheKey + console.log "hit", completionCacheKey if @debug + return callback @completionCache.get completionCacheKey + + fetchSuggestions = (callback) => + engine = @lookupEngine searchUrl + url = engine.getUrl queryTerms + console.log "get", url if @debug + query = queryTerms.join(" ").toLowerCase() + @get searchUrl, url, (xhr = null) => + # Parsing the response may fail if we receive an unexpected or an unexpectedly-formatted response. In + # all cases, we fall back to the catch clause, below. + try + suggestions = engine.parse xhr + # Make sure we really do have an iterable of strings. + suggestions = (suggestion for suggestion in suggestions when "string" == typeof suggestion) + # Filter out the query itself. It's not adding anything. + suggestions = (suggestion for suggestion in suggestions when suggestion.toLowerCase() != query) + catch + suggestions = [] + # We cache failures, but remove them after just ten minutes. This (it is hoped) avoids repeated + # XMLHttpRequest failures over a short period of time. + removeCompletionCacheKey = => @completionCache.set completionCacheKey, null + setTimeout removeCompletionCacheKey, 10 * 60 * 1000 # Ten minutes. + + callback suggestions + + # We pause in case the user is still typing. + Utils.setTimeout 200, handler = @mostRecentHandler = => + if handler != @mostRecentHandler # Bail if another completion has begun, or the user is typing. + console.log "bail", completionCacheKey if @debug + return callback [] + # Don't allow duplicate identical active requests. This can happen, for example, when the user enters or + # removes a space, or when they enter a character and immediately delete it. + @inTransit ?= {} + unless @inTransit[completionCacheKey]?.push callback + queue = @inTransit[completionCacheKey] = [] + fetchSuggestions (suggestions) => + callback @completionCache.set completionCacheKey, suggestions + delete @inTransit[completionCacheKey] + console.log "callbacks", queue.length, completionCacheKey if @debug and 0 < queue.length + callback suggestions for callback in queue + + userIsTyping: -> + console.log "reset (typing)" if @debug and @mostRecentHandler? + @mostRecentHandler = null + +root = exports ? window +root.CompletionEngines = CompletionEngines diff --git a/background_scripts/search_engines.coffee b/background_scripts/search_engines.coffee deleted file mode 100644 index 63c61a47..00000000 --- a/background_scripts/search_engines.coffee +++ /dev/null @@ -1,219 +0,0 @@ - -# A completion engine provides search suggestions for a search engine. A search engine is identified by a -# "searchUrl", e.g. Settings.get("searchUrl"), or a custom search engine. -# -# Each completion engine defines three functions: -# -# 1. "match" - This takes a searchUrl, and returns a boolean indicating whether this completion engine can -# perform completion for the given search engine. -# -# 2. "getUrl" - This takes a list of query terms (queryTerms) and generates a completion URL, that is, a URL -# which will provide completions for this completion engine. -# -# 3. "parse" - This takes a successful XMLHttpRequest object (the request has completed successfully), and -# returns a list of suggestions (a list of strings). -# -# The main completion entry point is SearchEngines.complete(). This implements all lookup and caching -# logic. It is possible to add new completion engines without changing the SearchEngines infrastructure -# itself. - -# A base class for common regexp-based matching engines. -class RegexpEngine - constructor: (@regexps) -> - match: (searchUrl) -> Utils.matchesAnyRegexp @regexps, searchUrl - -# Several Google completion engines package responses in this way. -class GoogleXMLRegexpEngine extends RegexpEngine - parse: (xhr) -> - for suggestion in xhr.responseXML.getElementsByTagName "suggestion" - continue unless suggestion = suggestion.getAttribute "data" - suggestion - -class Google extends GoogleXMLRegexpEngine - # Example search URL: http://www.google.com/search?q=%s - constructor: -> - super [ - # We match the major English-speaking TLDs. - new RegExp "^https?://[a-z]+\.google\.(com|ie|co\.uk|ca|com\.au)/" - new RegExp "localhost/cgi-bin/booky" # Only for smblott. - ] - - getUrl: (queryTerms) -> - "http://suggestqueries.google.com/complete/search?ss_protocol=legace&client=toolbar&q=#{Utils.createSearchQuery queryTerms}" - -class Youtube extends GoogleXMLRegexpEngine - # Example search URL: http://www.youtube.com/results?search_query=%s - constructor: -> - super [ new RegExp "^https?://[a-z]+\.youtube\.com/results" ] - - getUrl: (queryTerms) -> - "http://suggestqueries.google.com/complete/search?client=youtube&ds=yt&xml=t&q=#{Utils.createSearchQuery queryTerms}" - -class Wikipedia extends RegexpEngine - # Example search URL: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s - constructor: -> - super [ new RegExp "^https?://[a-z]+\.wikipedia\.org/" ] - - getUrl: (queryTerms) -> - "https://en.wikipedia.org/w/api.php?action=opensearch&format=json&search=#{Utils.createSearchQuery queryTerms}" - - parse: (xhr) -> - JSON.parse(xhr.responseText)[1] - -## class GoogleMaps extends RegexpEngine -## # Example search URL: https://www.google.com/maps/search/%s -## constructor: -> -## super [ new RegExp "^https?://www\.google\.com/maps/search/" ] -## -## getUrl: (queryTerms) -> -## console.log "xxxxxxxxxxxxxxxxxxxxx" -## "https://www.google.com/s?tbm=map&fp=1&gs_ri=maps&source=hp&suggest=p&authuser=0&hl=en&pf=p&tch=1&ech=2&q=#{Utils.createSearchQuery queryTerms}" -## -## parse: (xhr) -> -## console.log "yyy", xhr.responseText -## data = JSON.parse xhr.responseText -## console.log "zzz" -## console.log data -## [] - -class Bing extends RegexpEngine - # Example search URL: https://www.bing.com/search?q=%s - constructor: -> super [ new RegExp "^https?://www\.bing\.com/search" ] - getUrl: (queryTerms) -> "http://api.bing.com/osjson.aspx?query=#{Utils.createSearchQuery queryTerms}" - parse: (xhr) -> JSON.parse(xhr.responseText)[1] - -class Amazon extends RegexpEngine - # Example search URL: http://www.amazon.com/s/?field-keywords=%s - constructor: -> super [ new RegExp "^https?://www\.amazon\.(com|co.uk|ca|com.au)/s/" ] - getUrl: (queryTerms) -> "https://completion.amazon.com/search/complete?method=completion&search-alias=aps&client=amazon-search-ui&mkt=1&q=#{Utils.createSearchQuery queryTerms}" - parse: (xhr) -> JSON.parse(xhr.responseText)[1] - -class DuckDuckGo extends RegexpEngine - # Example search URL: https://duckduckgo.com/?q=%s - constructor: -> super [ new RegExp "^https?://([a-z]+\.)?duckduckgo\.com/" ] - getUrl: (queryTerms) -> "https://duckduckgo.com/ac/?q=#{Utils.createSearchQuery queryTerms}" - parse: (xhr) -> - suggestion.phrase for suggestion in JSON.parse xhr.responseText - -# A dummy search engine which is guaranteed to match any search URL, but never produces completions. This -# allows the rest of the logic to be written knowing that there will be a search engine match. -class DummySearchEngine - match: -> true - # We return a useless URL which we know will succeed, but which won't generate any network traffic. - getUrl: -> chrome.runtime.getURL "content_scripts/vimium.css" - parse: -> [] - -completionEngines = [ - Youtube - Google - DuckDuckGo - Wikipedia - Bing - Amazon - DummySearchEngine -] - -SearchEngines = - debug: true - - get: (searchUrl, url, callback) -> - xhr = new XMLHttpRequest() - xhr.open "GET", url, true - xhr.timeout = 1000 - xhr.ontimeout = xhr.onerror = -> callback null - xhr.send() - - xhr.onreadystatechange = -> - if xhr.readyState == 4 - callback(if xhr.status == 200 then xhr else null) - - # Look up the search-completion engine for this searchUrl. Because of DummySearchEngine, above, we know - # there will always be a match. Imagining that there may be many completion engines, and knowing that this - # is called for every input event in the vomnibar, we cache the result. - lookupEngine: (searchUrl) -> - @engineCache ?= new SimpleCache 30 * 60 * 60 * 1000 # 30 hours (these are small, we can keep them longer). - if @engineCache.has searchUrl - @engineCache.get searchUrl - else - for engine in completionEngines - engine = new engine() - return @engineCache.set searchUrl, engine if engine.match searchUrl - - # This is the main (actually, the only) entry point. - # - searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"), or a custome search engine's URL. - # This is only used as a key for determining the relevant completion engine. - # - queryTerms are the queryTerms. - # - callback will be applied to a list of suggestion strings (which may be an empty list, if anything goes - # wrong). - complete: (searchUrl, queryTerms, callback) -> - @mostRecentHandler = null - - # We can't complete empty queries. - return callback [] unless 0 < queryTerms.length - - if 1 == queryTerms.length - # We don't complete URLs. - return callback [] if Utils.isUrl queryTerms[0] - # We don't complete less then three characters: the results are usually useless. This also prevents - # one- and two-character custom search engine keywords from being sent to the default completer (e.g. - # the initial "w" before typing "w something"). - return callback [] unless 2 < queryTerms[0].length - - # We don't complete Javascript URLs. - return callback [] if Utils.hasJavascriptPrefix queryTerms[0] - - # Cache completions. However, completions depend upon both the searchUrl and the query terms. So we need - # to generate a key. We mix in some junk generated by pwgen. A key clash is possible, but vanishingly - # unlikely. - junk = "//Zi?ei5;o//" - completionCacheKey = searchUrl + junk + queryTerms.join junk - @completionCache ?= new SimpleCache 60 * 60 * 1000, 2000 # One hour, 2000 entries. - if @completionCache.has completionCacheKey - console.log "hit", completionCacheKey if @debug - return callback @completionCache.get completionCacheKey - - fetchSuggestions = (callback) => - engine = @lookupEngine searchUrl - url = engine.getUrl queryTerms - console.log "get", url if @debug - query = queryTerms.join(" ").toLowerCase() - @get searchUrl, url, (xhr = null) => - # Parsing the response may fail if we receive an unexpected or an unexpectedly-formatted response. In - # all cases, we fall back to the catch clause, below. - try - suggestions = engine.parse xhr - # Make sure we really do have an iterable of strings. - suggestions = (suggestion for suggestion in suggestions when "string" == typeof suggestion) - # Filter out the query itself. It's not adding anything. - suggestions = (suggestion for suggestion in suggestions when suggestion.toLowerCase() != query) - catch - suggestions = [] - # We cache failures, but remove them after just ten minutes. This (it is hoped) avoids repeated - # XMLHttpRequest failures over a short period of time. - removeCompletionCacheKey = => @completionCache.set completionCacheKey, null - setTimeout removeCompletionCacheKey, 10 * 60 * 1000 # Ten minutes. - - callback suggestions - - # We pause in case the user is still typing. - Utils.setTimeout 200, handler = @mostRecentHandler = => - if handler != @mostRecentHandler # Bail if another completion has begun, or the user is typing. - console.log "bail", completionCacheKey if @debug - return callback [] - # Don't allow duplicate identical active requests. This can happen, for example, when the user enters or - # removes a space, or when they enter a character and immediately delete it. - @inTransit ?= {} - unless @inTransit[completionCacheKey]?.push callback - queue = @inTransit[completionCacheKey] = [] - fetchSuggestions (suggestions) => - callback @completionCache.set completionCacheKey, suggestions - delete @inTransit[completionCacheKey] - console.log "callbacks", queue.length, completionCacheKey if @debug and 0 < queue.length - callback suggestions for callback in queue - - userIsTyping: -> - console.log "reset (typing)" if @debug and @mostRecentHandler? - @mostRecentHandler = null - -root = exports ? window -root.SearchEngines = SearchEngines -- cgit v1.2.3 From 7d11b1699454366bf99e8e5033ba39b127687fcb Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Wed, 6 May 2015 04:55:22 +0100 Subject: Search completion; simplify messaging code. This eliminates the need to repeatedly install and remove listeners for @filterPort in the vomnibar. It also eleiminates the need for "keepAlive" in reponses. All as suggested by @mrmr1993 in #1635. --- background_scripts/completion.coffee | 4 ++-- background_scripts/main.coffee | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 729e86ab..c91825b5 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -478,12 +478,12 @@ class MultiCompleter # (ie. a SearchEngineCompleter). This prevents hiding the vomnibar briefly before showing it # again, which looks ugly. unless shouldRunContinuation and suggestions.length == 0 - onComplete @prepareSuggestions(queryTerms, suggestions), keepAlive: shouldRunContinuation + onComplete @prepareSuggestions queryTerms, suggestions # Allow subsequent queries to begin. @filterInProgress = false if shouldRunContinuation continuation suggestions, (newSuggestions) => - onComplete @prepareSuggestions queryTerms, suggestions.concat(newSuggestions) + onComplete @prepareSuggestions queryTerms, suggestions.concat newSuggestions else @filter @mostRecentQuery.queryTerms, @mostRecentQuery.onComplete if @mostRecentQuery diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index 44644769..09a6b89f 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -226,8 +226,8 @@ filterCompleter = (args, port) -> if args.id? and args.name? and args.query? queryTerms = if (args.query == "") then [] else args.query.split(whitespaceRegexp) - completers[args.name].filter queryTerms, (results, extra = {}) -> - port.postMessage extend extra, id: args.id, results: results + completers[args.name].filter queryTerms, (results) -> + port.postMessage id: args.id, results: results chrome.tabs.onSelectionChanged.addListener (tabId, selectionInfo) -> if (selectionChangedHandlers.length > 0) -- cgit v1.2.3 From 1f97221aef5cfe28200df81a68a139a3f2b07784 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Wed, 6 May 2015 05:18:59 +0100 Subject: Search completion; move all filter messages to a single port. --- background_scripts/completion.coffee | 2 +- background_scripts/main.coffee | 26 ++++++++++++-------------- 2 files changed, 13 insertions(+), 15 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index c91825b5..8a69b645 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -331,7 +331,7 @@ class SearchEngineCompleter searchEngines: {} userIsTyping: -> - SearchEngines.userIsTyping() + CompletionEngines.userIsTyping() filter: (queryTerms, onComplete) -> { keyword: keyword, url: url, description: description } = @getSearchEngineMatches queryTerms diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index 09a6b89f..066e4cb6 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -60,6 +60,17 @@ completers = bookmarks: new MultiCompleter([completionSources.bookmarks]) tabs: new MultiCompleter([completionSources.tabs]) +completionHandlers = + filter: (completer, args, port) -> + queryTerms = args.query.split(/\s+/).filter (s) -> 0 < s.length + completer.filter queryTerms, (results) -> port.postMessage id: args.id, results: results + + refreshCompleter: (completer) -> completer.refresh() + userIsTyping: (completer) -> completer.userIsTyping() + +handleCompletions = (args, port) -> + completionHandlers[args.handler] completers[args.name], args, port + chrome.runtime.onConnect.addListener (port, name) -> senderTabId = if port.sender.tab then port.sender.tab.id else null # If this is a tab we've been waiting to open, execute any "tab loaded" handlers, e.g. to restore @@ -217,18 +228,6 @@ handleSettings = (request, port) -> values[key] = Settings.get key for own key of values port.postMessage { values } -refreshCompleter = (request) -> completers[request.name].refresh() - -whitespaceRegexp = /\s+/ -filterCompleter = (args, port) -> - if args.name? and args.userIsTyping - completers[args.name].userIsTyping?() - - if args.id? and args.name? and args.query? - queryTerms = if (args.query == "") then [] else args.query.split(whitespaceRegexp) - completers[args.name].filter queryTerms, (results) -> - port.postMessage id: args.id, results: results - chrome.tabs.onSelectionChanged.addListener (tabId, selectionInfo) -> if (selectionChangedHandlers.length > 0) selectionChangedHandlers.pop().call() @@ -643,7 +642,7 @@ bgLog = (request, sender) -> portHandlers = keyDown: handleKeyDown, settings: handleSettings, - filterCompleter: filterCompleter + completions: handleCompletions sendRequestHandlers = getCompletionKeys: getCompletionKeysRequest @@ -661,7 +660,6 @@ sendRequestHandlers = pasteFromClipboard: pasteFromClipboard isEnabledForUrl: isEnabledForUrl selectSpecificTab: selectSpecificTab - refreshCompleter: refreshCompleter createMark: Marks.create.bind(Marks) gotoMark: Marks.goto.bind(Marks) setIcon: setIcon -- cgit v1.2.3 From 372e65d7b44c3f0ad3f522bd6b8c9cfacd693186 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Wed, 6 May 2015 07:32:19 +0100 Subject: Search completion; many tweaks and refactor Suggestion constructor. --- background_scripts/completion.coffee | 166 +++++++++++++++++++---------------- background_scripts/settings.coffee | 4 +- 2 files changed, 94 insertions(+), 76 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 8a69b645..4ae0c44b 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -5,31 +5,38 @@ # The Vomnibox frontend script makes a "filterCompleter" request to the background page, which in turn calls # filter() on each these completers. # -# A completer is a class which has two functions: +# A completer is a class which has three functions: # - filter(query, onComplete): "query" will be whatever the user typed into the Vomnibox. # - refresh(): (optional) refreshes the completer's data source (e.g. refetches the list of bookmarks). - -# A Suggestion is a bookmark or history entry which matches the current query. -# It also has an attached "computeRelevancyFunction" which determines how well this item matches the given -# query terms. +# - userIsTyping(): (optional) informs the completer that the user is typing (and pending completions may no +# longer be needed). class Suggestion showRelevancy: true # Set this to true to render relevancy when debugging the ranking scores. - # - type: one of [bookmark, history, tab]. - # - computeRelevancyFunction: a function which takes a Suggestion and returns a relevancy score - # between [0, 1] - # - extraRelevancyData: data (like the History item itself) which may be used by the relevancy function. - constructor: (@queryTerms, @type, @url, @title, @computeRelevancyFunction, @extraRelevancyData) -> - @title ||= "" + constructor: (@options) -> + # Required options. + @queryTerms = null + @type = null + @url = null + @relevancyFunction = null + # Other options. + @title = "" + # Extra data which will be available to the relevancy function. + @relevancyData = null # When @autoSelect is truthy, the suggestion is automatically pre-selected in the vomnibar. @autoSelect = false - # If @noHighlightTerms is falsy, then we don't highlight matched terms in the title and URL. - @noHighlightTerms = false + # If @highlightTerms is true, then we highlight matched terms in the title and URL. + @highlightTerms = true # If @insertText is a string, then the indicated text is inserted into the vomnibar input when the # suggestion is selected. @insertText = null - computeRelevancy: -> @relevancy = @computeRelevancyFunction(this) + extend this, @options + + computeRelevancy: -> + # We assume that, once the relevancy has been set, it won't change. Completers must set either @relevancy + # or @relevancyFunction. + @relevancy ?= @relevancyFunction this generateHtml: -> return @html if @html @@ -39,10 +46,10 @@ class Suggestion """
#{@type} - #{@highlightTerms Utils.escapeHtml @title} + #{@highlightQueryTerms Utils.escapeHtml @title}
- #{@shortenUrl @highlightTerms Utils.escapeHtml @url} + #{@shortenUrl @highlightQueryTerms Utils.escapeHtml @url} #{relevancyHtml}
""" @@ -82,8 +89,8 @@ class Suggestion textPosition += matchedText.length # Wraps each occurence of the query terms in the given string in a . - highlightTerms: (string) -> - return string if @noHighlightTerms + highlightQueryTerms: (string) -> + return string unless @highlightTerms ranges = [] escapedTerms = @queryTerms.map (term) -> Utils.escapeHtml(term) for term in escapedTerms @@ -139,11 +146,15 @@ class BookmarkCompleter else [] suggestions = results.map (bookmark) => - suggestionTitle = if usePathAndTitle then bookmark.pathAndTitle else bookmark.title - new Suggestion(@currentSearch.queryTerms, "bookmark", bookmark.url, suggestionTitle, @computeRelevancy) + new Suggestion + queryTerms: @currentSearch.queryTerms + type: "bookmark" + url: bookmark.url + title: if usePathAndTitle then bookmark.pathAndTitle else bookmark.title + relevancyFunction: @computeRelevancy onComplete = @currentSearch.onComplete @currentSearch = null - onComplete(suggestions) + onComplete suggestions refresh: -> @bookmarks = null @@ -188,17 +199,21 @@ class HistoryCompleter else [] suggestions = results.map (entry) => - new Suggestion(queryTerms, "history", entry.url, entry.title, @computeRelevancy, entry) - onComplete(suggestions) + new Suggestion + queryTerms: queryTerms + type: "history" + url: entry.url + title: entry.title + relevancyFunction: @computeRelevancy + relevancyData: entry + onComplete suggestions computeRelevancy: (suggestion) -> - historyEntry = suggestion.extraRelevancyData + historyEntry = suggestion.relevancyData recencyScore = RankingUtils.recencyScore(historyEntry.lastVisitTime) wordRelevancy = RankingUtils.wordRelevancy(suggestion.queryTerms, suggestion.url, suggestion.title) # Average out the word score and the recency. Recency has the ability to pull the score up, but not down. - score = (wordRelevancy + Math.max(recencyScore, wordRelevancy)) / 2 - - refresh: -> + (wordRelevancy + Math.max recencyScore, wordRelevancy) / 2 # The domain completer is designed to match a single-word query which looks like it is a domain. This supports # the user experience where they quickly type a partial domain, hit tab -> enter, and expect to arrive there. @@ -222,16 +237,21 @@ class DomainCompleter domains = @sortDomainsByRelevancy(queryTerms, domainCandidates) return onComplete([]) if domains.length == 0 topDomain = domains[0][0] - onComplete([new Suggestion(queryTerms, "domain", topDomain, null, @computeRelevancy)]) + suggestion = new Suggestion + queryTerms: queryTerms + type: "domain" + url: topDomain + relevancy: 1 + onComplete [ suggestion ] # Returns a list of domains of the form: [ [domain, relevancy], ... ] sortDomainsByRelevancy: (queryTerms, domainCandidates) -> - results = [] - for domain in domainCandidates - recencyScore = RankingUtils.recencyScore(@domains[domain].entry.lastVisitTime || 0) - wordRelevancy = RankingUtils.wordRelevancy(queryTerms, domain, null) - score = (wordRelevancy + Math.max(recencyScore, wordRelevancy)) / 2 - results.push([domain, score]) + results = + for domain in domainCandidates + recencyScore = RankingUtils.recencyScore(@domains[domain].entry.lastVisitTime || 0) + wordRelevancy = RankingUtils.wordRelevancy queryTerms, domain, null + score = (wordRelevancy + Math.max(recencyScore, wordRelevancy)) / 2 + [domain, score] results.sort (a, b) -> b[1] - a[1] results @@ -264,9 +284,6 @@ class DomainCompleter parseDomainAndScheme: (url) -> Utils.hasFullUrlPrefix(url) and not Utils.hasChromePrefix(url) and url.split("/",3).join "/" - # Suggestions from the Domain completer have the maximum relevancy. They should be shown first in the list. - computeRelevancy: -> 1 - # TabRecency associates a logical timestamp with each tab id. These are used to provide an initial # recency-based ordering in the tabs vomnibar (which allows jumping quickly between recently-visited tabs). class TabRecency @@ -316,10 +333,14 @@ class TabCompleter chrome.tabs.query {}, (tabs) => results = tabs.filter (tab) -> RankingUtils.matches(queryTerms, tab.url, tab.title) suggestions = results.map (tab) => - suggestion = new Suggestion(queryTerms, "tab", tab.url, tab.title, @computeRelevancy) - suggestion.tabId = tab.id - suggestion - onComplete(suggestions) + new Suggestion + queryTerms: queryTerms + type: "tab" + url: tab.url + title: tab.title + relevancyFunction: @computeRelevancy + tabId: tab.id + onComplete suggestions computeRelevancy: (suggestion) -> if suggestion.queryTerms.length @@ -334,27 +355,26 @@ class SearchEngineCompleter CompletionEngines.userIsTyping() filter: (queryTerms, onComplete) -> - { keyword: keyword, url: url, description: description } = @getSearchEngineMatches queryTerms - custom = url? suggestions = [] - mkUrl = - if custom - (string) -> url.replace /%s/g, Utils.createSearchQuery string.split /\s+/ - else - (string) -> Utils.createSearchUrl string.split /\s+/ - - haveDescription = description? and 0 < description.trim().length - type = if haveDescription then description else "search" - searchUrl = if custom then url else Settings.get "searchUrl" + { keyword, searchUrl, description } = @getSearchEngineMatches queryTerms + custom = searchUrl? and keyword? + searchUrl ?= Settings.get("searchUrl") + "%s" + haveDescription = description? and 0 < description.length + description ||= "#{if custom then "custom " else ""}search" # For custom search engines, we add an auto-selected suggestion. if custom - query = queryTerms[1..].join " " - title = if haveDescription then query else keyword + ": " + query - suggestions.push @mkSuggestion null, queryTerms, type, mkUrl(query), title, @computeRelevancy, 1 - suggestions[0].autoSelect = true queryTerms = queryTerms[1..] + query = queryTerms.join " " + suggestions.push new Suggestion + queryTerms: queryTerms + type: description + url: searchUrl.replace /%s/g, Utils.createSearchQuery query.split /\s+/ + title: if haveDescription then query else "#{keyword}: #{query}" + relevancy: 1 + autoSelect: true + highlightTerms: false else query = queryTerms.join " " @@ -366,30 +386,36 @@ class SearchEngineCompleter # For custom search-engine queries, this adds suggestions only if we have a completer. For other queries, # this adds suggestions for the default search engine (if we have a completer for that). - # Scoring: - # - The score does not depend upon the actual suggestion (so, it does not depend upon word + # Relevancy: + # - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word # relevancy). We assume that the completion engine has already factored that in. Also, completion # engines often handle spelling mistakes, in which case we wouldn't find the query terms in the # suggestion anyway. - # - The score is higher if the query term is longer. The idea is that search suggestions are more + # - The relavancy is higher if the query term is longer. The idea is that search suggestions are more # likely to be relevant if, after typing some number of characters, the user hasn't yet found # a useful suggestion from another completer. # - Scores are weighted such that they retain the order provided by the completion engine. characterCount = query.length - queryTerms.length + 1 - score = 0.6 * (Math.min(characterCount, 10.0)/10.0) + relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) if 0 < existingSuggestions.length existingSuggestionMinScore = existingSuggestions[existingSuggestions.length-1].relevancy - if score < existingSuggestionMinScore and MultiCompleter.maxResults <= existingSuggestions.length - # No suggestion we propose will have a high enough score to beat the existing suggestions, so bail + if relavancy < existingSuggestionMinScore and MultiCompleter.maxResults <= existingSuggestions.length + # No suggestion we propose will have a high enough relavancy to beat the existing suggestions, so bail # immediately. return onComplete [] CompletionEngines.complete searchUrl, queryTerms, (searchSuggestions = []) => for suggestion in searchSuggestions - insertText = if custom then "#{keyword} #{suggestion}" else suggestion - suggestions.push @mkSuggestion insertText, queryTerms, type, mkUrl(suggestion), suggestion, @computeRelevancy, score - score *= 0.9 + suggestions.push new Suggestion + queryTerms: queryTerms + type: description + url: searchUrl.replace /%s/g, Utils.createSearchQuery suggestion.split /\s+/ + title: suggestion + relevancy: relavancy + insertText: if custom then "#{keyword} #{suggestion}" else suggestion + highlightTerms: false + relavancy *= 0.9 # We keep at least three suggestions (if possible) and at most six. We keep more than three only if # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions @@ -398,14 +424,6 @@ class SearchEngineCompleter count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length onComplete suggestions[...count] - # FIXME(smblott) Refactor Suggestion constructor as per @mrmr1993's comment in #1635. - mkSuggestion: (insertText, args...) -> - suggestion = new Suggestion args... - extend suggestion, insertText: insertText, noHighlightTerms: true - - # The score is computed in filter() and provided here via suggestion.extraRelevancyData. - computeRelevancy: (suggestion) -> suggestion.extraRelevancyData - refresh: -> @searchEngines = SearchEngineCompleter.getSearchEngines() @@ -426,7 +444,7 @@ class SearchEngineCompleter continue unless keywords.length == 2 and not keywords[1] # So, like: [ "w", "" ]. searchEnginesMap[keywords[0]] = keyword: keywords[0] - url: tokens[1] + searchUrl: tokens[1] description: tokens[2..].join(" ") # Fetch the search-engine map, building it if necessary. diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index b802937e..ce812970 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -98,11 +98,11 @@ root.Settings = Settings = "g: http://www.google.com/search?q=%s Google" "l: http://www.google.com/search?q=%s&btnI I'm feeling lucky..." "w: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedia" - "t: http://www.youtube.com/results?search_query=%s Youtube" + "y: http://www.youtube.com/results?search_query=%s Youtube" + "t: http://www.youtube.com/results?search_query=%s" "m: https://www.google.com/maps/search/%s Google Maps" "b: https://www.bing.com/search?q=%s Bing" "d: https://duckduckgo.com/?q=%s DuckDuckGo" - "y: http://www.youtube.com/results?search_query=%s Youtube" "az: http://www.amazon.com/s/?field-keywords=%s Amazon" ].join "\n\n" newTabUrl: "chrome://newtab" -- cgit v1.2.3 From 5752c0ead0a65fc2329515509f66e00bd6ee2f60 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Wed, 6 May 2015 07:40:46 +0100 Subject: Search completion; more tweaks. --- background_scripts/completion.coffee | 37 ++++++++++++++-------------- background_scripts/completion_engines.coffee | 26 +++++++++++++------ 2 files changed, 36 insertions(+), 27 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 4ae0c44b..bffb9700 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -363,23 +363,22 @@ class SearchEngineCompleter haveDescription = description? and 0 < description.length description ||= "#{if custom then "custom " else ""}search" + queryTerms = queryTerms[1..] if custom + query = queryTerms.join " " + + if queryTerms.length == 0 + return onComplete suggestions + # For custom search engines, we add an auto-selected suggestion. if custom - queryTerms = queryTerms[1..] - query = queryTerms.join " " suggestions.push new Suggestion queryTerms: queryTerms type: description - url: searchUrl.replace /%s/g, Utils.createSearchQuery query.split /\s+/ - title: if haveDescription then query else "#{keyword}: #{query}" + url: Utils.createSearchUrl queryTerms, searchUrl + title: if haveDescription then query else "#{keyword}: #{query}" relevancy: 1 - autoSelect: true highlightTerms: false - else - query = queryTerms.join " " - - if queryTerms.length == 0 - return onComplete suggestions + autoSelect: true onComplete suggestions, (existingSuggestions, onComplete) => suggestions = [] @@ -399,23 +398,22 @@ class SearchEngineCompleter relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) if 0 < existingSuggestions.length - existingSuggestionMinScore = existingSuggestions[existingSuggestions.length-1].relevancy - if relavancy < existingSuggestionMinScore and MultiCompleter.maxResults <= existingSuggestions.length + existingSuggestionsMinScore = existingSuggestions[existingSuggestions.length-1].relevancy + if relavancy < existingSuggestionsMinScore and MultiCompleter.maxResults <= existingSuggestions.length # No suggestion we propose will have a high enough relavancy to beat the existing suggestions, so bail # immediately. return onComplete [] - CompletionEngines.complete searchUrl, queryTerms, (searchSuggestions = []) => - for suggestion in searchSuggestions + CompletionEngines.complete searchUrl, queryTerms, (completionSuggestions = []) => + for suggestion in completionSuggestions suggestions.push new Suggestion queryTerms: queryTerms type: description - url: searchUrl.replace /%s/g, Utils.createSearchQuery suggestion.split /\s+/ + url: Utils.createSearchUrl suggestion, searchUrl title: suggestion - relevancy: relavancy - insertText: if custom then "#{keyword} #{suggestion}" else suggestion + relevancy: relavancy *= 0.9 highlightTerms: false - relavancy *= 0.9 + insertText: if custom then "#{keyword} #{suggestion}" else suggestion # We keep at least three suggestions (if possible) and at most six. We keep more than three only if # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions @@ -501,7 +499,8 @@ class MultiCompleter @filterInProgress = false if shouldRunContinuation continuation suggestions, (newSuggestions) => - onComplete @prepareSuggestions queryTerms, suggestions.concat newSuggestions + if 0 < newSuggestions.length + onComplete @prepareSuggestions queryTerms, suggestions.concat newSuggestions else @filter @mostRecentQuery.queryTerms, @mostRecentQuery.onComplete if @mostRecentQuery diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 0177806a..1386256f 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -116,6 +116,9 @@ completionEngines = [ CompletionEngines = debug: true + # The amount of time to wait for new completions before launching the HTTP request. + delay: 250 + get: (searchUrl, url, callback) -> xhr = new XMLHttpRequest() xhr.open "GET", url, true @@ -139,7 +142,7 @@ CompletionEngines = engine = new engine() return @engineCache.set searchUrl, engine if engine.match searchUrl - # This is the main (actually, the only) entry point. + # This is the main entry point. # - searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"), or a custome search engine's URL. # This is only used as a key for determining the relevant completion engine. # - queryTerms are the queryTerms. @@ -169,34 +172,41 @@ CompletionEngines = completionCacheKey = searchUrl + junk + queryTerms.join junk @completionCache ?= new SimpleCache 60 * 60 * 1000, 2000 # One hour, 2000 entries. if @completionCache.has completionCacheKey - console.log "hit", completionCacheKey if @debug - return callback @completionCache.get completionCacheKey + # We add a short delay, even for a cache hit. This avoids an ugly flicker when the additional + # suggestions are posted. It also makes the vomnibar behave similarly regardless of whether there's a + # cache hit. + Utils.setTimeout @delay, => + console.log "hit", completionCacheKey if @debug + callback @completionCache.get completionCacheKey + return fetchSuggestions = (callback) => engine = @lookupEngine searchUrl url = engine.getUrl queryTerms - console.log "get", url if @debug query = queryTerms.join(" ").toLowerCase() @get searchUrl, url, (xhr = null) => # Parsing the response may fail if we receive an unexpected or an unexpectedly-formatted response. In - # all cases, we fall back to the catch clause, below. + # all cases, we fall back to the catch clause, below. Therefore, we "fail safe" in the case of + # incorrect or out-of-date completion engines. try suggestions = engine.parse xhr # Make sure we really do have an iterable of strings. suggestions = (suggestion for suggestion in suggestions when "string" == typeof suggestion) # Filter out the query itself. It's not adding anything. suggestions = (suggestion for suggestion in suggestions when suggestion.toLowerCase() != query) + console.log "GET", url if @debug catch suggestions = [] - # We cache failures, but remove them after just ten minutes. This (it is hoped) avoids repeated - # XMLHttpRequest failures over a short period of time. + # We allow failures to be cached, but remove them after just ten minutes. This (it is hoped) avoids + # repeated unnecessary XMLHttpRequest failures over a short period of time. removeCompletionCacheKey = => @completionCache.set completionCacheKey, null setTimeout removeCompletionCacheKey, 10 * 60 * 1000 # Ten minutes. + console.log "fail", url if @debug callback suggestions # We pause in case the user is still typing. - Utils.setTimeout 200, handler = @mostRecentHandler = => + Utils.setTimeout @delay, handler = @mostRecentHandler = => if handler != @mostRecentHandler # Bail if another completion has begun, or the user is typing. console.log "bail", completionCacheKey if @debug return callback [] -- cgit v1.2.3 From f330e3243bc27f1a19040fb386fc877fe82fbefe Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Wed, 6 May 2015 11:50:28 +0100 Subject: Search completion; yet more tweaks. --- background_scripts/completion.coffee | 11 +++-- background_scripts/completion_engines.coffee | 63 +++++++++++++++------------- background_scripts/main.coffee | 4 +- 3 files changed, 41 insertions(+), 37 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index bffb9700..18bfafd1 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -8,8 +8,7 @@ # A completer is a class which has three functions: # - filter(query, onComplete): "query" will be whatever the user typed into the Vomnibox. # - refresh(): (optional) refreshes the completer's data source (e.g. refetches the list of bookmarks). -# - userIsTyping(): (optional) informs the completer that the user is typing (and pending completions may no -# longer be needed). +# - cancel(): (optional) cancels any pending, cancelable action. class Suggestion showRelevancy: true # Set this to true to render relevancy when debugging the ranking scores. @@ -351,8 +350,8 @@ class TabCompleter class SearchEngineCompleter searchEngines: {} - userIsTyping: -> - CompletionEngines.userIsTyping() + cancel: -> + CompletionEngines.cancel() filter: (queryTerms, onComplete) -> suggestions = [] @@ -462,8 +461,8 @@ class MultiCompleter refresh: -> completer.refresh?() for completer in @completers - userIsTyping: -> - completer.userIsTyping?() for completer in @completers + cancel: -> + completer.cancel?() for completer in @completers filter: (queryTerms, onComplete) -> # Allow only one query to run at a time. diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 1386256f..b5caadd7 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -22,8 +22,9 @@ class RegexpEngine constructor: (@regexps) -> match: (searchUrl) -> Utils.matchesAnyRegexp @regexps, searchUrl -# Several Google completion engines package responses in this way. +# Several Google completion engines package XML responses in this way. class GoogleXMLRegexpEngine extends RegexpEngine + doNotCache: true parse: (xhr) -> for suggestion in xhr.responseXML.getElementsByTagName "suggestion" continue unless suggestion = suggestion.getAttribute "data" @@ -50,6 +51,7 @@ class Youtube extends GoogleXMLRegexpEngine "http://suggestqueries.google.com/complete/search?client=youtube&ds=yt&xml=t&q=#{Utils.createSearchQuery queryTerms}" class Wikipedia extends RegexpEngine + doNotCache: true # Example search URL: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s constructor: -> super [ new RegExp "^https?://[a-z]+\.wikipedia\.org/" ] @@ -96,8 +98,8 @@ class DuckDuckGo extends RegexpEngine suggestion.phrase for suggestion in JSON.parse xhr.responseText # A dummy search engine which is guaranteed to match any search URL, but never produces completions. This -# allows the rest of the logic to be written knowing that there will be a search engine match. -class DummySearchEngine +# allows the rest of the logic to be written knowing that there will always be a completion engine match. +class DummyCompletionEngine match: -> true # We return a useless URL which we know will succeed, but which won't generate any network traffic. getUrl: -> chrome.runtime.getURL "content_scripts/vimium.css" @@ -110,13 +112,19 @@ completionEngines = [ Wikipedia Bing Amazon - DummySearchEngine + DummyCompletionEngine ] +# A note on caching. +# Some completion engines allow caching, and Chrome serves up cached responses to requests (e.g. Google, +# Wikipedia, YouTube). Others do not (e.g. Bing, DuckDuckGo, Amazon). A completion engine can set +# @doNotCache to a truthy value to disable caching in cases where it is unnecessary. + CompletionEngines = debug: true - # The amount of time to wait for new completions before launching the HTTP request. + # The amount of time to wait for new requests before launching the HTTP request. The intention is to cut + # down on the number of HTTP requests we issue. delay: 250 get: (searchUrl, url, callback) -> @@ -128,11 +136,12 @@ CompletionEngines = xhr.onreadystatechange = -> if xhr.readyState == 4 + console.log xhr.getAllResponseHeaders() callback(if xhr.status == 200 then xhr else null) - # Look up the search-completion engine for this searchUrl. Because of DummySearchEngine, above, we know - # there will always be a match. Imagining that there may be many completion engines, and knowing that this - # is called for every input event in the vomnibar, we cache the result. + # Look up the completion engine for this searchUrl. Because of DummyCompletionEngine, above, we know there + # will always be a match. Imagining that there may be many completion engines, and knowing that this is + # called for every input event in the vomnibar, we cache the result. lookupEngine: (searchUrl) -> @engineCache ?= new SimpleCache 30 * 60 * 60 * 1000 # 30 hours (these are small, we can keep them longer). if @engineCache.has searchUrl @@ -145,29 +154,25 @@ CompletionEngines = # This is the main entry point. # - searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"), or a custome search engine's URL. # This is only used as a key for determining the relevant completion engine. - # - queryTerms are the queryTerms. + # - queryTerms are the query terms. # - callback will be applied to a list of suggestion strings (which may be an empty list, if anything goes # wrong). complete: (searchUrl, queryTerms, callback) -> @mostRecentHandler = null + query = queryTerms.join "" - # We can't complete empty queries. - return callback [] unless 0 < queryTerms.length - - if 1 == queryTerms.length - # We don't complete URLs. - return callback [] if Utils.isUrl queryTerms[0] - # We don't complete less then three characters: the results are usually useless. This also prevents - # one- and two-character custom search engine keywords from being sent to the default completer (e.g. - # the initial "w" before typing "w something"). - return callback [] unless 2 < queryTerms[0].length + # We don't complete less then three characters: the results are usually useless. This also prevents + # one- and two-character custom search engine keywords from being sent to the default completer (e.g. + # the initial "w" before typing "w something" for Wikipedia). + return callback [] unless 3 <= query.length - # We don't complete Javascript URLs. - return callback [] if Utils.hasJavascriptPrefix queryTerms[0] + # We don't complete regular URLs or Javascript URLs. + return callback [] if 1 == queryTerms.length and Utils.isUrl query + return callback [] if Utils.hasJavascriptPrefix query # Cache completions. However, completions depend upon both the searchUrl and the query terms. So we need - # to generate a key. We mix in some junk generated by pwgen. A key clash is possible, but vanishingly - # unlikely. + # to generate a key. We mix in some junk generated by pwgen. A key clash might be possible, but + # vanishingly unlikely. junk = "//Zi?ei5;o//" completionCacheKey = searchUrl + junk + queryTerms.join junk @completionCache ?= new SimpleCache 60 * 60 * 1000, 2000 # One hour, 2000 entries. @@ -180,8 +185,7 @@ CompletionEngines = callback @completionCache.get completionCacheKey return - fetchSuggestions = (callback) => - engine = @lookupEngine searchUrl + fetchSuggestions = (engine, callback) => url = engine.getUrl queryTerms query = queryTerms.join(" ").toLowerCase() @get searchUrl, url, (xhr = null) => @@ -215,14 +219,15 @@ CompletionEngines = @inTransit ?= {} unless @inTransit[completionCacheKey]?.push callback queue = @inTransit[completionCacheKey] = [] - fetchSuggestions (suggestions) => - callback @completionCache.set completionCacheKey, suggestions + engine = @lookupEngine searchUrl + fetchSuggestions engine, (suggestions) => + callback @completionCache.set completionCacheKey, suggestions unless engine.doNotCache delete @inTransit[completionCacheKey] console.log "callbacks", queue.length, completionCacheKey if @debug and 0 < queue.length callback suggestions for callback in queue - userIsTyping: -> - console.log "reset (typing)" if @debug and @mostRecentHandler? + cancel: -> + console.log "cancel (user is typing)" if @debug and @mostRecentHandler? @mostRecentHandler = null root = exports ? window diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index 066e4cb6..c1be9f8f 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -65,8 +65,8 @@ completionHandlers = queryTerms = args.query.split(/\s+/).filter (s) -> 0 < s.length completer.filter queryTerms, (results) -> port.postMessage id: args.id, results: results - refreshCompleter: (completer) -> completer.refresh() - userIsTyping: (completer) -> completer.userIsTyping() + refresh: (completer) -> completer.refresh() + cancel: (completer) -> completer.cancel() handleCompletions = (args, port) -> completionHandlers[args.handler] completers[args.name], args, port -- cgit v1.2.3 From c7fb211f0d1a6504bd0cbb94395f9437f858f2c0 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Wed, 6 May 2015 11:56:16 +0100 Subject: Search completion; fix yet more tweaks. --- background_scripts/completion_engines.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index b5caadd7..8a880930 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -221,7 +221,8 @@ CompletionEngines = queue = @inTransit[completionCacheKey] = [] engine = @lookupEngine searchUrl fetchSuggestions engine, (suggestions) => - callback @completionCache.set completionCacheKey, suggestions unless engine.doNotCache + @completionCache.set completionCacheKey, suggestions unless engine.doNotCache + callback suggestions delete @inTransit[completionCacheKey] console.log "callbacks", queue.length, completionCacheKey if @debug and 0 < queue.length callback suggestions for callback in queue -- cgit v1.2.3 From 430bdc8bdf7c109a3007104fc06abeeed1891529 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Thu, 7 May 2015 09:25:15 +0100 Subject: Search completion; tweak domain completer. --- background_scripts/completion.coffee | 25 +++++++++++-------------- background_scripts/completion_engines.coffee | 7 ++++--- 2 files changed, 15 insertions(+), 17 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 18bfafd1..746e662d 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -197,7 +197,7 @@ class HistoryCompleter history.filter (entry) -> RankingUtils.matches(queryTerms, entry.url, entry.title) else [] - suggestions = results.map (entry) => + onComplete results.map (entry) => new Suggestion queryTerms: queryTerms type: "history" @@ -205,7 +205,6 @@ class HistoryCompleter title: entry.title relevancyFunction: @computeRelevancy relevancyData: entry - onComplete suggestions computeRelevancy: (suggestion) -> historyEntry = suggestion.relevancyData @@ -232,16 +231,15 @@ class DomainCompleter performSearch: (queryTerms, onComplete) -> query = queryTerms[0] - domainCandidates = (domain for domain of @domains when domain.indexOf(query) >= 0) - domains = @sortDomainsByRelevancy(queryTerms, domainCandidates) - return onComplete([]) if domains.length == 0 - topDomain = domains[0][0] - suggestion = new Suggestion - queryTerms: queryTerms - type: "domain" - url: topDomain - relevancy: 1 - onComplete [ suggestion ] + domains = (domain for domain of @domains when 0 <= domain.indexOf query) + domains = @sortDomainsByRelevancy queryTerms, domains + onComplete [ + new Suggestion + queryTerms: queryTerms + type: "domain" + url: domains[0]?[0] ? "" # This is the URL or an empty string, but not null. + relevancy: 1 + ].filter (s) -> 0 < s.url.length # Returns a list of domains of the form: [ [domain, relevancy], ... ] sortDomainsByRelevancy: (queryTerms, domainCandidates) -> @@ -656,8 +654,7 @@ HistoryCache = @callbacks = null use: (callback) -> - return @fetchHistory(callback) unless @history? - callback(@history) + if @history? then callback @history else @fetchHistory callback fetchHistory: (callback) -> return @callbacks.push(callback) if @callbacks diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 8a880930..3e762e32 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -136,7 +136,6 @@ CompletionEngines = xhr.onreadystatechange = -> if xhr.readyState == 4 - console.log xhr.getAllResponseHeaders() callback(if xhr.status == 200 then xhr else null) # Look up the completion engine for this searchUrl. Because of DummyCompletionEngine, above, we know there @@ -214,6 +213,7 @@ CompletionEngines = if handler != @mostRecentHandler # Bail if another completion has begun, or the user is typing. console.log "bail", completionCacheKey if @debug return callback [] + @mostRecentHandler = null # Don't allow duplicate identical active requests. This can happen, for example, when the user enters or # removes a space, or when they enter a character and immediately delete it. @inTransit ?= {} @@ -228,8 +228,9 @@ CompletionEngines = callback suggestions for callback in queue cancel: -> - console.log "cancel (user is typing)" if @debug and @mostRecentHandler? - @mostRecentHandler = null + if @mostRecentHandler? + @mostRecentHandler = null + console.log "cancel (user is typing)" if @debug root = exports ? window root.CompletionEngines = CompletionEngines -- cgit v1.2.3 From 723b66e24f9b3f4e8dd9843e21352eeceb7c96a9 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 8 May 2015 06:17:19 +0100 Subject: Search completion; tweak timeouts and caching. --- background_scripts/completion_engines.coffee | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 3e762e32..19a18ecd 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -24,7 +24,7 @@ class RegexpEngine # Several Google completion engines package XML responses in this way. class GoogleXMLRegexpEngine extends RegexpEngine - doNotCache: true + doNotCache: false # true (disbaled, experimental) parse: (xhr) -> for suggestion in xhr.responseXML.getElementsByTagName "suggestion" continue unless suggestion = suggestion.getAttribute "data" @@ -51,7 +51,7 @@ class Youtube extends GoogleXMLRegexpEngine "http://suggestqueries.google.com/complete/search?client=youtube&ds=yt&xml=t&q=#{Utils.createSearchQuery queryTerms}" class Wikipedia extends RegexpEngine - doNotCache: true + doNotCache: false # true (disbaled, experimental) # Example search URL: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s constructor: -> super [ new RegExp "^https?://[a-z]+\.wikipedia\.org/" ] @@ -125,7 +125,7 @@ CompletionEngines = # The amount of time to wait for new requests before launching the HTTP request. The intention is to cut # down on the number of HTTP requests we issue. - delay: 250 + delay: 200 get: (searchUrl, url, callback) -> xhr = new XMLHttpRequest() @@ -177,9 +177,8 @@ CompletionEngines = @completionCache ?= new SimpleCache 60 * 60 * 1000, 2000 # One hour, 2000 entries. if @completionCache.has completionCacheKey # We add a short delay, even for a cache hit. This avoids an ugly flicker when the additional - # suggestions are posted. It also makes the vomnibar behave similarly regardless of whether there's a - # cache hit. - Utils.setTimeout @delay, => + # suggestions are posted. + Utils.setTimeout 75, => console.log "hit", completionCacheKey if @debug callback @completionCache.get completionCacheKey return @@ -227,6 +226,7 @@ CompletionEngines = console.log "callbacks", queue.length, completionCacheKey if @debug and 0 < queue.length callback suggestions for callback in queue + # Cancel any pending (ie. blocked on @delay) queries. Does not cancel in-flight queries. cancel: -> if @mostRecentHandler? @mostRecentHandler = null -- cgit v1.2.3 From 908eac76062a45e5c8f686fccc9f91b16d8a23bf Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 8 May 2015 06:56:09 +0100 Subject: Search completion; refactor query terms. --- background_scripts/main.coffee | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index c1be9f8f..a3ddb48c 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -62,8 +62,7 @@ completers = completionHandlers = filter: (completer, args, port) -> - queryTerms = args.query.split(/\s+/).filter (s) -> 0 < s.length - completer.filter queryTerms, (results) -> port.postMessage id: args.id, results: results + completer.filter args.queryTerms, (results) -> port.postMessage id: args.id, results: results refresh: (completer) -> completer.refresh() cancel: (completer) -> completer.cancel() -- cgit v1.2.3 From 2dfcd17ea485484cedf636a94b9c89c527e2e0b7 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 8 May 2015 09:03:56 +0100 Subject: Search completion; add front end cache. --- background_scripts/completion.coffee | 15 ++++++++++----- background_scripts/completion_engines.coffee | 8 ++++++-- background_scripts/main.coffee | 18 +++++++++--------- 3 files changed, 25 insertions(+), 16 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 746e662d..f17ca28c 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -491,22 +491,27 @@ class MultiCompleter # (ie. a SearchEngineCompleter). This prevents hiding the vomnibar briefly before showing it # again, which looks ugly. unless shouldRunContinuation and suggestions.length == 0 - onComplete @prepareSuggestions queryTerms, suggestions + onComplete + results: @prepareSuggestions queryTerms, suggestions + callerMayCacheResults: not shouldRunContinuation # Allow subsequent queries to begin. @filterInProgress = false if shouldRunContinuation continuation suggestions, (newSuggestions) => if 0 < newSuggestions.length - onComplete @prepareSuggestions queryTerms, suggestions.concat newSuggestions + suggestions.push newSuggestions... + onComplete + results: @prepareSuggestions queryTerms, suggestions + callerMayCacheResults: true else @filter @mostRecentQuery.queryTerms, @mostRecentQuery.onComplete if @mostRecentQuery prepareSuggestions: (queryTerms, suggestions) -> suggestion.computeRelevancy queryTerms for suggestion in suggestions suggestions.sort (a, b) -> b.relevancy - a.relevancy - suggestions = suggestions[0...@maxResults] - suggestion.generateHtml() for suggestion in suggestions - suggestions + for suggestion in suggestions[0...@maxResults] + suggestion.generateHtml() + suggestion # Utilities which help us compute a relevancy score for a given item. RankingUtils = diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 19a18ecd..52db90d0 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -209,9 +209,13 @@ CompletionEngines = # We pause in case the user is still typing. Utils.setTimeout @delay, handler = @mostRecentHandler = => - if handler != @mostRecentHandler # Bail if another completion has begun, or the user is typing. + if handler != @mostRecentHandler + # Bail! Another completion has begun, or the user is typing. + # NOTE: We do *not* call the callback (because we are not providing results, and we don't want allow + # any higher-level component to cache the results -- specifically, the vomnibar itself, via + # callerMayCacheResults). console.log "bail", completionCacheKey if @debug - return callback [] + return @mostRecentHandler = null # Don't allow duplicate identical active requests. This can happen, for example, when the user enters or # removes a space, or when they enter a character and immediately delete it. diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index a3ddb48c..1a3281bf 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -43,26 +43,26 @@ chrome.storage.local.set vimiumSecret: Math.floor Math.random() * 2000000000 completionSources = - bookmarks: new BookmarkCompleter() - history: new HistoryCompleter() - domains: new DomainCompleter() - tabs: new TabCompleter() - searchEngines: new SearchEngineCompleter() + bookmarks: new BookmarkCompleter + history: new HistoryCompleter + domains: new DomainCompleter + tabs: new TabCompleter + searchEngines: new SearchEngineCompleter completers = omni: new MultiCompleter [ completionSources.bookmarks completionSources.history completionSources.domains - # This comes last, because it delivers additional, asynchronous results. completionSources.searchEngines ] - bookmarks: new MultiCompleter([completionSources.bookmarks]) - tabs: new MultiCompleter([completionSources.tabs]) + bookmarks: new MultiCompleter [completionSources.bookmarks] + tabs: new MultiCompleter [completionSources.tabs] completionHandlers = filter: (completer, args, port) -> - completer.filter args.queryTerms, (results) -> port.postMessage id: args.id, results: results + completer.filter args.queryTerms, (response) -> + port.postMessage extend args, response refresh: (completer) -> completer.refresh() cancel: (completer) -> completer.cancel() -- cgit v1.2.3 From 3e890adbbf7b6a9671b42536b6ee6221389b9443 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 8 May 2015 11:02:55 +0100 Subject: Search completion; add debug code to completer dispatch. --- background_scripts/completion.coffee | 65 ++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 29 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index f17ca28c..48039cd2 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -465,46 +465,53 @@ class MultiCompleter filter: (queryTerms, onComplete) -> # Allow only one query to run at a time. if @filterInProgress - @mostRecentQuery = { queryTerms: queryTerms, onComplete: onComplete } + @mostRecentQuery = [ queryTerms, onComplete ] return RegexpCache.clear() @mostRecentQuery = null @filterInProgress = true suggestions = [] - completersFinished = 0 continuation = null + activeCompleters = [0...@completers.length] # Call filter() on every source completer and wait for them all to finish before returning results. # At most one of the completers (SearchEngineCompleter) may pass a continuation function, which will be # called after the results of all of the other completers have been posted. Any additional results # from this continuation will be added to the existing results and posted later. We don't call the # continuation if another query is already waiting. - for completer in @completers - do (completer) => - Utils.nextTick => - completer.filter queryTerms, (newSuggestions, cont = null) => - suggestions = suggestions.concat newSuggestions - continuation = cont if cont? - if @completers.length <= ++completersFinished - shouldRunContinuation = continuation? and not @mostRecentQuery - console.log "skip continuation" if continuation? and not shouldRunContinuation - # We don't post results immediately if there are none, and we're going to run a continuation - # (ie. a SearchEngineCompleter). This prevents hiding the vomnibar briefly before showing it - # again, which looks ugly. - unless shouldRunContinuation and suggestions.length == 0 - onComplete - results: @prepareSuggestions queryTerms, suggestions - callerMayCacheResults: not shouldRunContinuation - # Allow subsequent queries to begin. - @filterInProgress = false - if shouldRunContinuation - continuation suggestions, (newSuggestions) => - if 0 < newSuggestions.length - suggestions.push newSuggestions... - onComplete - results: @prepareSuggestions queryTerms, suggestions - callerMayCacheResults: true - else - @filter @mostRecentQuery.queryTerms, @mostRecentQuery.onComplete if @mostRecentQuery + for completer, index in @completers + do (completer, index) => + completer.filter queryTerms, (newSuggestions, newContinuation = null) => + if index not in activeCompleters + # NOTE(smblott) I suspect one of the completers is calling onComplete more than once. (And the + # legacy code had ">=" where "==" should have sufficed.) This is just to track that case down. + console.log "XXXXXXXXXXXXXXX, onComplete called twice!" + console.log completer + activeCompleters = activeCompleters.filter (i) -> i != index + suggestions.push newSuggestions... + continuation = continuation ? newContinuation + if activeCompleters.length == 0 + shouldRunContinuation = continuation? and not @mostRecentQuery + console.log "skip continuation" if continuation? and not shouldRunContinuation + # We don't post results immediately if there are none, and we're going to run a continuation + # (ie. a SearchEngineCompleter). This collapsing the vomnibar briefly before expanding it + # again, which looks ugly. + unless shouldRunContinuation and suggestions.length == 0 + onComplete + results: @prepareSuggestions queryTerms, suggestions + callerMayCacheResults: not shouldRunContinuation + # Allow subsequent queries to begin. + @filterInProgress = false + if shouldRunContinuation + continuation suggestions, (newSuggestions) => + if 0 < newSuggestions.length + suggestions.push newSuggestions... + onComplete + results: @prepareSuggestions queryTerms, suggestions + callerMayCacheResults: true + else + if @mostRecentQuery + console.log "running pending query:", @mostRecentQuery[0] + @filter @mostRecentQuery... prepareSuggestions: (queryTerms, suggestions) -> suggestion.computeRelevancy queryTerms for suggestion in suggestions -- cgit v1.2.3 From b882213019792a7fb47352a920a54d468d352c86 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 8 May 2015 14:02:21 +0100 Subject: Search completion; exclusivity. If we have a custom search engine with a completer, then exclude suggestions from other completion engines. --- background_scripts/completion.coffee | 219 +++++++++++++++------------ background_scripts/completion_engines.coffee | 7 +- 2 files changed, 130 insertions(+), 96 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 48039cd2..b9efb034 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -22,8 +22,12 @@ class Suggestion @title = "" # Extra data which will be available to the relevancy function. @relevancyData = null - # When @autoSelect is truthy, the suggestion is automatically pre-selected in the vomnibar. + # If @autoSelect is truthy, then this suggestion is automatically pre-selected in the vomnibar. There may + # be at most one such suggestion. @autoSelect = false + # If truthy (and @autoSelect is truthy too), then this suggestion is always pre-selected when the query + # changes. There may be at most one such suggestion. + @forceAutoSelect = false # If @highlightTerms is true, then we highlight matched terms in the title and URL. @highlightTerms = true # If @insertText is a string, then the indicated text is inserted into the vomnibar input when the @@ -356,7 +360,7 @@ class SearchEngineCompleter { keyword, searchUrl, description } = @getSearchEngineMatches queryTerms custom = searchUrl? and keyword? - searchUrl ?= Settings.get("searchUrl") + "%s" + searchUrl ?= Settings.get "searchUrl" haveDescription = description? and 0 < description.length description ||= "#{if custom then "custom " else ""}search" @@ -364,7 +368,7 @@ class SearchEngineCompleter query = queryTerms.join " " if queryTerms.length == 0 - return onComplete suggestions + return onComplete [] # For custom search engines, we add an auto-selected suggestion. if custom @@ -376,48 +380,52 @@ class SearchEngineCompleter relevancy: 1 highlightTerms: false autoSelect: true - - onComplete suggestions, (existingSuggestions, onComplete) => - suggestions = [] - # For custom search-engine queries, this adds suggestions only if we have a completer. For other queries, - # this adds suggestions for the default search engine (if we have a completer for that). - - # Relevancy: - # - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word - # relevancy). We assume that the completion engine has already factored that in. Also, completion - # engines often handle spelling mistakes, in which case we wouldn't find the query terms in the - # suggestion anyway. - # - The relavancy is higher if the query term is longer. The idea is that search suggestions are more - # likely to be relevant if, after typing some number of characters, the user hasn't yet found - # a useful suggestion from another completer. - # - Scores are weighted such that they retain the order provided by the completion engine. - characterCount = query.length - queryTerms.length + 1 - relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) - - if 0 < existingSuggestions.length - existingSuggestionsMinScore = existingSuggestions[existingSuggestions.length-1].relevancy - if relavancy < existingSuggestionsMinScore and MultiCompleter.maxResults <= existingSuggestions.length - # No suggestion we propose will have a high enough relavancy to beat the existing suggestions, so bail - # immediately. - return onComplete [] - - CompletionEngines.complete searchUrl, queryTerms, (completionSuggestions = []) => - for suggestion in completionSuggestions - suggestions.push new Suggestion - queryTerms: queryTerms - type: description - url: Utils.createSearchUrl suggestion, searchUrl - title: suggestion - relevancy: relavancy *= 0.9 - highlightTerms: false - insertText: if custom then "#{keyword} #{suggestion}" else suggestion - - # We keep at least three suggestions (if possible) and at most six. We keep more than three only if - # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions - # from other completers. That would potentially be a problem because there is no relationship - # between the relevancy scores produced here and those produced by other completers. - count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length - onComplete suggestions[...count] + # Always reset the selection to this suggestion on query change. The UX is weird otherwise. + forceAutoSelect: true + + onComplete suggestions, + exclusive: if custom and CompletionEngines.haveCompletionEngine searchUrl then description else null + continuation: (existingSuggestions, onComplete) => + suggestions = [] + # For custom search-engine queries, this adds suggestions only if we have a completer. For other queries, + # this adds suggestions for the default search engine (if we have a completer for that). + + # Relevancy: + # - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word + # relevancy). We assume that the completion engine has already factored that in. Also, completion + # engines often handle spelling mistakes, in which case we wouldn't find the query terms in the + # suggestion anyway. + # - The relavancy is higher if the query term is longer. The idea is that search suggestions are more + # likely to be relevant if, after typing some number of characters, the user hasn't yet found + # a useful suggestion from another completer. + # - Scores are weighted such that they retain the order provided by the completion engine. + characterCount = query.length - queryTerms.length + 1 + relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) + + if 0 < existingSuggestions.length + existingSuggestionsMinScore = existingSuggestions[existingSuggestions.length-1].relevancy + if relavancy < existingSuggestionsMinScore and MultiCompleter.maxResults <= existingSuggestions.length + # No suggestion we propose will have a high enough relavancy to beat the existing suggestions, so bail + # immediately. + return onComplete [] + + CompletionEngines.complete searchUrl, queryTerms, (completionSuggestions = []) => + for suggestion in completionSuggestions + suggestions.push new Suggestion + queryTerms: queryTerms + type: description + url: Utils.createSearchUrl suggestion, searchUrl + title: suggestion + relevancy: relavancy *= 0.9 + highlightTerms: false + insertText: if custom then "#{keyword} #{suggestion}" else suggestion + + # We keep at least three suggestions (if possible) and at most six. We keep more than three only if + # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions + # from other completers. That would potentially be a problem because there is no relationship + # between the relevancy scores produced here and those produced by other completers. + count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length + onComplete suggestions[...count] refresh: -> @searchEngines = SearchEngineCompleter.getSearchEngines() @@ -462,56 +470,77 @@ class MultiCompleter cancel: -> completer.cancel?() for completer in @completers - filter: (queryTerms, onComplete) -> - # Allow only one query to run at a time. - if @filterInProgress - @mostRecentQuery = [ queryTerms, onComplete ] - return - RegexpCache.clear() - @mostRecentQuery = null - @filterInProgress = true - suggestions = [] - continuation = null - activeCompleters = [0...@completers.length] - # Call filter() on every source completer and wait for them all to finish before returning results. - # At most one of the completers (SearchEngineCompleter) may pass a continuation function, which will be - # called after the results of all of the other completers have been posted. Any additional results - # from this continuation will be added to the existing results and posted later. We don't call the - # continuation if another query is already waiting. - for completer, index in @completers - do (completer, index) => - completer.filter queryTerms, (newSuggestions, newContinuation = null) => - if index not in activeCompleters - # NOTE(smblott) I suspect one of the completers is calling onComplete more than once. (And the - # legacy code had ">=" where "==" should have sufficed.) This is just to track that case down. - console.log "XXXXXXXXXXXXXXX, onComplete called twice!" - console.log completer - activeCompleters = activeCompleters.filter (i) -> i != index - suggestions.push newSuggestions... - continuation = continuation ? newContinuation - if activeCompleters.length == 0 - shouldRunContinuation = continuation? and not @mostRecentQuery - console.log "skip continuation" if continuation? and not shouldRunContinuation - # We don't post results immediately if there are none, and we're going to run a continuation - # (ie. a SearchEngineCompleter). This collapsing the vomnibar briefly before expanding it - # again, which looks ugly. - unless shouldRunContinuation and suggestions.length == 0 - onComplete - results: @prepareSuggestions queryTerms, suggestions - callerMayCacheResults: not shouldRunContinuation - # Allow subsequent queries to begin. - @filterInProgress = false - if shouldRunContinuation - continuation suggestions, (newSuggestions) => - if 0 < newSuggestions.length - suggestions.push newSuggestions... - onComplete - results: @prepareSuggestions queryTerms, suggestions - callerMayCacheResults: true - else - if @mostRecentQuery - console.log "running pending query:", @mostRecentQuery[0] - @filter @mostRecentQuery... + filter: do -> + defaultCallbackOptions = + # At most one of the completers (SearchEngineCompleter) may pass a continuation function, which will be + # called after the results of all of the other completers have been posted. Any additional results + # from this continuation will be added to the existing results and posted later. We don't call the + # continuation if another query is already waiting. + continuation: null + # If truthy, completions from other completers should be discarded. The truthy value should be the type + # of the completer (e.g. "custom search"). + exclusive: false + + (queryTerms, onComplete) -> + # Allow only one query to run at a time. + if @filterInProgress + @mostRecentQuery = [ queryTerms, onComplete ] + return + RegexpCache.clear() + @mostRecentQuery = null + @filterInProgress = true + suggestions = [] + continuation = null + exclusive = null + activeCompleters = [0...@completers.length] + # Call filter() on every source completer and wait for them all to finish before returning results. + for completer, index in @completers + do (completer, index) => + completer.filter queryTerms, (newSuggestions, options = defaultCallbackOptions) => + if index not in activeCompleters + # NOTE(smblott) I suspect one of the completers is calling onComplete more than once. (And the + # legacy code had ">=" where "==" should have sufficed.) This is just to track that case down. + console.log "XXXXXXXXXXXXXXX, onComplete called twice!" + console.log completer + activeCompleters = activeCompleters.filter (i) -> i != index + suggestions.push newSuggestions... + continuation = continuation ? options.continuation + exclusive = options.exclusive if options.exclusive? + + if activeCompleters.length == 0 + # All the completers have now returned; we combine the results, post them and call any + # continuation. + shouldRunContinuation = continuation? and not @mostRecentQuery + console.log "skip continuation" if continuation? and not shouldRunContinuation + + # If one completer has claimed exclusivity (SearchEngineCompleter), then filter out results from + # other completers. + if exclusive + suggestions = suggestions.filter (suggestion) -> suggestion.type == exclusive + + # We don't post results immediately if there are none, and we're going to run a continuation + # (ie. a SearchEngineCompleter). This collapsing the vomnibar briefly before expanding it + # again, which looks ugly. + unless shouldRunContinuation and suggestions.length == 0 + onComplete + results: @prepareSuggestions queryTerms, suggestions + callerMayCacheResults: not shouldRunContinuation + + # Allow subsequent queries to begin. + @filterInProgress = false + + # Launch continuation or any pending query. + if shouldRunContinuation + continuation suggestions, (newSuggestions) => + if 0 < newSuggestions.length + suggestions.push newSuggestions... + onComplete + results: @prepareSuggestions queryTerms, suggestions + callerMayCacheResults: true + else + if @mostRecentQuery + console.log "running pending query:", @mostRecentQuery[0] + @filter @mostRecentQuery... prepareSuggestions: (queryTerms, suggestions) -> suggestion.computeRelevancy queryTerms for suggestion in suggestions diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 52db90d0..ff10f4b5 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -100,6 +100,7 @@ class DuckDuckGo extends RegexpEngine # A dummy search engine which is guaranteed to match any search URL, but never produces completions. This # allows the rest of the logic to be written knowing that there will always be a completion engine match. class DummyCompletionEngine + dummy: true match: -> true # We return a useless URL which we know will succeed, but which won't generate any network traffic. getUrl: -> chrome.runtime.getURL "content_scripts/vimium.css" @@ -140,7 +141,7 @@ CompletionEngines = # Look up the completion engine for this searchUrl. Because of DummyCompletionEngine, above, we know there # will always be a match. Imagining that there may be many completion engines, and knowing that this is - # called for every input event in the vomnibar, we cache the result. + # called for every query, we cache the result. lookupEngine: (searchUrl) -> @engineCache ?= new SimpleCache 30 * 60 * 60 * 1000 # 30 hours (these are small, we can keep them longer). if @engineCache.has searchUrl @@ -150,6 +151,10 @@ CompletionEngines = engine = new engine() return @engineCache.set searchUrl, engine if engine.match searchUrl + # True if we have a completion engine for this search URL, undefined otherwise. + haveCompletionEngine: (searchUrl) -> + not @lookupEngine(searchUrl).dummy + # This is the main entry point. # - searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"), or a custome search engine's URL. # This is only used as a key for determining the relevant completion engine. -- cgit v1.2.3 From 4ab7881e46084a5946bae6d29fb1d0ab9677542a Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 8 May 2015 14:41:01 +0100 Subject: Search completion; fix unit tests. --- background_scripts/settings.coffee | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index ce812970..2a21b0c9 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -5,8 +5,6 @@ root = exports ? window root.Settings = Settings = get: (key) -> - # FIXME(smblott). Remove this line. - return @defaults.searchEngines if key == "searchEngines" if (key of localStorage) then JSON.parse(localStorage[key]) else @defaults[key] set: (key, value) -> @@ -93,18 +91,16 @@ root.Settings = Settings = searchUrl: "http://www.google.com/search?q=" # put in an example search engine searchEngines: [ - # FIXME(smblott) Comment these out before merge. - "# THESE ARE HARD WIRED.\n# YOU CANNOT CHANGE THEM IN THIS VERSION.\n# FOR DEVELOPMENT ONLY." - "g: http://www.google.com/search?q=%s Google" - "l: http://www.google.com/search?q=%s&btnI I'm feeling lucky..." "w: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedia" - "y: http://www.youtube.com/results?search_query=%s Youtube" - "t: http://www.youtube.com/results?search_query=%s" - "m: https://www.google.com/maps/search/%s Google Maps" - "b: https://www.bing.com/search?q=%s Bing" - "d: https://duckduckgo.com/?q=%s DuckDuckGo" - "az: http://www.amazon.com/s/?field-keywords=%s Amazon" - ].join "\n\n" + "# Examples:" + "# g: http://www.google.com/search?q=%s Google" + "# l: http://www.google.com/search?q=%s&btnI I'm feeling lucky..." + "# y: http://www.youtube.com/results?search_query=%s Youtube" + "# m: https://www.google.com/maps/search/%s Google Maps" + "# b: https://www.bing.com/search?q=%s Bing" + "# d: https://duckduckgo.com/?q=%s DuckDuckGo" + "# az: http://www.amazon.com/s/?field-keywords=%s Amazon" + ].join "\n" newTabUrl: "chrome://newtab" grabBackFocus: false -- cgit v1.2.3 From 9887c8d763bf7b58e459a48f34531f6877ffebf4 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 8 May 2015 14:54:15 +0100 Subject: Search completion; tweak example text. --- background_scripts/settings.coffee | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index 2a21b0c9..89b26bff 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -92,7 +92,9 @@ root.Settings = Settings = # put in an example search engine searchEngines: [ "w: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedia" - "# Examples:" + "" + "# More examples:" + "#" "# g: http://www.google.com/search?q=%s Google" "# l: http://www.google.com/search?q=%s&btnI I'm feeling lucky..." "# y: http://www.youtube.com/results?search_query=%s Youtube" -- cgit v1.2.3 From 5e6fa4ccfc103750b84df02a35f42a6acef78fa1 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 8 May 2015 15:39:13 +0100 Subject: Search completion; suppress custom search keyword. --- background_scripts/completion.coffee | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index b9efb034..9d249198 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -382,6 +382,8 @@ class SearchEngineCompleter autoSelect: true # Always reset the selection to this suggestion on query change. The UX is weird otherwise. forceAutoSelect: true + # Suppress the "w" from "w query terms" in the vomnibar input. + suppressLeadingQueryTerm: true onComplete suggestions, exclusive: if custom and CompletionEngines.haveCompletionEngine searchUrl then description else null @@ -418,7 +420,7 @@ class SearchEngineCompleter title: suggestion relevancy: relavancy *= 0.9 highlightTerms: false - insertText: if custom then "#{keyword} #{suggestion}" else suggestion + insertText: suggestion # We keep at least three suggestions (if possible) and at most six. We keep more than three only if # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions -- cgit v1.2.3 From 82d25b5df76c8526d4ccb5352c0905cc28371199 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 8 May 2015 16:47:24 +0100 Subject: Search completion; search keyword on SPACE. --- background_scripts/completion.coffee | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 9d249198..ba19970f 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -131,7 +131,8 @@ class BookmarkCompleter # These bookmarks are loaded asynchronously when refresh() is called. bookmarks: null - filter: (@queryTerms, @onComplete) -> + filter: (queryTerms, @onComplete) -> + @queryTerms = queryTerms.filter (t) -> 0 < t.length @currentSearch = { queryTerms: @queryTerms, onComplete: @onComplete } @performSearch() if @bookmarks @@ -193,6 +194,7 @@ class BookmarkCompleter class HistoryCompleter filter: (queryTerms, onComplete) -> + queryTerms = queryTerms.filter (t) -> 0 < t.length @currentSearch = { queryTerms: @queryTerms, onComplete: @onComplete } results = [] HistoryCache.use (history) => @@ -227,6 +229,7 @@ class DomainCompleter domains: null filter: (queryTerms, onComplete) -> + queryTerms = queryTerms.filter (t) -> 0 < t.length return onComplete([]) unless queryTerms.length == 1 if @domains @performSearch(queryTerms, onComplete) @@ -329,6 +332,7 @@ tabRecency = new TabRecency() # Searches through all open tabs, matching on title and URL. class TabCompleter filter: (queryTerms, onComplete) -> + queryTerms = queryTerms.filter (t) -> 0 < t.length # NOTE(philc): We search all tabs, not just those in the current window. I'm not sure if this is the # correct UX. chrome.tabs.query {}, (tabs) => @@ -366,9 +370,7 @@ class SearchEngineCompleter queryTerms = queryTerms[1..] if custom query = queryTerms.join " " - - if queryTerms.length == 0 - return onComplete [] + return onComplete [] if queryTerms.length == 0 # For custom search engines, we add an auto-selected suggestion. if custom @@ -385,6 +387,10 @@ class SearchEngineCompleter # Suppress the "w" from "w query terms" in the vomnibar input. suppressLeadingQueryTerm: true + # We filter out the empty strings late so that we can distinguish between, for example, "w" and "w ". + queryTerms = queryTerms.filter (t) -> 0 < t.length + return onComplete suggestions if queryTerms.length == 0 + onComplete suggestions, exclusive: if custom and CompletionEngines.haveCompletionEngine searchUrl then description else null continuation: (existingSuggestions, onComplete) => @@ -404,6 +410,7 @@ class SearchEngineCompleter characterCount = query.length - queryTerms.length + 1 relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) + queryTerms = queryTerms.filter (t) -> 0 < t.length if 0 < existingSuggestions.length existingSuggestionsMinScore = existingSuggestions[existingSuggestions.length-1].relevancy if relavancy < existingSuggestionsMinScore and MultiCompleter.maxResults <= existingSuggestions.length @@ -477,10 +484,11 @@ class MultiCompleter # At most one of the completers (SearchEngineCompleter) may pass a continuation function, which will be # called after the results of all of the other completers have been posted. Any additional results # from this continuation will be added to the existing results and posted later. We don't call the - # continuation if another query is already waiting. + # continuation if another query is already waiting. This is for slow tasks which should be done + # asynchronously (e.g. HTTP GET). continuation: null - # If truthy, completions from other completers should be discarded. The truthy value should be the type - # of the completer (e.g. "custom search"). + # If truthy, completions from other completers should be suppressed. The truthy value should be the + # type of the completer (e.g. "custom search"). All other completion types are suppressed. exclusive: false (queryTerms, onComplete) -> -- cgit v1.2.3 From 21db0f353257e5e7848d9d884ed93e717120e88d Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 8 May 2015 17:28:09 +0100 Subject: Search completion; search keyword on SPACE. --- background_scripts/completion.coffee | 20 ++++++++++++-------- background_scripts/main.coffee | 14 +++++++------- 2 files changed, 19 insertions(+), 15 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index ba19970f..dc5b2737 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -359,6 +359,13 @@ class SearchEngineCompleter cancel: -> CompletionEngines.cancel() + refresh: (port) -> + @searchEngines = SearchEngineCompleter.getSearchEngines() + # Let the vomnibar know the custom search engine keywords. + port.postMessage + handler: "customSearchEngineKeywords" + keywords: key for own key of @searchEngines + filter: (queryTerms, onComplete) -> suggestions = [] @@ -385,7 +392,7 @@ class SearchEngineCompleter # Always reset the selection to this suggestion on query change. The UX is weird otherwise. forceAutoSelect: true # Suppress the "w" from "w query terms" in the vomnibar input. - suppressLeadingQueryTerm: true + suppressLeadingKeyword: true # We filter out the empty strings late so that we can distinguish between, for example, "w" and "w ". queryTerms = queryTerms.filter (t) -> 0 < t.length @@ -436,9 +443,6 @@ class SearchEngineCompleter count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length onComplete suggestions[...count] - refresh: -> - @searchEngines = SearchEngineCompleter.getSearchEngines() - getSearchEngineMatches: (queryTerms) -> (1 < queryTerms.length and @searchEngines[queryTerms[0]]) or {} @@ -473,11 +477,11 @@ class MultiCompleter constructor: (@completers) -> @maxResults = MultiCompleter.maxResults - refresh: -> - completer.refresh?() for completer in @completers + refresh: (port) -> + completer.refresh? port for completer in @completers - cancel: -> - completer.cancel?() for completer in @completers + cancel: (port) -> + completer.cancel? port for completer in @completers filter: do -> defaultCallbackOptions = diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index 1a3281bf..34db5a20 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -60,15 +60,15 @@ completers = tabs: new MultiCompleter [completionSources.tabs] completionHandlers = - filter: (completer, args, port) -> - completer.filter args.queryTerms, (response) -> - port.postMessage extend args, response + filter: (completer, request, port) -> + completer.filter request.queryTerms, (response) -> + port.postMessage extend request, extend response, handler: "completions" - refresh: (completer) -> completer.refresh() - cancel: (completer) -> completer.cancel() + refresh: (completer, _, port) -> completer.refresh port + cancel: (completer, _, port) -> completer.cancel port -handleCompletions = (args, port) -> - completionHandlers[args.handler] completers[args.name], args, port +handleCompletions = (request, port) -> + completionHandlers[request.handler] completers[request.name], request, port chrome.runtime.onConnect.addListener (port, name) -> senderTabId = if port.sender.tab then port.sender.tab.id else null -- cgit v1.2.3 From cf993efaf69fcf3482cf0f54881fcf87f0108a0d Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 8 May 2015 17:53:39 +0100 Subject: Search completion; exclusion [WIP]. --- background_scripts/completion.coffee | 32 ++++++++++++++++++++++------ background_scripts/completion_engines.coffee | 6 ++---- 2 files changed, 27 insertions(+), 11 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index dc5b2737..6e57f0d4 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -63,6 +63,11 @@ class Suggestion a.href = url a.protocol + "//" + a.hostname + getHostname: (url) -> + a = document.createElement 'a' + a.href = url + a.hostname + shortenUrl: (url) -> @stripTrailingSlash(url).replace(/^https?:\/\//, "") stripTrailingSlash: (url) -> @@ -396,10 +401,24 @@ class SearchEngineCompleter # We filter out the empty strings late so that we can distinguish between, for example, "w" and "w ". queryTerms = queryTerms.filter (t) -> 0 < t.length - return onComplete suggestions if queryTerms.length == 0 + # NOTE(smblott) I'm having difficulty figuring out to do the filtering, here. Exclusive should mean + # exclusive to what? + exclusive = if custom and CompletionEngines.haveCompletionEngine searchUrl then description else null + # exclusive = + # if custom and CompletionEngines.haveCompletionEngine searchUrl + # suggestions[0].getHostname suggestions[0].url + # else + # null + # exclusive = + # if custom and CompletionEngines.haveCompletionEngine searchUrl + # searchUrl.split("%s")?[0] + # else + # null + if queryTerms.length == 0 + return onComplete suggestions, { exclusive } onComplete suggestions, - exclusive: if custom and CompletionEngines.haveCompletionEngine searchUrl then description else null + exclusive: exclusive continuation: (existingSuggestions, onComplete) => suggestions = [] # For custom search-engine queries, this adds suggestions only if we have a completer. For other queries, @@ -491,9 +510,8 @@ class MultiCompleter # continuation if another query is already waiting. This is for slow tasks which should be done # asynchronously (e.g. HTTP GET). continuation: null - # If truthy, completions from other completers should be suppressed. The truthy value should be the - # type of the completer (e.g. "custom search"). All other completion types are suppressed. - exclusive: false + # If truthy, non-matching completions from other completers should be suppressed. + exclusive: null (queryTerms, onComplete) -> # Allow only one query to run at a time. @@ -518,8 +536,8 @@ class MultiCompleter console.log completer activeCompleters = activeCompleters.filter (i) -> i != index suggestions.push newSuggestions... - continuation = continuation ? options.continuation - exclusive = options.exclusive if options.exclusive? + continuation ?= options.continuation + exclusive ?= options.exclusive if activeCompleters.length == 0 # All the completers have now returned; we combine the results, post them and call any diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index ff10f4b5..badae126 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -165,10 +165,8 @@ CompletionEngines = @mostRecentHandler = null query = queryTerms.join "" - # We don't complete less then three characters: the results are usually useless. This also prevents - # one- and two-character custom search engine keywords from being sent to the default completer (e.g. - # the initial "w" before typing "w something" for Wikipedia). - return callback [] unless 3 <= query.length + # We don't complete single characters: the results are usually useless. + return callback [] unless 1 < query.length # We don't complete regular URLs or Javascript URLs. return callback [] if 1 == queryTerms.length and Utils.isUrl query -- cgit v1.2.3 From d94e7c74d78ce788cf3deabcd1c99c3859240566 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Fri, 8 May 2015 17:57:29 +0100 Subject: Search completion; reduce delay. --- background_scripts/completion_engines.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index badae126..85062c49 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -126,7 +126,7 @@ CompletionEngines = # The amount of time to wait for new requests before launching the HTTP request. The intention is to cut # down on the number of HTTP requests we issue. - delay: 200 + delay: 100 get: (searchUrl, url, callback) -> xhr = new XMLHttpRequest() -- cgit v1.2.3 From 8c308d47cc615b9fa0eed47e4ecddd1fd9d125eb Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 9 May 2015 06:56:02 +0100 Subject: Search completion; refactor MultiCompleter. --- background_scripts/completion.coffee | 129 +++++++++++++++++------------------ 1 file changed, 62 insertions(+), 67 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 6e57f0d4..011ff72b 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -401,24 +401,19 @@ class SearchEngineCompleter # We filter out the empty strings late so that we can distinguish between, for example, "w" and "w ". queryTerms = queryTerms.filter (t) -> 0 < t.length - # NOTE(smblott) I'm having difficulty figuring out to do the filtering, here. Exclusive should mean - # exclusive to what? - exclusive = if custom and CompletionEngines.haveCompletionEngine searchUrl then description else null - # exclusive = - # if custom and CompletionEngines.haveCompletionEngine searchUrl - # suggestions[0].getHostname suggestions[0].url - # else - # null - # exclusive = - # if custom and CompletionEngines.haveCompletionEngine searchUrl - # searchUrl.split("%s")?[0] - # else - # null + + # Exclude results from other completers if this is a custom search engine and we have a completer. + filter = + if custom and CompletionEngines.haveCompletionEngine searchUrl + (suggestion) -> suggestion.type == description + else + null + if queryTerms.length == 0 - return onComplete suggestions, { exclusive } + return onComplete suggestions, { filter } onComplete suggestions, - exclusive: exclusive + filter: filter continuation: (existingSuggestions, onComplete) => suggestions = [] # For custom search-engine queries, this adds suggestions only if we have a completer. For other queries, @@ -504,75 +499,75 @@ class MultiCompleter filter: do -> defaultCallbackOptions = - # At most one of the completers (SearchEngineCompleter) may pass a continuation function, which will be - # called after the results of all of the other completers have been posted. Any additional results - # from this continuation will be added to the existing results and posted later. We don't call the - # continuation if another query is already waiting. This is for slow tasks which should be done - # asynchronously (e.g. HTTP GET). + # Completers may provide a continuation function. This will be run after all completers have posted + # their suggestions, and is used to post additional (slow) asynchronous suggestions (e.g. search-engine + # completions fetched over HTTP). continuation: null - # If truthy, non-matching completions from other completers should be suppressed. - exclusive: null + # Completers may provide a filter function. This allows one completer to filter out suggestions from + # other completers. + filter: null (queryTerms, onComplete) -> - # Allow only one query to run at a time. - if @filterInProgress - @mostRecentQuery = [ queryTerms, onComplete ] - return + @debug = true + # Allow only one query to run at a time, and remember the most recent query. + return @mostRecentQuery = arguments if @filterInProgress + RegexpCache.clear() @mostRecentQuery = null @filterInProgress = true suggestions = [] - continuation = null - exclusive = null + continuations = [] + filters = [] activeCompleters = [0...@completers.length] - # Call filter() on every source completer and wait for them all to finish before returning results. + + # Call filter() on every completer and wait for them all to finish before filtering and posting the + # results, then calling any continuations. for completer, index in @completers - do (completer, index) => - completer.filter queryTerms, (newSuggestions, options = defaultCallbackOptions) => - if index not in activeCompleters - # NOTE(smblott) I suspect one of the completers is calling onComplete more than once. (And the - # legacy code had ">=" where "==" should have sufficed.) This is just to track that case down. - console.log "XXXXXXXXXXXXXXX, onComplete called twice!" - console.log completer - activeCompleters = activeCompleters.filter (i) -> i != index + do (index) => + completer.filter queryTerms, (newSuggestions = [], { continuation, filter } = defaultCallbackOptions) => + + # Store the results. suggestions.push newSuggestions... - continuation ?= options.continuation - exclusive ?= options.exclusive + continuations.push continuation if continuation? + filters.push filter if filter? + activeCompleters = activeCompleters.filter (i) -> i != index if activeCompleters.length == 0 - # All the completers have now returned; we combine the results, post them and call any - # continuation. - shouldRunContinuation = continuation? and not @mostRecentQuery - console.log "skip continuation" if continuation? and not shouldRunContinuation - - # If one completer has claimed exclusivity (SearchEngineCompleter), then filter out results from - # other completers. - if exclusive - suggestions = suggestions.filter (suggestion) -> suggestion.type == exclusive - - # We don't post results immediately if there are none, and we're going to run a continuation - # (ie. a SearchEngineCompleter). This collapsing the vomnibar briefly before expanding it - # again, which looks ugly. - unless shouldRunContinuation and suggestions.length == 0 - onComplete - results: @prepareSuggestions queryTerms, suggestions - callerMayCacheResults: not shouldRunContinuation + # All the completers have now yielded their (initial) results, we're good to go. - # Allow subsequent queries to begin. - @filterInProgress = false + # Apply filters. + suggestions = suggestions.filter filter for filter in filters + + # Should we run continuations? + shouldRunContinuations = 0 < continuations.length and not @mostRecentQuery? - # Launch continuation or any pending query. - if shouldRunContinuation - continuation suggestions, (newSuggestions) => - if 0 < newSuggestions.length + # Post results, unless there are none AND we will be running a continuation. This avoids + # collapsing the vomnibar briefly before expanding it again, which looks ugly. + unless suggestions.length == 0 and shouldRunContinuations + onComplete + results: @prepareSuggestions queryTerms, suggestions + mayCacheResults: continuations.length == 0 + expectMoreResults: shouldRunContinuations + + # Run any continuations, unless there's a pending query. + if shouldRunContinuations + for continuation in continuations + console.log "launching continuation..." if @debug + continuation suggestions, (newSuggestions) => + console.log "posting continuation" if @debug suggestions.push newSuggestions... onComplete results: @prepareSuggestions queryTerms, suggestions - callerMayCacheResults: true - else - if @mostRecentQuery - console.log "running pending query:", @mostRecentQuery[0] - @filter @mostRecentQuery... + # FIXME(smblott) This currently assumes that there is at most one continuation. We + # should really be counting pending/completed continuations. + mayCacheResults: true + expectMoreResults: false + + # Admit subsequent queries, and launch any pending query. + @filterInProgress = false + if @mostRecentQuery + console.log "running pending query:", @mostRecentQuery[0] if @debug + @filter @mostRecentQuery... prepareSuggestions: (queryTerms, suggestions) -> suggestion.computeRelevancy queryTerms for suggestion in suggestions -- cgit v1.2.3 From 6fd0e15b96325222abf1a19886bd5e0fb48fcdbb Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 9 May 2015 08:30:29 +0100 Subject: Search completion; refactor SearchCompleter activation. --- background_scripts/completion.coffee | 39 +++++++++++++++++------------------- background_scripts/main.coffee | 2 +- 2 files changed, 19 insertions(+), 22 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 011ff72b..c1f76b81 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -136,8 +136,7 @@ class BookmarkCompleter # These bookmarks are loaded asynchronously when refresh() is called. bookmarks: null - filter: (queryTerms, @onComplete) -> - @queryTerms = queryTerms.filter (t) -> 0 < t.length + filter: ({ @queryTerms }, @onComplete) -> @currentSearch = { queryTerms: @queryTerms, onComplete: @onComplete } @performSearch() if @bookmarks @@ -198,8 +197,7 @@ class BookmarkCompleter RankingUtils.wordRelevancy(suggestion.queryTerms, suggestion.url, suggestion.title) class HistoryCompleter - filter: (queryTerms, onComplete) -> - queryTerms = queryTerms.filter (t) -> 0 < t.length + filter: ({ queryTerms }, onComplete) -> @currentSearch = { queryTerms: @queryTerms, onComplete: @onComplete } results = [] HistoryCache.use (history) => @@ -233,8 +231,7 @@ class DomainCompleter # If `referenceCount` goes to zero, the domain entry can and should be deleted. domains: null - filter: (queryTerms, onComplete) -> - queryTerms = queryTerms.filter (t) -> 0 < t.length + filter: ({ queryTerms }, onComplete) -> return onComplete([]) unless queryTerms.length == 1 if @domains @performSearch(queryTerms, onComplete) @@ -336,8 +333,7 @@ tabRecency = new TabRecency() # Searches through all open tabs, matching on title and URL. class TabCompleter - filter: (queryTerms, onComplete) -> - queryTerms = queryTerms.filter (t) -> 0 < t.length + filter: ({ queryTerms }, onComplete) -> # NOTE(philc): We search all tabs, not just those in the current window. I'm not sure if this is the # correct UX. chrome.tabs.query {}, (tabs) => @@ -371,10 +367,11 @@ class SearchEngineCompleter handler: "customSearchEngineKeywords" keywords: key for own key of @searchEngines - filter: (queryTerms, onComplete) -> + filter: ({ queryTerms, query }, onComplete) -> + return onComplete [] if queryTerms.length == 0 suggestions = [] - { keyword, searchUrl, description } = @getSearchEngineMatches queryTerms + { keyword, searchUrl, description } = @getSearchEngineMatches queryTerms, query custom = searchUrl? and keyword? searchUrl ?= Settings.get "searchUrl" haveDescription = description? and 0 < description.length @@ -382,7 +379,6 @@ class SearchEngineCompleter queryTerms = queryTerms[1..] if custom query = queryTerms.join " " - return onComplete [] if queryTerms.length == 0 # For custom search engines, we add an auto-selected suggestion. if custom @@ -399,9 +395,6 @@ class SearchEngineCompleter # Suppress the "w" from "w query terms" in the vomnibar input. suppressLeadingKeyword: true - # We filter out the empty strings late so that we can distinguish between, for example, "w" and "w ". - queryTerms = queryTerms.filter (t) -> 0 < t.length - # Exclude results from other completers if this is a custom search engine and we have a completer. filter = if custom and CompletionEngines.haveCompletionEngine searchUrl @@ -431,7 +424,6 @@ class SearchEngineCompleter characterCount = query.length - queryTerms.length + 1 relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) - queryTerms = queryTerms.filter (t) -> 0 < t.length if 0 < existingSuggestions.length existingSuggestionsMinScore = existingSuggestions[existingSuggestions.length-1].relevancy if relavancy < existingSuggestionsMinScore and MultiCompleter.maxResults <= existingSuggestions.length @@ -457,8 +449,14 @@ class SearchEngineCompleter count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length onComplete suggestions[...count] - getSearchEngineMatches: (queryTerms) -> - (1 < queryTerms.length and @searchEngines[queryTerms[0]]) or {} + getSearchEngineMatches: (queryTerms, query = queryTerms.join " ") -> + # To allow users to write queries with leading search-engine keywords, leading whitespace disables custom + # search engines; for example, " w" is a regular query. + return {} if /^\s/.test query + # Trailing whitespace is significant when activating a custom search engine; for example, "w" (just a + # regular query) is different from "w " (a custom search engine). + length = queryTerms.length + (if /\s$/.test query then 1 else 0) + (1 < length and @searchEngines[queryTerms[0]]) or {} # Static data and methods for parsing the configured search engines. We keep a cache of the search-engine # mapping in @searchEnginesMap. @@ -507,11 +505,12 @@ class MultiCompleter # other completers. filter: null - (queryTerms, onComplete) -> + (request, onComplete) -> @debug = true # Allow only one query to run at a time, and remember the most recent query. return @mostRecentQuery = arguments if @filterInProgress + { queryTerms } = request RegexpCache.clear() @mostRecentQuery = null @filterInProgress = true @@ -524,7 +523,7 @@ class MultiCompleter # results, then calling any continuations. for completer, index in @completers do (index) => - completer.filter queryTerms, (newSuggestions = [], { continuation, filter } = defaultCallbackOptions) => + completer.filter request, (newSuggestions = [], { continuation, filter } = defaultCallbackOptions) => # Store the results. suggestions.push newSuggestions... @@ -546,7 +545,6 @@ class MultiCompleter unless suggestions.length == 0 and shouldRunContinuations onComplete results: @prepareSuggestions queryTerms, suggestions - mayCacheResults: continuations.length == 0 expectMoreResults: shouldRunContinuations # Run any continuations, unless there's a pending query. @@ -560,7 +558,6 @@ class MultiCompleter results: @prepareSuggestions queryTerms, suggestions # FIXME(smblott) This currently assumes that there is at most one continuation. We # should really be counting pending/completed continuations. - mayCacheResults: true expectMoreResults: false # Admit subsequent queries, and launch any pending query. diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index 34db5a20..612f6170 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -61,7 +61,7 @@ completers = completionHandlers = filter: (completer, request, port) -> - completer.filter request.queryTerms, (response) -> + completer.filter request, (response) -> port.postMessage extend request, extend response, handler: "completions" refresh: (completer, _, port) -> completer.refresh port -- cgit v1.2.3 From 3bc555eb02b228f35647884575189eed40625c52 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 9 May 2015 08:37:05 +0100 Subject: Search completion; tweaks. --- background_scripts/completion.coffee | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index c1f76b81..f800c818 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -395,14 +395,16 @@ class SearchEngineCompleter # Suppress the "w" from "w query terms" in the vomnibar input. suppressLeadingKeyword: true - # Exclude results from other completers if this is a custom search engine and we have a completer. + haveCompletionEngine = CompletionEngines.haveCompletionEngine searchUrl + # If this is a custom search engine and we have a completer, then exclude results from other completers. filter = - if custom and CompletionEngines.haveCompletionEngine searchUrl + if custom and haveCompletionEngine (suggestion) -> suggestion.type == description else null - if queryTerms.length == 0 + # Post suggestions and bail if there is no prospect of adding further suggestions. + if queryTerms.length == 0 or not haveCompletionEngine return onComplete suggestions, { filter } onComplete suggestions, -- cgit v1.2.3 From 75051d53536ddb9f247501b4509306cae1734184 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 9 May 2015 09:50:13 +0100 Subject: Search completion; reintroduce vomnibar cache. --- background_scripts/completion.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index f800c818..8c73c658 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -547,7 +547,7 @@ class MultiCompleter unless suggestions.length == 0 and shouldRunContinuations onComplete results: @prepareSuggestions queryTerms, suggestions - expectMoreResults: shouldRunContinuations + mayCacheResult: continuations.length == 0 # Run any continuations, unless there's a pending query. if shouldRunContinuations @@ -560,7 +560,7 @@ class MultiCompleter results: @prepareSuggestions queryTerms, suggestions # FIXME(smblott) This currently assumes that there is at most one continuation. We # should really be counting pending/completed continuations. - expectMoreResults: false + mayCacheResult: true # Admit subsequent queries, and launch any pending query. @filterInProgress = false -- cgit v1.2.3 From d73775057d443a53668f6a93fe45cc4a4b412de7 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 9 May 2015 11:30:59 +0100 Subject: Search completion; complete commmon search term. --- background_scripts/completion.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 8c73c658..850a257d 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -547,7 +547,7 @@ class MultiCompleter unless suggestions.length == 0 and shouldRunContinuations onComplete results: @prepareSuggestions queryTerms, suggestions - mayCacheResult: continuations.length == 0 + mayCacheResults: continuations.length == 0 # Run any continuations, unless there's a pending query. if shouldRunContinuations @@ -560,7 +560,7 @@ class MultiCompleter results: @prepareSuggestions queryTerms, suggestions # FIXME(smblott) This currently assumes that there is at most one continuation. We # should really be counting pending/completed continuations. - mayCacheResult: true + mayCacheResults: true # Admit subsequent queries, and launch any pending query. @filterInProgress = false -- cgit v1.2.3 From 311b35e416053a0d5d03eaf7eb894375f6e0f20d Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 9 May 2015 16:47:28 +0100 Subject: Search completion; tweaks and tests. --- background_scripts/completion.coffee | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 850a257d..7966452d 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -375,33 +375,37 @@ class SearchEngineCompleter custom = searchUrl? and keyword? searchUrl ?= Settings.get "searchUrl" haveDescription = description? and 0 < description.length - description ||= "#{if custom then "custom " else ""}search" + description ||= "search#{if custom then " [#{keyword}]" else ""}" queryTerms = queryTerms[1..] if custom query = queryTerms.join " " + haveCompletionEngine = CompletionEngines.haveCompletionEngine searchUrl + # If this is a custom search engine and we have a completer, then exclude results from other completers. + filter = + if custom and haveCompletionEngine + (suggestion) -> suggestion.type == description + else + null + # For custom search engines, we add an auto-selected suggestion. if custom suggestions.push new Suggestion queryTerms: queryTerms type: description url: Utils.createSearchUrl queryTerms, searchUrl - title: if haveDescription then query else "#{keyword}: #{query}" + title: query relevancy: 1 highlightTerms: false - autoSelect: true + insertText: query + # NOTE (smblott) Disbaled pending consideration of how to handle text selection within the vomnibar + # itself. + # autoSelect: true # Always reset the selection to this suggestion on query change. The UX is weird otherwise. - forceAutoSelect: true + # forceAutoSelect: true # Suppress the "w" from "w query terms" in the vomnibar input. suppressLeadingKeyword: true - - haveCompletionEngine = CompletionEngines.haveCompletionEngine searchUrl - # If this is a custom search engine and we have a completer, then exclude results from other completers. - filter = - if custom and haveCompletionEngine - (suggestion) -> suggestion.type == description - else - null + completeSuggestions: filter? # Post suggestions and bail if there is no prospect of adding further suggestions. if queryTerms.length == 0 or not haveCompletionEngine -- cgit v1.2.3 From 0c6b6e53d60a1c4b694d1515fdb7e43080bbf0d3 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 9 May 2015 16:53:41 +0100 Subject: Search completion; tweak default setting. --- background_scripts/settings.coffee | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index 89b26bff..e5604b78 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -93,7 +93,9 @@ root.Settings = Settings = searchEngines: [ "w: http://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedia" "" - "# More examples:" + "# More examples." + "#" + "# (Vimium has built-in completion for these.)" "#" "# g: http://www.google.com/search?q=%s Google" "# l: http://www.google.com/search?q=%s&btnI I'm feeling lucky..." -- cgit v1.2.3 From b332710f8395582809b1a1e2c436628b7f6e8c2a Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 9 May 2015 17:04:40 +0100 Subject: Search completion; tweak default setting. --- background_scripts/settings.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index e5604b78..44ed897d 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -100,10 +100,13 @@ root.Settings = Settings = "# g: http://www.google.com/search?q=%s Google" "# l: http://www.google.com/search?q=%s&btnI I'm feeling lucky..." "# y: http://www.youtube.com/results?search_query=%s Youtube" - "# m: https://www.google.com/maps/search/%s Google Maps" "# b: https://www.bing.com/search?q=%s Bing" "# d: https://duckduckgo.com/?q=%s DuckDuckGo" "# az: http://www.amazon.com/s/?field-keywords=%s Amazon" + "#" + "# Another example (for Vimium does not have completion)." + "#" + "# m: https://www.google.com/maps/search/%s Google Maps" ].join "\n" newTabUrl: "chrome://newtab" grabBackFocus: false -- cgit v1.2.3 From 275a91f203086b8f81542c174e736748dce68628 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sat, 9 May 2015 18:06:12 +0100 Subject: Search completion; tweak for engines without completers. --- background_scripts/completion.coffee | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 7966452d..cc334d78 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -382,6 +382,8 @@ class SearchEngineCompleter haveCompletionEngine = CompletionEngines.haveCompletionEngine searchUrl # If this is a custom search engine and we have a completer, then exclude results from other completers. + # The truthiness of filter? also serves to distinguish two very different vomnibar behaviors, one for + # custom search engines with completers (filter? is true), and one for those without (filter? is false). filter = if custom and haveCompletionEngine (suggestion) -> suggestion.type == description @@ -390,22 +392,26 @@ class SearchEngineCompleter # For custom search engines, we add an auto-selected suggestion. if custom + # Note. This suggestion always appears at the top of the suggestion list. Its settings serves to serve + # to configure various vomnibar behaviors. suggestions.push new Suggestion queryTerms: queryTerms type: description url: Utils.createSearchUrl queryTerms, searchUrl title: query relevancy: 1 - highlightTerms: false - insertText: query - # NOTE (smblott) Disbaled pending consideration of how to handle text selection within the vomnibar - # itself. - # autoSelect: true - # Always reset the selection to this suggestion on query change. The UX is weird otherwise. - # forceAutoSelect: true - # Suppress the "w" from "w query terms" in the vomnibar input. + insertText: if filter? then query else null + # For all custom search engines, we suppress the leading keyword, for example "w something" becomes + # "something" in the vomnibar. suppressLeadingKeyword: true + # We complete suggestions only for custom search engines where we have an associated completion + # engine. completeSuggestions: filter? + # We only use autoSelect and highlight query terms (on custom search engines) when we do not have a + # completer. + autoSelect: not filter? + forceAutoSelect: not filter? + highlightTerms: not filter? # Post suggestions and bail if there is no prospect of adding further suggestions. if queryTerms.length == 0 or not haveCompletionEngine -- cgit v1.2.3 From 313a1f96d666f23c2bc75ef340f0f828319e127c Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 05:35:50 +0100 Subject: Search completion; refactor searchEngineCompleter. This revamps how search-engine configuration is handled, and revises some rather strange legacy code. --- background_scripts/completion.coffee | 254 +++++++++++++++++------------------ background_scripts/settings.coffee | 3 - 2 files changed, 123 insertions(+), 134 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index cc334d78..94109b84 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -355,143 +355,135 @@ class TabCompleter tabRecency.recencyScore(suggestion.tabId) class SearchEngineCompleter - searchEngines: {} - - cancel: -> - CompletionEngines.cancel() + searchEngineConfig: null refresh: (port) -> - @searchEngines = SearchEngineCompleter.getSearchEngines() - # Let the vomnibar know the custom search engine keywords. - port.postMessage - handler: "customSearchEngineKeywords" - keywords: key for own key of @searchEngines + # Load and parse the search-engine configuration. + @searchEngineConfig = new AsyncDataFetcher (callback) -> + engines = {} + for line in Settings.get("searchEngines").split "\n" + line = line.trim() + continue if /^[#"]/.test line + tokens = line.split /\s+/ + continue unless 2 <= tokens.length + keyword = tokens[0].split(":")[0] + description = tokens[2..].join(" ") || "search (#{keyword})" + engines[keyword] = + keyword: keyword + searchUrl: tokens[1] + description: description + + # Deliver the resulting engines lookup table. + callback engines + + # Let the vomnibar know the custom search engine keywords. + port.postMessage + handler: "customSearchEngineKeywords" + keywords: key for own key of engines filter: ({ queryTerms, query }, onComplete) -> return onComplete [] if queryTerms.length == 0 - suggestions = [] - - { keyword, searchUrl, description } = @getSearchEngineMatches queryTerms, query - custom = searchUrl? and keyword? - searchUrl ?= Settings.get "searchUrl" - haveDescription = description? and 0 < description.length - description ||= "search#{if custom then " [#{keyword}]" else ""}" - - queryTerms = queryTerms[1..] if custom - query = queryTerms.join " " - - haveCompletionEngine = CompletionEngines.haveCompletionEngine searchUrl - # If this is a custom search engine and we have a completer, then exclude results from other completers. - # The truthiness of filter? also serves to distinguish two very different vomnibar behaviors, one for - # custom search engines with completers (filter? is true), and one for those without (filter? is false). - filter = - if custom and haveCompletionEngine - (suggestion) -> suggestion.type == description - else - null - # For custom search engines, we add an auto-selected suggestion. - if custom - # Note. This suggestion always appears at the top of the suggestion list. Its settings serves to serve - # to configure various vomnibar behaviors. - suggestions.push new Suggestion - queryTerms: queryTerms - type: description - url: Utils.createSearchUrl queryTerms, searchUrl - title: query - relevancy: 1 - insertText: if filter? then query else null - # For all custom search engines, we suppress the leading keyword, for example "w something" becomes - # "something" in the vomnibar. - suppressLeadingKeyword: true - # We complete suggestions only for custom search engines where we have an associated completion - # engine. - completeSuggestions: filter? - # We only use autoSelect and highlight query terms (on custom search engines) when we do not have a - # completer. - autoSelect: not filter? - forceAutoSelect: not filter? - highlightTerms: not filter? - - # Post suggestions and bail if there is no prospect of adding further suggestions. - if queryTerms.length == 0 or not haveCompletionEngine - return onComplete suggestions, { filter } - - onComplete suggestions, - filter: filter - continuation: (existingSuggestions, onComplete) => - suggestions = [] - # For custom search-engine queries, this adds suggestions only if we have a completer. For other queries, - # this adds suggestions for the default search engine (if we have a completer for that). - - # Relevancy: - # - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word - # relevancy). We assume that the completion engine has already factored that in. Also, completion - # engines often handle spelling mistakes, in which case we wouldn't find the query terms in the - # suggestion anyway. - # - The relavancy is higher if the query term is longer. The idea is that search suggestions are more - # likely to be relevant if, after typing some number of characters, the user hasn't yet found - # a useful suggestion from another completer. - # - Scores are weighted such that they retain the order provided by the completion engine. - characterCount = query.length - queryTerms.length + 1 - relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) - - if 0 < existingSuggestions.length - existingSuggestionsMinScore = existingSuggestions[existingSuggestions.length-1].relevancy - if relavancy < existingSuggestionsMinScore and MultiCompleter.maxResults <= existingSuggestions.length - # No suggestion we propose will have a high enough relavancy to beat the existing suggestions, so bail - # immediately. - return onComplete [] - - CompletionEngines.complete searchUrl, queryTerms, (completionSuggestions = []) => - for suggestion in completionSuggestions - suggestions.push new Suggestion - queryTerms: queryTerms - type: description - url: Utils.createSearchUrl suggestion, searchUrl - title: suggestion - relevancy: relavancy *= 0.9 - highlightTerms: false - insertText: suggestion - - # We keep at least three suggestions (if possible) and at most six. We keep more than three only if - # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions - # from other completers. That would potentially be a problem because there is no relationship - # between the relevancy scores produced here and those produced by other completers. - count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length - onComplete suggestions[...count] - - getSearchEngineMatches: (queryTerms, query = queryTerms.join " ") -> - # To allow users to write queries with leading search-engine keywords, leading whitespace disables custom - # search engines; for example, " w" is a regular query. - return {} if /^\s/.test query - # Trailing whitespace is significant when activating a custom search engine; for example, "w" (just a - # regular query) is different from "w " (a custom search engine). - length = queryTerms.length + (if /\s$/.test query then 1 else 0) - (1 < length and @searchEngines[queryTerms[0]]) or {} - - # Static data and methods for parsing the configured search engines. We keep a cache of the search-engine - # mapping in @searchEnginesMap. - @searchEnginesMap: null - - # Parse the custom search engines setting and cache it in SearchEngineCompleter.searchEnginesMap. - @parseSearchEngines: (searchEnginesText) -> - searchEnginesMap = SearchEngineCompleter.searchEnginesMap = {} - for line in searchEnginesText.split /\n/ - tokens = line.trim().split /\s+/ - continue if tokens.length < 2 or tokens[0].startsWith('"') or tokens[0].startsWith("#") - keywords = tokens[0].split ":" - continue unless keywords.length == 2 and not keywords[1] # So, like: [ "w", "" ]. - searchEnginesMap[keywords[0]] = - keyword: keywords[0] - searchUrl: tokens[1] - description: tokens[2..].join(" ") - - # Fetch the search-engine map, building it if necessary. - @getSearchEngines: -> - unless SearchEngineCompleter.searchEnginesMap? - SearchEngineCompleter.parseSearchEngines Settings.get "searchEngines" - SearchEngineCompleter.searchEnginesMap + @searchEngineConfig.use (engines) => + keyword = queryTerms[0] + + { custom, searchUrl, description, queryTerms } = + if engines[keyword]? and (1 < queryTerms.length or /\s$/.test query) + { searchUrl, description } = engines[keyword] + custom: true + searchUrl: searchUrl + description: description + queryTerms: queryTerms[1..] + else + custom: false + searchUrl: Settings.get "searchUrl" + description: "search" + queryTerms: queryTerms + + query = queryTerms.join " " + haveCompletionEngine = CompletionEngines.haveCompletionEngine searchUrl + + # This distinguishes two very different kinds of vomnibar baviours, the newer bahviour (true) and the + # legacy behavior (false). We retain the latter for the default search engine, and for custom search + # engines for which we do not have a completion engine. + version2 = custom and haveCompletionEngine + + # If this is a custom search engine and we have a completer, then we exclude results from other + # completers. + filter = if version2 then (suggestion) -> suggestion.type == description else null + + suggestions = [] + + # For custom search engines, we add a single, top-ranked entry for the unmodified query. This + # suggestion always appears at the top of the suggestion list. Its setting serve to define various + # vomnibar behaviors. + if custom + suggestions.push new Suggestion + queryTerms: queryTerms + type: description + url: Utils.createSearchUrl queryTerms, searchUrl + title: query + relevancy: 1 + insertText: if version2 then query else null + # We suppress the leading keyword, for example "w something" becomes "something" in the vomnibar. + suppressLeadingKeyword: true + # Should we highlight (via the selection) the longest continuation of the current query which is + # contained in all completions? + completeSuggestions: version2 + # Toggles for the legacy behaviour. + autoSelect: not version2 + forceAutoSelect: not version2 + highlightTerms: not version2 + + # Post suggestions and bail if there is no prospect of adding further suggestions. + if queryTerms.length == 0 or not haveCompletionEngine + return onComplete suggestions, { filter } + + # Post any initial suggestion, and then deliver suggestions from completion engines as a continuation + # (so, asynchronously). + onComplete suggestions, + filter: filter + continuation: (existingSuggestions, onComplete) => + suggestions = [] + # Relevancy: + # - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word + # relevancy). We assume that the completion engine has already factored that in. Also, completion + # engines often handle spelling mistakes, in which case we wouldn't find the query terms in the + # suggestion anyway. + # - The relavancy is higher if the query term is longer. The idea is that search suggestions are more + # likely to be relevant if, after typing some number of characters, the user hasn't yet found + # a useful suggestion from another completer. + # - Scores are weighted such that they retain the order provided by the completion engine. + characterCount = query.length - queryTerms.length + 1 + relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) + + if 0 < existingSuggestions.length + existingSuggestionsMinScore = existingSuggestions[existingSuggestions.length-1].relevancy + if relavancy < existingSuggestionsMinScore and MultiCompleter.maxResults <= existingSuggestions.length + # No suggestion we propose will have a high enough relavancy to beat the existing suggestions, so bail + # immediately. + return onComplete [] + + CompletionEngines.complete searchUrl, queryTerms, (completionSuggestions = []) => + for suggestion in completionSuggestions + suggestions.push new Suggestion + queryTerms: queryTerms + type: description + url: Utils.createSearchUrl suggestion, searchUrl + title: suggestion + relevancy: relavancy *= 0.9 + highlightTerms: false + insertText: suggestion + + # We keep at least three suggestions (if possible) and at most six. We keep more than three only if + # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions + # from other completers. That would potentially be a problem because there is no relationship + # between the relevancy scores produced here and those produced by other completers. + count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length + onComplete suggestions[...count] + + cancel: -> + CompletionEngines.cancel() # A completer which calls filter() on many completers, aggregates the results, ranks them, and returns the top # 10. Queries from the vomnibar frontend script come through a multi completer. diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index 44ed897d..11f492d7 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -32,9 +32,6 @@ root.Settings = Settings = root.Commands.parseCustomKeyMappings value root.refreshCompletionKeysAfterMappingSave() - searchEngines: (value) -> - root.SearchEngineCompleter.parseSearchEngines value - exclusionRules: (value) -> root.Exclusions.postUpdateHook value -- cgit v1.2.3 From 5fdbb8e579c068a54e9a397097d87063a3d8a146 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 05:57:47 +0100 Subject: Search completion; rework SimpleCache. --- background_scripts/completion.coffee | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 94109b84..024ea54c 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -355,11 +355,14 @@ class TabCompleter tabRecency.recencyScore(suggestion.tabId) class SearchEngineCompleter - searchEngineConfig: null + searchEngines: null + + cancel: -> + CompletionEngines.cancel() refresh: (port) -> # Load and parse the search-engine configuration. - @searchEngineConfig = new AsyncDataFetcher (callback) -> + @searchEngines = new AsyncDataFetcher (callback) -> engines = {} for line in Settings.get("searchEngines").split "\n" line = line.trim() @@ -373,18 +376,18 @@ class SearchEngineCompleter searchUrl: tokens[1] description: description - # Deliver the resulting engines lookup table. + # Deliver the resulting engines AsyncDataFetcher lookup table. callback engines # Let the vomnibar know the custom search engine keywords. port.postMessage - handler: "customSearchEngineKeywords" + handler: "keywords" keywords: key for own key of engines filter: ({ queryTerms, query }, onComplete) -> return onComplete [] if queryTerms.length == 0 - @searchEngineConfig.use (engines) => + @searchEngines.use (engines) => keyword = queryTerms[0] { custom, searchUrl, description, queryTerms } = @@ -482,9 +485,6 @@ class SearchEngineCompleter count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length onComplete suggestions[...count] - cancel: -> - CompletionEngines.cancel() - # A completer which calls filter() on many completers, aggregates the results, ranks them, and returns the top # 10. Queries from the vomnibar frontend script come through a multi completer. class MultiCompleter -- cgit v1.2.3 From 198dd8fa89148f3389c289bf71c40b9ab1a5681f Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 06:45:37 +0100 Subject: Search completion; refactor job-running logic. --- background_scripts/completion.coffee | 130 ++++++++++++++++------------------- 1 file changed, 60 insertions(+), 70 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 024ea54c..40123337 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -499,76 +499,66 @@ class MultiCompleter cancel: (port) -> completer.cancel? port for completer in @completers - filter: do -> - defaultCallbackOptions = - # Completers may provide a continuation function. This will be run after all completers have posted - # their suggestions, and is used to post additional (slow) asynchronous suggestions (e.g. search-engine - # completions fetched over HTTP). - continuation: null - # Completers may provide a filter function. This allows one completer to filter out suggestions from - # other completers. - filter: null - - (request, onComplete) -> - @debug = true - # Allow only one query to run at a time, and remember the most recent query. - return @mostRecentQuery = arguments if @filterInProgress - - { queryTerms } = request - RegexpCache.clear() - @mostRecentQuery = null - @filterInProgress = true - suggestions = [] - continuations = [] - filters = [] - activeCompleters = [0...@completers.length] - - # Call filter() on every completer and wait for them all to finish before filtering and posting the - # results, then calling any continuations. - for completer, index in @completers - do (index) => - completer.filter request, (newSuggestions = [], { continuation, filter } = defaultCallbackOptions) => - - # Store the results. - suggestions.push newSuggestions... - continuations.push continuation if continuation? - filters.push filter if filter? - - activeCompleters = activeCompleters.filter (i) -> i != index - if activeCompleters.length == 0 - # All the completers have now yielded their (initial) results, we're good to go. - - # Apply filters. - suggestions = suggestions.filter filter for filter in filters - - # Should we run continuations? - shouldRunContinuations = 0 < continuations.length and not @mostRecentQuery? - - # Post results, unless there are none AND we will be running a continuation. This avoids - # collapsing the vomnibar briefly before expanding it again, which looks ugly. - unless suggestions.length == 0 and shouldRunContinuations - onComplete - results: @prepareSuggestions queryTerms, suggestions - mayCacheResults: continuations.length == 0 - - # Run any continuations, unless there's a pending query. - if shouldRunContinuations - for continuation in continuations - console.log "launching continuation..." if @debug - continuation suggestions, (newSuggestions) => - console.log "posting continuation" if @debug - suggestions.push newSuggestions... - onComplete - results: @prepareSuggestions queryTerms, suggestions - # FIXME(smblott) This currently assumes that there is at most one continuation. We - # should really be counting pending/completed continuations. - mayCacheResults: true - - # Admit subsequent queries, and launch any pending query. - @filterInProgress = false - if @mostRecentQuery - console.log "running pending query:", @mostRecentQuery[0] if @debug - @filter @mostRecentQuery... + filter: (request, onComplete) -> + @debug = true + # Allow only one query to run at a time. + return @mostRecentQuery = arguments if @filterInProgress + + RegexpCache.clear() + { queryTerms } = request + + @mostRecentQuery = null + @filterInProgress = true + + suggestions = [] + continuations = [] + filters = [] + + # Run each of the completers (asynchronously). + jobs = new JobRunner @completers.map (completer) -> + (callback) -> + completer.filter request, (newSuggestions = [], { continuation, filter } = {}) -> + suggestions.push newSuggestions... + continuations.push continuation if continuation? + filters.push filter if filter? + callback() + + # Once all completers have finished, process and post the results, and run any continuations or pending + # queries. + jobs.onReady => + # Apply filters. + suggestions = suggestions.filter filter for filter in filters + + # Should we run continuations? + shouldRunContinuations = 0 < continuations.length and not @mostRecentQuery? + + # Post results, unless there are none AND we will be running a continuation. This avoids + # collapsing the vomnibar briefly before expanding it again, which looks ugly. + unless suggestions.length == 0 and shouldRunContinuations + onComplete + results: @prepareSuggestions queryTerms, suggestions + mayCacheResults: continuations.length == 0 + + # Run any continuations (asynchronously). + if shouldRunContinuations + continuationJobs = new JobRunner continuations.map (continuation) -> + (callback) -> + continuation suggestions, (newSuggestions) -> + suggestions.push newSuggestions... + callback() + + continuationJobs.onReady => + # We post these results even if a new query has started. The vomnibar will not display the + # completions, but will cache the results. + onComplete + results: @prepareSuggestions queryTerms, suggestions + mayCacheResults: true + + # Admit subsequent queries, and launch any pending query. + @filterInProgress = false + if @mostRecentQuery + console.log "running pending query:", @mostRecentQuery[0] if @debug + @filter @mostRecentQuery... prepareSuggestions: (queryTerms, suggestions) -> suggestion.computeRelevancy queryTerms for suggestion in suggestions -- cgit v1.2.3 From ef4d9cb7a6f5ddceb643deaeb94edc5886baad93 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 06:55:30 +0100 Subject: Search completion; minor tweaks. --- background_scripts/completion.coffee | 4 ++-- background_scripts/main.coffee | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 40123337..447c47a2 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -376,10 +376,10 @@ class SearchEngineCompleter searchUrl: tokens[1] description: description - # Deliver the resulting engines AsyncDataFetcher lookup table. + # Deliver the resulting engines AsyncDataFetcher table/data. callback engines - # Let the vomnibar know the custom search engine keywords. + # Let the vomnibar in the front end know the custom search engine keywords. port.postMessage handler: "keywords" keywords: key for own key of engines diff --git a/background_scripts/main.coffee b/background_scripts/main.coffee index 612f6170..d0411f6e 100644 --- a/background_scripts/main.coffee +++ b/background_scripts/main.coffee @@ -62,7 +62,11 @@ completers = completionHandlers = filter: (completer, request, port) -> completer.filter request, (response) -> - port.postMessage extend request, extend response, handler: "completions" + # We use try here because this may fail if the sender has already navigated away from the original page. + # This can happen, for example, when posting completion suggestions from the SearchEngineCompleter + # (which can be slow). + try + port.postMessage extend request, extend response, handler: "completions" refresh: (completer, _, port) -> completer.refresh port cancel: (completer, _, port) -> completer.cancel port -- cgit v1.2.3 From 0fa0f17190459bb37bfe8752f98cd8b37e689437 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 07:15:55 +0100 Subject: Search completion; highlight also for default search engine. --- background_scripts/completion.coffee | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 447c47a2..c529f376 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -437,6 +437,8 @@ class SearchEngineCompleter autoSelect: not version2 forceAutoSelect: not version2 highlightTerms: not version2 + # Do not use this entry for vomnibar completion. + highlightCommonMatches: false # Post suggestions and bail if there is no prospect of adding further suggestions. if queryTerms.length == 0 or not haveCompletionEngine @@ -477,6 +479,8 @@ class SearchEngineCompleter relevancy: relavancy *= 0.9 highlightTerms: false insertText: suggestion + # Do use this entry for vomnibar completion. + highlightCommonMatches: true # We keep at least three suggestions (if possible) and at most six. We keep more than three only if # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions -- cgit v1.2.3 From 1a337a261a6dd6deffa836cbd949bb036e103f36 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 08:59:14 +0100 Subject: Search completion; reuse previous query. --- background_scripts/completion_engines.coffee | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 85062c49..425ff47e 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -176,7 +176,7 @@ CompletionEngines = # to generate a key. We mix in some junk generated by pwgen. A key clash might be possible, but # vanishingly unlikely. junk = "//Zi?ei5;o//" - completionCacheKey = searchUrl + junk + queryTerms.join junk + completionCacheKey = searchUrl + junk + queryTerms.map((s) -> s.toLowerCase()).join junk @completionCache ?= new SimpleCache 60 * 60 * 1000, 2000 # One hour, 2000 entries. if @completionCache.has completionCacheKey # We add a short delay, even for a cache hit. This avoids an ugly flicker when the additional @@ -186,6 +186,21 @@ CompletionEngines = callback @completionCache.get completionCacheKey return + if @mostRecentQuery? and @mostRecentSuggestions? + # If the user appears to be typing a continuation of the characters in all of the most recent query, + # then we can re-use the results of the previous query. + reusePreviousSuggestions = do (query) => + query = queryTerms.join(" ").toLowerCase() + return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase() + previousSuggestions = @mostRecentSuggestions.map (s) -> s.toLowerCase() + return false unless query.length <= Utils.longestCommonPrefix previousSuggestions + true + + if reusePreviousSuggestions + console.log "reuse previous query", @mostRecentQuery if @debug + @mostRecentQuery = queryTerms.join " " + return callback @completionCache.set completionCacheKey, @mostRecentSuggestions + fetchSuggestions = (engine, callback) => url = engine.getUrl queryTerms query = queryTerms.join(" ").toLowerCase() @@ -227,6 +242,8 @@ CompletionEngines = queue = @inTransit[completionCacheKey] = [] engine = @lookupEngine searchUrl fetchSuggestions engine, (suggestions) => + @mostRecentQuery = queryTerms.join " " + @mostRecentSuggestions = suggestions @completionCache.set completionCacheKey, suggestions unless engine.doNotCache callback suggestions delete @inTransit[completionCacheKey] -- cgit v1.2.3 From 146957c9dae54c31ce4676b5a4b60a0000c05487 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 09:49:09 +0100 Subject: Search completion; use cached suggestions synchronously. --- background_scripts/completion.coffee | 49 ++++++++++++++++++++-------- background_scripts/completion_engines.coffee | 30 ++++++++++++----- 2 files changed, 57 insertions(+), 22 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index c529f376..4f0401f0 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -406,6 +406,18 @@ class SearchEngineCompleter query = queryTerms.join " " haveCompletionEngine = CompletionEngines.haveCompletionEngine searchUrl + # Relevancy: + # - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word + # relevancy). We assume that the completion engine has already factored that in. Also, completion + # engines often handle spelling mistakes, in which case we wouldn't find the query terms in the + # suggestion anyway. + # - The relavancy is higher if the query term is longer. The idea is that search suggestions are more + # likely to be relevant if, after typing some number of characters, the user hasn't yet found + # a useful suggestion from another completer. + # - Scores are weighted such that they retain the order provided by the completion engine. + characterCount = query.length - queryTerms.length + 1 + relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) + # This distinguishes two very different kinds of vomnibar baviours, the newer bahviour (true) and the # legacy behavior (false). We retain the latter for the default search engine, and for custom search # engines for which we do not have a completion engine. @@ -440,8 +452,30 @@ class SearchEngineCompleter # Do not use this entry for vomnibar completion. highlightCommonMatches: false - # Post suggestions and bail if there is no prospect of adding further suggestions. - if queryTerms.length == 0 or not haveCompletionEngine + mkSuggestion = do -> + (suggestion) -> + new Suggestion + queryTerms: queryTerms + type: description + url: Utils.createSearchUrl suggestion, searchUrl + title: suggestion + relevancy: relavancy *= 0.9 + highlightTerms: false + insertText: suggestion + # Do use this entry for vomnibar completion. + highlightCommonMatches: true + + # If we have cached suggestions, then we can bundle them immediately (otherwise we'll have to do an HTTP + # request, which we do asynchronously). This is a synchronous call (for cached suggestions only) + # because no callback is provided. + cachedSuggestions = CompletionEngines.complete searchUrl, queryTerms + + # Post suggestions and bail if we already have all of the suggestions, or if there is no prospect of + # adding further suggestions. + if queryTerms.length == 0 or cachedSuggestions? or not haveCompletionEngine + if cachedSuggestions? + console.log "using cached suggestions" + suggestions.push cachedSuggestions.map(mkSuggestion)... return onComplete suggestions, { filter } # Post any initial suggestion, and then deliver suggestions from completion engines as a continuation @@ -450,17 +484,6 @@ class SearchEngineCompleter filter: filter continuation: (existingSuggestions, onComplete) => suggestions = [] - # Relevancy: - # - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word - # relevancy). We assume that the completion engine has already factored that in. Also, completion - # engines often handle spelling mistakes, in which case we wouldn't find the query terms in the - # suggestion anyway. - # - The relavancy is higher if the query term is longer. The idea is that search suggestions are more - # likely to be relevant if, after typing some number of characters, the user hasn't yet found - # a useful suggestion from another completer. - # - Scores are weighted such that they retain the order provided by the completion engine. - characterCount = query.length - queryTerms.length + 1 - relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) if 0 < existingSuggestions.length existingSuggestionsMinScore = existingSuggestions[existingSuggestions.length-1].relevancy diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 425ff47e..51799971 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -161,10 +161,16 @@ CompletionEngines = # - queryTerms are the query terms. # - callback will be applied to a list of suggestion strings (which may be an empty list, if anything goes # wrong). - complete: (searchUrl, queryTerms, callback) -> + complete: (searchUrl, queryTerms, callback = null) -> @mostRecentHandler = null query = queryTerms.join "" + # If no callback is provided, then we're to provide suggestions only if we can do so synchronously (ie. + # from a cache). In this case we return the results and don't call callback. Return null if we cannot + # service the request synchronously. + returnResultsOnlyFromCache = not callback? + callback ?= (suggestions) -> suggestions + # We don't complete single characters: the results are usually useless. return callback [] unless 1 < query.length @@ -179,16 +185,20 @@ CompletionEngines = completionCacheKey = searchUrl + junk + queryTerms.map((s) -> s.toLowerCase()).join junk @completionCache ?= new SimpleCache 60 * 60 * 1000, 2000 # One hour, 2000 entries. if @completionCache.has completionCacheKey - # We add a short delay, even for a cache hit. This avoids an ugly flicker when the additional - # suggestions are posted. - Utils.setTimeout 75, => - console.log "hit", completionCacheKey if @debug - callback @completionCache.get completionCacheKey - return + if returnResultsOnlyFromCache + return @completionCache.get completionCacheKey + else + # We add a short delay, even for a cache hit. This avoids an ugly flicker when the additional + # suggestions are posted. + Utils.setTimeout 75, => + console.log "hit", completionCacheKey if @debug + callback @completionCache.get completionCacheKey + return if @mostRecentQuery? and @mostRecentSuggestions? - # If the user appears to be typing a continuation of the characters in all of the most recent query, - # then we can re-use the results of the previous query. + # If the user appears to be typing a continuation of the characters of the most recent query, and those + # characters are also common to all of the most recent suggestions, then we can re-use the previous + # suggestions. reusePreviousSuggestions = do (query) => query = queryTerms.join(" ").toLowerCase() return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase() @@ -201,6 +211,8 @@ CompletionEngines = @mostRecentQuery = queryTerms.join " " return callback @completionCache.set completionCacheKey, @mostRecentSuggestions + return null if returnResultsOnlyFromCache + fetchSuggestions = (engine, callback) => url = engine.getUrl queryTerms query = queryTerms.join(" ").toLowerCase() -- cgit v1.2.3 From 5191ee986615f45bfaedb389e3b34fd0a1ce9ca9 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 08:59:14 +0100 Subject: Search completion; reuse previous query. --- background_scripts/completion_engines.coffee | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'background_scripts') diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 51799971..729c68b0 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -213,6 +213,21 @@ CompletionEngines = return null if returnResultsOnlyFromCache + if @mostRecentQuery? and @mostRecentSuggestions? + # If the user appears to be typing a continuation of the characters in all of the most recent query, + # then we can re-use the results of the previous query. + reusePreviousSuggestions = do (query) => + query = queryTerms.join(" ").toLowerCase() + return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase() + previousSuggestions = @mostRecentSuggestions.map (s) -> s.toLowerCase() + return false unless query.length <= Utils.longestCommonPrefix previousSuggestions + true + + if reusePreviousSuggestions + console.log "reuse previous query", @mostRecentQuery if @debug + @mostRecentQuery = queryTerms.join " " + return callback @completionCache.set completionCacheKey, @mostRecentSuggestions + fetchSuggestions = (engine, callback) => url = engine.getUrl queryTerms query = queryTerms.join(" ").toLowerCase() -- cgit v1.2.3 From e56b5fc0129dd67d7978ee6453d366cb3642a4e2 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 09:49:09 +0100 Subject: Search completion; use cached suggestions synchronously. Conflicts: background_scripts/completion_engines.coffee --- background_scripts/completion_engines.coffee | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 729c68b0..78782513 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -214,8 +214,9 @@ CompletionEngines = return null if returnResultsOnlyFromCache if @mostRecentQuery? and @mostRecentSuggestions? - # If the user appears to be typing a continuation of the characters in all of the most recent query, - # then we can re-use the results of the previous query. + # If the user appears to be typing a continuation of the characters of the most recent query, and those + # characters are also common to all of the most recent suggestions, then we can re-use the previous + # suggestions. reusePreviousSuggestions = do (query) => query = queryTerms.join(" ").toLowerCase() return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase() @@ -228,6 +229,8 @@ CompletionEngines = @mostRecentQuery = queryTerms.join " " return callback @completionCache.set completionCacheKey, @mostRecentSuggestions + return null if returnResultsOnlyFromCache + fetchSuggestions = (engine, callback) => url = engine.getUrl queryTerms query = queryTerms.join(" ").toLowerCase() -- cgit v1.2.3 From cdf87b7a870303df9baee7161b15362242f65dcb Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 11:48:38 +0100 Subject: Search completion; strip duplicated code. --- background_scripts/completion_engines.coffee | 18 ------------------ 1 file changed, 18 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 78782513..51799971 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -213,24 +213,6 @@ CompletionEngines = return null if returnResultsOnlyFromCache - if @mostRecentQuery? and @mostRecentSuggestions? - # If the user appears to be typing a continuation of the characters of the most recent query, and those - # characters are also common to all of the most recent suggestions, then we can re-use the previous - # suggestions. - reusePreviousSuggestions = do (query) => - query = queryTerms.join(" ").toLowerCase() - return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase() - previousSuggestions = @mostRecentSuggestions.map (s) -> s.toLowerCase() - return false unless query.length <= Utils.longestCommonPrefix previousSuggestions - true - - if reusePreviousSuggestions - console.log "reuse previous query", @mostRecentQuery if @debug - @mostRecentQuery = queryTerms.join " " - return callback @completionCache.set completionCacheKey, @mostRecentSuggestions - - return null if returnResultsOnlyFromCache - fetchSuggestions = (engine, callback) => url = engine.getUrl queryTerms query = queryTerms.join(" ").toLowerCase() -- cgit v1.2.3 From 09b2aad039c7894c6023100d03c9941649843b77 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 13:03:32 +0100 Subject: Search completion; more minor tweaks. --- background_scripts/completion.coffee | 27 ++++++++++++++------------- background_scripts/completion_engines.coffee | 17 ++++++++++------- 2 files changed, 24 insertions(+), 20 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 4f0401f0..4fa91b9d 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -384,7 +384,7 @@ class SearchEngineCompleter handler: "keywords" keywords: key for own key of engines - filter: ({ queryTerms, query }, onComplete) -> + filter: ({ queryTerms, query, maxResults }, onComplete) -> return onComplete [] if queryTerms.length == 0 @searchEngines.use (engines) => @@ -421,11 +421,11 @@ class SearchEngineCompleter # This distinguishes two very different kinds of vomnibar baviours, the newer bahviour (true) and the # legacy behavior (false). We retain the latter for the default search engine, and for custom search # engines for which we do not have a completion engine. - version2 = custom and haveCompletionEngine + useExclusiveVomnibar = custom and haveCompletionEngine # If this is a custom search engine and we have a completer, then we exclude results from other # completers. - filter = if version2 then (suggestion) -> suggestion.type == description else null + filter = if useExclusiveVomnibar then (suggestion) -> suggestion.type == description else null suggestions = [] @@ -439,17 +439,17 @@ class SearchEngineCompleter url: Utils.createSearchUrl queryTerms, searchUrl title: query relevancy: 1 - insertText: if version2 then query else null + insertText: if useExclusiveVomnibar then query else null # We suppress the leading keyword, for example "w something" becomes "something" in the vomnibar. suppressLeadingKeyword: true # Should we highlight (via the selection) the longest continuation of the current query which is # contained in all completions? - completeSuggestions: version2 + completeSuggestions: useExclusiveVomnibar # Toggles for the legacy behaviour. - autoSelect: not version2 - forceAutoSelect: not version2 - highlightTerms: not version2 - # Do not use this entry for vomnibar completion. + autoSelect: not useExclusiveVomnibar + forceAutoSelect: not useExclusiveVomnibar + highlightTerms: not useExclusiveVomnibar + # Do not use this entry for vomnibar completion (highlighting the common text of the suggestions). highlightCommonMatches: false mkSuggestion = do -> @@ -487,9 +487,10 @@ class SearchEngineCompleter if 0 < existingSuggestions.length existingSuggestionsMinScore = existingSuggestions[existingSuggestions.length-1].relevancy - if relavancy < existingSuggestionsMinScore and MultiCompleter.maxResults <= existingSuggestions.length + if relavancy < existingSuggestionsMinScore and maxResults <= existingSuggestions.length # No suggestion we propose will have a high enough relavancy to beat the existing suggestions, so bail # immediately. + console.log "skip: cannot add completions" if @debug return onComplete [] CompletionEngines.complete searchUrl, queryTerms, (completionSuggestions = []) => @@ -509,16 +510,15 @@ class SearchEngineCompleter # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions # from other completers. That would potentially be a problem because there is no relationship # between the relevancy scores produced here and those produced by other completers. - count = Math.min 6, Math.max 3, MultiCompleter.maxResults - existingSuggestions.length + count = Math.min 6, Math.max 3, maxResults - existingSuggestions.length onComplete suggestions[...count] # A completer which calls filter() on many completers, aggregates the results, ranks them, and returns the top # 10. Queries from the vomnibar frontend script come through a multi completer. class MultiCompleter - @maxResults: 10 + maxResults: 10 constructor: (@completers) -> - @maxResults = MultiCompleter.maxResults refresh: (port) -> completer.refresh? port for completer in @completers @@ -533,6 +533,7 @@ class MultiCompleter RegexpCache.clear() { queryTerms } = request + request.maxResults = @maxResults @mostRecentQuery = null @filterInProgress = true diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 51799971..ac5c86aa 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -126,7 +126,8 @@ CompletionEngines = # The amount of time to wait for new requests before launching the HTTP request. The intention is to cut # down on the number of HTTP requests we issue. - delay: 100 + # delay: 100 + delay: 0 get: (searchUrl, url, callback) -> xhr = new XMLHttpRequest() @@ -190,20 +191,22 @@ CompletionEngines = else # We add a short delay, even for a cache hit. This avoids an ugly flicker when the additional # suggestions are posted. - Utils.setTimeout 75, => + Utils.setTimeout 50, => console.log "hit", completionCacheKey if @debug callback @completionCache.get completionCacheKey return if @mostRecentQuery? and @mostRecentSuggestions? - # If the user appears to be typing a continuation of the characters of the most recent query, and those - # characters are also common to all of the most recent suggestions, then we can re-use the previous - # suggestions. + # If the user appears to be typing a continuation of the characters of the most recent query, then we + # can re-use the previous suggestions. reusePreviousSuggestions = do (query) => query = queryTerms.join(" ").toLowerCase() + # Verify that the previous query is a prefix of the current query. return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase() - previousSuggestions = @mostRecentSuggestions.map (s) -> s.toLowerCase() - return false unless query.length <= Utils.longestCommonPrefix previousSuggestions + # Ensure that every previous suggestion contains the text of the new query. + for suggestion in (@mostRecentSuggestions.map (s) -> s.toLowerCase()) + return false unless 0 <= suggestion.indexOf query + # Ok. Re-use the suggestion. true if reusePreviousSuggestions -- cgit v1.2.3 From 8493811a4279950194cc8b1f5941cf9730cda1f0 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 15:08:01 +0100 Subject: Search completion; even more minor tweaks. --- background_scripts/completion.coffee | 2 +- background_scripts/completion_engines.coffee | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 4fa91b9d..ccb5e4e6 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -474,7 +474,7 @@ class SearchEngineCompleter # adding further suggestions. if queryTerms.length == 0 or cachedSuggestions? or not haveCompletionEngine if cachedSuggestions? - console.log "using cached suggestions" + console.log "using cached suggestions:", query suggestions.push cachedSuggestions.map(mkSuggestion)... return onComplete suggestions, { filter } diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index ac5c86aa..638d7f60 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -210,7 +210,7 @@ CompletionEngines = true if reusePreviousSuggestions - console.log "reuse previous query", @mostRecentQuery if @debug + console.log "reuse previous query:", @mostRecentQuery if @debug @mostRecentQuery = queryTerms.join " " return callback @completionCache.set completionCacheKey, @mostRecentSuggestions -- cgit v1.2.3 From 6b52c9e6397ac0a040c1bd46b7e6825a0a8415d2 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 15:19:25 +0100 Subject: Search completion; move completion engines to their own file. --- background_scripts/completion.coffee | 8 +- background_scripts/completion_engines.coffee | 170 ++------------------------- background_scripts/completion_search.coffee | 140 ++++++++++++++++++++++ 3 files changed, 153 insertions(+), 165 deletions(-) create mode 100644 background_scripts/completion_search.coffee (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index ccb5e4e6..7d39ccae 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -358,7 +358,7 @@ class SearchEngineCompleter searchEngines: null cancel: -> - CompletionEngines.cancel() + CompletionSearch.cancel() refresh: (port) -> # Load and parse the search-engine configuration. @@ -404,7 +404,7 @@ class SearchEngineCompleter queryTerms: queryTerms query = queryTerms.join " " - haveCompletionEngine = CompletionEngines.haveCompletionEngine searchUrl + haveCompletionEngine = CompletionSearch.haveCompletionEngine searchUrl # Relevancy: # - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word @@ -468,7 +468,7 @@ class SearchEngineCompleter # If we have cached suggestions, then we can bundle them immediately (otherwise we'll have to do an HTTP # request, which we do asynchronously). This is a synchronous call (for cached suggestions only) # because no callback is provided. - cachedSuggestions = CompletionEngines.complete searchUrl, queryTerms + cachedSuggestions = CompletionSearch.complete searchUrl, queryTerms # Post suggestions and bail if we already have all of the suggestions, or if there is no prospect of # adding further suggestions. @@ -493,7 +493,7 @@ class SearchEngineCompleter console.log "skip: cannot add completions" if @debug return onComplete [] - CompletionEngines.complete searchUrl, queryTerms, (completionSuggestions = []) => + CompletionSearch.complete searchUrl, queryTerms, (completionSuggestions = []) => for suggestion in completionSuggestions suggestions.push new Suggestion queryTerms: queryTerms diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 638d7f60..07ecfa26 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -4,18 +4,20 @@ # # Each completion engine defines three functions: # -# 1. "match" - This takes a searchUrl, and returns a boolean indicating whether this completion engine can +# 1. "match" - This takes a searchUrl and returns a boolean indicating whether this completion engine can # perform completion for the given search engine. # # 2. "getUrl" - This takes a list of query terms (queryTerms) and generates a completion URL, that is, a URL # which will provide completions for this completion engine. # # 3. "parse" - This takes a successful XMLHttpRequest object (the request has completed successfully), and -# returns a list of suggestions (a list of strings). +# returns a list of suggestions (a list of strings). This method is always executed within the context +# of a try/catch block, so errors do not propagate. +# +# Each new completion engine must be add to the list "CompletionEngines" at the bottom of this file. +# +# The lookup logic which uses these completion engines is in "./completion_search.coffee". # -# The main completion entry point is CompletionEngines.complete(). This implements all lookup and caching -# logic. It is possible to add new completion engines without changing the CompletionEngines infrastructure -# itself. # A base class for common regexp-based matching engines. class RegexpEngine @@ -106,7 +108,8 @@ class DummyCompletionEngine getUrl: -> chrome.runtime.getURL "content_scripts/vimium.css" parse: -> [] -completionEngines = [ +# Note: Order matters here. +CompletionEngines = [ Youtube Google DuckDuckGo @@ -116,160 +119,5 @@ completionEngines = [ DummyCompletionEngine ] -# A note on caching. -# Some completion engines allow caching, and Chrome serves up cached responses to requests (e.g. Google, -# Wikipedia, YouTube). Others do not (e.g. Bing, DuckDuckGo, Amazon). A completion engine can set -# @doNotCache to a truthy value to disable caching in cases where it is unnecessary. - -CompletionEngines = - debug: true - - # The amount of time to wait for new requests before launching the HTTP request. The intention is to cut - # down on the number of HTTP requests we issue. - # delay: 100 - delay: 0 - - get: (searchUrl, url, callback) -> - xhr = new XMLHttpRequest() - xhr.open "GET", url, true - xhr.timeout = 1000 - xhr.ontimeout = xhr.onerror = -> callback null - xhr.send() - - xhr.onreadystatechange = -> - if xhr.readyState == 4 - callback(if xhr.status == 200 then xhr else null) - - # Look up the completion engine for this searchUrl. Because of DummyCompletionEngine, above, we know there - # will always be a match. Imagining that there may be many completion engines, and knowing that this is - # called for every query, we cache the result. - lookupEngine: (searchUrl) -> - @engineCache ?= new SimpleCache 30 * 60 * 60 * 1000 # 30 hours (these are small, we can keep them longer). - if @engineCache.has searchUrl - @engineCache.get searchUrl - else - for engine in completionEngines - engine = new engine() - return @engineCache.set searchUrl, engine if engine.match searchUrl - - # True if we have a completion engine for this search URL, undefined otherwise. - haveCompletionEngine: (searchUrl) -> - not @lookupEngine(searchUrl).dummy - - # This is the main entry point. - # - searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"), or a custome search engine's URL. - # This is only used as a key for determining the relevant completion engine. - # - queryTerms are the query terms. - # - callback will be applied to a list of suggestion strings (which may be an empty list, if anything goes - # wrong). - complete: (searchUrl, queryTerms, callback = null) -> - @mostRecentHandler = null - query = queryTerms.join "" - - # If no callback is provided, then we're to provide suggestions only if we can do so synchronously (ie. - # from a cache). In this case we return the results and don't call callback. Return null if we cannot - # service the request synchronously. - returnResultsOnlyFromCache = not callback? - callback ?= (suggestions) -> suggestions - - # We don't complete single characters: the results are usually useless. - return callback [] unless 1 < query.length - - # We don't complete regular URLs or Javascript URLs. - return callback [] if 1 == queryTerms.length and Utils.isUrl query - return callback [] if Utils.hasJavascriptPrefix query - - # Cache completions. However, completions depend upon both the searchUrl and the query terms. So we need - # to generate a key. We mix in some junk generated by pwgen. A key clash might be possible, but - # vanishingly unlikely. - junk = "//Zi?ei5;o//" - completionCacheKey = searchUrl + junk + queryTerms.map((s) -> s.toLowerCase()).join junk - @completionCache ?= new SimpleCache 60 * 60 * 1000, 2000 # One hour, 2000 entries. - if @completionCache.has completionCacheKey - if returnResultsOnlyFromCache - return @completionCache.get completionCacheKey - else - # We add a short delay, even for a cache hit. This avoids an ugly flicker when the additional - # suggestions are posted. - Utils.setTimeout 50, => - console.log "hit", completionCacheKey if @debug - callback @completionCache.get completionCacheKey - return - - if @mostRecentQuery? and @mostRecentSuggestions? - # If the user appears to be typing a continuation of the characters of the most recent query, then we - # can re-use the previous suggestions. - reusePreviousSuggestions = do (query) => - query = queryTerms.join(" ").toLowerCase() - # Verify that the previous query is a prefix of the current query. - return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase() - # Ensure that every previous suggestion contains the text of the new query. - for suggestion in (@mostRecentSuggestions.map (s) -> s.toLowerCase()) - return false unless 0 <= suggestion.indexOf query - # Ok. Re-use the suggestion. - true - - if reusePreviousSuggestions - console.log "reuse previous query:", @mostRecentQuery if @debug - @mostRecentQuery = queryTerms.join " " - return callback @completionCache.set completionCacheKey, @mostRecentSuggestions - - return null if returnResultsOnlyFromCache - - fetchSuggestions = (engine, callback) => - url = engine.getUrl queryTerms - query = queryTerms.join(" ").toLowerCase() - @get searchUrl, url, (xhr = null) => - # Parsing the response may fail if we receive an unexpected or an unexpectedly-formatted response. In - # all cases, we fall back to the catch clause, below. Therefore, we "fail safe" in the case of - # incorrect or out-of-date completion engines. - try - suggestions = engine.parse xhr - # Make sure we really do have an iterable of strings. - suggestions = (suggestion for suggestion in suggestions when "string" == typeof suggestion) - # Filter out the query itself. It's not adding anything. - suggestions = (suggestion for suggestion in suggestions when suggestion.toLowerCase() != query) - console.log "GET", url if @debug - catch - suggestions = [] - # We allow failures to be cached, but remove them after just ten minutes. This (it is hoped) avoids - # repeated unnecessary XMLHttpRequest failures over a short period of time. - removeCompletionCacheKey = => @completionCache.set completionCacheKey, null - setTimeout removeCompletionCacheKey, 10 * 60 * 1000 # Ten minutes. - console.log "fail", url if @debug - - callback suggestions - - # We pause in case the user is still typing. - Utils.setTimeout @delay, handler = @mostRecentHandler = => - if handler != @mostRecentHandler - # Bail! Another completion has begun, or the user is typing. - # NOTE: We do *not* call the callback (because we are not providing results, and we don't want allow - # any higher-level component to cache the results -- specifically, the vomnibar itself, via - # callerMayCacheResults). - console.log "bail", completionCacheKey if @debug - return - @mostRecentHandler = null - # Don't allow duplicate identical active requests. This can happen, for example, when the user enters or - # removes a space, or when they enter a character and immediately delete it. - @inTransit ?= {} - unless @inTransit[completionCacheKey]?.push callback - queue = @inTransit[completionCacheKey] = [] - engine = @lookupEngine searchUrl - fetchSuggestions engine, (suggestions) => - @mostRecentQuery = queryTerms.join " " - @mostRecentSuggestions = suggestions - @completionCache.set completionCacheKey, suggestions unless engine.doNotCache - callback suggestions - delete @inTransit[completionCacheKey] - console.log "callbacks", queue.length, completionCacheKey if @debug and 0 < queue.length - callback suggestions for callback in queue - - # Cancel any pending (ie. blocked on @delay) queries. Does not cancel in-flight queries. - cancel: -> - if @mostRecentHandler? - @mostRecentHandler = null - console.log "cancel (user is typing)" if @debug - root = exports ? window root.CompletionEngines = CompletionEngines diff --git a/background_scripts/completion_search.coffee b/background_scripts/completion_search.coffee new file mode 100644 index 00000000..eb27c076 --- /dev/null +++ b/background_scripts/completion_search.coffee @@ -0,0 +1,140 @@ + +CompletionSearch = + debug: true + + # The amount of time to wait for new requests before launching the HTTP request. The intention is to cut + # down on the number of HTTP requests we issue. + delay: 100 + + get: (searchUrl, url, callback) -> + xhr = new XMLHttpRequest() + xhr.open "GET", url, true + xhr.timeout = 1000 + xhr.ontimeout = xhr.onerror = -> callback null + xhr.send() + + xhr.onreadystatechange = -> + if xhr.readyState == 4 + callback(if xhr.status == 200 then xhr else null) + + # Look up the completion engine for this searchUrl. Because of DummyCompletionEngine, above, we know there + # will always be a match. Imagining that there may be many completion engines, and knowing that this is + # called for every query, we cache the result. + lookupEngine: (searchUrl) -> + @engineCache ?= new SimpleCache 30 * 60 * 60 * 1000 # 30 hours (these are small, we can keep them longer). + if @engineCache.has searchUrl + @engineCache.get searchUrl + else + for engine in CompletionEngines + engine = new engine() + return @engineCache.set searchUrl, engine if engine.match searchUrl + + # True if we have a completion engine for this search URL, undefined otherwise. + haveCompletionEngine: (searchUrl) -> + not @lookupEngine(searchUrl).dummy + + # This is the main entry point. + # - searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"), or a custome search engine's URL. + # This is only used as a key for determining the relevant completion engine. + # - queryTerms are the query terms. + # - callback will be applied to a list of suggestion strings (which may be an empty list, if anything goes + # wrong). + complete: (searchUrl, queryTerms, callback = null) -> + query = queryTerms.join "" + + # If no callback is provided, then we're to provide suggestions only if we can do so synchronously (ie. + # from a cache). In this case we just return the results. Return null if we cannot service the request + # synchronously. + returnResultsOnlyFromCache = not callback? + callback ?= (suggestions) -> suggestions + + # We don't complete single characters: the results are usually useless. + return callback [] unless 1 < query.length + + # We don't complete regular URLs or Javascript URLs. + return callback [] if 1 == queryTerms.length and Utils.isUrl query + return callback [] if Utils.hasJavascriptPrefix query + + # Cache completions. However, completions depend upon both the searchUrl and the query terms. So we need + # to generate a key. We mix in some junk generated by pwgen. A key clash might be possible, but + # vanishingly unlikely. + junk = "//Zi?ei5;o//" + completionCacheKey = searchUrl + junk + queryTerms.map((s) -> s.toLowerCase()).join junk + @completionCache ?= new SimpleCache 60 * 60 * 1000, 2000 # One hour, 2000 entries. + if @completionCache.has completionCacheKey + if returnResultsOnlyFromCache + return callback @completionCache.get completionCacheKey + else + # We add a short delay, even for a cache hit. This avoids an ugly flicker when the additional + # suggestions are posted. + Utils.setTimeout 50, => + console.log "hit", completionCacheKey if @debug + callback @completionCache.get completionCacheKey + return + + # If the user appears to be typing a continuation of the characters of the most recent query, then we can + # re-use the previous suggestions. + if @mostRecentQuery? and @mostRecentSuggestions? + reusePreviousSuggestions = do (query) => + query = queryTerms.join(" ").toLowerCase() + # Verify that the previous query is a prefix of the current query. + return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase() + # Ensure that every previous suggestion contains the text of the new query. + for suggestion in (@mostRecentSuggestions.map (s) -> s.toLowerCase()) + return false unless 0 <= suggestion.indexOf query + # Ok. Re-use the suggestion. + true + + if reusePreviousSuggestions + console.log "reuse previous query:", @mostRecentQuery if @debug + @mostRecentQuery = queryTerms.join " " + return callback @completionCache.set completionCacheKey, @mostRecentSuggestions + + # That's all of the caches we can try. Bail if the caller is looking for synchronous results. + return callback null if returnResultsOnlyFromCache + + fetchSuggestions = (engine, callback) => + url = engine.getUrl queryTerms + query = queryTerms.join(" ").toLowerCase() + @get searchUrl, url, (xhr = null) => + # Parsing the response may fail if we receive an unexpected or an unexpectedly-formatted response. In + # all cases, we fall back to the catch clause, below. Therefore, we "fail safe" in the case of + # incorrect or out-of-date completion engines. + try + suggestions = engine.parse xhr + # Filter out the query itself. It's not adding anything. + suggestions = (suggestion for suggestion in suggestions when suggestion.toLowerCase() != query) + console.log "GET", url if @debug + catch + suggestions = [] + # We cache failures too, but remove them after just thirty minutes. + Utils.setTimeout 30 * 60 * 1000, => @completionCache.set completionCacheKey, null + console.log "fail", url if @debug + + callback suggestions + + # We pause in case the user is still typing. + Utils.setTimeout @delay, handler = @mostRecentHandler = => + if handler == @mostRecentHandler + @mostRecentHandler = null + + # Share duplicate requests. First fetch the suggestions... + @inTransit ?= {} + @inTransit[completionCacheKey] ?= new AsyncDataFetcher (callback) => + fetchSuggestions @lookupEngine(searchUrl), callback + + # ... then use the suggestions. + @inTransit[completionCacheKey].use (suggestions) => + @mostRecentQuery = queryTerms.join " " + @mostRecentSuggestions = suggestions + callback @completionCache.set completionCacheKey, suggestions + delete @inTransit[completionCacheKey] + + # Cancel any pending (ie. blocked on @delay) queries. Does not cancel in-flight queries. + cancel: -> + if @mostRecentHandler? + @mostRecentHandler = null + console.log "cancel (user is typing)" if @debug + +root = exports ? window +root.CompletionSearch = CompletionSearch -- cgit v1.2.3 From 91da4285f4d89f9ec43f5f8a05eaa741899d24ee Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 17:16:08 +0100 Subject: Search completion; even more minor tweaks. --- background_scripts/completion.coffee | 123 +++++++++------------------- background_scripts/completion_search.coffee | 97 ++++++++++------------ 2 files changed, 85 insertions(+), 135 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 7d39ccae..4663c091 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -361,7 +361,7 @@ class SearchEngineCompleter CompletionSearch.cancel() refresh: (port) -> - # Load and parse the search-engine configuration. + # Parse the search-engine configuration. @searchEngines = new AsyncDataFetcher (callback) -> engines = {} for line in Settings.get("searchEngines").split "\n" @@ -370,16 +370,17 @@ class SearchEngineCompleter tokens = line.split /\s+/ continue unless 2 <= tokens.length keyword = tokens[0].split(":")[0] + url = tokens[1] description = tokens[2..].join(" ") || "search (#{keyword})" + continue unless Utils.hasFullUrlPrefix url engines[keyword] = keyword: keyword - searchUrl: tokens[1] + searchUrl: url description: description - # Deliver the resulting engines AsyncDataFetcher table/data. callback engines - # Let the vomnibar in the front end know the custom search engine keywords. + # Let the front-end vomnibar know the search-engine keywords. port.postMessage handler: "keywords" keywords: key for own key of engines @@ -388,6 +389,7 @@ class SearchEngineCompleter return onComplete [] if queryTerms.length == 0 @searchEngines.use (engines) => + suggestions = [] keyword = queryTerms[0] { custom, searchUrl, description, queryTerms } = @@ -408,30 +410,27 @@ class SearchEngineCompleter # Relevancy: # - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word - # relevancy). We assume that the completion engine has already factored that in. Also, completion - # engines often handle spelling mistakes, in which case we wouldn't find the query terms in the - # suggestion anyway. + # relevancy, say). We assume that the completion engine has already factored that in. Also, + # completion engines often handle spelling mistakes, in which case we wouldn't find the query terms + # in the suggestion anyway. + # - Scores are weighted such that they retain the order provided by the completion engine. # - The relavancy is higher if the query term is longer. The idea is that search suggestions are more # likely to be relevant if, after typing some number of characters, the user hasn't yet found # a useful suggestion from another completer. - # - Scores are weighted such that they retain the order provided by the completion engine. + # characterCount = query.length - queryTerms.length + 1 relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) # This distinguishes two very different kinds of vomnibar baviours, the newer bahviour (true) and the # legacy behavior (false). We retain the latter for the default search engine, and for custom search - # engines for which we do not have a completion engine. + # engines for which we do not have a completion engine. By "exclusive vomnibar", we mean suggestions + # from other completers are suppressed (so the vomnibar "exclusively" uses suggestions from this search + # engine). useExclusiveVomnibar = custom and haveCompletionEngine - - # If this is a custom search engine and we have a completer, then we exclude results from other - # completers. filter = if useExclusiveVomnibar then (suggestion) -> suggestion.type == description else null - suggestions = [] - # For custom search engines, we add a single, top-ranked entry for the unmodified query. This - # suggestion always appears at the top of the suggestion list. Its setting serve to define various - # vomnibar behaviors. + # suggestion always appears at the top of the list. if custom suggestions.push new Suggestion queryTerms: queryTerms @@ -442,15 +441,11 @@ class SearchEngineCompleter insertText: if useExclusiveVomnibar then query else null # We suppress the leading keyword, for example "w something" becomes "something" in the vomnibar. suppressLeadingKeyword: true - # Should we highlight (via the selection) the longest continuation of the current query which is - # contained in all completions? - completeSuggestions: useExclusiveVomnibar + selectCommonMatches: false # Toggles for the legacy behaviour. autoSelect: not useExclusiveVomnibar forceAutoSelect: not useExclusiveVomnibar highlightTerms: not useExclusiveVomnibar - # Do not use this entry for vomnibar completion (highlighting the common text of the suggestions). - highlightCommonMatches: false mkSuggestion = do -> (suggestion) -> @@ -460,14 +455,12 @@ class SearchEngineCompleter url: Utils.createSearchUrl suggestion, searchUrl title: suggestion relevancy: relavancy *= 0.9 - highlightTerms: false insertText: suggestion - # Do use this entry for vomnibar completion. - highlightCommonMatches: true + highlightTerms: false + selectCommonMatches: true - # If we have cached suggestions, then we can bundle them immediately (otherwise we'll have to do an HTTP - # request, which we do asynchronously). This is a synchronous call (for cached suggestions only) - # because no callback is provided. + # If we have cached suggestions, then we can bundle them immediately (otherwise we'll have to fetch them + # asynchronously). cachedSuggestions = CompletionSearch.complete searchUrl, queryTerms # Post suggestions and bail if we already have all of the suggestions, or if there is no prospect of @@ -476,42 +469,15 @@ class SearchEngineCompleter if cachedSuggestions? console.log "using cached suggestions:", query suggestions.push cachedSuggestions.map(mkSuggestion)... - return onComplete suggestions, { filter } + return onComplete suggestions, { filter, continuation: null } - # Post any initial suggestion, and then deliver suggestions from completion engines as a continuation - # (so, asynchronously). + # Post any initial suggestion, and then deliver the rest of the suggestions as a continuation (so, + # asynchronously). onComplete suggestions, filter: filter - continuation: (existingSuggestions, onComplete) => - suggestions = [] - - if 0 < existingSuggestions.length - existingSuggestionsMinScore = existingSuggestions[existingSuggestions.length-1].relevancy - if relavancy < existingSuggestionsMinScore and maxResults <= existingSuggestions.length - # No suggestion we propose will have a high enough relavancy to beat the existing suggestions, so bail - # immediately. - console.log "skip: cannot add completions" if @debug - return onComplete [] - - CompletionSearch.complete searchUrl, queryTerms, (completionSuggestions = []) => - for suggestion in completionSuggestions - suggestions.push new Suggestion - queryTerms: queryTerms - type: description - url: Utils.createSearchUrl suggestion, searchUrl - title: suggestion - relevancy: relavancy *= 0.9 - highlightTerms: false - insertText: suggestion - # Do use this entry for vomnibar completion. - highlightCommonMatches: true - - # We keep at least three suggestions (if possible) and at most six. We keep more than three only if - # there are enough slots. The idea is that these suggestions shouldn't wholly displace suggestions - # from other completers. That would potentially be a problem because there is no relationship - # between the relevancy scores produced here and those produced by other completers. - count = Math.min 6, Math.max 3, maxResults - existingSuggestions.length - onComplete suggestions[...count] + continuation: (onComplete) => + CompletionSearch.complete searchUrl, queryTerms, (suggestions = []) => + onComplete suggestions.map mkSuggestion # A completer which calls filter() on many completers, aggregates the results, ranks them, and returns the top # 10. Queries from the vomnibar frontend script come through a multi completer. @@ -520,11 +486,8 @@ class MultiCompleter constructor: (@completers) -> - refresh: (port) -> - completer.refresh? port for completer in @completers - - cancel: (port) -> - completer.cancel? port for completer in @completers + refresh: (port) -> completer.refresh? port for completer in @completers + cancel: (port) -> completer.cancel? port for completer in @completers filter: (request, onComplete) -> @debug = true @@ -533,14 +496,9 @@ class MultiCompleter RegexpCache.clear() { queryTerms } = request - request.maxResults = @maxResults - - @mostRecentQuery = null - @filterInProgress = true - suggestions = [] - continuations = [] - filters = [] + [ @mostRecentQuery, @filterInProgress ] = [ null, true ] + [ suggestions, continuations, filters ] = [ [], [], [] ] # Run each of the completers (asynchronously). jobs = new JobRunner @completers.map (completer) -> @@ -554,38 +512,37 @@ class MultiCompleter # Once all completers have finished, process and post the results, and run any continuations or pending # queries. jobs.onReady => - # Apply filters. suggestions = suggestions.filter filter for filter in filters - - # Should we run continuations? shouldRunContinuations = 0 < continuations.length and not @mostRecentQuery? - # Post results, unless there are none AND we will be running a continuation. This avoids + # Post results, unless there are none and we will be running a continuation. This avoids # collapsing the vomnibar briefly before expanding it again, which looks ugly. unless suggestions.length == 0 and shouldRunContinuations + suggestions = @prepareSuggestions queryTerms, suggestions onComplete - results: @prepareSuggestions queryTerms, suggestions + results: suggestions mayCacheResults: continuations.length == 0 # Run any continuations (asynchronously). if shouldRunContinuations - continuationJobs = new JobRunner continuations.map (continuation) -> + jobs = new JobRunner continuations.map (continuation) -> (callback) -> - continuation suggestions, (newSuggestions) -> + continuation (newSuggestions) -> suggestions.push newSuggestions... callback() - continuationJobs.onReady => + jobs.onReady => + suggestions = @prepareSuggestions queryTerms, suggestions # We post these results even if a new query has started. The vomnibar will not display the - # completions, but will cache the results. + # completions (they're arriving too late), but it will cache them. onComplete - results: @prepareSuggestions queryTerms, suggestions + results: suggestions mayCacheResults: true # Admit subsequent queries, and launch any pending query. @filterInProgress = false if @mostRecentQuery - console.log "running pending query:", @mostRecentQuery[0] if @debug + console.log "running pending query:", @mostRecentQuery[0].query if @debug @filter @mostRecentQuery... prepareSuggestions: (queryTerms, suggestions) -> diff --git a/background_scripts/completion_search.coffee b/background_scripts/completion_search.coffee index eb27c076..46533833 100644 --- a/background_scripts/completion_search.coffee +++ b/background_scripts/completion_search.coffee @@ -1,9 +1,12 @@ CompletionSearch = debug: true + inTransit: {} + completionCache: new SimpleCache 2 * 60 * 60 * 1000, 5000 # Two hour, 5000 entries. + engineCache:new SimpleCache 1000 * 60 * 60 * 1000 # 1000 hours. - # The amount of time to wait for new requests before launching the HTTP request. The intention is to cut - # down on the number of HTTP requests we issue. + # The amount of time to wait for new requests before launching the current request (for example, if the user + # is still typing). delay: 100 get: (searchUrl, url, callback) -> @@ -15,13 +18,11 @@ CompletionSearch = xhr.onreadystatechange = -> if xhr.readyState == 4 - callback(if xhr.status == 200 then xhr else null) + callback if xhr.status == 200 then xhr else null - # Look up the completion engine for this searchUrl. Because of DummyCompletionEngine, above, we know there - # will always be a match. Imagining that there may be many completion engines, and knowing that this is - # called for every query, we cache the result. + # Look up the completion engine for this searchUrl. Because of DummyCompletionEngine, we know there will + # always be a match. lookupEngine: (searchUrl) -> - @engineCache ?= new SimpleCache 30 * 60 * 60 * 1000 # 30 hours (these are small, we can keep them longer). if @engineCache.has searchUrl @engineCache.get searchUrl else @@ -29,7 +30,7 @@ CompletionSearch = engine = new engine() return @engineCache.set searchUrl, engine if engine.match searchUrl - # True if we have a completion engine for this search URL, undefined otherwise. + # True if we have a completion engine for this search URL, false otherwise. haveCompletionEngine: (searchUrl) -> not @lookupEngine(searchUrl).dummy @@ -39,17 +40,19 @@ CompletionSearch = # - queryTerms are the query terms. # - callback will be applied to a list of suggestion strings (which may be an empty list, if anything goes # wrong). + # + # If no callback is provided, then we're to provide suggestions only if we can do so synchronously (ie. + # from a cache). In this case we just return the results. Returns null if we cannot service the request + # synchronously. + # complete: (searchUrl, queryTerms, callback = null) -> - query = queryTerms.join "" + query = queryTerms.join(" ").toLowerCase() - # If no callback is provided, then we're to provide suggestions only if we can do so synchronously (ie. - # from a cache). In this case we just return the results. Return null if we cannot service the request - # synchronously. returnResultsOnlyFromCache = not callback? callback ?= (suggestions) -> suggestions - # We don't complete single characters: the results are usually useless. - return callback [] unless 1 < query.length + # We don't complete queries which are too short: the results are usually useless. + return callback [] unless 3 < query.length # We don't complete regular URLs or Javascript URLs. return callback [] if 1 == queryTerms.length and Utils.isUrl query @@ -57,26 +60,18 @@ CompletionSearch = # Cache completions. However, completions depend upon both the searchUrl and the query terms. So we need # to generate a key. We mix in some junk generated by pwgen. A key clash might be possible, but - # vanishingly unlikely. + # is vanishingly unlikely. junk = "//Zi?ei5;o//" completionCacheKey = searchUrl + junk + queryTerms.map((s) -> s.toLowerCase()).join junk - @completionCache ?= new SimpleCache 60 * 60 * 1000, 2000 # One hour, 2000 entries. + if @completionCache.has completionCacheKey - if returnResultsOnlyFromCache - return callback @completionCache.get completionCacheKey - else - # We add a short delay, even for a cache hit. This avoids an ugly flicker when the additional - # suggestions are posted. - Utils.setTimeout 50, => - console.log "hit", completionCacheKey if @debug - callback @completionCache.get completionCacheKey - return + console.log "hit", completionCacheKey if @debug + return callback @completionCache.get completionCacheKey # If the user appears to be typing a continuation of the characters of the most recent query, then we can # re-use the previous suggestions. if @mostRecentQuery? and @mostRecentSuggestions? - reusePreviousSuggestions = do (query) => - query = queryTerms.join(" ").toLowerCase() + reusePreviousSuggestions = do => # Verify that the previous query is a prefix of the current query. return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase() # Ensure that every previous suggestion contains the text of the new query. @@ -93,44 +88,42 @@ CompletionSearch = # That's all of the caches we can try. Bail if the caller is looking for synchronous results. return callback null if returnResultsOnlyFromCache - fetchSuggestions = (engine, callback) => - url = engine.getUrl queryTerms - query = queryTerms.join(" ").toLowerCase() - @get searchUrl, url, (xhr = null) => - # Parsing the response may fail if we receive an unexpected or an unexpectedly-formatted response. In - # all cases, we fall back to the catch clause, below. Therefore, we "fail safe" in the case of - # incorrect or out-of-date completion engines. - try - suggestions = engine.parse xhr - # Filter out the query itself. It's not adding anything. - suggestions = (suggestion for suggestion in suggestions when suggestion.toLowerCase() != query) - console.log "GET", url if @debug - catch - suggestions = [] - # We cache failures too, but remove them after just thirty minutes. - Utils.setTimeout 30 * 60 * 1000, => @completionCache.set completionCacheKey, null - console.log "fail", url if @debug - - callback suggestions - # We pause in case the user is still typing. Utils.setTimeout @delay, handler = @mostRecentHandler = => if handler == @mostRecentHandler @mostRecentHandler = null - # Share duplicate requests. First fetch the suggestions... - @inTransit ?= {} + # Elide duplicate requests. First fetch the suggestions... @inTransit[completionCacheKey] ?= new AsyncDataFetcher (callback) => - fetchSuggestions @lookupEngine(searchUrl), callback + engine = @lookupEngine searchUrl + url = engine.getUrl queryTerms + + @get searchUrl, url, (xhr = null) => + # Parsing the response may fail if we receive an unexpected or an unexpectedly-formatted response. + # In all cases, we fall back to the catch clause, below. Therefore, we "fail safe" in the case of + # incorrect or out-of-date completion engines. + try + suggestions = engine.parse xhr + # Filter out the query itself. It's not adding anything. + suggestions = (suggestion for suggestion in suggestions when suggestion.toLowerCase() != query) + console.log "GET", url if @debug + catch + suggestions = [] + # We cache failures too, but remove them after just thirty minutes. + Utils.setTimeout 30 * 60 * 1000, => @completionCache.set completionCacheKey, null + console.log "fail", url if @debug + + callback suggestions # ... then use the suggestions. @inTransit[completionCacheKey].use (suggestions) => - @mostRecentQuery = queryTerms.join " " + @mostRecentQuery = query @mostRecentSuggestions = suggestions callback @completionCache.set completionCacheKey, suggestions delete @inTransit[completionCacheKey] - # Cancel any pending (ie. blocked on @delay) queries. Does not cancel in-flight queries. + # Cancel any pending (ie. blocked on @delay) queries. Does not cancel in-flight queries. This is called + # whenever the user is typing. cancel: -> if @mostRecentHandler? @mostRecentHandler = null -- cgit v1.2.3 From b733790d6b1b0f4110f3910e317e0be32b962fa4 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 19:07:04 +0100 Subject: Search completion; refactor search-engine detection. --- background_scripts/completion.coffee | 191 ++++++++++++++++++----------------- 1 file changed, 101 insertions(+), 90 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 4663c091..3ddb21d2 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -360,6 +360,16 @@ class SearchEngineCompleter cancel: -> CompletionSearch.cancel() + triageRequest: (request) -> + @searchEngines.use (engines) => + { queryTerms, query } = request + keyword = queryTerms[0] + if keyword and engines[keyword] and (1 < queryTerms.length or /\s$/.test query) + extend request, + queryTerms: queryTerms[1..] + keyword: keyword + engine: engines[keyword] + refresh: (port) -> # Parse the search-engine configuration. @searchEngines = new AsyncDataFetcher (callback) -> @@ -385,99 +395,95 @@ class SearchEngineCompleter handler: "keywords" keywords: key for own key of engines - filter: ({ queryTerms, query, maxResults }, onComplete) -> - return onComplete [] if queryTerms.length == 0 + filter: ({ queryTerms, query, engine }, onComplete) -> + suggestions = [] - @searchEngines.use (engines) => - suggestions = [] - keyword = queryTerms[0] - - { custom, searchUrl, description, queryTerms } = - if engines[keyword]? and (1 < queryTerms.length or /\s$/.test query) - { searchUrl, description } = engines[keyword] - custom: true - searchUrl: searchUrl - description: description - queryTerms: queryTerms[1..] - else - custom: false - searchUrl: Settings.get "searchUrl" - description: "search" - queryTerms: queryTerms - - query = queryTerms.join " " - haveCompletionEngine = CompletionSearch.haveCompletionEngine searchUrl - - # Relevancy: - # - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word - # relevancy, say). We assume that the completion engine has already factored that in. Also, - # completion engines often handle spelling mistakes, in which case we wouldn't find the query terms - # in the suggestion anyway. - # - Scores are weighted such that they retain the order provided by the completion engine. - # - The relavancy is higher if the query term is longer. The idea is that search suggestions are more - # likely to be relevant if, after typing some number of characters, the user hasn't yet found - # a useful suggestion from another completer. - # - characterCount = query.length - queryTerms.length + 1 - relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) - - # This distinguishes two very different kinds of vomnibar baviours, the newer bahviour (true) and the - # legacy behavior (false). We retain the latter for the default search engine, and for custom search - # engines for which we do not have a completion engine. By "exclusive vomnibar", we mean suggestions - # from other completers are suppressed (so the vomnibar "exclusively" uses suggestions from this search - # engine). - useExclusiveVomnibar = custom and haveCompletionEngine - filter = if useExclusiveVomnibar then (suggestion) -> suggestion.type == description else null - - # For custom search engines, we add a single, top-ranked entry for the unmodified query. This - # suggestion always appears at the top of the list. - if custom - suggestions.push new Suggestion + { custom, searchUrl, description } = + if engine + { keyword, searchUrl, description } = engine + custom: true + searchUrl: searchUrl + description: description + else + custom: false + searchUrl: Settings.get "searchUrl" + description: "search" + + return onComplete [] unless custom or 0 < queryTerms.length + + query = queryTerms.join " " + haveCompletionEngine = CompletionSearch.haveCompletionEngine searchUrl + + # Relevancy: + # - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word + # relevancy, say). We assume that the completion engine has already factored that in. Also, + # completion engines often handle spelling mistakes, in which case we wouldn't find the query terms + # in the suggestion anyway. + # - Scores are weighted such that they retain the order provided by the completion engine. + # - The relavancy is higher if the query term is longer. The idea is that search suggestions are more + # likely to be relevant if, after typing some number of characters, the user hasn't yet found + # a useful suggestion from another completer. + # + characterCount = query.length - queryTerms.length + 1 + relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) + + # This distinguishes two very different kinds of vomnibar baviours, the newer bahviour (true) and the + # legacy behavior (false). We retain the latter for the default search engine, and for custom search + # engines for which we do not have a completion engine. By "exclusive vomnibar", we mean suggestions + # from other completers are suppressed (so the vomnibar "exclusively" uses suggestions from this search + # engine). + useExclusiveVomnibar = custom and haveCompletionEngine + filter = if useExclusiveVomnibar then (suggestion) -> suggestion.type == description else null + + # For custom search engines, we add a single, top-ranked entry for the unmodified query. This + # suggestion always appears at the top of the list. + if custom + suggestions.push new Suggestion + queryTerms: queryTerms + type: description + url: Utils.createSearchUrl queryTerms, searchUrl + title: query + relevancy: 1 + insertText: if useExclusiveVomnibar then query else null + # We suppress the leading keyword, for example "w something" becomes "something" in the vomnibar. + suppressLeadingKeyword: true + selectCommonMatches: false + # Toggles for the legacy behaviour. + autoSelect: not useExclusiveVomnibar + forceAutoSelect: not useExclusiveVomnibar + highlightTerms: not useExclusiveVomnibar + + mkSuggestion = do -> + (suggestion) -> + new Suggestion queryTerms: queryTerms type: description - url: Utils.createSearchUrl queryTerms, searchUrl - title: query - relevancy: 1 - insertText: if useExclusiveVomnibar then query else null - # We suppress the leading keyword, for example "w something" becomes "something" in the vomnibar. - suppressLeadingKeyword: true - selectCommonMatches: false - # Toggles for the legacy behaviour. - autoSelect: not useExclusiveVomnibar - forceAutoSelect: not useExclusiveVomnibar - highlightTerms: not useExclusiveVomnibar - - mkSuggestion = do -> - (suggestion) -> - new Suggestion - queryTerms: queryTerms - type: description - url: Utils.createSearchUrl suggestion, searchUrl - title: suggestion - relevancy: relavancy *= 0.9 - insertText: suggestion - highlightTerms: false - selectCommonMatches: true - - # If we have cached suggestions, then we can bundle them immediately (otherwise we'll have to fetch them - # asynchronously). - cachedSuggestions = CompletionSearch.complete searchUrl, queryTerms - - # Post suggestions and bail if we already have all of the suggestions, or if there is no prospect of - # adding further suggestions. - if queryTerms.length == 0 or cachedSuggestions? or not haveCompletionEngine - if cachedSuggestions? - console.log "using cached suggestions:", query - suggestions.push cachedSuggestions.map(mkSuggestion)... - return onComplete suggestions, { filter, continuation: null } - - # Post any initial suggestion, and then deliver the rest of the suggestions as a continuation (so, - # asynchronously). - onComplete suggestions, - filter: filter - continuation: (onComplete) => - CompletionSearch.complete searchUrl, queryTerms, (suggestions = []) => - onComplete suggestions.map mkSuggestion + url: Utils.createSearchUrl suggestion, searchUrl + title: suggestion + relevancy: relavancy *= 0.9 + insertText: suggestion + highlightTerms: false + selectCommonMatches: true + + # If we have cached suggestions, then we can bundle them immediately (otherwise we'll have to fetch them + # asynchronously). + cachedSuggestions = CompletionSearch.complete searchUrl, queryTerms + + # Post suggestions and bail if we already have all of the suggestions, or if there is no prospect of + # adding further suggestions. + if queryTerms.length == 0 or cachedSuggestions? or not haveCompletionEngine + if cachedSuggestions? + console.log "using cached suggestions:", query + suggestions.push cachedSuggestions.map(mkSuggestion)... + return onComplete suggestions, { filter, continuation: null } + + # Post any initial suggestion, and then deliver the rest of the suggestions as a continuation (so, + # asynchronously). + onComplete suggestions, + filter: filter + continuation: (onComplete) => + CompletionSearch.complete searchUrl, queryTerms, (suggestions = []) => + onComplete suggestions.map mkSuggestion # A completer which calls filter() on many completers, aggregates the results, ranks them, and returns the top # 10. Queries from the vomnibar frontend script come through a multi completer. @@ -494,6 +500,11 @@ class MultiCompleter # Allow only one query to run at a time. return @mostRecentQuery = arguments if @filterInProgress + # Provide each completer with an opportunity to see (and possibly alter) the request before it is + # launched. + for completer in @completers + completer.triageRequest? request + RegexpCache.clear() { queryTerms } = request -- cgit v1.2.3 From 21895866c43c09caff096ba2ef5503ee220bab73 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 20:15:39 +0100 Subject: Search completion; minor changes to comments. --- background_scripts/completion.coffee | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 3ddb21d2..d0058538 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -360,6 +360,7 @@ class SearchEngineCompleter cancel: -> CompletionSearch.cancel() + # Look up the search engine and, if one is found, then note it and remove its keyword from the query terms. triageRequest: (request) -> @searchEngines.use (engines) => { queryTerms, query } = request @@ -501,9 +502,10 @@ class MultiCompleter return @mostRecentQuery = arguments if @filterInProgress # Provide each completer with an opportunity to see (and possibly alter) the request before it is - # launched. - for completer in @completers - completer.triageRequest? request + # launched. This is primarily for SearchEngineCompleter, which notes a search query and removes any + # keyword from the queryTerms. Then, other completers don't include keywords in their matching and + # relevancy scores. + completer.triageRequest? request for completer in @completers RegexpCache.clear() { queryTerms } = request -- cgit v1.2.3 From 447115fdc5eccf0edebd3c24eec9dd7f94a2eae2 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 20:49:10 +0100 Subject: Search completion; alternative vomnibar completion. --- background_scripts/completion.coffee | 2 ++ 1 file changed, 2 insertions(+) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index d0058538..e09d01ef 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -449,6 +449,7 @@ class SearchEngineCompleter # We suppress the leading keyword, for example "w something" becomes "something" in the vomnibar. suppressLeadingKeyword: true selectCommonMatches: false + custonSearchEnginePrimarySuggestion: true # Toggles for the legacy behaviour. autoSelect: not useExclusiveVomnibar forceAutoSelect: not useExclusiveVomnibar @@ -465,6 +466,7 @@ class SearchEngineCompleter insertText: suggestion highlightTerms: false selectCommonMatches: true + custonSearchEngineCompletionSuggestion: true # If we have cached suggestions, then we can bundle them immediately (otherwise we'll have to fetch them # asynchronously). -- cgit v1.2.3 From 6d0e3a0f019e85fd76a92cb015ffcfb0d44a4e07 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 22:01:39 +0100 Subject: Search completion; alternative vomnibar completion. --- background_scripts/completion.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index e09d01ef..018740c0 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -466,7 +466,7 @@ class SearchEngineCompleter insertText: suggestion highlightTerms: false selectCommonMatches: true - custonSearchEngineCompletionSuggestion: true + customSearchEngineCompletionSuggestion: true # If we have cached suggestions, then we can bundle them immediately (otherwise we'll have to fetch them # asynchronously). -- cgit v1.2.3 From d9d2e1ed9286523081a49705e4827425f565c202 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Sun, 10 May 2015 22:40:47 +0100 Subject: Search completion; tweak selection color. --- background_scripts/completion_search.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'background_scripts') diff --git a/background_scripts/completion_search.coffee b/background_scripts/completion_search.coffee index 46533833..841990c9 100644 --- a/background_scripts/completion_search.coffee +++ b/background_scripts/completion_search.coffee @@ -74,7 +74,7 @@ CompletionSearch = reusePreviousSuggestions = do => # Verify that the previous query is a prefix of the current query. return false unless 0 == query.indexOf @mostRecentQuery.toLowerCase() - # Ensure that every previous suggestion contains the text of the new query. + # Verify that every previous suggestion contains the text of the new query. for suggestion in (@mostRecentSuggestions.map (s) -> s.toLowerCase()) return false unless 0 <= suggestion.indexOf query # Ok. Re-use the suggestion. -- cgit v1.2.3 From c628508ce66f9f32ae88f36ae57a6b44f900ef88 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 11 May 2015 08:47:57 +0100 Subject: Domain completer should not complete with trailing whitespace. --- background_scripts/completion.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 3ddb21d2..a12f1bcf 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -231,8 +231,8 @@ class DomainCompleter # If `referenceCount` goes to zero, the domain entry can and should be deleted. domains: null - filter: ({ queryTerms }, onComplete) -> - return onComplete([]) unless queryTerms.length == 1 + filter: ({ queryTerms, query }, onComplete) -> + return onComplete [] unless queryTerms.length == 1 and not /\s$/.test query if @domains @performSearch(queryTerms, onComplete) else -- cgit v1.2.3 From 9ea1dc3a69180ffa7778c7f2cf1879f22fe60a92 Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 11 May 2015 09:17:45 +0100 Subject: Search completion; more efficient filtering. --- background_scripts/completion.coffee | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index a12f1bcf..175bc27d 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -365,6 +365,7 @@ class SearchEngineCompleter { queryTerms, query } = request keyword = queryTerms[0] if keyword and engines[keyword] and (1 < queryTerms.length or /\s$/.test query) + request.completers = [ this ] extend request, queryTerms: queryTerms[1..] keyword: keyword @@ -501,9 +502,12 @@ class MultiCompleter return @mostRecentQuery = arguments if @filterInProgress # Provide each completer with an opportunity to see (and possibly alter) the request before it is - # launched. - for completer in @completers - completer.triageRequest? request + # launched. The completer is provided with a list of the completers we're using (request.completers), and + # may change that list to override the default. + request.completers = @completers + completer.triageRequest? request for completer in @completers + completers = request.completers + delete request.completers RegexpCache.clear() { queryTerms } = request @@ -512,7 +516,7 @@ class MultiCompleter [ suggestions, continuations, filters ] = [ [], [], [] ] # Run each of the completers (asynchronously). - jobs = new JobRunner @completers.map (completer) -> + jobs = new JobRunner completers.map (completer) -> (callback) -> completer.filter request, (newSuggestions = [], { continuation, filter } = {}) -> suggestions.push newSuggestions... -- cgit v1.2.3 From c3134f6496f9b0136f1fa454a2c5f81683713a3a Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 11 May 2015 12:48:08 +0100 Subject: Search completion; pre-merge tweaks. --- background_scripts/completion.coffee | 29 +++++++++++++++------------- background_scripts/completion_engines.coffee | 5 +---- background_scripts/completion_search.coffee | 4 ++-- 3 files changed, 19 insertions(+), 19 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 68cd52fc..25fdf44e 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -355,12 +355,14 @@ class TabCompleter tabRecency.recencyScore(suggestion.tabId) class SearchEngineCompleter + @debug: false searchEngines: null cancel: -> CompletionSearch.cancel() - # Look up the search engine and, if one is found, then note it and remove its keyword from the query terms. + # This looks up the custom search engine and, if one is found, then notes it and removes its keyword from + # the query terms. It also sets request.completers to indicate that only this completer should run. triageRequest: (request) -> @searchEngines.use (engines) => { queryTerms, query } = request @@ -450,7 +452,7 @@ class SearchEngineCompleter # We suppress the leading keyword, for example "w something" becomes "something" in the vomnibar. suppressLeadingKeyword: true selectCommonMatches: false - custonSearchEnginePrimarySuggestion: true + customSearchEnginePrimarySuggestion: true # Toggles for the legacy behaviour. autoSelect: not useExclusiveVomnibar forceAutoSelect: not useExclusiveVomnibar @@ -477,7 +479,7 @@ class SearchEngineCompleter # adding further suggestions. if queryTerms.length == 0 or cachedSuggestions? or not haveCompletionEngine if cachedSuggestions? - console.log "using cached suggestions:", query + console.log "cached suggestions:", cachedSuggestions.length, query if SearchEngineCompleter.debug suggestions.push cachedSuggestions.map(mkSuggestion)... return onComplete suggestions, { filter, continuation: null } @@ -487,6 +489,7 @@ class SearchEngineCompleter filter: filter continuation: (onComplete) => CompletionSearch.complete searchUrl, queryTerms, (suggestions = []) => + console.log "fetched suggestions:", suggestions.length, query if SearchEngineCompleter.debug onComplete suggestions.map mkSuggestion # A completer which calls filter() on many completers, aggregates the results, ranks them, and returns the top @@ -495,18 +498,17 @@ class MultiCompleter maxResults: 10 constructor: (@completers) -> - refresh: (port) -> completer.refresh? port for completer in @completers cancel: (port) -> completer.cancel? port for completer in @completers filter: (request, onComplete) -> - @debug = true # Allow only one query to run at a time. return @mostRecentQuery = arguments if @filterInProgress # Provide each completer with an opportunity to see (and possibly alter) the request before it is - # launched. Each completer is provided with a list of all of the completers we're using - # (request.completers), and may change that list to override the default. + # launched. Each completer is also provided with a list of all of the completers we're using + # (request.completers), and may change that list to override the default (for example, the + # search-engine completer does this if it wants to be the *only* completer). request.completers = @completers completer.triageRequest? request for completer in @completers completers = request.completers @@ -527,8 +529,8 @@ class MultiCompleter filters.push filter if filter? callback() - # Once all completers have finished, process and post the results, and run any continuations or pending - # queries. + # Once all completers have finished, process the results and post them, and run any continuations or + # pending queries. jobs.onReady => suggestions = suggestions.filter filter for filter in filters shouldRunContinuations = 0 < continuations.length and not @mostRecentQuery? @@ -541,7 +543,9 @@ class MultiCompleter results: suggestions mayCacheResults: continuations.length == 0 - # Run any continuations (asynchronously). + # Run any continuations (asynchronously); for example, the search-engine completer + # (SearchEngineCompleter) uses a continuation to fetch suggestions from completion engines + # asynchronously. if shouldRunContinuations jobs = new JobRunner continuations.map (continuation) -> (callback) -> @@ -551,8 +555,8 @@ class MultiCompleter jobs.onReady => suggestions = @prepareSuggestions queryTerms, suggestions - # We post these results even if a new query has started. The vomnibar will not display the - # completions (they're arriving too late), but it will cache them. + # We post these results even if a new query has started. The vomnibar will not display them + # (because they're arriving too late), but it will cache them. onComplete results: suggestions mayCacheResults: true @@ -560,7 +564,6 @@ class MultiCompleter # Admit subsequent queries, and launch any pending query. @filterInProgress = false if @mostRecentQuery - console.log "running pending query:", @mostRecentQuery[0].query if @debug @filter @mostRecentQuery... prepareSuggestions: (queryTerms, suggestions) -> diff --git a/background_scripts/completion_engines.coffee b/background_scripts/completion_engines.coffee index 07ecfa26..14e65692 100644 --- a/background_scripts/completion_engines.coffee +++ b/background_scripts/completion_engines.coffee @@ -64,20 +64,17 @@ class Wikipedia extends RegexpEngine parse: (xhr) -> JSON.parse(xhr.responseText)[1] +## Does not work... ## class GoogleMaps extends RegexpEngine ## # Example search URL: https://www.google.com/maps/search/%s ## constructor: -> ## super [ new RegExp "^https?://www\.google\.com/maps/search/" ] ## ## getUrl: (queryTerms) -> -## console.log "xxxxxxxxxxxxxxxxxxxxx" ## "https://www.google.com/s?tbm=map&fp=1&gs_ri=maps&source=hp&suggest=p&authuser=0&hl=en&pf=p&tch=1&ech=2&q=#{Utils.createSearchQuery queryTerms}" ## ## parse: (xhr) -> -## console.log "yyy", xhr.responseText ## data = JSON.parse xhr.responseText -## console.log "zzz" -## console.log data ## [] class Bing extends RegexpEngine diff --git a/background_scripts/completion_search.coffee b/background_scripts/completion_search.coffee index 841990c9..2d2ee439 100644 --- a/background_scripts/completion_search.coffee +++ b/background_scripts/completion_search.coffee @@ -1,6 +1,6 @@ CompletionSearch = - debug: true + debug: false inTransit: {} completionCache: new SimpleCache 2 * 60 * 60 * 1000, 5000 # Two hour, 5000 entries. engineCache:new SimpleCache 1000 * 60 * 60 * 1000 # 1000 hours. @@ -109,7 +109,7 @@ CompletionSearch = console.log "GET", url if @debug catch suggestions = [] - # We cache failures too, but remove them after just thirty minutes. + # We allow failures to be cached too, but remove them after just thirty minutes. Utils.setTimeout 30 * 60 * 1000, => @completionCache.set completionCacheKey, null console.log "fail", url if @debug -- cgit v1.2.3 From 3975c13fe040639beb56582e50d951ad4839afbb Mon Sep 17 00:00:00 2001 From: Stephen Blott Date: Mon, 11 May 2015 14:37:57 +0100 Subject: Search completion; add weigthing option. --- background_scripts/completion.coffee | 6 ++++-- background_scripts/settings.coffee | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'background_scripts') diff --git a/background_scripts/completion.coffee b/background_scripts/completion.coffee index 25fdf44e..23526f85 100644 --- a/background_scripts/completion.coffee +++ b/background_scripts/completion.coffee @@ -416,7 +416,9 @@ class SearchEngineCompleter return onComplete [] unless custom or 0 < queryTerms.length query = queryTerms.join " " + factor = Settings.get "omniSearchWeight" haveCompletionEngine = CompletionSearch.haveCompletionEngine searchUrl + haveCompletionEngine = false unless 0.0 < factor # Relevancy: # - Relevancy does not depend upon the actual suggestion (so, it does not depend upon word @@ -429,7 +431,7 @@ class SearchEngineCompleter # a useful suggestion from another completer. # characterCount = query.length - queryTerms.length + 1 - relavancy = 0.6 * (Math.min(characterCount, 10.0)/10.0) + relavancy = factor * (Math.min(characterCount, 10.0)/10.0) # This distinguishes two very different kinds of vomnibar baviours, the newer bahviour (true) and the # legacy behavior (false). We retain the latter for the default search engine, and for custom search @@ -478,7 +480,7 @@ class SearchEngineCompleter # Post suggestions and bail if we already have all of the suggestions, or if there is no prospect of # adding further suggestions. if queryTerms.length == 0 or cachedSuggestions? or not haveCompletionEngine - if cachedSuggestions? + if cachedSuggestions? and 0 < factor console.log "cached suggestions:", cachedSuggestions.length, query if SearchEngineCompleter.debug suggestions.push cachedSuggestions.map(mkSuggestion)... return onComplete suggestions, { filter, continuation: null } diff --git a/background_scripts/settings.coffee b/background_scripts/settings.coffee index 11f492d7..e042eded 100644 --- a/background_scripts/settings.coffee +++ b/background_scripts/settings.coffee @@ -43,6 +43,7 @@ root.Settings = Settings = # or strings defaults: scrollStepSize: 60 + omniSearchWeight: 0.6 smoothScroll: true keyMappings: "# Insert your preferred key mappings here." linkHintCharacters: "sadfjklewcmpgh" -- cgit v1.2.3