Class: Queries::Otu::Autocomplete

Inherits:
Query::Autocomplete show all
Defined in:
lib/queries/otu/autocomplete.rb

Overview

See Query::Autocomplete for optimization strategy per name. There are 4 classes of name, each which has the same strategy: OTU name, Original TaxonName, TaxonName, CommonName We then apply a global priority pulling the best names from each sub-strategy to the top.

Constant Summary collapse

DEFAULT_LIMIT = 
40
QUERIES =

Keys are method names. Existence of method is checked before requesting the query

{
  # OTU
  autocomplete_taxon_name_and_otu_name: {priority: 1},
  otu_name_exact: {priority: 2}, # Was 1
  autocomplete_exact_id: {priority: 2},
  autocomplete_identifier_cached_exact: {priority: 3},
  otu_name_start_match: {priority: 200},
  otu_name_similarity: {priority: 220},

  # TaxonName
  autocomplete_taxon_name: {priority: nil}, # Priority is slotted from 10 .. 20
  # These are all approximately covered in the blanket taxon_name autocomplete
  # taxon_name_name_exact: {priority: 10},
  # taxon_name_identifier_exact: {priority: 10},
  # taxon_name_name_start_match: {priority: 100},
  # taxon_name_name_high_cuttoff: {priority: 200},

  # CommonName
  # These should all be covered/moved to common_name_autocomplete,
  autocomplete_common_name_exact: {priority: 300},
  autocomplete_common_name_like: {priority: 1000}
  # common_name_identifier_exact: {priority: 10},
  # common_name_name_start_match: {priority: 100},
  # common_name_name_similarity: {priority: 200},
}.freeze

Constants inherited from Query::Autocomplete

Query::Autocomplete::DELEGATED_DEPTH, Query::Autocomplete::DELEGATED_MINIMUM_LENGTH, Query::Autocomplete::LITERAL_RESTRICTION_MAX

Instance Attribute Summary collapse

Attributes inherited from Query::Autocomplete

#delegated_results, #dynamic_limit, #limit, #project_id, #query_string, #restrict_to

Attributes inherited from Query

#query_string, #terms

Instance Method Summary collapse

Methods inherited from Query::Autocomplete

#apply_restriction, #asserted_object_autocomplete, #asserted_object_restriction, #autocomplete_cached, #autocomplete_cached_wildcard_anywhere, #autocomplete_common_name_exact, #autocomplete_common_name_like, #autocomplete_exact_id, #autocomplete_exactly_named, #autocomplete_named, #autocomplete_ordered_wildcard_pieces_in_cached, #cached_facet, #combine_or_clauses, #common_name_name, #common_name_table, #common_name_wild_pieces, #delegated_autocomplete, #exactly_named, #fragments, #integers, #least_levenshtein, #match_wildcard_end_in_cached, #match_wildcard_in_cached, #named, #only_ids, #only_integers?, #order_by_id_rank, #parent, #parent_child_join, #parent_child_where, #pieces, #safe_integers, #scope, #string_fragments, #wildcard_wrapped_integers, #wildcard_wrapped_years, #with_cached, #with_cached_like, #with_id, #with_project_id, #year_letter, #years

Methods inherited from Query

#alphabetic_strings, #alphanumeric_strings, base_name, #base_name, #build_terms, #cached_facet, #end_wildcard, #levenshtein_distance, #match_ordered_wildcard_pieces_in_cached, #no_terms?, referenced_klass, #referenced_klass, #referenced_klass_except, #referenced_klass_intersection, #referenced_klass_union, #start_and_end_wildcard, #start_wildcard, #table, #wildcard_pieces

Constructor Details

#initialize(string, project_id: nil, having_taxon_name_only: false, with_taxon_name: nil, exact: 'false', include_common_names: false, include_taxon_name: false, restrict_to: nil, limit: nil) ⇒ Autocomplete

Returns a new instance of Autocomplete.



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/queries/otu/autocomplete.rb', line 72

def initialize(
  string, project_id: nil, having_taxon_name_only: false,
  with_taxon_name: nil, exact: 'false', include_common_names: false,
  include_taxon_name: false, restrict_to: nil, limit: nil
)

  super(string, project_id:, restrict_to:, limit:)
  @having_taxon_name_only = boolean_param({having_taxon_name_only:}, :having_taxon_name_only)
  @with_taxon_name = boolean_param({with_taxon_name:}, :with_taxon_name)

  # TODO: move to mode
  @exact = boolean_param({exact:}, :exact)

  @include_common_names = boolean_param({include_common_names:}, :include_common_names)
  @include_taxon_name = boolean_param({include_taxon_name:}, :include_taxon_name)
end

Instance Attribute Details

#exact ⇒ Boolean

Returns &exact=<"true"|"false"> if 'true' then only #name = query_string results are returned (no fuzzy matching).

Returns:

  • (Boolean) —

    &exact=<"true"|"false"> if 'true' then only #name = query_string results are returned (no fuzzy matching)



29
30
31
 
# File 'lib/queries/otu/autocomplete.rb', line 29

def exact
  @exact
end

#having_taxon_name_only ⇒ Object

Returns Boolean, nil true - only return Otus with name = nil false,nil - no effect.

Returns:

  • Boolean, nil true - only return Otus with name = nil false,nil - no effect



18
19
20
 
# File 'lib/queries/otu/autocomplete.rb', line 18

def having_taxon_name_only
  @having_taxon_name_only
end

#include_common_names ⇒ Object

Returns Boolean, nil true - 'pre-load' common names with otus false/nil - ignored.

Returns:

  • Boolean, nil true - 'pre-load' common names with otus false/nil - ignored



34
35
36
 
# File 'lib/queries/otu/autocomplete.rb', line 34

def include_common_names
  @include_common_names
end

#include_taxon_name ⇒ Object

Only applied pertinent to the TaxonName autocomplete Only applied pertinent to the TaxonName autocomplete

Returns:

  • Boolean, nil true - 'pre-load' taxon name with otus false/nil - ignored



42
43
44
 
# File 'lib/queries/otu/autocomplete.rb', line 42

def include_taxon_name
  @include_taxon_name
end

#with_taxon_name ⇒ Object

Returns Boolean, nil true - OTU must have taxon name false - OTU must not have taxon name nil - ignored.

Returns:

  • Boolean, nil true - OTU must have taxon name false - OTU must not have taxon name nil - ignored



24
25
26
 
# File 'lib/queries/otu/autocomplete.rb', line 24

def with_taxon_name
  @with_taxon_name
end

Instance Method Details

#api_autocomplete ⇒ Object

DEPRECATED Maintains valid_taxon_name_id needed for API.

Considerations:

otus -> taxon names -> valid taxon name_id <- otu can return more OTUs than the original query
 because there can be multiple OTUs for the valid name of an invalid original result.
 right now we pick the first valid OTU for the name with distinct on()


185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
 
# File 'lib/queries/otu/autocomplete.rb', line 185

def api_autocomplete
  @with_taxon_name = true

  # This limit() has more impact now. Since all
  # names are loaded large matches can swamp exact names
  # before priority ordering is applied. May require tuning.
  otus = compact_priorities( autocomplete_base.limit(30) )

  otu_order = otus.map(&:id).uniq

  f = ::Otu.where(id: otu_order)
    .joins('left join taxon_names t1 on otus.taxon_name_id = t1.id')
    .joins('left join otus o2 on t1.cached_valid_taxon_name_id = o2.taxon_name_id')
    .select('distinct on (otus.id) otus.id, otus.name, otus.taxon_name_id, COALESCE(o2.id, otus.id) as otu_valid_id')

  f.sort_by.with_index { |item, idx| [(otu_order.index(item.id) || 9999), (idx || 9999)] }
end

#api_autocomplete_extended ⇒ Array

An autocomplete result that permits displaying the TaxonName as originally matched. Note that otu: is really only useful when displaying otus without &having_taxon_name_only=true. We don't, for example make use of this element there.

Returns:

  • (Array) —

    of { otu:, label_target:, otu_valid_id: }



259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
 
# File 'lib/queries/otu/autocomplete.rb', line 259

def api_autocomplete_extended
  otu_queries = QUERIES.dup
  otu_queries.delete(:autocomplete_taxon_name)

  base_otus = autocomplete_base(otu_queries).limit(30)
  taxon_name_otus = autocomplete_taxon_name_extended

  r = []

  base_otus.each do |o|
    r.push({
      otu: o, # contains priority
      label_target: o
    })
  end

  taxon_name_otus.each do |o|
    r.push({
      otu: o,
      label_target: (o.label_target_taxon_name_id ? ::TaxonName.find(o.label_target_taxon_name_id) : o.taxon_name )  # is o.taxon_name true?!
    })
  end

  # Keep a unique set of otu + label (to render)
  seen = Set.new

  # The compacted result
  compact = []

  r.each do |h|
    g = h[:label_target].id.to_s + h[:label_target].class.name
    m = [ h[:otu].id, g ]
    next if seen.include?( m )
    seen << m
    compact.push h
  end

  compact.sort!{|c,d| (c[:otu].priority || 999) <=> (d[:otu].priority || 999 )}

  # TODO: Refactor to remove extra query and assignment of otu_valid_id.  This is ugly.
  otu_order = compact.collect{|d| d[:otu].id}

  # Extra query is painful.
  f = ::Otu.where(id: otu_order)
    .joins('left join taxon_names t1 on otus.taxon_name_id = t1.id')
#         .joins('left join otus o2 on t1.cached_valid_taxon_name_id = o2.taxon_name_id')
    .joins('left join otus o2 on t1.cached_valid_taxon_name_id = o2.taxon_name_id and o2.taxon_name_id <> otus.taxon_name_id') # See https://github.com/sfg-taxonpages/orthoptera/issues/90
    .select('distinct on (otus.id) otus.id, otus.name, otus.taxon_name_id, COALESCE(o2.id, otus.id) as otu_valid_id')

  compact.each do |h|
    h[:otu_valid_id] = f.select{|j| j.id == h[:otu].id}.first.otu_valid_id
  end

  compact
end

#autocomplete ⇒ Object 



332
333
334
 
# File 'lib/queries/otu/autocomplete.rb', line 332

def autocomplete
  distinct_autocomplete_base.limit(limit).to_a
end

#autocomplete_base(targets = QUERIES) ⇒ Scope

DEPRECATED API only (#api_autocomplete, #api_autocomplete_extended), otherwise use #distinct_autocomplete_base, which builds on it.

Returns:

  • (Scope) —

    an Otu once for each query it matches, at that query's priority, so a limit on it counts matches, not Otus



343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
 
# File 'lib/queries/otu/autocomplete.rb', line 343

def autocomplete_base(targets = QUERIES)
  queries = []

  targets.each do |q, p|
    if self.respond_to?(q)

      a = send(q)
      next if a.nil? # query has returned nil

      y = p[:priority]

      a = scope_autocomplete(a)

      # This is probably messing with the eager load
      # Need to refine all auto auto complete/select to include taxon name, parent, and valid name within the query
      a = a.select("otus.*, #{y} as priority") unless y.nil?

      queries.push a
    end
  end

  queries.compact!

  include_associations(referenced_klass_union(queries).order('priority'))
end

#autocomplete_taxon_name ⇒ Scope

Returns Pull the result of a TaxonName autocomplete. Maintain the order returned, and re-cast the result in terms of an OTU query. Expensive but maintaining order is key.

Returns:

  • (Scope) —

    Pull the result of a TaxonName autocomplete. Maintain the order returned, and re-cast the result in terms of an OTU query. Expensive but maintaining order is key.



161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/queries/otu/autocomplete.rb', line 161

def autocomplete_taxon_name
  taxon_names = taxon_name_autocomplete # an array, not a query

  ids = taxon_names.collect{|n| otu_taxon_name_id(n)} # TODO: Experiment with :cached_valid_taxon_name_id) # We assume we want to land on Valid OTUs, but see #
  return nil if ids.empty?

  min = 10.0
  max = 20.0
  scale = (max - min) / ids.count.to_f

  # TODO: optimize *
  base_query.select("otus.*, ((#{min} + row_number() OVER ())::float * #{scale}) as priority") # small incrementing numbers for priority
    .joins("INNER JOIN ( SELECT unnest(ARRAY[#{ids.join(',')}]) AS id, row_number() OVER () AS row_num ) AS id_order ON otus.taxon_name_id = id_order.id")
    .order('id_order.row_num')
end

#autocomplete_taxon_name_and_otu_name ⇒ Object

For names like Tapinoma CASC_2231

The query is a prefix match on the taxon name (without authorship) followed by a prefix match on the otu name - in that order. Only exactly two terms are supported.



122
123
124
125
126
127
128
129
130
131
132
 
# File 'lib/queries/otu/autocomplete.rb', line 122

def autocomplete_taxon_name_and_otu_name
  words = query_string.to_s.split(/\s+/)
  return nil unless words.length == 2

  genus_term, otu_term = words

  base_query
    .joins(:taxon_name)
    .where('taxon_names.cached ILIKE ? AND otus.name ILIKE ?', "#{genus_term}%", "#{otu_term}%")
    .order('taxon_names.cached, otus.name, length(taxon_names.cached), length(otus.name)')
end

#autocomplete_taxon_name_extended ⇒ Object 



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
 
# File 'lib/queries/otu/autocomplete.rb', line 203

def autocomplete_taxon_name_extended
  taxon_names = taxon_name_autocomplete # an array, not a query

  ids = taxon_names.collect{|n|
    [
      otu_taxon_name_id(n), # Points to the OTU target, if there is one
      n.id,  # points to the label target
    ]
  }

  return ::Otu.none if ids.empty?

  ids.uniq!

  min = 10.0
  max = 20.0
  scale = (max - min) / ids.count.to_f

  # TODO: optimize *
  otus = base_query
    .select(<<~SQL.squish)
    .joins(<<~SQL.squish)
      INNER JOIN (
        SELECT
          unnest(ARRAY[#{ids.map(&:first).join(',')}]) AS id,
          unnest(ARRAY[#{ids.map(&:last).join(',')}]) AS label_target_taxon_name_id,
          row_number() OVER () AS row_num
      ) AS id_order
      ON otus.taxon_name_id = id_order.id
    SQL
    .order('id_order.row_num')

  otus = scope_autocomplete(otus)

  # We could currently get away with using .includes here, but if we were
  # to ever filter or group `otus` on a non-otu table like id_order then
  # .includes would do a join on the associated table below and we could
  # get duplicate otu.id result rows that would be de-duplicated by rails,
  # losing vital (non-dup) id_order info. So just always do preload here
  # instead.
  otus = include_taxon_name ? otus.preload(:taxon_name) : otus
  otus = include_common_names ? otus.preload(:common_names) : otus

  otus
end

#base_query ⇒ Object 



89
90
91
92
93
 
# File 'lib/queries/otu/autocomplete.rb', line 89

def base_query
  q = ::Otu.all
  q = q.where(project_id:) if project_id.any? # TODO: this needs to be a wrapping layer check, definitely not here
  q
end

#compact_priorities(otus) ⇒ Object

Doesn't work for extended, as we can have the same OTU with different labels



318
319
320
321
322
323
324
325
326
327
328
329
330
 
# File 'lib/queries/otu/autocomplete.rb', line 318

def compact_priorities(otus)
  # Mmmmarg!
  # We may have the same name at different priorities, strike all but the highest/first.
  r = []
  i = {}

  otus.each do |o|
    next if i[o.id]
    r.push o
    i[o.id] = true
  end
  r
end

#distinct_autocomplete_base(targets = QUERIES) ⇒ Scope

Returns #autocomplete_base with each Otu once, at its best (lowest) priority, so that a limit on it counts Otus, not matches (an Otu can match more than one query, e.g. by a Protonym and its Combination).

Returns:

  • (Scope) —

    #autocomplete_base with each Otu once, at its best (lowest) priority, so that a limit on it counts Otus, not matches (an Otu can match more than one query, e.g. by a Protonym and its Combination)



373
374
375
376
377
378
379
380
381
382
383
 
# File 'lib/queries/otu/autocomplete.rb', line 373

def distinct_autocomplete_base(targets = QUERIES)
  matches = autocomplete_base(targets).unscope(:order)

  include_associations(
    ::Otu.from(<<~SQL.squish).order('priority')
      (SELECT DISTINCT ON (otus.id) otus.*
        FROM (#{matches.to_sql}) AS otus
        ORDER BY otus.id, otus.priority) AS otus
    SQL
  )
end

#include_associations(q) ⇒ Scope

Returns:

  • (Scope) 


386
387
388
389
 
# File 'lib/queries/otu/autocomplete.rb', line 386

def include_associations(q)
  q = include_common_names ? q.includes(:common_names) : q
  include_taxon_name ? q.includes(:taxon_name) : q
end

#otu_name_exact ⇒ Object 



95
96
97
 
# File 'lib/queries/otu/autocomplete.rb', line 95

def otu_name_exact
  base_query.where(otus: {name: query_string})
end

#otu_name_similarity ⇒ Object

All records that meet the similarity cuttoff

  • this is intended as a generic replacement for wildcarded results

Observations:

- was similarity(), experimenting with word_similarity
- 3 letter matches are going to be low probability, matches kick in at 4


110
111
112
113
114
115
 
# File 'lib/queries/otu/autocomplete.rb', line 110

def otu_name_similarity
  base_query
    .where('otus.name % ?', query_string)
    .where( ApplicationRecord.sanitize_sql_array(["word_similarity('%s', otus.name) > 0.33", query_string]))
    .order('otus.name, length(otus.name)')
end

#otu_name_start_match ⇒ Object 



99
100
101
 
# File 'lib/queries/otu/autocomplete.rb', line 99

def otu_name_start_match
  base_query.where('otus.name ilike ?', query_string + '%')
end

#otu_taxon_name_id(taxon_name) ⇒ Integer

Returns the id of the TaxonName whose Otus a TaxonName autocomplete result resolves to.

Returns:

  • (Integer) —

    the id of the TaxonName whose Otus a TaxonName autocomplete result resolves to



137
138
139
 
# File 'lib/queries/otu/autocomplete.rb', line 137

def otu_taxon_name_id(taxon_name)
  taxon_name.is_combination? ? taxon_name.cached_valid_taxon_name_id : taxon_name.id
end

#scope_autocomplete(query) ⇒ Object 



391
392
393
394
395
396
397
 
# File 'lib/queries/otu/autocomplete.rb', line 391

def scope_autocomplete(query)
  query = apply_restriction(query)
  query = query.joins(:taxon_name) if with_taxon_name
  query = query.where.missing(:taxon_name) if with_taxon_name == false
  query = query.where(otus: {name: nil}) if having_taxon_name_only
  query
end

#taxon_name_autocomplete ⇒ Array<TaxonName>

Returns the TaxonName autocomplete results for #autocomplete_taxon_name(_extended), when restricted only those that resolve to a restrict_to Otu, see #delegated_autocomplete.

Returns:

  • (Array<TaxonName>) —

    the TaxonName autocomplete results for #autocomplete_taxon_name(_extended), when restricted only those that resolve to a restrict_to Otu, see #delegated_autocomplete



145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/queries/otu/autocomplete.rb', line 145

def taxon_name_autocomplete
  @taxon_name_autocomplete ||= delegated_autocomplete(
    build: ->(l) { Queries::TaxonName::Autocomplete.new(query_string, exact:, project_id:, limit: l) },
    key: [query_string, exact, project_id],
    keep: ->(names) {
      allowed = apply_restriction(::Otu.where(taxon_name_id: names.map { otu_taxon_name_id(_1) }))
        .distinct.pluck(:taxon_name_id).to_set

      names.select { allowed.include?(otu_taxon_name_id(_1)) }
    }
  )
end