Class: BatchLoad::Import

Inherits:
Object
  • Object
show all
Defined in:
lib/batch_load/import.rb

Overview

A generic object for managing CSV based imports

Direct Known Subclasses

AssertedDistributions, CollectingEvents, CollectingEvents::CastorInterpreter, CollectingEvents::GpxInterpreter, CollectionObjects, CollectionObjects::BufferedInterpreter, CollectionObjects::CastorInterpreter, DWCA, Otus, Otus::IdentifiersInterpreter, TaxonNames, TaxonNames::CastorInterpreter, TaxonifiToTaxonworks, Import::Namespaces::SimpleInterpreter, Import::SequenceRelationships::PrimersInterpreter, Import::Sequences::GenbankInterpreter, Import::Sequences::PrimersInterpreter

Defined Under Namespace

Classes: AssertedDistributions, CollectingEvents, CollectionObjects, DWCA, Otus, TaxonNames, TaxonifiToTaxonworks

Instance Attribute Summary (collapse)

Instance Method Summary (collapse)

Constructor Details

- (Import) initialize(project_id: nil, user_id: nil, file: nil, process: true, import_level: :warn, user_header_map: {})

Returns a new instance of Import

Parameters:

  • args (Hash)


61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
# File 'lib/batch_load/import.rb', line 61

def initialize(project_id: nil, user_id: nil, file: nil, process: true, import_level: :warn, user_header_map: {})
  @processed = false
  @import_level = import_level
  @project_id = project_id
  @user_id = user_id
  @file = file

  @user_header_map = user_header_map

  @processed_rows  = {}
  @successful_rows = nil

  @user = User.find(@user_id)

  @file_errors = []
  @errors ||= [] # can be set in subclasses in some cases

  @create_attempted = false

  process && build
end

Instance Attribute Details

- (Object) create_attempted

An attempt was made to create new records



29
30
31
# File 'lib/batch_load/import.rb', line 29

def create_attempted
  @create_attempted
end

- (CSV?) csv

Returns:

  • (CSV, nil)


49
50
51
# File 'lib/batch_load/import.rb', line 49

def csv
  @csv
end

- (Object) errors

Errors from the import process itself.



55
56
57
# File 'lib/batch_load/import.rb', line 55

def errors
  @errors
end

- (Object) file

The input file, as it comes in on the form



46
47
48
# File 'lib/batch_load/import.rb', line 46

def file
  @file
end

- (Object) file_errors

Errors with the file itself, rather than its content



52
53
54
# File 'lib/batch_load/import.rb', line 52

def file_errors
  @file_errors
end

- (Object) import_level

How forgiving the import process is

:warn -> all possible names will be added, with those not validating ignored
:line_strict -> there is one record assumed / line, and each line must have a single valid record
:strict -> all processed records must be valid


43
44
45
# File 'lib/batch_load/import.rb', line 43

def import_level
  @import_level
end

- (Object) processed

File is processable, at the basic level, and is ready for preview/created



26
27
28
# File 'lib/batch_load/import.rb', line 26

def processed
  @processed
end

- (Object) processed_rows

An index of all rows for which some data was present, index is line number, points to a RowParse instance



18
19
20
# File 'lib/batch_load/import.rb', line 18

def processed_rows
  @processed_rows
end

- (Object) project

Returns the value of attribute project



31
32
33
# File 'lib/batch_load/import.rb', line 31

def project
  @project
end

- (Object) successful_rows

return [Array] the line numbers that resulted in saved records



23
24
25
# File 'lib/batch_load/import.rb', line 23

def successful_rows
  @successful_rows
end

- (Object) total_data_lines

return [Integer] the total lines with data



37
38
39
# File 'lib/batch_load/import.rb', line 37

def total_data_lines
  @total_data_lines
end

- (Object) total_lines

The number of non-header rows in the file



34
35
36
# File 'lib/batch_load/import.rb', line 34

def total_lines
  @total_lines
end

- (Object) user

Returns the value of attribute user



31
32
33
# File 'lib/batch_load/import.rb', line 31

def user
  @user
end

- (Object) user_header_map

User provided map of their header (key) to our attribute (value)



58
59
60
# File 'lib/batch_load/import.rb', line 58

def user_header_map
  @user_header_map
end

Instance Method Details

- (Object) all_objects

return [Array] all objects (parsed records)



243
244
245
# File 'lib/batch_load/import.rb', line 243

def all_objects
  processed_rows.collect { |_i, rp| rp.all_objects }.flatten
end

- (Object) build



202
203
204
# File 'lib/batch_load/import.rb', line 202

def build
  raise 'This method must be provided in each respective subclass.'
end

- (Boolean) create

Iterates in line order and attempts to save each record return [true]

Returns:

  • (Boolean)


170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
# File 'lib/batch_load/import.rb', line 170

def create
  @create_attempted = true
  if ready_to_create?

    # TODO: DRY
    if a = save_order
      sorted_processed_rows.each_value do |rp|
        a.each do |k|
          rp.objects[k].each do |o|
            o.save unless o.persisted?
          end
        end
      end

    else
      sorted_processed_rows.each_value do |rp|
        rp.objects.each_value do |objs|
          objs.each do |o|
            o.save
          end
        end
      end
    end

  else
    @errors << "Import level #{import_level} has prevented creation." unless import_level_ok?
    @errors << 'CSV has not been processed.' unless processed?
    @errors << 'One of user_id, project_id or file has not been provided.' unless valid?
  end
  true
end

- (Boolean) create_attempted?

return [Boolean] whether an attempt at creating records has occured

Returns:

  • (Boolean)


207
208
209
# File 'lib/batch_load/import.rb', line 207

def create_attempted?
  create_attempted
end

- (Boolean) import_level_ok?

Returns:

  • (Boolean)


136
137
138
139
140
141
142
143
144
145
146
147
# File 'lib/batch_load/import.rb', line 136

def import_level_ok?
  case import_level.to_sym
    when :warn
      warn_level_ok?
    when :strict
      strict_level_ok?
    when :line_strict
      line_strict_level_ok?
    else
      false
  end
end

- (Boolean) line_strict_level_ok?

Returns:

  • (Boolean)


163
164
165
# File 'lib/batch_load/import.rb', line 163

def line_strict_level_ok?
  total_data_lines == valid_objects.size
end

- (Boolean) processed?

return [Boolean] whether an attempt to process the input file has occured

Returns:

  • (Boolean)


212
213
214
# File 'lib/batch_load/import.rb', line 212

def processed?
  processed
end

- (Boolean) ready_to_create?

return [Boolean] whether the instance is configured

Returns:

  • (Boolean)


131
132
133
# File 'lib/batch_load/import.rb', line 131

def ready_to_create?
  valid? && processed? && import_level_ok?
end

- (Object) save_order



247
248
249
# File 'lib/batch_load/import.rb', line 247

def save_order
  self.class.const_defined?('SAVE_ORDER') ? self.class::SAVE_ORDER : nil
end

- (Object) sorted_processed_rows

return [Hash] processed rows, sorted by line number

?! key order might not persist ?!


233
234
235
# File 'lib/batch_load/import.rb', line 233

def sorted_processed_rows
  @processed_rows.sort.to_h
end

- (Boolean) strict_level_ok?

Returns:

  • (Boolean)


155
156
157
158
159
160
# File 'lib/batch_load/import.rb', line 155

def strict_level_ok?
  all_objects.each do |o|
    return false unless o.valid?
  end
  true
end

- (Object) total_records_created

return [Integer] the total number of records created



227
228
229
# File 'lib/batch_load/import.rb', line 227

def total_records_created
  successful_rows.inject(t = 0) { |t, i| t += processed_rows[i].persisted_objects.size }
end

- (String) user_map(h)

Parameters:

  • h (String)

Returns:

  • (String)


119
120
121
# File 'lib/batch_load/import.rb', line 119

def user_map(h)
  @user_header_map[h] ? @user_header_map[h] : h
end

- (Boolean) valid?

Returns:

  • (Boolean)


124
125
126
127
# File 'lib/batch_load/import.rb', line 124

def valid?
  return false unless @project_id && @user && @file && csv && errors.empty? && file_errors.empty?
  true
end

- (Object) valid_objects

return [Array] all objects (parsed records) that are .valid?



238
239
240
# File 'lib/batch_load/import.rb', line 238

def valid_objects
  all_objects.select { |o| o.valid? }
end

- (Boolean) warn_level_ok?

Returns:

  • (Boolean)


150
151
152
# File 'lib/batch_load/import.rb', line 150

def warn_level_ok?
  true
end