Class: BatchLoad::Import

Inherits:
Object
  • Object
show all
Defined in:
lib/batch_load/import.rb

Overview

A generic object for managing CSV based imports

Defined Under Namespace

Classes: AssertedDistributions, CollectingEvents, CollectionObjects, Descriptors, Otus, TaxonNames, TaxonifiToTaxonworks

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(project_id: nil, user_id: nil, file: nil, process: true, import_level: :warn, user_header_map: {}) ⇒ Import

Returns a new instance of Import.

Parameters:

  • args (Hash) 


70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
 
# File 'lib/batch_load/import.rb', line 70

def initialize(project_id: nil, user_id: nil, file: nil, process: true, import_level: :warn, user_header_map: {})
  @processed = false
  @import_level = import_level
  @project_id = project_id&.to_i
  @user_id = user_id&.to_i
  @file = file

  @user_header_map = user_header_map

  @processed_rows  = {}
  @successful_rows = nil

  @user = User.find(@user_id)

  @file_errors = []
  @errors ||= [] # can be set in subclasses in some cases

  @create_attempted = false

  process && build
end

Instance Attribute Details

#create_attempted ⇒ Object

An attempt was made to create new records



29
30
31
 
# File 'lib/batch_load/import.rb', line 29

def create_attempted
  @create_attempted
end

#csv ⇒ CSV?

Returns:

  • (CSV, nil) 


58
59
60
 
# File 'lib/batch_load/import.rb', line 58

def csv
  @csv
end

#errors ⇒ Object

Errors from the import process itself.



64
65
66
 
# File 'lib/batch_load/import.rb', line 64

def errors
  @errors
end

#file ⇒ Object

The input file, as it comes in on the form



55
56
57
 
# File 'lib/batch_load/import.rb', line 55

def file
  @file
end

#file_errors ⇒ Object

Errors with the file itself, rather than its content



61
62
63
 
# File 'lib/batch_load/import.rb', line 61

def file_errors
  @file_errors
end

#import_level ⇒ Object

How forgiving the import process is

:warn -> all possible names will be added, with those not validating ignored
:line_strict -> there is one record assumed / line, and each line must have a single valid record
:strict -> all processed records must be valid



52
53
54
 
# File 'lib/batch_load/import.rb', line 52

def import_level
  @import_level
end

#processed ⇒ Object

File is processable, at the basic level, and is ready for preview/created



26
27
28
 
# File 'lib/batch_load/import.rb', line 26

def processed
  @processed
end

#processed_rows ⇒ Object

An index of all rows for which some data was present, index is line number, points to a RowParse instance



18
19
20
 
# File 'lib/batch_load/import.rb', line 18

def processed_rows
  @processed_rows
end

#project ⇒ Object

Returns the value of attribute project.



31
32
33
 
# File 'lib/batch_load/import.rb', line 31

def project
  @project
end

#project_id ⇒ Integer

Returns:

  • (Integer) 


34
35
36
 
# File 'lib/batch_load/import.rb', line 34

def project_id
  @project_id
end

#successful_rows ⇒ Object

return [Array] the line numbers that resulted in saved records



23
24
25
 
# File 'lib/batch_load/import.rb', line 23

def successful_rows
  @successful_rows
end

#total_data_lines ⇒ Object

return [Integer] the total lines with data



46
47
48
 
# File 'lib/batch_load/import.rb', line 46

def total_data_lines
  @total_data_lines
end

#total_lines ⇒ Object

The number of non-header rows in the file



43
44
45
 
# File 'lib/batch_load/import.rb', line 43

def total_lines
  @total_lines
end

#user ⇒ Object

Returns User instance.

Returns:

  • User instance



40
41
42
 
# File 'lib/batch_load/import.rb', line 40

def user
  @user
end

#user_header_map ⇒ Object

User provided map of their header (key) to our attribute (value)



67
68
69
 
# File 'lib/batch_load/import.rb', line 67

def user_header_map
  @user_header_map
end

#user_id ⇒ Integer

Returns:

  • (Integer) 


37
38
39
 
# File 'lib/batch_load/import.rb', line 37

def user_id
  @user_id
end

Instance Method Details

#all_objects ⇒ Object

return [Array] all objects (parsed records)



291
292
293
 
# File 'lib/batch_load/import.rb', line 291

def all_objects
  processed_rows.collect { |_i, rp| rp.all_objects }.flatten
end

#build ⇒ Object 



250
251
252
 
# File 'lib/batch_load/import.rb', line 250

def build
  raise 'This method must be provided in each respective subclass.'
end

#create ⇒ true

Iterates in line order and attempts to save each record, loggine the result.

Returns:

  • (true) 


189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
 
# File 'lib/batch_load/import.rb', line 189

def create
  @create_attempted = true

  if ready_to_create?
    # TODO: DRY
    if a = save_order

      a.each do |k|
        sorted_processed_rows.each_value do |rp|
          rp.objects[k].each do |o|
            begin
              # puts o.name
              o.save! unless o.persisted?

            rescue ActiveRecord::RecordInvalid => o

              a = o.record
              o.record
            end
          end
        end
      end

    else
      sorted_processed_rows.each_value do |rp|
        rp.objects.each_value do |objs|
          objs.each do |o|

            if o.kind_of?(TaxonName)
              o.valid?

              # If the only errors are on invalid children then save this
              # record ignoring those errors.
              # TODO: does this skip call-backs ?!
              if o.errors.full_messages == ['Children is invalid']

                c = o.children.to_a
                o.children.clear

                o.save
              else
                o.save
              end

            else
              o.save
            end
          end
        end
      end
    end

  else
    @errors << "Import level #{import_level} has prevented creation." unless import_level_ok?
    @errors << 'CSV has not been processed.' unless processed?
    @errors << 'One of user_id, project_id or file has not been provided.' unless valid?
  end

  true
end

#create_attempted? ⇒ Boolean

return [Boolean] whether an attempt at creating records has occured

Returns:

  • (Boolean) 


255
256
257
 
# File 'lib/batch_load/import.rb', line 255

def create_attempted?
  create_attempted
end

#import_level_ok? ⇒ Boolean

Returns:

  • (Boolean) 


154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/batch_load/import.rb', line 154

def import_level_ok?
  case import_level.to_sym
  when :warn
    warn_level_ok?
  when :strict
    strict_level_ok?
  when :line_strict
    line_strict_level_ok?
  else
    false
  end
end

#line_strict_level_ok? ⇒ Boolean

Returns:

  • (Boolean) 


181
182
183
 
# File 'lib/batch_load/import.rb', line 181

def line_strict_level_ok?
  total_data_lines == valid_objects.size
end

#processed? ⇒ Boolean

return [Boolean] whether an attempt to process the input file has occured

Returns:

  • (Boolean) 


260
261
262
 
# File 'lib/batch_load/import.rb', line 260

def processed?
  processed
end

#ready_to_create? ⇒ Boolean

return [Boolean] whether the instance is configured

Returns:

  • (Boolean) 


149
150
151
 
# File 'lib/batch_load/import.rb', line 149

def ready_to_create?
  valid? && processed? && import_level_ok?
end

#save_order ⇒ Object

Save order is by ROW only, not by type



296
297
298
 
# File 'lib/batch_load/import.rb', line 296

def save_order
  self.class.const_defined?('SAVE_ORDER') ? self.class::SAVE_ORDER : nil
end

#sorted_processed_rows ⇒ Object

return [Hash] processed rows, sorted by line number

?! key order might not persist ?!



281
282
283
 
# File 'lib/batch_load/import.rb', line 281

def sorted_processed_rows
  processed_rows.sort.to_h
end

#strict_level_ok? ⇒ Boolean

Returns:

  • (Boolean) 


173
174
175
176
177
178
 
# File 'lib/batch_load/import.rb', line 173

def strict_level_ok?
  all_objects.each do |o|
    return false unless o.valid?
  end
  true
end

#total_records_created ⇒ Object

return [Integer] the total number of records created



275
276
277
 
# File 'lib/batch_load/import.rb', line 275

def total_records_created
  successful_rows.inject(t = 0) { |t, i| t += processed_rows[i].persisted_objects.size }
end

#user_map(h) ⇒ String

Parameters:

  • h (String)

Returns:

  • (String) 


137
138
139
 
# File 'lib/batch_load/import.rb', line 137

def user_map(h)
  user_header_map[h] ? user_header_map[h] : h
end

#valid? ⇒ Boolean

Returns:

  • (Boolean) 


142
143
144
145
 
# File 'lib/batch_load/import.rb', line 142

def valid?
  return false unless project_id && user && file && csv && errors.empty? && file_errors.empty?
  true
end

#valid_objects ⇒ Object

return [Array] all objects (parsed records) that are .valid?



286
287
288
 
# File 'lib/batch_load/import.rb', line 286

def valid_objects
  all_objects.select { |o| o.valid? }
end

#warn_level_ok? ⇒ Boolean

Returns:

  • (Boolean) 


168
169
170
 
# File 'lib/batch_load/import.rb', line 168

def warn_level_ok?
  true
end