Class: BatchLoad::Import

Inherits:
Object
  • Object
show all
Defined in:
lib/batch_load/import.rb

Overview

A generic object for managing CSV based imports

Defined Under Namespace

Classes: AssertedDistributions, CollectingEvents, CollectionObjects, Descriptors, Otus, TaxonNames, TaxonifiToTaxonworks

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(project_id: nil, user_id: nil, file: nil, process: true, import_level: :warn, user_header_map: {}) ⇒ Import

Returns a new instance of Import.

Parameters:

  • args (Hash) 


67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/batch_load/import.rb', line 67

def initialize(project_id: nil, user_id: nil, file: nil, process: true, import_level: :warn, user_header_map: {})
  @processed = false
  @import_level = import_level
  @project_id = project_id&.to_i
  @user_id = user_id&.to_i
  @file = file

  @user_header_map = user_header_map

  @processed_rows  = {}
  @successful_rows = nil

  @user = User.find(@user_id)

  @file_errors = []
  @errors ||= [] # can be set in subclasses in some cases

  @create_attempted = false

  process && build
end

Instance Attribute Details

#create_attempted ⇒ Object

An attempt was made to create new records



29
30
31
 
# File 'lib/batch_load/import.rb', line 29

def create_attempted
  @create_attempted
end

#csv ⇒ CSV?

Returns:

  • (CSV, nil) 


55
56
57
 
# File 'lib/batch_load/import.rb', line 55

def csv
  @csv
end

#errors ⇒ Object

Errors from the import process itself.



61
62
63
 
# File 'lib/batch_load/import.rb', line 61

def errors
  @errors
end

#file ⇒ Object

The input file, as it comes in on the form



52
53
54
 
# File 'lib/batch_load/import.rb', line 52

def file
  @file
end

#file_errors ⇒ Object

Errors with the file itself, rather than its content



58
59
60
 
# File 'lib/batch_load/import.rb', line 58

def file_errors
  @file_errors
end

#import_level ⇒ Object

How forgiving the import process is

:warn -> all possible names will be added, with those not validating ignored
:line_strict -> there is one record assumed / line, and each line must have a single valid record
:strict -> all processed records must be valid



49
50
51
 
# File 'lib/batch_load/import.rb', line 49

def import_level
  @import_level
end

#processed ⇒ Object

File is processable, at the basic level, and is ready for preview/created



26
27
28
 
# File 'lib/batch_load/import.rb', line 26

def processed
  @processed
end

#processed_rows ⇒ Object

An index of all rows for which some data was present, index is line number, points to a RowParse instance



18
19
20
 
# File 'lib/batch_load/import.rb', line 18

def processed_rows
  @processed_rows
end

#project ⇒ Object

Returns the value of attribute project.



31
32
33
 
# File 'lib/batch_load/import.rb', line 31

def project
  @project
end

#project_id ⇒ Integer

Returns:

  • (Integer) 


34
35
36
 
# File 'lib/batch_load/import.rb', line 34

def project_id
  @project_id
end

#successful_rows ⇒ Object

return [Array] the line numbers that resulted in saved records



23
24
25
 
# File 'lib/batch_load/import.rb', line 23

def successful_rows
  @successful_rows
end

#total_data_lines ⇒ Object

return [Integer] the total lines with data



43
44
45
 
# File 'lib/batch_load/import.rb', line 43

def total_data_lines
  @total_data_lines
end

#total_lines ⇒ Object

The number of non-header rows in the file



40
41
42
 
# File 'lib/batch_load/import.rb', line 40

def total_lines
  @total_lines
end

#user ⇒ Object

Returns the value of attribute user.



31
32
33
 
# File 'lib/batch_load/import.rb', line 31

def user
  @user
end

#user_header_map ⇒ Object

User provided map of their header (key) to our attribute (value)



64
65
66
 
# File 'lib/batch_load/import.rb', line 64

def user_header_map
  @user_header_map
end

#user_id ⇒ Integer

Returns:

  • (Integer) 


37
38
39
 
# File 'lib/batch_load/import.rb', line 37

def user_id
  @user_id
end

Instance Method Details

#all_objects ⇒ Object

return [Array] all objects (parsed records)



260
261
262
 
# File 'lib/batch_load/import.rb', line 260

def all_objects
  processed_rows.collect { |_i, rp| rp.all_objects }.flatten
end

#build ⇒ Object 



219
220
221
 
# File 'lib/batch_load/import.rb', line 219

def build
  raise 'This method must be provided in each respective subclass.'
end

#create ⇒ Boolean

Iterates in line order and attempts to save each record return [true]

Returns:

  • (Boolean) 


179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/batch_load/import.rb', line 179

def create
  @create_attempted = true

  if ready_to_create?
    # TODO: DRY
    if a = save_order

      a.each do |k|
        sorted_processed_rows.each_value do |rp|
            rp.objects[k].each do |o|
              begin
                # puts o.name
                o.save! unless o.persisted?
               rescue ActiveRecord::RecordInvalid => o
                 a = o.record
                 o.record
              end
            end
          end
      end

    else

      sorted_processed_rows.each_value do |rp|
        rp.objects.each_value do |objs|
          objs.each do |o|
            o.save
          end
        end
      end

    end
  else
    @errors << "Import level #{import_level} has prevented creation." unless import_level_ok?
    @errors << 'CSV has not been processed.' unless processed?
    @errors << 'One of user_id, project_id or file has not been provided.' unless valid?
  end
  true
end

#create_attempted? ⇒ Boolean

return [Boolean] whether an attempt at creating records has occured

Returns:

  • (Boolean) 


224
225
226
 
# File 'lib/batch_load/import.rb', line 224

def create_attempted?
  create_attempted
end

#import_level_ok? ⇒ Boolean

Returns:

  • (Boolean) 


145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/batch_load/import.rb', line 145

def import_level_ok?
  case import_level.to_sym
  when :warn
    warn_level_ok?
  when :strict
    strict_level_ok?
  when :line_strict
    line_strict_level_ok?
  else
    false
  end
end

#line_strict_level_ok? ⇒ Boolean

Returns:

  • (Boolean) 


172
173
174
 
# File 'lib/batch_load/import.rb', line 172

def line_strict_level_ok?
  total_data_lines == valid_objects.size
end

#processed? ⇒ Boolean

return [Boolean] whether an attempt to process the input file has occured

Returns:

  • (Boolean) 


229
230
231
 
# File 'lib/batch_load/import.rb', line 229

def processed?
  processed
end

#ready_to_create? ⇒ Boolean

return [Boolean] whether the instance is configured

Returns:

  • (Boolean) 


140
141
142
 
# File 'lib/batch_load/import.rb', line 140

def ready_to_create?
  valid? && processed? && import_level_ok?
end

#save_order ⇒ Object

Save order is by ROW only, not by type



265
266
267
 
# File 'lib/batch_load/import.rb', line 265

def save_order
  self.class.const_defined?('SAVE_ORDER') ? self.class::SAVE_ORDER : nil
end

#sorted_processed_rows ⇒ Object

return [Hash] processed rows, sorted by line number

?! key order might not persist ?!



250
251
252
 
# File 'lib/batch_load/import.rb', line 250

def sorted_processed_rows
  processed_rows.sort.to_h
end

#strict_level_ok? ⇒ Boolean

Returns:

  • (Boolean) 


164
165
166
167
168
169
 
# File 'lib/batch_load/import.rb', line 164

def strict_level_ok?
  all_objects.each do |o|
    return false unless o.valid?
  end
  true
end

#total_records_created ⇒ Object

return [Integer] the total number of records created



244
245
246
 
# File 'lib/batch_load/import.rb', line 244

def total_records_created
  successful_rows.inject(t = 0) { |t, i| t += processed_rows[i].persisted_objects.size }
end

#user_map(h) ⇒ String

Parameters:

  • h (String)

Returns:

  • (String) 


128
129
130
 
# File 'lib/batch_load/import.rb', line 128

def user_map(h)
  @user_header_map[h] ? @user_header_map[h] : h
end

#valid? ⇒ Boolean

Returns:

  • (Boolean) 


133
134
135
136
 
# File 'lib/batch_load/import.rb', line 133

def valid?
  return false unless @project_id && @user && @file && csv && errors.empty? && file_errors.empty?
  true
end

#valid_objects ⇒ Object

return [Array] all objects (parsed records) that are .valid?



255
256
257
 
# File 'lib/batch_load/import.rb', line 255

def valid_objects
  all_objects.select { |o| o.valid? }
end

#warn_level_ok? ⇒ Boolean

Returns:

  • (Boolean) 


159
160
161
 
# File 'lib/batch_load/import.rb', line 159

def warn_level_ok?
  true
end