Class: BatchLoad::Import

Inherits:
Object
  • Object
show all
Defined in:
lib/batch_load/import.rb

Overview

A generic object for managing CSV based imports

Defined Under Namespace

Classes: AssertedDistributions, CollectingEvents, CollectionObjects, Otus, TaxonNames, TaxonifiToTaxonworks

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(project_id: nil, user_id: nil, file: nil, process: true, import_level: :warn, user_header_map: {}) ⇒ Import

Returns a new instance of Import.

Parameters:

  • args (Hash)


61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
# File 'lib/batch_load/import.rb', line 61

def initialize(project_id: nil, user_id: nil, file: nil, process: true, import_level: :warn, user_header_map: {})
  @processed = false
  @import_level = import_level
  @project_id = project_id
  @user_id = user_id
  @file = file

  @user_header_map = user_header_map

  @processed_rows  = {}
  @successful_rows = nil

  @user = User.find(@user_id)

  @file_errors = []
  @errors ||= [] # can be set in subclasses in some cases

  @create_attempted = false

  process && build
end

Instance Attribute Details

#create_attemptedObject

An attempt was made to create new records



29
30
31
# File 'lib/batch_load/import.rb', line 29

def create_attempted
  @create_attempted
end

#csvCSV?

Returns:

  • (CSV, nil)


49
50
51
# File 'lib/batch_load/import.rb', line 49

def csv
  @csv
end

#errorsObject

Errors from the import process itself.



55
56
57
# File 'lib/batch_load/import.rb', line 55

def errors
  @errors
end

#fileObject

The input file, as it comes in on the form



46
47
48
# File 'lib/batch_load/import.rb', line 46

def file
  @file
end

#file_errorsObject

Errors with the file itself, rather than its content



52
53
54
# File 'lib/batch_load/import.rb', line 52

def file_errors
  @file_errors
end

#import_levelObject

How forgiving the import process is

:warn -> all possible names will be added, with those not validating ignored
:line_strict -> there is one record assumed / line, and each line must have a single valid record
:strict -> all processed records must be valid


43
44
45
# File 'lib/batch_load/import.rb', line 43

def import_level
  @import_level
end

#processedObject

File is processable, at the basic level, and is ready for preview/created



26
27
28
# File 'lib/batch_load/import.rb', line 26

def processed
  @processed
end

#processed_rowsObject

An index of all rows for which some data was present, index is line number, points to a RowParse instance



18
19
20
# File 'lib/batch_load/import.rb', line 18

def processed_rows
  @processed_rows
end

#projectObject

Returns the value of attribute project.



31
32
33
# File 'lib/batch_load/import.rb', line 31

def project
  @project
end

#successful_rowsObject

return [Array] the line numbers that resulted in saved records



23
24
25
# File 'lib/batch_load/import.rb', line 23

def successful_rows
  @successful_rows
end

#total_data_linesObject

return [Integer] the total lines with data



37
38
39
# File 'lib/batch_load/import.rb', line 37

def total_data_lines
  @total_data_lines
end

#total_linesObject

The number of non-header rows in the file



34
35
36
# File 'lib/batch_load/import.rb', line 34

def total_lines
  @total_lines
end

#userObject

Returns the value of attribute user.



31
32
33
# File 'lib/batch_load/import.rb', line 31

def user
  @user
end

#user_header_mapObject

User provided map of their header (key) to our attribute (value)



58
59
60
# File 'lib/batch_load/import.rb', line 58

def user_header_map
  @user_header_map
end

Instance Method Details

#all_objectsObject

return [Array] all objects (parsed records)



245
246
247
# File 'lib/batch_load/import.rb', line 245

def all_objects
  processed_rows.collect { |_i, rp| rp.all_objects }.flatten
end

#buildObject



204
205
206
# File 'lib/batch_load/import.rb', line 204

def build
  raise 'This method must be provided in each respective subclass.'
end

#createBoolean

Iterates in line order and attempts to save each record return [true]

Returns:

  • (Boolean)


172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
# File 'lib/batch_load/import.rb', line 172

def create
  @create_attempted = true
  if ready_to_create?

    # TODO: DRY
    if a = save_order
      sorted_processed_rows.each_value do |rp|
        a.each do |k|
          rp.objects[k].each do |o|
            o.save unless o.persisted?
          end
        end
      end

    else
      sorted_processed_rows.each_value do |rp|
        rp.objects.each_value do |objs|
          objs.each do |o|
            o.save
          end
        end
      end
    end

  else
    @errors << "Import level #{import_level} has prevented creation." unless import_level_ok?
    @errors << 'CSV has not been processed.' unless processed?
    @errors << 'One of user_id, project_id or file has not been provided.' unless valid?
  end
  true
end

#create_attempted?Boolean

return [Boolean] whether an attempt at creating records has occured

Returns:

  • (Boolean)


209
210
211
# File 'lib/batch_load/import.rb', line 209

def create_attempted?
  create_attempted
end

#import_level_ok?Boolean

Returns:

  • (Boolean)


138
139
140
141
142
143
144
145
146
147
148
149
# File 'lib/batch_load/import.rb', line 138

def import_level_ok?
  case import_level.to_sym
    when :warn
      warn_level_ok?
    when :strict
      strict_level_ok?
    when :line_strict
      line_strict_level_ok?
    else
      false
  end
end

#line_strict_level_ok?Boolean

Returns:

  • (Boolean)


165
166
167
# File 'lib/batch_load/import.rb', line 165

def line_strict_level_ok?
  total_data_lines == valid_objects.size
end

#processed?Boolean

return [Boolean] whether an attempt to process the input file has occured

Returns:

  • (Boolean)


214
215
216
# File 'lib/batch_load/import.rb', line 214

def processed?
  processed
end

#ready_to_create?Boolean

return [Boolean] whether the instance is configured

Returns:

  • (Boolean)


133
134
135
# File 'lib/batch_load/import.rb', line 133

def ready_to_create?
  valid? && processed? && import_level_ok?
end

#save_orderObject



249
250
251
# File 'lib/batch_load/import.rb', line 249

def save_order
  self.class.const_defined?('SAVE_ORDER') ? self.class::SAVE_ORDER : nil
end

#sorted_processed_rowsObject

return [Hash] processed rows, sorted by line number

?! key order might not persist ?!


235
236
237
# File 'lib/batch_load/import.rb', line 235

def sorted_processed_rows
  @processed_rows.sort.to_h
end

#strict_level_ok?Boolean

Returns:

  • (Boolean)


157
158
159
160
161
162
# File 'lib/batch_load/import.rb', line 157

def strict_level_ok?
  all_objects.each do |o|
    return false unless o.valid?
  end
  true
end

#total_records_createdObject

return [Integer] the total number of records created



229
230
231
# File 'lib/batch_load/import.rb', line 229

def total_records_created
  successful_rows.inject(t = 0) { |t, i| t += processed_rows[i].persisted_objects.size }
end

#user_map(h) ⇒ String

Parameters:

  • h (String)

Returns:

  • (String)


121
122
123
# File 'lib/batch_load/import.rb', line 121

def user_map(h)
  @user_header_map[h] ? @user_header_map[h] : h
end

#valid?Boolean

Returns:

  • (Boolean)


126
127
128
129
# File 'lib/batch_load/import.rb', line 126

def valid?
  return false unless @project_id && @user && @file && csv && errors.empty? && file_errors.empty?
  true
end

#valid_objectsObject

return [Array] all objects (parsed records) that are .valid?



240
241
242
# File 'lib/batch_load/import.rb', line 240

def valid_objects
  all_objects.select { |o| o.valid? }
end

#warn_level_ok?Boolean

Returns:

  • (Boolean)


152
153
154
# File 'lib/batch_load/import.rb', line 152

def warn_level_ok?
  true
end