Class: BatchLoad::Import

Inherits:
Object
  • Object
show all
Defined in:
lib/batch_load/import.rb

Overview

A generic object for managing CSV based imports

Direct Known Subclasses

AssertedDistributions, CollectingEvents, CollectionObjects, DWCA, Otus, TaxonifiToTaxonworks, Import::Namespaces::SimpleInterpreter

Defined Under Namespace

Classes: AssertedDistributions, CollectingEvents, CollectionObjects, DWCA, Otus, TaxonifiToTaxonworks

Instance Attribute Summary (collapse)

Instance Method Summary (collapse)

Constructor Details

- (Import) initialize(project_id: nil, user_id: nil, file: nil, process: true, import_level: :warn, user_header_map: {})

Returns a new instance of Import



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
# File 'lib/batch_load/import.rb', line 62

def initialize(project_id: nil, user_id: nil, file: nil, process: true, import_level: :warn, user_header_map: {})
  @processed    = false
  @import_level = import_level
  @project_id   = project_id
  @user_id      = user_id
  @file         = file

  @user_header_map = user_header_map

  @processed_rows  = {}
  @successful_rows = nil

  @user = User.find(@user_id)

  @file_errors = []
  @errors      ||= [] # can be set in subclasses in some cases

  @create_attempted = false

  process && build
end

Instance Attribute Details

- (Object) create_attempted

An attempt was made to create new records



31
32
33
# File 'lib/batch_load/import.rb', line 31

def create_attempted
  @create_attempted
end

- (Object) csv

The resultant csv table



51
52
53
# File 'lib/batch_load/import.rb', line 51

def csv
  @csv
end

- (Object) errors

Errors from the import process itself.



57
58
59
# File 'lib/batch_load/import.rb', line 57

def errors
  @errors
end

- (Object) file

The input file, as it comes in on the form



48
49
50
# File 'lib/batch_load/import.rb', line 48

def file
  @file
end

- (Object) file_errors

Errors with the file itself, rather than its content



54
55
56
# File 'lib/batch_load/import.rb', line 54

def file_errors
  @file_errors
end

- (Object) import_level

How forgiving the import process is

:warn -> all possible names will be added, with those not validating ignored
:line_strict -> there is one record assumed / line, and each line must have a single valid record
:strict -> all processed records must be valid


45
46
47
# File 'lib/batch_load/import.rb', line 45

def import_level
  @import_level
end

- (Object) processed

File is processable, at the basic level, and is ready for preview/created



28
29
30
# File 'lib/batch_load/import.rb', line 28

def processed
  @processed
end

- (Object) processed_rows

An index of all rows for which some data was present, index is line number, points to a RowParse instance



20
21
22
# File 'lib/batch_load/import.rb', line 20

def processed_rows
  @processed_rows
end

- (Object) project

Returns the value of attribute project



33
34
35
# File 'lib/batch_load/import.rb', line 33

def project
  @project
end

- (Object) successful_rows

return [Array] the line numbers that resulted in saved records



25
26
27
# File 'lib/batch_load/import.rb', line 25

def successful_rows
  @successful_rows
end

- (Object) total_data_lines

return [Integer] the total lines with data



39
40
41
# File 'lib/batch_load/import.rb', line 39

def total_data_lines
  @total_data_lines
end

- (Object) total_lines

The number of non-header rows in the file



36
37
38
# File 'lib/batch_load/import.rb', line 36

def total_lines
  @total_lines
end

- (Object) user

Returns the value of attribute user



33
34
35
# File 'lib/batch_load/import.rb', line 33

def user
  @user
end

- (Object) user_header_map

User provided map of their header (key) to our attribute (value)



60
61
62
# File 'lib/batch_load/import.rb', line 60

def user_header_map
  @user_header_map
end

Instance Method Details

- (Object) all_objects

return [Array] all objects (parsed records)



220
221
222
# File 'lib/batch_load/import.rb', line 220

def all_objects
  processed_rows.collect { |i, rp| rp.all_objects }.flatten
end

- (Object) build



179
180
181
# File 'lib/batch_load/import.rb', line 179

def build
  raise 'This method must be provided in each respective subclass.'
end

- (Object) create

Iterates in line order and attempts to save each record return [true]



160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
# File 'lib/batch_load/import.rb', line 160

def create
  @create_attempted = true
  if ready_to_create?
    # TODO: (additional!?) enumerate order per klass for save off
    sorted_processed_rows.each do |i, rp|
      rp.objects.each do |object_type, objs|
        objs.each do |o|
          o.save
        end
      end
    end
  else
    @errors << "Import level #{import_level} has prevented creation." unless import_level_ok?
    @errors << 'CSV has not been processed.' unless processed?
    @errors << 'One of user_id, project_id or file has not been provided.' unless valid?
  end
  true
end

- (Boolean) create_attempted?

return [Boolean] whether an attempt at creating records has occured

Returns:

  • (Boolean)


184
185
186
# File 'lib/batch_load/import.rb', line 184

def create_attempted?
  create_attempted
end

- (Boolean) import_level_ok?

Returns:

  • (Boolean)


130
131
132
133
134
135
136
137
138
139
140
141
# File 'lib/batch_load/import.rb', line 130

def import_level_ok?
  case import_level.to_sym
    when :warn
      warn_level_ok?
    when :strict
      strict_level_ok?
    when :line_strict
      line_strict_level_ok?
    else
      false
  end
end

- (Boolean) line_strict_level_ok?

Returns:

  • (Boolean)


154
155
156
# File 'lib/batch_load/import.rb', line 154

def line_strict_level_ok?
  total_data_lines == valid_objects.size
end

- (Boolean) processed?

return [Boolean] whether an attempt to process the input file has occured

Returns:

  • (Boolean)


189
190
191
# File 'lib/batch_load/import.rb', line 189

def processed?
  processed
end

- (Boolean) ready_to_create?

return [Boolean] whether the instance is configured

Returns:

  • (Boolean)


126
127
128
# File 'lib/batch_load/import.rb', line 126

def ready_to_create?
  valid? && processed? && import_level_ok?
end

- (Object) sorted_processed_rows

return [Hash] processed rows, sorted by line number

?! key order might not persist ?!


210
211
212
# File 'lib/batch_load/import.rb', line 210

def sorted_processed_rows
  @processed_rows.sort.to_h
end

- (Boolean) strict_level_ok?

Returns:

  • (Boolean)


147
148
149
150
151
152
# File 'lib/batch_load/import.rb', line 147

def strict_level_ok?
  all_objects.each do |o|
    return false unless o.valid?
  end
  true
end

- (Object) total_records_created

return [Integer] the total number of records created



204
205
206
# File 'lib/batch_load/import.rb', line 204

def total_records_created
  successful_rows.inject(t = 0) { |t, i| t += processed_rows[i].persisted_objects.size }
end

- (Object) user_map(h)



116
117
118
# File 'lib/batch_load/import.rb', line 116

def user_map(h)
  @user_header_map[h] ? @user_header_map[h] : h
end

- (Boolean) valid?

Returns:

  • (Boolean)


120
121
122
123
# File 'lib/batch_load/import.rb', line 120

def valid?
  return false unless @project_id && @user && @file && csv && errors.empty? && file_errors.empty?
  true
end

- (Object) valid_objects

return [Array] all objects (parsed records) that are .valid?



215
216
217
# File 'lib/batch_load/import.rb', line 215

def valid_objects
  all_objects.select { |o| o.valid? }
end

- (Boolean) warn_level_ok?

Returns:

  • (Boolean)


143
144
145
# File 'lib/batch_load/import.rb', line 143

def warn_level_ok?
  true
end