Class: BatchLoad::Import

Inherits:
Object
  • Object
show all
Defined in:
lib/batch_load/import.rb

Overview

A generic object for managing CSV based imports

Direct Known Subclasses

AssertedDistributions, CollectingEvents, CollectingEvents::CastorInterpreter, CollectionObjects, CollectionObjects::CastorInterpreter, DWCA, Otus, Otus::IdentifiersInterpreter, TaxonNames, TaxonNames::CastorInterpreter, TaxonifiToTaxonworks, Import::Namespaces::SimpleInterpreter, Import::SequenceRelationships::PrimersInterpreter, Import::Sequences::GenbankInterpreter, Import::Sequences::PrimersInterpreter

Defined Under Namespace

Classes: AssertedDistributions, CollectingEvents, CollectionObjects, DWCA, Otus, TaxonNames, TaxonifiToTaxonworks

Instance Attribute Summary (collapse)

Instance Method Summary (collapse)

Constructor Details

- (Import) initialize(project_id: nil, user_id: nil, file: nil, process: true, import_level: :warn, user_header_map: {})

Returns a new instance of Import

Parameters:

  • args (Hash)


61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
# File 'lib/batch_load/import.rb', line 61

def initialize(project_id: nil, user_id: nil, file: nil, process: true, import_level: :warn, user_header_map: {})
  @processed    = false
  @import_level = import_level
  @project_id   = project_id
  @user_id      = user_id
  @file         = file

  @user_header_map = user_header_map

  @processed_rows  = {}
  @successful_rows = nil

  @user = User.find(@user_id)

  @file_errors = []
  @errors      ||= [] # can be set in subclasses in some cases

  @create_attempted = false

  process && build
end

Instance Attribute Details

- (Object) create_attempted

An attempt was made to create new records



29
30
31
# File 'lib/batch_load/import.rb', line 29

def create_attempted
  @create_attempted
end

- (CSV?) csv

Returns:

  • (CSV, nil)


49
50
51
# File 'lib/batch_load/import.rb', line 49

def csv
  @csv
end

- (Object) errors

Errors from the import process itself.



55
56
57
# File 'lib/batch_load/import.rb', line 55

def errors
  @errors
end

- (Object) file

The input file, as it comes in on the form



46
47
48
# File 'lib/batch_load/import.rb', line 46

def file
  @file
end

- (Object) file_errors

Errors with the file itself, rather than its content



52
53
54
# File 'lib/batch_load/import.rb', line 52

def file_errors
  @file_errors
end

- (Object) import_level

How forgiving the import process is

:warn -> all possible names will be added, with those not validating ignored
:line_strict -> there is one record assumed / line, and each line must have a single valid record
:strict -> all processed records must be valid


43
44
45
# File 'lib/batch_load/import.rb', line 43

def import_level
  @import_level
end

- (Object) processed

File is processable, at the basic level, and is ready for preview/created



26
27
28
# File 'lib/batch_load/import.rb', line 26

def processed
  @processed
end

- (Object) processed_rows

An index of all rows for which some data was present, index is line number, points to a RowParse instance



18
19
20
# File 'lib/batch_load/import.rb', line 18

def processed_rows
  @processed_rows
end

- (Object) project

Returns the value of attribute project



31
32
33
# File 'lib/batch_load/import.rb', line 31

def project
  @project
end

- (Object) successful_rows

return [Array] the line numbers that resulted in saved records



23
24
25
# File 'lib/batch_load/import.rb', line 23

def successful_rows
  @successful_rows
end

- (Object) total_data_lines

return [Integer] the total lines with data



37
38
39
# File 'lib/batch_load/import.rb', line 37

def total_data_lines
  @total_data_lines
end

- (Object) total_lines

The number of non-header rows in the file



34
35
36
# File 'lib/batch_load/import.rb', line 34

def total_lines
  @total_lines
end

- (Object) user

Returns the value of attribute user



31
32
33
# File 'lib/batch_load/import.rb', line 31

def user
  @user
end

- (Object) user_header_map

User provided map of their header (key) to our attribute (value)



58
59
60
# File 'lib/batch_load/import.rb', line 58

def user_header_map
  @user_header_map
end

Instance Method Details

- (Object) all_objects

return [Array] all objects (parsed records)



244
245
246
# File 'lib/batch_load/import.rb', line 244

def all_objects
  processed_rows.collect { |_i, rp| rp.all_objects }.flatten
end

- (Object) build



203
204
205
# File 'lib/batch_load/import.rb', line 203

def build
  raise 'This method must be provided in each respective subclass.'
end

- (Boolean) create

Iterates in line order and attempts to save each record return [true]

Returns:

  • (Boolean)


171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
# File 'lib/batch_load/import.rb', line 171

def create
  @create_attempted = true
  if ready_to_create?

    # TODO: DRY
    if a = save_order
      sorted_processed_rows.each_value do |rp|
        a.each do |k|
          rp.objects[k].each do |o|
            o.save unless o.persisted?
          end
        end
      end

    else
      sorted_processed_rows.each_value do |rp|
        rp.objects.each_value do |objs|
          objs.each do |o|
            o.save
          end
        end
      end
    end

  else
    @errors << "Import level #{import_level} has prevented creation." unless import_level_ok?
    @errors << 'CSV has not been processed.' unless processed?
    @errors << 'One of user_id, project_id or file has not been provided.' unless valid?
  end
  true
end

- (Boolean) create_attempted?

return [Boolean] whether an attempt at creating records has occured

Returns:

  • (Boolean)


208
209
210
# File 'lib/batch_load/import.rb', line 208

def create_attempted?
  create_attempted
end

- (Boolean) import_level_ok?

Returns:

  • (Boolean)


137
138
139
140
141
142
143
144
145
146
147
148
# File 'lib/batch_load/import.rb', line 137

def import_level_ok?
  case import_level.to_sym
    when :warn
      warn_level_ok?
    when :strict
      strict_level_ok?
    when :line_strict
      line_strict_level_ok?
    else
      false
  end
end

- (Boolean) line_strict_level_ok?

Returns:

  • (Boolean)


164
165
166
# File 'lib/batch_load/import.rb', line 164

def line_strict_level_ok?
  total_data_lines == valid_objects.size
end

- (Boolean) processed?

return [Boolean] whether an attempt to process the input file has occured

Returns:

  • (Boolean)


213
214
215
# File 'lib/batch_load/import.rb', line 213

def processed?
  processed
end

- (Boolean) ready_to_create?

return [Boolean] whether the instance is configured

Returns:

  • (Boolean)


132
133
134
# File 'lib/batch_load/import.rb', line 132

def ready_to_create?
  valid? && processed? && import_level_ok?
end

- (Object) save_order



248
249
250
# File 'lib/batch_load/import.rb', line 248

def save_order
  self.class.const_defined?('SAVE_ORDER') ? self.class::SAVE_ORDER : nil
end

- (Object) sorted_processed_rows

return [Hash] processed rows, sorted by line number

?! key order might not persist ?!


234
235
236
# File 'lib/batch_load/import.rb', line 234

def sorted_processed_rows
  @processed_rows.sort.to_h
end

- (Boolean) strict_level_ok?

Returns:

  • (Boolean)


156
157
158
159
160
161
# File 'lib/batch_load/import.rb', line 156

def strict_level_ok?
  all_objects.each do |o|
    return false unless o.valid?
  end
  true
end

- (Object) total_records_created

return [Integer] the total number of records created



228
229
230
# File 'lib/batch_load/import.rb', line 228

def total_records_created
  successful_rows.inject(t = 0) { |t, i| t += processed_rows[i].persisted_objects.size }
end

- (String) user_map(h)

Parameters:

  • h (String)

Returns:

  • (String)


120
121
122
# File 'lib/batch_load/import.rb', line 120

def user_map(h)
  @user_header_map[h] ? @user_header_map[h] : h
end

- (Boolean) valid?

Returns:

  • (Boolean)


125
126
127
128
# File 'lib/batch_load/import.rb', line 125

def valid?
  return false unless @project_id && @user && @file && csv && errors.empty? && file_errors.empty?
  true
end

- (Object) valid_objects

return [Array] all objects (parsed records) that are .valid?



239
240
241
# File 'lib/batch_load/import.rb', line 239

def valid_objects
  all_objects.select { |o| o.valid? }
end

- (Boolean) warn_level_ok?

Returns:

  • (Boolean)


151
152
153
# File 'lib/batch_load/import.rb', line 151

def warn_level_ok?
  true
end