Class: Support::ProjectStatsDiff

Inherits:
Object
  • Object
show all
Defined in:
lib/support/project_stats_diff.rb

Overview

Compare two TSVs written by rake tw:audit:project_stats and summarize what changed between them: row count changes per project and table, tables emptied or newly populated, projects whose data appeared or disappeared, and tables whose rows changed without a change in count.

The snapshots hold only count, max_id, max_created_at and max_updated_at, so "signals" report only what those prove:

:inserted - max_id grew
:deleted  - count fell, or rows were inserted without the count rising
:updated  - max_updated_at got later without any inserts (deleting
          the newest rows moves it earlier, which is not an update)

Written mostly by Claude.

Defined Under Namespace

Classes: Change

Constant Summary collapse

HEADERS = 
%w[project_id project_name model table count max_created_at max_updated_at max_id].freeze

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(start_rows, end_rows, start_label: 'start', end_label: 'end', project_id: nil) ⇒ ProjectStatsDiff

Returns a new instance of ProjectStatsDiff.

Parameters:

  • start_rows (Array of Hash) —

    rows keyed by HEADERS

  • end_rows (Array of Hash)
  • project_id (Integer, nil) (defaults to: nil) —

    limit the diff to this project



111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
# File 'lib/support/project_stats_diff.rb', line 111

def initialize(start_rows, end_rows, start_label: 'start', end_label: 'end', project_id: nil)
  @start_label = start_label
  @end_label = end_label

  start_by_key = index(start_rows, project_id)
  end_by_key = index(end_rows, project_id)

  all = (start_by_key.keys | end_by_key.keys).map do |key|
    s = start_by_key[key]
    e = end_by_key[key]
    Change.new(
      project_id: key.first,
      project_name: (e || s)['project_name'],
      model: (e || s)['model'],
      table: key.last,
      start_row: s,
      end_row: e
    )
  end

  @changes = all.select(&:changed?)
  @unchanged_count = all.size - @changes.size
  @project_names = { start: names(start_rows), end: names(end_rows) }
  @start_totals = project_totals(start_by_key.values)
  @end_totals = project_totals(end_by_key.values)
end

Instance Attribute Details

#changes ⇒ Object (readonly)

Returns the value of attribute changes.



84
85
86
 
# File 'lib/support/project_stats_diff.rb', line 84

def changes
  @changes
end

#end_label ⇒ Object (readonly)

Returns the value of attribute end_label.



84
85
86
 
# File 'lib/support/project_stats_diff.rb', line 84

def end_label
  @end_label
end

#start_label ⇒ Object (readonly)

Returns the value of attribute start_label.



84
85
86
 
# File 'lib/support/project_stats_diff.rb', line 84

def start_label
  @start_label
end

#unchanged_count ⇒ Object (readonly)

Returns the value of attribute unchanged_count.



84
85
86
 
# File 'lib/support/project_stats_diff.rb', line 84

def unchanged_count
  @unchanged_count
end

Class Method Details

.from_files(start_path, end_path, project_id: nil) ⇒ ProjectStatsDiff

Parameters:

  • start_path (String)
  • end_path (String)

Returns:



89
90
91
92
93
94
 
# File 'lib/support/project_stats_diff.rb', line 89

def self.from_files(start_path, end_path, project_id: nil)
  new(
    read(start_path), read(end_path),
    start_label: start_path, end_label: end_path, project_id:
  )
end

.read(path) ⇒ Array of Hash

Returns one Hash per TSV row, keyed by HEADERS.

Returns:

  • (Array of Hash) —

    one Hash per TSV row, keyed by HEADERS

Raises:

  • (ArgumentError) 


97
98
99
100
101
102
103
104
105
106
 
# File 'lib/support/project_stats_diff.rb', line 97

def self.read(path)
  raise ArgumentError, "File not found: #{path}" unless File.exist?(path)

  table = CSV.read(path, col_sep: "\t", headers: true)
  unless table.headers == HEADERS
    raise ArgumentError, "#{path} has unexpected headers #{table.headers.inspect}, expected #{HEADERS.inspect}"
  end

  table.map(&:to_h)
end

Instance Method Details

#columns(headers, rows, right: [], indent: 2) ⇒ Object (private) 



276
277
278
279
280
281
282
 
# File 'lib/support/project_stats_diff.rb', line 276

def columns(headers, rows, right: [], indent: 2)
  widths = headers.each_index.map { |i| ([headers] + rows).map { |r| r[i].to_s.length }.max }
  ([headers] + rows).map do |r|
    cells = r.each_with_index.map { |v, i| right.include?(i) ? v.to_s.rjust(widths[i]) : v.to_s.ljust(widths[i]) }
    (' ' * indent + cells.join('  ')).rstrip
  end
end

#delimit(number) ⇒ Object (private) 



291
292
293
 
# File 'lib/support/project_stats_diff.rb', line 291

def delimit(number)
  number.to_s.reverse.scan(/\d{1,3}/).join(',').reverse.prepend(number < 0 ? '-' : '')
end

#flat_section(out, title, changes) ⇒ Object (private) 



251
252
253
254
255
256
257
258
259
 
# File 'lib/support/project_stats_diff.rb', line 251

def flat_section(out, title, changes)
  return if changes.nil? || changes.empty?
  out << title
  rows = sort_by_project(changes).map do |c|
    [project_label(c.project_id, c.project_name), c.table, c.model, delimit(c.start_count), delimit(c.end_count)]
  end
  out.concat(columns(%w[project table model start end], rows, right: [3, 4]))
  out << ''
end

#grouped_section(out, title, changes, headers, right) ⇒ Object (private) 



261
262
263
264
265
266
267
268
269
270
 
# File 'lib/support/project_stats_diff.rb', line 261

def grouped_section(out, title, changes, headers, right)
  return if changes.nil? || changes.empty?
  out << title
  sort_by_project(changes).group_by { |c| [c.project_id, c.project_name] }.each do |(id, name), cs|
    out << "  #{project_label(id, name)}"
    rows = cs.sort_by { |c| [-c.delta.abs, c.table] }.map { |c| yield(c) }
    out.concat(columns(headers, rows, right:, indent: 4))
  end
  out << ''
end

#index(rows, project_id) ⇒ Object (private) 



214
215
216
217
218
219
220
 
# File 'lib/support/project_stats_diff.rb', line 214

def index(rows, project_id)
  rows.each_with_object({}) do |r, h|
    pid = r['project_id'].to_s.empty? ? nil : r['project_id'].to_i
    next if project_id && pid != project_id
    h[[pid, r['table']]] = r
  end
end

#names(rows) ⇒ Object (private) 



222
223
224
225
226
227
 
# File 'lib/support/project_stats_diff.rb', line 222

def names(rows)
  rows.each_with_object({}) do |r, h|
    next if r['project_id'].to_s.empty?
    h[r['project_id'].to_i] = r['project_name']
  end
end

#percent(change) ⇒ Object (private) 



284
285
286
287
288
289
 
# File 'lib/support/project_stats_diff.rb', line 284

def percent(change)
  p = change.percent
  return '' if p.nil?
  sign = change.delta > 0 ? '+' : '-'
  p.zero? ? "#{sign}<0.1%" : "#{sign}#{p.abs}%"
end

#project_label(id, name) ⇒ Object (private) 



238
239
240
 
# File 'lib/support/project_stats_diff.rb', line 238

def project_label(id, name)
  id ? "[#{id}] #{name}" : '[-] not project-scoped'
end

#project_section(out, title, projects) ⇒ Object (private) 



242
243
244
245
246
247
248
249
 
# File 'lib/support/project_stats_diff.rb', line 242

def project_section(out, title, projects)
  return if projects.empty?
  out << title
  projects.sort.each do |id, t|
    out << "  #{project_label(id, t[:name])}  (#{delimit(t[:rows])} rows across #{t[:tables]} #{t[:tables] == 1 ? 'table' : 'tables'})"
  end
  out << ''
end

#project_totals(rows) ⇒ Object (private) 



229
230
231
232
233
234
235
236
 
# File 'lib/support/project_stats_diff.rb', line 229

def project_totals(rows)
  rows.each_with_object({}) do |r, h|
    next if r['project_id'].to_s.empty? || r['count'].to_i.zero?
    t = (h[r['project_id'].to_i] ||= { name: r['project_name'], rows: 0, tables: 0 })
    t[:rows] += r['count'].to_i
    t[:tables] += 1
  end
end

#projects_added ⇒ Hash

Returns project_id => { name:, rows:, tables: } for projects with rows only at the end.

Returns:

  • (Hash) —

    project_id => { name:, rows:, tables: } for projects with rows only at the end



152
153
154
 
# File 'lib/support/project_stats_diff.rb', line 152

def projects_added
  @end_totals.reject { |id, _| @start_totals.key?(id) }
end

#projects_gone ⇒ Hash

Returns project_id => { name:, rows:, tables: } for projects with rows only at the start.

Returns:

  • (Hash) —

    project_id => { name:, rows:, tables: } for projects with rows only at the start



147
148
149
 
# File 'lib/support/project_stats_diff.rb', line 147

def projects_gone
  @start_totals.reject { |id, _| @end_totals.key?(id) }
end

#projects_renamed ⇒ Hash

Returns project_id => [start name, end name].

Returns:

  • (Hash) —

    project_id => [start name, end name]



157
158
159
160
161
162
 
# File 'lib/support/project_stats_diff.rb', line 157

def projects_renamed
  @project_names[:start].each_with_object({}) do |(id, name), h|
    end_name = @project_names[:end][id]
    h[id] = [name, end_name] if end_name && end_name != name
  end
end

#report ⇒ String

Returns a human readable summary.

Returns:

  • (String) —

    a human readable summary



165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/support/project_stats_diff.rb', line 165

def report
  out = []
  by_kind = changes.group_by(&:kind)

  out << 'Project stats diff'
  out << "  start: #{start_label}"
  out << "  end:   #{end_label}"
  out << ''

  out << 'Summary'
  out << "  #{changes.size} of #{changes.size + unchanged_count} project/table pairs changed"
  out << "  rows: #{signed(rows_added)} added, #{signed(-rows_removed)} removed, net #{signed(rows_added - rows_removed)}"
  out << "  #{by_kind.fetch(:emptied, []).size} emptied, #{by_kind.fetch(:new, []).size} newly populated, " \
         "#{projects_gone.size} projects with no rows left, #{projects_added.size} projects added"
  out << ''

  if changes.empty? && projects_renamed.empty?
    out << 'No changes.'
    return out.join("\n") + "\n"
  end

  project_section(out, 'Projects with no rows left', projects_gone)
  project_section(out, 'Projects added', projects_added)

  if projects_renamed.any?
    out << 'Projects renamed'
    projects_renamed.sort.each { |id, (from, to)| out << "  [#{id}] #{from} -> #{to}" }
    out << ''
  end

  flat_section(out, 'Emptied', by_kind[:emptied])
  flat_section(out, 'Newly populated', by_kind[:new])

  grouped_section(out, 'Count changes', by_kind[:count], %w[table model start end delta % signals], [2, 3, 4, 5]) do |c|
    [c.table, c.model, delimit(c.start_count), delimit(c.end_count), signed(c.delta), percent(c), c.signals.join(', ')]
  end

  grouped_section(out, 'Same count, rows changed', by_kind[:same_count], %w[table model count signals], [2]) do |c|
    [c.table, c.model, delimit(c.end_count), c.signals.join(', ')]
  end

  out << 'Signals state only what the snapshots prove: inserted = max_id grew; ' \
         'deleted = count fell, or did not rise despite inserts; updated = max_updated_at got later with no inserts.'

  out.join("\n") + "\n"
end

#rows_added ⇒ Object 



138
139
140
 
# File 'lib/support/project_stats_diff.rb', line 138

def rows_added
  changes.sum { |c| c.delta > 0 ? c.delta : 0 }
end

#rows_removed ⇒ Object 



142
143
144
 
# File 'lib/support/project_stats_diff.rb', line 142

def rows_removed
  changes.sum { |c| c.delta < 0 ? -c.delta : 0 }
end

#signed(number) ⇒ Object (private) 



295
296
297
 
# File 'lib/support/project_stats_diff.rb', line 295

def signed(number)
  number > 0 ? "+#{delimit(number)}" : delimit(number)
end

#sort_by_project(changes) ⇒ Object (private) 



272
273
274
 
# File 'lib/support/project_stats_diff.rb', line 272

def sort_by_project(changes)
  changes.sort_by { |c| [c.project_id || -1, c.table] }
end