From 3342da13c6746ebab265e3e22e94716614f0e427 Mon Sep 17 00:00:00 2001 From: "connorc@orbitnode.net" Date: Sun, 19 Mar 2023 16:49:18 -0600 Subject: [PATCH] feat: Enhance MySQL backup script with customizable retention and logging Refactor backup method to dump each database individually Update delete_old_backups method to remove local backups based on user-defined retention days Add Loggman class for logging program actions to a logfile and implement log deletion for logs older than a week Modify MysqlDatabaseConfig class to ask the user for local and B2 backup retention days Update MysqlDatabaseBackup class to use user-defined retention days for local backups Refactor upload_to_b2 method to delete B2 backups based on user-defined retention days Implement various code improvements and refactoring for better readability and maintainability --- loggman.rb | 44 ++++++++++ mysql_database_backup.rb | 171 ++++++++++++++++++++++----------------- mysql_database_config.rb | 128 +++++++++++++++-------------- starter.rb | 24 +++--- 4 files changed, 218 insertions(+), 149 deletions(-) create mode 100644 loggman.rb diff --git a/loggman.rb b/loggman.rb new file mode 100644 index 0000000..3fdae8e --- /dev/null +++ b/loggman.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'time' +require 'fileutils' + +# Loggman class +class Loggman + def initialize(logfile) + @logfile = logfile + end + + def log(message, level = :info) + delete_old_logs + timestamp = Time.now.strftime('%Y-%m-%d %H:%M:%S') + File.open(@logfile, 'a') do |file| + file.puts "[#{timestamp}] [#{level.to_s.upcase}] #{message}" + end + end + + def info(message) + log(message, :info) + end + + def warn(message) + log(message, :warn) + end + + def error(message) + log(message, :error) + end + + def debug(message) + log(message, :debug) + end + + def delete_old_logs + max_age_days = 7 + max_age_seconds = max_age_days * 24 * 60 * 60 + + if File.exist?(@logfile) && Time.now - File.mtime(@logfile) > max_age_seconds + FileUtils.rm(@logfile) + end + end +end diff --git a/mysql_database_backup.rb b/mysql_database_backup.rb index 2b49c00..f630c02 100644 --- a/mysql_database_backup.rb +++ b/mysql_database_backup.rb @@ -1,75 +1,96 @@ -# frozen_string_literal: true - -require 'json' - -# class for creating, managing and deleting backups both locally and in B2 -class MysqlDatabaseBackup - def initialize(config_file) - config = JSON.parse(File.read(config_file)) - @host = config['mysql']['host'] - @username = config['mysql']['username'] - @password = config['mysql']['password'] - @backup_dir = config['backup_dir'] || '.' - @b2_enabled = config['b2_enabled'] || false - @b2_key_id = config['b2']&.dig('key_id') - @b2_application_key = config['b2']&.dig('application_key') - @b2_bucket_name = config['b2']&.dig('bucket_name') - end - - def backup - puts 'Backing up sql' - timestamp = Time.now.strftime('%Y-%m-%d_%H-%M-%S') - puts "Timestamp = #{timestamp}" - backup_file = File.join(@backup_dir, "database-backup_#{timestamp}.sql") - puts "backup_file = #{backup_file}" - puts "MySQL Info = #{@host} #{@username} #{@password} #{backup_file}" - - `mysqldump --host=#{@host} --user=#{@username} --password='#{@password}' --all-databases > #{backup_file}` - - delete_old_backups - - return unless @b2_enabled - - upload_to_b2(backup_file) - end - - def delete_old_backups - max_age_hours = 48 - max_age_seconds = max_age_hours * 60 * 60 - backups = Dir[File.join(@backup_dir, 'database-backup_*.sql')] - - return if backups.empty? - - backups.each do |backup| - age_seconds = Time.now - File.mtime(backup) - - if age_seconds > max_age_seconds - puts "Deleted old backup: #{backup}" - File.delete(backup) - end - end - end - - def upload_to_b2(backup_file) - b2_file_name = File.basename(backup_file) - b2_file_url = "b2://#{@b2_bucket_name}/#{b2_file_name}" - # Check if a backup file with the same name already exists in the B2 bucket - - existing_files = `b2 ls #{@b2_bucket_name}` - unless existing_files.empty? - existing_files.each_line do |line| - file_name = line.split(' ').last.strip - next unless file_name != b2_file_name - - file_id = line.match(/"fileId": "([^"]+)"/)[1] - `b2 delete-file-version #{@b2_bucket_name} #{file_name} #{file_id}` - puts "Deleted existing backup file from B2 bucket: #{file_name}" - end - end - - # Upload the backup file to the B2 bucket - `b2 upload-file #{@b2_bucket_name} #{backup_file} #{b2_file_name}` - puts "Uploaded backup file to B2 bucket: #{b2_file_url}" - end - -end +# frozen_string_literal: true + +require 'json' + +# class for creating, managing and deleting backups both locally and in B2 +class MysqlDatabaseBackup + def initialize(config_file) + config = JSON.parse(File.read(config_file)) + @host = config['mysql']['host'] + @username = config['mysql']['username'] + @password = config['mysql']['password'] + @backup_dir = config['backup_dir'] || '.' + @b2_enabled = config['b2_enabled'] || false + @b2_key_id = config['b2']&.dig('key_id') + @b2_application_key = config['b2']&.dig('application_key') + @b2_bucket_name = config['b2']&.dig('bucket_name') + @local_retention_days = config['local_retention_days'] || 30 + @b2_retention_days = config['b2']&.dig('retention_days') || 30 + end + + def backup + puts 'Backing up sql' + timestamp = Time.now.strftime('%Y-%m-%d_%H-%M-%S') + puts "Timestamp = #{timestamp}" + + databases = get_databases + + databases.each do |database_name| + backup_file = File.join(@backup_dir, "#{database_name}_#{timestamp}.sql") + puts "backup_file = #{backup_file}" + puts "MySQL Info = #{@host} #{@username} #{@password} #{backup_file}" + + `mysqldump --host=#{@host} --user=#{@username} --password='#{@password}' --databases #{database_name} > #{backup_file}` + + delete_old_backups + + upload_to_b2(backup_file) if @b2_enabled + end + end + + def get_databases + databases_output = `mysql --host=#{@host} --user=#{@username} --password='#{@password}' --execute='SHOW DATABASES;'` + databases = databases_output.split("\n")[1..] # Ignore the first line (header) + databases.reject { |db| %w[information_schema performance_schema mysql sys].include?(db) } + end + + def delete_old_backups + max_age_days = @local_retention_days + max_age_seconds = max_age_days * 24 * 60 * 60 + backups = Dir[File.join(@backup_dir, '*_*.sql')] + + return if backups.empty? + + backups.each do |backup| + age_seconds = Time.now - File.mtime(backup) + + if age_seconds > max_age_seconds + puts "Deleted old backup: #{backup}" + File.delete(backup) + end + end + end + + def upload_to_b2(backup_file) + b2_file_name = File.basename(backup_file) + b2_file_url = "b2://#{@b2_bucket_name}/#{b2_file_name}" + + # Upload the backup file to the B2 bucket + `b2 upload-file #{@b2_bucket_name} #{backup_file} #{b2_file_name}` + puts "Uploaded backup file to B2 bucket: #{b2_file_url}" + + # Calculate the cutoff date based on b2_retention_days + max_age_days = @b2_retention_days + cutoff_date = Time.now - (max_age_days * 24 * 60 * 60) + + existing_files = `b2 ls #{@b2_bucket_name}` + + return if existing_files.empty? + + existing_files.each_line do |line| + file_name = line.split(' ').last.strip + file_timestamp_str = file_name.match(/_(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})\.sql/)&.captures&.first + + # Skip files that don't match the expected filename format + next if file_timestamp_str.nil? + + file_timestamp = Time.strptime(file_timestamp_str, '%Y-%m-%d_%H-%M-%S') + + next unless file_timestamp < cutoff_date + + file_id = line.match(/"fileId": "([^"]+)"/)[1] + `b2 delete-file-version #{@b2_bucket_name} #{file_name} #{file_id}` + puts "Deleted old backup file from B2 bucket: #{file_name}" + end + end +end diff --git a/mysql_database_config.rb b/mysql_database_config.rb index fabc37a..c1176cf 100644 --- a/mysql_database_config.rb +++ b/mysql_database_config.rb @@ -1,62 +1,66 @@ -# frozen_string_literal: true - -# class for generating the mysql config if it doesn't exist -require 'json' - -# class for generating our config -class MysqlDatabaseConfig - def initialize(config_file) - @config_file = config_file - end - - def generate - if File.exist?(@config_file) - puts 'Config file already exists, skipping generation.' - else - mysql_host = prompt('MySQL Host') - mysql_username = prompt('MySQL Username') - mysql_password = prompt('MySQL Password') - - backup_dir = prompt('Backup Directory', default: '.') - - @config = { - 'mysql' => { - 'host' => mysql_host, - 'username' => mysql_username, - 'password' => mysql_password - }, - 'backup_dir' => backup_dir - } - - b2_enabled = prompt_bool('Enable Backblaze B2?', default: false) - @config['b2_enabled'] = b2_enabled - if b2_enabled - @b2_key_id = prompt('B2 Key ID') - @b2_application_key = prompt('B2 Application Key') - @b2_bucket_name = prompt('B2 Bucket Name') - @config['b2'] = { - 'key_id' => @b2_key_id, - 'application_key' => @b2_application_key, - 'bucket_name' => @b2_bucket_name - } - end - - File.write(@config_file, JSON.pretty_generate(@config)) - puts "Config file generated: #{@config_file}" - end - end - - private - - def prompt(message, default: nil) - print message.to_s - print " [#{default}]" if default - print ': ' - value = gets.chomp - value.empty? ? default : value - end - - def prompt_bool(message, default: false) - prompt("#{message} (y/n)", default: default) =~ /y|yes/i - end -end +# frozen_string_literal: true + +# class for generating the mysql config if it doesn't exist +require 'json' + +# class for generating our config +class MysqlDatabaseConfig + def initialize(config_file) + @config_file = config_file + end + + def generate + if File.exist?(@config_file) + puts 'Config file already exists, skipping generation.' + else + mysql_host = prompt('MySQL Host') + mysql_username = prompt('MySQL Username') + mysql_password = prompt('MySQL Password') + + backup_dir = prompt('Backup Directory', default: '.') + + local_retention_days = prompt('Local backup retention in days (1 day minimum)', default: '1').to_i + + @config = { + 'mysql' => { + 'host' => mysql_host, + 'username' => mysql_username, + 'password' => mysql_password + }, + 'backup_dir' => backup_dir, + 'local_retention_days' => local_retention_days + } + b2_enabled = prompt_bool('Enable Backblaze B2?', default: false) + @config['b2_enabled'] = b2_enabled + if b2_enabled + @b2_key_id = prompt('B2 Key ID') + @b2_application_key = prompt('B2 Application Key') + @b2_bucket_name = prompt('B2 Bucket Name') + b2_retention_days = prompt('B2 backup retention in days (1 day minimum)', default: '1').to_i + @config['b2'] = { + 'key_id' => @b2_key_id, + 'application_key' => @b2_application_key, + 'bucket_name' => @b2_bucket_name, + 'retention_days' => b2_retention_days + } + end + + File.write(@config_file, JSON.pretty_generate(@config)) + puts "Config file generated: #{@config_file}" + end + end + + private + + def prompt(message, default: nil) + print message.to_s + print " [#{default}]" if default + print ': ' + value = gets.chomp + value.empty? ? default : value + end + + def prompt_bool(message, default: false) + prompt("#{message} (y/n)", default: default) =~ /y|yes/i + end +end diff --git a/starter.rb b/starter.rb index ce3d691..cf75eb3 100644 --- a/starter.rb +++ b/starter.rb @@ -1,12 +1,12 @@ -# frozen_string_literal: true - -require_relative 'mysql_database_config' -require_relative 'mysql_database_backup' - -config_file = 'config.json' - -config_generator = MysqlDatabaseConfig.new(config_file) -config_generator.generate - -backup = MysqlDatabaseBackup.new(config_file) -backup.backup +# frozen_string_literal: true + +require_relative 'mysql_database_config' +require_relative 'mysql_database_backup' + +config_file = 'config.json' + +config_generator = MysqlDatabaseConfig.new(config_file) +config_generator.generate + +backup = MysqlDatabaseBackup.new(config_file) +backup.backup