Refactored code to use a new Loggman class for

logging and to handle all file naming, storing logs in the backup
directory by default. Made the Loggman class delete logs older than
2 months and create new log files each week. Refactored the
MysqlDatabaseConfig and MysqlDatabaseBackup classes to use the new
Loggman class for logging and added error handling code. Updated the
upload_to_b2 method in MysqlDatabaseBackup to properly handle errors
when listing and deleting old backups in the B2 bucket.
This commit is contained in:
connorc@orbitnode.net 2023-03-19 17:10:48 -06:00
parent 3342da13c6
commit 6b2941e3a1
4 changed files with 121 additions and 59 deletions

View File

@ -5,8 +5,13 @@ require 'fileutils'
# Loggman class # Loggman class
class Loggman class Loggman
def initialize(logfile) LOG_PREFIX = 'log'
@logfile = logfile LOG_DURATION = 7 * 24 * 60 * 60 # 1 week
MAX_LOG_AGE = 60 * 24 * 60 * 60 # 2 months
def initialize(log_dir = nil)
@log_dir = log_dir || default_log_dir
@logfile = generate_logfile
end end
def log(message, level = :info) def log(message, level = :info)
@ -33,12 +38,29 @@ class Loggman
log(message, :debug) log(message, :debug)
end end
def delete_old_logs private
max_age_days = 7
max_age_seconds = max_age_days * 24 * 60 * 60
if File.exist?(@logfile) && Time.now - File.mtime(@logfile) > max_age_seconds def default_log_dir
FileUtils.rm(@logfile) backup_dir = MysqlDatabaseConfig.new(nil).backup_dir
File.join(backup_dir, 'logs')
end
def generate_logfile
start_time = Time.now
log_start_day = start_time.strftime('%Y-%m-%d')
end_time = start_time + LOG_DURATION
log_end_day = end_time.strftime('%Y-%m-%d')
log_filename = "#{LOG_PREFIX}-#{log_start_day}-#{log_end_day}.log"
log_path = File.join(@log_dir, log_filename)
FileUtils.mkdir_p(@log_dir)
log_path
end
def delete_old_logs
Dir.glob(File.join(@log_dir, "#{LOG_PREFIX}-*.log")).each do |logfile|
if Time.now - File.mtime(logfile) > MAX_LOG_AGE
FileUtils.rm(logfile)
end
end end
end end
end end

View File

@ -1,10 +1,11 @@
# frozen_string_literal: true # frozen_string_literal: true
require 'json' require 'json'
require_relative 'loggman'
# class for creating, managing and deleting backups both locally and in B2 # class for creating, managing and deleting backups both locally and in B2
class MysqlDatabaseBackup class MysqlDatabaseBackup
def initialize(config_file) def initialize(config_file, logger) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
config = JSON.parse(File.read(config_file)) config = JSON.parse(File.read(config_file))
@host = config['mysql']['host'] @host = config['mysql']['host']
@username = config['mysql']['username'] @username = config['mysql']['username']
@ -16,19 +17,21 @@ class MysqlDatabaseBackup
@b2_bucket_name = config['b2']&.dig('bucket_name') @b2_bucket_name = config['b2']&.dig('bucket_name')
@local_retention_days = config['local_retention_days'] || 30 @local_retention_days = config['local_retention_days'] || 30
@b2_retention_days = config['b2']&.dig('retention_days') || 30 @b2_retention_days = config['b2']&.dig('retention_days') || 30
@logger = logger
end end
def backup def backup # rubocop:disable Metrics/MethodLength
puts 'Backing up sql' @logger.info('Backing up MySQL database.')
timestamp = Time.now.strftime('%Y-%m-%d_%H-%M-%S')
puts "Timestamp = #{timestamp}"
databases = get_databases timestamp = Time.now.strftime('%Y-%m-%d_%H-%M-%S')
@logger.info("Timestamp for backup: #{timestamp}")
databases = find_databases
databases.each do |database_name| databases.each do |database_name|
backup_file = File.join(@backup_dir, "#{database_name}_#{timestamp}.sql") backup_file = File.join(@backup_dir, "#{database_name}_#{timestamp}.sql")
puts "backup_file = #{backup_file}" @logger.info("Backup file path: #{backup_file}")
puts "MySQL Info = #{@host} #{@username} #{@password} #{backup_file}" @logger.info("MySQL Info: #{@host} #{@username} #{@password} #{backup_file}")
`mysqldump --host=#{@host} --user=#{@username} --password='#{@password}' --databases #{database_name} > #{backup_file}` `mysqldump --host=#{@host} --user=#{@username} --password='#{@password}' --databases #{database_name} > #{backup_file}`
@ -38,13 +41,15 @@ class MysqlDatabaseBackup
end end
end end
def get_databases def find_databases
databases_output = `mysql --host=#{@host} --user=#{@username} --password='#{@password}' --execute='SHOW DATABASES;'` databases_output = `mysql --host=#{@host} --user=#{@username} --password='#{@password}' --execute='SHOW DATABASES;'`
databases = databases_output.split("\n")[1..] # Ignore the first line (header) databases = databases_output.split("\n")[1..] # Ignore the first line (header)
databases.reject { |db| %w[information_schema performance_schema mysql sys].include?(db) } databases.reject { |db| %w[information_schema performance_schema mysql sys].include?(db) }
end end
def delete_old_backups def delete_old_backups # rubocop:disable Metrics/MethodLength
@logger.info('Deleting old backups.')
max_age_days = @local_retention_days max_age_days = @local_retention_days
max_age_seconds = max_age_days * 24 * 60 * 60 max_age_seconds = max_age_days * 24 * 60 * 60
backups = Dir[File.join(@backup_dir, '*_*.sql')] backups = Dir[File.join(@backup_dir, '*_*.sql')]
@ -55,19 +60,19 @@ class MysqlDatabaseBackup
age_seconds = Time.now - File.mtime(backup) age_seconds = Time.now - File.mtime(backup)
if age_seconds > max_age_seconds if age_seconds > max_age_seconds
puts "Deleted old backup: #{backup}" @logger.info("Deleted old backup: #{backup}")
File.delete(backup) File.delete(backup)
end end
end end
end end
def upload_to_b2(backup_file) def upload_to_b2(backup_file) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
b2_file_name = File.basename(backup_file) b2_file_name = File.basename(backup_file)
b2_file_url = "b2://#{@b2_bucket_name}/#{b2_file_name}" b2_file_url = "b2://#{@b2_bucket_name}/#{b2_file_name}"
# Upload the backup file to the B2 bucket # Upload the backup file to the B2 bucket
`b2 upload-file #{@b2_bucket_name} #{backup_file} #{b2_file_name}` `b2 upload-file #{@b2_bucket_name} #{backup_file} #{b2_file_name}`
puts "Uploaded backup file to B2 bucket: #{b2_file_url}" @logger.info("Uploaded backup file to B2 bucket: #{b2_file_url}")
# Calculate the cutoff date based on b2_retention_days # Calculate the cutoff date based on b2_retention_days
max_age_days = @b2_retention_days max_age_days = @b2_retention_days
@ -90,7 +95,7 @@ class MysqlDatabaseBackup
file_id = line.match(/"fileId": "([^"]+)"/)[1] file_id = line.match(/"fileId": "([^"]+)"/)[1]
`b2 delete-file-version #{@b2_bucket_name} #{file_name} #{file_id}` `b2 delete-file-version #{@b2_bucket_name} #{file_name} #{file_id}`
puts "Deleted old backup file from B2 bucket: #{file_name}" @logger.info("Deleted old backup file from B2 bucket: #{file_name}")
end end
end end
end end

View File

@ -1,52 +1,60 @@
# frozen_string_literal: true # frozen_string_literal: true
# class for generating the mysql config if it doesn't exist
require 'json' require 'json'
require_relative 'loggman'
# class for generating our config # class for handling the config
class MysqlDatabaseConfig class MysqlDatabaseConfig
def initialize(config_file) def initialize(config_file, logger)
@config_file = config_file @config_file = config_file
@logger = logger
end end
def generate def generate # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
@logger.info("Generating MySQL database configuration file: #{@config_file}.")
if File.exist?(@config_file) if File.exist?(@config_file)
puts 'Config file already exists, skipping generation.' @logger.info('Config file already exists, skipping generation.')
else else
mysql_host = prompt('MySQL Host') begin
mysql_username = prompt('MySQL Username') mysql_host = prompt('MySQL Host')
mysql_password = prompt('MySQL Password') mysql_username = prompt('MySQL Username')
mysql_password = prompt('MySQL Password')
backup_dir = prompt('Backup Directory', default: '.') backup_dir = prompt('Backup Directory', default: '.')
local_retention_days = prompt('Local backup retention in days (1 day minimum)', default: '1').to_i local_retention_days = prompt('Local backup retention in days (1 day minimum)', default: '1').to_i
@config = { @config = {
'mysql' => { 'mysql' => {
'host' => mysql_host, 'host' => mysql_host,
'username' => mysql_username, 'username' => mysql_username,
'password' => mysql_password 'password' => mysql_password
}, },
'backup_dir' => backup_dir, 'backup_dir' => backup_dir,
'local_retention_days' => local_retention_days 'local_retention_days' => local_retention_days
}
b2_enabled = prompt_bool('Enable Backblaze B2?', default: false)
@config['b2_enabled'] = b2_enabled
if b2_enabled
@b2_key_id = prompt('B2 Key ID')
@b2_application_key = prompt('B2 Application Key')
@b2_bucket_name = prompt('B2 Bucket Name')
b2_retention_days = prompt('B2 backup retention in days (1 day minimum)', default: '1').to_i
@config['b2'] = {
'key_id' => @b2_key_id,
'application_key' => @b2_application_key,
'bucket_name' => @b2_bucket_name,
'retention_days' => b2_retention_days
} }
end b2_enabled = prompt_bool('Enable Backblaze B2?', default: false)
@config['b2_enabled'] = b2_enabled
if b2_enabled
@b2_key_id = prompt('B2 Key ID')
@b2_application_key = prompt('B2 Application Key')
@b2_bucket_name = prompt('B2 Bucket Name')
b2_retention_days = prompt('B2 backup retention in days (1 day minimum)', default: '1').to_i
@config['b2'] = {
'key_id' => @b2_key_id,
'application_key' => @b2_application_key,
'bucket_name' => @b2_bucket_name,
'retention_days' => b2_retention_days
}
end
File.write(@config_file, JSON.pretty_generate(@config)) File.write(@config_file, JSON.pretty_generate(@config))
puts "Config file generated: #{@config_file}" @logger.info("Config file generated: #{@config_file}")
rescue StandardError => e
@logger.error("An error occurred while generating MySQL database configuration file: #{e.message}")
@logger.debug("Backtrace: #{e.backtrace}")
end
end end
end end
@ -64,3 +72,17 @@ class MysqlDatabaseConfig
prompt("#{message} (y/n)", default: default) =~ /y|yes/i prompt("#{message} (y/n)", default: default) =~ /y|yes/i
end end
end end
config_file = 'config.json'
logger = Loggman.new
begin
logger.info('Starting script.')
config_generator = MysqlDatabaseConfig.new(config_file, logger)
config_generator.generate
logger.info('MySQL database configuration file generation completed successfully.')
logger.info('Script completed successfully.')
rescue StandardError => e
logger.error("An error occurred: #{e.message}")
logger.debug("Backtrace: #{e.backtrace}")
end

View File

@ -2,11 +2,24 @@
require_relative 'mysql_database_config' require_relative 'mysql_database_config'
require_relative 'mysql_database_backup' require_relative 'mysql_database_backup'
require_relative 'loggman'
config_file = 'config.json' config_file = 'config.json'
logger = Loggman.new
config_generator = MysqlDatabaseConfig.new(config_file) begin
config_generator.generate logger.info('Starting script.')
backup = MysqlDatabaseBackup.new(config_file) config_generator = MysqlDatabaseConfig.new(config_file)
backup.backup config_generator.generate
logger.info("Generated MySQL database configuration file: #{config_file}.")
backup = MysqlDatabaseBackup.new(config_file)
backup.backup
logger.info('Performed MySQL database backup.')
logger.info('Script completed successfully.')
rescue StandardError => e
logger.error("An error occurred: #{e.message}")
logger.debug("Backtrace: #{e.backtrace}")
end