Hello.
We recently reach an inode use limit for /var/lib/one on our frontend and I found that there is no builtin tool to cleanup old VM directories (/var/lib/one/vms/[0-9]*) and log files (/var/lib/one/[0-9]*.log).
So, to celebrate our 6.2.0-1.ce migration, here is the little tool I wrote to cleanup them, called monthly in oneadmin crontab.
EDIT: It was not the final version with working --no-directory, --no-log-file and --no-db options.
#!/usr/bin/ruby
# frozen_string_literal: true
############################################################################
# Copyright 2022, Équipe EOLE <eole@ac-dijon.fr> #
# Author: Daniel Dehennin <daniel.dehennin@ac-dijon.fr> #
# #
# This program is free software: you can redistribute it and/or modify #
# it under the terms of the GNU Affero General Public License as #
# published by the Free Software Foundation, either version 3 of the #
# License, or (at your option) any later version. #
# #
# This program is distributed in the hope that it will be useful, #
# but WITHOUT ANY WARRANTY; without even the implied warranty of #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
# GNU Affero General Public License for more details. #
# #
# You should have received a copy of the GNU Affero General Public License #
# along with this program. If not, see <http://www.gnu.org/licenses/>. #
############################################################################
############################################################################
ONE_LOCATION = ENV['ONE_LOCATION']
if !ONE_LOCATION
RUBY_LIB_LOCATION = '/usr/lib/one/ruby'
GEMS_LOCATION = '/usr/share/one/gems'
else
RUBY_LIB_LOCATION = ONE_LOCATION + '/lib/ruby'
GEMS_LOCATION = ONE_LOCATION + '/share/gems'
end
# %%RUBYGEMS_SETUP_BEGIN%%
if File.directory?(GEMS_LOCATION)
real_gems_path = File.realpath(GEMS_LOCATION)
if !defined?(Gem) || Gem.path != [real_gems_path]
$LOAD_PATH.reject! { |l| l =~ /vendor_ruby/ }
# Suppress warnings from Rubygems
# https://github.com/OpenNebula/one/issues/5379
begin
verb = $VERBOSE
$VERBOSE = nil
require 'rubygems'
Gem.use_paths(real_gems_path)
ensure
$VERBOSE = verb
end
end
end
# %%RUBYGEMS_SETUP_END%%
$LOAD_PATH << RUBY_LIB_LOCATION
$LOAD_PATH << RUBY_LIB_LOCATION + '/onedb'
$LOAD_PATH << RUBY_LIB_LOCATION + '/cli'
require 'fileutils'
require 'optparse'
require 'optparse/time'
require 'cli/one_helper' # Required for OneDBLive.purge_done_vm
require 'opennebula'
require 'onedb_live'
VMS_DIR = '/var/lib/one/vms'
LOG_DIR = '/var/log/one'
DONE_STATE = OpenNebula::VirtualMachine::VM_STATE.index('DONE')
NOW = Time.now
# Older than 30 days ago at midnight
DEFAULT_TIME_LIMIT = NOW - 30 * 86_400 - NOW.hour * 3600 - NOW.min * 60 - NOW.sec - NOW.subsec
def main
options = parse_opts
begin
client = OpenNebula::Client.new
rescue StandardError => e
puts "OpenNebula error: #{e}"
exit(-1)
end
vm_pool = OpenNebula::VirtualMachinePool.new(client, OpenNebula::Pool::INFO_ALL)
clean_done_vm_files(vm_pool, options)
clean_unknow_vm_files(vm_pool, options)
purge_old_done_vms(options)
end
def clean_done_vm_files(vm_pool, options)
# Lookup all DONE VMs
rc = vm_pool.info_search(state: DONE_STATE)
raise "OpenNebula error: #{rc.message}" if OpenNebula.is_error?(rc)
vm_pool.each do |vm|
vm_dir = "#{VMS_DIR}/#{vm.id}"
log_file = "#{LOG_DIR}/#{vm.id}.log"
etime = Time.at(vm.retrieve_elements('ETIME').first.to_i)
if etime > options[:time]
puts "#{vm.id}/#{vm.name}: skip: #{parse_remaining_time(etime - options[:time])} remaining" if options[:debug]
next
end
clean_vm_dir(vm_dir, vm, options)
clean_vm_logfile(log_file, vm, options)
end
end
def clean_unknow_vm_files(vm_pool, options)
vms_glob = "#{VMS_DIR}/[0-9]*"
log_glob = "#{LOG_DIR}/[0-9]*.log"
Dir[vms_glob].each do |vm_dir|
next unless File.directory?(vm_dir)
vm_id = File.basename(vm_dir).to_i
vm = lookup_vm(vm_id, vm_pool)
if vm.nil?
clean_vm_dir(vm_dir, vm, options)
elsif options[:debug]
puts "#{vm.id}/#{vm.name}: keep #{vm.state_str} VM directory #{vm_dir}"
end
end
Dir[log_glob].each do |log_file|
next unless File.file?(log_file)
vm_id = File.basename(log_file, '.log').to_i
vm = lookup_vm(vm_id, vm_pool)
if vm.nil?
clean_vm_logfile(log_file, vm, options)
elsif options[:debug]
puts "#{vm.id}/#{vm.name}: keep #{vm.state_str} VM log file #{log_file}"
end
end
end
def purge_old_done_vms(options)
return unless options[:db]
if options[:dryrun]
puts "Would purge VM in DONE state older than '#{options[:time]}' from OpenNebula database"
return
end
puts "Purge VM in DONE state older than '#{options[:time]}' from OpenNebula database" if options[:verbose]
purge_options = {
end_time: options[:time],
verbose: options[:verbose]
}
action = OneDBLive.new
rc = action.purge_done_vm(purge_options)
raise "OpenNebula error: #{rc.message}" if OpenNebula.is_error?(rc)
end
def lookup_vm(vm_id, vm_pool)
rc = vm_pool.info_set(vm_id.to_s, false)
raise "OpenNebula error: #{rc.message}" if OpenNebula.is_error?(rc)
vm_pool.first
end
def clean_vm_dir(vm_dir, vm, options)
return unless options[:directory]
log_prefix = if vm.nil?
'Unknown VM: '
else
"#{vm.id}/#{vm.name}: "
end
if File.writable?(vm_dir)
puts "#{log_prefix}#{options[:action_str]} directory #{vm_dir}" if options[:verbose] || options[:dryrun]
FileUtils.rm_rf(vm_dir) unless options[:dryrun]
else
puts "#{log_prefix}directory #{vm_dir} does not exists" if options[:debug]
end
end
def clean_vm_logfile(log_file, vm, options)
return unless options[:logfile]
log_prefix = if vm.nil?
'Unknown VM: '
else
"#{vm.id}/#{vm.name}: "
end
if File.writable?(log_file)
puts "#{log_prefix}#{options[:action_str]} log file #{log_file}" if options[:verbose] || options[:dryrun]
FileUtils.rm_f(log_file) unless options[:dryrun]
else
puts "#{log_prefix}log file #{log_file} does not exists" if options[:debug]
end
end
def parse_remaining_time(seconds)
labels = %w[day hour minute second]
reduced = [60, 60, 24].reduce([seconds]) do |m, o|
m.unshift(m.shift.divmod(o)).flatten
end
remaining_string = ''
reduced.zip(labels).each do |pair|
next if pair[0].zero?
plural = 's' if pair[0] > 1
comma = ', ' unless remaining_string.empty?
remaining_string += if pair[1] == 'seconds'
" and #{pair[0]} #{pair[1]}#{plural}"
else
"#{comma}#{pair[0]} #{pair[1]}#{plural}"
end
end
remaining_string
end
def parse_opts
options = {
time: DEFAULT_TIME_LIMIT,
directory: true,
logfile: true,
db: true,
dryrun: true,
verbose: false,
debug: false
}
OptionParser.new do |parser|
parser.banner = "Usage: #{File.basename(__FILE__)}"
parser.on(
'-t',
'--time TIME',
Time,
"Cleanup VMs older than this date, default to '#{DEFAULT_TIME_LIMIT}'"
) do |value|
options[:time] = value
end
parser.on(
'--no-directory',
"Do not cleanup the VMs directories under '#{VMS_DIR}'"
) do |value|
options[:directory] = value
end
parser.on(
'--no-log-file',
"Do not cleanup the VMs log file under '#{LOG_DIR}'"
) do |value|
options[:logfile] = value
end
parser.on(
'--no-db',
'Do not cleanup the VMs in DONE state from OpenNebula database'
) do |value|
options[:db] = value
end
parser.on(
'-v',
'--verbose',
'Show verbose informations'
) do |value|
options[:verbose] = value
end
parser.on(
'-d',
'--debug',
'Show debug informations'
) do |value|
options[:debug] = value
end
parser.on(
'--no-dry-run',
'Do not run in dry-run mode'
) do |value|
options[:dryrun] = value
end
end.parse!
# Debug enable verbose
options[:verbose] = options[:debug] || options[:verbose]
options[:action_str] = if options[:dryrun]
'Would remove'
else
'Remove'
end
options
end
main