require 'rubyqa' include RubyQA require 'time' require 'English' info_template = ERB.new <" ============================================== <% @tests.each do |hostname,data| -%> <%= hostname %> ---------------------------------------------- Serial: <%= data[:context][:serial] %> System Date: <%= data[:context][:date] %> HWCLOCK: <%= data[:context][:hwclock] %> Timezone: <%= data[:context][:timezone] %> MM: <%= data[:context][:maintenance_mode] %> Standby: <%= data[:context][:standby] %> Gold-Images: <% data[:context][:gold_images].each do |image| -%> - <%= image %> <% end -%> Full-Backups: <% data[:context][:full_backups].each do |image| -%> - <%= image %> <% end -%> DRBD Syncs: <% data[:context][:drbd_syncs].each do |sync_name, sync_data| -%> - <%= sync_name %> (Sync Status: <%= sync_data['disk_state'] %>) <% end -%> <% end -%> EOF Manager.new_test("Get Information", template: info_template){|host,context| context[:serial] = host.resources['facts']['values']['dmi']['product']['serial_number'].downcase context[:date] = Time.parse(host.exec('date').strip) context[:hwclock] = Time.parse(host.exec('sudo hwclock').strip) context[:timezone] = host.resources['facts']['values']['timezone'] context[:gold_images] = host.exec('sudo ls -1 /mnt/gold-images/ | grep tar.gz').split(/\n/).map(&:strip).reject(&:empty?) context[:full_backups] = host.exec('sudo ls -1 /mnt/kvm_backups/ | grep tar.gz').split(/\n/).map(&:strip).reject(&:empty?) context[:drbd_syncs] = host.resources['facts']['values']['drbd_status'].sort_by{|name,data| name} context[:maintenance_mode] = host.exec("sudo pcs property show maintenance-mode | awk '/maint/ { print $2 }'").strip context[:standby] = host.resources['facts']['values']['corosync']['is_standby'] ? "true" : "false" } network_template = ERB.new <-' ============================================== Test : "<%= @name %>" ============================================== <% @tests.each do |hostname, data| -%> <%= hostname %>: ---------------------------------------------- <% data[:context][:interfaces].each do |name,info| -%> <%= name %> <% if info.has_key? :mode %>(<%= info[:mode] %>)<% end %>: <% if info[:mode] == "static" -%> <% if info.has_key? :address -%> IP: <%= info[:address] %> <% end -%> <% if info.has_key? :gateway -%> Gateway: <%= info[:gateway] %> <% end -%> <% if info.has_key? :netmask -%> Netmask: <%= info[:netmask] %> <% end -%> <% if info.has_key? :dns -%> DNS: <%= info[:dns] %> <% end -%> <% end -%> <% end -%> <% end -%> EOF Manager.new_test("Networking Configurations Host", template: network_template) {|host,context| contents = host.exec('cat /etc/network/interfaces') interfaces = Hash.new interface_regex = /iface (?\S+) inet (?\S+)/ address_regex = /address (?
\S+)/ netmask_regex = /netmask (?\S+)/ gateway_regex = /gateway (?\S+)/ current_interface = "" contents.split(/\n/).each do |line| case line when interface_regex current_interface = $LAST_MATCH_INFO['name'] mode = $LAST_MATCH_INFO['mode'] interfaces[current_interface]=Hash.new interfaces[current_interface][:mode] = mode when address_regex interfaces[current_interface][:address] = $LAST_MATCH_INFO['address'] when netmask_regex interfaces[current_interface][:netmask] = $LAST_MATCH_INFO['netmask'] when gateway_regex interfaces[current_interface][:gateway] = $LAST_MATCH_INFO['gateway'] end end contents = host.exec('sudo ipmitool lan print') current_interface="IPMI" interfaces[current_interface]=Hash.new mode_regex = /IP Address Source[[:space:]]+:[[:space:]](?\S+)/ address_regex = /IP Address[[:space:]]+:[[:space:]]+(?
\S+)/ netmask_regex = /Subnet Mask[[:space:]]+:[[:space:]]+(?\S+)/ gateway_regex = /Default Gateway IP[[:space:]]+:[[:space:]]+(?\S+)/ contents.split(/\n/).each do |line| case line when mode_regex interfaces[current_interface][:mode] = $LAST_MATCH_INFO['mode'].downcase when address_regex interfaces[current_interface][:address] = $LAST_MATCH_INFO['address'] when netmask_regex interfaces[current_interface][:netmask] = $LAST_MATCH_INFO['netmask'] when gateway_regex interfaces[current_interface][:gateway] = $LAST_MATCH_INFO['gateway'] end end ipconfig_script=<[A-Z]{3}[0-9]+).xml/ CACHE_MODE_REGEX = /cache=\'(?\S+)\'/ Manager.new_test("Check VM disk cache is writeback"){|host, context| pass = true failed_vms = Array.new host.exec("sudo grep cache /etc/libvirt/qemu/*.xml").split(/\n/).each do |line| name = VM_NAME_REGEX.match(line)['name'] mode = CACHE_MODE_REGEX.match(line)['mode'] if mode != "writeback" pass = false failed_vms << name end end if ! failed_vms.empty? context[:note] = "failed vms: #{failed_vms.join(",")}" end pass } Manager.new_test("Manual RVC Patch Applied to domain.rb"){|host, context| pass = true pass = host.exec("sudo grep Log.error /opt/rvc/lib/rvc/domain.rb") =~ /Log.error[[:space:]]*\{[[:space:]]*e(\.message)?[[:space:]]*\}/ if not pass context[:note] = "Manual Patch not applied (sudo sed -i 's/Log.error(e)/Log.error { e }/g' /opt/rvc/lib/rvc/domain.rb)" end pass } Manager.new_test("Test IPMI Credentials on self (over network)") {|host,context| fencenode=host.resources['facts']['values']['pcs_resources']["fence_#{host.name}"]['instance_attributes'] pass = true test_result = host.exec("set +o history; ipmitool -I lanplus -H #{fencenode["ipaddr"]} -U #{fencenode['userid']} -P #{fencenode['passwd']} chassis status >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip case test_result when 'yes' pass = true when 'no' pass = false end if pass == false ping_test = host.exec("ping -c 2 #{fencenode["ipaddr"]} >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip case ping_test when 'yes' context[:note] = "IPMI Pingable, Creds Failed" when 'no' context[:note] = "IPMI Not Pingable. Possibly unplugged" end end pass } Manager.new_test("Test IPMI Credentials on partner") {|host,context| if host.name =~ /red1/ hostname = host.name.sub('red1','red0') elsif host.name =~ /red0/ hostname = host.name.sub('red0','red1') end pass = true begin fencenode=host.resources['facts']['values']['pcs_resources']["fence_#{hostname}"]['instance_attributes'] rescue puts "Fact retrieval failed on #{host.name}. Failing test" return false end test_result = host.exec("set +o history; ipmitool -I lanplus -H #{fencenode["ipaddr"]} -U #{fencenode['userid']} -P #{fencenode['passwd']} chassis status >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip case test_result when 'yes' pass = true when 'no' pass = false end if pass == false ping_test = host.exec("ping -c 2 #{fencenode["ipaddr"]} >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip case ping_test when 'yes' context[:note] = "Other Node IPMI is pingable, Creds Failed" when 'no' context[:note] = "Other Node IPMI is not Pingable. Possibly unplugged" end end pass } Manager.new_test("No Cluster Irregularities", description: "Nothing Disabled that shouldn't be"){|node,context| pass = true disabled_items = Array.new ## Stonith ## .values.pcs_resources.fence_avo00000223red0.meta_attributes["target-role"] begin node_stonith_state = node.resources['facts']['values']['pcs_resources']["fence_#{node.name}"]['meta_attributes']['target-role'] rescue puts "Fact retrieval failed on #{host.name}. Failing test" return false end if node_stonith_state == "Stopped" disabled_items << "Stonith" end if disabled_items.count !=0 context[:note] = "Disabled Items: #{disabled_items.join(',')}" pass = false end pass } Manager.new_test("Puppet Agent disabled (pre-ship only)"){|node,context| pass = true ssl_dir_exists = node.exec('[[ -f $( sudo puppet config print agent_disabled_lockfile ) ]] && echo "yes" || echo "no"') case ssl_dir_exists when /no/ pass = false context[:note] = "Before shipping node you need to disable the puppet agent on #{node.name} (sudo puppet agent --disable)" end pass } Manager.new_test("SSL Directory Cleaned (pre-ship only)"){|node,context| pass = true ssl_dir_exists = node.exec('[[ -d /etc/puppetlabs/puppet/ssl ]] && echo "yes" || echo "no"') case ssl_dir_exists when /yes/ pass = false context[:note] = "Before shipping node you need to delete the ssl dir on #{node.name} (sudo rm -rf /etc/puppetlabs/puppet/ssl)" end pass } Manager.new_test("Ensure VMs not running (pre-ship only)"){|host,context| pass = true vms = host.exec("sudo virsh list --name").split(/\n/).map(&:strip).sort! if vms.count > 0 pass = false context[:note] = "VMs are still running: #{vms.join(',')} shut them down before shipping (for VM in #{vms.join(' ')}; do sudo rvc kvm shutdown $VM; done" end pass } Manager.new_test("Cluster in Pre-Ship Condition (pre-ship only)"){|host,context| pass = true maintenance_mode = host.exec("sudo pcs property show maintenance-mode | awk '/maint/ { print $2 }'").strip standby = host.resources['facts']['values']['corosync']['is_standby'] ? "true" : "false" case host.name when /red1/ if maintenance_mode =~ /false/ and standby =~ /false/ context[:note] = "Node is not in standby and cluster is not in maintenance-mode" pass = false elsif maintenance_mode =~ /false/ and standby =~ /true/ context[:note] = "Cluster isn't in maintenance-mode" pass = false elsif maintenance_mode =~ /true/ and standby =~ /false/ context[:note] = "Node isn't in standby" pass = false end when /red0/ if maintenance_mode =~ /false/ context[:note] = "Cluster isn't in maintenance-mode" pass = false end end pass }