467 lines
15 KiB
Ruby
Executable File
467 lines
15 KiB
Ruby
Executable File
require 'rubyqa'
|
|
|
|
include RubyQA
|
|
|
|
require 'time'
|
|
require 'English'
|
|
|
|
info_template = ERB.new <<EOF, trim_mode: '-'
|
|
==============================================
|
|
Test : "<%= @name %>"
|
|
==============================================
|
|
|
|
<% @tests.each do |hostname,data| -%>
|
|
<%= hostname %>
|
|
----------------------------------------------
|
|
Serial: <%= data[:context][:serial] %>
|
|
System Date: <%= data[:context][:date] %>
|
|
HWCLOCK: <%= data[:context][:hwclock] %>
|
|
Timezone: <%= data[:context][:timezone] %>
|
|
MM: <%= data[:context][:maintenance_mode] %>
|
|
Standby: <%= data[:context][:standby] %>
|
|
Gold-Images:
|
|
<% data[:context][:gold_images].each do |image| -%>
|
|
- <%= image %>
|
|
<% end -%>
|
|
Full-Backups:
|
|
<% data[:context][:full_backups].each do |image| -%>
|
|
- <%= image %>
|
|
<% end -%>
|
|
DRBD Syncs:
|
|
<% data[:context][:drbd_syncs].each do |sync_name, sync_data| -%>
|
|
- <%= sync_name %> (Sync Status: <%= sync_data['disk_state'] %>)
|
|
<% end -%>
|
|
|
|
<% end -%>
|
|
EOF
|
|
|
|
Manager.new_test("Get Information", template: info_template){|host,context|
|
|
context[:serial] = host.resources['facts']['values']['dmi']['product']['serial_number'].downcase
|
|
context[:date] = Time.parse(host.exec('date').strip)
|
|
context[:hwclock] = Time.parse(host.exec('sudo hwclock').strip)
|
|
context[:timezone] = host.resources['facts']['values']['timezone']
|
|
context[:gold_images] = host.exec('sudo ls -1 /mnt/gold-images/ | grep tar.gz').split(/\n/).map(&:strip).reject(&:empty?)
|
|
context[:full_backups] = host.exec('sudo ls -1 /mnt/kvm_backups/ | grep tar.gz').split(/\n/).map(&:strip).reject(&:empty?)
|
|
context[:drbd_syncs] = host.resources['facts']['values']['drbd_status'].sort_by{|name,data| name}
|
|
context[:maintenance_mode] = host.exec("sudo pcs property show maintenance-mode | awk '/maint/ { print $2 }'").strip
|
|
context[:standby] = host.resources['facts']['values']['corosync']['is_standby'] ? "true" : "false"
|
|
}
|
|
|
|
|
|
network_template = ERB.new <<EOF, trim_mode: '<>-'
|
|
==============================================
|
|
Test : "<%= @name %>"
|
|
==============================================
|
|
<% @tests.each do |hostname, data| -%>
|
|
<%= hostname %>:
|
|
----------------------------------------------
|
|
<% data[:context][:interfaces].each do |name,info| -%>
|
|
<%= name %> <% if info.has_key? :mode %>(<%= info[:mode] %>)<% end %>:
|
|
<% if info[:mode] == "static" -%>
|
|
<% if info.has_key? :address -%>
|
|
IP: <%= info[:address] %>
|
|
<% end -%>
|
|
<% if info.has_key? :gateway -%>
|
|
Gateway: <%= info[:gateway] %>
|
|
<% end -%>
|
|
<% if info.has_key? :netmask -%>
|
|
Netmask: <%= info[:netmask] %>
|
|
<% end -%>
|
|
<% if info.has_key? :dns -%>
|
|
DNS: <%= info[:dns] %>
|
|
<% end -%>
|
|
<% end -%>
|
|
<% end -%>
|
|
|
|
<% end -%>
|
|
EOF
|
|
|
|
Manager.new_test("Networking Configurations Host", template: network_template) {|host,context|
|
|
contents = host.exec('cat /etc/network/interfaces')
|
|
interfaces = Hash.new
|
|
interface_regex = /iface (?<name>\S+) inet (?<mode>\S+)/
|
|
address_regex = /address (?<address>\S+)/
|
|
netmask_regex = /netmask (?<netmask>\S+)/
|
|
gateway_regex = /gateway (?<gateway>\S+)/
|
|
|
|
|
|
current_interface = ""
|
|
contents.split(/\n/).each do |line|
|
|
case line
|
|
when interface_regex
|
|
current_interface = $LAST_MATCH_INFO['name']
|
|
mode = $LAST_MATCH_INFO['mode']
|
|
interfaces[current_interface]=Hash.new
|
|
interfaces[current_interface][:mode] = mode
|
|
|
|
when address_regex
|
|
interfaces[current_interface][:address] = $LAST_MATCH_INFO['address']
|
|
|
|
when netmask_regex
|
|
interfaces[current_interface][:netmask] = $LAST_MATCH_INFO['netmask']
|
|
|
|
when gateway_regex
|
|
interfaces[current_interface][:gateway] = $LAST_MATCH_INFO['gateway']
|
|
end
|
|
end
|
|
|
|
contents = host.exec('sudo ipmitool lan print')
|
|
current_interface="IPMI"
|
|
interfaces[current_interface]=Hash.new
|
|
mode_regex = /IP Address Source[[:space:]]+:[[:space:]](?<mode>\S+)/
|
|
address_regex = /IP Address[[:space:]]+:[[:space:]]+(?<address>\S+)/
|
|
netmask_regex = /Subnet Mask[[:space:]]+:[[:space:]]+(?<netmask>\S+)/
|
|
gateway_regex = /Default Gateway IP[[:space:]]+:[[:space:]]+(?<gateway>\S+)/
|
|
|
|
contents.split(/\n/).each do |line|
|
|
case line
|
|
when mode_regex
|
|
interfaces[current_interface][:mode] = $LAST_MATCH_INFO['mode'].downcase
|
|
|
|
when address_regex
|
|
interfaces[current_interface][:address] = $LAST_MATCH_INFO['address']
|
|
|
|
when netmask_regex
|
|
interfaces[current_interface][:netmask] = $LAST_MATCH_INFO['netmask']
|
|
|
|
when gateway_regex
|
|
interfaces[current_interface][:gateway] = $LAST_MATCH_INFO['gateway']
|
|
end
|
|
end
|
|
|
|
ipconfig_script=<<EOF
|
|
ipconfig(){
|
|
local KVM=${1:?"$FUNCNAME: No VM provided"}
|
|
local -i inc=0
|
|
local output=""
|
|
|
|
handle_return=$(sudo virsh qemu-agent-command "$KVM" '{"execute":"guest-exec", "arguments":{"path":"cmd.exe","arg":["/c","ipconfig /all"],"capture-output":true}}'| grep -o '[[:digit:]]*')
|
|
until [[ $output = *\\"exited\\":true* || "$inc" -ge "42" ]] ; do
|
|
output=$(sudo virsh qemu-agent-command "$KVM" '{"execute":"guest-exec-status","arguments":{"pid":'${handle_return}'}}')
|
|
sleep .1 ; ((inc++))
|
|
done
|
|
cut -d '"' -f8 <<< "$output" | base64 -d
|
|
}
|
|
EOF
|
|
|
|
get_vm_info_script=<<EOF
|
|
get_vm_info () {
|
|
local awk_script='
|
|
BEGIN { ip=""; subnet=""; dns=""; gateway=""; FS=":" }
|
|
/IPv4 Address/ {gsub(/[[:space:]]/,"", $2); gsub("[(]Preferred[)]", "", $2); ip = $2 }
|
|
/Subnet Mask/ { gsub(/[[:space:]]/,"", $2); subnet = $2 }
|
|
/Gateway/ { gsub(/[[:space:]]/,"", $2); gateway = $2 }
|
|
/DNS Servers/ { gsub(/[[:space:]]/,"", $2); dns=$2}
|
|
/^[[:space:]]+[0-9]/ { gsub(/[[:space:]]/, "", $1); dns = dns "," $1 }
|
|
/NetBIOS/ { print name"|"ip"|"subnet"|"gateway"|"dns; name=""; ip=""; subnet=""; dns=""; gateway=""; }
|
|
'
|
|
vms=( $( sudo virsh list --name | sort -V ) )
|
|
if [[ ${#vms[@]} -gt 0 ]]; then
|
|
for VM in ${vms[@]}; do
|
|
ipconfig $VM | awk -v name=$VM "$awk_script"
|
|
done
|
|
fi
|
|
}
|
|
EOF
|
|
|
|
info = host.exec([get_vm_info_script, ipconfig_script, "get_vm_info"].join("\n")).split(/\n/)
|
|
|
|
info.select{|line| line =~ /.*\|.*/}.each do |line|
|
|
name, ip, subnet, gateway, dns = line.split("|")
|
|
interfaces[name] = Hash.new
|
|
interfaces[name][:gateway] = gateway
|
|
interfaces[name][:netmask] = subnet
|
|
interfaces[name][:address] = ip
|
|
interfaces[name][:dns] = dns
|
|
interfaces[name][:mode] = "static"
|
|
end
|
|
|
|
context[:interfaces]=interfaces
|
|
}
|
|
|
|
Manager.new_test("OpenVPN Certs Deployed to Node") {|host, context|
|
|
pass = true
|
|
|
|
buildserver_client_cert_exist = `sudo lxc-attach -n avobuildaus -- bash -c '[[ -f /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.crt ]] && echo "yes" || echo "no"'`.strip
|
|
cert_exist = (buildserver_client_cert_exist =~ /yes/ ) ? true : false
|
|
buildserver_client_key_exist = `sudo lxc-attach -n avobuildaus -- bash -c '[[ -f /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.key ]] && echo "yes" || echo "no"'`.strip
|
|
key_exist = (buildserver_client_key_exist =~ /yes/ ) ? true : false
|
|
|
|
if not cert_exist or not key_exist
|
|
context[:note] = "Nodes Cert and/or key were not found on buildserver, inform ENG that certs haven't been generated for #{host.name}"
|
|
pass = false
|
|
else
|
|
openvpn_client_cert_md5 = host.exec("sudo md5sum /etc/openvpn/mgmt/easy-rsa/keys/client.crt | awk '{ print $1 }'").strip
|
|
openvpn_client_key_md5 = host.exec("sudo md5sum /etc/openvpn/mgmt/easy-rsa/keys/client.key | awk '{ print $1 }'").strip
|
|
buildserver_client_cert_md5 = `sudo lxc-attach -n avobuildaus -- md5sum /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.crt | awk '{ print $1 }'`.strip
|
|
buildserver_client_key_md5 = `sudo lxc-attach -n avobuildaus -- md5sum /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.key | awk '{ print $1 }'`.strip
|
|
|
|
md5_fails = Array.new
|
|
|
|
if not openvpn_client_cert_md5 == buildserver_client_cert_md5
|
|
md5_fails << "cert"
|
|
end
|
|
|
|
if not openvpn_client_key_md5 == buildserver_client_key_md5
|
|
md5_fails << "key"
|
|
end
|
|
|
|
if md5_fails.count != 0
|
|
pass = false
|
|
context[:note] = "Failed: #{md5_fails.join(',')}"
|
|
end
|
|
end
|
|
|
|
pass
|
|
}
|
|
|
|
NETMASK_32="255.255.255.255"
|
|
FILTER_INTERFACES=/eno(0|1|2|6)/
|
|
Manager.new_test("No /32 addresses") {|host, context|
|
|
pass = true
|
|
facts = host.resources['facts'].data
|
|
interfaces = facts['values']['networking']['interfaces'].select{|name,val| name =~ FILTER_INTERFACES}
|
|
failed_interfaces = Array.new
|
|
|
|
interfaces.each do |name,data|
|
|
if data["bindings"].is_a? Array
|
|
data["bindings"].each do |binding|
|
|
if binding['netmask'] == NETMASK_32
|
|
pass = false
|
|
puts "#{host.name}: Was found to have a /32 netmask on #{name} with ip #{binding['address']}"
|
|
if ! failed_interfaces.include? name
|
|
failed_interfaces << name
|
|
end
|
|
end
|
|
end
|
|
elsif data['bindings'].is_a? Hash
|
|
if data['bindings']['netmask'] == NETMASK_32
|
|
pass = false
|
|
puts "#{host.name}: Was found to have a /32 netmask on #{name} with ip #{data['address']}"
|
|
failed_interfaces << name
|
|
end
|
|
end
|
|
end
|
|
if ! failed_interfaces.empty?
|
|
context[:note] = "failed: #{failed_interfaces.join(',')}"
|
|
end
|
|
pass
|
|
}
|
|
|
|
Manager.new_test("Puppet Env is Pilot"){ |host|
|
|
host.exec("cat /etc/puppetlabs/puppet/puppet.conf") =~ /environment = pilot/
|
|
}
|
|
|
|
NETMASK_30="255.255.255.252"
|
|
RED1_ENO6="172.16.16.2"
|
|
RED0_ENO6="172.16.16.1"
|
|
Manager.new_test("DRBD Interfaces Configured Correctly"){|host|
|
|
facts = host.resources['facts'].data
|
|
networking = facts['values']['networking']
|
|
binding = networking['interfaces']['eno6']['bindings'].first
|
|
address = binding['address']
|
|
netmask = binding['netmask']
|
|
|
|
if host.name =~ /red1/
|
|
address == RED1_ENO6 && netmask == NETMASK_30
|
|
elsif host.name =~ /red0/
|
|
address == RED0_ENO6 && netmask == NETMASK_30
|
|
else
|
|
false
|
|
end
|
|
}
|
|
|
|
VM_NAME_REGEX = /(?<name>[A-Z]{3}[0-9]+).xml/
|
|
CACHE_MODE_REGEX = /cache=\'(?<mode>\S+)\'/
|
|
Manager.new_test("Check VM disk cache is writeback"){|host, context|
|
|
pass = true
|
|
failed_vms = Array.new
|
|
|
|
host.exec("sudo grep cache /etc/libvirt/qemu/*.xml").split(/\n/).each do |line|
|
|
name = VM_NAME_REGEX.match(line)['name']
|
|
mode = CACHE_MODE_REGEX.match(line)['mode']
|
|
|
|
if mode != "writeback"
|
|
pass = false
|
|
failed_vms << name
|
|
end
|
|
end
|
|
|
|
if ! failed_vms.empty?
|
|
context[:note] = "failed vms: #{failed_vms.join(",")}"
|
|
end
|
|
|
|
pass
|
|
}
|
|
|
|
Manager.new_test("Manual RVC Patch Applied to domain.rb"){|host, context|
|
|
pass = true
|
|
pass = host.exec("sudo grep Log.error /opt/rvc/lib/rvc/domain.rb") =~ /Log.error[[:space:]]*\{[[:space:]]*e(\.message)?[[:space:]]*\}/
|
|
if not pass
|
|
context[:note] = "Manual Patch not applied (sudo sed -i 's/Log.error(e)/Log.error { e }/g' /opt/rvc/lib/rvc/domain.rb)"
|
|
end
|
|
pass
|
|
}
|
|
|
|
Manager.new_test("Test IPMI Credentials on self (over network)") {|host,context|
|
|
fencenode=host.resources['facts']['values']['pcs_resources']["fence_#{host.name}"]['instance_attributes']
|
|
|
|
pass = true
|
|
|
|
test_result = host.exec("set +o history; ipmitool -I lanplus -H #{fencenode["ipaddr"]} -U #{fencenode['userid']} -P #{fencenode['passwd']} chassis status >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
|
|
|
|
case test_result
|
|
when 'yes'
|
|
pass = true
|
|
when 'no'
|
|
pass = false
|
|
end
|
|
|
|
if pass == false
|
|
ping_test = host.exec("ping -c 2 #{fencenode["ipaddr"]} >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
|
|
case ping_test
|
|
when 'yes'
|
|
context[:note] = "IPMI Pingable, Creds Failed"
|
|
when 'no'
|
|
context[:note] = "IPMI Not Pingable. Possibly unplugged"
|
|
end
|
|
end
|
|
|
|
pass
|
|
}
|
|
|
|
Manager.new_test("Test IPMI Credentials on partner") {|host,context|
|
|
if host.name =~ /red1/
|
|
hostname = host.name.sub('red1','red0')
|
|
elsif host.name =~ /red0/
|
|
hostname = host.name.sub('red0','red1')
|
|
end
|
|
|
|
pass = true
|
|
|
|
begin
|
|
fencenode=host.resources['facts']['values']['pcs_resources']["fence_#{hostname}"]['instance_attributes']
|
|
rescue
|
|
puts "Fact retrieval failed on #{host.name}. Failing test"
|
|
return false
|
|
end
|
|
|
|
test_result = host.exec("set +o history; ipmitool -I lanplus -H #{fencenode["ipaddr"]} -U #{fencenode['userid']} -P #{fencenode['passwd']} chassis status >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
|
|
|
|
case test_result
|
|
when 'yes'
|
|
pass = true
|
|
when 'no'
|
|
pass = false
|
|
end
|
|
|
|
if pass == false
|
|
ping_test = host.exec("ping -c 2 #{fencenode["ipaddr"]} >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
|
|
case ping_test
|
|
when 'yes'
|
|
context[:note] = "Other Node IPMI is pingable, Creds Failed"
|
|
when 'no'
|
|
context[:note] = "Other Node IPMI is not Pingable. Possibly unplugged"
|
|
end
|
|
end
|
|
|
|
pass
|
|
}
|
|
|
|
Manager.new_test("No Cluster Irregularities", description: "Nothing Disabled that shouldn't be"){|node,context|
|
|
pass = true
|
|
disabled_items = Array.new
|
|
|
|
## Stonith
|
|
## .values.pcs_resources.fence_avo00000223red0.meta_attributes["target-role"]
|
|
begin
|
|
node_stonith_state = node.resources['facts']['values']['pcs_resources']["fence_#{node.name}"]['meta_attributes']['target-role']
|
|
rescue
|
|
puts "Fact retrieval failed on #{host.name}. Failing test"
|
|
return false
|
|
end
|
|
|
|
|
|
if node_stonith_state == "Stopped"
|
|
disabled_items << "Stonith"
|
|
end
|
|
|
|
if disabled_items.count !=0
|
|
context[:note] = "Disabled Items: #{disabled_items.join(',')}"
|
|
pass = false
|
|
end
|
|
|
|
pass
|
|
}
|
|
|
|
Manager.new_test("Puppet Agent disabled (pre-ship only)"){|node,context|
|
|
pass = true
|
|
|
|
ssl_dir_exists = node.exec('[[ -f $( sudo puppet config print agent_disabled_lockfile ) ]] && echo "yes" || echo "no"')
|
|
|
|
case ssl_dir_exists
|
|
when /no/
|
|
pass = false
|
|
context[:note] = "Before shipping node you need to disable the puppet agent on #{node.name} (sudo puppet agent --disable)"
|
|
end
|
|
|
|
pass
|
|
}
|
|
|
|
|
|
Manager.new_test("SSL Directory Cleaned (pre-ship only)"){|node,context|
|
|
pass = true
|
|
|
|
ssl_dir_exists = node.exec('[[ -d /etc/puppetlabs/puppet/ssl ]] && echo "yes" || echo "no"')
|
|
|
|
case ssl_dir_exists
|
|
when /yes/
|
|
pass = false
|
|
context[:note] = "Before shipping node you need to delete the ssl dir on #{node.name} (sudo rm -rf /etc/puppetlabs/puppet/ssl)"
|
|
end
|
|
|
|
pass
|
|
}
|
|
|
|
Manager.new_test("Ensure VMs not running (pre-ship only)"){|host,context|
|
|
pass = true
|
|
vms = host.exec("sudo virsh list --name").split(/\n/).map(&:strip).sort!
|
|
|
|
if vms.count > 0
|
|
pass = false
|
|
context[:note] = "VMs are still running: #{vms.join(',')} shut them down before shipping (for VM in #{vms.join(' ')}; do sudo rvc kvm shutdown $VM; done"
|
|
end
|
|
|
|
pass
|
|
}
|
|
|
|
|
|
Manager.new_test("Cluster in Pre-Ship Condition (pre-ship only)"){|host,context|
|
|
pass = true
|
|
maintenance_mode = host.exec("sudo pcs property show maintenance-mode | awk '/maint/ { print $2 }'").strip
|
|
standby = host.resources['facts']['values']['corosync']['is_standby'] ? "true" : "false"
|
|
|
|
case host.name
|
|
when /red1/
|
|
if maintenance_mode =~ /false/ and standby =~ /false/
|
|
context[:note] = "Node is not in standby and cluster is not in maintenance-mode"
|
|
pass = false
|
|
elsif maintenance_mode =~ /false/ and standby =~ /true/
|
|
context[:note] = "Cluster isn't in maintenance-mode"
|
|
pass = false
|
|
elsif maintenance_mode =~ /true/ and standby =~ /false/
|
|
context[:note] = "Node isn't in standby"
|
|
pass = false
|
|
end
|
|
|
|
when /red0/
|
|
if maintenance_mode =~ /false/
|
|
context[:note] = "Cluster isn't in maintenance-mode"
|
|
pass = false
|
|
end
|
|
end
|
|
|
|
pass
|
|
}
|
|
|