tristan/ruby-qa
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New changes

Browse Source
This commit is contained in:
Tristan Ancelet
2024-12-11 10:25:13 -06:00
parent d663ee765a
commit 12c82281dc
16 changed files with 837 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

466
lib/rubyqa/tests/avo.rb Executable file
View File

@@ -0,0 +1,466 @@
require 'rubyqa'
include RubyQA
require 'time'
require 'English'
info_template = ERB.new <<EOF, trim_mode: '-'
==============================================
Test : "<%= @name %>"
==============================================
<% @tests.each do |hostname,data| -%>
<%= hostname %>
----------------------------------------------
Serial: <%= data[:context][:serial] %>
System Date: <%= data[:context][:date] %>
HWCLOCK: <%= data[:context][:hwclock] %>
Timezone: <%= data[:context][:timezone] %>
MM: <%= data[:context][:maintenance_mode] %>
Standby: <%= data[:context][:standby] %>
Gold-Images:
<% data[:context][:gold_images].each do |image| -%>
- <%= image %>
<% end -%>
Full-Backups:
<% data[:context][:full_backups].each do |image| -%>
- <%= image %>
<% end -%>
DRBD Syncs:
<% data[:context][:drbd_syncs].each do |sync_name, sync_data| -%>
- <%= sync_name %> (Sync Status: <%= sync_data['disk_state'] %>)
<% end -%>
<% end -%>
EOF
Manager.new_test("Get Information", template: info_template){|host,context|
context[:serial] = host.resources['facts']['values']['dmi']['product']['serial_number'].downcase
context[:date] = Time.parse(host.exec('date').strip)
context[:hwclock] = Time.parse(host.exec('sudo hwclock').strip)
context[:timezone] = host.resources['facts']['values']['timezone']
context[:gold_images] = host.exec('sudo ls -1 /mnt/gold-images/ | grep tar.gz').split(/\n/).map(&:strip).reject(&:empty?)
context[:full_backups] = host.exec('sudo ls -1 /mnt/kvm_backups/ | grep tar.gz').split(/\n/).map(&:strip).reject(&:empty?)
context[:drbd_syncs] = host.resources['facts']['values']['drbd_status'].sort_by{|name,data| name}
context[:maintenance_mode] = host.exec("sudo pcs property show maintenance-mode | awk '/maint/ { print $2 }'").strip
context[:standby] = host.resources['facts']['values']['corosync']['is_standby'] ? "true" : "false"
}
network_template = ERB.new <<EOF, trim_mode: '<>-'
==============================================
Test : "<%= @name %>"
==============================================
<% @tests.each do |hostname, data| -%>
<%= hostname %>:
----------------------------------------------
<% data[:context][:interfaces].each do |name,info| -%>
<%= name %> <% if info.has_key? :mode %>(<%= info[:mode] %>)<% end %>:
<% if info[:mode] == "static" -%>
<% if info.has_key? :address -%>
IP: <%= info[:address] %>
<% end -%>
<% if info.has_key? :gateway -%>
Gateway: <%= info[:gateway] %>
<% end -%>
<% if info.has_key? :netmask -%>
Netmask: <%= info[:netmask] %>
<% end -%>
<% if info.has_key? :dns -%>
DNS: <%= info[:dns] %>
<% end -%>
<% end -%>
<% end -%>
<% end -%>
EOF
Manager.new_test("Networking Configurations Host", template: network_template) {|host,context|
contents = host.exec('cat /etc/network/interfaces')
interfaces = Hash.new
interface_regex = /iface (?<name>\S+) inet (?<mode>\S+)/
address_regex = /address (?<address>\S+)/
netmask_regex = /netmask (?<netmask>\S+)/
gateway_regex = /gateway (?<gateway>\S+)/
current_interface = ""
contents.split(/\n/).each do |line|
case line
when interface_regex
current_interface = $LAST_MATCH_INFO['name']
mode = $LAST_MATCH_INFO['mode']
interfaces[current_interface]=Hash.new
interfaces[current_interface][:mode] = mode
when address_regex
interfaces[current_interface][:address] = $LAST_MATCH_INFO['address']
when netmask_regex
interfaces[current_interface][:netmask] = $LAST_MATCH_INFO['netmask']
when gateway_regex
interfaces[current_interface][:gateway] = $LAST_MATCH_INFO['gateway']
end
end
contents = host.exec('sudo ipmitool lan print')
current_interface="IPMI"
interfaces[current_interface]=Hash.new
mode_regex = /IP Address Source[[:space:]]+:[[:space:]](?<mode>\S+)/
address_regex = /IP Address[[:space:]]+:[[:space:]]+(?<address>\S+)/
netmask_regex = /Subnet Mask[[:space:]]+:[[:space:]]+(?<netmask>\S+)/
gateway_regex = /Default Gateway IP[[:space:]]+:[[:space:]]+(?<gateway>\S+)/
contents.split(/\n/).each do |line|
case line
when mode_regex
interfaces[current_interface][:mode] = $LAST_MATCH_INFO['mode'].downcase
when address_regex
interfaces[current_interface][:address] = $LAST_MATCH_INFO['address']
when netmask_regex
interfaces[current_interface][:netmask] = $LAST_MATCH_INFO['netmask']
when gateway_regex
interfaces[current_interface][:gateway] = $LAST_MATCH_INFO['gateway']
end
end
ipconfig_script=<<EOF
ipconfig(){
local KVM=${1:?"$FUNCNAME: No VM provided"}
local -i inc=0
local output=""
handle_return=$(sudo virsh qemu-agent-command "$KVM" '{"execute":"guest-exec", "arguments":{"path":"cmd.exe","arg":["/c","ipconfig /all"],"capture-output":true}}'| grep -o '[[:digit:]]*')
until [[ $output = *\\"exited\\":true* || "$inc" -ge "42" ]] ; do
output=$(sudo virsh qemu-agent-command "$KVM" '{"execute":"guest-exec-status","arguments":{"pid":'${handle_return}'}}')
sleep .1 ; ((inc++))
done
cut -d '"' -f8 <<< "$output" | base64 -d
}
EOF
get_vm_info_script=<<EOF
get_vm_info () {
local awk_script='
BEGIN { ip=""; subnet=""; dns=""; gateway=""; FS=":" }
/IPv4 Address/ {gsub(/[[:space:]]/,"", $2); gsub("[(]Preferred[)]", "", $2); ip = $2 }
/Subnet Mask/ { gsub(/[[:space:]]/,"", $2); subnet = $2 }
/Gateway/ { gsub(/[[:space:]]/,"", $2); gateway = $2 }
/DNS Servers/ { gsub(/[[:space:]]/,"", $2); dns=$2}
/^[[:space:]]+[0-9]/ { gsub(/[[:space:]]/, "", $1); dns = dns "," $1 }
/NetBIOS/ { print name"|"ip"|"subnet"|"gateway"|"dns; name=""; ip=""; subnet=""; dns=""; gateway=""; }
'
vms=( $( sudo virsh list --name | sort -V ) )
if [[ ${#vms[@]} -gt 0 ]]; then
for VM in ${vms[@]}; do
ipconfig $VM | awk -v name=$VM "$awk_script"
done
fi
}
EOF
info = host.exec([get_vm_info_script, ipconfig_script, "get_vm_info"].join("\n")).split(/\n/)
info.select{|line| line =~ /.*\|.*/}.each do |line|
name, ip, subnet, gateway, dns = line.split("|")
interfaces[name] = Hash.new
interfaces[name][:gateway] = gateway
interfaces[name][:netmask] = subnet
interfaces[name][:address] = ip
interfaces[name][:dns] = dns
interfaces[name][:mode] = "static"
end
context[:interfaces]=interfaces
}
Manager.new_test("OpenVPN Certs Deployed to Node") {|host, context|
pass = true
buildserver_client_cert_exist = `sudo lxc-attach -n avobuildaus -- bash -c '[[ -f /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.crt ]] && echo "yes" || echo "no"'`.strip
cert_exist = (buildserver_client_cert_exist =~ /yes/ ) ? true : false
buildserver_client_key_exist = `sudo lxc-attach -n avobuildaus -- bash -c '[[ -f /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.key ]] && echo "yes" || echo "no"'`.strip
key_exist = (buildserver_client_key_exist =~ /yes/ ) ? true : false
if not cert_exist or not key_exist
context[:note] = "Nodes Cert and/or key were not found on buildserver, inform ENG that certs haven't been generated for #{host.name}"
pass = false
else
openvpn_client_cert_md5 = host.exec("sudo md5sum /etc/openvpn/mgmt/easy-rsa/keys/client.crt | awk '{ print $1 }'").strip
openvpn_client_key_md5 = host.exec("sudo md5sum /etc/openvpn/mgmt/easy-rsa/keys/client.key | awk '{ print $1 }'").strip
buildserver_client_cert_md5 = `sudo lxc-attach -n avobuildaus -- md5sum /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.crt | awk '{ print $1 }'`.strip
buildserver_client_key_md5 = `sudo lxc-attach -n avobuildaus -- md5sum /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.key | awk '{ print $1 }'`.strip
md5_fails = Array.new
if not openvpn_client_cert_md5 == buildserver_client_cert_md5
md5_fails << "cert"
end
if not openvpn_client_key_md5 == buildserver_client_key_md5
md5_fails << "key"
end
if md5_fails.count != 0
pass = false
context[:note] = "Failed: #{md5_fails.join(',')}"
end
end
pass
}
NETMASK_32="255.255.255.255"
FILTER_INTERFACES=/eno(0|1|2|6)/
Manager.new_test("No /32 addresses") {|host, context|
pass = true
facts = host.resources['facts'].data
interfaces = facts['values']['networking']['interfaces'].select{|name,val| name =~ FILTER_INTERFACES}
failed_interfaces = Array.new
interfaces.each do |name,data|
if data["bindings"].is_a? Array
data["bindings"].each do |binding|
if binding['netmask'] == NETMASK_32
pass = false
puts "#{host.name}: Was found to have a /32 netmask on #{name} with ip #{binding['address']}"
if ! failed_interfaces.include? name
failed_interfaces << name
end
end
end
elsif data['bindings'].is_a? Hash
if data['bindings']['netmask'] == NETMASK_32
pass = false
puts "#{host.name}: Was found to have a /32 netmask on #{name} with ip #{data['address']}"
failed_interfaces << name
end
end
end
if ! failed_interfaces.empty?
context[:note] = "failed: #{failed_interfaces.join(',')}"
end
pass
}
Manager.new_test("Puppet Env is Pilot"){ |host|
host.exec("cat /etc/puppetlabs/puppet/puppet.conf") =~ /environment = pilot/
}
NETMASK_30="255.255.255.252"
RED1_ENO6="172.16.16.2"
RED0_ENO6="172.16.16.1"
Manager.new_test("DRBD Interfaces Configured Correctly"){|host|
facts = host.resources['facts'].data
networking = facts['values']['networking']
binding = networking['interfaces']['eno6']['bindings'].first
address = binding['address']
netmask = binding['netmask']
if host.name =~ /red1/
address == RED1_ENO6 && netmask == NETMASK_30
elsif host.name =~ /red0/
address == RED0_ENO6 && netmask == NETMASK_30
else
false
end
}
VM_NAME_REGEX = /(?<name>[A-Z]{3}[0-9]+).xml/
CACHE_MODE_REGEX = /cache=\'(?<mode>\S+)\'/
Manager.new_test("Check VM disk cache is writeback"){|host, context|
pass = true
failed_vms = Array.new
host.exec("sudo grep cache /etc/libvirt/qemu/*.xml").split(/\n/).each do |line|
name = VM_NAME_REGEX.match(line)['name']
mode = CACHE_MODE_REGEX.match(line)['mode']
if mode != "writeback"
pass = false
failed_vms << name
end
end
if ! failed_vms.empty?
context[:note] = "failed vms: #{failed_vms.join(",")}"
end
pass
}
Manager.new_test("Manual RVC Patch Applied to domain.rb"){|host, context|
pass = true
pass = host.exec("sudo grep Log.error /opt/rvc/lib/rvc/domain.rb") =~ /Log.error[[:space:]]*\{[[:space:]]*e(\.message)?[[:space:]]*\}/
if not pass
context[:note] = "Manual Patch not applied (sudo sed -i 's/Log.error(e)/Log.error { e }/g' /opt/rvc/lib/rvc/domain.rb)"
end
pass
}
Manager.new_test("Test IPMI Credentials on self (over network)") {|host,context|
fencenode=host.resources['facts']['values']['pcs_resources']["fence_#{host.name}"]['instance_attributes']
pass = true
test_result = host.exec("set +o history; ipmitool -I lanplus -H #{fencenode["ipaddr"]} -U #{fencenode['userid']} -P #{fencenode['passwd']} chassis status >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
case test_result
when 'yes'
pass = true
when 'no'
pass = false
end
if pass == false
ping_test = host.exec("ping -c 2 #{fencenode["ipaddr"]} >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
case ping_test
when 'yes'
context[:note] = "IPMI Pingable, Creds Failed"
when 'no'
context[:note] = "IPMI Not Pingable. Possibly unplugged"
end
end
pass
}
Manager.new_test("Test IPMI Credentials on partner") {|host,context|
if host.name =~ /red1/
hostname = host.name.sub('red1','red0')
elsif host.name =~ /red0/
hostname = host.name.sub('red0','red1')
end
pass = true
begin
fencenode=host.resources['facts']['values']['pcs_resources']["fence_#{hostname}"]['instance_attributes']
rescue
puts "Fact retrieval failed on #{host.name}. Failing test"
return false
end
test_result = host.exec("set +o history; ipmitool -I lanplus -H #{fencenode["ipaddr"]} -U #{fencenode['userid']} -P #{fencenode['passwd']} chassis status >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
case test_result
when 'yes'
pass = true
when 'no'
pass = false
end
if pass == false
ping_test = host.exec("ping -c 2 #{fencenode["ipaddr"]} >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
case ping_test
when 'yes'
context[:note] = "Other Node IPMI is pingable, Creds Failed"
when 'no'
context[:note] = "Other Node IPMI is not Pingable. Possibly unplugged"
end
end
pass
}
Manager.new_test("No Cluster Irregularities", description: "Nothing Disabled that shouldn't be"){|node,context|
pass = true
disabled_items = Array.new
## Stonith
## .values.pcs_resources.fence_avo00000223red0.meta_attributes["target-role"]
begin
node_stonith_state = node.resources['facts']['values']['pcs_resources']["fence_#{node.name}"]['meta_attributes']['target-role']
rescue
puts "Fact retrieval failed on #{host.name}. Failing test"
return false
end
if node_stonith_state == "Stopped"
disabled_items << "Stonith"
end
if disabled_items.count !=0
context[:note] = "Disabled Items: #{disabled_items.join(',')}"
pass = false
end
pass
}
Manager.new_test("Puppet Agent disabled (pre-ship only)"){|node,context|
pass = true
ssl_dir_exists = node.exec('[[ -f $( sudo puppet config print agent_disabled_lockfile ) ]] && echo "yes" || echo "no"')
case ssl_dir_exists
when /no/
pass = false
context[:note] = "Before shipping node you need to disable the puppet agent on #{node.name} (sudo puppet agent --disable)"
end
pass
}
Manager.new_test("SSL Directory Cleaned (pre-ship only)"){|node,context|
pass = true
ssl_dir_exists = node.exec('[[ -d /etc/puppetlabs/puppet/ssl ]] && echo "yes" || echo "no"')
case ssl_dir_exists
when /yes/
pass = false
context[:note] = "Before shipping node you need to delete the ssl dir on #{node.name} (sudo rm -rf /etc/puppetlabs/puppet/ssl)"
end
pass
}
Manager.new_test("Ensure VMs not running (pre-ship only)"){|host,context|
pass = true
vms = host.exec("sudo virsh list --name").split(/\n/).map(&:strip).sort!
if vms.count > 0
pass = false
context[:note] = "VMs are still running: #{vms.join(',')} shut them down before shipping (for VM in #{vms.join(' ')}; do sudo rvc kvm shutdown $VM; done"
end
pass
}
Manager.new_test("Cluster in Pre-Ship Condition (pre-ship only)"){|host,context|
pass = true
maintenance_mode = host.exec("sudo pcs property show maintenance-mode | awk '/maint/ { print $2 }'").strip
standby = host.resources['facts']['values']['corosync']['is_standby'] ? "true" : "false"
case host.name
when /red1/
if maintenance_mode =~ /false/ and standby =~ /false/
context[:note] = "Node is not in standby and cluster is not in maintenance-mode"
pass = false
elsif maintenance_mode =~ /false/ and standby =~ /true/
context[:note] = "Cluster isn't in maintenance-mode"
pass = false
elsif maintenance_mode =~ /true/ and standby =~ /false/
context[:note] = "Node isn't in standby"
pass = false
end
when /red0/
if maintenance_mode =~ /false/
context[:note] = "Cluster isn't in maintenance-mode"
pass = false
end
end
pass
}

166
lib/rubyqa/tests/gpc.rb Normal file
View File

@@ -0,0 +1,166 @@
#!/usr/bin/ruby
require 'rubyqa'
require 'erb'
include RubyQA
info_template = ERB.new <<EOF, trim_mode: '-'
==============================================
Test : <%= @name %>
==============================================
<% @tests.each do |hostname,data| -%>
<%= hostname %>
----------------------------------------------
Serials: <%= data[:context][:serial] %>
System Date: <%= data[:context][:date] %>
HWCLOCK: <%= data[:context][:hwclock] %>
Timezone: <%= data[:context][:timezone] %>
eSocket Status: <%= data[:context][:esocket_status] %>
eSocket Patch: <%= data[:context][:esocket_patch] %>
keystore md5s:
<% data[:context][:keystore_md5s].each do |line| -%>
- <%= line %>
<% end -%>
<% end -%>
EOF
Manager.new_test("Get Information", template: info_template){|host,context|
serial_matcher = /\"(\S+)\"/
serials = host.exec("sudo facter 2>/dev/null | grep serial | cut -d '>' -f 2 | tr -d ','")
context[:serial] = serials.scan(serial_matcher).map{|serial| "\"#{serial.first}\""}.join(',')
context[:date] = Time.parse(host.exec('date').strip)
context[:hwclock] = Time.parse(host.exec('sudo hwclock').strip)
context[:timezone] = host.exec('readlink -f /etc/localtime').strip.gsub(/(\/usr\/share|\/etc)\/zoneinfo\//,'')
context[:esocket_status] = host.exec('sudo lxc-attach -n ${HOSTNAME/red?/aci0} -- systemctl is-active esocket').strip
context[:esocket_patch] = host.exec('sudo lxc-attach -n ${HOSTNAME/red?/aci0} -- ls -1 /home/esocket/eSocket.POS').split(/\n/).select{|line| line =~ /patch/}.last
context[:keystore_md5s] = host.exec('sudo lxc-attach -n ${HOSTNAME/red?/aci0} -- md5sum /home/esocket/eSocket.POS/keystore/*').split(/\n/).map(&:strip)
}
firewall_template = ERB.new <<EOF, trim_mode: '-'
==============================================
Test : <%= @name %>
==============================================
<% @tests.each do |hostname,data| -%>
<%= hostname %>
----------------------------------------------
<% data[:context][:chains].each do |name, info| -%>
Chain <%= name %>
<% info.each do |rule| -%>
<%= rule[:line] %>
<% end -%>
<% end -%>
<% end -%>
EOF
filter_regex = /^target/
chain_regex = /Chain (?<name>\S+)/
rule_regex = /(?<target>\S+)[[:space:]]+(?<protocols>\S+)[[:space:]]+(?<opts>\S+)[[:space:]]+(?<source>\S+)[[:space:]]+(?<destination>\S+)/
Manager.new_test("Firewall Forward Rules", template: firewall_template) {|host,context|
host_firewall_rules = host.exec("sudo itptables -L FORWARD")
context[:chains]=Hash.new
current_chain = ""
host_firewall_rules.split(/\n/).each do |line|
case line
when chain_regex
info = chain_regex.match(line)
current_chain = info['name']
context[:chains][current_chain] = Array.new
when filter_regex
next
when rule_regex
rule = Hash.new
info = rule_regex.match(line)
rule[:target]=info['target']
rule[:source]=info['source']
rule[:destination]=info['destination']
rule[:protocols]=info['protocols']
rule[:line]=line
context[:chains][current_chain] << rule
end
end
}
Manager.new_test("Check if hostname is correctly configured") {|host,context|
pass = true
name = host.name
hosts = host.exec("cat /etc/hosts")
hostname = host.exec("cat /etc/hostname").strip
not_configured_in = Array.new
if not hosts =~ /127\.0\.0\.1.*#{name}/
pass = false
not_configured_in << "/etc/hosts"
end
if not hostname =~ /#{name}/
pass = false
not_configured_in << "/etc/hostname"
end
if not_configured_in.count > 0
context[:note] = "Hostname was not configured in #{not_configured_in.join(',')}"
end
pass
}
Manager.new_test("Puppet Certs Generated") {|host|
}
Manager.new_test("OpenVPN Certs Generated") {|host, context|
listing = host.exec('ls -1 /etc/openvpn/mgmt/easy-rsa/keys/')
files_not_generated = Array.new
pass = true
if ! listing =~ /my.crt/
pass = false
files_not_generated << 'my.crt'
end
if ! listing =~ /my.key/
pass = false
files_not_generated << 'my.key'
end
if not pass
context[:note] = "Files not created: #{files_not_generated.join(',')}"
end
pass
}
Manager.new_test("Ensure netcat installed on aci, prx & host"){|host, context|
pass = true
aci_listing = host.exec("sudo lxc-attach -n ${HOSTNAME/red?/aci0} -- apt list --installed")
prx_listing = host.exec("sudo lxc-attach -n ${HOSTNAME/red?/prx0} -- apt list --installed")
host_listing = host.exec("sudo apt list --installed")
not_installed_on = Array.new
if not aci_listing =~ /netcat/
not_installed_on << "aci"
end
if not prx_listing =~ /netcat/
not_installed_on << "prx"
end
if not host_listing =~ /netcat/
not_installed_on << "host"
end
if not_installed_on.count > 0
pass = false
context[:note] = "netcat not installed on #{not_installed_on.join(',')}"
end
pass
}