tristan/ruby-qa
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: tristan:master
Branches Tags
tristan:master
tristan:feature_modular_command_runners
..
compare: tristan:feature_modular_command_runners
Branches Tags
tristan:master
tristan:feature_modular_command_runners

No commits in common. "master" and "feature_modular_command_runners" have entirely different histories.
master ... feature_mo

16 changed files with 169 additions and 1011 deletions
Show Stats Expand all files Collapse all files

9
.gitignore vendored
View File

@ -1,9 +0,0 @@
*
!.gitignore
!lib
!lib/**
!rubyqa.gemspec
!templates
!templates/*
!design.adoc
!README.md

17
lib/rubyqa/host.rb → lib/host.rb
View File

@ -1,4 +1,5 @@
module RubyQA
require 'net/ssh'
class Host
DEFAULT_HOSTDATA={
:hostname => "",
@ -12,6 +13,7 @@ module RubyQA
:runner => SSH_Runner
}
attr_accessor *DEFAULT_HOSTDATA.keys
attr_reader :client, :data, :resources
def initialize( data = {} )
@ -19,36 +21,29 @@ module RubyQA
@data = DEFAULT_HOSTDATA.merge data
if not @data[:hostname].empty? and @data[:site].empty?
data[:site]=@data[:hostname][0,3]
end
## initialize client for usage tests
@client = @data[:runner].new(@data)
end
def exec (command)
## Send a command from the host
@client.exec(command)
end
def exec_sudo (command)
## Send a sudo command from the host
@client.exec_sudo(command)
end
def [](key)
## Provide Access to data (:ip, etc)
@data[key]
end
def name
if @data[:hostname] != ""
@data[:hostname]
elsif @resources['facts'].data['hostname'] != ""
@resources['facts'].data['hostname']
return @data[:hostname]
elsif @resources['facts'].data['networking']['hostname'] != ""
return @resources['facts'].data['networking']['hostname']
else
self.exec('hostname')
return @data[:ip]
end
end

52
lib/manager.rb Normal file
View File

@ -0,0 +1,52 @@
module RubyQA
class Manager
attr_reader :resources, :hosts
@@tests = Array.new
@@hosts = Array.new
def self.tests
@@tests
end
def self.hosts
@@hosts
end
def self.add_host(host)
@@hosts << host
register_resources(host)
update_resources(host)
end
def self.register_resources (host)
Resource.all_resources.each do |resource|
host.add_resource(resource)
end
end
def self.update_resources (host)
host.update_resources
end
def self.new_test(name, options={}, &test_proc)
@@tests << Test.new(name, **options, &test_proc)
end
def self.run_tests
@@hosts.each do |host|
tests = @@tests.select{|test| test.valid_host(host)}
tests.each do |test|
test.run(host)
end
end
end
def self.report (template = nil)
if template
puts template.result(binding)
else
puts @@tests.map(&:report).join("\n")
end
end
end
end

65
lib/resource.rb Normal file
View File

@ -0,0 +1,65 @@
module RubyQA
class Resource
attr_reader :name, :data
REQUIREMENTS = {}
def initialize (host)
@host = host
@data = Hash.new
@gather_command = ""
end
def gather
if not @gather_command.empty?
output = @host.exec(@gather_command)
parse(output)
else
raise "@gather_command was not defined on Resource[#{@name}]"
end
end
def parse(output)
raise "parse not yet implemented on Resource[#{@name}]"
end
## This will allow me to iterate through all subclasses without
## having to manually define them elsewhere
def self.all_resources
ObjectSpace.each_object(Class).select{|klass| klass < self}
end
end
## Gathers the facts from the remote machine via the facter utility
require 'json'
class Facts < Resource
REQUIREMENTS = {}
def initialize (host)
super host
@name = 'facts'
@gather_command = "facter -j"
end
def parse (output)
@data = JSON.load(output)
end
end
class DRDB < Resource
REQUIREMENTS = {
:cluster => true
}
def initialize (host)
super host
@name = 'facts'
@gather_command = "sudo drbdadm"
end
def parse (output)
@data = output
end
end
end

12
lib/rubyqa.rb
View File

@ -1,9 +1,7 @@
#!/usr/bin/env ruby
require_relative 'rubyqa/overrides'
require_relative 'rubyqa/templates'
require_relative 'rubyqa/manager'
require_relative 'rubyqa/resource'
require_relative 'rubyqa/runner'
require_relative 'rubyqa/host'
require_relative 'rubyqa/test'
$LOAD_PATH << __dir__
require 'manager'
require 'resource'
require 'host'
require 'test'

51
lib/rubyqa/interaction.rb
View File

@ -1,51 +0,0 @@
module RubyQA
module Interactions
def get_string_response(prompt)
puts "#{prompt}: "
while true
answer = STDIN.gets.strip
if get_user_verification "Is \"#{answer}\" correct?"
return answer
end
end
end
def get_user_verification(prompt)
print "#{prompt} (y/n): "
while true
answer = STDIN.gets.strip.downcase
case answer
when /(y|yes)/
return true
when /(n|no)/
return false
else
puts "#{answer} is not a valid option. Please try again"
end
end
end
def get_integer_response(prompt)
print "#{prompt}: "
while true
answer = STDIN.gets.strip
if not answer =~ /^[[:digit:]]+$/
puts "#{answer} is not a valid number. Please try again"
continue
end
answer = answer.to_i
if get_user_verification "Is #{answer} correct?"
return answer
end
end
end
end
end

61
lib/rubyqa/manager.rb
View File

@ -1,61 +0,0 @@
module RubyQA
class Manager
attr_reader :resources, :hosts
@@tests = Array.new
@@hosts = Array.new
class << self
def tests
@@tests
end
def hosts
@@hosts
end
def add_host(host)
@@hosts << host
register_resources(host)
update_resources(host)
end
def register_resources (host)
Resource.all_resources.each do |resource|
host.add_resource(resource)
end
end
def update_resources (host)
host.update_resources
end
def new_test(name, options={}, &test_proc)
if not options.has_key? :disabled
@@tests << Test.new(name, **options, &test_proc)
elsif options[:disabled] != true
@@tests << Test.new(name, **options, &test_proc)
end
end
def run_tests
@@hosts.each do |host|
tests = @@tests.select{|test| test.valid_host(host)}
tests.each do |test|
test.run(host)
end
end
end
def report (template = nil)
if not template.nil?
template.result(binding)
else
render
end
end
end
end
RubyQA::Template::BASE_REPORT_TEMPLATE.def_method(RubyQA::Manager.class, 'render')
end

22
lib/rubyqa/overrides.rb
View File

@ -1,22 +0,0 @@
module RubyQA
class ::String
def red () "\e[0;31m#{self}\e[0m" end
def red_background ()"\e[0;41m#{self}\e[0m" end
def green () "\e[0;32m#{self}\e[0m" end
def green_background ()"\e[0;42m#{self}\e[0m" end
def yellow () "\e[0;33m#{self}\e[0m" end
def yellow_background ()"\e[0;43m#{self}\e[0m" end
def blue () "\e[0;34m#{self}\e[0m" end
def blue_background ()"\e[0;44m#{self}\e[0m" end
def magenta () "\e[0;35m#{self}\e[0m" end
def magenta_background ()"\e[0;45m#{self}\e[0m" end
def cyan () "\e[0;36m#{self}\e[0m" end
def cyan_background ()"\e[0;46m#{self}\e[0m" end
def white () "\e[0;37m#{self}\e[0m" end
def white_background ()"\e[0;47m#{self}\e[0m" end
def blinking () "\e[5m#{self}\e[25m" end
def hidden () "\e[8m#{self}\e[28m" end
def bold ()"\e[1m#{self}\e[22m" end
def underline () "\e[4m#{self}\e[24m" end
end
end

109
lib/rubyqa/resource.rb
View File

@ -1,109 +0,0 @@
module RubyQA
class Resource
# This is responsible for data gathering & parsing to provide the test-writers direct access to
# resource information without needing to manually specify them in their code.
attr_reader :name, :data
# REQUIREMENTS
# This is a Hash that is meant to serve as a way of determining if a resource needs to be
# added to a host. If a key is defined here it will be checked against the information
# configured on the host. If it does not find matching keys it will not be added
#
# If empty it will just be assumed that this resource can be added to the host.
#
# This MUST be defined in each Resourced subclass, as otherwise manager will error when
# adding this resource to host.
REQUIREMENTS = {}
def initialize (host)
@host = host
@data = Hash.new
@gather_command = ""
end
def [](key)
@data[key]
end
# TODO: Create validate_host method to replace REQUIREMENTS for a bit more flexiblity on
# determining if a host is valid for this resource to be added to it.
# TODO: Add a discovery method for the resource to find out if the necessary utilities or
# otherwise are on the host
def gather
# Handles gathering the information from the host. Provided the developer has provided a
# @gather_command it will run that command on the remote host & provide the output to the
# user/dev implemented parse command. This can be overwritten by the sublcass to make it all in one in cases
# where something needs to be done differently
if not @gather_command.empty?
output = @host.exec(@gather_command)
parse(output)
else
raise "@gather_command was not defined on Resource[#{@name}]"
end
end
def parse(output)
# Subclass Implemented method that is supposed to handle parsing the gathered data on the remote node.
raise "parse not yet implemented on Resource[#{@name}]"
end
def self.all_resources
## This will allow me to iterate through all subclasses without
## having to manually define them elsewhere
ObjectSpace.each_object(Class).select{|klass| klass < self}
end
end
## Gathers the facts from the remote machine via the facter utility
class Facts < Resource
require 'json'
REQUIREMENTS = {}
def initialize (host)
super host
@name = 'facts'
@gather_command = "sudo puppet facts"
end
def parse (output)
# TODO: Find a better way of doing this.
## Only reason this exists is becuase when doing the command over telnet some extra special characters get added
## to the beginning of the opening bracket. Only quick way I found to remove it is to just split, reasssign
## & join it back into a string.
if @host.client.is_a? Telnet_Runner
output_lines = output.split(/\n/)
output_lines[0]='{'
output = output_lines.join("\n")
end
@data = JSON.parse(output)
end
end
class DRDB < Resource
# Handles gathering DRBD related information from the drbdadm utility.
REQUIREMENTS = {
cluster: true
}
def initialize (host)
super host
@name = 'facts'
@gather_command = "sudo drbdadm"
end
# TODO: Add a XML parser to requires & have the parse method load the config from the `sudo drbdadm dump-xml`
# command
def parse (output)
@data = output
end
end
end

40
lib/rubyqa/templates.rb
View File

@ -1,40 +0,0 @@
module RubyQA
module Template
require 'erb'
BASE_REPORT_TEMPLATE=ERB.new(<<ERB, trim_mode: '-')
RubyQA Test Report
QA Date: <%= Time.now.strftime("%Y/%m/%d %l:%M:%S %p %Z") %>
QA Runner: <%= ENV['USER'] %>
QA Tests: <%= Manager.tests.count %>
<% Manager.tests.each do |test| -%>
- "<%= test.name %>"
<% end -%>
QA Hosts:
<% Manager.hosts.each do |host| -%>
- <%= host.name %> (<%= host[:ip] %>)
<% end -%>
###############
# Tests Begin #
###############
<% Manager.tests.each do |test| -%>
<%= test.report %>
<% end -%>
ERB
BASE_TEST_TEMPLATE = ERB.new(<<EOF, :trim_mode => '-')
==============================================
Test : "<%= @name %>"
<%- if not @description.empty? -%>
Description : "<%= @description %>"
<% end -%>
==============================================
<% @tests.each do |hostname,data| -%>
<%= hostname %> : <%= data[:status] %> <% if data[:context].has_key? :note %> (<%= data[:context][:note] %>) <% end %>
<% end -%>
EOF
end
end

466
lib/rubyqa/tests/avo.rb
View File

@ -1,466 +0,0 @@
require 'rubyqa'
include RubyQA
require 'time'
require 'English'
info_template = ERB.new <<EOF, trim_mode: '-'
==============================================
Test : "<%= @name %>"
==============================================
<% @tests.each do |hostname,data| -%>
<%= hostname %>
----------------------------------------------
Serial: <%= data[:context][:serial] %>
System Date: <%= data[:context][:date] %>
HWCLOCK: <%= data[:context][:hwclock] %>
Timezone: <%= data[:context][:timezone] %>
MM: <%= data[:context][:maintenance_mode] %>
Standby: <%= data[:context][:standby] %>
Gold-Images:
<% data[:context][:gold_images].each do |image| -%>
- <%= image %>
<% end -%>
Full-Backups:
<% data[:context][:full_backups].each do |image| -%>
- <%= image %>
<% end -%>
DRBD Syncs:
<% data[:context][:drbd_syncs].each do |sync_name, sync_data| -%>
- <%= sync_name %> (Sync Status: <%= sync_data['disk_state'] %>)
<% end -%>
<% end -%>
EOF
Manager.new_test("Get Information", template: info_template){|host,context|
context[:serial] = host.resources['facts']['values']['dmi']['product']['serial_number'].downcase
context[:date] = Time.parse(host.exec('date').strip)
context[:hwclock] = Time.parse(host.exec('sudo hwclock').strip)
context[:timezone] = host.resources['facts']['values']['timezone']
context[:gold_images] = host.exec('sudo ls -1 /mnt/gold-images/ | grep tar.gz').split(/\n/).map(&:strip).reject(&:empty?)
context[:full_backups] = host.exec('sudo ls -1 /mnt/kvm_backups/ | grep tar.gz').split(/\n/).map(&:strip).reject(&:empty?)
context[:drbd_syncs] = host.resources['facts']['values']['drbd_status'].sort_by{|name,data| name}
context[:maintenance_mode] = host.exec("sudo pcs property show maintenance-mode | awk '/maint/ { print $2 }'").strip
context[:standby] = host.resources['facts']['values']['corosync']['is_standby'] ? "true" : "false"
}
network_template = ERB.new <<EOF, trim_mode: '<>-'
==============================================
Test : "<%= @name %>"
==============================================
<% @tests.each do |hostname, data| -%>
<%= hostname %>:
----------------------------------------------
<% data[:context][:interfaces].each do |name,info| -%>
<%= name %> <% if info.has_key? :mode %>(<%= info[:mode] %>)<% end %>:
<% if info[:mode] == "static" -%>
<% if info.has_key? :address -%>
IP: <%= info[:address] %>
<% end -%>
<% if info.has_key? :gateway -%>
Gateway: <%= info[:gateway] %>
<% end -%>
<% if info.has_key? :netmask -%>
Netmask: <%= info[:netmask] %>
<% end -%>
<% if info.has_key? :dns -%>
DNS: <%= info[:dns] %>
<% end -%>
<% end -%>
<% end -%>
<% end -%>
EOF
Manager.new_test("Networking Configurations Host", template: network_template) {|host,context|
contents = host.exec('cat /etc/network/interfaces')
interfaces = Hash.new
interface_regex = /iface (?<name>\S+) inet (?<mode>\S+)/
address_regex = /address (?<address>\S+)/
netmask_regex = /netmask (?<netmask>\S+)/
gateway_regex = /gateway (?<gateway>\S+)/
current_interface = ""
contents.split(/\n/).each do |line|
case line
when interface_regex
current_interface = $LAST_MATCH_INFO['name']
mode = $LAST_MATCH_INFO['mode']
interfaces[current_interface]=Hash.new
interfaces[current_interface][:mode] = mode
when address_regex
interfaces[current_interface][:address] = $LAST_MATCH_INFO['address']
when netmask_regex
interfaces[current_interface][:netmask] = $LAST_MATCH_INFO['netmask']
when gateway_regex
interfaces[current_interface][:gateway] = $LAST_MATCH_INFO['gateway']
end
end
contents = host.exec('sudo ipmitool lan print')
current_interface="IPMI"
interfaces[current_interface]=Hash.new
mode_regex = /IP Address Source[[:space:]]+:[[:space:]](?<mode>\S+)/
address_regex = /IP Address[[:space:]]+:[[:space:]]+(?<address>\S+)/
netmask_regex = /Subnet Mask[[:space:]]+:[[:space:]]+(?<netmask>\S+)/
gateway_regex = /Default Gateway IP[[:space:]]+:[[:space:]]+(?<gateway>\S+)/
contents.split(/\n/).each do |line|
case line
when mode_regex
interfaces[current_interface][:mode] = $LAST_MATCH_INFO['mode'].downcase
when address_regex
interfaces[current_interface][:address] = $LAST_MATCH_INFO['address']
when netmask_regex
interfaces[current_interface][:netmask] = $LAST_MATCH_INFO['netmask']
when gateway_regex
interfaces[current_interface][:gateway] = $LAST_MATCH_INFO['gateway']
end
end
ipconfig_script=<<EOF
ipconfig(){
local KVM=${1:?"$FUNCNAME: No VM provided"}
local -i inc=0
local output=""
handle_return=$(sudo virsh qemu-agent-command "$KVM" '{"execute":"guest-exec", "arguments":{"path":"cmd.exe","arg":["/c","ipconfig /all"],"capture-output":true}}'| grep -o '[[:digit:]]*')
until [[ $output = *\\"exited\\":true* || "$inc" -ge "42" ]] ; do
output=$(sudo virsh qemu-agent-command "$KVM" '{"execute":"guest-exec-status","arguments":{"pid":'${handle_return}'}}')
sleep .1 ; ((inc++))
done
cut -d '"' -f8 <<< "$output" | base64 -d
}
EOF
get_vm_info_script=<<EOF
get_vm_info () {
local awk_script='
BEGIN { ip=""; subnet=""; dns=""; gateway=""; FS=":" }
/IPv4 Address/ {gsub(/[[:space:]]/,"", $2); gsub("[(]Preferred[)]", "", $2); ip = $2 }
/Subnet Mask/ { gsub(/[[:space:]]/,"", $2); subnet = $2 }
/Gateway/ { gsub(/[[:space:]]/,"", $2); gateway = $2 }
/DNS Servers/ { gsub(/[[:space:]]/,"", $2); dns=$2}
/^[[:space:]]+[0-9]/ { gsub(/[[:space:]]/, "", $1); dns = dns "," $1 }
/NetBIOS/ { print name"|"ip"|"subnet"|"gateway"|"dns; name=""; ip=""; subnet=""; dns=""; gateway=""; }
'
vms=( $( sudo virsh list --name | sort -V ) )
if [[ ${#vms[@]} -gt 0 ]]; then
for VM in ${vms[@]}; do
ipconfig $VM | awk -v name=$VM "$awk_script"
done
fi
}
EOF
info = host.exec([get_vm_info_script, ipconfig_script, "get_vm_info"].join("\n")).split(/\n/)
info.select{|line| line =~ /.*\|.*/}.each do |line|
name, ip, subnet, gateway, dns = line.split("|")
interfaces[name] = Hash.new
interfaces[name][:gateway] = gateway
interfaces[name][:netmask] = subnet
interfaces[name][:address] = ip
interfaces[name][:dns] = dns
interfaces[name][:mode] = "static"
end
context[:interfaces]=interfaces
}
Manager.new_test("OpenVPN Certs Deployed to Node") {|host, context|
pass = true
buildserver_client_cert_exist = `sudo lxc-attach -n avobuildaus -- bash -c '[[ -f /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.crt ]] && echo "yes" || echo "no"'`.strip
cert_exist = (buildserver_client_cert_exist =~ /yes/ ) ? true : false
buildserver_client_key_exist = `sudo lxc-attach -n avobuildaus -- bash -c '[[ -f /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.key ]] && echo "yes" || echo "no"'`.strip
key_exist = (buildserver_client_key_exist =~ /yes/ ) ? true : false
if not cert_exist or not key_exist
context[:note] = "Nodes Cert and/or key were not found on buildserver, inform ENG that certs haven't been generated for #{host.name}"
pass = false
else
openvpn_client_cert_md5 = host.exec("sudo md5sum /etc/openvpn/mgmt/easy-rsa/keys/client.crt | awk '{ print $1 }'").strip
openvpn_client_key_md5 = host.exec("sudo md5sum /etc/openvpn/mgmt/easy-rsa/keys/client.key | awk '{ print $1 }'").strip
buildserver_client_cert_md5 = `sudo lxc-attach -n avobuildaus -- md5sum /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.crt | awk '{ print $1 }'`.strip
buildserver_client_key_md5 = `sudo lxc-attach -n avobuildaus -- md5sum /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.key | awk '{ print $1 }'`.strip
md5_fails = Array.new
if not openvpn_client_cert_md5 == buildserver_client_cert_md5
md5_fails << "cert"
end
if not openvpn_client_key_md5 == buildserver_client_key_md5
md5_fails << "key"
end
if md5_fails.count != 0
pass = false
context[:note] = "Failed: #{md5_fails.join(',')}"
end
end
pass
}
NETMASK_32="255.255.255.255"
FILTER_INTERFACES=/eno(0|1|2|6)/
Manager.new_test("No /32 addresses") {|host, context|
pass = true
facts = host.resources['facts'].data
interfaces = facts['values']['networking']['interfaces'].select{|name,val| name =~ FILTER_INTERFACES}
failed_interfaces = Array.new
interfaces.each do |name,data|
if data["bindings"].is_a? Array
data["bindings"].each do |binding|
if binding['netmask'] == NETMASK_32
pass = false
puts "#{host.name}: Was found to have a /32 netmask on #{name} with ip #{binding['address']}"
if ! failed_interfaces.include? name
failed_interfaces << name
end
end
end
elsif data['bindings'].is_a? Hash
if data['bindings']['netmask'] == NETMASK_32
pass = false
puts "#{host.name}: Was found to have a /32 netmask on #{name} with ip #{data['address']}"
failed_interfaces << name
end
end
end
if ! failed_interfaces.empty?
context[:note] = "failed: #{failed_interfaces.join(',')}"
end
pass
}
Manager.new_test("Puppet Env is Pilot"){ |host|
host.exec("cat /etc/puppetlabs/puppet/puppet.conf") =~ /environment = pilot/
}
NETMASK_30="255.255.255.252"
RED1_ENO6="172.16.16.2"
RED0_ENO6="172.16.16.1"
Manager.new_test("DRBD Interfaces Configured Correctly"){|host|
facts = host.resources['facts'].data
networking = facts['values']['networking']
binding = networking['interfaces']['eno6']['bindings'].first
address = binding['address']
netmask = binding['netmask']
if host.name =~ /red1/
address == RED1_ENO6 && netmask == NETMASK_30
elsif host.name =~ /red0/
address == RED0_ENO6 && netmask == NETMASK_30
else
false
end
}
VM_NAME_REGEX = /(?<name>[A-Z]{3}[0-9]+).xml/
CACHE_MODE_REGEX = /cache=\'(?<mode>\S+)\'/
Manager.new_test("Check VM disk cache is writeback"){|host, context|
pass = true
failed_vms = Array.new
host.exec("sudo grep cache /etc/libvirt/qemu/*.xml").split(/\n/).each do |line|
name = VM_NAME_REGEX.match(line)['name']
mode = CACHE_MODE_REGEX.match(line)['mode']
if mode != "writeback"
pass = false
failed_vms << name
end
end
if ! failed_vms.empty?
context[:note] = "failed vms: #{failed_vms.join(",")}"
end
pass
}
Manager.new_test("Manual RVC Patch Applied to domain.rb"){|host, context|
pass = true
pass = host.exec("sudo grep Log.error /opt/rvc/lib/rvc/domain.rb") =~ /Log.error[[:space:]]*\{[[:space:]]*e(\.message)?[[:space:]]*\}/
if not pass
context[:note] = "Manual Patch not applied (sudo sed -i 's/Log.error(e)/Log.error { e }/g' /opt/rvc/lib/rvc/domain.rb)"
end
pass
}
Manager.new_test("Test IPMI Credentials on self (over network)") {|host,context|
fencenode=host.resources['facts']['values']['pcs_resources']["fence_#{host.name}"]['instance_attributes']
pass = true
test_result = host.exec("set +o history; ipmitool -I lanplus -H #{fencenode["ipaddr"]} -U #{fencenode['userid']} -P #{fencenode['passwd']} chassis status >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
case test_result
when 'yes'
pass = true
when 'no'
pass = false
end
if pass == false
ping_test = host.exec("ping -c 2 #{fencenode["ipaddr"]} >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
case ping_test
when 'yes'
context[:note] = "IPMI Pingable, Creds Failed"
when 'no'
context[:note] = "IPMI Not Pingable. Possibly unplugged"
end
end
pass
}
Manager.new_test("Test IPMI Credentials on partner") {|host,context|
if host.name =~ /red1/
hostname = host.name.sub('red1','red0')
elsif host.name =~ /red0/
hostname = host.name.sub('red0','red1')
end
pass = true
begin
fencenode=host.resources['facts']['values']['pcs_resources']["fence_#{hostname}"]['instance_attributes']
rescue
puts "Fact retrieval failed on #{host.name}. Failing test"
return false
end
test_result = host.exec("set +o history; ipmitool -I lanplus -H #{fencenode["ipaddr"]} -U #{fencenode['userid']} -P #{fencenode['passwd']} chassis status >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
case test_result
when 'yes'
pass = true
when 'no'
pass = false
end
if pass == false
ping_test = host.exec("ping -c 2 #{fencenode["ipaddr"]} >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
case ping_test
when 'yes'
context[:note] = "Other Node IPMI is pingable, Creds Failed"
when 'no'
context[:note] = "Other Node IPMI is not Pingable. Possibly unplugged"
end
end
pass
}
Manager.new_test("No Cluster Irregularities", description: "Nothing Disabled that shouldn't be"){|node,context|
pass = true
disabled_items = Array.new
## Stonith
## .values.pcs_resources.fence_avo00000223red0.meta_attributes["target-role"]
begin
node_stonith_state = node.resources['facts']['values']['pcs_resources']["fence_#{node.name}"]['meta_attributes']['target-role']
rescue
puts "Fact retrieval failed on #{host.name}. Failing test"
return false
end
if node_stonith_state == "Stopped"
disabled_items << "Stonith"
end
if disabled_items.count !=0
context[:note] = "Disabled Items: #{disabled_items.join(',')}"
pass = false
end
pass
}
Manager.new_test("Puppet Agent disabled (pre-ship only)"){|node,context|
pass = true
ssl_dir_exists = node.exec('[[ -f $( sudo puppet config print agent_disabled_lockfile ) ]] && echo "yes" || echo "no"')
case ssl_dir_exists
when /no/
pass = false
context[:note] = "Before shipping node you need to disable the puppet agent on #{node.name} (sudo puppet agent --disable)"
end
pass
}
Manager.new_test("SSL Directory Cleaned (pre-ship only)"){|node,context|
pass = true
ssl_dir_exists = node.exec('[[ -d /etc/puppetlabs/puppet/ssl ]] && echo "yes" || echo "no"')
case ssl_dir_exists
when /yes/
pass = false
context[:note] = "Before shipping node you need to delete the ssl dir on #{node.name} (sudo rm -rf /etc/puppetlabs/puppet/ssl)"
end
pass
}
Manager.new_test("Ensure VMs not running (pre-ship only)"){|host,context|
pass = true
vms = host.exec("sudo virsh list --name").split(/\n/).map(&:strip).sort!
if vms.count > 0
pass = false
context[:note] = "VMs are still running: #{vms.join(',')} shut them down before shipping (for VM in #{vms.join(' ')}; do sudo rvc kvm shutdown $VM; done"
end
pass
}
Manager.new_test("Cluster in Pre-Ship Condition (pre-ship only)"){|host,context|
pass = true
maintenance_mode = host.exec("sudo pcs property show maintenance-mode | awk '/maint/ { print $2 }'").strip
standby = host.resources['facts']['values']['corosync']['is_standby'] ? "true" : "false"
case host.name
when /red1/
if maintenance_mode =~ /false/ and standby =~ /false/
context[:note] = "Node is not in standby and cluster is not in maintenance-mode"
pass = false
elsif maintenance_mode =~ /false/ and standby =~ /true/
context[:note] = "Cluster isn't in maintenance-mode"
pass = false
elsif maintenance_mode =~ /true/ and standby =~ /false/
context[:note] = "Node isn't in standby"
pass = false
end
when /red0/
if maintenance_mode =~ /false/
context[:note] = "Cluster isn't in maintenance-mode"
pass = false
end
end
pass
}

166
lib/rubyqa/tests/gpc.rb
View File

@ -1,166 +0,0 @@
#!/usr/bin/ruby
require 'rubyqa'
require 'erb'
include RubyQA
info_template = ERB.new <<EOF, trim_mode: '-'
==============================================
Test : <%= @name %>
==============================================
<% @tests.each do |hostname,data| -%>
<%= hostname %>
----------------------------------------------
Serials: <%= data[:context][:serial] %>
System Date: <%= data[:context][:date] %>
HWCLOCK: <%= data[:context][:hwclock] %>
Timezone: <%= data[:context][:timezone] %>
eSocket Status: <%= data[:context][:esocket_status] %>
eSocket Patch: <%= data[:context][:esocket_patch] %>
keystore md5s:
<% data[:context][:keystore_md5s].each do |line| -%>
- <%= line %>
<% end -%>
<% end -%>
EOF
Manager.new_test("Get Information", template: info_template){|host,context|
serial_matcher = /\"(\S+)\"/
serials = host.exec("sudo facter 2>/dev/null | grep serial | cut -d '>' -f 2 | tr -d ','")
context[:serial] = serials.scan(serial_matcher).map{|serial| "\"#{serial.first}\""}.join(',')
context[:date] = Time.parse(host.exec('date').strip)
context[:hwclock] = Time.parse(host.exec('sudo hwclock').strip)
context[:timezone] = host.exec('readlink -f /etc/localtime').strip.gsub(/(\/usr\/share|\/etc)\/zoneinfo\//,'')
context[:esocket_status] = host.exec('sudo lxc-attach -n ${HOSTNAME/red?/aci0} -- systemctl is-active esocket').strip
context[:esocket_patch] = host.exec('sudo lxc-attach -n ${HOSTNAME/red?/aci0} -- ls -1 /home/esocket/eSocket.POS').split(/\n/).select{|line| line =~ /patch/}.last
context[:keystore_md5s] = host.exec('sudo lxc-attach -n ${HOSTNAME/red?/aci0} -- md5sum /home/esocket/eSocket.POS/keystore/*').split(/\n/).map(&:strip)
}
firewall_template = ERB.new <<EOF, trim_mode: '-'
==============================================
Test : <%= @name %>
==============================================
<% @tests.each do |hostname,data| -%>
<%= hostname %>
----------------------------------------------
<% data[:context][:chains].each do |name, info| -%>
Chain <%= name %>
<% info.each do |rule| -%>
<%= rule[:line] %>
<% end -%>
<% end -%>
<% end -%>
EOF
filter_regex = /^target/
chain_regex = /Chain (?<name>\S+)/
rule_regex = /(?<target>\S+)[[:space:]]+(?<protocols>\S+)[[:space:]]+(?<opts>\S+)[[:space:]]+(?<source>\S+)[[:space:]]+(?<destination>\S+)/
Manager.new_test("Firewall Forward Rules", template: firewall_template) {|host,context|
host_firewall_rules = host.exec("sudo itptables -L FORWARD")
context[:chains]=Hash.new
current_chain = ""
host_firewall_rules.split(/\n/).each do |line|
case line
when chain_regex
info = chain_regex.match(line)
current_chain = info['name']
context[:chains][current_chain] = Array.new
when filter_regex
next
when rule_regex
rule = Hash.new
info = rule_regex.match(line)
rule[:target]=info['target']
rule[:source]=info['source']
rule[:destination]=info['destination']
rule[:protocols]=info['protocols']
rule[:line]=line
context[:chains][current_chain] << rule
end
end
}
Manager.new_test("Check if hostname is correctly configured") {|host,context|
pass = true
name = host.name
hosts = host.exec("cat /etc/hosts")
hostname = host.exec("cat /etc/hostname").strip
not_configured_in = Array.new
if not hosts =~ /127\.0\.0\.1.*#{name}/
pass = false
not_configured_in << "/etc/hosts"
end
if not hostname =~ /#{name}/
pass = false
not_configured_in << "/etc/hostname"
end
if not_configured_in.count > 0
context[:note] = "Hostname was not configured in #{not_configured_in.join(',')}"
end
pass
}
Manager.new_test("Puppet Certs Generated") {|host|
}
Manager.new_test("OpenVPN Certs Generated") {|host, context|
listing = host.exec('ls -1 /etc/openvpn/mgmt/easy-rsa/keys/')
files_not_generated = Array.new
pass = true
if ! listing =~ /my.crt/
pass = false
files_not_generated << 'my.crt'
end
if ! listing =~ /my.key/
pass = false
files_not_generated << 'my.key'
end
if not pass
context[:note] = "Files not created: #{files_not_generated.join(',')}"
end
pass
}
Manager.new_test("Ensure netcat installed on aci, prx & host"){|host, context|
pass = true
aci_listing = host.exec("sudo lxc-attach -n ${HOSTNAME/red?/aci0} -- apt list --installed")
prx_listing = host.exec("sudo lxc-attach -n ${HOSTNAME/red?/prx0} -- apt list --installed")
host_listing = host.exec("sudo apt list --installed")
not_installed_on = Array.new
if not aci_listing =~ /netcat/
not_installed_on << "aci"
end
if not prx_listing =~ /netcat/
not_installed_on << "prx"
end
if not host_listing =~ /netcat/
not_installed_on << "host"
end
if not_installed_on.count > 0
pass = false
context[:note] = "netcat not installed on #{not_installed_on.join(',')}"
end
pass
}

56
lib/rubyqa/runner.rb → lib/runner.rb
View File

@ -1,5 +1,4 @@
module RubyQA
require 'base64'
class CommandRunner
# Handles creating an interface for running arbitrary commands on a host/target
#
@ -40,17 +39,24 @@ module RubyQA
if @data[:password].empty?
raise "Password was not provided for host"
end
encoded_command = Base64.encode(command)
sudo_command = <<EOT
echo "#{@data[:password]}" | sudo -S bash -c "echo \"#{encoded_command}\" | base64 -d | bash"
new_command = <<EOT
if [[ $UID -ne 0 ]]; then
command_file=/tmp/sudo-command-$(date +%Y-%m-%d_%H-%M)
cat >$command_file <<EOF
#{command}
EOF
echo "#{@data[:password]}" | sudo -S /bin/bash $command_file 2>/dev/null
rm $command_file
else
#{command}
fi
EOT
run_command(sudo_command)
run_command(new_command)
end
end
class SSH_Runner < CommandRunner
require 'net/ssh'
def initialize_runner
begin
if @data[:password]
@ -67,44 +73,6 @@ EOT
def run_command(command)
@client.exec!(command).strip
end
end
class Telnet_Runner < CommandRunner
require 'net/ssh/telnet'
LOGIN_HANDLER=nil
TELNET_OPTS={
"Timeout" => 10,
"Prompt" => /[$:#]/
}
def initialize_runner
begin
@client = Net::SSH::Telnet.new( "Host" => @data[:ip], "Username" => @data[:user], "Password" => @data[:password])
# Since this is telnet, it doesn't filter out command sequences, so we have to set the PS1/prompt to something
# that is easily matched for filtering purposes
@client.cmd({"String" => "PS1='<filter_line>'; PROMPT_COMMANDS='';", "Match" => /filter_line/})
rescue SocketError
raise "Failed to make ssh client for #{@data[:hostname]}"
end
end
def run_command(command)
output = ""
## Doing intitial command (doesn't send white-space/newline)
lines = Array.new
filter_regex=/(^|#{command}|^\n$)/
output = @client.cmd({"String" => command, "Waittime" => 100, "Match" => /filter_line/})
# Have to clean output since telnet session will return all output, both stdin & stdout.
output = output.split(/\n/).reject do |line|
line.match?(/filter_line/) || line.match(/#{command}/)
end.join("\n")
output
end
end
end

31
lib/rubyqa/test.rb → lib/test.rb
View File

@ -48,9 +48,9 @@ module RubyQA
test_status = "N/A"
context = Hash.new
if @test.call(host, context)
test_status = "PASSED".green
test_status = "PASSED"
else
test_status = "FAILED".red
test_status = "FAILED"
end
@tests[host.name]={
:host => host,
@ -63,15 +63,28 @@ module RubyQA
if @tests.count == 0
return
end
template = nil
if @template.nil?
self.render
template = ERB.new <<EOF, :trim_mode => '-'
==============================================
Test : "<%= @name %>"
<% if not @description.empty? -%>
Description : "<%= @description %>
<% end -%>
==============================================
<% @tests.each do |hostname,data| -%>
<%= hostname %> : <%= data[:status] %> <% if data[:context].has_key? :note %> (<%= data[:context][:note] %>) <% end %>
<% end -%>
EOF
else
@template.result(binding)
end
end
end
RubyQA::Template::BASE_TEST_TEMPLATE.def_method(RubyQA::Test, 'render')
template = @template
end
template.result(binding)
end
end
end

13
rubyqa.gemspec
View File

@ -1,17 +1,10 @@
require 'find'
Gem::Specification.new do |s|
s.name = "RubyQA"
s.version = '0.1.0'
s.summary = 'A Ruby framework for QA-ing *NIX system builds'
s.summary = 'A Ruby framework for QA *NIX system builds'
s.description = 'A Ruby based framework defining & orchestrating tests against a host or hosts'
s.authors = ['Tristan Ancelet']
s.email = 'tristanancelet@yahoo.com'
s.homepage = 'https://git.arcanium.tech/tristan/RubyQA'
s.executables += [ 'generate-qa','impromptu-qa']
s.files = Find::find('lib').select{|file| file =~ /rb/}
s.license = 'MIT'
s.required_ruby_version = ">= 2.7.0"
s.add_dependency 'net-ssh', '~> 7.1', '>= 7.1.0'
s.add_dependency 'net-ssh-telnet', '~> 0.3', '>= 0.3.0'
s.homepatge = 'https://git.arcanium.tech/tristan/RubyQA'
s.files = Dir.glob('lib/*.rb')
end

10
templates/base.erb Executable file → Normal file
View File

@ -1,6 +1,6 @@
RubyQA Test Report
QA Date: <%= Time.now.strftime("%Y/%m/%d %l:%M:%S %p %Z") %>
QA Date: <%= Time.now %>
QA Runner: <%= ENV['USER'] %>
QA Tests: <%= Manager.tests.count %>
<% Manager.tests.each do |test| -%>
@ -8,13 +8,11 @@ QA Tests: <%= Manager.tests.count %>
<% end -%>
QA Hosts:
<% Manager.hosts.each do |host| -%>
- <%= host.name %> (<%= host[:ip] %>)
- <%= host.name %> (<%= host.data[:ip] %>)
<% end -%>
###############
# Tests Begin #
###############
Tests Begin
-----------
<% Manager.tests.each do |test| -%>
<%= test.report %>
<% end -%>