New changes

.gitignore

@@ -0,0 +1,9 @@
+*
+!.gitignore
+!lib
+!lib/**
+!rubyqa.gemspec
+!templates
+!templates/*
+!design.adoc
+!README.md

lib/manager.rb

@@ -1,52 +0,0 @@
-module RubyQA
-  class Manager
-    attr_reader :resources, :hosts
-    @@tests = Array.new
-    @@hosts = Array.new
-
-    def self.tests
-      @@tests
-    end
-
-    def self.hosts
-      @@hosts
-    end
-
-    def self.add_host(host)
-      @@hosts << host
-      register_resources(host)
-      update_resources(host)
-    end
-
-    def self.register_resources (host)
-      Resource.all_resources.each do |resource|
-        host.add_resource(resource)
-      end
-    end
-
-    def self.update_resources (host)
-      host.update_resources
-    end
-
-    def self.new_test(name, options={}, &test_proc)
-      @@tests << Test.new(name, **options, &test_proc)
-    end
-
-    def self.run_tests
-      @@hosts.each do |host|
-        tests = @@tests.select{|test| test.valid_host(host)}
-        tests.each do |test|
-          test.run(host)
-        end
-      end
-    end
-
-    def self.report (template = nil)
-      if template
-        puts template.result(binding)
-      else
-        puts @@tests.map(&:report).join("\n")
-      end
-    end
-  end
-end

lib/rubyqa.rb

@@ -1,7 +1,11 @@
 #!/usr/bin/env ruby
 
-$LOAD_PATH << __dir__
-require 'manager'
-require 'resource'
-require 'host'
-require 'test'
+require_relative 'rubyqa/overrides'
+require_relative 'rubyqa/templates/report'
+require_relative 'rubyqa/templates/test'
+require_relative 'rubyqa/interaction'
+require_relative 'rubyqa/manager'
+require_relative 'rubyqa/resource'
+require_relative 'rubyqa/runner'
+require_relative 'rubyqa/host'
+require_relative 'rubyqa/test'

lib/rubyqa/host.rb

@@ -1,5 +1,4 @@
 module RubyQA
-  require 'net/ssh'
   class Host
     DEFAULT_HOSTDATA={
       :hostname      => "",
@@ -13,7 +12,6 @@ module RubyQA
       :runner        => SSH_Runner
     }
 
-    attr_accessor *DEFAULT_HOSTDATA.keys
     attr_reader :client, :data, :resources
 
     def initialize( data = {} )
@@ -21,29 +19,36 @@ module RubyQA
 
       @data = DEFAULT_HOSTDATA.merge data
 
+      if not @data[:hostname].empty? and @data[:site].empty?
+        data[:site]=@data[:hostname][0,3]
+      end
+
       ## initialize client for usage tests
       @client = @data[:runner].new(@data)
     end
 
     def exec (command)
+      ## Send a command from the host
       @client.exec(command)
     end
 
     def exec_sudo (command)
+      ## Send a sudo command from the host
       @client.exec_sudo(command)
     end
 
     def [](key)
+      ## Provide Access to data (:ip, etc)
       @data[key]
     end
 
     def name
       if @data[:hostname] != ""
-        return @data[:hostname] 
-      elsif @resources['facts'].data['networking']['hostname'] != ""
-        return @resources['facts'].data['networking']['hostname']
+        @data[:hostname] 
+      elsif @resources['facts'].data['hostname'] != ""
+        @resources['facts'].data['hostname']
       else
-        return @data[:ip]
+        self.exec('hostname')
       end
     end
 

lib/rubyqa/interaction.rb

@@ -0,0 +1,51 @@
+module RubyQA
+  module Interactions
+
+    def get_string_response(prompt)
+      puts "#{prompt}: "
+
+      while true
+        answer = STDIN.gets.strip
+
+        if get_user_verification "Is \"#{answer}\" correct?"
+          return answer
+        end
+      end
+    end
+
+    def get_user_verification(prompt)
+      print "#{prompt} (y/n): "
+      while true
+        answer = STDIN.gets.strip.downcase
+
+        case answer
+          when /(y|yes)/
+            return true
+          when /(n|no)/
+            return false
+          else
+            puts "#{answer} is not a valid option. Please try again"
+        end
+      end
+    end
+
+    def get_integer_response(prompt)
+      print "#{prompt}: "
+      while true
+        answer = STDIN.gets.strip
+
+        if not answer =~ /^[[:digit:]]+$/
+          puts "#{answer} is not a valid number. Please try again"
+          continue
+        end
+
+        answer = answer.to_i
+
+        if get_user_verification "Is #{answer} correct?"
+          return answer
+        end
+      end
+    end
+
+  end
+end

lib/rubyqa/manager.rb

@@ -0,0 +1,61 @@
+module RubyQA
+
+  class Manager
+    attr_reader :resources, :hosts
+    @@tests = Array.new
+    @@hosts = Array.new
+
+    class << self
+      def tests
+        @@tests
+      end
+
+      def hosts
+        @@hosts
+      end
+
+      def add_host(host)
+        @@hosts << host
+        register_resources(host)
+        update_resources(host)
+      end
+
+      def register_resources (host)
+        Resource.all_resources.each do |resource|
+          host.add_resource(resource)
+        end
+      end
+
+      def update_resources (host)
+        host.update_resources
+      end
+
+      def new_test(name, options={}, &test_proc)
+        if not options.has_key? :disabled 
+          @@tests << Test.new(name, **options, &test_proc)
+        elsif options[:disabled] != true
+          @@tests << Test.new(name, **options, &test_proc)
+        end
+      end
+
+      def run_tests
+        @@hosts.each do |host|
+          tests = @@tests.select{|test| test.valid_host(host)}
+          tests.each do |test|
+            test.run(host)
+          end
+        end
+      end
+
+      def report (template = nil)
+        if template
+          template.result(binding)
+        else
+          @@tests.map(&:report).join("\n")
+        end
+      end
+    end
+  end
+  RubyQA::Template::BASE_REPORT_TEMPLATE.def_method(RubyQA::Manager, 'render')
+
+end

lib/rubyqa/overrides.rb

@@ -0,0 +1,22 @@
+module RubyQA::Overrides
+  class ::String
+    def red () "\e[0;31m#{self}\e[0m" end
+    def red_background ()"\e[0;41m#{self}\e[0m" end
+    def green () "\e[0;32m#{self}\e[0m" end
+    def green_background ()"\e[0;42m#{self}\e[0m" end
+    def yellow () "\e[0;33m#{self}\e[0m" end
+    def yellow_background ()"\e[0;43m#{self}\e[0m" end
+    def blue () "\e[0;34m#{self}\e[0m" end
+    def blue_background ()"\e[0;44m#{self}\e[0m" end
+    def magenta () "\e[0;35m#{self}\e[0m" end
+    def magenta_background ()"\e[0;45m#{self}\e[0m" end
+    def cyan () "\e[0;36m#{self}\e[0m" end
+    def cyan_background ()"\e[0;46m#{self}\e[0m" end
+    def white () "\e[0;37m#{self}\e[0m" end
+    def white_background ()"\e[0;47m#{self}\e[0m" end
+    def blinking () "\e[5m#{self}\e[25m" end
+    def hidden () "\e[8m#{self}\e[28m" end
+    def bold ()"\e[1m#{self}\e[22m" end
+    def underline () "\e[4m#{self}\e[24m" end
+  end
+end

lib/rubyqa/resource.rb

@@ -22,6 +22,11 @@ module RubyQA
       @gather_command = ""
     end
 
+    def [](key)
+      @data[key]
+    end
+
+
     # TODO: Create validate_host method to replace REQUIREMENTS for a bit more flexiblity on 
     #       determining if a host is valid for this resource to be added to it.
 
@@ -57,16 +62,26 @@ module RubyQA
   ## Gathers the facts from the remote machine via the facter utility
   class Facts < Resource
     require 'json'
-
     REQUIREMENTS = {}
     def initialize (host)
       super host
       @name = 'facts'
-      @gather_command = "facter -j"
+      @gather_command = "sudo puppet facts"
     end
 
     def parse (output)
-      @data = JSON.load(output)
+
+      # TODO: Find a better way of doing this.
+      ## Only reason this exists is becuase when doing the command over telnet some extra special characters get added 
+      ## to the beginning of the opening bracket. Only quick way I found to remove it is to just split, reasssign 
+      ## & join it back into a string.
+      if @host.client.is_a? Telnet_Runner
+        output_lines = output.split(/\n/)
+        output_lines[0]='{'
+        output = output_lines.join("\n")
+      end
+
+      @data = JSON.parse(output)
     end
   end
 
@@ -74,7 +89,7 @@ module RubyQA
     # Handles gathering DRBD related information from the drbdadm utility. 
     
     REQUIREMENTS = {
-      :cluster => true
+      cluster: true
     }
 
     def initialize (host)
@@ -91,5 +106,4 @@ module RubyQA
     end
   end
 
-
 end

lib/rubyqa/runner.rb

@@ -1,4 +1,5 @@
 module RubyQA
+  require 'base64'
   class CommandRunner
     # Handles creating an interface for running arbitrary commands on a host/target
     #
@@ -39,24 +40,17 @@ module RubyQA
       if @data[:password].empty?
         raise "Password was not provided for host"
       end
-      new_command = <<EOT
-if [[ $UID -ne 0 ]]; then
-command_file=/tmp/sudo-command-$(date +%Y-%m-%d_%H-%M)
-cat >$command_file <<EOF
-#{command}
-EOF
-echo "#{@data[:password]}" | sudo -S /bin/bash $command_file 2>/dev/null
-rm $command_file
-else
-#{command}
-fi
+      encoded_command = Base64.encode(command)
+      sudo_command = <<EOT
+echo "#{@data[:password]}" | sudo -S bash -c "echo \"#{encoded_command}\" | base64 -d | bash" 
 EOT
-    run_command(new_command)
+      run_command(sudo_command)
     end
   end
 
 
   class SSH_Runner < CommandRunner
+    require 'net/ssh'
     def initialize_runner
       begin
         if @data[:password]
@@ -73,6 +67,44 @@ EOT
     def run_command(command)
       @client.exec!(command).strip
     end
+
+  end
+
+
+  class Telnet_Runner < CommandRunner
+    require 'net/ssh/telnet'
+    LOGIN_HANDLER=nil
+    TELNET_OPTS={
+      "Timeout" => 10,
+      "Prompt" => /[$:#]/
+    }
+    def initialize_runner
+      begin
+          @client = Net::SSH::Telnet.new( "Host" => @data[:ip], "Username" => @data[:user], "Password" => @data[:password])
+          # Since this is telnet, it doesn't filter out command sequences, so we have to set the PS1/prompt to something
+          # that is easily matched for filtering purposes
+          @client.cmd({"String" => "PS1='<filter_line>'; PROMPT_COMMANDS='';", "Match" => /filter_line/})
+      rescue SocketError
+        raise "Failed to make ssh client for #{@data[:hostname]}"
+      end
+    end
+
+    def run_command(command)
+      output = ""
+
+      ## Doing intitial command (doesn't send white-space/newline)
+      lines = Array.new
+      filter_regex=/(^|#{command}|^\n$)/
+
+      output = @client.cmd({"String" => command, "Waittime" => 100, "Match" => /filter_line/}) 
+
+      # Have to clean output since telnet session will return all output, both stdin & stdout.
+      output = output.split(/\n/).reject do |line|
+        line.match?(/filter_line/) || line.match(/#{command}/)
+      end.join("\n")
+
+      output
+    end
   end
 
 end

lib/rubyqa/templates/report.rb

@@ -0,0 +1,24 @@
+module RubyQA::Templates
+  require 'erb'
+  BASE_REPORT=ERB.new(<<ERB, trim_mode: '-')
+RubyQA Test Report
+
+QA Date: <%= Time.now.strftime("%Y/%m/%d %l:%M:%S %p %Z") %>
+QA Runner: <%= ENV['USER'] %>
+QA Tests: <%= Manager.tests.count %>
+<% Manager.tests.each  do |test| -%>
+- "<%= test.name %>"
+<% end -%>
+QA Hosts: 
+<% Manager.hosts.each do |host| -%>
+- <%= host.name %> (<%= host[:ip] %>)
+<% end -%>
+
+###############
+# Tests Begin #
+###############
+
+<% Manager.tests.each do |test| -%>
+<%= test.report %>
+<% end -%>
+ERB

lib/rubyqa/templates/test.rb

@@ -0,0 +1,12 @@
+module RubyQA::Template
+       BASE_TEST_TEMPLATE = ERB.new <<EOF, :trim_mode => '-'
+==============================================
+Test : "<%= @name %>"
+<%- if not @description.empty? -%> 
+Description : "<%= @description %>"
+<% end -%>
+==============================================
+<% @tests.each do |hostname,data| -%>
+<%= hostname %> : <%= data[:status] %> <% if data[:context].has_key? :note %> (<%= data[:context][:note] %>) <% end %>
+<% end -%>
+EOF

lib/rubyqa/test.rb

@@ -10,7 +10,7 @@ module RubyQA
 
     attr_reader :name, :options, :proc
 
-    def initialize (name, options ={}, &test_block)
+    def initialize (name, options = {}, &test_block)
       @tests = Hash.new
       @name = name
       @options = DEFAULT_OPTIONS.merge(options)
@@ -48,9 +48,9 @@ module RubyQA
       test_status = "N/A"
       context = Hash.new
       if @test.call(host, context)
-        test_status = "PASSED"
+        test_status = "PASSED".green
       else
-        test_status = "FAILED"
+        test_status = "FAILED".red
       end
       @tests[host.name]={
         :host    => host,
@@ -63,28 +63,15 @@ module RubyQA
       if @tests.count == 0
         return
       end
-      template = nil
 
       if @template.nil?
-      template = ERB.new <<EOF, :trim_mode => '-'
-==============================================
-Test : "<%= @name %>"
-<% if  not @description.empty? -%> 
-Description : "<%= @description %> 
-<% end  -%>
-==============================================
-<% @tests.each  do |hostname,data| -%>
-<%= hostname %> : <%= data[:status] %> <% if data[:context].has_key? :note %> (<%= data[:context][:note] %>) <% end %>
-<% end  -%>
-
-EOF
-
-    else 
-      template = @template
-    end
-
-    template.result(binding)
+        self.render
+      else 
+        @template.result(binding)
+      end
     end
   end
+  RubyQA::Template::BASE_TEST_TEMPLATE.def_method(RubyQA::Test, 'render')
+
 end
 

lib/rubyqa/tests/avo.rb

@@ -0,0 +1,466 @@
+require 'rubyqa'
+
+include RubyQA
+
+require 'time'
+require 'English'
+
+info_template = ERB.new <<EOF, trim_mode: '-'
+==============================================
+Test : "<%= @name %>"
+==============================================
+
+<% @tests.each  do |hostname,data| -%>
+<%= hostname %>
+----------------------------------------------
+       Serial: <%= data[:context][:serial] %>
+  System Date: <%= data[:context][:date] %>
+      HWCLOCK: <%= data[:context][:hwclock] %>
+     Timezone: <%= data[:context][:timezone] %>
+           MM: <%= data[:context][:maintenance_mode] %>
+      Standby: <%= data[:context][:standby] %>
+  Gold-Images:
+<% data[:context][:gold_images].each  do |image| -%>
+    - <%= image %>
+<% end  -%>
+ Full-Backups:
+<% data[:context][:full_backups].each do |image| -%>
+    - <%= image %>
+<% end -%>
+   DRBD Syncs:
+<% data[:context][:drbd_syncs].each do |sync_name, sync_data| -%>
+    - <%= sync_name %> (Sync Status: <%= sync_data['disk_state'] %>)
+<% end  -%>
+
+<% end -%>
+EOF
+
+Manager.new_test("Get Information", template: info_template){|host,context|
+  context[:serial]   = host.resources['facts']['values']['dmi']['product']['serial_number'].downcase
+  context[:date]     = Time.parse(host.exec('date').strip)
+  context[:hwclock]  = Time.parse(host.exec('sudo hwclock').strip)
+  context[:timezone] = host.resources['facts']['values']['timezone']
+  context[:gold_images] = host.exec('sudo ls -1 /mnt/gold-images/ | grep tar.gz').split(/\n/).map(&:strip).reject(&:empty?)
+  context[:full_backups] = host.exec('sudo ls -1 /mnt/kvm_backups/ | grep tar.gz').split(/\n/).map(&:strip).reject(&:empty?)
+  context[:drbd_syncs] = host.resources['facts']['values']['drbd_status'].sort_by{|name,data| name}
+  context[:maintenance_mode] = host.exec("sudo pcs property show maintenance-mode | awk '/maint/ { print $2 }'").strip
+  context[:standby] = host.resources['facts']['values']['corosync']['is_standby'] ? "true" : "false"
+}
+
+
+network_template = ERB.new <<EOF, trim_mode: '<>-'
+==============================================
+Test : "<%= @name %>"
+==============================================
+<% @tests.each  do |hostname, data| -%>
+<%= hostname %>:
+----------------------------------------------
+<% data[:context][:interfaces].each  do |name,info| -%>
+  <%= name %> <% if  info.has_key? :mode %>(<%= info[:mode] %>)<% end  %>:
+<% if info[:mode] == "static" -%>
+<% if  info.has_key? :address -%>
+    IP: <%= info[:address] %>
+<% end  -%>
+<% if info.has_key? :gateway -%>
+    Gateway: <%= info[:gateway] %>
+<% end -%>
+<% if info.has_key? :netmask -%>
+    Netmask: <%= info[:netmask] %>
+<% end -%>
+<% if  info.has_key? :dns -%>
+    DNS: <%= info[:dns] %>
+<% end  -%>
+<% end -%>
+<% end  -%>
+
+<% end -%>
+EOF
+
+Manager.new_test("Networking Configurations Host", template: network_template) {|host,context|
+  contents = host.exec('cat /etc/network/interfaces')
+  interfaces = Hash.new
+  interface_regex = /iface (?<name>\S+) inet (?<mode>\S+)/
+  address_regex = /address (?<address>\S+)/
+  netmask_regex = /netmask (?<netmask>\S+)/
+  gateway_regex = /gateway (?<gateway>\S+)/
+
+
+  current_interface = ""
+  contents.split(/\n/).each do |line|
+    case line
+      when interface_regex
+        current_interface = $LAST_MATCH_INFO['name']
+        mode = $LAST_MATCH_INFO['mode']
+        interfaces[current_interface]=Hash.new
+        interfaces[current_interface][:mode] = mode
+
+      when address_regex
+        interfaces[current_interface][:address] = $LAST_MATCH_INFO['address']
+
+      when netmask_regex
+        interfaces[current_interface][:netmask] = $LAST_MATCH_INFO['netmask']
+
+      when gateway_regex
+        interfaces[current_interface][:gateway] = $LAST_MATCH_INFO['gateway']
+    end
+  end
+
+  contents = host.exec('sudo ipmitool lan print')
+  current_interface="IPMI"
+  interfaces[current_interface]=Hash.new
+  mode_regex = /IP Address Source[[:space:]]+:[[:space:]](?<mode>\S+)/
+  address_regex = /IP Address[[:space:]]+:[[:space:]]+(?<address>\S+)/
+  netmask_regex = /Subnet Mask[[:space:]]+:[[:space:]]+(?<netmask>\S+)/
+  gateway_regex = /Default Gateway IP[[:space:]]+:[[:space:]]+(?<gateway>\S+)/
+
+  contents.split(/\n/).each do |line|
+    case line
+      when mode_regex
+        interfaces[current_interface][:mode] = $LAST_MATCH_INFO['mode'].downcase
+
+      when address_regex
+        interfaces[current_interface][:address] = $LAST_MATCH_INFO['address']
+
+      when netmask_regex
+        interfaces[current_interface][:netmask] = $LAST_MATCH_INFO['netmask']
+
+      when gateway_regex
+        interfaces[current_interface][:gateway] = $LAST_MATCH_INFO['gateway']
+    end
+  end
+
+  ipconfig_script=<<EOF
+ipconfig(){
+  local KVM=${1:?"$FUNCNAME: No VM provided"}
+  local -i inc=0
+  local output=""
+
+  handle_return=$(sudo virsh qemu-agent-command "$KVM" '{"execute":"guest-exec", "arguments":{"path":"cmd.exe","arg":["/c","ipconfig /all"],"capture-output":true}}'| grep -o '[[:digit:]]*')
+  until [[ $output = *\\"exited\\":true* || "$inc" -ge "42" ]] ; do
+    output=$(sudo virsh qemu-agent-command "$KVM" '{"execute":"guest-exec-status","arguments":{"pid":'${handle_return}'}}')
+    sleep .1 ; ((inc++))
+  done
+  cut -d '"' -f8 <<< "$output" | base64 -d
+}
+EOF
+
+get_vm_info_script=<<EOF
+get_vm_info () {
+        local awk_script='
+BEGIN { ip=""; subnet=""; dns=""; gateway=""; FS=":" }
+/IPv4 Address/ {gsub(/[[:space:]]/,"", $2);  gsub("[(]Preferred[)]", "", $2); ip = $2 }
+/Subnet Mask/ { gsub(/[[:space:]]/,"", $2); subnet = $2 }
+/Gateway/ {  gsub(/[[:space:]]/,"", $2); gateway = $2 }
+/DNS Servers/ { gsub(/[[:space:]]/,"", $2); dns=$2}
+/^[[:space:]]+[0-9]/ { gsub(/[[:space:]]/, "", $1); dns = dns "," $1 }
+/NetBIOS/ { print name"|"ip"|"subnet"|"gateway"|"dns; name=""; ip=""; subnet=""; dns=""; gateway=""; }
+'
+        vms=( $( sudo virsh list --name | sort -V ) )
+        if [[ ${#vms[@]} -gt 0 ]]; then
+                for VM in ${vms[@]}; do
+                        ipconfig $VM | awk -v name=$VM "$awk_script"
+                done
+        fi
+}
+EOF
+
+  info = host.exec([get_vm_info_script, ipconfig_script, "get_vm_info"].join("\n")).split(/\n/)
+
+  info.select{|line| line =~ /.*\|.*/}.each do |line|
+    name, ip, subnet, gateway, dns = line.split("|")
+    interfaces[name] = Hash.new
+    interfaces[name][:gateway] = gateway
+    interfaces[name][:netmask] = subnet
+    interfaces[name][:address] = ip
+    interfaces[name][:dns] = dns
+    interfaces[name][:mode] = "static"
+  end
+
+  context[:interfaces]=interfaces
+}
+
+Manager.new_test("OpenVPN Certs Deployed to Node") {|host, context|
+  pass = true
+
+  buildserver_client_cert_exist = `sudo lxc-attach -n avobuildaus -- bash -c '[[ -f /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.crt ]] && echo "yes" || echo "no"'`.strip 
+  cert_exist = (buildserver_client_cert_exist =~ /yes/ ) ? true : false
+  buildserver_client_key_exist = `sudo lxc-attach -n avobuildaus -- bash -c '[[ -f /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.key ]] && echo "yes" || echo "no"'`.strip
+  key_exist = (buildserver_client_key_exist =~ /yes/ ) ? true : false
+
+  if not cert_exist or not key_exist 
+    context[:note] = "Nodes Cert and/or key were not found on buildserver, inform ENG that certs haven't been generated for #{host.name}"
+    pass = false
+  else
+    openvpn_client_cert_md5 = host.exec("sudo md5sum /etc/openvpn/mgmt/easy-rsa/keys/client.crt | awk '{ print $1 }'").strip
+    openvpn_client_key_md5 = host.exec("sudo md5sum /etc/openvpn/mgmt/easy-rsa/keys/client.key | awk '{ print $1 }'").strip
+    buildserver_client_cert_md5 = `sudo lxc-attach -n avobuildaus -- md5sum /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.crt | awk '{ print $1 }'`.strip
+    buildserver_client_key_md5 = `sudo lxc-attach -n avobuildaus -- md5sum /etc/puppetlabs/puppet/openvpn/openvpn-client-mgmt-#{host.name}.key | awk '{ print $1 }'`.strip
+
+    md5_fails = Array.new
+
+    if not openvpn_client_cert_md5 == buildserver_client_cert_md5
+      md5_fails << "cert"
+    end
+
+    if not openvpn_client_key_md5 == buildserver_client_key_md5
+      md5_fails << "key"
+    end
+
+    if md5_fails.count != 0
+      pass = false
+      context[:note] = "Failed: #{md5_fails.join(',')}"
+    end
+  end
+
+  pass
+}
+
+NETMASK_32="255.255.255.255"
+FILTER_INTERFACES=/eno(0|1|2|6)/
+Manager.new_test("No /32 addresses") {|host, context|
+  pass = true
+  facts = host.resources['facts'].data
+  interfaces = facts['values']['networking']['interfaces'].select{|name,val| name =~ FILTER_INTERFACES}
+  failed_interfaces = Array.new
+
+  interfaces.each do |name,data| 
+    if data["bindings"].is_a? Array
+      data["bindings"].each do |binding|
+        if binding['netmask'] == NETMASK_32
+          pass = false
+          puts "#{host.name}: Was found to have a /32 netmask on #{name} with ip #{binding['address']}"
+          if ! failed_interfaces.include? name
+            failed_interfaces << name
+          end
+        end
+      end
+    elsif data['bindings'].is_a? Hash
+      if data['bindings']['netmask'] == NETMASK_32
+        pass = false
+        puts "#{host.name}: Was found to have a /32 netmask on #{name} with ip #{data['address']}"
+        failed_interfaces << name
+      end
+    end
+  end
+  if ! failed_interfaces.empty?
+    context[:note] = "failed: #{failed_interfaces.join(',')}"
+  end
+  pass
+}
+
+Manager.new_test("Puppet Env is Pilot"){ |host|
+  host.exec("cat /etc/puppetlabs/puppet/puppet.conf") =~ /environment = pilot/
+}
+
+NETMASK_30="255.255.255.252"
+RED1_ENO6="172.16.16.2"
+RED0_ENO6="172.16.16.1"
+Manager.new_test("DRBD Interfaces Configured Correctly"){|host|
+  facts = host.resources['facts'].data
+  networking = facts['values']['networking']
+  binding = networking['interfaces']['eno6']['bindings'].first
+  address = binding['address']
+  netmask = binding['netmask']
+
+  if host.name =~ /red1/
+    address == RED1_ENO6 && netmask == NETMASK_30
+  elsif host.name =~ /red0/
+    address == RED0_ENO6 && netmask == NETMASK_30
+  else
+    false
+  end
+}
+
+VM_NAME_REGEX = /(?<name>[A-Z]{3}[0-9]+).xml/
+CACHE_MODE_REGEX = /cache=\'(?<mode>\S+)\'/
+Manager.new_test("Check VM disk cache is writeback"){|host, context|
+  pass = true
+  failed_vms = Array.new
+
+  host.exec("sudo grep cache /etc/libvirt/qemu/*.xml").split(/\n/).each do |line|
+    name = VM_NAME_REGEX.match(line)['name']
+    mode = CACHE_MODE_REGEX.match(line)['mode']
+
+    if mode != "writeback"
+       pass = false
+       failed_vms << name
+    end
+  end
+
+  if ! failed_vms.empty?
+    context[:note] = "failed vms: #{failed_vms.join(",")}"
+  end
+
+  pass
+}
+
+Manager.new_test("Manual RVC Patch Applied to domain.rb"){|host, context|
+  pass = true
+  pass = host.exec("sudo grep Log.error /opt/rvc/lib/rvc/domain.rb") =~ /Log.error[[:space:]]*\{[[:space:]]*e(\.message)?[[:space:]]*\}/
+  if not pass
+    context[:note] = "Manual Patch not applied (sudo sed -i 's/Log.error(e)/Log.error { e }/g' /opt/rvc/lib/rvc/domain.rb)"
+  end
+  pass
+}
+
+Manager.new_test("Test IPMI Credentials on self (over network)") {|host,context|
+  fencenode=host.resources['facts']['values']['pcs_resources']["fence_#{host.name}"]['instance_attributes']
+  
+  pass = true
+
+  test_result = host.exec("set +o history; ipmitool -I lanplus -H #{fencenode["ipaddr"]} -U #{fencenode['userid']} -P #{fencenode['passwd']} chassis status >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
+  
+  case test_result
+    when 'yes'
+      pass = true
+    when 'no'
+      pass = false
+  end
+
+  if pass == false
+    ping_test = host.exec("ping -c 2 #{fencenode["ipaddr"]} >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
+    case ping_test
+      when 'yes'
+        context[:note] = "IPMI Pingable, Creds Failed"
+      when 'no'
+        context[:note] = "IPMI Not Pingable. Possibly unplugged"
+    end
+  end
+
+  pass
+}
+
+Manager.new_test("Test IPMI Credentials on partner") {|host,context|
+  if host.name =~ /red1/
+    hostname = host.name.sub('red1','red0')
+  elsif host.name =~ /red0/
+    hostname = host.name.sub('red0','red1')
+  end
+
+  pass = true
+
+  begin 
+    fencenode=host.resources['facts']['values']['pcs_resources']["fence_#{hostname}"]['instance_attributes']
+  rescue 
+    puts "Fact retrieval failed on #{host.name}. Failing test"
+    return false
+  end
+
+  test_result = host.exec("set +o history; ipmitool -I lanplus -H #{fencenode["ipaddr"]} -U #{fencenode['userid']} -P #{fencenode['passwd']} chassis status >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
+  
+  case test_result
+    when 'yes'
+      pass = true
+    when 'no'
+      pass = false
+  end
+
+  if pass == false
+    ping_test = host.exec("ping -c 2 #{fencenode["ipaddr"]} >/dev/null 2>&1 && echo 'yes' || echo 'no'").strip
+    case ping_test
+      when 'yes'
+        context[:note] = "Other Node IPMI is pingable, Creds Failed"
+      when 'no'
+        context[:note] = "Other Node IPMI is not Pingable. Possibly unplugged"
+    end
+  end
+
+  pass
+}
+
+Manager.new_test("No Cluster Irregularities", description: "Nothing Disabled that shouldn't be"){|node,context|
+  pass = true
+  disabled_items = Array.new
+
+  ## Stonith
+  ## .values.pcs_resources.fence_avo00000223red0.meta_attributes["target-role"]
+  begin 
+    node_stonith_state = node.resources['facts']['values']['pcs_resources']["fence_#{node.name}"]['meta_attributes']['target-role']
+  rescue
+    puts "Fact retrieval failed on #{host.name}. Failing test"
+    return false
+  end
+
+
+  if node_stonith_state == "Stopped"
+    disabled_items << "Stonith"
+  end
+
+  if disabled_items.count !=0 
+    context[:note] = "Disabled Items: #{disabled_items.join(',')}"
+    pass = false
+  end
+
+  pass
+}
+
+Manager.new_test("Puppet Agent disabled (pre-ship only)"){|node,context|
+  pass = true
+
+  ssl_dir_exists = node.exec('[[ -f $( sudo puppet config print agent_disabled_lockfile ) ]] && echo "yes" || echo "no"')
+
+  case ssl_dir_exists
+    when /no/
+      pass = false
+      context[:note] = "Before shipping node you need to disable the puppet agent on #{node.name} (sudo puppet agent --disable)"
+  end
+
+  pass
+}
+
+
+Manager.new_test("SSL Directory Cleaned (pre-ship only)"){|node,context|
+  pass = true
+
+  ssl_dir_exists = node.exec('[[ -d /etc/puppetlabs/puppet/ssl ]] && echo "yes" || echo "no"')
+
+  case ssl_dir_exists
+    when /yes/
+      pass = false
+      context[:note] = "Before shipping node you need to delete the ssl dir on #{node.name} (sudo rm -rf /etc/puppetlabs/puppet/ssl)"
+  end
+
+  pass
+}
+
+Manager.new_test("Ensure VMs not running (pre-ship only)"){|host,context|
+  pass = true
+  vms = host.exec("sudo virsh list --name").split(/\n/).map(&:strip).sort!
+
+  if vms.count > 0 
+    pass = false
+    context[:note] = "VMs are still running: #{vms.join(',')} shut them down before shipping (for VM in #{vms.join(' ')}; do sudo rvc kvm shutdown $VM; done"
+  end
+
+  pass
+}
+
+
+Manager.new_test("Cluster in Pre-Ship Condition (pre-ship only)"){|host,context|
+  pass = true
+  maintenance_mode = host.exec("sudo pcs property show maintenance-mode | awk '/maint/ { print $2 }'").strip
+  standby = host.resources['facts']['values']['corosync']['is_standby'] ? "true" : "false"
+
+  case host.name
+    when /red1/
+    if maintenance_mode =~ /false/ and standby =~ /false/
+      context[:note] = "Node is not in standby and cluster is not in maintenance-mode"
+      pass = false
+    elsif maintenance_mode =~ /false/ and standby =~ /true/
+      context[:note] = "Cluster isn't in maintenance-mode"
+      pass = false
+    elsif maintenance_mode =~ /true/ and standby =~ /false/
+      context[:note] = "Node isn't in standby"
+      pass = false
+    end
+
+  when /red0/
+    if maintenance_mode =~ /false/
+      context[:note] = "Cluster isn't in maintenance-mode"
+      pass = false
+    end
+  end
+
+  pass
+}
+

lib/rubyqa/tests/gpc.rb

@@ -0,0 +1,166 @@
+#!/usr/bin/ruby
+
+require 'rubyqa'
+require 'erb'
+
+include RubyQA
+
+info_template = ERB.new <<EOF, trim_mode: '-'
+==============================================
+Test : <%= @name %>
+==============================================
+
+<% @tests.each  do |hostname,data| -%>
+<%= hostname %>
+----------------------------------------------
+         Serials: <%= data[:context][:serial] %>
+     System Date: <%= data[:context][:date] %>
+         HWCLOCK: <%= data[:context][:hwclock] %>
+        Timezone: <%= data[:context][:timezone] %>
+  eSocket Status: <%= data[:context][:esocket_status] %>
+   eSocket Patch: <%= data[:context][:esocket_patch] %>
+   keystore md5s: 
+<% data[:context][:keystore_md5s].each  do |line| -%>
+     - <%= line %>
+<% end  -%>
+<% end -%>
+EOF
+Manager.new_test("Get Information", template: info_template){|host,context|
+  serial_matcher = /\"(\S+)\"/
+  serials = host.exec("sudo facter 2>/dev/null | grep serial | cut -d '>' -f 2 | tr -d ','")
+
+  context[:serial]   = serials.scan(serial_matcher).map{|serial| "\"#{serial.first}\""}.join(',')
+  context[:date]     = Time.parse(host.exec('date').strip)
+  context[:hwclock]  = Time.parse(host.exec('sudo hwclock').strip)
+  context[:timezone] = host.exec('readlink -f /etc/localtime').strip.gsub(/(\/usr\/share|\/etc)\/zoneinfo\//,'')
+  context[:esocket_status] = host.exec('sudo lxc-attach -n ${HOSTNAME/red?/aci0} -- systemctl is-active esocket').strip
+  context[:esocket_patch] = host.exec('sudo lxc-attach -n ${HOSTNAME/red?/aci0} -- ls -1 /home/esocket/eSocket.POS').split(/\n/).select{|line| line =~ /patch/}.last
+  context[:keystore_md5s] = host.exec('sudo lxc-attach -n ${HOSTNAME/red?/aci0} -- md5sum /home/esocket/eSocket.POS/keystore/*').split(/\n/).map(&:strip)
+}
+
+firewall_template = ERB.new <<EOF, trim_mode: '-'
+==============================================
+Test : <%= @name %>
+==============================================
+<% @tests.each  do |hostname,data| -%>
+<%= hostname %>
+----------------------------------------------
+<% data[:context][:chains].each  do |name, info| -%>
+  Chain <%= name %>
+<% info.each  do |rule| -%>
+    <%= rule[:line] %>  
+<% end  -%>
+<% end -%>
+
+<% end  -%>
+EOF
+
+filter_regex = /^target/
+chain_regex = /Chain (?<name>\S+)/
+rule_regex = /(?<target>\S+)[[:space:]]+(?<protocols>\S+)[[:space:]]+(?<opts>\S+)[[:space:]]+(?<source>\S+)[[:space:]]+(?<destination>\S+)/
+Manager.new_test("Firewall Forward Rules", template: firewall_template) {|host,context|
+  host_firewall_rules = host.exec("sudo itptables -L FORWARD")
+  context[:chains]=Hash.new
+
+  current_chain = ""
+  host_firewall_rules.split(/\n/).each do |line|
+    case line
+      when chain_regex
+        info = chain_regex.match(line)
+        current_chain = info['name']
+        context[:chains][current_chain] = Array.new
+
+      when filter_regex
+        next
+
+      when rule_regex
+        rule = Hash.new
+        info = rule_regex.match(line)
+        rule[:target]=info['target']
+        rule[:source]=info['source']
+        rule[:destination]=info['destination']
+        rule[:protocols]=info['protocols']
+        rule[:line]=line
+        context[:chains][current_chain] << rule
+    end
+  end
+}
+
+Manager.new_test("Check if hostname is correctly configured") {|host,context|
+  pass = true
+  name = host.name
+  hosts = host.exec("cat /etc/hosts")
+  hostname = host.exec("cat /etc/hostname").strip
+  not_configured_in = Array.new
+
+  if not hosts =~ /127\.0\.0\.1.*#{name}/
+    pass = false
+    not_configured_in << "/etc/hosts"
+  end
+
+  if not hostname =~ /#{name}/
+    pass = false
+    not_configured_in << "/etc/hostname"
+  end
+
+  if not_configured_in.count > 0
+    context[:note] = "Hostname was not configured in #{not_configured_in.join(',')}"
+  end
+
+  pass
+}
+
+Manager.new_test("Puppet Certs Generated") {|host|
+
+}
+
+Manager.new_test("OpenVPN Certs Generated") {|host, context|
+  listing = host.exec('ls -1 /etc/openvpn/mgmt/easy-rsa/keys/')
+  files_not_generated = Array.new
+  pass = true
+
+  if ! listing =~ /my.crt/
+    pass = false
+    files_not_generated << 'my.crt'
+  end
+
+  if ! listing =~ /my.key/
+    pass = false
+    files_not_generated << 'my.key'
+  end
+
+  if not pass
+    context[:note] = "Files not created: #{files_not_generated.join(',')}"
+  end
+
+  pass
+  
+}
+
+Manager.new_test("Ensure netcat installed on aci, prx & host"){|host, context|
+  pass = true
+
+  aci_listing = host.exec("sudo lxc-attach -n ${HOSTNAME/red?/aci0} -- apt list --installed")
+  prx_listing = host.exec("sudo lxc-attach -n ${HOSTNAME/red?/prx0} -- apt list --installed")
+  host_listing = host.exec("sudo apt list --installed")
+  not_installed_on = Array.new
+
+  if not aci_listing =~ /netcat/
+    not_installed_on << "aci"
+  end
+
+  if not prx_listing =~ /netcat/
+    not_installed_on << "prx"
+  end
+
+  if not host_listing =~ /netcat/
+    not_installed_on << "host"
+  end
+
+  if not_installed_on.count > 0
+    pass = false
+    context[:note] = "netcat not installed on #{not_installed_on.join(',')}"
+  end
+
+  pass
+}

rubyqa.gemspec

@@ -1,10 +1,15 @@
+require 'find'
 Gem::Specification.new do |s|
   s.name        = "RubyQA"
   s.version     = '0.1.0'
-  s.summary     = 'A Ruby framework for QA *NIX system builds'
+  s.summary     = 'A Ruby framework for QA-ing *NIX system builds'
   s.description = 'A Ruby based framework defining & orchestrating tests against a host or hosts'
   s.authors     = ['Tristan Ancelet']
   s.email       = 'tristanancelet@yahoo.com'
   s.homepatge   = 'https://git.arcanium.tech/tristan/RubyQA'
-  s.files       = Dir.glob('lib/*.rb')
+  s.executables += [ 'generate-qa','impromptu-qa']
+  s.files       = Find::find('lib').select{|file| file =~ /rb/}
+
+  s.add_dependency 'net-ssh', '~> 7.1', '>= 7.1.0'
+  s.add_dependency 'net-ssh-telnet', '~> 0.3', '>= 0.3.0'
 end

templates/base.erb

@@ -1,6 +1,6 @@
 RubyQA Test Report
 
-QA Date: <%= Time.now.strftime("%Y/%m/%d %l:%M:%S %Z") %>
+QA Date: <%= Time.now.strftime("%Y/%m/%d %l:%M:%S %p %Z") %>
 QA Runner: <%= ENV['USER'] %>
 QA Tests: <%= Manager.tests.count %>
 <% Manager.tests.each do |test| -%>
@@ -8,7 +8,7 @@ QA Tests: <%= Manager.tests.count %>
 <% end -%>
 QA Hosts: 
 <% Manager.hosts.each do |host| -%>
-- <%= host.name %> (<%= host.data[:ip] %>)
+- <%= host.name %> (<%= host[:ip] %>)
 <% end -%>
 
 ###############