Add support for Pbench agent installation

README.md

@@ -184,3 +184,35 @@ For OSP deploy with Ceph using Composable Roles, After setting the above specifi
 
 Note: User can customize [internal.yml.j2](templates/internal.yml.j2) template for Ceph deployment based on their
       requirement if needed
+2) set composable_roles: true true in group_vars/all.yml
+
+## Pbench agent Installations
+To install Pbench agent on the compute nodes and vm's (i.e client and server),you have to follow the below steps and set the below params install/group_vars/all.yml
+
+```
+ pbench_key_url: <pbench_key_url>
+ pbench_config_url: <pbench_config_url>
+
+ #check the infrared inv to update the value
+ id_rsa_overcloud: ~/.infrared/.workspaces/<update_value>/id_rsa_overcloud
+ undercloud_hostname: <undercloud hostname>
+```
+
+`cd  {{ ansible_user_dir}}/jetpack/install`
+
+1) To create the client and server
+     `ansible-playbook boot_vms.yml`
+
+To install Pbench agent on Trafficgen host, set the below params
+
+```
+trafficgen_host:
+  ip: <ip>
+  ssh_pass: password
+  ssh_user: root
+```
+2) Install Pbench agent bootstrapping
+     `ansible-playbook pbench_agent_bootstrap.yml -e "repos=<perf-dept-repo>" `
+3) Install Pbench agent install
+     `ansible-playbook pbench_agent_install.yml`
+

install/ansible.cfg

@@ -0,0 +1,4 @@
+[defaults]
+host_key_checking = false
+inventory = ~/.infrared/.workspaces/active/hosts
+timeout = 120

install/boot_vms.yml

@@ -0,0 +1,100 @@
+---
+
+# Playbook to create client and server vm on Openstack
+#
+# ansible-playbook boot_vms.yml
+
+- hosts: undercloud
+  gather_facts: yes
+  become: true
+  vars:
+    vm_password: centos
+    external_network: public
+  pre_tasks:
+    - name: generate pass_auth.sh
+      template:
+        src: pass_auth.sh.j2
+        dest: "{{ user_dir }}/pass_auth.sh"
+        mode: a+rwx
+
+    - name: install packages
+      package:
+        name: "{{ item }}"
+        state: present
+      loop:
+        - wget
+        - libguestfs-tools
+        - sshpass
+        - libvirt
+
+    - name: restart libvirt service
+      service:
+        name: libvirtd
+        state: restarted
+
+  roles:
+    - images
+    - flavors
+    - network
+    - create_vm
+
+  post_tasks:
+    - name: check router exists
+      shell: |
+        . {{ overcloudrc }}
+        openstack router list | grep 'r1'
+      register: router_exists
+      ignore_errors: yes
+
+    - name: router
+      shell: |
+        . {{ overcloudrc }}
+        openstack router create r1
+        cidr=10.0.0.0
+        subnet_id=`openstack subnet list -c ID -c Subnet -f value | grep $cidr | cut -d' ' -f1`
+        neutron router-interface-add r1 $subnet_id
+        neutron router-gateway-set r1 {{ external_network }}
+      when: router_exists.stdout == ""
+
+    - name: create floating ip for client and server
+      shell: |
+        . {{ overcloudrc }}
+        client_port_id=`openstack port show {{ ports[0] }} -c id -f value`
+        server_port_id=`openstack port show {{ ports[1] }} -c id -f value`
+        neutron floatingip-create --port-id $client_port_id {{ external_network }}
+        neutron floatingip-create --port-id $server_port_id {{ external_network }}
+      ignore_errors: yes
+
+    - name: get floating ip of server and client
+      shell: |
+        . {{ overcloudrc }}
+        openstack server list | grep {{ item }} | cut -d \| -f 5 | cut -d \, -f 2 | awk '{$1=$1;print}'
+      register: vms_fip
+      with_items:
+        - "{{ vm_names }}"
+
+    - name: update user to root
+      lineinfile:
+        path: "{{ user_dir}}/.ssh/config"
+        regexp: '  User heat-admin'
+        line: '  User root'
+
+    - name: copy the ssh keys
+      shell: |
+        . {{ overcloudrc }}
+        echo '{{ vm_password }}' | sshpass ssh-copy-id -i {{ user_dir }}/.ssh/id_rsa -o 'StrictHostKeyChecking no' -f root@{{ item.stdout }}
+      with_items: "{{  vms_fip.results }}"
+      changed_when: false
+
+    - name: get default route
+      shell: |
+        ip r | grep default | cut -d ' ' -f5
+      register: default_route
+
+    - name: flush iptables on undercloud and add masquerade rule
+      become: true
+      shell: |
+        iptables -F
+        iptables -t nat -L --line-numbers
+        iptables -t nat -A POSTROUTING -o {{ default_route.stdout }} -j MASQUERADE
+        iptables -t nat -A POSTROUTING -o br-ctlplane -j MASQUERADE

install/filter_plugins/install_filters.py

@@ -0,0 +1,10 @@
+def dict_remove_item( dict, item):
+    """Remove an item from a dictionary."""
+    del dict[item]
+    return dict
+
+class FilterModule(object):
+    def filters(self):
+      return {
+        'dict_remove_item': dict_remove_item,
+        }

install/group_vars/all.yml

@@ -0,0 +1,41 @@
+---
+guest_image_name: pbench_image
+# user can add the guest_image url required below
+guest_images:
+  pbench_image:
+    name: pbench_image
+    url_centos8: http://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.2.2004-20200611.2.x86_64.qcow2
+    url_centos7: http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2
+    type: qcow2
+    convert_to_raw: false
+
+# rhel 8
+epel_repo:
+  el8: https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+  el7: https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+
+pbench_image_rhel: false
+
+#name for the client and server
+vm_names: [ client, server ]
+# port names
+ports: [ 'client_port', 'server_port']
+timeout: 15
+
+# trafficgen host ip address
+#trafficgen_host:
+#  ip: <ip>
+#  ssh_pass: password
+#  ssh_user: root
+
+user_dir: /home/stack
+overcloudrc: /home/stack/overcloudrc
+stackrc: /home/stack/stackrc
+
+# update the values to install pbench agent
+#check the infrared inv
+#id_rsa_overcloud: ~/.infrared/.workspaces/<update_value>/id_rsa_overcloud
+#undercloud_hostname:
+#pbench_key_url:
+#pbench_config_url:
+pbench_configuration_environment: "production"

install/pbench_agent_bootstrap.yml

@@ -0,0 +1,65 @@
+---
+#
+# Playbook for installing pbench_bootstrap
+#
+# Usage:
+#   ansible-playbook pbench_agent_bootstrap.yml -e "repos=<perf-dept-repo>"
+
+- hosts: compute
+  tasks:
+    - name: copy the keys
+      shell: |
+        sudo cp -r ~/.ssh /root/
+
+- hosts: undercloud
+  gather_facts: yes
+  tasks:
+    - name: clone perf-dept
+      shell: |
+        GIT_SSL_NO_VERIFY=true git clone {{ repos }}
+      ignore_errors: yes
+
+    - name: copy keys to trafficgen host
+      shell: |
+        echo '{{ trafficgen_host.ssh_pass }}' | sshpass ssh-copy-id -i ~/.ssh/id_rsa -o 'StrictHostKeyChecking no' -f {{ trafficgen_host.ssh_user }}@{{ trafficgen_host.ip }}
+      register: ssh_result
+      changed_when: false
+      ignore_errors: true
+      when: trafficgen_host is defined
+
+    - name: get computes ip
+      shell: |
+        . {{ stackrc }}
+        nova list | grep -i "compute" | cut -d \| -f 7 |cut -d \= -f 2 | awk '{$1=$1;print}'
+      register: computes_ip
+
+    - name: get floating ip of server and client
+      shell: |
+        . {{ overcloudrc }}
+        openstack server list | grep {{ item }} | cut -d \| -f 5 | cut -d \, -f 2 | awk '{$1=$1;print}'
+      register: vms_fip
+      with_items:
+        - "{{ vm_names }}"
+
+    - name: copy the keys
+      shell: |
+        sudo cp -r ~/.ssh /root/
+
+    - name: create inventories
+      file:
+        path: "{{ user_dir }}/.config/Inventories/"
+        state: directory
+
+    - name: generate a inventory
+      template:
+        src: repo_bootstrap.hosts.j2
+        dest: "{{ user_dir }}/.config/Inventories/repo-bootstrap.hosts"
+
+    - name: run repo-bootstrap.yml
+      shell: |
+        ansible-playbook  --user=root -i "{{ user_dir}}/.config/Inventories/repo-bootstrap.hosts" repo-bootstrap.yml
+      register: repo_bootstrap
+      args:
+        chdir: "{{ user_dir }}/perf-dept/sysadmin/Ansible"
+    - debug:
+        msg: "{{ repo_bootstrap }}"

install/pbench_agent_install.yml

@@ -0,0 +1,98 @@
+# ansible-playbook pbench_agent_install.yml
+
+- hosts: compute
+  gather_facts: yes
+  tasks:
+    - name: install latest epel and perl-Data-UUID
+      yum:
+        name: "{{ epel_repo.el7 }}"
+        state: present
+      become: yes
+      loop:
+        - "{{ epel_repo.el7 }}"
+        - perl-Data-UUID
+      when: ansible_distribution_major_version < '8'
+
+    - name: install perl-Data-UUID on Compute nodes
+      become: true
+      dnf:
+        name: perl-Data-UUID
+        state: present
+      when: ansible_distribution_major_version == '8'
+
+- hosts: undercloud
+  gather_facts: yes
+  tasks:
+    - name: get floating ip of server and client
+      shell: |
+        . {{ overcloudrc }}
+        openstack server list | grep {{ item }} | cut -d \| -f 5 | cut -d \, -f 2 | awk '{$1=$1;print}'
+      register: vms_fip
+      with_items:
+        - "{{ vm_names }}"
+
+- hosts: localhost
+  vars:
+    pbench_agent_install: true
+    vms_fip: "{{ hostvars['undercloud']['vms_fip'] }}"
+    host_file: "{{ ansible_user_dir }}/.config/hosts"
+    infrared_inv: "{{ ansible_user_dir }}/.infrared/.workspaces/active/hosts"
+
+  tasks:
+    - name: copy keys to trafficgen host
+      shell: |
+        echo '{{ trafficgen_host.ssh_pass }}' | sshpass ssh-copy-id -i ~/.ssh/id_rsa -o 'StrictHostKeyChecking no' -f {{ trafficgen_host.ssh_user }}@{{ trafficgen_host.ip }}
+      register: ssh_result
+      changed_when: false
+      ignore_errors: true
+      when: trafficgen_host is defined
+
+    - name: Install the pbench-agent roles from Galaxy
+      shell: |
+        ansible-galaxy collection install pbench.agent
+        export ANSIBLE_ROLES_PATH=$HOME/.ansible/collections/ansible_collections/pbench/agent/roles:$ANSIBLE_ROLES_PATH
+
+    - name: generate pbench agent playbook
+      template:
+        src: pbench_agent_install.yml.j2
+        dest: "{{ ansible_user_dir }}/.ansible/collections/ansible_collections/pbench/agent/roles/pbench_agent_install.yml"
+
+    - name: create .config dir
+      file:
+        path: "{{ ansible_user_dir }}/.config/"
+        state: directory
+
+    - name: copy the infrared inv in ~/.config
+      shell: |
+        cp {{ infrared_inv }} {{ host_file }}
+
+    - name: Add client and server in inventory
+      blockinfile:
+        path: "{{ host_file }}"
+        marker: "# {mark} ANSIBLE MANAGED BLOCK {{ item[0] }}"
+        regexp: "[local]"
+        block: |
+          {{ item[0] }} original_name= ansible_host={{ item[1].stdout }} ansible_connection=ssh ansible_user=root ansible_ssh_private_key_file={{ id_rsa_overcloud }} ansible_ssh_common_args='-o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -i {{ ansible_user_dir }}/.ssh/id_rsa stack@{{ undercloud_hostname }}"'
+      with_together:
+        - "{{ vm_names }}"
+        - "{{ vms_fip.results }}"
+
+    - name: create a inventory file
+      template:
+        src: my.hosts.j2
+        dest: "{{ ansible_user_dir }}/.config/myhosts.inv"
+
+    - name: generate a pbench_agent_install inv
+      shell: |
+        cat "{{ ansible_user_dir }}/.config/myhosts.inv" >> {{ host_file }}
+
+    - name: run the playbook pbench_agent_install.yml
+      shell: |
+        export ANSIBLE_HOST_KEY_CHECKING=False
+        ansible-playbook -i {{ host_file }} pbench_agent_install.yml -T {{ timeout }} --user=root
+      register: agent_install
+      args:
+        chdir: "{{ ansible_user_dir }}/.ansible/collections/ansible_collections/pbench/agent/roles/"
+    - debug:
+        msg: "{{ agent_install }}"
+

install/roles/create_vm/tasks/main.yml

@@ -0,0 +1,19 @@
+---
+- name: create a vm's
+  shell: . {{ overcloudrc }}; openstack server create --flavor pbench_flavor --image {{ guest_image_name }} --nic port-id={{ item[0].stdout }} --user-data {{ user_dir }}/pass_auth.sh {{ item[1] }}
+  with_together:
+    - "{{ port_ids.results }}"
+    - "{{ vm_names }}"
+  ignore_errors: yes
+
+- name: create a vm's
+  shell: . {{ overcloudrc }}; openstack server create --flavor pbench_flavor --image {{ guest_image_name }} --nic port-id={{ item[0].stdout }} --user-data {{ user_dir }}/pass_auth.sh {{ item[1] }}
+  with_together:
+    - "{{ get_port_ids.results }}"
+    - "{{ vm_names }}"
+  ignore_errors: yes
+
+
+- name: wait for vm to boot
+  wait_for:
+     timeout: "{{ timeout }}"

install/roles/flavors/tasks/main.yml

@@ -0,0 +1,8 @@
+--- 
+# Add flavors to OpenStack Cloud
+# Ignore errors here incase the flavors already exist.
+
+- name: Add flavors to overcloud
+  shell: . {{ overcloudrc }}; openstack flavor create --public {{ item.name }} --id auto --ram {{ item.ram }} --disk {{ item.disk }} --vcpus {{ item.cpu }}
+  with_items: "{{ pbench_flavors }}"
+  ignore_errors: true

install/roles/flavors/vars/main.yml

@@ -0,0 +1,7 @@
+---
+pbench_flavors:
+  - name: pbench_flavor
+    cpu: 8
+    disk: 160 
+    ram: 16384
+