diff --git a/README.rst b/README.rst index 8dbdb88d..282f39d4 100644 --- a/README.rst +++ b/README.rst @@ -252,4 +252,4 @@ You can file bugs on the storyboard devstack-gate project:: https://storyboard.openstack.org/#!/project/712 -And you can chat with us on Freenode in #openstack-dev or #openstack-infra. +And you can chat with us on Freenode in #openstack-qa or #openstack-infra. diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index e62b1256..0b8a2cbe 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -37,6 +37,11 @@ source $WORKSPACE/devstack-gate/functions.sh start_timer +# Save the PROJECTS variable as it was passed in. This is needed for reproduce.sh +# incase the job definition contains items that are not in the "global" list +# below. +# See: https://bugs.launchpad.net/openstack-gate/+bug/1544827 +JOB_PROJECTS="$PROJECTS" PROJECTS="openstack-infra/devstack-gate $PROJECTS" PROJECTS="openstack-dev/devstack $PROJECTS" PROJECTS="openstack-dev/pbr $PROJECTS" @@ -71,10 +76,12 @@ PROJECTS="openstack/neutron-vpnaas $PROJECTS" PROJECTS="openstack/nova $PROJECTS" PROJECTS="openstack/os-apply-config $PROJECTS" PROJECTS="openstack/os-brick $PROJECTS" +PROJECTS="openstack/os-client-config $PROJECTS" PROJECTS="openstack/os-cloud-config $PROJECTS" PROJECTS="openstack/os-collect-config $PROJECTS" PROJECTS="openstack/os-net-config $PROJECTS" PROJECTS="openstack/os-refresh-config $PROJECTS" +PROJECTS="openstack/osc-lib $PROJECTS" PROJECTS="openstack/oslo.cache $PROJECTS" PROJECTS="openstack/oslo.concurrency $PROJECTS" PROJECTS="openstack/oslo.config $PROJECTS" @@ -163,8 +170,15 @@ export DEVSTACK_GATE_TEMPEST_DISABLE_TENANT_ISOLATION=${DEVSTACK_GATE_TEMPEST_DI # Set to 1 to enable Cinder secure delete. # False by default to avoid dd problems on Precise. # https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1023755 +# TODO(mriedem): CINDER_SECURE_DELETE is deprecated in devstack as of liberty. +# Remove after kilo-eol happens in devstack. export DEVSTACK_CINDER_SECURE_DELETE=${DEVSTACK_CINDER_SECURE_DELETE:-0} +# Should cinder perform secure deletion of volumes? +# Defaults to none to avoid bug 1023755. Can also be set to zero or shred. +# Only applicable to stable/liberty+ devstack. +export DEVSTACK_CINDER_VOLUME_CLEAR=${DEVSTACK_CINDER_VOLUME_CLEAR:-none} + # Set to 1 to run neutron instead of nova network # Only applicable to master branch export DEVSTACK_GATE_NEUTRON=${DEVSTACK_GATE_NEUTRON:-0} @@ -269,6 +283,9 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then export GRENADE_NEW_BRANCH="stable/liberty" elif [[ "$GRENADE_BASE_BRANCH" == "stable/liberty" ]]; then export GRENADE_OLD_BRANCH="stable/liberty" + export GRENADE_NEW_BRANCH="stable/mitaka" + elif [[ "$GRENADE_BASE_BRANCH" == "stable/mitaka" ]]; then + export GRENADE_OLD_BRANCH="stable/mitaka" export GRENADE_NEW_BRANCH="$GIT_BRANCH" fi ;; @@ -296,8 +313,11 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then elif [[ "$GRENADE_BASE_BRANCH" == "stable/liberty" ]]; then export GRENADE_OLD_BRANCH="stable/kilo" export GRENADE_NEW_BRANCH="stable/liberty" - else # master + elif [[ "$GRENADE_BASE_BRANCH" == "stable/mitaka" ]]; then export GRENADE_OLD_BRANCH="stable/liberty" + export GRENADE_NEW_BRANCH="stable/mitaka" + else # master + export GRENADE_OLD_BRANCH="stable/mitaka" export GRENADE_NEW_BRANCH="$GIT_BRANCH" fi ;; @@ -416,7 +436,7 @@ fi if ! function_exists "gate_hook"; then # the command we use to run the gate function gate_hook { - $BASE/new/devstack-gate/devstack-vm-gate.sh + $WORKSPACE/devstack-gate/devstack-vm-gate.sh } export -f gate_hook fi @@ -434,7 +454,9 @@ set -x # Install ansible sudo -H pip install virtualenv virtualenv /tmp/ansible -/tmp/ansible/bin/pip install ansible==$ANSIBLE_VERSION +# NOTE(emilien): workaround to avoid installing cryptography +# https://github.com/ansible/ansible/issues/15665 +sudo -H /tmp/ansible/bin/pip install paramiko==1.16.0 ansible==$ANSIBLE_VERSION export ANSIBLE=/tmp/ansible/bin/ansible # Write inventory file with groupings @@ -468,7 +490,7 @@ $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m file \ -a "path='$WORKSPACE/logs' state=directory" # Record a file to reproduce this build -reproduce +reproduce "$JOB_PROJECTS" # Run ansible to do setup_host on all nodes. echo "Setting up the hosts" @@ -552,15 +574,23 @@ fi # devstack-vm-gate-wrap.sh will not automagically run the hooks on each node. # Run pre test hook if we have one with_timeout call_hook_if_defined "pre_test_hook" +GATE_RETVAL=$? +if [ $GATE_RETVAL -ne 0 ]; then + echo "ERROR: the pre-test setup script run by this job failed - exit code: $GATE_RETVAL" +fi # Run the gate function -echo "Running gate_hook" -with_timeout "gate_hook" -GATE_RETVAL=$? +if [ $GATE_RETVAL -eq 0 ]; then + echo "Running gate_hook" + with_timeout "gate_hook" + GATE_RETVAL=$? + if [ $GATE_RETVAL -ne 0 ]; then + echo "ERROR: the main setup script run by this job failed - exit code: $GATE_RETVAL" + fi +fi RETVAL=$GATE_RETVAL if [ $GATE_RETVAL -ne 0 ]; then - echo "ERROR: the main setup script run by this job failed - exit code: $GATE_RETVAL" echo " please look at the relevant log files to determine the root cause" echo "Running devstack worlddump.py" sudo $BASE/new/devstack/tools/worlddump.py -d $BASE/logs diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 068a2fe4..98cd8dcd 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -50,7 +50,11 @@ PUBLIC_NETWORK_GATEWAY=${DEVSTACK_GATE_PUBLIC_NETWORK_GATEWAY:-172.24.5.1} FLOATING_HOST_PREFIX=${DEVSTACK_GATE_FLOATING_HOST_PREFIX:-172.24.4} FLOATING_HOST_MASK=${DEVSTACK_GATE_FLOATING_HOST_MASK:-23} -EXTERNAL_BRIDGE_MTU=1450 +# Get the smallest local MTU +LOCAL_MTU=$(ip link show | sed -ne 's/.*mtu \([0-9]\+\).*/\1/p' | sort -n | head -1) +# 50 bytes is overhead for vxlan (which is greater than GRE +# allowing us to use either overlay option with this MTU. +EXTERNAL_BRIDGE_MTU=$((LOCAL_MTU - 50)) function setup_ssh { local path=$1 @@ -71,15 +75,19 @@ function setup_nova_net_networking { # issue with nova net configuring br100 to take over eth0 # by default. # TODO (clarkb): figure out how to make bridge setup sane with ansible. - ovs_vxlan_bridge "br_pub" $primary_node "True" 1 \ - $FLOATING_HOST_PREFIX $FLOATING_HOST_MASK \ - $sub_nodes - ovs_vxlan_bridge "br_flat" $primary_node "False" 128 \ - $sub_nodes - cat <>"$localrc" + if [[ "$DEVSTACK_GATE_VIRT_DRIVER" != "xenapi" ]]; then + # The following work around shouldn't be applied on xenapi, as xenserver + # has vmnet connected on eth3. + ovs_vxlan_bridge "br_pub" $primary_node "True" 1 \ + $FLOATING_HOST_PREFIX $FLOATING_HOST_MASK \ + $sub_nodes + ovs_vxlan_bridge "br_flat" $primary_node "False" 128 \ + $sub_nodes + cat <>"$localrc" FLAT_INTERFACE=br_flat PUBLIC_INTERFACE=br_pub EOF + fi } function setup_multinode_connectivity { @@ -210,17 +218,16 @@ function setup_localrc { local original_enabled_services=$MY_ENABLED_SERVICES # TODO(afazekas): Move to the feature grid - # TODO(afazekas): add c-vol if [[ $role = sub ]]; then - MY_ENABLED_SERVICES="n-cpu,ceilometer-acompute,dstat" + MY_ENABLED_SERVICES="n-cpu,ceilometer-acompute,dstat,c-vol,c-bak" if [[ "$DEVSTACK_GATE_NEUTRON" -eq "1" ]]; then MY_ENABLED_SERVICES+=",q-agt" if [[ "$DEVSTACK_GATE_NEUTRON_DVR" -eq "1" ]]; then # As per reference architecture described in # https://wiki.openstack.org/wiki/Neutron/DVR # for DVR multi-node, add the following services - # on all compute nodes (q-fwaas being optional): - MY_ENABLED_SERVICES+=",q-l3,q-fwaas,q-meta" + # on all compute nodes: + MY_ENABLED_SERVICES+=",q-l3,q-meta" fi else MY_ENABLED_SERVICES+=",n-net,n-api-meta" @@ -307,9 +314,15 @@ DATABASE_QUERY_LOGGING=True EBTABLES_RACE_FIX=True EOF + if [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]] && [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then + # Reduce the MTU on br-ex to match the MTU of underlying tunnels + echo "PUBLIC_BRIDGE_MTU=$EXTERNAL_BRIDGE_MTU" >>"$localrc_file" + fi + if [[ "$DEVSTACK_CINDER_SECURE_DELETE" -eq "0" ]]; then echo "CINDER_SECURE_DELETE=False" >>"$localrc_file" fi + echo "CINDER_VOLUME_CLEAR=${DEVSTACK_CINDER_VOLUME_CLEAR}" >>"$localrc_file" if [[ "$DEVSTACK_GATE_TEMPEST_HEAT_SLOW" -eq "1" ]]; then echo "HEAT_CREATE_TEST_IMAGE=False" >>"$localrc_file" @@ -332,31 +345,31 @@ EOF fi if [[ "$DEVSTACK_GATE_VIRT_DRIVER" == "ironic" ]]; then - export TEMPEST_OS_TEST_TIMEOUT=900 - echo "VIRT_DRIVER=ironic" >>"$localrc_file" + export TEMPEST_OS_TEST_TIMEOUT=${DEVSTACK_GATE_OS_TEST_TIMEOUT:-1200} + echo "IRONIC_DEPLOY_DRIVER=$DEVSTACK_GATE_IRONIC_DRIVER" >>"$localrc_file" echo "IRONIC_BAREMETAL_BASIC_OPS=True" >>"$localrc_file" echo "IRONIC_VM_LOG_DIR=$BASE/$localrc_oldnew/ironic-bm-logs" >>"$localrc_file" echo "DEFAULT_INSTANCE_TYPE=baremetal" >>"$localrc_file" - echo "BUILD_TIMEOUT=600" >>"$localrc_file" + echo "BUILD_TIMEOUT=${DEVSTACK_GATE_TEMPEST_BAREMETAL_BUILD_TIMEOUT:-600}" >>"$localrc_file" echo "IRONIC_CALLBACK_TIMEOUT=600" >>"$localrc_file" echo "Q_AGENT=openvswitch" >>"$localrc_file" echo "Q_ML2_TENANT_NETWORK_TYPE=vxlan" >>"$localrc_file" if [[ "$DEVSTACK_GATE_IRONIC_BUILD_RAMDISK" -eq 0 ]]; then echo "IRONIC_BUILD_DEPLOY_RAMDISK=False" >>"$localrc_file" + else + echo "IRONIC_BUILD_DEPLOY_RAMDISK=True" >>"$localrc_file" fi - if [[ "$DEVSTACK_GATE_IRONIC_DRIVER" == "agent_ssh" ]]; then + if [[ -z "${DEVSTACK_GATE_IRONIC_DRIVER%%agent*}" ]]; then echo "SWIFT_ENABLE_TEMPURLS=True" >>"$localrc_file" echo "SWIFT_TEMPURL_KEY=secretkey" >>"$localrc_file" echo "IRONIC_ENABLED_DRIVERS=fake,agent_ssh,agent_ipmitool" >>"$localrc_file" - echo "IRONIC_DEPLOY_DRIVER=agent_ssh" >>"$localrc_file" # agent driver doesn't support ephemeral volumes yet echo "IRONIC_VM_EPHEMERAL_DISK=0" >>"$localrc_file" # agent CoreOS ramdisk is a little heavy echo "IRONIC_VM_SPECS_RAM=1024" >>"$localrc_file" - echo "IRONIC_VM_COUNT=1" >>"$localrc_file" else + echo "IRONIC_ENABLED_DRIVERS=fake,pxe_ssh,pxe_ipmitool" >>"$localrc_file" echo "IRONIC_VM_EPHEMERAL_DISK=1" >>"$localrc_file" - echo "IRONIC_VM_COUNT=3" >>"$localrc_file" fi fi @@ -366,15 +379,15 @@ EOF exit 1 fi cat >> "$localrc_file" << EOF +# Need to force devstack to run because we currently use saucy +FORCE=yes + SKIP_EXERCISES=${SKIP_EXERCISES},volumes XENAPI_PASSWORD=${DEVSTACK_GATE_XENAPI_PASSWORD} XENAPI_CONNECTION_URL=http://${DEVSTACK_GATE_XENAPI_DOM0_IP} VNCSERVER_PROXYCLIENT_ADDRESS=${DEVSTACK_GATE_XENAPI_DOM0_IP} VIRT_DRIVER=xenserver -# A separate xapi network is created with this name-label -FLAT_NETWORK_BRIDGE=vmnet - # A separate xapi network on eth4 serves the purpose of the public network. # This interface is added in Citrix's XenServer environment as an internal # interface @@ -400,9 +413,17 @@ EXTRA_OPTS=("xenapi_disable_agent=True") # Add a separate device for volumes VOLUME_BACKING_DEVICE=/dev/xvdb -# Set multi-host config -MULTI_HOST=1 +# We use single host for XenAPI test +MULTI_HOST=False EOF + + # neutron network will set FLAT_NETWORK_BRIDGE in pre_test_hook + if [[ $DEVSTACK_GATE_NEUTRON -ne "1" ]]; then + cat >> "$localrc_file" << EOF +# A separate xapi network is created with this name-label +FLAT_NETWORK_BRIDGE=vmnet +EOF + fi fi if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then @@ -511,8 +532,20 @@ EOF # a way to pass through arbitrary devstack config options so that # we don't need to add new devstack-gate options every time we # want to create a new config. - if [[ -n "$DEVSTACK_LOCAL_CONFIG" ]]; then - echo "$DEVSTACK_LOCAL_CONFIG" >>"$localrc_file" + if [[ "$role" = sub ]]; then + # If we are in a multinode environment, we may want to specify 2 + # different sets of plugins + if [[ -n "$DEVSTACK_SUBNODE_CONFIG" ]]; then + echo "$DEVSTACK_SUBNODE_CONFIG" >>"$localrc_file" + else + if [[ -n "$DEVSTACK_LOCAL_CONFIG" ]]; then + echo "$DEVSTACK_LOCAL_CONFIG" >>"$localrc_file" + fi + fi + else + if [[ -n "$DEVSTACK_LOCAL_CONFIG" ]]; then + echo "$DEVSTACK_LOCAL_CONFIG" >>"$localrc_file" + fi fi } @@ -537,6 +570,7 @@ TARGET_RUN_SMOKE=False SAVE_DIR=\$BASE_RELEASE_DIR/save DO_NOT_UPGRADE_SERVICES=$DO_NOT_UPGRADE_SERVICES TEMPEST_CONCURRENCY=$TEMPEST_CONCURRENCY +OS_TEST_TIMEOUT=$DEVSTACK_GATE_OS_TEST_TIMEOUT VERBOSE=False PLUGIN_DIR=\$TARGET_RELEASE_DIR EOF @@ -549,9 +583,10 @@ EOF fi if [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]]; then - echo -e "[[post-config|\$NOVA_CONF]]\n[libvirt]\ncpu_mode=custom\ncpu_model=gate64" >> local.conf + # ensure local.conf exists to remove conditional logic + touch local.conf if [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then - echo -e "[[post-config|\$NEUTRON_CONF]]\n[DEFAULT]\nnetwork_device_mtu=$EXTERNAL_BRIDGE_MTU" >> local.conf + echo -e "[[post-config|\$NEUTRON_CONF]]\n[DEFAULT]\nglobal_physnet_mtu=$EXTERNAL_BRIDGE_MTU" >> local.conf fi # get this in our base config @@ -586,9 +621,10 @@ else cd $BASE/new/devstack setup_localrc "new" "localrc" "primary" if [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]]; then - echo -e "[[post-config|\$NOVA_CONF]]\n[libvirt]\ncpu_mode=custom\ncpu_model=gate64" >> local.conf + # ensure local.conf exists to remove conditional logic + touch local.conf if [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then - echo -e "[[post-config|\$NEUTRON_CONF]]\n[DEFAULT]\nnetwork_device_mtu=$EXTERNAL_BRIDGE_MTU" >> local.conf + echo -e "[[post-config|\$NEUTRON_CONF]]\n[DEFAULT]\nglobal_physnet_mtu=$EXTERNAL_BRIDGE_MTU" >> local.conf fi fi @@ -640,19 +676,6 @@ else exit 1 fi fi - - if [[ "$DEVSTACK_GATE_TOPOLOGY" != "aio" ]] && [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then - # NOTE(afazekas): The cirros lp#1301958 does not support MTU setting via dhcp, - # simplest way the have tunneling working, with dvsm, without increasing the host system MTU - # is to decreasion the MTU on br-ex - # TODO(afazekas): Configure the mtu smarter on the devstack side - MTU_NODES=primary - if [[ "$DEVSTACK_GATE_NEUTRON_DVR" -eq "1" ]]; then - MTU_NODES=all - fi - $ANSIBLE "$MTU_NODES" -f 5 -i "$WORKSPACE/inventory" -m shell \ - -a "sudo ip link set mtu $EXTERNAL_BRIDGE_MTU dev br-ex" - fi fi if [[ "$DEVSTACK_GATE_UNSTACK" -eq "1" ]]; then @@ -708,7 +731,7 @@ if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then sudo chmod -R o+rx $BASE/new/devstack/files fi - # In the future we might want to increase the number of compute nodes. + # In the future we might want to increase the number of compute nodes. # This will ensure that multinode jobs consist of 2 nodes. # As a part of tempest configuration, it should be executed # before the DEVSTACK_GATE_TEMPEST_NOTESTS check, because the DEVSTACK_GATE_TEMPEST @@ -723,6 +746,16 @@ if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then exit 0 fi + # There are some parts of devstack that call the neutron api to verify the + # extension. We should not ever trust this for gate testing. This checks to + # ensure on master we always are using the default value. (on stable we hard + # code a list of available extensions so we can't use this) + neutron_extensions=$(iniget "$BASE/new/tempest/etc/tempest.conf" "neutron-feature-enabled" "api_extensions") + if [[ $GIT_BRANCH == 'master' && ($neutron_extensions == 'all' || $neutron_extensions == '') ]] ; then + echo "Devstack misconfugred tempest and changed the value of api_extensions" + exit 1 + fi + # From here until the end we rely on the fact that all the code fails if # something is wrong, to enforce exit on bad test results. set -o errexit @@ -736,14 +769,16 @@ if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then if [[ "$DEVSTACK_GATE_TEMPEST_REGEX" != "" ]] ; then if [[ "$DEVSTACK_GATE_TEMPEST_ALL_PLUGINS" -eq "1" ]]; then echo "Running tempest with plugins and a custom regex filter" - $TEMPEST_COMMAND -eall-plugin -- --concurrency=$TEMPEST_CONCURRENCY $DEVSTACK_GATE_TEMPEST_REGEX + $TEMPEST_COMMAND -eall-plugin -- $DEVSTACK_GATE_TEMPEST_REGEX --concurrency=$TEMPEST_CONCURRENCY + sudo -H -u tempest .tox/all-plugin/bin/tempest list-plugins else echo "Running tempest with a custom regex filter" - $TEMPEST_COMMAND -eall -- --concurrency=$TEMPEST_CONCURRENCY $DEVSTACK_GATE_TEMPEST_REGEX + $TEMPEST_COMMAND -eall -- $DEVSTACK_GATE_TEMPEST_REGEX --concurrency=$TEMPEST_CONCURRENCY fi elif [[ "$DEVSTACK_GATE_TEMPEST_ALL_PLUGINS" -eq "1" ]]; then echo "Running tempest all-plugins test suite" $TEMPEST_COMMAND -eall-plugin -- --concurrency=$TEMPEST_CONCURRENCY + sudo -H -u tempest .tox/all-plugin/bin/tempest list-plugins elif [[ "$DEVSTACK_GATE_TEMPEST_ALL" -eq "1" ]]; then echo "Running tempest all test suite" $TEMPEST_COMMAND -eall -- --concurrency=$TEMPEST_CONCURRENCY diff --git a/features.yaml b/features.yaml index 50b35f19..5b391f86 100644 --- a/features.yaml +++ b/features.yaml @@ -1,10 +1,12 @@ config: default: - master: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] - liberty: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] - kilo: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] - juno: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] - icehouse: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] + master: [default, glance, horizon, nova, swift, cinder, keystone] + ocata: [default, glance, horizon, nova, swift, cinder, keystone] + mitaka: [default, glance, horizon, nova, swift, cinder, keystone] + liberty: [default, glance, horizon, nova, swift, cinder, keystone] + kilo: [default, glance, horizon, nova, swift, cinder, keystone] + juno: [default, glance, horizon, nova, swift, cinder, keystone] + icehouse: [default, glance, horizon, nova, swift, cinder, keystone] # This can be used by functional jobs that only want their dependencies installed # and don't need to incur the overhead of installing all services in the process. no_services: [default] @@ -43,12 +45,16 @@ config: features: [ceph] heat: features: [heat] + tlsproxy: + features: [tlsproxy] + cinder_mn_grenade: + features: [cinder-mn-grenade] branches: # The value of ""default" is the name of the "trunk" branch default: master # Normalized branch names only here, e.g. stable/icehouse => icehouse - allowed: [master, liberty, kilo, juno, icehouse] + allowed: [master, ocata, mitaka, liberty, kilo, juno, icehouse] features: default: @@ -93,7 +99,7 @@ features: neutron-adv: base: - services: [q-lbaas, q-fwaas] + services: [q-lbaas] rm-services: [n-net] kilo: services: [q-vpn] @@ -106,6 +112,12 @@ features: base: services: [cinder, c-api, c-vol, c-sch, c-bak] + # This will be used to disable c-vol on primary node when running multinode grenade + # job that will test compatibility of new c-api, c-sch (primary) and old c-vol and c-bak (sub). + cinder-mn-grenade: + base: + rm-services: [c-vol, c-bak] + heat: base: services: [heat, h-api, h-api-cfn, h-api-cw, h-eng] @@ -154,3 +166,7 @@ features: ceph: base: services: [ceph] + + tlsproxy: + base: + services: [tls-proxy] diff --git a/functions.sh b/functions.sh index 6bb9af06..e06dda49 100644 --- a/functions.sh +++ b/functions.sh @@ -99,7 +99,7 @@ function tsfilter { function _ping_check { local host=$1 local times=${2:-20} - echo "Testing ICMP connectivit to $host" + echo "Testing ICMP connectivity to $host" ping -c $times $host } @@ -140,20 +140,7 @@ function start_timer { # first make sure the time is right, so we don't go into crazy land # later if the system decides to apply an ntp date and we jump forward # 4 hrs (which has happened) - if is_fedora; then - local ntp_service='ntpd' - elif uses_debs; then - local ntp_service='ntp' - else - echo "Unsupported platform, can't determine ntp service" - exit 1 - fi - local default_ntp_server=$( - grep ^server /etc/ntp.conf | head -1 | awk '{print $2}') - local ntp_server=${NTP_SERVER:-$default_ntp_server} - sudo service $ntp_service stop - sudo /usr/sbin/ntpdate $ntp_server - sudo service $ntp_service start + sudo /usr/sbin/ntp-wait -v -n 100 sleep 1 START_TIME=`date +%s` } @@ -171,14 +158,34 @@ function remaining_time { # Create a script to reproduce this build function reproduce { + local xtrace=$(set +o | grep xtrace) + set +o xtrace + + JOB_PROJECTS=$1 cat > $WORKSPACE/logs/reproduce.sh <> $WORKSPACE/logs/reproduce.sh + # first get all keys that match our filter and then output the whole line + # that will ensure that multi-line env vars are set properly + for KEY in $(printenv | grep '\(DEVSTACK\|ZUUL\)' | sed 's/\(.*\)=.*/\1/'); do + echo "declare -x ${KEY}=\"${!KEY}\"" >> $WORKSPACE/logs/reproduce.sh + done + if [ -n "$JOB_PROJECTS" ] ; then + echo "declare -x PROJECTS=\"$JOB_PROJECTS\"" >> $WORKSPACE/logs/reproduce.sh + fi cat >> $WORKSPACE/logs/reproduce.sh <$TEMPFILE @@ -668,6 +696,39 @@ function process_testr_artifacts { fi } +function process_stackviz { + local project=$1 + local path_prefix=${2:-new} + + local project_path=$BASE/$path_prefix/$project + local log_path=$BASE/logs + if [[ "$path_prefix" != "new" ]]; then + log_path=$BASE/logs/$path_prefix + fi + + local stackviz_path=/opt/stackviz + if [ -d $stackviz_path/build ]; then + sudo pip install -U $stackviz_path + + # static html+js should be prebuilt during image creation + cp -r $stackviz_path/build $log_path/stackviz + + pushd $project_path + if [ -f $BASE/new/dstat-csv.txt ]; then + sudo testr last --subunit | stackviz-export \ + --dstat $BASE/new/dstat-csv.txt \ + --end --stdin \ + $log_path/stackviz/data + else + sudo testr last --subunit | stackviz-export \ + --env --stdin \ + $log_path/stackviz/data + fi + sudo chown -R $USER:$USER $log_path/stackviz + popd + fi +} + function cleanup_host { # TODO: clean this up to be errexit clean local errexit=$(set +o | grep errexit) @@ -683,15 +744,16 @@ function cleanup_host { sleep 2 # No matter what, archive logs and config files - if uses_debs; then - sudo cp /var/log/syslog $BASE/logs/syslog.txt - sudo cp /var/log/kern.log $BASE/logs/kern_log.txt - elif is_fedora; then + if which journalctl ; then # the journal gives us syslog() and kernel output, so is like # a concatenation of the above. sudo journalctl --no-pager \ --since="$(cat $BASE/log-start-timestamp.txt)" \ | sudo tee $BASE/logs/syslog.txt > /dev/null + else + # assume rsyslog + sudo cp /var/log/syslog $BASE/logs/syslog.txt + sudo cp /var/log/kern.log $BASE/logs/kern_log.txt fi # apache logs; including wsgi stuff like horizon, keystone, etc. @@ -731,8 +793,10 @@ function cleanup_host { sudo cp /etc/sudoers $BASE/logs/sudoers.txt # Archive config files + # NOTE(mriedem): 'openstack' is added separately since it's not a project + # but it is where clouds.yaml is stored in dsvm runs that use it. sudo mkdir $BASE/logs/etc/ - for PROJECT in $PROJECTS; do + for PROJECT in $PROJECTS openstack; do proj=`basename $PROJECT` if [ -d /etc/$proj ]; then sudo cp -r /etc/$proj $BASE/logs/etc/ @@ -764,10 +828,16 @@ function cleanup_host { sudo cp $BASE/old/devstacklog.txt $BASE/logs/old/ sudo cp $BASE/old/devstack/localrc $BASE/logs/old/localrc.txt sudo cp $BASE/old/tempest/etc/tempest.conf $BASE/logs/old/tempest_conf.txt - if -f [ $BASE/old/devstack/tempest.log ] ; then + if [ -f $BASE/old/devstack/tempest.log ] ; then sudo cp $BASE/old/devstack/tempest.log $BASE/logs/old/verify_tempest_conf.log fi + # Copy Ironic nodes console logs if they exist + if [ -d $BASE/old/ironic-bm-logs ] ; then + sudo mkdir -p $BASE/logs/old/ironic-bm-logs + sudo cp $BASE/old/ironic-bm-logs/*.log $BASE/logs/old/ironic-bm-logs/ + fi + # dstat CSV log if [ -f $BASE/old/dstat-csv.log ]; then sudo cp $BASE/old/dstat-csv.log $BASE/logs/old/ @@ -831,6 +901,9 @@ function cleanup_host { # Copy tempest config file sudo cp $BASE/new/tempest/etc/tempest.conf $NEWLOGTARGET/tempest_conf.txt + if [ -f $BASE/new/tempest/etc/accounts.yaml ] ; then + sudo cp $BASE/new/tempest/etc/accounts.yaml $NEWLOGTARGET/accounts_yaml.txt + fi # Copy dstat CSV log if it exists if [ -f $BASE/new/dstat-csv.log ]; then @@ -839,9 +912,14 @@ function cleanup_host { sudo iptables-save > $WORKSPACE/iptables.txt df -h > $WORKSPACE/df.txt - pip freeze > $WORKSPACE/pip-freeze.txt - sudo mv $WORKSPACE/iptables.txt $WORKSPACE/df.txt \ - $WORKSPACE/pip-freeze.txt $BASE/logs/ + sudo mv $WORKSPACE/iptables.txt $WORKSPACE/df.txt $BASE/logs/ + + for py_ver in 2 3; do + if [[ `which python${py_ver}` ]]; then + python${py_ver} -m pip freeze > $WORKSPACE/pip${py_ver}-freeze.txt + sudo mv $WORKSPACE/pip${py_ver}-freeze.txt $BASE/logs/ + fi + done if [ `command -v dpkg` ]; then dpkg -l> $WORKSPACE/dpkg-l.txt @@ -854,6 +932,8 @@ function cleanup_host { sudo mv $WORKSPACE/rpm-qa.txt.gz $BASE/logs/ fi + process_stackviz tempest + process_testr_artifacts tempest process_testr_artifacts tempest old @@ -878,7 +958,23 @@ function cleanup_host { # Make sure the current user can read all the logs and configs sudo chown -R $USER:$USER $BASE/logs/ - sudo chmod a+r $BASE/logs/ $BASE/logs/etc + # (note X not x ... execute/search only if the file is a directory + # or already has execute permission for some user) + sudo chmod -R a+rX $BASE/logs/ + + # Collect all the deprecation related messages into a single file. + # strip out date(s), timestamp(s), pid(s), context information and + # remove duplicates as well so we have a limited set of lines to + # look through. The fancy awk is used instead of a "sort | uniq -c" + # to preserve the order in which we find the lines in a specific + # log file. + grep -i deprecat $BASE/logs/*.log $BASE/logs/apache/*.log | \ + sed -r 's/[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}\.[0-9]{1,3}/ /g' | \ + sed -r 's/[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}/ /g' | \ + sed -r 's/[0-9]{1,4}-[0-9]{1,2}-[0-9]{1,4}/ /g' | + sed -r 's/\[.*\]/ /g' | \ + sed -r 's/\s[0-9]+\s/ /g' | \ + awk '{if ($0 in seen) {seen[$0]++} else {out[++n]=$0;seen[$0]=1}} END { for (i=1; i<=n; i++) print seen[out[i]]" :: " out[i] }' > $BASE/logs/deprecations.log # rename files to .txt; this is so that when displayed via # logs.openstack.org clicking results in the browser shows the @@ -969,7 +1065,7 @@ function enable_netconsole { # out to the world is specify the default gw as the remote # destination. local default_gw=$(ip route | grep default | awk '{print $3}') - local gw_mac=$(arp $default_gw | grep $default_gw | awk '{print $3}') + local gw_mac=$(arp -n $default_gw | grep $default_gw | awk '{print $3}') local gw_dev=$(ip route | grep default | awk '{print $5}') # turn up message output @@ -1107,3 +1203,8 @@ function with_timeout { function iniset { $(source $BASE/new/devstack/inc/ini-config; iniset $@) } + +# Iniget imported from devstack +function iniget { + $(source $BASE/new/devstack/inc/ini-config; iniget $@) +} diff --git a/help/tempest-logs.html b/help/tempest-logs.html index e76c339a..cfdc2adb 100644 --- a/help/tempest-logs.html +++ b/help/tempest-logs.html @@ -73,7 +73,7 @@

Types of logs

  • screen-q-meta.txt.gz: neutron-metadata-agent
  • screen-q-metering.txt.gz: neutron-metering-agent
  • screen-q-svc.txt.gz: neutron-server -
  • screen-q-vpn.txt.gz: neutron-vpn-agent +
  • screen-q-l3.txt.gz: neutron-l3-agent
  • swift
      diff --git a/multinode_setup_info.txt b/multinode_setup_info.txt index 70b4bc81..31fc7487 100644 --- a/multinode_setup_info.txt +++ b/multinode_setup_info.txt @@ -38,9 +38,9 @@ Nova Network ||br_pub| | br_flat| | ||br_pub | |br_flat| | ||br_pub | |br_flat| | |+--+---+ +---+----+ | |+---+---+ +---+---+ | |+---+---+ +---+---+ | | | | | | | | | | | | | -| | +-------------------gre-tunnel--+-----------------gre-tunnel---+ | +| | +------------------vxlan-tunnel-+-----------------vxlan-tunnel-+ | | | | | | | | | | -| +----------gre-tunnel-----------+----------gre-tunnel----------+ | +| +--------vxlan-tunnel-----------+--------vxlan-tunnel----------+ | | | | | | | +--------------------------+ +--------------------------+ +--------------------------+ @@ -72,9 +72,9 @@ This is what it all looks like after you run devstack and boot some nodes. ||br_pub| | br_flat| | ||br_pub | |br_flat| | ||br_pub | |br_flat| | |+--+---+ +---+----+ | |+---+---+ +---+---+ | |+---+---+ +---+---+ | | | | | | | | | | | | | -| | +-------------------gre-tunnel--+-----------------gre-tunnel---+ | +| | +------------------vxlan-tunnel-+-----------------vxlan-tunnel-+ | | | | | | | | | | -| +----------gre-tunnel-----------+----------gre-tunnel----------+ | +| +--------vxlan-tunnel-----------+--------vxlan-tunnel----------+ | | | | | | | +--------------------------+ +--------------------------+ +--------------------------+ @@ -104,7 +104,7 @@ happens here. |+--+---+ | |+---+---+ | |+---+---+ | | | | | | | | | | | | | | | | | | | -| +----------gre-tunnel-----------+----------gre-tunnel----------+ | +| +--------vxlan-tunnel-----------+--------vxlan-tunnel----------+ | | | | | | | +--------------------------+ +--------------------------+ +--------------------------+ @@ -139,6 +139,6 @@ This is what it all looks like after you run devstack and boot some nodes. |+--+---+ | |+---+---+ | |+---+---+ | | | | | | | | | | | | | | | | | | | -| +----------gre-tunnel-----------+----------gre-tunnel----------+ | +| +--------vxlan-tunnel-----------+--------vxlan-tunnel----------+ | | | | | | | +--------------------------+ +--------------------------+ +--------------------------+ diff --git a/test-features.sh b/test-features.sh index cb1d3116..a7d75469 100755 --- a/test-features.sh +++ b/test-features.sh @@ -18,11 +18,11 @@ ERRORS=0 TEMPEST_FULL_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,n-net" -TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-fwaas,q-metering" +TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-metering" -TEMPEST_NEUTRON_KILO="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-fwaas,q-metering,q-vpn" +TEMPEST_NEUTRON_KILO="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-metering,q-vpn" -TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-fwaas,q-metering" +TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-metering" GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" diff --git a/test-functions.sh b/test-functions.sh index 80295eec..bebba624 100755 --- a/test-functions.sh +++ b/test-functions.sh @@ -515,6 +515,51 @@ function test_call_hook_if_defined { rm -rf $save_dir } +# test that reproduce file is populated correctly +function test_reproduce { + # expected result + read -d '' EXPECTED_VARS << EOF +declare -x ZUUL_VAR="zuul-var" +declare -x DEVSTACK_VAR="devstack-var" +declare -x ZUUL_VAR_MULTILINE="zuul-var-setting1 +zuul-var-setting2" +declare -x DEVSTACK_VAR_MULTILINE="devstack-var-setting1 +devstack-var-setting2" +EOF + + # prepare environment for test + WORKSPACE=. + export DEVSTACK_VAR=devstack-var + export DEVSTACK_VAR_MULTILINE="devstack-var-setting1 +devstack-var-setting2" + export ZUUL_VAR=zuul-var + export ZUUL_VAR_MULTILINE="zuul-var-setting1 +zuul-var-setting2" + JOB_NAME=test-job + mkdir $WORKSPACE/logs + + # execute call and assert + reproduce + + [[ -e $WORKSPACE/logs/reproduce.sh ]] + file_exists=$? + assert_equal $file_exists 0 + + result_expected=`cat $WORKSPACE/logs/reproduce.sh | grep "$EXPECTED_VARS"` + [[ ${#result_expected} -eq "0" ]] + assert_equal $? 1 + + # clean up environment + rm -rf $WORKSPACE/logs + rm -rf $WORKSPACE/workspace + unset WORKSPACE + unset DEVSTACK_VAR + unset DEVSTACK_VAR_MULTILINE + unset ZUUL_VAR + unset ZUUL_VAR_MULTILINE + unset JOB_NAME +} + # Run tests: #set -o xtrace test_branch_override @@ -530,6 +575,7 @@ test_periodic_no_branch test_two_on_master test_workspace_branch_arg test_call_hook_if_defined +test_reproduce if [[ ! -z "$ERROR" ]]; then echo diff --git a/test-matrix.py b/test-matrix.py index 48fff355..17170f2f 100755 --- a/test-matrix.py +++ b/test-matrix.py @@ -65,7 +65,7 @@ def normalize_branch(branch): def configs_from_env(): configs = [] - for k, v in os.environ.iteritems(): + for k, v in os.environ.items(): if k.startswith('DEVSTACK_GATE_'): if v not in FALSE_VALUES: f = k.split('DEVSTACK_GATE_')[1] @@ -145,7 +145,7 @@ def main(): LOG.debug("Services: %s " % services) if opts.mode == "services": - print ",".join(services) + print(",".join(services)) if __name__ == "__main__": diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000..d46548ad --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1 @@ +PyYAML>=3.1.0 diff --git a/tox.ini b/tox.ini index 303fbcca..cf770882 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,11 @@ skipsdist = True install_command = pip install -U {opts} {packages} setenv = VIRTUAL_ENV={envdir} +[testenv:run-tests] +deps = -r{toxinidir}/test-requirements.txt +commands = + bash -c "./run-tests.sh" + [testenv:bashate] whitelist_externals = bash