From 4f4bc22510c1458b2012198defee7983f43e29b1 Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Thu, 29 Oct 2015 10:23:37 -0700 Subject: [PATCH 001/148] Start using CINDER_VOLUME_CLEAR to replace CINDER_SECURE_DELETE CINDER_SECURE_DELETE was deprecated in devstack in liberty with dfcc3871c022516330b6afb6c74751ac42a87480. Start using CINDER_VOLUME_CLEAR (default to 'none' to avoid the same bug that CINDER_SECURE_DELETE is used for). Add a TODO about removing DEVSTACK_CINDER_SECURE_DELETE once kilo-eol happens in devstack. Change-Id: I131c1aee665d6bd4e38dcfd0aec1b4796adc64c5 --- devstack-vm-gate-wrap.sh | 7 +++++++ devstack-vm-gate.sh | 1 + 2 files changed, 8 insertions(+) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index ff3b3a75..29b9d575 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -176,8 +176,15 @@ export DEVSTACK_GATE_TEMPEST_DISABLE_TENANT_ISOLATION=${DEVSTACK_GATE_TEMPEST_DI # Set to 1 to enable Cinder secure delete. # False by default to avoid dd problems on Precise. # https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1023755 +# TODO(mriedem): CINDER_SECURE_DELETE is deprecated in devstack as of liberty. +# Remove after kilo-eol happens in devstack. export DEVSTACK_CINDER_SECURE_DELETE=${DEVSTACK_CINDER_SECURE_DELETE:-0} +# Should cinder perform secure deletion of volumes? +# Defaults to none to avoid bug 1023755. Can also be set to zero or shred. +# Only applicable to stable/liberty+ devstack. +export DEVSTACK_CINDER_VOLUME_CLEAR=${DEVSTACK_CINDER_VOLUME_CLEAR:-none} + # Set to 1 to run neutron instead of nova network # Only applicable to master branch export DEVSTACK_GATE_NEUTRON=${DEVSTACK_GATE_NEUTRON:-0} diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 82b2f88b..11cbb620 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -308,6 +308,7 @@ EOF if [[ "$DEVSTACK_CINDER_SECURE_DELETE" -eq "0" ]]; then echo "CINDER_SECURE_DELETE=False" >>"$localrc_file" fi + echo "CINDER_VOLUME_CLEAR=${DEVSTACK_CINDER_VOLUME_CLEAR}" >>"$localrc_file" if [[ "$DEVSTACK_GATE_TEMPEST_HEAT_SLOW" -eq "1" ]]; then echo "HEAT_CREATE_TEST_IMAGE=False" >>"$localrc_file" From 78591f5d9ab34a96eba4e06a6ad4ce98573509c8 Mon Sep 17 00:00:00 2001 From: "John L. Villalovos" Date: Tue, 5 Jan 2016 14:59:49 -0800 Subject: [PATCH 002/148] TRIVIAL: Fix some misspellings Fix some misspellings Change-Id: Id9ac593a585f8d348667e6f208b8327d72502d17 --- functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions.sh b/functions.sh index 489c6172..7cdd3665 100644 --- a/functions.sh +++ b/functions.sh @@ -99,7 +99,7 @@ function tsfilter { function _ping_check { local host=$1 local times=${2:-20} - echo "Testing ICMP connectivit to $host" + echo "Testing ICMP connectivity to $host" ping -c $times $host } @@ -554,7 +554,7 @@ function setup_host { # Create a stack user for devstack to run as, so that we can # revoke sudo permissions from that user when appropriate. sudo useradd -U -s /bin/bash -d $BASE/new -m stack - # Use 755 mode on the user dir regarless to the /etc/login.defs setting + # Use 755 mode on the user dir regardless of the /etc/login.defs setting sudo chmod 755 $BASE/new TEMPFILE=`mktemp` echo "stack ALL=(root) NOPASSWD:ALL" >$TEMPFILE From 4ee1b9d78e70762bae0c6f7ae0b608c51f1053f9 Mon Sep 17 00:00:00 2001 From: OpenStack Jenkins Date: Mon, 8 Jun 2015 14:42:45 +0000 Subject: [PATCH 003/148] Add tls-proxy as an available service devstack can configure a TLS proxy for a number of services if the tls-proxy service is enabled. Make this service available to the gate. Change-Id: I654f06c34a60e56ff98b9b6720d0f9a3fe5d240b --- features.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/features.yaml b/features.yaml index 50b35f19..dcb45078 100644 --- a/features.yaml +++ b/features.yaml @@ -43,6 +43,8 @@ config: features: [ceph] heat: features: [heat] + tlsproxy: + features: [tlsproxy] branches: # The value of ""default" is the name of the "trunk" branch @@ -154,3 +156,7 @@ features: ceph: base: services: [ceph] + + tlsproxy: + base: + services: [tls-proxy] From ad2170adf3ab23910ab0991fb83bb2ea04948eb0 Mon Sep 17 00:00:00 2001 From: "John L. Villalovos" Date: Fri, 22 Jan 2016 10:21:29 -0800 Subject: [PATCH 004/148] Fix '-f: command not found' error The left bracket '[' was incorrectly placed. And '-f' was trying to be executed instead of the test executable '[' Change-Id: Ib947fa6aab651e427e44c8edc3552a02f3b505a4 --- functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index 1d380177..aa122d63 100644 --- a/functions.sh +++ b/functions.sh @@ -744,7 +744,7 @@ function cleanup_host { sudo cp $BASE/old/devstacklog.txt $BASE/logs/old/ sudo cp $BASE/old/devstack/localrc $BASE/logs/old/localrc.txt sudo cp $BASE/old/tempest/etc/tempest.conf $BASE/logs/old/tempest_conf.txt - if -f [ $BASE/old/devstack/tempest.log ] ; then + if [ -f $BASE/old/devstack/tempest.log ] ; then sudo cp $BASE/old/devstack/tempest.log $BASE/logs/old/verify_tempest_conf.log fi From 98ccf4037ead6c8abd0abf1c36fe78b4c6423e73 Mon Sep 17 00:00:00 2001 From: "John L. Villalovos" Date: Fri, 22 Jan 2016 10:40:03 -0800 Subject: [PATCH 005/148] Update IRC channel to use #openstack-qa Change #openstack-dev to #openstack-qa in README.rst Use #openstack-qa as that is the channel that patch submission notices get broadcast on. Change-Id: I8ea331bbb325182e465f68f91930886822ba3c9c --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a8e04689..c10504e3 100644 --- a/README.rst +++ b/README.rst @@ -253,4 +253,4 @@ You can file bugs on the storyboard devstack-gate project:: https://storyboard.openstack.org/#!/project/712 -And you can chat with us on Freenode in #openstack-dev or #openstack-infra. +And you can chat with us on Freenode in #openstack-qa or #openstack-infra. From e16f854e388a9aff08a10c67d14a60321c012def Mon Sep 17 00:00:00 2001 From: "John L. Villalovos" Date: Fri, 22 Jan 2016 09:18:37 -0800 Subject: [PATCH 006/148] Set PS4 to provide additional debug info Add: export PS4='+ ${BASH_SOURCE:-}:${FUNCNAME[0]:-}:L${LINENO:-}: ' To devstack-vm-gate-wrap.sh to provide additional debug info. This will show the filename, line number, function name, and the content of the line. Same as how it is done in openstack/devstack An example output line: + ./safe-devstack-vm-gate-wrap.sh:main:L547: RETVAL=1 Change-Id: I2c745fd139be63d56ff89ab8e3f60eb795ab82ce --- devstack-vm-gate-wrap.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 235405d6..be3fea94 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -27,6 +27,8 @@ GIT_BRANCH=${GIT_BRANCH:-master} # sshd may have been compiled with a default path excluding */sbin export PATH=$PATH:/usr/local/sbin:/usr/sbin +# When doing xtrace (set -x / set -o xtrace), provide more debug output +export PS4='+ ${BASH_SOURCE:-}:${FUNCNAME[0]:-}:L${LINENO:-}: ' source $WORKSPACE/devstack-gate/functions.sh From 3630eae36ff75347c12598b35e091ba8c71ee04f Mon Sep 17 00:00:00 2001 From: Lucas Alvares Gomes Date: Wed, 24 Feb 2016 09:43:01 +0000 Subject: [PATCH 007/148] Add support for ipmitool driver in Ironic The commit 7ffa1d58ce9fd9e32cdab142a1f12feae38c3097 added support to configure nodes with ipmitool drivers in DevStack ironic to run with virtual machines. The main goal of that patch was to allow us to test the production ipmitool drivers in the gate replacing the *_ssh testing drivers in the future. Later a patch to project-config 2e577710e975983b594f6df59b504b4cae85e594 added two new jobs to the ironic gate: pxe_ipmitool and agent_ipmitool. This patch contains the final pieces to enable ipmitool drivers to run on gate. Change-Id: I2adff0d7195d3f53e19b600b2d91163a8271bf2c Related-Bug: #1544642 --- devstack-vm-gate.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 068a2fe4..a6e5c3d7 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -333,6 +333,7 @@ EOF if [[ "$DEVSTACK_GATE_VIRT_DRIVER" == "ironic" ]]; then export TEMPEST_OS_TEST_TIMEOUT=900 + echo "IRONIC_DEPLOY_DRIVER=$DEVSTACK_GATE_IRONIC_DRIVER" >>"$localrc_file" echo "VIRT_DRIVER=ironic" >>"$localrc_file" echo "IRONIC_BAREMETAL_BASIC_OPS=True" >>"$localrc_file" echo "IRONIC_VM_LOG_DIR=$BASE/$localrc_oldnew/ironic-bm-logs" >>"$localrc_file" @@ -344,17 +345,17 @@ EOF if [[ "$DEVSTACK_GATE_IRONIC_BUILD_RAMDISK" -eq 0 ]]; then echo "IRONIC_BUILD_DEPLOY_RAMDISK=False" >>"$localrc_file" fi - if [[ "$DEVSTACK_GATE_IRONIC_DRIVER" == "agent_ssh" ]]; then + if [[ -z "${DEVSTACK_GATE_IRONIC_DRIVER%%agent*}" ]]; then echo "SWIFT_ENABLE_TEMPURLS=True" >>"$localrc_file" echo "SWIFT_TEMPURL_KEY=secretkey" >>"$localrc_file" echo "IRONIC_ENABLED_DRIVERS=fake,agent_ssh,agent_ipmitool" >>"$localrc_file" - echo "IRONIC_DEPLOY_DRIVER=agent_ssh" >>"$localrc_file" # agent driver doesn't support ephemeral volumes yet echo "IRONIC_VM_EPHEMERAL_DISK=0" >>"$localrc_file" # agent CoreOS ramdisk is a little heavy echo "IRONIC_VM_SPECS_RAM=1024" >>"$localrc_file" echo "IRONIC_VM_COUNT=1" >>"$localrc_file" else + echo "IRONIC_ENABLED_DRIVERS=fake,pxe_ssh,pxe_ipmitool" >>"$localrc_file" echo "IRONIC_VM_EPHEMERAL_DISK=1" >>"$localrc_file" echo "IRONIC_VM_COUNT=3" >>"$localrc_file" fi @@ -708,7 +709,7 @@ if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then sudo chmod -R o+rx $BASE/new/devstack/files fi - # In the future we might want to increase the number of compute nodes. + # In the future we might want to increase the number of compute nodes. # This will ensure that multinode jobs consist of 2 nodes. # As a part of tempest configuration, it should be executed # before the DEVSTACK_GATE_TEMPEST_NOTESTS check, because the DEVSTACK_GATE_TEMPEST From e0fa9fced95a55d4b354714a7ff25abfec226e7a Mon Sep 17 00:00:00 2001 From: Jeremy Stanley Date: Tue, 1 Mar 2016 23:10:57 +0000 Subject: [PATCH 008/148] Update multinode setup doc to VXLAN We switched from GRE to VXLAN in the implementation as of I951e7e1bd98eeaf347e85296c12e027e225c2c97 so the diagrams should match accordingly. Change-Id: I0b0cef3a7217530573e694de54bf6dc68e295856 --- multinode_setup_info.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/multinode_setup_info.txt b/multinode_setup_info.txt index 70b4bc81..31fc7487 100644 --- a/multinode_setup_info.txt +++ b/multinode_setup_info.txt @@ -38,9 +38,9 @@ Nova Network ||br_pub| | br_flat| | ||br_pub | |br_flat| | ||br_pub | |br_flat| | |+--+---+ +---+----+ | |+---+---+ +---+---+ | |+---+---+ +---+---+ | | | | | | | | | | | | | -| | +-------------------gre-tunnel--+-----------------gre-tunnel---+ | +| | +------------------vxlan-tunnel-+-----------------vxlan-tunnel-+ | | | | | | | | | | -| +----------gre-tunnel-----------+----------gre-tunnel----------+ | +| +--------vxlan-tunnel-----------+--------vxlan-tunnel----------+ | | | | | | | +--------------------------+ +--------------------------+ +--------------------------+ @@ -72,9 +72,9 @@ This is what it all looks like after you run devstack and boot some nodes. ||br_pub| | br_flat| | ||br_pub | |br_flat| | ||br_pub | |br_flat| | |+--+---+ +---+----+ | |+---+---+ +---+---+ | |+---+---+ +---+---+ | | | | | | | | | | | | | -| | +-------------------gre-tunnel--+-----------------gre-tunnel---+ | +| | +------------------vxlan-tunnel-+-----------------vxlan-tunnel-+ | | | | | | | | | | -| +----------gre-tunnel-----------+----------gre-tunnel----------+ | +| +--------vxlan-tunnel-----------+--------vxlan-tunnel----------+ | | | | | | | +--------------------------+ +--------------------------+ +--------------------------+ @@ -104,7 +104,7 @@ happens here. |+--+---+ | |+---+---+ | |+---+---+ | | | | | | | | | | | | | | | | | | | -| +----------gre-tunnel-----------+----------gre-tunnel----------+ | +| +--------vxlan-tunnel-----------+--------vxlan-tunnel----------+ | | | | | | | +--------------------------+ +--------------------------+ +--------------------------+ @@ -139,6 +139,6 @@ This is what it all looks like after you run devstack and boot some nodes. |+--+---+ | |+---+---+ | |+---+---+ | | | | | | | | | | | | | | | | | | | -| +----------gre-tunnel-----------+----------gre-tunnel----------+ | +| +--------vxlan-tunnel-----------+--------vxlan-tunnel----------+ | | | | | | | +--------------------------+ +--------------------------+ +--------------------------+ From 2a65d2ba8671a5e910cc69833d6e7a81d4a55185 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Thu, 3 Mar 2016 08:20:27 -0800 Subject: [PATCH 009/148] Set the bridge MTU dynamically Don't always assume our MTU will always be 1500 when calculating the MTUs of overlays running inside our test nodes. Instead determine our smallest interface MTU, subtract 50 from that for vxlan overhead (and GRE too since GRE has less overhead than vxlan), and use this new number on the overlay networks. Change-Id: If5facee99f03c0ec4773241f872477b922a6674e --- devstack-vm-gate.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 068a2fe4..736c1bab 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -50,7 +50,11 @@ PUBLIC_NETWORK_GATEWAY=${DEVSTACK_GATE_PUBLIC_NETWORK_GATEWAY:-172.24.5.1} FLOATING_HOST_PREFIX=${DEVSTACK_GATE_FLOATING_HOST_PREFIX:-172.24.4} FLOATING_HOST_MASK=${DEVSTACK_GATE_FLOATING_HOST_MASK:-23} -EXTERNAL_BRIDGE_MTU=1450 +# Get the smallest local MTU +LOCAL_MTU=$(ip link show | sed -ne 's/.*mtu \([0-9]\+\).*/\1/p' | sort -n | head -1) +# 50 bytes is overhead for vxlan (which is greater than GRE +# allowing us to use either overlay option with this MTU. +EXTERNAL_BRIDGE_MTU=$((LOCAL_MTU - 50)) function setup_ssh { local path=$1 From ceb9cd120b3f2129d4322c0fc283971b4dc6aec8 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 8 Mar 2016 10:21:21 -0500 Subject: [PATCH 010/148] Run tests in venv Previously we had wrongly assumed that python yaml package will be available in the images and the run-tests.sh script ran a set of things which included a script that used the pyyaml package. With the recent bindep changes, this assumption broke and the gate-devstack-gate-unittests failed. So we need to install the pyyaml package and then run the tests in this venv for run-tests.sh to work. Change-Id: I541c8add8041348a259b3b789d24cf24ba122ce0 --- test-requirements.txt | 1 + tox.ini | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 test-requirements.txt diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000..d46548ad --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1 @@ +PyYAML>=3.1.0 diff --git a/tox.ini b/tox.ini index 303fbcca..cf770882 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,11 @@ skipsdist = True install_command = pip install -U {opts} {packages} setenv = VIRTUAL_ENV={envdir} +[testenv:run-tests] +deps = -r{toxinidir}/test-requirements.txt +commands = + bash -c "./run-tests.sh" + [testenv:bashate] whitelist_externals = bash From ea710eeb58a9f0c947826ffa520d7955b40efb3a Mon Sep 17 00:00:00 2001 From: "John L. Villalovos" Date: Tue, 8 Mar 2016 14:45:03 -0800 Subject: [PATCH 011/148] Archive Ironic VM nodes console logs for 'old' If the ironic VM node console logs exist in the /opt/stack/old/ directory then archive those logs as part of cleanup_host(). Change-Id: I5349f33c94654534825a7b0a88eb2e5bc8092394 --- functions.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/functions.sh b/functions.sh index 6bb9af06..49b9a290 100644 --- a/functions.sh +++ b/functions.sh @@ -768,6 +768,12 @@ function cleanup_host { sudo cp $BASE/old/devstack/tempest.log $BASE/logs/old/verify_tempest_conf.log fi + # Copy Ironic nodes console logs if they exist + if [ -d $BASE/old/ironic-bm-logs ] ; then + sudo mkdir -p $BASE/logs/old/ironic-bm-logs + sudo cp $BASE/old/ironic-bm-logs/*.log $BASE/logs/old/ironic-bm-logs/ + fi + # dstat CSV log if [ -f $BASE/old/dstat-csv.log ]; then sudo cp $BASE/old/dstat-csv.log $BASE/logs/old/ From 416f60410613a42c9fabe44f9dde1caddde9573e Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 8 Mar 2016 07:43:22 -0500 Subject: [PATCH 012/148] Generate a log with deprecation warnings Makes it easier for folks to go look in one spot for deprecation warnings spit out in various process(es). This will also help us warn folks to go fix deprecated stuff before they become a problem. Change-Id: I757821bd6f19a168888df92d416d7af16f41637b --- functions.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/functions.sh b/functions.sh index 6bb9af06..5e13a55c 100644 --- a/functions.sh +++ b/functions.sh @@ -880,6 +880,21 @@ function cleanup_host { sudo chown -R $USER:$USER $BASE/logs/ sudo chmod a+r $BASE/logs/ $BASE/logs/etc + + # Collect all the deprecation related messages into a single file. + # strip out date(s), timestamp(s), pid(s), context information and + # remove duplicates as well so we have a limited set of lines to + # look through. The fancy awk is used instead of a "sort | uniq -c" + # to preserve the order in which we find the lines in a specific + # log file. + grep -i deprecat $BASE/logs/*.log | \ + sed -r 's/[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}\.[0-9]{1,3}/ /g' | \ + sed -r 's/[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}/ /g' | \ + sed -r 's/[0-9]{1,4}-[0-9]{1,2}-[0-9]{1,4}/ /g' | + sed -r 's/\[.*\]/ /g' | \ + sed -r 's/\s[0-9]+\s/ /g' | \ + awk '{if ($0 in seen) {seen[$0]++} else {out[++n]=$0;seen[$0]=1}} END { for (i=1; i<=n; i++) print seen[out[i]]" :: " out[i] }' > $BASE/logs/deprecations.log + # rename files to .txt; this is so that when displayed via # logs.openstack.org clicking results in the browser shows the # files, rather than trying to send it to another app or make you From d10c22f360fb28e519fde76fbc9c9c568e951baa Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 9 Mar 2016 12:09:09 -0500 Subject: [PATCH 013/148] add mitaka and ocata branches to feature list Change-Id: Ie4f86c812d4de5cd5987833f377efde4921875b3 Signed-off-by: Doug Hellmann --- features.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/features.yaml b/features.yaml index dcb45078..9f300052 100644 --- a/features.yaml +++ b/features.yaml @@ -1,6 +1,8 @@ config: default: master: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] + ocata: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] + mitaka: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] liberty: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] kilo: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] juno: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] @@ -50,7 +52,7 @@ branches: # The value of ""default" is the name of the "trunk" branch default: master # Normalized branch names only here, e.g. stable/icehouse => icehouse - allowed: [master, liberty, kilo, juno, icehouse] + allowed: [master, ocata, mitaka, liberty, kilo, juno, icehouse] features: default: From 23f2dc4fb4c9c2122261a28fed1bd373b5615120 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Thu, 10 Mar 2016 12:00:38 -0500 Subject: [PATCH 014/148] Log deprecations in horizon and keystone Oops! missed the location for the Apache based logs (horizon and keystone). Change-Id: I7d20e95b3ed9f37bfe04865e8809795964a49455 --- functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index 5e13a55c..9beb38c2 100644 --- a/functions.sh +++ b/functions.sh @@ -887,7 +887,7 @@ function cleanup_host { # look through. The fancy awk is used instead of a "sort | uniq -c" # to preserve the order in which we find the lines in a specific # log file. - grep -i deprecat $BASE/logs/*.log | \ + grep -i deprecat $BASE/logs/*.log $BASE/logs/apache/*.log | \ sed -r 's/[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}\.[0-9]{1,3}/ /g' | \ sed -r 's/[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}/ /g' | \ sed -r 's/[0-9]{1,4}-[0-9]{1,2}-[0-9]{1,4}/ /g' | From 75966977607404026e5e8c872abc41be075c2657 Mon Sep 17 00:00:00 2001 From: David Moreau Simard Date: Wed, 9 Mar 2016 13:51:50 -0500 Subject: [PATCH 015/148] Use dd to create swapfile, ensure swap amount == 8GB It is not possible to use a swapfile generated by fallocate on a xfs filesystem which is shipped by default on CentOS >7. So since [1], we effectively ran jobs on CentOS VMs without any swap. Use dd as a way of creating the swapfile that works on both Ubuntu and CentOS with a slight performance hit and ensure it's mounted properly so we don't have a silent failure. Also manage a new specific case, Vexxhost VMs ship with 2GB of swap which is not in line with the expectation that all VMs should have 8GB. If there is less than 8GB of swap (or zero), we will make up for the difference. [1]: https://review.openstack.org/#/c/278557/ Change-Id: Iaf4de66475d747f62bd49dfe7318131cbeb03be8 --- functions.sh | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/functions.sh b/functions.sh index 6bb9af06..094208d3 100644 --- a/functions.sh +++ b/functions.sh @@ -314,14 +314,19 @@ function fix_etc_hosts { } function fix_disk_layout { - # HPCloud and Rackspace performance nodes provide no swap, but do have - # ephemeral disks we can use. For providers with no ephemeral disks, such - # as OVH or Internap, create and use a sparse swapfile on the root - # filesystem. - # HPCloud also doesn't have enough space on / for two devstack installs, + # Don't attempt to fix disk layout more than once + [[ -e /etc/fixed_disk_layout ]] && return 0 || sudo touch /etc/fixed_disk_layout + + # Ensure virtual machines from different providers all have at least 8GB of + # swap. + # Use an ephemeral disk if there is one or create and use a swapfile. + # Rackspace also doesn't have enough space on / for two devstack installs, # so we partition the disk and mount it on /opt, syncing the previous # contents of /opt over. - if [ `grep SwapTotal /proc/meminfo | awk '{ print $2; }'` -eq 0 ]; then + SWAPSIZE=8192 + swapcurrent=$(( $(grep SwapTotal /proc/meminfo | awk '{ print $2; }') / 1024 )) + + if [[ $swapcurrent -lt $SWAPSIZE ]]; then if [ -b /dev/xvde ]; then DEV='/dev/xvde' else @@ -341,7 +346,7 @@ function fix_disk_layout { sudo umount ${DEV} fi sudo parted ${DEV} --script -- mklabel msdos - sudo parted ${DEV} --script -- mkpart primary linux-swap 1 8192 + sudo parted ${DEV} --script -- mkpart primary linux-swap 1 ${SWAPSIZE} sudo parted ${DEV} --script -- mkpart primary ext2 8192 -1 sudo mkswap ${DEV}1 sudo mkfs.ext4 ${DEV}2 @@ -350,15 +355,24 @@ function fix_disk_layout { sudo find /opt/ -mindepth 1 -maxdepth 1 -exec mv {} /mnt/ \; sudo umount /mnt sudo mount ${DEV}2 /opt + + # Sanity check + grep -q ${DEV}1 /proc/swaps || exit 1 + grep -q ${DEV}2 /proc/mounts || exit 1 else # If no ephemeral devices are available, use root filesystem # Don't use sparse device to avoid wedging when disk space and # memory are both unavailable. local swapfile='/root/swapfile' - sudo fallocate -l 8192M ${swapfile} + swapdiff=$(( $SWAPSIZE - $swapcurrent )) + + sudo dd if=/dev/zero of=${swapfile} bs=1M count=${swapdiff} sudo chmod 600 ${swapfile} sudo mkswap ${swapfile} sudo swapon ${swapfile} + + # Sanity check + grep -q ${swapfile} /proc/swaps || exit 1 fi fi From 589ca4f1c28990fd51dc78382542b0388ff2ebbc Mon Sep 17 00:00:00 2001 From: Yolanda Robla Date: Tue, 15 Mar 2016 11:13:40 +0100 Subject: [PATCH 016/148] Fix bug in populating multiline env vars on reproduce The script to populate reproduce.sh is failing when populating multi-line env vars. That is the case of the enable_plugin lines, when more than one plugin is enabled. Add this workaround to properly populate all needed settings. Also add a test to ensure that reproduce method is working as expected. Change-Id: I5287098a47162321efadbde7d069fa2a4e7eabf7 --- functions.sh | 6 +++++- test-functions.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index d34ebd44..d23bb4f0 100644 --- a/functions.sh +++ b/functions.sh @@ -178,7 +178,11 @@ exec 0> $WORKSPACE/logs/reproduce.sh + # first get all keys that match our filter and then output the whole line + # that will ensure that multi-line env vars are set properly + for KEY in $(printenv | grep '\(DEVSTACK\|ZUUL\)' | sed 's/\(.*\)=.*/\1/'); do + echo "declare -x ${KEY}=\"${!KEY}\"" >> $WORKSPACE/logs/reproduce.sh + done cat >> $WORKSPACE/logs/reproduce.sh < Date: Wed, 16 Mar 2016 12:38:11 -0600 Subject: [PATCH 017/148] Remove deletion of ~/workspace-cache Fix so that ~/workspace-cache is no longer deleted. Makes instructions in README work again. Change-Id: I71668132820137f50b32494bc78d58a80867be92 --- functions.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/functions.sh b/functions.sh index d34ebd44..a3f47aec 100644 --- a/functions.sh +++ b/functions.sh @@ -503,9 +503,6 @@ function setup_workspace { sudo mkdir -p $DEST sudo chown -R $USER:$USER $DEST - #TODO(jeblair): remove when this is no longer created by the image - rm -fr ~/workspace-cache/ - # The vm template update job should cache the git repos # Move them to where we expect: echo "Using branch: $base_branch" From a95902068e974d24518ea71b09a978039169ac85 Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Thu, 17 Mar 2016 15:10:55 -0700 Subject: [PATCH 018/148] Document prereqs/usage for reproduce script This documents the prerequsites and requirements for reproducing devstack-gate in Ubuntu Trusty. Known issue: Doesn't document this for CentOS/Fedora. Change-Id: I6c483d6af7afda4d4ebb914dc7bd6ee862ad874f --- functions.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/functions.sh b/functions.sh index d23bb4f0..e06e8b0b 100644 --- a/functions.sh +++ b/functions.sh @@ -173,6 +173,15 @@ function remaining_time { function reproduce { cat > $WORKSPACE/logs/reproduce.sh < Date: Fri, 18 Mar 2016 13:08:08 +0100 Subject: [PATCH 019/148] Added env variable for subnode conf for multinode As of now, there is no way to address 2 different set of configuration for the primary node and the subnodes when using the multinode tests. Hence, this patchset adds a new environment variable called DEVSTACK_SUBNODE_CONFIG that appends its content into the local.conf file of the subnodes (only). Besides, the existing DEVSTACK_LOCAL_CONFIG variable is still used to set up the primary node. Change-Id: I1603b747b2cf4767cd3ed71fe4d56ae54920d19e --- devstack-vm-gate.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 6b52ac23..59a88a77 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -516,8 +516,20 @@ EOF # a way to pass through arbitrary devstack config options so that # we don't need to add new devstack-gate options every time we # want to create a new config. - if [[ -n "$DEVSTACK_LOCAL_CONFIG" ]]; then - echo "$DEVSTACK_LOCAL_CONFIG" >>"$localrc_file" + if [[ "$role" = sub ]]; then + # If we are in a multinode environment, we may want to specify 2 + # different sets of plugins + if [[ -n "$DEVSTACK_SUBNODE_CONFIG" ]]; then + echo "$DEVSTACK_SUBNODE_CONFIG" >>"$localrc_file" + else + if [[ -n "$DEVSTACK_LOCAL_CONFIG" ]]; then + echo "$DEVSTACK_LOCAL_CONFIG" >>"$localrc_file" + fi + fi + else + if [[ -n "$DEVSTACK_LOCAL_CONFIG" ]]; then + echo "$DEVSTACK_LOCAL_CONFIG" >>"$localrc_file" + fi fi } From be06eea9ec7d2b947373e9209f353e11da4e2d11 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Thu, 24 Mar 2016 06:36:57 -0400 Subject: [PATCH 020/148] Adjust grenade branches for mitaka This makes grenade work with stable mitaka, both upgrading to it, and upgrading from it in master. Depends-On: Ic9fc5cde67691ce03b3d612d86da7207608e20b1 grenade fixes for OS_PROJECT_NAME Change-Id: I73d2f7f83f700e430d231235e6d2277100036cd5 --- devstack-vm-gate-wrap.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index e62b1256..58845408 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -269,6 +269,9 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then export GRENADE_NEW_BRANCH="stable/liberty" elif [[ "$GRENADE_BASE_BRANCH" == "stable/liberty" ]]; then export GRENADE_OLD_BRANCH="stable/liberty" + export GRENADE_NEW_BRANCH="stable/mitaka" + elif [[ "$GRENADE_BASE_BRANCH" == "stable/mitaka" ]]; then + export GRENADE_OLD_BRANCH="stable/mitaka" export GRENADE_NEW_BRANCH="$GIT_BRANCH" fi ;; @@ -296,8 +299,11 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then elif [[ "$GRENADE_BASE_BRANCH" == "stable/liberty" ]]; then export GRENADE_OLD_BRANCH="stable/kilo" export GRENADE_NEW_BRANCH="stable/liberty" + elif [[ "$GRENADE_BASE_BRANCH" == "stable/mitaka" ]]; then + export GRENADE_OLD_BRANCH="stable/kilo" + export GRENADE_NEW_BRANCH="stable/mitaka" else # master - export GRENADE_OLD_BRANCH="stable/liberty" + export GRENADE_OLD_BRANCH="stable/mitaka" export GRENADE_NEW_BRANCH="$GIT_BRANCH" fi ;; From 7447c61345924c586eaee54ede74dd1a0879ca33 Mon Sep 17 00:00:00 2001 From: "watanabe.isao" Date: Fri, 25 Mar 2016 09:27:30 +0900 Subject: [PATCH 021/148] Remove doubled configuration in local.conf When DEVSTACK_GATE_VIRT_DRIVER has been set to ironic, in function setup_localrc, VIRT_DRIVER=ironic will be set to local.conf twice. This fix remove the second write. Change-Id: I6c0e7649366e3567b9342e4f83489dbae0229c2b Closes-Bug: 1561279 --- devstack-vm-gate.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 6b52ac23..035f732a 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -338,7 +338,6 @@ EOF if [[ "$DEVSTACK_GATE_VIRT_DRIVER" == "ironic" ]]; then export TEMPEST_OS_TEST_TIMEOUT=900 echo "IRONIC_DEPLOY_DRIVER=$DEVSTACK_GATE_IRONIC_DRIVER" >>"$localrc_file" - echo "VIRT_DRIVER=ironic" >>"$localrc_file" echo "IRONIC_BAREMETAL_BASIC_OPS=True" >>"$localrc_file" echo "IRONIC_VM_LOG_DIR=$BASE/$localrc_oldnew/ironic-bm-logs" >>"$localrc_file" echo "DEFAULT_INSTANCE_TYPE=baremetal" >>"$localrc_file" From a9c64e30326ed5cca33517a76ccab7de5e132544 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Thu, 24 Mar 2016 18:31:18 -0700 Subject: [PATCH 022/148] Use the right stable branch Bad copy and paste, this should have been liberty. Change-Id: I42e0971bbbb2e14bf75f5399811122c64594661e --- devstack-vm-gate-wrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 58845408..09ffd955 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -300,7 +300,7 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then export GRENADE_OLD_BRANCH="stable/kilo" export GRENADE_NEW_BRANCH="stable/liberty" elif [[ "$GRENADE_BASE_BRANCH" == "stable/mitaka" ]]; then - export GRENADE_OLD_BRANCH="stable/kilo" + export GRENADE_OLD_BRANCH="stable/liberty" export GRENADE_NEW_BRANCH="stable/mitaka" else # master export GRENADE_OLD_BRANCH="stable/mitaka" From 258d8c86bda09065b4b27f57af1f98de588b2782 Mon Sep 17 00:00:00 2001 From: Tony Breeds Date: Tue, 29 Mar 2016 15:08:17 +1100 Subject: [PATCH 023/148] Log locale during setup_host A recent bug was due to the locale configuration of our nodes. While it is unlikely we'll have that issue again it's still worth logging the locale for future analysis. Change-Id: I25c945b2a1e13fb1012c653f56ecd5c14e320677 Related-Bug: 1560808 --- functions.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/functions.sh b/functions.sh index d23bb4f0..c0b8f9bd 100644 --- a/functions.sh +++ b/functions.sh @@ -566,6 +566,9 @@ function setup_host { echo "NProc has discovered $(nproc) CPUs" cat /proc/cpuinfo + # Capture locale configuration + locale + # This is necessary to keep sudo from complaining fix_etc_hosts From 4184fe2f3bdca813d5047abf9741a49e674721b6 Mon Sep 17 00:00:00 2001 From: vsaienko Date: Mon, 28 Mar 2016 23:49:05 +0300 Subject: [PATCH 024/148] Allow to pass TEMPEST_OS_TEST_TIMEOUT from the job 'ironic-multitenant-network' job build/delete 2 instances and can't be completed in current TEMPEST_OS_TEST_TIMEOUT 900 sec. This patch allow to pass TEMPEST_OS_TEST_TIMEOUT from the gate job via DEVSTACK_GATE_OS_TEST_TIMEOUT variable to Ironic jobs. If DEVSTACK_GATE_OS_TEST_TIMEOUT is not set, timeout will be configured to current value 900 sec. Change-Id: I3f0833a4d3371846cb67eeb1cfe0946abc3d3772 --- devstack-vm-gate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 6b52ac23..e9ccbb58 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -336,7 +336,7 @@ EOF fi if [[ "$DEVSTACK_GATE_VIRT_DRIVER" == "ironic" ]]; then - export TEMPEST_OS_TEST_TIMEOUT=900 + export TEMPEST_OS_TEST_TIMEOUT=${DEVSTACK_GATE_OS_TEST_TIMEOUT:-900} echo "IRONIC_DEPLOY_DRIVER=$DEVSTACK_GATE_IRONIC_DRIVER" >>"$localrc_file" echo "VIRT_DRIVER=ironic" >>"$localrc_file" echo "IRONIC_BAREMETAL_BASIC_OPS=True" >>"$localrc_file" From f34dab53e79bdd703a03da84f5f5ebe56a741a24 Mon Sep 17 00:00:00 2001 From: Tony Breeds Date: Mon, 15 Feb 2016 09:42:18 +1100 Subject: [PATCH 025/148] Save PROJECTS form the job defintion for use in reproduce.sh If the Jenkins Job definition adds items to the PROJECTS variable that are not included as we build up PROJECTS reproduce.sh will ultimately fail. The fix saves the PROJECTS as it is passed into devstack-gate-wrap and includes that in reproduce.sh. Change-Id: Iea45f53b8c26985cb0da9720a631fcf8725522cf Closes-Bug: bug/1544827 --- devstack-vm-gate-wrap.sh | 7 ++++++- functions.sh | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index e62b1256..6e18c5e8 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -37,6 +37,11 @@ source $WORKSPACE/devstack-gate/functions.sh start_timer +# Save the PROJECTS variable as it was passed in. This is needed for reproduce.sh +# incase the job definition contains items that are not in the "global" list +# below. +# See: https://bugs.launchpad.net/openstack-gate/+bug/1544827 +JOB_PROJECTS="$PROJECTS" PROJECTS="openstack-infra/devstack-gate $PROJECTS" PROJECTS="openstack-dev/devstack $PROJECTS" PROJECTS="openstack-dev/pbr $PROJECTS" @@ -468,7 +473,7 @@ $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m file \ -a "path='$WORKSPACE/logs' state=directory" # Record a file to reproduce this build -reproduce +reproduce "$JOB_PROJECTS" # Run ansible to do setup_host on all nodes. echo "Setting up the hosts" diff --git a/functions.sh b/functions.sh index d23bb4f0..eb8ce947 100644 --- a/functions.sh +++ b/functions.sh @@ -171,6 +171,7 @@ function remaining_time { # Create a script to reproduce this build function reproduce { + JOB_PROJECTS=$1 cat > $WORKSPACE/logs/reproduce.sh <> $WORKSPACE/logs/reproduce.sh done + if [ -n "$JOB_PROJECTS" ] ; then + echo "declare -x PROJECTS=\"$JOB_PROJECTS\"" >> $WORKSPACE/logs/reproduce.sh + fi cat >> $WORKSPACE/logs/reproduce.sh < Date: Wed, 30 Mar 2016 16:44:29 -0700 Subject: [PATCH 026/148] Use ntp-wait instead of ntpdate to set time ntpdate is deprecated and should not be used anymore. Move away from ntpdate and use ntp-wait to wait for ntpd to snychronize the time instead. Change-Id: I29a101eb9ea5c2d0da6b7762fe93366f0c0a8999 --- functions.sh | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/functions.sh b/functions.sh index af37ed5c..c489f2bf 100644 --- a/functions.sh +++ b/functions.sh @@ -140,20 +140,7 @@ function start_timer { # first make sure the time is right, so we don't go into crazy land # later if the system decides to apply an ntp date and we jump forward # 4 hrs (which has happened) - if is_fedora; then - local ntp_service='ntpd' - elif uses_debs; then - local ntp_service='ntp' - else - echo "Unsupported platform, can't determine ntp service" - exit 1 - fi - local default_ntp_server=$( - grep ^server /etc/ntp.conf | head -1 | awk '{print $2}') - local ntp_server=${NTP_SERVER:-$default_ntp_server} - sudo service $ntp_service stop - sudo /usr/sbin/ntpdate $ntp_server - sudo service $ntp_service start + sudo /usr/sbin/ntp-wait -v sleep 1 START_TIME=`date +%s` } From 4abc5d6f338041fc9aa1c60402b27512dfd31f72 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 30 Mar 2016 16:55:38 -0700 Subject: [PATCH 027/148] Use journald when it is present Previously we only assumed journald was present on fedora, but now it exists on centos7 and ubuntu xenial. Instead of using the distro to determine if journald is present test for the presence of journalctl. If we don't hvae a journalctl fall back to the old behavior and assume rsyslog is in use. Change-Id: I4e1a3bdda7d46722b0a16ebb0b442917d4cb9b78 --- functions.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/functions.sh b/functions.sh index af37ed5c..07bcaa76 100644 --- a/functions.sh +++ b/functions.sh @@ -586,7 +586,12 @@ function setup_host { sudo mkdir -p $BASE # Start with a fresh syslog - if uses_debs; then + if which journalctl ; then + # save timestamp and use journalctl to dump everything since + # then at the end + date +"%Y-%m-%d %H:%M:%S" | sudo tee $BASE/log-start-timestamp.txt + else + # Assume rsyslog, move old logs aside then restart the service. sudo stop rsyslog sudo mv /var/log/syslog /var/log/syslog-pre-devstack sudo mv /var/log/kern.log /var/log/kern_log-pre-devstack @@ -599,10 +604,6 @@ function setup_host { sudo chmod /var/log/kern.log --ref /var/log/kern_log-pre-devstack sudo chmod a+r /var/log/kern.log sudo start rsyslog - elif is_fedora; then - # save timestamp and use journalctl to dump everything since - # then at the end - date +"%Y-%m-%d %H:%M:%S" | sudo tee $BASE/log-start-timestamp.txt fi # Create a stack user for devstack to run as, so that we can @@ -714,15 +715,16 @@ function cleanup_host { sleep 2 # No matter what, archive logs and config files - if uses_debs; then - sudo cp /var/log/syslog $BASE/logs/syslog.txt - sudo cp /var/log/kern.log $BASE/logs/kern_log.txt - elif is_fedora; then + if which journalctl ; then # the journal gives us syslog() and kernel output, so is like # a concatenation of the above. sudo journalctl --no-pager \ --since="$(cat $BASE/log-start-timestamp.txt)" \ | sudo tee $BASE/logs/syslog.txt > /dev/null + else + # assume rsyslog + sudo cp /var/log/syslog $BASE/logs/syslog.txt + sudo cp /var/log/kern.log $BASE/logs/kern_log.txt fi # apache logs; including wsgi stuff like horizon, keystone, etc. From 3254d688df01e23cd56786b8f5b783e3980c1129 Mon Sep 17 00:00:00 2001 From: Doug Wiegley Date: Sun, 13 Mar 2016 17:21:44 -0600 Subject: [PATCH 028/148] Fail the test if the pre test hook fails Change-Id: I92f34d064c76da05a0ef7b9d458673543da3f40e --- devstack-vm-gate-wrap.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index e62b1256..61eec324 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -552,15 +552,23 @@ fi # devstack-vm-gate-wrap.sh will not automagically run the hooks on each node. # Run pre test hook if we have one with_timeout call_hook_if_defined "pre_test_hook" +GATE_RETVAL=$? +if [ $GATE_RETVAL -ne 0 ]; then + echo "ERROR: the pre-test setup script run by this job failed - exit code: $GATE_RETVAL" +fi # Run the gate function -echo "Running gate_hook" -with_timeout "gate_hook" -GATE_RETVAL=$? +if [ $GATE_RETVAL -eq 0 ]; then + echo "Running gate_hook" + with_timeout "gate_hook" + GATE_RETVAL=$? + if [ $GATE_RETVAL -ne 0 ]; then + echo "ERROR: the main setup script run by this job failed - exit code: $GATE_RETVAL" + fi +fi RETVAL=$GATE_RETVAL if [ $GATE_RETVAL -ne 0 ]; then - echo "ERROR: the main setup script run by this job failed - exit code: $GATE_RETVAL" echo " please look at the relevant log files to determine the root cause" echo "Running devstack worlddump.py" sudo $BASE/new/devstack/tools/worlddump.py -d $BASE/logs From 60d480ca9b369ca5a2d8f1a39ce8ebe2d5be6bcc Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Sun, 10 Apr 2016 10:49:23 -0400 Subject: [PATCH 029/148] Archive /etc/openstack/clouds.yaml in gate runs gate-novaclient-dsvm-functional uses a clouds.yaml file for keystone auth so we should archive that if it's available for debugging. Change-Id: I87b313bbb9e9f865be5936e8465ec73e5975a65c Related-Bug: #1568527 --- functions.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index 07bcaa76..08956741 100644 --- a/functions.sh +++ b/functions.sh @@ -764,8 +764,10 @@ function cleanup_host { sudo cp /etc/sudoers $BASE/logs/sudoers.txt # Archive config files + # NOTE(mriedem): 'openstack' is added separately since it's not a project + # but it is where clouds.yaml is stored in dsvm runs that use it. sudo mkdir $BASE/logs/etc/ - for PROJECT in $PROJECTS; do + for PROJECT in $PROJECTS openstack; do proj=`basename $PROJECT` if [ -d /etc/$proj ]; then sudo cp -r /etc/$proj $BASE/logs/etc/ From bd3ca1f091ec94e59e4d410ebd25f9b1f6b481b9 Mon Sep 17 00:00:00 2001 From: Eyal Date: Tue, 12 Apr 2016 15:33:54 +0300 Subject: [PATCH 030/148] support py3 Change-Id: Ied6e8c9bb44557677041854c6974929d7383a484 --- test-matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-matrix.py b/test-matrix.py index 48fff355..17170f2f 100755 --- a/test-matrix.py +++ b/test-matrix.py @@ -65,7 +65,7 @@ def normalize_branch(branch): def configs_from_env(): configs = [] - for k, v in os.environ.iteritems(): + for k, v in os.environ.items(): if k.startswith('DEVSTACK_GATE_'): if v not in FALSE_VALUES: f = k.split('DEVSTACK_GATE_')[1] @@ -145,7 +145,7 @@ def main(): LOG.debug("Services: %s " % services) if opts.mode == "services": - print ",".join(services) + print(",".join(services)) if __name__ == "__main__": From fb8739202f49151c77785d66a4807c892e62c29e Mon Sep 17 00:00:00 2001 From: fumihiko kakuma Date: Mon, 16 Nov 2015 15:39:05 +0900 Subject: [PATCH 031/148] Consider to log a package list for pip3 When we run devstack with USE_PYTHON3=True, a package may be installed by pip2 or pip3 depending on package classifier. So we should try to log a package list by pip2 and pip3. Change-Id: I06e2559e852f0dabbbbb8f6c0dc081cedb745b60 --- functions.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/functions.sh b/functions.sh index 08956741..eba6e6c2 100644 --- a/functions.sh +++ b/functions.sh @@ -874,9 +874,14 @@ function cleanup_host { sudo iptables-save > $WORKSPACE/iptables.txt df -h > $WORKSPACE/df.txt - pip freeze > $WORKSPACE/pip-freeze.txt - sudo mv $WORKSPACE/iptables.txt $WORKSPACE/df.txt \ - $WORKSPACE/pip-freeze.txt $BASE/logs/ + sudo mv $WORKSPACE/iptables.txt $WORKSPACE/df.txt $BASE/logs/ + + for py_ver in 2 3; do + if [[ `which python${py_ver}` ]]; then + python${py_ver} -m pip freeze > $WORKSPACE/pip${py_ver}-freeze.txt + sudo mv $WORKSPACE/pip${py_ver}-freeze.txt $BASE/logs/ + fi + done if [ `command -v dpkg` ]; then dpkg -l> $WORKSPACE/dpkg-l.txt From 41b076c5e3904fc3a762e8645cfd71a12dc1a2b5 Mon Sep 17 00:00:00 2001 From: vsaienko Date: Wed, 13 Apr 2016 16:21:44 +0300 Subject: [PATCH 032/148] Increase default OS_TEST_TIMEOUT to 1200 Both build and terminate instance timeout are set to 600sec for baremetal instances. As result maximum legimate time is 1200 if do not take info account power/state change and other timeouts. The job timeout is set to 180 minutes [0] so tempest test timeout value can be safely increased to 1200sec. Closes-Bug: #1576100 Change-Id: I5343486fa7f4bc7007d8cabd906ef1a136e76e8c --- devstack-vm-gate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 600e5dd9..1c99cf78 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -336,7 +336,7 @@ EOF fi if [[ "$DEVSTACK_GATE_VIRT_DRIVER" == "ironic" ]]; then - export TEMPEST_OS_TEST_TIMEOUT=${DEVSTACK_GATE_OS_TEST_TIMEOUT:-900} + export TEMPEST_OS_TEST_TIMEOUT=${DEVSTACK_GATE_OS_TEST_TIMEOUT:-1200} echo "IRONIC_DEPLOY_DRIVER=$DEVSTACK_GATE_IRONIC_DRIVER" >>"$localrc_file" echo "VIRT_DRIVER=ironic" >>"$localrc_file" echo "IRONIC_BAREMETAL_BASIC_OPS=True" >>"$localrc_file" From 033036ebcd77430bc88e302f7caf60e48f140127 Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Thu, 28 Apr 2016 12:48:29 -0700 Subject: [PATCH 033/148] Fixed link to l3 agent in tempest-logs q-vpn log is not expected for non vpnaas jobs. Even more so, vpnaas tempest gate does not even run q-vpn service, instead relying on neutron-vpnaas service defined in devstack plugin. So just replace q-vpn log (which never exists) with l3 agent log, which is a reasonable default for neutron gate logdirs. Change-Id: Ie91b3a724c22f9262aa1925be5505fe118d93751 --- help/tempest-logs.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/help/tempest-logs.html b/help/tempest-logs.html index e76c339a..cfdc2adb 100644 --- a/help/tempest-logs.html +++ b/help/tempest-logs.html @@ -73,7 +73,7 @@

Types of logs

  • screen-q-meta.txt.gz: neutron-metadata-agent
  • screen-q-metering.txt.gz: neutron-metering-agent
  • screen-q-svc.txt.gz: neutron-server -
  • screen-q-vpn.txt.gz: neutron-vpn-agent +
  • screen-q-l3.txt.gz: neutron-l3-agent
  • swift
      From 5af29c0d5bb0c65493266d52676766879f4e2b69 Mon Sep 17 00:00:00 2001 From: Tim Buckley Date: Tue, 16 Feb 2016 15:44:42 -0700 Subject: [PATCH 034/148] Install and run StackViz during devstack-gate host cleanup StackViz is a utility for visualizing the performance of DevStack and Tempest executions. This adds a new function, `process_stackviz`, that copies a static site template built during nodepool image build (see dependent patch), and then installs and runs a small Python module (already cached on the filesystem) to process subunit data. This data, along with the static template, is then copied into the log output directory so that it can be uploaded to the log server when the job finishes. Change-Id: Ia5bee69f2874ca50ac6148494d2926ef4ab73b6b Depends-On: I9bc50b8eba696264d2b97ed20a37f28f4768abb8 --- functions.sh | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/functions.sh b/functions.sh index 72314c51..96fe5216 100644 --- a/functions.sh +++ b/functions.sh @@ -657,6 +657,39 @@ function process_testr_artifacts { fi } +function process_stackviz { + local project=$1 + local path_prefix=${2:-new} + + local project_path=$BASE/$path_prefix/$project + local log_path=$BASE/logs + if [[ "$path_prefix" != "new" ]]; then + log_path=$BASE/logs/$path_prefix + fi + + local stackviz_path=/opt/stackviz + if [ -d $stackviz_path/build ]; then + sudo pip install -U $stackviz_path + + # static html+js should be prebuilt during image creation + cp -r $stackviz_path/build $log_path/stackviz + + pushd $project_path + if [ -f $BASE/new/dstat-csv.txt ]; then + sudo testr last --subunit | stackviz-export \ + --dstat $BASE/new/dstat-csv.txt \ + --end --stdin \ + $log_path/stackviz/data + else + sudo testr last --subunit | stackviz-export \ + --env --stdin \ + $log_path/stackviz/data + fi + sudo chown -R $USER:$USER $log_path/stackviz + popd + fi +} + function cleanup_host { # TODO: clean this up to be errexit clean local errexit=$(set +o | grep errexit) @@ -843,6 +876,8 @@ function cleanup_host { sudo mv $WORKSPACE/rpm-qa.txt.gz $BASE/logs/ fi + process_stackviz tempest + process_testr_artifacts tempest process_testr_artifacts tempest old From 42a9a9d8bb0b98bd340ab5b6e2b52b5e740095ee Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Mon, 2 May 2016 10:33:24 -0400 Subject: [PATCH 035/148] devstack-vm-gate-wrap: pin paramiko==1.16.0 paramiki > 1.16.0 requires python-cryptography, but TripleO CI does not handle libffi-devel and openssl-devel packages, it's a WIP in DIB. Until we fix it in DIB, this patch is a workaround so we can unblock CIs that use this script, TripleO CI included. Change-Id: I05755e5374472fb95a4ff9d0dc03dbd33bba627f --- devstack-vm-gate-wrap.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index ec938b2b..34996b9e 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -445,7 +445,9 @@ set -x # Install ansible sudo -H pip install virtualenv virtualenv /tmp/ansible -/tmp/ansible/bin/pip install ansible==$ANSIBLE_VERSION +# NOTE(emilien): workaround to avoid installing cryptography +# https://github.com/ansible/ansible/issues/15665 +/tmp/ansible/bin/pip install paramiko==1.16.0 ansible==$ANSIBLE_VERSION export ANSIBLE=/tmp/ansible/bin/ansible # Write inventory file with groupings From 2de2483d4611c877972908b1645bb90f041b700b Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 4 May 2016 10:18:39 -0700 Subject: [PATCH 036/148] Use fallocate for swapfile if on ext fallocate is much quicker than writing an 8GB file of zeros but it only works reliably for swapfiles on ext filesystems. Check what the filesystem type is and use fallocate if on an ext fs. Change-Id: I8861005a842bd82151c250edef99c2017f09a81a --- functions.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index f47aded6..ca188a7b 100644 --- a/functions.sh +++ b/functions.sh @@ -381,9 +381,15 @@ function fix_disk_layout { # Don't use sparse device to avoid wedging when disk space and # memory are both unavailable. local swapfile='/root/swapfile' + sudo touch ${swapfile} swapdiff=$(( $SWAPSIZE - $swapcurrent )) - sudo dd if=/dev/zero of=${swapfile} bs=1M count=${swapdiff} + if sudo df -T ${swapfile} | grep -q ext ; then + sudo fallocate -l ${swapdiff}M ${swapfile} + else + # Cannot fallocate on filesystems like XFS + sudo dd if=/dev/zero of=${swapfile} bs=1M count=${swapdiff} + fi sudo chmod 600 ${swapfile} sudo mkswap ${swapfile} sudo swapon ${swapfile} From baf68bb7edfa2d7cc50be5e702304b9c3b15e212 Mon Sep 17 00:00:00 2001 From: vsaienko Date: Tue, 10 May 2016 15:24:58 +0300 Subject: [PATCH 037/148] Do not set IRONIC_VM_COUNT from devstack-gate IRONIC_VM_COUNT is set directly from the project-config [0]. Reference: [0] https://github.com/openstack-infra/project-config/blob/0e958003cdcabd7b8c23bd1ded4d79385936556d/jenkins/jobs/devstack-gate.yaml#L681 Change-Id: If5be4eb2d478b624a9dea97d80b0ab37cb8256dd Closes-Bug: #1580157 --- devstack-vm-gate.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 6c53ad60..7356686e 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -356,11 +356,9 @@ EOF echo "IRONIC_VM_EPHEMERAL_DISK=0" >>"$localrc_file" # agent CoreOS ramdisk is a little heavy echo "IRONIC_VM_SPECS_RAM=1024" >>"$localrc_file" - echo "IRONIC_VM_COUNT=1" >>"$localrc_file" else echo "IRONIC_ENABLED_DRIVERS=fake,pxe_ssh,pxe_ipmitool" >>"$localrc_file" echo "IRONIC_VM_EPHEMERAL_DISK=1" >>"$localrc_file" - echo "IRONIC_VM_COUNT=3" >>"$localrc_file" fi fi From cf8b7ae704f9e387d4cbc626ed966e3d542ab723 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 23 Mar 2016 15:29:07 +1100 Subject: [PATCH 038/148] Make reproduce function quiet In every console log (e.g. [1]) we've got a long section of looping around variables and writing them out being traced. This is working fine, so turn down the tracing on this to give better signal/noise ratio in logs. [1] http://logs.openstack.org/29/295629/3/check/gate-tempest-dsvm-full/f7fe416/console.html#_2016-03-22_14_53_45_466 Change-Id: Ieac461609a1a1866c1c3c461e75230406dffc649 --- functions.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/functions.sh b/functions.sh index 07bcaa76..c3e38a01 100644 --- a/functions.sh +++ b/functions.sh @@ -171,6 +171,9 @@ function remaining_time { # Create a script to reproduce this build function reproduce { + local xtrace=$(set +o | grep xtrace) + set +o xtrace + JOB_PROJECTS=$1 cat > $WORKSPACE/logs/reproduce.sh < Date: Fri, 20 May 2016 17:34:28 -0400 Subject: [PATCH 039/148] Print installed tempest-plugins after tempest run The ALL_PLUGINS tox job runs tempest in a venv with system site-packages enabled. This can make it very confusing to figure out exactly which plugins are installed when tempest is run. Especially given many projects' terrible choice to bundle the plugin in the project repo. This commit adds running 'tempest list-plugins' inside the venv tox makes for that job to show at the end of a run all the plugins tempest was run with. Change-Id: I8a65c851cdf5a4081ac32afa17f69ee001ff40c7 --- devstack-vm-gate.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 773cbdf2..28eac92e 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -754,6 +754,7 @@ if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then if [[ "$DEVSTACK_GATE_TEMPEST_ALL_PLUGINS" -eq "1" ]]; then echo "Running tempest with plugins and a custom regex filter" $TEMPEST_COMMAND -eall-plugin -- --concurrency=$TEMPEST_CONCURRENCY $DEVSTACK_GATE_TEMPEST_REGEX + sudo -H -u tempest .tox/all-plugin/bin/tempest list-plugins else echo "Running tempest with a custom regex filter" $TEMPEST_COMMAND -eall -- --concurrency=$TEMPEST_CONCURRENCY $DEVSTACK_GATE_TEMPEST_REGEX @@ -761,6 +762,7 @@ if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then elif [[ "$DEVSTACK_GATE_TEMPEST_ALL_PLUGINS" -eq "1" ]]; then echo "Running tempest all-plugins test suite" $TEMPEST_COMMAND -eall-plugin -- --concurrency=$TEMPEST_CONCURRENCY + sudo -H -u tempest .tox/all-plugin/bin/tempest list-plugins elif [[ "$DEVSTACK_GATE_TEMPEST_ALL" -eq "1" ]]; then echo "Running tempest all test suite" $TEMPEST_COMMAND -eall -- --concurrency=$TEMPEST_CONCURRENCY From b81c4a68b381346d0ee610d1302b3f228c131a62 Mon Sep 17 00:00:00 2001 From: vsaienko Date: Thu, 12 May 2016 14:37:30 +0300 Subject: [PATCH 040/148] Allow to set Ironic provision timeout from the job Launching smoke tempest tests building several instances in parallel. As result grenade job that uses tempest smoke tests need to have higher provision/unprovision timeout values value. This patch allows to set BUILD_TIMEOUT from the job. Change-Id: Idd454b7dea2a6d37a6bc048132fc2b3765d37429 --- devstack-vm-gate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index cebec23c..28f47d65 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -342,7 +342,7 @@ EOF echo "IRONIC_BAREMETAL_BASIC_OPS=True" >>"$localrc_file" echo "IRONIC_VM_LOG_DIR=$BASE/$localrc_oldnew/ironic-bm-logs" >>"$localrc_file" echo "DEFAULT_INSTANCE_TYPE=baremetal" >>"$localrc_file" - echo "BUILD_TIMEOUT=600" >>"$localrc_file" + echo "BUILD_TIMEOUT=${DEVSTACK_GATE_TEMPEST_BAREMETAL_BUILD_TIMEOUT:-600}" >>"$localrc_file" echo "IRONIC_CALLBACK_TIMEOUT=600" >>"$localrc_file" echo "Q_AGENT=openvswitch" >>"$localrc_file" echo "Q_ML2_TENANT_NETWORK_TYPE=vxlan" >>"$localrc_file" From 3250b2096772f7d02be73ab935cd4e4215343b84 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Wed, 25 May 2016 07:52:25 -0400 Subject: [PATCH 041/148] don't set libvirt cpu_model Nova was incorrectly checking cpu model compatiblity when running qemu guests, which isn't needed, as they can emulate any instruction under the sun. This removes this setting from our testing after Nova lands a patch to fix this. Depends-On: I8b782678a6dfa7bcfc203381037e1df4475c3194 Change-Id: I41b4c0371515c88d4ab9fc127e5abc9b3fb97f31 --- devstack-vm-gate.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index cebec23c..95e9f6b8 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -564,7 +564,8 @@ EOF fi if [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]]; then - echo -e "[[post-config|\$NOVA_CONF]]\n[libvirt]\ncpu_mode=custom\ncpu_model=gate64" >> local.conf + # ensure local.conf exists to remove conditional logic + touch local.conf if [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then echo -e "[[post-config|\$NEUTRON_CONF]]\n[DEFAULT]\nnetwork_device_mtu=$EXTERNAL_BRIDGE_MTU" >> local.conf fi @@ -601,7 +602,8 @@ else cd $BASE/new/devstack setup_localrc "new" "localrc" "primary" if [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]]; then - echo -e "[[post-config|\$NOVA_CONF]]\n[libvirt]\ncpu_mode=custom\ncpu_model=gate64" >> local.conf + # ensure local.conf exists to remove conditional logic + touch local.conf if [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then echo -e "[[post-config|\$NEUTRON_CONF]]\n[DEFAULT]\nnetwork_device_mtu=$EXTERNAL_BRIDGE_MTU" >> local.conf fi From 40c2ba7e637c20326f86e543a66e65dfe6aec0b1 Mon Sep 17 00:00:00 2001 From: "Sean M. Collins" Date: Wed, 25 May 2016 10:51:24 -0700 Subject: [PATCH 042/148] Stop using DevStack fwaas code We should transition to the DevStack fwaas plugin, similar to what was done in c59dfacef8c211b7e114e634ca17f225bde6c3e0 Change-Id: Iadcee07e873fcb4f099ebccc2e33780e74438140 Related-Change: If6c6e032689392fecc8c24517666128c8c103a7b Related-Change: Ic60cd1fa90c19dfac00be583e2ddc5633dbb68a3 --- devstack-vm-gate.sh | 4 ++-- features.yaml | 2 +- test-features.sh | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index cebec23c..32746ed2 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -223,8 +223,8 @@ function setup_localrc { # As per reference architecture described in # https://wiki.openstack.org/wiki/Neutron/DVR # for DVR multi-node, add the following services - # on all compute nodes (q-fwaas being optional): - MY_ENABLED_SERVICES+=",q-l3,q-fwaas,q-meta" + # on all compute nodes: + MY_ENABLED_SERVICES+=",q-l3,q-meta" fi else MY_ENABLED_SERVICES+=",n-net,n-api-meta" diff --git a/features.yaml b/features.yaml index 9f300052..d11d12ca 100644 --- a/features.yaml +++ b/features.yaml @@ -97,7 +97,7 @@ features: neutron-adv: base: - services: [q-lbaas, q-fwaas] + services: [q-lbaas] rm-services: [n-net] kilo: services: [q-vpn] diff --git a/test-features.sh b/test-features.sh index cb1d3116..a7d75469 100755 --- a/test-features.sh +++ b/test-features.sh @@ -18,11 +18,11 @@ ERRORS=0 TEMPEST_FULL_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,n-net" -TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-fwaas,q-metering" +TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-metering" -TEMPEST_NEUTRON_KILO="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-fwaas,q-metering,q-vpn" +TEMPEST_NEUTRON_KILO="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-metering,q-vpn" -TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-fwaas,q-metering" +TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-metering" GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" From 077314510c90e29637c7864357e2cf137c1eb766 Mon Sep 17 00:00:00 2001 From: "Andrea Frittoli (andreaf)" Date: Sat, 28 May 2016 10:13:27 +0100 Subject: [PATCH 043/148] Save tempest accounts.yaml in logs Some test jobs use pre-provisioned credentials. It can be useful to have the credetials file for troubleshooting purposes, so copying it to the logs folder when available. Change-Id: I9fcb3497387fb32f3e515cda44a1a105921a04c2 --- functions.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/functions.sh b/functions.sh index 43ae0579..ba18ee51 100644 --- a/functions.sh +++ b/functions.sh @@ -876,6 +876,9 @@ function cleanup_host { # Copy tempest config file sudo cp $BASE/new/tempest/etc/tempest.conf $NEWLOGTARGET/tempest_conf.txt + if [ -f $BASE/new/tempest/etc/accounts.yaml ] ; then + sudo cp $BASE/new/tempest/etc/accounts.yaml $NEWLOGTARGET/accounts_yaml.txt + fi # Copy dstat CSV log if it exists if [ -f $BASE/new/dstat-csv.log ]; then From 3ef48cec70c771242d743467768f22d1ecb55ecb Mon Sep 17 00:00:00 2001 From: vsaienko Date: Mon, 16 May 2016 13:20:26 +0300 Subject: [PATCH 044/148] Allow to pass OS_TEST_TIMEOUT for grenade job OS_TEST_TIMEOUT may be passed via DEVSTACK_GATE_OS_TEST_TIMEOUT project-config variable. Change-Id: Id81e9060fb0b7800cba1345ea3c8f28b1a691759 --- devstack-vm-gate.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 6a7041dd..971915e9 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -552,6 +552,7 @@ TARGET_RUN_SMOKE=False SAVE_DIR=\$BASE_RELEASE_DIR/save DO_NOT_UPGRADE_SERVICES=$DO_NOT_UPGRADE_SERVICES TEMPEST_CONCURRENCY=$TEMPEST_CONCURRENCY +OS_TEST_TIMEOUT=$DEVSTACK_GATE_OS_TEST_TIMEOUT VERBOSE=False PLUGIN_DIR=\$TARGET_RELEASE_DIR EOF From ec3e4b40a73da8748fa05543c487b0e0029891f9 Mon Sep 17 00:00:00 2001 From: "John L. Villalovos" Date: Tue, 14 Jun 2016 08:36:48 -0700 Subject: [PATCH 045/148] Remove 'partial' support for Grenade jobs Remove 'partial' support for Grenade as it doesn't work anymore. There is no support for the DO_NOT_UPGRADE_SERVICES setting in Grenade anymore. Grenade does not support 'partial' jobs anymore. If a project wants to test rolling upgrades they should use multinode grenade. Change-Id: I369b12b612b852c48bad662b2fb04c4832712c36 --- devstack-vm-gate-wrap.sh | 15 --------------- devstack-vm-gate.sh | 1 - 2 files changed, 16 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 35c514da..4c745e4a 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -228,10 +228,6 @@ export DEVSTACK_PROJECT_FROM_GIT=${DEVSTACK_PROJECT_FROM_GIT:-} # i.e. stable/juno: # pullup means stable/icehouse => stable/juno # forward means stable/juno => master (or stable/kilo if that's out) -# partial-ncpu means stable/icehouse => stable/juno but keep nova -# compute at stable/icehouse -# partial-ironic means stable/icehouse => stable/juno but keep ironic -# compute at stable/icehouse export DEVSTACK_GATE_GRENADE=${DEVSTACK_GATE_GRENADE:-} # the branch name for selecting grenade branches @@ -288,17 +284,6 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then fi ;; - # partial upgrades are like normal upgrades except they leave - # certain services behind. We use the base 4 operator ';&' - # here to fall trhough to the next conditionals - partial-*) - if [[ "$DEVSTACK_GATE_GRENADE" == "partial-ncpu" ]]; then - export DO_NOT_UPGRADE_SERVICES=[n-cpu] - elif [[ "$DEVSTACK_GATE_GRENADE" == "partial-ironic" ]]; then - export DO_NOT_UPGRADE_SERVICES=[ir-api,ir-cond] - fi - ;& - # pullup upgrades are our normal upgrade test. Can you upgrade # to the current patch from the last stable. pullup) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 971915e9..8756087e 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -550,7 +550,6 @@ TARGET_DEVSTACK_DIR=\$TARGET_RELEASE_DIR/devstack TARGET_DEVSTACK_BRANCH=$GRENADE_NEW_BRANCH TARGET_RUN_SMOKE=False SAVE_DIR=\$BASE_RELEASE_DIR/save -DO_NOT_UPGRADE_SERVICES=$DO_NOT_UPGRADE_SERVICES TEMPEST_CONCURRENCY=$TEMPEST_CONCURRENCY OS_TEST_TIMEOUT=$DEVSTACK_GATE_OS_TEST_TIMEOUT VERBOSE=False From 8aa9194ed9d263196e9286cff34f4f3f1b229496 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Fri, 17 Jun 2016 11:04:15 +1000 Subject: [PATCH 046/148] Set read/search perms recursively With the new zuul rsync copy, the apache/ directory disappeared from view as its permissions are copied verbatim into the log directory and it's not world readable. Correct permissions on $BASE/logs so that everything is readable. Change-Id: Ia034c5176937f579e0f19504c6f0ef9c118869b8 --- functions.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/functions.sh b/functions.sh index 6712ac06..202cb01d 100644 --- a/functions.sh +++ b/functions.sh @@ -966,8 +966,9 @@ function cleanup_host { # Make sure the current user can read all the logs and configs sudo chown -R $USER:$USER $BASE/logs/ - sudo chmod a+r $BASE/logs/ $BASE/logs/etc - + # (note X not x ... execute/search only if the file is a directory + # or already has execute permission for some user) + sudo chmod -R a+rX $BASE/logs/ # Collect all the deprecation related messages into a single file. # strip out date(s), timestamp(s), pid(s), context information and From 4f3bc0ef76bf42195b54a421c51579a076a7c33a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Dulko?= Date: Thu, 9 Jun 2016 13:55:52 +0200 Subject: [PATCH 047/148] Enabling Cinder multinode testing Cinder team is planning to start multinode testing in Newton cycle. We want to be able to test volume migration/retype and partial grenade. This commit enables c-vol and c-bak on the subnode in regular multinode setups (this should work as I4d2960d92f388ac689dfa6b436dc8bfc1e129fbf is merged now). Also a new features.yaml elements are introduced that would disable c-vol and c-bak on primary node. This will be required for multinode grenade jobs that will test compatibility of master c-api and c-sch with stable c-vol and c-bak. Co-Authored-By: Karthik Prabhu Vinod Change-Id: I4532bdcfa845cf0a405f68371c61559b0a70f9c1 --- devstack-vm-gate.sh | 3 +-- features.yaml | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 971915e9..e18068a6 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -214,9 +214,8 @@ function setup_localrc { local original_enabled_services=$MY_ENABLED_SERVICES # TODO(afazekas): Move to the feature grid - # TODO(afazekas): add c-vol if [[ $role = sub ]]; then - MY_ENABLED_SERVICES="n-cpu,ceilometer-acompute,dstat" + MY_ENABLED_SERVICES="n-cpu,ceilometer-acompute,dstat,c-vol,c-bak" if [[ "$DEVSTACK_GATE_NEUTRON" -eq "1" ]]; then MY_ENABLED_SERVICES+=",q-agt" if [[ "$DEVSTACK_GATE_NEUTRON_DVR" -eq "1" ]]; then diff --git a/features.yaml b/features.yaml index 9f300052..27fa9fc5 100644 --- a/features.yaml +++ b/features.yaml @@ -47,6 +47,8 @@ config: features: [heat] tlsproxy: features: [tlsproxy] + cinder_mn_grenade: + features: [cinder-mn-grenade] branches: # The value of ""default" is the name of the "trunk" branch @@ -110,6 +112,12 @@ features: base: services: [cinder, c-api, c-vol, c-sch, c-bak] + # This will be used to disable c-vol on primary node when running multinode grenade + # job that will test compatibility of new c-api, c-sch (primary) and old c-vol and c-bak (sub). + cinder-mn-grenade: + base: + rm-services: [c-vol, c-bak] + heat: base: services: [heat, h-api, h-api-cfn, h-api-cw, h-eng] From f5d3bdb99fab09d39eaa2014abff78c8efb695b5 Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Fri, 15 Jul 2016 19:19:09 +0200 Subject: [PATCH 048/148] Configure infrastructure MTU with global_physnet_mtu The network_device_mtu option will be removed from Neutron in Newton. The global_physnet_mtu option was introduced in Mitaka. Before Mitaka, Neutron did not have any multinode jobs that would require setting the infrastructure MTU for inter-node connectivity. Change-Id: I337b284076a794027fbd63796119d56bd1923cf2 Related-Bug: #1603493 --- devstack-vm-gate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 048163b7..d146ea8e 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -568,7 +568,7 @@ EOF # ensure local.conf exists to remove conditional logic touch local.conf if [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then - echo -e "[[post-config|\$NEUTRON_CONF]]\n[DEFAULT]\nnetwork_device_mtu=$EXTERNAL_BRIDGE_MTU" >> local.conf + echo -e "[[post-config|\$NEUTRON_CONF]]\n[DEFAULT]\nglobal_physnet_mtu=$EXTERNAL_BRIDGE_MTU" >> local.conf fi # get this in our base config @@ -606,7 +606,7 @@ else # ensure local.conf exists to remove conditional logic touch local.conf if [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then - echo -e "[[post-config|\$NEUTRON_CONF]]\n[DEFAULT]\nnetwork_device_mtu=$EXTERNAL_BRIDGE_MTU" >> local.conf + echo -e "[[post-config|\$NEUTRON_CONF]]\n[DEFAULT]\nglobal_physnet_mtu=$EXTERNAL_BRIDGE_MTU" >> local.conf fi fi From e76e156c4001eb910ad0ce4f6e685363e038dbac Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Fri, 15 Jul 2016 10:54:33 -0700 Subject: [PATCH 049/148] Always set br-ex mtu including for grenade runs Previously we were only setting the mtu on br-ex if not running grenade. But now neutron runs multinode grenade tests so need to have br-ex's mtu set appropriately for grenade and devstack both. This is done by using PUBLIC_DEVICE_MTU devstack variable that will guarantee the bridge is set appropriately in both devstack and grenade cases. Change-Id: I9bcc47303a6ce1f0b3125ad374adcc2e7ca654ac Depends-On: I38a80bac18673a30842a7b997d0669fed5aff976 Related-Bug: #1603268 --- devstack-vm-gate.sh | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 048163b7..b5cafa21 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -311,6 +311,11 @@ DATABASE_QUERY_LOGGING=True EBTABLES_RACE_FIX=True EOF + if [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]] && [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then + # Reduce the MTU on br-ex to match the MTU of underlying tunnels + echo "PUBLIC_BRIDGE_MTU=$EXTERNAL_BRIDGE_MTU" >>"$localrc_file" + fi + if [[ "$DEVSTACK_CINDER_SECURE_DELETE" -eq "0" ]]; then echo "CINDER_SECURE_DELETE=False" >>"$localrc_file" fi @@ -658,19 +663,6 @@ else exit 1 fi fi - - if [[ "$DEVSTACK_GATE_TOPOLOGY" != "aio" ]] && [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then - # NOTE(afazekas): The cirros lp#1301958 does not support MTU setting via dhcp, - # simplest way the have tunneling working, with dvsm, without increasing the host system MTU - # is to decreasion the MTU on br-ex - # TODO(afazekas): Configure the mtu smarter on the devstack side - MTU_NODES=primary - if [[ "$DEVSTACK_GATE_NEUTRON_DVR" -eq "1" ]]; then - MTU_NODES=all - fi - $ANSIBLE "$MTU_NODES" -f 5 -i "$WORKSPACE/inventory" -m shell \ - -a "sudo ip link set mtu $EXTERNAL_BRIDGE_MTU dev br-ex" - fi fi if [[ "$DEVSTACK_GATE_UNSTACK" -eq "1" ]]; then From a35ab289eb9f1dbc78826d6c7d7f700e4caede1a Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Thu, 28 Jul 2016 11:35:27 +1000 Subject: [PATCH 050/148] Use "arp -n" when finding gateway MAC for netconsole arp can resolve the gateway address to something unhelpful like "gateway" which fails the grep. Avoid this. Change-Id: I236b7080774fef8700d0ec056d9703eb98f90204 --- functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index 0b254380..777b09ce 100644 --- a/functions.sh +++ b/functions.sh @@ -1060,7 +1060,7 @@ function enable_netconsole { # out to the world is specify the default gw as the remote # destination. local default_gw=$(ip route | grep default | awk '{print $3}') - local gw_mac=$(arp $default_gw | grep $default_gw | awk '{print $3}') + local gw_mac=$(arp -n $default_gw | grep $default_gw | awk '{print $3}') local gw_dev=$(ip route | grep default | awk '{print $5}') # turn up message output From 6415658e38774337db9a2a3cefef5030eb930965 Mon Sep 17 00:00:00 2001 From: Patrick East Date: Fri, 29 Jul 2016 12:40:58 -0700 Subject: [PATCH 051/148] Copy local.conf to the archived logs directory Not so important for upstream jenkins that doesn't use local.conf, but very nice for 3rd party CI's that override configuration primarily through local.conf's. Change-Id: Id53921a4a417e90ad012713cfe8265ac7764512d --- functions.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/functions.sh b/functions.sh index 777b09ce..e6e51bc9 100644 --- a/functions.sh +++ b/functions.sh @@ -878,6 +878,9 @@ function cleanup_host { xargs -0 -I {} sudo cp {} $NEWLOGTARGET/ sudo cp $BASE/new/devstacklog.txt $NEWLOGTARGET/ sudo cp $BASE/new/devstack/localrc $NEWLOGTARGET/localrc.txt + if [ -f $BASE/new/devstack/local.conf ]; then + sudo cp $BASE/new/devstack/local.conf $NEWLOGTARGET/local.conf.txt + fi if [ -f $BASE/new/devstack/tempest.log ]; then sudo cp $BASE/new/devstack/tempest.log $NEWLOGTARGET/verify_tempest_conf.log fi From 54b30a1b6a87baeb3a9bcf99624baca51fd1523b Mon Sep 17 00:00:00 2001 From: Sagi Shnaidman Date: Sun, 31 Jul 2016 15:26:42 +0300 Subject: [PATCH 052/148] Use ntp wait time as 10 min instead of 100 By default ntp-wait waits 6 sec * 1000 tries = 100 mins It breaks long jobs like in TripleO projects because of waiting for 1 hour 40 minutes. If ntp wasn't synced during first 10 min it most likely won't be synced at all, so no reason to wait so long time and break jobs that date is not critical for them Change-Id: Iae28a03829a255f0990934685d8c2e563a7a8492 Partial-Bug: 1608226 --- functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index 777b09ce..65f37137 100644 --- a/functions.sh +++ b/functions.sh @@ -140,7 +140,7 @@ function start_timer { # first make sure the time is right, so we don't go into crazy land # later if the system decides to apply an ntp date and we jump forward # 4 hrs (which has happened) - sudo /usr/sbin/ntp-wait -v + sudo /usr/sbin/ntp-wait -v -n 100 sleep 1 START_TIME=`date +%s` } From 59aeed26cd09f8c42b247050bd1407a0e10f591b Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Wed, 3 Aug 2016 14:38:50 -0700 Subject: [PATCH 053/148] Explicitly set IRONIC_BUILD_DEPLOY_RAMDISK We'd like to change the default for this in our local devstack plugin, but until devstack-gate sets the desired value in all cases we can't. Change-Id: I25c2ece32b7e8b601fe5ed7f96dcfbad4798e376 Related-bug: #1607981 --- devstack-vm-gate.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 8c7f5955..3f3b9d0a 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -353,6 +353,8 @@ EOF echo "Q_ML2_TENANT_NETWORK_TYPE=vxlan" >>"$localrc_file" if [[ "$DEVSTACK_GATE_IRONIC_BUILD_RAMDISK" -eq 0 ]]; then echo "IRONIC_BUILD_DEPLOY_RAMDISK=False" >>"$localrc_file" + else + echo "IRONIC_BUILD_DEPLOY_RAMDISK=True" >>"$localrc_file" fi if [[ -z "${DEVSTACK_GATE_IRONIC_DRIVER%%agent*}" ]]; then echo "SWIFT_ENABLE_TEMPURLS=True" >>"$localrc_file" From dcd690265146e89c363acd1b070644c4eb1f82fe Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Tue, 9 Aug 2016 14:32:05 -0400 Subject: [PATCH 054/148] Add sanity check on devstack generated tempest.conf There are some parts of devstack that call the service api to verify the extension list we pass to tempest. This is fine for the out of gate case if it makes life easier for users. But, we should not ever trust this for gate testing. This commit adds a check on master to ensure we always are using the default value of 'all'. On stable we hard code a list of available extensions so this isn't a concern there. Change-Id: I01be3c1392fd4642bdd4633ba5624ddacb608bc5 --- devstack-vm-gate.sh | 10 ++++++++++ functions.sh | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index a807bdac..2491eacc 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -734,6 +734,16 @@ if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then exit 0 fi + # There are some parts of devstack that call the neutron api to verify the + # extension. We should not ever trust this for gate testing. This checks to + # ensure on master we always are using the default value. (on stable we hard + # code a list of available extensions so we can't use this) + neutron_extensions=$(iniget "$BASE/new/tempest/etc/tempest.conf" "neutron-feature-enabled" "api_extensions") + if [[ $GIT_BRANCH == 'master' && ($neutron_extensions == 'all' || $neutron_extensions == '') ]] ; then + echo "Devstack misconfugred tempest and changed the value of api_extensions" + exit 1 + fi + # From here until the end we rely on the fact that all the code fails if # something is wrong, to enforce exit on bad test results. set -o errexit diff --git a/functions.sh b/functions.sh index 65f37137..f61c4539 100644 --- a/functions.sh +++ b/functions.sh @@ -1198,3 +1198,8 @@ function with_timeout { function iniset { $(source $BASE/new/devstack/inc/ini-config; iniset $@) } + +# Iniget imported from devstack +function iniget { + $(source $BASE/new/devstack/inc/ini-config; iniget $@) +} From 88a41dab7a56dd96b7abb4f8fcc986d2aeb65cf0 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 12 Aug 2016 10:32:27 -0500 Subject: [PATCH 055/148] Add osc-lib and os-client-config to PROJECTS Both libraries are needed in gate jobs to be able to make sure we don't break the gate broadly. Change-Id: Idb2a766069bbea088d402cb86d7ded86e93f3096 Depends-On: I747480b6063a62e82ca2b030f274d3e87bf28b3b --- devstack-vm-gate-wrap.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 35c514da..39cc7d22 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -76,10 +76,12 @@ PROJECTS="openstack/neutron-vpnaas $PROJECTS" PROJECTS="openstack/nova $PROJECTS" PROJECTS="openstack/os-apply-config $PROJECTS" PROJECTS="openstack/os-brick $PROJECTS" +PROJECTS="openstack/os-client-config $PROJECTS" PROJECTS="openstack/os-cloud-config $PROJECTS" PROJECTS="openstack/os-collect-config $PROJECTS" PROJECTS="openstack/os-net-config $PROJECTS" PROJECTS="openstack/os-refresh-config $PROJECTS" +PROJECTS="openstack/osc-lib $PROJECTS" PROJECTS="openstack/oslo.cache $PROJECTS" PROJECTS="openstack/oslo.concurrency $PROJECTS" PROJECTS="openstack/oslo.config $PROJECTS" From 67bc7606bce4be0757e2fb21f5aafb8e69f202b9 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Thu, 18 Aug 2016 16:10:41 -0400 Subject: [PATCH 056/148] Change default for FORCE_CONFIG_DRIVE to False devstack defaults always run the metadata server and have access to config drive. Devstack, however, supports an override so that guests never access the metadata server. Tempest tests already do specific config drive tests, so changing to this default lets us test both. Change-Id: I141bb7a44b5c34620479dd56496b21e0d95d9937 --- devstack-vm-gate-wrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 39cc7d22..9db4c1e1 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -208,7 +208,7 @@ export DEVSTACK_GATE_SAHARA=${DEVSTACK_GATE_SAHARA:-0} export DEVSTACK_GATE_TROVE=${DEVSTACK_GATE_TROVE:-0} # Set to 0 to disable config_drive and use the metadata server instead -export DEVSTACK_GATE_CONFIGDRIVE=${DEVSTACK_GATE_CONFIGDRIVE:-1} +export DEVSTACK_GATE_CONFIGDRIVE=${DEVSTACK_GATE_CONFIGDRIVE:-0} # Set to 1 to enable installing test requirements export DEVSTACK_GATE_INSTALL_TESTONLY=${DEVSTACK_GATE_INSTALL_TESTONLY:-0} From ed81ff26a970a34fb2751f56da322d7f2c0ab61c Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Fri, 19 Aug 2016 13:11:23 -0700 Subject: [PATCH 057/148] Revert "Use ntp-wait instead of ntpdate to set time" This reverts commit aaa0890e4f1b47f1d455aa333aa4cdb1bb978e18. Turns out using ntp-wait does not always result in a stepped clock like we want. Go back to the old ntpdate setup. Depends-On: Ib3fd4afe5a89d8a799cc15c57254aaf11b6aa3e5 Change-Id: I6721f614c56298f9c9b476044705fa5fab6d63fc --- functions.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index f61c4539..6264d4d2 100644 --- a/functions.sh +++ b/functions.sh @@ -140,7 +140,20 @@ function start_timer { # first make sure the time is right, so we don't go into crazy land # later if the system decides to apply an ntp date and we jump forward # 4 hrs (which has happened) - sudo /usr/sbin/ntp-wait -v -n 100 + if is_fedora; then + local ntp_service='ntpd' + elif uses_debs; then + local ntp_service='ntp' + else + echo "Unsupported platform, can't determine ntp service" + exit 1 + fi + local default_ntp_server=$( + grep ^server /etc/ntp.conf | head -1 | awk '{print $2}') + local ntp_server=${NTP_SERVER:-$default_ntp_server} + sudo service $ntp_service stop + sudo /usr/sbin/ntpdate $ntp_server + sudo service $ntp_service start sleep 1 START_TIME=`date +%s` } From cb2b8092de574527188da5d2c329e77d6f54cbd6 Mon Sep 17 00:00:00 2001 From: Doug Wiegley Date: Sat, 20 Aug 2016 17:37:55 +0100 Subject: [PATCH 058/148] Remove q-lbaas from tempest pre-installed stuff. LBaaS tests are no longer in tempest. Change-Id: I59d9e3ac756a91936f598b683bc3da4dc6ba9ba0 --- features.yaml | 5 ++++- test-features.sh | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/features.yaml b/features.yaml index 66b4b650..e0e011e2 100644 --- a/features.yaml +++ b/features.yaml @@ -99,8 +99,11 @@ features: neutron-adv: base: - services: [q-lbaas] rm-services: [n-net] + mitaka: + services: [q-lbaas] + liberty: + services: [q-lbaas] kilo: services: [q-vpn] diff --git a/test-features.sh b/test-features.sh index a7d75469..81efaa22 100755 --- a/test-features.sh +++ b/test-features.sh @@ -18,11 +18,11 @@ ERRORS=0 TEMPEST_FULL_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,n-net" -TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-metering" +TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering" -TEMPEST_NEUTRON_KILO="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-metering,q-vpn" +TEMPEST_NEUTRON_KILO="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,q-vpn" -TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-lbaas,q-metering" +TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering" GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" From 34cf66727c58d72b3d1446b4ac0d8cb50e40ff8c Mon Sep 17 00:00:00 2001 From: Jeremy Stanley Date: Tue, 23 Aug 2016 17:27:15 +0000 Subject: [PATCH 059/148] Update link to nodepool.yaml in README.rst Our nodepool.yaml moved from being a template in the system-config repo to a plain file in the project-config repo. Change-Id: Iaadaa9e4780d5277f222a2df127fe05b59ecb650 --- README.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 282f39d4..45bf36a5 100644 --- a/README.rst +++ b/README.rst @@ -140,9 +140,8 @@ By comparison, a provider settings file for HPCloud:: Note: The image regularly changes as new images are uploaded, for the specific image name currently used for tests, see -`nodepool.yaml.erb `_. +`nodepool.yaml `_. Source the provider settings, boot a server named "testserver" (chosen arbitrarily for this example) with your SSH key allowed, and log into From f7e3f3bfa85f1e5277703ef19ddfb3140358a563 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Thu, 25 Aug 2016 09:55:18 -0400 Subject: [PATCH 060/148] don't start horizon in grenade horizon is not really tested in grenade, so it's extra effort for no real validation. We should not do this by default as it will shorten the job length. Depends-On: I2cb3c75bdd738a8e19796456f0aed14237ef755e (enables horizon to be conditionally upgraded only if it exists) Change-Id: Ifcf4dad0366a2763bb311f77ab63de1e5b3fafe7 --- features.yaml | 2 +- test-features.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/features.yaml b/features.yaml index e0e011e2..9994b0a6 100644 --- a/features.yaml +++ b/features.yaml @@ -17,7 +17,7 @@ config: features: [postgresql] # feature changes for different test matrixes grenade: - rm-features: [trove, sahara, neutron-adv] + rm-features: [trove, sahara, neutron-adv, horizon] tempest: features: [tempest] # feature changes for different configs of existing services diff --git a/test-features.sh b/test-features.sh index 81efaa22..dc0e5547 100755 --- a/test-features.sh +++ b/test-features.sh @@ -24,11 +24,11 @@ TEMPEST_NEUTRON_KILO="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horiz TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering" -GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" +GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" -GRENADE_JUNO_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" +GRENADE_JUNO_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" -GRENADE_ICEHOUSE_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" +GRENADE_ICEHOUSE_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" TEMPEST_FULL_JUNO="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,n-net" From 98a8b3f5ef5c37097dcb4c09433ac8726e6822e2 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Thu, 25 Aug 2016 15:52:35 -0400 Subject: [PATCH 061/148] remove old tests remove tests for feature grid for releases that are eol. Change-Id: Iab92aa60d4d417fa04d2c97f5764f637cab88732 --- test-features.sh | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/test-features.sh b/test-features.sh index dc0e5547..0a9d4a85 100755 --- a/test-features.sh +++ b/test-features.sh @@ -20,18 +20,10 @@ TEMPEST_FULL_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizo TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering" -TEMPEST_NEUTRON_KILO="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,q-vpn" - TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering" GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" -GRENADE_JUNO_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" - -GRENADE_ICEHOUSE_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" - -TEMPEST_FULL_JUNO="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,n-net" - # Utility function for tests function assert_list_equal { local source=$(echo $1 | awk 'BEGIN{RS=",";} {print $1}' | sort -V | xargs echo) @@ -57,21 +49,11 @@ function test_full_feature_ec { assert_list_equal $TEMPEST_FULL_MASTER $results } -function test_full_juno { - local results=$(DEVSTACK_GATE_TEMPEST=1 ./test-matrix.py -b stable/juno) - assert_list_equal $TEMPEST_FULL_JUNO $results -} - function test_neutron_master { local results=$(DEVSTACK_GATE_NEUTRON=1 DEVSTACK_GATE_TEMPEST=1 ./test-matrix.py) assert_list_equal $TEMPEST_NEUTRON_MASTER $results } -function test_neutron_kilo { - local results=$(DEVSTACK_GATE_NEUTRON=1 DEVSTACK_GATE_TEMPEST=1 ./test-matrix.py -b stable/kilo) - assert_list_equal $TEMPEST_NEUTRON_KILO $results -} - function test_heat_slow_master { local results=$(DEVSTACK_GATE_TEMPEST_HEAT_SLOW=1 DEVSTACK_GATE_NEUTRON=1 DEVSTACK_GATE_TEMPEST=1 ./test-matrix.py) assert_list_equal $TEMPEST_HEAT_SLOW_MASTER $results @@ -82,25 +64,11 @@ function test_grenade_new_master { assert_list_equal $GRENADE_NEW_MASTER $results } -function test_grenade_juno_master { - local results=$(DEVSTACK_GATE_GRENADE=pullup DEVSTACK_GATE_TEMPEST=1 ./test-matrix.py -b stable/juno) - assert_list_equal $GRENADE_JUNO_MASTER $results -} - -function test_grenade_icehouse_master { - local results=$(DEVSTACK_GATE_GRENADE=pullup DEVSTACK_GATE_TEMPEST=1 ./test-matrix.py -b stable/icehouse) - assert_list_equal $GRENADE_ICEHOUSE_MASTER $results -} - test_full_master test_full_feature_ec test_neutron_master -test_neutron_kilo test_heat_slow_master test_grenade_new_master -test_grenade_juno_master -test_grenade_icehouse_master -test_full_juno if [[ "$ERRORS" -ne 0 ]]; then echo "Errors detected, job failed" From f624a73604178956123dcded5b6fe025fdc61422 Mon Sep 17 00:00:00 2001 From: Paul Belanger Date: Thu, 19 May 2016 10:32:18 -0400 Subject: [PATCH 062/148] Use local AFS mirror for network_sanity_check Rather then pin to rax, lets use our local AFS mirrors. We don't have signed packages in AFS yet, so use Release. Change-Id: I4e79fa026f4d4a2e4c5be46d6c9a74b87683de7e Signed-off-by: Paul Belanger --- functions.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/functions.sh b/functions.sh index f61c4539..807dd614 100644 --- a/functions.sh +++ b/functions.sh @@ -130,9 +130,13 @@ function network_sanity_check { _http_check $pypi_url fi - # rax ubuntu mirror - _ping_check mirror.rackspace.com - _http_check http://mirror.rackspace.com/ubuntu/dists/trusty/Release.gpg + # AFS ubuntu mirror + source /etc/nodepool/provider + NODEPOOL_MIRROR_HOST=${NODEPOOL_MIRROR_HOST:-mirror.$NODEPOOL_REGION.$NODEPOOL_CLOUD.openstack.org} + NODEPOOL_MIRROR_HOST=$(echo $NODEPOOL_MIRROR_HOST|tr '[:upper:]' '[:lower:]') + + _ping_check $NODEPOOL_MIRROR_HOST + _http_check http://$NODEPOOL_MIRROR_HOST/ubuntu/dists/trusty/Release } # create the start timer for when the job began From 46ce759f4dc9334e589733bd1586dfc6ee156ccc Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 29 Aug 2016 16:51:41 -0400 Subject: [PATCH 063/148] set up stable/newton Change-Id: I848a38f00a1fe8cd599b862b9c4e59175bbb9d2f Signed-off-by: Doug Hellmann --- features.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/features.yaml b/features.yaml index 9994b0a6..e48c0cad 100644 --- a/features.yaml +++ b/features.yaml @@ -2,6 +2,7 @@ config: default: master: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] ocata: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] + newton: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] mitaka: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] liberty: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] kilo: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] From a7b1cba9a73ef36ac026741502be40b39b28ba60 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 30 Aug 2016 14:00:40 -0400 Subject: [PATCH 064/148] add newton to the allowed list Change-Id: I6b30d0d7f1d577002ae98a38f7264aac5b38b74f Signed-off-by: Doug Hellmann --- features.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features.yaml b/features.yaml index e48c0cad..5112071e 100644 --- a/features.yaml +++ b/features.yaml @@ -55,7 +55,7 @@ branches: # The value of ""default" is the name of the "trunk" branch default: master # Normalized branch names only here, e.g. stable/icehouse => icehouse - allowed: [master, ocata, mitaka, liberty, kilo, juno, icehouse] + allowed: [master, ocata, newton, mitaka, liberty, kilo, juno, icehouse] features: default: From c50c32ef5d7b5edf8c1d5b23959604b4411d22ce Mon Sep 17 00:00:00 2001 From: Lucas Alvares Gomes Date: Tue, 30 Aug 2016 14:24:39 +0100 Subject: [PATCH 065/148] Ironic: Copy all deployment logs Prior to this patch the cleanup_host function was copying only the *.log files when collecting the logs from Ironic, this patch is removing this filter and copying the whole directory instead. The ironic-bm-logs might contain another directory inside called "deploy_logs" which contain some troubleshooting information from the deploy ramdisk. Change-Id: Icb6952ac96ccc840423d0f3df01a3233611a60ce --- functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index 6264d4d2..3eb594b3 100644 --- a/functions.sh +++ b/functions.sh @@ -904,7 +904,7 @@ function cleanup_host { # Copy Ironic nodes console logs if they exist if [ -d $BASE/new/ironic-bm-logs ] ; then sudo mkdir -p $BASE/logs/ironic-bm-logs - sudo cp $BASE/new/ironic-bm-logs/*.log $BASE/logs/ironic-bm-logs/ + sudo cp -r $BASE/new/ironic-bm-logs/* $BASE/logs/ironic-bm-logs/ fi # Copy tempest config file From 7ee1b835e3ae21a55b46ae70c541376dfc4a4046 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 5 Sep 2016 11:33:42 +1000 Subject: [PATCH 066/148] Update to bashate 0.5.0 Update to bashate 0.5.0. Document options. Fix one simple error about (( )) return code. This introduces warnings about "local" hiding return values; fix in a follow-on. Change-Id: Id907a566e046ed65f7e02776f2be4382829afe8e --- functions.sh | 2 +- tox.ini | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/functions.sh b/functions.sh index 6264d4d2..917d994b 100644 --- a/functions.sh +++ b/functions.sh @@ -1160,7 +1160,7 @@ function ovs_vxlan_bridge { fi fi for node_ip in $peer_ips; do - (( offset++ )) + offset=$(( offset+1 )) # For reference on how to setup a tunnel using OVS see: # http://openvswitch.org/support/config-cookbooks/port-tunneling/ # The command below is equivalent to the sequence of ip/brctl commands diff --git a/tox.ini b/tox.ini index cf770882..eab1bb0d 100644 --- a/tox.ini +++ b/tox.ini @@ -13,9 +13,12 @@ commands = bash -c "./run-tests.sh" [testenv:bashate] -whitelist_externals = - bash deps= - {env:BASHATE_INSTALL_PATH:bashate==0.3.1} + {env:BASHATE_INSTALL_PATH:bashate==0.5.0} +whitelist_externals= + bash +# bashate options: +# -i E006 : ignore long lines +# -e E005 : error if not starting with #! commands = - bash -c "ls *.sh | xargs bashate -v {posargs}" + bash -c "ls *.sh | xargs bashate -v {posargs} -iE006 -eE005" From e3585e1c3de73c7f127bb8a111f67d6b03df0e3b Mon Sep 17 00:00:00 2001 From: Kevin Benton Date: Thu, 8 Sep 2016 19:08:44 -0700 Subject: [PATCH 067/148] Don't use 1:1000 VNI range for node-to-node tunnels These ranges are used by Neutron by default with a devstack deployment. So if we conflict with a Neutron network's VNI allocation, it usually breaks traffic between nodes for that network. Change-Id: I94f985446577e2cb71849a7b42f96ffd409c08ae Closes-Bug: #1599329 --- functions.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/functions.sh b/functions.sh index 96108a17..f56091fd 100644 --- a/functions.sh +++ b/functions.sh @@ -1151,6 +1151,8 @@ function ovs_vxlan_bridge { shift 4 fi local peer_ips=$@ + # neutron uses 1:1000 with default devstack configuration, avoid overlap + local additional_vni_offset=1000000 eval $install_ovs_deps # create a bridge, just like you would with 'brctl addbr' # if the bridge exists, --may-exist prevents ovs from returning an error @@ -1167,6 +1169,7 @@ function ovs_vxlan_bridge { fi for node_ip in $peer_ips; do offset=$(( offset+1 )) + vni=$(( offset + additional_vni_offset )) # For reference on how to setup a tunnel using OVS see: # http://openvswitch.org/support/config-cookbooks/port-tunneling/ # The command below is equivalent to the sequence of ip/brctl commands @@ -1180,7 +1183,7 @@ function ovs_vxlan_bridge { ${bridge_name}_${node_ip} \ -- set interface ${bridge_name}_${node_ip} type=vxlan \ options:remote_ip=${node_ip} \ - options:key=${offset} \ + options:key=${vni} \ options:local_ip=${host_ip} # Now complete the vxlan tunnel setup for the Compute Node: # Similarly this establishes the tunnel in the reverse direction @@ -1191,7 +1194,7 @@ function ovs_vxlan_bridge { ${bridge_name}_${host_ip} \ -- set interface ${bridge_name}_${host_ip} type=vxlan \ options:remote_ip=${host_ip} \ - options:key=${offset} \ + options:key=${vni} \ options:local_ip=${node_ip} if [[ "$set_ips" == "True" ]] ; then if ! remote_command $node_ip sudo ip addr show dev ${bridge_name} | \ From c6aad9ff4bae04464341faf9e81ce5a9f089c108 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Fri, 9 Sep 2016 21:30:58 -0400 Subject: [PATCH 068/148] Put the regex first This commit swaps the argument order of tempest tox jobs that use a regex to specify a subset of tests to run. This is necessary for some future changes to the tempest tox definitions that will switch it from using a testr wrapping bash script to the tempest run command instead. With tempest run you specify a regex with a '--regex' parameter and to keep backwards compatibility this is added to the tempest tox definitions. However this breaks on devstack-gate because we pass the concurrency in before the regex. This ends up having a call equivalent to: tempest run --regex --concurrency=2 $REGEX which obviously is incorrect. Simply switching the arg order should work in both cases. It's also worth pointing out that this is a temporary step as we will migrate devstack-gate to call tempest init and tempest run directly. Change-Id: I8786b9709584378aaf48393115dda94397e22544 Need-by: I3684fce66a799579fa68af119652cafef25a9f03 --- devstack-vm-gate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 2491eacc..78aa859f 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -757,11 +757,11 @@ if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then if [[ "$DEVSTACK_GATE_TEMPEST_REGEX" != "" ]] ; then if [[ "$DEVSTACK_GATE_TEMPEST_ALL_PLUGINS" -eq "1" ]]; then echo "Running tempest with plugins and a custom regex filter" - $TEMPEST_COMMAND -eall-plugin -- --concurrency=$TEMPEST_CONCURRENCY $DEVSTACK_GATE_TEMPEST_REGEX + $TEMPEST_COMMAND -eall-plugin -- $DEVSTACK_GATE_TEMPEST_REGEX --concurrency=$TEMPEST_CONCURRENCY sudo -H -u tempest .tox/all-plugin/bin/tempest list-plugins else echo "Running tempest with a custom regex filter" - $TEMPEST_COMMAND -eall -- --concurrency=$TEMPEST_CONCURRENCY $DEVSTACK_GATE_TEMPEST_REGEX + $TEMPEST_COMMAND -eall -- $DEVSTACK_GATE_TEMPEST_REGEX --concurrency=$TEMPEST_CONCURRENCY fi elif [[ "$DEVSTACK_GATE_TEMPEST_ALL_PLUGINS" -eq "1" ]]; then echo "Running tempest all-plugins test suite" From 99c191fb892b86c16f61f2a359c77ba6afeaaa35 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 13 Sep 2016 11:56:48 -0700 Subject: [PATCH 069/148] Handle multiline env vars in reproduce.sh Our old reproduce.sh env var processing was line based which meant we mangled multiline env var values. Switch to using null delimiters when processing the env vars so that we can preserve any multiline env vars. Change-Id: Ia0756baf0dfea790d489f7cbff68c0ec5887b3d1 --- functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index f61c4539..2b56751d 100644 --- a/functions.sh +++ b/functions.sh @@ -180,7 +180,7 @@ EOF # first get all keys that match our filter and then output the whole line # that will ensure that multi-line env vars are set properly - for KEY in $(printenv | grep '\(DEVSTACK\|ZUUL\)' | sed 's/\(.*\)=.*/\1/'); do + for KEY in $(printenv -0 | grep -z -Z '\(DEVSTACK\|ZUUL\)' | sed -z -n 's/^\([^=]\+\)=.*/\1\n/p'); do echo "declare -x ${KEY}=\"${!KEY}\"" >> $WORKSPACE/logs/reproduce.sh done if [ -n "$JOB_PROJECTS" ] ; then From 25e81473204478f5b144ec103d57171590a2fc7c Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 14 Sep 2016 14:28:16 -0700 Subject: [PATCH 070/148] Explicitly set overlay bridge ints as up Notice that nova net multinode jobs fail due to a lack of connectivity from tempest to cirros VMs. On further inspections br_pub is DOWN on this host. Setting it up after the fact seems to make it happy so add an explicit UP state transition to our setup. Change-Id: I09d3ebdc7459dccf22aa4f75aa24777715f7ffad --- functions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/functions.sh b/functions.sh index 107828c5..8f6fdd36 100644 --- a/functions.sh +++ b/functions.sh @@ -1167,6 +1167,7 @@ function ovs_vxlan_bridge { dev ${bridge_name} fi fi + sudo ip link set dev $bridge_name up for node_ip in $peer_ips; do offset=$(( offset+1 )) vni=$(( offset + additional_vni_offset )) @@ -1204,6 +1205,7 @@ function ovs_vxlan_bridge { dev ${bridge_name} fi fi + remote_command $node_ip sudo ip link set dev $bridge_name up done } From e326d0a7fa9ceb04c5cb663097f2f29d35ee4ed7 Mon Sep 17 00:00:00 2001 From: Vasyl Saienko Date: Tue, 9 Aug 2016 16:40:51 +0300 Subject: [PATCH 071/148] Add c-vol,c-bak on subnode when c-api enabled Cinder is not installed on primary node in Ironic jobs. When c-vol,c-bak were added to MY_ENABLED_SERVICES on subnode it caused ironic multinode job to failure. This patch adds c-vol,c-bak to enabled_services only when c-api is enabled on primary node. Reference: [0] https://review.openstack.org/#/c/327619/ Co-Authored-By: Vladyslav Drok Change-Id: I7807fe5f76d9fa8b778371299d177cb0bfafc78b Closes-Bug: #1611381 --- devstack-vm-gate.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 2491eacc..e8d8a414 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -215,7 +215,10 @@ function setup_localrc { # TODO(afazekas): Move to the feature grid if [[ $role = sub ]]; then - MY_ENABLED_SERVICES="n-cpu,ceilometer-acompute,dstat,c-vol,c-bak" + MY_ENABLED_SERVICES="n-cpu,ceilometer-acompute,dstat" + if [[ "$original_enabled_services" =~ "c-api" ]]; then + MY_ENABLED_SERVICES+=",c-vol,c-bak" + fi if [[ "$DEVSTACK_GATE_NEUTRON" -eq "1" ]]; then MY_ENABLED_SERVICES+=",q-agt" if [[ "$DEVSTACK_GATE_NEUTRON_DVR" -eq "1" ]]; then From 562325eaf5de64a57040274724bf1ccb3fe98239 Mon Sep 17 00:00:00 2001 From: Vasyl Saienko Date: Thu, 30 Jun 2016 15:30:47 +0300 Subject: [PATCH 072/148] Preparing multinode networking for Ironic At the gates Ironic uses VMs instead of real hardware servers. Those VMs are not managed by Nova and is used for emulating hardware servers for Ironic. In multinode topology those VMs are created on all nodes primary and subnodes. This patch adds an extra vxlan tunnel and new bridge br_ironic_vxlan to provide connectivity between VMs on different nodes. Change-Id: I8892a2418e3bee35dacce1d5959faa30dd38e971 --- devstack-vm-gate.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index e8d8a414..b40fec39 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -132,6 +132,14 @@ EOF $sub_nodes fi + if [[ "$DEVSTACK_GATE_IRONIC" -eq '1' ]]; then + # NOTE(vsaienko) Ironic VMs will be connected to this bridge + # in order to have access to VMs on another nodes. + ovs_vxlan_bridge "br_ironic_vxlan" $primary_node "False" 128 \ + $sub_nodes + + fi + echo "Preparing cross node connectivity" setup_ssh $BASE/new/.ssh setup_ssh ~root/.ssh From 2b2fc889fee77f7f0aafc01f28381514638d2209 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 29 Aug 2016 16:51:48 -0400 Subject: [PATCH 073/148] update grenade settings for stable/newton Depends-On: Id12d230720a8c472856e94ee60033d78feee95c8 Depends-On: Ied33f52bb0e3265a2ca4fcf76658f1b5c373cd07 Change-Id: Iaed754f559fa44181b88c6036b209e92ee23c984 Signed-off-by: Doug Hellmann --- devstack-vm-gate-wrap.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 6ac5418e..3a06f191 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -284,6 +284,9 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then export GRENADE_NEW_BRANCH="stable/mitaka" elif [[ "$GRENADE_BASE_BRANCH" == "stable/mitaka" ]]; then export GRENADE_OLD_BRANCH="stable/mitaka" + export GRENADE_NEW_BRANCH="stable/newton" + elif [[ "$GRENADE_BASE_BRANCH" == "stable/newton" ]]; then + export GRENADE_OLD_BRANCH="stable/newton" export GRENADE_NEW_BRANCH="$GIT_BRANCH" fi ;; @@ -303,8 +306,11 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then elif [[ "$GRENADE_BASE_BRANCH" == "stable/mitaka" ]]; then export GRENADE_OLD_BRANCH="stable/liberty" export GRENADE_NEW_BRANCH="stable/mitaka" - else # master + elif [[ "$GRENADE_BASE_BRANCH" == "stable/newton" ]]; then export GRENADE_OLD_BRANCH="stable/mitaka" + export GRENADE_NEW_BRANCH="stable/newton" + else # master + export GRENADE_OLD_BRANCH="stable/newton" export GRENADE_NEW_BRANCH="$GIT_BRANCH" fi ;; From 504179d3a56c8679cb708ae274afdad13825111e Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Fri, 23 Sep 2016 17:36:46 -0700 Subject: [PATCH 074/148] Enable tlsproxy on ocata and above Since tlsproxy wasn't working in older branches specifically only enable tlsproxy if >=ocata. This allows us to set the flag to turn this on in the JJB config without a bunch of branch checks. Depends-On: I2e26c188e723002e8eb058cfb2d881ab858cfaa4 Change-Id: Ic21c0213c8ea91f3d7c30f03227af7b5a107b527 --- features.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/features.yaml b/features.yaml index 5112071e..976283fa 100644 --- a/features.yaml +++ b/features.yaml @@ -174,3 +174,10 @@ features: tlsproxy: base: services: [tls-proxy] + # TLS proxy didn't work properly until ocata + liberty: + rm-services: [tls-proxy] + mitaka: + rm-services: [tls-proxy] + newton: + rm-services: [tls-proxy] From 7d7e84df09d154cb5660e7a864582a188de6b4b6 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 28 Apr 2015 13:50:10 -0700 Subject: [PATCH 075/148] Remove tracing since ansible seems to be working We no longer need to trace everything which we put in place temporarily to keep track of what ansible is doing. Remove the tracing. Change-Id: Ib98ba5c699d2f18e75783aace5e788f7777d0029 --- devstack-vm-gate-wrap.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 3a06f191..4e641eb9 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -441,9 +441,6 @@ with $DEVSTACK_GATE_TIMEOUT_BUFFER minutes reserved for cleanup." echo "Available disk space on this host:" indent df -h -# Enable tracing while we transition to using ansible to run -# setup across multiple nodes. -set -x # Install ansible sudo -H pip install virtualenv virtualenv /tmp/ansible From c2516e95167f92f056a9acc6c1427ba61ca54468 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 20 Sep 2016 05:02:54 -0700 Subject: [PATCH 076/148] Copy ssl certs for multinode This updates the version of ansible because the older version had a bug that broke the synchronize module for us. As a side effect of this we can no longer rely on ansible nooping if an inventory group is empty. This is now an error so now we check if the subnodes list is empty before running commands against them. Change-Id: I66b17561cac62b9bcedd67a724153612744aead6 --- devstack-vm-gate-wrap.sh | 24 ++++++++++++++---------- devstack-vm-gate.sh | 28 ++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 6ac5418e..779cd2d3 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -28,7 +28,7 @@ GIT_BRANCH=${GIT_BRANCH:-master} # We're using enough ansible specific features that it's extremely # possible that new ansible releases can break us. As such we should # be very deliberate about which ansible we use. -ANSIBLE_VERSION=${ANSIBLE_VERSION:-2.0.0.2} +ANSIBLE_VERSION=${ANSIBLE_VERSION:-2.1.1.0} # sshd may have been compiled with a default path excluding */sbin export PATH=$PATH:/usr/local/sbin:/usr/sbin @@ -451,7 +451,7 @@ COUNTER=1 echo "[primary]" > "$WORKSPACE/inventory" echo "localhost ansible_connection=local host_counter=$COUNTER" >> "$WORKSPACE/inventory" echo "[subnodes]" >> "$WORKSPACE/inventory" -SUBNODES=$(cat /etc/nodepool/sub_nodes_private) +export SUBNODES=$(cat /etc/nodepool/sub_nodes_private) for SUBNODE in $SUBNODES ; do let COUNTER=COUNTER+1 echo "$SUBNODE host_counter=$COUNTER" >> "$WORKSPACE/inventory" @@ -462,13 +462,15 @@ done # vars to ansible shell commands. This may need to change in the future but # for now the current setup is simple, consistent and easy to understand. -# Copy bootstrap to remote hosts -# It is in brackets for avoiding inheriting a huge environment variable +# This is in brackets for avoiding inheriting a huge environment variable (export PROJECTS; export > "$WORKSPACE/test_env.sh") -$ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m copy \ - -a "src='$WORKSPACE/devstack-gate' dest='$WORKSPACE'" -$ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m copy \ - -a "src='$WORKSPACE/test_env.sh' dest='$WORKSPACE/test_env.sh'" +if [ -n "$SUBNODES" ] ; then + # Copy bootstrap to remote hosts + $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m copy \ + -a "src='$WORKSPACE/devstack-gate' dest='$WORKSPACE'" + $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m copy \ + -a "src='$WORKSPACE/test_env.sh' dest='$WORKSPACE/test_env.sh'" +fi # Make a directory to store logs $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m file \ @@ -601,8 +603,10 @@ echo "Cleaning up host" echo "... this takes 3 - 4 minutes (logs at logs/devstack-gate-cleanup-host.txt.gz)" $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ -a "$(run_command cleanup_host)" &> "$WORKSPACE/devstack-gate-cleanup-host.txt" -$ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m synchronize \ - -a "mode=pull src='$BASE/logs/' dest='$BASE/logs/subnode-{{ host_counter }}'" +if [ -n "$SUBNODES" ] ; then + $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m synchronize \ + -a "mode=pull src='$BASE/logs/' dest='$BASE/logs/subnode-{{ host_counter }}'" +fi sudo mv $WORKSPACE/devstack-gate-cleanup-host.txt $BASE/logs/ exit $RETVAL diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 6abc0ef3..ce34eed3 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -631,12 +631,28 @@ else $ANSIBLE primary -f 5 -i "$WORKSPACE/inventory" -m shell \ -a "cd '$BASE/new/devstack' && sudo -H -u stack stdbuf -oL -eL ./stack.sh executable=/bin/bash" \ &> "$WORKSPACE/logs/devstack-early.txt" - # Run non controller setup after controller is up. This is necessary - # because services like nova apparently expect to have the controller in - # place before anything else. - $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m shell \ - -a "cd '$BASE/new/devstack' && sudo -H -u stack stdbuf -oL -eL ./stack.sh executable=/bin/bash" \ - &> "$WORKSPACE/logs/devstack-subnodes-early.txt" + if [ -n "$SUBNODES" ] ; then + if [ -d "$BASE/data/CA" ] && [ -f "$BASE/data/ca-bundle.pem" ] ; then + # Sync any data files which include certificates to be used if + # TLS is enabled + $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" --sudo -m file \ + -a "path='$BASE/data' state=directory owner=stack group=stack mode=0755" + $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" --sudo -m file \ + -a "path='$BASE/data/CA' state=directory owner=stack group=stack mode=0755" + $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" \ + --sudo -m synchronize \ + -a "mode=push src='$BASE/data/ca-bundle.pem' dest='$BASE/data/ca-bundle.pem'" + sudo $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" \ + --sudo -u $USER -m synchronize \ + -a "mode=push src='$BASE/data/CA' dest='$BASE/data'" + fi + # Run non controller setup after controller is up. This is necessary + # because services like nova apparently expect to have the controller in + # place before anything else. + $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "cd '$BASE/new/devstack' && sudo -H -u stack stdbuf -oL -eL ./stack.sh executable=/bin/bash" \ + &> "$WORKSPACE/logs/devstack-subnodes-early.txt" + fi end=$(date +%s) took=$((($end - $start) / 60)) if [[ "$took" -gt 20 ]]; then From 8e6edce98dfa5e83281ca2ad0075b0e7ffa76c42 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 5 Oct 2016 10:11:10 -0700 Subject: [PATCH 077/148] Enable tls-proxy on subnode if enabled on primary We need to ensure tls-proxy is enabled on the subnode so that the subnode configs get setup to use services fronted with tls. This was not caught in testing because we used a devstack change that always turned on tls-proxy regardless to test. Change-Id: I5f603b22c0e4f1e9c9a984c215680aec3a146703 --- devstack-vm-gate.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index d0fa94a2..a62f6b42 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -227,6 +227,9 @@ function setup_localrc { if [[ "$original_enabled_services" =~ "c-api" ]]; then MY_ENABLED_SERVICES+=",c-vol,c-bak" fi + if [[ "$original_enabled_services" =~ "tls-proxy" ]]; then + MY_ENABLED_SERVICES+=",tls-proxy" + fi if [[ "$DEVSTACK_GATE_NEUTRON" -eq "1" ]]; then MY_ENABLED_SERVICES+=",q-agt" if [[ "$DEVSTACK_GATE_NEUTRON_DVR" -eq "1" ]]; then From 3bf06b0b0df8cd36447bfdf89f6402e321545bb7 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 3 Oct 2016 12:28:48 -0700 Subject: [PATCH 078/148] Use new ansible 2.2 for bugfixes The ansible 2.2 release returns the old ansible 2.0 behavior of nooping on an empty inventory group. This greatly simplifies the logic around running things for single node and multinode testing. Be prepared for this release when it arrives. Change-Id: I3b2bc9adc0f581b60adb0a58787461418cb25a0c --- devstack-vm-gate-wrap.sh | 20 ++++++++----------- devstack-vm-gate.sh | 42 +++++++++++++++++++--------------------- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index e5fc2323..c7cb6ddb 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -28,7 +28,7 @@ GIT_BRANCH=${GIT_BRANCH:-master} # We're using enough ansible specific features that it's extremely # possible that new ansible releases can break us. As such we should # be very deliberate about which ansible we use. -ANSIBLE_VERSION=${ANSIBLE_VERSION:-2.1.1.0} +ANSIBLE_VERSION=${ANSIBLE_VERSION:-2.2.0.0} # sshd may have been compiled with a default path excluding */sbin export PATH=$PATH:/usr/local/sbin:/usr/sbin @@ -470,13 +470,11 @@ done # This is in brackets for avoiding inheriting a huge environment variable (export PROJECTS; export > "$WORKSPACE/test_env.sh") -if [ -n "$SUBNODES" ] ; then - # Copy bootstrap to remote hosts - $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m copy \ - -a "src='$WORKSPACE/devstack-gate' dest='$WORKSPACE'" - $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m copy \ - -a "src='$WORKSPACE/test_env.sh' dest='$WORKSPACE/test_env.sh'" -fi +# Copy bootstrap to remote hosts +$ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m copy \ + -a "src='$WORKSPACE/devstack-gate' dest='$WORKSPACE'" +$ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m copy \ + -a "src='$WORKSPACE/test_env.sh' dest='$WORKSPACE/test_env.sh'" # Make a directory to store logs $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m file \ @@ -609,10 +607,8 @@ echo "Cleaning up host" echo "... this takes 3 - 4 minutes (logs at logs/devstack-gate-cleanup-host.txt.gz)" $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ -a "$(run_command cleanup_host)" &> "$WORKSPACE/devstack-gate-cleanup-host.txt" -if [ -n "$SUBNODES" ] ; then - $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m synchronize \ - -a "mode=pull src='$BASE/logs/' dest='$BASE/logs/subnode-{{ host_counter }}'" -fi +$ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m synchronize \ + -a "mode=pull src='$BASE/logs/' dest='$BASE/logs/subnode-{{ host_counter }}'" sudo mv $WORKSPACE/devstack-gate-cleanup-host.txt $BASE/logs/ exit $RETVAL diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index a62f6b42..dccfed77 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -645,28 +645,26 @@ else $ANSIBLE primary -f 5 -i "$WORKSPACE/inventory" -m shell \ -a "cd '$BASE/new/devstack' && sudo -H -u stack stdbuf -oL -eL ./stack.sh executable=/bin/bash" \ &> "$WORKSPACE/logs/devstack-early.txt" - if [ -n "$SUBNODES" ] ; then - if [ -d "$BASE/data/CA" ] && [ -f "$BASE/data/ca-bundle.pem" ] ; then - # Sync any data files which include certificates to be used if - # TLS is enabled - $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" --sudo -m file \ - -a "path='$BASE/data' state=directory owner=stack group=stack mode=0755" - $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" --sudo -m file \ - -a "path='$BASE/data/CA' state=directory owner=stack group=stack mode=0755" - $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" \ - --sudo -m synchronize \ - -a "mode=push src='$BASE/data/ca-bundle.pem' dest='$BASE/data/ca-bundle.pem'" - sudo $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" \ - --sudo -u $USER -m synchronize \ - -a "mode=push src='$BASE/data/CA' dest='$BASE/data'" - fi - # Run non controller setup after controller is up. This is necessary - # because services like nova apparently expect to have the controller in - # place before anything else. - $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m shell \ - -a "cd '$BASE/new/devstack' && sudo -H -u stack stdbuf -oL -eL ./stack.sh executable=/bin/bash" \ - &> "$WORKSPACE/logs/devstack-subnodes-early.txt" - fi + if [ -d "$BASE/data/CA" ] && [ -f "$BASE/data/ca-bundle.pem" ] ; then + # Sync any data files which include certificates to be used if + # TLS is enabled + $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" --sudo -m file \ + -a "path='$BASE/data' state=directory owner=stack group=stack mode=0755" + $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" --sudo -m file \ + -a "path='$BASE/data/CA' state=directory owner=stack group=stack mode=0755" + $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" \ + --sudo -m synchronize \ + -a "mode=push src='$BASE/data/ca-bundle.pem' dest='$BASE/data/ca-bundle.pem'" + sudo $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" \ + --sudo -u $USER -m synchronize \ + -a "mode=push src='$BASE/data/CA' dest='$BASE/data'" + fi + # Run non controller setup after controller is up. This is necessary + # because services like nova apparently expect to have the controller in + # place before anything else. + $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "cd '$BASE/new/devstack' && sudo -H -u stack stdbuf -oL -eL ./stack.sh executable=/bin/bash" \ + &> "$WORKSPACE/logs/devstack-subnodes-early.txt" end=$(date +%s) took=$((($end - $start) / 60)) if [[ "$took" -gt 20 ]]; then From 4b70a096360f3983f76d26660b74ba234ce0556c Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Fri, 7 Oct 2016 14:09:40 -0700 Subject: [PATCH 079/148] Make sure the d-g hooks end up in reproduce.sh Currently we don't have any handling for devstack-gate hooks in the reproduce.sh script so jobs that rely on them don't generate accurate reproduce.sh scripts. Fix this by explicitly adding each of the three hooks to the reproduce.sh script if they are defined. Change-Id: Iae6191dfc2dab81e7ca172762728ede2946f5f62 --- functions.sh | 5 +++++ test-functions.sh | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/functions.sh b/functions.sh index 7b3077f5..72ef1bbe 100644 --- a/functions.sh +++ b/functions.sh @@ -203,6 +203,11 @@ EOF if [ -n "$JOB_PROJECTS" ] ; then echo "declare -x PROJECTS=\"$JOB_PROJECTS\"" >> $WORKSPACE/logs/reproduce.sh fi + for fun in pre_test_hook gate_hook post_test_hook ; do + if function_exists $fun ; then + declare -fp $fun >> $WORKSPACE/logs/reproduce.sh + fi + done cat >> $WORKSPACE/logs/reproduce.sh < Date: Thu, 13 Oct 2016 10:12:56 -0700 Subject: [PATCH 080/148] Put the devstacklog.txt.summary file in the logs directory The summary log file for devstack was not being put into the logs directory. Put it in the directory as devstacklog.summary.txt. Change-Id: I2c33ae685bca04add6e7b689ffe617051146ec2a --- functions.sh | 2 ++ help/tempest-logs.html | 1 + 2 files changed, 3 insertions(+) diff --git a/functions.sh b/functions.sh index 7b3077f5..d8db0df3 100644 --- a/functions.sh +++ b/functions.sh @@ -844,6 +844,7 @@ function cleanup_host { find $BASE/old/screen-logs -type l -print0 | \ xargs -0 -I {} sudo cp {} $BASE/logs/old sudo cp $BASE/old/devstacklog.txt $BASE/logs/old/ + sudo cp $BASE/old/devstacklog.txt.summary $BASE/logs/old/devstacklog.summary.txt sudo cp $BASE/old/devstack/localrc $BASE/logs/old/localrc.txt sudo cp $BASE/old/tempest/etc/tempest.conf $BASE/logs/old/tempest_conf.txt if [ -f $BASE/old/devstack/tempest.log ] ; then @@ -900,6 +901,7 @@ function cleanup_host { find $BASE/new/screen-logs -type l -print0 | \ xargs -0 -I {} sudo cp {} $NEWLOGTARGET/ sudo cp $BASE/new/devstacklog.txt $NEWLOGTARGET/ + sudo cp $BASE/new/devstacklog.txt.summary $NEWLOGTARGET/devstacklog.summary.txt sudo cp $BASE/new/devstack/localrc $NEWLOGTARGET/localrc.txt if [ -f $BASE/new/devstack/tempest.log ]; then sudo cp $BASE/new/devstack/tempest.log $NEWLOGTARGET/verify_tempest_conf.log diff --git a/help/tempest-logs.html b/help/tempest-logs.html index cfdc2adb..a8f2579c 100644 --- a/help/tempest-logs.html +++ b/help/tempest-logs.html @@ -107,6 +107,7 @@

      Types of logs

    • devstack

    From 01b55c8412f463b2da0569e43dfcad5614da8559 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 17 Oct 2016 11:32:36 -0700 Subject: [PATCH 081/148] Reorganize PROJECTS list to show repos to remove We want to reduce the size of the PROJECTS list by default to more closely match things done by a default devstack + tempest test run. The reason for that is every project in the PROJECTS list has to be cloned, updated, and checked out to the proper ref as part of setup. This can be slow so reduce the total number of projects that need this by default. In this change we split the list into two halves, the first is the half that should remain and the second is the half that we should work to remove. Hopefully this also discourages new changes to add projects to this list. Change-Id: Iaec3d0ad54879c22cd14907c15c8352ba1c63ec8 --- devstack-vm-gate-wrap.sh | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index e5fc2323..5a7503c3 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -47,17 +47,9 @@ JOB_PROJECTS="$PROJECTS" PROJECTS="openstack-infra/devstack-gate $PROJECTS" PROJECTS="openstack-dev/devstack $PROJECTS" PROJECTS="openstack-dev/pbr $PROJECTS" -PROJECTS="openstack-infra/tripleo-ci $PROJECTS" -PROJECTS="openstack/automaton $PROJECTS" PROJECTS="openstack/ceilometer $PROJECTS" PROJECTS="openstack/ceilometermiddleware $PROJECTS" PROJECTS="openstack/cinder $PROJECTS" -PROJECTS="openstack/cliff $PROJECTS" -PROJECTS="openstack/debtcollector $PROJECTS" -PROJECTS="openstack/dib-utils $PROJECTS" -PROJECTS="openstack/diskimage-builder $PROJECTS" -PROJECTS="openstack/django_openstack_auth $PROJECTS" -PROJECTS="openstack/futurist $PROJECTS" PROJECTS="openstack/glance $PROJECTS" PROJECTS="openstack/glance_store $PROJECTS" PROJECTS="openstack/heat $PROJECTS" @@ -67,15 +59,33 @@ PROJECTS="openstack/horizon $PROJECTS" PROJECTS="openstack/keystone $PROJECTS" PROJECTS="openstack/keystoneauth $PROJECTS" PROJECTS="openstack/keystonemiddleware $PROJECTS" +PROJECTS="openstack/neutron $PROJECTS" +PROJECTS="openstack/nova $PROJECTS" +PROJECTS="openstack/requirements $PROJECTS" +PROJECTS="openstack/swift $PROJECTS" +PROJECTS="openstack/tempest $PROJECTS" +PROJECTS="openstack/tempest-lib $PROJECTS" +# Everything below this line in the PROJECTS list is for non +# default devstack runs. Overtime we should remove items from +# below and add them explicitly to the jobs that need them. The +# reason for this is to reduce job runtimes, every git repo +# has to be cloned and updated and checked out to the proper ref +# which is not free. +PROJECTS="openstack-infra/tripleo-ci $PROJECTS" +PROJECTS="openstack/automaton $PROJECTS" +PROJECTS="openstack/cliff $PROJECTS" +PROJECTS="openstack/debtcollector $PROJECTS" +PROJECTS="openstack/dib-utils $PROJECTS" +PROJECTS="openstack/diskimage-builder $PROJECTS" +PROJECTS="openstack/django_openstack_auth $PROJECTS" +PROJECTS="openstack/futurist $PROJECTS" PROJECTS="openstack/manila $PROJECTS" PROJECTS="openstack/manila-ui $PROJECTS" PROJECTS="openstack/zaqar $PROJECTS" -PROJECTS="openstack/neutron $PROJECTS" PROJECTS="openstack/neutron-fwaas $PROJECTS" PROJECTS="openstack/neutron-lbaas $PROJECTS" PROJECTS="openstack/octavia $PROJECTS" PROJECTS="openstack/neutron-vpnaas $PROJECTS" -PROJECTS="openstack/nova $PROJECTS" PROJECTS="openstack/os-apply-config $PROJECTS" PROJECTS="openstack/os-brick $PROJECTS" PROJECTS="openstack/os-client-config $PROJECTS" @@ -102,14 +112,10 @@ PROJECTS="openstack/oslo.service $PROJECTS" PROJECTS="openstack/oslo.versionedobjects $PROJECTS" PROJECTS="openstack/oslo.vmware $PROJECTS" PROJECTS="openstack/pycadf $PROJECTS" -PROJECTS="openstack/requirements $PROJECTS" PROJECTS="openstack/sahara $PROJECTS" PROJECTS="openstack/sahara-dashboard $PROJECTS" PROJECTS="openstack/stevedore $PROJECTS" -PROJECTS="openstack/swift $PROJECTS" PROJECTS="openstack/taskflow $PROJECTS" -PROJECTS="openstack/tempest $PROJECTS" -PROJECTS="openstack/tempest-lib $PROJECTS" PROJECTS="openstack/tooz $PROJECTS" PROJECTS="openstack/tripleo-heat-templates $PROJECTS" PROJECTS="openstack/tripleo-image-elements $PROJECTS" From 83105d3c7dd7173c63c8af82894f697aaa74b76a Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Fri, 22 Jul 2016 15:43:44 +1000 Subject: [PATCH 082/148] Remove dib-utils & diskimage-builder from PROJECTS No projects using devstack-gate currently install dib from git; these are actually more like the python-* clients in that they are just installed via pip as dependencies. I do have a change out to the nodepool dsvm job to use dib from git (480dcb201aa93bc3d208e2bf5e96bcee6dd7df09) but since that is the only place we are doing this, we can add it to PROJECTS in that job and clean it up from here. Change-Id: Ia2b39193f481c879b454d10610ca1135959241b1 --- devstack-vm-gate-wrap.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 5a7503c3..dea0d686 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -75,8 +75,6 @@ PROJECTS="openstack-infra/tripleo-ci $PROJECTS" PROJECTS="openstack/automaton $PROJECTS" PROJECTS="openstack/cliff $PROJECTS" PROJECTS="openstack/debtcollector $PROJECTS" -PROJECTS="openstack/dib-utils $PROJECTS" -PROJECTS="openstack/diskimage-builder $PROJECTS" PROJECTS="openstack/django_openstack_auth $PROJECTS" PROJECTS="openstack/futurist $PROJECTS" PROJECTS="openstack/manila $PROJECTS" From 6172191497c01d524cad950654eee5916ef91146 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 18 Oct 2016 07:18:40 -0500 Subject: [PATCH 083/148] Re-add dib repos to conditionally if heat is being used DEVSTACK_GATE_HEAT=1 otherwise breaks jobs. Change-Id: I59e9834d249ccda0beeb715a8d6271140fb144f4 --- devstack-vm-gate-wrap.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index dea0d686..f72501da 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -75,6 +75,11 @@ PROJECTS="openstack-infra/tripleo-ci $PROJECTS" PROJECTS="openstack/automaton $PROJECTS" PROJECTS="openstack/cliff $PROJECTS" PROJECTS="openstack/debtcollector $PROJECTS" +# The devstack heat plugin uses these repos +if [[ "$DEVSTACK_GATE_HEAT" -eq "1" ]] ; then + PROJECTS="openstack/dib-utils $PROJECTS" + PROJECTS="openstack/diskimage-builder $PROJECTS" +fi PROJECTS="openstack/django_openstack_auth $PROJECTS" PROJECTS="openstack/futurist $PROJECTS" PROJECTS="openstack/manila $PROJECTS" From 350c8b1e1b9473d0341bb6b90a51902526ab278e Mon Sep 17 00:00:00 2001 From: Vasyl Saienko Date: Fri, 2 Sep 2016 13:29:13 +0300 Subject: [PATCH 084/148] Setup ssh-key on subnodes for Ironic Ironic conductor manages VMs via ssh. When conductor is located on primary node and VMs are located on subnodes we need to have ssh connection between nodes. Copy the master SSH key and place it in the Ironic's default key directory so all conductors (primary and subnodes) share the same SSH keys. Update the setup_ssh function to allow specifying the destination file name of the copies SSH key. It will now create the id_rsa.pub file too. Co-Authored-By: John L. Villalovos Change-Id: I80a01ac8714dbf569692935cea7319c50f3af3dd --- devstack-vm-gate.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index a62f6b42..75eedc58 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -57,13 +57,19 @@ LOCAL_MTU=$(ip link show | sed -ne 's/.*mtu \([0-9]\+\).*/\1/p' | sort -n | head EXTERNAL_BRIDGE_MTU=$((LOCAL_MTU - 50)) function setup_ssh { + # Copy the SSH key from /etc/nodepool/id_rsa{.pub} to the specified + # directory on 'all' the nodes. 'all' the nodes consists of the primary + # node and all of the subnodes. local path=$1 + local dest_file=${2:-id_rsa} $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m file \ -a "path='$path' mode=0700 state=directory" $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m copy \ -a "src=/etc/nodepool/id_rsa.pub dest='$path/authorized_keys' mode=0600" $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m copy \ - -a "src=/etc/nodepool/id_rsa dest='$path/id_rsa' mode=0400" + -a "src=/etc/nodepool/id_rsa.pub dest='$path/${dest_file}.pub' mode=0600" + $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m copy \ + -a "src=/etc/nodepool/id_rsa dest='$path/${dest_file}' mode=0400" } function setup_nova_net_networking { @@ -169,6 +175,16 @@ EOF remote_copy_file /tmp/tmp_sub_localrc $NODE:$devstack_dir/localrc remote_copy_file $localconf $NODE:$localconf done + + # NOTE(vsaienko) we need to have ssh connection among nodes to manage + # VMs from ironic-conductor or setup networking from networking-generic-switch + if [[ "$DEVSTACK_GATE_IRONIC" -eq '1' ]]; then + echo "Copy ironic key among nodes" + # NOTE(vsaienko) setup_ssh() set 700 to all parent directories when they doesn't + # exist. Keep ironic keys in other directory than /opt/stack/data to avoid setting + # 700 on /opt/stack/data + setup_ssh $BASE/new/.ssh ironic_key + fi } function setup_networking { From 63406fbbe3529ba8fc48ff74dab5a580b8aada80 Mon Sep 17 00:00:00 2001 From: Vasyl Saienko Date: Mon, 12 Sep 2016 10:48:52 +0300 Subject: [PATCH 085/148] Update ENABLED_SERVICE on subnode with ironic This patch adds ir-api and ir-cond on subnode when DEVSTACK_GATE_IRONIC is set to 1. Change-Id: Icf177d0c7f55d92299e3e1b3f1b346ffdbb707ca --- devstack-vm-gate.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 75eedc58..77bec96a 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -258,6 +258,9 @@ function setup_localrc { else MY_ENABLED_SERVICES+=",n-net,n-api-meta" fi + if [[ "$DEVSTACK_GATE_IRONIC" -eq "1" ]]; then + MY_ENABLED_SERVICES+=",ir-api,ir-cond" + fi fi # Allow optional injection of ENABLED_SERVICES from the calling context From 885ac7585f99e54fd3b98058e6c8debaeebe7f7c Mon Sep 17 00:00:00 2001 From: Vasyl Saienko Date: Tue, 9 Aug 2016 11:58:51 +0300 Subject: [PATCH 086/148] Update localrc for ironic-multinode case If running multinode Ironic then add variables to get passed to devstack for the primary and subnodes. Change-Id: I7e4ab0fec79f027302baea7257c9851f34d8be39 Co-Authored-By: John L. Villalovos --- devstack-vm-gate.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 77bec96a..2767dd73 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -144,6 +144,25 @@ EOF ovs_vxlan_bridge "br_ironic_vxlan" $primary_node "False" 128 \ $sub_nodes + cat <>"$sub_localrc" +HOST_TOPOLOGY=multinode +HOST_TOPOLOGY_ROLE=subnode +# NOTE(vsaienko) we assume for now that we using only 1 subnode, +# each subnode should have different switch name (bridge) as it is used +# by networking-generic-switch to uniquely identify switch. +IRONIC_VM_NETWORK_BRIDGE=sub1brbm +OVS_PHYSICAL_BRIDGE=sub1brbm +ENABLE_TENANT_TUNNELS=False +IRONIC_KEY_FILE="$BASE/new/.ssh/ironic_key" +EOF + cat <>"$localrc" +HOST_TOPOLOGY=multinode +HOST_TOPOLOGY_ROLE=primary +HOST_TOPOLOGY_SUBNODES="$sub_nodes" +IRONIC_KEY_FILE="$BASE/new/.ssh/ironic_key" +GENERIC_SWITCH_KEY_FILE="$BASE/new/.ssh/ironic_key" +ENABLE_TENANT_TUNNELS=False +EOF fi echo "Preparing cross node connectivity" From e297c1215231df06bd8d103fb18fc25f783c4030 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 2 Nov 2016 11:22:14 -0700 Subject: [PATCH 087/148] Make reproduce.sh header more accurate We run on more than ubuntu trusty now, try to reflect that information and tell people where to determine exactly what distro to run on to properly reproduce a job. Also update the package list to include libssl-dev as it is now needed for cryptography things. Change-Id: I635b133b1775a6a931f520404a5fa8a23514e60f --- functions.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/functions.sh b/functions.sh index 2e40359e..7639d510 100644 --- a/functions.sh +++ b/functions.sh @@ -185,8 +185,12 @@ function reproduce { # Script to reproduce devstack-gate run. # # Prerequisites: -# - Fresh install of Ubuntu Trusty, with basic internet access -# - Must have python-dev, build-essential, and git installed from apt +# - Fresh install of current Ubuntu LTS, with basic internet access. +# Note we can and do run devstack-gate on other distros double check +# where your job ran (will be recorded in console.html) to reproduce +# as accurately as possible. +# - Must have python-all-dev, build-essential, git, libssl-dev installed +# from apt, or their equivalents on other distros. # - Must have virtualenv installed from pip # - Must be run as root # From 1d6cc0771a3399300117f488e9d71e7ea46a4d82 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 2 Nov 2016 11:53:22 -0700 Subject: [PATCH 088/148] Smartly set neutron default To aid in the deprecation of nova net attempt to enable neutron more globally against the master branch and intelligently keep old nova net default for old branches. The cells v1 job is explicitly using nova-net due to change: If86b061be095ed5cfc7a5f5594f84abae68baab0 The nova-net grenade jobs are using nova-net explicitly due to change: I33e8c81bb5366d2d1feccc144baa54ed3ed51e33 The integrated-gate-nova-net jobs are pinned to only run on stable/newton due to change: I3c13cb525da126e272789d0db6301f5e58c13b51 For everything else like designate/trove/ceph jobs that don't specify if they use neutron or not, those will start using neutron on master (Ocata) changes. Any fallout will have to be dealt with individually. Co-Authored-By: Matt Riedemann Change-Id: I6ece30e823c0cf21fbb2f37568c669dd7d3c6a08 --- devstack-vm-gate-wrap.sh | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 4dff19fd..6a3a47d2 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -190,9 +190,27 @@ export DEVSTACK_CINDER_SECURE_DELETE=${DEVSTACK_CINDER_SECURE_DELETE:-0} # Only applicable to stable/liberty+ devstack. export DEVSTACK_CINDER_VOLUME_CLEAR=${DEVSTACK_CINDER_VOLUME_CLEAR:-none} +# Set this to override the branch selected for testing (in +# single-branch checkouts; not used for grenade) +export OVERRIDE_ZUUL_BRANCH=${OVERRIDE_ZUUL_BRANCH:-$ZUUL_BRANCH} + +stable_compare="stable/[a-n]" + # Set to 1 to run neutron instead of nova network -# Only applicable to master branch -export DEVSTACK_GATE_NEUTRON=${DEVSTACK_GATE_NEUTRON:-0} +# This is a bit complicated to handle the deprecation of nova net across +# repos with branches from this branchless job runner. +if [ -n "$DEVSTACK_GATE_NEUTRON" ] ; then + # If someone has made a choice externally honor it + export DEVSTACK_GATE_NEUTRON=$DEVSTACK_GATE_NEUTRON +elif [[ "$OVERRIDE_ZUUL_BRANCH" =~ $stable_compare ]] ; then + # Default to no neutron on older stable branches because nova net + # was the default all that time. + export DEVSTACK_GATE_NEUTRON=0 +else + # For everything else there is neutron + export DEVSTACK_GATE_NEUTRON=1 +fi + # Set to 1 to run neutron distributed virtual routing export DEVSTACK_GATE_NEUTRON_DVR=${DEVSTACK_GATE_NEUTRON_DVR:-0} @@ -385,10 +403,6 @@ export DEVSTACK_GATE_REMOVE_STACK_SUDO=${DEVSTACK_GATE_REMOVE_STACK_SUDO:-1} # dependency-only installation. export DEVSTACK_GATE_UNSTACK=${DEVSTACK_GATE_UNSTACK:-0} -# Set this to override the branch selected for testing (in -# single-branch checkouts; not used for grenade) -export OVERRIDE_ZUUL_BRANCH=${OVERRIDE_ZUUL_BRANCH:-$ZUUL_BRANCH} - # Set Ceilometer backend to override the default one. It could be mysql, # postgresql, mongodb. export DEVSTACK_GATE_CEILOMETER_BACKEND=${DEVSTACK_GATE_CEILOMETER_BACKEND:-mysql} From 913fd7e2bf36d2ca008543f2d5c4454d3eb05745 Mon Sep 17 00:00:00 2001 From: "John L. Villalovos" Date: Fri, 11 Nov 2016 09:56:36 -0800 Subject: [PATCH 089/148] Add GRENADE_PLUGINRC and TEMPEST_CONCURRENCY to reproduce.sh For jobs that are running a grenade plugin (for example the Ironic grenade job), the environment variable GRENADE_PLUGINRC is used. Therefore if that variable exists it needs to be part of reproduce.sh. If TEMPEST_CONCURRENCY has been explicitly set to 1, as is the case for Ironic grenade jobs, then also write it to reproduce.sh. Change-Id: Ib038053633e8b43430c8116daa27b1d6f5bce913 --- functions.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index 7639d510..277f5ba3 100644 --- a/functions.sh +++ b/functions.sh @@ -201,9 +201,13 @@ EOF # first get all keys that match our filter and then output the whole line # that will ensure that multi-line env vars are set properly - for KEY in $(printenv -0 | grep -z -Z '\(DEVSTACK\|ZUUL\)' | sed -z -n 's/^\([^=]\+\)=.*/\1\n/p'); do + for KEY in $(printenv -0 | grep -z -Z '\(DEVSTACK\|GRENADE_PLUGINRC\|ZUUL\)' | sed -z -n 's/^\([^=]\+\)=.*/\1\n/p'); do echo "declare -x ${KEY}=\"${!KEY}\"" >> $WORKSPACE/logs/reproduce.sh done + # If TEMPEST_CONCURRENCY has been explicitly set to 1, then save it in reproduce.sh + if [ "${TEMPEST_CONCURRENCY}" -eq 1 ]; then + echo "declare -x TEMPEST_CONCURRENCY=\"${TEMPEST_CONCURRENCY}\"" >> $WORKSPACE/logs/reproduce.sh + fi if [ -n "$JOB_PROJECTS" ] ; then echo "declare -x PROJECTS=\"$JOB_PROJECTS\"" >> $WORKSPACE/logs/reproduce.sh fi From ad13acabfdb8655520f39ce0ebd6bd5f3961a826 Mon Sep 17 00:00:00 2001 From: Varun Gadiraju Date: Thu, 10 Nov 2016 21:41:37 +0000 Subject: [PATCH 090/148] Exit early if WORKSPACE var is undefined Check to see if the WORKSPACE environment variable is defined. If it is undefined print an error message and exit. This is useful for users who may run this on their own. Co-Authored-By: John L. Villalovos Change-Id: I8c569469b2358c49790498bdb33eb135889d7b21 --- devstack-vm-gate-wrap.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 6a3a47d2..5f0608c2 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -35,6 +35,12 @@ export PATH=$PATH:/usr/local/sbin:/usr/sbin # When doing xtrace (set -x / set -o xtrace), provide more debug output export PS4='+ ${BASH_SOURCE:-}:${FUNCNAME[0]:-}:L${LINENO:-}: ' +#check to see if WORKSPACE var is defined +if [ -z ${WORKSPACE} ]; then + echo "The 'WORKSPACE' variable is undefined. It must be defined for this script to work" + exit 1 +fi + source $WORKSPACE/devstack-gate/functions.sh start_timer From f283a62020615f3d266058b9d145b50fe83737db Mon Sep 17 00:00:00 2001 From: Michelle Mandel Date: Thu, 21 Jul 2016 14:52:09 -0400 Subject: [PATCH 091/148] Enable n-novnc service The service n-novnc is not enabled by default. This will enable it, allowing tests associated with NoVNC to be executed. This also specifies to install noVNC from packages instead of git cloning it for gate jobs. Change-Id: I09aed8de28f1ba2637382e870134ced38808df29 --- devstack-vm-gate.sh | 2 ++ features.yaml | 2 +- test-features.sh | 8 ++++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index b89f551a..aeff6904 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -335,6 +335,8 @@ ROOTSLEEP=0 # to correctly do testing. Otherwise you are not testing # the code you have posted for review. ERROR_ON_CLONE=True +# Since git clone can't be used for novnc in gates, force it to install the packages +NOVNC_FROM_PACKAGE=True ENABLED_SERVICES=$MY_ENABLED_SERVICES SKIP_EXERCISES=$SKIP_EXERCISES SERVICE_HOST=127.0.0.1 diff --git a/features.yaml b/features.yaml index 976283fa..cd2a96d4 100644 --- a/features.yaml +++ b/features.yaml @@ -80,7 +80,7 @@ features: nova: base: - services: [n-api, n-cond, n-cpu, n-crt, n-net, n-obj, n-sch] + services: [n-api, n-cauth, n-cond, n-cpu, n-crt, n-net, n-novnc, n-obj, n-sch] icehouse: compute-ext: diff --git a/test-features.sh b/test-features.sh index 0a9d4a85..69d96cfe 100755 --- a/test-features.sh +++ b/test-features.sh @@ -16,13 +16,13 @@ ERRORS=0 -TEMPEST_FULL_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,n-net" +TEMPEST_FULL_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,n-net" -TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering" +TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering" -TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering" +TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering" -GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" +GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" # Utility function for tests function assert_list_equal { From 338beb5bdbe61055b676acb54c0238a3c345f43d Mon Sep 17 00:00:00 2001 From: Sagi Shnaidman Date: Sun, 20 Nov 2016 15:08:42 +0200 Subject: [PATCH 092/148] Make chown and chmode to work on symlinks When clenup functions collects logs it use recursive chown and chmod, but it doesn't affect symlinks. There are symlinks with absolute path that have root permissions only and are failed to be copied with rsync later. On other hand rsync fails and doesn't recover on this error, which fails the whole taks of copying logs. Let's use -L option for chown, which means traverse symlinks. For chmod it's better to use "find" command, which will execute on each file and directory. See story: https://storyboard-dev.openstack.org/#!/story/55 Change-Id: I776cb8bc952c81b98cf0b1124eee9f28a92eb7b6 --- functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions.sh b/functions.sh index 7639d510..a8e25876 100644 --- a/functions.sh +++ b/functions.sh @@ -986,10 +986,10 @@ function cleanup_host { fi # Make sure the current user can read all the logs and configs - sudo chown -R $USER:$USER $BASE/logs/ + sudo chown -RL $USER:$USER $BASE/logs/ # (note X not x ... execute/search only if the file is a directory # or already has execute permission for some user) - sudo chmod -R a+rX $BASE/logs/ + sudo find $BASE/logs/ -exec chmod a+rX {} \; # Collect all the deprecation related messages into a single file. # strip out date(s), timestamp(s), pid(s), context information and From a211da5fc78b655022354334424179f9ff4a2ee3 Mon Sep 17 00:00:00 2001 From: Sagi Shnaidman Date: Tue, 22 Nov 2016 15:03:07 +0200 Subject: [PATCH 093/148] Find and delete broken symlinks in logs When rsync meet broken symlink with args --archive and --copy-link it fails to copy it and breaks all logs task. Find and delete all broken symlinks in logs folder before copy. See story: https://storyboard-dev.openstack.org/#!/story/56 Also move glusterfs logs part upper to be affected by chown,chmod and other logs directory changes. Change-Id: Ibf06dc83d697a7c5d6b8f3ae84334a53b7c49c7c --- devstack-vm-gate-wrap.sh | 2 +- functions.sh | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 6a3a47d2..00aeba57 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -631,7 +631,7 @@ echo "... this takes 3 - 4 minutes (logs at logs/devstack-gate-cleanup-host.txt. $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ -a "$(run_command cleanup_host)" &> "$WORKSPACE/devstack-gate-cleanup-host.txt" $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m synchronize \ - -a "mode=pull src='$BASE/logs/' dest='$BASE/logs/subnode-{{ host_counter }}'" + -a "mode=pull src='$BASE/logs/' dest='$BASE/logs/subnode-{{ host_counter }}' copy_links=yes" sudo mv $WORKSPACE/devstack-gate-cleanup-host.txt $BASE/logs/ exit $RETVAL diff --git a/functions.sh b/functions.sh index a8e25876..d20c7b52 100644 --- a/functions.sh +++ b/functions.sh @@ -985,11 +985,22 @@ function cleanup_host { sudo cp -r /var/log/openvswitch $BASE/logs/ fi + # glusterfs logs and config + if [ -d /var/log/glusterfs ] ; then + sudo cp -r /var/log/glusterfs $BASE/logs/ + fi + if [ -f /etc/glusterfs/glusterd.vol ] ; then + sudo cp /etc/glusterfs/glusterd.vol $BASE/logs/ + fi + # Make sure the current user can read all the logs and configs sudo chown -RL $USER:$USER $BASE/logs/ # (note X not x ... execute/search only if the file is a directory # or already has execute permission for some user) sudo find $BASE/logs/ -exec chmod a+rX {} \; + # Remove all broken symlinks, which point to non existing files + # They could be copied by rsync + sudo find $BASE/logs/ -type l -exec test ! -e {} \; -delete # Collect all the deprecation related messages into a single file. # strip out date(s), timestamp(s), pid(s), context information and @@ -1033,14 +1044,6 @@ function cleanup_host { done fi - # glusterfs logs and config - if [ -d /var/log/glusterfs ] ; then - sudo cp -r /var/log/glusterfs $BASE/logs/ - fi - if [ -f /etc/glusterfs/glusterd.vol ] ; then - sudo cp /etc/glusterfs/glusterd.vol $BASE/logs/ - fi - # final memory usage and process list ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > $BASE/logs/ps.txt From 8503e97bd727154f67d06f9007f3d8cbae875b7c Mon Sep 17 00:00:00 2001 From: Andrea Frittoli Date: Thu, 24 Nov 2016 18:04:18 +0000 Subject: [PATCH 094/148] Add ntp/ntpdate to reproduce.sh help text Starting from a fresh Xenial image, ntp and ntpdate must be installed for reproduce.sh to work. Adding them to the help text. Story: 2000808 Change-Id: I75ea28d4ee359f26059d6b0e1c001ae5c186f5f6 --- functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions.sh b/functions.sh index 7639d510..de115107 100644 --- a/functions.sh +++ b/functions.sh @@ -189,8 +189,8 @@ function reproduce { # Note we can and do run devstack-gate on other distros double check # where your job ran (will be recorded in console.html) to reproduce # as accurately as possible. -# - Must have python-all-dev, build-essential, git, libssl-dev installed -# from apt, or their equivalents on other distros. +# - Must have python-all-dev, build-essential, git, libssl-dev, ntp, ntpdate +# installed from apt, or their equivalents on other distros. # - Must have virtualenv installed from pip # - Must be run as root # From c579986564f0cd397e30ab92629fd52134272dd8 Mon Sep 17 00:00:00 2001 From: Vasyl Saienko Date: Mon, 12 Sep 2016 17:15:32 +0300 Subject: [PATCH 095/148] Drop SERVICE_HOST=127.0.0.1 from setup_localrc() SERVICE_HOST default is set in stackrc [0] This patch removes hardcoded 127.0.0.1 value from setup_localrc() function to pick default value specified in stackrc. Reference: [0] https://git.openstack.org/cgit/openstack-dev/devstack/tree/stackrc?id=d7c9d2e#n812 Change-Id: I0812e16583dfd678c8f349724a159b412f4efb55 --- devstack-vm-gate.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index b89f551a..0f4b8183 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -337,7 +337,6 @@ ROOTSLEEP=0 ERROR_ON_CLONE=True ENABLED_SERVICES=$MY_ENABLED_SERVICES SKIP_EXERCISES=$SKIP_EXERCISES -SERVICE_HOST=127.0.0.1 # Screen console logs will capture service logs. SYSLOG=False SCREEN_LOGDIR=$BASE/$localrc_oldnew/screen-logs From 08da6c623014b9d530e127f1d17fb59bd3b7f515 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 16 Nov 2016 16:14:01 -0600 Subject: [PATCH 096/148] Consume new variable IPV4_ADDRS_SAFE_TO_USE Change-Id: Ic6720677a4a496d246a8f9be5ede1a53821f56dc Depends-On: I7b32804d47bec743c0b13e434e6a7958728896ea --- devstack-vm-gate.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index b89f551a..ac0aecc0 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -33,7 +33,9 @@ source $TOP_DIR/functions.sh echo $PPID > $WORKSPACE/gate.pid source `dirname "$(readlink -f "$0")"`/functions.sh +# Need to set FIXED_RANGE for pre-ocata devstack FIXED_RANGE=${DEVSTACK_GATE_FIXED_RANGE:-10.1.0.0/20} +IPV4_ADDRS_SAFE_TO_USE=${DEVSTACK_GATE_IPV4_ADDRS_SAFE_TO_USE:-${DEVSTACK_GATE_FIXED_RANGE:-10.1.0.0/20}} FLOATING_RANGE=${DEVSTACK_GATE_FLOATING_RANGE:-172.24.5.0/24} PUBLIC_NETWORK_GATEWAY=${DEVSTACK_GATE_PUBLIC_NETWORK_GATEWAY:-172.24.5.1} # The next two values are used in multinode testing and are related @@ -344,6 +346,7 @@ SCREEN_LOGDIR=$BASE/$localrc_oldnew/screen-logs LOGFILE=$BASE/$localrc_oldnew/devstacklog.txt VERBOSE=True FIXED_RANGE=$FIXED_RANGE +IPV4_ADDRS_SAFE_TO_USE=$IPV4_ADDRS_SAFE_TO_USE FLOATING_RANGE=$FLOATING_RANGE PUBLIC_NETWORK_GATEWAY=$PUBLIC_NETWORK_GATEWAY FIXED_NETWORK_SIZE=4096 From 18f51e5d86a5b7ff24aa7cc81fc9441f85f6abd6 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Wed, 23 Nov 2016 21:46:42 +0000 Subject: [PATCH 097/148] Add setup_host.yaml playbook and ansibly gather_host_info This change adds the setup_host.yaml playbook, along with a playbook and role for gathering host info. The setup_host playbook is meant to replace the setup_host function from functions.sh. In this initial change it also invokes the gather_host_info playbook/role, when this lands I'll continue removing portions of logic from setup_host function and put equivalent plays/roles on the setup_host playbook. Change-Id: I916e075d8ed3ee6079e4c4621f6eb10f1d30bf95 --- devstack-vm-gate-wrap.sh | 7 +++++-- functions.sh | 10 ---------- playbooks/roles/gather_host_info/tasks/main.yaml | 7 +++++++ playbooks/setup_host.yaml | 5 +++++ 4 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 playbooks/roles/gather_host_info/tasks/main.yaml create mode 100644 playbooks/setup_host.yaml diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 6a3a47d2..109ef980 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -474,6 +474,7 @@ virtualenv /tmp/ansible # https://github.com/ansible/ansible/issues/15665 /tmp/ansible/bin/pip install paramiko==1.16.0 ansible==$ANSIBLE_VERSION export ANSIBLE=/tmp/ansible/bin/ansible +export ANSIBLE_PLAYBOOK=/tmp/ansible/bin/ansible-playbook # Write inventory file with groupings COUNTER=1 @@ -532,8 +533,10 @@ EOF } echo "... this takes a few seconds (logs at logs/devstack-gate-setup-host.txt.gz)" +$ANSIBLE_PLAYBOOK -f 5 -i "$WORKSPACE/inventory" "$WORKSPACE/devstack-gate/playbooks/setup_host.yaml" \ + &> "$WORKSPACE/logs/devstack-gate-setup-host.txt" $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ - -a "$(run_command setup_host)" &> "$WORKSPACE/logs/devstack-gate-setup-host.txt" + -a "$(run_command setup_host)" &>> "$WORKSPACE/logs/devstack-gate-setup-host.txt" if [ -n "$DEVSTACK_GATE_GRENADE" ]; then start=$(date +%s) @@ -631,7 +634,7 @@ echo "... this takes 3 - 4 minutes (logs at logs/devstack-gate-cleanup-host.txt. $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ -a "$(run_command cleanup_host)" &> "$WORKSPACE/devstack-gate-cleanup-host.txt" $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m synchronize \ - -a "mode=pull src='$BASE/logs/' dest='$BASE/logs/subnode-{{ host_counter }}'" + -a "mode=pull src='$BASE/logs/' dest='$BASE/logs/subnode-{{ host_counter }}' copy_links=yes" sudo mv $WORKSPACE/devstack-gate-cleanup-host.txt $BASE/logs/ exit $RETVAL diff --git a/functions.sh b/functions.sh index 7639d510..f43b8253 100644 --- a/functions.sh +++ b/functions.sh @@ -592,16 +592,6 @@ function setup_host { local xtrace=$(set +o | grep xtrace) set -o xtrace - echo "What's our kernel?" - uname -a - - # capture # of cpus - echo "NProc has discovered $(nproc) CPUs" - cat /proc/cpuinfo - - # Capture locale configuration - locale - # This is necessary to keep sudo from complaining fix_etc_hosts diff --git a/playbooks/roles/gather_host_info/tasks/main.yaml b/playbooks/roles/gather_host_info/tasks/main.yaml new file mode 100644 index 00000000..0f0a0007 --- /dev/null +++ b/playbooks/roles/gather_host_info/tasks/main.yaml @@ -0,0 +1,7 @@ +--- +- debug: msg="The running kernel version is {{ ansible_kernel }}" +- debug: msg="This instance has {{ ansible_processor_vcpus }} vcpu(s)" +- command: locale + register: locale_out +- name: "Locale is:" + debug: var=locale_out.stdout_lines diff --git a/playbooks/setup_host.yaml b/playbooks/setup_host.yaml new file mode 100644 index 00000000..137d6119 --- /dev/null +++ b/playbooks/setup_host.yaml @@ -0,0 +1,5 @@ +--- +- hosts: all + gather_facts: yes + roles: + - gather_host_info From 8ef2f9800d5e82774165a7ec5c12bba62d17afd7 Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Tue, 29 Nov 2016 14:00:38 -0500 Subject: [PATCH 098/148] Remove setting CEPH_LOOPBACK_DISK_SIZE There are two issues with setting this value in devstack-gate: 1. It's already hard-coded and set to the same value in the devstack-plugin-ceph repo which we are now using in all ceph jobs after I44c3a0c35e8dbcc49c0ac1800e9075cc3a169fdd. 2. The 8GB value is not large enough compared to non-ceph jobs which use VOLUME_BACKING_FILE_SIZE=24GB. That is being fixed in devstack-plugin-ceph to use VOLUME_BACKING_FILE_SIZE to set CEPH_LOOPBACK_DISK_SIZE in change I788eefa6c1d427bf51d2d3d40be4abe0336443e7. We can remove this now before the devstack-plugin-ceph change is merged because the plugin is already hard-coding the same value. Change-Id: I71be308c8373e9ac429b901c374100c6b3c1e59d Partial-Bug: #1645530 --- devstack-vm-gate.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 0e0191e0..87534fc8 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -293,7 +293,6 @@ function setup_localrc { if [[ "$DEVSTACK_GATE_CEPH" == "1" ]]; then echo "CINDER_ENABLED_BACKENDS=ceph:ceph" >>"$localrc_file" echo "TEMPEST_STORAGE_PROTOCOL=ceph" >>"$localrc_file" - echo "CEPH_LOOPBACK_DISK_SIZE=8G" >>"$localrc_file" fi # the exercises we *don't* want to test on for devstack From c0fdb2c2b05a6c4f6736e4fb43983e87a32cdce8 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 29 Nov 2016 09:57:46 -0600 Subject: [PATCH 099/148] Add ansible callback plugin to make output nicer Based on the in-tree minimal plugin, this reduces the amount of output, and also adds timestamps. It also automatically outputs stdout, so we can skip adding that ourselves. Finally - it gives us the framework to poke with it, should we desire to morph things further. The assumption is that CWD is $WORKSPACE when we're executing ansible-playbook Change-Id: I8378880d1a722be21b61311f3979483909b87fea --- devstack-vm-gate-wrap.sh | 7 + playbooks/plugins/callback/devstack.py | 141 ++++++++++++++++++ .../roles/gather_host_info/tasks/main.yaml | 7 +- 3 files changed, 150 insertions(+), 5 deletions(-) create mode 100644 playbooks/plugins/callback/devstack.py diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 109ef980..232a9339 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -487,6 +487,13 @@ for SUBNODE in $SUBNODES ; do echo "$SUBNODE host_counter=$COUNTER" >> "$WORKSPACE/inventory" done +# Write ansible config file +cat > "$WORKSPACE/ansible.cfg" < +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# Make coding more python3-ish +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from ansible import constants as C +from ansible.plugins.callback import CallbackBase +from ansible.vars import strip_internal_keys + +import datetime +import yaml + + +def _get_timestamp(): + return str(datetime.datetime.now())[:-3] + + +class CallbackModule(CallbackBase): + + '''Callback plugin for devstack-gate. + + Based on the minimal callback plugin from the ansible tree. Adds + timestamps to the start of the lines, squishes responses that are only + messages, returns facts in yaml not json format and strips facter facts + from the reported facts. + ''' + + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'stdout' + CALLBACK_NAME = 'devstack' + + def _command_generic_msg(self, host, result, task, caption): + '''output the result of a command run''' + + if caption == 'SUCCESS': + buf = "%s | %s | %s | %s >>\n" % ( + _get_timestamp(), host, caption, task.get_name().strip()) + else: + buf = "%s | %s | %s | %s | rc=%s >>\n" % ( + _get_timestamp(), host, caption, task.get_name().strip(), + result.get('rc', 0)) + buf += result.get('stdout', '') + buf += result.get('stderr', '') + buf += result.get('msg', '') + + return buf + "\n" + + def v2_runner_on_failed(self, result, ignore_errors=False): + if 'exception' in result._result: + self._display.display( + "An exception occurred during task execution." + " The full traceback is:\n" + result._result['exception']) + + if result._task.action in C.MODULE_NO_JSON: + self._display.display( + self._command_generic_msg( + result._host.get_name(), result._result, result._task, + "FAILED")) + else: + self._display.display( + "%s | %s | FAILED! => %s" % ( + _get_timestamp(), + result._host.get_name(), self._dump_results( + result._result, indent=4))) + + def v2_runner_on_ok(self, result): + self._clean_results(result._result, result._task.action) + if 'ansible_facts' in result._result: + return + elif 'hostvars[inventory_hostname]' in result._result: + facts = result._result['hostvars[inventory_hostname]'] + facter_keys = [k for k in facts.keys() if k.startswith('facter_')] + for key in facter_keys: + del facts[key] + result._result['ansible_facts'] = facts + self._display.display( + "%s | %s | Gathered facts:\n%s" % ( + _get_timestamp(), + result._host.get_name(), + yaml.safe_dump(facts, default_flow_style=False))) + return + + if result._task.action in C.MODULE_NO_JSON: + self._display.display( + self._command_generic_msg( + result._host.get_name(), result._result, result._task, + "SUCCESS")) + else: + if 'changed' in result._result and result._result['changed']: + self._display.display( + "%s | %s | SUCCESS => %s" % ( + _get_timestamp(), + result._host.get_name(), self._dump_results( + result._result, indent=4))) + else: + abriged_result = strip_internal_keys(result._result) + if 'msg' in abriged_result and len(abriged_result.keys()) == 1: + result_text = result._result['msg'] + else: + result_text = self._dump_results(result._result, indent=4) + + self._display.display( + "%s | %s | %s | %s" % ( + _get_timestamp(), + result._host.get_name(), + result._task.get_name().strip(), + result_text)) + self._handle_warnings(result._result) + + def v2_runner_on_skipped(self, result): + self._display.display( + "%s | %s | SKIPPED" % ( + _get_timestamp(), result._host.get_name())) + + def v2_runner_on_unreachable(self, result): + self._display.display( + "%s | %s | UNREACHABLE! => %s" % ( + _get_timestamp(), + result._host.get_name(), self._dump_results( + result._result, indent=4))) + + def v2_on_file_diff(self, result): + if 'diff' in result._result and result._result['diff']: + self._display.display(self._get_diff(result._result['diff'])) diff --git a/playbooks/roles/gather_host_info/tasks/main.yaml b/playbooks/roles/gather_host_info/tasks/main.yaml index 0f0a0007..bccc6895 100644 --- a/playbooks/roles/gather_host_info/tasks/main.yaml +++ b/playbooks/roles/gather_host_info/tasks/main.yaml @@ -1,7 +1,4 @@ --- -- debug: msg="The running kernel version is {{ ansible_kernel }}" -- debug: msg="This instance has {{ ansible_processor_vcpus }} vcpu(s)" +- debug: var=hostvars[inventory_hostname] - command: locale - register: locale_out -- name: "Locale is:" - debug: var=locale_out.stdout_lines + name: "Gather locale" From c2631249a0f6fbc12ee26a60153ba9643e7fb04b Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 29 Nov 2016 17:17:49 -0600 Subject: [PATCH 100/148] Add note to README about GPL in callback plugin The introduction of the Ansible callback plugin introduced a GPLv3 file into the repo. While it has a license header - go ahead and mention that in the contributing section so that people are aware. Change-Id: Ic6835ad261986235ef58bd672efc193496140863 --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 45bf36a5..8063d078 100644 --- a/README.rst +++ b/README.rst @@ -252,3 +252,7 @@ You can file bugs on the storyboard devstack-gate project:: https://storyboard.openstack.org/#!/project/712 And you can chat with us on Freenode in #openstack-qa or #openstack-infra. + +It's worth noting that, while devstack-gate is generally licensed under the +Apache license, `playbooks/plugins/callback/devstack.py` is GPLv3 due to having +derived from the Ansible source code. From f3b545bd6e6da996040dee063c5cabcc8be8c822 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Wed, 30 Nov 2016 21:21:59 +0000 Subject: [PATCH 101/148] Add devstack_gate_vars.yaml to hold varibles We want to force users that re-use d-g roles to provide their own variables, instead of having per-role defaults. We will set the variables used by our playbooks on devstack_gate_vars.yaml, and feed it to the playbooks via either vars_files or -e on the CLI. For this initial change, add the BASE variable, which will be looked up from envvars and default to /opt/stack if not defined. Change-Id: I9b6b76fbe9866d29327a0c087051e10a80ef55eb --- playbooks/devstack_gate_vars.yaml | 2 ++ playbooks/setup_host.yaml | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 playbooks/devstack_gate_vars.yaml diff --git a/playbooks/devstack_gate_vars.yaml b/playbooks/devstack_gate_vars.yaml new file mode 100644 index 00000000..760581a7 --- /dev/null +++ b/playbooks/devstack_gate_vars.yaml @@ -0,0 +1,2 @@ +--- +BASE: "{{ lookup('env', 'BASE')|default('/opt/stack', true) }}" diff --git a/playbooks/setup_host.yaml b/playbooks/setup_host.yaml index 137d6119..8f9a01a4 100644 --- a/playbooks/setup_host.yaml +++ b/playbooks/setup_host.yaml @@ -1,5 +1,7 @@ --- - hosts: all gather_facts: yes + vars_files: + - devstack_gate_vars.yaml roles: - gather_host_info From 33a326691466ad41693598b9e9705778ee0c1032 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Thu, 24 Nov 2016 11:22:10 +0000 Subject: [PATCH 102/148] Ansibly fix_etc_hosts Replace fix_etc_hosts bash function for fix_etc_hosts role Change-Id: Icfabbfeec649b43328b5251c5b9e481f1f58b002 --- functions.sh | 15 --------------- playbooks/roles/fix_etc_hosts/tasks/main.yaml | 11 +++++++++++ playbooks/setup_host.yaml | 1 + 3 files changed, 12 insertions(+), 15 deletions(-) create mode 100644 playbooks/roles/fix_etc_hosts/tasks/main.yaml diff --git a/functions.sh b/functions.sh index 300ffcb4..d1cb1aa6 100644 --- a/functions.sh +++ b/functions.sh @@ -339,18 +339,6 @@ function git_clone_and_cd { cd $short_project } -function fix_etc_hosts { - # HPcloud stopped adding the hostname to /etc/hosts with their - # precise images. - - HOSTNAME=`/bin/hostname` - if ! grep $HOSTNAME /etc/hosts >/dev/null; then - echo "Need to add hostname to /etc/hosts" - sudo bash -c 'echo "127.0.1.1 $HOSTNAME" >>/etc/hosts' - fi - -} - function fix_disk_layout { # Don't attempt to fix disk layout more than once [[ -e /etc/fixed_disk_layout ]] && return 0 || sudo touch /etc/fixed_disk_layout @@ -596,9 +584,6 @@ function setup_host { local xtrace=$(set +o | grep xtrace) set -o xtrace - # This is necessary to keep sudo from complaining - fix_etc_hosts - # We set some home directories under $BASE, make sure it exists. sudo mkdir -p $BASE diff --git a/playbooks/roles/fix_etc_hosts/tasks/main.yaml b/playbooks/roles/fix_etc_hosts/tasks/main.yaml new file mode 100644 index 00000000..0eb59df1 --- /dev/null +++ b/playbooks/roles/fix_etc_hosts/tasks/main.yaml @@ -0,0 +1,11 @@ +--- +- name: Check whether /etc/hosts contains hostname + command: grep {{ ansible_hostname }} /etc/hosts + changed_when: False + failed_when: False + register: grep_out + +- name: Add hostname to /etc/hosts + lineinfile: dest=/etc/hosts insertafter=EOF line='127.0.1.1 {{ ansible_hostname }}' + become: yes + when: grep_out.rc != 0 diff --git a/playbooks/setup_host.yaml b/playbooks/setup_host.yaml index 8f9a01a4..f4169591 100644 --- a/playbooks/setup_host.yaml +++ b/playbooks/setup_host.yaml @@ -5,3 +5,4 @@ - devstack_gate_vars.yaml roles: - gather_host_info + - fix_etc_hosts From c920f1ad9856ab4320417d240d9f4305adb92c56 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Thu, 24 Nov 2016 15:42:15 +0000 Subject: [PATCH 103/148] Ansibly BASE folder creation Replace BASE folder creation function for create_base_folder role Change-Id: Ia775c1ac110b0df7fd8c47e17e1945eb16a92779 --- functions.sh | 3 --- playbooks/roles/create_base_folder/tasks/main.yaml | 3 +++ playbooks/setup_host.yaml | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 playbooks/roles/create_base_folder/tasks/main.yaml diff --git a/functions.sh b/functions.sh index d1cb1aa6..f5221a1d 100644 --- a/functions.sh +++ b/functions.sh @@ -584,9 +584,6 @@ function setup_host { local xtrace=$(set +o | grep xtrace) set -o xtrace - # We set some home directories under $BASE, make sure it exists. - sudo mkdir -p $BASE - # Start with a fresh syslog if which journalctl ; then # save timestamp and use journalctl to dump everything since diff --git a/playbooks/roles/create_base_folder/tasks/main.yaml b/playbooks/roles/create_base_folder/tasks/main.yaml new file mode 100644 index 00000000..d9dbc068 --- /dev/null +++ b/playbooks/roles/create_base_folder/tasks/main.yaml @@ -0,0 +1,3 @@ +- name: Create BASE folder + file: path={{ BASE }} state=directory + become: yes diff --git a/playbooks/setup_host.yaml b/playbooks/setup_host.yaml index f4169591..d0805be2 100644 --- a/playbooks/setup_host.yaml +++ b/playbooks/setup_host.yaml @@ -6,3 +6,4 @@ roles: - gather_host_info - fix_etc_hosts + - create_base_folder From 161fa62663977692cbc5fb2a93cdbb8b4634b05c Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Thu, 24 Nov 2016 15:43:59 +0000 Subject: [PATCH 104/148] Ansibly the start of fresh logging Replace start fresh logging bash logic for start_fresh_logging role Change-Id: I0177fcce93087b6a7081f6f96ac1fec3fdd9f53f --- functions.sh | 21 ------- .../roles/start_fresh_logging/tasks/main.yaml | 57 +++++++++++++++++++ playbooks/setup_host.yaml | 1 + 3 files changed, 58 insertions(+), 21 deletions(-) create mode 100644 playbooks/roles/start_fresh_logging/tasks/main.yaml diff --git a/functions.sh b/functions.sh index f5221a1d..aa1f8c3f 100644 --- a/functions.sh +++ b/functions.sh @@ -584,27 +584,6 @@ function setup_host { local xtrace=$(set +o | grep xtrace) set -o xtrace - # Start with a fresh syslog - if which journalctl ; then - # save timestamp and use journalctl to dump everything since - # then at the end - date +"%Y-%m-%d %H:%M:%S" | sudo tee $BASE/log-start-timestamp.txt - else - # Assume rsyslog, move old logs aside then restart the service. - sudo stop rsyslog - sudo mv /var/log/syslog /var/log/syslog-pre-devstack - sudo mv /var/log/kern.log /var/log/kern_log-pre-devstack - sudo touch /var/log/syslog - sudo chown /var/log/syslog --ref /var/log/syslog-pre-devstack - sudo chmod /var/log/syslog --ref /var/log/syslog-pre-devstack - sudo chmod a+r /var/log/syslog - sudo touch /var/log/kern.log - sudo chown /var/log/kern.log --ref /var/log/kern_log-pre-devstack - sudo chmod /var/log/kern.log --ref /var/log/kern_log-pre-devstack - sudo chmod a+r /var/log/kern.log - sudo start rsyslog - fi - # Create a stack user for devstack to run as, so that we can # revoke sudo permissions from that user when appropriate. sudo useradd -U -s /bin/bash -d $BASE/new -m stack diff --git a/playbooks/roles/start_fresh_logging/tasks/main.yaml b/playbooks/roles/start_fresh_logging/tasks/main.yaml new file mode 100644 index 00000000..500537c1 --- /dev/null +++ b/playbooks/roles/start_fresh_logging/tasks/main.yaml @@ -0,0 +1,57 @@ +--- +- name: Check for /bin/journalctl file + command: which journalctl + changed_when: False + failed_when: False + register: which_out + +- block: + - name: Get current date + command: date +"%Y-%m-%d %H:%M:%S" + register: date_out + + - name: Copy current date to log-start-timestamp.txt + copy: + dest: "{{ BASE }}/log-start-timestamp.txt" + content: "{{ date_out.stdout }}" + when: which_out.rc == 0 + become: yes + +- block: + - name: Stop rsyslog + service: name=rsyslog state=stopped + + - name: Save syslog file prior to devstack run + command: mv /var/log/syslog /var/log/syslog-pre-devstack + + - name: Save kern.log file prior to devstack run + command: mv /var/log/kern.log /var/log/kern_log-pre-devstack + + - name: Recreate syslog file + file: name=/var/log/syslog state=touch + + - name: Recreate syslog file owner and group + command: chown /var/log/syslog --ref /var/log/syslog-pre-devstack + + - name: Recreate syslog file permissions + command: chmod /var/log/syslog --ref /var/log/syslog-pre-devstack + + - name: Add read permissions to all on syslog file + file: name=/var/log/syslog mode=a+r + + - name: Recreate kern.log file + file: name=/var/log/kern.log state=touch + + - name: Recreate kern.log file owner and group + command: chown /var/log/kern.log --ref /var/log/kern_log-pre-devstack + + - name: Recreate kern.log file permissions + command: chmod /var/log/kern.log --ref /var/log/kern_log-pre-devstack + + - name: Add read permissions to all on kern.log file + file: name=/var/log/kern.log mode=a+r + + - name: Start rsyslog + service: name=rsyslog state=started + when: which_out.rc == 1 + become: yes diff --git a/playbooks/setup_host.yaml b/playbooks/setup_host.yaml index d0805be2..51a0e69c 100644 --- a/playbooks/setup_host.yaml +++ b/playbooks/setup_host.yaml @@ -7,3 +7,4 @@ - gather_host_info - fix_etc_hosts - create_base_folder + - start_fresh_logging From 5241ba6ebb2bbfbf9998cb85e0b9294a705a8d11 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Mon, 28 Nov 2016 12:25:06 +0000 Subject: [PATCH 105/148] Ansibly the setup of stack user Replace the setup of stack user bash logic for setup_stack_user role Change-Id: Ibec64b8ac97324282c710242c2e3e89e31b75fb0 --- functions.sh | 16 --------------- .../roles/setup_stack_user/files/50_stack_sh | 1 + .../roles/setup_stack_user/tasks/main.yaml | 20 +++++++++++++++++++ playbooks/setup_host.yaml | 1 + 4 files changed, 22 insertions(+), 16 deletions(-) create mode 100644 playbooks/roles/setup_stack_user/files/50_stack_sh create mode 100644 playbooks/roles/setup_stack_user/tasks/main.yaml diff --git a/functions.sh b/functions.sh index aa1f8c3f..295ead23 100644 --- a/functions.sh +++ b/functions.sh @@ -584,22 +584,6 @@ function setup_host { local xtrace=$(set +o | grep xtrace) set -o xtrace - # Create a stack user for devstack to run as, so that we can - # revoke sudo permissions from that user when appropriate. - sudo useradd -U -s /bin/bash -d $BASE/new -m stack - # Use 755 mode on the user dir regardless of the /etc/login.defs setting - sudo chmod 755 $BASE/new - TEMPFILE=`mktemp` - echo "stack ALL=(root) NOPASSWD:ALL" >$TEMPFILE - chmod 0440 $TEMPFILE - sudo chown root:root $TEMPFILE - sudo mv $TEMPFILE /etc/sudoers.d/50_stack_sh - - # Create user's ~/.cache directory with proper permissions, ensuring later - # 'sudo pip install's do not create it owned by root. - sudo mkdir -p $BASE/new/.cache - sudo chown -R stack:stack $BASE/new/.cache - # Create a tempest user for tempest to run as, so that we can # revoke sudo permissions from that user when appropriate. # NOTE(sdague): we should try to get the state dump to be a diff --git a/playbooks/roles/setup_stack_user/files/50_stack_sh b/playbooks/roles/setup_stack_user/files/50_stack_sh new file mode 100644 index 00000000..4c6b46bd --- /dev/null +++ b/playbooks/roles/setup_stack_user/files/50_stack_sh @@ -0,0 +1 @@ +stack ALL=(root) NOPASSWD:ALL diff --git a/playbooks/roles/setup_stack_user/tasks/main.yaml b/playbooks/roles/setup_stack_user/tasks/main.yaml new file mode 100644 index 00000000..6748d5a7 --- /dev/null +++ b/playbooks/roles/setup_stack_user/tasks/main.yaml @@ -0,0 +1,20 @@ +--- +- name: Create stack group + group: name=stack state=present + become: yes + +- name: Create stack user + user: name=stack shell=/bin/bash home={{ BASE }}/new group=stack + become: yes + +- name: Set home folder permissions + file: path={{ BASE }}/new mode=0755 + become: yes + +- name: Copy 50_stack_sh file to /etc/sudoers.d + copy: src=50_stack_sh dest=/etc/sudoers.d mode=0440 owner=root group=root + become: yes + +- name: Create new/.cache folder within BASE + file: path={{ BASE }}/new/.cache state=directory owner=stack group=stack + become: yes diff --git a/playbooks/setup_host.yaml b/playbooks/setup_host.yaml index 51a0e69c..78f3353c 100644 --- a/playbooks/setup_host.yaml +++ b/playbooks/setup_host.yaml @@ -8,3 +8,4 @@ - fix_etc_hosts - create_base_folder - start_fresh_logging + - setup_stack_user From 7a3c30ded5d080c4c7ba6b96ebf95a0c6aaa384c Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Mon, 28 Nov 2016 14:16:01 +0000 Subject: [PATCH 106/148] Ansibly the setup of the tempest user Replace setup of tempest user bash logic for setup_tempest_user role Change-Id: I0e3f3ff9e3386e078cddc05b9b0d7887d66428f0 --- functions.sh | 17 ----------------- .../setup_tempest_user/files/51_tempest_sh | 3 +++ .../roles/setup_tempest_user/tasks/main.yaml | 12 ++++++++++++ playbooks/setup_host.yaml | 1 + 4 files changed, 16 insertions(+), 17 deletions(-) create mode 100644 playbooks/roles/setup_tempest_user/files/51_tempest_sh create mode 100644 playbooks/roles/setup_tempest_user/tasks/main.yaml diff --git a/functions.sh b/functions.sh index 295ead23..111dbac7 100644 --- a/functions.sh +++ b/functions.sh @@ -584,23 +584,6 @@ function setup_host { local xtrace=$(set +o | grep xtrace) set -o xtrace - # Create a tempest user for tempest to run as, so that we can - # revoke sudo permissions from that user when appropriate. - # NOTE(sdague): we should try to get the state dump to be a - # neutron API call in Icehouse to remove this. - sudo useradd -U -s /bin/bash -m tempest - TEMPFILE=`mktemp` - echo "tempest ALL=(root) NOPASSWD:/sbin/ip" >$TEMPFILE - echo "tempest ALL=(root) NOPASSWD:/sbin/iptables" >>$TEMPFILE - echo "tempest ALL=(root) NOPASSWD:/usr/bin/ovsdb-client" >>$TEMPFILE - chmod 0440 $TEMPFILE - sudo chown root:root $TEMPFILE - sudo mv $TEMPFILE /etc/sudoers.d/51_tempest_sh - - # Future useradd calls should strongly consider also updating - # ~/.pydisutils.cfg in the copy_mirror_config - # function if tox/pip will be used at all. - # If we will be testing OpenVZ, make sure stack is a member of the vz group if [ "$DEVSTACK_GATE_VIRT_DRIVER" == "openvz" ]; then sudo usermod -a -G vz stack diff --git a/playbooks/roles/setup_tempest_user/files/51_tempest_sh b/playbooks/roles/setup_tempest_user/files/51_tempest_sh new file mode 100644 index 00000000..f88ff9f4 --- /dev/null +++ b/playbooks/roles/setup_tempest_user/files/51_tempest_sh @@ -0,0 +1,3 @@ +tempest ALL=(root) NOPASSWD:/sbin/ip +tempest ALL=(root) NOPASSWD:/sbin/iptables +tempest ALL=(root) NOPASSWD:/usr/bin/ovsdb-client diff --git a/playbooks/roles/setup_tempest_user/tasks/main.yaml b/playbooks/roles/setup_tempest_user/tasks/main.yaml new file mode 100644 index 00000000..b6af7ed2 --- /dev/null +++ b/playbooks/roles/setup_tempest_user/tasks/main.yaml @@ -0,0 +1,12 @@ +--- +- name: Create tempest group + group: name=tempest state=present + become: yes + +- name: Create tempest user + user: name=tempest shell=/bin/bash group=tempest + become: yes + +- name: Copy 51_tempest_sh to /etc/sudoers.d + copy: src=51_tempest_sh dest=/etc/sudoers.d owner=root group=root mode=0440 + become: yes diff --git a/playbooks/setup_host.yaml b/playbooks/setup_host.yaml index 78f3353c..36a9e08b 100644 --- a/playbooks/setup_host.yaml +++ b/playbooks/setup_host.yaml @@ -9,3 +9,4 @@ - create_base_folder - start_fresh_logging - setup_stack_user + - setup_tempest_user From 47ef35d1670c613ec72630c98ad80ea30c772038 Mon Sep 17 00:00:00 2001 From: David Moreau-Simard Date: Thu, 1 Dec 2016 15:11:42 -0500 Subject: [PATCH 107/148] Add ARA callback and reporting This adds the ARA [1] callback in order to record the playbook executions and generate a report that will be retrieved as part of the logs at the end of the jobs. [1]: https://github.com/openstack/ara Change-Id: I101f74223b5277b34825ba0e46b3365a544c5435 --- devstack-vm-gate-wrap.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 3b099d16..bbc4ea2c 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -469,7 +469,7 @@ sudo -H pip install virtualenv virtualenv /tmp/ansible # NOTE(emilien): workaround to avoid installing cryptography # https://github.com/ansible/ansible/issues/15665 -/tmp/ansible/bin/pip install paramiko==1.16.0 ansible==$ANSIBLE_VERSION +/tmp/ansible/bin/pip install paramiko==1.16.0 ansible==$ANSIBLE_VERSION ara export ANSIBLE=/tmp/ansible/bin/ansible export ANSIBLE_PLAYBOOK=/tmp/ansible/bin/ansible-playbook @@ -487,7 +487,7 @@ done # Write ansible config file cat > "$WORKSPACE/ansible.cfg" < Date: Wed, 12 Oct 2016 13:21:31 -0400 Subject: [PATCH 108/148] Introduce roles into the feature matrix test-matrix.py has a new cli option, for which role the node will play. If it is a subnode, the starting set of services will be emitted, that matches the current logic. Logic regarding flags like DEVSTACK_GATE_NEUTRON, DEVSTACK_GATE_NEUTRON_DVR, DEVSTACK_GATE_IRONIC is now moved to the feature matrix. Co-Authored-By: Julia Varlamova Co-Authored-By: Michal Dulko Change-Id: Ief1b74cbc01235d9f52d08add930781f3a0ae71b --- devstack-vm-gate.sh | 30 +++--------------- features.yaml | 75 ++++++++++++++++++++++++++++++++++++++++++++- test-matrix.py | 23 +++++++++----- 3 files changed, 94 insertions(+), 34 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index d64278dc..a8726ccd 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -255,35 +255,15 @@ function setup_localrc { sudo yum install -y PyYAML fi fi - MY_ENABLED_SERVICES=`cd $BASE/new/devstack-gate && ./test-matrix.py -b $branch_for_matrix -f $DEVSTACK_GATE_FEATURE_MATRIX` - local original_enabled_services=$MY_ENABLED_SERVICES - # TODO(afazekas): Move to the feature grid + local test_matrix_role='primary' if [[ $role = sub ]]; then - MY_ENABLED_SERVICES="n-cpu,ceilometer-acompute,dstat" - if [[ "$original_enabled_services" =~ "c-api" ]]; then - MY_ENABLED_SERVICES+=",c-vol,c-bak" - fi - if [[ "$original_enabled_services" =~ "tls-proxy" ]]; then - MY_ENABLED_SERVICES+=",tls-proxy" - fi - if [[ "$DEVSTACK_GATE_NEUTRON" -eq "1" ]]; then - MY_ENABLED_SERVICES+=",q-agt" - if [[ "$DEVSTACK_GATE_NEUTRON_DVR" -eq "1" ]]; then - # As per reference architecture described in - # https://wiki.openstack.org/wiki/Neutron/DVR - # for DVR multi-node, add the following services - # on all compute nodes: - MY_ENABLED_SERVICES+=",q-l3,q-meta" - fi - else - MY_ENABLED_SERVICES+=",n-net,n-api-meta" - fi - if [[ "$DEVSTACK_GATE_IRONIC" -eq "1" ]]; then - MY_ENABLED_SERVICES+=",ir-api,ir-cond" - fi + test_matrix_role='subnode' fi + MY_ENABLED_SERVICES=$(cd $BASE/new/devstack-gate && ./test-matrix.py -b $branch_for_matrix -f $DEVSTACK_GATE_FEATURE_MATRIX -r $test_matrix_role) + local original_enabled_services=$(cd $BASE/new/devstack-gate && ./test-matrix.py -b $branch_for_matrix -f $DEVSTACK_GATE_FEATURE_MATRIX -r primary) + # Allow optional injection of ENABLED_SERVICES from the calling context if [[ ! -z $ENABLED_SERVICES ]] ; then MY_ENABLED_SERVICES+=,$ENABLED_SERVICES diff --git a/features.yaml b/features.yaml index cd2a96d4..467cf140 100644 --- a/features.yaml +++ b/features.yaml @@ -50,6 +50,8 @@ config: features: [tlsproxy] cinder_mn_grenade: features: [cinder-mn-grenade] + neutron_dvr: + features: [neutron-dvr] branches: # The value of ""default" is the name of the "trunk" branch @@ -57,7 +59,7 @@ branches: # Normalized branch names only here, e.g. stable/icehouse => icehouse allowed: [master, ocata, newton, mitaka, liberty, kilo, juno, icehouse] -features: +primary: default: base: services: [mysql, rabbit, dstat] @@ -181,3 +183,74 @@ features: rm-services: [tls-proxy] newton: rm-services: [tls-proxy] + +subnode: + default: + base: + services: [dstat] + + ceilometer: + base: + services: [ceilometer-acompute] + + cinder: + base: + services: [c-vol, c-bak] + + cinder-mn-grenade: + base: + services: [] + + glance: + base: + services: [] + + horizon: + base: + services: [] + + ironic: + base: + rm-services: [c-vol, c-bak] + services: [ir-api, ir-cond] + + keystone: + base: + services: [] + + neutron: + base: + rm-services: [n-net, n-api-meta] + services: [q-agt] + + neutron-adv: + base: + services: [] + + neutron-dvr: + base: + rm-services: [n-net, n-api-meta] + services: [q-agt, q-l3, q-meta] + + nova: + base: + services: [n-cpu, n-net, n-api-meta] + + swift: + base: + services: [] + + tempest: + base: + services: [] + + tlsproxy: + base: + services: [tls-proxy] + # TLS proxy didn't work properly until ocata + liberty: + rm-services: [tls-proxy] + mitaka: + rm-services: [tls-proxy] + newton: + rm-services: [tls-proxy] diff --git a/test-matrix.py b/test-matrix.py index 17170f2f..2210db38 100755 --- a/test-matrix.py +++ b/test-matrix.py @@ -73,22 +73,24 @@ def configs_from_env(): return configs -def calc_services(branch, features): +def calc_services(branch, features, role): services = set() for feature in features: - services.update(GRID['features'][feature]['base'].get('services', [])) - if branch in GRID['features'][feature]: + grid_feature = GRID[role][feature] + services.update(grid_feature['base'].get('services', [])) + if branch in grid_feature: services.update( - GRID['features'][feature][branch].get('services', [])) + grid_feature[branch].get('services', [])) # deletes always trump adds for feature in features: + grid_feature = GRID[role][feature] services.difference_update( - GRID['features'][feature]['base'].get('rm-services', [])) + grid_feature['base'].get('rm-services', [])) - if branch in GRID['features'][feature]: + if branch in grid_feature: services.difference_update( - GRID['features'][feature][branch].get('rm-services', [])) + grid_feature[branch].get('rm-services', [])) return sorted(list(services)) @@ -127,6 +129,10 @@ def get_opts(): parser.add_argument('-m', '--mode', default="services", help="What to return (services, compute-ext)") + parser.add_argument('-r', '--role', + default='primary', + help="What role this node will have", + choices=['primary', 'subnode']) return parser.parse_args() @@ -137,11 +143,12 @@ def main(): GRID = parse_features(opts.features) ALLOWED_BRANCHES = GRID['branches']['allowed'] branch = normalize_branch(opts.branch) + role = opts.role features = calc_features(branch, configs_from_env()) LOG.debug("Features: %s " % features) - services = calc_services(branch, features) + services = calc_services(branch, features, role) LOG.debug("Services: %s " % services) if opts.mode == "services": From c1b64743b8404dad93d4c06bcb26cf87cbc5240c Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Mon, 12 Dec 2016 12:24:51 -0500 Subject: [PATCH 109/148] enable placement api by default for ocata and beyond Make placement API the default in ocata and later. Depends-On: I04a655fbc58913b3d607400a7f677be299499142 Change-Id: Ibd760c642e3c1ffff2dd61be48e30530b0d24720 --- features.yaml | 12 ++++++++++-- test-features.sh | 8 ++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/features.yaml b/features.yaml index 467cf140..c7b53c94 100644 --- a/features.yaml +++ b/features.yaml @@ -1,7 +1,7 @@ config: default: - master: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] - ocata: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] + master: [default, ceilometer, glance, horizon, nova, placement, swift, cinder, keystone] + ocata: [default, ceilometer, glance, horizon, nova, placement, swift, cinder, keystone] newton: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] mitaka: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] liberty: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] @@ -95,6 +95,10 @@ primary: services: [n-cell] rm-compute-ext: [agregates, hosts] + placement: + base: + services: [placement-api] + neutron: base: services: [quantum, q-svc, q-agt, q-dhcp, q-l3, q-meta, q-metering] @@ -236,6 +240,10 @@ subnode: base: services: [n-cpu, n-net, n-api-meta] + placement: + base: + services: [placement-client] + swift: base: services: [] diff --git a/test-features.sh b/test-features.sh index 69d96cfe..cd45c7a8 100755 --- a/test-features.sh +++ b/test-features.sh @@ -16,13 +16,13 @@ ERRORS=0 -TEMPEST_FULL_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,n-net" +TEMPEST_FULL_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,n-net,placement-api" -TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering" +TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,placement-api" -TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering" +TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,placement-api" -GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification" +GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,placement-api" # Utility function for tests function assert_list_equal { From f96ca3c980a4fdcbee3dcc98b0b9fb7f792d7f69 Mon Sep 17 00:00:00 2001 From: Sagi Shnaidman Date: Sat, 17 Dec 2016 15:15:27 +0200 Subject: [PATCH 110/148] Propagate remaining time to job We want to know within a job how much time remained before zuul finishes the job. For local preparements and end job tasks. Change-Id: Ib4c7df73efaca42db30c56a8d1eb3b790b86cb99 --- functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index 111dbac7..d1357dff 100644 --- a/functions.sh +++ b/functions.sh @@ -165,7 +165,7 @@ function start_timer { function remaining_time { local now=`date +%s` local elapsed=$(((now - START_TIME) / 60)) - REMAINING_TIME=$((DEVSTACK_GATE_TIMEOUT - elapsed - 5)) + export REMAINING_TIME=$((DEVSTACK_GATE_TIMEOUT - elapsed - 5)) echo "Job timeout set to: $REMAINING_TIME minutes" if [ ${REMAINING_TIME} -le 0 ]; then echo "Already timed out." From 86c144412183d3a9b352f3c5dd6ed6a21012b542 Mon Sep 17 00:00:00 2001 From: "John L. Villalovos" Date: Sat, 17 Dec 2016 19:03:25 -0800 Subject: [PATCH 111/148] Fix DVR gate job breakage The Neutron DVR gate jobs are broken due to commit 7fac6b68380f3f0ce44eefd257e571844a8b741d. Add an empty 'neutron-dvr' entry in the 'primary' section of features.yaml Also add logging on what the features have been set to. Change-Id: I7f57de5d43f6943863b1df3e2c367587b519fc05 --- devstack-vm-gate.sh | 2 ++ features.yaml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index a8726ccd..0c28b2a1 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -263,6 +263,8 @@ function setup_localrc { MY_ENABLED_SERVICES=$(cd $BASE/new/devstack-gate && ./test-matrix.py -b $branch_for_matrix -f $DEVSTACK_GATE_FEATURE_MATRIX -r $test_matrix_role) local original_enabled_services=$(cd $BASE/new/devstack-gate && ./test-matrix.py -b $branch_for_matrix -f $DEVSTACK_GATE_FEATURE_MATRIX -r primary) + echo "MY_ENABLED_SERVICES: ${MY_ENABLED_SERVICES}" + echo "original_enabled_services: ${original_enabled_services}" # Allow optional injection of ENABLED_SERVICES from the calling context if [[ ! -z $ENABLED_SERVICES ]] ; then diff --git a/features.yaml b/features.yaml index 467cf140..367617eb 100644 --- a/features.yaml +++ b/features.yaml @@ -110,6 +110,10 @@ primary: kilo: services: [q-vpn] + neutron-dvr: + base: + services: [] + swift: base: services: [s-proxy, s-account, s-container, s-object] From ab356c10fb5d2699927f83529919370a8d24d544 Mon Sep 17 00:00:00 2001 From: Jeremy Stanley Date: Wed, 14 Dec 2016 21:07:23 +0000 Subject: [PATCH 112/148] Only check connectivity to mirrors when defined If nodepool configuration is not present or does not define a mirror host, then avoid running network checks against it. Not all uses of devstack-gate happen within nodepool-managed test systems. Change-Id: I1932a72c3234b466c9e1c0ab5a09e93cda40a21e --- functions.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/functions.sh b/functions.sh index f5221a1d..cf61ed54 100644 --- a/functions.sh +++ b/functions.sh @@ -130,13 +130,17 @@ function network_sanity_check { _http_check $pypi_url fi - # AFS ubuntu mirror - source /etc/nodepool/provider - NODEPOOL_MIRROR_HOST=${NODEPOOL_MIRROR_HOST:-mirror.$NODEPOOL_REGION.$NODEPOOL_CLOUD.openstack.org} - NODEPOOL_MIRROR_HOST=$(echo $NODEPOOL_MIRROR_HOST|tr '[:upper:]' '[:lower:]') - - _ping_check $NODEPOOL_MIRROR_HOST - _http_check http://$NODEPOOL_MIRROR_HOST/ubuntu/dists/trusty/Release + if [[ -f /etc/nodepool/provider ]]; then + # AFS ubuntu mirror + source /etc/nodepool/provider + if [[ -n "$NODEPOOL_MIRROR_HOST" || ( -n "$NODEPOOL_REGION" && -n "$NODEPOOL_CLOUD" ) ]]; then + NODEPOOL_MIRROR_HOST=${NODEPOOL_MIRROR_HOST:-mirror.$NODEPOOL_REGION.$NODEPOOL_CLOUD.openstack.org} + NODEPOOL_MIRROR_HOST=$(echo $NODEPOOL_MIRROR_HOST|tr '[:upper:]' '[:lower:]') + + _ping_check $NODEPOOL_MIRROR_HOST + _http_check http://$NODEPOOL_MIRROR_HOST/ubuntu/dists/trusty/Release + fi + fi } # create the start timer for when the job began From a540ed08e1d4085f29c0e8b999fce9ce1a8770e1 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Mon, 28 Nov 2016 19:08:37 +0000 Subject: [PATCH 113/148] Remove unused openvz code from d-g This logic should not be in d-g, but in devstack itself. Change-Id: I3ff325fab3182c9a6d1499c9620b96d4feb1a57a --- devstack-vm-gate.sh | 8 -------- functions.sh | 5 ----- 2 files changed, 13 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 0c28b2a1..146d26e0 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -369,14 +369,6 @@ EOF fi fi - if [[ "$DEVSTACK_GATE_VIRT_DRIVER" == "openvz" ]]; then - echo "SKIP_EXERCISES=${SKIP_EXERCISES},volumes" >>"$localrc_file" - echo "DEFAULT_INSTANCE_TYPE=m1.small" >>"$localrc_file" - echo "DEFAULT_INSTANCE_USER=root" >>"$localrc_file" - echo "DEFAULT_INSTANCE_TYPE=m1.small" >>exerciserc - echo "DEFAULT_INSTANCE_USER=root" >>exerciserc - fi - if [[ "$DEVSTACK_GATE_VIRT_DRIVER" == "ironic" ]]; then export TEMPEST_OS_TEST_TIMEOUT=${DEVSTACK_GATE_OS_TEST_TIMEOUT:-1200} echo "IRONIC_DEPLOY_DRIVER=$DEVSTACK_GATE_IRONIC_DRIVER" >>"$localrc_file" diff --git a/functions.sh b/functions.sh index d1357dff..1456491c 100644 --- a/functions.sh +++ b/functions.sh @@ -584,11 +584,6 @@ function setup_host { local xtrace=$(set +o | grep xtrace) set -o xtrace - # If we will be testing OpenVZ, make sure stack is a member of the vz group - if [ "$DEVSTACK_GATE_VIRT_DRIVER" == "openvz" ]; then - sudo usermod -a -G vz stack - fi - # Ensure that all of the users have the openstack mirror config copy_mirror_config From 8d680c44de55cd00a01968ecd453b8ac57da7ab4 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Thu, 22 Dec 2016 10:31:09 -0500 Subject: [PATCH 114/148] pass-through USE_PYTHON3 to devstack If USE_PYTHON3 is specified in the CI job definitions, then we should pass that through to devstack. Change-Id: Id48e1b328230fcdf97ed1cb4b97f4c3f9cf6eb8a --- devstack-vm-gate.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 146d26e0..f754fb43 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -272,6 +272,10 @@ function setup_localrc { fi fi + if [[ ! -z $USE_PYTHON3 ]] ; then + echo "USE_PYTHON3=$USE_PYTHON3" >>"$localrc_file" + fi + if [[ "$DEVSTACK_GATE_CEPH" == "1" ]]; then echo "CINDER_ENABLED_BACKENDS=ceph:ceph" >>"$localrc_file" echo "TEMPEST_STORAGE_PROTOCOL=ceph" >>"$localrc_file" From 6d8b7d9edf64c527735b8175eaa5ba1bc1095bcc Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Thu, 5 Jan 2017 08:49:32 -0500 Subject: [PATCH 115/148] Drop pre-seeding results from old runs Per @mtreinish, "Preloading the scheduler never seemed to make a noticeable difference" Also this interferes with python3 experiments. So let's just yank it out. Change-Id: I0df5c95017b5bbcd611acd7ef499c58686c2868d --- devstack-vm-gate.sh | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index f754fb43..7b9e7170 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -729,28 +729,7 @@ if [[ "$DEVSTACK_GATE_EXERCISES" -eq "1" ]]; then -a "cd '$BASE/new/devstack' && sudo -H -u stack ./exercise.sh" fi -function load_subunit_stream { - local stream=$1; - pushd $BASE/new/tempest/ - sudo testr load --force-init $stream - popd -} - - if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then - #TODO(mtreinish): This if block can be removed after all the nodepool images - # are built using with streams dir instead - echo "Loading previous tempest runs subunit streams into testr" - if [[ -f /opt/git/openstack/tempest/.testrepository/0 ]]; then - temp_stream=`mktemp` - subunit-1to2 /opt/git/openstack/tempest/.testrepository/0 > $temp_stream - load_subunit_stream $temp_stream - elif [[ -d /opt/git/openstack/tempest/preseed-streams ]]; then - for stream in /opt/git/openstack/tempest/preseed-streams/* ; do - load_subunit_stream $stream - done - fi - # under tempest isolation tempest will need to write .tox dir, log files if [[ -d "$BASE/new/tempest" ]]; then sudo chown -R tempest:stack $BASE/new/tempest From 8740b6075b53e3c9bfda76d022fcc53904594e9c Mon Sep 17 00:00:00 2001 From: "John L. Villalovos" Date: Thu, 5 Jan 2017 23:15:13 -0800 Subject: [PATCH 116/148] Change USE_PYTHON3 to DEVSTACK_GATE_USE_PYTHON3 Change USE_PYTHON3 to DEVSTACK_GATE_USE_PYTHON3 so that reproduce.sh will work correctly. Support usage of USE_PYTHON3 for backwards compatibility. But leave comment that it should be removed once all jobs have been updated. Change-Id: Ie42a2652da4dcf21103f85d60a4a041b0d436dbd --- devstack-vm-gate-wrap.sh | 5 +++++ devstack-vm-gate.sh | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index bbc4ea2c..6dc75c8f 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -424,6 +424,11 @@ export DEVSTACK_GATE_TOPOLOGY=${DEVSTACK_GATE_TOPOLOGY:-aio} # for jobs that know exactly which repos they need. export DEVSTACK_GATE_PROJECTS_OVERRIDE=${DEVSTACK_GATE_PROJECTS_OVERRIDE:-""} +# By default we don't use Python 3 +# NOTE(jlvillal) 6-Jan-2017: Support USE_PYTHON3 for backwards compatibility, +# but should remove once all jobs have been fixed. +export DEVSTACK_GATE_USE_PYTHON3=${DEVSTACK_GATE_USE_PYTHON3:-${USE_PYTHON3:-False}} + # Set this to enable remote logging of the console via UDP packets to # a specified ipv4 ip:port (note; not hostname -- ip address only). # This can be extremely useful if a host is oopsing or dropping off diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 7b9e7170..fe107829 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -272,8 +272,8 @@ function setup_localrc { fi fi - if [[ ! -z $USE_PYTHON3 ]] ; then - echo "USE_PYTHON3=$USE_PYTHON3" >>"$localrc_file" + if [[ ! -z $DEVSTACK_GATE_USE_PYTHON3 ]] ; then + echo "USE_PYTHON3=$DEVSTACK_GATE_USE_PYTHON3" >>"$localrc_file" fi if [[ "$DEVSTACK_GATE_CEPH" == "1" ]]; then From 20d9f850258dc64ba13b9d9441d65d4b089678a1 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Fri, 6 Jan 2017 16:20:13 -0800 Subject: [PATCH 117/148] Add tox target for python3 test env As devstack moves towards testing with python3 it would be potentially useful to run devstack-gate without depending on python2. There are likely other items that need porting/testing, but this is a start in that process and makes sure we can test the test-matrix.py script with python2 and 3. Change-Id: Ida4906bfebc839bb7a0df11f53966a6b8c45689f --- tox.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tox.ini b/tox.ini index eab1bb0d..4e61e445 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,12 @@ deps = -r{toxinidir}/test-requirements.txt commands = bash -c "./run-tests.sh" +[testenv:py3-run-tests] +basepython = python3 +deps = -r{toxinidir}/test-requirements.txt +commands = + bash -c "./run-tests.sh" + [testenv:bashate] deps= {env:BASHATE_INSTALL_PATH:bashate==0.5.0} From a3255a7253abf56ed8ea71284037b77d167f33ef Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Sun, 8 Jan 2017 20:59:56 -0500 Subject: [PATCH 118/148] Remove USE_PYTHON3 usage Once we cleanup the usages, we can cleanup the backwards compat env var. Also, Let's avoid forcing it to False and let projects specify this if needed in their pre_test_hook or let the devstack choose the default. Depends-On: Ie42a2652da4dcf21103f85d60a4a041b0d436dbd Change-Id: I79cd956165eb8fc66b2c07f987e1c60deac8b4f7 --- devstack-vm-gate-wrap.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 6dc75c8f..c37f2c71 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -424,10 +424,10 @@ export DEVSTACK_GATE_TOPOLOGY=${DEVSTACK_GATE_TOPOLOGY:-aio} # for jobs that know exactly which repos they need. export DEVSTACK_GATE_PROJECTS_OVERRIDE=${DEVSTACK_GATE_PROJECTS_OVERRIDE:-""} -# By default we don't use Python 3 -# NOTE(jlvillal) 6-Jan-2017: Support USE_PYTHON3 for backwards compatibility, -# but should remove once all jobs have been fixed. -export DEVSTACK_GATE_USE_PYTHON3=${DEVSTACK_GATE_USE_PYTHON3:-${USE_PYTHON3:-False}} +# Set this to "True" to force devstack to pick python 3.x. "False" will cause +# devstack to pick python 2.x. We should leave this empty for devstack to +# pick the default. +export DEVSTACK_GATE_USE_PYTHON3=${DEVSTACK_GATE_USE_PYTHON3:-""} # Set this to enable remote logging of the console via UDP packets to # a specified ipv4 ip:port (note; not hostname -- ip address only). From 35e6c8a998a4e37bd29e494c9bd20582c0985577 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Mon, 27 Jun 2016 10:13:10 -0400 Subject: [PATCH 119/148] Add save_* functions for clearer saving of artifacts The cleanup function has a log of if [[ -r ... ]] logic around files to be able to operate in a non fatal way. This creates a couple of functions that make saving a file or directory more straight forward. It migrates a number of existing calls to this model. Co-Authored-By: YAMAMOTO Takashi Change-Id: I9ad412fc8a4ba7b046f2fa95226ca0d5fc796b23 --- functions.sh | 136 ++++++++++++++++++++++++--------------------------- 1 file changed, 64 insertions(+), 72 deletions(-) diff --git a/functions.sh b/functions.sh index fb5d2ae0..1d5106cc 100644 --- a/functions.sh +++ b/functions.sh @@ -677,6 +677,30 @@ function process_stackviz { fi } +function save_file { + local from=$1 + local to=$2 + if [[ -z "$to" ]]; then + to=$(basename $from) + if [[ "$to" != *.txt ]]; then + to=${to/\./_} + to="$to.txt" + fi + fi + if [[ -f $from ]]; then + sudo cp $from $BASE/logs/$to + fi +} + +function save_dir { + local from=$1 + local to=$2 + if [[ -d $from ]]; then + sudo cp -r $from $BASE/logs/$to + fi +} + + function cleanup_host { # TODO: clean this up to be errexit clean local errexit=$(set +o | grep errexit) @@ -700,8 +724,8 @@ function cleanup_host { | sudo tee $BASE/logs/syslog.txt > /dev/null else # assume rsyslog - sudo cp /var/log/syslog $BASE/logs/syslog.txt - sudo cp /var/log/kern.log $BASE/logs/kern_log.txt + save_file /var/log/syslog + save_file /var/log/kern.log fi # apache logs; including wsgi stuff like horizon, keystone, etc. @@ -713,9 +737,7 @@ function cleanup_host { sudo cp -r ${apache_logs} $BASE/logs/apache # rabbitmq logs - if [ -d /var/log/rabbitmq ]; then - sudo cp -r /var/log/rabbitmq $BASE/logs - fi + save_dir /var/log/rabbitmq # db logs if [ -d /var/log/postgresql ] ; then @@ -723,22 +745,15 @@ function cleanup_host { # deleted sudo cp /var/log/postgresql/*log $BASE/logs/postgres.log fi - if [ -f /var/log/mysql.err ] ; then - sudo cp /var/log/mysql.err $BASE/logs/mysql_err.log - fi - if [ -f /var/log/mysql.log ] ; then - sudo cp /var/log/mysql.log $BASE/logs/ - fi + save_file /var/log/mysql.err + save_file /var/log/mysql.log # libvirt - if [ -d /var/log/libvirt ] ; then - sudo cp -r /var/log/libvirt $BASE/logs/ - sudo cp -r /usr/share/libvirt/cpu_map.xml $BASE/logs/libvirt/cpu_map.xml - fi + save_dir /var/log/libvirt # sudo config - sudo cp -r /etc/sudoers.d $BASE/logs/ - sudo cp /etc/sudoers $BASE/logs/sudoers.txt + save_dir /etc/sudoers.d + save_file /etc/sudoers # Archive config files # NOTE(mriedem): 'openstack' is added separately since it's not a project @@ -746,9 +761,7 @@ function cleanup_host { sudo mkdir $BASE/logs/etc/ for PROJECT in $PROJECTS openstack; do proj=`basename $PROJECT` - if [ -d /etc/$proj ]; then - sudo cp -r /etc/$proj $BASE/logs/etc/ - fi + save_dir /etc/$proj etc/ done # Archive Apache config files @@ -773,13 +786,12 @@ function cleanup_host { # avoid excessively long file-names. find $BASE/old/screen-logs -type l -print0 | \ xargs -0 -I {} sudo cp {} $BASE/logs/old - sudo cp $BASE/old/devstacklog.txt $BASE/logs/old/ - sudo cp $BASE/old/devstacklog.txt.summary $BASE/logs/old/devstacklog.summary.txt - sudo cp $BASE/old/devstack/localrc $BASE/logs/old/localrc.txt - sudo cp $BASE/old/tempest/etc/tempest.conf $BASE/logs/old/tempest_conf.txt - if [ -f $BASE/old/devstack/tempest.log ] ; then - sudo cp $BASE/old/devstack/tempest.log $BASE/logs/old/verify_tempest_conf.log - fi + save_file $BASE/old/devstacklog.txt old/devstacklog.txt + save_file $BASE/old/devstacklog.txt.summary old/devstacklog.summary.txt + save_file $BASE/old/devstack/localrc old/localrc.txt + save_file $BASE/old/devstack/local.conf old/local_conf.txt + save_file $BASE/old/tempest/etc/tempest.conf old/tempest_conf.txt + save_file $BASE/old/devstack/tempest.log old/verify_tempest_conf.log # Copy Ironic nodes console logs if they exist if [ -d $BASE/old/ironic-bm-logs ] ; then @@ -788,12 +800,10 @@ function cleanup_host { fi # dstat CSV log - if [ -f $BASE/old/dstat-csv.log ]; then - sudo cp $BASE/old/dstat-csv.log $BASE/logs/old/ - fi + save_file $BASE/old/dstat-csv.log old/ # grenade logs - sudo cp $BASE/new/grenade/localrc $BASE/logs/grenade_localrc.txt + save_file $BASE/new/grenade/localrc grenade_localrc.txt # grenade saved state files - resources created during upgrade tests # use this directory to dump arbitrary configuration/state files. @@ -804,9 +814,7 @@ function cleanup_host { # grenade pluginrc - external grenade plugins use this file to # communicate with grenade, capture for posterity - if -f [ $BASE/new/grenade/pluginrc ]; then - sudo cp $BASE/new/grenade/pluginrc $BASE/logs/grenade_pluginrc.txt - fi + save_file $BASE/new/grenade/pluginrc grenade_pluginrc.txt # grenade logs directly and uses similar timestampped files to # devstack. So temporarily copy out & rename the latest log @@ -814,31 +822,26 @@ function cleanup_host { # over time-stampped files and put the interesting logs back at # top-level for easy access sudo mkdir -p $BASE/logs/grenade - sudo cp $BASE/logs/grenade.sh.log $BASE/logs/grenade/ - sudo cp $BASE/logs/grenade.sh.log.summary \ - $BASE/logs/grenade/grenade.sh.summary.log + save_file $BASE/logs/grenade.sh.log grenade/grenade.sh + save_file $BASE/logs/grenade.sh.log.summary \ + grenade/grenade.sh.summary.log sudo rm $BASE/logs/grenade.sh.* sudo mv $BASE/logs/grenade/*.log $BASE/logs sudo rm -rf $BASE/logs/grenade - if [ -f $BASE/new/grenade/javelin.log ] ; then - sudo cp $BASE/new/grenade/javelin.log $BASE/logs/javelin.log - fi + save_file $BASE/new/grenade/javelin.log javelin.log - NEWLOGTARGET=$BASE/logs/new + NEWLOGPREFIX=new/ else - NEWLOGTARGET=$BASE/logs + NEWLOGPREFIX= fi + NEWLOGTARGET=$BASE/logs/$NEWLOGPREFIX find $BASE/new/screen-logs -type l -print0 | \ xargs -0 -I {} sudo cp {} $NEWLOGTARGET/ - sudo cp $BASE/new/devstacklog.txt $NEWLOGTARGET/ - sudo cp $BASE/new/devstacklog.txt.summary $NEWLOGTARGET/devstacklog.summary.txt - sudo cp $BASE/new/devstack/localrc $NEWLOGTARGET/localrc.txt - if [ -f $BASE/new/devstack/local.conf ]; then - sudo cp $BASE/new/devstack/local.conf $NEWLOGTARGET/local.conf.txt - fi - if [ -f $BASE/new/devstack/tempest.log ]; then - sudo cp $BASE/new/devstack/tempest.log $NEWLOGTARGET/verify_tempest_conf.log - fi + save_file $BASE/new/devstacklog.txt ${NEWLOGPREFIX}devstacklog.txt + save_file $BASE/new/devstacklog.txt.summary ${NEWLOGPREFIX}devstacklog.summary.txt + save_file $BASE/new/devstack/localrc ${NEWLOGPREFIX}localrc.txt + save_file $BASE/new/devstack/local.conf ${NEWLOGPREFIX}local.conf.txt + save_file $BASE/new/devstack/tempest.log ${NEWLOGPREFIX}verify_tempest_conf.log # Copy failure files if they exist if [ $(ls $BASE/status/stack/*.failure | wc -l) -gt 0 ]; then @@ -853,24 +856,21 @@ function cleanup_host { fi # Copy tempest config file - sudo cp $BASE/new/tempest/etc/tempest.conf $NEWLOGTARGET/tempest_conf.txt - if [ -f $BASE/new/tempest/etc/accounts.yaml ] ; then - sudo cp $BASE/new/tempest/etc/accounts.yaml $NEWLOGTARGET/accounts_yaml.txt - fi + save_file $BASE/new/tempest/etc/tempest.conf ${NEWLOGPREFIX}tempest_conf.txt + save_file $BASE/new/tempest/etc/accounts.yaml ${NEWLOGPREFIX}accounts_yaml.txt # Copy dstat CSV log if it exists - if [ -f $BASE/new/dstat-csv.log ]; then - sudo cp $BASE/new/dstat-csv.log $BASE/logs/ - fi + save_file $BASE/new/dstat-csv.log sudo iptables-save > $WORKSPACE/iptables.txt df -h > $WORKSPACE/df.txt - sudo mv $WORKSPACE/iptables.txt $WORKSPACE/df.txt $BASE/logs/ + save_file $WORKSPACE/iptables.txt + save_file $WORKSPACE/df.txt for py_ver in 2 3; do if [[ `which python${py_ver}` ]]; then python${py_ver} -m pip freeze > $WORKSPACE/pip${py_ver}-freeze.txt - sudo mv $WORKSPACE/pip${py_ver}-freeze.txt $BASE/logs/ + save_file $WORKSPACE/pip${py_ver}-freeze.txt fi done @@ -890,20 +890,14 @@ function cleanup_host { process_testr_artifacts tempest process_testr_artifacts tempest old - if [ -f $BASE/new/tempest/tempest.log ] ; then - sudo cp $BASE/new/tempest/tempest.log $BASE/logs/tempest.log - fi - if [ -f $BASE/old/tempest/tempest.log ] ; then - sudo cp $BASE/old/tempest/tempest.log $BASE/logs/old/tempest.log - fi + save_file $BASE/new/tempest/tempest.log tempest.log + save_file $BASE/old/tempest/tempest.log old/tempest.log # ceph logs and config if [ -d /var/log/ceph ] ; then sudo cp -r /var/log/ceph $BASE/logs/ fi - if [ -f /etc/ceph/ceph.conf ] ; then - sudo cp /etc/ceph/ceph.conf $BASE/logs/ceph_conf.txt - fi + save_file /etc/ceph/ceph.conf if [ -d /var/log/openvswitch ] ; then sudo cp -r /var/log/openvswitch $BASE/logs/ @@ -913,9 +907,7 @@ function cleanup_host { if [ -d /var/log/glusterfs ] ; then sudo cp -r /var/log/glusterfs $BASE/logs/ fi - if [ -f /etc/glusterfs/glusterd.vol ] ; then - sudo cp /etc/glusterfs/glusterd.vol $BASE/logs/ - fi + save_file /etc/glusterfs/glusterd.vol glusterd.vol # Make sure the current user can read all the logs and configs sudo chown -RL $USER:$USER $BASE/logs/ From df0968e96587e9152b8f16175f1d890ccee6b5ef Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Tue, 10 Jan 2017 07:40:59 -0500 Subject: [PATCH 120/148] save off grenade.sh.log correctly Change-Id: Ia19d64ba240adc642284e4b942f6ce12310f862b --- functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index 1d5106cc..7f05297e 100644 --- a/functions.sh +++ b/functions.sh @@ -822,7 +822,7 @@ function cleanup_host { # over time-stampped files and put the interesting logs back at # top-level for easy access sudo mkdir -p $BASE/logs/grenade - save_file $BASE/logs/grenade.sh.log grenade/grenade.sh + save_file $BASE/logs/grenade.sh.log grenade/grenade.sh.log save_file $BASE/logs/grenade.sh.log.summary \ grenade/grenade.sh.summary.log sudo rm $BASE/logs/grenade.sh.* From 60e957c75797bf69cf8e2c5b9beb215d85998949 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Tue, 17 Jan 2017 11:41:26 -0500 Subject: [PATCH 121/148] functions: sort rpms list in cleanup_host In cleanup_host, we find useful to sort the rpms by alphabetical name so it's easier to use when people copy/paste the list with another list to make packages diff list. It's also easier for readability when going through the list to find a package. Change-Id: I9baac97f94f9dbc8d96b2d625fc1be8f4cb9c6a6 --- functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index 7f05297e..d27ecfd2 100644 --- a/functions.sh +++ b/functions.sh @@ -880,7 +880,7 @@ function cleanup_host { sudo mv $WORKSPACE/dpkg-l.txt.gz $BASE/logs/ fi if [ `command -v rpm` ]; then - rpm -qa > $WORKSPACE/rpm-qa.txt + rpm -qa | sort > $WORKSPACE/rpm-qa.txt gzip -9 rpm-qa.txt sudo mv $WORKSPACE/rpm-qa.txt.gz $BASE/logs/ fi From b4885f9a6728125b8bf6c9cc3d4ae1caaa6b9bad Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Wed, 18 Jan 2017 15:37:26 -0500 Subject: [PATCH 122/148] Create and propogate the DSTOOLS_VERSION devstack-tools will be used to do local.conf manipulation, however we need a sane way to synchronize the version in the event of a breakage. This creates a d-g master variable and pushes that out to all stack.sh and grenade.sh calls Change-Id: I426295baec4f639b0846e9e2d59c36f9bf997b67 --- devstack-vm-gate-wrap.sh | 1 + devstack-vm-gate.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 36093414..d1981b40 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -29,6 +29,7 @@ GIT_BRANCH=${GIT_BRANCH:-master} # possible that new ansible releases can break us. As such we should # be very deliberate about which ansible we use. ANSIBLE_VERSION=${ANSIBLE_VERSION:-2.2.0.0} +export DSTOOLS_VERSION=${DSTOOLS_VERSION:-0.1.4} # sshd may have been compiled with a default path excluding */sbin export PATH=$PATH:/usr/local/sbin:/usr/sbin diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index fe107829..1a4a20cd 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -634,7 +634,7 @@ EOF echo "Running grenade ..." echo "This takes a good 30 minutes or more" cd $BASE/new/grenade - sudo -H -u stack stdbuf -oL -eL ./grenade.sh + sudo -H -u stack DSTOOLS_VERSION=$DSTOOLS_VERSION stdbuf -oL -eL ./grenade.sh cd $BASE/new/devstack else @@ -662,7 +662,7 @@ else echo "... this takes 10 - 15 minutes (logs in logs/devstacklog.txt.gz)" start=$(date +%s) $ANSIBLE primary -f 5 -i "$WORKSPACE/inventory" -m shell \ - -a "cd '$BASE/new/devstack' && sudo -H -u stack stdbuf -oL -eL ./stack.sh executable=/bin/bash" \ + -a "cd '$BASE/new/devstack' && sudo -H -u stack DSTOOLS_VERSION=$DSTOOLS_VERSION stdbuf -oL -eL ./stack.sh executable=/bin/bash" \ &> "$WORKSPACE/logs/devstack-early.txt" if [ -d "$BASE/data/CA" ] && [ -f "$BASE/data/ca-bundle.pem" ] ; then # Sync any data files which include certificates to be used if @@ -682,7 +682,7 @@ else # because services like nova apparently expect to have the controller in # place before anything else. $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m shell \ - -a "cd '$BASE/new/devstack' && sudo -H -u stack stdbuf -oL -eL ./stack.sh executable=/bin/bash" \ + -a "cd '$BASE/new/devstack' && sudo -H -u stack DSTOOLS_VERSION=$DSTOOLS_VERSION stdbuf -oL -eL ./stack.sh executable=/bin/bash" \ &> "$WORKSPACE/logs/devstack-subnodes-early.txt" end=$(date +%s) took=$((($end - $start) / 60)) From 6c3752000feb57366d423ebb84cc3fc60df3d519 Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Fri, 20 Jan 2017 14:13:26 -0500 Subject: [PATCH 123/148] Make the workspace logs owned by stack user for grenade also This makes the workspace, including logs, owned by the stack user. This is needed for writing logs during a grenade run also. Change-Id: I4a7b23a7ef083a8570c21acbb468fc63705aaebc --- devstack-vm-gate.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 1a4a20cd..0f76594d 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -571,6 +571,19 @@ EOF } +# This makes the stack user own the $BASE files and also changes the +# permissions on the logs directory so we can write to the logs when running +# devstack or grenade. This must be called AFTER setup_localrc. +function setup_access_for_stack_user { + # Make the workspace owned by the stack user + # It is not clear if the ansible file module can do this for us + $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "chown -R stack:stack '$BASE'" + # allow us to add logs + $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "chmod 777 '$WORKSPACE/logs'" +} + if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then cd $BASE/old/devstack setup_localrc "old" "localrc" "primary" @@ -626,10 +639,7 @@ EOF setup_networking "grenade" - # Make the workspace owned by the stack user - # It is not clear if the ansible file module can do this for us - $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m shell \ - -a "chown -R stack:stack '$BASE'" + setup_access_for_stack_user echo "Running grenade ..." echo "This takes a good 30 minutes or more" @@ -650,13 +660,7 @@ else setup_networking - # Make the workspace owned by the stack user - # It is not clear if the ansible file module can do this for us - $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m shell \ - -a "chown -R stack:stack '$BASE'" - # allow us to add logs - $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m shell \ - -a "chmod 777 '$WORKSPACE/logs'" + setup_access_for_stack_user echo "Running devstack" echo "... this takes 10 - 15 minutes (logs in logs/devstacklog.txt.gz)" From f1025531af1726e4fff54b31e16d472bae8193de Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Thu, 19 Jan 2017 21:55:00 -0500 Subject: [PATCH 124/148] Run discover_hosts to register subnodes in a multinode setup CellsV2 requires that compute hosts are registered with a cell. In a single-node devstack run the 'nova-manage cell_v2 simple_cell_setup' command will create a cell from the main (only) cell database, discover the compute node and map it to the cell. In a multinode setup, we need to run discover_hosts on the primary node after the subnode is setup since the subnode doesn't know which cell it's going to be in and doesn't (or can't rely on) have the database connection to auto-register. The discover_hosts command is idempotent so running this from a single-node or multi-node setup will make no difference. Depends-On: Icc595d60de373471aa7ee8fb9f3a81fc12d80438 Change-Id: I8938fd37f71498e87aa348a81c357eef5c1a8093 --- devstack-vm-gate.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 0f76594d..cf8599af 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -224,6 +224,22 @@ function setup_networking { fi } +# Discovers compute nodes (subnodes) and maps them to cells. +# NOTE(mriedem): We want to remove this if/when nova supports auto-registration +# of computes with cells, but that's not happening in Ocata. +function discover_hosts { + # We have to run this on the primary node AFTER the subnodes have been + # setup. Since discover_hosts is really only needed for Ocata, this checks + # to see if the script exists in the devstack installation first. + # NOTE(danms): This is ||'d with an assertion that the script does not exist, + # so that if we actually failed the script, we'll exit nonzero here instead + # of ignoring failures along with the case where there is no script. + # TODO(mriedem): Would be nice to do this with wrapped lines. + $ANSIBLE primary -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "cd $BASE/new/devstack/ && (test -f tools/discover_hosts.sh && sudo -H -u stack DSTOOLS_VERSION=$DSTOOLS_VERSION stdbuf -oL -eL ./tools/discover_hosts.sh) || (! test -f tools/discover_hosts.sh)" \ + &> "$WORKSPACE/logs/devstack-gate-discover-hosts.txt" +} + function setup_localrc { local localrc_oldnew=$1; local localrc_file=$2 @@ -694,6 +710,9 @@ else echo "WARNING: devstack run took > 20 minutes, this is a very slow node." fi + # Discover the hosts on a cells v2 deployment. + discover_hosts + # provide a check that the right db was running # the path are different for fedora and red hat. if [[ -f /usr/bin/yum ]]; then From 8545702eab1f5456e9e78b5319e1a9cf8bc610af Mon Sep 17 00:00:00 2001 From: Kevin Benton Date: Thu, 26 Jan 2017 14:47:01 -0800 Subject: [PATCH 125/148] Increase swappiness to try to avoid OOM killer The OOM killer is being triggered even with significant swap space remaining. This patch increases the swappiness value from 10 to 30. This is still half of the 60 default so it should avoid excessive swapping. Change-Id: I0d81057d66dbee0dbd3650db9f34b598e1b3356b Closes-Bug: #1656386 --- functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index ed346494..ebf40210 100644 --- a/functions.sh +++ b/functions.sh @@ -429,7 +429,7 @@ function fix_disk_layout { sudo sed -i '/vm.swappiness/d' /etc/sysctl.conf # This sets swappiness low; we really don't want to be relying on # cloud I/O based swap during our runs - sudo sysctl -w vm.swappiness=10 + sudo sysctl -w vm.swappiness=30 } # Set up a project in accordance with the future state proposed by From b9aba1f6dacd150d5d4489364886c4319382b892 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Fri, 27 Jan 2017 11:06:21 -0500 Subject: [PATCH 126/148] stop tracing db queries We're no longer even collecting mysql logs at this point (not sure when that broke, but they aren't anywhere to be found), so doing all the extra work for query logging seems like a way to just slow things down and possibly drive mysql into crazy land. Change-Id: I5e030dfcb93b581e9424373674f94278cbbc58f6 --- devstack-vm-gate.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index cf8599af..e400764e 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -361,7 +361,6 @@ CINDER_PERIODIC_INTERVAL=10 export OS_NO_CACHE=True CEILOMETER_BACKEND=$DEVSTACK_GATE_CEILOMETER_BACKEND LIBS_FROM_GIT=$DEVSTACK_PROJECT_FROM_GIT -DATABASE_QUERY_LOGGING=True # set this until all testing platforms have libvirt >= 1.2.11 # see bug #1501558 EBTABLES_RACE_FIX=True From 48f3d704f9acfc7aa445d76004fdbfdf7131f111 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Tue, 31 Jan 2017 11:39:49 -0500 Subject: [PATCH 127/148] don't set "export ..." lines in localrc export lines in localrc end up not matching in devstack-tools for config file merging. They shouldn't be added that way, as there is no benefit over just setting them. Change-Id: I582c262f2280fbaa015bfa7624a129a0f86deac0 --- devstack-vm-gate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index e400764e..4ca99a92 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -539,8 +539,8 @@ EOF if [[ "$DEVSTACK_GATE_TOPOLOGY" != "aio" ]]; then echo "NOVA_ALLOW_MOVE_TO_SAME_HOST=False" >> "$localrc_file" - echo "export LIVE_MIGRATION_AVAILABLE=True" >> "$localrc_file" - echo "export USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION=True" >> "$localrc_file" + echo "LIVE_MIGRATION_AVAILABLE=True" >> "$localrc_file" + echo "USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION=True" >> "$localrc_file" local primary_node=`cat /etc/nodepool/primary_node_private` echo "SERVICE_HOST=$primary_node" >>"$localrc_file" From 634da15ca9365de171e299c50ccd60dec448c11b Mon Sep 17 00:00:00 2001 From: Masayuki Igawa Date: Wed, 21 Sep 2016 11:34:53 +0200 Subject: [PATCH 128/148] Remove unused tempest runnings This commit removes unused tempest running - large-ops and heat-slow. These were removed before from Tempest[1][2]. We can't use these anymore. [1] I582c697d59d121bdd43d9c41eeba1b67c64a4861 [2] I93b2fb33e97381f7c1e0cb1ef09ebc5c42c16ecc Change-Id: I770a0838755b5ff5c903dfc18d8cb08df5c923bb --- devstack-vm-gate.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index e400764e..853f0f73 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -826,12 +826,6 @@ if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then elif [[ "$DEVSTACK_GATE_TEMPEST_STRESS" -eq "1" ]] ; then echo "Running stress tests" $TEMPEST_COMMAND -estress -- $DEVSTACK_GATE_TEMPEST_STRESS_ARGS - elif [[ "$DEVSTACK_GATE_TEMPEST_HEAT_SLOW" -eq "1" ]] ; then - echo "Running slow heat tests" - $TEMPEST_COMMAND -eheat-slow -- --concurrency=$TEMPEST_CONCURRENCY - elif [[ "$DEVSTACK_GATE_TEMPEST_LARGE_OPS" -ge "1" ]] ; then - echo "Running large ops tests" - $TEMPEST_COMMAND -elarge-ops -- --concurrency=$TEMPEST_CONCURRENCY elif [[ "$DEVSTACK_GATE_SMOKE_SERIAL" -eq "1" ]] ; then echo "Running tempest smoke tests" $TEMPEST_COMMAND -esmoke-serial From c435a724bd257b22a2e39e8e9125c11302a8c81d Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Mon, 28 Nov 2016 14:25:09 +0000 Subject: [PATCH 129/148] Ansibly the copy of mirror config Replace the copy of mirror config bash logic for copy_mirror_config role Change-Id: I2fa4a19dd2db1c531c75b3ed14b5f1e4380c70a5 --- functions.sh | 15 --------------- playbooks/devstack_gate_vars.yaml | 1 + .../roles/copy_mirror_config/tasks/main.yaml | 12 ++++++++++++ playbooks/setup_host.yaml | 1 + 4 files changed, 14 insertions(+), 15 deletions(-) create mode 100644 playbooks/roles/copy_mirror_config/tasks/main.yaml diff --git a/functions.sh b/functions.sh index ebf40210..fbe7e877 100644 --- a/functions.sh +++ b/functions.sh @@ -571,26 +571,11 @@ function setup_workspace { $xtrace } -function copy_mirror_config { - # The pydistutils.cfg file is added by Puppet. Some CIs may not rely on - # Puppet to do the base node installation - if [ -f ~/.pydistutils.cfg ]; then - sudo install -D -m0644 -o root -g root ~/.pydistutils.cfg ~root/.pydistutils.cfg - - sudo install -D -m0644 -o stack -g stack ~/.pydistutils.cfg ~stack/.pydistutils.cfg - - sudo install -D -m0644 -o tempest -g tempest ~/.pydistutils.cfg ~tempest/.pydistutils.cfg - fi -} - function setup_host { # Enabled detailed logging, since output of this function is redirected local xtrace=$(set +o | grep xtrace) set -o xtrace - # Ensure that all of the users have the openstack mirror config - copy_mirror_config - # perform network sanity check so that we can characterize the # state of the world network_sanity_check diff --git a/playbooks/devstack_gate_vars.yaml b/playbooks/devstack_gate_vars.yaml index 760581a7..8a0da981 100644 --- a/playbooks/devstack_gate_vars.yaml +++ b/playbooks/devstack_gate_vars.yaml @@ -1,2 +1,3 @@ --- BASE: "{{ lookup('env', 'BASE')|default('/opt/stack', true) }}" +CI_USER: "{{ lookup('env', 'CI_USER')|default(ansible_user_id, true) }}" diff --git a/playbooks/roles/copy_mirror_config/tasks/main.yaml b/playbooks/roles/copy_mirror_config/tasks/main.yaml new file mode 100644 index 00000000..385dcd4b --- /dev/null +++ b/playbooks/roles/copy_mirror_config/tasks/main.yaml @@ -0,0 +1,12 @@ +- name: Get status of pydistutils.cfg file + stat: path={{ '~' + CI_USER | expanduser }}/.pydistutils.cfg + register: st +- block: + - name: Install CI_USER .pydistutils on root home folder + command: install -D -m0644 -o root -g root {{ '~' + CI_USER | expanduser }}/.pydistutils.cfg /root/.pydistutils.cfg + - name: Install CI_USER .pydistutils on stack home folder + command: install -D -m0644 -o stack -g stack {{ '~' + CI_USER | expanduser }}/.pydistutils.cfg {{ BASE }}/new/.pydistutils.cfg + - name: Install CI_USER .pydistutils on tempest home folder + command: install -D -m0644 -o tempest -g tempest {{ '~' + CI_USER | expanduser }}/.pydistutils.cfg /home/tempest/.pydistutils.cfg + when: st.stat.exists + become: yes diff --git a/playbooks/setup_host.yaml b/playbooks/setup_host.yaml index 36a9e08b..26decd47 100644 --- a/playbooks/setup_host.yaml +++ b/playbooks/setup_host.yaml @@ -10,3 +10,4 @@ - start_fresh_logging - setup_stack_user - setup_tempest_user + - copy_mirror_config From c8cd51705255dd8ea643ee684508a1b5489804da Mon Sep 17 00:00:00 2001 From: Dan Smith Date: Wed, 8 Feb 2017 10:10:03 -0800 Subject: [PATCH 130/148] Run discover_hosts.sh in post_stack.sh for grenade Change-Id: Iaf14c226a3be72a076cc58690bdae8b6b51d6120 --- devstack-vm-gate.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 06663a42..c815792b 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -648,6 +648,10 @@ set -x $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m shell \ -a "cd '$BASE/old/devstack' && stdbuf -oL -eL ./stack.sh" + +if [[ -e "$BASE/old/devstack/tools/discover_hosts.sh" ]]; then + $BASE/old/devstack/tools/discover_hosts.sh +fi EOF sudo chmod a+x $BASE/new/grenade/post-stack.sh fi From 9c752b02fbd57c7021a7c9295bf4d68a0d1973a8 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 1 Feb 2017 15:41:54 -0500 Subject: [PATCH 131/148] update grenade settings for stable/ocata Depends-On: Ib17b61641cfeb5f10bf7d01f9b02019fb1e9dbf8 Change-Id: Iec68201755bdb2985c1d1b2fbddc9fdaf82a8864 Signed-off-by: Doug Hellmann --- devstack-vm-gate-wrap.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index d1981b40..ba4ad232 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -322,6 +322,9 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then elif [[ "$GRENADE_BASE_BRANCH" == "stable/newton" ]]; then export GRENADE_OLD_BRANCH="stable/newton" export GRENADE_NEW_BRANCH="$GIT_BRANCH" + elif [[ "$GRENADE_BASE_BRANCH" == "stable/ocata" ]]; then + export GRENADE_OLD_BRANCH="stable/ocata" + export GRENADE_NEW_BRANCH="$GIT_BRANCH" fi ;; @@ -343,8 +346,11 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then elif [[ "$GRENADE_BASE_BRANCH" == "stable/newton" ]]; then export GRENADE_OLD_BRANCH="stable/mitaka" export GRENADE_NEW_BRANCH="stable/newton" - else # master + elif [[ "$GRENADE_BASE_BRANCH" == "stable/ocata" ]]; then export GRENADE_OLD_BRANCH="stable/newton" + export GRENADE_NEW_BRANCH="stable/ocata" + else # master + export GRENADE_OLD_BRANCH="stable/ocata" export GRENADE_NEW_BRANCH="$GIT_BRANCH" fi ;; From 00a74cb1b73c9492cad07c2fd2e27dea647feb4b Mon Sep 17 00:00:00 2001 From: David Moreau-Simard Date: Mon, 13 Feb 2017 12:28:47 -0500 Subject: [PATCH 132/148] Replace "ara generate" by "ara generate html" The newest version of ARA, 0.11, deprecated "ara generate" in favor of "ara generate html". "ara generate" will be removed sometime in the future so let's adjust it right away. Change-Id: I21eab6508b642714770c5a8530bdc98df11ed64c --- devstack-vm-gate-wrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index ba4ad232..75504bf4 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -660,6 +660,6 @@ $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m synchronize \ sudo mv $WORKSPACE/devstack-gate-cleanup-host.txt $BASE/logs/ # Generate ARA report -/tmp/ansible/bin/ara generate $BASE/logs/ara +/tmp/ansible/bin/ara generate html $BASE/logs/ara exit $RETVAL From eb288f579fea433a55f7ca5008e5f6274ede7787 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Mon, 13 Feb 2017 09:46:16 -0500 Subject: [PATCH 133/148] Upgrade devstack-tools to 0.2.1 This provides a more comprehensive merge solution. Change-Id: I9932e3d0a68035b7aee205e037e605a50e5fbae7 --- devstack-vm-gate-wrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index ba4ad232..1fdd1e87 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -29,7 +29,7 @@ GIT_BRANCH=${GIT_BRANCH:-master} # possible that new ansible releases can break us. As such we should # be very deliberate about which ansible we use. ANSIBLE_VERSION=${ANSIBLE_VERSION:-2.2.0.0} -export DSTOOLS_VERSION=${DSTOOLS_VERSION:-0.1.4} +export DSTOOLS_VERSION=${DSTOOLS_VERSION:-0.2.1} # sshd may have been compiled with a default path excluding */sbin export PATH=$PATH:/usr/local/sbin:/usr/sbin From dfe8aa213919670fc0daa3a2bb9c280c4ed2021b Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Thu, 16 Feb 2017 18:48:25 -0500 Subject: [PATCH 134/148] Set LIBVIRT_TYPE in the subnode localrc setup_localrc was not setting LIBVIRT_TYPE into the subnode localrc file, which means we run multinode (live migration) tests with nested virt using kvm rather than qemu on the subnode and nested virt with kvm has lots of problems, like just crazy random failures. Change-Id: Ifa11d305cb0dfea932ae16108a894d747d78167c Closes-Bug: #1665487 --- devstack-vm-gate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index c815792b..c6112f5e 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -384,7 +384,7 @@ EOF if [[ "$DEVSTACK_GATE_VIRT_DRIVER" == "libvirt" ]]; then if [[ -n "$DEVSTACK_GATE_LIBVIRT_TYPE" ]]; then - echo "LIBVIRT_TYPE=${DEVSTACK_GATE_LIBVIRT_TYPE}" >>localrc + echo "LIBVIRT_TYPE=${DEVSTACK_GATE_LIBVIRT_TYPE}" >>"$localrc_file" fi fi From c4612ff98d8241ea7f4b33c6869191b2dd63585b Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Mon, 20 Feb 2017 10:04:03 +0000 Subject: [PATCH 135/148] Correct ovs_vxlan_bridge function error message Copy-pasta error, s/ntp/openvswitch Change-Id: I7c2873057c70c808e97efd4fc6174e5445bebb03 --- functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions.sh b/functions.sh index fbe7e877..d4f96dde 100644 --- a/functions.sh +++ b/functions.sh @@ -1051,7 +1051,7 @@ function ovs_vxlan_bridge { local ovs_package='openvswitch-switch' local ovs_service='openvswitch-switch' else - echo "Unsupported platform, can't determine ntp service" + echo "Unsupported platform, can't determine openvswitch service" exit 1 fi local install_ovs_deps="source $BASE/new/devstack/functions-common; \ From 4d7b92ca16c5ac04af71512d39230004abf3b4db Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Tue, 29 Nov 2016 14:27:20 +0000 Subject: [PATCH 136/148] Ansibly the network sanity check Replace the network sanity check bash logic for the network_sanity_check role. This is the last step of replacing the setup_host bash function, thus removing also the function altogether. Change-Id: I8814cf284e572dda66455eea7a10f61037979a9b --- devstack-vm-gate-wrap.sh | 2 - functions.sh | 39 ------------------- playbooks/devstack_gate_vars.yaml | 3 ++ .../tasks/http_check.yaml | 5 +++ .../network_sanity_check/tasks/main.yaml | 28 +++++++++++++ .../tasks/ping_check.yaml | 3 ++ playbooks/setup_host.yaml | 1 + 7 files changed, 40 insertions(+), 41 deletions(-) create mode 100644 playbooks/roles/network_sanity_check/tasks/http_check.yaml create mode 100644 playbooks/roles/network_sanity_check/tasks/main.yaml create mode 100644 playbooks/roles/network_sanity_check/tasks/ping_check.yaml diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 12834238..bf6eaa99 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -557,8 +557,6 @@ EOF echo "... this takes a few seconds (logs at logs/devstack-gate-setup-host.txt.gz)" $ANSIBLE_PLAYBOOK -f 5 -i "$WORKSPACE/inventory" "$WORKSPACE/devstack-gate/playbooks/setup_host.yaml" \ &> "$WORKSPACE/logs/devstack-gate-setup-host.txt" -$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ - -a "$(run_command setup_host)" &>> "$WORKSPACE/logs/devstack-gate-setup-host.txt" if [ -n "$DEVSTACK_GATE_GRENADE" ]; then start=$(date +%s) diff --git a/functions.sh b/functions.sh index fbe7e877..3857b0bc 100644 --- a/functions.sh +++ b/functions.sh @@ -117,32 +117,6 @@ function _http_check { done } -# do a few network tests to baseline how bad we are -function network_sanity_check { - echo "Performing network sanity check..." - PIP_CONFIG_FILE=/etc/pip.conf - if [[ -f $PIP_CONFIG_FILE ]]; then - line=$(cat $PIP_CONFIG_FILE|grep --max-count 1 index-url) - pypi_url=${line#*=} - pypi_host=$(echo $pypi_url|grep -Po '.*?//\K.*?(?=/)') - - _ping_check $pypi_host - _http_check $pypi_url - fi - - if [[ -f /etc/nodepool/provider ]]; then - # AFS ubuntu mirror - source /etc/nodepool/provider - if [[ -n "$NODEPOOL_MIRROR_HOST" || ( -n "$NODEPOOL_REGION" && -n "$NODEPOOL_CLOUD" ) ]]; then - NODEPOOL_MIRROR_HOST=${NODEPOOL_MIRROR_HOST:-mirror.$NODEPOOL_REGION.$NODEPOOL_CLOUD.openstack.org} - NODEPOOL_MIRROR_HOST=$(echo $NODEPOOL_MIRROR_HOST|tr '[:upper:]' '[:lower:]') - - _ping_check $NODEPOOL_MIRROR_HOST - _http_check http://$NODEPOOL_MIRROR_HOST/ubuntu/dists/trusty/Release - fi - fi -} - # create the start timer for when the job began function start_timer { # first make sure the time is right, so we don't go into crazy land @@ -571,19 +545,6 @@ function setup_workspace { $xtrace } -function setup_host { - # Enabled detailed logging, since output of this function is redirected - local xtrace=$(set +o | grep xtrace) - set -o xtrace - - # perform network sanity check so that we can characterize the - # state of the world - network_sanity_check - - # Disable detailed logging as we return to the main script - $xtrace -} - function archive_test_artifact { local filename=$1 diff --git a/playbooks/devstack_gate_vars.yaml b/playbooks/devstack_gate_vars.yaml index 8a0da981..62f5ff21 100644 --- a/playbooks/devstack_gate_vars.yaml +++ b/playbooks/devstack_gate_vars.yaml @@ -1,3 +1,6 @@ --- BASE: "{{ lookup('env', 'BASE')|default('/opt/stack', true) }}" CI_USER: "{{ lookup('env', 'CI_USER')|default(ansible_user_id, true) }}" +PING_TIMES: 20 +HTTP_TIMES: 10 +PIP_CONFIG_FILE: "{{ lookup('env', 'PIP_CONFIG_FILE')|default('/etc/pip.conf', true) }}" diff --git a/playbooks/roles/network_sanity_check/tasks/http_check.yaml b/playbooks/roles/network_sanity_check/tasks/http_check.yaml new file mode 100644 index 00000000..555d9a41 --- /dev/null +++ b/playbooks/roles/network_sanity_check/tasks/http_check.yaml @@ -0,0 +1,5 @@ +- name: Perform HTTP check + uri: url={{ url }} + register: uri_result + until: uri_result['status'] == 200 + retries: "{{ HTTP_TIMES }}" diff --git a/playbooks/roles/network_sanity_check/tasks/main.yaml b/playbooks/roles/network_sanity_check/tasks/main.yaml new file mode 100644 index 00000000..091f91ce --- /dev/null +++ b/playbooks/roles/network_sanity_check/tasks/main.yaml @@ -0,0 +1,28 @@ +--- +- name: Get status of file PIP_CONFIG_FILE + stat: path={{ PIP_CONFIG_FILE }} + register: st +- block: + - name: Set pypi_url variable + set_fact: pypi_url={{ lookup('ini', 'index-url section=global file=' + PIP_CONFIG_FILE) }} + - name: Set pypi_host variable + set_fact: pypi_host={{ pypi_url.split('/')[2] }} + - include: ping_check.yaml host={{ pypi_host }} + - include: http_check.yaml url={{ pypi_url }} + when: st.stat.exists +- name: Get NODEPOOL_MIRROR_HOST from /etc/nodepool/provider + shell: grep NODEPOOL_MIRROR_HOST /etc/nodepool/provider | cut -d "=" -f2 + register: grep_mirror_host + changed_when: False +- name: Get NODEPOOL_REGION from /etc/nodepool/provider + shell: grep NODEPOOL_REGION /etc/nodepool/provider | cut -d "=" -f2 + register: grep_region + changed_when: False +- name: Get NODEPOOL_CLOUD from /etc/nodepool/provider + shell: grep NODEPOOL_CLOUD /etc/nodepool/provider | cut -d "=" -f2 + register: grep_cloud + changed_when: False +- name: Build NODEPOOL_MIRROR_HOST variable with region and cloud if undefined + set_fact: NODEPOOL_MIRROR_HOST={{ grep_mirror_host.stdout|default("mirror." + grep_region.stdout|lower + "." + grep_cloud.stdout + ".openstack.org", true) }} +- include: ping_check.yaml host={{ NODEPOOL_MIRROR_HOST }} +- include: http_check.yaml url=http://{{ NODEPOOL_MIRROR_HOST }}/ubuntu/dists/trusty/Release diff --git a/playbooks/roles/network_sanity_check/tasks/ping_check.yaml b/playbooks/roles/network_sanity_check/tasks/ping_check.yaml new file mode 100644 index 00000000..57ae849d --- /dev/null +++ b/playbooks/roles/network_sanity_check/tasks/ping_check.yaml @@ -0,0 +1,3 @@ +- name: Perform ping check + command: ping -c {{ PING_TIMES }} {{ host }} + changed_when: False diff --git a/playbooks/setup_host.yaml b/playbooks/setup_host.yaml index 26decd47..2a140e5d 100644 --- a/playbooks/setup_host.yaml +++ b/playbooks/setup_host.yaml @@ -11,3 +11,4 @@ - setup_stack_user - setup_tempest_user - copy_mirror_config + - network_sanity_check From 72cc599a977cb83f8c68dffff87aafbabac8a06e Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Wed, 22 Feb 2017 07:16:49 +0000 Subject: [PATCH 137/148] Cleaned up 'quantum' service logic There is no such service (as a separate entity) in devstack since at least kilo (probably even earlier). Instead, neutron services are controlled separately by q-* (legacy) or neutron-* (new devstack library module). Change-Id: I69a8866315acce1d192738873939d82bf397b918 --- features.yaml | 2 +- test-features.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/features.yaml b/features.yaml index 0a001cb3..80cc0437 100644 --- a/features.yaml +++ b/features.yaml @@ -101,7 +101,7 @@ primary: neutron: base: - services: [quantum, q-svc, q-agt, q-dhcp, q-l3, q-meta, q-metering] + services: [q-svc, q-agt, q-dhcp, q-l3, q-meta, q-metering] rm-services: [n-net] neutron-adv: diff --git a/test-features.sh b/test-features.sh index cd45c7a8..5fea6b2d 100755 --- a/test-features.sh +++ b/test-features.sh @@ -18,9 +18,9 @@ ERRORS=0 TEMPEST_FULL_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,n-net,placement-api" -TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,placement-api" +TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,placement-api" -TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,quantum,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,placement-api" +TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,placement-api" GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,placement-api" From 728ee0c2ca1085572d0a54ed6ad6a3ede20198ba Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Wed, 22 Feb 2017 07:11:43 +0000 Subject: [PATCH 138/148] Cleaned up code from Icehouse and Juno logic There are no such branches for devstack anymore, so keeping the code that supports them in gate doesn't make sense. Change-Id: I2d7b7845b31e860ee9e8b64c581768711fa73a78 --- devstack-vm-gate-wrap.sh | 22 +++++----------------- devstack-vm-gate.sh | 18 ------------------ features.yaml | 8 ++------ 3 files changed, 7 insertions(+), 41 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 12834238..c4775c31 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -263,9 +263,9 @@ export DEVSTACK_PROJECT_FROM_GIT=${DEVSTACK_PROJECT_FROM_GIT:-} # for a stable branch we want to both try to upgrade forward n => n+1 as # well as upgrade from last n-1 => n. # -# i.e. stable/juno: -# pullup means stable/icehouse => stable/juno -# forward means stable/juno => master (or stable/kilo if that's out) +# i.e. stable/ocata: +# pullup means stable/newton => stable/ocata +# forward means stable/ocata => master (or stable/pike if that's out) export DEVSTACK_GATE_GRENADE=${DEVSTACK_GATE_GRENADE:-} # the branch name for selecting grenade branches @@ -304,13 +304,7 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then # forward upgrades are an attempt to migrate up from an # existing stable branch to the next release. forward) - if [[ "$GRENADE_BASE_BRANCH" == "stable/icehouse" ]]; then - export GRENADE_OLD_BRANCH="stable/icehouse" - export GRENADE_NEW_BRANCH="stable/juno" - elif [[ "$GRENADE_BASE_BRANCH" == "stable/juno" ]]; then - export GRENADE_OLD_BRANCH="stable/juno" - export GRENADE_NEW_BRANCH="stable/kilo" - elif [[ "$GRENADE_BASE_BRANCH" == "stable/kilo" ]]; then + if [[ "$GRENADE_BASE_BRANCH" == "stable/kilo" ]]; then export GRENADE_OLD_BRANCH="stable/kilo" export GRENADE_NEW_BRANCH="stable/liberty" elif [[ "$GRENADE_BASE_BRANCH" == "stable/liberty" ]]; then @@ -331,13 +325,7 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then # pullup upgrades are our normal upgrade test. Can you upgrade # to the current patch from the last stable. pullup) - if [[ "$GRENADE_BASE_BRANCH" == "stable/juno" ]]; then - export GRENADE_OLD_BRANCH="stable/icehouse" - export GRENADE_NEW_BRANCH="stable/juno" - elif [[ "$GRENADE_BASE_BRANCH" == "stable/kilo" ]]; then - export GRENADE_OLD_BRANCH="stable/juno" - export GRENADE_NEW_BRANCH="stable/kilo" - elif [[ "$GRENADE_BASE_BRANCH" == "stable/liberty" ]]; then + if [[ "$GRENADE_BASE_BRANCH" == "stable/liberty" ]]; then export GRENADE_OLD_BRANCH="stable/kilo" export GRENADE_NEW_BRANCH="stable/liberty" elif [[ "$GRENADE_BASE_BRANCH" == "stable/mitaka" ]]; then diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index c6112f5e..f0616b71 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -486,24 +486,6 @@ EOF else echo "GRENADE_PHASE=target" >> "$localrc_file" fi - # services deployed with mod wsgi cannot be upgraded or migrated - # until https://launchpad.net/bugs/1365105 is resolved. - case $GRENADE_NEW_BRANCH in - "stable/icehouse") - ;& - "stable/juno") - echo "KEYSTONE_USE_MOD_WSGI=False" >> "$localrc_file" - ;; - "stable/kilo") - # while both juno and kilo can run under wsgi, they - # can't run a code only upgrade because the - # configuration assumes copying python files around - # during config stage. This might be addressed by - # keystone team later, hence separate comment and code - # block. - echo "KEYSTONE_USE_MOD_WSGI=False" >> "$localrc_file" - ;; - esac echo "CEILOMETER_USE_MOD_WSGI=False" >> "$localrc_file" fi diff --git a/features.yaml b/features.yaml index 80cc0437..c7333e1d 100644 --- a/features.yaml +++ b/features.yaml @@ -6,8 +6,6 @@ config: mitaka: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] liberty: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] kilo: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] - juno: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] - icehouse: [default, ceilometer, glance, horizon, nova, swift, cinder, keystone] # This can be used by functional jobs that only want their dependencies installed # and don't need to incur the overhead of installing all services in the process. no_services: [default] @@ -56,8 +54,8 @@ config: branches: # The value of ""default" is the name of the "trunk" branch default: master - # Normalized branch names only here, e.g. stable/icehouse => icehouse - allowed: [master, ocata, newton, mitaka, liberty, kilo, juno, icehouse] + # Normalized branch names only here, e.g. stable/ocata => ocata + allowed: [master, ocata, newton, mitaka, liberty, kilo] primary: default: @@ -83,8 +81,6 @@ primary: nova: base: services: [n-api, n-cauth, n-cond, n-cpu, n-crt, n-net, n-novnc, n-obj, n-sch] - icehouse: - compute-ext: nova-md: base: From 57b3b8912989496fb11e7ce30d54cddb601f785e Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Fri, 10 Feb 2017 12:29:03 +0000 Subject: [PATCH 139/148] Enable peakmem_tracker in gate We experience unexplained behaviour lately in some gate jobs where oom-killer is triggered while swap is not used. We suspect that it may be because some process mlocked some pages in RAM. This service should hopefully help us to understand if that's the case. Change-Id: Ia5bbc3d5dc405af0417624b92816269775809e1a --- features.yaml | 4 ++-- test-features.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/features.yaml b/features.yaml index c7333e1d..73eaf29a 100644 --- a/features.yaml +++ b/features.yaml @@ -60,7 +60,7 @@ branches: primary: default: base: - services: [mysql, rabbit, dstat] + services: [mysql, rabbit, dstat, peakmem_tracker] ceilometer: base: @@ -191,7 +191,7 @@ primary: subnode: default: base: - services: [dstat] + services: [dstat, peakmem_tracker] ceilometer: base: diff --git a/test-features.sh b/test-features.sh index 5fea6b2d..bdda8024 100755 --- a/test-features.sh +++ b/test-features.sh @@ -16,13 +16,13 @@ ERRORS=0 -TEMPEST_FULL_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,n-net,placement-api" +TEMPEST_FULL_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,peakmem_tracker,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,n-net,placement-api" -TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,placement-api" +TEMPEST_NEUTRON_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,peakmem_tracker,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,placement-api" -TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,placement-api" +TEMPEST_HEAT_SLOW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,horizon,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,peakmem_tracker,tempest,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,placement-api" -GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,placement-api" +GRENADE_NEW_MASTER="n-api,n-crt,n-obj,n-cpu,n-sch,n-cond,n-novnc,n-cauth,g-api,g-reg,key,c-api,c-vol,c-sch,c-bak,cinder,s-proxy,s-account,s-container,s-object,mysql,rabbit,dstat,peakmem_tracker,tempest,n-net,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api,ceilometer-alarm-notifier,ceilometer-alarm-evaluator,ceilometer-anotification,placement-api" # Utility function for tests function assert_list_equal { From 19561a4cb44fe3a20a0eab50ce91d83790d0333d Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Mon, 27 Jun 2016 10:15:11 -0400 Subject: [PATCH 140/148] support local.conf instead of localrc This moves the bulk of devstack-gate to local.conf instead of localrc, which makes it so that we can do more interesting and complicated things with the devstack setup. grenade local.conf support Depends-On: I9bf5ac24dc87404e054068d7ed643e0bf0fac311 Change-Id: I111a6df3adc07ed426ac714cb9e64667a63e0e5a --- devstack-vm-gate.sh | 350 +++++++++++++++++++++----------------------- 1 file changed, 170 insertions(+), 180 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index f0616b71..e1c99e64 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -20,6 +20,7 @@ # limitations under the License. set -o errexit +set -o xtrace # Keep track of the devstack directory TOP_DIR=$(cd $(dirname "$0") && pwd) @@ -29,6 +30,8 @@ TOP_DIR=$(cd $(dirname "$0") && pwd) # Import common functions source $TOP_DIR/functions.sh +# Get access to iniset and friends +source $BASE/new/devstack/inc/ini-config echo $PPID > $WORKSPACE/gate.pid source `dirname "$(readlink -f "$0")"`/functions.sh @@ -88,10 +91,8 @@ function setup_nova_net_networking { $sub_nodes ovs_vxlan_bridge "br_flat" $primary_node "False" 128 \ $sub_nodes - cat <>"$localrc" -FLAT_INTERFACE=br_flat -PUBLIC_INTERFACE=br_pub -EOF + localrc_set $localrc "FLAT_INTERFACE" "br_flat" + localrc_set $localrc "PUBLIC_INTERFACE" "br_pub" } function setup_multinode_connectivity { @@ -114,11 +115,11 @@ function setup_multinode_connectivity { # set explicit paths on all conf files we're writing so that # current working directory doesn't introduce subtle bugs. local devstack_dir=$BASE/$old_or_new/devstack - local sub_localrc=$devstack_dir/sub_localrc + local sub_localconf=$devstack_dir/sub_local.conf local localconf=$devstack_dir/local.conf set -x # for now enabling debug and do not turn it off - setup_localrc $old_or_new "$sub_localrc" "sub" + setup_localrc $old_or_new "$sub_localconf" "sub" local primary_node primary_node=$(cat /etc/nodepool/primary_node_private) @@ -126,14 +127,11 @@ function setup_multinode_connectivity { sub_nodes=$(cat /etc/nodepool/sub_nodes_private) if [[ "$DEVSTACK_GATE_NEUTRON" -ne '1' ]]; then setup_nova_net_networking $localrc $primary_node $sub_nodes - cat <>"$sub_localrc" -FLAT_INTERFACE=br_flat -PUBLIC_INTERFACE=br_pub -MULTI_HOST=True -EOF - cat <>"$localrc" -MULTI_HOST=True -EOF + localrc_set $sub_localconf "FLAT_INTERFACE" "br_flat" + localrc_set $sub_localconf "PUBLIC_INTERFACE" "br_pub" + localrc_set $sub_localconf "MULTI_HOST" "True" + # and on the master + localrc_set $localconf "MULTI_HOST" "True" elif [[ "$DEVSTACK_GATE_NEUTRON_DVR" -eq '1' ]]; then ovs_vxlan_bridge "br-ex" $primary_node "True" 1 \ $FLOATING_HOST_PREFIX $FLOATING_HOST_MASK \ @@ -191,10 +189,9 @@ EOF for NODE in $sub_nodes; do remote_copy_file /tmp/tmp_hosts $NODE:/tmp/tmp_hosts remote_command $NODE "cat /tmp/tmp_hosts | sudo tee --append /etc/hosts > /dev/null" - cp $sub_localrc /tmp/tmp_sub_localrc - echo "HOST_IP=$NODE" >> /tmp/tmp_sub_localrc - remote_copy_file /tmp/tmp_sub_localrc $NODE:$devstack_dir/localrc - remote_copy_file $localconf $NODE:$localconf + cp $sub_localconf /tmp/tmp_sub_localconf + localrc_set /tmp/tmp_sub_localconf "HOST_IP" "$NODE" + remote_copy_file /tmp/tmp_sub_localconf $NODE:$devstack_dir/local.conf done # NOTE(vsaienko) we need to have ssh connection among nodes to manage @@ -214,11 +211,12 @@ function setup_networking { # sauce to function. if [[ "$DEVSTACK_GATE_TOPOLOGY" != "multinode" ]] && \ [[ "$DEVSTACK_GATE_NEUTRON" -ne '1' ]]; then - local localrc=$BASE/new/devstack/localrc if [[ "$mode" == "grenade" ]]; then - localrc=$BASE/new/grenade/devstack.localrc + setup_nova_net_networking "$BASE/new/grenade/devstack.local.conf.base" "127.0.0.1" + setup_nova_net_networking "$BASE/new/grenade/devstack.local.conf.target" "127.0.0.1" + else + setup_nova_net_networking "$BASE/new/devstack/local.conf" "127.0.0.1" fi - setup_nova_net_networking "$localrc" "127.0.0.1" elif [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]]; then setup_multinode_connectivity $mode fi @@ -289,131 +287,129 @@ function setup_localrc { fi if [[ ! -z $DEVSTACK_GATE_USE_PYTHON3 ]] ; then - echo "USE_PYTHON3=$DEVSTACK_GATE_USE_PYTHON3" >>"$localrc_file" + localrc_set $localrc_file "USE_PYTHON3" "$DEVSTACK_GATE_USE_PYTHON3" fi if [[ "$DEVSTACK_GATE_CEPH" == "1" ]]; then - echo "CINDER_ENABLED_BACKENDS=ceph:ceph" >>"$localrc_file" - echo "TEMPEST_STORAGE_PROTOCOL=ceph" >>"$localrc_file" + localrc_set $localrc_file "CINDER_ENABLED_BACKENDS" "ceph:ceph" + localrc_set $localrc_file "TEMPEST_STORAGE_PROTOCOL" "ceph" fi # the exercises we *don't* want to test on for devstack SKIP_EXERCISES=boot_from_volume,bundle,client-env,euca if [[ "$DEVSTACK_GATE_NEUTRON" -eq "1" ]]; then - echo "Q_USE_DEBUG_COMMAND=True" >>"$localrc_file" - echo "NETWORK_GATEWAY=10.1.0.1" >>"$localrc_file" + localrc_set $localrc_file "Q_USE_DEBUG_COMMAND" "True" + localrc_set $localrc_file "NETWORK_GATEWAY" "10.1.0.1" fi if [[ "$DEVSTACK_GATE_NEUTRON_DVR" -eq "1" ]]; then if [[ "$DEVSTACK_GATE_TOPOLOGY" != "aio" ]] && [[ $role = sub ]]; then # The role for L3 agents running on compute nodes is 'dvr' - echo "Q_DVR_MODE=dvr" >>"$localrc_file" + localrc_set $localrc_file "Q_DVR_MODE" "dvr" else # The role for L3 agents running on controller nodes is 'dvr_snat' - echo "Q_DVR_MODE=dvr_snat" >>"$localrc_file" + localrc_set $localrc_file "Q_DVR_MODE" "dvr_snat" fi fi - cat <>"$localrc_file" -USE_SCREEN=False -DEST=$BASE/$localrc_oldnew -# move DATA_DIR outside of DEST to keep DEST a bit cleaner -DATA_DIR=$BASE/data -ACTIVE_TIMEOUT=90 -BOOT_TIMEOUT=90 -ASSOCIATE_TIMEOUT=60 -TERMINATE_TIMEOUT=60 -MYSQL_PASSWORD=secretmysql -DATABASE_PASSWORD=secretdatabase -RABBIT_PASSWORD=secretrabbit -ADMIN_PASSWORD=secretadmin -SERVICE_PASSWORD=secretservice -SERVICE_TOKEN=111222333444 -SWIFT_HASH=1234123412341234 -ROOTSLEEP=0 -# ERROR_ON_CLONE should never be set to FALSE in gate jobs. -# Setting up git trees must be done by zuul -# because it needs specific git references directly from gerrit -# to correctly do testing. Otherwise you are not testing -# the code you have posted for review. -ERROR_ON_CLONE=True -# Since git clone can't be used for novnc in gates, force it to install the packages -NOVNC_FROM_PACKAGE=True -ENABLED_SERVICES=$MY_ENABLED_SERVICES -SKIP_EXERCISES=$SKIP_EXERCISES -# Screen console logs will capture service logs. -SYSLOG=False -SCREEN_LOGDIR=$BASE/$localrc_oldnew/screen-logs -LOGFILE=$BASE/$localrc_oldnew/devstacklog.txt -VERBOSE=True -FIXED_RANGE=$FIXED_RANGE -IPV4_ADDRS_SAFE_TO_USE=$IPV4_ADDRS_SAFE_TO_USE -FLOATING_RANGE=$FLOATING_RANGE -PUBLIC_NETWORK_GATEWAY=$PUBLIC_NETWORK_GATEWAY -FIXED_NETWORK_SIZE=4096 -VIRT_DRIVER=$DEVSTACK_GATE_VIRT_DRIVER -SWIFT_REPLICAS=1 -LOG_COLOR=False -# Don't reset the requirements.txt files after g-r updates -UNDO_REQUIREMENTS=False -CINDER_PERIODIC_INTERVAL=10 -export OS_NO_CACHE=True -CEILOMETER_BACKEND=$DEVSTACK_GATE_CEILOMETER_BACKEND -LIBS_FROM_GIT=$DEVSTACK_PROJECT_FROM_GIT -# set this until all testing platforms have libvirt >= 1.2.11 -# see bug #1501558 -EBTABLES_RACE_FIX=True -EOF + localrc_set "$localrc_file" "USE_SCREEN" "False" + localrc_set "$localrc_file" "DEST" "$BASE/$localrc_oldnew" + # move DATA_DIR outside of DEST to keep DEST a bit cleaner + localrc_set "$localrc_file" "DATA_DIR" "$BASE/data" + localrc_set "$localrc_file" "ACTIVE_TIMEOUT" "90" + localrc_set "$localrc_file" "BOOT_TIMEOUT" "90" + localrc_set "$localrc_file" "ASSOCIATE_TIMEOUT" "60" + localrc_set "$localrc_file" "TERMINATE_TIMEOUT" "60" + localrc_set "$localrc_file" "MYSQL_PASSWORD" "secretmysql" + localrc_set "$localrc_file" "DATABASE_PASSWORD" "secretdatabase" + localrc_set "$localrc_file" "RABBIT_PASSWORD" "secretrabbit" + localrc_set "$localrc_file" "ADMIN_PASSWORD" "secretadmin" + localrc_set "$localrc_file" "SERVICE_PASSWORD" "secretservice" + localrc_set "$localrc_file" "SERVICE_TOKEN" "111222333444" + localrc_set "$localrc_file" "SWIFT_HASH" "1234123412341234" + localrc_set "$localrc_file" "ROOTSLEEP" "0" + # ERROR_ON_CLONE should never be set to FALSE in gate jobs. + # Setting up git trees must be done by zuul + # because it needs specific git references directly from gerrit + # to correctly do testing. Otherwise you are not testing + # the code you have posted for review. + localrc_set "$localrc_file" "ERROR_ON_CLONE" "True" + # Since git clone can't be used for novnc in gates, force it to install the packages + localrc_set "$localrc_file" "NOVNC_FROM_PACKAGE" "True" + localrc_set "$localrc_file" "ENABLED_SERVICES" "$MY_ENABLED_SERVICES" + localrc_set "$localrc_file" "SKIP_EXERCISES" "$SKIP_EXERCISES" + # Screen console logs will capture service logs. + localrc_set "$localrc_file" "SYSLOG" "False" + localrc_set "$localrc_file" "SCREEN_LOGDIR" "$BASE/$localrc_oldnew/screen-logs" + localrc_set "$localrc_file" "LOGFILE" "$BASE/$localrc_oldnew/devstacklog.txt" + localrc_set "$localrc_file" "VERBOSE" "True" + localrc_set "$localrc_file" "FIXED_RANGE" "$FIXED_RANGE" + localrc_set "$localrc_file" "IPV4_ADDRS_SAFE_TO_USE" "$IPV4_ADDRS_SAFE_TO_USE" + localrc_set "$localrc_file" "FLOATING_RANGE" "$FLOATING_RANGE" + localrc_set "$localrc_file" "PUBLIC_NETWORK_GATEWAY" "$PUBLIC_NETWORK_GATEWAY" + localrc_set "$localrc_file" "FIXED_NETWORK_SIZE" "4096" + localrc_set "$localrc_file" "VIRT_DRIVER" "$DEVSTACK_GATE_VIRT_DRIVER" + localrc_set "$localrc_file" "SWIFT_REPLICAS" "1" + localrc_set "$localrc_file" "LOG_COLOR" "False" + # Don't reset the requirements.txt files after g-r updates + localrc_set "$localrc_file" "UNDO_REQUIREMENTS" "False" + localrc_set "$localrc_file" "CINDER_PERIODIC_INTERVAL" "10" + localrc_set "$localrc_file" "export OS_NO_CACHE" "True" + localrc_set "$localrc_file" "CEILOMETER_BACKEND" "$DEVSTACK_GATE_CEILOMETER_BACKEND" + localrc_set "$localrc_file" "LIBS_FROM_GIT" "$DEVSTACK_PROJECT_FROM_GIT" + # set this until all testing platforms have libvirt >= 1.2.11 + # see bug #1501558 + localrc_set "$localrc_file" "EBTABLES_RACE_FIX" "True" if [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]] && [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then # Reduce the MTU on br-ex to match the MTU of underlying tunnels - echo "PUBLIC_BRIDGE_MTU=$EXTERNAL_BRIDGE_MTU" >>"$localrc_file" + localrc_set "$localrc_file" "PUBLIC_BRIDGE_MTU" "$EXTERNAL_BRIDGE_MTU" fi if [[ "$DEVSTACK_CINDER_SECURE_DELETE" -eq "0" ]]; then - echo "CINDER_SECURE_DELETE=False" >>"$localrc_file" + localrc_set "$localrc_file" "CINDER_SECURE_DELETE" "False" fi - echo "CINDER_VOLUME_CLEAR=${DEVSTACK_CINDER_VOLUME_CLEAR}" >>"$localrc_file" + localrc_set "$localrc_file" "CINDER_VOLUME_CLEAR" "${DEVSTACK_CINDER_VOLUME_CLEAR}" if [[ "$DEVSTACK_GATE_TEMPEST_HEAT_SLOW" -eq "1" ]]; then - echo "HEAT_CREATE_TEST_IMAGE=False" >>"$localrc_file" + localrc_set "$localrc_file" "HEAT_CREATE_TEST_IMAGE" "False" # Use Fedora 20 for heat test image, it has heat-cfntools pre-installed - echo "HEAT_FETCHED_TEST_IMAGE=Fedora-i386-20-20131211.1-sda" >>"$localrc_file" + localrc_set "$localrc_file" "HEAT_FETCHED_TEST_IMAGE" "Fedora-i386-20-20131211.1-sda" fi if [[ "$DEVSTACK_GATE_VIRT_DRIVER" == "libvirt" ]]; then if [[ -n "$DEVSTACK_GATE_LIBVIRT_TYPE" ]]; then - echo "LIBVIRT_TYPE=${DEVSTACK_GATE_LIBVIRT_TYPE}" >>"$localrc_file" + localrc_set "$localrc_file" "LIBVIRT_TYPE" "${DEVSTACK_GATE_LIBVIRT_TYPE}" fi fi if [[ "$DEVSTACK_GATE_VIRT_DRIVER" == "ironic" ]]; then export TEMPEST_OS_TEST_TIMEOUT=${DEVSTACK_GATE_OS_TEST_TIMEOUT:-1200} - echo "IRONIC_DEPLOY_DRIVER=$DEVSTACK_GATE_IRONIC_DRIVER" >>"$localrc_file" - echo "IRONIC_BAREMETAL_BASIC_OPS=True" >>"$localrc_file" - echo "IRONIC_VM_LOG_DIR=$BASE/$localrc_oldnew/ironic-bm-logs" >>"$localrc_file" - echo "DEFAULT_INSTANCE_TYPE=baremetal" >>"$localrc_file" - echo "BUILD_TIMEOUT=${DEVSTACK_GATE_TEMPEST_BAREMETAL_BUILD_TIMEOUT:-600}" >>"$localrc_file" - echo "IRONIC_CALLBACK_TIMEOUT=600" >>"$localrc_file" - echo "Q_AGENT=openvswitch" >>"$localrc_file" - echo "Q_ML2_TENANT_NETWORK_TYPE=vxlan" >>"$localrc_file" + localrc_set "$localrc_file" "IRONIC_DEPLOY_DRIVER" "$DEVSTACK_GATE_IRONIC_DRIVER" + localrc_set "$localrc_file" "IRONIC_BAREMETAL_BASIC_OPS" "True" + localrc_set "$localrc_file" "IRONIC_VM_LOG_DIR" "$BASE/$localrc_oldnew/ironic-bm-logs" + localrc_set "$localrc_file" "DEFAULT_INSTANCE_TYPE" "baremetal" + localrc_set "$localrc_file" "BUILD_TIMEOUT" "${DEVSTACK_GATE_TEMPEST_BAREMETAL_BUILD_TIMEOUT:-600}" + localrc_set "$localrc_file" "IRONIC_CALLBACK_TIMEOUT" "600" + localrc_set "$localrc_file" "Q_AGENT" "openvswitch" + localrc_set "$localrc_file" "Q_ML2_TENANT_NETWORK_TYPE" "vxlan" if [[ "$DEVSTACK_GATE_IRONIC_BUILD_RAMDISK" -eq 0 ]]; then - echo "IRONIC_BUILD_DEPLOY_RAMDISK=False" >>"$localrc_file" + localrc_set "$localrc_file" "IRONIC_BUILD_DEPLOY_RAMDISK" "False" else - echo "IRONIC_BUILD_DEPLOY_RAMDISK=True" >>"$localrc_file" + localrc_set "$localrc_file" "IRONIC_BUILD_DEPLOY_RAMDISK" "True" fi if [[ -z "${DEVSTACK_GATE_IRONIC_DRIVER%%agent*}" ]]; then - echo "SWIFT_ENABLE_TEMPURLS=True" >>"$localrc_file" - echo "SWIFT_TEMPURL_KEY=secretkey" >>"$localrc_file" - echo "IRONIC_ENABLED_DRIVERS=fake,agent_ssh,agent_ipmitool" >>"$localrc_file" + localrc_set "$localrc_file" "SWIFT_ENABLE_TEMPURLS" "True" + localrc_set "$localrc_file" "SWIFT_TEMPURL_KEY" "secretkey" + localrc_set "$localrc_file" "IRONIC_ENABLED_DRIVERS" "fake,agent_ssh,agent_ipmitool" # agent driver doesn't support ephemeral volumes yet - echo "IRONIC_VM_EPHEMERAL_DISK=0" >>"$localrc_file" + localrc_set "$localrc_file" "IRONIC_VM_EPHEMERAL_DISK" "0" # agent CoreOS ramdisk is a little heavy - echo "IRONIC_VM_SPECS_RAM=1024" >>"$localrc_file" + localrc_set "$localrc_file" "IRONIC_VM_SPECS_RAM" "1024" else - echo "IRONIC_ENABLED_DRIVERS=fake,pxe_ssh,pxe_ipmitool" >>"$localrc_file" - echo "IRONIC_VM_EPHEMERAL_DISK=1" >>"$localrc_file" + localrc_set "$localrc_file" "IRONIC_ENABLED_DRIVERS" "fake,pxe_ssh,pxe_ipmitool" + localrc_set "$localrc_file" "IRONIC_VM_EPHEMERAL_DISK" "1" fi fi @@ -422,44 +418,42 @@ EOF echo "XenAPI must have DEVSTACK_GATE_XENAPI_DOM0_IP, DEVSTACK_GATE_XENAPI_DOMU_IP and DEVSTACK_GATE_XENAPI_PASSWORD all set" exit 1 fi - cat >> "$localrc_file" << EOF -SKIP_EXERCISES=${SKIP_EXERCISES},volumes -XENAPI_PASSWORD=${DEVSTACK_GATE_XENAPI_PASSWORD} -XENAPI_CONNECTION_URL=http://${DEVSTACK_GATE_XENAPI_DOM0_IP} -VNCSERVER_PROXYCLIENT_ADDRESS=${DEVSTACK_GATE_XENAPI_DOM0_IP} -VIRT_DRIVER=xenserver - -# A separate xapi network is created with this name-label -FLAT_NETWORK_BRIDGE=vmnet - -# A separate xapi network on eth4 serves the purpose of the public network. -# This interface is added in Citrix's XenServer environment as an internal -# interface -PUBLIC_INTERFACE=eth4 - -# The xapi network "vmnet" is connected to eth3 in domU -# We need to explicitly specify these, as the devstack/xenserver driver -# sets GUEST_INTERFACE_DEFAULT -VLAN_INTERFACE=eth3 -FLAT_INTERFACE=eth3 - -# Explicitly set HOST_IP, so that it will be passed down to xapi, -# thus it will be able to reach glance -HOST_IP=${DEVSTACK_GATE_XENAPI_DOMU_IP} -SERVICE_HOST=${DEVSTACK_GATE_XENAPI_DOMU_IP} - -# Disable firewall -XEN_FIREWALL_DRIVER=nova.virt.firewall.NoopFirewallDriver - -# Disable agent -EXTRA_OPTS=("xenapi_disable_agent=True") - -# Add a separate device for volumes -VOLUME_BACKING_DEVICE=/dev/xvdb - -# Set multi-host config -MULTI_HOST=1 -EOF + localrc_set "$localrc_file" "SKIP_EXERCISES" "${SKIP_EXERCISES},volumes" + localrc_set "$localrc_file" "XENAPI_PASSWORD" "${DEVSTACK_GATE_XENAPI_PASSWORD}" + localrc_set "$localrc_file" "XENAPI_CONNECTION_URL" "http://${DEVSTACK_GATE_XENAPI_DOM0_IP}" + localrc_set "$localrc_file" "VNCSERVER_PROXYCLIENT_ADDRESS" "${DEVSTACK_GATE_XENAPI_DOM0_IP}" + localrc_set "$localrc_file" "VIRT_DRIVER" "xenserver" + + # A separate xapi network is created with this name-label + localrc_set "$localrc_file" "FLAT_NETWORK_BRIDGE" "vmnet" + + # A separate xapi network on eth4 serves the purpose of the public network. + # This interface is added in Citrix's XenServer environment as an internal + # interface + localrc_set "$localrc_file" "PUBLIC_INTERFACE" "eth4" + + # The xapi network "vmnet" is connected to eth3 in domU + # We need to explicitly specify these, as the devstack/xenserver driver + # sets GUEST_INTERFACE_DEFAULT + localrc_set "$localrc_file" "VLAN_INTERFACE" "eth3" + localrc_set "$localrc_file" "FLAT_INTERFACE" "eth3" + + # Explicitly set HOST_IP, so that it will be passed down to xapi, + # thus it will be able to reach glance + localrc_set "$localrc_file" "HOST_IP" "${DEVSTACK_GATE_XENAPI_DOMU_IP}" + localrc_set "$localrc_file" "SERVICE_HOST" "${DEVSTACK_GATE_XENAPI_DOMU_IP}" + + # Disable firewall + localrc_set "$localrc_file" "XEN_FIREWALL_DRIVER" "nova.virt.firewall.NoopFirewallDriver" + + # Disable agent + localrc_set "$localrc_file" "EXTRA_OPTS" "(\"xenapi_disable_agent=True\")" + + # Add a separate device for volumes + localrc_set "$localrc_file" "VOLUME_BACKING_DEVICE" "/dev/xvdb" + + # Set multi-host config + localrc_set "$localrc_file" "MULTI_HOST" "1" fi if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then @@ -469,81 +463,81 @@ EOF # # The 24G setting is expected to be enough even # in parallel run. - echo "VOLUME_BACKING_FILE_SIZE=24G" >> "$localrc_file" + localrc_set "$localrc_file" "VOLUME_BACKING_FILE_SIZE" "24G" # in order to ensure glance http tests don't time out, we # specify the TEMPEST_HTTP_IMAGE address that's in infra on a # service we need to be up for anything to work anyway. - echo "TEMPEST_HTTP_IMAGE=http://git.openstack.org/static/openstack.png" >> "$localrc_file" + localrc_set "$localrc_file" "TEMPEST_HTTP_IMAGE" "http://git.openstack.org/static/openstack.png" fi if [[ "$DEVSTACK_GATE_TEMPEST_DISABLE_TENANT_ISOLATION" -eq "1" ]]; then - echo "TEMPEST_ALLOW_TENANT_ISOLATION=False" >>"$localrc_file" + localrc_set "$localrc_file" "TEMPEST_ALLOW_TENANT_ISOLATION" "False" fi if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then if [[ "$localrc_oldnew" == "old" ]]; then - echo "GRENADE_PHASE=base" >> "$localrc_file" + localrc_set "$localrc_file" "GRENADE_PHASE" "base" else - echo "GRENADE_PHASE=target" >> "$localrc_file" + localrc_set "$localrc_file" "GRENADE_PHASE" "target" fi - echo "CEILOMETER_USE_MOD_WSGI=False" >> "$localrc_file" + localrc_set "$localrc_file" "CEILOMETER_USE_MOD_WSGI" "False" fi if [[ "$DEVSTACK_GATE_TEMPEST_LARGE_OPS" -eq "1" ]]; then # NOTE(danms): Temporary transition to =NUM_RESOURCES - echo "VIRT_DRIVER=fake" >> "$localrc_file" - echo "TEMPEST_LARGE_OPS_NUMBER=50" >>"$localrc_file" + localrc_set "$localrc_file" "VIRT_DRIVER" "fake" + localrc_set "$localrc_file" "TEMPEST_LARGE_OPS_NUMBER" "50" elif [[ "$DEVSTACK_GATE_TEMPEST_LARGE_OPS" -gt "1" ]]; then # use fake virt driver and 10 copies of nova-compute - echo "VIRT_DRIVER=fake" >> "$localrc_file" + localrc_set "$localrc_file" "VIRT_DRIVER" "fake" # To make debugging easier, disabled until bug 1218575 is fixed. # echo "NUMBER_FAKE_NOVA_COMPUTE=10" >>"$localrc_file" - echo "TEMPEST_LARGE_OPS_NUMBER=$DEVSTACK_GATE_TEMPEST_LARGE_OPS" >>"$localrc_file" + localrc_set "$localrc_file" "TEMPEST_LARGE_OPS_NUMBER" "$DEVSTACK_GATE_TEMPEST_LARGE_OPS" fi if [[ "$DEVSTACK_GATE_CONFIGDRIVE" -eq "1" ]]; then - echo "FORCE_CONFIG_DRIVE=True" >>"$localrc_file" + localrc_set "$localrc_file" "FORCE_CONFIG_DRIVE" "True" else - echo "FORCE_CONFIG_DRIVE=False" >>"$localrc_file" + localrc_set "$localrc_file" "FORCE_CONFIG_DRIVE" "False" fi if [[ "$CEILOMETER_NOTIFICATION_TOPICS" ]]; then # Add specified ceilometer notification topics to localrc # Set to notifications,profiler to enable profiling - echo "CEILOMETER_NOTIFICATION_TOPICS=$CEILOMETER_NOTIFICATION_TOPICS" >>"$localrc_file" + localrc_set "$localrc_file" "CEILOMETER_NOTIFICATION_TOPICS" "$CEILOMETER_NOTIFICATION_TOPICS" fi if [[ "$DEVSTACK_GATE_INSTALL_TESTONLY" -eq "1" ]]; then # Sometimes we do want the test packages - echo "INSTALL_TESTONLY_PACKAGES=True" >> "$localrc_file" + localrc_set "$localrc_file" "INSTALL_TESTONLY_PACKAGES" "True" fi if [[ "$DEVSTACK_GATE_TOPOLOGY" != "aio" ]]; then - echo "NOVA_ALLOW_MOVE_TO_SAME_HOST=False" >> "$localrc_file" - echo "LIVE_MIGRATION_AVAILABLE=True" >> "$localrc_file" - echo "USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION=True" >> "$localrc_file" + localrc_set "$localrc_file" "NOVA_ALLOW_MOVE_TO_SAME_HOST" "False" + localrc_set "$localrc_file" "LIVE_MIGRATION_AVAILABLE" "True" + localrc_set "$localrc_file" "USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION" "True" local primary_node=`cat /etc/nodepool/primary_node_private` - echo "SERVICE_HOST=$primary_node" >>"$localrc_file" + localrc_set "$localrc_file" "SERVICE_HOST" "$primary_node" if [[ "$role" = sub ]]; then if [[ $original_enabled_services =~ "qpid" ]]; then - echo "QPID_HOST=$primary_node" >>"$localrc_file" + localrc_set "$localrc_file" "QPID_HOST" "$primary_node" fi if [[ $original_enabled_services =~ "rabbit" ]]; then - echo "RABBIT_HOST=$primary_node" >>"$localrc_file" + localrc_set "$localrc_file" "RABBIT_HOST" "$primary_node" fi - echo "DATABASE_HOST=$primary_node" >>"$localrc_file" + localrc_set "$localrc_file" "DATABASE_HOST" "$primary_node" if [[ $original_enabled_services =~ "mysql" ]]; then - echo "DATABASE_TYPE=mysql" >>"$localrc_file" + localrc_set "$localrc_file" "DATABASE_TYPE" "mysql" else - echo "DATABASE_TYPE=postgresql" >>"$localrc_file" + localrc_set "$localrc_file" "DATABASE_TYPE" "postgresql" fi - echo "GLANCE_HOSTPORT=$primary_node:9292" >>"$localrc_file" - echo "Q_HOST=$primary_node" >>"$localrc_file" + localrc_set "$localrc_file" "GLANCE_HOSTPORT" "$primary_node:9292" + localrc_set "$localrc_file" "Q_HOST" "$primary_node" # Set HOST_IP in subnodes before copying localrc to each node else - echo "HOST_IP=$primary_node" >>"$localrc_file" + localrc_set "$localrc_file" "HOST_IP" "$primary_node" fi fi @@ -554,7 +548,7 @@ EOF # If we are in a multinode environment, we may want to specify 2 # different sets of plugins if [[ -n "$DEVSTACK_SUBNODE_CONFIG" ]]; then - echo "$DEVSTACK_SUBNODE_CONFIG" >>"$localrc_file" + localrc_set "$localrc_file" "$DEVSTACK_SUBNODE_CONFIG" >>"$localrc_file" else if [[ -n "$DEVSTACK_LOCAL_CONFIG" ]]; then echo "$DEVSTACK_LOCAL_CONFIG" >>"$localrc_file" @@ -582,11 +576,9 @@ function setup_access_for_stack_user { } if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then - cd $BASE/old/devstack - setup_localrc "old" "localrc" "primary" - - cd $BASE/new/devstack - setup_localrc "new" "localrc" "primary" + cd $BASE/new/grenade + setup_localrc "old" "devstack.local.conf.base" "primary" + setup_localrc "new" "devstack.local.conf.target" "primary" cat <$BASE/new/grenade/localrc BASE_RELEASE=old @@ -614,14 +606,13 @@ EOF if [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]]; then # ensure local.conf exists to remove conditional logic - touch local.conf if [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then - echo -e "[[post-config|\$NEUTRON_CONF]]\n[DEFAULT]\nglobal_physnet_mtu=$EXTERNAL_BRIDGE_MTU" >> local.conf + localconf_set "devstack.local.conf.base" "post-config" "\$NEUTRON_CONF" \ + "DEFAULT" "global_physnet_mtu" "$EXTERNAL_BRIDGE_MTU" + localconf_set "devstack.local.conf.target" "post-config" "\$NEUTRON_CONF" \ + "DEFAULT" "global_physnet_mtu" "$EXTERNAL_BRIDGE_MTU" fi - # get this in our base config - cp local.conf $BASE/old/devstack - # build the post-stack.sh config, this will be run as stack user so no sudo required cat > $BASE/new/grenade/post-stack.sh <> local.conf + localconf_set "local.conf" "post-config" "\$NEUTRON_CONF" \ + "DEFAULT" "global_physnet_mtu" "$EXTERNAL_BRIDGE_MTU" fi fi From 0ef46186dd26a5eecd104f7cf73dca5625ba0586 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Wed, 22 Feb 2017 10:47:49 -0500 Subject: [PATCH 141/148] install devstack-tools and use it for conf setting Instead of using devstack bash functions, which are error prone, us dsconf, which is much more tested for setting values in ini and local.conf. Change-Id: I565f5420743fbd95e9d744010c488953c54dd8a4 --- devstack-vm-gate-wrap.sh | 4 +++- devstack-vm-gate.sh | 29 +++++++++++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index f76177ae..8336a847 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -475,9 +475,11 @@ sudo -H pip install virtualenv virtualenv /tmp/ansible # NOTE(emilien): workaround to avoid installing cryptography # https://github.com/ansible/ansible/issues/15665 -/tmp/ansible/bin/pip install paramiko==1.16.0 ansible==$ANSIBLE_VERSION ara +/tmp/ansible/bin/pip install paramiko==1.16.0 ansible==$ANSIBLE_VERSION \ + devstack-tools==$DSTOOLS_VERSION ara export ANSIBLE=/tmp/ansible/bin/ansible export ANSIBLE_PLAYBOOK=/tmp/ansible/bin/ansible-playbook +export DSCONF=/tmp/ansible/bin/dsconf # Write inventory file with groupings COUNTER=1 diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index e1c99e64..13c61a8e 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -31,8 +31,19 @@ TOP_DIR=$(cd $(dirname "$0") && pwd) # Import common functions source $TOP_DIR/functions.sh # Get access to iniset and friends + +# NOTE(sdague): as soon as we put +# iniget into dsconf, we can remove this. source $BASE/new/devstack/inc/ini-config +# redefine localrc_set to use dsconf +function localrc_set { + local lcfile=$1 + local key=$2 + local value=$3 + $DSCONF setlc "$1" "$2" "$3" +} + echo $PPID > $WORKSPACE/gate.pid source `dirname "$(readlink -f "$0")"`/functions.sh @@ -548,15 +559,21 @@ function setup_localrc { # If we are in a multinode environment, we may want to specify 2 # different sets of plugins if [[ -n "$DEVSTACK_SUBNODE_CONFIG" ]]; then - localrc_set "$localrc_file" "$DEVSTACK_SUBNODE_CONFIG" >>"$localrc_file" + echo "[[local|localrc]]" > /tmp/ds-subnode-localrc + echo $DEVSTACK_SUBNODE_CONFIG >> /tmp/ds-subnode-localrc + $DSCONF merge_lc "$localrc_file" /tmp/ds-subnode-localrc else if [[ -n "$DEVSTACK_LOCAL_CONFIG" ]]; then - echo "$DEVSTACK_LOCAL_CONFIG" >>"$localrc_file" + echo "[[local|localrc]]" > /tmp/ds-localrc + echo $DEVSTACK_LOCAL_CONFIG >> /tmp/ds-localrc + $DSCONF merge_lc "$localrc_file" /tmp/ds-localrc fi fi else if [[ -n "$DEVSTACK_LOCAL_CONFIG" ]]; then - echo "$DEVSTACK_LOCAL_CONFIG" >>"$localrc_file" + echo "[[local|localrc]]" > /tmp/ds-localrc + echo $DEVSTACK_LOCAL_CONFIG >> /tmp/ds-localrc + $DSCONF merge_lc "$localrc_file" /tmp/ds-localrc fi fi @@ -607,9 +624,9 @@ EOF if [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]]; then # ensure local.conf exists to remove conditional logic if [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then - localconf_set "devstack.local.conf.base" "post-config" "\$NEUTRON_CONF" \ + $DSCONF setlc_conf "devstack.local.conf.base" "post-config" "\$NEUTRON_CONF" \ "DEFAULT" "global_physnet_mtu" "$EXTERNAL_BRIDGE_MTU" - localconf_set "devstack.local.conf.target" "post-config" "\$NEUTRON_CONF" \ + $DSCONF setlc_conf "devstack.local.conf.target" "post-config" "\$NEUTRON_CONF" \ "DEFAULT" "global_physnet_mtu" "$EXTERNAL_BRIDGE_MTU" fi @@ -749,7 +766,7 @@ if [[ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]]; then # guarantees that tempest should be configured, no matter should # tests be executed or not. if [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]]; then - iniset -sudo $BASE/new/tempest/etc/tempest.conf compute min_compute_nodes 2 + sudo $DSCONF iniset $BASE/new/tempest/etc/tempest.conf compute min_compute_nodes 2 fi # if set, we don't need to run Tempest at all From 37065f778967fcc7430a8887e28807c016151f39 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Tue, 7 Feb 2017 15:00:42 -0500 Subject: [PATCH 142/148] Support local_conf from project-config This adds the support for merging in local_conf from project config into local.conf files deployed in jobs. Change-Id: I81616a5b381fe203aef47628b52371609db5e5eb --- devstack-vm-gate.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 13c61a8e..ff9a6c75 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -552,6 +552,28 @@ function setup_localrc { fi fi + # If you specify a section of a project-config job with + # + # local_conf: + # conf: | + # [[local|localrc]] + # foo=a + # [[post-config|$NEUTRON_CONF]] + # [DEFAULT] + # global_physnet_mtu = 1400 + # + # Then that whole local.conf fragment will get carried through to + # this special file, and we'll merge those values into *all* + # local.conf files in the job. That includes subnodes, and new & + # old in grenade. + # + # NOTE(sdague): the name of this file should be considered + # internal only, and jobs should not write to it directly, they + # should only use the project-config stanza. + if [[ -e "/tmp/dg-local.conf" ]]; then + $DSCONF merge_lc "$localrc_file" "/tmp/dg-local.conf" + fi + # a way to pass through arbitrary devstack config options so that # we don't need to add new devstack-gate options every time we # want to create a new config. From b9154881aa0627be4824c8107c510d92af6d1af7 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Thu, 23 Feb 2017 10:45:41 -0500 Subject: [PATCH 143/148] use setlc_raw to inject DEVSTACK_LOCAL_CONFIG into files This was actually a suggestion on the way to do this originally, and it helps with the fact that we don't have to deal with bash handling of line endings. Change-Id: Ie64b9a20adccdb19352a6cc8fa402acc4c3f5e5b --- devstack-vm-gate.sh | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 13c61a8e..290484b8 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -559,21 +559,15 @@ function setup_localrc { # If we are in a multinode environment, we may want to specify 2 # different sets of plugins if [[ -n "$DEVSTACK_SUBNODE_CONFIG" ]]; then - echo "[[local|localrc]]" > /tmp/ds-subnode-localrc - echo $DEVSTACK_SUBNODE_CONFIG >> /tmp/ds-subnode-localrc - $DSCONF merge_lc "$localrc_file" /tmp/ds-subnode-localrc + $DSCONF setlc_raw "$localrc_file" "$DEVSTACK_SUBNODE_CONFIG" else if [[ -n "$DEVSTACK_LOCAL_CONFIG" ]]; then - echo "[[local|localrc]]" > /tmp/ds-localrc - echo $DEVSTACK_LOCAL_CONFIG >> /tmp/ds-localrc - $DSCONF merge_lc "$localrc_file" /tmp/ds-localrc + $DSCONF setlc_raw "$localrc_file" "$DEVSTACK_LOCAL_CONFIG" fi fi else if [[ -n "$DEVSTACK_LOCAL_CONFIG" ]]; then - echo "[[local|localrc]]" > /tmp/ds-localrc - echo $DEVSTACK_LOCAL_CONFIG >> /tmp/ds-localrc - $DSCONF merge_lc "$localrc_file" /tmp/ds-localrc + $DSCONF setlc_raw "$localrc_file" "$DEVSTACK_LOCAL_CONFIG" fi fi From e7d4e5c81bfcd2341beb26059f4c9e62fe5f7174 Mon Sep 17 00:00:00 2001 From: Masayuki Igawa Date: Wed, 22 Feb 2017 15:58:03 -0500 Subject: [PATCH 144/148] Fix dstat log file path for stackviz This commit fixes dstat log file path for stackviz. stackviz has a feature to illustrate a dstat graphs. However it doesn't work because the dstat log path is wrong. Change-Id: I510921cf926fe49065d03c161b0a9735bb772908 --- functions.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions.sh b/functions.sh index fb037249..77f20b2b 100644 --- a/functions.sh +++ b/functions.sh @@ -608,10 +608,10 @@ function process_stackviz { cp -r $stackviz_path/build $log_path/stackviz pushd $project_path - if [ -f $BASE/new/dstat-csv.txt ]; then + if [ -f $log_path/dstat-csv_log.txt ]; then sudo testr last --subunit | stackviz-export \ - --dstat $BASE/new/dstat-csv.txt \ - --end --stdin \ + --dstat $log_path/dstat-csv_log.txt \ + --env --stdin \ $log_path/stackviz/data else sudo testr last --subunit | stackviz-export \ From 28c7b5b2cef90d753dde7af57c648f1b93b90c06 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Thu, 23 Feb 2017 14:51:24 -0500 Subject: [PATCH 145/148] Fix creating the sublocal.conf for ironic During the rebasing we apparently missed ironic for the sublocalrc. This fixes this with the localrc_set calls instead of heredocs. Change-Id: I85d63fadea49096f48406297ec07d2111e5dfe93 --- devstack-vm-gate.sh | 48 ++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 290484b8..ff706bef 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -117,17 +117,20 @@ function setup_multinode_connectivity { # ``old_or_new`` - should the subnodes be computed on the old side # or new side. For grenade where we don't upgrade them, calculate # on the old side. - local localrc=$BASE/new/devstack/localrc local old_or_new="new" + local localconf + local devstack_dir if [[ "$mode" == "grenade" ]]; then - localrc=$BASE/new/grenade/devstack.localrc + localconf=$BASE/new/grenade/devstack.localrc old_or_new="old" + devstack_dir=$BASE/$old_or_new/devstack + else + devstack_dir=$BASE/$old_or_new/devstack + localconf=$devstack_dir/local.conf fi # set explicit paths on all conf files we're writing so that # current working directory doesn't introduce subtle bugs. - local devstack_dir=$BASE/$old_or_new/devstack local sub_localconf=$devstack_dir/sub_local.conf - local localconf=$devstack_dir/local.conf set -x # for now enabling debug and do not turn it off setup_localrc $old_or_new "$sub_localconf" "sub" @@ -137,7 +140,7 @@ function setup_multinode_connectivity { local sub_nodes sub_nodes=$(cat /etc/nodepool/sub_nodes_private) if [[ "$DEVSTACK_GATE_NEUTRON" -ne '1' ]]; then - setup_nova_net_networking $localrc $primary_node $sub_nodes + setup_nova_net_networking $localconf $primary_node $sub_nodes localrc_set $sub_localconf "FLAT_INTERFACE" "br_flat" localrc_set $sub_localconf "PUBLIC_INTERFACE" "br_pub" localrc_set $sub_localconf "MULTI_HOST" "True" @@ -155,25 +158,22 @@ function setup_multinode_connectivity { ovs_vxlan_bridge "br_ironic_vxlan" $primary_node "False" 128 \ $sub_nodes - cat <>"$sub_localrc" -HOST_TOPOLOGY=multinode -HOST_TOPOLOGY_ROLE=subnode -# NOTE(vsaienko) we assume for now that we using only 1 subnode, -# each subnode should have different switch name (bridge) as it is used -# by networking-generic-switch to uniquely identify switch. -IRONIC_VM_NETWORK_BRIDGE=sub1brbm -OVS_PHYSICAL_BRIDGE=sub1brbm -ENABLE_TENANT_TUNNELS=False -IRONIC_KEY_FILE="$BASE/new/.ssh/ironic_key" -EOF - cat <>"$localrc" -HOST_TOPOLOGY=multinode -HOST_TOPOLOGY_ROLE=primary -HOST_TOPOLOGY_SUBNODES="$sub_nodes" -IRONIC_KEY_FILE="$BASE/new/.ssh/ironic_key" -GENERIC_SWITCH_KEY_FILE="$BASE/new/.ssh/ironic_key" -ENABLE_TENANT_TUNNELS=False -EOF + localrc_set "$sub_localconf" "HOST_TOPOLOGY" "multinode" + localrc_set "$sub_localconf" "HOST_TOPOLOGY_ROLE" "subnode" + # NOTE(vsaienko) we assume for now that we using only 1 subnode, + # each subnode should have different switch name (bridge) as it is used + # by networking-generic-switch to uniquely identify switch. + localrc_set "$sub_localconf" "IRONIC_VM_NETWORK_BRIDGE" "sub1brbm" + localrc_set "$sub_localconf" "OVS_PHYSICAL_BRIDGE" "sub1brbm" + localrc_set "$sub_localconf" "ENABLE_TENANT_TUNNELS" "False" + localrc_set "$sub_localconf" "IRONIC_KEY_FILE" "$BASE/new/.ssh/ironic_key" + + localrc_set "$localconf" "HOST_TOPOLOGY" "multinode" + localrc_set "$localconf" "HOST_TOPOLOGY_ROLE" "primary" + localrc_set "$localconf" "HOST_TOPOLOGY_SUBNODES" "$sub_nodes" + localrc_set "$localconf" "IRONIC_KEY_FILE" "$BASE/new/.ssh/ironic_key" + localrc_set "$localconf" "GENERIC_SWITCH_KEY_FILE" "$BASE/new/.ssh/ironic_key" + localrc_set "$localconf" "ENABLE_TENANT_TUNNELS" "False" fi echo "Preparing cross node connectivity" From 24a6ed073b547fbbd484157e544b4bc10dda8880 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Mon, 27 Feb 2017 10:44:09 -0500 Subject: [PATCH 146/148] bump dstools to 0.3.0 This includes a fix that ensures that setlc works correctly if the local.conf file doesn't have a [[local|localrc]] section at all. That was tripping up some existing 3rd party CI systems. Change-Id: I8f855ec627c2a28db755c52ab590fccb8ef55b5d --- devstack-vm-gate-wrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 8336a847..25a522c3 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -29,7 +29,7 @@ GIT_BRANCH=${GIT_BRANCH:-master} # possible that new ansible releases can break us. As such we should # be very deliberate about which ansible we use. ANSIBLE_VERSION=${ANSIBLE_VERSION:-2.2.0.0} -export DSTOOLS_VERSION=${DSTOOLS_VERSION:-0.2.1} +export DSTOOLS_VERSION=${DSTOOLS_VERSION:-0.3.0} # sshd may have been compiled with a default path excluding */sbin export PATH=$PATH:/usr/local/sbin:/usr/sbin From f4c9f1049acaf7bf39ff325c075f1dd6bdc7906c Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Fri, 10 Feb 2017 05:47:44 +0000 Subject: [PATCH 147/148] Gather /proc/cpuinfo This information was once collected for gate jobs, but was removed by I916e075d8ed3ee6079e4c4621f6eb10f1d30bf95 rework. Turned out that the info collected by ansible itself does not provide some crucial data, like cpu flags supported by the machine that may become handy when debugging some performance related gate issues. This patch restores collecting the info, using ansible. Change-Id: Icd2bc68eb723ba2f3fc0d8a113fb247bc478121a --- playbooks/roles/gather_host_info/tasks/main.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playbooks/roles/gather_host_info/tasks/main.yaml b/playbooks/roles/gather_host_info/tasks/main.yaml index bccc6895..09d599c6 100644 --- a/playbooks/roles/gather_host_info/tasks/main.yaml +++ b/playbooks/roles/gather_host_info/tasks/main.yaml @@ -1,4 +1,9 @@ --- +# this is what prints the facts to the logs - debug: var=hostvars[inventory_hostname] + - command: locale name: "Gather locale" + +- command: cat /proc/cpuinfo + name: "Gather kernel cpu info" From 8b2ee25eeb37e3de1780c59056a1e9307d48ef08 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Tue, 28 Feb 2017 08:02:50 -0500 Subject: [PATCH 148/148] Support setting a variable to get local.conf merged late The current mechanisms for changing local.conf make it so that they all happen before jobs get running. However projects like neutron do some rather complicated per job configuration that has to happen on top of devstack-gate configs. So expose a way for this to merge late. This does remain a bit error prone because you are unaware of what is being configured in devstack-gate first. It might be better to make local.conf configuration either d-g or project oriented. Change-Id: I53119876b82eeaec19382dbe115cb1740bed0cec --- devstack-vm-gate.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 38de526c..2245b83d 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -577,6 +577,10 @@ function setup_localrc { # a way to pass through arbitrary devstack config options so that # we don't need to add new devstack-gate options every time we # want to create a new config. + # + # NOTE(sdague): this assumes these are old school "localrc" + # sections, we should probably figure out a way to warn over using + # these. if [[ "$role" = sub ]]; then # If we are in a multinode environment, we may want to specify 2 # different sets of plugins @@ -593,6 +597,17 @@ function setup_localrc { fi fi + # NOTE(sdague): new style local.conf declarations which need to + # merge late. Projects like neutron build up a lot of scenarios + # based on this, but they have to apply them late. + # + # TODO(sdague): subnode support. + if [[ -n "$DEVSTACK_LOCALCONF" ]]; then + local ds_conf_late="/tmp/ds-conf-late.conf" + echo "$DEVSTACK_LOCALCONF" > "$ds_conf_late" + $DSCONF merge_lc "$localrc_file" "$ds_conf_late" + fi + } # This makes the stack user own the $BASE files and also changes the