diff --git a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb index c2bdf0930f..df96cb2f65 100644 --- a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb +++ b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb @@ -19,6 +19,34 @@ # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the # bash block to a recipe that uses the pyenv resource. +command = "pip install . --no-build-isolation" + +dependency_package_name = "pypi-node-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}" +dependency_folder_name = dependency_package_name +if platform?('amazon') && node['platform_version'] == "2" + dependency_package_name = "node-dependencies" + dependency_folder_name = "node" +end + +remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz" + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing +end + +bash 'pip install' do + user 'root' + group 'root' + cwd "#{node['cluster']['base_dir']}" + code <<-REQ + set -e + tar xzf node-dependencies.tgz + cd #{dependency_folder_name} + #{node_virtualenv_path}/bin/pip install * -f ./ --no-index + REQ +end bash "install custom aws-parallelcluster-node" do cwd Chef::Config[:file_cache_path] @@ -38,7 +66,7 @@ mkdir aws-parallelcluster-custom-node tar -xzf aws-parallelcluster-node.tgz --directory aws-parallelcluster-custom-node cd aws-parallelcluster-custom-node/*aws-parallelcluster-node* - pip install . + #{command} deactivate NODE end diff --git a/cookbooks/aws-parallelcluster-computefleet/spec/unit/recipes/custom_parallelcluster_node_spec.rb b/cookbooks/aws-parallelcluster-computefleet/spec/unit/recipes/custom_parallelcluster_node_spec.rb new file mode 100644 index 0000000000..41bbccfbcf --- /dev/null +++ b/cookbooks/aws-parallelcluster-computefleet/spec/unit/recipes/custom_parallelcluster_node_spec.rb @@ -0,0 +1,91 @@ +require 'spec_helper' + +describe 'aws-parallelcluster-computefleet::custom_parallelcluster_node' do + for_all_oses do |platform, version| + context "on #{platform}#{version}" do + cached(:s3_url) { 's3://url' } + cached(:base_dir) { 'base_dir' } + cached(:arch) { 'x86_64' } + cached(:region) { 'any-region' } + cached(:python_version) { 'python_version' } + cached(:dependency_pkg_name_suffix) do + if platform == 'amazon' && version == '2' + 'node-dependencies' + else + "pypi-node-dependencies-#{python_version}-#{arch}" + end + end + cached(:dependency_folder_name_suffix) do + if platform == 'amazon' && version == '2' + "node" + else + dependency_pkg_name_suffix + end + end + cached(:virtualenv_path) { "#{base_dir}/pyenv/versions/#{python_version}/envs/node_virtualenv" } + cached(:cookbook_virtualenv_path) { "#{base_dir}/pyenv/versions/#{python_version}/envs/cookbook_virtualenv" } + cached(:custom_node_s3_url) { "#{s3_url}/pyenv/versions/#{python_version}/envs/node_virtualenv" } + cached(:pip_install_bash_code) do + <<-REQ + set -e + tar xzf node-dependencies.tgz + cd #{dependency_folder_name_suffix} + #{virtualenv_path}/bin/pip install * -f ./ --no-index + REQ + end + cached(:node_bash_code) do + <<-NODE + set -e + [[ ":$PATH:" != *":/usr/local/bin:"* ]] && PATH="/usr/local/bin:${PATH}" + echo "PATH is $PATH" + source #{virtualenv_path}/bin/activate + pip uninstall --yes aws-parallelcluster-node + if [[ "#{custom_node_s3_url}" =~ ^s3:// ]]; then + custom_package_url=$(#{cookbook_virtualenv_path}/bin/aws s3 presign #{custom_node_s3_url} --region #{region}) + else + custom_package_url=#{custom_node_s3_url} + fi + curl --retry 3 -L -o aws-parallelcluster-node.tgz ${custom_package_url} + rm -fr aws-parallelcluster-custom-node + mkdir aws-parallelcluster-custom-node + tar -xzf aws-parallelcluster-node.tgz --directory aws-parallelcluster-custom-node + cd aws-parallelcluster-custom-node/*aws-parallelcluster-node* + pip install . --no-build-isolation + deactivate + NODE + end + cached(:chef_run) do + runner = runner(platform: platform, version: version) do |node| + node.override['kernel']['machine'] = arch + node.override['cluster']['python-major-minor-version'] = python_version + node.override['cluster']['python-version'] = python_version + node.override['cluster']['base_dir'] = base_dir + node.override['cluster']['region'] = region + node.override['cluster']['artifacts_s3_url'] = s3_url + node.override['cluster']['custom_node_package'] = custom_node_s3_url + end + allow(File).to receive(:exist?).with("#{virtualenv_path}/bin/activate").and_return(true) + runner.converge(described_recipe) + end + + it 'downloads tarball' do + is_expected.to create_if_missing_remote_file("base_dir/node-dependencies.tgz") + .with(source: "#{s3_url}/dependencies/PyPi/#{arch}/#{dependency_pkg_name_suffix}.tgz") + .with(mode: '0644') + .with(retries: 3) + .with(retry_delay: 5) + end + + it 'pip installs' do + is_expected.to run_bash('pip install') + .with(cwd: base_dir) + .with(code: pip_install_bash_code.gsub(/^ /, ' ')) + end + + it 'install custom aws-parallelcluster-node' do + is_expected.to run_bash('install custom aws-parallelcluster-node') + .with(code: node_bash_code.gsub(/^ /, ' ')) + end + end + end +end diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index 3985418b74..4d504a8621 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -33,6 +33,33 @@ not_if { ::File.exist?("#{virtualenv_path}/bin/activate") } end +dependency_package_name = "pypi-cfn-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}" +dependency_folder_name = dependency_package_name +if platform?('amazon') && node['platform_version'] == "2" + dependency_package_name = "cfn-dependencies" + dependency_folder_name = "cfn" +end + +remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz" + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing +end + +bash 'pip install' do + user 'root' + group 'root' + cwd "#{node['cluster']['base_dir']}" + code <<-REQ + set -e + tar xzf cfn-dependencies.tgz + cd #{dependency_folder_name} + #{virtualenv_path}/bin/pip install * -f ./ --no-index + REQ +end + cfnbootstrap_version = '2.0-33' cfnbootstrap_package = "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" @@ -51,11 +78,13 @@ retry_delay 5 end +command = "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package} --no-build-isolation" + bash "Install CloudFormation helpers from #{cfnbootstrap_package}" do user 'root' group 'root' cwd '/tmp' - code "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}" + code command creates "#{virtualenv_path}/bin/cfn-hup" end diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb index b2ab45bc1b..6af950560c 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb @@ -21,3 +21,55 @@ use 'partial/_redhat_based' use 'partial/_install_from_tar' use 'partial/_mount_umount' + +def adc_install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) + <<-EFSUTILSINSTALL + set -e + tar xf #{efs_utils_tarball} + mv efs-proxy-dependencies-#{efs_utils_version}.tar.gz efs-utils-#{efs_utils_version}/src/proxy/ + cd efs-utils-#{efs_utils_version}/src/proxy/ + tar -xf efs-proxy-dependencies-#{efs_utils_version}.tar.gz + cargo build --offline + cd ../.. + make rpm + yum -y install ./build/#{efs_utils_package}*rpm + EFSUTILSINSTALL +end + +def prerequisites + %w(rpm-build make rust cargo openssl-devel) +end + +action :install_efs_utils do + package_name = "amazon-efs-utils" + package_version = _efs_utils_version + efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" + + if aws_region.start_with?("us-iso") + + efs_proxy_deps = "efs-proxy-dependencies-#{package_version}.tar.gz" + efs_proxy_deps_tarball = "#{node['cluster']['sources_dir']}/#{efs_proxy_deps}" + efs_proxy_deps_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/#{efs_proxy_deps}" + remote_file efs_proxy_deps_tarball do + source efs_proxy_deps_url + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing + end + + bash "install efs utils" do + cwd node['cluster']['sources_dir'] + code adc_install_script_code(efs_utils_tarball, package_name, package_version) + end + + else + # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html + bash "install efs utils" do + cwd node['cluster']['sources_dir'] + code install_script_code(efs_utils_tarball, package_name, package_version) + end + end + + action_increase_poll_interval +end diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb index 618e91d3fb..612edabe20 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb @@ -2,15 +2,12 @@ default_action :install_utils -property :efs_utils_version, String -property :efs_utils_checksum, String - def _efs_utils_version - efs_utils_version || node['cluster']['efs']['version'] + node['cluster']['efs']['version'] end def _efs_utils_checksum - efs_utils_checksum || node['cluster']['efs']['sha256'] + node['cluster']['efs']['sha256'] end def already_installed?(package_name, expected_version) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index dea1449ac1..2337b33917 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -12,8 +12,13 @@ # or in the "LICENSE.txt" file accompanying this file. # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. +package_name = "amazon-efs-utils" action :install_utils do + package_version = _efs_utils_version + efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" + efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" + package_repos 'update package repositories' do action :update end @@ -29,11 +34,6 @@ return if redhat_on_docker? - package_name = "amazon-efs-utils" - package_version = _efs_utils_version - efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" - efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" - # Do not install efs-utils if a same or newer version is already installed. return if already_installed?(package_name, package_version) @@ -50,10 +50,22 @@ action :create_if_missing end + action_install_efs_utils + + # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html + # bash "install efs utils" do + # cwd node['cluster']['sources_dir'] + # code install_script_code(efs_utils_tarball, package_name, package_version) + # end + action_increase_poll_interval +end + +action :install_efs_utils do + package_version = _efs_utils_version + efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html bash "install efs utils" do cwd node['cluster']['sources_dir'] code install_script_code(efs_utils_tarball, package_name, package_version) end - action_increase_poll_interval end diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb index 1e47a13200..274f3d41cf 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb @@ -4,18 +4,46 @@ for_all_oses do |platform, version| context "on #{platform}#{version}" do cached(:cfnbootstrap_version) { '2.0-33' } - cached(:cfnbootstrap_package) { "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" } + cached(:arch) { 'x86_64' } + cached(:s3_url) { 's3://url' } + cached(:base_dir) { 'base_dir' } cached(:python_version) { "#{node['cluster']['python-version']}" } + cached(:dependecy_package_name_suffix) do + if platform == 'amazon' && version == '2' + "cfn-dependencies" + else + "pypi-cfn-dependencies-#{node['cluster']['python-major-minor-version']}-#{arch}" + end + end + cached(:dependecy_folder_name) do + if platform == 'amazon' && version == '2' + "cfn" + else + dependecy_package_name_suffix + end + end + cached(:cfnbootstrap_package) { "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" } cached(:system_pyenv_root) { 'system_pyenv_root' } cached(:virtualenv_path) { "system_pyenv_root/versions/#{python_version}/envs/cfn_bootstrap_virtualenv" } cached(:timeout) { 1800 } + cached(:dependency_bash_code) do + <<-REQ + set -e + tar xzf cfn-dependencies.tgz + cd #{dependecy_folder_name} + #{virtualenv_path}/bin/pip install * -f ./ --no-index + REQ + end context "when cfn_bootstrap virtualenv not installed yet" do cached(:chef_run) do runner = runner(platform: platform, version: version) do |node| node.override['cluster']['system_pyenv_root'] = system_pyenv_root node.override['cluster']['region'] = 'non_china' + node.override['cluster']['base_dir'] = base_dir node.override['cluster']['compute_node_bootstrap_timeout'] = timeout + node.override['cluster']['artifacts_s3_url'] = s3_url + node.override['kernel']['machine'] = arch end runner.converge(described_recipe) end @@ -37,6 +65,22 @@ is_expected.to write_node_attributes('dump node attributes') end + it 'downloads cfn_dependecies package from s3' do + is_expected.to create_if_missing_remote_file("#{base_dir}/cfn-dependencies.tgz") + .with(source: "#{s3_url}/dependencies/PyPi/#{arch}/#{dependecy_package_name_suffix}.tgz") + .with(mode: '0644') + .with(retries: 3) + .with(retry_delay: 5) + end + + it 'pip installs dependencies' do + is_expected.to run_bash('pip install') + .with(user: 'root') + .with(group: 'root') + .with(cwd: base_dir) + .with(code: dependency_bash_code) + end + it 'downloads cfn_bootstrap package from s3' do is_expected.to create_remote_file("/tmp/#{cfnbootstrap_package}").with( source: "https://s3.amazonaws.com/cloudformation-examples/#{cfnbootstrap_package}" @@ -48,7 +92,7 @@ user: 'root', group: 'root', cwd: '/tmp', - code: "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}", + code: "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package} --no-build-isolation", creates: "#{virtualenv_path}/bin/cfn-hup" ) end diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb index 738999a3eb..8ad07f0644 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb @@ -1,11 +1,9 @@ require 'spec_helper' class ConvergeEfs - def self.install_utils(chef_run, efs_utils_version:, tarball_checksum:) + def self.install_utils(chef_run) chef_run.converge_dsl('aws-parallelcluster-environment') do efs 'install_utils' do - efs_utils_checksum tarball_checksum - efs_utils_version efs_utils_version action :install_utils end end @@ -28,14 +26,17 @@ def mock_already_installed(package, expected_version, installed) context "on amazon2" do cached(:efs_utils_version) { '1.2.3' } cached(:tarball_checksum) { 'tarball_checksum' } - let(:chef_run) do - runner(platform: 'amazon', version: '2', step_into: ['efs']) - end context "when same version of amazon-efs-utils already installed" do + cached(:chef_run) do + runner(platform: 'amazon', version: '2', step_into: ['efs']) + end + cached(:node) { chef_run.node } before do + node.override['cluster']['efs']['version'] = efs_utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum mock_get_package_version('amazon-efs-utils', efs_utils_version) - ConvergeEfs.install_utils(chef_run, efs_utils_version: efs_utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(chef_run) end it 'does not install amazon-efs-utils' do @@ -44,9 +45,13 @@ def mock_already_installed(package, expected_version, installed) end context "when newer version of amazon-efs-utils already installed" do + cached(:chef_run) do + runner(platform: 'amazon', version: '2', step_into: ['efs']) + end + cached(:node) { chef_run.node } before do mock_get_package_version('amazon-efs-utils', '1.3.2') - ConvergeEfs.install_utils(chef_run, efs_utils_version: efs_utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(chef_run) end it 'does not install amazon-efs-utils' do @@ -55,9 +60,15 @@ def mock_already_installed(package, expected_version, installed) end context "when amazon-efs-utils not installed" do + cached(:chef_run) do + runner(platform: 'amazon', version: '2', step_into: ['efs']) + end + cached(:node) { chef_run.node } before do + node.override['cluster']['efs']['version'] = efs_utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum mock_get_package_version('amazon-efs-utils', '') - ConvergeEfs.install_utils(chef_run, efs_utils_version: efs_utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(chef_run) end it 'installs amazon-efs-utils' do @@ -67,9 +78,15 @@ def mock_already_installed(package, expected_version, installed) end context "when older version of amazon-efs-utils installed" do + cached(:chef_run) do + runner(platform: 'amazon', version: '2', step_into: ['efs']) + end + cached(:node) { chef_run.node } before do + node.override['cluster']['efs']['version'] = efs_utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum mock_get_package_version('amazon-efs-utils', '1.1.4') - ConvergeEfs.install_utils(chef_run, efs_utils_version: efs_utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(chef_run) end it 'installs amazon-efs-utils' do @@ -105,8 +122,10 @@ def mock_already_installed(package, expected_version, installed) node.override['cluster']['efs_utils']['tarball_path'] = tarball_path node.override['cluster']['sources_dir'] = source_dir node.override['cluster']['region'] = aws_region + node.override['cluster']['efs']['version'] = utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum end - ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(runner) end cached(:node) { chef_run.node } @@ -140,8 +159,10 @@ def mock_already_installed(package, expected_version, installed) runner = runner(platform: platform, version: version, step_into: ['efs']) do |node| node.override['cluster']['efs_utils']['tarball_path'] = tarball_path node.override['cluster']['sources_dir'] = source_dir + node.override['cluster']['efs']['version'] = utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum end - ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(runner) end cached(:node) { chef_run.node } @@ -190,9 +211,11 @@ def mock_already_installed(package, expected_version, installed) runner = runner(platform: platform, version: version, step_into: ['efs']) do |node| node.override['cluster']['efs_utils']['tarball_path'] = tarball_path node.override['cluster']['sources_dir'] = source_dir + node.override['cluster']['efs']['version'] = utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum node.override['cluster']['region'] = aws_region end - ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(runner) end it 'creates sources dir' do @@ -230,9 +253,11 @@ def mock_already_installed(package, expected_version, installed) mock_already_installed('amazon-efs-utils', utils_version, true) runner = runner(platform: platform, version: version, step_into: ['efs']) do |node| node.override['cluster']['efs_utils']['tarball_path'] = tarball_path + node.override['cluster']['efs']['version'] = utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum node.override['cluster']['sources_dir'] = source_dir end - ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(runner) end it 'does not download tarball' do diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb index cdc557df5f..e2063eb56f 100644 --- a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb +++ b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb @@ -12,8 +12,10 @@ # limitations under the License. virtualenv_path = cookbook_virtualenv_path -pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz" +dependency_package_name = "pypi-cookbook-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}" +pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz" if platform?('amazon') && node['platform_version'] == "2" + dependency_package_name = "dependencies" pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz" end @@ -44,9 +46,9 @@ group 'root' cwd "#{node['cluster']['base_dir']}" code <<-REQ - set -e - tar xzf cookbook-dependencies.tgz - cd dependencies - #{virtualenv_path}/bin/pip install * -f ./ --no-index - REQ + set -e + tar xzf cookbook-dependencies.tgz + cd #{dependency_package_name} + #{virtualenv_path}/bin/pip install * -f ./ --no-index + REQ end diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_imex/partial/_nvidia_imex_common.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_imex/partial/_nvidia_imex_common.rb index f791eb5f1d..454ea6e99b 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_imex/partial/_nvidia_imex_common.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_imex/partial/_nvidia_imex_common.rb @@ -17,7 +17,7 @@ action :install do return unless nvidia_enabled_or_installed? - return if on_docker? || imex_installed? || aws_region.start_with?("us-iso") + return if on_docker? || imex_installed? action_install_imex diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb index 65b83d509e..4ed242ed53 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb @@ -6,12 +6,15 @@ cached(:python_version) { 'python_version' } cached(:system_pyenv_root) { 'system_pyenv_root' } cached(:virtualenv_path) { 'system_pyenv_root/versions/python_version/envs/cookbook_virtualenv' } + cached(:aws_region) { 'any-region' } context "when cookbook virtualenv not installed yet" do cached(:chef_run) do runner = runner(platform: platform, version: version) do |node| + allow_any_instance_of(Object).to receive(:aws_region).and_return(aws_region) node.override['cluster']['system_pyenv_root'] = system_pyenv_root node.override['cluster']['python-version'] = python_version + node.override['cluster']['region'] = aws_region end runner.converge(described_recipe) end @@ -32,7 +35,6 @@ expect(node.default['cluster']['cookbook_virtualenv_path']).to eq(virtualenv_path) is_expected.to write_node_attributes('dump node attributes') end - it 'installs python packages' do is_expected.to run_bash("pip install").with( user: 'root', diff --git a/util/upload-cookbook.sh b/util/upload-cookbook.sh index c2169a4c3e..864ce56959 100755 --- a/util/upload-cookbook.sh +++ b/util/upload-cookbook.sh @@ -93,21 +93,22 @@ main() { # Create archive and md5 _cwd=$(pwd) pushd "${_srcdir}" > /dev/null || exit + GIT_REF=$(git rev-parse HEAD) _stashName=$(git stash create) - git archive --format tar --prefix="aws-parallelcluster-cookbook-${_version}/" "${_stashName:-HEAD}" | gzip > "${_cwd}/aws-parallelcluster-cookbook-${_version}.tgz" + git archive --format tar --prefix="aws-parallelcluster-cookbook-${_version}/" "${_stashName:-HEAD}" | gzip > "${_cwd}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz" #tar zcvf "${_cwd}/aws-parallelcluster-cookbook-${_version}.tgz" --transform "s,^aws-parallelcluster-cookbook/,aws-parallelcluster-cookbook-${_version}/," ../aws-parallelcluster-cookbook popd > /dev/null || exit - md5sum aws-parallelcluster-cookbook-${_version}.tgz > aws-parallelcluster-cookbook-${_version}.md5 + md5sum aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz > aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 # upload packages _key_path="parallelcluster/${_version}/cookbooks" if [ -n "${_scope}" ]; then _key_path="${_key_path}/${_scope}" fi - aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.tgz s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz || _error_exit 'Failed to push cookbook to S3' - aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.md5 s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.md5 || _error_exit 'Failed to push cookbook md5 to S3' - aws ${_profile} --region "${_region}" s3api head-object --bucket ${_bucket} --key ${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz --output text --query LastModified > aws-parallelcluster-cookbook-${_version}.tgz.date || _error_exit 'Failed to fetch LastModified date' - aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.tgz.date s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz.date || _error_exit 'Failed to push cookbook date' + aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz || _error_exit 'Failed to push cookbook to S3' + aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 || _error_exit 'Failed to push cookbook md5 to S3' + aws ${_profile} --region "${_region}" s3api head-object --bucket ${_bucket} --key ${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz --output text --query LastModified > aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date || _error_exit 'Failed to fetch LastModified date' + aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date || _error_exit 'Failed to push cookbook date' _bucket_region=$(aws ${_profile} s3api get-bucket-location --bucket ${_bucket} --output text) if [ ${_bucket_region} = "None" ]; then