From 915cf6050f20961134c9ef209d84d3992f05e1a2 Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 10 Aug 2015 07:07:29 -0700 Subject: [PATCH 01/17] Initial HBase / Phoenix support. --- files/init.d/hbase-master | 180 ++++++++++++++++++ files/init.d/hbase-regionserver | 180 ++++++++++++++++++ modules/hbase_client/manifests/init.pp | 58 ++++++ .../hbase_client/templates/hbase-env.sh.erb | 60 ++++++ .../hbase_client/templates/hbase-site.xml.erb | 46 +++++ .../templates/log4j.properties.erb | 113 +++++++++++ .../hbase_client/templates/regionservers.erb | 8 + modules/hbase_master/manifests/init.pp | 43 +++++ modules/hbase_regionserver/manifests/init.pp | 43 +++++ modules/phoenix_client/manifests/init.pp | 26 +++ profiles/1node-hbase-nonsecure.profile | 15 ++ profiles/3node-hbase-nonsecure.profile | 16 ++ 12 files changed, 788 insertions(+) create mode 100755 files/init.d/hbase-master create mode 100755 files/init.d/hbase-regionserver create mode 100644 modules/hbase_client/manifests/init.pp create mode 100644 modules/hbase_client/templates/hbase-env.sh.erb create mode 100644 modules/hbase_client/templates/hbase-site.xml.erb create mode 100644 modules/hbase_client/templates/log4j.properties.erb create mode 100644 modules/hbase_client/templates/regionservers.erb create mode 100644 modules/hbase_master/manifests/init.pp create mode 100644 modules/hbase_regionserver/manifests/init.pp create mode 100644 modules/phoenix_client/manifests/init.pp create mode 100644 profiles/1node-hbase-nonsecure.profile create mode 100644 profiles/3node-hbase-nonsecure.profile diff --git a/files/init.d/hbase-master b/files/init.d/hbase-master new file mode 100755 index 0000000..131b1a1 --- /dev/null +++ b/files/init.d/hbase-master @@ -0,0 +1,180 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Starts the local HBase Master +# +# chkconfig: 345 85 15 +# description: HBase Master +# +### BEGIN INIT INFO +# Provides: hbase-master +# Short-Description: HBase Master +# Default-Start: 3 4 5 +# Default-Stop: 0 1 2 6 +# Required-Start: $syslog $remote_fs +# Required-Stop: $syslog $remote_fs +# Should-Start: +# Should-Stop: +### END INIT INFO + +. /lib/lsb/init-functions + +RETVAL_SUCCESS=0 + +STATUS_RUNNING=0 +STATUS_DEAD=1 +STATUS_DEAD_AND_LOCK=2 +STATUS_NOT_RUNNING=3 +STATUS_OTHER_ERROR=102 + + +ERROR_PROGRAM_NOT_INSTALLED=5 +ERROR_PROGRAM_NOT_CONFIGURED=6 + + +RETVAL=0 +SLEEP_TIME=5 +PROC_NAME="java" + +DAEMON="hbase-master" +DESC="HBase Master" +EXEC_PATH="/usr/hdp/current/hbase-master/bin/hbase-daemon.sh" +DAEMON_FLAGS="master" +CONF_DIR="/etc/hbase/conf" +LOCKDIR="/var/lock/subsys" +LOCKFILE="$LOCKDIR/hbase-master" +WORKING_DIR="/var/run/hbase" +SVC_USER=hbase + +# set user since the pid/log dir often depend on it +USER="$SVC_USER" +. $CONF_DIR/hbase-env.sh +PIDFILE="$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-master.pid" + +if [ -n "$HBASE_SECURE_DN_USER" ]; then + TARGET_USER=root +else + TARGET_USER=${HBASE_REGIONSERVER_USER:-hbase} +fi + +install -d -m 0755 -o hbase -g hbase /var/lib/hadoop-hbase 1>/dev/null 2>&1 || : +[ -d "$LOCKDIR" ] || install -d -m 0755 $LOCKDIR 1>/dev/null 2>&1 || : +start() { + [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED + [ -d $CONF_DIR ] || exit $ERROR_PROGRAM_NOT_CONFIGURED + log_success_msg "Starting ${DESC}: " + + su -s /bin/bash $TARGET_USER -c "$EXEC_PATH --config '$CONF_DIR' start $DAEMON_FLAGS" + + # Some processes are slow to start + sleep $SLEEP_TIME + checkstatusofproc + RETVAL=$? + + [ $RETVAL -eq $RETVAL_SUCCESS ] && touch $LOCKFILE + return $RETVAL +} + +stop() { + log_success_msg "Stopping ${DESC}: " + su -s /bin/bash $TARGET_USER \ + -c "$EXEC_PATH --config $CONF_DIR stop $DAEMON_FLAGS" + RETVAL=$? + + [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE +} + +restart() { + stop + start +} + +checkstatusofproc(){ + pidofproc -p $PIDFILE $PROC_NAME > /dev/null +} + +checkstatus(){ + checkstatusofproc + status=$? + + case "$status" in + $STATUS_RUNNING) + log_success_msg "${DESC} is running" + ;; + $STATUS_DEAD) + log_failure_msg "${DESC} is dead and pid file exists" + ;; + $STATUS_DEAD_AND_LOCK) + log_failure_msg "${DESC} is dead and lock file exists" + ;; + $STATUS_NOT_RUNNING) + log_failure_msg "${DESC} is not running" + ;; + *) + log_failure_msg "${DESC} status is unknown" + ;; + esac + return $status +} + +condrestart(){ + [ -e $LOCKFILE ] && restart || : +} + +check_for_root() { + if [ $(id -ur) -ne 0 ]; then + echo 'Error: root user required' + echo + exit 1 + fi +} + +service() { + case "$1" in + start) + check_for_root + start + ;; + stop) + check_for_root + stop + ;; + status) + checkstatus + RETVAL=$? + ;; + restart) + check_for_root + restart + ;; + condrestart|try-restart) + check_for_root + condrestart + ;; + rollback) + DAEMON_FLAGS="$DAEMON_FLAGS -${1}" + start + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart|rollback}" + exit 1 + esac +} + +service "$1" + +exit $RETVAL diff --git a/files/init.d/hbase-regionserver b/files/init.d/hbase-regionserver new file mode 100755 index 0000000..3984ba5 --- /dev/null +++ b/files/init.d/hbase-regionserver @@ -0,0 +1,180 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Starts the local HBase Regionserver +# +# chkconfig: 345 85 15 +# description: HBase Regionserver +# +### BEGIN INIT INFO +# Provides: hbase-regionserver +# Short-Description: HBase Regionserver +# Default-Start: 3 4 5 +# Default-Stop: 0 1 2 6 +# Required-Start: $syslog $remote_fs +# Required-Stop: $syslog $remote_fs +# Should-Start: +# Should-Stop: +### END INIT INFO + +. /lib/lsb/init-functions + +RETVAL_SUCCESS=0 + +STATUS_RUNNING=0 +STATUS_DEAD=1 +STATUS_DEAD_AND_LOCK=2 +STATUS_NOT_RUNNING=3 +STATUS_OTHER_ERROR=102 + + +ERROR_PROGRAM_NOT_INSTALLED=5 +ERROR_PROGRAM_NOT_CONFIGURED=6 + + +RETVAL=0 +SLEEP_TIME=5 +PROC_NAME="java" + +DAEMON="hbase-regionserver" +DESC="HBase Regionserver" +EXEC_PATH="/usr/hdp/current/hbase-regionserver/bin/hbase-daemon.sh" +DAEMON_FLAGS="regionserver" +CONF_DIR="/etc/hbase/conf" +LOCKDIR="/var/lock/subsys" +LOCKFILE="$LOCKDIR/hbase-regionserver" +WORKING_DIR="/var/run/hbase" +SVC_USER=hbase + +# set user since the pid/log dir often depend on it +USER="$SVC_USER" +. $CONF_DIR/hbase-env.sh +PIDFILE="$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-regionserver.pid" + +if [ -n "$HBASE_SECURE_DN_USER" ]; then + TARGET_USER=root +else + TARGET_USER=${HBASE_REGIONSERVER_USER:-hbase} +fi + +install -d -m 0755 -o hbase -g hbase /var/lib/hadoop-hbase 1>/dev/null 2>&1 || : +[ -d "$LOCKDIR" ] || install -d -m 0755 $LOCKDIR 1>/dev/null 2>&1 || : +start() { + [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED + [ -d $CONF_DIR ] || exit $ERROR_PROGRAM_NOT_CONFIGURED + log_success_msg "Starting ${DESC}: " + + su -s /bin/bash $TARGET_USER -c "$EXEC_PATH --config '$CONF_DIR' start $DAEMON_FLAGS" + + # Some processes are slow to start + sleep $SLEEP_TIME + checkstatusofproc + RETVAL=$? + + [ $RETVAL -eq $RETVAL_SUCCESS ] && touch $LOCKFILE + return $RETVAL +} + +stop() { + log_success_msg "Stopping ${DESC}: " + su -s /bin/bash $TARGET_USER \ + -c "$EXEC_PATH --config $CONF_DIR stop $DAEMON_FLAGS" + RETVAL=$? + + [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE +} + +restart() { + stop + start +} + +checkstatusofproc(){ + pidofproc -p $PIDFILE $PROC_NAME > /dev/null +} + +checkstatus(){ + checkstatusofproc + status=$? + + case "$status" in + $STATUS_RUNNING) + log_success_msg "${DESC} is running" + ;; + $STATUS_DEAD) + log_failure_msg "${DESC} is dead and pid file exists" + ;; + $STATUS_DEAD_AND_LOCK) + log_failure_msg "${DESC} is dead and lock file exists" + ;; + $STATUS_NOT_RUNNING) + log_failure_msg "${DESC} is not running" + ;; + *) + log_failure_msg "${DESC} status is unknown" + ;; + esac + return $status +} + +condrestart(){ + [ -e $LOCKFILE ] && restart || : +} + +check_for_root() { + if [ $(id -ur) -ne 0 ]; then + echo 'Error: root user required' + echo + exit 1 + fi +} + +service() { + case "$1" in + start) + check_for_root + start + ;; + stop) + check_for_root + stop + ;; + status) + checkstatus + RETVAL=$? + ;; + restart) + check_for_root + restart + ;; + condrestart|try-restart) + check_for_root + condrestart + ;; + rollback) + DAEMON_FLAGS="$DAEMON_FLAGS -${1}" + start + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart|rollback}" + exit 1 + esac +} + +service "$1" + +exit $RETVAL diff --git a/modules/hbase_client/manifests/init.pp b/modules/hbase_client/manifests/init.pp new file mode 100644 index 0000000..1a033e1 --- /dev/null +++ b/modules/hbase_client/manifests/init.pp @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class hbase_client { + require hdfs_client + require zookeeper_client + require phoenix_client + + $path="/usr/bin" + + if $security == "true" { + require kerberos_http + + file { "${hbase_client::keytab_dir}/hbase.keytab": + ensure => file, + source => "/vagrant/generated/keytabs/${hostname}/hbase.keytab", + owner => hbase, + group => hadoop, + mode => '400', + } + } + + package { "hbase_${rpm_version}": + ensure => installed, + } + -> + file { '/etc/hbase/conf/hbase-env.sh': + ensure => file, + content => template('hbase_client/hbase-env.sh.erb'), + } + -> + file { '/etc/hbase/conf/hbase-site.xml': + ensure => file, + content => template('hbase_client/hbase-site.xml.erb'), + } + -> + file { '/etc/hbase/conf/log4j.properties': + ensure => file, + content => template('hbase_client/log4j.properties.erb'), + } + -> + file { '/etc/hbase/conf/regionservers': + ensure => file, + content => template('hbase_client/regionservers.erb'), + } +} diff --git a/modules/hbase_client/templates/hbase-env.sh.erb b/modules/hbase_client/templates/hbase-env.sh.erb new file mode 100644 index 0000000..b151e75 --- /dev/null +++ b/modules/hbase_client/templates/hbase-env.sh.erb @@ -0,0 +1,60 @@ +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +# export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk.x86_64 + +# HBase Configuration directory +export HBASE_CONF_DIR=${HBASE_CONF_DIR:-/usr/hdp/current/hbase-regionserver/conf} + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=${HBASE_CLASSPATH} + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/var/log/hbase/gc.log-`date +'%Y%m%d%H%M'`" +# Uncomment below to enable java garbage collection logging. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size +# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +export HBASE_LOG_DIR=/var/log/hbase + +# A string representing this instance of hbase. $USER by default. +export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +export HBASE_PID_DIR=/var/run/hbase + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false + +export HBASE_OPTS="$HBASE_OPTS -XX:ErrorFile=/var/log/hbase/hs_err_pid%p.log" +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx<%= @hbase_master_mem %>m" +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmx<%= @hbase_regionserver_mem %>m" diff --git a/modules/hbase_client/templates/hbase-site.xml.erb b/modules/hbase_client/templates/hbase-site.xml.erb new file mode 100644 index 0000000..0d845e2 --- /dev/null +++ b/modules/hbase_client/templates/hbase-site.xml.erb @@ -0,0 +1,46 @@ +<% @namenode = eval(@nodes). + select {|node| node[:roles].include? 'nn'}[0][:hostname] + "." + @domain; + @zookeeper_servers = eval(@nodes). + select {|node| node[:roles].include? 'zk'}. + map{|node| node[:hostname] + "." + @domain + ":2181"}.join(","); + -%> + + + + + + hbase.rootdir + hdfs://<%= @namenode %>/apps/hbase + + + + phoenix.functions.allowUserDefinedFunctions + true + + + phoenix.query.timeoutMs + 60000 + + + phoenix.sequence.saltBuckets + 8 + + + hbase.regionserver.wal.codec + org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec + + + + hbase.cluster.distributed + true + + + hbase.zookeeper.quorum + <%= @zookeeper_servers %> + + + zookeeper.znode.parent + /hbase + + + diff --git a/modules/hbase_client/templates/log4j.properties.erb b/modules/hbase_client/templates/log4j.properties.erb new file mode 100644 index 0000000..121de00 --- /dev/null +++ b/modules/hbase_client/templates/log4j.properties.erb @@ -0,0 +1,113 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.security.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Rolling File Appender properties +hbase.log.maxfilesize=256MB +hbase.log.maxbackupindex=20 + +# Rolling File Appender +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file} + +log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize} +log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex} + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# +# Security audit appender +# +hbase.security.log.file=SecurityAuth.audit +hbase.security.log.maxfilesize=256MB +hbase.security.log.maxbackupindex=20 +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file} +log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize} +log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.category.SecurityLogger=${hbase.security.logger} +log4j.additivity.SecurityLogger=false +#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE + +# +# Null Appender +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Custom Logging levels + +log4j.logger.org.apache.zookeeper=INFO +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.hbase=INFO +# Make these two classes INFO-level. Make them DEBUG to see more zk debug. +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO +#log4j.logger.org.apache.hadoop.dfs=DEBUG +# Set this class to log INFO only otherwise its OTT +# Enable this to get detailed connection error/retry logging. +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE + + +# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) +#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG + +# Uncomment the below if you want to remove logging of client region caching' +# and scan of .META. messages +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO +# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO + + \ No newline at end of file diff --git a/modules/hbase_client/templates/regionservers.erb b/modules/hbase_client/templates/regionservers.erb new file mode 100644 index 0000000..a3fb2cc --- /dev/null +++ b/modules/hbase_client/templates/regionservers.erb @@ -0,0 +1,8 @@ +<% + regionservers = eval(@nodes). + select {|node| node[:roles].include? 'hbase-regionserver'}. + map{|node| node[:hostname] + "." + @domain}; + -%> +<% regionservers.each do |node| -%> +<%= node %> +<% end -%> diff --git a/modules/hbase_master/manifests/init.pp b/modules/hbase_master/manifests/init.pp new file mode 100644 index 0000000..a146b34 --- /dev/null +++ b/modules/hbase_master/manifests/init.pp @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class hbase_master { + require hdfs_client + require zookeeper_client + require hbase_client + + $path="/usr/bin" + package { "hbase_${rpm_version}-master" : + ensure => installed, + } + -> + exec { "hdp-select set hbase-master ${hdp_version}": + cwd => "/", + path => "$path", + } + -> + file { "/etc/init.d/hbase-master": + ensure => file, + source => "puppet:///files/init.d/hbase-master", + owner => root, + group => root, + } + -> + service {"hbase-master": + ensure => running, + enable => true, + subscribe => File['/etc/hbase/conf/hbase-site.xml'], + } +} diff --git a/modules/hbase_regionserver/manifests/init.pp b/modules/hbase_regionserver/manifests/init.pp new file mode 100644 index 0000000..a8ea3eb --- /dev/null +++ b/modules/hbase_regionserver/manifests/init.pp @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class hbase_regionserver { + require hdfs_client + require zookeeper_client + require hbase_client + + $path="/usr/bin" + package { "hbase_${rpm_version}-regionserver" : + ensure => installed, + } + -> + exec { "hdp-select set hbase-regionserver ${hdp_version}": + cwd => "/", + path => "$path", + } + -> + file { "/etc/init.d/hbase-regionserver": + ensure => file, + source => "puppet:///files/init.d/hbase-regionserver", + owner => root, + group => root, + } + -> + service {"hbase-regionserver": + ensure => running, + enable => true, + subscribe => File['/etc/hbase/conf/hbase-site.xml'], + } +} diff --git a/modules/phoenix_client/manifests/init.pp b/modules/phoenix_client/manifests/init.pp new file mode 100644 index 0000000..69d2326 --- /dev/null +++ b/modules/phoenix_client/manifests/init.pp @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class phoenix_client { + $path="/usr/bin" + + file { "/etc/environment": + content => inline_template("HBASE_CONF_PATH=/etc/hbase/conf") + } + + package { "phoenix_${rpm_version}": + ensure => installed, + } +} diff --git a/profiles/1node-hbase-nonsecure.profile b/profiles/1node-hbase-nonsecure.profile new file mode 100644 index 0000000..733fcdc --- /dev/null +++ b/profiles/1node-hbase-nonsecure.profile @@ -0,0 +1,15 @@ +{ + "domain": "example.com", + "realm": "EXAMPLE.COM", + "security": false, + "vm_mem": 3072, + "server_mem": 300, + "client_mem": 200, + "hbase_master_mem": 1024, + "hbase_regionserver_mem": 1024, + "clients" : [ "hdfs", "zk", "phoenix" ], + "nodes": [ + {"hostname": "nn", "ip": "240.0.0.11", + "roles": ["nn", "zk", "hbase-master", "hbase-regionserver", "client", "slave"]} + ] +} diff --git a/profiles/3node-hbase-nonsecure.profile b/profiles/3node-hbase-nonsecure.profile new file mode 100644 index 0000000..be400ed --- /dev/null +++ b/profiles/3node-hbase-nonsecure.profile @@ -0,0 +1,16 @@ +{ + "domain": "example.com", + "realm": "EXAMPLE.COM", + "security": false, + "vm_mem": 2048, + "server_mem": 300, + "client_mem": 200, + "hbase_master_mem": 1024, + "hbase_regionserver_mem": 1024, + "clients" : [ "hdfs", "zk", "phoenix" ], + "nodes": [ + { "hostname": "master", "ip": "240.0.0.10", "roles": ["nn", "zk", "hbase-master", "hbase-regionserver", "client", "slave"] }, + { "hostname": "rs1", "ip": "240.0.0.11", "roles": [ "hbase-regionserver", "client", "slave" ] }, + { "hostname": "rs2", "ip": "240.0.0.12", "roles": [ "hbase-regionserver", "client", "slave" ] } + ] +} From ac33e1bec5685311b3ef013d557cc044d95cb755 Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Tue, 11 Aug 2015 11:58:36 -0700 Subject: [PATCH 02/17] New repos for Ambari 2.1 and other HDP versions. --- files/repos/ambari.repo | 14 ++++---------- files/repos/ambari.repo.1.6.1 | 15 +++++++++++++++ files/repos/hdp.repo | 12 ++++++------ files/repos/hdp.repo.2.2.4 | 17 +++++++++++++++++ files/repos/hdp.repo.2.2.6 | 17 +++++++++++++++++ 5 files changed, 59 insertions(+), 16 deletions(-) create mode 100644 files/repos/ambari.repo.1.6.1 create mode 100644 files/repos/hdp.repo.2.2.4 create mode 100644 files/repos/hdp.repo.2.2.6 diff --git a/files/repos/ambari.repo b/files/repos/ambari.repo index c9b70e1..680093b 100644 --- a/files/repos/ambari.repo +++ b/files/repos/ambari.repo @@ -1,14 +1,8 @@ -[ambari-1.x] -name=Ambari 1.x -baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/1.x/GA -gpgcheck=1 -gpgkey=http://public-repo-1.hortonworks.com/ambari/centos6/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins -enabled=1 -priority=1 +#VERSION_NUMBER=2.1.0-1470 -[Updates-ambari-1.6.1] -name=ambari-1.6.1 - Updates -baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/1.x/updates/1.6.1 +[Updates-ambari-2.1.0] +name=ambari-2.1.0 - Updates +baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.1.0 gpgcheck=1 gpgkey=http://public-repo-1.hortonworks.com/ambari/centos6/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins enabled=1 diff --git a/files/repos/ambari.repo.1.6.1 b/files/repos/ambari.repo.1.6.1 new file mode 100644 index 0000000..c9b70e1 --- /dev/null +++ b/files/repos/ambari.repo.1.6.1 @@ -0,0 +1,15 @@ +[ambari-1.x] +name=Ambari 1.x +baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/1.x/GA +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/ambari/centos6/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 + +[Updates-ambari-1.6.1] +name=ambari-1.6.1 - Updates +baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/1.x/updates/1.6.1 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/ambari/centos6/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 diff --git a/files/repos/hdp.repo b/files/repos/hdp.repo index dc062bf..aee3023 100644 --- a/files/repos/hdp.repo +++ b/files/repos/hdp.repo @@ -1,9 +1,9 @@ -#VERSION_NUMBER=2.2.6.0-2800 -[HDP-2.2.6.0] -name=HDP Version - HDP-2.2.6.0 -baseurl=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.6.0 +#VERSION_NUMBER=2.2.4.2-2 +[HDP-2.2.4.2] +name=HDP Version - HDP-2.2.4.2 +baseurl=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.4.2 gpgcheck=1 -gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.6.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.4.2/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins enabled=1 priority=1 @@ -12,6 +12,6 @@ priority=1 name=HDP Utils Version - HDP-UTILS-1.1.0.20 baseurl=http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6 gpgcheck=1 -gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.6.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.4.2/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins enabled=1 priority=1 diff --git a/files/repos/hdp.repo.2.2.4 b/files/repos/hdp.repo.2.2.4 new file mode 100644 index 0000000..aee3023 --- /dev/null +++ b/files/repos/hdp.repo.2.2.4 @@ -0,0 +1,17 @@ +#VERSION_NUMBER=2.2.4.2-2 +[HDP-2.2.4.2] +name=HDP Version - HDP-2.2.4.2 +baseurl=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.4.2 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.4.2/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 + + +[HDP-UTILS-1.1.0.20] +name=HDP Utils Version - HDP-UTILS-1.1.0.20 +baseurl=http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.4.2/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 diff --git a/files/repos/hdp.repo.2.2.6 b/files/repos/hdp.repo.2.2.6 new file mode 100644 index 0000000..dc062bf --- /dev/null +++ b/files/repos/hdp.repo.2.2.6 @@ -0,0 +1,17 @@ +#VERSION_NUMBER=2.2.6.0-2800 +[HDP-2.2.6.0] +name=HDP Version - HDP-2.2.6.0 +baseurl=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.6.0 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.6.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 + + +[HDP-UTILS-1.1.0.20] +name=HDP Utils Version - HDP-UTILS-1.1.0.20 +baseurl=http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.6.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 From c55b61258fe01a63d43b31b96a4d78d3f1b447cb Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Tue, 11 Aug 2015 11:59:56 -0700 Subject: [PATCH 03/17] 2 node profile. --- profiles/ambari-nonsecure-2-node.profile | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 profiles/ambari-nonsecure-2-node.profile diff --git a/profiles/ambari-nonsecure-2-node.profile b/profiles/ambari-nonsecure-2-node.profile new file mode 100644 index 0000000..7e44ad1 --- /dev/null +++ b/profiles/ambari-nonsecure-2-node.profile @@ -0,0 +1,13 @@ +{ + "domain": "example.com", + "realm": "EXAMPLE.COM", + "security": false, + "vm_mem": 2048, + "server_mem": 300, + "client_mem": 200, + "clients" : [ ], + "nodes": [ + { "hostname": "ambari", "ip": "10.0.10.10", "roles": [ "ambari-server" ] }, + { "hostname": "slave1", "ip": "10.0.10.11", "roles": [ "ambari-agent" ] } + ] +} From 923f6566066891f60c66d00aa4f9fb3fba8059c8 Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Tue, 11 Aug 2015 12:00:27 -0700 Subject: [PATCH 04/17] More hbase changes --- Vagrantfile | 2 ++ manifests/default.pp | 10 ++++++++++ modules/hdfs_namenode/manifests/init.pp | 21 ++++++++++++++++++++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 14758f4..3f5d412 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -91,6 +91,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| "clients" => profile[:clients], "server_mem" => profile[:server_mem], "client_mem" => profile[:client_mem], + "hbase_master_mem" => profile[:hbase_master_mem], + "hbase_regionserver_mem" => profile[:hbase_regionserver_mem], "profile" => profile } end diff --git a/manifests/default.pp b/manifests/default.pp index 35fe31f..337707f 100644 --- a/manifests/default.pp +++ b/manifests/default.pp @@ -41,6 +41,9 @@ if hasrole($clients, 'pig') { include pig_client } + if hasrole($clients, 'phoenix') { + include phoenix_client + } if hasrole($clients, 'tez') { include tez_client } @@ -78,6 +81,13 @@ include knox_gateway } +if hasrole($roles, 'hbase-master') { + include hbase_master +} +if hasrole($roles, 'hbase-regionserver') { + include hbase_regionserver +} + if hasrole($roles, 'ambari-server') { include ambari_server } diff --git a/modules/hdfs_namenode/manifests/init.pp b/modules/hdfs_namenode/manifests/init.pp index 93ddc49..35d8fa4 100644 --- a/modules/hdfs_namenode/manifests/init.pp +++ b/modules/hdfs_namenode/manifests/init.pp @@ -155,6 +155,25 @@ user => "hdfs", } -> + exec {"hbase-warehouse": + command => "hadoop fs -mkdir -p /apps/hbase", + unless => "hadoop fs -test -e /apps/hbase", + path => "$PATH", + user => "hdfs", + } + -> + exec {"hbase-warehouse-chown": + command => "hadoop fs -chown hbase:hbase /apps/hbase", + path => "$PATH", + user => "hdfs", + } + -> + exec {"hbase-warehouse-chmod": + command => "hadoop fs -chmod 1777 /apps/hbase", + path => "$PATH", + user => "hdfs", + } + -> exec {"hdfs-tmp": command => "hadoop fs -mkdir /tmp", unless => "hadoop fs -test -e /tmp", @@ -201,4 +220,4 @@ path => "$PATH", user => "hdfs", } -} \ No newline at end of file +} From 875ade296d8617ab63d22aae3f9d1f2274bcca4d Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 17 Aug 2015 08:44:47 -0700 Subject: [PATCH 05/17] Spark support. --- modules/spark/manifests/init.pp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 modules/spark/manifests/init.pp diff --git a/modules/spark/manifests/init.pp b/modules/spark/manifests/init.pp new file mode 100644 index 0000000..64471a2 --- /dev/null +++ b/modules/spark/manifests/init.pp @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class spark { + $path="/usr/bin" + package { "spark_${rpm_version}" : + ensure => installed, + } + -> + exec { "hdp-select set spark ${hdp_version}": + cwd => "/", + path => "$path", + } +} From 2c02f0800bf0b53290ecd431754abd5cb3c0f02d Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 17 Aug 2015 08:45:42 -0700 Subject: [PATCH 06/17] Multi HDP/Ambari/Java support. --- manifests/default.pp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/manifests/default.pp b/manifests/default.pp index 337707f..12905e6 100644 --- a/manifests/default.pp +++ b/manifests/default.pp @@ -13,12 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -include repos_setup +# Initializations. +stage { 'pre': + before => Stage["main"], +} +class { 'repos_setup': + stage => 'pre', +} -> +class { 'jdk': + stage => 'pre', +} + include vm_users include ip_setup include selinux include weak_random include ntp +include ssh_keygen if $security == "true" { include kerberos_client @@ -88,6 +99,10 @@ include hbase_regionserver } +if hasrole($roles, 'spark') { + include spark +} + if hasrole($roles, 'ambari-server') { include ambari_server } From 71ffc00866d91e05584affe83981fed000e2c239 Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 17 Aug 2015 08:46:05 -0700 Subject: [PATCH 07/17] Multi HDP/Ambari/Java support. --- modules/jdk/manifests/init.pp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/modules/jdk/manifests/init.pp b/modules/jdk/manifests/init.pp index 6ffd59a..6872118 100644 --- a/modules/jdk/manifests/init.pp +++ b/modules/jdk/manifests/init.pp @@ -14,15 +14,14 @@ # limitations under the License. class jdk { - $HOME = "/usr/java/default" - - file { "${HOME}": - ensure => "link", - target => "/usr/java/jdk1.6.0_31", + class { 'java': + package => $java_version, } - file { "/etc/profile.d/java.sh": - ensure => "file", - content => template('jdk/java.erb'), + ensure => file, + content => "export JAVA_HOME=/etc/alternatives/jre\n", + owner => root, + group => root, + mode => '644', } -} \ No newline at end of file +} From f14355f58050e1799b39c89db1c06dbd15f426e6 Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 17 Aug 2015 08:48:44 -0700 Subject: [PATCH 08/17] Multi HDP/Ambari/Java support --- modules/ambari_agent/manifests/init.pp | 1 - modules/ambari_server/manifests/init.pp | 3 +-- modules/hdfs_client/manifests/init.pp | 5 ++--- modules/hdfs_client/templates/hadoop-env.erb | 4 ++-- modules/hive_client/templates/hive-env.erb | 2 +- modules/hive_meta/templates/hive-metastore.erb | 2 +- modules/kerberos_http/manifests/init.pp | 4 ++-- modules/knox_gateway/manifests/init.pp | 3 +-- modules/pig_client/templates/pig-env.erb | 2 +- modules/repos_setup/manifests/init.pp | 4 ++-- modules/ssl_ca/manifests/init.pp | 7 ++----- modules/tez_client/templates/tez-env.erb | 2 +- modules/yarn_client/manifests/init.pp | 3 +-- modules/yarn_client/templates/yarn-env.erb | 2 +- modules/zookeeper_client/manifests/init.pp | 1 - modules/zookeeper_client/templates/zookeeper-env.erb | 4 ++-- 16 files changed, 20 insertions(+), 29 deletions(-) diff --git a/modules/ambari_agent/manifests/init.pp b/modules/ambari_agent/manifests/init.pp index 25d55d4..0c9fc40 100644 --- a/modules/ambari_agent/manifests/init.pp +++ b/modules/ambari_agent/manifests/init.pp @@ -14,7 +14,6 @@ # limitations under the License. class ambari_agent { - require repos_setup $tmp_dir = "/tmp" diff --git a/modules/ambari_server/manifests/init.pp b/modules/ambari_server/manifests/init.pp index af0a0d2..c323c18 100644 --- a/modules/ambari_server/manifests/init.pp +++ b/modules/ambari_server/manifests/init.pp @@ -14,7 +14,6 @@ # limitations under the License. class ambari_server { - require repos_setup package { "ambari-server": @@ -22,7 +21,7 @@ } -> exec { "ambari-server-setup": - command => "/usr/sbin/ambari-server setup --silent" + command => "/usr/sbin/ambari-server setup --silent -j /etc/alternatives/jre" } -> exec { "ambari-server-start": diff --git a/modules/hdfs_client/manifests/init.pp b/modules/hdfs_client/manifests/init.pp index 1c5cc85..37495e5 100644 --- a/modules/hdfs_client/manifests/init.pp +++ b/modules/hdfs_client/manifests/init.pp @@ -16,10 +16,9 @@ class hdfs_client { require repos_setup require hdp_select - require jdk $conf_dir="/etc/hadoop/hdp" - $path="${jdk::HOME}/bin:/bin:/usr/bin" + $path="${java_home}/bin:/bin:/usr/bin" $log_dir="/var/log/hadoop" $data_dir="/var/lib/hadoop" $pid_dir="/var/run/pid" @@ -130,7 +129,7 @@ require ssl_ca # bless the generated ca cert for java clients - exec {"keytool -importcert -noprompt -alias horton-ca -keystore ${jdk::HOME}/jre/lib/security/cacerts -storepass changeit -file ca.crt": + exec {"keytool -importcert -noprompt -alias horton-ca -keystore ${java_home}/jre/lib/security/cacerts -storepass changeit -file ca.crt": cwd => "/vagrant/generated/ssl-ca", path => "$path", } diff --git a/modules/hdfs_client/templates/hadoop-env.erb b/modules/hdfs_client/templates/hadoop-env.erb index a723f7f..84bb16f 100644 --- a/modules/hdfs_client/templates/hadoop-env.erb +++ b/modules/hdfs_client/templates/hadoop-env.erb @@ -17,7 +17,7 @@ # Set Hadoop-specific environment variables here. # The java implementation to use. Required. -export JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> +export JAVA_HOME=<%= scope.lookupvar('java_home') %> export HADOOP_HOME_WARN_SUPPRESS=1 # The maximum amount of heap to use, in MB. Default is 1000. @@ -81,4 +81,4 @@ export JSVC_HOME=/usr/lib/bigtop-utils # On secure datanodes, user to run the datanode as after dropping privileges export HADOOP_SECURE_DN_USER=hdfs -<% end -%> \ No newline at end of file +<% end -%> diff --git a/modules/hive_client/templates/hive-env.erb b/modules/hive_client/templates/hive-env.erb index 4431a34..0fc2c05 100755 --- a/modules/hive_client/templates/hive-env.erb +++ b/modules/hive_client/templates/hive-env.erb @@ -45,6 +45,6 @@ export HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-1024} # Hive Configuration Directory can be controlled by: export HIVE_CONF_DIR=/etc/hive/conf -export JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> +export JAVA_HOME=<%= scope.lookupvar('java_home') %> # Folder containing extra ibraries required for hive compilation/execution can be controlled by: export HIVE_AUX_JARS_PATH= diff --git a/modules/hive_meta/templates/hive-metastore.erb b/modules/hive_meta/templates/hive-metastore.erb index 9aee022..cff87b8 100755 --- a/modules/hive_meta/templates/hive-metastore.erb +++ b/modules/hive_meta/templates/hive-metastore.erb @@ -37,7 +37,7 @@ BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default} [ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hadoop ] && . ${BIGTOP_DEFAULTS_DIR}/hadoop [ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hive-metastore ] && . ${BIGTOP_DEFAULTS_DIR}/hive-metastore -export JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> +export JAVA_HOME=<%= scope.lookupvar('java_home') %> RETVAL_SUCCESS=0 diff --git a/modules/kerberos_http/manifests/init.pp b/modules/kerberos_http/manifests/init.pp index 7826609..dea094c 100644 --- a/modules/kerberos_http/manifests/init.pp +++ b/modules/kerberos_http/manifests/init.pp @@ -22,7 +22,7 @@ Class['kerberos_kdc'] -> Class['kerberos_http'] } - $path = "${jdk::HOME}/bin:/bin:/usr/bin" + $path = "${java_home}/bin:/bin:/usr/bin" file { "${hdfs_client::keytab_dir}": ensure => directory, @@ -68,4 +68,4 @@ mode => '640', content => template('kerberos_http/ssl-server.erb'), } -} \ No newline at end of file +} diff --git a/modules/knox_gateway/manifests/init.pp b/modules/knox_gateway/manifests/init.pp index 9ab4bc3..8a154de 100644 --- a/modules/knox_gateway/manifests/init.pp +++ b/modules/knox_gateway/manifests/init.pp @@ -15,7 +15,6 @@ class knox_gateway { require repos_setup - require jdk package { "knox.noarch" : ensure => installed, @@ -42,4 +41,4 @@ ensure => running, enable => true, } -} \ No newline at end of file +} diff --git a/modules/pig_client/templates/pig-env.erb b/modules/pig_client/templates/pig-env.erb index 5432d0a..c3636b2 100755 --- a/modules/pig_client/templates/pig-env.erb +++ b/modules/pig_client/templates/pig-env.erb @@ -14,4 +14,4 @@ See the License for the specific language governing permissions and limitations under the License. -%> -JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> +JAVA_HOME=<%= scope.lookupvar('java_home') %> diff --git a/modules/repos_setup/manifests/init.pp b/modules/repos_setup/manifests/init.pp index 6a02010..89c02ac 100644 --- a/modules/repos_setup/manifests/init.pp +++ b/modules/repos_setup/manifests/init.pp @@ -16,11 +16,11 @@ class repos_setup { file { '/etc/yum.repos.d/hdp.repo': ensure => file, - source => 'puppet:///files/repos/hdp.repo', + source => "puppet:///files/repos/hdp.repo.${hdp_short_version}", } file { '/etc/yum.repos.d/ambari.repo': ensure => file, - source => 'puppet:///files/repos/ambari.repo', + source => "puppet:///files/repos/ambari.repo.${ambari_version}", } package { 'epel-release-6-8': ensure => absent, diff --git a/modules/ssl_ca/manifests/init.pp b/modules/ssl_ca/manifests/init.pp index bd0703c..d0c1299 100644 --- a/modules/ssl_ca/manifests/init.pp +++ b/modules/ssl_ca/manifests/init.pp @@ -16,10 +16,7 @@ # This module create a generated certificate authority that can be used to # make certificates for all of the servers. class ssl_ca { - require jdk - - $java="/usr/java/default" - $path="${java}/bin:/bin:/usr/bin" + $path="${java_home}/bin:/bin:/usr/bin" $cadir="/vagrant/generated/ssl-ca" file { "${cadir}": @@ -44,4 +41,4 @@ content => "01", mode => "600", } -} \ No newline at end of file +} diff --git a/modules/tez_client/templates/tez-env.erb b/modules/tez_client/templates/tez-env.erb index adf3109..c20ba1c 100644 --- a/modules/tez_client/templates/tez-env.erb +++ b/modules/tez_client/templates/tez-env.erb @@ -18,4 +18,4 @@ export TEZ_CONF_DIR=/etc/tez/conf # The java implementation to use. -export JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> +export JAVA_HOME=<%= scope.lookupvar('java_home') %> diff --git a/modules/yarn_client/manifests/init.pp b/modules/yarn_client/manifests/init.pp index b59306b..971eac4 100644 --- a/modules/yarn_client/manifests/init.pp +++ b/modules/yarn_client/manifests/init.pp @@ -15,7 +15,6 @@ class yarn_client { require repos_setup - require jdk require hdfs_client $user_logs = "/user/yarn/" @@ -73,4 +72,4 @@ ensure => file, content => template('yarn_client/yarn-site.erb'), } -} \ No newline at end of file +} diff --git a/modules/yarn_client/templates/yarn-env.erb b/modules/yarn_client/templates/yarn-env.erb index 98fe0a5..4abe4f6 100755 --- a/modules/yarn_client/templates/yarn-env.erb +++ b/modules/yarn_client/templates/yarn-env.erb @@ -17,7 +17,7 @@ export YARN_LOG_DIR=<%= scope.lookupvar('hdfs_client::log_dir') %>/$USER export YARN_PID_DIR=<%= scope.lookupvar('hdfs_client::pid_dir') %>/$USER -export JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> +export JAVA_HOME=<%= scope.lookupvar('java_home') %> export YARN_IDENT_STRING="$HADOOP_IDENT_STRING" # User for YARN daemons diff --git a/modules/zookeeper_client/manifests/init.pp b/modules/zookeeper_client/manifests/init.pp index 49006c9..8cc5b7f 100644 --- a/modules/zookeeper_client/manifests/init.pp +++ b/modules/zookeeper_client/manifests/init.pp @@ -16,7 +16,6 @@ class zookeeper_client { require repos_setup require hdp_select - require jdk $conf_dir="/etc/zookeeper/hdp" $log_dir="/var/log/zookeeper" diff --git a/modules/zookeeper_client/templates/zookeeper-env.erb b/modules/zookeeper_client/templates/zookeeper-env.erb index e088796..14a8277 100755 --- a/modules/zookeeper_client/templates/zookeeper-env.erb +++ b/modules/zookeeper_client/templates/zookeeper-env.erb @@ -14,11 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. -%> -export JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> +export JAVA_HOME=<%= scope.lookupvar('java_home') %> export ZOO_LOG_DIR=<%= @log_dir %> export ZOOPIDFILE=<%= @pid_dir %>/zookeeper-server.pid export SERVER_JVMFLAGS=-Xmx<%= @server_mem %>m <% if @security == "true" -%> export SERVER_JVMFLAGS="-Djava.security.auth.login.config=/etc/zookeeper/conf/zookeeper-server.jaas" export CLIENT_JVMFLAGS="-Djava.security.auth.login.config=/etc/zookeeper/conf/zookeeper-client.jaas" -<% end -%> \ No newline at end of file +<% end -%> From 2ea8877bf47ec9382ef598532e1511095cc1c80d Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 17 Aug 2015 08:49:14 -0700 Subject: [PATCH 09/17] Deploy ssh keys for easier Ambari deploys. --- modules/ssh_keygen/files/authorized_keys | 1 + modules/ssh_keygen/files/id_rsa | 27 ++++++++++++ modules/ssh_keygen/files/id_rsa.pub | 1 + modules/ssh_keygen/manifests/init.pp | 53 ++++++++++++++++++++++++ 4 files changed, 82 insertions(+) create mode 100644 modules/ssh_keygen/files/authorized_keys create mode 100644 modules/ssh_keygen/files/id_rsa create mode 100644 modules/ssh_keygen/files/id_rsa.pub create mode 100644 modules/ssh_keygen/manifests/init.pp diff --git a/modules/ssh_keygen/files/authorized_keys b/modules/ssh_keygen/files/authorized_keys new file mode 100644 index 0000000..8dce3fa --- /dev/null +++ b/modules/ssh_keygen/files/authorized_keys @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDWb1HUKRCUhVnHe+Qm7yA/iKJkucOL7lTIVKwspK3Qylp55M6cZ87XngfGvbgthO5ytnNboRWBGhxDoXFXTMUBdofiE5cXooSWhpp1YKlYZx/Ewl/muThk77myS0jjAMUDIvu67K6tfAYJ8kiuQsyKtjiiMCoL+3nbLXuW9RPLEOIXALKooXCJhFzPj+cFJGDy7qHK5QbKI011C462nvFkl3bhSUyfkbRTZXW5Bm6TAHrxQRXwQmEW0GB8e6PQkHwH90HHP9TiaGFB8OdP1pu+u2Q4xPn1YRfqCSX/Jpi/zn3KUySbjgAM4bmALbka2Gex1tFS6Yk3gr5VBhJFXckR carter@HW11972.local diff --git a/modules/ssh_keygen/files/id_rsa b/modules/ssh_keygen/files/id_rsa new file mode 100644 index 0000000..5c42a14 --- /dev/null +++ b/modules/ssh_keygen/files/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA1m9R1CkQlIVZx3vkJu8gP4iiZLnDi+5UyFSsLKSt0MpaeeTO +nGfO154Hxr24LYTucrZzW6EVgRocQ6FxV0zFAXaH4hOXF6KEloaadWCpWGcfxMJf +5rk4ZO+5sktI4wDFAyL7uuyurXwGCfJIrkLMirY4ojAqC/t52y17lvUTyxDiFwCy +qKFwiYRcz4/nBSRg8u6hyuUGyiNNdQuOtp7xZJd24UlMn5G0U2V1uQZukwB68UEV +8EJhFtBgfHuj0JB8B/dBxz/U4mhhQfDnT9abvrtkOMT59WEX6gkl/yaYv859ylMk +m44ADOG5gC25GthnsdbRUumJN4K+VQYSRV3JEQIDAQABAoIBAASAkddChXaY1im6 +EDBBFQxAcihrPg/SETWjkXnee0fG999XTn86Lr2YM9GYaiWKhPcjnYhu+WSch7tj +yF1s3zYIpxG4F5QHdYEzkkqknOmeQKyA6EKP4P89iUiVDBK4EWrfcSmbjo9V60qK +K7x86o5/BPFi0X8u6IARlGa2/pNk3X4Iqy+LXI4ipow+MRB3bwdWTR0C9Tl3QcE/ +FgB+O4h7deGhpbCcEdEYfJb4blhkNXmQvQZC57Io40UYaLkqn87j21hzPEeQwYrN +C3albHKWlxeOGMUqlAk/XdmNrPfbP8KMgO3upV0jFOOATXtCu7n/uUPQFddoVDhu +EQSYigECgYEA+oESO8nmkZHnuhwYouX6y+xJou8LO/pQ0Sw/yvxpWk85ADcEDJBV +KV+rWM+wK6Lv2QAcPRptZJv5b4VFrzZjPrhUUsOEvGGzGfAyP+yYJDrdFk1mP29S +d5bdKWRkmvcW3CLbOD9+VgqVxI1ptgEex/nk+uTRgtxrY5zGZbv7YJUCgYEA2yOr +QrzEht/yFv/6vmr6DFMMWoWDDPcsrU6Xz7Zx3UasKc+RSJHiyf23kRsB7HWLE3jr +EBdVXcUmTZ62XH7DYj0YGWBGH2xsJGzMX0JVtIlBtnNVvOSwopcA/ev2XAK5MPKV +CymJGO7dcL8x+ATZEXFZapv7cB/lkCjwhVf6e40CgYEAx+GXb/eRL5RRZ3Ilwk1p +ia+3rfe5ahk4xT7ehBRV6AaQv/gmFfRQz5uTuGC3/ImdgfQRlDvzD7VZGcAosz7d +f91LA2MZFcuLQmYjYlhhqX8CjFoj/PfRwLSzMeT2VaRVLD/v1fjr0LHPgS8pK13H +dt+R8xQDjLxeP/SHZgPR9+UCgYBVYRpS0DVJz2sFJwrMseVmc9z2T9p1bMBNbbec +SBso+WYOjD6wpZZU2LNyp7EiLdHX7l63UEAEXtHDoDiw5Pq/Bcqj6GypxZaEwpcF +7lRyGKAvs9QN2pQvZOIQDKVZx/3bYkhLpw0xV6+TRAr19P2V3DHYZZmbX/ITLsA3 +U8XV6QKBgQCNNQh+UvgzNyu1YFbB8FjNHN7ZYehb/a04rxVwng2V0ejmx+yhR/VC +hO96FcsxkEnX1HLmtGwcdVgATRA8ryp64/B3aSgJ6dAs7Oe4VPHX+L8OU8CleG3l +YjUk8VYjcOpIFIDJ+qEWg0shAXZQBS0mrgtjsqkP+QHyoBEU/gnOPg== +-----END RSA PRIVATE KEY----- diff --git a/modules/ssh_keygen/files/id_rsa.pub b/modules/ssh_keygen/files/id_rsa.pub new file mode 100644 index 0000000..8dce3fa --- /dev/null +++ b/modules/ssh_keygen/files/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDWb1HUKRCUhVnHe+Qm7yA/iKJkucOL7lTIVKwspK3Qylp55M6cZ87XngfGvbgthO5ytnNboRWBGhxDoXFXTMUBdofiE5cXooSWhpp1YKlYZx/Ewl/muThk77myS0jjAMUDIvu67K6tfAYJ8kiuQsyKtjiiMCoL+3nbLXuW9RPLEOIXALKooXCJhFzPj+cFJGDy7qHK5QbKI011C462nvFkl3bhSUyfkbRTZXW5Bm6TAHrxQRXwQmEW0GB8e6PQkHwH90HHP9TiaGFB8OdP1pu+u2Q4xPn1YRfqCSX/Jpi/zn3KUySbjgAM4bmALbka2Gex1tFS6Yk3gr5VBhJFXckR carter@HW11972.local diff --git a/modules/ssh_keygen/manifests/init.pp b/modules/ssh_keygen/manifests/init.pp new file mode 100644 index 0000000..ce7f956 --- /dev/null +++ b/modules/ssh_keygen/manifests/init.pp @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class ssh_keygen { + $path = "/bin:/usr/bin" + + file { '/root/.ssh': + ensure => directory, + mode => 'go-rwx', + } + -> + file { '/root/.ssh/id_rsa': + ensure => file, + source => 'puppet:///modules/ssh_keygen/id_rsa', + owner => root, + group => root, + mode => '600', + } + -> + file { '/root/.ssh/id_rsa.pub': + ensure => file, + source => 'puppet:///modules/ssh_keygen/id_rsa.pub', + owner => root, + group => root, + mode => '644', + } + -> + file { '/root/.ssh/authorized_keys': + ensure => file, + source => 'puppet:///modules/ssh_keygen/authorized_keys', + owner => root, + group => root, + mode => '644', + } + -> + exec { 'known_hosts' : + path => $path, + command => 'ssh-keyscan -H `hostname` > /root/.ssh/known_hosts', + unless => 'test -f /root/.ssh/known_hosts', + } +} From 87f906507fbc24246ec856b0c27c29730f221ba9 Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 17 Aug 2015 08:49:50 -0700 Subject: [PATCH 10/17] Multi HDP/Ambari/Java versions --- files/repos/ambari.repo.1.7.0 | 15 +++++++++++++++ files/repos/ambari.repo.2.0.2 | 7 +++++++ files/repos/ambari.repo.2.1.0 | 9 +++++++++ files/repos/hdp.repo.1.3.10 | 17 +++++++++++++++++ files/repos/hdp.repo.2.0.13 | 17 +++++++++++++++++ files/repos/hdp.repo.2.1.15 | 17 +++++++++++++++++ files/repos/hdp.repo.2.3.0 | 17 +++++++++++++++++ 7 files changed, 99 insertions(+) create mode 100644 files/repos/ambari.repo.1.7.0 create mode 100644 files/repos/ambari.repo.2.0.2 create mode 100644 files/repos/ambari.repo.2.1.0 create mode 100644 files/repos/hdp.repo.1.3.10 create mode 100644 files/repos/hdp.repo.2.0.13 create mode 100644 files/repos/hdp.repo.2.1.15 create mode 100644 files/repos/hdp.repo.2.3.0 diff --git a/files/repos/ambari.repo.1.7.0 b/files/repos/ambari.repo.1.7.0 new file mode 100644 index 0000000..d1ccd28 --- /dev/null +++ b/files/repos/ambari.repo.1.7.0 @@ -0,0 +1,15 @@ +[ambari-1.x] +name=Ambari 1.x +baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/1.x/GA +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/ambari/centos6/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 + +[Updates-ambari-1.7.0] +name=ambari-1.7.0 - Updates +baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/1.x/updates/1.7.0 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/ambari/centos6/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 diff --git a/files/repos/ambari.repo.2.0.2 b/files/repos/ambari.repo.2.0.2 new file mode 100644 index 0000000..2cfdc47 --- /dev/null +++ b/files/repos/ambari.repo.2.0.2 @@ -0,0 +1,7 @@ +[Updates-ambari-2.0.2] +name=ambari-2.0.2 - Updates +baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.0.2 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/ambari/centos6/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 diff --git a/files/repos/ambari.repo.2.1.0 b/files/repos/ambari.repo.2.1.0 new file mode 100644 index 0000000..680093b --- /dev/null +++ b/files/repos/ambari.repo.2.1.0 @@ -0,0 +1,9 @@ +#VERSION_NUMBER=2.1.0-1470 + +[Updates-ambari-2.1.0] +name=ambari-2.1.0 - Updates +baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.1.0 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/ambari/centos6/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 diff --git a/files/repos/hdp.repo.1.3.10 b/files/repos/hdp.repo.1.3.10 new file mode 100644 index 0000000..38c144b --- /dev/null +++ b/files/repos/hdp.repo.1.3.10 @@ -0,0 +1,17 @@ +#VERSION_NUMBER=1.3.10.0-24 +[HDP-1.3.10.0] +name=Hortonworks Data Platform Version - HDP-1.3.10.0 +baseurl=http://public-repo-1.hortonworks.com/HDP/centos6/1.x/updates/1.3.10.0 +gpgcheck=0 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/1.x/updates/1.3.10.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 + + +[HDP-UTILS-1.1.0.16] +name=Hortonworks Data Platform Utils Version - HDP-UTILS-1.1.0.16 +baseurl=http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.16/repos/centos6 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/1.x/updates/1.3.10.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 diff --git a/files/repos/hdp.repo.2.0.13 b/files/repos/hdp.repo.2.0.13 new file mode 100644 index 0000000..b747415 --- /dev/null +++ b/files/repos/hdp.repo.2.0.13 @@ -0,0 +1,17 @@ +#VERSION_NUMBER=2.0.13.0-43 +[HDP-2.0.13.0] +name=Hortonworks Data Platform Version - HDP-2.0.13.0 +baseurl=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.13.0 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.13.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 + + +[HDP-UTILS-1.1.0.16] +name=Hortonworks Data Platform Utils Version - HDP-UTILS-1.1.0.16 +baseurl=http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.16/repos/centos6 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.13.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 diff --git a/files/repos/hdp.repo.2.1.15 b/files/repos/hdp.repo.2.1.15 new file mode 100644 index 0000000..cd910ea --- /dev/null +++ b/files/repos/hdp.repo.2.1.15 @@ -0,0 +1,17 @@ +#VERSION_NUMBER=2.1.15.0-946 +[HDP-2.1.15.0] +name=Hortonworks Data Platform Version - HDP-2.1.15.0 +baseurl=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.1.15.0 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.1.15.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 + + +[HDP-UTILS-1.1.0.19] +name=Hortonworks Data Platform Utils Version - HDP-UTILS-1.1.0.19 +baseurl=http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.19/repos/centos6 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.1.15.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 diff --git a/files/repos/hdp.repo.2.3.0 b/files/repos/hdp.repo.2.3.0 new file mode 100644 index 0000000..ffbefd1 --- /dev/null +++ b/files/repos/hdp.repo.2.3.0 @@ -0,0 +1,17 @@ +#VERSION_NUMBER=2.3.0.0-2557 +[HDP-2.3.0.0] +name=HDP Version - HDP-2.3.0.0 +baseurl=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 + + +[HDP-UTILS-1.1.0.20] +name=HDP Utils Version - HDP-UTILS-1.1.0.20 +baseurl=http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 From 8fa9c6ddfe001f9539c9b8d310fa08166cd54d0d Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 17 Aug 2015 08:50:37 -0700 Subject: [PATCH 11/17] Ambari manage Ambari --- profiles/ambari-nonsecure-2-node.profile | 4 +++- profiles/ambari-nonsecure-4-nodes.profile | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/profiles/ambari-nonsecure-2-node.profile b/profiles/ambari-nonsecure-2-node.profile index 7e44ad1..c4f7489 100644 --- a/profiles/ambari-nonsecure-2-node.profile +++ b/profiles/ambari-nonsecure-2-node.profile @@ -1,4 +1,6 @@ { + "ambari_version": "2.1.0", + "java_version": "java-1.8.0-openjdk", "domain": "example.com", "realm": "EXAMPLE.COM", "security": false, @@ -8,6 +10,6 @@ "clients" : [ ], "nodes": [ { "hostname": "ambari", "ip": "10.0.10.10", "roles": [ "ambari-server" ] }, - { "hostname": "slave1", "ip": "10.0.10.11", "roles": [ "ambari-agent" ] } + { "hostname": "slave1", "ip": "10.0.10.11", "roles": [ ] } ] } diff --git a/profiles/ambari-nonsecure-4-nodes.profile b/profiles/ambari-nonsecure-4-nodes.profile index 1f7f28c..bce7c0b 100644 --- a/profiles/ambari-nonsecure-4-nodes.profile +++ b/profiles/ambari-nonsecure-4-nodes.profile @@ -8,8 +8,8 @@ "clients" : [ ], "nodes": [ { "hostname": "ambari", "ip": "10.0.10.10", "roles": [ "ambari-server" ] }, - { "hostname": "master", "ip": "10.0.10.11", "roles": [ "ambari-agent" ] }, - { "hostname": "slave1", "ip": "10.0.10.12", "roles": [ "ambari-agent" ] }, - { "hostname": "slave2", "ip": "10.0.10.13", "roles": [ "ambari-agent" ] } + { "hostname": "master", "ip": "10.0.10.11", "roles": [ ] }, + { "hostname": "slave1", "ip": "10.0.10.12", "roles": [ ] }, + { "hostname": "slave2", "ip": "10.0.10.13", "roles": [ ] } ] } From dc41e855c74ce5e2e29eba205bdcacb6103298f0 Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 17 Aug 2015 08:51:14 -0700 Subject: [PATCH 12/17] Spark profile --- profiles/1node-spark-nonsecure.profile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 profiles/1node-spark-nonsecure.profile diff --git a/profiles/1node-spark-nonsecure.profile b/profiles/1node-spark-nonsecure.profile new file mode 100644 index 0000000..b20e22c --- /dev/null +++ b/profiles/1node-spark-nonsecure.profile @@ -0,0 +1,14 @@ +{ + "java_version": "java-1.8.0-openjdk", + "hdp_short_version": "2.3.0", + "domain": "example.com", + "realm": "EXAMPLE.COM", + "security": false, + "vm_mem": 3072, + "server_mem": 300, + "client_mem": 200, + "clients" : [ "hdfs", "zk" ], + "nodes": [ + {"hostname": "spark1", "ip": "240.0.0.11", "roles": ["nn", "zk", "client", "slave", "spark"]} + ] +} From 0563ea552929b82f8a997bdc87014e88e5a378ac Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 17 Aug 2015 08:51:46 -0700 Subject: [PATCH 13/17] Multi HDP/Ambari/Java support --- Vagrantfile | 49 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 3f5d412..ac5a195 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -21,6 +21,11 @@ VAGRANTFILE_API_VERSION = "2" $profile_path = ["current.profile", "profiles/3node-nonsecure.profile"] +# Default versions. +default_hdp_short_version = "2.3.0" +default_ambari_version = "2.1.0" +default_java_version = "java-1.7.0-openjdk" + ############################################################################### # Loads a profile, which is a JSON file describing a specific configuation. # @@ -36,20 +41,27 @@ def loadProfile() end # Pull the HDP version out of the hdp.repo file -def findVersion() - fileObj = File.new('files/repos/hdp.repo', 'r') +def findVersion(version) + fileObj = File.new('files/repos/hdp.repo.%s' % version, 'r') match = /^#VERSION_NUMBER=(?[-0-9.]*)/.match(fileObj.gets) fileObj.close() result = match['ver'] - puts "HDP Version = %s\n" % result + puts "HDP Build = %s\n" % result return result end ############################################################################### # Define cluster - profile = loadProfile() -hdp_version = findVersion() + +# Versions +hdp_short_version = profile[:hdp_short_version] || default_hdp_short_version +ambari_version = profile[:ambari_version] || default_ambari_version +java_version = profile[:java_version] || default_java_version +java_home = "/etc/alternatives/jre" +puts "Ambari Version = %s\n" % ambari_version +puts "Java Version = %s\n" % java_version +hdp_version = findVersion(hdp_short_version) rpm_version = hdp_version.gsub /[.-]/, '_' Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| @@ -75,24 +87,39 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| node_config.vm.hostname = node[:hostname] + "." + profile[:domain] node_config.vm.network :private_network, ip: node[:ip] node_config.ssh.forward_agent = true + + node_config.vm.provision :shell do |shell| + shell.inline = " + if [ ! -d /etc/puppet/modules/java ] ; then + mkdir -p /etc/puppet/modules; + puppet module install puppetlabs/java; + fi" + end + node_config.vm.provision "puppet" do |puppet| puppet.module_path = "modules" puppet.options = ["--libdir", "/vagrant", "--fileserverconfig=/vagrant/fileserver.conf"] puppet.facter = { + "hdp_short_version" => hdp_short_version, + "hdp_version" => hdp_version, + "ambari_version" => ambari_version, + "java_version" => java_version, + "java_home" => java_home, + "rpm_version" => rpm_version, + + "server_mem" => profile[:server_mem], + "client_mem" => profile[:client_mem], + "hbase_master_mem" => profile[:hbase_master_mem], + "hbase_regionserver_mem" => profile[:hbase_regionserver_mem], + "hostname" => node[:hostname], "roles" => node[:roles], "nodes" => profile[:nodes], - "hdp_version" => hdp_version, - "rpm_version" => rpm_version, "domain" => profile[:domain], "security" => profile[:security], "realm" => profile[:realm], "clients" => profile[:clients], - "server_mem" => profile[:server_mem], - "client_mem" => profile[:client_mem], - "hbase_master_mem" => profile[:hbase_master_mem], - "hbase_regionserver_mem" => profile[:hbase_regionserver_mem], "profile" => profile } end From 4f0ebf9de36bb3bb2b8d993db80fb1381b934d0a Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 7 Sep 2015 22:27:40 -0700 Subject: [PATCH 14/17] Ubuntu support. --- files/repos/hdp.list.2.2.6 | 3 + modules/hive_db/files/my.cnf | 127 +++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 files/repos/hdp.list.2.2.6 create mode 100644 modules/hive_db/files/my.cnf diff --git a/files/repos/hdp.list.2.2.6 b/files/repos/hdp.list.2.2.6 new file mode 100644 index 0000000..4e82755 --- /dev/null +++ b/files/repos/hdp.list.2.2.6 @@ -0,0 +1,3 @@ +#VERSION_NUMBER=2.2.6.0-2800 +deb http://public-repo-1.hortonworks.com/HDP/ubuntu12/2.x/updates/2.2.6.0 HDP main +deb http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/ubuntu12 HDP-UTILS main diff --git a/modules/hive_db/files/my.cnf b/modules/hive_db/files/my.cnf new file mode 100644 index 0000000..37b2e9a --- /dev/null +++ b/modules/hive_db/files/my.cnf @@ -0,0 +1,127 @@ +# +# The MySQL database server configuration file. +# +# You can copy this to one of: +# - "/etc/mysql/my.cnf" to set global options, +# - "~/.my.cnf" to set user-specific options. +# +# One can use all long options that the program supports. +# Run program with --help to get a list of available options and with +# --print-defaults to see which it would actually understand and use. +# +# For explanations see +# http://dev.mysql.com/doc/mysql/en/server-system-variables.html + +# This will be passed to all mysql clients +# It has been reported that passwords should be enclosed with ticks/quotes +# escpecially if they contain "#" chars... +# Remember to edit /etc/mysql/debian.cnf when changing the socket location. +[client] +port = 3306 +socket = /var/run/mysqld/mysqld.sock + +# Here is entries for some specific programs +# The following values assume you have at least 32M ram + +# This was formally known as [safe_mysqld]. Both versions are currently parsed. +[mysqld_safe] +socket = /var/run/mysqld/mysqld.sock +nice = 0 + +[mysqld] +# +# * Basic Settings +# +user = mysql +pid-file = /var/run/mysqld/mysqld.pid +socket = /var/run/mysqld/mysqld.sock +port = 3306 +basedir = /usr +datadir = /var/lib/mysql +tmpdir = /tmp +lc-messages-dir = /usr/share/mysql +skip-external-locking +# +# Instead of skip-networking the default is now to listen only on +# localhost which is more compatible and is not less secure. +bind-address = 0.0.0.0 +# +# * Fine Tuning +# +key_buffer = 16M +max_allowed_packet = 16M +thread_stack = 192K +thread_cache_size = 8 +# This replaces the startup script and checks MyISAM tables if needed +# the first time they are touched +myisam-recover = BACKUP +#max_connections = 100 +#table_cache = 64 +#thread_concurrency = 10 +# +# * Query Cache Configuration +# +query_cache_limit = 1M +query_cache_size = 16M +# +# * Logging and Replication +# +# Both location gets rotated by the cronjob. +# Be aware that this log type is a performance killer. +# As of 5.1 you can enable the log at runtime! +#general_log_file = /var/log/mysql/mysql.log +#general_log = 1 +# +# Error log - should be very few entries. +# +log_error = /var/log/mysql/error.log +# +# Here you can see queries with especially long duration +#log_slow_queries = /var/log/mysql/mysql-slow.log +#long_query_time = 2 +#log-queries-not-using-indexes +# +# The following can be used as easy to replay backup logs or for replication. +# note: if you are setting up a replication slave, see README.Debian about +# other settings you may need to change. +#server-id = 1 +#log_bin = /var/log/mysql/mysql-bin.log +expire_logs_days = 10 +max_binlog_size = 100M +#binlog_do_db = include_database_name +#binlog_ignore_db = include_database_name +# +# * InnoDB +# +# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. +# Read the manual for more InnoDB related options. There are many! +# +# * Security Features +# +# Read the manual, too, if you want chroot! +# chroot = /var/lib/mysql/ +# +# For generating SSL certificates I recommend the OpenSSL GUI "tinyca". +# +# ssl-ca=/etc/mysql/cacert.pem +# ssl-cert=/etc/mysql/server-cert.pem +# ssl-key=/etc/mysql/server-key.pem + + + +[mysqldump] +quick +quote-names +max_allowed_packet = 16M + +[mysql] +#no-auto-rehash # faster start of mysql but no tab completition + +[isamchk] +key_buffer = 16M + +# +# * IMPORTANT: Additional settings that can override those from this file! +# The files must end with '.cnf', otherwise they'll be ignored. +# +!includedir /etc/mysql/conf.d/ From bfa654cf84a6c22835f6b11e262bd6e79757e4c8 Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 7 Sep 2015 22:31:07 -0700 Subject: [PATCH 15/17] Ubuntu support. --- Vagrantfile | 77 +++++++++---------- files/repos/ambari.repo | 9 --- modules/ambari_agent/manifests/init.pp | 1 + modules/ambari_server/manifests/init.pp | 3 +- modules/hbase_client/manifests/init.pp | 6 +- modules/hbase_master/manifests/init.pp | 23 +++++- modules/hbase_regionserver/manifests/init.pp | 37 ++++++--- modules/hdfs_client/manifests/init.pp | 44 +++++++---- modules/hdfs_client/templates/hadoop-env.erb | 4 +- modules/hdfs_datanode/manifests/init.pp | 8 +- modules/hdfs_namenode/manifests/init.pp | 8 +- modules/hive_client/manifests/init.pp | 8 +- modules/hive_client/templates/hive-env.erb | 2 +- modules/hive_db/manifests/init.pp | 39 ++++++++-- modules/hive_meta/manifests/init.pp | 6 +- .../hive_meta/templates/hive-metastore.erb | 2 +- modules/ip_setup/manifests/init.pp | 19 ++--- modules/jdk/manifests/init.pp | 35 +++++++-- modules/kerberos_client/manifests/init.pp | 38 ++++++--- modules/kerberos_http/manifests/init.pp | 4 +- modules/kerberos_kdc/manifests/init.pp | 54 ++++++++++--- modules/knox_gateway/manifests/init.pp | 3 +- modules/ntp/manifests/init.pp | 19 +++-- modules/oozie_client/manifests/init.pp | 4 +- modules/oozie_server/manifests/init.pp | 41 +++++++++- modules/pig_client/manifests/init.pp | 6 +- modules/pig_client/templates/pig-env.erb | 2 +- modules/repos_setup/manifests/init.pp | 43 ++++++++--- modules/selinux/manifests/init.pp | 10 ++- modules/tez_client/manifests/init.pp | 4 +- modules/tez_client/templates/tez-env.erb | 2 +- modules/yarn_client/manifests/init.pp | 5 +- modules/yarn_client/templates/yarn-env.erb | 2 +- modules/yarn_node_manager/manifests/init.pp | 22 ++++-- .../yarn_resource_manager/manifests/init.pp | 14 ++-- modules/zookeeper_client/manifests/init.pp | 5 +- .../templates/zookeeper-env.erb | 4 +- modules/zookeeper_server/manifests/init.pp | 38 +++++++-- 38 files changed, 447 insertions(+), 204 deletions(-) delete mode 100644 files/repos/ambari.repo diff --git a/Vagrantfile b/Vagrantfile index b6d7e1c..ec775cc 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -21,11 +21,6 @@ VAGRANTFILE_API_VERSION = "2" $profile_path = ["current.profile", "profiles/3node-nonsecure.profile"] -# Default versions. -default_hdp_short_version = "2.3.0" -default_ambari_version = "2.1.0" -default_java_version = "java-1.7.0-openjdk" - ############################################################################### # Loads a profile, which is a JSON file describing a specific configuation. # @@ -40,9 +35,15 @@ def loadProfile() } end -# Pull the HDP version out of the hdp.repo file -def findVersion(version) - fileObj = File.new('files/repos/hdp.repo.%s' % version, 'r') +# Pull the HDP version out of the repository file. +def findVersion(profile) + if (profile[:os] == "centos") + path = 'files/repos/hdp.repo.%s' % profile[:hdp_short_version] + elsif (profile[:os] == "ubuntu") + path = 'files/repos/hdp.list.%s' % profile[:hdp_short_version] + end + + fileObj = File.new(path, 'r') match = /^#VERSION_NUMBER=(?[-0-9.]*)/.match(fileObj.gets) fileObj.close() result = match['ver'] @@ -52,17 +53,20 @@ end ############################################################################### # Define cluster + profile = loadProfile() -# Versions -hdp_short_version = profile[:hdp_short_version] || default_hdp_short_version -ambari_version = profile[:ambari_version] || default_ambari_version -java_version = profile[:java_version] || default_java_version -java_home = "/etc/alternatives/jre" -puts "Ambari Version = %s\n" % ambari_version -puts "Java Version = %s\n" % java_version -hdp_version = findVersion(hdp_short_version) -rpm_version = hdp_version.gsub /[.-]/, '_' +# Set defaults. +default_os = "centos" +default_hdp_short_version = "2.2.6" +default_ambari_version = "2.1.0" +default_java_version = "java-1.7.0-openjdk" + +profile[:hdp_short_version] ||= default_hdp_short_version +profile[:ambari_version] ||= default_ambari_version +profile[:java_version] ||= default_java_version +profile[:os] ||= default_os +hdp_version = findVersion(profile) Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| if Vagrant.has_plugin?("vagrant-cachier") @@ -72,7 +76,15 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # All Vagrant configuration is done here. The most common configuration # Every Vagrant virtual environment requires a box to build off of. - config.vm.box = "omalley/centos6_x64" + if (profile[:os] == "centos") + config.vm.box = "omalley/centos6_x64" + package_version = "_" + (hdp_version.gsub /[.-]/, '_') + start_script_path = "rc.d/init.d" + elsif (profile[:os] == "ubuntu") + config.vm.box = "ubuntu/trusty64" + package_version = "-" + (hdp_version.gsub /[.-]/, '-') + start_script_path = "init.d" + end config.vm.provider :virtualbox do |vb| vb.customize ["modifyvm", :id, "--memory", profile[:vm_mem] ] @@ -87,41 +99,28 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| node_config.vm.hostname = node[:hostname] + "." + profile[:domain] node_config.vm.network :private_network, ip: node[:ip] node_config.ssh.forward_agent = true - - node_config.vm.provision :shell do |shell| - shell.inline = " - if [ ! -d /etc/puppet/modules/java ] ; then - mkdir -p /etc/puppet/modules; - puppet module install puppetlabs/java; - fi" - end - node_config.vm.provision "puppet" do |puppet| puppet.module_path = "modules" puppet.options = ["--libdir", "/vagrant", "--verbose", "--debug", "--fileserverconfig=/vagrant/fileserver.conf"] puppet.facter = { - "hdp_short_version" => hdp_short_version, - "hdp_version" => hdp_version, - "ambari_version" => ambari_version, - "java_version" => java_version, - "java_home" => java_home, - "rpm_version" => rpm_version, - - "server_mem" => profile[:server_mem], - "client_mem" => profile[:client_mem], - "hbase_master_mem" => profile[:hbase_master_mem], - "hbase_regionserver_mem" => profile[:hbase_regionserver_mem], + "hdp_short_version" => profile[:hdp_short_version], + "ambari_version" => profile[:ambari_version], + "package_version" => package_version, + "start_script_path" => start_script_path, "hostname" => node[:hostname], "roles" => node[:roles], "nodes" => profile[:nodes], + "hdp_version" => hdp_version, "domain" => profile[:domain], "security" => profile[:security], "realm" => profile[:realm], "clients" => profile[:clients], - "profile" => profile + "server_mem" => profile[:server_mem], + "client_mem" => profile[:client_mem], + "profile" => profile, } end end diff --git a/files/repos/ambari.repo b/files/repos/ambari.repo deleted file mode 100644 index 680093b..0000000 --- a/files/repos/ambari.repo +++ /dev/null @@ -1,9 +0,0 @@ -#VERSION_NUMBER=2.1.0-1470 - -[Updates-ambari-2.1.0] -name=ambari-2.1.0 - Updates -baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.1.0 -gpgcheck=1 -gpgkey=http://public-repo-1.hortonworks.com/ambari/centos6/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins -enabled=1 -priority=1 diff --git a/modules/ambari_agent/manifests/init.pp b/modules/ambari_agent/manifests/init.pp index 0c9fc40..25d55d4 100644 --- a/modules/ambari_agent/manifests/init.pp +++ b/modules/ambari_agent/manifests/init.pp @@ -14,6 +14,7 @@ # limitations under the License. class ambari_agent { + require repos_setup $tmp_dir = "/tmp" diff --git a/modules/ambari_server/manifests/init.pp b/modules/ambari_server/manifests/init.pp index c323c18..af0a0d2 100644 --- a/modules/ambari_server/manifests/init.pp +++ b/modules/ambari_server/manifests/init.pp @@ -14,6 +14,7 @@ # limitations under the License. class ambari_server { + require repos_setup package { "ambari-server": @@ -21,7 +22,7 @@ } -> exec { "ambari-server-setup": - command => "/usr/sbin/ambari-server setup --silent -j /etc/alternatives/jre" + command => "/usr/sbin/ambari-server setup --silent" } -> exec { "ambari-server-start": diff --git a/modules/hbase_client/manifests/init.pp b/modules/hbase_client/manifests/init.pp index bec1056..b1cbaa2 100644 --- a/modules/hbase_client/manifests/init.pp +++ b/modules/hbase_client/manifests/init.pp @@ -35,14 +35,14 @@ content => template('hbase_client/zk-jaas.erb'), } -> - Package["hbase_${rpm_version}"] + Package["hbase${package_version}"] } - package { "hbase_${rpm_version}": + package { "hbase${package_version}": ensure => installed, } -> - package { "phoenix_${rpm_version}": + package { "phoenix${package_version}": ensure => installed, } -> diff --git a/modules/hbase_master/manifests/init.pp b/modules/hbase_master/manifests/init.pp index dc86bd8..54fe277 100644 --- a/modules/hbase_master/manifests/init.pp +++ b/modules/hbase_master/manifests/init.pp @@ -16,10 +16,27 @@ class hbase_master { require hbase_server - $path="/usr/bin" + $path="/bin:/sbin:/usr/bin" - package { "hbase_${rpm_version}-master" : - ensure => installed, + case $operatingsystem { + 'centos': { + package { "hbase${package_version}-master" : + ensure => installed, + } + } + # XXX: Work around BUG-39010. + 'ubuntu': { + exec { "apt-get download hbase${package_version}-master": + cwd => "/tmp", + path => "$path", + } + -> + exec { "dpkg -i --force-overwrite hbase${package_version}*master*.deb": + cwd => "/tmp", + path => "$path", + user => "root", + } + } } -> exec { "hdp-select set hbase-master ${hdp_version}": diff --git a/modules/hbase_regionserver/manifests/init.pp b/modules/hbase_regionserver/manifests/init.pp index e36cbe6..4327891 100644 --- a/modules/hbase_regionserver/manifests/init.pp +++ b/modules/hbase_regionserver/manifests/init.pp @@ -14,20 +14,35 @@ # limitations under the License. class hbase_regionserver { -<<<<<<< HEAD - require hdfs_client - require zookeeper_client - require hbase_client - - $path="/usr/bin" -======= require hbase_server - $path="/usr/bin" + $path="/bin:/sbin:/usr/bin" ->>>>>>> upstream/master - package { "hbase_${rpm_version}-regionserver" : - ensure => installed, + case $operatingsystem { + 'centos': { + package { "hbase${package_version}-regionserver" : + ensure => installed, + } + } + # XXX: Work around BUG-39010. + 'ubuntu': { + exec { "apt-get download hbase${package_version}-regionserver": + cwd => "/tmp", + path => "$path", + } + -> + exec { "dpkg -i --force-overwrite hbase${package_version}*.deb": + cwd => "/tmp", + path => "$path", + user => "root", + } + # Fix incorrect startup script permissions (XXX: Is a bug filed for this?). + -> + file { "/usr/hdp/${hdp_version}/etc/init.d/hbase-regionserver": + ensure => file, + mode => '755', + } + } } -> exec { "hdp-select set hbase-regionserver ${hdp_version}": diff --git a/modules/hdfs_client/manifests/init.pp b/modules/hdfs_client/manifests/init.pp index 87eb91b..1d4e66b 100644 --- a/modules/hdfs_client/manifests/init.pp +++ b/modules/hdfs_client/manifests/init.pp @@ -16,15 +16,16 @@ class hdfs_client { require repos_setup require hdp_select + require jdk $conf_dir="/etc/hadoop/hdp" - $path="${java_home}/bin:/bin:/usr/bin" + $path="${jdk::HOME}/bin:/bin:/usr/bin" $log_dir="/var/log/hadoop" $data_dir="/var/lib/hadoop" $pid_dir="/var/run/pid" $keytab_dir="/etc/security/hadoop" - package { "hadoop_${rpm_version}": + package { "hadoop${package_version}": ensure => installed, } -> @@ -33,22 +34,30 @@ path => "$path", } - package { "hadoop_${rpm_version}-libhdfs": - ensure => installed, - require => Package["hadoop_${rpm_version}"], + if ($operatingsystem == "centos") { + package { "hadoop${package_version}-libhdfs": + ensure => installed, + require => Package["hadoop${package_version}"], + } + } + elsif ($operatingsystem == "ubuntu") { + package { "libhdfs0${package_version}": + ensure => installed, + require => Package["hadoop${package_version}"], + } } - package { "hadoop_${rpm_version}-client": + package { "hadoop${package_version}-client": ensure => installed, - require => Package["hadoop_${rpm_version}"], + require => Package["hadoop${package_version}"], } - package { "hadooplzo_${rpm_version}": + package { "hadooplzo${package_version}": ensure => installed, - require => Package["hadoop_${rpm_version}"], + require => Package["hadoop${package_version}"], } -> - package { "hadooplzo_${rpm_version}-native": + package { "hadooplzo${package_version}-native": ensure => installed, } @@ -60,8 +69,15 @@ ensure => installed, } - package { 'lzo': - ensure => installed, + if ($operatingsystem == "centos") { + package { 'lzo': + ensure => installed, + } + } + elsif ($operatingsystem == "ubuntu") { + package { 'liblzo2-2': + ensure => installed, + } } file { '/etc/hadoop': @@ -75,7 +91,7 @@ file { '/etc/hadoop/conf': ensure => 'link', target => "${conf_dir}", - require => Package["hadoop_${rpm_version}"], + require => Package["hadoop${package_version}"], force => true } @@ -129,7 +145,7 @@ require ssl_ca # bless the generated ca cert for java clients - exec {"keytool -importcert -noprompt -alias horton-ca -keystore ${java_home}/jre/lib/security/cacerts -storepass changeit -file ca.crt": + exec {"keytool -importcert -noprompt -alias horton-ca -keystore ${jdk::HOME}/jre/lib/security/cacerts -storepass changeit -file ca.crt": cwd => "/vagrant/generated/ssl-ca", path => "$path", unless => "keytool -list -alias horton-ca -keystore /usr/java/default/jre/lib/security/cacerts -storepass changeit", diff --git a/modules/hdfs_client/templates/hadoop-env.erb b/modules/hdfs_client/templates/hadoop-env.erb index 84bb16f..a723f7f 100644 --- a/modules/hdfs_client/templates/hadoop-env.erb +++ b/modules/hdfs_client/templates/hadoop-env.erb @@ -17,7 +17,7 @@ # Set Hadoop-specific environment variables here. # The java implementation to use. Required. -export JAVA_HOME=<%= scope.lookupvar('java_home') %> +export JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> export HADOOP_HOME_WARN_SUPPRESS=1 # The maximum amount of heap to use, in MB. Default is 1000. @@ -81,4 +81,4 @@ export JSVC_HOME=/usr/lib/bigtop-utils # On secure datanodes, user to run the datanode as after dropping privileges export HADOOP_SECURE_DN_USER=hdfs -<% end -%> +<% end -%> \ No newline at end of file diff --git a/modules/hdfs_datanode/manifests/init.pp b/modules/hdfs_datanode/manifests/init.pp index 0f0dfeb..f9b6129 100644 --- a/modules/hdfs_datanode/manifests/init.pp +++ b/modules/hdfs_datanode/manifests/init.pp @@ -30,10 +30,10 @@ mode => '400', } -> - Package["hadoop_${rpm_version}-hdfs-datanode"] + Package["hadoop${package_version}-hdfs-datanode"] } - package { "hadoop_${rpm_version}-hdfs-datanode" : + package { "hadoop${package_version}-hdfs-datanode" : ensure => installed, } -> @@ -44,11 +44,11 @@ -> file { "/etc/init.d/hadoop-hdfs-datanode": ensure => 'link', - target => "/usr/hdp/current/hadoop-hdfs-datanode/../etc/rc.d/init.d/hadoop-hdfs-datanode", + target => "/usr/hdp/current/hadoop-hdfs-datanode/../etc/${start_script_path}/hadoop-hdfs-datanode", } -> service {"hadoop-hdfs-datanode": ensure => running, enable => true, } -} \ No newline at end of file +} diff --git a/modules/hdfs_namenode/manifests/init.pp b/modules/hdfs_namenode/manifests/init.pp index a5cc75f..964240a 100644 --- a/modules/hdfs_namenode/manifests/init.pp +++ b/modules/hdfs_namenode/manifests/init.pp @@ -34,10 +34,10 @@ user => hdfs, } -> - Package["hadoop_${rpm_version}-hdfs-namenode"] + Package["hadoop${package_version}-hdfs-namenode"] } - package { "hadoop_${rpm_version}-hdfs-namenode" : + package { "hadoop${package_version}-hdfs-namenode" : ensure => installed, } -> @@ -48,7 +48,7 @@ -> file { "/etc/init.d/hadoop-hdfs-namenode": ensure => 'link', - target => "/usr/hdp/current/hadoop-hdfs-namenode/../etc/rc.d/init.d/hadoop-hdfs-namenode", + target => "/usr/hdp/current/hadoop-hdfs-namenode/../etc/${start_script_path}/hadoop-hdfs-namenode", } -> exec {"namenode-format": @@ -56,7 +56,7 @@ path => "$PATH", creates => "${hdfs_client::data_dir}/hdfs/namenode", user => "hdfs", - require => Package["hadoop_${rpm_version}-hdfs-namenode"], + require => Package["hadoop${package_version}-hdfs-namenode"], } -> service {"hadoop-hdfs-namenode": diff --git a/modules/hive_client/manifests/init.pp b/modules/hive_client/manifests/init.pp index 87b3cd7..374126e 100644 --- a/modules/hive_client/manifests/init.pp +++ b/modules/hive_client/manifests/init.pp @@ -16,7 +16,7 @@ class hive_client { require yarn_client - package { "hive_${rpm_version}": + package { "hive${package_version}": ensure => installed, } @@ -31,7 +31,7 @@ file { '/etc/hive/conf': ensure => 'link', target => '/etc/hive/hdp', - require => Package["hive_${rpm_version}"], + require => Package["hive${package_version}"], force => true } @@ -57,6 +57,6 @@ file { "/usr/hdp/${hdp_version}/hive/lib/mysql-connector-java.jar": ensure => 'link', target => '/usr/share/java/mysql-connector-java.jar', - require => Package["hive_${rpm_version}"], + require => Package["hive${package_version}"], } -} \ No newline at end of file +} diff --git a/modules/hive_client/templates/hive-env.erb b/modules/hive_client/templates/hive-env.erb index 0fc2c05..4431a34 100755 --- a/modules/hive_client/templates/hive-env.erb +++ b/modules/hive_client/templates/hive-env.erb @@ -45,6 +45,6 @@ export HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-1024} # Hive Configuration Directory can be controlled by: export HIVE_CONF_DIR=/etc/hive/conf -export JAVA_HOME=<%= scope.lookupvar('java_home') %> +export JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> # Folder containing extra ibraries required for hive compilation/execution can be controlled by: export HIVE_AUX_JARS_PATH= diff --git a/modules/hive_db/manifests/init.pp b/modules/hive_db/manifests/init.pp index 4043801..ccb95c1 100644 --- a/modules/hive_db/manifests/init.pp +++ b/modules/hive_db/manifests/init.pp @@ -16,13 +16,36 @@ class hive_db { $PATH = "/bin:/usr/bin" - package { 'mysql-server': - ensure => installed, - } - -> - service { 'mysqld': - ensure => running, - enable => true, + case $operatingsystem { + 'centos': { + package { 'mysql-server': + ensure => installed, + } + -> + service { 'mysqld': + ensure => running, + enable => true, + } + } + 'ubuntu': { + package { 'mysql-server': + ensure => installed, + } + -> + file { '/etc/mysql/my.cnf': + ensure => file, + source => 'puppet:///modules/hive_db/my.cnf', + owner => root, + group => root, + mode => '644', + notify => Service["mysql"], + } + + service { 'mysql': + ensure => running, + enable => true, + } + } } -> exec { "secure-mysqld": @@ -42,4 +65,4 @@ cwd => "/vagrant/modules/hive_db", creates => "/var/lib/mysql/hive", } -} \ No newline at end of file +} diff --git a/modules/hive_meta/manifests/init.pp b/modules/hive_meta/manifests/init.pp index be635eb..4036a7b 100644 --- a/modules/hive_meta/manifests/init.pp +++ b/modules/hive_meta/manifests/init.pp @@ -28,10 +28,10 @@ mode => '400', } -> - Package["hive_${rpm_version}-metastore"] + Package["hive${package_version}-metastore"] } - package { "hive_${rpm_version}-metastore": + package { "hive${package_version}-metastore": ensure => installed, } -> @@ -56,4 +56,4 @@ ensure => running, enable => true, } -} \ No newline at end of file +} diff --git a/modules/hive_meta/templates/hive-metastore.erb b/modules/hive_meta/templates/hive-metastore.erb index cff87b8..9aee022 100755 --- a/modules/hive_meta/templates/hive-metastore.erb +++ b/modules/hive_meta/templates/hive-metastore.erb @@ -37,7 +37,7 @@ BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default} [ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hadoop ] && . ${BIGTOP_DEFAULTS_DIR}/hadoop [ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hive-metastore ] && . ${BIGTOP_DEFAULTS_DIR}/hive-metastore -export JAVA_HOME=<%= scope.lookupvar('java_home') %> +export JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> RETVAL_SUCCESS=0 diff --git a/modules/ip_setup/manifests/init.pp b/modules/ip_setup/manifests/init.pp index 6f29b8d..bdddf61 100644 --- a/modules/ip_setup/manifests/init.pp +++ b/modules/ip_setup/manifests/init.pp @@ -14,19 +14,20 @@ # limitations under the License. class ip_setup { + if ($operatingsystem == "centos") { + service {"iptables": + ensure => stopped, + enable => false, + } - service {"iptables": - ensure => stopped, - enable => false, - } - - service {"ip6tables": - ensure => stopped, - enable => false, + service {"ip6tables": + ensure => stopped, + enable => false, + } } file { '/etc/hosts': ensure => file, content => template('ip_setup/hosts.erb'), } -} \ No newline at end of file +} diff --git a/modules/jdk/manifests/init.pp b/modules/jdk/manifests/init.pp index 76a8a81..3bca9a6 100644 --- a/modules/jdk/manifests/init.pp +++ b/modules/jdk/manifests/init.pp @@ -16,16 +16,39 @@ class jdk { $HOME = "/usr/lib/jvm/java" - package { "java-1.7.0-openjdk": - ensure => installed, - } + if ($operatingsystem == "centos") { + package { "java-1.7.0-openjdk": + ensure => installed, + } - package { "java-1.7.0-openjdk-devel": - ensure => installed, + package { "java-1.7.0-openjdk-devel": + ensure => installed, + } + } + elsif ($operatingsystem == "ubuntu") { + package { "openjdk-7-jdk": + ensure => installed, + } + -> + file { $HOME: + ensure => 'link', + target => '/usr/lib/jvm/java-7-openjdk-amd64', + force => true + } + -> + file { "/usr/java": + ensure => 'directory' + } + -> + file { "/usr/java/default": + ensure => 'link', + target => '/usr/lib/jvm/java', + force => true + } } file { "/etc/profile.d/java.sh": ensure => "file", content => template('jdk/java.erb'), } -} \ No newline at end of file +} diff --git a/modules/kerberos_client/manifests/init.pp b/modules/kerberos_client/manifests/init.pp index b7ee859..fac97a1 100644 --- a/modules/kerberos_client/manifests/init.pp +++ b/modules/kerberos_client/manifests/init.pp @@ -16,21 +16,35 @@ class kerberos_client { require ntp - - package { 'krb5-auth-dialog': - ensure => installed, - } - - package { 'krb5-workstation': - ensure => installed, + case $operatingsystem { + 'centos': { + package { 'krb5-auth-dialog': + ensure => installed, + before => File['/etc/krb5.conf'], + } + package { 'krb5-workstation': + ensure => installed, + before => File['/etc/krb5.conf'], + } + package { 'krb5-libs': + ensure => installed, + before => File['/etc/krb5.conf'], + } + } + 'ubuntu': { + package { 'krb5-user': + ensure => installed, + before => File['/etc/krb5.conf'], + } + package { 'krb5-config': + ensure => installed, + before => File['/etc/krb5.conf'], + } + } } - package { 'krb5-libs': - ensure => installed, - } - -> file { '/etc/krb5.conf': ensure => file, content => template('kerberos_client/krb5.erb'), } -} \ No newline at end of file +} diff --git a/modules/kerberos_http/manifests/init.pp b/modules/kerberos_http/manifests/init.pp index dea094c..7826609 100644 --- a/modules/kerberos_http/manifests/init.pp +++ b/modules/kerberos_http/manifests/init.pp @@ -22,7 +22,7 @@ Class['kerberos_kdc'] -> Class['kerberos_http'] } - $path = "${java_home}/bin:/bin:/usr/bin" + $path = "${jdk::HOME}/bin:/bin:/usr/bin" file { "${hdfs_client::keytab_dir}": ensure => directory, @@ -68,4 +68,4 @@ mode => '640', content => template('kerberos_http/ssl-server.erb'), } -} +} \ No newline at end of file diff --git a/modules/kerberos_kdc/manifests/init.pp b/modules/kerberos_kdc/manifests/init.pp index 3ad05c1..907a2ba 100644 --- a/modules/kerberos_kdc/manifests/init.pp +++ b/modules/kerberos_kdc/manifests/init.pp @@ -18,11 +18,33 @@ $path="/bin:/usr/bin:/sbin:/usr/sbin" $password="vagrant" - package { 'krb5-server': - ensure => installed, + if ($operatingsystem == "centos") { + $kdcpath = "/var/kerberos/krb5kdc" } - -> - file { '/var/kerberos/krb5kdc/kdc.conf': + elsif ($operatingsystem == "ubuntu") { + $kdcpath = "/etc/krb5kdc" + } + + case $operatingsystem { + 'centos': { + package { 'krb5-server': + ensure => installed, + before => File["$kdcpath/kdc.conf"] + } + } + 'ubuntu': { + package { 'krb5-kdc': + ensure => installed, + } + -> + package { 'krb5-admin-server': + ensure => installed, + before => File["$kdcpath/kdc.conf"] + } + } + } + + file { "$kdcpath/kdc.conf": ensure => file, content => template('kerberos_kdc/kdc.erb'), } @@ -40,12 +62,24 @@ -> exec { 'kdc-init': command => "/vagrant/generated/create-kerberos-db", - creates => "/var/kerberos/krb5kdc/principal", + creates => "$kdcpath/principal", path => $path, } - -> - service { 'krb5kdc': - ensure => running, - enable => true, + + case $operatingsystem { + 'centos': { + service { 'krb5kdc': + ensure => running, + enable => true, + require => Exec['kdc-init'], + } + } + 'ubuntu': { + service { 'krb5-kdc': + ensure => running, + enable => true, + require => Exec['kdc-init'], + } + } } -} \ No newline at end of file +} diff --git a/modules/knox_gateway/manifests/init.pp b/modules/knox_gateway/manifests/init.pp index 8a154de..9ab4bc3 100644 --- a/modules/knox_gateway/manifests/init.pp +++ b/modules/knox_gateway/manifests/init.pp @@ -15,6 +15,7 @@ class knox_gateway { require repos_setup + require jdk package { "knox.noarch" : ensure => installed, @@ -41,4 +42,4 @@ ensure => running, enable => true, } -} +} \ No newline at end of file diff --git a/modules/ntp/manifests/init.pp b/modules/ntp/manifests/init.pp index d6dcedf..a719c50 100644 --- a/modules/ntp/manifests/init.pp +++ b/modules/ntp/manifests/init.pp @@ -17,9 +17,18 @@ package { "ntp": ensure => installed, } - service { "ntp": - name => "ntpd", - ensure => running, - enable => true, + if ($operatingsystem == "centos") { + service { "ntp": + name => "ntpd", + ensure => running, + enable => true, + } } -} \ No newline at end of file + elsif ($operatingsystem == "ubuntu") { + service { "ntp": + name => "ntp", + ensure => running, + enable => true, + } + } +} diff --git a/modules/oozie_client/manifests/init.pp b/modules/oozie_client/manifests/init.pp index 65d8037..f128135 100644 --- a/modules/oozie_client/manifests/init.pp +++ b/modules/oozie_client/manifests/init.pp @@ -17,7 +17,7 @@ require repos_setup require hdp_select - package { "oozie_${rpm_version}-client": + package { "oozie${package_version}-client": ensure => installed, } @@ -25,4 +25,4 @@ ensure => file, content => template('oozie_client/oozie.erb'), } -} \ No newline at end of file +} diff --git a/modules/oozie_server/manifests/init.pp b/modules/oozie_server/manifests/init.pp index d7f7e01..aa245cc 100644 --- a/modules/oozie_server/manifests/init.pp +++ b/modules/oozie_server/manifests/init.pp @@ -39,20 +39,55 @@ user => 'oozie', } -> - Package["oozie_${rpm_version}"] + Package["oozie${package_version}"] $prepare_war_opts = "-secure" } - package { "oozie_${rpm_version}": + # XXX: Uncaptured dependencies on Ubuntu (needs Bug filed). + case $operatingsystem { + 'ubuntu': { + package { "zip": + ensure => installed, + before => Package["oozie${package_version}"], + } + package { "unzip": + ensure => installed, + before => Package["oozie${package_version}"], + } + } + default: {} + } + + package { "oozie${package_version}": ensure => installed, } -> + case $operatingsystem { + # XXX: More packaging bugs. + 'ubuntu': { + file { "/var/run/oozie": + ensure => directory, + owner => oozie, + group => oozie, + mode => '755', + } + file { "/var/tmp/oozie": + ensure => directory, + owner => oozie, + group => oozie, + mode => '755', + } + } + default: {} + } + file { "/etc/oozie": ensure => directory, owner => "root", group => "oozie", mode => "0750", + require => Package["oozie${package_version}"], } -> file { "${conf_dir}/adminusers.txt": @@ -151,4 +186,4 @@ ensure => running, enable => true, } -} \ No newline at end of file +} diff --git a/modules/pig_client/manifests/init.pp b/modules/pig_client/manifests/init.pp index bcf08a3..947f828 100644 --- a/modules/pig_client/manifests/init.pp +++ b/modules/pig_client/manifests/init.pp @@ -19,7 +19,7 @@ $conf_dir="/etc/pig/hdp" $path="/usr/bin" - package { "pig_${rpm_version}": + package { "pig${package_version}": ensure => present, } @@ -34,7 +34,7 @@ file { '/etc/pig/conf': ensure => 'link', target => "${conf_dir}", - require => Package["pig_${rpm_version}"], + require => Package["pig${package_version}"], force => true } @@ -52,4 +52,4 @@ ensure => file, content => template('pig_client/pig.erb'), } -} \ No newline at end of file +} diff --git a/modules/pig_client/templates/pig-env.erb b/modules/pig_client/templates/pig-env.erb index c3636b2..5432d0a 100755 --- a/modules/pig_client/templates/pig-env.erb +++ b/modules/pig_client/templates/pig-env.erb @@ -14,4 +14,4 @@ See the License for the specific language governing permissions and limitations under the License. -%> -JAVA_HOME=<%= scope.lookupvar('java_home') %> +JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> diff --git a/modules/repos_setup/manifests/init.pp b/modules/repos_setup/manifests/init.pp index 89c02ac..3881130 100644 --- a/modules/repos_setup/manifests/init.pp +++ b/modules/repos_setup/manifests/init.pp @@ -14,15 +14,40 @@ # limitations under the License. class repos_setup { - file { '/etc/yum.repos.d/hdp.repo': - ensure => file, - source => "puppet:///files/repos/hdp.repo.${hdp_short_version}", - } - file { '/etc/yum.repos.d/ambari.repo': - ensure => file, - source => "puppet:///files/repos/ambari.repo.${ambari_version}", + $PATH="/bin:/usr/bin" + + if ($operatingsystem == "centos") { + file { '/etc/yum.repos.d/hdp.repo': + ensure => file, + source => "puppet:///files/repos/hdp.repo.${hdp_short_version}", + } + file { '/etc/yum.repos.d/ambari.repo': + ensure => file, + source => "puppet:///files/repos/ambari.repo.${ambari_version}", + } + package { 'epel-release-6-8': + ensure => absent, + } } - package { 'epel-release-6-8': - ensure => absent, + elsif ($operatingsystem == "ubuntu") { + file { '/etc/apt/sources.list.d/hdp.list': + ensure => file, + source => "puppet:///files/repos/hdp.list.${hdp_short_version}", + } + -> + exec { "gpg-updates-import": + command => "gpg --keyserver pgp.mit.edu --recv-keys B9733A7A07513CAD", + path => "$PATH", + } + -> + exec { "gpg-updates-aptkey": + command => "gpg -a --export 07513CAD | apt-key add -", + path => "$PATH", + } + -> + exec { "refresh-apt-cache": + command => "apt-get update", + path => "$PATH", + } } } diff --git a/modules/selinux/manifests/init.pp b/modules/selinux/manifests/init.pp index 77d6eb4..bf831ea 100644 --- a/modules/selinux/manifests/init.pp +++ b/modules/selinux/manifests/init.pp @@ -15,8 +15,10 @@ # Turns of selinux so that mapreduce's task controller will work correctly class selinux { - file { '/etc/selinux/config': - ensure => file, - content => template('selinux/selinux.erb'), + if ($operatingsystem == "centos") { + file { '/etc/selinux/config': + ensure => file, + content => template('selinux/selinux.erb'), + } } -} \ No newline at end of file +} diff --git a/modules/tez_client/manifests/init.pp b/modules/tez_client/manifests/init.pp index b6c29bd..475d090 100644 --- a/modules/tez_client/manifests/init.pp +++ b/modules/tez_client/manifests/init.pp @@ -18,7 +18,7 @@ $conf_dir="/etc/tez/hdp" - package { "tez_${rpm_version}": + package { "tez${package_version}": ensure => installed, } -> @@ -45,4 +45,4 @@ ensure => file, content => template('tez_client/tez-site.erb'), } -} \ No newline at end of file +} diff --git a/modules/tez_client/templates/tez-env.erb b/modules/tez_client/templates/tez-env.erb index c20ba1c..adf3109 100644 --- a/modules/tez_client/templates/tez-env.erb +++ b/modules/tez_client/templates/tez-env.erb @@ -18,4 +18,4 @@ export TEZ_CONF_DIR=/etc/tez/conf # The java implementation to use. -export JAVA_HOME=<%= scope.lookupvar('java_home') %> +export JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> diff --git a/modules/yarn_client/manifests/init.pp b/modules/yarn_client/manifests/init.pp index 971eac4..51fa251 100644 --- a/modules/yarn_client/manifests/init.pp +++ b/modules/yarn_client/manifests/init.pp @@ -15,12 +15,13 @@ class yarn_client { require repos_setup + require jdk require hdfs_client $user_logs = "/user/yarn/" $path="/usr/bin" - package { "hadoop_${rpm_version}-yarn": + package { "hadoop${package_version}-yarn": ensure => installed, } -> @@ -29,7 +30,7 @@ target => "/usr/hdp/${hdp_version}/hadoop/libexec", } - package { "hadoop_${rpm_version}-mapreduce": + package { "hadoop${package_version}-mapreduce": ensure => installed, } -> diff --git a/modules/yarn_client/templates/yarn-env.erb b/modules/yarn_client/templates/yarn-env.erb index 4abe4f6..98fe0a5 100755 --- a/modules/yarn_client/templates/yarn-env.erb +++ b/modules/yarn_client/templates/yarn-env.erb @@ -17,7 +17,7 @@ export YARN_LOG_DIR=<%= scope.lookupvar('hdfs_client::log_dir') %>/$USER export YARN_PID_DIR=<%= scope.lookupvar('hdfs_client::pid_dir') %>/$USER -export JAVA_HOME=<%= scope.lookupvar('java_home') %> +export JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> export YARN_IDENT_STRING="$HADOOP_IDENT_STRING" # User for YARN daemons diff --git a/modules/yarn_node_manager/manifests/init.pp b/modules/yarn_node_manager/manifests/init.pp index f0cded4..9083bb2 100644 --- a/modules/yarn_node_manager/manifests/init.pp +++ b/modules/yarn_node_manager/manifests/init.pp @@ -17,7 +17,7 @@ require yarn_client require hadoop_server - $path="/usr/bin" + $path="/bin:/usr/bin" if $security == "true" { require kerberos_http @@ -38,10 +38,10 @@ mode => 400, } -> - Package["hadoop_${rpm_version}-yarn-nodemanager"] + Package["hadoop${package_version}-yarn-nodemanager"] } - package { "hadoop_${rpm_version}-yarn-nodemanager" : + package { "hadoop${package_version}-yarn-nodemanager" : ensure => installed, } -> @@ -52,7 +52,19 @@ -> file { "/etc/init.d/hadoop-yarn-nodemanager": ensure => 'link', - target => "/usr/hdp/current/hadoop-yarn-nodemanager/../etc/rc.d/init.d/hadoop-yarn-nodemanager", + target => "/usr/hdp/current/hadoop-yarn-nodemanager/../etc/${start_script_path}/hadoop-yarn-nodemanager", + } + -> + exec { "chgrp yarn /usr/hdp/${hdp_version}/hadoop-yarn/bin/container-executor": + # Bug: The Ubuntu packages don't work on secure cluster due to wrong group membership. + cwd => "/", + path => "$path", + } + -> + exec { "chmod 6050 /usr/hdp/${hdp_version}/hadoop-yarn/bin/container-executor": + # Bug: Puppet can't deal with a mode of 6050. + cwd => "/", + path => "$path", } -> service {"hadoop-yarn-nodemanager": @@ -60,4 +72,4 @@ enable => true, } -} \ No newline at end of file +} diff --git a/modules/yarn_resource_manager/manifests/init.pp b/modules/yarn_resource_manager/manifests/init.pp index a9ce079..1b10805 100644 --- a/modules/yarn_resource_manager/manifests/init.pp +++ b/modules/yarn_resource_manager/manifests/init.pp @@ -30,7 +30,7 @@ mode => '400', } -> - Package["hadoop_${rpm_version}-mapreduce-historyserver"] + Package["hadoop${package_version}-mapreduce-historyserver"] file { "${hdfs_client::keytab_dir}/jhs.keytab": ensure => file, @@ -40,10 +40,10 @@ mode => '400', } -> - Package["hadoop_${rpm_version}-yarn-resourcemanager"] + Package["hadoop${package_version}-yarn-resourcemanager"] } - package { "hadoop_${rpm_version}-yarn-resourcemanager" : + package { "hadoop${package_version}-yarn-resourcemanager" : ensure => installed, } -> @@ -54,7 +54,7 @@ -> file { "/etc/init.d/hadoop-yarn-resourcemanager": ensure => 'link', - target => "/usr/hdp/current/hadoop-yarn-resourcemanager/../etc/rc.d/init.d/hadoop-yarn-resourcemanager", + target => "/usr/hdp/current/hadoop-yarn-resourcemanager/../etc/${start_script_path}/hadoop-yarn-resourcemanager", } -> service {"hadoop-yarn-resourcemanager": @@ -62,7 +62,7 @@ enable => true, } - package { "hadoop_${rpm_version}-mapreduce-historyserver" : + package { "hadoop${package_version}-mapreduce-historyserver" : ensure => installed, } -> @@ -73,11 +73,11 @@ -> file { "/etc/init.d/hadoop-mapreduce-historyserver": ensure => 'link', - target => "/usr/hdp/current/hadoop-mapreduce-historyserver/../etc/rc.d/init.d/hadoop-mapreduce-historyserver", + target => "/usr/hdp/current/hadoop-mapreduce-historyserver/../etc/${start_script_path}/hadoop-mapreduce-historyserver", } -> service {"hadoop-mapreduce-historyserver": ensure => running, enable => true, } -} \ No newline at end of file +} diff --git a/modules/zookeeper_client/manifests/init.pp b/modules/zookeeper_client/manifests/init.pp index 8cc5b7f..643e781 100644 --- a/modules/zookeeper_client/manifests/init.pp +++ b/modules/zookeeper_client/manifests/init.pp @@ -16,6 +16,7 @@ class zookeeper_client { require repos_setup require hdp_select + require jdk $conf_dir="/etc/zookeeper/hdp" $log_dir="/var/log/zookeeper" @@ -23,7 +24,7 @@ $pid_dir="/var/run/pid/zookeeper" $path="/usr/bin" - package { "zookeeper_${rpm_version}": + package { "zookeeper${package_version}": ensure => installed, } -> @@ -43,7 +44,7 @@ file { '/etc/zookeeper/conf': ensure => 'link', target => "${conf_dir}", - require => Package["zookeeper_${rpm_version}"], + require => Package["zookeeper${package_version}"], force => true } diff --git a/modules/zookeeper_client/templates/zookeeper-env.erb b/modules/zookeeper_client/templates/zookeeper-env.erb index 14a8277..e088796 100755 --- a/modules/zookeeper_client/templates/zookeeper-env.erb +++ b/modules/zookeeper_client/templates/zookeeper-env.erb @@ -14,11 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. -%> -export JAVA_HOME=<%= scope.lookupvar('java_home') %> +export JAVA_HOME=<%= scope.lookupvar('jdk::HOME') %> export ZOO_LOG_DIR=<%= @log_dir %> export ZOOPIDFILE=<%= @pid_dir %>/zookeeper-server.pid export SERVER_JVMFLAGS=-Xmx<%= @server_mem %>m <% if @security == "true" -%> export SERVER_JVMFLAGS="-Djava.security.auth.login.config=/etc/zookeeper/conf/zookeeper-server.jaas" export CLIENT_JVMFLAGS="-Djava.security.auth.login.config=/etc/zookeeper/conf/zookeeper-client.jaas" -<% end -%> +<% end -%> \ No newline at end of file diff --git a/modules/zookeeper_server/manifests/init.pp b/modules/zookeeper_server/manifests/init.pp index b461ea4..8293468 100644 --- a/modules/zookeeper_server/manifests/init.pp +++ b/modules/zookeeper_server/manifests/init.pp @@ -16,14 +16,14 @@ class zookeeper_server { require zookeeper_client - $path="/usr/bin" + $path="/bin:/sbin:/usr/bin" if $security == "true" { file { "${zookeeper_client::conf_dir}/zookeeper-server.jaas": ensure => file, content => template('zookeeper_server/zookeeper-server.erb'), } - -> Package["zookeeper_${rpm_version}-server"] + -> Service["zookeeper-server"] file { "${hdfs_client::keytab_dir}/zookeeper.keytab": ensure => file, @@ -32,12 +32,34 @@ group => hadoop, mode => '400', } - -> - Package["zookeeper_${rpm_version}-server"] + -> Service["zookeeper-server"] } - package { "zookeeper_${rpm_version}-server": - ensure => installed, + case $operatingsystem { + 'centos': { + package { "zookeeper${package_version}-server": + ensure => installed, + } + } + # XXX: Work around BUG-39010. + 'ubuntu': { + exec { "apt-get download zookeeper${package_version}-server": + cwd => "/tmp", + path => "$path", + } + -> + exec { "dpkg -i --force-overwrite zookeeper${package_version}*.deb": + cwd => "/tmp", + path => "$path", + user => "root", + } + # Fix incorrect startup script permissions (XXX: Is a bug filed for this?). + -> + file { "/usr/hdp/${hdp_version}/etc/init.d/zookeeper-server": + ensure => file, + mode => '755', + } + } } -> exec { "hdp-select set zookeeper-server ${hdp_version}": @@ -52,7 +74,7 @@ -> file { "/etc/init.d/zookeeper-server": ensure => 'link', - target => "/usr/hdp/current/zookeeper-server/../etc/rc.d/init.d/zookeeper-server", + target => "/usr/hdp/current/zookeeper-server/../etc/${start_script_path}/zookeeper-server", } -> file { "${zookeeper_client::data_dir}": @@ -85,4 +107,4 @@ ensure => running, enable => true, } -} \ No newline at end of file +} From 22afe1de25c46b90f1b08480e74d238f844a7bc6 Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 7 Sep 2015 22:33:38 -0700 Subject: [PATCH 16/17] Removing modules not related to Ubuntu. --- modules/phoenix_client/manifests/init.pp | 26 ------------------------ modules/spark/manifests/init.pp | 26 ------------------------ 2 files changed, 52 deletions(-) delete mode 100644 modules/phoenix_client/manifests/init.pp delete mode 100644 modules/spark/manifests/init.pp diff --git a/modules/phoenix_client/manifests/init.pp b/modules/phoenix_client/manifests/init.pp deleted file mode 100644 index 69d2326..0000000 --- a/modules/phoenix_client/manifests/init.pp +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class phoenix_client { - $path="/usr/bin" - - file { "/etc/environment": - content => inline_template("HBASE_CONF_PATH=/etc/hbase/conf") - } - - package { "phoenix_${rpm_version}": - ensure => installed, - } -} diff --git a/modules/spark/manifests/init.pp b/modules/spark/manifests/init.pp deleted file mode 100644 index 64471a2..0000000 --- a/modules/spark/manifests/init.pp +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class spark { - $path="/usr/bin" - package { "spark_${rpm_version}" : - ensure => installed, - } - -> - exec { "hdp-select set spark ${hdp_version}": - cwd => "/", - path => "$path", - } -} From 2b62f64d8f48054ca7ed8aeb817964a20797f168 Mon Sep 17 00:00:00 2001 From: Carter Shanklin Date: Mon, 7 Sep 2015 22:34:38 -0700 Subject: [PATCH 17/17] Removing things not related to Ubuntu. --- profiles/1node-spark-nonsecure.profile | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 profiles/1node-spark-nonsecure.profile diff --git a/profiles/1node-spark-nonsecure.profile b/profiles/1node-spark-nonsecure.profile deleted file mode 100644 index b20e22c..0000000 --- a/profiles/1node-spark-nonsecure.profile +++ /dev/null @@ -1,14 +0,0 @@ -{ - "java_version": "java-1.8.0-openjdk", - "hdp_short_version": "2.3.0", - "domain": "example.com", - "realm": "EXAMPLE.COM", - "security": false, - "vm_mem": 3072, - "server_mem": 300, - "client_mem": 200, - "clients" : [ "hdfs", "zk" ], - "nodes": [ - {"hostname": "spark1", "ip": "240.0.0.11", "roles": ["nn", "zk", "client", "slave", "spark"]} - ] -}