From 76ce379d3a3a2b766612cf78048dd8566a86b461 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Thu, 6 Dec 2012 13:47:42 -0800 Subject: [PATCH 01/22] default to localhost for mongodb instance --- conf/conf.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/conf.ini b/conf/conf.ini index c0655bf..68b19e9 100644 --- a/conf/conf.ini +++ b/conf/conf.ini @@ -1,4 +1,4 @@ [mongodb_info] -mongodb_servers = arch-mongod-s1-01, arch-mongod-s1-02, arch-mongod-s1-03, arch-mongod-s1-04 +mongodb_servers = localhost mongodb_db_name = instances mongodb_collection_name = puppet_enc From 0353fc03c67da263640a8be82cddefaea697382d Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Thu, 6 Dec 2012 13:51:27 -0800 Subject: [PATCH 02/22] search FHS compliant location for config file --- scripts/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/config.py b/scripts/config.py index fb2fcf7..0dcdc76 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -32,7 +32,10 @@ def main(): """ This script adds nodes to the mongodb enc """ parser = SafeConfigParser() - config = os.path.join(os.path.dirname(__file__),"../conf/conf.ini") + if os.path.isfile('/etc/mongodb-enc/conf.ini'): + config = '/etc/mongodb-enc/conf.ini' + else: + config = os.path.join(os.path.dirname(__file__),"../conf/conf.ini") parser.read(config) database = parser.get('mongodb_info', 'mongodb_db_name') collection = parser.get('mongodb_info', 'mongodb_collection_name') From 79aa1829ab11be11ca95663fcb3e01f4d3057f8c Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Thu, 6 Dec 2012 14:10:14 -0800 Subject: [PATCH 03/22] add a spec file to use for building RPMs --- packaging/rpm/mongo-enc.spec | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 packaging/rpm/mongo-enc.spec diff --git a/packaging/rpm/mongo-enc.spec b/packaging/rpm/mongo-enc.spec new file mode 100644 index 0000000..240b3d9 --- /dev/null +++ b/packaging/rpm/mongo-enc.spec @@ -0,0 +1,46 @@ +Name: mongo-enc +Version: 0.1 +Release: 1%{dist} +Summary: MongoDB driven External Node Classifier (ENC) +License: GPLv3 +URL: https://github.com/bcarpio/mongodb-enc +Group: System Environment/Base +Source0: %{name}-%{version}.tar.gz +BuildRoot: %{_tmppath}/%{name}-%{version}-root-%(%{__id_u} -n) +BuildArch: noarch + +Requires: python +Requires: pymongo +Requires: PyYAML +Requires: python-argparse + +%description +A set of scripts which can be used to leverage mongodb as an external node +classifier for puppet. + +%prep +%setup -q -n %{name} + +%install +rm -rf %{buildroot} + +%{__mkdir_p} %{buildroot}%{_bindir}/mongo-enc +%{__mkdir_p} %{buildroot}%{_sysconfdir}/mongo-enc +%{__mkdir_p} %{buildroot}%{_localstatedir}/log/mongo-enc +cp -r ./scripts/* %{buildroot}%{_bindir}/mongo-enc/ +cp -r ./conf/* %{buildroot}%{_sysconfidir}/mongo-enc/ + +%files +%{_bindir}/mongo-enc/* +%{_sysconfdir}/mongo-enc/* + +%pre + +%post + +%clean +rm -rf %{buildroot} + +%changelog +* Thu Dec 6 2012 David Wahlstrom - 0.1-1 +- initial packaging of mongo-enc From ebbec021aac8a062bf278bdb6244560d4394e257 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Fri, 7 Dec 2012 08:08:05 -0800 Subject: [PATCH 04/22] setup.py and add_node.py now accept parameterized classes, but can no longer take multiple classes on a single call (instead, use append for each class) --- scripts/add_node.py | 30 +++++++++++++++++++++--------- scripts/setup.py | 27 +++++++++++++++++++-------- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/scripts/add_node.py b/scripts/add_node.py index d50c855..cd52b3f 100755 --- a/scripts/add_node.py +++ b/scripts/add_node.py @@ -35,8 +35,9 @@ def main(): cmd_parser = argparse.ArgumentParser(description='Add Nodes To Mongodb ENC') cmd_parser.add_argument('-a', '--action', dest='puppet_action', choices=['append', 'new'], help='Append Or Recreate Default Node', required=True) cmd_parser.add_argument('-n', '--node', dest='puppet_node', help='Puppet Node Hostname', required=True) - cmd_parser.add_argument('-c', '--class', dest='puppet_classes', help='Can specify multiple classes each with -c', action='append') - cmd_parser.add_argument('-p', '--param', dest='puppet_param', help='Can specify multiple parameters each with -p', action='append') + cmd_parser.add_argument('-c', '--class', dest='puppet_class', help='Specify a class to add to a node', action='store') + cmd_parser.add_argument('-m', '--classparameters', dest='class_params', help='Can specify multiple class paramters each with -m. Requires a class to be specified', action='append') + cmd_parser.add_argument('-p', '--param', dest='puppet_param', help='Can specify multiple parameters (global variables) each with -p', action='append') cmd_parser.add_argument('-i', '--inherit', dest='puppet_inherit', help='Define a node to inherit classes from', action='store') cmd_parser.add_argument('-e', '--environment', dest='environment', help='Optional, defaults to "production"', default='production') args = cmd_parser.parse_args() @@ -45,10 +46,10 @@ def main(): print "ERROR: Node name and inherit name can not be the same" sys.exit(1) - if args.puppet_classes: + if args.puppet_class: c = {} - for pclass in args.puppet_classes: + for pclass in args.puppet_class: c[pclass] = '' if args.puppet_param: @@ -71,9 +72,21 @@ def main(): print args.puppet_node+" Exists In Mongodb. Please Remove Node" - if args.puppet_classes: - d = { 'node' : args.puppet_node, 'enc' : { 'classes': c, 'environment' : args.environment }} - + if args.puppet_class: + paramclass = {} + if not args.class_params: + paramkeyvalue = '' + else: + paramkeyvalue = {} + for param in args.class_params: + paramvalue = [] + paramkey = '' + for pkey in param.split(','): + paramkey = pkey.split('=')[0] + paramvalue.append(pkey.split('=')[1]) + paramkeyvalue[paramkey] = paramvalue + paramclass[args.puppet_class] = paramkeyvalue + d = { 'node' : args.puppet_node, 'enc' : { 'classes': paramclass, 'environment' : args.environment }} else: d = { 'node' : args.puppet_node, 'enc' : { 'environment' : args.environment }} @@ -82,7 +95,6 @@ def main(): if args.puppet_inherit: d['inherit'] = args.puppet_inherit - col.ensure_index('node', unique=True) col.insert(d) @@ -94,7 +106,7 @@ def main(): print "ERROR: Not Node In Mongo ENC. Please Use -a new" sys.exit(1) - if args.puppet_classes: + if args.puppet_class: if 'classes' in node['enc']: node['enc']['classes'].update(c) diff --git a/scripts/setup.py b/scripts/setup.py index 1a9c0d0..8441654 100755 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -36,29 +36,40 @@ def main(): cmd_parser = argparse.ArgumentParser(description='Add Default Node To Mongodb ENC') cmd_parser.add_argument('-a', '--action', dest='puppet_action', choices=['append', 'new'], help='Append Or Recreate Default Node', required=True) - cmd_parser.add_argument('-c', '--class', dest='puppet_classes', help='Can specify multiple classes each with -c', action='append', required=True) + cmd_parser.add_argument('-c', '--class', dest='puppet_class', help='Specify a puppet class', action='store', required=True) + cmd_parser.add_argument('-m', '--classparameters', dest='class_params', help='Specify multiple parameters for a class with each -m', action='append', required=False) args = cmd_parser.parse_args() - - c = {} + paramclass = {} + paramclass[args.puppet_class] = '' col.ensure_index('node', unique=True) - for pclass in args.puppet_classes: - c[pclass] = '' + if not args.class_params: + paramkeyvalue = '' + else: + paramkeyvalue = {} + for param in args.class_params: + paramvalue = [] + paramkey = '' + for pkey in param.split(','): + paramkey = pkey.split('=')[0] + paramvalue.append(pkey.split('=')[1]) + paramkeyvalue[paramkey] = paramvalue + paramclass[args.puppet_class] = paramkeyvalue if args.puppet_action == 'append': - d = { 'node' : 'default', 'enc' : { 'classes': c }} + d = { 'node' : 'default', 'enc' : { 'classes': paramclass }} check = col.find_one({ 'node' : 'default' }, {'node': 1}) if not check: print "Default Node Doesn't Exist, Please Add It First" sys.exit(1) ec = col.find_one({ 'node' : 'default'}) - ec['enc']['classes'].update(c) + ec['enc']['classes'].update(paramclass) col.remove({ 'node' : 'default'}) col.insert(ec) if args.puppet_action == 'new': - d = { 'node' : 'default', 'enc' : { 'classes': c }} + d = { 'node' : 'default', 'enc' : { 'classes': paramclass }} check = col.find_one({ 'node' : 'default' }, {'node': 1}) if check: col.remove({ 'node' : 'default'}) From 01f72450786dcf33edb24ecf1bb6b0ab44a80649 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Fri, 7 Dec 2012 08:26:16 -0800 Subject: [PATCH 05/22] accept parameters without values --- scripts/setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/setup.py b/scripts/setup.py index 8441654..39db30e 100755 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -53,7 +53,10 @@ def main(): paramkey = '' for pkey in param.split(','): paramkey = pkey.split('=')[0] - paramvalue.append(pkey.split('=')[1]) + if "=" in pkey: + paramvalue.append(pkey.split('=')[1]) + else: + paramvalue = '' paramkeyvalue[paramkey] = paramvalue paramclass[args.puppet_class] = paramkeyvalue From d9cf15d0ceb72910e0704b33e33d0cd7af584f29 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Fri, 7 Dec 2012 08:45:06 -0800 Subject: [PATCH 06/22] update packaging name references from mongo-enc to mongodb-enc --- .../rpm/{mongo-enc.spec => mongodb-enc.spec} | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) rename packaging/rpm/{mongo-enc.spec => mongodb-enc.spec} (66%) diff --git a/packaging/rpm/mongo-enc.spec b/packaging/rpm/mongodb-enc.spec similarity index 66% rename from packaging/rpm/mongo-enc.spec rename to packaging/rpm/mongodb-enc.spec index 240b3d9..ca9a71f 100644 --- a/packaging/rpm/mongo-enc.spec +++ b/packaging/rpm/mongodb-enc.spec @@ -1,4 +1,4 @@ -Name: mongo-enc +Name: mongodb-enc Version: 0.1 Release: 1%{dist} Summary: MongoDB driven External Node Classifier (ENC) @@ -24,15 +24,15 @@ classifier for puppet. %install rm -rf %{buildroot} -%{__mkdir_p} %{buildroot}%{_bindir}/mongo-enc -%{__mkdir_p} %{buildroot}%{_sysconfdir}/mongo-enc -%{__mkdir_p} %{buildroot}%{_localstatedir}/log/mongo-enc -cp -r ./scripts/* %{buildroot}%{_bindir}/mongo-enc/ -cp -r ./conf/* %{buildroot}%{_sysconfidir}/mongo-enc/ +%{__mkdir_p} %{buildroot}%{_bindir}/mongodb-enc +%{__mkdir_p} %{buildroot}%{_sysconfdir}/mongodb-enc +%{__mkdir_p} %{buildroot}%{_localstatedir}/log/mongodb-enc +cp -r ./scripts/* %{buildroot}%{_bindir}/mongodb-enc/ +cp -r ./conf/* %{buildroot}%{_sysconfidir}/mongodb-enc/ %files -%{_bindir}/mongo-enc/* -%{_sysconfdir}/mongo-enc/* +%{_bindir}/mongodb-enc/* +%{_sysconfdir}/mongodb-enc/* %pre @@ -43,4 +43,4 @@ rm -rf %{buildroot} %changelog * Thu Dec 6 2012 David Wahlstrom - 0.1-1 -- initial packaging of mongo-enc +- initial packaging of mongodb-enc From c9d2b1ed3994bbce45df0b5ef101f3cdce600404 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Fri, 7 Dec 2012 08:48:02 -0800 Subject: [PATCH 07/22] fix typo in sysconfdir --- packaging/rpm/mongodb-enc.spec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging/rpm/mongodb-enc.spec b/packaging/rpm/mongodb-enc.spec index ca9a71f..fb8c891 100644 --- a/packaging/rpm/mongodb-enc.spec +++ b/packaging/rpm/mongodb-enc.spec @@ -28,7 +28,7 @@ rm -rf %{buildroot} %{__mkdir_p} %{buildroot}%{_sysconfdir}/mongodb-enc %{__mkdir_p} %{buildroot}%{_localstatedir}/log/mongodb-enc cp -r ./scripts/* %{buildroot}%{_bindir}/mongodb-enc/ -cp -r ./conf/* %{buildroot}%{_sysconfidir}/mongodb-enc/ +cp -r ./conf/* %{buildroot}%{_sysconfdir}/mongodb-enc/ %files %{_bindir}/mongodb-enc/* @@ -44,3 +44,4 @@ rm -rf %{buildroot} %changelog * Thu Dec 6 2012 David Wahlstrom - 0.1-1 - initial packaging of mongodb-enc + From 94a3e211f2c7ac366eccb762e47c5686eed23c8c Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Sun, 9 Dec 2012 07:36:36 -0800 Subject: [PATCH 08/22] provide a default host of "none" when running setup.p, so that there can be an empty host to inherit from --- scripts/add_node.py | 10 ++++------ scripts/setup.py | 4 ++++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/add_node.py b/scripts/add_node.py index cd52b3f..0d7dd1f 100755 --- a/scripts/add_node.py +++ b/scripts/add_node.py @@ -62,9 +62,6 @@ def main(): sys.exit(1) if args.puppet_action == 'new': - if not args.puppet_inherit: - print "ERROR: You Need To Define PUPPET_INHERIT" - sys.exit(1) check = col.find({ 'node' : args.puppet_node }, {'node': 1}) for document in check: node = document['node'] @@ -95,6 +92,8 @@ def main(): if args.puppet_inherit: d['inherit'] = args.puppet_inherit + else: + d['inherit'] = 'none' col.ensure_index('node', unique=True) col.insert(d) @@ -124,9 +123,8 @@ def main(): p = node['enc']['parameters'] col.update({ 'node' : args.puppet_node}, { '$set': {'enc.parameters' : p}}) - if args.puppet_inherit: - node['enc']['inherit'] = args.puppet_inherit - col.update({ 'node' : args.puppet_node}, { '$set' : {'inherit' : args.puppet_inherit}}) + node['enc']['inherit'] = args.puppet_inherit + col.update({ 'node' : args.puppet_node}, { '$set' : {'inherit' : args.puppet_inherit}}) if __name__ == "__main__": diff --git a/scripts/setup.py b/scripts/setup.py index 39db30e..ffb106f 100755 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -78,5 +78,9 @@ def main(): col.remove({ 'node' : 'default'}) col.insert(d) + #empty node so that inheritance can be effectively disabled + empty = { 'node' : 'none', 'enc' : ''} + col.insert(empty) + if __name__ == "__main__": main() From aed402c352cbcec5806768cea1efb85d74bd98cf Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Mon, 10 Dec 2012 08:11:43 -0800 Subject: [PATCH 09/22] apply the inheritance in the correct order, so that node overrides do not get lost --- scripts/mongodb_node_classifier.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/mongodb_node_classifier.py b/scripts/mongodb_node_classifier.py index 7c0c4e4..7d0d5a9 100755 --- a/scripts/mongodb_node_classifier.py +++ b/scripts/mongodb_node_classifier.py @@ -52,15 +52,21 @@ def main(): if 'inherit' in n: i = True while i == True: + # grab the info from the inheritance node inode = n['inherit'] if not col.find_one({"node" : inode}): print "ERROR: Inheritance Node "+inode+" Not Found In ENC" sys.exit(1) idict = col.find_one({"node": inode}) if 'classes' in idict['enc']: + # grab the classes from the inheritance node iclass = idict['enc']['classes'] + # apply inheritance node classes to the requested node if 'classes' in n['enc']: - d['enc']['classes'].update(iclass) + #d['enc']['classes'].update(iclass) + tmp_class_store = d['enc']['classes'] + d['enc']['classes'] = iclass + d['enc']['classes'].update(tmp_class_store) else: d['enc']['classes'] = iclass n = col.find_one({"node": inode}) From a0e6b7f9df3561b8be960d345d10a0725638a138 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Mon, 10 Dec 2012 08:13:36 -0800 Subject: [PATCH 10/22] supply some comments as to what is happening --- scripts/mongodb_node_classifier.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/mongodb_node_classifier.py b/scripts/mongodb_node_classifier.py index 7d0d5a9..ec6ac1f 100755 --- a/scripts/mongodb_node_classifier.py +++ b/scripts/mongodb_node_classifier.py @@ -63,9 +63,11 @@ def main(): iclass = idict['enc']['classes'] # apply inheritance node classes to the requested node if 'classes' in n['enc']: - #d['enc']['classes'].update(iclass) + # grab the requested node's classes tmp_class_store = d['enc']['classes'] + # apply the inheritance node classes d['enc']['classes'] = iclass + # apply the requested node's classes and overrides d['enc']['classes'].update(tmp_class_store) else: d['enc']['classes'] = iclass From 4d1314c0fbaff7233f0eda52cbce9b0004ce43d4 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Mon, 10 Dec 2012 08:53:26 -0800 Subject: [PATCH 11/22] parameterized class support for appending hosts via add_node.py and roll over inheritance if one was not supplied --- scripts/add_node.py | 45 +++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/scripts/add_node.py b/scripts/add_node.py index 0d7dd1f..3b8a0b9 100755 --- a/scripts/add_node.py +++ b/scripts/add_node.py @@ -46,12 +46,6 @@ def main(): print "ERROR: Node name and inherit name can not be the same" sys.exit(1) - if args.puppet_class: - - c = {} - for pclass in args.puppet_class: - c[pclass] = '' - if args.puppet_param: args.puppet_param = dict([arg.split('=') for arg in args.puppet_param]) @@ -68,7 +62,6 @@ def main(): if node == args.puppet_node: print args.puppet_node+" Exists In Mongodb. Please Remove Node" - if args.puppet_class: paramclass = {} if not args.class_params: @@ -97,7 +90,7 @@ def main(): col.ensure_index('node', unique=True) col.insert(d) - + if args.puppet_action == 'append': node = col.find_one({ 'node' : args.puppet_node}) @@ -106,16 +99,32 @@ def main(): sys.exit(1) if args.puppet_class: - + paramclass = {} + if not args.class_params: + paramkeyvalue = '' + else: + paramkeyvalue = {} + for param in args.class_params: + paramvalue = [] + paramkey = '' + for pkey in param.split(','): + paramkey = pkey.split('=')[0] + if "=" in pkey: + paramvalue.append(pkey.split('=')[1]) + else: + paramvalue = '' + paramkeyvalue[paramkey] = paramvalue + paramclass[args.puppet_class] = paramkeyvalue + if 'classes' in node['enc']: - node['enc']['classes'].update(c) + node['enc']['classes'].update(paramclass) else: - node['enc']['classes'] = c - c = node['enc']['classes'] - col.update({ 'node' : args.puppet_node}, { '$set': { 'enc.classes' : c }}) + node['enc']['classes'] = paramclass + paramclass = node['enc']['classes'] + col.update({ 'node' : args.puppet_node}, { '$set': { 'enc.classes' : paramclass }}) if args.puppet_param: - + if 'parameters' in node['enc']: node['enc']['parameters'].update(args.puppet_param) else: @@ -124,8 +133,12 @@ def main(): col.update({ 'node' : args.puppet_node}, { '$set': {'enc.parameters' : p}}) node['enc']['inherit'] = args.puppet_inherit - col.update({ 'node' : args.puppet_node}, { '$set' : {'inherit' : args.puppet_inherit}}) - + if args.puppet_inherit: + pinherit = args.puppet_inherit + else: + pinherit = node['inherit'] + col.update({ 'node' : args.puppet_node}, { '$set' : {'inherit' : pinherit}}) + if __name__ == "__main__": main() From 1d3d8fa592fbd53cdadadb7f9124640118453223 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Mon, 10 Dec 2012 10:13:22 -0800 Subject: [PATCH 12/22] remove lists from single items to ensure that the data is stored correct in the db --- scripts/add_node.py | 13 +++++++++++++ scripts/setup.py | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/scripts/add_node.py b/scripts/add_node.py index 3b8a0b9..c34e82b 100755 --- a/scripts/add_node.py +++ b/scripts/add_node.py @@ -76,6 +76,12 @@ def main(): paramvalue.append(pkey.split('=')[1]) paramkeyvalue[paramkey] = paramvalue paramclass[args.puppet_class] = paramkeyvalue + + for puppetclass in paramclass: + for param in paramclass[puppetclass]: + if type(paramclass[puppetclass][param]) is type(list()): + if len(paramclass[puppetclass][param]) == 1: + paramclass[puppetclass][param] = paramclass[puppetclass][param][0] d = { 'node' : args.puppet_node, 'enc' : { 'classes': paramclass, 'environment' : args.environment }} else: d = { 'node' : args.puppet_node, 'enc' : { 'environment' : args.environment }} @@ -116,11 +122,18 @@ def main(): paramkeyvalue[paramkey] = paramvalue paramclass[args.puppet_class] = paramkeyvalue + for puppetclass in paramclass: + for param in paramclass[puppetclass]: + if type(paramclass[puppetclass][param]) is type(list()): + if len(paramclass[puppetclass][param]) == 1: + paramclass[puppetclass][param] = paramclass[puppetclass][param][0] + if 'classes' in node['enc']: node['enc']['classes'].update(paramclass) else: node['enc']['classes'] = paramclass paramclass = node['enc']['classes'] + col.update({ 'node' : args.puppet_node}, { '$set': { 'enc.classes' : paramclass }}) if args.puppet_param: diff --git a/scripts/setup.py b/scripts/setup.py index ffb106f..10ca645 100755 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -60,6 +60,12 @@ def main(): paramkeyvalue[paramkey] = paramvalue paramclass[args.puppet_class] = paramkeyvalue + for puppetclass in paramclass: + for param in paramclass[puppetclass]: + if type(paramclass[puppetclass][param]) is type(list()): + if len(paramclass[puppetclass][param]) == 1: + paramclass[puppetclass][param] = paramclass[puppetclass][param][0] + if args.puppet_action == 'append': d = { 'node' : 'default', 'enc' : { 'classes': paramclass }} check = col.find_one({ 'node' : 'default' }, {'node': 1}) From d1173809ffde1660aeba1d3ce58b4a496e488f22 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Wed, 12 Dec 2012 07:35:17 -0800 Subject: [PATCH 13/22] pretty much a complete re-write. mongodb_enc.py is an object oriented script that handles adding, modifying, and removing hosts from the database. setup.py was really just adding hosts, so it has been replaced by mongodb_enc.py as well --- README.md | 4 +- scripts/add_node.py | 157 ---------------------- scripts/config.py | 48 ------- scripts/mongodb_enc.py | 207 +++++++++++++++++++++++++++++ scripts/mongodb_node_classifier.py | 21 ++- scripts/remove_node.py | 49 ------- scripts/setup.py | 92 ------------- 7 files changed, 219 insertions(+), 359 deletions(-) delete mode 100755 scripts/add_node.py delete mode 100644 scripts/config.py create mode 100755 scripts/mongodb_enc.py delete mode 100755 scripts/remove_node.py delete mode 100755 scripts/setup.py diff --git a/README.md b/README.md index 1e3ade7..c4d127b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Puppet ENC Using Mongodb +This originally started as a fork of bcarpio/mongodb-enc, however it has turned in to a complete re-write. I no longer consider this a fork, but a new project entirely. + ## Documentation Please refer to the [Wiki](https://github.com/bcarpio/mongodb-enc/wiki). @@ -13,4 +15,4 @@ This project is still in **Alpha**. This means at any given time the code is und Website: [Brian Carpio](http://www.briancarpio.com) -Twitter: [Linsys](https://twitter.com/linsys) \ No newline at end of file +Twitter: [Linsys](https://twitter.com/linsys) diff --git a/scripts/add_node.py b/scripts/add_node.py deleted file mode 100755 index c34e82b..0000000 --- a/scripts/add_node.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/python -# vim: set expandtab: -""" -********************************************************************** -GPL License -*********************************************************************** -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -***********************************************************************/ - -:author: Brian Carpio -:email: bcarpio@thetek.net -:web: http://www.briancarpio.com - -""" -import sys -import argparse -import config - -def main(): - """ This script adds nodes to the mongodb enc """ - - col = config.main() - cmd_parser = argparse.ArgumentParser(description='Add Nodes To Mongodb ENC') - cmd_parser.add_argument('-a', '--action', dest='puppet_action', choices=['append', 'new'], help='Append Or Recreate Default Node', required=True) - cmd_parser.add_argument('-n', '--node', dest='puppet_node', help='Puppet Node Hostname', required=True) - cmd_parser.add_argument('-c', '--class', dest='puppet_class', help='Specify a class to add to a node', action='store') - cmd_parser.add_argument('-m', '--classparameters', dest='class_params', help='Can specify multiple class paramters each with -m. Requires a class to be specified', action='append') - cmd_parser.add_argument('-p', '--param', dest='puppet_param', help='Can specify multiple parameters (global variables) each with -p', action='append') - cmd_parser.add_argument('-i', '--inherit', dest='puppet_inherit', help='Define a node to inherit classes from', action='store') - cmd_parser.add_argument('-e', '--environment', dest='environment', help='Optional, defaults to "production"', default='production') - args = cmd_parser.parse_args() - - if args.puppet_node == args.puppet_inherit != 'default' : - print "ERROR: Node name and inherit name can not be the same" - sys.exit(1) - - if args.puppet_param: - args.puppet_param = dict([arg.split('=') for arg in args.puppet_param]) - - if args.puppet_inherit: - ck = col.find_one({ "node" : args.puppet_inherit}) - if not ck: - print "ERROR: Inherit node does not exist, please add "+args.puppet_inherit+" and then retry" - sys.exit(1) - - if args.puppet_action == 'new': - check = col.find({ 'node' : args.puppet_node }, {'node': 1}) - for document in check: - node = document['node'] - if node == args.puppet_node: - print args.puppet_node+" Exists In Mongodb. Please Remove Node" - - if args.puppet_class: - paramclass = {} - if not args.class_params: - paramkeyvalue = '' - else: - paramkeyvalue = {} - for param in args.class_params: - paramvalue = [] - paramkey = '' - for pkey in param.split(','): - paramkey = pkey.split('=')[0] - paramvalue.append(pkey.split('=')[1]) - paramkeyvalue[paramkey] = paramvalue - paramclass[args.puppet_class] = paramkeyvalue - - for puppetclass in paramclass: - for param in paramclass[puppetclass]: - if type(paramclass[puppetclass][param]) is type(list()): - if len(paramclass[puppetclass][param]) == 1: - paramclass[puppetclass][param] = paramclass[puppetclass][param][0] - d = { 'node' : args.puppet_node, 'enc' : { 'classes': paramclass, 'environment' : args.environment }} - else: - d = { 'node' : args.puppet_node, 'enc' : { 'environment' : args.environment }} - - if args.puppet_param: - d['enc']['parameters'] = args.puppet_param - - if args.puppet_inherit: - d['inherit'] = args.puppet_inherit - else: - d['inherit'] = 'none' - - col.ensure_index('node', unique=True) - col.insert(d) - - if args.puppet_action == 'append': - - node = col.find_one({ 'node' : args.puppet_node}) - if node == None: - print "ERROR: Not Node In Mongo ENC. Please Use -a new" - sys.exit(1) - - if args.puppet_class: - paramclass = {} - if not args.class_params: - paramkeyvalue = '' - else: - paramkeyvalue = {} - for param in args.class_params: - paramvalue = [] - paramkey = '' - for pkey in param.split(','): - paramkey = pkey.split('=')[0] - if "=" in pkey: - paramvalue.append(pkey.split('=')[1]) - else: - paramvalue = '' - paramkeyvalue[paramkey] = paramvalue - paramclass[args.puppet_class] = paramkeyvalue - - for puppetclass in paramclass: - for param in paramclass[puppetclass]: - if type(paramclass[puppetclass][param]) is type(list()): - if len(paramclass[puppetclass][param]) == 1: - paramclass[puppetclass][param] = paramclass[puppetclass][param][0] - - if 'classes' in node['enc']: - node['enc']['classes'].update(paramclass) - else: - node['enc']['classes'] = paramclass - paramclass = node['enc']['classes'] - - col.update({ 'node' : args.puppet_node}, { '$set': { 'enc.classes' : paramclass }}) - - if args.puppet_param: - - if 'parameters' in node['enc']: - node['enc']['parameters'].update(args.puppet_param) - else: - node['enc']['parameters'] = args.puppet_param - p = node['enc']['parameters'] - col.update({ 'node' : args.puppet_node}, { '$set': {'enc.parameters' : p}}) - - node['enc']['inherit'] = args.puppet_inherit - if args.puppet_inherit: - pinherit = args.puppet_inherit - else: - pinherit = node['inherit'] - col.update({ 'node' : args.puppet_node}, { '$set' : {'inherit' : pinherit}}) - - -if __name__ == "__main__": - main() diff --git a/scripts/config.py b/scripts/config.py deleted file mode 100644 index 0dcdc76..0000000 --- a/scripts/config.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/python -# vim: set expandtab: -""" -********************************************************************** -GPL License -*********************************************************************** -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -***********************************************************************/ - -:author: Brian Carpio -:email: bcarpio@thetek.net -:web: http://www.briancarpio.com - -""" -from pymongo import Connection -import os -from ConfigParser import SafeConfigParser - -def main(): - """ This script adds nodes to the mongodb enc """ - - parser = SafeConfigParser() - if os.path.isfile('/etc/mongodb-enc/conf.ini'): - config = '/etc/mongodb-enc/conf.ini' - else: - config = os.path.join(os.path.dirname(__file__),"../conf/conf.ini") - parser.read(config) - database = parser.get('mongodb_info', 'mongodb_db_name') - collection = parser.get('mongodb_info', 'mongodb_collection_name') - host = parser.get('mongodb_info', 'mongodb_servers') - con = Connection(host) - col = con[database][collection] - return col - -if __name__ == "__main__": - main() diff --git a/scripts/mongodb_enc.py b/scripts/mongodb_enc.py new file mode 100755 index 0000000..14ae7bc --- /dev/null +++ b/scripts/mongodb_enc.py @@ -0,0 +1,207 @@ +#!/usr/bin/python +# vim: set expandtab: +""" +********************************************************************** +GPL License +*********************************************************************** +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +***********************************************************************/ + +:author: David Wahlstrom +:email: david.wahlstrom@gmail.com + +""" + + +class Node(object): + """Class to facilitate adding/modifying a node""" + + def __init__(self, nodename, nodeclass=None, classparams=None, nodeparam=None, puppet_inherit=None, environment=None): + """Initialize variables for use""" + + #create an initial connection to the mongodb + self.mongo_collection = self.configure() + + #verify the inherit node exists + if puppet_inherit: + if self.verifynode(puppet_inherit) is False: + print "ERROR: Inherit node does not exist, please add %s and then retry" % puppet_inherit + else: + self.puppet_inherit = puppet_inherit + self.node = nodename + if nodeclass: + self.nodeclass = nodeclass + else: + self.nodeclass = '' + self.classparams = classparams + self.nodeparam = nodeparam + if puppet_inherit: + self.puppet_inherit = puppet_inherit + else: + self.puppet_inherit = 'none' + if environment: + self.environment = environment + else: + self.environment = '' + + def configure(self): + """Read configuration file and intialize connection to the mongodb instance""" + + from pymongo import Connection + import os + from ConfigParser import SafeConfigParser + + parser = SafeConfigParser() + if os.path.isfile('/etc/mongodb-enc/conf.ini'): + config = '/etc/mongodb-enc/conf.ini' + else: + config = os.path.join(os.path.dirname(__file__), "../conf/conf.ini") + parser.read(config) + database = parser.get('mongodb_info', 'mongodb_db_name') + collection = parser.get('mongodb_info', 'mongodb_collection_name') + host = parser.get('mongodb_info', 'mongodb_servers') + con = Connection(host) + col = con[database][collection] + return col + + def verifynode(self, vernode=None): + """Verify that the node exists in the DB. Returns True if it does exist, Return False if it does not""" + + if not vernode: + testnode = self.node + else: + testnode = vernode + + docnode = self.mongo_collection.find_one({'node': testnode}) + if docnode is None: + return False + else: + if docnode['node'] == testnode: + return True + else: + return False + + def parse_node_classification(self, puppet_class=None, class_params=None, parameters=None, environment=None, inherit=None): + """Parse puppet_class, class_params, parameters, and environment, and return a dict with the result""" + + puppet_enc = { + 'classes': '', + 'environment': '', + 'parameters': '', + } + + if not puppet_class: + puppet_class = self.nodeclass + puppet_enc['classes'] = puppet_class + + if not class_params: + class_params = self.classparams + + if parameters: + puppet_enc['parameters'] = parameters + else: + if self.nodeparam: + puppet_enc['parameters'] = self.nodeparam + else: + puppet_enc['parameters'] = '' + + if environment: + puppet_enc['environment'] = environment + else: + puppet_enc['environment'] = self.environment + + paramclass = {} + try: + paramclass = self.mongo_collection.find_one({'node': self.node})['enc']['classes'] + except: + paramclass = {} + + if not paramclass: + paramclass = {} + + #Pull parameters for each class + if puppet_class: + paramkeyvalue = {} + paramvalue = [] + if not class_params: + paramkeyvalue = '' + else: + for param in class_params.split(','): + paramkey = '' + paramkey = param.split('=')[0] + paramvalue.append(param.split('=')[1]) + paramkeyvalue[paramkey] = paramvalue + + paramclass[puppet_class] = paramkeyvalue + + #Since we stored them as lists above, reduce single item lists to strings + for puppetclass in paramclass: + for param in paramclass[puppetclass]: + if type(paramclass[puppetclass][param]) is type(list()): + if len(paramclass[puppetclass][param]) == 1: + paramclass[puppetclass][param] = paramclass[puppetclass][param][0] + + puppet_enc['classes'] = paramclass + + return puppet_enc + + def update(self, enc, inherit=None): + """Add/modify "enc" properties on "node" """ + + if inherit: + tmp_inherit = inherit + else: + tmp_inherit = self.puppet_inherit + + if self.verifynode('none'): + self.mongo_collection.update({'node': self.node}, {"$set": {'enc': enc, 'inherit': tmp_inherit}}, True) + else: + self.mongo_collection.update({'node': 'none'}, {"$set": {'enc': '', 'inherit': ''}}, True) + self.mongo_collection.update({'node': self.node}, {"$set": {'enc': enc, 'inherit': tmp_inherit}}, True) + + def remove(self, rmnode=None): + """Remove node from mongodb""" + + if rmnode: + nodename = rmnode + else: + nodename = self.node + + nodes_with_inheritance = self.mongo_collection.find({'inherit': nodename}) + for inheritnode in nodes_with_inheritance: + print "%s inherits the node you are trying to remove." % inheritnode['node'] + + self.mongo_collection.remove({'node': nodename}) + +if __name__ == "__main__": + + import argparse + + cmd_parser = argparse.ArgumentParser(description='Add/remove/modify nodes in MongoDB ENC.') + cmd_parser.add_argument('-n', '--node', dest='puppet_node', help='Puppet node hostname.', required=True) + cmd_parser.add_argument('-r', '--remove', dest='remove_attr', help='Remove supplied attributes from host.', action='store_true', default=False) + cmd_parser.add_argument('-c', '--class', dest='puppet_class', help='Apply (or remove with -r) class on node (-n). Parameters can be passed to the class with -m.', action='store', default=None) + cmd_parser.add_argument('-m', '--classparameters', dest='class_params', help='Apply (or remove with -r) class parameters to class (-c) on node (-n).', action='store', default=None) + cmd_parser.add_argument('-p', '--param', dest='puppet_param', help='Apply (or remove with -r) parameters (global variables) to node (-n).', action='store', default=None) + cmd_parser.add_argument('-i', '--inherit', dest='puppet_inherit', help='Apply (or remove with -r) inherit node. Only a single node can be set to inherit from.', action='store', default=None) + cmd_parser.add_argument('-e', '--environment', dest='environment', help='Apply (or remove with -r) puppet agent environment.', default=None) + args = cmd_parser.parse_args() + + node = Node(args.puppet_node, args.puppet_class, args.class_params, args.puppet_param, args.puppet_inherit, args.environment) + + if args.remove_attr is True: + node.remove(node.node) + else: + node.update(node.parse_node_classification()) diff --git a/scripts/mongodb_node_classifier.py b/scripts/mongodb_node_classifier.py index ec6ac1f..bacc986 100755 --- a/scripts/mongodb_node_classifier.py +++ b/scripts/mongodb_node_classifier.py @@ -31,20 +31,17 @@ def main(): """ This script is called by puppet """ if (len(sys.argv) < 2): - print "ERROR: Please Supply A Hostname or FQDN" + print "ERROR: Please supply a hostname or FQDN" sys.exit(1) col = config.main() - # Probably want to remove this. This is because I don't use FQDNs in my current puppet manifest. - # also made this easier for me to test. node = sys.argv[1] - #node = node.split('.')[0] # Find the node given at a command line argument d = col.find_one({"node": node}) if d == None: - print "ERROR: Node "+node+" Not Found In ENC" + print "ERROR: Node "+node+" not found in ENC" sys.exit(1) # Check if the node requiers inheritance @@ -52,22 +49,22 @@ def main(): if 'inherit' in n: i = True while i == True: - # grab the info from the inheritance node + # Grab the info from the inheritance node inode = n['inherit'] if not col.find_one({"node" : inode}): - print "ERROR: Inheritance Node "+inode+" Not Found In ENC" + print "ERROR: Inheritance node "+inode+" not found in ENC" sys.exit(1) idict = col.find_one({"node": inode}) if 'classes' in idict['enc']: - # grab the classes from the inheritance node + # Grab the classes from the inheritance node iclass = idict['enc']['classes'] - # apply inheritance node classes to the requested node + # Apply inheritance node classes to the requested node if 'classes' in n['enc']: - # grab the requested node's classes + # Grab the requested node's classes tmp_class_store = d['enc']['classes'] - # apply the inheritance node classes + # Apply the inheritance node classes d['enc']['classes'] = iclass - # apply the requested node's classes and overrides + # Apply the requested node's classes and overrides d['enc']['classes'].update(tmp_class_store) else: d['enc']['classes'] = iclass diff --git a/scripts/remove_node.py b/scripts/remove_node.py deleted file mode 100755 index d0383fc..0000000 --- a/scripts/remove_node.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/python -# vim: set expandtab: -""" -********************************************************************** -GPL License -*********************************************************************** -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -***********************************************************************/ - -:author: Brian Carpio -:email: bcarpio@thetek.net -:web: http://www.briancarpio.com - -""" -import os -import argparse -import config - -def main(): - - """ This script removes nodes from mongodb """ - col = config.main() - - cmd_parser = argparse.ArgumentParser(description='Remove Nodes To Mongodb ENC') - cmd_parser.add_argument('-n', '--node', dest='puppet_node', help='Puppet Node Hostname', required=True) - args = cmd_parser.parse_args() - - isinode = col.find_one({ "inherit" : args.puppet_node }) - if isinode: - isinode = col.find({ "inherit" : args.puppet_node }) - for node in isinode: - print "ERROR: "+args.puppet_node+" is inherited by "+node['node'] - else: - col.remove({ 'node' : args.puppet_node}) - -if __name__ == "__main__": - main() diff --git a/scripts/setup.py b/scripts/setup.py deleted file mode 100755 index 10ca645..0000000 --- a/scripts/setup.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/python -# vim: set expandtab: -""" -********************************************************************** -GPL License -*********************************************************************** -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -***********************************************************************/ - -:author: Brian Carpio -:email: bcarpio@thetek.net -:web: http://www.briancarpio.com - -""" -import sys -import argparse -import config - - -def main(): - """ This script creates the default node definition """ - - col = config.main() - - cmd_parser = argparse.ArgumentParser(description='Add Default Node To Mongodb ENC') - cmd_parser.add_argument('-a', '--action', dest='puppet_action', choices=['append', 'new'], help='Append Or Recreate Default Node', required=True) - cmd_parser.add_argument('-c', '--class', dest='puppet_class', help='Specify a puppet class', action='store', required=True) - cmd_parser.add_argument('-m', '--classparameters', dest='class_params', help='Specify multiple parameters for a class with each -m', action='append', required=False) - args = cmd_parser.parse_args() - - paramclass = {} - paramclass[args.puppet_class] = '' - col.ensure_index('node', unique=True) - - if not args.class_params: - paramkeyvalue = '' - else: - paramkeyvalue = {} - for param in args.class_params: - paramvalue = [] - paramkey = '' - for pkey in param.split(','): - paramkey = pkey.split('=')[0] - if "=" in pkey: - paramvalue.append(pkey.split('=')[1]) - else: - paramvalue = '' - paramkeyvalue[paramkey] = paramvalue - paramclass[args.puppet_class] = paramkeyvalue - - for puppetclass in paramclass: - for param in paramclass[puppetclass]: - if type(paramclass[puppetclass][param]) is type(list()): - if len(paramclass[puppetclass][param]) == 1: - paramclass[puppetclass][param] = paramclass[puppetclass][param][0] - - if args.puppet_action == 'append': - d = { 'node' : 'default', 'enc' : { 'classes': paramclass }} - check = col.find_one({ 'node' : 'default' }, {'node': 1}) - if not check: - print "Default Node Doesn't Exist, Please Add It First" - sys.exit(1) - ec = col.find_one({ 'node' : 'default'}) - ec['enc']['classes'].update(paramclass) - col.remove({ 'node' : 'default'}) - col.insert(ec) - - if args.puppet_action == 'new': - d = { 'node' : 'default', 'enc' : { 'classes': paramclass }} - check = col.find_one({ 'node' : 'default' }, {'node': 1}) - if check: - col.remove({ 'node' : 'default'}) - col.insert(d) - - #empty node so that inheritance can be effectively disabled - empty = { 'node' : 'none', 'enc' : ''} - col.insert(empty) - -if __name__ == "__main__": - main() From a53efd8b129cedf1581327fd9f1bd646eb6ba414 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Wed, 12 Dec 2012 08:32:39 -0800 Subject: [PATCH 14/22] rename variable in config file to make more sense --- conf/conf.ini | 2 +- scripts/mongodb_enc.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/conf.ini b/conf/conf.ini index 68b19e9..d8e4f74 100644 --- a/conf/conf.ini +++ b/conf/conf.ini @@ -1,4 +1,4 @@ [mongodb_info] -mongodb_servers = localhost +mongodb_server = localhost mongodb_db_name = instances mongodb_collection_name = puppet_enc diff --git a/scripts/mongodb_enc.py b/scripts/mongodb_enc.py index 14ae7bc..acd9de3 100755 --- a/scripts/mongodb_enc.py +++ b/scripts/mongodb_enc.py @@ -64,14 +64,14 @@ def configure(self): from ConfigParser import SafeConfigParser parser = SafeConfigParser() - if os.path.isfile('/etc/mongodb-enc/conf.ini'): - config = '/etc/mongodb-enc/conf.ini' + if os.path.isfile('/etc/mongodb_enc/conf.ini'): + config = '/etc/mongodb_enc/conf.ini' else: config = os.path.join(os.path.dirname(__file__), "../conf/conf.ini") parser.read(config) database = parser.get('mongodb_info', 'mongodb_db_name') collection = parser.get('mongodb_info', 'mongodb_collection_name') - host = parser.get('mongodb_info', 'mongodb_servers') + host = parser.get('mongodb_info', 'mongodb_server') con = Connection(host) col = con[database][collection] return col From 4b10963591e508ffe428aaf29b8fa089a9f659d3 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Wed, 12 Dec 2012 08:56:19 -0800 Subject: [PATCH 15/22] since config.py is gone, add config parsing to mongodb_node_classifier.py. also handle missing inherits more gracefully --- scripts/mongodb_enc.py | 9 ++-- scripts/mongodb_node_classifier.py | 71 ++++++++++++++++++------------ 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/scripts/mongodb_enc.py b/scripts/mongodb_enc.py index acd9de3..da913d3 100755 --- a/scripts/mongodb_enc.py +++ b/scripts/mongodb_enc.py @@ -165,11 +165,10 @@ def update(self, enc, inherit=None): else: tmp_inherit = self.puppet_inherit - if self.verifynode('none'): - self.mongo_collection.update({'node': self.node}, {"$set": {'enc': enc, 'inherit': tmp_inherit}}, True) - else: - self.mongo_collection.update({'node': 'none'}, {"$set": {'enc': '', 'inherit': ''}}, True) - self.mongo_collection.update({'node': self.node}, {"$set": {'enc': enc, 'inherit': tmp_inherit}}, True) + #if not self.verifynode('none'): + #self.mongo_collection.update({'node': 'none'}, {"$set": {'enc': '', 'inherit': ''}}, True) + + self.mongo_collection.update({'node': self.node}, {"$set": {'enc': enc, 'inherit': tmp_inherit}}, True) def remove(self, rmnode=None): """Remove node from mongodb""" diff --git a/scripts/mongodb_node_classifier.py b/scripts/mongodb_node_classifier.py index bacc986..00374f8 100755 --- a/scripts/mongodb_node_classifier.py +++ b/scripts/mongodb_node_classifier.py @@ -26,7 +26,26 @@ """ import yaml import sys -import config + +def configure(): + """Read configuration file and intialize connection to the mongodb instance""" + + from pymongo import Connection + import os + from ConfigParser import SafeConfigParser + + parser = SafeConfigParser() + if os.path.isfile('/etc/mongodb_enc/conf.ini'): + config = '/etc/mongodb_enc/conf.ini' + else: + config = os.path.join(os.path.dirname(__file__), "../conf/conf.ini") + parser.read(config) + database = parser.get('mongodb_info', 'mongodb_db_name') + collection = parser.get('mongodb_info', 'mongodb_collection_name') + host = parser.get('mongodb_info', 'mongodb_server') + con = Connection(host) + col = con[database][collection] + return col def main(): """ This script is called by puppet """ @@ -34,7 +53,7 @@ def main(): print "ERROR: Please supply a hostname or FQDN" sys.exit(1) - col = config.main() + col = configure() node = sys.argv[1] @@ -46,31 +65,29 @@ def main(): # Check if the node requiers inheritance n = col.find_one({"node": node}) - if 'inherit' in n: - i = True - while i == True: - # Grab the info from the inheritance node - inode = n['inherit'] - if not col.find_one({"node" : inode}): - print "ERROR: Inheritance node "+inode+" not found in ENC" - sys.exit(1) - idict = col.find_one({"node": inode}) - if 'classes' in idict['enc']: - # Grab the classes from the inheritance node - iclass = idict['enc']['classes'] - # Apply inheritance node classes to the requested node - if 'classes' in n['enc']: - # Grab the requested node's classes - tmp_class_store = d['enc']['classes'] - # Apply the inheritance node classes - d['enc']['classes'] = iclass - # Apply the requested node's classes and overrides - d['enc']['classes'].update(tmp_class_store) - else: - d['enc']['classes'] = iclass - n = col.find_one({"node": inode}) - if 'inherit' not in n: - i = False + if n['inherit']: + # Grab the info from the inheritance node + inode = n['inherit'] + if not col.find_one({"node" : inode}): + print "ERROR: Inheritance node "+inode+" not found in ENC" + sys.exit(1) + idict = col.find_one({"node": inode}) + if 'classes' in idict['enc']: + # Grab the classes from the inheritance node + iclass = idict['enc']['classes'] + # Apply inheritance node classes to the requested node + #if 'classes' in n['enc']: + if iclass: + # Grab the requested node's classes + tmp_class_store = d['enc']['classes'] + # Apply the inheritance node classes + d['enc']['classes'] = iclass + # Apply the requested node's classes and overrides + d['enc']['classes'].update(tmp_class_store) + else: + d['enc']['classes'] = n['enc']['classes'] + else: + d = n print yaml.safe_dump(d['enc'], default_flow_style=False) From dfff93a6659b945e85dc06aefac0ea0bb89a76fa Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Wed, 12 Dec 2012 09:21:15 -0800 Subject: [PATCH 16/22] handle key error on inherit --- scripts/mongodb_node_classifier.py | 45 +++++++++++++++--------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/scripts/mongodb_node_classifier.py b/scripts/mongodb_node_classifier.py index 00374f8..bf9c916 100755 --- a/scripts/mongodb_node_classifier.py +++ b/scripts/mongodb_node_classifier.py @@ -65,28 +65,29 @@ def main(): # Check if the node requiers inheritance n = col.find_one({"node": node}) - if n['inherit']: - # Grab the info from the inheritance node - inode = n['inherit'] - if not col.find_one({"node" : inode}): - print "ERROR: Inheritance node "+inode+" not found in ENC" - sys.exit(1) - idict = col.find_one({"node": inode}) - if 'classes' in idict['enc']: - # Grab the classes from the inheritance node - iclass = idict['enc']['classes'] - # Apply inheritance node classes to the requested node - #if 'classes' in n['enc']: - if iclass: - # Grab the requested node's classes - tmp_class_store = d['enc']['classes'] - # Apply the inheritance node classes - d['enc']['classes'] = iclass - # Apply the requested node's classes and overrides - d['enc']['classes'].update(tmp_class_store) - else: - d['enc']['classes'] = n['enc']['classes'] - else: + try: + if n['inherit']: + # Grab the info from the inheritance node + inode = n['inherit'] + if not col.find_one({"node" : inode}): + print "ERROR: Inheritance node "+inode+" not found in ENC" + sys.exit(1) + idict = col.find_one({"node": inode}) + if 'classes' in idict['enc']: + # Grab the classes from the inheritance node + iclass = idict['enc']['classes'] + # Apply inheritance node classes to the requested node + #if 'classes' in n['enc']: + if iclass: + # Grab the requested node's classes + tmp_class_store = d['enc']['classes'] + # Apply the inheritance node classes + d['enc']['classes'] = iclass + # Apply the requested node's classes and overrides + d['enc']['classes'].update(tmp_class_store) + else: + d['enc']['classes'] = n['enc']['classes'] + except KeyError: d = n print yaml.safe_dump(d['enc'], default_flow_style=False) From 862dce55b1b00398f4956c4514738b3458a60b58 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Thu, 13 Dec 2012 08:14:15 -0800 Subject: [PATCH 17/22] puppet prefers empty arrays to be returned instead of empty strings --- scripts/mongodb_enc.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/mongodb_enc.py b/scripts/mongodb_enc.py index da913d3..e8d914c 100755 --- a/scripts/mongodb_enc.py +++ b/scripts/mongodb_enc.py @@ -122,6 +122,15 @@ def parse_node_classification(self, puppet_class=None, class_params=None, parame else: puppet_enc['environment'] = self.environment + if not puppet_enc['classes']: + puppet_enc['classes'] = [] + + if not puppet_enc['parameters']: + puppet_enc['parameters'] = {} + + if not puppet_enc['environment']: + del puppet_enc['environment'] + paramclass = {} try: paramclass = self.mongo_collection.find_one({'node': self.node})['enc']['classes'] From 082ea73158df093529fe62377791a5f2c9160e31 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Thu, 13 Dec 2012 10:02:42 -0800 Subject: [PATCH 18/22] update URL in spec file --- packaging/rpm/mongodb-enc.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/rpm/mongodb-enc.spec b/packaging/rpm/mongodb-enc.spec index fb8c891..28a0164 100644 --- a/packaging/rpm/mongodb-enc.spec +++ b/packaging/rpm/mongodb-enc.spec @@ -3,7 +3,7 @@ Version: 0.1 Release: 1%{dist} Summary: MongoDB driven External Node Classifier (ENC) License: GPLv3 -URL: https://github.com/bcarpio/mongodb-enc +URL: https://github.com/drwahl/mongodb-enc Group: System Environment/Base Source0: %{name}-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-root-%(%{__id_u} -n) From e4dd7afa7ea092f464a4463796497508614aac09 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Wed, 19 Dec 2012 08:59:43 -0800 Subject: [PATCH 19/22] update README.md --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c4d127b..19cb7ad 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,8 @@ Please refer to the [Wiki](https://github.com/bcarpio/mongodb-enc/wiki). ## Project Status -This project is still in **Alpha**. This means at any given time the code is under active development. There is still one big change coming where I change the mongodb schema. I currently don't like how I have the mongodb laid out. If you are using this code and would like to make feature requests please do so in the issue tracker. +This project is still in **Alpha**. This means at any given time the code is under active development. If you are using this code and would like to make feature requests please do so in the issue tracker. ## Author Info -Website: [Brian Carpio](http://www.briancarpio.com) - -Twitter: [Linsys](https://twitter.com/linsys) +Website: [David Wahlstrom](http://www.linkedin.com/in/dwahlstrom/) From 1d0671d4dda644123376f2fc3520879b7048eb18 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Fri, 21 Dec 2012 06:35:53 -0800 Subject: [PATCH 20/22] provide empty class for node "none" --- scripts/mongodb_enc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/mongodb_enc.py b/scripts/mongodb_enc.py index e8d914c..e54472b 100755 --- a/scripts/mongodb_enc.py +++ b/scripts/mongodb_enc.py @@ -174,8 +174,8 @@ def update(self, enc, inherit=None): else: tmp_inherit = self.puppet_inherit - #if not self.verifynode('none'): - #self.mongo_collection.update({'node': 'none'}, {"$set": {'enc': '', 'inherit': ''}}, True) + if not self.verifynode('none'): + self.mongo_collection.update({'node': 'none'}, {"$set": {'enc': {'classes': []}, 'inherit': ''}}, True) self.mongo_collection.update({'node': self.node}, {"$set": {'enc': enc, 'inherit': tmp_inherit}}, True) From adbc7686ad6ea40c6870da962ad69a24e30a37e5 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Fri, 21 Dec 2012 08:28:05 -0800 Subject: [PATCH 21/22] re-write mongo_node_classifier.py to follow multiple layers of inheritance --- scripts/mongodb_node_classifier.py | 76 ++++++++++++++++-------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/scripts/mongodb_node_classifier.py b/scripts/mongodb_node_classifier.py index bf9c916..5bc196d 100755 --- a/scripts/mongodb_node_classifier.py +++ b/scripts/mongodb_node_classifier.py @@ -19,9 +19,8 @@ ***********************************************************************/ -:author: Brian Carpio -:email: bcarpio@thetek.net -:web: http://www.briancarpio.com +:author: David Wahlstrom +:email: david.wahlstrom@gmail.com """ import yaml @@ -47,51 +46,58 @@ def configure(): col = con[database][collection] return col -def main(): - """ This script is called by puppet """ +def classify(cnode): + """Classify the requested node, including inheritances""" + + col = configure() + try: + node_classes = col.find_one({'node': cnode})['enc']['classes'] + except TypeError: + node_classes = {} + + try: + # Grab the info from the inheritance node + inode = col.find_one({'node': cnode}) + if not inode['inherit']: + raise KeyError + inode_classes = classify(inode['inherit']) + if not col.find_one({"node" : cnode}): + print "ERROR: Inheritance node " + cnode + " not found in ENC" + sys.exit(1) + node_classes = col.find_one({"node": cnode})['enc']['classes'] + # Grab the requested node's classes + tmp_class_store = node_classes + # Apply the inheritance node classes + node_classes = dict(inode_classes) + # Apply the requested node's classes and overrides + node_classes.update(tmp_class_store) + except KeyError: + pass + except TypeError: + pass + + return node_classes + +def main(node): + """This script is called by puppet""" + if (len(sys.argv) < 2): print "ERROR: Please supply a hostname or FQDN" sys.exit(1) col = configure() - node = sys.argv[1] - # Find the node given at a command line argument d = col.find_one({"node": node}) if d == None: - print "ERROR: Node "+node+" not found in ENC" + print "ERROR: Node %s not found in ENC" % node sys.exit(1) - # Check if the node requiers inheritance - n = col.find_one({"node": node}) - try: - if n['inherit']: - # Grab the info from the inheritance node - inode = n['inherit'] - if not col.find_one({"node" : inode}): - print "ERROR: Inheritance node "+inode+" not found in ENC" - sys.exit(1) - idict = col.find_one({"node": inode}) - if 'classes' in idict['enc']: - # Grab the classes from the inheritance node - iclass = idict['enc']['classes'] - # Apply inheritance node classes to the requested node - #if 'classes' in n['enc']: - if iclass: - # Grab the requested node's classes - tmp_class_store = d['enc']['classes'] - # Apply the inheritance node classes - d['enc']['classes'] = iclass - # Apply the requested node's classes and overrides - d['enc']['classes'].update(tmp_class_store) - else: - d['enc']['classes'] = n['enc']['classes'] - except KeyError: - d = n + # Classify node + d['enc']['classes'] = classify(node) print yaml.safe_dump(d['enc'], default_flow_style=False) if __name__ == "__main__": - main() + main(sys.argv[1]) From 6a028028899e9cc8d43fc90998a7365b0b85b4f4 Mon Sep 17 00:00:00 2001 From: David Wahlstrom Date: Wed, 2 Jan 2013 07:56:03 -0800 Subject: [PATCH 22/22] supply note of the new location of code development (yape) --- README.md | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 19cb7ad..330aef5 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,3 @@ -# Puppet ENC Using Mongodb +THIS PROJECT HAS BEEN MOVED TO https://github.com/drwahl/yape.git -This originally started as a fork of bcarpio/mongodb-enc, however it has turned in to a complete re-write. I no longer consider this a fork, but a new project entirely. - -## Documentation - -Please refer to the [Wiki](https://github.com/bcarpio/mongodb-enc/wiki). - - -## Project Status - -This project is still in **Alpha**. This means at any given time the code is under active development. If you are using this code and would like to make feature requests please do so in the issue tracker. - -## Author Info - -Website: [David Wahlstrom](http://www.linkedin.com/in/dwahlstrom/) +This originally started as a fork of bcarpio/mongodb-enc, however it has turned in to a complete re-write. The only thing that has been kept is the basic schema. Since the two code bases no longer share any code in common, I decided to break off the fork and create an entirely new project with a new name. All further development will be completed at https://github.com/drwahl/yape.git.