diff --git a/.gitignore b/.gitignore index ba698c7..8129ef9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .vscode/launch.json +__pycache__/* \ No newline at end of file diff --git a/comanage_person_schema_utils.py b/comanage_person_schema_utils.py new file mode 100644 index 0000000..d7fba5e --- /dev/null +++ b/comanage_person_schema_utils.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 + +CO_PERSON = { + "co_id": None, + "timezone": None, + "dateofbirth": None, + "status": None + } + +IDENTIFIER = { + "identifier": None, + "type": None, + "login": False, + "status": None, +} + +NAME = { + "honorific": None, + "given": "None", + "middle": None, + "family": "None", + "suffix": None, + "type": "official", + "language": None, + "primary_name": True, +} + + +GROUP = { + "co_group_id": None, + "member": True, + "owner": False, +} + + +COU = { + "co_id": None, + "name": None, + "description": None, + "parent_id": None, +} + + +ORG_IDENTITY = { + "co_id": None, + "title": None, + # Organization for OrgID + "o": None, + # Department for OrgID + "ou": None, + "valid_from": None, + "valid_through": None, + "status": "", + "affiliation": None, + "date_of_birth": None, + "Address": [], + "AdHocAttribute": [], + "EmailAddress": [], + "Identifier": [], + "Name": [], + "TelephoneNumber": [], + "Url": [], +} + + +ROLE = { + "cou_id": None, + "title": "", + "o": "", + "ou": "", + "valid_from": None, + "valid_through": None, + "status": "A", + "sponsor_co_person_id": None, + "affiliation": "member", + "ordr": 1, +} + + +EMAIL_ADDRESS = { + "mail": None, + "type": None, + "verified": None, +} + + +UNIX_CLUSTER_ACCOUNT = { + "sync_mode": "F", + "status": None, + "username": None, + "uid": None, + "gecos": None, + "login_shell": "/bin/bash", + "home_directory": None, + "primary_co_group_id": None, + "valid_from": None, + "valid_through": None, + "unix_cluster_id": None, +} + + +SSHKEY = { + "type": None, + "skey": None, + "comment": "", + "ssh_key_authenticator_id": None, +} + + +def co_person_schema(co_id, timezone=None, dob=None, status="Active"): + person_data = CO_PERSON.copy() + person_data["co_id"] = co_id + person_data["timezone"] = timezone + person_data["dateofbirth"] = dob + person_data["status"] = status + return person_data + + +def co_person_identifier(identifier, type, login=False, status="Active"): + identifier_data = IDENTIFIER.copy() + identifier_data["identifier"] = identifier + identifier_data["type"] = type + identifier_data["login"] = login + identifier_data["status"] = status + return identifier_data + + +def co_person_name(given, family=None, middle=None, type="official", primary=False): + name_data = NAME.copy() + name_data["given"] = given + name_data["family"] = family + name_data["middle"] = middle + name_data["type"] = type + name_data["primary_name"] = primary + return name_data + + +def name_split(whole_name): + name_sections = str(whole_name).split() + parts_count = len(name_sections) + if parts_count == 1: + return co_person_name(given=whole_name) + elif parts_count == 2: + return co_person_name(given=name_sections[0], family=name_sections[1]) + else: + return co_person_name(given=name_sections[0], family=name_sections[parts_count-1], middle=" ".join(name_sections[1:parts_count-1])) + + +def name_unsplit(name_id): + if name_id["given"] is None: + return "" + elif name_id["family"] is None: + return name_id["given"] + elif name_id["middle"] is None: + return f'{name_id["given"]} {name_id["family"]}' + else: + return f'{name_id["given"]} {name_id["middle"]} {name_id["family"]}' + + + +def co_person_group_member(group_id, member=True, owner=False): + group_member = GROUP.copy() + group_member["co_group_id"] = group_id + group_member["member"] = member + group_member["owner"] = owner + return group_member + + +def co_person_org_id( + osg_co_id, name, organization="", department="", title="", affiliation="member", id_list=[] +): + #org_id = {"co_id" : osg_co_id} + org_id = ORG_IDENTITY.copy() + org_id["co_id"] = osg_co_id + org_id["title"] = title + org_id["o"] = organization + org_id["ou"] = department + org_id["affiliation"] = affiliation + org_id["Identifier"] = id_list + org_id["Name"] = name + return org_id + + +def co_person_role(cou, title, affiliation, order): + role = ROLE.copy() + role["cou_id"] = cou + role["title"] = title + role["affiliation"] = affiliation + role["ordr"] = order + return role + +def co_person_email_address(mail, type="delivery", verified=False): + email = EMAIL_ADDRESS.copy() + email["mail"] = mail + email["type"] = type + email["verified"] = verified + return email + + +def co_person_unix_cluster_acc(unix_cluster_id, username, uid, name, group_id, status="A"): + uca = UNIX_CLUSTER_ACCOUNT.copy() + uca["unix_cluster_id"] = unix_cluster_id + uca["username"] = username + uca["uid"] = uid + uca["status"] = status + uca["gecos"] = name + uca["home_directory"] = f"/home/{username}" + uca["primary_co_group_id"] = group_id + return uca + + +def co_person_sshkey(type, skey, comment, auth_id): + sshkey_data = SSHKEY.copy() + sshkey_data["type"] = type + sshkey_data["skey"] = skey + sshkey_data["comment"] = comment + sshkey_data["ssh_key_authenticator_id"] = auth_id + return sshkey_data + + +# def merge_schema(base_data, new_data, type): +# temp = base_data +# for field in type.keys(): +# for entry in new_data[field]: +# if "meta" in entry: +# +# +# return temp diff --git a/comanage_utils.py b/comanage_utils.py index 238b0ec..9d5e5b1 100644 --- a/comanage_utils.py +++ b/comanage_utils.py @@ -30,6 +30,8 @@ TIMEOUT_BASE = 5 MAX_ATTEMPTS = 5 +# HTTP return codes we shouldn't attempt to retry +HTTP_NO_RETRY_CODES = {401, 404, 405, 500} GET = "GET" PUT = "PUT" @@ -47,6 +49,16 @@ class URLRequestError(Error): pass +class HTTPRequestError(URLRequestError): + """Class for exceptions due to not being able to fulfill a HTTPRequest""" + def __init__(self, message, code): + self.message = message + self.code = code + + def __str__(self): + return self.message + + def getpw(user, passfd, passfile): if ":" in user: user, pw = user.split(":", 1) @@ -100,6 +112,14 @@ def call_api3(method, target, data, endpoint, authstr, **kw): # exception catching, mainly for request timeouts, "Service Temporarily Unavailable" (Rate limiting), and DNS failures. except urllib.error.URLError as exception: req_attempts += 1 + # If the exception was an HTTPError, with a code like 404 that won't change on a retry, fail fast + if issubclass(exception.__class__, urllib.error.HTTPError) and exception.code in HTTP_NO_RETRY_CODES: + raise HTTPRequestError( + "Exception raised due to api call error status" + + f"Exception reason: {exception}.\n Request: {req.full_url}", + code=exception.code + ) + # Since we think the exception *might* be transient, continue with retry logic if req_attempts >= MAX_ATTEMPTS: raise URLRequestError( "Exception raised after maximum number of retries reached after total backoff of " + @@ -140,6 +160,18 @@ def get_co_group(gid, endpoint, authstr): return grouplist[0] +def core_api_co_person_read(identifier, coid, endpoint, authstr): + return call_api(f"api/co/{coid}/core/v1/people/{identifier}", endpoint, authstr) + + +def core_api_co_person_create(data, coid, endpoint, authstr): + return call_api3(POST, f"api/co/{coid}/core/v1/people/", data, endpoint, authstr) + + +def core_api_co_person_update(identifier, coid, data, endpoint, authstr): + return call_api3(PUT, f"api/co/{coid}/core/v1/people/{identifier}", data, endpoint, authstr) + + def get_identifier(id_, endpoint, authstr): resp_data = call_api("identifiers/%s.json" % id_, endpoint, authstr) idfs = get_datalist(resp_data, "Identifiers") @@ -157,10 +189,37 @@ def get_unix_cluster_groups_ids(ucid, endpoint, authstr): return set(group["CoGroupId"] for group in unix_cluster_groups["UnixClusterGroups"]) +def update_co_person_identifier(id_, type, identifier, person_id, endpoint, authstr, provisioning_target): + id_data = { + "RequestType":"Identifiers", + "Version":"1.0", + "Identifiers": + [ + { + "Version":"1.0", + "Type":type, + "Identifier":identifier, + "Login":False, + "Person":{"Type":"CO","Id":person_id}, + "CoProvisioningTargetId":provisioning_target, + "Status":"Active" + } + ] + } + return call_api3(PUT, "/api/v2/identifiers" % id_, id_data, endpoint, authstr, ) + #return call_api3(PUT, "identifiers/%s.json" % id_, id_data, endpoint, authstr) + + def delete_identifier(id_, endpoint, authstr): return call_api2(DELETE, "identifiers/%s.json" % id_, endpoint, authstr) +def get_co_group_members_pids(gid, endpoint, authstr): + resp_data = get_co_group_members(gid, endpoint, authstr) + data = get_datalist(resp_data, "CoGroupMembers") + return [m["Person"]["Id"] for m in data] + + def get_datalist(data, listname): return data[listname] if data else [] @@ -183,6 +242,14 @@ def identifier_from_list(id_list, id_type): except ValueError: return None +def full_identifier_from_list(id_list, id_type): + id_type_list = [id["Type"] for id in id_list] + try: + id_index = id_type_list.index(id_type) + return id_list[id_index] + except ValueError: + return None + def identifier_matches(id_list, id_type, regex_string): pattern = re.compile(regex_string) @@ -190,6 +257,23 @@ def identifier_matches(id_list, id_type, regex_string): return (value is not None) and (pattern.match(value) is not None) +def create_co_group(groupname, description, coId, endpoint, authstr, open=False,): + group_info = { + "Version" : "1.0", + "CoId" : coId, + "Name" : groupname, + "Description" : description, + "Open" : open, + "Status" : "Active", + } + data = { + "CoGroups" : [group_info], + "RequestType" : "CoGroups", + "Version" : "1.0" + } + return call_api3(POST, "co_groups/.json", data, endpoint, authstr) + + def rename_co_group(gid, group, newname, endpoint, authstr): # minimal edit CoGroup Request includes Name+CoId+Status+Version new_group_info = { diff --git a/mass_person_create_modify.py b/mass_person_create_modify.py new file mode 100644 index 0000000..fe71ebd --- /dev/null +++ b/mass_person_create_modify.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 + +import os +import sys +import json +import getopt +import comanage_utils as utils +import comanage_person_schema_utils as schema_utils + +SCRIPT = os.path.basename(__file__) +ENDPOINT = "https://registry.cilogon.org/registry/" +OSG_CO_ID = 7 +LDAP_TARGET_ID = 6 + +_usage = f"""\ +usage: [PASS=...] {SCRIPT} [OPTIONS] +""" + +def usage(msg=None): + if msg: + print(msg + "\n", file=sys.stderr) + + print(_usage, file=sys.stderr) + sys.exit() + +class Options: + endpoint = ENDPOINT + user = "co_7.project_script" + osg_co_id = OSG_CO_ID + authstr = None + input_file = None + mapping_file = None + ssh_key_authenticator = 1 + unix_cluster_id = 1 + provisioning_target = LDAP_TARGET_ID + import_group_id = None + import_cou_id = None + + +options = Options() + +def parse_options(args): + try: + ops, args = getopt.getopt(args, 'u:c:d:f:e:i:m:g:o:h') + except getopt.GetoptError: + usage() + + if args: + usage("Extra arguments: %s" % repr(args)) + + passfd = None + passfile = None + + for op, arg in ops: + if op == '-h': usage() + if op == '-u': options.user = arg + if op == '-c': options.osg_co_id = int(arg) + if op == '-d': passfd = int(arg) + if op == '-f': passfile = arg + if op == '-e': options.endpoint = arg + if op == '-i': options.input_file = arg + if op == '-m': options.mapping_file = arg + if op == '-g': options.import_group_id = arg + if op == '-o': options.import_cou_id = arg + + try: + user, passwd = utils.getpw(options.user, passfd, passfile) + options.authstr = utils.mkauthstr(user, passwd) + except PermissionError: + usage("PASS required") + + +def read_data_dump(): + data_json = [] + with open(options.input_file, 'r', encoding='utf-8') as input_file: + data_json = json.load(input_file) + for entry in range(len(data_json)): + for key_index in range(len(data_json[entry]["public_keys"])): + key = data_json[entry]["public_keys"][key_index] + key_sections = str(key).split() + if len(key_sections) >= 2: + data_json[entry]["public_keys"][key_index] = {"type" : key_sections[0], "pkey" : key_sections[1]} + if len(key_sections) >= 3: + data_json[entry]["public_keys"][key_index].update({"authenticator" : key_sections[2]}) + with open(options.mapping_file, 'r', encoding='utf-8') as mapping_file: + mapping_json = json.load(mapping_file) + return data_json, mapping_json + + +def build_co_person_record(entry, mapping_json : dict): + record = {} + record.update({"CoPerson" : schema_utils.co_person_schema(options.osg_co_id, status="A")}) + + names = [] + names.append(schema_utils.name_split(entry["name"])) + record.update({"Name" : names}) + + identifiers = [] + + # UC Connect Username + identifiers.append(schema_utils.co_person_identifier(entry["username"], "ucconnectuser", status="A")) + # UC Connect UID + identifiers.append(schema_utils.co_person_identifier(entry["uid"], "ucconnectuid", status="A")) + #globus id + identifiers.append(schema_utils.co_person_identifier(entry["globus_id"], "ucconnectglobusid", status="A")) + #cilogon id + if not (entry["cilogon_oidc_sub"] is None or entry["cilogon_oidc_sub"] == ""): + identifiers.append(schema_utils.co_person_identifier(entry["cilogon_oidc_sub"], "oidcsub", status="A")) + else: + print(f"Warning: user {entry['username']} lacks a cilogon id.") + # With our current LDAP Provisioner configuration, the CO Person still needs an oidcsub identifier, even if it's junk + identifiers.append(schema_utils.co_person_identifier(f"dummy-text-for-provisioning-{entry['username']}", "oidcsub", status="A", login=False)) + + record.update({"Identifier" : identifiers }) + + group_memberships = [] + group_memberships.append(schema_utils.co_person_group_member(options.import_group_id)) + + for user_membership in entry["groups"]: + if not mapping_json.get(user_membership) is None: + group_memberships.append(schema_utils.co_person_group_member(mapping_json.get(user_membership))) + else: + print(f"Warning: could not find group id for group {user_membership}, user {entry['username']}.") + + # Group Memberships + record.update({"CoGroupMember" : group_memberships }) + + roles = [] + + roles.append(schema_utils.co_person_role(options.import_cou_id, "UC Connect User", "member", 1)) + record.update({"CoPersonRole" : roles }) + + emails = [] + + emails.append(schema_utils.co_person_email_address(entry["email"])) + record.update({"EmailAddress" : emails}) + + org_ids = [] + + org_ids.append(schema_utils.co_person_org_id(options.osg_co_id, names, entry["institution"], id_list=identifiers)) + record.update({"OrgIdentity" : org_ids}) + + ssh_keys = [] + + for ssh_key in entry["public_keys"]: + key_type = ssh_key["type"] + public_key = ssh_key["pkey"] + comment = "" + if "authenticator" in dict(ssh_key): + comment = f"SSH Key for {ssh_key['authenticator']}" + auth_id = options.ssh_key_authenticator + ssh_keys.append(schema_utils.co_person_sshkey(type=key_type, skey=public_key, comment=comment, auth_id=auth_id)) + record.update({"SshKey" : ssh_keys}) + + return record + +def fix_username(co_person_record, new_username): + record = co_person_record + + for identifier in co_person_record["Identifier"]: + if identifier["type"] == "osguser": + identifier["identifier"] = new_username + + return record + + +def create_unix_cluster_group(co_person_record): + identifiers_list = co_person_record["Identifier"] + username = next((item["identifier"] for item in identifiers_list if item["type"] == "osguser")) + uid = next((item["identifier"] for item in identifiers_list if item["type"] == "uid")) + description = f"Unix Cluster Group for {username}" + result = utils.create_co_group(username, description, options.osg_co_id, options.endpoint, options.authstr) + ucg = None + if not (result is None) and ("ResponseType" in result) and (result["ResponseType"] == "NewObject"): + group_id = result["Id"] + utils.add_identifier_to_group(group_id, "osggid", uid, options.endpoint, options.authstr) + utils.add_identifier_to_group(group_id, "osggroup", username, options.endpoint, options.authstr) + ucg = utils.add_unix_cluster_group(group_id, options.unix_cluster_id, options.endpoint, options.authstr) + utils.provision_group(group_id, options.provisioning_target, options.endpoint, options.authstr) + #TODO throw catch on new group creation + if not (ucg is None) and ("ResponseType" in ucg) and (ucg["ResponseType"] == "NewObject"): + return(result["Id"]) + else: + raise ValueError(f"Failed to create CO Group for Unix Cluster Group, results were: {result} and {ucg}") + + +def add_unix_cluster_account(co_person_record): + identifiers_list = co_person_record["Identifier"] + names_list = co_person_record["Name"] + username = next((item["identifier"] for item in identifiers_list if item["type"] == "osguser")) + uid = next((item["identifier"] for item in identifiers_list if item["type"] == "uid")) + name_id = next((item for item in names_list if item["primary_name"] == True)) + name = schema_utils.name_unsplit(name_id) + default_group_id = -1 + default_group_id = create_unix_cluster_group(co_person_record) + ucg_membership = schema_utils.co_person_group_member(default_group_id) + if "CoGroupMember" in co_person_record: + co_person_record["CoGroupMember"].append(ucg_membership) + else: + co_person_record.update({"CoGroupMember" : [ucg_membership]}) + if default_group_id != -1: + uca = schema_utils.co_person_unix_cluster_acc(options.unix_cluster_id, username, uid, name, default_group_id) + if "UnixClusterAccount" in co_person_record: + co_person_record["UnixClusterAccount"].append(uca) + else: + co_person_record.update({"UnixClusterAccount" : [uca]}) + return co_person_record, default_group_id + + +def main(args): + parse_options(args) + + co_person_records = dict() + + data_dump_json, mapping_json = read_data_dump() + + #data_dump_json = [data_dump_json[0], data_dump_json[1], data_dump_json[2], data_dump_json[3], data_dump_json[8], data_dump_json[9]] + + for entry in data_dump_json: + co_person_records.update({entry["username"] : build_co_person_record(entry, mapping_json)}) + + usernames = list(co_person_records.keys()) + + for user in usernames: + co_person_data = None + results_create = None + + try: + try: + #If the CO Person record exists, stop creating/modifying (TODO: switch to modifying existing user rather than trying to create) + if utils.core_api_co_person_read(user, options.osg_co_id, options.endpoint, options.authstr): + continue + except utils.HTTPRequestError as e: + # If the record *doesn't* exist, pass and make it. Else, some other error happened on our read, like 403 or 500 and we'll try again on another run. + if e.code == 404: + pass + else: + break + print(f"CREATING RECORDS FOR USER: {user}") + results_create = utils.core_api_co_person_create(data=co_person_records[user], coid=options.osg_co_id, endpoint=options.endpoint, authstr=options.authstr) + + co_person_data = utils.core_api_co_person_read(user, options.osg_co_id, options.endpoint, options.authstr) + + co_person_data = fix_username(co_person_data, user) + utils.core_api_co_person_update(user, options.osg_co_id, co_person_data, options.endpoint, options.authstr) + + co_person_data, gid = add_unix_cluster_account(co_person_data) + + utils.core_api_co_person_update(user, options.osg_co_id, co_person_data, options.endpoint, options.authstr) + + utils.provision_group(gid, options.provisioning_target, options.endpoint, options.authstr) + except Exception as e: + print(f"\tException for user {user}.") + print(f"\t{e}") + if results_create: + print(f"\t{results_create}") + if co_person_data: + print(f"\t{co_person_data}") + + # Read in dump to build / update users from + # select which field of the dump co-responds to the identifier we'll use to index the corresponding CO Person + # mapping file from dump attributes to COmanage object types, so we know what each dump attribute should become + + # Loop over entries in dump + # try a CO Person Read based on specified field + # if Read succeeds, we're doing an update + # else if we fail due to the record not existing, we're doing a create + # else, something went wrong and we should log about it + + # Needs: + # Create base CO Person record with at least enough information to read them at a later date + # Read CO Person via index identifier + # Write CO Person with specified full schema + # "matching identifier/key/name already exists in CO Person entry?" function + # + # Data -> Write schema functions, with argument for existing field to merge data in from, for + # Base CO Person + # Identifiers + # Email + # Name + # SSHKey + # Org ID + # CO Person Role + # UNIX Cluster Account + # GroupMembership (?) + + # Some Schema Info: + # See comanage_person_schema_utils.py and COmanage docs on co person schema and associated record types + # https://spaces.at.internet2.edu/spaces/COmanage/pages/25859280/cm_co_people + # CoPerson + # { + # 'meta': {'id': , 'created': , 'modified': , 'co_person_id': , 'revision': , 'deleted': , 'actor_identifier': }, + # 'co_id': <8 for test / 7 for prod>, + # 'status': 'A', + # 'timezone': None, + # 'date_of_birth': None + # } + # Identifier + # { + # 'meta': {'id': , 'created': , 'modified': , 'co_person_id': , 'revision': , 'deleted': , 'actor_identifier': }, + #"identifier": None, + #"type": None, + #"login": False, + # } + # CoGroupMember + # Array of { + # 'meta': {'id': 9119, 'source_org_identity_id': None, 'created': '2025-05-14 21:47:15', 'modified': '2025-05-14 21:47:15', 'co_group_member_id': None, 'revision': 0, 'deleted': False, 'actor_identifier': 'co_8.william_test'}, + # 'co_group_id': 101, + # 'member': True, + # 'owner': False, + # 'valid_from': None, + # 'valid_through': None, + # 'co_group_nesting_id': None + # } + # Name + # Array of { + # 'honorific': None, + # 'given': 'TEST_PERSON', + # 'middle': None, + # 'family': 'Wanson', + # 'suffix': None, + # 'type': 'official', + # 'language': None, + # 'primary_name': True + # } + # SSHKEY + # Array of { + # 'type': '', + # 'skey': '', + # 'ssh_key_authenticator_id': 5 + # } + # NOTE: the "5" is from https://registry-test.cilogon.org/registry/ssh_key_authenticator/ssh_key_authenticators/edit/5 + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except Exception as e: + sys.exit(e)