minor version update - ran oslcquery tests against 7.0.2SR1

Author: barny

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.11.3
+current_version = 0.13.0
 commit = True
 tag = True
 

README.md

@@ -8,7 +8,7 @@
 
  SPDX-License-Identifier: MIT
 
- version="0.11.3"
+ version="0.13.0"
 
 
 Introduction

elmclient/__meta__.py

@@ -9,7 +9,7 @@
 
 app = 'elmoslcquery'
 description = 'Commandline OSLC query for ELM'
-version = '0.11.3'
+version = '0.13.0'
 license = 'MIT'
 author_name = 'Ian Barnard'
 author_mail = 'ian.barnard@uk.ibm.com'

elmclient/examples/oslcquery.py

@@ -18,6 +18,7 @@
 import time
 import urllib3
 import webbrowser
+import concurrent.futures
 
 import cryptography
 import cryptography.fernet
@@ -79,9 +80,9 @@ def do_oslc_query(inputargs=None):
     parser.add_argument('-A', '--appstrings', default=None, help=f'A comma-seperated list of apps, the query goes to the first entry, default "rm". Each entry must be a domain or domain:contextroot e.g. rm or rm:rm1 - Default can be set using environemnt variable QUERY_APPSTRINGS')
     parser.add_argument('-C', '--component', help='The local component (optional, you *have* to specify the local configuration using -F)')
     parser.add_argument('-D', '--delaybetweenpages', type=float,default=0.0, help="Delay in seconds between each page of results - use this to reduce overall server load particularly for large result sets or when retrieving many properties")
-    parser.add_argument('-E', '--globalproject', default=None, help="The global configuration project - needed if the globalconfiguration isn't unique")
-    parser.add_argument('-F', '--configuration', default=None, help='The local configuration')
-    parser.add_argument('-G', '--globalconfiguration', default=None, help='The global configuration (you must not specify local config as well!) - you can specify the id, the full URI, or the config name (not implemented yet)')
+    parser.add_argument('-E', '--globalproject', default=None, help="The global configuration project - optional if the globalconfiguration is unique in the gcm app")
+    parser.add_argument('-F', '--configuration', default=None, help='The local configuration name')
+    parser.add_argument('-G', '--globalconfiguration', default=None, help='The global configuration (you must not specify local config as well!) - you can specify the id, the full URI, or the config name')
     parser.add_argument('-H', '--saveconfigs', default=None, help='Name of CSV file to save details of the local project components and configurations')
     parser.add_argument('-I', '--totalize', action="store_true", help="For any column with multiple results, put in the total instead of the results")
     parser.add_argument("-J", "--jazzurl", default=JAZZURL, help=f"jazz server url (without the /jts!) default {JAZZURL} - Default can be set using environemnt variable QUERY_JAZZURL - defaults to https://jazz.ibm.com:9443 which DOESN'T EXIST")
@@ -111,7 +112,8 @@ def do_oslc_query(inputargs=None):
     parser.add_argument('--saveprocessedresults', default=None, help="Save the processed results as JSON to this path/file" )
     parser.add_argument('--percontribution', action="store_true", help="When querying a GC, query once for each app-domain contribution in the GC tree, with added component and configuration columns in the result")
     parser.add_argument('--cacheable', action="store_true", help="Query results can be cached - use when you know the data isn't changing and you need faster re-run")
-    parser.add_argument('--crossproject', action="store_true", help="For --percontriubtion GC queries follow gc contributions to other projects and query those too (requires access permission of course)")
+    parser.add_argument('--crossproject', action="store_true", help="For --percontribution GC queries follow gc contributions to other projects and query those too (requires access permission of course)")
+    parser.add_argument('--threading', action="store_true", help="For --percontriubtion GC queries, use threading to parallelize queries with processing results UNTESTED")
 
     # saved credentials
     parser.add_argument('-0', '--savecreds', default=None, help="Save obfuscated credentials file for use with readcreds, then exit - this stores jazzurl, appstring, username and password")
@@ -474,32 +476,66 @@ def do_oslc_query(inputargs=None):
 
     if args.percontribution:
         results = {}
-        # get all the contributions in this domain, and the component they're in - these will be added to the results
-        contribs = p.get_our_contributions(gcconfiguri)
-        for i,(contriburi,compuri) in enumerate(contribs):
-            print( f"{i+1}/{len(contribs)} {contriburi=} {compuri=}" )
-            # find the component in teh config
-            queryon = p.find_local_component(compuri)
-            if queryon is None:
-                print( f"Component not found from {compuri}" )
-                if args.crossproject:
-                    # this component isn't in our current project
-                    # try to create a component for it and add to current project
-                    queryon = p.add_external_component(compuri)
-                    if queryon is None:
-                        burp
-                        continue
+        futureresults = []
+        workers = 4 if args.threading else 1
+        def thread_fn(i,queryon,configuri):
+            print( f"Thread start {i}" )
+            queryon.set_local_config(configuri)
+            try:
+                thisresults = queryon.do_complex_query( args.resourcetype, querystring=args.query, searchterms=args.searchterms, select=args.select, isnulls=args.null, isnotnulls=args.value
+                            ,orderby=args.orderby
+                            ,show_progress=args.noprogressbar
+                            ,verbose=args.verbose
+                            ,maxresults=args.maxresults
+                            ,delaybetweenpages=args.delaybetweenpages
+                            ,pagesize=args.pagesize
+                            ,resolvenames = args.resolvenames
+                            ,totalize=args.totalize
+                            ,saverawresults=args.saverawresults
+                            ,addcolumns={'$contriburi':contriburi,'$compuri':compuri}
+                            ,cacheable=args.cacheable
+                            )
+            except KeyboardInterrupt:
+                raise Exception( "Control-c" )
+            print( f"Thread finish {i=}" )
+            return thisresults
+        with concurrent.futures.ThreadPoolExecutor(max_workers = workers) as executor:
+            # get all the contributions in this domain, and the component they're in - these will be added to the results
+            contribs = p.get_our_contributions(gcconfiguri)
+            for i,(contriburi,compuri) in enumerate(sorted(contribs,key=lambda v:v[0])):
+                print( f"{i+1}/{len(contribs)} {contriburi=} {compuri=}" )
+                # find the component in teh config
+                queryon = p.find_local_component(compuri)
+                if queryon is None:
+                    print( f"Component not found from {compuri}" )
+                    if args.crossproject:
+                        # this component isn't in our current project
+                        # try to create a component for it and add to current project
+                        queryon = p.add_external_component(compuri)
+                        if queryon is None:
+                            raise Exception( f"Can't add external component for {compuri}" )
+                        else:
+                            print( f"Added external component {queryon=} for {compuri}" )
                     else:
-                        print( f"Added external component {queryon=}" )
-            # check the comonent is accessible (may have been achived!)
-            if not app.is_accessible( compuri ):
-                print( f"Archived component {compuri} !")
-                continue
-            # check if the config is accessible (may have been archived!)
-            if not app.is_accessible( contriburi ):
-                print( f"Archived configuration {contriburi} !")
-                continue
-            
+                        raise Exception( "Component {compuri} not found in current project - maybe you need to use --crossproject?" )
+                # check the comonent is accessible (may have been achived!)
+                if not app.is_accessible( compuri ):
+                    print( f"**** Archived component {compuri} !")
+                    continue
+                # check if the config is accessible (may have been archived!)
+                if not app.is_accessible( contriburi ):
+                    print( f"**** Archived configuration {contriburi} !")
+                    continue
+                if not args.threading:
+                    # work synchronously
+                    results.update(thread_fn(i,queryon,contriburi))
+                else:
+                    # get the results later
+                    futureresults.append(executor.submit(thread_fn,i,queryon,contriburi))
+            # now if working asynchronously retrieve the results
+            for res in futureresults:
+                results.update(res.result())
+        if False:
             # set the config ready to do the query
             queryon.set_local_config(contriburi)
             # now do a query for each contribution

elmclient/httpops.py

@@ -91,6 +91,17 @@ def to_binary(text, encoding=None, errors='strict'):
         result = codecs.encode(text, encoding=encoding, errors=errors)
     return result
 
+#################################################################################################
+
+def getcookievalue( cookies, cookiename, defaultvalue=None):
+    print( f"gcv {cookies=} {cookiename=} {defaultvalue=}" )
+    for c in cookies:
+        if c.name == cookiename:
+            print( f"Found {cookiename} {c.value}" )
+            return c.value
+    print( f"Not found {cookiename}" )
+    return defaultvalue
+
 ##############################################################################################
 
 class _FormParser(html.parser.HTMLParser):
@@ -203,6 +214,7 @@ def execute_get_binary( self, reluri, *, params=None, headers=None, **kwargs):
         return response
 
     def execute_post_content( self, uri, *, params=None, data=None, headers={}, put=False, **kwargs):
+        logger.debug("+++++++++++++++++++")
         data = data if data is not None else ""
         reqheaders = {}
         if headers is not None:
@@ -211,6 +223,7 @@ def execute_post_content( self, uri, *, params=None, data=None, headers={}, put=
         if put:
             request.method = "PUT"
         response = request.execute( **kwargs )
+        logger.debug("-----------------")
         return response
 
     def execute_get(self, reluri, *, params=None, headers=None, **kwargs):
@@ -308,7 +321,7 @@ def _execute_request( self, *, no_error_log=False, close=False, cacheable=True,
         for wait_dur in [2, 5, 10, 0]:
             try:
                 if not cacheable:
-                    # add a header so the response isnm't cached
+                    # add a header so the response isn't cached
                     self._req.headers['Cache-Control'] = "no-store, max-age=0"
                 result = self._execute_one_request_with_login( no_error_log=no_error_log, close=close, **kwargs)
                 return result
@@ -319,6 +332,35 @@ def _execute_request( self, *, no_error_log=False, close=False, cacheable=True,
                 logger.warning( f'RETRY: Retry after {wait_dur} seconds... URL: {self._req.url}' )
                 time.sleep(wait_dur)
         raise Exception('programming error this point should never be reached')
+
+    def _execute_request_newbutunused( self, *, no_error_log=False, close=False, cacheable=True, **kwargs ):
+        logger.debug( f"execute_preventing_csrf {cacheable=}",)  
+        try:
+            if not cacheable:
+                # add a header so the response isn't cached
+                self._req.headers['Cache-Control'] = "no-store, max-age=0"
+            result = self._execute_one_request_with_login(no_error_log=no_error_log, close=close, **kwargs)
+            logger.debug( f"execute_request result {result}" )
+            return result
+        except requests.HTTPError as e:
+            logger.debug( f"execute_request Exception {e}" )
+            if (e.response.status_code==http.client.FORBIDDEN or e.response.status_code==http.client.CONFLICT) and e.response.text.find('X-Jazz-CSRF-Prevent'):
+                # Add special header and try again
+                print( f"{e.response=}" )
+                print( f"{e.request._cookies=}" )
+                jsessionid = getcookievalue( e.request._cookies, 'JSESSIONID',None)
+                if not jsessionid:
+                    logger.debug("Retrying request with CSRF header, but coudln't get JSESSIONID from the cookie for url [%s]." % request.url)
+                    raise
+                
+#                e.close()
+                logger.debug("  Retrying request with CSRF header...")
+                self._req.headers['X-Jazz-CSRF-Prevent'] = jsessionid
+                return self._execute_one_request_with_login( close=close, **kwargs )
+            else:
+                raise
+
+
 
     # log a request/response, which may be the result of one or more redirections, so first log each of their request/response
     def log_redirection_history( self, response, intent, action=None, donotlogbody=False ):
@@ -443,6 +485,29 @@ def _is_retryable_error( self, e ):
                                         ]:
                 return True
         return False
+        
+    def get_auth_path(self, request_url, response):
+        request_url_parsed = urllib.parse.urlparse(request_url)
+        form_auth_path = [c.path for c in response.cookies if c.name == 'JazzFormAuth']
+        auth_app_context = form_auth_path[0] if len(form_auth_path) == 1 else request_url_parsed.path.split('/')[1]
+        return auth_app_context
+        
+        
+    def tidy_cookies(self):
+        '''
+        LQE 7.0.2SR1 and 7.0.3 has the unpleasant habit of including double-quotes in the auth cookie path so it looks like "/lqe" (which includes the quotation marks in the path) rather than /lqe, and then the path is never matched so authentication is lost
+        This code cleans up the path on all cookies on the session4
+        # return True if any cookie changed
+        '''
+        result = False
+        for cookie in self._session.cookies:
+            if cookie.path.startswith('%22'):
+                oldvalue = cookie.path
+                cookie.path = cookie.path.replace("%22","")
+                logger.debug( f"REVISED cookie {cookie.name} path {oldvalue} to {cookie.path=}" )
+                result = True
+        return result
+
 
     # execute a request once, except:
     #  1. if the response indicates login is required then login and try the request again
@@ -489,6 +554,7 @@ def _execute_one_request_with_login( self, *, no_error_log=False, close=False, d
         try:
             prepped = self._session.prepare_request( request )
             response = self._session.send( prepped )
+                                                 
             self.log_redirection_history( response, intent=intent, action=action )
 
             response.raise_for_status()
@@ -512,12 +578,14 @@ def _execute_one_request_with_login( self, *, no_error_log=False, close=False, d
                 self._session.is_authenticated = False
                 auth_url = e.response.headers['X-jazz-web-oauth-url']
                 login_response = self._login(auth_url)
+                                   
                 if login_response:
                     logger.trace("WIRE: NOT retrying")
                     response = login_response
                 else:
                     logger.trace("WIRE: retrying")
                     logger.trace( f"Auth completed (in theory) result - 1" )
+                                                                     
                     retry_after_login_needed = True
             elif e.response.status_code == 401 and 'X-JSA-AUTHORIZATION-REDIRECT' in e.response.headers:
                 logger.trace("WIRE: JAS Login required!")
@@ -605,12 +673,14 @@ def _login(self, auth_url):
                 return None  # no more auth required
             login_url = auth_url_response.url  # Take the redirected URL and login action URL
             self._authorize(login_url)
+            
             logger.trace("authorized")
         else:
             logger.error('''Something has changed since this script was written. I can no longer determine where to authorize myself.''')
             raise Exception("Login not possible(1)!")
 
         try:
+                
             # Now we should have the proper oauth cookies, so try again
             response = self._session.get(auth_url)
         except requests.exceptions.RequestException as e:
@@ -700,4 +770,3 @@ def _jazz_form_authorize(self, request_url, prev_request, prev_response):
             logger.info( f"Failed to jazz_authorize with auth URL {auth_url} with exception {e}" )  # was logger.error despite subsequent authentication success
             raise Exception("Jazz FORM authorize not possible!")
         return response
-