diff --git a/Python/intelx/intelx_identity.py b/Python/intelx/intelx_identity.py index bc64e81..27763f8 100644 --- a/Python/intelx/intelx_identity.py +++ b/Python/intelx/intelx_identity.py @@ -5,23 +5,34 @@ class IdentityService(intelx): - def __init__(self, api_key, user_agent='IX-Python/0.6'): + def __init__(self, api_key, user_agent="IX-Python/0.6"): super().__init__(api_key, user_agent) - self.API_ROOT = 'https://3.intelx.io' - self.HEADERS = {'X-Key': self.API_KEY, 'User-Agent': self.USER_AGENT} + self.API_ROOT = "https://3.intelx.io" + self.HEADERS = {"X-Key": self.API_KEY, "User-Agent": self.USER_AGENT} self.PAUSE_BETWEEN_REQUESTS = 1 def get_search_results(self, id, format=1, maxresults=100): - params = {'id': id, 'format': format, 'limit': maxresults} - r = requests.get(self.API_ROOT + '/live/search/result', - params, headers=self.HEADERS) + params = {"id": id, "format": format, "limit": maxresults} + r = requests.get( + self.API_ROOT + "/live/search/result", params, headers=self.HEADERS + ) if r.status_code == 200: return r.json() else: return r.status_code - def idsearch(self, term, maxresults=100, buckets="", timeout=5, datefrom="", dateto="", - terminate=[], analyze=False, skip_invalid=False): + def idsearch( + self, + term, + maxresults=100, + buckets="", + timeout=5, + datefrom="", + dateto="", + terminate=[], + analyze=False, + skip_invalid=False, + ): p = { "selector": term, "bucket": buckets, @@ -34,45 +45,54 @@ def idsearch(self, term, maxresults=100, buckets="", timeout=5, datefrom="", dat } done = False results = [] - r = requests.get(self.API_ROOT + '/live/search/internal', - headers=self.HEADERS, params=p) + r = requests.get( + self.API_ROOT + "/live/search/internal", headers=self.HEADERS, params=p + ) if r.status_code == 200: - search_id = r.json()['id'] + search_id = r.json()["id"] else: return (r.status_code, r.text) - if (len(str(search_id)) <= 3): - print( - f"[!] intelx.IDENTITY_SEARCH() Received {self.get_error(search_id)}") + if len(str(search_id)) <= 3: + print(f"[!] intelx.IDENTITY_SEARCH() Received {self.get_error(search_id)}") while not done: time.sleep(self.PAUSE_BETWEEN_REQUESTS) r = self.get_search_results(search_id, maxresults=maxresults) - if (r["status"] == 0 and r["records"]): - for a in r['records']: + if r["status"] == 0 and r["records"]: + for a in r["records"]: results.append(a) - maxresults -= len(r['records']) - if (r['status'] == 2 or maxresults <= 0): - for a in r['records']: + maxresults -= len(r["records"]) + if r["status"] == 2 or maxresults <= 0: + for a in r["records"]: results.append(a) - if (maxresults <= 0): + if maxresults <= 0: self.terminate_search(search_id) done = True - if r['status'] == 3: + if r["status"] == 3: self.terminate_search(search_id) done = True - return {'records': results} + return {"records": results} def terminate_search(self, id): p = { "id": id, } - r = requests.get(self.API_ROOT + '/live/search/internal', - headers=self.HEADERS, params=p) + r = requests.get( + self.API_ROOT + "/live/search/internal", headers=self.HEADERS, params=p + ) if r.status_code == 204: return (r.status_code, r.text) else: return (r.status_code, r.text) - def export_accounts(self, term, datefrom=None, dateto=None, maxresults=10, buckets="", terminate=None): + def export_accounts( + self, + term, + datefrom=None, + dateto=None, + maxresults=10, + buckets="", + terminate=None, + ): p = { "selector": term, "bucket": buckets, @@ -83,24 +103,70 @@ def export_accounts(self, term, datefrom=None, dateto=None, maxresults=10, bucke } done = False results = [] - r = requests.get(self.API_ROOT + '/accounts/csv', - headers=self.HEADERS, params=p) + r = requests.get( + self.API_ROOT + "/accounts/csv", headers=self.HEADERS, params=p + ) if r.status_code == 200: - search_id = r.json()['id'] - if (len(str(search_id)) <= 3): + search_id = r.json()["id"] + if len(str(search_id)) <= 3: print( - f"[!] intelx.IDENTITY_EXPORT() Received {self.get_error(search_id)}") + f"[!] intelx.IDENTITY_EXPORT() Received {self.get_error(search_id)}" + ) while not done: time.sleep(self.PAUSE_BETWEEN_REQUESTS) r = self.get_search_results(search_id, maxresults=maxresults) - if (r["status"] == 0 and r["records"]): - for a in r['records']: + if r["status"] == 0 and r["records"]: + for a in r["records"]: results.append(a) - maxresults -= len(r['records']) - if (r['status'] == 2 or maxresults <= 0): - if (maxresults <= 0): + maxresults -= len(r["records"]) + if r["status"] == 2 or maxresults <= 0: + if r["records"]: + for a in r["records"]: + results.append(a) + maxresults -= len(r["records"]) + if maxresults <= 0: self.terminate_search(search_id) done = True - return {'records': results} + return {"records": results} + else: + return (r.status_code, r.text) + + def reverse_domain( + self, term, maxresults=10, datefrom=None, dateto=None, terminate=None + ): + p = { + "selector": term, + "limit": maxresults, + "datefrom": datefrom, # "YYYY-MM-DD HH:MM:SS", + "dateto": dateto, # "YYYY-MM-DD HH:MM:SS" + "terminate": terminate, + } + done = False + results = [] + r = requests.get( + self.API_ROOT + "/reverse/domain", headers=self.HEADERS, params=p + ) + if r.status_code == 200: + search_id = r.json()["id"] + if len(str(search_id)) <= 3: + print( + f"[!] intelx.IDENTITY_DOMAIN() Received {self.get_error(search_id)}" + ) + while not done: + time.sleep(self.PAUSE_BETWEEN_REQUESTS) + r = self.get_search_results(search_id, maxresults=maxresults) + if r["status"] == 0 and r["records"]: + for a in r["records"]: + results.append(a) + maxresults -= len(r["records"]) + if r["status"] == 2 or maxresults <= 0: + if r["records"]: + for a in r["records"]: + results.append(a) + maxresults -= len(r["records"]) + if maxresults <= 0: + self.terminate_search(search_id) + done = True + return {"records": results} else: return (r.status_code, r.text) diff --git a/Python/scripts/intelx.py b/Python/scripts/intelx.py index 8647b79..3140d4e 100644 --- a/Python/scripts/intelx.py +++ b/Python/scripts/intelx.py @@ -16,10 +16,10 @@ from pygments.lexers import JsonLexer from pygments.formatters import TerminalFormatter -BOLD = '\033[1m' -END = '\033[0m' +BOLD = "\033[1m" +END = "\033[0m" -banner = r''' +banner = r""" {} _____ _ ___ __ |_ _| | | | \ \ / / | | _ __ | |_ ___| |\ V / @@ -30,30 +30,80 @@ a command line client for intelx.io {} -'''.format(BOLD, END) +""".format( + BOLD, END +) def rightnow(): return time.strftime("%H:%M:%S") -def search(ix, query, maxresults=100, buckets=[], timeout=5, datefrom="", dateto="", sort=4, media=0, terminate=[]): +def search( + ix, + query, + maxresults=100, + buckets=[], + timeout=5, + datefrom="", + dateto="", + sort=4, + media=0, + terminate=[], +): if not args.raw: - print(colored(f"[{rightnow()}] Starting search of \"{args.search}\".", 'green')) - s = ix.search(args.search, maxresults, buckets, timeout, datefrom, dateto, sort, media, terminate) + print(colored(f'[{rightnow()}] Starting search of "{args.search}".', "green")) + s = ix.search( + args.search, + maxresults, + buckets, + timeout, + datefrom, + dateto, + sort, + media, + terminate, + ) return s -def pbsearch(ix, query, maxresults=100, buckets=[], timeout=5, datefrom="", dateto="", sort=4, media=0, terminate=[], target=0): +def pbsearch( + ix, + query, + maxresults=100, + buckets=[], + timeout=5, + datefrom="", + dateto="", + sort=4, + media=0, + terminate=[], + target=0, +): if not args.raw: - print(colored(f"[{rightnow()}] Starting phonebook search of \"{args.search}\".", 'green')) - s = ix.phonebooksearch(args.search, maxresults, buckets, timeout, datefrom, dateto, sort, media, terminate, target) + print( + colored( + f'[{rightnow()}] Starting phonebook search of "{args.search}".', "green" + ) + ) + s = ix.phonebooksearch( + args.search, + maxresults, + buckets, + timeout, + datefrom, + dateto, + sort, + media, + terminate, + target, + ) return s def get_stats(stats): if not args.raw: - print(colored(f"[{rightnow()}] Gathering stats from search.\n", 'green')) + print(colored(f"[{rightnow()}] Gathering stats from search.\n", "green")) stats = json.dumps(ix.stats(search), indent=4, sort_keys=True) print(stats) @@ -64,50 +114,84 @@ def format_list(content): def quick_search_results(ix, search, limit): - for idx, result in enumerate(search['records']): - if(idx == limit): + for idx, result in enumerate(search["records"]): + if idx == limit: sys.exit() else: if args.view: - viewtext = ix.FILE_VIEW(result['type'], result['media'], result['storageid'], result['bucket']) + viewtext = ix.FILE_VIEW( + result["type"], + result["media"], + result["storageid"], + result["bucket"], + ) elif not args.nopreview: - viewtext = ix.FILE_PREVIEW(result['type'], result['media'], 0, result['storageid'], result['bucket']) - if(len(result['name']) == 0): - result['name'] = "Untitled Document" - print(f"{BOLD}________________________________________________________________________________{END}") - print(f"{BOLD}> Name:{END}", html.unescape(result['name'])) - print(f"{BOLD}> Date:{END}", result['date']) - print(f"{BOLD}> Size:{END}", result['size'], "bytes") - print(f"{BOLD}> Media:{END}", result['mediah']) - print(f"{BOLD}> Bucket:{END}", result['bucketh']) - print(f"{BOLD}> ID:{END}", result['systemid']) + viewtext = ix.FILE_PREVIEW( + result["type"], + result["media"], + 0, + result["storageid"], + result["bucket"], + ) + if len(result["name"]) == 0: + result["name"] = "Untitled Document" + print( + f"{BOLD}________________________________________________________________________________{END}" + ) + print(f"{BOLD}> Name:{END}", html.unescape(result["name"])) + print(f"{BOLD}> Date:{END}", result["date"]) + print(f"{BOLD}> Size:{END}", result["size"], "bytes") + print(f"{BOLD}> Media:{END}", result["mediah"]) + print(f"{BOLD}> Bucket:{END}", result["bucketh"]) + print(f"{BOLD}> ID:{END}", result["systemid"]) if len(viewtext) > 0: print("") print(viewtext) - print(f"{BOLD}________________________________________________________________________________{END}") + print( + f"{BOLD}________________________________________________________________________________{END}" + ) def pb_search_results(ix, search): headers = ["Type", "Value"] data = [] for block in search: - for result in block['selectors']: - data.append([result['selectortypeh'], result['selectorvalue']]) + for result in block["selectors"]: + data.append([result["selectortypeh"], result["selectorvalue"]]) print(tabulate.tabulate(sorted(data), headers=headers, tablefmt="fancy_grid")) def pb_search_results_emails(ix, search): for block in search: - for result in block['selectors']: - if result['selectortype'] == 1: - print(result['selectorvalue']) - -def idsearch(identity_ix, query, maxresults=100, buckets=[], timeout=5, datefrom="", dateto="", terminate=[]): + for result in block["selectors"]: + if result["selectortype"] == 1: + print(result["selectorvalue"]) + + +def idsearch( + identity_ix, + query, + maxresults=100, + buckets=[], + timeout=5, + datefrom="", + dateto="", + terminate=[], +): if not args.raw: - print(colored(f"[{rightnow()}] Starting search of \"{args.search}\".", 'green')) - s = identity_ix.search(term=query, maxresults=maxresults, buckets=buckets, timeout=timeout, datefrom=datefrom, dateto=dateto, terminate=terminate) + print(colored(f'[{rightnow()}] Starting search of "{args.search}".', "green")) + s = identity_ix.search( + term=query, + maxresults=maxresults, + buckets=buckets, + timeout=timeout, + datefrom=datefrom, + dateto=dateto, + terminate=terminate, + ) return s + def main(argv=None): global search @@ -117,40 +201,71 @@ def main(argv=None): # get the argument parser ready parser = argparse.ArgumentParser( description="Command line interface for https://intelx.io", - epilog="Usage: intelx.py -search 'riseup.net' -buckets 'pastes, darknet'" + epilog="Usage: intelx.py -search 'riseup.net' -buckets 'pastes, darknet'", ) - parser.add_argument('-apikey', help="set the api key via command line") - parser.add_argument('-search', help="search query") - parser.add_argument('-identity', help="search only accounts on identity service") - parser.add_argument('-buckets', help="set which buckets to search") - parser.add_argument('-limit', help="set the amount of results to show") - parser.add_argument('-timeout', help="set the timeout value") - parser.add_argument('-datefrom', help="begin search starting from state") - parser.add_argument('-dateto', help="begin search ending from date") - parser.add_argument('-sort', help="set the sort value") - parser.add_argument('-media', help="set the media value") - parser.add_argument('-lines', help="set the number of lines displayed in the preview") - parser.add_argument('-download', help="download the specified item specified by its ID") - parser.add_argument('-bucket', help="download from this bucket (must be specified with -download)") - parser.add_argument('-name', help="set the filename to save the item as") - parser.add_argument('--dataleaks', help="searches for a domain or email address to find data leaks", action="store_true") - parser.add_argument('--exportaccounts', help="searches for a domain or email address to find leaked accounts.", action="store_true") - parser.add_argument('--nopreview', help="do not show text preview snippets of search results", action="store_true") - parser.add_argument('--view', help="show full contents of search results", action="store_true") - parser.add_argument('--phonebook', help="set the search type to a phonebook search") - parser.add_argument('--emails', help="show only emails from phonebook results", action="store_true") - parser.add_argument('--capabilities', help="show your account's capabilities", action="store_true") - parser.add_argument('--stats', help="show stats of search results", action="store_true") - parser.add_argument('--raw', help="show raw json", action="store_true") + parser.add_argument("-apikey", help="set the api key via command line") + parser.add_argument("-search", help="search query") + parser.add_argument("-identity", help="search only accounts on identity service") + parser.add_argument("-buckets", help="set which buckets to search") + parser.add_argument("-limit", help="set the amount of results to show") + parser.add_argument("-timeout", help="set the timeout value") + parser.add_argument("-datefrom", help="begin search starting from state") + parser.add_argument("-dateto", help="begin search ending from date") + parser.add_argument("-sort", help="set the sort value") + parser.add_argument("-media", help="set the media value") + parser.add_argument( + "-lines", help="set the number of lines displayed in the preview" + ) + parser.add_argument( + "-download", help="download the specified item specified by its ID" + ) + parser.add_argument( + "-bucket", help="download from this bucket (must be specified with -download)" + ) + parser.add_argument("-name", help="set the filename to save the item as") + parser.add_argument( + "--dataleaks", + help="searches for a domain or email address to find data leaks", + action="store_true", + ) + parser.add_argument( + "--exportaccounts", + help="searches for a domain or email address to find leaked accounts.", + action="store_true", + ) + parser.add_argument( + "--reversedomain", + help="searches for a domain to discover session stealer activity.", + action="store_true", + ) + parser.add_argument( + "--nopreview", + help="do not show text preview snippets of search results", + action="store_true", + ) + parser.add_argument( + "--view", help="show full contents of search results", action="store_true" + ) + parser.add_argument("--phonebook", help="set the search type to a phonebook search") + parser.add_argument( + "--emails", help="show only emails from phonebook results", action="store_true" + ) + parser.add_argument( + "--capabilities", help="show your account's capabilities", action="store_true" + ) + parser.add_argument( + "--stats", help="show stats of search results", action="store_true" + ) + parser.add_argument("--raw", help="show raw json", action="store_true") args = parser.parse_args(argv) # configure IX & the API key - if 'INTELX_KEY' in os.environ: + if "INTELX_KEY" in os.environ: if args.identity: - ix = IdentityService(os.environ['INTELX_KEY']) + ix = IdentityService(os.environ["INTELX_KEY"]) else: - ix = intelx(os.environ['INTELX_KEY']) + ix = intelx(os.environ["INTELX_KEY"]) elif args.apikey: if args.identity: @@ -160,13 +275,15 @@ def main(argv=None): else: print(banner) - print('intelx.py v' + str(version('intelx'))) - exit('No API key specified. Please use the "-apikey" parameter or set the environment variable "INTELX_KEY".') + print("intelx.py v" + str(version("intelx"))) + exit( + 'No API key specified. Please use the "-apikey" parameter or set the environment variable "INTELX_KEY".' + ) # main application flow if not args.raw: print(banner) - print('intelx.py v' + str(version('intelx'))) + print("intelx.py v" + str(version("intelx"))) if len(sys.argv) < 2: print('Usage: intelx.py -search "riseup.net"') @@ -175,7 +292,12 @@ def main(argv=None): if not args.limit and not args.stats and not args.phonebook: if not args.raw: - print(colored(f"[{rightnow()}] Limit argument not supplied, setting default to 10 results.", 'yellow')) + print( + colored( + f"[{rightnow()}] Limit argument not supplied, setting default to 10 results.", + "yellow", + ) + ) args.limit = 10 maxresults = 100 @@ -201,57 +323,130 @@ def main(argv=None): media = int(args.media) if args.exportaccounts: - print(colored(f"[{rightnow()}] Starting account export of \"{args.identity}\".", 'green')) + print( + colored( + f'[{rightnow()}] Starting account export of "{args.identity}".', + "green", + ) + ) account = IdentityService.export_accounts( - ix, - args.identity, - maxresults=maxresults, - buckets=buckets, - datefrom=datefrom, - dateto=dateto, - terminate=terminate + ix, + args.identity, + maxresults=maxresults, + buckets=buckets, + datefrom=datefrom, + dateto=dateto, + terminate=terminate, ) headers = ["User", "Password", "Password Type", "Source Short"] data = [] for block in account: for result in account[block]: - data.append([result['user'], result['password'], result['passwordtype'], result['sourceshort']]) - print(tabulate.tabulate(sorted(data), headers=headers, tablefmt="fancy_grid")) - exporttsv=tabulate.tabulate(data, tablefmt="tsv") + data.append( + [ + result["user"], + result["password"], + result["passwordtype"], + result["sourceshort"], + ] + ) + print( + tabulate.tabulate(sorted(data), headers=headers, tablefmt="fancy_grid") + ) + exporttsv = tabulate.tabulate(data, tablefmt="tsv") tsv_filename = "intelx-output-" + args.identity + "-export_accounts.tsv" - tsv_file=open(tsv_filename,"w") + tsv_file = open(tsv_filename, "w") tsv_file.write(exporttsv) tsv_file.close() - print(colored(f"[{rightnow()}] Exported output to \"{tsv_filename}\".", 'green')) + print( + colored(f'[{rightnow()}] Exported output to "{tsv_filename}".', "green") + ) + + if args.reversedomain: + print( + colored( + f'[{rightnow()}] Starting reverse domain export of "{args.identity}".', + "green", + ) + ) + account = IdentityService.reverse_domain( + ix, + args.identity, + datefrom=datefrom, + dateto=dateto, + terminate=terminate, + ) + headers = ["User", "Password", "Password Type", "Source Short"] + data = [] + for block in account: + for result in account[block]: + data.append( + [ + result["user"], + result["password"], + result["url"], + result["sourceshort"], + ] + ) + print( + tabulate.tabulate(sorted(data), headers=headers, tablefmt="fancy_grid") + ) + exporttsv = tabulate.tabulate(data, tablefmt="tsv") + tsv_filename = "intelx-output-" + args.identity + "-export_accounts.tsv" + tsv_file = open(tsv_filename, "w") + tsv_file.write(exporttsv) + tsv_file.close() + print( + colored(f'[{rightnow()}] Exported output to "{tsv_filename}".', "green") + ) if args.dataleaks: - print(colored(f"[{rightnow()}] Starting data leaks search of \"{args.identity}\".", 'green')) - search = IdentityService.idsearch( - ix, - args.identity, - maxresults=maxresults, - buckets=buckets, - datefrom=datefrom, - dateto=dateto, - terminate=terminate + print( + colored( + f'[{rightnow()}] Starting data leaks search of "{args.identity}".', + "green", ) - headers = ["Name", "Date", "Bucket", "Line"] - data = [] - for records in search: - for result in search[records]: - data.append([result['item']['name'], result['item']['date'], result['item']['bucket'], result['linea']]) - exporttsv=tabulate.tabulate(data, tablefmt="tsv") - tsv_filename = "intelx-output-" + args.identity + "-data_leaks.tsv" - tsv_file=open(tsv_filename,"w") - tsv_file.write(exporttsv) - tsv_file.close() - print(colored(f"[{rightnow()}] Exported output to \"{tsv_filename}\".", 'green')) + ) + search = IdentityService.idsearch( + ix, + args.identity, + maxresults=maxresults, + buckets=buckets, + datefrom=datefrom, + dateto=dateto, + terminate=terminate, + ) + headers = ["Name", "Date", "Bucket", "Line"] + data = [] + for records in search: + for result in search[records]: + data.append( + [ + result["item"]["name"], + result["item"]["date"], + result["item"]["bucket"], + result["linea"], + ] + ) + exporttsv = tabulate.tabulate(data, tablefmt="tsv") + tsv_filename = "intelx-output-" + args.identity + "-data_leaks.tsv" + tsv_file = open(tsv_filename, "w") + tsv_file.write(exporttsv) + tsv_file.close() + print( + colored(f'[{rightnow()}] Exported output to "{tsv_filename}".', "green") + ) if args.search: if not args.limit and not args.stats and not args.phonebook: if not args.raw: - print(colored(f"[{rightnow()}] Limit argument not supplied, setting default to 10 results.", 'yellow')) + print( + colored( + f"[{rightnow()}] Limit argument not supplied, setting default to 10 results.", + "yellow", + ) + ) args.limit = 10 maxresults = 100 @@ -290,15 +485,15 @@ def main(argv=None): dateto=dateto, sort=sort, media=media, - terminate=terminate + terminate=terminate, ) elif args.phonebook: - if(args.phonebook == 'domains'): + if args.phonebook == "domains": targetval = 1 - elif(args.phonebook == 'emails'): + elif args.phonebook == "emails": targetval = 2 - elif(args.phonebook == 'urls'): + elif args.phonebook == "urls": targetval = 3 else: targetval = 0 @@ -314,7 +509,7 @@ def main(argv=None): sort=sort, media=media, terminate=terminate, - target=targetval + target=targetval, ) if args.raw: @@ -336,20 +531,40 @@ def main(argv=None): if args.download: if not args.bucket: - print(colored(f"[{rightnow()}] Failed to download item {args.download} missing bucket name.\n", 'red')) + print( + colored( + f"[{rightnow()}] Failed to download item {args.download} missing bucket name.\n", + "red", + ) + ) else: fname = args.download + ".bin" if args.name: fname = args.name - if(ix.FILE_READ(args.download, bucket=args.bucket, filename=fname)): - print(colored(f"[{rightnow()}] Successfully downloaded the file '{fname}'.\n", 'green')) + if ix.FILE_READ(args.download, bucket=args.bucket, filename=fname): + print( + colored( + f"[{rightnow()}] Successfully downloaded the file '{fname}'.\n", + "green", + ) + ) else: - print(colored(f"[{rightnow()}] Failed to download item {args.download}.\n", 'red')) + print( + colored( + f"[{rightnow()}] Failed to download item {args.download}.\n", + "red", + ) + ) if args.capabilities: - print(colored(f"[{rightnow()}] Getting your API capabilities.\n", 'green')) + print(colored(f"[{rightnow()}] Getting your API capabilities.\n", "green")) capabilities = ix.GET_CAPABILITIES() - print(highlight(json.dumps(capabilities, indent=4), JsonLexer(), TerminalFormatter())) + print( + highlight( + json.dumps(capabilities, indent=4), JsonLexer(), TerminalFormatter() + ) + ) + if __name__ == "__main__": - sys.exit(main()) + sys.exit(main())