diff --git a/scripts/artifacts/FacebookMessenger.py b/scripts/artifacts/FacebookMessenger.py index b1b22c58..7057b668 100755 --- a/scripts/artifacts/FacebookMessenger.py +++ b/scripts/artifacts/FacebookMessenger.py @@ -1,3 +1,4 @@ +# FacebookMessenger.py #2023-02-03: Added support for new msys_database format - Kevin Pagano import os @@ -64,34 +65,67 @@ def get_FacebookMessenger(files_found, report_folder, seeker, wrap_text): source_file = file_found.replace(seeker.data_folder, '') db = open_sqlite_db_readonly(file_found) cursor = db.cursor() - cursor.execute(''' - select - datetime(messages.timestamp_ms/1000,'unixepoch') as "Message Time", - contacts.name as "Sender", - messages.sender_id as "Sender Account ID", - messages.thread_key as "Thread Key", - messages.text as "Message", - attachments.title_text as "Snippet", - attachments.subtitle_text as "Call/Location Information", - attachments.filename as "Attachment File Name", - attachments.playable_url_mime_type as "Attachment Type", - attachments.playable_url as "Attachment URL", - attachment_ctas.native_url as "Location Lat/Long", - reactions.reaction as "Reaction", - datetime(reactions.reaction_creation_timestamp_ms/1000,'unixepoch') as "Reaction Time", - case - when messages.is_admin_message = 1 then "Yes" - when messages.is_admin_message = 0 then "No" - else messages.is_admin_message - end as "Is Admin Message", - messages.message_id as "Message ID" - from messages - join contacts on contacts.id = messages.sender_id - left join attachments on attachments.message_id = messages.message_id - left join attachment_ctas on messages.message_id = attachment_ctas.message_id - left join reactions on reactions.message_id = messages.message_id - order by "Message Time" ASC - ''') + full_schema = True + try: + cursor.execute(''' + select + datetime(messages.timestamp_ms/1000,'unixepoch') as "Message Time", + contacts.name as "Sender", + messages.sender_id as "Sender Account ID", + messages.thread_key as "Thread Key", + messages.text as "Message", + attachments.title_text as "Snippet", + attachments.subtitle_text as "Call/Location Information", + attachments.filename as "Attachment File Name", + attachments.playable_url_mime_type as "Attachment Type", + attachments.playable_url as "Attachment URL", + attachment_ctas.native_url as "Location Lat/Long", + reactions.reaction as "Reaction", + datetime(reactions.reaction_creation_timestamp_ms/1000,'unixepoch') as "Reaction Time", + case + when messages.is_admin_message = 1 then "Yes" + when messages.is_admin_message = 0 then "No" + else messages.is_admin_message + end as "Is Admin Message", + messages.message_id as "Message ID" + from messages + join contacts on contacts.id = messages.sender_id + left join attachments on attachments.message_id = messages.message_id + left join attachment_ctas on messages.message_id = attachment_ctas.message_id + left join reactions on reactions.message_id = messages.message_id + order by "Message Time" ASC + ''') + + except sqlite3.OperationalError as e: + logfunc(f'Error retrieving chats from msys_database: {e}. Attempting alternative query without Reaction Time.') + full_schema = False + cursor.execute(''' + select + datetime(messages.timestamp_ms/1000,'unixepoch') as "Message Time", + contacts.name as "Sender", + messages.sender_id as "Sender Account ID", + messages.thread_key as "Thread Key", + messages.text as "Message", + attachments.title_text as "Snippet", + attachments.subtitle_text as "Call/Location Information", + attachments.filename as "Attachment File Name", + attachments.playable_url_mime_type as "Attachment Type", + attachments.playable_url as "Attachment URL", + attachment_ctas.native_url as "Location Lat/Long", + reactions.reaction as "Reaction", + case + when messages.is_admin_message = 1 then "Yes" + when messages.is_admin_message = 0 then "No" + else messages.is_admin_message + end as "Is Admin Message", + messages.message_id as "Message ID" + from messages + join contacts on contacts.id = messages.sender_id + left join attachments on attachments.message_id = messages.message_id + left join attachment_ctas on messages.message_id = attachment_ctas.message_id + left join reactions on reactions.message_id = messages.message_id + order by "Message Time" ASC + ''') all_rows = cursor.fetchall() usageentries = len(all_rows) @@ -99,10 +133,17 @@ def get_FacebookMessenger(files_found, report_folder, seeker, wrap_text): report = ArtifactHtmlReport('Facebook - Chats - msys_database') report.start_artifact_report(report_folder, f'Facebook{typeof}- Chats{usernum} - msys_database') report.add_script() - data_headers = ('Message Timestamp','Sender','Sender ID','Thread Key','Message','Snippet','Call/Location Information','Attachment Name','Attachment Type','Attachment URL','Location Lat/Long','Reaction','Reaction Timestamp','Is Admin Message','Message ID') # Don't remove the comma, that is required to make this a tuple as there is only 1 element - data_list = [] - for row in all_rows: - data_list.append((row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8],row[9],row[10],row[11],row[12],row[13],row[14])) + + if full_schema: + data_headers = ('Message Timestamp','Sender','Sender ID','Thread Key','Message','Snippet','Call/Location Information','Attachment Name','Attachment Type','Attachment URL','Location Lat/Long','Reaction','Reaction Timestamp','Is Admin Message','Message ID') # 15 columns + data_list = [] + for row in all_rows: + data_list.append((row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8],row[9],row[10],row[11],row[12],row[13],row[14])) + else: + data_headers = ('Message Timestamp','Sender','Sender ID','Thread Key','Message','Snippet','Call/Location Information','Attachment Name','Attachment Type','Attachment URL','Location Lat/Long','Reaction','Is Admin Message','Message ID') # 14 columns + data_list = [] + for row in all_rows: + data_list.append((row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8],row[9],row[10],row[11],row[12],row[13])) report.write_artifact_data_table(data_headers, data_list, file_found) report.end_artifact_report() @@ -197,7 +238,7 @@ def get_FacebookMessenger(files_found, report_folder, seeker, wrap_text): tsv(report_folder, data_headers, data_list, tsvname, source_file) else: - logfunc(f'No Facebook{typeof}- Contacts{usernum} - msys_database data available') + logfunc(f'No Facebook{typeof}- Contacts{usernum} - msys_database data available') db.close() @@ -207,13 +248,15 @@ def get_FacebookMessenger(files_found, report_folder, seeker, wrap_text): source_file = file_found.replace(seeker.data_folder, '') db = open_sqlite_db_readonly(file_found) cursor = db.cursor() + + snippet = 0 try: cursor.execute(''' select case messages.timestamp_ms when 0 then '' else datetime(messages.timestamp_ms/1000,'unixepoch') - End as Datestamp, + End as Datestamp, (select json_extract (messages.sender, '$.name')) as "Sender", substr((select json_extract (messages.sender, '$.user_key')),10) as "Sender ID", messages.thread_key, @@ -231,14 +274,16 @@ def get_FacebookMessenger(files_found, report_folder, seeker, wrap_text): where messages.thread_key=threads.thread_key and generic_admin_message_extensible_data IS NULL and msg_type != -1 order by messages.thread_key, datestamp; ''') - snippet = 1 - except: + snippet = 1 + except sqlite3.OperationalError as e: + logfunc(f'Handling Error from threads_db2: {e}. current Query without snippet and Reaction Timestamp.') + cursor.execute(''' select case messages.timestamp_ms when 0 then '' else datetime(messages.timestamp_ms/1000,'unixepoch') - End as Datestamp, + End as Datestamp, (select json_extract (messages.sender, '$.name')) as "Sender", substr((select json_extract (messages.sender, '$.user_key')),10) as "Sender ID", messages.thread_key, @@ -248,14 +293,13 @@ def get_FacebookMessenger(files_found, report_folder, seeker, wrap_text): (select json_extract (messages.shares, '$[0].description')) as ShareDesc, (select json_extract (messages.shares, '$[0].href')) as ShareLink, message_reactions.reaction as "Message Reaction", - datetime(message_reactions.reaction_timestamp/1000,'unixepoch') as "Message Reaction Timestamp", messages.msg_id from messages, threads left join message_reactions on message_reactions.msg_id = messages.msg_id where messages.thread_key=threads.thread_key and generic_admin_message_extensible_data IS NULL and msg_type != -1 order by messages.thread_key, datestamp; ''') - + all_rows = cursor.fetchall() usageentries = len(all_rows) if usageentries > 0: @@ -269,9 +313,9 @@ def get_FacebookMessenger(files_found, report_folder, seeker, wrap_text): for row in all_rows: data_list.append((row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8],row[9],row[10],row[11],row[12])) else: - data_headers = ('Timestamp','Sender Name','Sender ID','Thread Key','Message','Attachment Name','Share Name','Share Description','Share Link','Message Reaction','Message Reaction Timestamp','Message ID') + data_headers = ('Timestamp','Sender Name','Sender ID','Thread Key','Message','Attachment Name','Share Name','Share Description','Share Link','Message Reaction','Message ID') for row in all_rows: - data_list.append((row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8],row[9],row[10],row[11])) + data_list.append((row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8],row[9],row[10])) report.write_artifact_data_table(data_headers, data_list, file_found) report.end_artifact_report() @@ -307,7 +351,7 @@ def get_FacebookMessenger(files_found, report_folder, seeker, wrap_text): report = ArtifactHtmlReport('Facebook - Calls - threads_db2') report.start_artifact_report(report_folder, f'Facebook{typeof}- Calls{usernum} - threads_db2') report.add_script() - data_headers = ('Timestamp','Call Duration','Caller ID','Receiver Name','Receiver ID','Video Call','Thread Key') # Don't remove the comma, that is required to make this a tuple as there is only 1 element + data_headers = ('Call Timestamp','Call Duration','Caller ID','Receiver Name','Receiver ID','Video Call','Thread Key') # Don't remove the comma, that is required to make this a tuple as there is only 1 element data_list = [] for row in all_rows: data_list.append((row[0],row[1],row[2],row[3],row[4],row[5],row[6])) @@ -323,25 +367,49 @@ def get_FacebookMessenger(files_found, report_folder, seeker, wrap_text): else: logfunc(f'No Facebook{typeof}- Calls{usernum} - threads_db2 data available') - cursor.execute(''' - select - substr(user_key,10), - first_name, - last_name, - username, - (select json_extract (profile_pic_square, '$[0].url')) as profile_pic_square, - case is_messenger_user - when 0 then '' - else 'Yes' - end is_messenger_user, - case is_friend - when 0 then 'No' - when 1 then 'Yes' - end is_friend, - friendship_status, - contact_relationship_status - from thread_users - ''') + contacts_full_schema = True + try: + cursor.execute(''' + select + substr(user_key,10), + first_name, + last_name, + username, + (select json_extract (profile_pic_square, '$[0].url')) as profile_pic_square, + case is_messenger_user + when 0 then '' + else 'Yes' + end is_messenger_user, + case is_friend + when 0 then 'No' + when 1 then 'Yes' + end is_friend, + friendship_status, + contact_relationship_status + from thread_users + ''') + + except sqlite3.OperationalError as e: + logfunc(f'Handling Error : {e}. current Query without friendship_status/contact_relationship_status.') + contacts_full_schema = False + cursor.execute(''' + select + substr(user_key,10), + first_name, + last_name, + username, + (select json_extract (profile_pic_square, '$[0].url')) as profile_pic_square, + case is_messenger_user + when 0 then '' + else 'Yes' + end is_messenger_user, + case is_friend + when 0 then 'No' + when 1 then 'Yes' + end is_friend + from thread_users + ''') + # --- END threads_db2 CONTACTS EXCEPTION HANDLING --- all_rows = cursor.fetchall() usageentries = len(all_rows) @@ -349,10 +417,17 @@ def get_FacebookMessenger(files_found, report_folder, seeker, wrap_text): report = ArtifactHtmlReport('Facebook - Contacts - threads_db2') report.start_artifact_report(report_folder, f'Facebook{typeof}- Contacts{usernum} - threads_db2') report.add_script() - data_headers = ('User ID','First Name','Last Name','Username','Profile Pic URL','Is Messenger User','Is Friend','Friendship Status','Contact Relationship Status') # Don't remove the comma, that is required to make this a tuple as there is only 1 element data_list = [] - for row in all_rows: - data_list.append((row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8])) + + if contacts_full_schema: + data_headers = ('User ID','First Name','Last Name','Username','Profile Pic URL','Is Messenger User','Is Friend','Friendship Status','Contact Relationship Status') + for row in all_rows: + data_list.append((row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8])) + else: + data_headers = ('User ID','First Name','Last Name','Username','Profile Pic URL','Is Messenger User','Is Friend') + for row in all_rows: + data_list.append((row[0],row[1],row[2],row[3],row[4],row[5],row[6])) + report.write_artifact_data_table(data_headers, data_list, file_found) report.end_artifact_report() @@ -368,11 +443,11 @@ def get_FacebookMessenger(files_found, report_folder, seeker, wrap_text): continue else: - continue # Skip all other files + continue __artifacts__ = { "FacebookMessenger": ( "Facebook Messenger", ('*/*threads_db2*','*/msys_database*'), get_FacebookMessenger) -} +} \ No newline at end of file diff --git a/scripts/artifacts/deviceHealthServices_Battery.py b/scripts/artifacts/deviceHealthServices_Battery.py index 38f2c8df..9aa3316b 100644 --- a/scripts/artifacts/deviceHealthServices_Battery.py +++ b/scripts/artifacts/deviceHealthServices_Battery.py @@ -37,12 +37,18 @@ from scripts.artifact_report import ArtifactHtmlReport from scripts.ilapfuncs import artifact_processor, open_sqlite_db_readonly, convert_ts_human_to_utc, convert_utc_human_to_timezone +@artifact_processor @artifact_processor def Turbo_Battery(files_found, report_folder, seeker, wrap_text): source_file_turbo = '' turbo_db = '' data_list = [] - + + try: + time_offset + except NameError: + time_offset = '0' + for file_found in files_found: file_found = str(file_found) if file_found.lower().endswith('turbo.db'): @@ -80,7 +86,10 @@ def Turbo_Battery(files_found, report_folder, seeker, wrap_text): if timestamp is None: pass else: - timestamp = convert_utc_human_to_timezone(convert_ts_human_to_utc(timestamp),time_offset) + try: + timestamp = convert_utc_human_to_timezone(convert_ts_human_to_utc(timestamp),time_offset) + except Exception as e: + pass data_list.append((timestamp,row[1],row[2],row[3],row[4],file_found)) db.close() diff --git a/scripts/artifacts/googleCallScreen.py b/scripts/artifacts/googleCallScreen.py index ab1ee250..62dd6e09 100755 --- a/scripts/artifacts/googleCallScreen.py +++ b/scripts/artifacts/googleCallScreen.py @@ -1,3 +1,4 @@ +#googleCallScreen.py import blackboxprotobuf import os import shutil @@ -21,17 +22,40 @@ def get_googleCallScreen(files_found, report_folder, seeker, wrap_text): db = open_sqlite_db_readonly(file_found) cursor = db.cursor() - cursor.execute(''' - select - datetime(lastModifiedMillis/1000,'unixepoch'), - audioRecordingFilePath, - conversation, - id, - replace(audioRecordingFilePath, rtrim(audioRecordingFilePath, replace(audioRecordingFilePath, '/', '')), '') as 'File Name' - from Transcript - ''') + + try: + cursor.execute(''' + select + datetime(lastModifiedMillis/1000,'unixepoch'), -- Index 0 + audioRecordingFilePath, -- Index 1 + conversation, -- Index 2 + id, -- Index 3 + replace(audioRecordingFilePath, rtrim(audioRecordingFilePath, replace(audioRecordingFilePath, '/', '')), '') as 'File Name' -- Index 4 + from Transcript + ''') + logfunc('Using primary query (with all expected columns).') + all_rows = cursor.fetchall() + + except sqlite3.OperationalError as e: + if 'no such column' in str(e): + logfunc(f'Column not found in {file_found}. current Alternative') + + cursor.execute(''' + select + + NULL, -- Placeholder for the missing lastModifiedMillis timestamp (Index 0) + NULL, -- Placeholder for the missing audioRecordingFilePath (Index 1) + conversation, -- Actual value (Index 2) + id, -- Actual value (Index 3) + NULL -- Placeholder for 'File Name' (Index 4) + from Transcript + ''') + all_rows = cursor.fetchall() + else: + logfunc(f'An unexpected database error occurred: {e}') + db.close() + continue - all_rows = cursor.fetchall() usageentries = len(all_rows) data_list = [] @@ -56,11 +80,11 @@ def get_googleCallScreen(files_found, report_folder, seeker, wrap_text): if usageentries > 0: for row in all_rows: - lm_ts = row[0] - recording_path = row[1] - pb = row[2] + lm_ts = row[0] + recording_path = row[1] + pb = row[2] convo_id = row[3] - recording_filename = row[4] + recording_filename = row[4] audio_clip = '' conversation = '' @@ -72,10 +96,11 @@ def get_googleCallScreen(files_found, report_folder, seeker, wrap_text): convo_transcript = x['convo_text'] + '

' conversation += convo_timestamp + convo_transcript - for match in files_found: - if str(recording_filename) in match: - shutil.copy2(match, report_folder) - audio_clip = f'' + if recording_filename: + for match in files_found: + if str(recording_filename) in match: + shutil.copy2(match, report_folder) + audio_clip = f'' data_list.append((lm_ts,recording_path,conversation,audio_clip)) @@ -95,7 +120,7 @@ def get_googleCallScreen(files_found, report_folder, seeker, wrap_text): tlactivity = f'Google Call Screen' timeline(report_folder, tlactivity, data_list, data_headers) else: - logfunc('No Google Call Screen data available') + logfunc('Google Call Screen data not found') db.close() @@ -104,4 +129,4 @@ def get_googleCallScreen(files_found, report_folder, seeker, wrap_text): "Google Call Screen", ('*/com.google.android.dialer/databases/callscreen_transcripts*','*/com.google.android.dialer/files/callscreenrecordings/*.*'), get_googleCallScreen) -} +} \ No newline at end of file diff --git a/scripts/artifacts/keepNotes.py b/scripts/artifacts/keepNotes.py index a50ec7b3..b4b44be2 100644 --- a/scripts/artifacts/keepNotes.py +++ b/scripts/artifacts/keepNotes.py @@ -1,3 +1,4 @@ +#keepNotes.py __artifacts_v2__ = { "keepNotes": { "name": "Google Keep Notes", @@ -28,19 +29,57 @@ def get_keepNotes(files_found, report_folder, seeker, wrap_text): if filename.endswith('keep.db'): db = open_sqlite_db_readonly(file_found) cursor = db.cursor() - cursor.execute(''' - SELECT - datetime(tree_entity.time_created/1000, 'unixepoch') AS "Time Created", - datetime(tree_entity.time_last_updated/1000, 'unixepoch') AS "Time Last Updated", - datetime(tree_entity.user_edited_timestamp/1000, 'unixepoch') AS "User Edited Timestamp", - tree_entity.title AS Title, - text_search_note_content_content.c0text AS "Text", - tree_entity.last_modifier_email AS "Last Modifier Email" - FROM text_search_note_content_content - INNER JOIN tree_entity ON text_search_note_content_content.docid = tree_entity._id - ''') + all_rows = [] + + primary_query = ''' + SELECT + datetime(T1.time_created/1000, 'unixepoch') AS "Time Created", + datetime(T1.time_last_updated/1000, 'unixepoch') AS "Time Last Updated", + datetime(T1.user_edited_timestamp/1000, 'unixepoch') AS "User Edited Timestamp", + T1.title AS Title, + T2.c0text AS "Text", + T1.last_modifier_email AS "Last Modifier Email" + FROM tree_entity T1 + INNER JOIN text_search_note_content_content T2 ON T2.docid = T1._id + ''' + data_headers = ('Time Created', 'Time Last Updated', 'User Edited Timestamp', 'Title', 'Text', 'Last Modifier Email') + + try: + cursor.execute(primary_query) + all_rows = cursor.fetchall() + logfunc(f'Good main Query for {filename}.') + + except sqlite3.OperationalError as e: + if 'no such table' in str(e): + logfunc(f'Bad Query failed for {filename} missing table: {e}, alternative->') + + fallback_query = ''' + SELECT + datetime(T1.time_created/1000, 'unixepoch') AS "Time Created", + datetime(T1.time_last_updated/1000, 'unixepoch') AS "Time Last Updated", + datetime(T1.user_edited_timestamp/1000, 'unixepoch') AS "User Edited Timestamp", + T1.title AS Title, + T2.note_content_text AS "Text (From Note Changes)", + T1.last_modifier_email AS "Last Modifier Email" + FROM tree_entity T1 + LEFT JOIN note_changes T2 ON T1._id = T2.note_id + GROUP BY T1._id + ORDER BY T1.time_created ASC + ''' + data_headers = ('Time Created', 'Time Last Updated', 'User Edited Timestamp', 'Title', 'Text (From Note Changes)', 'Last Modifier Email') + + try: + cursor.execute(fallback_query) + all_rows = cursor.fetchall() + logfunc(f'Good Fallback for {filename}.') + except Exception as fe: + logfunc(f'Bad Fallback, with error: {fe}') + continue + + else: + logfunc(f'Error unhandled SQLite Operational Error in {filename}: {e}') + raise - all_rows = cursor.fetchall() usageentries = len(all_rows) if usageentries > 0: @@ -51,7 +90,6 @@ def get_keepNotes(files_found, report_folder, seeker, wrap_text): report = ArtifactHtmlReport('Google Keep Notes') report.start_artifact_report(report_folder, 'Google Keep Notes') report.add_script() - data_headers = ('Time Created', 'Time Last Updated', 'User Edited Timestamp', 'Title', 'Text', 'Last Modifier Email') report.write_artifact_data_table(data_headers, data_list, file_found, html_escape=False) report.end_artifact_report() @@ -62,5 +100,4 @@ def get_keepNotes(files_found, report_folder, seeker, wrap_text): timeline(report_folder, tlactivity, data_list, data_headers) else: - logfunc('No Google Keep Notes data available') - + logfunc(f'No Google Keep Notes data available') \ No newline at end of file diff --git a/scripts/artifacts/wellbeing.py b/scripts/artifacts/wellbeing.py index a2cdfc0c..1c6913bd 100755 --- a/scripts/artifacts/wellbeing.py +++ b/scripts/artifacts/wellbeing.py @@ -1,3 +1,4 @@ +#wellbeing.py __artifacts_v2__ = { "wellbeing": { "name": "Digital Wellbeing", diff --git a/scripts/ilapfuncs.py b/scripts/ilapfuncs.py index c9484834..680e1a31 100755 --- a/scripts/ilapfuncs.py +++ b/scripts/ilapfuncs.py @@ -292,8 +292,11 @@ def wrapper(files_found, report_folder, seeker, wrap_text): func_name = func.__name__ func_object = func.__globals__.get(func_name, {}) - artifact_info = func_object.artifact_info #get('artifact_info', {}) - + + artifact_info = func.__globals__.get('__artifacts_v2__', {}).get(func_name, {}) + if not artifact_info: + artifact_info = func_object.get(func_name, {}) + artifact_name = artifact_info.get('name', func_name) category = artifact_info.get('category', '') description = artifact_info.get('description', '') @@ -1306,6 +1309,4 @@ def check_internet_connection(): return True except: logfunc("Internet connection is not available.") - return False - - \ No newline at end of file + return False \ No newline at end of file