From 7febb549dec6d23770177132a5fa5887e1d38434 Mon Sep 17 00:00:00 2001 From: Funeoz Date: Fri, 12 Dec 2025 21:26:42 +0100 Subject: [PATCH 1/4] Add citymapper parser --- scripts/artifacts/citymapper.py | 576 ++++++++++++++++++++++++++++++++ 1 file changed, 576 insertions(+) create mode 100644 scripts/artifacts/citymapper.py diff --git a/scripts/artifacts/citymapper.py b/scripts/artifacts/citymapper.py new file mode 100644 index 00000000..a3bbfffd --- /dev/null +++ b/scripts/artifacts/citymapper.py @@ -0,0 +1,576 @@ +# Citymapper App (com.citymapper.app.release) +# Author : Funeoz +# Version : 0.0.1 + +# Tested with the following versions: +# 2025-12-12: Android 12.0, App: 11.43.2 + +# Requirements: Python 3.7 or higher, folium + +__artifacts_v2__ = { + "get_citymapperLocationHistory" : { + "name": "Citymapper - Location History", + "description": "Parses location history from the Citymapper App", + "author": "Funeoz", + "version": "0.0.1", + "date":"2025-12-12", + "requirements": "none", + "category" : "Citymapper", + "notes" : "", + "paths" : ('*/data/data/com.citymapper.app.release/databases/citymapper.db'), + "output_types": ['tsv', 'timeline', 'lava', 'kml'] # Exclude 'html' to use custom report + }, + "get_citymapperSavedTrips" : { + "name": "Citymapper - Saved Trips", + "description": "Parses saved trips (home/work) from the Citymapper App", + "author": "Funeoz", + "version": "0.0.1", + "date":"2025-12-12", + "requirements": "none", + "category" : "Citymapper", + "notes" : "", + "paths" : ('*/data/data/com.citymapper.app.release/databases/citymapper.db'), + "output_types": ['tsv', 'timeline', 'lava', 'kml'] # Exclude 'html' to use custom report + }, + "get_citymapperUserData" : { + "name": "Citymapper - User Data", + "description": "Parses user data from the Citymapper App", + "author": "Funeoz", + "version": "0.0.1", + "date":"2025-12-12", + "requirements": "none", + "category" : "Citymapper", + "notes" : "", + "paths" : ('*/data/data/com.citymapper.app.release/shared_prefs/superProperties.xml*', + '*/data/data/com.citymapper.app.release/shared_prefs/preferences.xml*', + '*/data/data/com.citymapper.app.release/shared_prefs/Session.xml*', + '*/data/data/com.citymapper.app.release/shared_prefs/no_backup_preferences.xml*' + ), + "output_types": ['tsv', 'timeline', 'lava', 'kml'] # Exclude 'html' to use custom report + } +} + +import os +import folium +import xml.etree.ElementTree as ET + +from scripts.artifact_report import ArtifactHtmlReport +from scripts.ilapfuncs import artifact_processor, logfunc, tsv, timeline, kmlgen, open_sqlite_db_readonly, convert_unix_ts_to_utc + +@artifact_processor +def get_citymapperLocationHistory(files_found, report_folder, _seeker, _wrap_text): + + location_data_list = [] + source = '' + + for file_found in files_found: + file_found = str(file_found) + + if file_found.endswith('citymapper.db'): + source = file_found + db = open_sqlite_db_readonly(file_found) + cursor = db.cursor() + + # Fetch location history entries + cursor.execute(''' + SELECT + id, + address, + date, + lat, + lng, + name, + role + FROM locationhistoryentry + ''') + + location_rows = cursor.fetchall() + for row in location_rows: + location_data_list.append((row[0], row[1], row[2], row[3], row[4], row[5], row[6])) + + db.close() + + if not location_data_list: + return ('No Data',), [], source + + location_headers = ('ID', 'Address', 'Timestamp', 'Latitude', 'Longitude', 'Name', 'Role') + + kmlgen(report_folder, 'Citymapper - Location History', location_data_list, location_headers) + + # Generate map for location history + try: + # Find center point (average of all coordinates) + valid_coords = [(row[3], row[4]) for row in location_data_list if row[3] and row[4]] + if valid_coords: + center_lat = sum(coord[0] for coord in valid_coords) / len(valid_coords) + center_lon = sum(coord[1] for coord in valid_coords) / len(valid_coords) + + m = folium.Map(location=[center_lat, center_lon], zoom_start=12, max_zoom=19) + + # Sort locations by date for chronological numbering + sorted_locations = sorted( + [row for row in location_data_list if row[3] and row[4]], + key=lambda x: x[2] if x[2] else '' + ) + + # Add numbered markers for each location + for idx, row in enumerate(sorted_locations, 1): + popup_text = f"#{idx}: {row[5] if row[5] else 'Location'}
" + popup_text += f"Address: {row[1] if row[1] else 'N/A'}
" + popup_text += f"Date: {row[2] if row[2] else 'N/A'}
" + popup_text += f"Role: {row[6] if row[6] else 'N/A'}" + + # Different colors based on role + color = 'blue' + if row[6] == 'home': + color = 'green' + elif row[6] == 'work': + color = 'red' + + folium.Marker( + [row[3], row[4]], + popup=popup_text, + icon=folium.DivIcon(html=f''' +
{idx}
+ ''') + ).add_to(m) + + # Save map + map_filename = 'Citymapper_Location_History_Map.html' + if os.name == 'nt': + map_path = os.path.join(report_folder, map_filename) + else: + map_path = report_folder + '/' + map_filename + + m.save(map_path) + + # Create custom HTML report + report = ArtifactHtmlReport('Citymapper - Location History') + report.start_artifact_report(report_folder, 'Citymapper - Location History') + report.add_script() + + # Source Path section + report.add_section_heading('Source Path', 'h3') + report.write_raw_html(f''' +
+
File Path
+
{source}
+
+ ''') + + # Summary section + report.add_section_heading('Location History Summary', 'h3') + report.write_raw_html(f''' +
+
Total Locations
+
{len(location_data_list)}
+
Locations with Coordinates
+
{len(valid_coords)}
+
+ ''') + + # Location Details Table + report.add_section_heading('Location Details', 'h3') + table_html = '' + for header in location_headers: + table_html += f'' + table_html += '' + + for row in location_data_list: + table_html += '' + for cell in row: + table_html += f'' + table_html += '' + + table_html += '
{header}
{cell if cell is not None else ""}
' + report.write_raw_html(table_html) + + # Add map section + report.add_section_heading('Location History Map') + report.add_map(f'') + + # End report + report.end_artifact_report() + + + except Exception as e: + logfunc(f"Error generating location history map: {e}") + + return location_headers, location_data_list, source + + +@artifact_processor +def get_citymapperSavedTrips(files_found, report_folder, _seeker, _wrap_text): + + saved_trip_data_list = [] + source = '' + + for file_found in files_found: + file_found = str(file_found) + + if file_found.endswith('citymapper.db'): + source = file_found + db = open_sqlite_db_readonly(file_found) + cursor = db.cursor() + + # Fetch saved trip entries + cursor.execute(''' + SELECT + id, + commuteType, + created, + homeLat, + homeLng, + workLat, + workLng, + tripData, + regionCode + FROM savedtripentry + ''') + + trip_rows = cursor.fetchall() + for row in trip_rows: + saved_trip_data_list.append((row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[8])) + + db.close() + + """if not saved_trip_data_list: + return ('No Data',), [], source""" + + trip_headers = ('ID', 'Commute Type', 'Timestamp', 'Home Latitude', 'Home Longitude', 'Work Latitude', 'Work Longitude', 'Region Code') + + # Create KML + kmlgen(report_folder, 'Citymapper - Saved Trips', saved_trip_data_list, trip_headers) + + # Generate map for saved trips + try: + # Collect home and work coordinates + home_work_coords = [] + for row in saved_trip_data_list: + if row[3] and row[4]: # home lat, lng + home_work_coords.append((row[3], row[4], 'Home', row[0])) + if row[5] and row[6]: # work lat, lng + home_work_coords.append((row[5], row[6], 'Work', row[0])) + + if home_work_coords: + # Find center point + center_lat = sum(coord[0] for coord in home_work_coords) / len(home_work_coords) + center_lon = sum(coord[1] for coord in home_work_coords) / len(home_work_coords) + + m = folium.Map(location=[center_lat, center_lon], zoom_start=11, max_zoom=19) + + # Add markers for home/work locations + for coord in home_work_coords: + lat, lon, place_type, trip_id = coord + popup_text = f"{place_type}
Trip ID: {trip_id}
Lat: {lat}, Lon: {lon}" + + color = 'green' if place_type == 'Home' else 'red' + icon = 'home' if place_type == 'Home' else 'briefcase' + + folium.Marker( + [lat, lon], + popup=popup_text, + icon=folium.Icon(color=color, icon=icon, prefix='fa') + ).add_to(m) + + # Draw lines between home and work for each trip + for row in saved_trip_data_list: + if row[3] and row[4] and row[5] and row[6]: + folium.PolyLine( + [[row[3], row[4]], [row[5], row[6]]], + color='blue', + weight=2, + opacity=0.6, + popup=f"Trip ID: {row[0]}" + ).add_to(m) + + # Save map + map_filename = 'Citymapper_Saved_Trips_Map.html' + if os.name == 'nt': + map_path = os.path.join(report_folder, map_filename) + else: + map_path = report_folder + '/' + map_filename + + m.save(map_path) + + # Create custom HTML report + report = ArtifactHtmlReport('Citymapper - Saved Trips') + report.start_artifact_report(report_folder, 'Citymapper - Saved Trips') + report.add_script() + + # Source Path section + report.add_section_heading('Source Path', 'h3') + report.write_raw_html(f''' +
+
File Path
+
{source}
+
+ ''') + + # Summary section + report.add_section_heading('Saved Trips Summary', 'h3') + home_count = sum(1 for row in saved_trip_data_list if row[3] and row[4]) + work_count = sum(1 for row in saved_trip_data_list if row[5] and row[6]) + report.write_raw_html(f''' +
+
Total Saved Trips
+
{len(saved_trip_data_list)}
+
+ ''') + + # Trip Details Table + report.add_section_heading('Trip Details', 'h3') + table_html = '' + for header in trip_headers: + table_html += f'' + table_html += '' + + for row in saved_trip_data_list: + table_html += '' + for cell in row: + table_html += f'' + table_html += '' + + table_html += '
{header}
{cell if cell is not None else ""}
' + report.write_raw_html(table_html) + + # Add map section + report.add_section_heading('Saved Trips Map') + report.add_map(f'') + + # End report + report.end_artifact_report() + + except Exception as e: + logfunc(f"Error generating saved trips map: {e}") + + return trip_headers, saved_trip_data_list, source + + +@artifact_processor +def get_citymapperUserData(files_found, report_folder, _seeker, _wrap_text): + + data_list = [] + source = '' + + # Dictionary to store parsed data from all files + user_data = {} + + for file_found in files_found: + file_found = str(file_found) + + if not source: + source = file_found + + try: + tree = ET.parse(file_found) + root = tree.getroot() + + for child in root: + name = child.attrib.get('name', '') + + # Handle different XML element types + if child.tag == 'string': + user_data[name] = child.text if child.text else '' + elif child.tag == 'long': + user_data[name] = child.attrib.get('value', '') + elif child.tag == 'boolean': + user_data[name] = child.attrib.get('value', '') + elif child.tag == 'set': + # Handle set elements (typically empty or with multiple values) + set_values = [item.text for item in child if item.text] + user_data[name] = ', '.join(set_values) if set_values else 'Empty' + + except Exception as e: + logfunc(f"Error parsing XML from {file_found}: {e}") + + # Extract and format key data fields with proper timestamp conversion + device_id = user_data.get('deviceID', '') + device_ip = user_data.get('deviceIp', '') + last_seen_version = user_data.get('lastSeenVersion', '') + earliest_seen_version = user_data.get('earliestSeenVersion', '') + last_location = user_data.get('LAST_LOCATION', '') + + # Convert timestamps + onboarding_date = user_data.get('onboarding_terms_accepted_date', '') + if onboarding_date: + onboarding_date = convert_unix_ts_to_utc(int(onboarding_date)) + + last_used_date = user_data.get('LastUsedDate', '') + if last_used_date: + last_used_date = convert_unix_ts_to_utc(int(last_used_date)) + + session_count = user_data.get('SessionCount', '') + + # App properties + language = user_data.get('Language', '') + app_installed = user_data.get('App Installed', '') + cm_region = user_data.get('CM Region', '') + connectivity_state = user_data.get('Connectivity State', '') + os_api_level = user_data.get('OS API Level', '') + build_flavor = user_data.get('Build Flavor', '') + + # Add main record with all important fields + data_list.append(( + device_id, + device_ip, + last_seen_version, + earliest_seen_version, + last_location, + onboarding_date, + last_used_date, + session_count, + language, + app_installed, + cm_region, + connectivity_state, + os_api_level, + build_flavor + )) + + data_headers = ( + 'Device ID', + 'Device IP', + 'Last Seen Version', + 'Earliest Seen Version', + 'Last Location (Lat,Lon)', + ('Onboarding Date', 'datetime'), + ('Last Used Date', 'datetime'), + 'Session Count', + 'Language', + 'App Installed', + 'CityMapper Region', + 'Connectivity State', + 'OS API Level', + 'Build Flavor' + ) + + # Generate and add folium map if location exists + if last_location and len(data_list) > 0: + try: + # Parse coordinates (format: "latitude,longitude") + coords = last_location.split(',') + if len(coords) == 2: + lat = float(coords[0]) + lon = float(coords[1]) + + # Create folium map + m = folium.Map(location=[lat, lon], zoom_start=13, max_zoom=19) + + # Add marker for last location + folium.Marker( + [lat, lon], + popup=f'Last Location
Lat: {lat}, Lon: {lon}', + icon=folium.Icon(color='red', icon='map-pin', prefix='fa') + ).add_to(m) + + # Save map to HTML file + map_filename = 'Citymapper_Last_Location_Map.html' + if os.name == 'nt': + map_path = os.path.join(report_folder, map_filename) + else: + map_path = report_folder + '/' + map_filename + + m.save(map_path) + + # Create report and add map section + report = ArtifactHtmlReport('Citymapper - User Data') + report.start_artifact_report(report_folder, 'Citymapper - User Data') + report.add_script() + + # Source Path section + report.add_section_heading('Source Path', 'h3') + report.write_raw_html(f''' +
+
File Path
+
{source}
+
+ ''') + + # Device Information + report.add_section_heading('Device Information', 'h3') + report.write_raw_html(f''' +
+
Device ID
+
{device_id}
+
Device IP
+
{device_ip}
+
+ ''') + + # App Version Information + report.add_section_heading('App Version Information', 'h3') + report.write_raw_html(f''' +
+
Last Seen Version
+
{last_seen_version}
+
Earliest Seen Version
+
{earliest_seen_version}
+
App Installed
+
{app_installed}
+
+ ''') + + # Location Information + report.add_section_heading('Location Information', 'h3') + report.write_raw_html(f''' +
+
Last Location (Lat,Lon)
+
{last_location}
+
CityMapper Region
+
{cm_region}
+
+ ''') + + # Session Information + report.add_section_heading('Session Information', 'h3') + report.write_raw_html(f''' +
+
Onboarding Date
+
{onboarding_date}
+
Last Used Date
+
{last_used_date}
+
Session Count
+
{session_count}
+
+ ''') + + # System & App Settings + report.add_section_heading('System & App Settings', 'h3') + report.write_raw_html(f''' +
+
Language
+
{language}
+
Connectivity State
+
{connectivity_state}
+
OS API Level
+
{os_api_level}
+
Build Flavor
+
{build_flavor}
+
+ ''') + + # Add map section + report.add_section_heading('Last Location Map') + report.add_map(f'') + + # End report + report.end_artifact_report() + + except Exception as e: + logfunc(f"Error generating map for last location: {e}") + + return data_headers, data_list, source \ No newline at end of file From 1bff0c41edf3d7d680e71dd1d6cf6b0748a25382 Mon Sep 17 00:00:00 2001 From: Funeoz Date: Fri, 12 Dec 2025 21:42:51 +0100 Subject: [PATCH 2/4] Rename user data function to app preferences and add file paths in report --- scripts/artifacts/citymapper.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/scripts/artifacts/citymapper.py b/scripts/artifacts/citymapper.py index a3bbfffd..c47b05c5 100644 --- a/scripts/artifacts/citymapper.py +++ b/scripts/artifacts/citymapper.py @@ -32,9 +32,9 @@ "paths" : ('*/data/data/com.citymapper.app.release/databases/citymapper.db'), "output_types": ['tsv', 'timeline', 'lava', 'kml'] # Exclude 'html' to use custom report }, - "get_citymapperUserData" : { - "name": "Citymapper - User Data", - "description": "Parses user data from the Citymapper App", + "get_citymapperAppPreferences" : { + "name": "Citymapper - App Preferences", + "description": "Parses app preferences from the Citymapper App", "author": "Funeoz", "version": "0.0.1", "date":"2025-12-12", @@ -55,7 +55,7 @@ import xml.etree.ElementTree as ET from scripts.artifact_report import ArtifactHtmlReport -from scripts.ilapfuncs import artifact_processor, logfunc, tsv, timeline, kmlgen, open_sqlite_db_readonly, convert_unix_ts_to_utc +from scripts.ilapfuncs import artifact_processor, logfunc, kmlgen, open_sqlite_db_readonly, convert_unix_ts_to_utc @artifact_processor def get_citymapperLocationHistory(files_found, report_folder, _seeker, _wrap_text): @@ -361,7 +361,7 @@ def get_citymapperSavedTrips(files_found, report_folder, _seeker, _wrap_text): @artifact_processor -def get_citymapperUserData(files_found, report_folder, _seeker, _wrap_text): +def get_citymapperAppPreferences(files_found, report_folder, _seeker, _wrap_text): data_list = [] source = '' @@ -492,13 +492,14 @@ def get_citymapperUserData(files_found, report_folder, _seeker, _wrap_text): report.add_script() # Source Path section - report.add_section_heading('Source Path', 'h3') - report.write_raw_html(f''' -
-
File Path
-
{source}
-
- ''') + report.add_section_heading('Source Paths', 'h3') + for file_found in files_found: + report.write_raw_html(f''' +
+
File Path
+
{file_found}
+
+ ''') # Device Information report.add_section_heading('Device Information', 'h3') @@ -528,7 +529,7 @@ def get_citymapperUserData(files_found, report_folder, _seeker, _wrap_text): report.add_section_heading('Location Information', 'h3') report.write_raw_html(f'''
-
Last Location (Lat,Lon)
+
Last Location (Latitude, Longitude)
{last_location}
CityMapper Region
{cm_region}
From 0306ade80f9b951fe458226d7decd75b0c86eb87 Mon Sep 17 00:00:00 2001 From: Funeoz Date: Fri, 12 Dec 2025 21:53:12 +0100 Subject: [PATCH 3/4] Rename report title from 'User Data' to 'App Preferences' in Citymapper report generation --- scripts/artifacts/citymapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/artifacts/citymapper.py b/scripts/artifacts/citymapper.py index c47b05c5..2cbb2984 100644 --- a/scripts/artifacts/citymapper.py +++ b/scripts/artifacts/citymapper.py @@ -487,8 +487,8 @@ def get_citymapperAppPreferences(files_found, report_folder, _seeker, _wrap_text m.save(map_path) # Create report and add map section - report = ArtifactHtmlReport('Citymapper - User Data') - report.start_artifact_report(report_folder, 'Citymapper - User Data') + report = ArtifactHtmlReport('Citymapper - App Preferences') + report.start_artifact_report(report_folder, 'Citymapper - App Preferences') report.add_script() # Source Path section From 84442cb82ea20730046ed9bb085e0eb4ca531546 Mon Sep 17 00:00:00 2001 From: Funeoz Date: Mon, 15 Dec 2025 18:45:26 +0100 Subject: [PATCH 4/4] Remove commented-out code and unused variables in get_citymapperSavedTrips function --- scripts/artifacts/citymapper.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/artifacts/citymapper.py b/scripts/artifacts/citymapper.py index 2cbb2984..f60994d0 100644 --- a/scripts/artifacts/citymapper.py +++ b/scripts/artifacts/citymapper.py @@ -247,9 +247,6 @@ def get_citymapperSavedTrips(files_found, report_folder, _seeker, _wrap_text): db.close() - """if not saved_trip_data_list: - return ('No Data',), [], source""" - trip_headers = ('ID', 'Commute Type', 'Timestamp', 'Home Latitude', 'Home Longitude', 'Work Latitude', 'Work Longitude', 'Region Code') # Create KML @@ -322,8 +319,6 @@ def get_citymapperSavedTrips(files_found, report_folder, _seeker, _wrap_text): # Summary section report.add_section_heading('Saved Trips Summary', 'h3') - home_count = sum(1 for row in saved_trip_data_list if row[3] and row[4]) - work_count = sum(1 for row in saved_trip_data_list if row[5] and row[6]) report.write_raw_html(f'''
Total Saved Trips