diff --git a/ana/pedestal/Promptdata_analysis.py b/ana/pedestal/Promptdata_analysis.py new file mode 100644 index 000000000..203e00cc3 --- /dev/null +++ b/ana/pedestal/Promptdata_analysis.py @@ -0,0 +1,1067 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import mplhep as hep +import os +import json + + +# In[2]: + + +class I2CConfig: + def __init__(self): + self.ETx_active = [0] # Which eTx (transmitter) columns are active + self.IdlePattern = '127ccc' # Idle pattern to detect between packets (hex) + self.HeaderMarker = '1e6' # Header marker bits to identify packet start (hex) + +i2c = I2CConfig() + + +# In[3]: + + +def to_hex(binary_str): + """Convert binary string to hexadecimal string with appropriate width.""" + if not binary_str: + return '' + + length_to_format = { + 16: '04X', + 24: '06X', + 32: '08X' + } + + fmt = length_to_format.get(len(binary_str), '08X') + return f'{int(binary_str, 2):{fmt}}' + + +to_hex_vectorized = np.vectorize(to_hex) + + +def parse_idle(idle_word): + """ + Parse idle word into its constituent parts. + + Returns: + pattern: Idle pattern (bits 8+) + rr: Ready/Reset bits (bits 6-7) + err: Error bits (bits 3-5) + buff_stat: Buffer status (bits 0-2) + """ + buff_stat = idle_word & 0x7 + err = (idle_word >> 3) & 0x7 + rr = (idle_word >> 6) & 0b11 + pattern = idle_word >> 8 + + return pattern, rr, err, buff_stat + + +def parse_header_word0(header_word, return_dict=False): + """ + Parse the first header word into its constituent fields. + + Args: + header_word: 32-bit integer header word + return_dict: If True, return dictionary; otherwise return tuple + + Returns: + Dictionary or tuple of: HdrMarker, PayloadLength, P, E, HT, EBO, M, T, Hamming + """ + hdr_marker = (header_word >> 23) & 0x1ff + payload_length = (header_word >> 14) & 0x1ff + p = (header_word >> 13) & 0x1 + e = (header_word >> 12) & 0x1 + ht = (header_word >> 10) & 0x3 + ebo = (header_word >> 8) & 0x3 + m = (header_word >> 7) & 0x1 + t = (header_word >> 6) & 0x1 + hamming = header_word & 0x3f + + if return_dict: + return { + "HdrMarker": hex(hdr_marker), + "PayloadLength": payload_length, + "P": p, + "E": e, + "HT": ht, + "EBO": ebo, + "M": m, + "T": t, + "Hamming": hamming + } + + return hdr_marker, payload_length, p, e, ht, ebo, m, t, hamming + + +def parse_header_word1(header_word, return_dict=False): + """ + Parse the second header word into its constituent fields. + + Args: + header_word: 32-bit integer header word + return_dict: If True, return dictionary; otherwise return tuple + + Returns: + Dictionary or tuple of: BX, L1A, Orb, S, RR, CRC + """ + bx = (header_word >> 20) & 0xfff + l1a = (header_word >> 14) & 0x3f + orb = (header_word >> 11) & 0x7 + s = (header_word >> 10) & 0x1 + rr = (header_word >> 8) & 0x3 + crc = header_word & 0xff + + if return_dict: + return { + "Bunch": bx, + "Event": l1a, + "Orbit": orb, + "S": s, + "RR": rr, + "CRC": crc + } + + return bx, l1a, orb, s, rr, crc + + +def convert_to_int(value): + """ + Convert various types to integer, handling hex strings and numpy types. + + Args: + value: String (hex), int, or numpy integer type + + Returns: + Integer value + + Raises: + TypeError: If value type is not supported + """ + # Handle string types (including numpy string types) + if isinstance(value, (str, np.str_, np.bytes_)): + return int(value, 16) + + # Handle numeric types (including numpy integer types) + if isinstance(value, (int, np.integer)): + return int(value) + + raise TypeError(f"Unsupported type for conversion: {type(value)}") + + +def parse_header_words(header_words, return_dict=False): + """ + Parse both header words and return combined fields. + + Args: + header_words: Array-like containing two header words (can be hex strings or ints) + return_dict: If True, return dictionary; otherwise return tuple + + Returns: + Dictionary or tuple of all header fields + """ + # Convert to integers if needed + hdr_0 = convert_to_int(header_words[0]) + hdr_1 = convert_to_int(header_words[1]) + + if return_dict: + fields = parse_header_word0(hdr_0, return_dict=True) + fields.update(parse_header_word1(hdr_1, return_dict=True)) + return fields + + return parse_header_word0(hdr_0) + parse_header_word1(hdr_1) + + +def parse_packet_header(packet_header0, packet_header1=0, as_hex=True, return_dict=False): + """ + Parse packet header(s) into constituent fields. + + Args: + packet_header0: First 32-bit packet header word + packet_header1: Second 32-bit packet header word (optional, for full packet) + as_hex: If True, return values as hex strings; otherwise as integers + return_dict: If True, return dictionary; otherwise return tuple + + Returns: + Dictionary or tuple of: Stat, Ham, F, CM0, CM1, E, ChMap + """ + stat = (packet_header0 >> 29) & 0x7 + ham = (packet_header0 >> 26) & 0x7 + f = (packet_header0 >> 25) & 0x1 + cm0 = (packet_header0 >> 15) & 0x3ff + cm1 = (packet_header0 >> 5) & 0x3ff + e = (packet_header0 >> 4) & 0x1 if f == 1 else 0 + chmap = ((packet_header0 & 0x1f) << 32) + packet_header1 + + if as_hex: + stat = f'{stat:01x}' + ham = f'{ham:01x}' + f = f'{f:01x}' + cm0 = f'{cm0:03x}' + cm1 = f'{cm1:03x}' + e = f'{e:01x}' + chmap = f'{chmap:010x}' + + if return_dict: + return { + "Stat": stat, + "Ham": ham, + "F": f, + "CM0": cm0, + "CM1": cm1, + "E": e, + "ChMap": chmap + } + + return stat, ham, f, cm0, cm1, e, chmap + + +# In[4]: + + +def parse_output_packets_no_idle(output_stream, header_marker='1e6'): + """ + Parse output data stream into individual packets WITHOUT relying on idle words. + Finds packets by detecting header markers and using PayloadLength. + + Args: + output_stream: Array of hex strings (e.g., from df['link0'].values) + header_marker: Hex string of header marker pattern (default: '1e6' for 0xf3xxxxxx) + + Returns: + List of packets (each packet is an array of hex strings) + """ + if isinstance(output_stream, np.ndarray): + output_stream = output_stream.flatten() + + output_stream_int = np.array([int(x, 16) if isinstance(x, str) else x + for x in output_stream]) + + # Find all header markers + header_marker_int = int(header_marker, 16) + is_header = (output_stream_int >> 23) == header_marker_int + header_indices = np.where(is_header)[0] + + if len(header_indices) == 0: + print("Warning: No header markers found in data stream") + return [] + + packets = [] + + for i, start_idx in enumerate(header_indices): + try: + # Parse header to get payload length + header_word0 = output_stream_int[start_idx] + header_word1 = output_stream_int[start_idx + 1] + + # Extract payload length from header word 0 + payload_length = (header_word0 >> 14) & 0x1ff + + # Calculate packet size: 2 (header) + payload_length + 1 (CRC) + packet_size = 2 + payload_length + 1 + + # Check if we have enough data + if start_idx + packet_size > len(output_stream): + print(f"Warning: Packet at index {start_idx} truncated " + f"(needs {packet_size} words, only {len(output_stream) - start_idx} available)") + break + + # Extract packet + packet_end = start_idx + packet_size + packet = output_stream[start_idx:packet_end] + packets.append(packet) + + except (IndexError, ValueError) as e: + print(f"Warning: Error parsing packet at index {start_idx}: {e}") + break + + return packets + + +def parse_output_packets(df_output, i2c): + """ + Parse output data stream into individual packets using idle word detection. + + Args: + df_output: DataFrame containing output stream data + i2c: Configuration object with ETx_active, IdlePattern, and HeaderMarker + + Returns: + List of packets (each packet is an array of hex strings) + """ + # Extract and flatten output stream from active eTx channels + output_stream = df_output.values[:, i2c.ETx_active][:, ::-1].flatten() + output_stream_int = np.vectorize(lambda x: int(x, 16))(output_stream) + + # Identify idle patterns and output headers + idle_pattern = int(i2c.IdlePattern, 16) + header_marker = int(i2c.HeaderMarker, 16) + + is_idle = (output_stream_int >> 8) == idle_pattern + is_output_header = (output_stream_int >> 23) == header_marker + + # Packets start when we see a header after an idle + is_packet_start = np.concatenate([[False], is_output_header[1:] & is_idle[:-1]]) + output_start_indices = np.where(is_packet_start)[0] + + # Extract individual packets + packets = [] + for start_idx in output_start_indices: + # Parse header to get payload length + header_info = parse_header_word0(output_stream_int[start_idx]) + payload_length = header_info[1] # PayloadLength is second element in tuple + + # Extract packet (header + payload) + packet_end = start_idx + payload_length + 2 + packet = output_stream[start_idx:packet_end] + packets.append(packet) + + return packets + + +# In[5]: + + +# Channel data compression code mappings +# Format: (code, bit_length, description, tctp_value) +COMPRESSION_CODES = { + '0000': (24, 'ADCm1 and ADC', '00', True, True, False), # has_adcm1, has_adc, has_toa + '0001': (16, 'ADC only', '00', False, True, False), + '0010': (24, 'ADCm1 and ADC', '01', True, True, False), + '0011': (24, 'ADC and TOA', '00', False, True, True), + '01': (32, 'All data passing ZS', '00', True, True, True), + '11': (32, 'All data', '11', True, True, True), + '10': (32, 'Invalid code', '10', True, True, True), +} + + +def to_binary_32bit(value): + """Convert hex string or int to 32-bit binary string.""" + if isinstance(value, str): + return f'{int(value, 16):032b}' + return f'{value:032b}' + + +def decode_channel_map(chmap_hex): + """ + Decode channel map to get list of active channels. + + Args: + chmap_hex: Hex string representing the 37-bit channel map + + Returns: + List of active channel indices (0-36) + """ + chmap_int = int(chmap_hex, 16) + return [i for i in range(37) if (chmap_int >> (36 - i)) & 0x1] + + +def decode_compressed_channel(bin_string, passthrough=False): + """ + Decode a single channel's compressed data. + + Args: + bin_string: Binary string containing compressed channel data + passthrough: If True, return raw 32 bits without decompression + + Returns: + tuple: (decoded_data, bits_consumed, remaining_binary_string) + decoded_data is 32-bit string: tctp(2) + adcm1(10) + adc(10) + toa(10) + """ + if passthrough: + return bin_string[:32], 32, bin_string[32:] + + # Determine compression code + code = bin_string[:2] + if code == '00': + code = bin_string[:4] + + if code not in COMPRESSION_CODES: + raise ValueError(f"Unknown compression code: {code}") + + bit_length, _, tctp, has_adcm1, has_adc, has_toa = COMPRESSION_CODES[code] + code_len = len(code) + + # Initialize components + adcm1 = '0' * 10 + adc = '0' * 10 + toa = '0' * 10 + + # Extract data based on compression format + offset = code_len + + if has_adcm1: + adcm1 = bin_string[offset:offset+10] + offset += 10 + + if has_adc: + adc = bin_string[offset:offset+10] + offset += 10 + + if has_toa: + toa = bin_string[offset:offset+10] + + decoded_data = tctp + adcm1 + adc + toa + remaining = bin_string[bit_length:] + + return decoded_data, bit_length, remaining + + +def unpack_single_packet(packet, active_links): + """ + Unpack a single packet into its constituent data. + + Args: + packet: Array of hex strings or integers representing the packet + active_links: List of active eRx link indices (0-11) + + Returns: + List of unpacked values in order: header fields, eRx data, channel data, CRC + """ + # Initialize storage for all 12 eRx units and 37 channels each + ch_data = np.full((12, 37), '', dtype=object) + erx_header_data = np.full((12, 7), '', dtype=object) + + # Parse packet header + header_info = parse_header_words(packet, return_dict=True) + + # Extract subpackets and CRC + subpackets = packet[2:-1] + crc = packet[-1] + + # Handle truncated packets + if header_info['T'] == 1: + assert len(subpackets) == 0, "Truncated packet should have no subpackets" + combined_data = np.concatenate([erx_header_data, ch_data], axis=1).flatten() + return list(header_info.values()) + list(combined_data) + [crc] + + # Convert subpackets to continuous binary string + bin_string = ''.join(np.vectorize(to_binary_32bit)(subpackets)) + + # Process each active eRx link + for erx_idx in active_links: + # Parse eRx header - first check F flag + header_word0 = int(bin_string[:32], 2) + f_flag = (header_word0 >> 25) & 0x1 + + if f_flag == 0: + # F=0: Full packet with 2-word header and channel data + header_word1 = int(bin_string[32:64], 2) + erx_header = parse_packet_header(header_word0, header_word1) + erx_header_data[erx_idx] = erx_header + bin_string = bin_string[64:] + else: + # F=1: Empty packet with 1-word header, no channel data, no ChMap + erx_header = parse_packet_header(header_word0, 0) + erx_header_data[erx_idx] = erx_header + bin_string = bin_string[32:] + # Skip channel processing for this eRx - no data! + continue + + # Decode channel map to find active channels + chmap_hex = erx_header[-1] # ChMap is last element + active_channels = decode_channel_map(chmap_hex) + + # Process each active channel + bits_consumed = 0 + is_passthrough = header_info['P'] == 1 + + for ch_idx in active_channels: + decoded, bits_used, bin_string = decode_compressed_channel( + bin_string, + passthrough=is_passthrough + ) + ch_data[erx_idx][ch_idx] = decoded + bits_consumed += bits_used + + # Handle padding to 32-bit boundary + padding_bits = (32 - (bits_consumed % 32)) % 32 + + # Verify padding is all zeros + assert bin_string[:padding_bits] == '0' * padding_bits, \ + f"Expected {padding_bits} zero padding bits, got: {bin_string[:padding_bits]}" + + bin_string = bin_string[padding_bits:] + + # Verify remaining data is 32-bit aligned + assert len(bin_string) % 32 == 0, \ + f"Remaining binary string not 32-bit aligned: {len(bin_string)} bits" + + # Combine all data into flat list + combined_data = np.concatenate([erx_header_data, ch_data], axis=1).flatten() + return list(header_info.values()) + list(combined_data) + [crc] + + +def unpack_packets(packet_list, active_links): + """ + Unpack multiple packets into a DataFrame. + + Args: + packet_list: List of packets (each packet is an array of hex strings/ints) + active_links: List of active eRx link indices (0-11) + + Returns: + DataFrame with columns for all header fields, eRx data, and channel data + """ + unpacked_data = [unpack_single_packet(p, active_links) for p in packet_list] + + # Build column names + columns = [ + 'HeaderMarker', 'PayloadLength', 'P', 'E', 'HT', 'EBO', + 'M', 'T', 'HdrHamming', 'BXNum', 'L1ANum', 'OrbNum', + 'S', 'RR', 'HdrCRC' + ] + + # Add columns for all 12 eRx units (even if not active) + for i in range(12): + # eRx header columns + columns.extend([ + f'eRx{i:02d}_Stat', + f'eRx{i:02d}_Ham', + f'eRx{i:02d}_F', + f'eRx{i:02d}_CM0', + f'eRx{i:02d}_CM1', + f'eRx{i:02d}_E', + f'eRx{i:02d}_ChMap' + ]) + + # Channel data columns (37 channels per eRx) + columns.extend([f'eRx{i:02d}_ChData{j:02d}' for j in range(37)]) + + columns.append('CRC') + + return pd.DataFrame(unpacked_data, columns=columns) + + +# In[31]: + + +def summarize_packet(result_df, event_idx=0): + """ + Summarize the content of an unpacked packet DataFrame. + + Args: + result_df: DataFrame from unpack_packets() + event_idx: Which event/row to summarize (default: 0) + """ + print("=" * 80) + print(f"PACKET SUMMARY - Event {event_idx}") + print("=" * 80) + + # Header Information + print("\n HEADER INFORMATION:") + print("-" * 80) + header_cols = ['HeaderMarker', 'PayloadLength', 'P', 'E', 'HT', 'EBO', + 'M', 'T', 'HdrHamming', 'BXNum', 'L1ANum', 'OrbNum', + 'S', 'RR', 'HdrCRC'] + + for col in header_cols: + if col in result_df.columns: + val = result_df[col].iloc[event_idx] + print(f" {col:15s}: {val}") + + # Check if truncated + is_truncated = result_df['T'].iloc[event_idx] == 1 + if is_truncated: + print("\nāš ļø TRUNCATED PACKET - No data available") + return + + # eRx Summary - Only show eRx units with actual data + print("\n" + "=" * 80) + print("šŸ“” eRx UNITS SUMMARY:") + print("=" * 80) + + active_erx_list = [] + empty_erx_list = [] + + for erx_idx in range(12): + erx_prefix = f'eRx{erx_idx:02d}_' + + # Check if this eRx has any columns + erx_cols = [col for col in result_df.columns if col.startswith(erx_prefix)] + if not erx_cols: + continue + + # Get F flag + f_flag = result_df[f'{erx_prefix}F'].iloc[event_idx] + + if f_flag == '1': + # Empty subpacket - just track it + stat = result_df[f'{erx_prefix}Stat'].iloc[event_idx] + ham = result_df[f'{erx_prefix}Ham'].iloc[event_idx] + empty_erx_list.append((erx_idx, stat, ham)) + continue + + # Check if this eRx actually has channel data + channels_with_data = [] + for ch in range(37): + chdata_col = f'{erx_prefix}ChData{ch:02d}' + chdata = result_df[chdata_col].iloc[event_idx] + if chdata and chdata != '': + channels_with_data.append(ch) + + if len(channels_with_data) == 0: + # Has F=0 but no actual data - skip it + continue + + active_erx_list.append(erx_idx) + + print(f"\nšŸ”¹ eRx {erx_idx:02d}:") + print(f" F flag: {f_flag} (HAS DATA)") + + # Full subpacket - show header info + stat = result_df[f'{erx_prefix}Stat'].iloc[event_idx] + ham = result_df[f'{erx_prefix}Ham'].iloc[event_idx] + cm0 = result_df[f'{erx_prefix}CM0'].iloc[event_idx] + cm1 = result_df[f'{erx_prefix}CM1'].iloc[event_idx] + e = result_df[f'{erx_prefix}E'].iloc[event_idx] + chmap = result_df[f'{erx_prefix}ChMap'].iloc[event_idx] + + print(f" Status: {stat}") + print(f" Hamming: {ham}") + + # Handle empty CM values + if cm0 and cm0 != '': + print(f" CM0 (Common Mode 0): {cm0} = {int(cm0, 16)}") + else: + print(f" CM0 (Common Mode 0): (empty)") + + if cm1 and cm1 != '': + print(f" CM1 (Common Mode 1): {cm1} = {int(cm1, 16)}") + else: + print(f" CM1 (Common Mode 1): (empty)") + + print(f" Error: {e}") + print(f" ChMap: {chmap}") + + # Decode active channels from ChMap + if chmap and chmap != '': + chmap_int = int(chmap, 16) + active_channels = [i for i in range(37) if (chmap_int >> (36 - i)) & 0x1] + else: + active_channels = [] + + print(f" Active channels ({len(active_channels)}): {active_channels}") + + # Check which channels actually have data + channels_with_data = [] + for ch in range(37): + chdata_col = f'{erx_prefix}ChData{ch:02d}' + chdata = result_df[chdata_col].iloc[event_idx] + if chdata and chdata != '': + channels_with_data.append(ch) + + print(f" Channels with data ({len(channels_with_data)}): {channels_with_data}") + + if len(channels_with_data) != len(active_channels): + print(f" āš ļø WARNING: ChMap indicates {len(active_channels)} channels, " + f"but {len(channels_with_data)} have data!") + + # Summary at the end + print(f"\n" + "-" * 80) + if active_erx_list: + print(f"āœ“ Active eRx units with data: {active_erx_list}") + else: + print(f"āœ— No eRx units with channel data") + + if empty_erx_list: + print(f"āŠ˜ Empty eRx units (F=1): {[e[0] for e in empty_erx_list]}") + for erx_idx, stat, ham in empty_erx_list: + print(f" eRx{erx_idx:02d}: Status={stat}, Hamming={ham}") + + + +# In[7]: + + +def read_files(folder, filename=None): + import glob + headers = ["link0","link1","link2","link3","link4","link5","link6"] + if filename: + csv_files = [os.path.join(folder, filename)] + else: + csv_files = glob.glob(folder+"/*.csv") # Update with your actual folder path + + df_list = [pd.read_csv(file, sep=" ", names=headers, skiprows=1) for file in csv_files] + final_df = pd.concat(df_list, ignore_index=True) + return final_df + + +# In[8]: + + +# data = read_files("data", "pedestal_20251208_115228_hexdump.csv") +data = read_files("data", "pedestal_prelevel_1.raw") +packet_data = data["link0"].values + +df_output = pd.DataFrame({'link0': packet_data}) + +# Use parseOutputPackets to split into individual packets +#packets = parse_output_packets(df_output, i2c) +packets = parse_output_packets_no_idle(packet_data, header_marker='1e6') + +print(f"Found {len(packets)} packets") + + +# In[9]: + + +packet_data + + +# In[10]: + + +with open('run40unwrapped.txt', 'r') as f: + lines = f.readlines() + +hex_values = [] +for line in lines: + line = line.strip() + # Check if line looks like a hex value (8 characters, all hex digits) + if len(line) == 8 and all(c in '0123456789abcdefABCDEF' for c in line): + hex_values.append(line.lower()) + +# Create numpy array with dtype=object +packet_data = np.array(hex_values, dtype=object) + +print(packet_data) +#print(f"\nTotal hex values: {len(hex_values)}") + + +# In[32]: + + +packets = parse_output_packets_no_idle(packet_data, header_marker='1e6') + +print(f"Found {len(packets)} packets") + +# Show packet sizes +for i, pkt in enumerate(packets): + print(f"Packet {i}: {len(pkt)} words") + +# Unpack each packet +for i, pkt in enumerate(packets): + print(f"\n{'='*80}") + print(f"PACKET {i}") + print(f"{'='*80}") + result = unpack_packets([pkt], active_links=[9,10]) + summarize_packet(result, event_idx=0) + + +# In[12]: + + +parse_header_words([packets[0][0], packets[0][1]], return_dict=True) + + +# In[13]: + + +parse_header_words([packets[1][0], packets[1][1]], return_dict=True) + + +# In[ ]: + + +packets[0] + + +# In[14]: + + +packets[1] + + +# In[15]: + + +summarize_packet(result, event_idx=0) + + +# In[ ]: + + +import crcmod +import codecs + +crc8 = crcmod.mkCrcFun(0x1a7, initCrc=0, xorOut=0, rev=False) + +def calculate_crc8_bits(data): + """ + Calculate 8-bit CRC using crcmod library. + + Args: + data: 64-bit integer + Returns: + 8-bit CRC value (integer) + """ + # Convert integer to 8 bytes (hex string, then to bytes) + hex_str = f'{data:016x}' + byte_data = codecs.decode(hex_str, 'hex') + crc_value = crc8(byte_data) + + return crc_value + + +def verify_header_crc(header_word0, header_word1, verbose=False): + """ + Verify the 8-bit CRC in the event header. + + The CRC is calculated over a 64-bit value constructed as: + [8 zeros][26 bits from word0, bits 6-31, excluding Hamming bits 0-5][6 zeros][24 bits from word1, bits 8-31, excluding CRC bits 0-7] + + Args: + header_word0: First 32-bit header word (hex string or int) + header_word1: Second 32-bit header word (hex string or int) + verbose: If True, print detailed information + + Returns: + tuple: (is_valid, transmitted_crc, calculated_crc) + """ + # Convert to integers if needed + if isinstance(header_word0, str): + header_word0 = int(header_word0, 16) + if isinstance(header_word1, str): + header_word1 = int(header_word1, 16) + + # Extract transmitted CRC from header word 1 (bits 0-7) + transmitted_crc = header_word1 & 0xFF + + # Build CRC calculation base (64 bits total): + # Format: 8_zeros + 26_bits_from_word0 + 6_zeros + 24_bits_from_word1 + + # Extract bits 6-31 from header_word0 (26 bits, excluding Hamming bits 0-5) + word0_data = (header_word0 >> 6) & 0x3FFFFFF # 26 bits + + # Extract bits 8-31 from header_word1 (24 bits, excluding CRC bits 0-7) + word1_data = (header_word1 >> 8) & 0xFFFFFF # 24 bits + + # Construct the 64-bit CRC base: + # [8 zeros][26 bits from word0][6 zeros][24 bits from word1] + header_crc_base = (word0_data << 30) | word1_data + + if verbose: + print(f"Header Word 0: 0x{header_word0:08x}") + print(f" 0b{header_word0:032b}") + print(f" {'D' * 26}{'H' * 6} (D=Data, H=Hamming)") + print() + + print(f"Header Word 1: 0x{header_word1:08x}") + print(f" 0b{header_word1:032b}") + print(f" {'D' * 24}{'C' * 8} (D=Data, C=CRC)") + print() + + print("Building CRC Base (64-bit):") + print(f" Word 0 [31:6]: 0b{word0_data:026b} (26 bits)") + print(f" Word 1 [31:8]: 0b{word1_data:024b} (24 bits)") + print() + print(f"CRC Base Structure:") + print(f" 0b{'0'*8}{word0_data:026b}{'0'*6}{word1_data:024b}") + print(f" {'8 zeros':<8}{'26 bits from W0':<26}{'6 zero':<6}{'24 bits from W1':<24}") + print() + print(f"CRC Base (64-bit): 0x{header_crc_base:016x}") + print(f" 0b{header_crc_base:064b}") + print() + print(f"Transmitted CRC: 0x{transmitted_crc:02x} = 0b{transmitted_crc:08b} = {transmitted_crc}") + + # Calculate CRC + calculated_crc = calculate_crc8_bits(header_crc_base) + + if verbose: + print(f"Calculated CRC: 0x{calculated_crc:02x} = 0b{calculated_crc:08b} = {calculated_crc}") + print(f"CRC Valid: {calculated_crc == transmitted_crc}") + + return calculated_crc == transmitted_crc, transmitted_crc, calculated_crc + + +def find_corrupted_bits(header_word0, header_word1): + """ + Determine which bits would need to change to make the CRC match. + Tests single-bit and two-bit errors. + + Args: + header_word0: First 32-bit header word (hex string or int) + header_word1: Second 32-bit header word (hex string or int) + + Returns: + Dictionary with possible corrections + """ + # Convert to integers + if isinstance(header_word0, str): + header_word0 = int(header_word0, 16) + if isinstance(header_word1, str): + header_word1 = int(header_word1, 16) + + transmitted_crc = header_word1 & 0xFF + + # Check if original is valid + is_valid_orig, _, calc_crc_orig = verify_header_crc(header_word0, header_word1) + + results = { + 'original': { + 'word0': f'0x{header_word0:08x}', + 'word1': f'0x{header_word1:08x}', + 'transmitted_crc': transmitted_crc, + 'calculated_crc': calc_crc_orig, + 'is_valid': is_valid_orig + }, + 'single_bit_flips': [], + 'two_bit_flips': [] + } + + if is_valid_orig: + print("Original CRC is already valid!") + return results + + print(f"Original CRC mismatch: transmitted={transmitted_crc}, calculated={calc_crc_orig}") + print("Searching for single-bit flips...") + + # Test single-bit flips in word 0 (bits 6-31, excluding Hamming bits 0-5) + for bit in range(6, 32): + test_word0 = header_word0 ^ (1 << bit) + is_valid, _, calc_crc = verify_header_crc(test_word0, header_word1) + if is_valid: + results['single_bit_flips'].append({ + 'location': f'Word0, bit {bit}', + 'corrected_word0': f'0x{test_word0:08x}', + 'corrected_word1': f'0x{header_word1:08x}', + 'original_bit': (header_word0 >> bit) & 1, + 'corrected_bit': (test_word0 >> bit) & 1, + 'crc': calc_crc + }) + + # Test single-bit flips in word 1 (bits 8-31, excluding CRC bits 0-7) + for bit in range(8, 32): + test_word1 = header_word1 ^ (1 << bit) + is_valid, _, calc_crc = verify_header_crc(header_word0, test_word1) + if is_valid: + results['single_bit_flips'].append({ + 'location': f'Word1, bit {bit}', + 'corrected_word0': f'0x{header_word0:08x}', + 'corrected_word1': f'0x{test_word1:08x}', + 'original_bit': (header_word1 >> bit) & 1, + 'corrected_bit': (test_word1 >> bit) & 1, + 'crc': calc_crc + }) + + # Only search two-bit flips if no single-bit solution found + if not results['single_bit_flips']: + print("No single-bit flip found, searching two-bit flips...") + + # Test two-bit flips in word 0 + for bit1 in range(6, 32): + for bit2 in range(bit1 + 1, 32): + test_word0 = header_word0 ^ (1 << bit1) ^ (1 << bit2) + is_valid, _, calc_crc = verify_header_crc(test_word0, header_word1) + if is_valid: + results['two_bit_flips'].append({ + 'location': f'Word0, bits {bit1} and {bit2}', + 'corrected_word0': f'0x{test_word0:08x}', + 'corrected_word1': f'0x{header_word1:08x}', + 'crc': calc_crc + }) + + # Test two-bit flips in word 1 + for bit1 in range(8, 32): + for bit2 in range(bit1 + 1, 32): + test_word1 = header_word1 ^ (1 << bit1) ^ (1 << bit2) + is_valid, _, calc_crc = verify_header_crc(header_word0, test_word1) + if is_valid: + results['two_bit_flips'].append({ + 'location': f'Word1, bits {bit1} and {bit2}', + 'corrected_word0': f'0x{header_word0:08x}', + 'corrected_word1': f'0x{test_word1:08x}', + 'crc': calc_crc + }) + + # Test one bit in each word + print("Searching for one bit flip in each word...") + for bit0 in range(6, 32): + for bit1 in range(8, 32): + test_word0 = header_word0 ^ (1 << bit0) + test_word1 = header_word1 ^ (1 << bit1) + is_valid, _, calc_crc = verify_header_crc(test_word0, test_word1) + if is_valid: + results['two_bit_flips'].append({ + 'location': f'Word0 bit {bit0} and Word1 bit {bit1}', + 'corrected_word0': f'0x{test_word0:08x}', + 'corrected_word1': f'0x{test_word1:08x}', + 'crc': calc_crc + }) + + return results + + +# In[34]: + + +#header0 = "f313f007" +#header1 = "d631c0ba" + +header0 = "f313f024" +header1 = "a0e22825" + +is_valid, tx, calc = verify_header_crc(header0, header1, verbose=True) + +if not is_valid: + print("\n" + "=" * 80) + print("Finding which bits need to change to fix CRC...") + print("=" * 80) + corrections = find_corrupted_bits(header0, header1) + + print(f"\nOriginal Header:") + print(f" Word 0: {corrections['original']['word0']}") + print(f" Word 1: {corrections['original']['word1']}") + print(f" Transmitted CRC: 0x{corrections['original']['transmitted_crc']:02x}") + + if corrections['single_bit_flips']: + print(f"\nāœ“ Found {len(corrections['single_bit_flips'])} single-bit flip solution(s):") + for i, fix in enumerate(corrections['single_bit_flips'], 1): + print(f"\n Solution {i}: Flip {fix['location']}") + print(f" Corrected Word 0: {fix['corrected_word0']}") + print(f" Corrected Word 1: {fix['corrected_word1']}") + print(f" CRC now matches: 0x{fix['crc']:02x}") + elif corrections['two_bit_flips']: + print(f"\nāœ“ Found {len(corrections['two_bit_flips'])} two-bit flip solution(s):") + for i, fix in enumerate(corrections['two_bit_flips'][:5], 1): # Show first 5 + print(f"\n Solution {i}: Flip {fix['location']}") + print(f" Corrected Word 0: {fix['corrected_word0']}") + print(f" Corrected Word 1: {fix['corrected_word1']}") + print(f" CRC now matches: 0x{fix['crc']:02x}") + if len(corrections['two_bit_flips']) > 5: + print(f"\n ... and {len(corrections['two_bit_flips']) - 5} more solutions") + else: + print("\nāœ— No simple single or two-bit flip solution found") + print(" This suggests multiple bit errors or a different error pattern") + + +# The hamming values are a parity check of different combinations of bits: +# +# ``` +# parityGroups = np.array([[56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, +# 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26], +# [56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, +# 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11], +# [56, 55, 54, 53, 52, 51, 50, 49, 40, 39, 38, 37, 36, 35, 34, 33, +# 25, 24, 23, 22, 21, 20, 19, 18, 10, 9, 8, 7, 6, 5, 4], +# [56, 55, 54, 53, 48, 47, 46, 45, 40, 39, 38, 37, 32, 31, 30, 29, +# 25, 24, 23, 22, 17, 16, 15, 14, 10, 9, 8, 7, 3, 2, 1], +# [56, 55, 52, 51, 48, 47, 44, 43, 40, 39, 36, 35, 32, 31, 28, 27, +# 25, 24, 21, 20, 17, 16, 13, 12, 10, 9, 6, 5, 3, 2, 0], +# [56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26, +# 25, 23, 21, 19, 17, 15, 13, 11, 10, 8, 6, 4, 3, 1, 0]]) +# ``` +# for these different groups of bits, a Hamming code !=0 is whether there is an even or odd number of 1's in the values + +# In[ ]: + + + + diff --git a/ana/pedestal/Promptdata_analysis.py.ipynb b/ana/pedestal/Promptdata_analysis.py.ipynb new file mode 100644 index 000000000..8e7cd5e69 --- /dev/null +++ b/ana/pedestal/Promptdata_analysis.py.ipynb @@ -0,0 +1,1930 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "df0537b7", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import mplhep as hep\n", + "import os \n", + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "14d3d589", + "metadata": {}, + "outputs": [], + "source": [ + "class I2CConfig:\n", + " def __init__(self):\n", + " self.ETx_active = [0] # Which eTx (transmitter) columns are active\n", + " self.IdlePattern = '127ccc' # Idle pattern to detect between packets (hex)\n", + " self.HeaderMarker = '1e6' # Header marker bits to identify packet start (hex)\n", + "\n", + "i2c = I2CConfig()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "70a28e64", + "metadata": {}, + "outputs": [], + "source": [ + "def to_hex(binary_str):\n", + " \"\"\"Convert binary string to hexadecimal string with appropriate width.\"\"\"\n", + " if not binary_str:\n", + " return ''\n", + " \n", + " length_to_format = {\n", + " 16: '04X',\n", + " 24: '06X',\n", + " 32: '08X'\n", + " }\n", + " \n", + " fmt = length_to_format.get(len(binary_str), '08X')\n", + " return f'{int(binary_str, 2):{fmt}}'\n", + "\n", + "\n", + "to_hex_vectorized = np.vectorize(to_hex)\n", + "\n", + "\n", + "def parse_idle(idle_word):\n", + " \"\"\"\n", + " Parse idle word into its constituent parts.\n", + " \n", + " Returns:\n", + " pattern: Idle pattern (bits 8+)\n", + " rr: Ready/Reset bits (bits 6-7)\n", + " err: Error bits (bits 3-5)\n", + " buff_stat: Buffer status (bits 0-2)\n", + " \"\"\"\n", + " buff_stat = idle_word & 0x7\n", + " err = (idle_word >> 3) & 0x7\n", + " rr = (idle_word >> 6) & 0b11\n", + " pattern = idle_word >> 8\n", + " \n", + " return pattern, rr, err, buff_stat\n", + "\n", + "\n", + "def parse_header_word0(header_word, return_dict=False):\n", + " \"\"\"\n", + " Parse the first header word into its constituent fields.\n", + " \n", + " Args:\n", + " header_word: 32-bit integer header word\n", + " return_dict: If True, return dictionary; otherwise return tuple\n", + " \n", + " Returns:\n", + " Dictionary or tuple of: HdrMarker, PayloadLength, P, E, HT, EBO, M, T, Hamming\n", + " \"\"\"\n", + " hdr_marker = (header_word >> 23) & 0x1ff\n", + " payload_length = (header_word >> 14) & 0x1ff\n", + " p = (header_word >> 13) & 0x1\n", + " e = (header_word >> 12) & 0x1\n", + " ht = (header_word >> 10) & 0x3\n", + " ebo = (header_word >> 8) & 0x3\n", + " m = (header_word >> 7) & 0x1\n", + " t = (header_word >> 6) & 0x1\n", + " hamming = header_word & 0x3f\n", + " \n", + " if return_dict:\n", + " return {\n", + " \"HdrMarker\": hex(hdr_marker),\n", + " \"PayloadLength\": payload_length,\n", + " \"P\": p,\n", + " \"E\": e,\n", + " \"HT\": ht,\n", + " \"EBO\": ebo,\n", + " \"M\": m,\n", + " \"T\": t,\n", + " \"Hamming\": hamming\n", + " }\n", + " \n", + " return hdr_marker, payload_length, p, e, ht, ebo, m, t, hamming\n", + "\n", + "\n", + "def parse_header_word1(header_word, return_dict=False):\n", + " \"\"\"\n", + " Parse the second header word into its constituent fields.\n", + " \n", + " Args:\n", + " header_word: 32-bit integer header word\n", + " return_dict: If True, return dictionary; otherwise return tuple\n", + " \n", + " Returns:\n", + " Dictionary or tuple of: BX, L1A, Orb, S, RR, CRC\n", + " \"\"\"\n", + " bx = (header_word >> 20) & 0xfff\n", + " l1a = (header_word >> 14) & 0x3f\n", + " orb = (header_word >> 11) & 0x7\n", + " s = (header_word >> 10) & 0x1\n", + " rr = (header_word >> 8) & 0x3\n", + " crc = header_word & 0xff\n", + " \n", + " if return_dict:\n", + " return {\n", + " \"Bunch\": bx,\n", + " \"Event\": l1a,\n", + " \"Orbit\": orb,\n", + " \"S\": s,\n", + " \"RR\": rr,\n", + " \"CRC\": crc\n", + " }\n", + " \n", + " return bx, l1a, orb, s, rr, crc\n", + "\n", + "\n", + "def convert_to_int(value):\n", + " \"\"\"\n", + " Convert various types to integer, handling hex strings and numpy types.\n", + " \n", + " Args:\n", + " value: String (hex), int, or numpy integer type\n", + " \n", + " Returns:\n", + " Integer value\n", + " \n", + " Raises:\n", + " TypeError: If value type is not supported\n", + " \"\"\"\n", + " # Handle string types (including numpy string types)\n", + " if isinstance(value, (str, np.str_, np.bytes_)):\n", + " return int(value, 16)\n", + " \n", + " # Handle numeric types (including numpy integer types)\n", + " if isinstance(value, (int, np.integer)):\n", + " return int(value)\n", + " \n", + " raise TypeError(f\"Unsupported type for conversion: {type(value)}\")\n", + "\n", + "\n", + "def parse_header_words(header_words, return_dict=False):\n", + " \"\"\"\n", + " Parse both header words and return combined fields.\n", + " \n", + " Args:\n", + " header_words: Array-like containing two header words (can be hex strings or ints)\n", + " return_dict: If True, return dictionary; otherwise return tuple\n", + " \n", + " Returns:\n", + " Dictionary or tuple of all header fields\n", + " \"\"\"\n", + " # Convert to integers if needed\n", + " hdr_0 = convert_to_int(header_words[0])\n", + " hdr_1 = convert_to_int(header_words[1])\n", + " \n", + " if return_dict:\n", + " fields = parse_header_word0(hdr_0, return_dict=True)\n", + " fields.update(parse_header_word1(hdr_1, return_dict=True))\n", + " return fields\n", + " \n", + " return parse_header_word0(hdr_0) + parse_header_word1(hdr_1)\n", + "\n", + "\n", + "def parse_packet_header(packet_header0, packet_header1=0, as_hex=True, return_dict=False):\n", + " \"\"\"\n", + " Parse packet header(s) into constituent fields.\n", + " \n", + " Args:\n", + " packet_header0: First 32-bit packet header word\n", + " packet_header1: Second 32-bit packet header word (optional, for full packet)\n", + " as_hex: If True, return values as hex strings; otherwise as integers\n", + " return_dict: If True, return dictionary; otherwise return tuple\n", + " \n", + " Returns:\n", + " Dictionary or tuple of: Stat, Ham, F, CM0, CM1, E, ChMap\n", + " \"\"\"\n", + " stat = (packet_header0 >> 29) & 0x7\n", + " ham = (packet_header0 >> 26) & 0x7\n", + " f = (packet_header0 >> 25) & 0x1\n", + " cm0 = (packet_header0 >> 15) & 0x3ff\n", + " cm1 = (packet_header0 >> 5) & 0x3ff\n", + " e = (packet_header0 >> 4) & 0x1 if f == 1 else 0\n", + " chmap = ((packet_header0 & 0x1f) << 32) + packet_header1\n", + " \n", + " if as_hex:\n", + " stat = f'{stat:01x}'\n", + " ham = f'{ham:01x}'\n", + " f = f'{f:01x}'\n", + " cm0 = f'{cm0:03x}'\n", + " cm1 = f'{cm1:03x}'\n", + " e = f'{e:01x}'\n", + " chmap = f'{chmap:010x}'\n", + " \n", + " if return_dict:\n", + " return {\n", + " \"Stat\": stat,\n", + " \"Ham\": ham,\n", + " \"F\": f,\n", + " \"CM0\": cm0,\n", + " \"CM1\": cm1,\n", + " \"E\": e,\n", + " \"ChMap\": chmap\n", + " }\n", + " \n", + " return stat, ham, f, cm0, cm1, e, chmap" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "f9608aa9", + "metadata": {}, + "outputs": [], + "source": [ + "def parse_output_packets_no_idle(output_stream, header_marker='1e6'):\n", + " \"\"\"\n", + " Parse output data stream into individual packets WITHOUT relying on idle words.\n", + " Finds packets by detecting header markers and using PayloadLength.\n", + " \n", + " Args:\n", + " output_stream: Array of hex strings (e.g., from df['link0'].values)\n", + " header_marker: Hex string of header marker pattern (default: '1e6' for 0xf3xxxxxx)\n", + " \n", + " Returns:\n", + " List of packets (each packet is an array of hex strings)\n", + " \"\"\"\n", + " if isinstance(output_stream, np.ndarray):\n", + " output_stream = output_stream.flatten()\n", + " \n", + " output_stream_int = np.array([int(x, 16) if isinstance(x, str) else x \n", + " for x in output_stream])\n", + " \n", + " # Find all header markers\n", + " header_marker_int = int(header_marker, 16)\n", + " is_header = (output_stream_int >> 23) == header_marker_int\n", + " header_indices = np.where(is_header)[0]\n", + " \n", + " if len(header_indices) == 0:\n", + " print(\"Warning: No header markers found in data stream\")\n", + " return []\n", + " \n", + " packets = []\n", + " \n", + " for i, start_idx in enumerate(header_indices):\n", + " try:\n", + " # Parse header to get payload length\n", + " header_word0 = output_stream_int[start_idx]\n", + " header_word1 = output_stream_int[start_idx + 1]\n", + " \n", + " # Extract payload length from header word 0\n", + " payload_length = (header_word0 >> 14) & 0x1ff\n", + " \n", + " # Calculate packet size: 2 (header) + payload_length + 1 (CRC)\n", + " packet_size = 2 + payload_length + 1\n", + " \n", + " # Check if we have enough data\n", + " if start_idx + packet_size > len(output_stream):\n", + " print(f\"Warning: Packet at index {start_idx} truncated \"\n", + " f\"(needs {packet_size} words, only {len(output_stream) - start_idx} available)\")\n", + " break\n", + " \n", + " # Extract packet\n", + " packet_end = start_idx + packet_size\n", + " packet = output_stream[start_idx:packet_end]\n", + " packets.append(packet)\n", + " \n", + " except (IndexError, ValueError) as e:\n", + " print(f\"Warning: Error parsing packet at index {start_idx}: {e}\")\n", + " break\n", + " \n", + " return packets\n", + "\n", + "\n", + "def parse_output_packets(df_output, i2c):\n", + " \"\"\"\n", + " Parse output data stream into individual packets using idle word detection.\n", + " \n", + " Args:\n", + " df_output: DataFrame containing output stream data\n", + " i2c: Configuration object with ETx_active, IdlePattern, and HeaderMarker\n", + " \n", + " Returns:\n", + " List of packets (each packet is an array of hex strings)\n", + " \"\"\"\n", + " # Extract and flatten output stream from active eTx channels\n", + " output_stream = df_output.values[:, i2c.ETx_active][:, ::-1].flatten()\n", + " output_stream_int = np.vectorize(lambda x: int(x, 16))(output_stream)\n", + " \n", + " # Identify idle patterns and output headers\n", + " idle_pattern = int(i2c.IdlePattern, 16)\n", + " header_marker = int(i2c.HeaderMarker, 16)\n", + " \n", + " is_idle = (output_stream_int >> 8) == idle_pattern\n", + " is_output_header = (output_stream_int >> 23) == header_marker\n", + " \n", + " # Packets start when we see a header after an idle\n", + " is_packet_start = np.concatenate([[False], is_output_header[1:] & is_idle[:-1]])\n", + " output_start_indices = np.where(is_packet_start)[0]\n", + " \n", + " # Extract individual packets\n", + " packets = []\n", + " for start_idx in output_start_indices:\n", + " # Parse header to get payload length\n", + " header_info = parse_header_word0(output_stream_int[start_idx])\n", + " payload_length = header_info[1] # PayloadLength is second element in tuple\n", + " \n", + " # Extract packet (header + payload)\n", + " packet_end = start_idx + payload_length + 2\n", + " packet = output_stream[start_idx:packet_end]\n", + " packets.append(packet)\n", + " \n", + " return packets" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "9ccfd744", + "metadata": {}, + "outputs": [], + "source": [ + "# Channel data compression code mappings\n", + "# Format: (code, bit_length, description, tctp_value)\n", + "COMPRESSION_CODES = {\n", + " '0000': (24, 'ADCm1 and ADC', '00', True, True, False), # has_adcm1, has_adc, has_toa\n", + " '0001': (16, 'ADC only', '00', False, True, False),\n", + " '0010': (24, 'ADCm1 and ADC', '01', True, True, False),\n", + " '0011': (24, 'ADC and TOA', '00', False, True, True),\n", + " '01': (32, 'All data passing ZS', '00', True, True, True),\n", + " '11': (32, 'All data', '11', True, True, True),\n", + " '10': (32, 'Invalid code', '10', True, True, True),\n", + "}\n", + "\n", + "\n", + "def to_binary_32bit(value):\n", + " \"\"\"Convert hex string or int to 32-bit binary string.\"\"\"\n", + " if isinstance(value, str):\n", + " return f'{int(value, 16):032b}'\n", + " return f'{value:032b}'\n", + "\n", + "\n", + "def decode_channel_map(chmap_hex):\n", + " \"\"\"\n", + " Decode channel map to get list of active channels.\n", + " \n", + " Args:\n", + " chmap_hex: Hex string representing the 37-bit channel map\n", + " \n", + " Returns:\n", + " List of active channel indices (0-36)\n", + " \"\"\"\n", + " chmap_int = int(chmap_hex, 16)\n", + " return [i for i in range(37) if (chmap_int >> (36 - i)) & 0x1]\n", + "\n", + "\n", + "def decode_compressed_channel(bin_string, passthrough=False):\n", + " \"\"\"\n", + " Decode a single channel's compressed data.\n", + " \n", + " Args:\n", + " bin_string: Binary string containing compressed channel data\n", + " passthrough: If True, return raw 32 bits without decompression\n", + " \n", + " Returns:\n", + " tuple: (decoded_data, bits_consumed, remaining_binary_string)\n", + " decoded_data is 32-bit string: tctp(2) + adcm1(10) + adc(10) + toa(10)\n", + " \"\"\"\n", + " if passthrough:\n", + " return bin_string[:32], 32, bin_string[32:]\n", + " \n", + " # Determine compression code\n", + " code = bin_string[:2]\n", + " if code == '00':\n", + " code = bin_string[:4]\n", + " \n", + " if code not in COMPRESSION_CODES:\n", + " raise ValueError(f\"Unknown compression code: {code}\")\n", + " \n", + " bit_length, _, tctp, has_adcm1, has_adc, has_toa = COMPRESSION_CODES[code]\n", + " code_len = len(code)\n", + " \n", + " # Initialize components\n", + " adcm1 = '0' * 10\n", + " adc = '0' * 10\n", + " toa = '0' * 10\n", + " \n", + " # Extract data based on compression format\n", + " offset = code_len\n", + " \n", + " if has_adcm1:\n", + " adcm1 = bin_string[offset:offset+10]\n", + " offset += 10\n", + " \n", + " if has_adc:\n", + " adc = bin_string[offset:offset+10]\n", + " offset += 10\n", + " \n", + " if has_toa:\n", + " toa = bin_string[offset:offset+10]\n", + " \n", + " decoded_data = tctp + adcm1 + adc + toa\n", + " remaining = bin_string[bit_length:]\n", + " \n", + " return decoded_data, bit_length, remaining\n", + "\n", + "\n", + "def unpack_single_packet(packet, active_links):\n", + " \"\"\"\n", + " Unpack a single packet into its constituent data.\n", + " \n", + " Args:\n", + " packet: Array of hex strings or integers representing the packet\n", + " active_links: List of active eRx link indices (0-11)\n", + " \n", + " Returns:\n", + " List of unpacked values in order: header fields, eRx data, channel data, CRC\n", + " \"\"\"\n", + " # Initialize storage for all 12 eRx units and 37 channels each\n", + " ch_data = np.full((12, 37), '', dtype=object)\n", + " erx_header_data = np.full((12, 7), '', dtype=object)\n", + " \n", + " # Parse packet header\n", + " header_info = parse_header_words(packet, return_dict=True)\n", + " \n", + " # Extract subpackets and CRC\n", + " subpackets = packet[2:-1]\n", + " crc = packet[-1]\n", + " \n", + " # Handle truncated packets\n", + " if header_info['T'] == 1:\n", + " assert len(subpackets) == 0, \"Truncated packet should have no subpackets\"\n", + " combined_data = np.concatenate([erx_header_data, ch_data], axis=1).flatten()\n", + " return list(header_info.values()) + list(combined_data) + [crc]\n", + " \n", + " # Convert subpackets to continuous binary string\n", + " bin_string = ''.join(np.vectorize(to_binary_32bit)(subpackets))\n", + " \n", + " # Process each active eRx link\n", + " for erx_idx in active_links:\n", + " # Parse eRx header - first check F flag\n", + " header_word0 = int(bin_string[:32], 2)\n", + " f_flag = (header_word0 >> 25) & 0x1\n", + " \n", + " if f_flag == 0:\n", + " # F=0: Full packet with 2-word header and channel data\n", + " header_word1 = int(bin_string[32:64], 2)\n", + " erx_header = parse_packet_header(header_word0, header_word1)\n", + " erx_header_data[erx_idx] = erx_header\n", + " bin_string = bin_string[64:]\n", + " else:\n", + " # F=1: Empty packet with 1-word header, no channel data, no ChMap\n", + " erx_header = parse_packet_header(header_word0, 0)\n", + " erx_header_data[erx_idx] = erx_header\n", + " bin_string = bin_string[32:]\n", + " # Skip channel processing for this eRx - no data!\n", + " continue\n", + " \n", + " # Decode channel map to find active channels\n", + " chmap_hex = erx_header[-1] # ChMap is last element\n", + " active_channels = decode_channel_map(chmap_hex)\n", + " \n", + " # Process each active channel\n", + " bits_consumed = 0\n", + " is_passthrough = header_info['P'] == 1\n", + " \n", + " for ch_idx in active_channels:\n", + " decoded, bits_used, bin_string = decode_compressed_channel(\n", + " bin_string, \n", + " passthrough=is_passthrough\n", + " )\n", + " ch_data[erx_idx][ch_idx] = decoded\n", + " bits_consumed += bits_used\n", + " \n", + " # Handle padding to 32-bit boundary\n", + " padding_bits = (32 - (bits_consumed % 32)) % 32\n", + " \n", + " # Verify padding is all zeros\n", + " assert bin_string[:padding_bits] == '0' * padding_bits, \\\n", + " f\"Expected {padding_bits} zero padding bits, got: {bin_string[:padding_bits]}\"\n", + " \n", + " bin_string = bin_string[padding_bits:]\n", + " \n", + " # Verify remaining data is 32-bit aligned\n", + " assert len(bin_string) % 32 == 0, \\\n", + " f\"Remaining binary string not 32-bit aligned: {len(bin_string)} bits\"\n", + " \n", + " # Combine all data into flat list\n", + " combined_data = np.concatenate([erx_header_data, ch_data], axis=1).flatten()\n", + " return list(header_info.values()) + list(combined_data) + [crc]\n", + "\n", + "\n", + "def unpack_packets(packet_list, active_links):\n", + " \"\"\"\n", + " Unpack multiple packets into a DataFrame.\n", + " \n", + " Args:\n", + " packet_list: List of packets (each packet is an array of hex strings/ints)\n", + " active_links: List of active eRx link indices (0-11)\n", + " \n", + " Returns:\n", + " DataFrame with columns for all header fields, eRx data, and channel data\n", + " \"\"\"\n", + " unpacked_data = [unpack_single_packet(p, active_links) for p in packet_list]\n", + " \n", + " # Build column names\n", + " columns = [\n", + " 'HeaderMarker', 'PayloadLength', 'P', 'E', 'HT', 'EBO', \n", + " 'M', 'T', 'HdrHamming', 'BXNum', 'L1ANum', 'OrbNum', \n", + " 'S', 'RR', 'HdrCRC'\n", + " ]\n", + " \n", + " # Add columns for all 12 eRx units (even if not active)\n", + " for i in range(12):\n", + " # eRx header columns\n", + " columns.extend([\n", + " f'eRx{i:02d}_Stat', \n", + " f'eRx{i:02d}_Ham', \n", + " f'eRx{i:02d}_F',\n", + " f'eRx{i:02d}_CM0', \n", + " f'eRx{i:02d}_CM1', \n", + " f'eRx{i:02d}_E', \n", + " f'eRx{i:02d}_ChMap'\n", + " ])\n", + " \n", + " # Channel data columns (37 channels per eRx)\n", + " columns.extend([f'eRx{i:02d}_ChData{j:02d}' for j in range(37)])\n", + " \n", + " columns.append('CRC')\n", + " \n", + " return pd.DataFrame(unpacked_data, columns=columns)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "ee95ca8c", + "metadata": {}, + "outputs": [], + "source": [ + "def summarize_packet(result_df, event_idx=0):\n", + " \"\"\"\n", + " Summarize the content of an unpacked packet DataFrame.\n", + " \n", + " Args:\n", + " result_df: DataFrame from unpack_packets()\n", + " event_idx: Which event/row to summarize (default: 0)\n", + " \"\"\"\n", + " print(\"=\" * 80)\n", + " print(f\"PACKET SUMMARY - Event {event_idx}\")\n", + " print(\"=\" * 80)\n", + " \n", + " # Header Information\n", + " print(\"\\n HEADER INFORMATION:\")\n", + " print(\"-\" * 80)\n", + " header_cols = ['HeaderMarker', 'PayloadLength', 'P', 'E', 'HT', 'EBO', \n", + " 'M', 'T', 'HdrHamming', 'BXNum', 'L1ANum', 'OrbNum', \n", + " 'S', 'RR', 'HdrCRC']\n", + " \n", + " for col in header_cols:\n", + " if col in result_df.columns:\n", + " val = result_df[col].iloc[event_idx]\n", + " print(f\" {col:15s}: {val}\")\n", + " \n", + " # Check if truncated\n", + " is_truncated = result_df['T'].iloc[event_idx] == 1\n", + " if is_truncated:\n", + " print(\"\\nāš ļø TRUNCATED PACKET - No data available\")\n", + " return\n", + " \n", + " # eRx Summary - Only show eRx units with actual data\n", + " print(\"\\n\" + \"=\" * 80)\n", + " print(\"šŸ“” eRx UNITS SUMMARY:\")\n", + " print(\"=\" * 80)\n", + " \n", + " active_erx_list = []\n", + " empty_erx_list = []\n", + " \n", + " for erx_idx in range(12):\n", + " erx_prefix = f'eRx{erx_idx:02d}_'\n", + " \n", + " # Check if this eRx has any columns\n", + " erx_cols = [col for col in result_df.columns if col.startswith(erx_prefix)]\n", + " if not erx_cols:\n", + " continue\n", + " \n", + " # Get F flag\n", + " f_flag = result_df[f'{erx_prefix}F'].iloc[event_idx]\n", + " \n", + " if f_flag == '1':\n", + " # Empty subpacket - just track it\n", + " stat = result_df[f'{erx_prefix}Stat'].iloc[event_idx]\n", + " ham = result_df[f'{erx_prefix}Ham'].iloc[event_idx]\n", + " empty_erx_list.append((erx_idx, stat, ham))\n", + " continue\n", + " \n", + " # Check if this eRx actually has channel data\n", + " channels_with_data = []\n", + " for ch in range(37):\n", + " chdata_col = f'{erx_prefix}ChData{ch:02d}'\n", + " chdata = result_df[chdata_col].iloc[event_idx]\n", + " if chdata and chdata != '':\n", + " channels_with_data.append(ch)\n", + " \n", + " if len(channels_with_data) == 0:\n", + " # Has F=0 but no actual data - skip it\n", + " continue\n", + " \n", + " active_erx_list.append(erx_idx)\n", + " \n", + " print(f\"\\nšŸ”¹ eRx {erx_idx:02d}:\")\n", + " print(f\" F flag: {f_flag} (HAS DATA)\")\n", + " \n", + " # Full subpacket - show header info\n", + " stat = result_df[f'{erx_prefix}Stat'].iloc[event_idx]\n", + " ham = result_df[f'{erx_prefix}Ham'].iloc[event_idx]\n", + " cm0 = result_df[f'{erx_prefix}CM0'].iloc[event_idx]\n", + " cm1 = result_df[f'{erx_prefix}CM1'].iloc[event_idx]\n", + " e = result_df[f'{erx_prefix}E'].iloc[event_idx]\n", + " chmap = result_df[f'{erx_prefix}ChMap'].iloc[event_idx]\n", + " \n", + " print(f\" Status: {stat}\")\n", + " print(f\" Hamming: {ham}\")\n", + " \n", + " # Handle empty CM values\n", + " if cm0 and cm0 != '':\n", + " print(f\" CM0 (Common Mode 0): {cm0} = {int(cm0, 16)}\")\n", + " else:\n", + " print(f\" CM0 (Common Mode 0): (empty)\")\n", + " \n", + " if cm1 and cm1 != '':\n", + " print(f\" CM1 (Common Mode 1): {cm1} = {int(cm1, 16)}\")\n", + " else:\n", + " print(f\" CM1 (Common Mode 1): (empty)\")\n", + " \n", + " print(f\" Error: {e}\")\n", + " print(f\" ChMap: {chmap}\")\n", + " \n", + " # Decode active channels from ChMap\n", + " if chmap and chmap != '':\n", + " chmap_int = int(chmap, 16)\n", + " active_channels = [i for i in range(37) if (chmap_int >> (36 - i)) & 0x1]\n", + " else:\n", + " active_channels = []\n", + " \n", + " print(f\" Active channels ({len(active_channels)}): {active_channels}\")\n", + " \n", + " # Check which channels actually have data\n", + " channels_with_data = []\n", + " for ch in range(37):\n", + " chdata_col = f'{erx_prefix}ChData{ch:02d}'\n", + " chdata = result_df[chdata_col].iloc[event_idx]\n", + " if chdata and chdata != '':\n", + " channels_with_data.append(ch)\n", + " \n", + " print(f\" Channels with data ({len(channels_with_data)}): {channels_with_data}\")\n", + " \n", + " if len(channels_with_data) != len(active_channels):\n", + " print(f\" āš ļø WARNING: ChMap indicates {len(active_channels)} channels, \"\n", + " f\"but {len(channels_with_data)} have data!\")\n", + " \n", + " # Summary at the end\n", + " print(f\"\\n\" + \"-\" * 80)\n", + " if active_erx_list:\n", + " print(f\"āœ“ Active eRx units with data: {active_erx_list}\")\n", + " else:\n", + " print(f\"āœ— No eRx units with channel data\")\n", + " \n", + " if empty_erx_list:\n", + " print(f\"āŠ˜ Empty eRx units (F=1): {[e[0] for e in empty_erx_list]}\")\n", + " for erx_idx, stat, ham in empty_erx_list:\n", + " print(f\" eRx{erx_idx:02d}: Status={stat}, Hamming={ham}\")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "859676ee", + "metadata": {}, + "outputs": [], + "source": [ + "def read_files(folder, filename=None):\n", + " import glob\n", + " headers = [\"link0\",\"link1\",\"link2\",\"link3\",\"link4\",\"link5\",\"link6\"]\n", + " if filename:\n", + " csv_files = [os.path.join(folder, filename)]\n", + " else:\n", + " csv_files = glob.glob(folder+\"/*.csv\") # Update with your actual folder path\n", + "\n", + " df_list = [pd.read_csv(file, sep=\" \", names=headers, skiprows=1) for file in csv_files]\n", + " final_df = pd.concat(df_list, ignore_index=True)\n", + " return final_df" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "4cb02c24", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Warning: Packet at index 16369 truncated (needs 82 words, only 30 available)\n", + "Found 199 packets\n" + ] + } + ], + "source": [ + "data = read_files(\"data\", \"pedestal_20251208_115228_hexdump.csv\")\n", + "packet_data = data[\"link0\"].values \n", + "\n", + "df_output = pd.DataFrame({'link0': packet_data})\n", + "\n", + "# Use parseOutputPackets to split into individual packets\n", + "#packets = parse_output_packets(df_output, i2c)\n", + "packets = parse_output_packets_no_idle(packet_data, header_marker='1e6')\n", + "\n", + "print(f\"Found {len(packets)} packets\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "7ebdc47d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['f313f039', '9421e842', 'e046801f', ..., '0df37c00', '0e339400',\n", + " '1fffe000'], dtype=object)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "packet_data" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "8523e2e9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['a5140500' '00000000' '1922971f' '000052c0' '10801051' 'f313f007'\n", + " 'd631c0ba' 'e058801f' 'ffffffff' '01103c00' '00300000' '00000000'\n", + " '00a01c00' '00000000' '00000000' '00000000' '00000000' '0c731800'\n", + " '00701800' '00000000' '00000000' '01905800' '00200400' '00000000'\n", + " '00100000' '01505c00' '0dc36800' '0dd37c00' '00000000' '01505400'\n", + " '00000000' '01705800' '00d03400' '00f03c00' '00601c00' '00100c00'\n", + " '0d334c00' '0ce33400' '0d736400' '0d635800' '0cb32c00' '0c631800'\n", + " '0b72e000' '0bf2fc00' '0b62d800' '0c330c00' 'e04a801f' 'ffffffff'\n", + " '00000000' '00000000' '00000000' '00000000' '0c931c00' '09f27c00'\n", + " '00000000' '0bf2fc00' '0c631400' '00000000' '00000000' '00000000'\n", + " '00000000' '0ba2ec00' '0df38800' '0cd32800' '0b42cc00' '0b32cc00'\n", + " '0ad2ac00' '00000000' '00000000' '00000000' '00000000' '00000000'\n", + " '00000000' '0c932c00' '00000000' '0cf33c00' '0c932800' '0b72d800'\n", + " '09f27c00' '09f28400' '0ca32c00' '0af2bc00' '09d26c00' '09123c00'\n", + " '09d27400' 'a7139ada' '1fffe801' '00000000' 'a5140500' '00000000'\n", + " '49edbb09' '000052c0' '10801051' 'f313f023' 'dc820022' 'e058801f'\n", + " 'ffffffff' '01104400' '00100800' '00000000' '00b02400' '00000000'\n", + " '00000000' '00000000' '00000000' '0c732000' '00901c00' '00000000'\n", + " '00000000' '01a06800' '00100400' '00000000' '00100400' '01505400'\n", + " '0dd36c00' '0de37c00' '00000000' '01605800' '00000000' '01605c00'\n", + " '00f03c00' '01104400' '00701c00' '00300800' '0d134800' '0cf33800'\n", + " '0db36400' '0d735400' '0cb33400' '0c732400' '0bb2ec00' '0bf2fc00'\n", + " '0b52dc00' '0c431400' 'e04a801f' 'ffffffff' '00000000' '00000000'\n", + " '00000000' '00000000' '0c932c00' '09f28800' '00000000' '0bf2fc00'\n", + " '0c531c00' '00000000' '00000000' '00000000' '00000000' '0bd2f400'\n", + " '0df38800' '0cd33400' '0b52d400' '0b52cc00' '0ab2b400' '00000000'\n", + " '00000000' '00000000' '00000000' '00000000' '00000000' '0cb32c00'\n", + " '00000000' '0cf34800' '0cd32800' '0b72dc00' '0a128800' '09f28400'\n", + " '0cb33000' '0af2bc00' '09b26c00' '09123c00' '09d27400' '278b8d3c'\n", + " '1fffe801' '00000000' 'a5140500' '00000000' '7a6d3c27' '000052c0'\n", + " '10801051' 'f313f038' '041248aa' 'e056801f' 'ffffffff' '00f03c00'\n", + " '00100800' '00000000' '00701c00' '00000000' '00000000' '00000000'\n", + " '00000000' '0c731400' '00702400' '00000000' '00000000' '01705c00'\n", + " '00100400' '00000000' '00000000' '01505400' '0db36800' '0df37800'\n", + " '00000000' '01305400' '00000000' '01605c00' '00d03800' '00f03c00'\n", + " '00301400' '00400400' '0d334800' '0cf33400' '0d935c00' '0d735c00'\n", + " '0cc32800' '0c631c00' '0b92e400' '0bf2fc00' '0b62d400' '0c331400'\n", + " 'e04a801f' 'ffffffff' '00000000' '00000000' '00000000' '00000000'\n", + " '0ca32400' '09f28800' '00000000' '0bf2fc00' '0c631c00' '00000000'\n", + " '00000000' '00000000' '00000000' '0bd2ec00' '0df37c00' '0cd33400'\n", + " '0b52d400' '0b32cc00' '0ae2b400' '00000000' '00000000' '00000000'\n", + " '00000000' '00000000' '00000000' '0cd32800' '00000000' '0d134400'\n", + " '0ca32800' '0b62dc00' '0a228800' '0a228c00' '0ce32c00' '0af2bc00'\n", + " '09d26c00' '08f24400' '09d27400' '1222c462' '1fffe801' '00000000'\n", + " 'a5140500' '00000000' 'a7721210' '000052c0' '10801051' 'f313f007'\n", + " '0a62882d' 'e057801f' 'ffffffff' '00f03c00' '00100400' '00000000'\n", + " '00b02400' '00000000' '00000000' '00000000' '00000000' '0c731c00'\n", + " '00a02400' '00000000' '00000000' '01706400' '00200400' '00000000'\n", + " '00000000' '01505800' '0dd37800' '0df37c00' '00000000' '01704c00'\n", + " '00000000' '01605c00' '00f03800' '01103c00' '00701c00' '00300c00'\n", + " '0d334800' '0ce33400' '0db36800' '0d435400' '0cd32c00' '0c731c00'\n", + " '0ba2e800' '0c22fc00' '0b72d400' '0c531400' 'e049801f' 'ffffffff'\n", + " '00000000' '00000000' '00000000' '00000000' '0ca31c00' '09f27c00'\n", + " '00000000' '0bf2fc00' '0c531400' '00000000' '00000000' '00000000'\n", + " '00000000' '0bd2f400' '0e137c00' '0cb33400' '0b32d400' '0b42cc00'\n", + " '0ac2b400' '00000000' '00000000' '00000000' '00000000' '00000000'\n", + " '00000000' '0cb32800' '00000000' '0cf33c00' '0c932c00' '0b72dc00'\n", + " '0a228400' '0a128800' '0cb32800' '0ae2bc00' '09d27400' '09123c00'\n", + " '09b27400' '086e69c4' '1fffe801' '00000000' 'a5140500' '00000000'\n", + " 'd24dae01' '000052c0' '10801051' 'f313f017' '10b2c8d3' 'e057801f'\n", + " 'ffffffff' '00f03c00' '00100000' '00000000' '00b01c00' '00000000'\n", + " '00000000' '00000000' '00000000' '0c931800' '00701c00' '00000000'\n", + " '00000000' '01705c00' '00000400' '00000000' '00000000' '01505400'\n", + " '0dd36800' '0df37c00' '00000000' '01504c00' '00000000' '01705c00'\n", + " '00d03800' '00f03c00' '00601400' '00300c00' '0d334800' '0ce33400'\n", + " '0d935c00' '0d735400' '0cb32c00' '0c531800' '0b72dc00' '0bf2fc00'\n", + " '0b62dc00' '0c331400' 'e04a801f' 'ffffffff' '00000000' '00000000'\n", + " '00000000' '00000000' '0cb32400' '0a227c00' '00000000' '0bf2fc00'\n", + " '0c631400' '00000000' '00000000' '00000000' '00000000' '0bf2ec00'\n", + " '0df37c00' '0cb33000' '0b52cc00' '0b32cc00' '0ab2ac00' '00000000'\n", + " '00000000' '00000000' '00000000' '00000000' '00000000' '0c932400'\n", + " '00000000' '0cf33c00' '0cb32400' '0b72d800' '0a127c00' '09f27c00'\n", + " '0cb33000' '0af2bc00' '09d26c00' '08f23c00' '09d26c00' '3ec77698'\n", + " '1fffe801' '00000000' 'a5140500' '00000000' '0468b610' '000052c1'\n", + " '10801051' 'f313f006' '17030872' 'e058801f' 'ffffffff' '00f03c00'\n", + " '00300800' '00000000' '00702c00' '00000000' '00000000' '00000000'\n", + " '00000000' '0c731c00' '00702800' '00000000' '00000000' '01905c00'\n", + " '00100400' '00000000' '00000000' '01605400' '0db37c00' '0de37c00'\n", + " '00000000' '01504c00' '00000000' '01705c00' '00f03400' '01103c00'\n", + " '00601800' '00300400' '0d334c00' '0ce33c00' '0db36c00' '0d735c00'\n", + " '0cb32c00' '0c731c00' '0b92dc00' '0bf2fc00' '0b72d400' '0c531400'\n", + " 'e04a801f' 'ffffffff' '00000000' '00000000' '00000000' '00000000'\n", + " '0c832400' '0a128c00' '00000000' '0bf2fc00' '0c531400' '00000000'\n", + " '00000000' '00000000' '00000000' '0bd2f400' '0df38800' '0ce32c00'\n", + " '0b62cc00' '0b32cc00' '0ab2b400' '00000000' '00000000' '00000000'\n", + " '00000000' '00000000' '00000000' '0cb32400' '00000000' '0cf33800'\n", + " '0c932400' '0b72d800' '0a128800' '0a127c00' '0cd32c00' '0af2bc00'\n", + " '09b26400' '08f24400' '09d27400' 'e79e3d82' '1fffe801' '00000000'\n", + " 'a5140500' '00000000' '285d9d18' '000052c1' '10801051' 'f313f028'\n", + " '1d53489d' 'e057801f' 'ffffffff' '00f03c00' '00100400' '00000000'\n", + " '00702400' '00000000' '00000000' '00000000' '00000000' '0c731c00'\n", + " '00701c00' '00000000' '00000000' '01705c00' '00000000' '00000000'\n", + " '00000000' '01305400' '0db36800' '0dd37c00' '00000000' '01504800'\n", + " '00000000' '01705800' '00d03800' '00f03c00' '00501400' '00100800'\n", + " '0d134400' '0ce33400' '0d736400' '0d735800' '0cb32800' '0c631800'\n", + " '0b92e400' '0bf2fc00' '0b32dc00' '0c231400' 'e049001f' 'ffffffff'\n", + " '00000000' '00000000' '00000000' '00000000' '0c932800' '0a327c00'\n", + " '00000000' '0bf2fc00' '0c731400' '00000000' '00000000' '00000000'\n", + " '00000000' '0bd2ec00' '0df37c00' '0cd32c00' '0b62cc00' '0b32d400'\n", + " '0ab2b000' '00000000' '00000000' '00000000' '00000000' '00000000'\n", + " '00000000' '0cb32400' '00000000' '0d133c00' '0cb32400' '0b62e400'\n", + " '0a227c00' '0a127c00' '0cb32c00' '0af2bc00' '09b26c00' '08f23c00'\n", + " '09d26c00' '51100361' '1fffe801' '00000000' 'a5140500' '00000000'\n", + " '5832560c' '000052c1' '10801051' 'f313f033' '026388d4' 'e057801f'\n", + " 'ffffffff' '01303c00' '00100400' '00000000' '00a02800' '00000000'\n", + " '00000000' '00000000' '00000000' '0c731800' '00701c00' '00000000'\n", + " '00000000' '01705c00' '00100400' '00000000' '00000000' '01604c00'\n", + " '0db36c00' '0de37c00' '00000000' '01504c00' '00000000' '01705c00'\n", + " '00e03c00' '00f04400' '00701800' '00300c00' '0d334400' '0cf33800'\n", + " '0d936000' '0d735c00' '0cb32800' '0c731800' '0b92e400' '0bf2fc00'\n", + " '0b62d400' '0c530c00' 'e049801f' 'ffffffff' '00000000' '00000000'\n", + " '00000000' '00000000' '0ca33000' '0a228800' '00000000' '0bf2fc00'\n", + " '0c532000' '00000000' '00000000' '00000000' '00000000' '0bb2f400'\n", + " '0df37c00' '0cd33400' '0b52d800' '0b32cc00' '0ad2ac00' '00000000'\n", + " '00000000' '00000000' '00000000' '00000000' '00000000' '0cb32800'\n", + " '00000000' '0cf33c00' '0cb32c00' '0b72d800' '09f28400' '0a228400'\n", + " '0cb32c00' '0af2bc00' '09a27400' '08f23c00' '09f27400' 'd9f48ee3'\n", + " '1fffe801' '00000000']\n" + ] + } + ], + "source": [ + "with open('run40unwrapped.txt', 'r') as f:\n", + " lines = f.readlines()\n", + "\n", + "hex_values = []\n", + "for line in lines:\n", + " line = line.strip()\n", + " # Check if line looks like a hex value (8 characters, all hex digits)\n", + " if len(line) == 8 and all(c in '0123456789abcdefABCDEF' for c in line):\n", + " hex_values.append(line.lower())\n", + "\n", + "# Create numpy array with dtype=object\n", + "packet_data = np.array(hex_values, dtype=object)\n", + "\n", + "print(packet_data)\n", + "#print(f\"\\nTotal hex values: {len(hex_values)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "ab109cb8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 8 packets\n", + "Packet 0: 82 words\n", + "Packet 1: 82 words\n", + "Packet 2: 82 words\n", + "Packet 3: 82 words\n", + "Packet 4: 82 words\n", + "Packet 5: 82 words\n", + "Packet 6: 82 words\n", + "Packet 7: 82 words\n", + "\n", + "================================================================================\n", + "PACKET 0\n", + "================================================================================\n", + "================================================================================\n", + "PACKET SUMMARY - Event 0\n", + "================================================================================\n", + "\n", + " HEADER INFORMATION:\n", + "--------------------------------------------------------------------------------\n", + " HeaderMarker : 0x1e6\n", + " PayloadLength : 79\n", + " P : 1\n", + " E : 1\n", + " HT : 0\n", + " EBO : 0\n", + " M : 0\n", + " T : 0\n", + " HdrHamming : 7\n", + " BXNum : 3427\n", + " L1ANum : 7\n", + " OrbNum : 0\n", + " S : 0\n", + " RR : 0\n", + " HdrCRC : 186\n", + "\n", + "================================================================================\n", + "šŸ“” eRx UNITS SUMMARY:\n", + "================================================================================\n", + "\n", + "šŸ”¹ eRx 09:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 0b1 = 177\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "šŸ”¹ eRx 10:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 095 = 149\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "--------------------------------------------------------------------------------\n", + "āœ“ Active eRx units with data: [9, 10]\n", + "\n", + "================================================================================\n", + "PACKET 1\n", + "================================================================================\n", + "================================================================================\n", + "PACKET SUMMARY - Event 0\n", + "================================================================================\n", + "\n", + " HEADER INFORMATION:\n", + "--------------------------------------------------------------------------------\n", + " HeaderMarker : 0x1e6\n", + " PayloadLength : 79\n", + " P : 1\n", + " E : 1\n", + " HT : 0\n", + " EBO : 0\n", + " M : 0\n", + " T : 0\n", + " HdrHamming : 35\n", + " BXNum : 3528\n", + " L1ANum : 8\n", + " OrbNum : 0\n", + " S : 0\n", + " RR : 0\n", + " HdrCRC : 34\n", + "\n", + "================================================================================\n", + "šŸ“” eRx UNITS SUMMARY:\n", + "================================================================================\n", + "\n", + "šŸ”¹ eRx 09:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 0b1 = 177\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "šŸ”¹ eRx 10:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 095 = 149\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "--------------------------------------------------------------------------------\n", + "āœ“ Active eRx units with data: [9, 10]\n", + "\n", + "================================================================================\n", + "PACKET 2\n", + "================================================================================\n", + "================================================================================\n", + "PACKET SUMMARY - Event 0\n", + "================================================================================\n", + "\n", + " HEADER INFORMATION:\n", + "--------------------------------------------------------------------------------\n", + " HeaderMarker : 0x1e6\n", + " PayloadLength : 79\n", + " P : 1\n", + " E : 1\n", + " HT : 0\n", + " EBO : 0\n", + " M : 0\n", + " T : 0\n", + " HdrHamming : 56\n", + " BXNum : 65\n", + " L1ANum : 9\n", + " OrbNum : 1\n", + " S : 0\n", + " RR : 0\n", + " HdrCRC : 170\n", + "\n", + "================================================================================\n", + "šŸ“” eRx UNITS SUMMARY:\n", + "================================================================================\n", + "\n", + "šŸ”¹ eRx 09:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 0ad = 173\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "šŸ”¹ eRx 10:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 095 = 149\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "--------------------------------------------------------------------------------\n", + "āœ“ Active eRx units with data: [9, 10]\n", + "\n", + "================================================================================\n", + "PACKET 3\n", + "================================================================================\n", + "================================================================================\n", + "PACKET SUMMARY - Event 0\n", + "================================================================================\n", + "\n", + " HEADER INFORMATION:\n", + "--------------------------------------------------------------------------------\n", + " HeaderMarker : 0x1e6\n", + " PayloadLength : 79\n", + " P : 1\n", + " E : 1\n", + " HT : 0\n", + " EBO : 0\n", + " M : 0\n", + " T : 0\n", + " HdrHamming : 7\n", + " BXNum : 166\n", + " L1ANum : 10\n", + " OrbNum : 1\n", + " S : 0\n", + " RR : 0\n", + " HdrCRC : 45\n", + "\n", + "================================================================================\n", + "šŸ“” eRx UNITS SUMMARY:\n", + "================================================================================\n", + "\n", + "šŸ”¹ eRx 09:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 0af = 175\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "šŸ”¹ eRx 10:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 093 = 147\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "--------------------------------------------------------------------------------\n", + "āœ“ Active eRx units with data: [9, 10]\n", + "\n", + "================================================================================\n", + "PACKET 4\n", + "================================================================================\n", + "================================================================================\n", + "PACKET SUMMARY - Event 0\n", + "================================================================================\n", + "\n", + " HEADER INFORMATION:\n", + "--------------------------------------------------------------------------------\n", + " HeaderMarker : 0x1e6\n", + " PayloadLength : 79\n", + " P : 1\n", + " E : 1\n", + " HT : 0\n", + " EBO : 0\n", + " M : 0\n", + " T : 0\n", + " HdrHamming : 23\n", + " BXNum : 267\n", + " L1ANum : 11\n", + " OrbNum : 1\n", + " S : 0\n", + " RR : 0\n", + " HdrCRC : 211\n", + "\n", + "================================================================================\n", + "šŸ“” eRx UNITS SUMMARY:\n", + "================================================================================\n", + "\n", + "šŸ”¹ eRx 09:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 0af = 175\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "šŸ”¹ eRx 10:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 095 = 149\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "--------------------------------------------------------------------------------\n", + "āœ“ Active eRx units with data: [9, 10]\n", + "\n", + "================================================================================\n", + "PACKET 5\n", + "================================================================================\n", + "================================================================================\n", + "PACKET SUMMARY - Event 0\n", + "================================================================================\n", + "\n", + " HEADER INFORMATION:\n", + "--------------------------------------------------------------------------------\n", + " HeaderMarker : 0x1e6\n", + " PayloadLength : 79\n", + " P : 1\n", + " E : 1\n", + " HT : 0\n", + " EBO : 0\n", + " M : 0\n", + " T : 0\n", + " HdrHamming : 6\n", + " BXNum : 368\n", + " L1ANum : 12\n", + " OrbNum : 1\n", + " S : 0\n", + " RR : 0\n", + " HdrCRC : 114\n", + "\n", + "================================================================================\n", + "šŸ“” eRx UNITS SUMMARY:\n", + "================================================================================\n", + "\n", + "šŸ”¹ eRx 09:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 0b1 = 177\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "šŸ”¹ eRx 10:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 095 = 149\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "--------------------------------------------------------------------------------\n", + "āœ“ Active eRx units with data: [9, 10]\n", + "\n", + "================================================================================\n", + "PACKET 6\n", + "================================================================================\n", + "================================================================================\n", + "PACKET SUMMARY - Event 0\n", + "================================================================================\n", + "\n", + " HEADER INFORMATION:\n", + "--------------------------------------------------------------------------------\n", + " HeaderMarker : 0x1e6\n", + " PayloadLength : 79\n", + " P : 1\n", + " E : 1\n", + " HT : 0\n", + " EBO : 0\n", + " M : 0\n", + " T : 0\n", + " HdrHamming : 40\n", + " BXNum : 469\n", + " L1ANum : 13\n", + " OrbNum : 1\n", + " S : 0\n", + " RR : 0\n", + " HdrCRC : 157\n", + "\n", + "================================================================================\n", + "šŸ“” eRx UNITS SUMMARY:\n", + "================================================================================\n", + "\n", + "šŸ”¹ eRx 09:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 0af = 175\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "šŸ”¹ eRx 10:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 092 = 146\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "--------------------------------------------------------------------------------\n", + "āœ“ Active eRx units with data: [9, 10]\n", + "\n", + "================================================================================\n", + "PACKET 7\n", + "================================================================================\n", + "================================================================================\n", + "PACKET SUMMARY - Event 0\n", + "================================================================================\n", + "\n", + " HEADER INFORMATION:\n", + "--------------------------------------------------------------------------------\n", + " HeaderMarker : 0x1e6\n", + " PayloadLength : 79\n", + " P : 1\n", + " E : 1\n", + " HT : 0\n", + " EBO : 0\n", + " M : 0\n", + " T : 0\n", + " HdrHamming : 51\n", + " BXNum : 38\n", + " L1ANum : 14\n", + " OrbNum : 1\n", + " S : 0\n", + " RR : 0\n", + " HdrCRC : 212\n", + "\n", + "================================================================================\n", + "šŸ“” eRx UNITS SUMMARY:\n", + "================================================================================\n", + "\n", + "šŸ”¹ eRx 09:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 0af = 175\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "šŸ”¹ eRx 10:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 093 = 147\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "--------------------------------------------------------------------------------\n", + "āœ“ Active eRx units with data: [9, 10]\n" + ] + } + ], + "source": [ + "packets = parse_output_packets_no_idle(packet_data, header_marker='1e6')\n", + "\n", + "print(f\"Found {len(packets)} packets\")\n", + "\n", + "# Show packet sizes\n", + "for i, pkt in enumerate(packets):\n", + " print(f\"Packet {i}: {len(pkt)} words\")\n", + " \n", + "# Unpack each packet\n", + "for i, pkt in enumerate(packets):\n", + " print(f\"\\n{'='*80}\")\n", + " print(f\"PACKET {i}\")\n", + " print(f\"{'='*80}\")\n", + " result = unpack_packets([pkt], active_links=[9,10])\n", + " summarize_packet(result, event_idx=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "2db829b1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'HdrMarker': '0x1e6',\n", + " 'PayloadLength': 79,\n", + " 'P': 1,\n", + " 'E': 1,\n", + " 'HT': 0,\n", + " 'EBO': 0,\n", + " 'M': 0,\n", + " 'T': 0,\n", + " 'Hamming': 7,\n", + " 'Bunch': 3427,\n", + " 'Event': 7,\n", + " 'Orbit': 0,\n", + " 'S': 0,\n", + " 'RR': 0,\n", + " 'CRC': 186}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parse_header_words([packets[0][0], packets[0][1]], return_dict=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "7bd9a089", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'HdrMarker': '0x1e6',\n", + " 'PayloadLength': 79,\n", + " 'P': 1,\n", + " 'E': 1,\n", + " 'HT': 0,\n", + " 'EBO': 0,\n", + " 'M': 0,\n", + " 'T': 0,\n", + " 'Hamming': 35,\n", + " 'Bunch': 3528,\n", + " 'Event': 8,\n", + " 'Orbit': 0,\n", + " 'S': 0,\n", + " 'RR': 0,\n", + " 'CRC': 34}" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parse_header_words([packets[1][0], packets[1][1]], return_dict=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "87111651", + "metadata": {}, + "outputs": [], + "source": [ + "packets[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0f3d1e23", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['f313f023', 'dc820022', 'e058801f', 'ffffffff', '01104400',\n", + " '00100800', '00000000', '00b02400', '00000000', '00000000',\n", + " '00000000', '00000000', '0c732000', '00901c00', '00000000',\n", + " '00000000', '01a06800', '00100400', '00000000', '00100400',\n", + " '01505400', '0dd36c00', '0de37c00', '00000000', '01605800',\n", + " '00000000', '01605c00', '00f03c00', '01104400', '00701c00',\n", + " '00300800', '0d134800', '0cf33800', '0db36400', '0d735400',\n", + " '0cb33400', '0c732400', '0bb2ec00', '0bf2fc00', '0b52dc00',\n", + " '0c431400', 'e04a801f', 'ffffffff', '00000000', '00000000',\n", + " '00000000', '00000000', '0c932c00', '09f28800', '00000000',\n", + " '0bf2fc00', '0c531c00', '00000000', '00000000', '00000000',\n", + " '00000000', '0bd2f400', '0df38800', '0cd33400', '0b52d400',\n", + " '0b52cc00', '0ab2b400', '00000000', '00000000', '00000000',\n", + " '00000000', '00000000', '00000000', '0cb32c00', '00000000',\n", + " '0cf34800', '0cd32800', '0b72dc00', '0a128800', '09f28400',\n", + " '0cb33000', '0af2bc00', '09b26c00', '09123c00', '09d27400',\n", + " '278b8d3c', '1fffe801'], dtype=object)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "packets[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "50f97698", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "================================================================================\n", + "PACKET SUMMARY - Event 0\n", + "================================================================================\n", + "\n", + "šŸ“‹ HEADER INFORMATION:\n", + "--------------------------------------------------------------------------------\n", + " HeaderMarker : 0x1e6\n", + " PayloadLength : 79\n", + " P : 1\n", + " E : 1\n", + " HT : 0\n", + " EBO : 0\n", + " M : 0\n", + " T : 0\n", + " HdrHamming : 51\n", + " BXNum : 38\n", + " L1ANum : 14\n", + " OrbNum : 1\n", + " S : 0\n", + " RR : 0\n", + " HdrCRC : 212\n", + "\n", + "================================================================================\n", + "šŸ“” eRx UNITS SUMMARY:\n", + "================================================================================\n", + "\n", + "šŸ”¹ eRx 09:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 0af = 175\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "šŸ”¹ eRx 10:\n", + " F flag: 0 (HAS DATA)\n", + " Status: 7\n", + " Hamming: 0\n", + " CM0 (Common Mode 0): 093 = 147\n", + " CM1 (Common Mode 1): 000 = 0\n", + " Error: 0\n", + " ChMap: 1fffffffff\n", + " Active channels (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + " Channels with data (37): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36]\n", + "\n", + "--------------------------------------------------------------------------------\n", + "āœ“ Active eRx units with data: [9, 10]\n" + ] + } + ], + "source": [ + "summarize_packet(result, event_idx=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c60588a", + "metadata": {}, + "outputs": [], + "source": [ + "import crcmod\n", + "import codecs\n", + "\n", + "crc8 = crcmod.mkCrcFun(0x1a7, initCrc=0, xorOut=0, rev=False)\n", + "\n", + "def calculate_crc8_bits(data):\n", + " \"\"\"\n", + " Calculate 8-bit CRC using crcmod library.\n", + "\n", + " Args:\n", + " data: 64-bit integer\n", + " Returns:\n", + " 8-bit CRC value (integer)\n", + " \"\"\"\n", + " # Convert integer to 8 bytes (hex string, then to bytes)\n", + " hex_str = f'{data:016x}'\n", + " byte_data = codecs.decode(hex_str, 'hex')\n", + " crc_value = crc8(byte_data)\n", + " \n", + " return crc_value\n", + "\n", + "\n", + "def verify_header_crc(header_word0, header_word1, verbose=False):\n", + " \"\"\"\n", + " Verify the 8-bit CRC in the event header.\n", + " \n", + " The CRC is calculated over a 64-bit value constructed as:\n", + " [8 zeros][26 bits from word0, bits 6-31, excluding Hamming bits 0-5][6 zeros][24 bits from word1, bits 8-31, excluding CRC bits 0-7]\n", + " \n", + " Args:\n", + " header_word0: First 32-bit header word (hex string or int)\n", + " header_word1: Second 32-bit header word (hex string or int)\n", + " verbose: If True, print detailed information\n", + " \n", + " Returns:\n", + " tuple: (is_valid, transmitted_crc, calculated_crc)\n", + " \"\"\"\n", + " # Convert to integers if needed\n", + " if isinstance(header_word0, str):\n", + " header_word0 = int(header_word0, 16)\n", + " if isinstance(header_word1, str):\n", + " header_word1 = int(header_word1, 16)\n", + " \n", + " # Extract transmitted CRC from header word 1 (bits 0-7)\n", + " transmitted_crc = header_word1 & 0xFF\n", + " \n", + " # Build CRC calculation base (64 bits total):\n", + " # Format: 8_zeros + 26_bits_from_word0 + 6_zeros + 24_bits_from_word1\n", + " \n", + " # Extract bits 6-31 from header_word0 (26 bits, excluding Hamming bits 0-5)\n", + " word0_data = (header_word0 >> 6) & 0x3FFFFFF # 26 bits\n", + " \n", + " # Extract bits 8-31 from header_word1 (24 bits, excluding CRC bits 0-7)\n", + " word1_data = (header_word1 >> 8) & 0xFFFFFF # 24 bits\n", + " \n", + " # Construct the 64-bit CRC base:\n", + " # [8 zeros][26 bits from word0][6 zeros][24 bits from word1]\n", + " header_crc_base = (word0_data << 30) | word1_data\n", + " \n", + " if verbose:\n", + " print(f\"Header Word 0: 0x{header_word0:08x}\")\n", + " print(f\" 0b{header_word0:032b}\")\n", + " print(f\" {'D' * 26}{'H' * 6} (D=Data, H=Hamming)\")\n", + " print()\n", + " \n", + " print(f\"Header Word 1: 0x{header_word1:08x}\")\n", + " print(f\" 0b{header_word1:032b}\")\n", + " print(f\" {'D' * 24}{'C' * 8} (D=Data, C=CRC)\")\n", + " print()\n", + " \n", + " print(\"Building CRC Base (64-bit):\")\n", + " print(f\" Word 0 [31:6]: 0b{word0_data:026b} (26 bits)\")\n", + " print(f\" Word 1 [31:8]: 0b{word1_data:024b} (24 bits)\")\n", + " print()\n", + " print(f\"CRC Base Structure:\")\n", + " print(f\" 0b{'0'*8}{word0_data:026b}{'0'*6}{word1_data:024b}\")\n", + " print(f\" {'8 zeros':<8}{'26 bits from W0':<26}{'6 zero':<6}{'24 bits from W1':<24}\")\n", + " print()\n", + " print(f\"CRC Base (64-bit): 0x{header_crc_base:016x}\")\n", + " print(f\" 0b{header_crc_base:064b}\")\n", + " print()\n", + " print(f\"Transmitted CRC: 0x{transmitted_crc:02x} = 0b{transmitted_crc:08b} = {transmitted_crc}\")\n", + " \n", + " # Calculate CRC\n", + " calculated_crc = calculate_crc8_bits(header_crc_base)\n", + " \n", + " if verbose:\n", + " print(f\"Calculated CRC: 0x{calculated_crc:02x} = 0b{calculated_crc:08b} = {calculated_crc}\")\n", + " print(f\"CRC Valid: {calculated_crc == transmitted_crc}\")\n", + " \n", + " return calculated_crc == transmitted_crc, transmitted_crc, calculated_crc\n", + "\n", + "\n", + "def find_corrupted_bits(header_word0, header_word1):\n", + " \"\"\"\n", + " Determine which bits would need to change to make the CRC match.\n", + " Tests single-bit and two-bit errors.\n", + " \n", + " Args:\n", + " header_word0: First 32-bit header word (hex string or int)\n", + " header_word1: Second 32-bit header word (hex string or int)\n", + " \n", + " Returns:\n", + " Dictionary with possible corrections\n", + " \"\"\"\n", + " # Convert to integers\n", + " if isinstance(header_word0, str):\n", + " header_word0 = int(header_word0, 16)\n", + " if isinstance(header_word1, str):\n", + " header_word1 = int(header_word1, 16)\n", + " \n", + " transmitted_crc = header_word1 & 0xFF\n", + " \n", + " # Check if original is valid\n", + " is_valid_orig, _, calc_crc_orig = verify_header_crc(header_word0, header_word1)\n", + " \n", + " results = {\n", + " 'original': {\n", + " 'word0': f'0x{header_word0:08x}',\n", + " 'word1': f'0x{header_word1:08x}',\n", + " 'transmitted_crc': transmitted_crc,\n", + " 'calculated_crc': calc_crc_orig,\n", + " 'is_valid': is_valid_orig\n", + " },\n", + " 'single_bit_flips': [],\n", + " 'two_bit_flips': []\n", + " }\n", + " \n", + " if is_valid_orig:\n", + " print(\"Original CRC is already valid!\")\n", + " return results\n", + " \n", + " print(f\"Original CRC mismatch: transmitted={transmitted_crc}, calculated={calc_crc_orig}\")\n", + " print(\"Searching for single-bit flips...\")\n", + " \n", + " # Test single-bit flips in word 0 (bits 6-31, excluding Hamming bits 0-5)\n", + " for bit in range(6, 32):\n", + " test_word0 = header_word0 ^ (1 << bit)\n", + " is_valid, _, calc_crc = verify_header_crc(test_word0, header_word1)\n", + " if is_valid:\n", + " results['single_bit_flips'].append({\n", + " 'location': f'Word0, bit {bit}',\n", + " 'corrected_word0': f'0x{test_word0:08x}',\n", + " 'corrected_word1': f'0x{header_word1:08x}',\n", + " 'original_bit': (header_word0 >> bit) & 1,\n", + " 'corrected_bit': (test_word0 >> bit) & 1,\n", + " 'crc': calc_crc\n", + " })\n", + " \n", + " # Test single-bit flips in word 1 (bits 8-31, excluding CRC bits 0-7)\n", + " for bit in range(8, 32):\n", + " test_word1 = header_word1 ^ (1 << bit)\n", + " is_valid, _, calc_crc = verify_header_crc(header_word0, test_word1)\n", + " if is_valid:\n", + " results['single_bit_flips'].append({\n", + " 'location': f'Word1, bit {bit}',\n", + " 'corrected_word0': f'0x{header_word0:08x}',\n", + " 'corrected_word1': f'0x{test_word1:08x}',\n", + " 'original_bit': (header_word1 >> bit) & 1,\n", + " 'corrected_bit': (test_word1 >> bit) & 1,\n", + " 'crc': calc_crc\n", + " })\n", + " \n", + " # Only search two-bit flips if no single-bit solution found\n", + " if not results['single_bit_flips']:\n", + " print(\"No single-bit flip found, searching two-bit flips...\")\n", + " \n", + " # Test two-bit flips in word 0\n", + " for bit1 in range(6, 32):\n", + " for bit2 in range(bit1 + 1, 32):\n", + " test_word0 = header_word0 ^ (1 << bit1) ^ (1 << bit2)\n", + " is_valid, _, calc_crc = verify_header_crc(test_word0, header_word1)\n", + " if is_valid:\n", + " results['two_bit_flips'].append({\n", + " 'location': f'Word0, bits {bit1} and {bit2}',\n", + " 'corrected_word0': f'0x{test_word0:08x}',\n", + " 'corrected_word1': f'0x{header_word1:08x}',\n", + " 'crc': calc_crc\n", + " })\n", + " \n", + " # Test two-bit flips in word 1\n", + " for bit1 in range(8, 32):\n", + " for bit2 in range(bit1 + 1, 32):\n", + " test_word1 = header_word1 ^ (1 << bit1) ^ (1 << bit2)\n", + " is_valid, _, calc_crc = verify_header_crc(header_word0, test_word1)\n", + " if is_valid:\n", + " results['two_bit_flips'].append({\n", + " 'location': f'Word1, bits {bit1} and {bit2}',\n", + " 'corrected_word0': f'0x{header_word0:08x}',\n", + " 'corrected_word1': f'0x{test_word1:08x}',\n", + " 'crc': calc_crc\n", + " })\n", + " \n", + " # Test one bit in each word\n", + " print(\"Searching for one bit flip in each word...\")\n", + " for bit0 in range(6, 32):\n", + " for bit1 in range(8, 32):\n", + " test_word0 = header_word0 ^ (1 << bit0)\n", + " test_word1 = header_word1 ^ (1 << bit1)\n", + " is_valid, _, calc_crc = verify_header_crc(test_word0, test_word1)\n", + " if is_valid:\n", + " results['two_bit_flips'].append({\n", + " 'location': f'Word0 bit {bit0} and Word1 bit {bit1}',\n", + " 'corrected_word0': f'0x{test_word0:08x}',\n", + " 'corrected_word1': f'0x{test_word1:08x}',\n", + " 'crc': calc_crc\n", + " })\n", + " \n", + " return results" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "cd41004c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Header Word 0: 0xf313f024\n", + " 0b11110011000100111111000000100100\n", + " DDDDDDDDDDDDDDDDDDDDDDDDDDHHHHHH (D=Data, H=Hamming)\n", + "\n", + "Header Word 1: 0xa0e22825\n", + " 0b10100000111000100010100000100101\n", + " DDDDDDDDDDDDDDDDDDDDDDDDCCCCCCCC (D=Data, C=CRC)\n", + "\n", + "Building CRC Base (64-bit):\n", + " Word 0 [31:6]: 0b11110011000100111111000000 (26 bits)\n", + " Word 1 [31:8]: 0b101000001110001000101000 (24 bits)\n", + "\n", + "CRC Base Structure:\n", + " 0b0000000011110011000100111111000000000000101000001110001000101000\n", + " 8 zeros 26 bits from W0 6 zero24 bits from W1 \n", + "\n", + "CRC Base (64-bit): 0x00f313f000a0e228\n", + " 0b0000000011110011000100111111000000000000101000001110001000101000\n", + "\n", + "Transmitted CRC: 0x25 = 0b00100101 = 37\n", + "Calculated CRC: 0x25 = 0b00100101 = 37\n", + "CRC Valid: True\n" + ] + } + ], + "source": [ + "#header0 = \"f313f007\"\n", + "#header1 = \"d631c0ba\"\n", + "\n", + "header0 = \"f313f024\"\n", + "header1 = \"a0e22825\"\n", + "\n", + "is_valid, tx, calc = verify_header_crc(header0, header1, verbose=True)\n", + "\n", + "if not is_valid:\n", + " print(\"\\n\" + \"=\" * 80)\n", + " print(\"Finding which bits need to change to fix CRC...\")\n", + " print(\"=\" * 80)\n", + " corrections = find_corrupted_bits(header0, header1)\n", + " \n", + " print(f\"\\nOriginal Header:\")\n", + " print(f\" Word 0: {corrections['original']['word0']}\")\n", + " print(f\" Word 1: {corrections['original']['word1']}\")\n", + " print(f\" Transmitted CRC: 0x{corrections['original']['transmitted_crc']:02x}\")\n", + " \n", + " if corrections['single_bit_flips']:\n", + " print(f\"\\nāœ“ Found {len(corrections['single_bit_flips'])} single-bit flip solution(s):\")\n", + " for i, fix in enumerate(corrections['single_bit_flips'], 1):\n", + " print(f\"\\n Solution {i}: Flip {fix['location']}\")\n", + " print(f\" Corrected Word 0: {fix['corrected_word0']}\")\n", + " print(f\" Corrected Word 1: {fix['corrected_word1']}\")\n", + " print(f\" CRC now matches: 0x{fix['crc']:02x}\")\n", + " elif corrections['two_bit_flips']:\n", + " print(f\"\\nāœ“ Found {len(corrections['two_bit_flips'])} two-bit flip solution(s):\")\n", + " for i, fix in enumerate(corrections['two_bit_flips'][:5], 1): # Show first 5\n", + " print(f\"\\n Solution {i}: Flip {fix['location']}\")\n", + " print(f\" Corrected Word 0: {fix['corrected_word0']}\")\n", + " print(f\" Corrected Word 1: {fix['corrected_word1']}\")\n", + " print(f\" CRC now matches: 0x{fix['crc']:02x}\")\n", + " if len(corrections['two_bit_flips']) > 5:\n", + " print(f\"\\n ... and {len(corrections['two_bit_flips']) - 5} more solutions\")\n", + " else:\n", + " print(\"\\nāœ— No simple single or two-bit flip solution found\")\n", + " print(\" This suggests multiple bit errors or a different error pattern\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "117d19d0", + "metadata": {}, + "source": [ + "The hamming values are a parity check of different combinations of bits:\n", + "\n", + "```\n", + "parityGroups = np.array([[56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41,\n", + " 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26],\n", + " [56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41,\n", + " 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11],\n", + " [56, 55, 54, 53, 52, 51, 50, 49, 40, 39, 38, 37, 36, 35, 34, 33,\n", + " 25, 24, 23, 22, 21, 20, 19, 18, 10, 9, 8, 7, 6, 5, 4],\n", + " [56, 55, 54, 53, 48, 47, 46, 45, 40, 39, 38, 37, 32, 31, 30, 29,\n", + " 25, 24, 23, 22, 17, 16, 15, 14, 10, 9, 8, 7, 3, 2, 1],\n", + " [56, 55, 52, 51, 48, 47, 44, 43, 40, 39, 36, 35, 32, 31, 28, 27,\n", + " 25, 24, 21, 20, 17, 16, 13, 12, 10, 9, 6, 5, 3, 2, 0],\n", + " [56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26,\n", + " 25, 23, 21, 19, 17, 15, 13, 11, 10, 8, 6, 4, 3, 1, 0]])\n", + "```\n", + "for these different groups of bits, a Hamming code !=0 is whether there is an even or odd number of 1's in the values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c39a12d8", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "@webio": { + "lastCommId": null, + "lastKernelId": null + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/app/tool/algorithm/get_toa_efficiencies.cxx b/app/tool/algorithm/get_toa_efficiencies.cxx index bf3e34980..c941a620b 100644 --- a/app/tool/algorithm/get_toa_efficiencies.cxx +++ b/app/tool/algorithm/get_toa_efficiencies.cxx @@ -1,20 +1,34 @@ #include "get_toa_efficiencies.h" +#include + #include "pflib/utility/efficiency.h" namespace pflib::algorithm { +// helper function +bool is_masked(int ch, const std::vector& masked_channels) { + return (count(masked_channels.begin(), masked_channels.end(), ch) > 0); +} + template std::array get_toa_efficiencies( - const std::vector& data) { + const std::vector& data, + const std::vector& masked_channels) { std::array efficiencies; /// reserve a vector of the appropriate size to avoid repeating allocation /// time for all 72 channels + // fill with NANs so that channels can be masked out + efficiencies.fill(std::numeric_limits::quiet_NaN()); std::vector toas(data.size()); int i_ch = 0; // 0ā€“35 int i_link = 0; for (int ch{0}; ch < 72; ch++) { + if (is_masked(ch, masked_channels)) { + continue; // leave NaN + } + i_link = (ch / 36); i_ch = ch % 36; for (std::size_t i{0}; i < toas.size(); i++) { @@ -46,9 +60,11 @@ std::array get_toa_efficiencies( // get toa efficiencies template std::array pflib::algorithm::get_toa_efficiencies( - const std::vector& data); + const std::vector& data, + const std::vector& masked_channels); template std::array pflib::algorithm::get_toa_efficiencies< pflib::packing::MultiSampleECONDEventPacket>( - const std::vector& data); + const std::vector& data, + const std::vector& masked_channels); } // namespace pflib::algorithm \ No newline at end of file diff --git a/app/tool/algorithm/get_toa_efficiencies.h b/app/tool/algorithm/get_toa_efficiencies.h index de8bcb272..200fc1d3e 100644 --- a/app/tool/algorithm/get_toa_efficiencies.h +++ b/app/tool/algorithm/get_toa_efficiencies.h @@ -16,7 +16,8 @@ namespace pflib::algorithm { // templated to match any event packet type template std::array get_toa_efficiencies( - const std::vector& data); + const std::vector& data, + const std::vector& masked_channels); // std::array get_toa_efficiencies( // const std::vector& data); diff --git a/app/tool/algorithm/level_pedestals.cxx b/app/tool/algorithm/level_pedestals.cxx index 6c35e208f..c81ef17f3 100644 --- a/app/tool/algorithm/level_pedestals.cxx +++ b/app/tool/algorithm/level_pedestals.cxx @@ -51,17 +51,35 @@ static int get_adc(const EventPacket& p, int ch) { * TOT/TOA and the sample Tp/Tc flags are ignored. */ template -static std::array get_adc_medians( - const std::vector& data) { +static std::array get_adc_medians(const std::vector& data, + std::vector& masked_channels) { std::array medians; /// reserve a vector of the appropriate size to avoid repeating allocation /// time for all 72 channels std::vector adcs(data.size()); for (int ch{0}; ch < 72; ch++) { + if (count(masked_channels.begin(), masked_channels.end(), ch) > 0) { + // Do Not include masked channels in the median calculation + medians[ch] = -999; // to maintain length of medians + continue; + } + for (std::size_t i{0}; i < adcs.size(); i++) { adcs[i] = get_adc(data[i], ch); + + // // print out multi sample event packet p here + // if constexpr (std::is_same_v< + // EventPacket, + // pflib::packing::MultiSampleECONDEventPacket>) { + // std::cout << "p.soi, ch, ilink, adc: " << data[i].i_soi << ", " << ch + // << ", " << ch / 36 << ", " << adcs[i] << std::endl; + // } else { + // std::cout << "NOT MULTI " << std::endl; + // } } - medians[ch] = pflib::utility::median(adcs); + // medians[ch] = pflib::utility::median(adcs); + // std::cout << "here: ch, median " << ch << ", " << medians[ch] << + // std::endl; } return medians; } @@ -72,7 +90,8 @@ template // any use of is a placeholder for static void pedestal_runs(Target* tgt, ROC& roc, std::array& baseline, std::array& highend, std::array& lowend, - std::array& target, size_t n_events) { + std::array& target, size_t n_events, + std::vector& masked_channels) { /// TODO for multi-ROC set ups, we could dynamically determine the number // of ROCs and the number of channels from the Target DecodeAndBuffer buffer{n_events, 2}; @@ -87,8 +106,12 @@ static void pedestal_runs(Target* tgt, ROC& roc, std::array& baseline, .apply(); daq_run(tgt, "PEDESTAL", buffer, n_events, 100); pflib_log(trace) << "baseline run done, getting channel medians"; - auto medians = get_adc_medians(buffer.get_buffer()); + auto medians = + get_adc_medians(buffer.get_buffer(), masked_channels); baseline = medians; + // for (int i = 0; i < 72; i++) { + // pflib_log(info) << "baseline: " << baseline[i]; + // } pflib_log(trace) << "got channel medians, getting link medians"; for (int i_link{0}; i_link < 2; i_link++) { auto start{medians.begin() + 36 * i_link}; @@ -108,7 +131,8 @@ static void pedestal_runs(Target* tgt, ROC& roc, std::array& baseline, .add_all_channels("TRIM_INV", 63) .apply(); daq_run(tgt, "PEDESTAL", buffer, n_events, 100); - highend = get_adc_medians(buffer.get_buffer()); + highend = + get_adc_medians(buffer.get_buffer(), masked_channels); } { // lowend run @@ -119,7 +143,7 @@ static void pedestal_runs(Target* tgt, ROC& roc, std::array& baseline, .add_all_channels("TRIM_INV", 0) .apply(); daq_run(tgt, "PEDESTAL", buffer, n_events, 100); - lowend = get_adc_medians(buffer.get_buffer()); + lowend = get_adc_medians(buffer.get_buffer(), masked_channels); } } @@ -127,6 +151,20 @@ std::map> level_pedestals( Target* tgt, ROC roc) { static auto the_log_{::pflib::logging::get("level_pedestals")}; + // Load CHANNEL MASK file + std::string mask_file_path = pftool::readline("Path to maskfile: ", ""); + bool use_mask = !mask_file_path.empty(); + std::vector masked_channels; + std::string line; + if (use_mask) { + std::ifstream mask_file(mask_file_path); + std::getline(mask_file, line); // ditch first line + while (std::getline(mask_file, line)) { + int int_ch = std::atoi(line.c_str()); + masked_channels.push_back(int_ch); + } + } + /// do three runs of 100 samples each to have well defined pedestals static const std::size_t n_events = 100; @@ -142,12 +180,12 @@ std::map> level_pedestals( if (pftool::state.daq_format_mode == Target::DaqFormat::SIMPLEROC) { pedestal_runs( - tgt, roc, baseline, highend, lowend, target, n_events); + tgt, roc, baseline, highend, lowend, target, n_events, masked_channels); } else if (pftool::state.daq_format_mode == Target::DaqFormat::ECOND_SW_HEADERS) { pedestal_runs( - tgt, roc, baseline, highend, lowend, target, n_events); + tgt, roc, baseline, highend, lowend, target, n_events, masked_channels); } else { pflib_log(warn) << "Unsupported DAQ format (" @@ -157,9 +195,30 @@ std::map> level_pedestals( pflib_log(info) << "sample collections done, deducing settings"; std::map> settings; + for (int ch{0}; ch < 72; ch++) { std::string page{pflib::utility::string_format("CH_%d", ch)}; + + // SKIP IF CHANNEL IS MASKED + if (count(masked_channels.begin(), masked_channels.end(), ch) > 0) { + pflib_log(info) << "Channel " << ch + << " is masked; skipping pedestal leveling."; + + // Explicitly mark it masked + // settings[page]["MASKED"] = 1; + + continue; + + // } else { + // settings[page]["MASKED"] = 0; + } + int i_link = ch / 36; + + pflib_log(info) << "Channel, Baseline, Lowend, Highend: " << ch << ", " + << baseline[ch] << ", " << lowend[ch] << ", " << highend[ch] + << std::endl; + if (baseline.at(ch) < target.at(i_link)) { pflib_log(debug) << "Channel " << ch << " is below target, setting TRIM_INV"; @@ -216,6 +275,7 @@ std::map> level_pedestals( << " is above target, setting SIGN_DAC=1 and DACB"; settings[page]["SIGN_DAC"] = 1; settings[page]["DACB"] = val; + pflib_log(info) << "Channel " << ch << " has val: " << val << "."; } } } diff --git a/app/tool/algorithm/toa_vref_scan.cxx b/app/tool/algorithm/toa_vref_scan.cxx index 1486a8a2e..cc6bf5077 100644 --- a/app/tool/algorithm/toa_vref_scan.cxx +++ b/app/tool/algorithm/toa_vref_scan.cxx @@ -8,7 +8,7 @@ namespace pflib::algorithm { -// Templated helpder function +// Templated helper function template static void toa_vref_runs(Target* tgt, ROC& roc, size_t n_events, std::array& target) { @@ -18,6 +18,20 @@ static void toa_vref_runs(Target* tgt, ROC& roc, size_t n_events, // loop over runs, from toa_vref = 0 to = 255 for (int toa_vref{0}; toa_vref < 256; toa_vref++) { + // Load CHANNEL MASK file + std::string mask_file_path = pftool::readline("Path to maskfile: ", ""); + bool use_mask = !mask_file_path.empty(); + std::vector masked_channels; + std::string line; + if (use_mask) { + std::ifstream mask_file(mask_file_path); + std::getline(mask_file, line); // ditch first line + while (std::getline(mask_file, line)) { + int int_ch = std::atoi(line.c_str()); + masked_channels.push_back(int_ch); + } + } + pflib_log(info) << "testing toa_vref = " << toa_vref; auto test_handle = roc.testParameters() .add("REFERENCEVOLTAGE_0", "TOA_VREF", toa_vref) @@ -27,7 +41,8 @@ static void toa_vref_runs(Target* tgt, ROC& roc, size_t n_events, daq_run(tgt, "PEDESTAL", buffer, n_events, 100); pflib_log(trace) << "finished toa_vref = " << toa_vref << ", getting efficiencies"; - auto efficiencies = get_toa_efficiencies(buffer.get_buffer()); + auto efficiencies = + get_toa_efficiencies(buffer.get_buffer(), masked_channels); pflib_log(trace) << "got channel efficiencies, getting max efficiency per link"; for (int i_link{0}; i_link < 2; i_link++) { diff --git a/app/tool/algorithm/trim_toa_scan.cxx b/app/tool/algorithm/trim_toa_scan.cxx index c5ecf2568..780dfb108 100644 --- a/app/tool/algorithm/trim_toa_scan.cxx +++ b/app/tool/algorithm/trim_toa_scan.cxx @@ -68,7 +68,8 @@ namespace pflib::algorithm { template static void trim_tof_runs( Target* tgt, ROC& roc, size_t n_events, - std::array, 8>, 200>& final_data) { + std::array, 8>, 200>& final_data, + std::vector& masked_channels) { // working in buffer, not in writer DecodeAndBuffer buffer{n_events, 2}; static auto the_log_{::pflib::logging::get("toa_vref_scan")}; @@ -98,7 +99,8 @@ static void trim_tof_runs( pflib_log(trace) << "finished trim_toa = " << trim_toa << ", and calib = " << calib << ", getting efficiencies"; - auto efficiencies = get_toa_efficiencies(buffer.get_buffer()); + auto efficiencies = + get_toa_efficiencies(buffer.get_buffer(), masked_channels); pflib_log(trace) << "got channel efficiencies, storing now"; for (int ch{0}; ch < 72; ch++) { // need to divide by 4 because index is value/4 from final_data @@ -124,6 +126,20 @@ std::map> trim_toa_scan( static const std::size_t n_events = 100; + // Load CHANNEL MASK file + std::string mask_file_path = pftool::readline("Path to maskfile: ", ""); + bool use_mask = !mask_file_path.empty(); + std::vector masked_channels; + std::string line; + if (use_mask) { + std::ifstream mask_file(mask_file_path); + std::getline(mask_file, line); // ditch first line + while (std::getline(mask_file, line)) { + int int_ch = std::atoi(line.c_str()); + masked_channels.push_back(int_ch); + } + } + tgt->setup_run(1, pftool::state.daq_format_mode, 1); // trim_toa is a channel-wise parameter (1 value per channel) @@ -134,12 +150,12 @@ std::map> trim_toa_scan( std::array, 8>, 200> final_data; if (pftool::state.daq_format_mode == Target::DaqFormat::SIMPLEROC) { - trim_tof_runs(tgt, roc, n_events, - final_data); + trim_tof_runs( + tgt, roc, n_events, final_data, masked_channels); } else if (pftool::state.daq_format_mode == Target::DaqFormat::ECOND_SW_HEADERS) { trim_tof_runs( - tgt, roc, n_events, final_data); + tgt, roc, n_events, final_data, masked_channels); } else { pflib_log(warn) << "Unsupported DAQ format (" << static_cast(pftool::state.daq_format_mode) diff --git a/build/hcal_channel_mask.csv b/build/hcal_channel_mask.csv new file mode 100644 index 000000000..29c6e5494 --- /dev/null +++ b/build/hcal_channel_mask.csv @@ -0,0 +1,25 @@ +Mask +8 +17 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +44 +53 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 diff --git a/level_peds_backup b/level_peds_backup new file mode 100644 index 000000000..e9d5ac335 --- /dev/null +++ b/level_peds_backup @@ -0,0 +1,264 @@ +#include "level_pedestals.h" + +#include "../daq_run.h" +#include "../tasks/level_pedestals.h" +#include "pflib/utility/median.h" +#include "pflib/utility/string_format.h" + +namespace pflib::algorithm { + +/** + * Retrieve the ADC sample for the input channel from the input event packet + */ +template +static int get_adc(const EventPacket& p, int ch) { + if constexpr (std::is_same_v) { + return p.channel(ch).adc(); + } else if constexpr (std::is_same_v< + EventPacket, + pflib::packing::MultiSampleECONDEventPacket>) { + // Use link specific channel calculation, this is done in + // singleROCEventPacket.cxx for the other case + // Use the "Sample Of Interest" inside the EventPacket + // TODO this is only true if we only have one ROC's channels enabled + // in the ECON-D. In the more realistic case, we should get the + // link indices depending on which ROC we are aligning + int i_link = ch / 36; // 0 or 1 + int i_ch = ch % 36; // 0 - 35 + + // ECONDEventPacket.h defines channel differently to SingleROCEventPacket.h + // because it can have more than 2 links readout + return p.samples[p.i_soi].channel(i_link, i_ch).adc(); + } else { + static_assert(sizeof(EventPacket) == 0, + "Unsupported packet type in get_adc()"); + } +} + +/** + * get the medians of the channel ADC values + * + * This may be helpful in some other contexts, but since it depends on the + * packing library it cannot go into utility. Just keeping it here for now, + * maybe move it into its own header/impl in algorithm. + * + * @param[in] data buffer of single-roc packet data + * @return array of channel ADC values + * + * @note We assume the caller knows what they are doing. + * Calib and Common Mode channels are ignored. + * TOT/TOA and the sample Tp/Tc flags are ignored. + */ +template +static std::array get_adc_medians(const std::vector& data, + std::vector& masked_channels) { + std::array medians; + /// reserve a vector of the appropriate size to avoid repeating allocation + /// time for all 72 channels + std::vector adcs(data.size()); + for (int ch{0}; ch < 72; ch++) { + // if (count(masked_channels.begin(), masked_channels.end(), ch) > 0) { + // // Do Not include masked channels in the median calculation + // medians[ch] = -0; // to maintain length of medians + // continue; + // } + + for (std::size_t i{0}; i < adcs.size(); i++) { + adcs[i] = get_adc(data[i], ch); + } + medians[ch] = pflib::utility::median(adcs); + } + return medians; +} + +// Helper function to pull the 3 runs +template // any use of is a placeholder for + // what the function gets called with. +static void pedestal_runs(Target* tgt, ROC& roc, std::array& baseline, + std::array& highend, + std::array& lowend, + std::array& target, size_t n_events, + std::vector& masked_channels) { + /// TODO for multi-ROC set ups, we could dynamically determine the number + // of ROCs and the number of channels from the Target + DecodeAndBuffer buffer{n_events, 2}; + static auto the_log_{::pflib::logging::get("level_pedestals")}; + + { // baseline run scope + pflib_log(info) << "100 event baseline run"; + auto test_handle = roc.testParameters() + .add_all_channels("SIGN_DAC", 0) + .add_all_channels("DACB", 0) + .add_all_channels("TRIM_INV", 0) + .apply(); + daq_run(tgt, "PEDESTAL", buffer, n_events, 100); + pflib_log(trace) << "baseline run done, getting channel medians"; + auto medians = + get_adc_medians(buffer.get_buffer(), masked_channels); + baseline = medians; + pflib_log(trace) << "got channel medians, getting link medians"; + for (int i_link{0}; i_link < 2; i_link++) { + auto start{medians.begin() + 36 * i_link}; + auto end{start + 36}; + auto halfway{start + 18}; + std::nth_element(start, halfway, end); + target[i_link] = *halfway; + } + pflib_log(trace) << "got link medians"; + } + + { // highend run scope + pflib_log(info) << "100 event highend run"; + auto test_handle = roc.testParameters() + .add_all_channels("SIGN_DAC", 0) + .add_all_channels("DACB", 0) + .add_all_channels("TRIM_INV", 63) + .apply(); + daq_run(tgt, "PEDESTAL", buffer, n_events, 100); + highend = + get_adc_medians(buffer.get_buffer(), masked_channels); + } + + { // lowend run + pflib_log(info) << "100 event lowend run"; + auto test_handle = roc.testParameters() + .add_all_channels("SIGN_DAC", 1) + .add_all_channels("DACB", 31) + .add_all_channels("TRIM_INV", 0) + .apply(); + daq_run(tgt, "PEDESTAL", buffer, n_events, 100); + lowend = get_adc_medians(buffer.get_buffer(), masked_channels); + } +} + +std::map> level_pedestals( + Target* tgt, ROC roc) { + static auto the_log_{::pflib::logging::get("level_pedestals")}; + + // Load CHANNEL MASK file + std::string mask_file_path = pftool::readline("Path to maskfile: ", ""); + bool use_mask = !mask_file_path.empty(); + std::vector masked_channels; + std::string line; + if (use_mask) { + std::ifstream mask_file(mask_file_path); + std::getline(mask_file, line); // ditch first line + while (std::getline(mask_file, line)) { + int int_ch = std::atoi(line.c_str()); + masked_channels.push_back(int_ch); + } + } + + /// do three runs of 100 samples each to have well defined pedestals + static const std::size_t n_events = 100; + + // tgt->setup_run(1, Target::DaqFormat::SIMPLEROC, 1); + // Use the DAQ format selected in the pftool DAQ->FORMAT menu so the + // format mode can be chosen interactively by the user. + tgt->setup_run(1, pftool::state.daq_format_mode, 1); + pflib_log(info) << "Using DAQ format mode: " + << static_cast(pftool::state.daq_format_mode); + + std::array target; + std::array baseline, highend, lowend; + + if (pftool::state.daq_format_mode == Target::DaqFormat::SIMPLEROC) { + pedestal_runs( + tgt, roc, baseline, highend, lowend, target, n_events, masked_channels); + + } else if (pftool::state.daq_format_mode == + Target::DaqFormat::ECOND_SW_HEADERS) { + pedestal_runs( + tgt, roc, baseline, highend, lowend, target, n_events, masked_channels); + + } else { + pflib_log(warn) << "Unsupported DAQ format (" + << static_cast(pftool::state.daq_format_mode) + << ") in level_pedestals. Skipping pedestal leveling..."; + } + + pflib_log(info) << "sample collections done, deducing settings"; + std::map> settings; + for (int ch{0}; ch < 72; ch++) { + std::string page{pflib::utility::string_format("CH_%d", ch)}; + + // // SKIP IF CHANNEL IS MASKED + // if (count(masked_channels.begin(), masked_channels.end(), ch) > 0) { + // pflib_log(info) << "Channel " << ch + // << " is masked; skipping pedestal leveling."; + + // // Explicitly mark it masked + // // settings[page]["MASKED"] = 1; + + // continue; + + // // } else { + // // settings[page]["MASKED"] = 0; + // } + + int i_link = ch / 36; + if (baseline.at(ch) < target.at(i_link)) { + pflib_log(debug) << "Channel " << ch + << " is below target, setting TRIM_INV"; + double scale = static_cast(target.at(i_link) - baseline.at(ch)) / + (highend.at(ch) - baseline.at(ch)); + if (scale < 0) { + pflib_log(warn) << "Channel " << ch + << " is below target but increasing TRIM_INV made it " + "lower??? Skipping..."; + continue; + } + if (scale > 1) { + pflib_log(warn) << "Channel " << ch + << " is so far below target that we cannot increase " + "TRIM_INV enough." + << " Setting TRIM_INV to its maximum."; + settings[page]["TRIM_INV"] = 63; + continue; + } + // scale is in [0,1] + double optim = scale * 63; + uint64_t val = static_cast(optim); + pflib_log(trace) << "Scale " << scale << " giving optimal value of " + << optim << " which rounds to " << val; + settings[page]["TRIM_INV"] = val; + } else { + double scale = static_cast(baseline.at(ch) - target.at(i_link)) / + (baseline.at(ch) - lowend.at(ch)); + if (scale < 0) { + pflib_log(warn) << "Channel " << ch + << " is above target but using SIGN_DAC=1 and " + "increasing DACB made it higher??? Skipping..."; + continue; + } + if (scale > 1) { + pflib_log(warn) + << "Channel " << ch + << " is so far above target that we cannot lower it enough." + << " Setting SIGN_DAC=1 and DACB to its maximum."; + settings[page]["SIGN_DAC"] = 1; + settings[page]["DACB"] = 31; + continue; + } + double optim = scale * 31; + uint64_t val = static_cast(optim); + pflib_log(trace) << "Scale " << scale << " giving optimal value of " + << optim << " which rounds to " << val; + if (val == 0) { + pflib_log(debug) + << "Channel " << ch + << " is above target but too close to use DACB to lower, skipping"; + } else { + pflib_log(debug) << "Channel " << ch + << " is above target, setting SIGN_DAC=1 and DACB"; + settings[page]["SIGN_DAC"] = 1; + settings[page]["DACB"] = val; + } + } + } + + return settings; +} + +} // namespace pflib::algorithm