diff --git a/src/dcs/CMakeLists.txt b/src/dcs/CMakeLists.txt index f2780d89..199af0c7 100644 --- a/src/dcs/CMakeLists.txt +++ b/src/dcs/CMakeLists.txt @@ -1,5 +1,5 @@ -add_library(dcs dcs.cc) +add_library(dcs dcs.cc sutron.cc) -add_executable(dcsdump dcsdump.cc) -add_sanitizers(dcsdump) -target_link_libraries(dcsdump lrit dcs m stdc++) +#add_executable(dcsdump dcsdump.cc) +#add_sanitizers(dcsdump) +#target_link_libraries(dcsdump lrit dcs m stdc++) diff --git a/src/dcs/DCPR_CS2_final_June09.pdf b/src/dcs/DCPR_CS2_final_June09.pdf new file mode 100644 index 00000000..106b5c0b Binary files /dev/null and b/src/dcs/DCPR_CS2_final_June09.pdf differ diff --git a/src/dcs/HRIT_DCS_File_Format_Rev1.pdf b/src/dcs/HRIT_DCS_File_Format_Rev1.pdf new file mode 100644 index 00000000..38db28d7 Binary files /dev/null and b/src/dcs/HRIT_DCS_File_Format_Rev1.pdf differ diff --git a/src/dcs/dcs.cc b/src/dcs/dcs.cc index 28335502..4155838d 100644 --- a/src/dcs/dcs.cc +++ b/src/dcs/dcs.cc @@ -1,168 +1,603 @@ #include "dcs.h" - #include #include +#include +#include -#include +#include "util/error.h" +#include "sutron.h" namespace dcs { int FileHeader::readFrom(const char* buf, size_t len) { - size_t nread = 0; + size_t nread = 0; + + // The DCS file header is 64 bytes - // 32 bytes hold the DCS file name (and trailing spaces) + // 3.1.1 FILE_NAME:=32 bytes { - constexpr unsigned n = 32; - ASSERT((len - nread) >= n); - name = std::string(buf + nread, n); - nread += n; + constexpr unsigned n = 32; + ASSERT((len - nread) >= n); + name = std::string(buf + nread, n); + nread += n; } - // 8 bytes holding length of payload + // 3.1.2 FIL_SIZE:= 8 bytes { - constexpr unsigned n = 8; - ASSERT((len - nread) >= n); - length = std::stoi(std::string(buf + nread, n)); - nread += n; + constexpr unsigned n = 8; + ASSERT((len - nread) >= n); + length = std::stoi(std::string(buf + nread, n)); + ASSERT(len == length); // Payload reported length does not equal what we actually received + nread += n; } - // 20 bytes holding some ASCII data + // 3.3.3 SOURCE:= 4 bytes { - constexpr unsigned n = 20; - ASSERT((len - nread) >= n); - misc1 = std::string(buf + nread, n); - nread += n; + constexpr unsigned n = 4; + ASSERT((len - nread) >= n); + source = std::string(buf + nread, n); + nread += n; } - // 8 bytes holding some binary data + // 3.1.4 TYPE:= 4 bytes ("DCSH") { - constexpr unsigned n = 8; - ASSERT((len - nread) >= n); - misc2.resize(n); - memcpy(misc2.data(), &buf[nread], n); - nread += n; + constexpr unsigned n = 4; + ASSERT((len - nread) >= n); + type = std::string(buf + nread, n); + nread += n; } - return nread; -} - -int Header::readFrom(const char* buf, size_t len) { - size_t nread = 0; - - // 8 hex digit DCP Address + // 3.1.4.1 EXP_FILL:= 12 bytes (EXP_FLD, EXP_FIELD) { - constexpr unsigned n = 8; - ASSERT((len - nread) >= 8); - auto rv = sscanf(buf + nread, "%08lx", &address); - ASSERT(rv == 1); - nread += n; + constexpr unsigned n = 12; + ASSERT((len - nread) >= n); + expansion = std::string(buf + nread, n); + nread += n; } - // YYDDDHHMMSS – Time the message arrived at the Wallops receive station. - // The day is represented as a three digit day of the year (julian day). + // 3.1.4.2 HDR_CRC32:= 4 bytes { - constexpr unsigned n = 11; - ASSERT((len - nread) >= n); - auto year = std::stoi(std::string(buf + nread, 2), nullptr, 10); - nread += 2; - auto day = std::stoi(std::string(buf + nread, 3), nullptr, 10); - nread += 3; - auto hour = std::stoi(std::string(buf + nread, 2), nullptr, 10); - nread += 2; - auto minute = std::stoi(std::string(buf + nread, 2), nullptr, 10); - nread += 2; - auto second = std::stoi(std::string(buf + nread, 2), nullptr, 10); - nread += 2; - - struct tm tm; - memset(&tm, 0, sizeof(tm)); - - // The number of years since 1900. - tm.tm_year = 100 + year; - - // Use offset at beginning of year and add parts - time.tv_sec = mktime(&tm) + ((((day * 24) + hour) * 60) + minute) * 60 + second; - time.tv_nsec = 0; - } - - // 1 character failure code - { - constexpr unsigned n = 1; - ASSERT((len - nread) >= n); - failure = buf[nread]; - nread += n; + constexpr unsigned n = 4; + ASSERT((len - nread) >= n); + memcpy(&headerCRC, &buf[nread], n); + nread += n; } - // 2 decimal digit signal strength + // 3.5 FILE_CRC32:= 4 bytes { - constexpr unsigned n = 2; - ASSERT((len - nread) >= n); - signalStrength = std::stoi(std::string(buf + nread, 2)); - nread += n; + constexpr unsigned n = 4; + ASSERT((len - nread) >= n); + memcpy(&fileCRC, &buf[length - n], n); // CRC is at the end of the DCS payload + nread += n; } - // 2 decimal digit frequency offset - { - constexpr unsigned n = 2; - ASSERT((len - nread) >= n); - // This can be +A or -A; don't know what it means... - if ((buf[nread] == '+' || buf[nread] == '-') && buf[nread+1] == 'A') { - frequencyOffset = 0; - } else { - frequencyOffset = std::stoi(std::string(buf + nread, n)); + + return nread; +} + +int DCPData::readFrom(const char* buf, size_t len) { + size_t nread = 64; // Start position of message block after file header + uint16_t pos = 0; + + + while (nread < len - 4) { // 4 for ending header CRC32 per spec + blocks.resize(pos + 1); + + // 1 byte contains blocks ID + { + constexpr unsigned n = 1; + ASSERT((len - nread) >= n); + memcpy(&blocks[pos].blockID, &buf[nread], n); + nread += n; + } + + // 2 bytes contain blocks length + { + constexpr unsigned n = 2; + ASSERT((len - nread) >= n); + memcpy(&blocks[pos].blockLength, &buf[nread], n); + nread += n; + } + if (blocks[pos].blockID != 1) + { + // Skip over Missed Messages Blocks (blockID == 2 per HRITS Doc 1/25/2019) + nread += blocks[pos].blockLength - 3; //Skip(header + 2(CRC16))-(ID+LNG) + // nread += 26; // Size of Missed Message Block (24(header) + 2(CRC16)) } - nread += n; + else + { + // Begin DCP Header section + + // 3 bytes contain sequence number + { + constexpr unsigned n = 3; + ASSERT((len - nread) >= n); + std::vector tmp; + tmp.resize(n); + memcpy(tmp.data(), &buf[nread], n); + blocks[pos].sequence = tmp.at(0) | (tmp.at(1) << 8) | (tmp.at(2) << 16); + + nread += n; + } + + // 1 byte contains message flags + { + constexpr unsigned n = 1; + ASSERT((len - nread) >= n); + uint8_t tmp; + memcpy(&tmp, &buf[nread], n); + blocks[pos].baudRate = toRate(tmp); + blocks[pos].platform = tmp & 0x8 >> 3; + blocks[pos].parityErrors = tmp & 0x10; + blocks[pos].missingEOT = tmp & 0x20; + + nread += n; + } + + // 1 byte contains abnormal message flags + { + constexpr unsigned n = 1; + ASSERT((len - nread) >= n); + uint8_t tmp; + memcpy(&tmp, &buf[nread], n); + blocks[pos].addrCorrected = tmp & 0x1; + blocks[pos].badAddr = tmp & 0x2; + blocks[pos].invalidAddr = tmp & 0x4; + blocks[pos].incompletePDT = tmp & 0x8; + blocks[pos].timingError = tmp & 0x10; + blocks[pos].unexpectedMessage = tmp & 0x20; + blocks[pos].wrongChannel = tmp & 0x40; + nread += n; + } + + // 4 bytes contain received platform (i.e. "ground station") address + { + constexpr unsigned n = 4; + ASSERT((len - nread) >= n); + memcpy(&blocks[pos].correctedAddr, &buf[nread], n); + nread += n; + } + + // 7 bytes contain carrier start time in BCD format. Time when signal energy received by space platform began. + { + constexpr unsigned n = 7; + ASSERT((len - nread) >= n); + std::vector tmp; + tmp.resize(n); + memcpy(tmp.data(), &buf[nread], n); + blocks[pos].carrierStart = toDateTime(tmp); + nread += n; + } + + // 7 bytes contain carrier end time in BCD format. Time when signal energy received by space platform stopped. + { + constexpr unsigned n = 7; + ASSERT((len - nread) >= n); + std::vector tmp; + tmp.resize(n); + memcpy(tmp.data(), &buf[nread], n); + blocks[pos].carrierEnd = toDateTime(tmp); + nread += n; + } + + // 2 bytes contain signal strength received at space platform (dBm EIRP) + { + constexpr unsigned n = 2; + ASSERT((len - nread) >= n); + uint16_t tmp; + memcpy(&tmp, &buf[nread], n); + blocks[pos].signalStrength = (float)(0x3ff & tmp) / 10.0; // Keep 10 LSBits and divide by 10 per data spec + nread += n; + } + + // 2 bytes contain received frequency offset from the expected carrier frequency + { + constexpr unsigned n = 2; + ASSERT((len - nread) >= n); + uint16_t tmp; + memcpy(&tmp, &buf[nread], n); + blocks[pos].freqOffset = (float)(int16_t)((tmp & 0x3fff) | (((tmp & 0x2000) << 2) | ((tmp & 0x2000) << 1))) / 10.0; // Keep 14 LSBits, add sign, and divide by 10 per data spec + nread += n; + } + + // 2 bytes contain received signal phase noise (degrees) and modulation index + { + constexpr unsigned n = 2; + ASSERT((len - nread) >= n); + uint16_t tmp; + memcpy(&tmp, &buf[nread], n); + blocks[pos].phaseNoise = (float)(tmp & 0xfff) / 100.0; // Keep 12 LSBits and divide by 100 per data spec + blocks[pos].phaseModQuality = toPhaseModQuality(tmp); // Keep 2 MSBits per data spec (00 = Unknown, 01 = Normal, 10 = High, 11 = Low (as of Jan 9, 2021)) + nread += n; + } + + // 1 byte contains a representation of the received signal quality (percentage) by the platform ('>=85% = GOOD', '70%<= <85% = FAIR', '<70% = POOR'). + { + constexpr unsigned n = 1; + ASSERT((len - nread) >= n); + uint8_t tmp; + memcpy(&tmp, &buf[nread], n); + blocks[pos].goodPhase = tmp / 2.0; + nread += n; + } + + // 2 bytes contain the space platform and channel number + { + constexpr unsigned n = 2; + ASSERT((len - nread) >= n); + uint16_t tmp; + memcpy(&tmp, &buf[nread], n); + blocks[pos].spacePlatform = toSpacePlatform(tmp); + blocks[pos].channelNumber = tmp & 0xFFF; + nread += n; + } + + // 2 bytes contain the source platform the message was received on. This is the not the client platform but instead the main "hub" platform. + // See spec document for a list of platforms. + { + constexpr unsigned n = 2; + ASSERT((len - nread) >= n); + std::string tmp; + tmp = std::string(buf + nread, n); + blocks[pos].sourcePlatform = tmp; + nread += n; + } + + // 2 bytes contain the secondary source (not used as of April 12, 2022) + { + constexpr unsigned n = 2; + ASSERT((len - nread) >= n); + std::string tmp; + tmp=""; // hack as there is nothing sent yet + tmp = std::string(buf + nread, n); + blocks[pos].sourceSecondary = tmp; + nread += n; + } + // End DCP Header section + + // Variables bytes contain the message data from the client ("ground station") + { + // Message data size. Total size subtract [DCP Header size (36(DCP Header) + 1(Block ID) + 2(Block Length) + 2(CRC16))] + uint16_t n = blocks[pos].blockLength - 41; + ASSERT((len - nread) >= n); + blocks[pos].DCPData.resize(n); + memcpy(blocks[pos].DCPData.data(), &buf[nread], n); + blocks[pos].DCPDataLength = n; + nread += n; + } + + // 2 bytes contain blocks CRC16 + { + constexpr unsigned n = 2; + ASSERT((len - nread) >= n); + memcpy(&blocks[pos].blockCRC, &buf[nread], n); + nread += n; + } + } + pos++; } + return nread; +} - // 1 character modulation index - { - constexpr unsigned n = 1; - ASSERT((len - nread) >= n); - modulationIndex = buf[nread]; - nread += n; +std::string toPhaseModQuality(const uint16_t& data) { + switch ((data & 0xc000) >> 14) { + case 0: return "Unknown"; + case 1: return "Normal"; + case 2: return "High"; + case 3: return "Low"; + default: return ""; + } +} +// Table 5 +std::string toSpacePlatform(const uint16_t& data) { + switch ((data & 0xF000) >> 12) { + case 0: return "Unknown"; + case 1: return "GOES-East"; + case 2: return "GOES-West"; + case 3: return "GOES-Central"; + default: return ""; } +} + +uint16_t toRate(const uint8_t& data) { + switch (data & 0x7) { + case 1: return 100; + case 2: return 300; + case 3: return 1200; + default: return 0; + } +} + +std::string toDateTime(const std::vector& data) { + + ASSERT(data.size() == 7); // Expect 7 bytes + + // YYDDDHHMMSSZZZ – The day is represented as a three digit day of the year (julian day) + auto year = (((data.at(6) & 0xF0) >> 4) * 10) + (data.at(6) & 0x0F); // Nibble math + auto day = (((data.at(5) & 0xF0) >> 4) * 100) + ((data.at(5) & 0x0F) * 10) + ((data.at(4) & 0xF0) >> 4); + auto hour = ((data.at(4) & 0x0F) * 10) + (((data.at(3) & 0xF0) >> 4)); + auto minute = ((data.at(3) & 0x0F) * 10) + ((data.at(2) & 0xF0) >> 4); + auto second = ((data.at(2) & 0x0F) * 10) + ((data.at(1) & 0xF0) >> 4); + auto subSecond = ((data.at(1) & 0x0F) * 100) + (((data.at(0) & 0xF0) >> 4) * 10) + (data.at(0) & 0x0F); + + struct tm tm; + struct timespec time; + memset(&tm, 0, sizeof(tm)); // Set values to zero + memset(&time, 0, sizeof(time)); + + // The number of years since 1900. + tm.tm_year = 100 + year; + + // Use offset at beginning of year and add parts + time.tv_sec = mktime(&tm) + ((((day * 24) + hour) * 60) + minute) * 60 + second; + time.tv_nsec = subSecond * 1000000; + + std::array buf; + memset(&buf, 0, sizeof(buf)); + auto len = strftime(buf.data(), buf.size(), "%c", gmtime(&time.tv_sec)); + + return std::string(buf.data(), len); +} + + std::vector formatData(const dcs::FileHeader &fh, const dcs::DCPData &dcp) { + std::vector buf; - // 1 character data quality indicator { - constexpr unsigned n = 1; - ASSERT((len - nread) >= n); - dataQuality = buf[nread]; - nread += n; + std::string tmp("----------[ DCS Header Info ]----------\nFilename: "); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + std::copy(fh.name.begin(), fh.name.end(), std::back_inserter(buf)); + buf.push_back('\n'); } - // 3 decimal digit GOES receive channel { - constexpr unsigned n = 3; - ASSERT((len - nread) >= n); - receiveChannel = std::stoi(std::string(buf + nread, n)); - nread += n; + std::string tmp = "File Size: " + std::to_string(fh.length); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); } - // 1 character GOES spacecraft indicator ('E' or 'W') { - constexpr unsigned n = 1; - ASSERT((len - nread) >= n); - spacecraft = buf[nread]; - nread += n; + std::string tmp("File Source: "); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + std::copy(fh.source.begin(), fh.source.end(), std::back_inserter(buf)); + buf.push_back('\n'); } - // 2 character data source code Data Source Code Table { - constexpr unsigned n = 2; - ASSERT((len - nread) >= n); - memcpy(dataSourceCode, buf + nread, n); - nread += n; + std::string tmp("File Type: "); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + std::copy(fh.type.begin(), fh.type.end(), std::back_inserter(buf)); + buf.push_back('\n'); } - // 5 decimal digit message data length + { - constexpr unsigned n = 5; - ASSERT((len - nread) >= n); - dataLength = std::stoi(std::string(buf + nread, n)); - nread += n; + { + std::string tmp="HDR_CRC32: " + to_string(fh.headerCRC); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + { + std::string tmp="FILE_CRC32: " + to_string(fh.fileCRC); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } } - return nread; + uint16_t pos = 0; + + while (pos < dcp.blocks.size()) { + + { + std::string tmp = "\n----------[ DCP Block ]----------\nHeader:\n\tBlock ID: " + std::to_string(dcp.blocks[pos].blockID); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + + { + std::string tmp = "\tBlock Length: " + std::to_string(dcp.blocks[pos].blockLength); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp = "\tSequence: " + std::to_string(dcp.blocks[pos].sequence); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp = "\tFlags:\n\t\tData Rate: " + std::to_string(dcp.blocks[pos].baudRate); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp = "\t\tPlatform: " + std::string(dcp.blocks[pos].platform ? "CS1" : "CS2"); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp = "\t\tParity Errors: " + std::string(dcp.blocks[pos].parityErrors ? "Yes" : "No"); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp = "\tARM Flag: "; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + } + + { + std::string tmp = std::string(dcp.blocks[pos].addrCorrected ? "A" : ""); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + } + + { + std::string tmp = std::string(dcp.blocks[pos].badAddr ? "B" : ""); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + } + + { + std::string tmp = std::string(dcp.blocks[pos].invalidAddr ? "I" : ""); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + } + + { + std::string tmp = std::string(dcp.blocks[pos].incompletePDT ? "N" : ""); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + } + + { + std::string tmp = std::string(dcp.blocks[pos].timingError ? "T" : ""); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + } + + { + std::string tmp = std::string(dcp.blocks[pos].unexpectedMessage ? "U" : ""); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + } + + { + std::string tmp = std::string(dcp.blocks[pos].wrongChannel ? "W" : "G"); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + } + + { + buf.push_back('\n'); + std::stringstream addr; + addr << std::hex << dcp.blocks[pos].correctedAddr; + std::string tmp = "\tCorrected Address: " + addr.str(); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp("\tCarrier Start (UTC): "); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + std::copy(dcp.blocks[pos].carrierStart.begin(), dcp.blocks[pos].carrierStart.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp("\tCarrier End (UTC): "); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + std::copy(dcp.blocks[pos].carrierEnd.begin(), dcp.blocks[pos].carrierEnd.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp = "\tSignal Strength(dBm EIRP): " + std::to_string(dcp.blocks[pos].signalStrength); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp = "\tFrequency Offset(Hz): " + std::to_string(dcp.blocks[pos].freqOffset); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp = "\tPhase Noise(Degrees RMS): " + std::to_string(dcp.blocks[pos].phaseNoise); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp("\tModulation Index: "); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + std::copy(dcp.blocks[pos].phaseModQuality.begin(), dcp.blocks[pos].phaseModQuality.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp = "\tGood Phase (%): " + std::to_string(dcp.blocks[pos].goodPhase); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp = "\tChannel: " + std::to_string(dcp.blocks[pos].channelNumber); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp("\tSpaceCraft: "); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + std::copy(dcp.blocks[pos].spacePlatform.begin(), dcp.blocks[pos].spacePlatform.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp("\tSource Code: "); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + std::copy(dcp.blocks[pos].sourcePlatform.begin(), dcp.blocks[pos].sourcePlatform.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + std::string tmp =( "\tSource Secondary: "); + std::copy(dcp.blocks[pos].sourceSecondary.begin(), dcp.blocks[pos].sourceSecondary.end(), std::back_inserter(buf)); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + { + + std::string tmp="BLOCK_CRC16: " + to_string(dcp.blocks[pos].blockCRC); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + buf.push_back('\n'); + + } + + { + // To output decoded ascii from Compaction, 6-bit packed ascii ("Pseudo-Binary") + std::string tmp("Data (Pseudo-Binary): \n"); + std::copy(tmp.begin(), tmp.end(), std::back_inserter(buf)); + std::string ascii, tmp2; + char unpacked[5]={0}; + for (unsigned j = 0; j < dcp.blocks[pos].DCPDataLength; j++) + { + // Read through 6-bit packed data + for(int i=0;i<4;++i) + { + unpacked[i]=dcp.blocks[pos].DCPData[j]>>(6*(3-i)) & 0x3F; // Divide into blocks of 6 + //Decode Compacted Pseudo Binary (25 % word size reduction). Compaction drops bits 6 & 7 + // Adding back bit 6 "0x40" where needed. ie any letter (bit 5 is 0) + if ( unpacked[i] < 0x20) + ascii=unpacked[i]+0x40; + else + ascii=unpacked[i]; + } + + std::copy(ascii.begin(), ascii.end(), std::back_inserter(tmp2)); + } + // From Appendix B DCS_CRS_final_June09.pdf. All data is Pseudo-Binary + // encoded. + // The first byte contains the format types of data. + // This word is used to decode the following data type structure. + // This is a possible 0-63, 64 different format types. + // Some are readable by humans. See https://github.com/opendcs for + // more info. + // + // Common types: Ascii, SHEF, Sutron Types: B,C,D .. 64 of them. + tmp2 = sutron(tmp2); // convert the known Sutron formats, to human + std::copy(tmp2.begin(), tmp2.end(), std::back_inserter(buf)); + buf.push_back('\n'); + } + + + + + buf.push_back('\n'); + pos++; + } + + buf.push_back('\n'); + return buf; } } // namespace dcs diff --git a/src/dcs/dcs.h b/src/dcs/dcs.h index 25998b54..9ea3ff31 100644 --- a/src/dcs/dcs.h +++ b/src/dcs/dcs.h @@ -1,3 +1,4 @@ +// From https://dcs1.noaa.gov/documents/HRIT%20DCS%20File%20Format%20Rev1.pdf #pragma once #include @@ -5,37 +6,127 @@ #include #include #include +#include namespace dcs { // Header at the beginning of an LRIT DCS file + +// 3.1 File Header 64 bytes struct FileHeader { + +// 3.1.1 FILE_NAME 32 bytes std::string name; + +// 3.1.2 FILE_SIZE:= 8 bytes uint32_t length; - // Don't know what these hold. - // First one looks ASCII and second one looks binary. - std::string misc1; - std::vector misc2; +// 3.1.3 SOURCE 4 bytes + std::string source; + +// 3.1.4 TYPE :="DCSH" 4 bytes + std::string type; + +// 3.1.4.1 EXP_FILL 12 bytes + std::string expansion; + +// 3.1.4.2 HDR_CRC32 4 bytes + uint32_t headerCRC; + +// 3.5 FILE_CRC32:= 4 Bytes + uint32_t fileCRC; int readFrom(const char* buf, size_t len); }; -// Header of every single DCS payload -struct Header { - uint64_t address; - struct timespec time; - char failure; - int signalStrength; - int frequencyOffset; - char modulationIndex; - char dataQuality; - int receiveChannel; - char spacecraft; - char dataSourceCode[2]; - int dataLength; +struct DCPData { + // Keep information in the provided spec format as much as possible. Formatting will be done in formatData for output + + struct blockData { + +// 3.2.1 BLK_ID 1 byte + uint8_t blockID; + +//3.2.2 BLK_LNG 2 bytes + uint16_t blockLength; + +// 3.3.1 Table + uint32_t sequence; + +// 3.3.1.1 Message Flags/Baud +// Parity errors used to define 0-ASCII, 1-Pseudo-Binary + uint16_t baudRate; + uint8_t platform; + bool parityErrors; + bool missingEOT; + bool msgflagb6; + bool msgflagb7; + +// 3.3.1.2 Message ARM Flag + bool addrCorrected; + bool badAddr; + bool invalidAddr; + bool incompletePDT; + bool timingError; + bool unexpectedMessage; + bool wrongChannel; + bool armflagb7; + +// 3.3.1.3 Corrected Address + uint32_t correctedAddr; + +// 3.3.1.4 Carrier Start + std::string carrierStart; + +// 3.3.1.5 Carrier End + std::string carrierEnd; +// 3.3.1.6 Signal Strength X10 + + float signalStrength; + +// 3.3.1.7 Frequency Offset X10 + float freqOffset; + +// 3.3.1.8 Phase Noise X100 and Modulation Index + float phaseNoise; + std::string phaseModQuality; + +// 3.3.1.9 Good Phase X2 + float goodPhase; + +// 3.3.1.10 Channel/Spacecraft + std::string spacePlatform; + uint16_t channelNumber; + +// 3.3.1.11 Source Code= 2 bytes + std::string sourcePlatform; + +// 3.3.1.2 Source Secondary:= 2 bytes + std::string sourceSecondary; + +// 3.3.2 DCP_DATA:=0.. + std::vector DCPData; + +// 3.2.2 BLK_LNG:=2 bytes + uint16_t DCPDataLength; + +// 3.2.3 BLK_DATA:=0..65,530 bytes + +// 3.2.4 BLK_CRC16:=2 bytes + uint16_t blockCRC; + }; + + std::unordered_map spMap; + std::vector blocks; int readFrom(const char* buf, size_t len); }; +void initMap(std::unordered_map &data); +std::string toPhaseModQuality(const uint16_t& data); +std::string toSpacePlatform(const uint16_t& data); +uint16_t toRate(const uint8_t& data); +std::string toDateTime(const std::vector& data); +std::vector formatData(const FileHeader &fh, const DCPData &dcp); + } // namespace dcs diff --git a/src/dcs/dcsdump.cc b/src/dcs/dcsdump.cc index aad70485..7d693f5d 100644 --- a/src/dcs/dcsdump.cc +++ b/src/dcs/dcsdump.cc @@ -1,3 +1,6 @@ +// January 16, 2021: "Deprecated" Needs to be modified to work with latest DCS algorithm implementation + + #include #include @@ -38,13 +41,13 @@ int main(int argc, char** argv) { while (nread < nbytes) { // Read DCS header - dcs::Header h; + dcs::FileHeader h; rv = h.readFrom(buf.get() + nread, nbytes - nread); ASSERT(rv > 0); nread += rv; // Skip over actual data - nread += h.dataLength; + nread += h.length; // Skip 14 characters for time with milliseconds (in DCS format). // Skip 1 whitepace. diff --git a/src/dcs/packedascii.cc b/src/dcs/packedascii.cc new file mode 100644 index 00000000..c5620de7 --- /dev/null +++ b/src/dcs/packedascii.cc @@ -0,0 +1,17 @@ +/*HART makes limited use of data compression in the form of Packed ASCII. Normally, there are 256 possible ASCII characters, so that a full byte is needed to represent a character. Packed ASCII is a subset of full ASCII and uses only 64 of the 256 possible characters. These 64 characters are the capitalized alphabet, numbers 0 through 9, and a few punctuation marks. Many HART parameters need only this limited ASCII set, which means that data can be compressed to 3/4 of normal. This improves transmission speed, especially if the textual parameter being communicated is a large one. + +Since only full bytes can be transmitted, the 3/4 compression is fully realized only when the number of uncompressed bytes is a multiple of 4. Any fractional part requires a whole byte. Thus, if U is the number of uncompressed bytes, and T the number of transmitted bytes; find T = (3*U)/4 and increase any fractional part to 1. As examples, U = 3, 7, 8, and 9 result in T = 3, 6, 6, and 7. + +The rule for converting from ASCII to Packed ASCII is just to remove bits 6 and 7 (two most significant). An example is the character "M". The full binary code to represent this is 0100,1101. The packed binary code is 00,1101. The rules for conversion from packed ASCII back to ASCII are (1) set bit 7 = 0 and (2) set bit 6 = complement of packed ASCII bit 5. +*/ +unsigned long packed; +read(device,&packed,3); +char unpacked[5]={0}; + +for(int i=0;i<4;++i) +{ + unpacked[i]=packed>>(6*(3-i)) & 0x3F; // Divide into blocks of 6 + char bit6 = (~unpacked[i] & 0x10) << 1; // set bit 6 = complement of packed ASCII bit 5. + unpacked[i]|=bit6; // bit 6 is set, bit 7 is set to 0 from the start. +} + diff --git a/src/dcs/sutron.cc b/src/dcs/sutron.cc new file mode 100644 index 00000000..8567800a --- /dev/null +++ b/src/dcs/sutron.cc @@ -0,0 +1,385 @@ +#include "sutron.h" + +std::vector split_delim(const std::string &s, char delim) +{ + std::stringstream ss(s); + std::string item; + std::vector elems; + while (std::getline(ss, item, delim)) + { + // elems.push_back(item); + elems.push_back(std::move(item)); // if C++11 (based on comment from @mchiasson) + } + return elems; +} + +constexpr size_t Hash(const char* str) +{ + const long long p = 131; + const long long m = 4294967291; // 2^32 - 5, largest 32 bit prime + long long total = 0; + long long current_multiplier = 1; + for (int i = 0; str[i] != '\0'; ++i) + { + total = (total + current_multiplier * str[i]) % m; + current_multiplier = (current_multiplier * p) % m; + } + return total; +} + +std::string sutron(std::string inbuf) +{ + std::string outbuf; + std::string key1(":YB "); + std::string key2(":BL "); + std::string key3(" "); + char coda[40]; + int battery; + std::size_t foundYB=inbuf.find(key1), foundBL=inbuf.find(key2), foundSP=inbuf.rfind(key3), len; +// battery is set to 1, to display bat: xx.xx Voltages. Else "0" don't show them, or are not present. + battery=1; + coda[0]=' '; + coda[1]='\0'; + // cout< len) + r=0; + else + { + r= (inbuf[i+2] - 64)&0x3F; // right of 18 bit percision + } + + if (i+1 > len) + m=0 <<6; + else + m= ((inbuf[i+1] - 64)&0x3F) <<6; // middle of 18 bit percision + + l= ((inbuf[i] - 64)&0x3F) <<12; // left of 18 bit percision + if ( (inbuf[i] & 0x20) != 0) + sign = -1; // negative + ival= l + m + r; + if (sign == -1) + ival=ival*sign+0x20000; // negative value + tmp=to_string(ival) + " "; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + } + bat=0; + bat=(inbuf[len]-64)*.234 + 10.6; + if ( bat < 0) + battery = 0; + if ( battery == 1) + + { + std::string s; + s=coda; + //tmp="opt: " + s + " bat: " + to_string(bat) + " Volts\n"; + tmp="opt: " + s + " bat: " + to_string( round(bat * 10)/10).substr(0,4) + " Volts\n"; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + } + else + { + std::string s; + s=coda; + tmp="opt: " + s + "\n"; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + } + + } + break; + + case Hash("C1"): + // case Hash("C2"): + // case Hash("C3"): + // case Hash("C4"): + { + std::vector word = split_delim(inbuf, '.'); + + std::string tmp(""); + std::string tmp2(""); + float bat; + long int ival,l,m,r; + int lenbuf=inbuf.length(); + len=word[0].length()+1; + + if (lenbuf -len > 0 ) + { + std::string s; + s=coda; + bat=0; + bat= (inbuf[len] -64)*.234 + 10.6; + tmp="opt: " + s + " bat: " + to_string( round(bat * 10)/10).substr(0,4) + " Volts\n"; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(tmp2)); + } + std::vector pword = split_delim(word[0], '+'); + if ( pword[0].empty()) + return inbuf; // see if it is really a C type, as it is required to have + + + std::string c(1,inbuf[2]); // convert 1 char group-id to string + tmp="block-id: C group-id: " + c + " "; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + std::string sensor; + std::string smeas, sj_day, stmin_in_day, stmin_int; + long s,jday,tday,tint; + //int j=1; // skip the "C1: piece + for (std::size_t j=1; j < pword.size(); j++) + { + std::string c(1,inbuf[2]); // convert 1 char group-id to string + std::string data(""); + sensor=pword[j]; + len=sensor.length(); + + for(std::size_t i=7;i len) + r=0; + else + { + r= (sensor[i+2] - 64)&0x3F; // right of 18 bit percision + } + + if (i+1 > len) + m=0 <<6; + else + { + m= ((sensor[i+1] - 64)&0x3F) <<6; // middle of 18 bit percision + } + + if (i > len) + l=0; + else + { + l= ((sensor[i] - 64)&0x3F) <<12; // left of 18 bit percision + } + if ( (inbuf[i] & 0x20) != 0) + sign = -1; + + ival= l + m + r; + if ( sign == -1) + ival=ival*sign+0x20000; //negative value + tmp=to_string(ival) + " "; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(data)); + } + + s=(sensor[0]-64)&0x3F; + smeas=to_string(s); + + l=((sensor[1]-64)&0x3F)<<6 ; + r=(sensor[2]-64)&0x3F; + jday=l+r; + sj_day=to_string(jday); + + l=((sensor[3]-64)&0x3F)<<6 ; + r=(sensor[4]-64)&0x3F; + tday=l+r; + stmin_in_day=to_string(tday); + + l=((sensor[5]-64)&0x3F)<<6 ; + r=(sensor[6]-64)&0x3F; + tint=l+r; + stmin_int=to_string(tint); + + tmp=" M" + smeas + ": " + "Julian Day: " + sj_day + + " Time Since Midnight(min): " + stmin_in_day + " Interval(min): " + + stmin_int + " data(x10^-2): " + data + " "; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + } + + std::copy(tmp2.begin(), tmp2.end(), std::back_inserter(outbuf)); + } + + break; + + case Hash("D1"): + case Hash("D2"): + case Hash("D3"): + case Hash("D4"): + { + std::string tmp(""); + tmp="block-id: D "; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + + long int ival,l,m,r; + float bat; + + std::string c(1,inbuf[2]); // convert 1 char to string + tmp="group-id: " + c + " "; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + +// -------------------> put in the jday and time mins <--------------- + r= (inbuf[4] - 64)&0x3F; // low byte jday + l= ((inbuf[3] - 64)&0x3F) <<6; // high byte jday + ival=l+r; + tmp=" Julian Day: " + to_string(ival) + " "; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + + r= (inbuf[6] - 64)&0x3F; // low byte mins from midnight + l= ((inbuf[5] - 64)&0x3F) <<6; // high byte mins from midnight + ival=l+r; + tmp=" Time Since Midnight(mins): " + to_string(ival) + " "; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + tmp=" data(x10^-2): "; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + +// --------------------------------- + for(std::size_t i=7;i len) + r=0; + else + { + r= (inbuf[i+2] - 64)&0x3F; // right of 18 bit percision + } + + if (i+1 > len) + m=0 <<6; + else + m= ((inbuf[i+1] - 64)&0x3F) <<6; // middle of 18 bit percision + + l= ((inbuf[i] - 64)&0x3F) <<12; // left of 18 bit percision + if ( (inbuf[i] & 0x20) != 0) + sign = -1; + ival= l + m + r; + if (sign == -1) + ival=ival*sign+0x20000; //negative value + tmp=to_string(ival) + " "; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + } + if ( inbuf[len] < 64) + battery=0; + bat=0; + bat=(inbuf[len]-64)*.234 + 10.6; + if ( bat < 0) + battery = 0; + if ( battery == 1) + + { + std::string s; + s=coda; + //tmp="opt: " + s + " bat: " + to_string(bat) + " Volts\n"; + tmp="opt: " + s + " bat: " + to_string( round(bat * 10)/10).substr(0,4) + " Volts\n"; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + } + else + { + std::string s; + s=coda; + tmp="opt: " + s + "\n"; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + } + + } + break; + + case Hash(" :"): + { + std::string tmp(""); + + tmp="block-id: SHEF\n"+ inbuf; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + } + break; + + case Hash("MJ"): + { + std::string tmp(""); + + inbuf=std::regex_replace(inbuf,std::regex("MJ"), "\r\n"); + tmp="block-id: ASCII\n" + inbuf; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + } + break; + + default: + { + std::string tmp(""); + tmp=inbuf; + std::copy(tmp.begin(), tmp.end(), std::back_inserter(outbuf)); + } + } // end switch() + + return outbuf; +} + diff --git a/src/dcs/sutron.h b/src/dcs/sutron.h new file mode 100644 index 00000000..484ee84a --- /dev/null +++ b/src/dcs/sutron.h @@ -0,0 +1,12 @@ +#include // for std::cout, stdio +using namespace std; // for standard io +#include // for rfind or find +#include // for std::regex_replace +#include // for round() +#include // for std::istringstream +#include // for std::vector + + +constexpr size_t Hash(const char* str); +std::vector split(const std::string &s, char delim); +std::string sutron(std::string obj); diff --git a/src/dcs/sutronTest.cc b/src/dcs/sutronTest.cc new file mode 100644 index 00000000..6389b749 --- /dev/null +++ b/src/dcs/sutronTest.cc @@ -0,0 +1,52 @@ +#include "sutron.h" +// Main function +int +main () +{ +// std::string data0(" B1@@Gt@Gs@Sx@Sr@@i@@iL"); +// std::string data0("\"BN^D<1@XXF5HBW BN_D< @XVF% B(?BNVD;9@XUF((B*\\BN\\D;Y@XRGI\\B);BN^D;Q@XPG\\LB&9BN#D:9@XMG\\LB$OBN%D:Q@XJGZ(B\"KBN?D95@XEG(,B %CV DO\\DTICVSDO[DTP"); +// std::string data0(" B\\A@AO@C?@DB@NX@NN@@I@@N@@#@^(@],@^@@@^@@^@@@@@@@A9@@JE"); +// std::string data0(" B[#@A\"@DD@DG@O@@N6@@B@@B@@G@7(@I,@[4@@C@@K@@@@@@@B@@@JH"); +// std::string data0(" B6E#@L@1DNI;@.(@@CAU,BLDK@H@'DPI0@.(@@CAU,BBD-@N@"); + std::string data0(" B*B@KX@KW@KY@KW@'&@&'@']@',@A<@A>@A?@A?@@A@@A@@A@@A@I.@I1@I1@I1@BBH"); + // std::string data0("\"B1$I/TLHG@A$@A$//////BF,BF,BF,BF,BF,BF,L(HL(H?GU?GS@0;@1^@1Y@03@1H@0F@2J@2O@0P@.#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@AH@A.@A\\@B-@BG@BY@A<@B8@CM@C-@C+@EA@AH@AB@@3@A)EV4E;XFR:E,&EA\\EO,C;/D0R@U @U!L"); +// std::string data1("\"B1@@Gt@Gs@Sx@Sr@@i@@iL"); +// std::string data1("\"B2@@Gt@Gs@Sx@Sr@@i@@iL"); + std::string data1("\"B1O@!&@!&@!'@!(@!'@!'@!'@!&@!&@!(@!&@!%@!&@!'@!&@!'JAH@@"); + std::string data2("\"C1+AA9L[@O@PA@PA@PA@PA@PA@P@@P@@P@.H 2102527 "); +// std::string data2("\"C1+ABeHq@A@E|@FG@FM+BBeHq@A@@O.K"); // look for "C1+. the terminator . is important +// std::string data2("\"C1+AA,T(@O @@ @@ @@ @@+BA,T(@O @@ @@ @@ @@+CA,T(@O @@ @@ @@ @@+DA,T(@O @@ @@ @@ @@+EA,T(@O @@ @@ @@ @@+FA,T(@O @@ @@ @@ @@+GA,T(@O @@ @@ @@ @@+HA,T(@O@T,@U6@U6@U6+IA,T(@O @@ @@ @@ @@."); +// std::string data2("\"C1+AA,TY@O@PL@PL@PL@PL@PL@PL@PL@PL.J 2102527"); +// std::string data2("\"C1+AA6O^@O@O?@O?@O?@O?@P@@O?@O?@O?.I 2102527"); +// std::string data2("\"C1+AA9U$@O@PB@PB@PB@PB@PB@PB@PB@PB.J 2102527 "); +// std::string data2(" C1+AA;BG@O@PA@PA@PA@PA@PA@PA@PA@PB.I2102527 "); +// std::string data2(" C3D*@YAZD^G\"@.:@@@AVWD3D=@OAGDXG=@.:@@@AVWCRDV@ZARDWG1@.:@@@AVWCYE@@TAPDOHT@.:@@@AVWCQE @\"AMDPH^@.:@@@AVWD/C4@M@5DPH$@.:@@@AVXDOD0@PAKDMH-@.:@@@AVXDGD*@M@2DGIR@.9@@@AVXC E$@YATDAI%@.:@@@AVYCTD]@\\A9C;I1@.:@@@AVYB>D/@TA\"C8JU@.:@@@AVYA=AC@XAHC5J*@.:@@@AVYBI"); +// std::string data2(" C4ES@R@&CRL/@.2@@[??PC+D&@\\@+CSL.@.2@@[??JC\"D@@^@3CTL-@.2@@[??FC&D'@_@4CTL+@.2@@[??AC'EH@\\@6CUL)@.2@@[?>;C)DP@^@9CVL(@.2@@[?>6C+D0@#@1C)DI@(AKCWL&@.2@@[?>,C(D @2ARCXL&@.2@@[?>)C,D_@1A\\CXL$@.3@@[?>'C%DN@*ACCYL\"@.3@@[?>$C*DJ@^AFCZL^@.3@@[?>!A?"); +// std::string data2("\"CS#!0101XX-@PM1457N&_&@D@\\@P@R@#@B@B@A@A-@0FPBVD,D+-(5?.F"); +// std::string data2("\"CS#!0101XX-@PM1457N&_&@D@\\@P@R@#@B@B@A@A-@0FPBVD,D+-(5?.@V@NNHYU@B@@@@@@@@ANC(M-@@@@@@@@AQB1ARB1@L@L@@@@BSC?@@A9+A@QZ@@W@U@IA;BG+B@TJ@@@@S@IBRBS+C@U9@@@@T@IBLBL+D@XK@@\"@V@JBFBF+E@W3@@=@V@JBBBB+F@V^@A!@X@KA>A>+G@ZG@A[@X@JA:A7+H@Z9@A0@W@JA5A/+I@[9@BP@W@KA1A)+J@];@BO@X@JA.A$+K@^2@BU@V@IA+A +L@^*@BI@U@JA+A^+M@^5@A6@Y@JA+A]+N@_7@BA@W@JA)A[+O@ Y@A=@T@IA&AZ+P@^=@A2@V@KA$AW+Q@]0@@[@[@NA#AU+R@\\E??O@ @NA#AV+S@X:?=6@ @NA%AX+T@XE?=U@_@OA%AY+U@T7?;.@ @MA(AX+V@Q\"@W@JA&AJ+!@C&?>M@T@IA+AK+\"@EH?=-@T@IA-AE+#@C1?>N@S@IA&AI+$@DM?>J@T@IA*AE+%@B8?>6@R@HA'AG+&@AM??_@P@HA_AJ+'@B[?>/@U@IA AK+(@@=??_@R@HA A.+)@@5??<@P@HAWBD+*@@%@@M@O@GAWBI++@@S@@F@P@GA!BG+,??9@@N@Q@GA#A8+-@@G@@@@Q@HA)A@+.??-??2@Q@GA.@3+/@@S??-@P@GA*@.+0@@V??/@Q@HA#@*"); +// std::string data3("\"D1D~A8@NI@NH@NG@NF@NE@DGF"); +// std::string data3("\"D1A,TT@GK@GK@GK@GK@GK@GK@GK@GL@GL@GK@GK@GK@@@@@@@@@@@@M"); +// std::string data3("\"D1A,T>@A5@A5@A5@A5@A5@A5@A5@A5@A5@A5@A5@A5@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@A0@A0@A0@A0@A0@A0@A0@A0@C?@C>@C=@C<@C;@C:@C9@C7J"); +// std::string data3("\"D1A,UBAR.AT1AV3AX3AZ-A\"A^TA_?A!\A#C@AY@ID@@@@U\""); +// std::string data3(" D7.E&\\@YRF@@B ZD65E'I@YHF@@B LD5"); + std::string data3("\"DOO@@@@$LK"); +// std::string data3(" D8IE,"); +// std::string data4(" :HG 1 #5 0.077 0.077 0.077 0.077 0.077 0.077 0.077 0.077 0.077 0.077 0.077 0.077 :VB 36 #60 13.6"); +// std::string data5("\"MJ0.41 12.48 442.60 13.13MJ0.41 12.52 442.60 13.21"); + std::string data5(" MJ100 MJ099 MJ100 MJ098 MJ100 MJ00.5 MJ00.5 MJ01.1 MJ00.5 MJ01.9 MJ028.5 MJ023 MJ017.5 MJ019 MJ001.0 MJ003.3 MJ05659.9 MJ14.2 MJ13.2 MJ14.3 MJ0.0 MJ20.3 MJ0.0 MJ"); + std::string data6("\":HG 7 #15 2.94 2.96 2.96 2.96 2.96 2.96 2.95 2.92 :BL 14.08"); + std::string data7("\"P85452401A1A>K@@^0T6V1AJ=@E@AFAH>AJZ4AM5BB6\"D // for boost::crc_basic, boost::crc_optimal +#include // for boost::uint_t +#include // for assert +#include // for std::size_t +#include // for std::cout +#include // for std::endl +// Main function +int +main () +{ + // This is "123456789" in ASCII + unsigned char const data[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39 }; + std::size_t const data_len = sizeof( data ) / sizeof( data[0] ); + + // The expected CRC for the given data + boost::uint16_t const expected = 0x29B1; + + // Simulate CRC-CCITT + boost::crc_basic<16> crc_ccitt1( 0x1021, 0xFFFF, 0, false, false ); + crc_ccitt1.process_bytes( data, data_len ); + assert( crc_ccitt1.checksum() == expected ); + + // Repeat with the optimal version (assuming a 16-bit type exists) + boost::crc_optimal<16, 0x1021, 0xFFFF, 0, false, false> crc_ccitt2; + crc_ccitt2 = std::for_each( data, data + data_len, crc_ccitt2 ); + assert( crc_ccitt2() == expected ); + + std::cout << "All tests passed." << std::endl; + return 0; +} diff --git a/src/dcs/tester-crc32.cc b/src/dcs/tester-crc32.cc new file mode 100644 index 00000000..234ebe56 --- /dev/null +++ b/src/dcs/tester-crc32.cc @@ -0,0 +1,63 @@ +#include // for boost::crc_32_type +#include // for EXIT_SUCCESS, EXIT_FAILURE +#include // for std::exception +#include // for std::ifstream +#include // for std::ios_base, etc. +#include // for std::cerr, std::cout +#include // for std::endl + + +// Redefine this to change to processing buffer size +#ifndef PRIVATE_BUFFER_SIZE +#define PRIVATE_BUFFER_SIZE 1024 +#endif + +// Global objects +std::streamsize const buffer_size = PRIVATE_BUFFER_SIZE; + + +// Main program +int +main +( + int argc, + char const * argv[] +) +try +{ + boost::crc_32_type result; + + for ( int i = 1 ; i < argc ; ++i ) + { + std::ifstream ifs( argv[i], std::ios_base::binary ); + + if ( ifs ) + { + do + { + char buffer[ buffer_size ]; + + ifs.read( buffer, buffer_size ); + result.process_bytes( buffer, ifs.gcount() ); + } while ( ifs ); + } + else + { + std::cerr << "Failed to open file '" << argv[i] << "'." + << std::endl; + } + } + + std::cout << std::hex << std::uppercase << result.checksum() << std::endl; + return EXIT_SUCCESS; +} +catch ( std::exception &e ) +{ + std::cerr << "Found an exception with '" << e.what() << "'." << std::endl; + return EXIT_FAILURE; +} +catch ( ... ) +{ + std::cerr << "Found an unknown exception." << std::endl; + return EXIT_FAILURE; +} diff --git a/src/goesproc/CMakeLists.txt b/src/goesproc/CMakeLists.txt index 8778c547..cc88e59c 100644 --- a/src/goesproc/CMakeLists.txt +++ b/src/goesproc/CMakeLists.txt @@ -10,6 +10,7 @@ set(GOESPROC_SRCS handler_goesr.cc handler_himawari8.cc handler_nws_image.cc + handler_dcs.cc handler_nws_text.cc handler_text.cc image.cc @@ -36,7 +37,7 @@ endif() add_executable(goesproc ${GOESPROC_SRCS}) add_sanitizers(goesproc) install(TARGETS goesproc COMPONENT goestools RUNTIME DESTINATION bin) -target_link_libraries(goesproc lrit util assembler packet_reader dir) +target_link_libraries(goesproc dcs lrit util assembler packet_reader dir) target_link_libraries(goesproc zip) target_link_libraries(goesproc nlohmann_json) target_link_libraries(goesproc timer) diff --git a/src/goesproc/goesproc.cc b/src/goesproc/goesproc.cc index a0cc72eb..5a3780aa 100644 --- a/src/goesproc/goesproc.cc +++ b/src/goesproc/goesproc.cc @@ -17,6 +17,7 @@ #include "handler_goesr.h" #include "handler_himawari8.h" #include "handler_nws_image.h" +#include "handler_dcs.h" #include "handler_nws_text.h" #include "handler_text.h" #include "options.h" @@ -87,7 +88,9 @@ int main(int argc, char** argv) { std::unique_ptr( new EMWINHandler(handler, fileWriter))); } else if (handler.type == "dcs") { - // TODO + handlers.push_back( + std::unique_ptr( + new DCSTextHandler(handler, fileWriter))); } else if (handler.type == "text") { const auto& origin = handler.origin; if (origin == "nws") { diff --git a/src/goesproc/handler_dcs.cc b/src/goesproc/handler_dcs.cc new file mode 100644 index 00000000..dc9ed5f1 --- /dev/null +++ b/src/goesproc/handler_dcs.cc @@ -0,0 +1,68 @@ +#include "handler_dcs.h" + +#include +#include + +#include "filename.h" +#include "string.h" +#include "time.h" + +DCSTextHandler::DCSTextHandler( + const Config::Handler& config, + const std::shared_ptr& fileWriter) + : config_(config), + fileWriter_(fileWriter) { +} + +void DCSTextHandler::handle(std::shared_ptr f) { + auto ph = f->getHeader(); + + // Filter DCS + if (ph.fileType != 130) { + return; + } + + // Filter DCS again if needed + auto nlh = f->getHeader(); + if (nlh.productID != 8) { + return; + } + + auto buf = f->read(); + const char* dcsData = buf.data(); + + int rv; + + // Read DCS file header (container for multiple DCS payloads) + dcs::FileHeader fh; + rv = fh.readFrom(dcsData, buf.size()); + + // Only process DCS data types. As of January 11, 2021 this appears to be the only type of data available. + if (fh.type != "DCSH") { + return; + } + + dcs::DCPData dcp; + rv = dcp.readFrom(dcsData, fh.length); + + struct timespec time = {0, 0}; + + rv = clock_gettime(CLOCK_REALTIME, &time); + ASSERT(rv >= 0); + + // Skip if the time could not be determined + if (time.tv_sec == 0) { + return; + } + + FilenameBuilder fb; + fb.dir = config_.dir; + fb.filename = removeSuffix(f->getHeader().text); + fb.time = time; + auto path = fb.build(config_.filename, "txt"); + const std::vector fd = dcs::formatData(fh, dcp); // Format data to output to file + fileWriter_->write(path, fd); + if (config_.json) { + fileWriter_->writeHeader(*f, path); + } +} diff --git a/src/goesproc/handler_dcs.h b/src/goesproc/handler_dcs.h new file mode 100644 index 00000000..0b9e2d16 --- /dev/null +++ b/src/goesproc/handler_dcs.h @@ -0,0 +1,21 @@ +#pragma once + +#include "config.h" +#include "file_writer.h" +#include "handler.h" +#include "types.h" + +#include "dcs/dcs.h" + +class DCSTextHandler : public Handler { +public: + explicit DCSTextHandler( + const Config::Handler& config, + const std::shared_ptr& fileWriter); + + virtual void handle(std::shared_ptr f); + +protected: + Config::Handler config_; + std::shared_ptr fileWriter_; +}; diff --git a/src/goesproc/map_drawer.cc b/src/goesproc/map_drawer.cc index 2ab1bcb3..13679baf 100644 --- a/src/goesproc/map_drawer.cc +++ b/src/goesproc/map_drawer.cc @@ -10,6 +10,8 @@ Proj longitudeToProj(float longitude) { args["h"] = "35786023.0"; args["lon_0"] = std::to_string(longitude); args["sweep"] = "x"; +// goestools-proj-compat.patch + args["ellps"] = "WGS84"; return Proj(args); }