diff --git a/CMakeLists.txt b/CMakeLists.txt index 89c0598e..959ed277 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -301,6 +301,68 @@ build_lua() if (NOT WITHOUT_QEMU) + # TODO(Jhieb) pull request changes to make sure capabilities are DWORD aligned. + CPMAddPackage( + NAME pcie_model + GIT_REPOSITORY "https://github.com/joshwhieb/pcie-model.git" + GIT_TAG "master" + GIT_SHALLOW TRUE + ) + + + set(LIBPCIE_DIR ${pcie_model_SOURCE_DIR}/libpcie) + set(LIBPCIE_C_SOURCES + ${LIBPCIE_DIR}/src/pcie/cosim_common.c + ${LIBPCIE_DIR}/src/pcie/cosim_socket.c + ${LIBPCIE_DIR}/src/pcie/cosim_tlp.c + ${LIBPCIE_DIR}/src/pcie/pcie.c + ${LIBPCIE_DIR}/src/pcie/pseudocore.c + ${LIBPCIE_DIR}/src/pcie/platform.c + ${LIBPCIE_DIR}/src/pcie/pcie_cfgspc.c + ${LIBPCIE_DIR}/src/pcie/pcie_cfgutil.c + ${LIBPCIE_DIR}/src/pcie/pcie_msix.c + ) + + # C++ sources from the TLM modules + set(LIBPCIE_CXX_SOURCES + ${pcie_model_SOURCE_DIR}/tlm-modules/pcie-controller.cc + ${pcie_model_SOURCE_DIR}/tlm-modules/libpcie-callbacks.cc + ) + + # Create the static library + add_library(libpcie STATIC + ${LIBPCIE_C_SOURCES} + ${LIBPCIE_CXX_SOURCES} + ) + + # Set the output name to libpcie.a (without the extra 'lib' prefix) + set_target_properties(libpcie PROPERTIES OUTPUT_NAME pcie) + + # Add compile definitions + target_compile_definitions(libpcie PRIVATE + __STDC_WANT_LIB_EXT2__=1 + CONFIG_TLM=1 + ) + + # Set include directories + target_include_directories(libpcie PUBLIC + ${LIBPCIE_DIR}/src + ${pcie_model_SOURCE_DIR} + ${systemclanguage_SOURCE_DIR}/src + ) + + # Set compile options (matching the original Makefile) + target_compile_options(libpcie PRIVATE + -fPIC + $<$:-g3 -O0> + $<$:-O2> + ) + + # Link against SystemC + target_link_libraries(libpcie PUBLIC + SystemC::systemc + ) + CPMAddPackage( NAME libslirp GIT_REPOSITORY https://gitlab.freedesktop.org/slirp/libslirp.git @@ -436,6 +498,8 @@ target_include_directories( $ $ ${LIBELF_INCLUDE_DIR} + ${pcie_model_SOURCE_DIR} + ${pcie_model_SOURCE_DIR}/libpcie/src ) target_link_libraries(${PROJECT_NAME} PUBLIC @@ -446,7 +510,8 @@ target_link_libraries(${PROJECT_NAME} PUBLIC ${CROW_DEP} zip ${PYBIND11_EMBED} - ${CMAKE_DL_LIBS} + ${CMAKE_DL_LIBS} + libpcie ) install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/systemc-components" DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/qemu-components/common/include/libqemu-cxx/libqemu-cxx.h b/qemu-components/common/include/libqemu-cxx/libqemu-cxx.h index 85e01284..2d418f48 100644 --- a/qemu-components/common/include/libqemu-cxx/libqemu-cxx.h +++ b/qemu-components/common/include/libqemu-cxx/libqemu-cxx.h @@ -318,6 +318,8 @@ class MemoryRegionOps struct MemTxAttrs { bool secure = false; bool debug = false; + /* Requester ID (for MSI for example) */ + unsigned int requester_id:16; MemTxAttrs() = default; MemTxAttrs(const ::MemTxAttrs& qemu_attrs); diff --git a/qemu-components/common/include/ports/target.h b/qemu-components/common/include/ports/target.h index a0333bd7..5f9f369c 100644 --- a/qemu-components/common/include/ports/target.h +++ b/qemu-components/common/include/ports/target.h @@ -14,6 +14,7 @@ #include "qemu-instance.h" #include "tlm-extensions/qemu-cpu-hint.h" #include "tlm-extensions/qemu-mr-hint.h" +#include "tlm-extensions/pcie_extension.h" #include class TlmTargetToQemuBridge : public tlm::tlm_fw_transport_if<> @@ -93,6 +94,13 @@ class TlmTargetToQemuBridge : public tlm::tlm_fw_transport_if<> return; } + // Extract PCIe extension if present + gs::PcieExtension* pcie_ext = nullptr; + trans.get_extension(pcie_ext); + if (pcie_ext) { + attrs.requester_id = pcie_ext->requester_id; + } + current_cpu_save = push_current_cpu(trans); switch (trans.get_command()) { diff --git a/qemu-components/common/include/tlm-extensions/pcie_extension.h b/qemu-components/common/include/tlm-extensions/pcie_extension.h new file mode 100644 index 00000000..0dddade0 --- /dev/null +++ b/qemu-components/common/include/tlm-extensions/pcie_extension.h @@ -0,0 +1,31 @@ +#ifndef _GREENSOCS_PCIE_EXTENSION_H +#define _GREENSOCS_PCIE_EXTENSION_H + +#include + +namespace gs { + +class PcieExtension : public tlm::tlm_extension +{ +public: + uint16_t requester_id; + uint8_t tag; + + PcieExtension() : requester_id(0), tag(0) {} + + virtual tlm_extension_base* clone() const override { + PcieExtension* ext = new PcieExtension(); + ext->requester_id = this->requester_id; + ext->tag = this->tag; + return ext; + } + + virtual void copy_from(tlm_extension_base const& ext) override { + requester_id = static_cast(ext).requester_id; + tag = static_cast(ext).tag; + } +}; + +} // namespace gs + +#endif \ No newline at end of file diff --git a/qemu-components/common/src/libqemu-cxx/memory.cc b/qemu-components/common/src/libqemu-cxx/memory.cc index 37be5c2d..0bb4b71e 100644 --- a/qemu-components/common/src/libqemu-cxx/memory.cc +++ b/qemu-components/common/src/libqemu-cxx/memory.cc @@ -301,6 +301,7 @@ AddressSpace::MemTxResult AddressSpace::write(uint64_t addr, const void* data, s ::MemTxResult qemu_res; qemu_attrs.secure = attrs.secure; + qemu_attrs.requester_id = attrs.requester_id; qemu_res = m_int->exports().address_space_write(m_as, addr, qemu_attrs, data, size); diff --git a/qemu-components/nvme/include/nvme.h b/qemu-components/nvme/include/nvme.h index 883ae057..561cf59a 100644 --- a/qemu-components/nvme/include/nvme.h +++ b/qemu-components/nvme/include/nvme.h @@ -26,11 +26,11 @@ class nvme_disk : public qemu_gpex::Device std::string m_drive_id; public: - nvme_disk(const sc_core::sc_module_name& name, sc_core::sc_object* o) - : nvme_disk(name, *(dynamic_cast(o))) + nvme_disk(const sc_core::sc_module_name& name, sc_core::sc_object* o, sc_core::sc_object* gpex) + : nvme_disk(name, *(dynamic_cast(o)), *(dynamic_cast(gpex))) { } - nvme_disk(const sc_core::sc_module_name& name, QemuInstance& inst) + nvme_disk(const sc_core::sc_module_name& name, QemuInstance& inst, qemu_gpex& gpex) : qemu_gpex::Device(name, inst, "nvme") , p_serial("serial", basename(), "Serial name of the nvme disk") , p_blob_file("blob_file", "", "Blob file to load as data storage") @@ -44,6 +44,7 @@ class nvme_disk : public qemu_gpex::Device opts << "if=sd,id=" << m_drive_id << ",file=" << file << ",format=raw"; m_inst.add_arg("-drive"); m_inst.add_arg(opts.str().c_str()); + gpex.add_device(*this); } void before_end_of_elaboration() override diff --git a/qemu-components/nvme/src/nvme.cc b/qemu-components/nvme/src/nvme.cc index 371e1d5d..80f133f9 100644 --- a/qemu-components/nvme/src/nvme.cc +++ b/qemu-components/nvme/src/nvme.cc @@ -8,4 +8,4 @@ #include "nvme.h" -void module_register() { GSC_MODULE_REGISTER_C(nvme_disk, sc_core::sc_object*); } \ No newline at end of file +void module_register() { GSC_MODULE_REGISTER_C(nvme_disk, sc_core::sc_object*, sc_core::sc_object*); } \ No newline at end of file diff --git a/systemc-components/CMakeLists.txt b/systemc-components/CMakeLists.txt index b1536234..0ae9335b 100644 --- a/systemc-components/CMakeLists.txt +++ b/systemc-components/CMakeLists.txt @@ -7,6 +7,7 @@ add_subdirectory(macs) add_subdirectory(gs_memory) add_subdirectory(memory_dumper) add_subdirectory(pass) +add_subdirectory(pci) if((NOT WITHOUT_PYTHON_BINDER) AND (NOT GS_ONLY)) add_subdirectory(python_binder) endif() diff --git a/systemc-components/pci/CMakeLists.txt b/systemc-components/pci/CMakeLists.txt new file mode 100644 index 00000000..5d8e5c10 --- /dev/null +++ b/systemc-components/pci/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(gs_gpex) +add_subdirectory(nvme_ssd) diff --git a/systemc-components/pci/gs_gpex/CMakeLists.txt b/systemc-components/pci/gs_gpex/CMakeLists.txt new file mode 100644 index 00000000..1079a943 --- /dev/null +++ b/systemc-components/pci/gs_gpex/CMakeLists.txt @@ -0,0 +1,6 @@ +gs_create_dymod(gs_gpex) + +target_include_directories( + gs_gpex PUBLIC + $ +) \ No newline at end of file diff --git a/systemc-components/pci/gs_gpex/include/gs_gpex.h b/systemc-components/pci/gs_gpex/include/gs_gpex.h new file mode 100644 index 00000000..79fac06f --- /dev/null +++ b/systemc-components/pci/gs_gpex/include/gs_gpex.h @@ -0,0 +1,1058 @@ +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Author: jhieb@micron.com 2025 + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _GREENSOCS_BASE_COMPONENTS_GPEX_H +#define _GREENSOCS_BASE_COMPONENTS_GPEX_H + + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include "tlm-modules/pcie-controller.h" + +#include "tlm-extensions/pcie_extension.h" + +#include "pci_regs.h" +#include "pci_ids.h" +#include "pci_tlps.h" +/* Size of the standard PCI config space */ +#define PCI_CONFIG_SPACE_SIZE 0x1000 + +namespace gs { + +/** + * @class gs_gpex + * + * @brief A Generic PCI host root component used for connecting SystemC PCI devices to. + * + * @details handles MMIO and ECAM operations and routing to child PCI objects. + * Will handle responses and DMA accesses upstream (including MSIX). + * Currently has to translate systemC transactions from router into + * TLP format (big endian) to be sent to the PCI controller(s). + * + * Currently using the xilinx pcie tlm model: https://github.com/Xilinx/pcie-model + */ +template +class gs_gpex : public sc_core::sc_module +{ + + SCP_LOGGER(()); + +private: + uint8_t config[PCI_CONFIG_SPACE_SIZE] = {}; + std::vector> e_wait_for_devices_resp; + std::vector dev_responses; + cci::cci_broker_handle m_broker; + uint8_t next_tag = 0; + // TODO(jhieb) any potential for overflow conditions with tag tracking? + std::unordered_map tag_to_device_map; // Maps tag -> device_id + + // BAR tracking structure + struct BARInfo { + uint64_t base_addr; + uint64_t size; + bool is_64bit; + bool is_io; + bool enabled; + uint8_t bar_index; // 0-5 + }; + + struct DeviceBARs { + uint8_t bus; + uint8_t device; + uint8_t function; + std::vector bars; // Up to 6 BARs per device + }; + + std::vector device_bar_map; + +protected: + + int find_device_by_mmio_addr(uint64_t addr) { + for (size_t dev_idx = 0; dev_idx < device_bar_map.size(); dev_idx++) { + for (const auto& bar : device_bar_map[dev_idx].bars) { + if (bar.enabled && !bar.is_io) { + uint64_t bar_end = bar.base_addr + bar.size - 1; + if (addr >= bar.base_addr && addr <= bar_end) { + return dev_idx; + } + } + } + } + return -1; // Not found + } + + // Helper to update BAR information from ECAM write + void update_bar_mapping(uint8_t bus, uint8_t device, uint8_t function, + uint8_t bar_index, uint32_t value) { + // Find or create device entry + DeviceBARs* dev_bars = nullptr; + for (auto& dev : device_bar_map) { + if (dev.bus == bus && dev.device == device && dev.function == function) { + dev_bars = &dev; + break; + } + } + + if (!dev_bars) { + device_bar_map.push_back({bus, device, function, std::vector(6)}); + dev_bars = &device_bar_map.back(); + if(device > devices.size()){ + SCP_FATAL(()) << "Device index is out of range."; + } + std::shared_ptr pci_device = devices[device-1]; + PhysFuncConfig dev_cfg = pci_device->GetPFConfig(); + std::vector bars_config = dev_cfg.GetBarConfigs(); + // Could also watch the ecam reads for the BAR size. Currently just reading the BAR size directly. + uint32_t bar_num; + for(size_t i = 0; i < bars_config.size(); i++){ + bar_num = bars_config[i].GetBARNum(); + // In some cases there's a strange BAR in the config list? + if(bar_num < dev_bars->bars.size()){ + dev_bars->bars[bar_num].size = bars_config[i].GetSize(); + } + } + } + BARInfo& bar = dev_bars->bars[bar_index]; + + // Check if this is a 64-bit BAR (bit 2:1 = 10b) + bool is_64bit = ((value & 0x6) == 0x4); + bool is_io = (value & 0x1); + + if (bar_index % 2 == 0) { // Even BAR index (or low 32-bits of 64-bit BAR) + bar.base_addr = value & (is_io ? 0xFFFFFFFC : 0xFFFFFFF0); + bar.is_64bit = is_64bit; + bar.is_io = is_io; + bar.bar_index = bar_index; + bar.enabled = (value != 0 && value != 0xFFFFFFFF); + + // For 64-bit BARs, mark the next BAR as part of this one + if (is_64bit && bar_index < 5) { + dev_bars->bars[bar_index + 1].enabled = false; // Upper 32-bits handled separately + } + } else if (bar_index > 0 && dev_bars->bars[bar_index - 1].is_64bit) { + // This is the upper 32-bits of a 64-bit BAR + BARInfo& lower_bar = dev_bars->bars[bar_index - 1]; + lower_bar.base_addr |= (static_cast(value) << 32); + } + + SCP_INFO() << "Updated BAR" << (int)bar_index + << " for device " << (int)device + << ": addr=0x" << std::hex << bar.base_addr + << " 64bit=" << bar.is_64bit + << " IO=" << bar.is_io; + } + + void get_bdf_from_address(uint64_t addr, uint8_t &bus, uint8_t &device, uint8_t &function){ + // BDF in ECAM address: ECAM_BASE + (Bus << 20) + (Device << 15) + (Function << 12) + bus = (addr >> 20) & 0xff; + device = (addr >> 15) & 0x1f; + function = (addr >> 12) & 0x7; + } + + void b_transport_ecam(int id, tlm::tlm_generic_payload& txn, sc_core::sc_time& delay){ + unsigned int len = txn.get_data_length(); + unsigned char* ptr = txn.get_data_ptr(); + sc_dt::uint64 addr = txn.get_address(); + unsigned char* byt = txn.get_byte_enable_ptr(); + unsigned int bel = txn.get_byte_enable_length(); + uint8_t bus,device,function; + + switch (txn.get_command()) { + case tlm::TLM_READ_COMMAND: + handle_ecam_read(txn, delay); + break; + case tlm::TLM_WRITE_COMMAND: + handle_ecam_write(txn, delay); + break; + default: + SCP_FATAL(()) << "TLM command not supported"; + break; + } + } + + void handle_ecam_write(tlm::tlm_generic_payload& txn, sc_core::sc_time& delay){ + unsigned int len = txn.get_data_length(); + unsigned char* ptr = txn.get_data_ptr(); + sc_dt::uint64 addr = txn.get_address(); + unsigned char* byt = txn.get_byte_enable_ptr(); + unsigned int bel = txn.get_byte_enable_length(); + uint8_t bus,device,function,tag; + + if(addr < PCI_CONFIG_SPACE_SIZE){ + memcpy(config+addr, ptr, len); + } else { + get_bdf_from_address(addr, bus, device, function); + uint32_t reg_offset = addr & 0xFFF; + // TODO(jhieb) ignore the 0xFFFF status write for now?? How is the root supposed to handle status reg writes? + if(reg_offset == 0x6){ + SCP_INFO() << "PCI STS WRITE"; + get_bdf_from_address(addr, bus, device, function); + txn.set_response_status(tlm::TLM_OK_RESPONSE); + return; + } + // Check if this is a BAR write (offsets 0x10-0x24) + if (reg_offset >= PCI_BASE_ADDRESS_0 && reg_offset <= PCI_BASE_ADDRESS_5) { + uint8_t bar_index = (reg_offset - PCI_BASE_ADDRESS_0) / 4; + uint32_t bar_value; + memcpy(&bar_value, ptr, sizeof(uint32_t)); + // Update our BAR tracking + update_bar_mapping(bus, device, function, bar_index, bar_value); + } + + if(device <= devices.size()){ + // Need to build out the TLP in the txn data for the pcie controller to process. + TLPHeader4DW tlp_header; + // DW0: Format and Type + tlp_header.dword0.fmt = 0x02; // 3DW header WITH data (10b) + tlp_header.dword0.type = 0x04; // Configuration Write Type 0 (00100b) + tlp_header.dword0.tc = 0x0; // Traffic Class 0 + tlp_header.dword0.length = 1; // Writing 1 DWORD (4 bytes) + tlp_header.dword0.at = 0x0; // Address Type: untranslated + tlp_header.dword0.attr = 0x0; // No Relaxed Ordering, No Snoop + tlp_header.dword0.ep = 0; // Not poisoned + tlp_header.dword0.td = 0; // No TLP digest + tlp_header.dword0.th = 0; // No processing hints + tlp_header.dword0.ln = 0; // Not a lightweight notification + + // DW1: Requester ID and Byte Enables + tlp_header.dword1.first_be = calculate_first_be(addr & 0x3FF, len); + tlp_header.dword1.last_be = calculate_last_be(addr & 0x3FF, len); // Single DWORD, so last_be = 0 + tlp_header.dword1.tag = next_tag; // Transaction tag + tag = tlp_header.dword1.tag; + tag_to_device_map[tlp_header.dword1.tag] = device; + next_tag++; + tlp_header.dword1.requester_id = 0x0000; // Root complex ID (Bus 0, Dev 0, Func 0) + + // DW2: Target BDF and Register Address + tlp_header.dword2.bus = bus; + tlp_header.dword2.device = device; + tlp_header.dword2.function = function; + tlp_header.dword2.reg_num = 0; // Must be 0 for DWORD-aligned access + tlp_header.dword2.reg_offset = (addr >> 2) & 0x3FF; // DWORD offset (bits 11:2) + tlp_header.dword2.reserved = 0; + + + uint64_t test_log = 0; + memcpy(&test_log, ptr, std::min(static_cast(len), sizeof(uint64_t))); + SCP_INFO() << "ECAM Root " << get_txn_command_str(txn) << " to address: 0x" << std::hex << addr << " len: 0x" << std::hex << len << " data: 0x" << std::setw(len*2) << std::setfill('0') << test_log; + + // DW3: Align data payload based on byte offset + uint8_t byte_offset = addr & 0x3; // Offset within DWORD + // Zero out the payload first + memset(&tlp_header.data_payload, 0, sizeof(tlp_header.data_payload)); + + // Copy data to the correct offset within the DWORD + uint8_t* payload_ptr = reinterpret_cast(&tlp_header.data_payload); + memcpy(payload_ptr + byte_offset, ptr, len); + + // Convert to big-endian (network byte order) for PCIe wire format + tlp_header.to_big_endian(); + + // Create NEW transaction instead of reusing txn + tlm::tlm_generic_payload tlp_txn; + std::vector tlp_data(16); // 4DW = 16 bytes + memcpy(tlp_data.data(), tlp_header.data, 16); + + tlp_txn.set_command(tlm::TLM_WRITE_COMMAND); + tlp_txn.set_address(addr & 0xFFF); + tlp_txn.set_data_ptr(tlp_data.data()); + tlp_txn.set_data_length(16); + tlp_txn.set_streaming_width(16); + tlp_txn.set_byte_enable_ptr(nullptr); + tlp_txn.set_dmi_allowed(false); + tlp_txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE); + + pci_device_initiator_socket[device-1]->b_transport(tlp_txn, delay); + wait(*e_wait_for_devices_resp[device-1]); + TLPCompletion3DW resp; + memcpy(resp.data, dev_responses[device-1]->get_data_ptr(), 12); // TODO(Jhieb) data length is wrong sometimes? + resp.from_big_endian(); // Do a byte swap. + if(resp.dword2.tag != tag){ + SCP_FATAL(()) << "Tag to completion tag don't match"; + } + if(resp.dword1.status != TLPCompletion3DW::SC){ + txn.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE); + return; + } + } + } + txn.set_response_status(tlm::TLM_OK_RESPONSE); + return; + } + + void handle_ecam_read(tlm::tlm_generic_payload& txn, sc_core::sc_time& delay){ + unsigned int len = txn.get_data_length(); + unsigned char* ptr = txn.get_data_ptr(); + sc_dt::uint64 addr = txn.get_address(); + unsigned char* byt = txn.get_byte_enable_ptr(); + unsigned int bel = txn.get_byte_enable_length(); + uint8_t bus,device,function,tag; + + if(addr < PCI_CONFIG_SPACE_SIZE){ + memcpy(ptr, config+addr, len); + } else { + get_bdf_from_address(addr, bus, device, function); + + if(device <= devices.size()){ + // Need to build out the TLP in the txn data for the pcie controller to process. + TLPHeader3DW tlp_header; + // DW0: Format and Type + tlp_header.dword0.fmt = 0x00; // 3DW header, no data + tlp_header.dword0.type = 0x04; // Configuration Read Type 0 + tlp_header.dword0.tc = 0x0; // Traffic Class 0 + tlp_header.dword0.length = 1; // Requesting 1 DWORD (4 bytes) + tlp_header.dword0.at = 0x0; // Address Type: untranslated + tlp_header.dword0.attr = 0x0; // No Relaxed Ordering, No Snoop + tlp_header.dword0.ep = 0; // Not poisoned + tlp_header.dword0.td = 0; // No TLP digest + tlp_header.dword0.th = 0; // No processing hints + tlp_header.dword0.ln = 0; // Not a lightweight notification + + // DW1: Requester ID and Byte Enables + tlp_header.dword1.first_be = calculate_first_be(addr & 0x3FF, len); // Enable all 4 bytes (DWORD-aligned read) + tlp_header.dword1.last_be = calculate_last_be(addr & 0x3FF, len); // Single DWORD, so last_be = 0 + tlp_header.dword1.tag = next_tag; // Transaction tag (could increment for tracking) + tag = tlp_header.dword1.tag; + tag_to_device_map[tlp_header.dword1.tag] = device; + next_tag++; + tlp_header.dword1.requester_id = 0x0000; // Root complex ID (Bus 0, Dev 0, Func 0) + + // DW2: Target BDF and Register Address + tlp_header.dword2.bus = bus; + tlp_header.dword2.device = device; + tlp_header.dword2.function = function; + tlp_header.dword2.reg_num = 0; // Must be 0 for DWORD-aligned access + tlp_header.dword2.reg_offset = (addr >> 2) & 0x3FF; // DWORD offset (bits 11:2) + tlp_header.dword2.reserved = 0; + + // Convert to big-endian (network byte order) for PCIe wire format + tlp_header.to_big_endian(); + + // Create NEW transaction with properly sized buffer + tlm::tlm_generic_payload tlp_txn; + std::vector tlp_data(12); // 3DW = 12 bytes + memcpy(tlp_data.data(), tlp_header.data, 12); + tlp_txn.set_command(tlm::TLM_READ_COMMAND); + tlp_txn.set_address(addr & 0xFFF); + tlp_txn.set_data_ptr(tlp_data.data()); + tlp_txn.set_data_length(12); + tlp_txn.set_streaming_width(12); + tlp_txn.set_byte_enable_ptr(nullptr); + tlp_txn.set_dmi_allowed(false); + tlp_txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE); + + pci_device_initiator_socket[device-1]->b_transport(tlp_txn, delay); + wait(*e_wait_for_devices_resp[device-1]); + TLPCompletion3DW resp; + memcpy(resp.data, dev_responses[device-1]->get_data_ptr(), dev_responses[device-1]->get_data_length()); + resp.from_big_endian(); // Do a byte swap. + if(resp.dword2.tag != tag){ + SCP_FATAL(()) << "Tag to completion tag don't match"; + } + if(resp.dword1.status == TLPCompletion3DW::SC){ + uint8_t* payload_ptr = reinterpret_cast(&resp.data_payload); + uint8_t byte_offset = addr & 0x3; + memcpy(ptr, payload_ptr + byte_offset, len); + uint64_t test_log = 0; + memcpy(&test_log, payload_ptr + byte_offset, std::min(static_cast(len), sizeof(uint64_t))); + SCP_INFO() << "ECAM Root " << get_txn_command_str(txn) << " to address: 0x" << std::hex << addr << " len: 0x" << std::hex << len << " data: 0x" << std::setw(len*2) << std::setfill('0') << test_log; + } else { + txn.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE); + return; + } + } + } + txn.set_response_status(tlm::TLM_OK_RESPONSE); + } + + void b_transport_mmio(int id, tlm::tlm_generic_payload& txn, sc_core::sc_time& delay){ + unsigned int len = txn.get_data_length(); + unsigned char* ptr = txn.get_data_ptr(); + sc_dt::uint64 addr = txn.get_address(); + unsigned char* byt = txn.get_byte_enable_ptr(); + unsigned int bel = txn.get_byte_enable_length(); + bool is_64bit_addr; + TLPCompletionWithData resp; + + SCP_INFO() << "Root MMIO " << get_txn_command_str(txn) << " to address: 0x" << std::hex << addr << " len: 0x" << std::hex << len; + switch (txn.get_command()) { + case tlm::TLM_READ_COMMAND: + handle_mmio_read(txn, delay); + break; + case tlm::TLM_WRITE_COMMAND: + handle_mmio_write(txn, delay); + break; + default: + SCP_FATAL(()) << "TLM command not supported"; + break; + } + } + + void b_transport_mmio_high(int id, tlm::tlm_generic_payload& txn, sc_core::sc_time& delay){ + unsigned int len = txn.get_data_length(); + unsigned char* ptr = txn.get_data_ptr(); + sc_dt::uint64 addr = txn.get_address(); + unsigned char* byt = txn.get_byte_enable_ptr(); + unsigned int bel = txn.get_byte_enable_length(); + + SCP_INFO() << "Root MMIO HIGH " << get_txn_command_str(txn) << " to address: 0x" << std::hex << addr << " len: 0x" << std::hex << len; + // TODO support and update if we end up needing 64 bit BARs. + txn.set_response_status(tlm::TLM_OK_RESPONSE); + return; + } + + void handle_mmio_write(tlm::tlm_generic_payload& txn, sc_core::sc_time& delay){ + unsigned int len = txn.get_data_length(); + unsigned char* ptr = txn.get_data_ptr(); + sc_dt::uint64 addr = txn.get_address(); + unsigned char* byt = txn.get_byte_enable_ptr(); + unsigned int bel = txn.get_byte_enable_length(); + bool is_64bit_addr; + TLPCompletionWithData resp; + // Build Memory Write TLP (similar structure) + is_64bit_addr = (addr > 0xFFFFFFFF); + + // Figure out which device BAR that this MMIO address relates to. + int index = find_device_by_mmio_addr(addr); + if(index < 0){ + SCP_FATAL(()) << "Couldn't find device bar index."; + } + uint32_t device = device_bar_map[index].device; + + // Create NEW transaction with properly sized buffer + tlm::tlm_generic_payload tlp_txn; + std::vector tlp_data; + + if (is_64bit_addr) { + // 4DW Memory Write with data + TLPMemoryWrite4DW tlp_header; + + tlp_header.dword0.fmt = 0x03; // 4DW header WITH data (11b) + tlp_header.dword0.type = 0x00; // Memory Write (00000b) + tlp_header.dword0.tc = 0x0; + tlp_header.dword0.length = (len + 3) / 4; + tlp_header.dword0.at = 0x0; + tlp_header.dword0.attr = 0x0; + tlp_header.dword0.ep = 0; + tlp_header.dword0.td = 0; + tlp_header.dword0.th = 0; + + uint8_t first_be = calculate_first_be(addr, len); + uint8_t last_be = calculate_last_be(addr, len); + + tlp_header.dword1.first_be = first_be; + tlp_header.dword1.last_be = last_be; + tlp_header.dword1.tag = next_tag; + tag_to_device_map[tlp_header.dword1.tag] = device; + next_tag++; + tlp_header.dword1.requester_id = 0x0000; + + tlp_header.dword2.addr_high = (addr >> 32) & 0xFFFFFFFF; + tlp_header.dword3.addr_low = addr & 0xFFFFFFFC; + + // Align data payload based on byte offset + uint8_t byte_offset = addr & 0x3; + memset(tlp_header.data_payload, 0, sizeof(tlp_header.data_payload)); + uint8_t* payload_ptr = reinterpret_cast(tlp_header.data_payload); + memcpy(payload_ptr + byte_offset, ptr, len); + + tlp_header.to_big_endian(); + + size_t total_size = 16 + ((len + 3) & ~3); // Header + padded payload + tlp_data.resize(total_size); + memcpy(tlp_data.data(), tlp_header.data, total_size); + + } else { + // 3DW Memory Write with data + TLPMemoryWrite3DW tlp_header; + + tlp_header.dword0.fmt = 0x02; // 3DW header WITH data (10b) + tlp_header.dword0.type = 0x00; // Memory Write (00000b) + tlp_header.dword0.tc = 0x0; + tlp_header.dword0.length = (len + 3) / 4; + tlp_header.dword0.at = 0x0; + tlp_header.dword0.attr = 0x0; + tlp_header.dword0.ep = 0; + tlp_header.dword0.td = 0; + tlp_header.dword0.th = 0; + + uint8_t first_be = calculate_first_be(addr, len); + uint8_t last_be = calculate_last_be(addr, len); + + tlp_header.dword1.first_be = first_be; + tlp_header.dword1.last_be = last_be; + tlp_header.dword1.tag = next_tag; + tag_to_device_map[tlp_header.dword1.tag] = device; + next_tag++; + tlp_header.dword1.requester_id = 0x0000; + + tlp_header.dword2.addr = addr & 0xFFFFFFFC; + + // Align data payload based on byte offset + uint8_t byte_offset = addr & 0x3; + memset(tlp_header.data_payload, 0, sizeof(tlp_header.data_payload)); + uint8_t* payload_ptr = reinterpret_cast(tlp_header.data_payload); + memcpy(payload_ptr + byte_offset, ptr, len); + + tlp_header.to_big_endian(); + + size_t total_size = 12 + ((len + 3) & ~3); // Header + padded payload + tlp_data.resize(total_size); + memcpy(tlp_data.data(), tlp_header.data, total_size); + } + + // Setup NEW transaction + tlp_txn.set_command(tlm::TLM_WRITE_COMMAND); + tlp_txn.set_address(addr); + tlp_txn.set_data_ptr(tlp_data.data()); + tlp_txn.set_data_length(tlp_data.size()); + tlp_txn.set_streaming_width(tlp_data.size()); + tlp_txn.set_byte_enable_ptr(nullptr); + tlp_txn.set_dmi_allowed(false); + tlp_txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE); + + // Send to device (Memory Writes typically don't get completions unless posted) + pci_device_initiator_socket[device-1]->b_transport(tlp_txn, delay); + + // TODO(jhieb) copy the original status back to the host one. + txn.set_response_status(tlm::TLM_OK_RESPONSE); + } + + void handle_mmio_read(tlm::tlm_generic_payload& txn, sc_core::sc_time& delay){ + unsigned int len = txn.get_data_length(); + unsigned char* ptr = txn.get_data_ptr(); + sc_dt::uint64 addr = txn.get_address(); + unsigned char* byt = txn.get_byte_enable_ptr(); + unsigned int bel = txn.get_byte_enable_length(); + bool is_64bit_addr; + uint8_t tag; + TLPCompletionWithData resp; + // Build Memory Read TLP (3DW or 4DW depending on address) + is_64bit_addr = (addr > 0xFFFFFFFF); + + // Figure out which device BAR that this MMIO address relates to. + int index = find_device_by_mmio_addr(addr); + if(index < 0){ + SCP_FATAL(()) << "Couldn't find device bar index."; + } + uint32_t device = device_bar_map[index].device; + + // Create NEW transaction with properly sized buffer + tlm::tlm_generic_payload tlp_txn; + std::vector tlp_data(16); // 4DW = 16 bytes + tlp_txn.set_command(tlm::TLM_READ_COMMAND); + tlp_txn.set_address(addr); + tlp_txn.set_data_ptr(tlp_data.data()); + tlp_txn.set_data_length(16); + tlp_txn.set_streaming_width(16); + tlp_txn.set_byte_enable_ptr(nullptr); + tlp_txn.set_dmi_allowed(false); + tlp_txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE); + tlp_txn.set_address(addr); + + if (is_64bit_addr) { + // 4DW Memory Read for 64-bit addresses + TLPMemoryRead4DW tlp_header; + + // DW0: Format and Type + tlp_header.dword0.fmt = 0x01; // 4DW header, no data (01b) + tlp_header.dword0.type = 0x00; // Memory Read (00000b) + tlp_header.dword0.tc = 0x0; // Traffic Class 0 + tlp_header.dword0.length = (len + 3) / 4; // Length in DWORDs (round up) + tlp_header.dword0.at = 0x0; // Address Type: untranslated + tlp_header.dword0.attr = 0x0; // No Relaxed Ordering, No Snoop + tlp_header.dword0.ep = 0; // Not poisoned + tlp_header.dword0.td = 0; // No TLP digest + tlp_header.dword0.th = 0; // No processing hints + + // DW1: Requester ID and Byte Enables + uint8_t first_be = calculate_first_be(addr, len); + uint8_t last_be = calculate_last_be(addr, len); + + tlp_header.dword1.first_be = first_be; + tlp_header.dword1.last_be = last_be; + tlp_header.dword1.tag = next_tag; // Transaction tag + tag = tlp_header.dword1.tag; + tag_to_device_map[tlp_header.dword1.tag] = device; + next_tag++; + tlp_header.dword1.requester_id = 0x0000; // Root complex ID + + // DW2-3: 64-bit Address + tlp_header.dword2.addr_high = (addr >> 32) & 0xFFFFFFFF; + tlp_header.dword3.addr_low = addr & 0xFFFFFFFC; // DWORD-aligned (bits [1:0] = 00) + + // Convert to big-endian + tlp_header.to_big_endian(); + + tlp_txn.set_data_length(16); // 4DW header + memcpy(tlp_data.data(), tlp_header.data, 16); + + } else { + // 3DW Memory Read for 32-bit addresses + TLPMemoryRead3DW tlp_header; + + // DW0: Format and Type + tlp_header.dword0.fmt = 0x00; // 3DW header, no data (00b) + tlp_header.dword0.type = 0x00; // Memory Read (00000b) + tlp_header.dword0.tc = 0x0; // Traffic Class 0 + tlp_header.dword0.length = (len + 3) / 4; // Length in DWORDs (round up) + tlp_header.dword0.at = 0x0; // Address Type: untranslated + tlp_header.dword0.attr = 0x0; // No Relaxed Ordering, No Snoop + tlp_header.dword0.ep = 0; // Not poisoned + tlp_header.dword0.td = 0; // No TLP digest + tlp_header.dword0.th = 0; // No processing hints + + // DW1: Requester ID and Byte Enables + uint8_t first_be = calculate_first_be(addr, len); + uint8_t last_be = calculate_last_be(addr, len); + + tlp_header.dword1.first_be = first_be; + tlp_header.dword1.last_be = last_be; + tlp_header.dword1.tag = next_tag; // Transaction tag + tag = tlp_header.dword1.tag; + tag_to_device_map[tlp_header.dword1.tag] = device; + next_tag++; + tlp_header.dword1.requester_id = 0x0000; // Root complex ID + + // DW2: 32-bit Address + tlp_header.dword2.addr = addr & 0xFFFFFFFC; // DWORD-aligned (bits [1:0] = 00) + + // Convert to big-endian + tlp_header.to_big_endian(); + + tlp_txn.set_data_length(12); // 3DW header + memcpy(tlp_data.data(), tlp_header.data, 12); + } + + // Send to device + pci_device_initiator_socket[device-1]->b_transport(tlp_txn, delay); + + // Wait for completion + wait(*e_wait_for_devices_resp[device-1]); + + // Parse completion TLP + memcpy(resp.data, dev_responses[device-1]->get_data_ptr(), dev_responses[device-1]->get_data_length()); + resp.from_big_endian(); + if(resp.dword2.tag != tag){ + SCP_FATAL(()) << "Tag to completion tag don't match"; + } + if (resp.dword1.status == TLPCompletionWithData::SC) { + // Extract data from completion + uint32_t byte_count = resp.dword1.byte_count; + uint32_t lower_addr = resp.dword2.lower_addr; + uint8_t* payload_ptr = reinterpret_cast(&resp.data_payload); + uint8_t byte_offset = addr & 0x3; + // Copy payload data back to original transaction + memcpy(ptr, payload_ptr + byte_offset, std::min(len, byte_count)); + + txn.set_response_status(tlm::TLM_OK_RESPONSE); + } else { + SCP_WARN() << "Memory read completion error, status: " << resp.dword1.status; + txn.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE); + return; + } + } + + void b_transport_pio(int id, tlm::tlm_generic_payload& txn, sc_core::sc_time& delay){ + unsigned int len = txn.get_data_length(); + unsigned char* ptr = txn.get_data_ptr(); + sc_dt::uint64 addr = txn.get_address(); + unsigned char* byt = txn.get_byte_enable_ptr(); + unsigned int bel = txn.get_byte_enable_length(); + + SCP_INFO() << "Root PIO " << get_txn_command_str(txn) << " to address: 0x" << std::hex << addr << " len: 0x" << std::hex << len; + txn.set_response_status(tlm::TLM_OK_RESPONSE); + return; + } + + // Helper to calculate first byte enable + uint8_t calculate_first_be(uint64_t addr, size_t len) { + uint8_t offset = addr & 0x3; // Byte offset within DWORD + uint8_t bytes_in_first_dword = std::min(size_t(4 - offset), len); + return ((1 << bytes_in_first_dword) - 1) << offset; + } + + // Helper to calculate last byte enable + uint8_t calculate_last_be(uint64_t addr, size_t len) { + if (len <= (4 - (addr & 0x3))) { + return 0x0; // Single DWORD access + } + uint8_t remaining = (addr + len) & 0x3; + if (remaining == 0) remaining = 4; + return (1 << remaining) - 1; + } + + const char* get_txn_command_str(const tlm::tlm_generic_payload& trans) + { + switch (trans.get_command()) { + case tlm::TLM_READ_COMMAND: + return "READ"; + case tlm::TLM_WRITE_COMMAND: + return "WRITE"; + case tlm::TLM_IGNORE_COMMAND: + return "IGNORE"; + default: + break; + } + return "UNKNOWN"; + } + + // Helper to get device_id from tag + int get_device_from_tag(uint8_t tag) { + auto it = tag_to_device_map.find(tag); + if (it != tag_to_device_map.end()) { + int device_id = it->second; + tag_to_device_map.erase(it); // Free the tag + return device_id; + } + return -1; // Tag not found + } + + void b_transport_device_response(int id, tlm::tlm_generic_payload& txn, sc_core::sc_time& delay){ + unsigned int len = txn.get_data_length(); + unsigned char* ptr = txn.get_data_ptr(); + sc_dt::uint64 addr = txn.get_address(); + unsigned char* byt = txn.get_byte_enable_ptr(); + unsigned int bel = txn.get_byte_enable_length(); + + if(len < 12){ + SCP_WARN() << "TLP too short: " << len << "bytes"; + txn.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE); + return; + } + + // Read the first DWORD to determine TLP type + uint32_t dword0; + memcpy(&dword0, ptr, 4); + dword0 = __builtin_bswap32(dword0); // Convert from big-endian + + uint8_t fmt = (dword0 >> 29) & 0x3; // Bits 30-29 + uint8_t type = (dword0 >> 24) & 0x1F; // Bits 28-24 + + + // Determine TLP category + if (type == 0x0A) { + // Parse the completion header to get tag and requester_id + uint32_t dword2; + memcpy(&dword2, ptr + 8, 4); + dword2 = __builtin_bswap32(dword2); + + uint32_t dword1; + memcpy(&dword1, ptr + 4, 4); + dword1 = __builtin_bswap32(dword1); + uint16_t completer_id = (dword1 >> 16) & 0xFFFF; + uint8_t tag = (dword2 >> 8) & 0xFF; // Bits [15:8] + + // TODO(jhieb) need to check for out of order responses. Will be more complex to handle appropriately. + int cmpltr_dev_id = (completer_id >> 3) & 0x1F; // Extract device number + int dev_from_tag = get_device_from_tag(tag); + if(dev_from_tag <= 0 || dev_from_tag > devices.size()){ + SCP_FATAL(()) << "Invalid device from response tag."; + } + dev_responses[dev_from_tag-1] = &txn; + e_wait_for_devices_resp[dev_from_tag-1]->notify(); + + } else if (type == 0x00 && (fmt == 0x02 || fmt == 0x03)) { + // **Memory Write TLP** (DMA write from device) + //SCP_DEBUG() << "Received DMA Memory Write from device " << id + // << " fmt=" << (int)fmt << " (3DW=" << (fmt==0x02) + // << ", 4DW=" << (fmt==0x03) << ")"; + handle_dma_write(txn, delay, fmt); + + } else if (type == 0x00 && (fmt == 0x00 || fmt == 0x01)) { + // **Memory Read TLP** (DMA read request from device) + //SCP_DEBUG() << "Received DMA Memory Read Request from device " << id; + handle_dma_read(txn, delay, fmt); + + } else { + SCP_WARN() << "Unknown TLP type from device " << id + << ": fmt=0x" << std::hex << (int)fmt + << " type=0x" << (int)type; + txn.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE); + } + + return; + } + + void handle_dma_write(tlm::tlm_generic_payload& txn, sc_core::sc_time& delay, uint8_t fmt) { + unsigned char* ptr = txn.get_data_ptr(); + + if (fmt == 0x02) { + // 3DW Memory Write + TLPMemoryWrite3DW tlp; + memcpy(tlp.data, ptr, txn.get_data_length()); + tlp.from_big_endian(); + + uint64_t target_addr = tlp.dword2.addr; + uint32_t length_dw = tlp.dword0.length; + uint32_t byte_len = length_dw * 4; + + // Forward to system memory via bus_master socket + tlm::tlm_generic_payload dma_txn; + dma_txn.set_command(tlm::TLM_WRITE_COMMAND); + dma_txn.set_address(target_addr); + dma_txn.set_data_ptr(tlp.data_payload); + dma_txn.set_data_length(byte_len); + dma_txn.set_streaming_width(byte_len); + dma_txn.set_byte_enable_ptr(nullptr); + dma_txn.set_dmi_allowed(false); + dma_txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE); + + // Attach PCIe extension with requester_id + PcieExtension* pcie_ext = new PcieExtension(); + pcie_ext->requester_id = tlp.dword1.requester_id; + dma_txn.set_extension(pcie_ext); + + bus_master->b_transport(dma_txn, delay); + txn.set_response_status(dma_txn.get_response_status()); + } else if (fmt == 0x03) { + // 4DW Memory Write (64-bit address) + TLPMemoryWrite4DW tlp; + memcpy(tlp.data, ptr, txn.get_data_length()); + tlp.from_big_endian(); + + uint64_t target_addr = (static_cast(tlp.dword2.addr_high) << 32) + | tlp.dword3.addr_low; + uint32_t length_dw = tlp.dword0.length; + uint32_t byte_len = length_dw * 4; + + //SCP_INFO() << "DMA Write (64-bit): addr=0x" << std::hex << target_addr + // << " len=" << std::dec << byte_len << " bytes"; + + tlm::tlm_generic_payload dma_txn; + dma_txn.set_command(tlm::TLM_WRITE_COMMAND); + dma_txn.set_address(target_addr); + dma_txn.set_data_ptr(tlp.data_payload); + dma_txn.set_data_length(byte_len); + dma_txn.set_streaming_width(byte_len); + dma_txn.set_byte_enable_ptr(nullptr); + dma_txn.set_dmi_allowed(false); + dma_txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE); + + // Attach PCIe extension with requester_id + PcieExtension* pcie_ext = new PcieExtension(); + pcie_ext->requester_id = tlp.dword1.requester_id; + dma_txn.set_extension(pcie_ext); + + bus_master->b_transport(dma_txn, delay); + txn.set_response_status(dma_txn.get_response_status()); + } + } + + + void handle_dma_read(tlm::tlm_generic_payload& txn, sc_core::sc_time& delay, uint8_t fmt) { + unsigned char* ptr = txn.get_data_ptr(); + + // Parse Memory Read Request TLP + uint64_t target_addr; + uint32_t length_dw; + uint16_t requester_id; + uint8_t tag; + + if (fmt == 0x00) { + // 3DW Memory Read + TLPMemoryRead3DW tlp; + memcpy(tlp.data, ptr, 12); + tlp.from_big_endian(); + + target_addr = tlp.dword2.addr; + length_dw = tlp.dword0.length; + requester_id = tlp.dword1.requester_id; + tag = tlp.dword1.tag; + + } else { + // 4DW Memory Read (64-bit) + TLPMemoryRead4DW tlp; + memcpy(tlp.data, ptr, 16); + tlp.from_big_endian(); + + target_addr = (static_cast(tlp.dword2.addr_high) << 32) + | tlp.dword3.addr_low; + length_dw = tlp.dword0.length; + requester_id = tlp.dword1.requester_id; + tag = tlp.dword1.tag; + } + + uint32_t byte_len = length_dw * 4; + //SCP_INFO() << "DMA Read Request: addr=0x" << std::hex << target_addr + // << " len=" << std::dec << byte_len + // << " tag=0x" << std::hex << (int)tag; + + // Read from system memory + std::vector read_data(byte_len); + tlm::tlm_generic_payload dma_txn; + dma_txn.set_command(tlm::TLM_READ_COMMAND); + dma_txn.set_address(target_addr); + dma_txn.set_data_ptr(read_data.data()); + dma_txn.set_data_length(byte_len); + dma_txn.set_streaming_width(byte_len); + dma_txn.set_byte_enable_ptr(nullptr); + dma_txn.set_dmi_allowed(false); + dma_txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE); + + bus_master->b_transport(dma_txn, delay); + + if (dma_txn.get_response_status() == tlm::TLM_OK_RESPONSE) { + // Send Completion with Data back to device + send_completion_with_data(requester_id, tag, read_data.data(), byte_len, delay); + } + txn.set_response_status(dma_txn.get_response_status()); + } + + void send_completion_with_data(uint16_t requester_id, uint8_t tag, + uint8_t* data, uint32_t byte_len, + sc_core::sc_time& delay) { + TLPCompletionWithData cpl; + + // Build completion header + cpl.dword0.fmt = 0x02; // 3DW with data + cpl.dword0.type = 0x0A; // Completion + cpl.dword0.length = (byte_len + 3) / 4; + + cpl.dword1.completer_id = 0x0000; // Root complex + cpl.dword1.status = TLPCompletionWithData::SC; // Successful completion + cpl.dword1.byte_count = byte_len; + + cpl.dword2.requester_id = requester_id; + cpl.dword2.tag = tag; + cpl.dword2.lower_addr = 0; // Assume DWORD-aligned + + // Copy payload + memcpy(cpl.data_payload, data, byte_len); + + cpl.to_big_endian(); + + // Send back to requesting device + tlm::tlm_generic_payload resp_txn; + resp_txn.set_command(tlm::TLM_WRITE_COMMAND); + resp_txn.set_data_ptr(cpl.data); + resp_txn.set_data_length(12 + byte_len); + resp_txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE); + + // Send to device that made the request + int device_id = (requester_id >> 3) & 0x1F; // Extract device number + if (device_id > 0 && device_id <= devices.size()) { + pci_device_initiator_socket[device_id - 1]->b_transport(resp_txn, delay); + } else { + SCP_FATAL(()) << "Invalid device ID for DMA."; + } + } + +public: + std::vector> devices; + + tlm_utils::multi_passthrough_target_socket, BUSWIDTH> ecam_iface_socket; + tlm_utils::multi_passthrough_target_socket, BUSWIDTH> mmio_iface_socket; + tlm_utils::multi_passthrough_target_socket, BUSWIDTH> mmio_iface_high_socket; + tlm_utils::multi_passthrough_target_socket, BUSWIDTH> pio_iface_socket; + + tlm_utils::multi_passthrough_target_socket, BUSWIDTH> pci_device_target_socket; + tlm_utils::multi_passthrough_initiator_socket, BUSWIDTH> pci_device_initiator_socket; + + TargetSignalSocket reset; + tlm_utils::simple_initiator_socket bus_master; + // TODO(jhieb) Setup IRQ pins?? Currently use MSIX. + //sc_core::sc_vector irq_out; + /* + * In qemu the gpex host is aware of which gic spi indexes it + * plugged into. This is optional and the indexes can be left to -1 + * but it removes some feature. + */ + //int irq_num[4]; + + gs_gpex(sc_core::sc_module_name name) + : m_broker(cci::cci_get_broker()) + , ecam_iface_socket("ecam_iface") + , mmio_iface_socket("mmio_iface") + , mmio_iface_high_socket("mmio_iface_high") + , pio_iface_socket("pio_iface") + , bus_master("bus_master") + , reset("reset") + //, irq_out("irq_out", 4) + //, irq_num{ -1, -1, -1, -1 } + { + // Set relative_addresses to false for MMIO sockets so we get the full addressing to map BARs in GPEX. + m_broker.set_preset_cci_value(std::string(sc_module::name()) + ".mmio_iface.relative_addresses", + cci::cci_value(false)); + m_broker.set_preset_cci_value(std::string(sc_module::name()) + ".mmio_iface_high.relative_addresses", + cci::cci_value(false)); + m_broker.set_preset_cci_value(std::string(sc_module::name()) + ".ecam_iface.relative_addresses", + cci::cci_value(true)); + + SCP_DEBUG(()) << "gs_gpex constructor"; + ecam_iface_socket.register_b_transport(this, &gs_gpex::b_transport_ecam); + mmio_iface_socket.register_b_transport(this, &gs_gpex::b_transport_mmio); + mmio_iface_high_socket.register_b_transport(this, &gs_gpex::b_transport_mmio_high); + pio_iface_socket.register_b_transport(this, &gs_gpex::b_transport_pio); + pci_device_target_socket.register_b_transport(this, &gs_gpex::b_transport_device_response); + + reset.register_value_changed_cb([&](bool value) { + if (value) { + SCP_WARN(()) << "Reset"; + } + }); + + // We need to setup the PCI configuration for the root so that it can be read by the host. + write_to_config(config + PCI_VENDOR_ID, 0x1b36, 2); // PCI_VENDOR_ID_REDHAT + write_to_config(config + PCI_DEVICE_ID, 0x0008, 2); // PCI_DEVICE_ID_REDHAT_PCIE_HOST + config[PCI_COMMAND] = 0x0; + config[PCI_STATUS] = 0x0; + config[PCI_REVISION_ID] = 0; + write_to_config(config + PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_HOST, 2); + config[PCI_CACHE_LINE_SIZE] = 0x0; + config[PCI_LATENCY_TIMER] = 0x0; + config[PCI_HEADER_TYPE] = 0x0; + config[PCI_BIST] = 0x0; + } + + void add_device(std::shared_ptr& pcie_ctlr){ + devices.push_back(pcie_ctlr); + e_wait_for_devices_resp.push_back(std::make_unique()); + dev_responses.push_back(nullptr); + pcie_ctlr->init_socket.bind(pci_device_target_socket); + pci_device_initiator_socket.bind(pcie_ctlr->tgt_socket); + } + + void write_to_config(uint8_t* addr, uint64_t val, uint8_t len = 1){ + if(len > 8){ + SCP_FATAL(()) << "Can't copy more than the 64 bit value passed in"; + } + memcpy(addr, &val, len); + } + + void before_end_of_elaboration() + { + } + + gs_gpex() = delete; + gs_gpex(const gs_gpex&) = delete; + + ~gs_gpex() {} +}; +} // namespace gs + +extern "C" void module_register(); +#endif diff --git a/systemc-components/pci/gs_gpex/include/pci_ids.h b/systemc-components/pci/gs_gpex/include/pci_ids.h new file mode 100644 index 00000000..f1a53fea --- /dev/null +++ b/systemc-components/pci/gs_gpex/include/pci_ids.h @@ -0,0 +1,293 @@ +/* + * PCI Class, Vendor and Device IDs + * + * Please keep sorted. + * + * Abbreviated version of linux/pci_ids.h + * + * QEMU-specific definitions belong in pci.h + */ + +#ifndef HW_PCI_IDS_H +#define HW_PCI_IDS_H + +/* Device classes and subclasses */ + +#define PCI_CLASS_NOT_DEFINED 0x0000 +#define PCI_CLASS_NOT_DEFINED_VGA 0x0001 + +#define PCI_BASE_CLASS_STORAGE 0x01 +#define PCI_CLASS_STORAGE_SCSI 0x0100 +#define PCI_CLASS_STORAGE_IDE 0x0101 +#define PCI_CLASS_STORAGE_FLOPPY 0x0102 +#define PCI_CLASS_STORAGE_IPI 0x0103 +#define PCI_CLASS_STORAGE_RAID 0x0104 +#define PCI_CLASS_STORAGE_ATA 0x0105 +#define PCI_CLASS_STORAGE_SATA 0x0106 +#define PCI_CLASS_STORAGE_SAS 0x0107 +#define PCI_CLASS_STORAGE_EXPRESS 0x0108 +#define PCI_CLASS_STORAGE_UFS 0x0109 +#define PCI_CLASS_STORAGE_OTHER 0x0180 + +#define PCI_BASE_CLASS_NETWORK 0x02 +#define PCI_CLASS_NETWORK_ETHERNET 0x0200 +#define PCI_CLASS_NETWORK_TOKEN_RING 0x0201 +#define PCI_CLASS_NETWORK_FDDI 0x0202 +#define PCI_CLASS_NETWORK_ATM 0x0203 +#define PCI_CLASS_NETWORK_ISDN 0x0204 +#define PCI_CLASS_NETWORK_WORLDFIP 0x0205 +#define PCI_CLASS_NETWORK_PICMG214 0x0206 +#define PCI_CLASS_NETWORK_OTHER 0x0280 + +#define PCI_BASE_CLASS_DISPLAY 0x03 +#define PCI_CLASS_DISPLAY_VGA 0x0300 +#define PCI_CLASS_DISPLAY_XGA 0x0301 +#define PCI_CLASS_DISPLAY_3D 0x0302 +#define PCI_CLASS_DISPLAY_OTHER 0x0380 + +#define PCI_BASE_CLASS_MULTIMEDIA 0x04 +#define PCI_CLASS_MULTIMEDIA_VIDEO 0x0400 +#define PCI_CLASS_MULTIMEDIA_AUDIO 0x0401 +#define PCI_CLASS_MULTIMEDIA_PHONE 0x0402 +#define PCI_CLASS_MULTIMEDIA_OTHER 0x0480 + +#define PCI_BASE_CLASS_MEMORY 0x05 +#define PCI_CLASS_MEMORY_RAM 0x0500 +#define PCI_CLASS_MEMORY_FLASH 0x0501 +#define PCI_CLASS_MEMORY_CXL 0x0502 +#define PCI_CLASS_MEMORY_OTHER 0x0580 + +#define PCI_BASE_CLASS_BRIDGE 0x06 +#define PCI_CLASS_BRIDGE_HOST 0x0600 +#define PCI_CLASS_BRIDGE_ISA 0x0601 +#define PCI_CLASS_BRIDGE_EISA 0x0602 +#define PCI_CLASS_BRIDGE_MC 0x0603 +#define PCI_CLASS_BRIDGE_PCI 0x0604 +#define PCI_CLASS_BRIDGE_PCI_INF_SUB 0x01 +#define PCI_CLASS_BRIDGE_PCMCIA 0x0605 +#define PCI_CLASS_BRIDGE_NUBUS 0x0606 +#define PCI_CLASS_BRIDGE_CARDBUS 0x0607 +#define PCI_CLASS_BRIDGE_RACEWAY 0x0608 +#define PCI_CLASS_BRIDGE_PCI_SEMITP 0x0609 +#define PCI_CLASS_BRIDGE_IB_PCI 0x060a +#define PCI_CLASS_BRIDGE_OTHER 0x0680 + +#define PCI_BASE_CLASS_COMMUNICATION 0x07 +#define PCI_CLASS_COMMUNICATION_SERIAL 0x0700 +#define PCI_CLASS_COMMUNICATION_PARALLEL 0x0701 +#define PCI_CLASS_COMMUNICATION_MULTISERIAL 0x0702 +#define PCI_CLASS_COMMUNICATION_MODEM 0x0703 +#define PCI_CLASS_COMMUNICATION_GPIB 0x0704 +#define PCI_CLASS_COMMUNICATION_SC 0x0705 +#define PCI_CLASS_COMMUNICATION_OTHER 0x0780 + +#define PCI_BASE_CLASS_SYSTEM 0x08 +#define PCI_CLASS_SYSTEM_PIC 0x0800 +#define PCI_CLASS_SYSTEM_PIC_IOAPIC 0x080010 +#define PCI_CLASS_SYSTEM_PIC_IOXAPIC 0x080020 +#define PCI_CLASS_SYSTEM_DMA 0x0801 +#define PCI_CLASS_SYSTEM_TIMER 0x0802 +#define PCI_CLASS_SYSTEM_RTC 0x0803 +#define PCI_CLASS_SYSTEM_PCI_HOTPLUG 0x0804 +#define PCI_CLASS_SYSTEM_SDHCI 0x0805 +#define PCI_CLASS_SYSTEM_OTHER 0x0880 + +#define PCI_BASE_CLASS_INPUT 0x09 +#define PCI_CLASS_INPUT_KEYBOARD 0x0900 +#define PCI_CLASS_INPUT_PEN 0x0901 +#define PCI_CLASS_INPUT_MOUSE 0x0902 +#define PCI_CLASS_INPUT_SCANNER 0x0903 +#define PCI_CLASS_INPUT_GAMEPORT 0x0904 +#define PCI_CLASS_INPUT_OTHER 0x0980 + +#define PCI_BASE_CLASS_DOCKING 0x0a +#define PCI_CLASS_DOCKING_GENERIC 0x0a00 +#define PCI_CLASS_DOCKING_OTHER 0x0a80 + +#define PCI_BASE_CLASS_PROCESSOR 0x0b +#define PCI_CLASS_PROCESSOR_PENTIUM 0x0b02 +#define PCI_CLASS_PROCESSOR_POWERPC 0x0b20 +#define PCI_CLASS_PROCESSOR_MIPS 0x0b30 +#define PCI_CLASS_PROCESSOR_CO 0x0b40 + +#define PCI_BASE_CLASS_SERIAL 0x0c +#define PCI_CLASS_SERIAL_FIREWIRE 0x0c00 +#define PCI_CLASS_SERIAL_ACCESS 0x0c01 +#define PCI_CLASS_SERIAL_SSA 0x0c02 +#define PCI_CLASS_SERIAL_USB 0x0c03 +#define PCI_CLASS_SERIAL_USB_UHCI 0x0c0300 +#define PCI_CLASS_SERIAL_USB_OHCI 0x0c0310 +#define PCI_CLASS_SERIAL_USB_EHCI 0x0c0320 +#define PCI_CLASS_SERIAL_USB_XHCI 0x0c0330 +#define PCI_CLASS_SERIAL_USB_UNKNOWN 0x0c0380 +#define PCI_CLASS_SERIAL_USB_DEVICE 0x0c03fe +#define PCI_CLASS_SERIAL_FIBER 0x0c04 +#define PCI_CLASS_SERIAL_SMBUS 0x0c05 +#define PCI_CLASS_SERIAL_IB 0x0c06 +#define PCI_CLASS_SERIAL_IPMI 0x0c07 +#define PCI_CLASS_SERIAL_SERCOS 0x0c08 +#define PCI_CLASS_SERIAL_CANBUS 0x0c09 + +#define PCI_BASE_CLASS_WIRELESS 0x0d +#define PCI_CLASS_WIRELESS_IRDA 0x0d00 +#define PCI_CLASS_WIRELESS_CIR 0x0d01 +#define PCI_CLASS_WIRELESS_RF_CONTROLLER 0x0d10 +#define PCI_CLASS_WIRELESS_BLUETOOTH 0x0d11 +#define PCI_CLASS_WIRELESS_BROADBAND 0x0d12 +#define PCI_CLASS_WIRELESS_OTHER 0x0d80 + +#define PCI_BASE_CLASS_SATELLITE 0x0f +#define PCI_CLASS_SATELLITE_TV 0x0f00 +#define PCI_CLASS_SATELLITE_AUDIO 0x0f01 +#define PCI_CLASS_SATELLITE_VOICE 0x0f03 +#define PCI_CLASS_SATELLITE_DATA 0x0f04 + +#define PCI_BASE_CLASS_CRYPT 0x10 +#define PCI_CLASS_CRYPT_NETWORK 0x1000 +#define PCI_CLASS_CRYPT_ENTERTAINMENT 0x1001 +#define PCI_CLASS_CRYPT_OTHER 0x1080 + +#define PCI_BASE_CLASS_SIGNAL_PROCESSING 0x11 +#define PCI_CLASS_SP_DPIO 0x1100 +#define PCI_CLASS_SP_PERF 0x1101 +#define PCI_CLASS_SP_SYNCH 0x1110 +#define PCI_CLASS_SP_MANAGEMENT 0x1120 +#define PCI_CLASS_SP_OTHER 0x1180 + +#define PCI_CLASS_OTHERS 0xff + +/* Vendors and devices. Sort key: vendor first, device next. */ + +/* Ref: PCIe r6.0 Table 6-32 */ +#define PCI_VENDOR_ID_PCI_SIG 0x0001 + +#define PCI_VENDOR_ID_LSI_LOGIC 0x1000 +#define PCI_DEVICE_ID_LSI_53C810 0x0001 +#define PCI_DEVICE_ID_LSI_53C895A 0x0012 +#define PCI_DEVICE_ID_LSI_SAS1068 0x0054 +#define PCI_DEVICE_ID_LSI_SAS1078 0x0060 +#define PCI_DEVICE_ID_LSI_SAS0079 0x0079 + +#define PCI_VENDOR_ID_DEC 0x1011 +#define PCI_DEVICE_ID_DEC_21143 0x0019 + +#define PCI_VENDOR_ID_CIRRUS 0x1013 + +#define PCI_VENDOR_ID_IBM 0x1014 + +#define PCI_VENDOR_ID_AMD 0x1022 +#define PCI_DEVICE_ID_AMD_LANCE 0x2000 +#define PCI_DEVICE_ID_AMD_SCSI 0x2020 + +#define PCI_VENDOR_ID_HP 0x103c + +#define PCI_VENDOR_ID_TI 0x104c + +#define PCI_VENDOR_ID_MOTOROLA 0x1057 +#define PCI_DEVICE_ID_MOTOROLA_MPC106 0x0002 +#define PCI_DEVICE_ID_MOTOROLA_RAVEN 0x4801 + +#define PCI_VENDOR_ID_APPLE 0x106b +#define PCI_DEVICE_ID_APPLE_UNI_N_AGP 0x0020 +#define PCI_DEVICE_ID_APPLE_U3_AGP 0x004b +#define PCI_DEVICE_ID_APPLE_UNI_N_GMAC 0x0021 + +#define PCI_VENDOR_ID_SUN 0x108e +#define PCI_DEVICE_ID_SUN_EBUS 0x1000 +#define PCI_DEVICE_ID_SUN_HME 0x1001 +#define PCI_DEVICE_ID_SUN_SIMBA 0x5000 +#define PCI_DEVICE_ID_SUN_SABRE 0xa000 + +#define PCI_VENDOR_ID_ORACLE 0x108e +#define PCI_DEVICE_ID_REMOTE_IOHUB 0xb000 + +#define PCI_VENDOR_ID_CMD 0x1095 +#define PCI_DEVICE_ID_CMD_646 0x0646 + +#define PCI_VENDOR_ID_REALTEK 0x10ec +#define PCI_DEVICE_ID_REALTEK_8139 0x8139 + +#define PCI_VENDOR_ID_XILINX 0x10ee + +#define PCI_VENDOR_ID_VIA 0x1106 +#define PCI_DEVICE_ID_VIA_82C686B_ISA 0x0686 +#define PCI_DEVICE_ID_VIA_IDE 0x0571 +#define PCI_DEVICE_ID_VIA_UHCI 0x3038 +#define PCI_DEVICE_ID_VIA_82C686B_PM 0x3057 +#define PCI_DEVICE_ID_VIA_AC97 0x3058 +#define PCI_DEVICE_ID_VIA_MC97 0x3068 +#define PCI_DEVICE_ID_VIA_8231_ISA 0x8231 +#define PCI_DEVICE_ID_VIA_8231_PM 0x8235 + +#define PCI_VENDOR_ID_MARVELL 0x11ab +#define PCI_DEVICE_ID_MARVELL_MV6436X 0x6460 + +#define PCI_VENDOR_ID_SILICON_MOTION 0x126f +#define PCI_DEVICE_ID_SM501 0x0501 + +#define PCI_VENDOR_ID_ENSONIQ 0x1274 +#define PCI_DEVICE_ID_ENSONIQ_ES1370 0x5000 + +#define PCI_VENDOR_ID_CHELSIO 0x1425 + +#define PCI_VENDOR_ID_FREESCALE 0x1957 +#define PCI_DEVICE_ID_MPC8533E 0x0030 + +#define PCI_VENDOR_ID_BAIDU 0x1d22 +#define PCI_DEVICE_ID_KUNLUN_VF 0x3685 + +#define PCI_VENDOR_ID_INTEL 0x8086 +#define PCI_DEVICE_ID_INTEL_82378 0x0484 +#define PCI_DEVICE_ID_INTEL_82441 0x1237 +#define PCI_DEVICE_ID_INTEL_82801AA_5 0x2415 +#define PCI_DEVICE_ID_INTEL_82801BA_11 0x244e +#define PCI_DEVICE_ID_INTEL_82801D 0x24CD +#define PCI_DEVICE_ID_INTEL_ESB_9 0x25ab +#define PCI_DEVICE_ID_INTEL_NVME 0x5845 +#define PCI_DEVICE_ID_INTEL_82371SB_0 0x7000 +#define PCI_DEVICE_ID_INTEL_82371SB_1 0x7010 +#define PCI_DEVICE_ID_INTEL_82371SB_2 0x7020 +#define PCI_DEVICE_ID_INTEL_82371AB_0 0x7110 +#define PCI_DEVICE_ID_INTEL_82371AB 0x7111 +#define PCI_DEVICE_ID_INTEL_82371AB_2 0x7112 +#define PCI_DEVICE_ID_INTEL_82371AB_3 0x7113 + +#define PCI_DEVICE_ID_INTEL_ICH9_0 0x2910 +#define PCI_DEVICE_ID_INTEL_ICH9_1 0x2917 +#define PCI_DEVICE_ID_INTEL_ICH9_2 0x2912 +#define PCI_DEVICE_ID_INTEL_ICH9_3 0x2913 +#define PCI_DEVICE_ID_INTEL_ICH9_4 0x2914 +#define PCI_DEVICE_ID_INTEL_ICH9_5 0x2919 +#define PCI_DEVICE_ID_INTEL_ICH9_6 0x2930 +#define PCI_DEVICE_ID_INTEL_ICH9_7 0x2916 +#define PCI_DEVICE_ID_INTEL_ICH9_8 0x2918 + +#define PCI_DEVICE_ID_INTEL_82801I_UHCI1 0x2934 +#define PCI_DEVICE_ID_INTEL_82801I_UHCI2 0x2935 +#define PCI_DEVICE_ID_INTEL_82801I_UHCI3 0x2936 +#define PCI_DEVICE_ID_INTEL_82801I_UHCI4 0x2937 +#define PCI_DEVICE_ID_INTEL_82801I_UHCI5 0x2938 +#define PCI_DEVICE_ID_INTEL_82801I_UHCI6 0x2939 +#define PCI_DEVICE_ID_INTEL_82801I_EHCI1 0x293a +#define PCI_DEVICE_ID_INTEL_82801I_EHCI2 0x293c +#define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed + +#define PCI_DEVICE_ID_INTEL_P35_MCH 0x29c0 + +#define PCI_VENDOR_ID_XEN 0x5853 +#define PCI_DEVICE_ID_XEN_PLATFORM 0x0001 + +#define PCI_VENDOR_ID_NEC 0x1033 +#define PCI_DEVICE_ID_NEC_UPD720200 0x0194 + +#define PCI_VENDOR_ID_TEWS 0x1498 +#define PCI_DEVICE_ID_TEWS_TPCI200 0x30C8 + +#define PCI_VENDOR_ID_VMWARE 0x15ad +#define PCI_DEVICE_ID_VMWARE_PVRDMA 0x0820 + +#define PCI_VENDOR_ID_SYNOPSYS 0x16C3 + +#define PCI_VENDOR_ID_NVIDIA 0x10de + +#endif diff --git a/systemc-components/pci/gs_gpex/include/pci_regs.h b/systemc-components/pci/gs_gpex/include/pci_regs.h new file mode 100644 index 00000000..94c00996 --- /dev/null +++ b/systemc-components/pci/gs_gpex/include/pci_regs.h @@ -0,0 +1,1157 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * PCI standard defines + * Copyright 1994, Drew Eckhardt + * Copyright 1997--1999 Martin Mares + * + * For more information, please consult the following manuals (look at + * http://www.pcisig.com/ for how to get them): + * + * PCI BIOS Specification + * PCI Local Bus Specification + * PCI to PCI Bridge Specification + * PCI System Design Guide + * + * For HyperTransport information, please consult the following manuals + * from http://www.hypertransport.org : + * + * The HyperTransport I/O Link Specification + */ + +#ifndef LINUX_PCI_REGS_H +#define LINUX_PCI_REGS_H + +/* + * Conventional PCI and PCI-X Mode 1 devices have 256 bytes of + * configuration space. PCI-X Mode 2 and PCIe devices have 4096 bytes of + * configuration space. + */ +#define PCI_CFG_SPACE_SIZE 256 +#define PCI_CFG_SPACE_EXP_SIZE 4096 + +/* + * Under PCI, each device has 256 bytes of configuration address space, + * of which the first 64 bytes are standardized as follows: + */ +#define PCI_STD_HEADER_SIZEOF 64 +#define PCI_STD_NUM_BARS 6 /* Number of standard BARs */ +#define PCI_VENDOR_ID 0x00 /* 16 bits */ +#define PCI_DEVICE_ID 0x02 /* 16 bits */ +#define PCI_COMMAND 0x04 /* 16 bits */ +#define PCI_COMMAND_IO 0x1 /* Enable response in I/O space */ +#define PCI_COMMAND_MEMORY 0x2 /* Enable response in Memory space */ +#define PCI_COMMAND_MASTER 0x4 /* Enable bus mastering */ +#define PCI_COMMAND_SPECIAL 0x8 /* Enable response to special cycles */ +#define PCI_COMMAND_INVALIDATE 0x10 /* Use memory write and invalidate */ +#define PCI_COMMAND_VGA_PALETTE 0x20 /* Enable palette snooping */ +#define PCI_COMMAND_PARITY 0x40 /* Enable parity checking */ +#define PCI_COMMAND_WAIT 0x80 /* Enable address/data stepping */ +#define PCI_COMMAND_SERR 0x100 /* Enable SERR */ +#define PCI_COMMAND_FAST_BACK 0x200 /* Enable back-to-back writes */ +#define PCI_COMMAND_INTX_DISABLE 0x400 /* INTx Emulation Disable */ + +#define PCI_STATUS 0x06 /* 16 bits */ +#define PCI_STATUS_IMM_READY 0x01 /* Immediate Readiness */ +#define PCI_STATUS_INTERRUPT 0x08 /* Interrupt status */ +#define PCI_STATUS_CAP_LIST 0x10 /* Support Capability List */ +#define PCI_STATUS_66MHZ 0x20 /* Support 66 MHz PCI 2.1 bus */ +#define PCI_STATUS_UDF 0x40 /* Support User Definable Features [obsolete] */ +#define PCI_STATUS_FAST_BACK 0x80 /* Accept fast-back to back */ +#define PCI_STATUS_PARITY 0x100 /* Detected parity error */ +#define PCI_STATUS_DEVSEL_MASK 0x600 /* DEVSEL timing */ +#define PCI_STATUS_DEVSEL_FAST 0x000 +#define PCI_STATUS_DEVSEL_MEDIUM 0x200 +#define PCI_STATUS_DEVSEL_SLOW 0x400 +#define PCI_STATUS_SIG_TARGET_ABORT 0x800 /* Set on target abort */ +#define PCI_STATUS_REC_TARGET_ABORT 0x1000 /* Master ack of " */ +#define PCI_STATUS_REC_MASTER_ABORT 0x2000 /* Set on master abort */ +#define PCI_STATUS_SIG_SYSTEM_ERROR 0x4000 /* Set when we drive SERR */ +#define PCI_STATUS_DETECTED_PARITY 0x8000 /* Set on parity error */ + +#define PCI_CLASS_REVISION 0x08 /* High 24 bits are class, low 8 revision */ +#define PCI_REVISION_ID 0x08 /* Revision ID */ +#define PCI_CLASS_PROG 0x09 /* Reg. Level Programming Interface */ +#define PCI_CLASS_DEVICE 0x0a /* Device class */ + +#define PCI_CACHE_LINE_SIZE 0x0c /* 8 bits */ +#define PCI_LATENCY_TIMER 0x0d /* 8 bits */ +#define PCI_HEADER_TYPE 0x0e /* 8 bits */ +#define PCI_HEADER_TYPE_MASK 0x7f +#define PCI_HEADER_TYPE_NORMAL 0 +#define PCI_HEADER_TYPE_BRIDGE 1 +#define PCI_HEADER_TYPE_CARDBUS 2 +#define PCI_HEADER_TYPE_MFD 0x80 /* Multi-Function Device (possible) */ + +#define PCI_BIST 0x0f /* 8 bits */ +#define PCI_BIST_CODE_MASK 0x0f /* Return result */ +#define PCI_BIST_START 0x40 /* 1 to start BIST, 2 secs or less */ +#define PCI_BIST_CAPABLE 0x80 /* 1 if BIST capable */ + +/* + * Base addresses specify locations in memory or I/O space. + * Decoded size can be determined by writing a value of + * 0xffffffff to the register, and reading it back. Only + * 1 bits are decoded. + */ +#define PCI_BASE_ADDRESS_0 0x10 /* 32 bits */ +#define PCI_BASE_ADDRESS_1 0x14 /* 32 bits [htype 0,1 only] */ +#define PCI_BASE_ADDRESS_2 0x18 /* 32 bits [htype 0 only] */ +#define PCI_BASE_ADDRESS_3 0x1c /* 32 bits */ +#define PCI_BASE_ADDRESS_4 0x20 /* 32 bits */ +#define PCI_BASE_ADDRESS_5 0x24 /* 32 bits */ +#define PCI_BASE_ADDRESS_SPACE 0x01 /* 0 = memory, 1 = I/O */ +#define PCI_BASE_ADDRESS_SPACE_IO 0x01 +#define PCI_BASE_ADDRESS_SPACE_MEMORY 0x00 +#define PCI_BASE_ADDRESS_MEM_TYPE_MASK 0x06 +#define PCI_BASE_ADDRESS_MEM_TYPE_32 0x00 /* 32 bit address */ +#define PCI_BASE_ADDRESS_MEM_TYPE_1M 0x02 /* Below 1M [obsolete] */ +#define PCI_BASE_ADDRESS_MEM_TYPE_64 0x04 /* 64 bit address */ +#define PCI_BASE_ADDRESS_MEM_PREFETCH 0x08 /* prefetchable? */ +#define PCI_BASE_ADDRESS_MEM_MASK (~0x0fUL) +#define PCI_BASE_ADDRESS_IO_MASK (~0x03UL) +/* bit 1 is reserved if address_space = 1 */ + +/* Header type 0 (normal devices) */ +#define PCI_CARDBUS_CIS 0x28 +#define PCI_SUBSYSTEM_VENDOR_ID 0x2c +#define PCI_SUBSYSTEM_ID 0x2e +#define PCI_ROM_ADDRESS 0x30 /* Bits 31..11 are address, 10..1 reserved */ +#define PCI_ROM_ADDRESS_ENABLE 0x01 +#define PCI_ROM_ADDRESS_MASK (~0x7ffU) + +#define PCI_CAPABILITY_LIST 0x34 /* Offset of first capability list entry */ + +/* 0x35-0x3b are reserved */ +#define PCI_INTERRUPT_LINE 0x3c /* 8 bits */ +#define PCI_INTERRUPT_PIN 0x3d /* 8 bits */ +#define PCI_MIN_GNT 0x3e /* 8 bits */ +#define PCI_MAX_LAT 0x3f /* 8 bits */ + +/* Header type 1 (PCI-to-PCI bridges) */ +#define PCI_PRIMARY_BUS 0x18 /* Primary bus number */ +#define PCI_SECONDARY_BUS 0x19 /* Secondary bus number */ +#define PCI_SUBORDINATE_BUS 0x1a /* Highest bus number behind the bridge */ +#define PCI_SEC_LATENCY_TIMER 0x1b /* Latency timer for secondary interface */ +#define PCI_IO_BASE 0x1c /* I/O range behind the bridge */ +#define PCI_IO_LIMIT 0x1d +#define PCI_IO_RANGE_TYPE_MASK 0x0fUL /* I/O bridging type */ +#define PCI_IO_RANGE_TYPE_16 0x00 +#define PCI_IO_RANGE_TYPE_32 0x01 +#define PCI_IO_RANGE_MASK (~0x0fUL) /* Standard 4K I/O windows */ +#define PCI_IO_1K_RANGE_MASK (~0x03UL) /* Intel 1K I/O windows */ +#define PCI_SEC_STATUS 0x1e /* Secondary status register, only bit 14 used */ +#define PCI_MEMORY_BASE 0x20 /* Memory range behind */ +#define PCI_MEMORY_LIMIT 0x22 +#define PCI_MEMORY_RANGE_TYPE_MASK 0x0fUL +#define PCI_MEMORY_RANGE_MASK (~0x0fUL) +#define PCI_PREF_MEMORY_BASE 0x24 /* Prefetchable memory range behind */ +#define PCI_PREF_MEMORY_LIMIT 0x26 +#define PCI_PREF_RANGE_TYPE_MASK 0x0fUL +#define PCI_PREF_RANGE_TYPE_32 0x00 +#define PCI_PREF_RANGE_TYPE_64 0x01 +#define PCI_PREF_RANGE_MASK (~0x0fUL) +#define PCI_PREF_BASE_UPPER32 0x28 /* Upper half of prefetchable memory range */ +#define PCI_PREF_LIMIT_UPPER32 0x2c +#define PCI_IO_BASE_UPPER16 0x30 /* Upper half of I/O addresses */ +#define PCI_IO_LIMIT_UPPER16 0x32 +/* 0x34 same as for htype 0 */ +/* 0x35-0x3b is reserved */ +#define PCI_ROM_ADDRESS1 0x38 /* Same as PCI_ROM_ADDRESS, but for htype 1 */ +/* 0x3c-0x3d are same as for htype 0 */ +#define PCI_BRIDGE_CONTROL 0x3e +#define PCI_BRIDGE_CTL_PARITY 0x01 /* Enable parity detection on secondary interface */ +#define PCI_BRIDGE_CTL_SERR 0x02 /* The same for SERR forwarding */ +#define PCI_BRIDGE_CTL_ISA 0x04 /* Enable ISA mode */ +#define PCI_BRIDGE_CTL_VGA 0x08 /* Forward VGA addresses */ +#define PCI_BRIDGE_CTL_MASTER_ABORT 0x20 /* Report master aborts */ +#define PCI_BRIDGE_CTL_BUS_RESET 0x40 /* Secondary bus reset */ +#define PCI_BRIDGE_CTL_FAST_BACK 0x80 /* Fast Back2Back enabled on secondary interface */ + +/* Header type 2 (CardBus bridges) */ +#define PCI_CB_CAPABILITY_LIST 0x14 +/* 0x15 reserved */ +#define PCI_CB_SEC_STATUS 0x16 /* Secondary status */ +#define PCI_CB_PRIMARY_BUS 0x18 /* PCI bus number */ +#define PCI_CB_CARD_BUS 0x19 /* CardBus bus number */ +#define PCI_CB_SUBORDINATE_BUS 0x1a /* Subordinate bus number */ +#define PCI_CB_LATENCY_TIMER 0x1b /* CardBus latency timer */ +#define PCI_CB_MEMORY_BASE_0 0x1c +#define PCI_CB_MEMORY_LIMIT_0 0x20 +#define PCI_CB_MEMORY_BASE_1 0x24 +#define PCI_CB_MEMORY_LIMIT_1 0x28 +#define PCI_CB_IO_BASE_0 0x2c +#define PCI_CB_IO_BASE_0_HI 0x2e +#define PCI_CB_IO_LIMIT_0 0x30 +#define PCI_CB_IO_LIMIT_0_HI 0x32 +#define PCI_CB_IO_BASE_1 0x34 +#define PCI_CB_IO_BASE_1_HI 0x36 +#define PCI_CB_IO_LIMIT_1 0x38 +#define PCI_CB_IO_LIMIT_1_HI 0x3a +#define PCI_CB_IO_RANGE_MASK (~0x03UL) +/* 0x3c-0x3d are same as for htype 0 */ +#define PCI_CB_BRIDGE_CONTROL 0x3e +#define PCI_CB_BRIDGE_CTL_PARITY 0x01 /* Similar to standard bridge control register */ +#define PCI_CB_BRIDGE_CTL_SERR 0x02 +#define PCI_CB_BRIDGE_CTL_ISA 0x04 +#define PCI_CB_BRIDGE_CTL_VGA 0x08 +#define PCI_CB_BRIDGE_CTL_MASTER_ABORT 0x20 +#define PCI_CB_BRIDGE_CTL_CB_RESET 0x40 /* CardBus reset */ +#define PCI_CB_BRIDGE_CTL_16BIT_INT 0x80 /* Enable interrupt for 16-bit cards */ +#define PCI_CB_BRIDGE_CTL_PREFETCH_MEM0 0x100 /* Prefetch enable for both memory regions */ +#define PCI_CB_BRIDGE_CTL_PREFETCH_MEM1 0x200 +#define PCI_CB_BRIDGE_CTL_POST_WRITES 0x400 +#define PCI_CB_SUBSYSTEM_VENDOR_ID 0x40 +#define PCI_CB_SUBSYSTEM_ID 0x42 +#define PCI_CB_LEGACY_MODE_BASE 0x44 /* 16-bit PC Card legacy mode base address (ExCa) */ +/* 0x48-0x7f reserved */ + +/* Capability lists */ + +#define PCI_CAP_LIST_ID 0 /* Capability ID */ +#define PCI_CAP_ID_PM 0x01 /* Power Management */ +#define PCI_CAP_ID_AGP 0x02 /* Accelerated Graphics Port */ +#define PCI_CAP_ID_VPD 0x03 /* Vital Product Data */ +#define PCI_CAP_ID_SLOTID 0x04 /* Slot Identification */ +#define PCI_CAP_ID_MSI 0x05 /* Message Signalled Interrupts */ +#define PCI_CAP_ID_CHSWP 0x06 /* CompactPCI HotSwap */ +#define PCI_CAP_ID_PCIX 0x07 /* PCI-X */ +#define PCI_CAP_ID_HT 0x08 /* HyperTransport */ +#define PCI_CAP_ID_VNDR 0x09 /* Vendor-Specific */ +#define PCI_CAP_ID_DBG 0x0A /* Debug port */ +#define PCI_CAP_ID_CCRC 0x0B /* CompactPCI Central Resource Control */ +#define PCI_CAP_ID_SHPC 0x0C /* PCI Standard Hot-Plug Controller */ +#define PCI_CAP_ID_SSVID 0x0D /* Bridge subsystem vendor/device ID */ +#define PCI_CAP_ID_AGP3 0x0E /* AGP Target PCI-PCI bridge */ +#define PCI_CAP_ID_SECDEV 0x0F /* Secure Device */ +#define PCI_CAP_ID_EXP 0x10 /* PCI Express */ +#define PCI_CAP_ID_MSIX 0x11 /* MSI-X */ +#define PCI_CAP_ID_SATA 0x12 /* SATA Data/Index Conf. */ +#define PCI_CAP_ID_AF 0x13 /* PCI Advanced Features */ +#define PCI_CAP_ID_EA 0x14 /* PCI Enhanced Allocation */ +#define PCI_CAP_ID_MAX PCI_CAP_ID_EA +#define PCI_CAP_LIST_NEXT 1 /* Next capability in the list */ +#define PCI_CAP_FLAGS 2 /* Capability defined flags (16 bits) */ +#define PCI_CAP_SIZEOF 4 + +/* Power Management Registers */ + +#define PCI_PM_PMC 2 /* PM Capabilities Register */ +#define PCI_PM_CAP_VER_MASK 0x0007 /* Version */ +#define PCI_PM_CAP_PME_CLOCK 0x0008 /* PME clock required */ +#define PCI_PM_CAP_RESERVED 0x0010 /* Reserved field */ +#define PCI_PM_CAP_DSI 0x0020 /* Device specific initialization */ +#define PCI_PM_CAP_AUX_POWER 0x01C0 /* Auxiliary power support mask */ +#define PCI_PM_CAP_D1 0x0200 /* D1 power state support */ +#define PCI_PM_CAP_D2 0x0400 /* D2 power state support */ +#define PCI_PM_CAP_PME 0x0800 /* PME pin supported */ +#define PCI_PM_CAP_PME_MASK 0xF800 /* PME Mask of all supported states */ +#define PCI_PM_CAP_PME_D0 0x0800 /* PME# from D0 */ +#define PCI_PM_CAP_PME_D1 0x1000 /* PME# from D1 */ +#define PCI_PM_CAP_PME_D2 0x2000 /* PME# from D2 */ +#define PCI_PM_CAP_PME_D3hot 0x4000 /* PME# from D3 (hot) */ +#define PCI_PM_CAP_PME_D3cold 0x8000 /* PME# from D3 (cold) */ +#define PCI_PM_CAP_PME_SHIFT 11 /* Start of the PME Mask in PMC */ +#define PCI_PM_CTRL 4 /* PM control and status register */ +#define PCI_PM_CTRL_STATE_MASK 0x0003 /* Current power state (D0 to D3) */ +#define PCI_PM_CTRL_NO_SOFT_RESET 0x0008 /* No reset for D3hot->D0 */ +#define PCI_PM_CTRL_PME_ENABLE 0x0100 /* PME pin enable */ +#define PCI_PM_CTRL_DATA_SEL_MASK 0x1e00 /* Data select (??) */ +#define PCI_PM_CTRL_DATA_SCALE_MASK 0x6000 /* Data scale (??) */ +#define PCI_PM_CTRL_PME_STATUS 0x8000 /* PME pin status */ +#define PCI_PM_PPB_EXTENSIONS 6 /* PPB support extensions (??) */ +#define PCI_PM_PPB_B2_B3 0x40 /* Stop clock when in D3hot (??) */ +#define PCI_PM_BPCC_ENABLE 0x80 /* Bus power/clock control enable (??) */ +#define PCI_PM_DATA_REGISTER 7 /* (??) */ +#define PCI_PM_SIZEOF 8 + +/* AGP registers */ + +#define PCI_AGP_VERSION 2 /* BCD version number */ +#define PCI_AGP_RFU 3 /* Rest of capability flags */ +#define PCI_AGP_STATUS 4 /* Status register */ +#define PCI_AGP_STATUS_RQ_MASK 0xff000000 /* Maximum number of requests - 1 */ +#define PCI_AGP_STATUS_SBA 0x0200 /* Sideband addressing supported */ +#define PCI_AGP_STATUS_64BIT 0x0020 /* 64-bit addressing supported */ +#define PCI_AGP_STATUS_FW 0x0010 /* FW transfers supported */ +#define PCI_AGP_STATUS_RATE4 0x0004 /* 4x transfer rate supported */ +#define PCI_AGP_STATUS_RATE2 0x0002 /* 2x transfer rate supported */ +#define PCI_AGP_STATUS_RATE1 0x0001 /* 1x transfer rate supported */ +#define PCI_AGP_COMMAND 8 /* Control register */ +#define PCI_AGP_COMMAND_RQ_MASK 0xff000000 /* Master: Maximum number of requests */ +#define PCI_AGP_COMMAND_SBA 0x0200 /* Sideband addressing enabled */ +#define PCI_AGP_COMMAND_AGP 0x0100 /* Allow processing of AGP transactions */ +#define PCI_AGP_COMMAND_64BIT 0x0020 /* Allow processing of 64-bit addresses */ +#define PCI_AGP_COMMAND_FW 0x0010 /* Force FW transfers */ +#define PCI_AGP_COMMAND_RATE4 0x0004 /* Use 4x rate */ +#define PCI_AGP_COMMAND_RATE2 0x0002 /* Use 2x rate */ +#define PCI_AGP_COMMAND_RATE1 0x0001 /* Use 1x rate */ +#define PCI_AGP_SIZEOF 12 + +/* Vital Product Data */ + +#define PCI_VPD_ADDR 2 /* Address to access (15 bits!) */ +#define PCI_VPD_ADDR_MASK 0x7fff /* Address mask */ +#define PCI_VPD_ADDR_F 0x8000 /* Write 0, 1 indicates completion */ +#define PCI_VPD_DATA 4 /* 32-bits of data returned here */ +#define PCI_CAP_VPD_SIZEOF 8 + +/* Slot Identification */ + +#define PCI_SID_ESR 2 /* Expansion Slot Register */ +#define PCI_SID_ESR_NSLOTS 0x1f /* Number of expansion slots available */ +#define PCI_SID_ESR_FIC 0x20 /* First In Chassis Flag */ +#define PCI_SID_CHASSIS_NR 3 /* Chassis Number */ + +/* Message Signaled Interrupt registers */ + +#define PCI_MSI_FLAGS 0x02 /* Message Control */ +#define PCI_MSI_FLAGS_ENABLE 0x0001 /* MSI feature enabled */ +#define PCI_MSI_FLAGS_QMASK 0x000e /* Maximum queue size available */ +#define PCI_MSI_FLAGS_QSIZE 0x0070 /* Message queue size configured */ +#define PCI_MSI_FLAGS_64BIT 0x0080 /* 64-bit addresses allowed */ +#define PCI_MSI_FLAGS_MASKBIT 0x0100 /* Per-vector masking capable */ +#define PCI_MSI_RFU 3 /* Rest of capability flags */ +#define PCI_MSI_ADDRESS_LO 0x04 /* Lower 32 bits */ +#define PCI_MSI_ADDRESS_HI 0x08 /* Upper 32 bits (if PCI_MSI_FLAGS_64BIT set) */ +#define PCI_MSI_DATA_32 0x08 /* 16 bits of data for 32-bit devices */ +#define PCI_MSI_MASK_32 0x0c /* Mask bits register for 32-bit devices */ +#define PCI_MSI_PENDING_32 0x10 /* Pending intrs for 32-bit devices */ +#define PCI_MSI_DATA_64 0x0c /* 16 bits of data for 64-bit devices */ +#define PCI_MSI_MASK_64 0x10 /* Mask bits register for 64-bit devices */ +#define PCI_MSI_PENDING_64 0x14 /* Pending intrs for 64-bit devices */ + +/* MSI-X registers (in MSI-X capability) */ +#define PCI_MSIX_FLAGS 2 /* Message Control */ +#define PCI_MSIX_FLAGS_QSIZE 0x07FF /* Table size */ +#define PCI_MSIX_FLAGS_MASKALL 0x4000 /* Mask all vectors for this function */ +#define PCI_MSIX_FLAGS_ENABLE 0x8000 /* MSI-X enable */ +#define PCI_MSIX_TABLE 4 /* Table offset */ +#define PCI_MSIX_TABLE_BIR 0x00000007 /* BAR index */ +#define PCI_MSIX_TABLE_OFFSET 0xfffffff8 /* Offset into specified BAR */ +#define PCI_MSIX_PBA 8 /* Pending Bit Array offset */ +#define PCI_MSIX_PBA_BIR 0x00000007 /* BAR index */ +#define PCI_MSIX_PBA_OFFSET 0xfffffff8 /* Offset into specified BAR */ +#define PCI_MSIX_FLAGS_BIRMASK PCI_MSIX_PBA_BIR /* deprecated */ +#define PCI_CAP_MSIX_SIZEOF 12 /* size of MSIX registers */ + +/* MSI-X Table entry format (in memory mapped by a BAR) */ +#define PCI_MSIX_ENTRY_SIZE 16 +#define PCI_MSIX_ENTRY_LOWER_ADDR 0x0 /* Message Address */ +#define PCI_MSIX_ENTRY_UPPER_ADDR 0x4 /* Message Upper Address */ +#define PCI_MSIX_ENTRY_DATA 0x8 /* Message Data */ +#define PCI_MSIX_ENTRY_VECTOR_CTRL 0xc /* Vector Control */ +#define PCI_MSIX_ENTRY_CTRL_MASKBIT 0x00000001 + +/* CompactPCI Hotswap Register */ + +#define PCI_CHSWP_CSR 2 /* Control and Status Register */ +#define PCI_CHSWP_DHA 0x01 /* Device Hiding Arm */ +#define PCI_CHSWP_EIM 0x02 /* ENUM# Signal Mask */ +#define PCI_CHSWP_PIE 0x04 /* Pending Insert or Extract */ +#define PCI_CHSWP_LOO 0x08 /* LED On / Off */ +#define PCI_CHSWP_PI 0x30 /* Programming Interface */ +#define PCI_CHSWP_EXT 0x40 /* ENUM# status - extraction */ +#define PCI_CHSWP_INS 0x80 /* ENUM# status - insertion */ + +/* PCI Advanced Feature registers */ + +#define PCI_AF_LENGTH 2 +#define PCI_AF_CAP 3 +#define PCI_AF_CAP_TP 0x01 +#define PCI_AF_CAP_FLR 0x02 +#define PCI_AF_CTRL 4 +#define PCI_AF_CTRL_FLR 0x01 +#define PCI_AF_STATUS 5 +#define PCI_AF_STATUS_TP 0x01 +#define PCI_CAP_AF_SIZEOF 6 /* size of AF registers */ + +/* PCI Enhanced Allocation registers */ + +#define PCI_EA_NUM_ENT 2 /* Number of Capability Entries */ +#define PCI_EA_NUM_ENT_MASK 0x3f /* Num Entries Mask */ +#define PCI_EA_FIRST_ENT 4 /* First EA Entry in List */ +#define PCI_EA_FIRST_ENT_BRIDGE 8 /* First EA Entry for Bridges */ +#define PCI_EA_ES 0x00000007 /* Entry Size */ +#define PCI_EA_BEI 0x000000f0 /* BAR Equivalent Indicator */ + +/* EA fixed Secondary and Subordinate bus numbers for Bridge */ +#define PCI_EA_SEC_BUS_MASK 0xff +#define PCI_EA_SUB_BUS_MASK 0xff00 +#define PCI_EA_SUB_BUS_SHIFT 8 + +/* 0-5 map to BARs 0-5 respectively */ +#define PCI_EA_BEI_BAR0 0 +#define PCI_EA_BEI_BAR5 5 +#define PCI_EA_BEI_BRIDGE 6 /* Resource behind bridge */ +#define PCI_EA_BEI_ENI 7 /* Equivalent Not Indicated */ +#define PCI_EA_BEI_ROM 8 /* Expansion ROM */ +/* 9-14 map to VF BARs 0-5 respectively */ +#define PCI_EA_BEI_VF_BAR0 9 +#define PCI_EA_BEI_VF_BAR5 14 +#define PCI_EA_BEI_RESERVED 15 /* Reserved - Treat like ENI */ +#define PCI_EA_PP 0x0000ff00 /* Primary Properties */ +#define PCI_EA_SP 0x00ff0000 /* Secondary Properties */ +#define PCI_EA_P_MEM 0x00 /* Non-Prefetch Memory */ +#define PCI_EA_P_MEM_PREFETCH 0x01 /* Prefetchable Memory */ +#define PCI_EA_P_IO 0x02 /* I/O Space */ +#define PCI_EA_P_VF_MEM_PREFETCH 0x03 /* VF Prefetchable Memory */ +#define PCI_EA_P_VF_MEM 0x04 /* VF Non-Prefetch Memory */ +#define PCI_EA_P_BRIDGE_MEM 0x05 /* Bridge Non-Prefetch Memory */ +#define PCI_EA_P_BRIDGE_MEM_PREFETCH 0x06 /* Bridge Prefetchable Memory */ +#define PCI_EA_P_BRIDGE_IO 0x07 /* Bridge I/O Space */ +/* 0x08-0xfc reserved */ +#define PCI_EA_P_MEM_RESERVED 0xfd /* Reserved Memory */ +#define PCI_EA_P_IO_RESERVED 0xfe /* Reserved I/O Space */ +#define PCI_EA_P_UNAVAILABLE 0xff /* Entry Unavailable */ +#define PCI_EA_WRITABLE 0x40000000 /* Writable: 1 = RW, 0 = HwInit */ +#define PCI_EA_ENABLE 0x80000000 /* Enable for this entry */ +#define PCI_EA_BASE 4 /* Base Address Offset */ +#define PCI_EA_MAX_OFFSET 8 /* MaxOffset (resource length) */ +/* bit 0 is reserved */ +#define PCI_EA_IS_64 0x00000002 /* 64-bit field flag */ +#define PCI_EA_FIELD_MASK 0xfffffffc /* For Base & Max Offset */ + +/* PCI-X registers (Type 0 (non-bridge) devices) */ + +#define PCI_X_CMD 2 /* Modes & Features */ +#define PCI_X_CMD_DPERR_E 0x0001 /* Data Parity Error Recovery Enable */ +#define PCI_X_CMD_ERO 0x0002 /* Enable Relaxed Ordering */ +#define PCI_X_CMD_READ_512 0x0000 /* 512 byte maximum read byte count */ +#define PCI_X_CMD_READ_1K 0x0004 /* 1Kbyte maximum read byte count */ +#define PCI_X_CMD_READ_2K 0x0008 /* 2Kbyte maximum read byte count */ +#define PCI_X_CMD_READ_4K 0x000c /* 4Kbyte maximum read byte count */ +#define PCI_X_CMD_MAX_READ 0x000c /* Max Memory Read Byte Count */ + /* Max # of outstanding split transactions */ +#define PCI_X_CMD_SPLIT_1 0x0000 /* Max 1 */ +#define PCI_X_CMD_SPLIT_2 0x0010 /* Max 2 */ +#define PCI_X_CMD_SPLIT_3 0x0020 /* Max 3 */ +#define PCI_X_CMD_SPLIT_4 0x0030 /* Max 4 */ +#define PCI_X_CMD_SPLIT_8 0x0040 /* Max 8 */ +#define PCI_X_CMD_SPLIT_12 0x0050 /* Max 12 */ +#define PCI_X_CMD_SPLIT_16 0x0060 /* Max 16 */ +#define PCI_X_CMD_SPLIT_32 0x0070 /* Max 32 */ +#define PCI_X_CMD_MAX_SPLIT 0x0070 /* Max Outstanding Split Transactions */ +#define PCI_X_CMD_VERSION(x) (((x) >> 12) & 3) /* Version */ +#define PCI_X_STATUS 4 /* PCI-X capabilities */ +#define PCI_X_STATUS_DEVFN 0x000000ff /* A copy of devfn */ +#define PCI_X_STATUS_BUS 0x0000ff00 /* A copy of bus nr */ +#define PCI_X_STATUS_64BIT 0x00010000 /* 64-bit device */ +#define PCI_X_STATUS_133MHZ 0x00020000 /* 133 MHz capable */ +#define PCI_X_STATUS_SPL_DISC 0x00040000 /* Split Completion Discarded */ +#define PCI_X_STATUS_UNX_SPL 0x00080000 /* Unexpected Split Completion */ +#define PCI_X_STATUS_COMPLEX 0x00100000 /* Device Complexity */ +#define PCI_X_STATUS_MAX_READ 0x00600000 /* Designed Max Memory Read Count */ +#define PCI_X_STATUS_MAX_SPLIT 0x03800000 /* Designed Max Outstanding Split Transactions */ +#define PCI_X_STATUS_MAX_CUM 0x1c000000 /* Designed Max Cumulative Read Size */ +#define PCI_X_STATUS_SPL_ERR 0x20000000 /* Rcvd Split Completion Error Msg */ +#define PCI_X_STATUS_266MHZ 0x40000000 /* 266 MHz capable */ +#define PCI_X_STATUS_533MHZ 0x80000000 /* 533 MHz capable */ +#define PCI_X_ECC_CSR 8 /* ECC control and status */ +#define PCI_CAP_PCIX_SIZEOF_V0 8 /* size of registers for Version 0 */ +#define PCI_CAP_PCIX_SIZEOF_V1 24 /* size for Version 1 */ +#define PCI_CAP_PCIX_SIZEOF_V2 PCI_CAP_PCIX_SIZEOF_V1 /* Same for v2 */ + +/* PCI-X registers (Type 1 (bridge) devices) */ + +#define PCI_X_BRIDGE_SSTATUS 2 /* Secondary Status */ +#define PCI_X_SSTATUS_64BIT 0x0001 /* Secondary AD interface is 64 bits */ +#define PCI_X_SSTATUS_133MHZ 0x0002 /* 133 MHz capable */ +#define PCI_X_SSTATUS_FREQ 0x03c0 /* Secondary Bus Mode and Frequency */ +#define PCI_X_SSTATUS_VERS 0x3000 /* PCI-X Capability Version */ +#define PCI_X_SSTATUS_V1 0x1000 /* Mode 2, not Mode 1 */ +#define PCI_X_SSTATUS_V2 0x2000 /* Mode 1 or Modes 1 and 2 */ +#define PCI_X_SSTATUS_266MHZ 0x4000 /* 266 MHz capable */ +#define PCI_X_SSTATUS_533MHZ 0x8000 /* 533 MHz capable */ +#define PCI_X_BRIDGE_STATUS 4 /* Bridge Status */ + +/* PCI Bridge Subsystem ID registers */ + +#define PCI_SSVID_VENDOR_ID 4 /* PCI Bridge subsystem vendor ID */ +#define PCI_SSVID_DEVICE_ID 6 /* PCI Bridge subsystem device ID */ + +/* PCI Express capability registers */ + +#define PCI_EXP_FLAGS 0x02 /* Capabilities register */ +#define PCI_EXP_FLAGS_VERS 0x000f /* Capability version */ +#define PCI_EXP_FLAGS_TYPE 0x00f0 /* Device/Port type */ +#define PCI_EXP_TYPE_ENDPOINT 0x0 /* Express Endpoint */ +#define PCI_EXP_TYPE_LEG_END 0x1 /* Legacy Endpoint */ +#define PCI_EXP_TYPE_ROOT_PORT 0x4 /* Root Port */ +#define PCI_EXP_TYPE_UPSTREAM 0x5 /* Upstream Port */ +#define PCI_EXP_TYPE_DOWNSTREAM 0x6 /* Downstream Port */ +#define PCI_EXP_TYPE_PCI_BRIDGE 0x7 /* PCIe to PCI/PCI-X Bridge */ +#define PCI_EXP_TYPE_PCIE_BRIDGE 0x8 /* PCI/PCI-X to PCIe Bridge */ +#define PCI_EXP_TYPE_RC_END 0x9 /* Root Complex Integrated Endpoint */ +#define PCI_EXP_TYPE_RC_EC 0xa /* Root Complex Event Collector */ +#define PCI_EXP_FLAGS_SLOT 0x0100 /* Slot implemented */ +#define PCI_EXP_FLAGS_IRQ 0x3e00 /* Interrupt message number */ +#define PCI_EXP_DEVCAP 0x04 /* Device capabilities */ +#define PCI_EXP_DEVCAP_PAYLOAD 0x00000007 /* Max_Payload_Size */ +#define PCI_EXP_DEVCAP_PHANTOM 0x00000018 /* Phantom functions */ +#define PCI_EXP_DEVCAP_EXT_TAG 0x00000020 /* Extended tags */ +#define PCI_EXP_DEVCAP_L0S 0x000001c0 /* L0s Acceptable Latency */ +#define PCI_EXP_DEVCAP_L1 0x00000e00 /* L1 Acceptable Latency */ +#define PCI_EXP_DEVCAP_ATN_BUT 0x00001000 /* Attention Button Present */ +#define PCI_EXP_DEVCAP_ATN_IND 0x00002000 /* Attention Indicator Present */ +#define PCI_EXP_DEVCAP_PWR_IND 0x00004000 /* Power Indicator Present */ +#define PCI_EXP_DEVCAP_RBER 0x00008000 /* Role-Based Error Reporting */ +#define PCI_EXP_DEVCAP_PWR_VAL 0x03fc0000 /* Slot Power Limit Value */ +#define PCI_EXP_DEVCAP_PWR_SCL 0x0c000000 /* Slot Power Limit Scale */ +#define PCI_EXP_DEVCAP_FLR 0x10000000 /* Function Level Reset */ +#define PCI_EXP_DEVCTL 0x08 /* Device Control */ +#define PCI_EXP_DEVCTL_CERE 0x0001 /* Correctable Error Reporting En. */ +#define PCI_EXP_DEVCTL_NFERE 0x0002 /* Non-Fatal Error Reporting Enable */ +#define PCI_EXP_DEVCTL_FERE 0x0004 /* Fatal Error Reporting Enable */ +#define PCI_EXP_DEVCTL_URRE 0x0008 /* Unsupported Request Reporting En. */ +#define PCI_EXP_DEVCTL_RELAX_EN 0x0010 /* Enable relaxed ordering */ +#define PCI_EXP_DEVCTL_PAYLOAD 0x00e0 /* Max_Payload_Size */ +#define PCI_EXP_DEVCTL_PAYLOAD_128B 0x0000 /* 128 Bytes */ +#define PCI_EXP_DEVCTL_PAYLOAD_256B 0x0020 /* 256 Bytes */ +#define PCI_EXP_DEVCTL_PAYLOAD_512B 0x0040 /* 512 Bytes */ +#define PCI_EXP_DEVCTL_PAYLOAD_1024B 0x0060 /* 1024 Bytes */ +#define PCI_EXP_DEVCTL_PAYLOAD_2048B 0x0080 /* 2048 Bytes */ +#define PCI_EXP_DEVCTL_PAYLOAD_4096B 0x00a0 /* 4096 Bytes */ +#define PCI_EXP_DEVCTL_EXT_TAG 0x0100 /* Extended Tag Field Enable */ +#define PCI_EXP_DEVCTL_PHANTOM 0x0200 /* Phantom Functions Enable */ +#define PCI_EXP_DEVCTL_AUX_PME 0x0400 /* Auxiliary Power PM Enable */ +#define PCI_EXP_DEVCTL_NOSNOOP_EN 0x0800 /* Enable No Snoop */ +#define PCI_EXP_DEVCTL_READRQ 0x7000 /* Max_Read_Request_Size */ +#define PCI_EXP_DEVCTL_READRQ_128B 0x0000 /* 128 Bytes */ +#define PCI_EXP_DEVCTL_READRQ_256B 0x1000 /* 256 Bytes */ +#define PCI_EXP_DEVCTL_READRQ_512B 0x2000 /* 512 Bytes */ +#define PCI_EXP_DEVCTL_READRQ_1024B 0x3000 /* 1024 Bytes */ +#define PCI_EXP_DEVCTL_READRQ_2048B 0x4000 /* 2048 Bytes */ +#define PCI_EXP_DEVCTL_READRQ_4096B 0x5000 /* 4096 Bytes */ +#define PCI_EXP_DEVCTL_BCR_FLR 0x8000 /* Bridge Configuration Retry / FLR */ +#define PCI_EXP_DEVSTA 0x0a /* Device Status */ +#define PCI_EXP_DEVSTA_CED 0x0001 /* Correctable Error Detected */ +#define PCI_EXP_DEVSTA_NFED 0x0002 /* Non-Fatal Error Detected */ +#define PCI_EXP_DEVSTA_FED 0x0004 /* Fatal Error Detected */ +#define PCI_EXP_DEVSTA_URD 0x0008 /* Unsupported Request Detected */ +#define PCI_EXP_DEVSTA_AUXPD 0x0010 /* AUX Power Detected */ +#define PCI_EXP_DEVSTA_TRPND 0x0020 /* Transactions Pending */ +#define PCI_CAP_EXP_RC_ENDPOINT_SIZEOF_V1 12 /* v1 endpoints without link end here */ +#define PCI_EXP_LNKCAP 0x0c /* Link Capabilities */ +#define PCI_EXP_LNKCAP_SLS 0x0000000f /* Supported Link Speeds */ +#define PCI_EXP_LNKCAP_SLS_2_5GB 0x00000001 /* LNKCAP2 SLS Vector bit 0 */ +#define PCI_EXP_LNKCAP_SLS_5_0GB 0x00000002 /* LNKCAP2 SLS Vector bit 1 */ +#define PCI_EXP_LNKCAP_SLS_8_0GB 0x00000003 /* LNKCAP2 SLS Vector bit 2 */ +#define PCI_EXP_LNKCAP_SLS_16_0GB 0x00000004 /* LNKCAP2 SLS Vector bit 3 */ +#define PCI_EXP_LNKCAP_SLS_32_0GB 0x00000005 /* LNKCAP2 SLS Vector bit 4 */ +#define PCI_EXP_LNKCAP_SLS_64_0GB 0x00000006 /* LNKCAP2 SLS Vector bit 5 */ +#define PCI_EXP_LNKCAP_MLW 0x000003f0 /* Maximum Link Width */ +#define PCI_EXP_LNKCAP_ASPMS 0x00000c00 /* ASPM Support */ +#define PCI_EXP_LNKCAP_ASPM_L0S 0x00000400 /* ASPM L0s Support */ +#define PCI_EXP_LNKCAP_ASPM_L1 0x00000800 /* ASPM L1 Support */ +#define PCI_EXP_LNKCAP_L0SEL 0x00007000 /* L0s Exit Latency */ +#define PCI_EXP_LNKCAP_L1EL 0x00038000 /* L1 Exit Latency */ +#define PCI_EXP_LNKCAP_CLKPM 0x00040000 /* Clock Power Management */ +#define PCI_EXP_LNKCAP_SDERC 0x00080000 /* Surprise Down Error Reporting Capable */ +#define PCI_EXP_LNKCAP_DLLLARC 0x00100000 /* Data Link Layer Link Active Reporting Capable */ +#define PCI_EXP_LNKCAP_LBNC 0x00200000 /* Link Bandwidth Notification Capability */ +#define PCI_EXP_LNKCAP_PN 0xff000000 /* Port Number */ +#define PCI_EXP_LNKCTL 0x10 /* Link Control */ +#define PCI_EXP_LNKCTL_ASPMC 0x0003 /* ASPM Control */ +#define PCI_EXP_LNKCTL_ASPM_L0S 0x0001 /* L0s Enable */ +#define PCI_EXP_LNKCTL_ASPM_L1 0x0002 /* L1 Enable */ +#define PCI_EXP_LNKCTL_RCB 0x0008 /* Read Completion Boundary */ +#define PCI_EXP_LNKCTL_LD 0x0010 /* Link Disable */ +#define PCI_EXP_LNKCTL_RL 0x0020 /* Retrain Link */ +#define PCI_EXP_LNKCTL_CCC 0x0040 /* Common Clock Configuration */ +#define PCI_EXP_LNKCTL_ES 0x0080 /* Extended Synch */ +#define PCI_EXP_LNKCTL_CLKREQ_EN 0x0100 /* Enable clkreq */ +#define PCI_EXP_LNKCTL_HAWD 0x0200 /* Hardware Autonomous Width Disable */ +#define PCI_EXP_LNKCTL_LBMIE 0x0400 /* Link Bandwidth Management Interrupt Enable */ +#define PCI_EXP_LNKCTL_LABIE 0x0800 /* Link Autonomous Bandwidth Interrupt Enable */ +#define PCI_EXP_LNKSTA 0x12 /* Link Status */ +#define PCI_EXP_LNKSTA_CLS 0x000f /* Current Link Speed */ +#define PCI_EXP_LNKSTA_CLS_2_5GB 0x0001 /* Current Link Speed 2.5GT/s */ +#define PCI_EXP_LNKSTA_CLS_5_0GB 0x0002 /* Current Link Speed 5.0GT/s */ +#define PCI_EXP_LNKSTA_CLS_8_0GB 0x0003 /* Current Link Speed 8.0GT/s */ +#define PCI_EXP_LNKSTA_CLS_16_0GB 0x0004 /* Current Link Speed 16.0GT/s */ +#define PCI_EXP_LNKSTA_CLS_32_0GB 0x0005 /* Current Link Speed 32.0GT/s */ +#define PCI_EXP_LNKSTA_CLS_64_0GB 0x0006 /* Current Link Speed 64.0GT/s */ +#define PCI_EXP_LNKSTA_NLW 0x03f0 /* Negotiated Link Width */ +#define PCI_EXP_LNKSTA_NLW_X1 0x0010 /* Current Link Width x1 */ +#define PCI_EXP_LNKSTA_NLW_X2 0x0020 /* Current Link Width x2 */ +#define PCI_EXP_LNKSTA_NLW_X4 0x0040 /* Current Link Width x4 */ +#define PCI_EXP_LNKSTA_NLW_X8 0x0080 /* Current Link Width x8 */ +#define PCI_EXP_LNKSTA_NLW_SHIFT 4 /* start of NLW mask in link status */ +#define PCI_EXP_LNKSTA_LT 0x0800 /* Link Training */ +#define PCI_EXP_LNKSTA_SLC 0x1000 /* Slot Clock Configuration */ +#define PCI_EXP_LNKSTA_DLLLA 0x2000 /* Data Link Layer Link Active */ +#define PCI_EXP_LNKSTA_LBMS 0x4000 /* Link Bandwidth Management Status */ +#define PCI_EXP_LNKSTA_LABS 0x8000 /* Link Autonomous Bandwidth Status */ +#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V1 20 /* v1 endpoints with link end here */ +#define PCI_EXP_SLTCAP 0x14 /* Slot Capabilities */ +#define PCI_EXP_SLTCAP_ABP 0x00000001 /* Attention Button Present */ +#define PCI_EXP_SLTCAP_PCP 0x00000002 /* Power Controller Present */ +#define PCI_EXP_SLTCAP_MRLSP 0x00000004 /* MRL Sensor Present */ +#define PCI_EXP_SLTCAP_AIP 0x00000008 /* Attention Indicator Present */ +#define PCI_EXP_SLTCAP_PIP 0x00000010 /* Power Indicator Present */ +#define PCI_EXP_SLTCAP_HPS 0x00000020 /* Hot-Plug Surprise */ +#define PCI_EXP_SLTCAP_HPC 0x00000040 /* Hot-Plug Capable */ +#define PCI_EXP_SLTCAP_SPLV 0x00007f80 /* Slot Power Limit Value */ +#define PCI_EXP_SLTCAP_SPLS 0x00018000 /* Slot Power Limit Scale */ +#define PCI_EXP_SLTCAP_EIP 0x00020000 /* Electromechanical Interlock Present */ +#define PCI_EXP_SLTCAP_NCCS 0x00040000 /* No Command Completed Support */ +#define PCI_EXP_SLTCAP_PSN 0xfff80000 /* Physical Slot Number */ +#define PCI_EXP_SLTCTL 0x18 /* Slot Control */ +#define PCI_EXP_SLTCTL_ABPE 0x0001 /* Attention Button Pressed Enable */ +#define PCI_EXP_SLTCTL_PFDE 0x0002 /* Power Fault Detected Enable */ +#define PCI_EXP_SLTCTL_MRLSCE 0x0004 /* MRL Sensor Changed Enable */ +#define PCI_EXP_SLTCTL_PDCE 0x0008 /* Presence Detect Changed Enable */ +#define PCI_EXP_SLTCTL_CCIE 0x0010 /* Command Completed Interrupt Enable */ +#define PCI_EXP_SLTCTL_HPIE 0x0020 /* Hot-Plug Interrupt Enable */ +#define PCI_EXP_SLTCTL_AIC 0x00c0 /* Attention Indicator Control */ +#define PCI_EXP_SLTCTL_ATTN_IND_SHIFT 6 /* Attention Indicator shift */ +#define PCI_EXP_SLTCTL_ATTN_IND_ON 0x0040 /* Attention Indicator on */ +#define PCI_EXP_SLTCTL_ATTN_IND_BLINK 0x0080 /* Attention Indicator blinking */ +#define PCI_EXP_SLTCTL_ATTN_IND_OFF 0x00c0 /* Attention Indicator off */ +#define PCI_EXP_SLTCTL_PIC 0x0300 /* Power Indicator Control */ +#define PCI_EXP_SLTCTL_PWR_IND_ON 0x0100 /* Power Indicator on */ +#define PCI_EXP_SLTCTL_PWR_IND_BLINK 0x0200 /* Power Indicator blinking */ +#define PCI_EXP_SLTCTL_PWR_IND_OFF 0x0300 /* Power Indicator off */ +#define PCI_EXP_SLTCTL_PCC 0x0400 /* Power Controller Control */ +#define PCI_EXP_SLTCTL_PWR_ON 0x0000 /* Power On */ +#define PCI_EXP_SLTCTL_PWR_OFF 0x0400 /* Power Off */ +#define PCI_EXP_SLTCTL_EIC 0x0800 /* Electromechanical Interlock Control */ +#define PCI_EXP_SLTCTL_DLLSCE 0x1000 /* Data Link Layer State Changed Enable */ +#define PCI_EXP_SLTCTL_ASPL_DISABLE 0x2000 /* Auto Slot Power Limit Disable */ +#define PCI_EXP_SLTCTL_IBPD_DISABLE 0x4000 /* In-band PD disable */ +#define PCI_EXP_SLTSTA 0x1a /* Slot Status */ +#define PCI_EXP_SLTSTA_ABP 0x0001 /* Attention Button Pressed */ +#define PCI_EXP_SLTSTA_PFD 0x0002 /* Power Fault Detected */ +#define PCI_EXP_SLTSTA_MRLSC 0x0004 /* MRL Sensor Changed */ +#define PCI_EXP_SLTSTA_PDC 0x0008 /* Presence Detect Changed */ +#define PCI_EXP_SLTSTA_CC 0x0010 /* Command Completed */ +#define PCI_EXP_SLTSTA_MRLSS 0x0020 /* MRL Sensor State */ +#define PCI_EXP_SLTSTA_PDS 0x0040 /* Presence Detect State */ +#define PCI_EXP_SLTSTA_EIS 0x0080 /* Electromechanical Interlock Status */ +#define PCI_EXP_SLTSTA_DLLSC 0x0100 /* Data Link Layer State Changed */ +#define PCI_EXP_RTCTL 0x1c /* Root Control */ +#define PCI_EXP_RTCTL_SECEE 0x0001 /* System Error on Correctable Error */ +#define PCI_EXP_RTCTL_SENFEE 0x0002 /* System Error on Non-Fatal Error */ +#define PCI_EXP_RTCTL_SEFEE 0x0004 /* System Error on Fatal Error */ +#define PCI_EXP_RTCTL_PMEIE 0x0008 /* PME Interrupt Enable */ +#define PCI_EXP_RTCTL_CRSSVE 0x0010 /* CRS Software Visibility Enable */ +#define PCI_EXP_RTCAP 0x1e /* Root Capabilities */ +#define PCI_EXP_RTCAP_CRSVIS 0x0001 /* CRS Software Visibility capability */ +#define PCI_EXP_RTSTA 0x20 /* Root Status */ +#define PCI_EXP_RTSTA_PME_RQ_ID 0x0000ffff /* PME Requester ID */ +#define PCI_EXP_RTSTA_PME 0x00010000 /* PME status */ +#define PCI_EXP_RTSTA_PENDING 0x00020000 /* PME pending */ +/* + * The Device Capabilities 2, Device Status 2, Device Control 2, + * Link Capabilities 2, Link Status 2, Link Control 2, + * Slot Capabilities 2, Slot Status 2, and Slot Control 2 registers + * are only present on devices with PCIe Capability version 2. + * Use pcie_capability_read_word() and similar interfaces to use them + * safely. + */ +#define PCI_EXP_DEVCAP2 0x24 /* Device Capabilities 2 */ +#define PCI_EXP_DEVCAP2_COMP_TMOUT_DIS 0x00000010 /* Completion Timeout Disable supported */ +#define PCI_EXP_DEVCAP2_ARI 0x00000020 /* Alternative Routing-ID */ +#define PCI_EXP_DEVCAP2_ATOMIC_ROUTE 0x00000040 /* Atomic Op routing */ +#define PCI_EXP_DEVCAP2_ATOMIC_COMP32 0x00000080 /* 32b AtomicOp completion */ +#define PCI_EXP_DEVCAP2_ATOMIC_COMP64 0x00000100 /* 64b AtomicOp completion */ +#define PCI_EXP_DEVCAP2_ATOMIC_COMP128 0x00000200 /* 128b AtomicOp completion */ +#define PCI_EXP_DEVCAP2_LTR 0x00000800 /* Latency tolerance reporting */ +#define PCI_EXP_DEVCAP2_OBFF_MASK 0x000c0000 /* OBFF support mechanism */ +#define PCI_EXP_DEVCAP2_OBFF_MSG 0x00040000 /* New message signaling */ +#define PCI_EXP_DEVCAP2_OBFF_WAKE 0x00080000 /* Re-use WAKE# for OBFF */ +#define PCI_EXP_DEVCAP2_EE_PREFIX 0x00200000 /* End-End TLP Prefix */ +#define PCI_EXP_DEVCTL2 0x28 /* Device Control 2 */ +#define PCI_EXP_DEVCTL2_COMP_TIMEOUT 0x000f /* Completion Timeout Value */ +#define PCI_EXP_DEVCTL2_COMP_TMOUT_DIS 0x0010 /* Completion Timeout Disable */ +#define PCI_EXP_DEVCTL2_ARI 0x0020 /* Alternative Routing-ID */ +#define PCI_EXP_DEVCTL2_ATOMIC_REQ 0x0040 /* Set Atomic requests */ +#define PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK 0x0080 /* Block atomic egress */ +#define PCI_EXP_DEVCTL2_IDO_REQ_EN 0x0100 /* Allow IDO for requests */ +#define PCI_EXP_DEVCTL2_IDO_CMP_EN 0x0200 /* Allow IDO for completions */ +#define PCI_EXP_DEVCTL2_LTR_EN 0x0400 /* Enable LTR mechanism */ +#define PCI_EXP_DEVCTL2_OBFF_MSGA_EN 0x2000 /* Enable OBFF Message type A */ +#define PCI_EXP_DEVCTL2_OBFF_MSGB_EN 0x4000 /* Enable OBFF Message type B */ +#define PCI_EXP_DEVCTL2_OBFF_WAKE_EN 0x6000 /* OBFF using WAKE# signaling */ +#define PCI_EXP_DEVSTA2 0x2a /* Device Status 2 */ +#define PCI_CAP_EXP_RC_ENDPOINT_SIZEOF_V2 0x2c /* end of v2 EPs w/o link */ +#define PCI_EXP_LNKCAP2 0x2c /* Link Capabilities 2 */ +#define PCI_EXP_LNKCAP2_SLS_2_5GB 0x00000002 /* Supported Speed 2.5GT/s */ +#define PCI_EXP_LNKCAP2_SLS_5_0GB 0x00000004 /* Supported Speed 5GT/s */ +#define PCI_EXP_LNKCAP2_SLS_8_0GB 0x00000008 /* Supported Speed 8GT/s */ +#define PCI_EXP_LNKCAP2_SLS_16_0GB 0x00000010 /* Supported Speed 16GT/s */ +#define PCI_EXP_LNKCAP2_SLS_32_0GB 0x00000020 /* Supported Speed 32GT/s */ +#define PCI_EXP_LNKCAP2_SLS_64_0GB 0x00000040 /* Supported Speed 64GT/s */ +#define PCI_EXP_LNKCAP2_CROSSLINK 0x00000100 /* Crosslink supported */ +#define PCI_EXP_LNKCTL2 0x30 /* Link Control 2 */ +#define PCI_EXP_LNKCTL2_TLS 0x000f +#define PCI_EXP_LNKCTL2_TLS_2_5GT 0x0001 /* Supported Speed 2.5GT/s */ +#define PCI_EXP_LNKCTL2_TLS_5_0GT 0x0002 /* Supported Speed 5GT/s */ +#define PCI_EXP_LNKCTL2_TLS_8_0GT 0x0003 /* Supported Speed 8GT/s */ +#define PCI_EXP_LNKCTL2_TLS_16_0GT 0x0004 /* Supported Speed 16GT/s */ +#define PCI_EXP_LNKCTL2_TLS_32_0GT 0x0005 /* Supported Speed 32GT/s */ +#define PCI_EXP_LNKCTL2_TLS_64_0GT 0x0006 /* Supported Speed 64GT/s */ +#define PCI_EXP_LNKCTL2_ENTER_COMP 0x0010 /* Enter Compliance */ +#define PCI_EXP_LNKCTL2_TX_MARGIN 0x0380 /* Transmit Margin */ +#define PCI_EXP_LNKCTL2_HASD 0x0020 /* HW Autonomous Speed Disable */ +#define PCI_EXP_LNKSTA2 0x32 /* Link Status 2 */ +#define PCI_EXP_LNKSTA2_FLIT 0x0400 /* Flit Mode Status */ +#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V2 0x32 /* end of v2 EPs w/ link */ +#define PCI_EXP_SLTCAP2 0x34 /* Slot Capabilities 2 */ +#define PCI_EXP_SLTCAP2_IBPD 0x00000001 /* In-band PD Disable Supported */ +#define PCI_EXP_SLTCTL2 0x38 /* Slot Control 2 */ +#define PCI_EXP_SLTSTA2 0x3a /* Slot Status 2 */ + +/* Extended Capabilities (PCI-X 2.0 and Express) */ +#define PCI_EXT_CAP_ID(header) (header & 0x0000ffff) +#define PCI_EXT_CAP_VER(header) ((header >> 16) & 0xf) +#define PCI_EXT_CAP_NEXT(header) ((header >> 20) & 0xffc) + +#define PCI_EXT_CAP_ID_ERR 0x01 /* Advanced Error Reporting */ +#define PCI_EXT_CAP_ID_VC 0x02 /* Virtual Channel Capability */ +#define PCI_EXT_CAP_ID_DSN 0x03 /* Device Serial Number */ +#define PCI_EXT_CAP_ID_PWR 0x04 /* Power Budgeting */ +#define PCI_EXT_CAP_ID_RCLD 0x05 /* Root Complex Link Declaration */ +#define PCI_EXT_CAP_ID_RCILC 0x06 /* Root Complex Internal Link Control */ +#define PCI_EXT_CAP_ID_RCEC 0x07 /* Root Complex Event Collector */ +#define PCI_EXT_CAP_ID_MFVC 0x08 /* Multi-Function VC Capability */ +#define PCI_EXT_CAP_ID_VC9 0x09 /* same as _VC */ +#define PCI_EXT_CAP_ID_RCRB 0x0A /* Root Complex RB? */ +#define PCI_EXT_CAP_ID_VNDR 0x0B /* Vendor-Specific */ +#define PCI_EXT_CAP_ID_CAC 0x0C /* Config Access - obsolete */ +#define PCI_EXT_CAP_ID_ACS 0x0D /* Access Control Services */ +#define PCI_EXT_CAP_ID_ARI 0x0E /* Alternate Routing ID */ +#define PCI_EXT_CAP_ID_ATS 0x0F /* Address Translation Services */ +#define PCI_EXT_CAP_ID_SRIOV 0x10 /* Single Root I/O Virtualization */ +#define PCI_EXT_CAP_ID_MRIOV 0x11 /* Multi Root I/O Virtualization */ +#define PCI_EXT_CAP_ID_MCAST 0x12 /* Multicast */ +#define PCI_EXT_CAP_ID_PRI 0x13 /* Page Request Interface */ +#define PCI_EXT_CAP_ID_AMD_XXX 0x14 /* Reserved for AMD */ +#define PCI_EXT_CAP_ID_REBAR 0x15 /* Resizable BAR */ +#define PCI_EXT_CAP_ID_DPA 0x16 /* Dynamic Power Allocation */ +#define PCI_EXT_CAP_ID_TPH 0x17 /* TPH Requester */ +#define PCI_EXT_CAP_ID_LTR 0x18 /* Latency Tolerance Reporting */ +#define PCI_EXT_CAP_ID_SECPCI 0x19 /* Secondary PCIe Capability */ +#define PCI_EXT_CAP_ID_PMUX 0x1A /* Protocol Multiplexing */ +#define PCI_EXT_CAP_ID_PASID 0x1B /* Process Address Space ID */ +#define PCI_EXT_CAP_ID_DPC 0x1D /* Downstream Port Containment */ +#define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */ +#define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */ +#define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */ +#define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */ +#define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */ +#define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */ +#define PCI_EXT_CAP_ID_DOE 0x2E /* Data Object Exchange */ +#define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_DOE + +#define PCI_EXT_CAP_DSN_SIZEOF 12 +#define PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF 40 + +/* Advanced Error Reporting */ +#define PCI_ERR_UNCOR_STATUS 0x04 /* Uncorrectable Error Status */ +#define PCI_ERR_UNC_UND 0x00000001 /* Undefined */ +#define PCI_ERR_UNC_DLP 0x00000010 /* Data Link Protocol */ +#define PCI_ERR_UNC_SURPDN 0x00000020 /* Surprise Down */ +#define PCI_ERR_UNC_POISON_TLP 0x00001000 /* Poisoned TLP */ +#define PCI_ERR_UNC_FCP 0x00002000 /* Flow Control Protocol */ +#define PCI_ERR_UNC_COMP_TIME 0x00004000 /* Completion Timeout */ +#define PCI_ERR_UNC_COMP_ABORT 0x00008000 /* Completer Abort */ +#define PCI_ERR_UNC_UNX_COMP 0x00010000 /* Unexpected Completion */ +#define PCI_ERR_UNC_RX_OVER 0x00020000 /* Receiver Overflow */ +#define PCI_ERR_UNC_MALF_TLP 0x00040000 /* Malformed TLP */ +#define PCI_ERR_UNC_ECRC 0x00080000 /* ECRC Error Status */ +#define PCI_ERR_UNC_UNSUP 0x00100000 /* Unsupported Request */ +#define PCI_ERR_UNC_ACSV 0x00200000 /* ACS Violation */ +#define PCI_ERR_UNC_INTN 0x00400000 /* internal error */ +#define PCI_ERR_UNC_MCBTLP 0x00800000 /* MC blocked TLP */ +#define PCI_ERR_UNC_ATOMEG 0x01000000 /* Atomic egress blocked */ +#define PCI_ERR_UNC_TLPPRE 0x02000000 /* TLP prefix blocked */ +#define PCI_ERR_UNCOR_MASK 0x08 /* Uncorrectable Error Mask */ + /* Same bits as above */ +#define PCI_ERR_UNCOR_SEVER 0x0c /* Uncorrectable Error Severity */ + /* Same bits as above */ +#define PCI_ERR_COR_STATUS 0x10 /* Correctable Error Status */ +#define PCI_ERR_COR_RCVR 0x00000001 /* Receiver Error Status */ +#define PCI_ERR_COR_BAD_TLP 0x00000040 /* Bad TLP Status */ +#define PCI_ERR_COR_BAD_DLLP 0x00000080 /* Bad DLLP Status */ +#define PCI_ERR_COR_REP_ROLL 0x00000100 /* REPLAY_NUM Rollover */ +#define PCI_ERR_COR_REP_TIMER 0x00001000 /* Replay Timer Timeout */ +#define PCI_ERR_COR_ADV_NFAT 0x00002000 /* Advisory Non-Fatal */ +#define PCI_ERR_COR_INTERNAL 0x00004000 /* Corrected Internal */ +#define PCI_ERR_COR_LOG_OVER 0x00008000 /* Header Log Overflow */ +#define PCI_ERR_COR_MASK 0x14 /* Correctable Error Mask */ + /* Same bits as above */ +#define PCI_ERR_CAP 0x18 /* Advanced Error Capabilities & Ctrl*/ +#define PCI_ERR_CAP_FEP(x) ((x) & 0x1f) /* First Error Pointer */ +#define PCI_ERR_CAP_ECRC_GENC 0x00000020 /* ECRC Generation Capable */ +#define PCI_ERR_CAP_ECRC_GENE 0x00000040 /* ECRC Generation Enable */ +#define PCI_ERR_CAP_ECRC_CHKC 0x00000080 /* ECRC Check Capable */ +#define PCI_ERR_CAP_ECRC_CHKE 0x00000100 /* ECRC Check Enable */ +#define PCI_ERR_HEADER_LOG 0x1c /* Header Log Register (16 bytes) */ +#define PCI_ERR_ROOT_COMMAND 0x2c /* Root Error Command */ +#define PCI_ERR_ROOT_CMD_COR_EN 0x00000001 /* Correctable Err Reporting Enable */ +#define PCI_ERR_ROOT_CMD_NONFATAL_EN 0x00000002 /* Non-Fatal Err Reporting Enable */ +#define PCI_ERR_ROOT_CMD_FATAL_EN 0x00000004 /* Fatal Err Reporting Enable */ +#define PCI_ERR_ROOT_STATUS 0x30 +#define PCI_ERR_ROOT_COR_RCV 0x00000001 /* ERR_COR Received */ +#define PCI_ERR_ROOT_MULTI_COR_RCV 0x00000002 /* Multiple ERR_COR */ +#define PCI_ERR_ROOT_UNCOR_RCV 0x00000004 /* ERR_FATAL/NONFATAL */ +#define PCI_ERR_ROOT_MULTI_UNCOR_RCV 0x00000008 /* Multiple FATAL/NONFATAL */ +#define PCI_ERR_ROOT_FIRST_FATAL 0x00000010 /* First UNC is Fatal */ +#define PCI_ERR_ROOT_NONFATAL_RCV 0x00000020 /* Non-Fatal Received */ +#define PCI_ERR_ROOT_FATAL_RCV 0x00000040 /* Fatal Received */ +#define PCI_ERR_ROOT_AER_IRQ 0xf8000000 /* Advanced Error Interrupt Message Number */ +#define PCI_ERR_ROOT_ERR_SRC 0x34 /* Error Source Identification */ + +/* Virtual Channel */ +#define PCI_VC_PORT_CAP1 0x04 +#define PCI_VC_CAP1_EVCC 0x00000007 /* extended VC count */ +#define PCI_VC_CAP1_LPEVCC 0x00000070 /* low prio extended VC count */ +#define PCI_VC_CAP1_ARB_SIZE 0x00000c00 +#define PCI_VC_PORT_CAP2 0x08 +#define PCI_VC_CAP2_32_PHASE 0x00000002 +#define PCI_VC_CAP2_64_PHASE 0x00000004 +#define PCI_VC_CAP2_128_PHASE 0x00000008 +#define PCI_VC_CAP2_ARB_OFF 0xff000000 +#define PCI_VC_PORT_CTRL 0x0c +#define PCI_VC_PORT_CTRL_LOAD_TABLE 0x00000001 +#define PCI_VC_PORT_STATUS 0x0e +#define PCI_VC_PORT_STATUS_TABLE 0x00000001 +#define PCI_VC_RES_CAP 0x10 +#define PCI_VC_RES_CAP_32_PHASE 0x00000002 +#define PCI_VC_RES_CAP_64_PHASE 0x00000004 +#define PCI_VC_RES_CAP_128_PHASE 0x00000008 +#define PCI_VC_RES_CAP_128_PHASE_TB 0x00000010 +#define PCI_VC_RES_CAP_256_PHASE 0x00000020 +#define PCI_VC_RES_CAP_ARB_OFF 0xff000000 +#define PCI_VC_RES_CTRL 0x14 +#define PCI_VC_RES_CTRL_LOAD_TABLE 0x00010000 +#define PCI_VC_RES_CTRL_ARB_SELECT 0x000e0000 +#define PCI_VC_RES_CTRL_ID 0x07000000 +#define PCI_VC_RES_CTRL_ENABLE 0x80000000 +#define PCI_VC_RES_STATUS 0x1a +#define PCI_VC_RES_STATUS_TABLE 0x00000001 +#define PCI_VC_RES_STATUS_NEGO 0x00000002 +#define PCI_CAP_VC_BASE_SIZEOF 0x10 +#define PCI_CAP_VC_PER_VC_SIZEOF 0x0c + +/* Power Budgeting */ +#define PCI_PWR_DSR 0x04 /* Data Select Register */ +#define PCI_PWR_DATA 0x08 /* Data Register */ +#define PCI_PWR_DATA_BASE(x) ((x) & 0xff) /* Base Power */ +#define PCI_PWR_DATA_SCALE(x) (((x) >> 8) & 3) /* Data Scale */ +#define PCI_PWR_DATA_PM_SUB(x) (((x) >> 10) & 7) /* PM Sub State */ +#define PCI_PWR_DATA_PM_STATE(x) (((x) >> 13) & 3) /* PM State */ +#define PCI_PWR_DATA_TYPE(x) (((x) >> 15) & 7) /* Type */ +#define PCI_PWR_DATA_RAIL(x) (((x) >> 18) & 7) /* Power Rail */ +#define PCI_PWR_CAP 0x0c /* Capability */ +#define PCI_PWR_CAP_BUDGET(x) ((x) & 1) /* Included in system budget */ +#define PCI_EXT_CAP_PWR_SIZEOF 0x10 + +/* Root Complex Event Collector Endpoint Association */ +#define PCI_RCEC_RCIEP_BITMAP 4 /* Associated Bitmap for RCiEPs */ +#define PCI_RCEC_BUSN 8 /* RCEC Associated Bus Numbers */ +#define PCI_RCEC_BUSN_REG_VER 0x02 /* Least version with BUSN present */ +#define PCI_RCEC_BUSN_NEXT(x) (((x) >> 8) & 0xff) +#define PCI_RCEC_BUSN_LAST(x) (((x) >> 16) & 0xff) + +/* Vendor-Specific (VSEC, PCI_EXT_CAP_ID_VNDR) */ +#define PCI_VNDR_HEADER 4 /* Vendor-Specific Header */ +#define PCI_VNDR_HEADER_ID(x) ((x) & 0xffff) +#define PCI_VNDR_HEADER_REV(x) (((x) >> 16) & 0xf) +#define PCI_VNDR_HEADER_LEN(x) (((x) >> 20) & 0xfff) + +/* + * HyperTransport sub capability types + * + * Unfortunately there are both 3 bit and 5 bit capability types defined + * in the HT spec, catering for that is a little messy. You probably don't + * want to use these directly, just use pci_find_ht_capability() and it + * will do the right thing for you. + */ +#define HT_3BIT_CAP_MASK 0xE0 +#define HT_CAPTYPE_SLAVE 0x00 /* Slave/Primary link configuration */ +#define HT_CAPTYPE_HOST 0x20 /* Host/Secondary link configuration */ + +#define HT_5BIT_CAP_MASK 0xF8 +#define HT_CAPTYPE_IRQ 0x80 /* IRQ Configuration */ +#define HT_CAPTYPE_REMAPPING_40 0xA0 /* 40 bit address remapping */ +#define HT_CAPTYPE_REMAPPING_64 0xA2 /* 64 bit address remapping */ +#define HT_CAPTYPE_UNITID_CLUMP 0x90 /* Unit ID clumping */ +#define HT_CAPTYPE_EXTCONF 0x98 /* Extended Configuration Space Access */ +#define HT_CAPTYPE_MSI_MAPPING 0xA8 /* MSI Mapping Capability */ +#define HT_MSI_FLAGS 0x02 /* Offset to flags */ +#define HT_MSI_FLAGS_ENABLE 0x1 /* Mapping enable */ +#define HT_MSI_FLAGS_FIXED 0x2 /* Fixed mapping only */ +#define HT_MSI_FIXED_ADDR 0x00000000FEE00000ULL /* Fixed addr */ +#define HT_MSI_ADDR_LO 0x04 /* Offset to low addr bits */ +#define HT_MSI_ADDR_LO_MASK 0xFFF00000 /* Low address bit mask */ +#define HT_MSI_ADDR_HI 0x08 /* Offset to high addr bits */ +#define HT_CAPTYPE_DIRECT_ROUTE 0xB0 /* Direct routing configuration */ +#define HT_CAPTYPE_VCSET 0xB8 /* Virtual Channel configuration */ +#define HT_CAPTYPE_ERROR_RETRY 0xC0 /* Retry on error configuration */ +#define HT_CAPTYPE_GEN3 0xD0 /* Generation 3 HyperTransport configuration */ +#define HT_CAPTYPE_PM 0xE0 /* HyperTransport power management configuration */ +#define HT_CAP_SIZEOF_LONG 28 /* slave & primary */ +#define HT_CAP_SIZEOF_SHORT 24 /* host & secondary */ + +/* Alternative Routing-ID Interpretation */ +#define PCI_ARI_CAP 0x04 /* ARI Capability Register */ +#define PCI_ARI_CAP_MFVC 0x0001 /* MFVC Function Groups Capability */ +#define PCI_ARI_CAP_ACS 0x0002 /* ACS Function Groups Capability */ +#define PCI_ARI_CAP_NFN(x) (((x) >> 8) & 0xff) /* Next Function Number */ +#define PCI_ARI_CTRL 0x06 /* ARI Control Register */ +#define PCI_ARI_CTRL_MFVC 0x0001 /* MFVC Function Groups Enable */ +#define PCI_ARI_CTRL_ACS 0x0002 /* ACS Function Groups Enable */ +#define PCI_ARI_CTRL_FG(x) (((x) >> 4) & 7) /* Function Group */ +#define PCI_EXT_CAP_ARI_SIZEOF 8 + +/* Address Translation Service */ +#define PCI_ATS_CAP 0x04 /* ATS Capability Register */ +#define PCI_ATS_CAP_QDEP(x) ((x) & 0x1f) /* Invalidate Queue Depth */ +#define PCI_ATS_MAX_QDEP 32 /* Max Invalidate Queue Depth */ +#define PCI_ATS_CAP_PAGE_ALIGNED 0x0020 /* Page Aligned Request */ +#define PCI_ATS_CTRL 0x06 /* ATS Control Register */ +#define PCI_ATS_CTRL_ENABLE 0x8000 /* ATS Enable */ +#define PCI_ATS_CTRL_STU(x) ((x) & 0x1f) /* Smallest Translation Unit */ +#define PCI_ATS_MIN_STU 12 /* shift of minimum STU block */ +#define PCI_EXT_CAP_ATS_SIZEOF 8 + +/* Page Request Interface */ +#define PCI_PRI_CTRL 0x04 /* PRI control register */ +#define PCI_PRI_CTRL_ENABLE 0x0001 /* Enable */ +#define PCI_PRI_CTRL_RESET 0x0002 /* Reset */ +#define PCI_PRI_STATUS 0x06 /* PRI status register */ +#define PCI_PRI_STATUS_RF 0x0001 /* Response Failure */ +#define PCI_PRI_STATUS_UPRGI 0x0002 /* Unexpected PRG index */ +#define PCI_PRI_STATUS_STOPPED 0x0100 /* PRI Stopped */ +#define PCI_PRI_STATUS_PASID 0x8000 /* PRG Response PASID Required */ +#define PCI_PRI_MAX_REQ 0x08 /* PRI max reqs supported */ +#define PCI_PRI_ALLOC_REQ 0x0c /* PRI max reqs allowed */ +#define PCI_EXT_CAP_PRI_SIZEOF 16 + +/* Process Address Space ID */ +#define PCI_PASID_CAP 0x04 /* PASID feature register */ +#define PCI_PASID_CAP_EXEC 0x0002 /* Exec permissions Supported */ +#define PCI_PASID_CAP_PRIV 0x0004 /* Privilege Mode Supported */ +#define PCI_PASID_CAP_WIDTH 0x1f00 +#define PCI_PASID_CTRL 0x06 /* PASID control register */ +#define PCI_PASID_CTRL_ENABLE 0x0001 /* Enable bit */ +#define PCI_PASID_CTRL_EXEC 0x0002 /* Exec permissions Enable */ +#define PCI_PASID_CTRL_PRIV 0x0004 /* Privilege Mode Enable */ +#define PCI_EXT_CAP_PASID_SIZEOF 8 + +/* Single Root I/O Virtualization */ +#define PCI_SRIOV_CAP 0x04 /* SR-IOV Capabilities */ +#define PCI_SRIOV_CAP_VFM 0x00000001 /* VF Migration Capable */ +#define PCI_SRIOV_CAP_INTR(x) ((x) >> 21) /* Interrupt Message Number */ +#define PCI_SRIOV_CTRL 0x08 /* SR-IOV Control */ +#define PCI_SRIOV_CTRL_VFE 0x0001 /* VF Enable */ +#define PCI_SRIOV_CTRL_VFM 0x0002 /* VF Migration Enable */ +#define PCI_SRIOV_CTRL_INTR 0x0004 /* VF Migration Interrupt Enable */ +#define PCI_SRIOV_CTRL_MSE 0x0008 /* VF Memory Space Enable */ +#define PCI_SRIOV_CTRL_ARI 0x0010 /* ARI Capable Hierarchy */ +#define PCI_SRIOV_STATUS 0x0a /* SR-IOV Status */ +#define PCI_SRIOV_STATUS_VFM 0x0001 /* VF Migration Status */ +#define PCI_SRIOV_INITIAL_VF 0x0c /* Initial VFs */ +#define PCI_SRIOV_TOTAL_VF 0x0e /* Total VFs */ +#define PCI_SRIOV_NUM_VF 0x10 /* Number of VFs */ +#define PCI_SRIOV_FUNC_LINK 0x12 /* Function Dependency Link */ +#define PCI_SRIOV_VF_OFFSET 0x14 /* First VF Offset */ +#define PCI_SRIOV_VF_STRIDE 0x16 /* Following VF Stride */ +#define PCI_SRIOV_VF_DID 0x1a /* VF Device ID */ +#define PCI_SRIOV_SUP_PGSIZE 0x1c /* Supported Page Sizes */ +#define PCI_SRIOV_SYS_PGSIZE 0x20 /* System Page Size */ +#define PCI_SRIOV_BAR 0x24 /* VF BAR0 */ +#define PCI_SRIOV_NUM_BARS 6 /* Number of VF BARs */ +#define PCI_SRIOV_VFM 0x3c /* VF Migration State Array Offset*/ +#define PCI_SRIOV_VFM_BIR(x) ((x) & 7) /* State BIR */ +#define PCI_SRIOV_VFM_OFFSET(x) ((x) & ~7) /* State Offset */ +#define PCI_SRIOV_VFM_UA 0x0 /* Inactive.Unavailable */ +#define PCI_SRIOV_VFM_MI 0x1 /* Dormant.MigrateIn */ +#define PCI_SRIOV_VFM_MO 0x2 /* Active.MigrateOut */ +#define PCI_SRIOV_VFM_AV 0x3 /* Active.Available */ +#define PCI_EXT_CAP_SRIOV_SIZEOF 0x40 + +#define PCI_LTR_MAX_SNOOP_LAT 0x4 +#define PCI_LTR_MAX_NOSNOOP_LAT 0x6 +#define PCI_LTR_VALUE_MASK 0x000003ff +#define PCI_LTR_SCALE_MASK 0x00001c00 +#define PCI_LTR_SCALE_SHIFT 10 +#define PCI_LTR_NOSNOOP_VALUE 0x03ff0000 /* Max No-Snoop Latency Value */ +#define PCI_LTR_NOSNOOP_SCALE 0x1c000000 /* Scale for Max Value */ +#define PCI_EXT_CAP_LTR_SIZEOF 8 + +/* Access Control Service */ +#define PCI_ACS_CAP 0x04 /* ACS Capability Register */ +#define PCI_ACS_SV 0x0001 /* Source Validation */ +#define PCI_ACS_TB 0x0002 /* Translation Blocking */ +#define PCI_ACS_RR 0x0004 /* P2P Request Redirect */ +#define PCI_ACS_CR 0x0008 /* P2P Completion Redirect */ +#define PCI_ACS_UF 0x0010 /* Upstream Forwarding */ +#define PCI_ACS_EC 0x0020 /* P2P Egress Control */ +#define PCI_ACS_DT 0x0040 /* Direct Translated P2P */ +#define PCI_ACS_EGRESS_BITS 0x05 /* ACS Egress Control Vector Size */ +#define PCI_ACS_CTRL 0x06 /* ACS Control Register */ +#define PCI_ACS_EGRESS_CTL_V 0x08 /* ACS Egress Control Vector */ + +#define PCI_VSEC_HDR 4 /* extended cap - vendor-specific */ +#define PCI_VSEC_HDR_LEN_SHIFT 20 /* shift for length field */ + +/* SATA capability */ +#define PCI_SATA_REGS 4 /* SATA REGs specifier */ +#define PCI_SATA_REGS_MASK 0xF /* location - BAR#/inline */ +#define PCI_SATA_REGS_INLINE 0xF /* REGS in config space */ +#define PCI_SATA_SIZEOF_SHORT 8 +#define PCI_SATA_SIZEOF_LONG 16 + +/* Resizable BARs */ +#define PCI_REBAR_CAP 4 /* capability register */ +#define PCI_REBAR_CAP_SIZES 0x00FFFFF0 /* supported BAR sizes */ +#define PCI_REBAR_CTRL 8 /* control register */ +#define PCI_REBAR_CTRL_BAR_IDX 0x00000007 /* BAR index */ +#define PCI_REBAR_CTRL_NBAR_MASK 0x000000E0 /* # of resizable BARs */ +#define PCI_REBAR_CTRL_NBAR_SHIFT 5 /* shift for # of BARs */ +#define PCI_REBAR_CTRL_BAR_SIZE 0x00001F00 /* BAR size */ +#define PCI_REBAR_CTRL_BAR_SHIFT 8 /* shift for BAR size */ + +/* Dynamic Power Allocation */ +#define PCI_DPA_CAP 4 /* capability register */ +#define PCI_DPA_CAP_SUBSTATE_MASK 0x1F /* # substates - 1 */ +#define PCI_DPA_BASE_SIZEOF 16 /* size with 0 substates */ + +/* TPH Requester */ +#define PCI_TPH_CAP 4 /* capability register */ +#define PCI_TPH_CAP_LOC_MASK 0x600 /* location mask */ +#define PCI_TPH_LOC_NONE 0x000 /* no location */ +#define PCI_TPH_LOC_CAP 0x200 /* in capability */ +#define PCI_TPH_LOC_MSIX 0x400 /* in MSI-X */ +#define PCI_TPH_CAP_ST_MASK 0x07FF0000 /* ST table mask */ +#define PCI_TPH_CAP_ST_SHIFT 16 /* ST table shift */ +#define PCI_TPH_BASE_SIZEOF 0xc /* size with no ST table */ + +/* Downstream Port Containment */ +#define PCI_EXP_DPC_CAP 0x04 /* DPC Capability */ +#define PCI_EXP_DPC_IRQ 0x001F /* Interrupt Message Number */ +#define PCI_EXP_DPC_CAP_RP_EXT 0x0020 /* Root Port Extensions */ +#define PCI_EXP_DPC_CAP_POISONED_TLP 0x0040 /* Poisoned TLP Egress Blocking Supported */ +#define PCI_EXP_DPC_CAP_SW_TRIGGER 0x0080 /* Software Triggering Supported */ +#define PCI_EXP_DPC_RP_PIO_LOG_SIZE 0x0F00 /* RP PIO Log Size */ +#define PCI_EXP_DPC_CAP_DL_ACTIVE 0x1000 /* ERR_COR signal on DL_Active supported */ + +#define PCI_EXP_DPC_CTL 0x06 /* DPC control */ +#define PCI_EXP_DPC_CTL_EN_FATAL 0x0001 /* Enable trigger on ERR_FATAL message */ +#define PCI_EXP_DPC_CTL_EN_NONFATAL 0x0002 /* Enable trigger on ERR_NONFATAL message */ +#define PCI_EXP_DPC_CTL_INT_EN 0x0008 /* DPC Interrupt Enable */ + +#define PCI_EXP_DPC_STATUS 0x08 /* DPC Status */ +#define PCI_EXP_DPC_STATUS_TRIGGER 0x0001 /* Trigger Status */ +#define PCI_EXP_DPC_STATUS_TRIGGER_RSN 0x0006 /* Trigger Reason */ +#define PCI_EXP_DPC_STATUS_TRIGGER_RSN_UNCOR 0x0000 /* Uncorrectable error */ +#define PCI_EXP_DPC_STATUS_TRIGGER_RSN_NFE 0x0002 /* Rcvd ERR_NONFATAL */ +#define PCI_EXP_DPC_STATUS_TRIGGER_RSN_FE 0x0004 /* Rcvd ERR_FATAL */ +#define PCI_EXP_DPC_STATUS_TRIGGER_RSN_IN_EXT 0x0006 /* Reason in Trig Reason Extension field */ +#define PCI_EXP_DPC_STATUS_INTERRUPT 0x0008 /* Interrupt Status */ +#define PCI_EXP_DPC_RP_BUSY 0x0010 /* Root Port Busy */ +#define PCI_EXP_DPC_STATUS_TRIGGER_RSN_EXT 0x0060 /* Trig Reason Extension */ +#define PCI_EXP_DPC_STATUS_TRIGGER_RSN_RP_PIO 0x0000 /* RP PIO error */ +#define PCI_EXP_DPC_STATUS_TRIGGER_RSN_SW_TRIGGER 0x0020 /* DPC SW Trigger bit */ +#define PCI_EXP_DPC_RP_PIO_FEP 0x1f00 /* RP PIO First Err Ptr */ + +#define PCI_EXP_DPC_SOURCE_ID 0x0A /* DPC Source Identifier */ + +#define PCI_EXP_DPC_RP_PIO_STATUS 0x0C /* RP PIO Status */ +#define PCI_EXP_DPC_RP_PIO_MASK 0x10 /* RP PIO Mask */ +#define PCI_EXP_DPC_RP_PIO_SEVERITY 0x14 /* RP PIO Severity */ +#define PCI_EXP_DPC_RP_PIO_SYSERROR 0x18 /* RP PIO SysError */ +#define PCI_EXP_DPC_RP_PIO_EXCEPTION 0x1C /* RP PIO Exception */ +#define PCI_EXP_DPC_RP_PIO_HEADER_LOG 0x20 /* RP PIO Header Log */ +#define PCI_EXP_DPC_RP_PIO_IMPSPEC_LOG 0x30 /* RP PIO ImpSpec Log */ +#define PCI_EXP_DPC_RP_PIO_TLPPREFIX_LOG 0x34 /* RP PIO TLP Prefix Log */ + +/* Precision Time Measurement */ +#define PCI_PTM_CAP 0x04 /* PTM Capability */ +#define PCI_PTM_CAP_REQ 0x00000001 /* Requester capable */ +#define PCI_PTM_CAP_RES 0x00000002 /* Responder capable */ +#define PCI_PTM_CAP_ROOT 0x00000004 /* Root capable */ +#define PCI_PTM_GRANULARITY_MASK 0x0000FF00 /* Clock granularity */ +#define PCI_PTM_CTRL 0x08 /* PTM Control */ +#define PCI_PTM_CTRL_ENABLE 0x00000001 /* PTM enable */ +#define PCI_PTM_CTRL_ROOT 0x00000002 /* Root select */ + +/* ASPM L1 PM Substates */ +#define PCI_L1SS_CAP 0x04 /* Capabilities Register */ +#define PCI_L1SS_CAP_PCIPM_L1_2 0x00000001 /* PCI-PM L1.2 Supported */ +#define PCI_L1SS_CAP_PCIPM_L1_1 0x00000002 /* PCI-PM L1.1 Supported */ +#define PCI_L1SS_CAP_ASPM_L1_2 0x00000004 /* ASPM L1.2 Supported */ +#define PCI_L1SS_CAP_ASPM_L1_1 0x00000008 /* ASPM L1.1 Supported */ +#define PCI_L1SS_CAP_L1_PM_SS 0x00000010 /* L1 PM Substates Supported */ +#define PCI_L1SS_CAP_CM_RESTORE_TIME 0x0000ff00 /* Port Common_Mode_Restore_Time */ +#define PCI_L1SS_CAP_P_PWR_ON_SCALE 0x00030000 /* Port T_POWER_ON scale */ +#define PCI_L1SS_CAP_P_PWR_ON_VALUE 0x00f80000 /* Port T_POWER_ON value */ +#define PCI_L1SS_CTL1 0x08 /* Control 1 Register */ +#define PCI_L1SS_CTL1_PCIPM_L1_2 0x00000001 /* PCI-PM L1.2 Enable */ +#define PCI_L1SS_CTL1_PCIPM_L1_1 0x00000002 /* PCI-PM L1.1 Enable */ +#define PCI_L1SS_CTL1_ASPM_L1_2 0x00000004 /* ASPM L1.2 Enable */ +#define PCI_L1SS_CTL1_ASPM_L1_1 0x00000008 /* ASPM L1.1 Enable */ +#define PCI_L1SS_CTL1_L1_2_MASK 0x00000005 +#define PCI_L1SS_CTL1_L1SS_MASK 0x0000000f +#define PCI_L1SS_CTL1_CM_RESTORE_TIME 0x0000ff00 /* Common_Mode_Restore_Time */ +#define PCI_L1SS_CTL1_LTR_L12_TH_VALUE 0x03ff0000 /* LTR_L1.2_THRESHOLD_Value */ +#define PCI_L1SS_CTL1_LTR_L12_TH_SCALE 0xe0000000 /* LTR_L1.2_THRESHOLD_Scale */ +#define PCI_L1SS_CTL2 0x0c /* Control 2 Register */ +#define PCI_L1SS_CTL2_T_PWR_ON_SCALE 0x00000003 /* T_POWER_ON Scale */ +#define PCI_L1SS_CTL2_T_PWR_ON_VALUE 0x000000f8 /* T_POWER_ON Value */ + +/* Designated Vendor-Specific (DVSEC, PCI_EXT_CAP_ID_DVSEC) */ +#define PCI_DVSEC_HEADER1 0x4 /* Designated Vendor-Specific Header1 */ +#define PCI_DVSEC_HEADER1_VID(x) ((x) & 0xffff) +#define PCI_DVSEC_HEADER1_REV(x) (((x) >> 16) & 0xf) +#define PCI_DVSEC_HEADER1_LEN(x) (((x) >> 20) & 0xfff) +#define PCI_DVSEC_HEADER2 0x8 /* Designated Vendor-Specific Header2 */ +#define PCI_DVSEC_HEADER2_ID(x) ((x) & 0xffff) + +/* Data Link Feature */ +#define PCI_DLF_CAP 0x04 /* Capabilities Register */ +#define PCI_DLF_EXCHANGE_ENABLE 0x80000000 /* Data Link Feature Exchange Enable */ + +/* Physical Layer 16.0 GT/s */ +#define PCI_PL_16GT_LE_CTRL 0x20 /* Lane Equalization Control Register */ +#define PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK 0x0000000F +#define PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK 0x000000F0 +#define PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT 4 + +/* Data Object Exchange */ +#define PCI_DOE_CAP 0x04 /* DOE Capabilities Register */ +#define PCI_DOE_CAP_INT_SUP 0x00000001 /* Interrupt Support */ +#define PCI_DOE_CAP_INT_MSG_NUM 0x00000ffe /* Interrupt Message Number */ +#define PCI_DOE_CTRL 0x08 /* DOE Control Register */ +#define PCI_DOE_CTRL_ABORT 0x00000001 /* DOE Abort */ +#define PCI_DOE_CTRL_INT_EN 0x00000002 /* DOE Interrupt Enable */ +#define PCI_DOE_CTRL_GO 0x80000000 /* DOE Go */ +#define PCI_DOE_STATUS 0x0c /* DOE Status Register */ +#define PCI_DOE_STATUS_BUSY 0x00000001 /* DOE Busy */ +#define PCI_DOE_STATUS_INT_STATUS 0x00000002 /* DOE Interrupt Status */ +#define PCI_DOE_STATUS_ERROR 0x00000004 /* DOE Error */ +#define PCI_DOE_STATUS_DATA_OBJECT_READY 0x80000000 /* Data Object Ready */ +#define PCI_DOE_WRITE 0x10 /* DOE Write Data Mailbox Register */ +#define PCI_DOE_READ 0x14 /* DOE Read Data Mailbox Register */ +#define PCI_DOE_CAP_SIZEOF 0x18 /* Size of DOE register block */ + +/* DOE Data Object - note not actually registers */ +#define PCI_DOE_DATA_OBJECT_HEADER_1_VID 0x0000ffff +#define PCI_DOE_DATA_OBJECT_HEADER_1_TYPE 0x00ff0000 +#define PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH 0x0003ffff + +#define PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX 0x000000ff +#define PCI_DOE_DATA_OBJECT_DISC_REQ_3_VER 0x0000ff00 +#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID 0x0000ffff +#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL 0x00ff0000 +#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX 0xff000000 + +/* Compute Express Link (CXL r3.1, sec 8.1.5) */ +#define PCI_DVSEC_CXL_PORT 3 +#define PCI_DVSEC_CXL_PORT_CTL 0x0c +#define PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR 0x00000001 + +#endif /* LINUX_PCI_REGS_H */ diff --git a/systemc-components/pci/gs_gpex/include/pci_tlps.h b/systemc-components/pci/gs_gpex/include/pci_tlps.h new file mode 100644 index 00000000..852cfd34 --- /dev/null +++ b/systemc-components/pci/gs_gpex/include/pci_tlps.h @@ -0,0 +1,450 @@ +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Author: jhieb@micron.com 2025 + * + * SPDX-License-Identifier: BSD-3-Clause + */ + + +#ifndef PCI_TLPS_H +#define PCI_TLPS_H + + +typedef union _TLPHeader3DW { + uint8_t data[12]; // 3 DWORDs = 12 bytes + uint32_t dwords[3]; // Access as DWORDs for byte swapping + struct { + // DW0 (Byte 0-3) + struct { + uint32_t length : 10; // Bits 0-9: Length in DWs + uint32_t at : 2; // Bits 10-11: Address Type (00 = untranslated) + uint32_t attr : 2; // Bits 12-13: Attributes (Relaxed Ordering, No Snoop) + uint32_t ep : 1; // Bit 14: Poisoned TLP + uint32_t td : 1; // Bit 15: TLP Digest present + uint32_t th : 1; // Bit 16: TLP Processing Hints + uint32_t ln : 1; // Bit 17: Lightweight Notification + uint32_t reserved1 : 2; // Bits 18-19: Reserved + uint32_t tc : 3; // Bits 20-22: Traffic Class + uint32_t reserved2 : 1; // Bit 23: Reserved + uint32_t type : 5; // Bits 24-28: Type (00100b = CfgRd0, 00101b = CfgRd1) + uint32_t fmt : 2; // Bits 29-30: Format (00b = 3DW, no data) + uint32_t reserved3 : 1; // Bit 31: Reserved + } dword0; + + // DW1 (Byte 4-7) + struct { + uint32_t first_be : 4; // Bits 0-3: First DW Byte Enable + uint32_t last_be : 4; // Bits 4-7: Last DW Byte Enable + uint32_t tag : 8; // Bits 8-15: Tag + uint32_t requester_id : 16; // Bits 16-31: Requester ID (Bus:Dev:Func) + } dword1; + + // DW2 (Byte 8-11) + struct { + uint32_t reg_num : 2; // Bits 0-1: Register Number (must be 00b) + uint32_t reg_offset : 10; // Bits 2-11: Extended Register Number (DWORD aligned) + uint32_t reserved : 4; // Bits 12-15: Reserved + uint32_t function : 3; // Bits 16-18: Function Number + uint32_t device : 5; // Bits 19-23: Device Number + uint32_t bus : 8; // Bits 24-31: Bus Number + } dword2; + }; + + _TLPHeader3DW() { memset(data, 0, sizeof(data)); } + + // Helper to byte-swap to big-endian (network byte order) + void to_big_endian() { + dwords[0] = __builtin_bswap32(dwords[0]); + dwords[1] = __builtin_bswap32(dwords[1]); + dwords[2] = __builtin_bswap32(dwords[2]); + } + +} TLPHeader3DW; + +typedef union _TLPCompletion3DW { + uint8_t data[16]; // 3 DW header + 1 DW data = 16 bytes + uint32_t dwords[4]; // Access as DWORDs for byte swapping + struct { + // DW0 (Byte 0-3) + struct { + uint32_t length : 10; // Bits 0-9: Length in DWs (1 for single DWORD) + uint32_t at : 2; // Bits 10-11: Address Type + uint32_t attr : 2; // Bits 12-13: Attributes + uint32_t ep : 1; // Bit 14: Poisoned completion + uint32_t td : 1; // Bit 15: TLP Digest present + uint32_t th : 1; // Bit 16: TLP Processing Hints + uint32_t ln : 1; // Bit 17: Lightweight Notification + uint32_t reserved1 : 2; // Bits 18-19: Reserved + uint32_t tc : 3; // Bits 20-22: Traffic Class + uint32_t reserved2 : 1; // Bit 23: Reserved + uint32_t type : 5; // Bits 24-28: Type (01010b = Completion with Data) + uint32_t fmt : 2; // Bits 29-30: Format (10b = 3DW header with data) + uint32_t reserved3 : 1; // Bit 31: Reserved + } dword0; + + // DW1 (Byte 4-7) + struct { + uint32_t byte_count : 12; // Bits 0-11: Byte Count (remaining bytes, typically 4) + uint32_t bcm : 1; // Bit 12: Byte Count Modified + uint32_t status : 3; // Bits 13-15: Completion Status (000b = SC - Successful) + uint32_t completer_id : 16; // Bits 16-31: Completer ID (Bus:Dev:Func) + } dword1; + + // DW2 (Byte 8-11) + struct { + uint32_t lower_addr : 7; // Bits 0-6: Lower Address (byte address within DWORD) + uint32_t reserved : 1; // Bit 7: Reserved + uint32_t tag : 8; // Bits 8-15: Tag (matches request tag) + uint32_t requester_id : 16; // Bits 16-31: Requester ID (original requester) + } dword2; + + // DW3 (Byte 12-15) - Data payload + uint32_t data_payload; + }; + + _TLPCompletion3DW() { memset(data, 0, sizeof(data)); } + + // Helper to byte-swap from big-endian (network byte order) + void from_big_endian() { + dwords[0] = __builtin_bswap32(dwords[0]); + dwords[1] = __builtin_bswap32(dwords[1]); + dwords[2] = __builtin_bswap32(dwords[2]); + // Not swapping payload - it's already in little-endian (PCI config data) + // dwords[3] = __builtin_bswap32(dwords[3]); + } + + // Helper to byte-swap to big-endian (network byte order) + void to_big_endian() { + dwords[0] = __builtin_bswap32(dwords[0]); + dwords[1] = __builtin_bswap32(dwords[1]); + dwords[2] = __builtin_bswap32(dwords[2]); + // Not swapping payload - it's already in little-endian (PCI config data) + // dwords[3] = __builtin_bswap32(dwords[3]); + } + + // Helper to get PCI config data in correct byte order + uint32_t get_pci_config_dword() const { + // Swap the two 16-bit words in the payload + return ((data_payload & 0xFFFF) << 16) | ((data_payload >> 16) & 0xFFFF); + } + + // Completion Status values + enum CompletionStatus { + SC = 0b000, // Successful Completion + UR = 0b001, // Unsupported Request + CRS = 0b010, // Configuration Request Retry Status + CA = 0b100 // Completer Abort + }; + +} TLPCompletion3DW; + +typedef union _TLPHeader4DW { + uint8_t data[16]; // 4 DWORDs = 16 bytes (header + data) + uint32_t dwords[4]; // Access as DWORDs for byte swapping + struct { + // DW0 (Byte 0-3) + struct { + uint32_t length : 10; // Bits 0-9: Length in DWs + uint32_t at : 2; // Bits 10-11: Address Type (00 = untranslated) + uint32_t attr : 2; // Bits 12-13: Attributes (Relaxed Ordering, No Snoop) + uint32_t ep : 1; // Bit 14: Poisoned TLP + uint32_t td : 1; // Bit 15: TLP Digest present + uint32_t th : 1; // Bit 16: TLP Processing Hints + uint32_t ln : 1; // Bit 17: Lightweight Notification + uint32_t reserved1 : 2; // Bits 18-19: Reserved + uint32_t tc : 3; // Bits 20-22: Traffic Class + uint32_t reserved2 : 1; // Bit 23: Reserved + uint32_t type : 5; // Bits 24-28: Type (00100b = CfgWr0) + uint32_t fmt : 2; // Bits 29-30: Format (10b = 3DW header with data) + uint32_t reserved3 : 1; // Bit 31: Reserved + } dword0; + + // DW1 (Byte 4-7) + struct { + uint32_t first_be : 4; // Bits 0-3: First DW Byte Enable + uint32_t last_be : 4; // Bits 4-7: Last DW Byte Enable + uint32_t tag : 8; // Bits 8-15: Tag + uint32_t requester_id : 16; // Bits 16-31: Requester ID (Bus:Dev:Func) + } dword1; + + // DW2 (Byte 8-11) + struct { + uint32_t reg_num : 2; // Bits 0-1: Register Number (must be 00b) + uint32_t reg_offset : 10; // Bits 2-11: Extended Register Number (DWORD aligned) + uint32_t reserved : 4; // Bits 12-15: Reserved + uint32_t function : 3; // Bits 16-18: Function Number + uint32_t device : 5; // Bits 19-23: Device Number + uint32_t bus : 8; // Bits 24-31: Bus Number + } dword2; + + // DW3 (Byte 12-15) - Data payload + uint32_t data_payload; + }; + + _TLPHeader4DW() { memset(data, 0, sizeof(data)); } + + // Helper to byte-swap to big-endian (network byte order) + void to_big_endian() { + dwords[0] = __builtin_bswap32(dwords[0]); + dwords[1] = __builtin_bswap32(dwords[1]); + dwords[2] = __builtin_bswap32(dwords[2]); + // Not swapping payload - it's already in little-endian (PCI config data) + // dwords[3] = __builtin_bswap32(dwords[3]); // Swap data payload too + } + +} TLPHeader4DW; + +// 3DW Memory Write TLP (for 32-bit addresses with data) +typedef union _TLPMemoryWrite3DW { + uint8_t data[12 + 4096]; // 3DW header + max payload + uint32_t dwords[3 + 1024]; + struct { + // DW0 (Byte 0-3) + struct { + uint32_t length : 10; // Bits 0-9: Length in DWs + uint32_t at : 2; // Bits 10-11: Address Type + uint32_t attr : 2; // Bits 12-13: Attributes + uint32_t ep : 1; // Bit 14: Poisoned TLP + uint32_t td : 1; // Bit 15: TLP Digest present + uint32_t th : 1; // Bit 16: TLP Processing Hints + uint32_t ln : 1; // Bit 17: Lightweight Notification + uint32_t reserved1 : 2; // Bits 18-19: Reserved + uint32_t tc : 3; // Bits 20-22: Traffic Class + uint32_t reserved2 : 1; // Bit 23: Reserved + uint32_t type : 5; // Bits 24-28: Type (00000b = Memory Write) + uint32_t fmt : 2; // Bits 29-30: Format (10b = 3DW with data) + uint32_t reserved3 : 1; // Bit 31: Reserved + } dword0; + + // DW1 (Byte 4-7) + struct { + uint32_t first_be : 4; // Bits 0-3: First DW Byte Enable + uint32_t last_be : 4; // Bits 4-7: Last DW Byte Enable + uint32_t tag : 8; // Bits 8-15: Tag + uint32_t requester_id : 16; // Bits 16-31: Requester ID + } dword1; + + // DW2 (Byte 8-11) + struct { + uint32_t addr : 32; // 32-bit address (bits [31:2] valid, [1:0] = 00) + } dword2; + + // Data payload (up to 4096 bytes) + uint8_t data_payload[4096]; + }; + + _TLPMemoryWrite3DW() { memset(data, 0, sizeof(data)); } + + void to_big_endian() { + for (int i = 0; i < 3; i++) { + dwords[i] = __builtin_bswap32(dwords[i]); + } + // Data payload stays in original byte order + } + void from_big_endian() { + for (int i = 0; i < 3; i++) dwords[i] = __builtin_bswap32(dwords[i]); + // Data payload stays in original byte order + } +} TLPMemoryWrite3DW; + +// 4DW Memory Write TLP (for 64-bit addresses with data) +typedef union _TLPMemoryWrite4DW { + uint8_t data[16 + 4096]; // 4DW header + max payload + uint32_t dwords[4 + 1024]; + struct { + // DW0 (Byte 0-3) + struct { + uint32_t length : 10; // Bits 0-9: Length in DWs + uint32_t at : 2; // Bits 10-11: Address Type + uint32_t attr : 2; // Bits 12-13: Attributes + uint32_t ep : 1; // Bit 14: Poisoned TLP + uint32_t td : 1; // Bit 15: TLP Digest present + uint32_t th : 1; // Bit 16: TLP Processing Hints + uint32_t ln : 1; // Bit 17: Lightweight Notification + uint32_t reserved1 : 2; // Bits 18-19: Reserved + uint32_t tc : 3; // Bits 20-22: Traffic Class + uint32_t reserved2 : 1; // Bit 23: Reserved + uint32_t type : 5; // Bits 24-28: Type (00000b = Memory Write) + uint32_t fmt : 2; // Bits 29-30: Format (11b = 4DW with data) + uint32_t reserved3 : 1; // Bit 31: Reserved + } dword0; + + // DW1 (Byte 4-7) + struct { + uint32_t first_be : 4; // Bits 0-3: First DW Byte Enable + uint32_t last_be : 4; // Bits 4-7: Last DW Byte Enable + uint32_t tag : 8; // Bits 8-15: Tag + uint32_t requester_id : 16; // Bits 16-31: Requester ID + } dword1; + + // DW2 (Byte 8-11) + struct { + uint32_t addr_high : 32; // Upper 32 bits of 64-bit address + } dword2; + + // DW3 (Byte 12-15) + struct { + uint32_t addr_low : 32; // Lower 32 bits (bits [31:2] valid, [1:0] = 00) + } dword3; + + // Data payload (up to 4096 bytes) + uint8_t data_payload[4096]; + }; + + _TLPMemoryWrite4DW() { memset(data, 0, sizeof(data)); } + + void to_big_endian() { + for (int i = 0; i < 4; i++) { + dwords[i] = __builtin_bswap32(dwords[i]); + } + // Data payload stays in original byte order + } + + void from_big_endian() { + for (int i = 0; i < 4; i++) dwords[i] = __builtin_bswap32(dwords[i]); + // Data payload stays in original byte order + } +} TLPMemoryWrite4DW; + +// 3DW Memory Read TLP +typedef union _TLPMemoryRead3DW { + uint8_t data[12]; + uint32_t dwords[3]; + struct { + struct { + uint32_t length : 10; + uint32_t at : 2; + uint32_t attr : 2; + uint32_t ep : 1; + uint32_t td : 1; + uint32_t th : 1; + uint32_t reserved1 : 3; + uint32_t tc : 3; + uint32_t reserved2 : 1; + uint32_t type : 5; // 00000b = Memory Read + uint32_t fmt : 2; // 00b = 3DW, no data + uint32_t reserved3 : 1; + } dword0; + + struct { + uint32_t first_be : 4; + uint32_t last_be : 4; + uint32_t tag : 8; + uint32_t requester_id : 16; + } dword1; + + struct { + uint32_t addr : 32; // Bits [31:2] valid, [1:0] = 00 + } dword2; + }; + + _TLPMemoryRead3DW() { memset(data, 0, sizeof(data)); } + void to_big_endian() { + for (int i = 0; i < 3; i++) dwords[i] = __builtin_bswap32(dwords[i]); + } + void from_big_endian() { + for (int i = 0; i < 3; i++) dwords[i] = __builtin_bswap32(dwords[i]); + } +} TLPMemoryRead3DW; + +// 4DW Memory Read TLP (for 64-bit addresses) +typedef union _TLPMemoryRead4DW { + uint8_t data[16]; + uint32_t dwords[4]; + struct { + struct { + uint32_t length : 10; + uint32_t at : 2; + uint32_t attr : 2; + uint32_t ep : 1; + uint32_t td : 1; + uint32_t th : 1; + uint32_t reserved1 : 3; + uint32_t tc : 3; + uint32_t reserved2 : 1; + uint32_t type : 5; // 00000b = Memory Read + uint32_t fmt : 2; // 01b = 4DW, no data + uint32_t reserved3 : 1; + } dword0; + + struct { + uint32_t first_be : 4; + uint32_t last_be : 4; + uint32_t tag : 8; + uint32_t requester_id : 16; + } dword1; + + struct { + uint32_t addr_high : 32; // Upper 32 bits of address + } dword2; + + struct { + uint32_t addr_low : 32; // Lower 32 bits, [1:0] = 00 + } dword3; + }; + + _TLPMemoryRead4DW() { memset(data, 0, sizeof(data)); } + void to_big_endian() { + for (int i = 0; i < 4; i++) dwords[i] = __builtin_bswap32(dwords[i]); + } + void from_big_endian() { + for (int i = 0; i < 4; i++) dwords[i] = __builtin_bswap32(dwords[i]); + } +} TLPMemoryRead4DW; + +// Completion with Data TLP +typedef union _TLPCompletionWithData { + uint8_t data[12 + 4096]; // Header + max payload + uint32_t dwords[3 + 1024]; + struct { + struct { + uint32_t length : 10; + uint32_t at : 2; + uint32_t attr : 2; + uint32_t ep : 1; + uint32_t td : 1; + uint32_t th : 1; + uint32_t reserved1 : 3; + uint32_t tc : 3; + uint32_t reserved2 : 1; + uint32_t type : 5; // 01010b = Completion with Data + uint32_t fmt : 2; // 10b = 3DW with data + uint32_t reserved3 : 1; + } dword0; + + struct { + uint32_t byte_count : 12; + uint32_t bcm : 1; + uint32_t status : 3; + uint32_t completer_id : 16; + } dword1; + + struct { + uint32_t lower_addr : 7; + uint32_t reserved : 1; + uint32_t tag : 8; + uint32_t requester_id : 16; + } dword2; + + uint8_t data_payload[4096]; + }; + + enum CompletionStatus { + SC = 0, // Successful Completion + UR = 1, // Unsupported Request + CRS = 2, // Configuration Request Retry Status + CA = 4 // Completer Abort + }; + + _TLPCompletionWithData() { memset(data, 0, sizeof(data)); } + void from_big_endian() { + for (int i = 0; i < 3; i++) dwords[i] = __builtin_bswap32(dwords[i]); + // Data payload stays in original byte order + } + void to_big_endian() { + for (int i = 0; i < 3; i++) dwords[i] = __builtin_bswap32(dwords[i]); + // Data payload stays in original byte order + } +} TLPCompletionWithData; + +#endif \ No newline at end of file diff --git a/systemc-components/pci/gs_gpex/src/gs_gpex.cc b/systemc-components/pci/gs_gpex/src/gs_gpex.cc new file mode 100644 index 00000000..c3262e40 --- /dev/null +++ b/systemc-components/pci/gs_gpex/src/gs_gpex.cc @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All Rights Reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include "gs_gpex.h" + +typedef gs::gs_gpex<> gs_gpex; + +void module_register() { GSC_MODULE_REGISTER_C(gs_gpex); } \ No newline at end of file diff --git a/systemc-components/pci/nvme_ssd/CMakeLists.txt b/systemc-components/pci/nvme_ssd/CMakeLists.txt new file mode 100644 index 00000000..198640ef --- /dev/null +++ b/systemc-components/pci/nvme_ssd/CMakeLists.txt @@ -0,0 +1,20 @@ +gs_create_dymod(nvme_ssd) + +target_sources( + nvme_ssd PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/nvme_ssd.cc + ${CMAKE_CURRENT_SOURCE_DIR}/src/bitset.cc + ${CMAKE_CURRENT_SOURCE_DIR}/src/config.cc + ${CMAKE_CURRENT_SOURCE_DIR}/src/def.cc + ${CMAKE_CURRENT_SOURCE_DIR}/src/disk.cc + ${CMAKE_CURRENT_SOURCE_DIR}/src/dma.cc + ${CMAKE_CURRENT_SOURCE_DIR}/src/interface.cc + ${CMAKE_CURRENT_SOURCE_DIR}/src/namespace.cc + ${CMAKE_CURRENT_SOURCE_DIR}/src/pci_device_base.cc + ${CMAKE_CURRENT_SOURCE_DIR}/src/queue.cc +) + +target_include_directories( + nvme_ssd PUBLIC + $ +) \ No newline at end of file diff --git a/systemc-components/pci/nvme_ssd/include/algorithm.h b/systemc-components/pci/nvme_ssd/include/algorithm.h new file mode 100644 index 00000000..aa99a6e7 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/algorithm.h @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#pragma once + +#ifndef __UTIL_ALGORITHM__ +#define __UTIL_ALGORITHM__ + +#include +#include +#include + +#ifdef _MSC_VER + +#include + +#include + +#define __builtin_bswap16 _byteswap_ushort +#define __builtin_bswap32 _byteswap_ulong +#define __builtin_bswap64 _byteswap_uint64 + +inline uint32_t __builtin_clzl(uint32_t val) { + unsigned long leadingZero = 0; + + if (_BitScanReverse(&leadingZero, val)) { + return 31 - leadingZero; + } + + return 32; +} + +inline uint32_t __builtin_ffsl(uint32_t val) { + unsigned long trailingZero = 0; + + if (_BitScanForward(&trailingZero, val)) { + return trailingZero + 1; + } + + return 0; +} + +#endif + +#ifndef MIN +#define MIN(x, y) ((x) > (y) ? (y) : (x)) +#endif + +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif + +#define DIVCEIL(x, y) (((x)-1) / (y) + 1) + +template +uint8_t popcount(T v) { + v = v - ((v >> 1) & (T) ~(T)0 / 3); + v = (v & (T) ~(T)0 / 15 * 3) + ((v >> 2) & (T) ~(T)0 / 15 * 3); + v = (v + (v >> 4)) & (T) ~(T)0 / 255 * 15; + v = (T)(v * ((T) ~(T)0 / 255)) >> (sizeof(T) - 1) * CHAR_BIT; + + return (uint8_t)v; +} + +inline uint64_t generateMask(uint32_t val, uint32_t &count) { + uint64_t mask = (uint64_t)-1; + uint32_t tmp = 0; + + if (val > 0) { + int shift = __builtin_clzl(val); + + if (shift + __builtin_ffsl(val) == 64) { + shift++; + } + + tmp = 64 - shift; + mask = (mask << tmp); + } + + mask = (~mask) << count; + count += tmp; + + return mask; +} + +inline uint16_t bswap16(uint16_t val) { + return __builtin_bswap16(val); +} + +#endif diff --git a/systemc-components/pci/nvme_ssd/include/base_config.h b/systemc-components/pci/nvme_ssd/include/base_config.h new file mode 100644 index 00000000..0ca91f96 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/base_config.h @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#pragma once + +#ifndef __SIM_BASE_CONFIG__ +#define __SIM_BASE_CONFIG__ + +#include +#include +#include + +namespace SimpleSSD { + +#define MATCH_SECTION(str) (strcmp(section, str) == 0) +#define MATCH_NAME(str) (strcmp(name, str) == 0) +#define MATCH_VALUE(str) (strcmp(value, str) == 0) + +class BaseConfig { + protected: + bool convertBool(const char *); + + public: + BaseConfig() {} + virtual ~BaseConfig() {} + + virtual bool setConfig(const char *, const char *) = 0; + virtual void update() {} + + virtual int64_t readInt(uint32_t) { return 0; } + virtual uint64_t readUint(uint32_t) { return 0; } + virtual float readFloat(uint32_t) { return 0.f; } + virtual std::string readString(uint32_t) { return std::string(); } + virtual bool readBoolean(uint32_t) { return false; } +}; + +} // namespace SimpleSSD + +#endif diff --git a/systemc-components/pci/nvme_ssd/include/bitset.h b/systemc-components/pci/nvme_ssd/include/bitset.h new file mode 100644 index 00000000..fca66971 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/bitset.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#pragma once + +#ifndef __UTIL_BITSET__ +#define __UTIL_BITSET__ + +#include +#include + +#include "systemc.h" + +//#include "sim/trace.hh" + +// TODO: use SIMD operation if possible +class Bitset { + private: + uint8_t *data; + uint32_t dataSize; + uint32_t allocSize; + + public: + Bitset(); + Bitset(uint32_t); + Bitset(const Bitset &); + Bitset(Bitset &&) noexcept; + ~Bitset(); + + bool test(uint32_t) noexcept; + bool all() noexcept; + bool any() noexcept; + bool none() noexcept; + uint32_t count() noexcept; + uint32_t size() noexcept; + void set() noexcept; + void set(uint32_t, bool = true) noexcept; + void reset() noexcept; + void reset(uint32_t) noexcept; + void flip() noexcept; + void flip(uint32_t) noexcept; + + bool operator[](uint32_t) noexcept; + Bitset &operator&=(const Bitset &); + Bitset &operator|=(const Bitset &); + Bitset &operator^=(const Bitset &); + Bitset &operator=(const Bitset &); + Bitset &operator=(Bitset &&) noexcept; + Bitset operator~() const; + + friend Bitset &operator&(Bitset lhs, const Bitset &rhs) { return lhs &= rhs; } + friend Bitset &operator|(Bitset lhs, const Bitset &rhs) { return lhs |= rhs; } + friend Bitset &operator^(Bitset lhs, const Bitset &rhs) { return lhs ^= rhs; } + friend bool operator==(const Bitset &lhs, const Bitset &rhs) { + bool ret = true; + + if (lhs.dataSize != rhs.dataSize) { + SC_REPORT_WARNING("bitset", "Size does not match"); + } + + for (uint32_t i = 0; i < lhs.allocSize; i++) { + if (lhs.data[i] != rhs.data[i]) { + ret = false; + + break; + } + } + + return ret; + } + + friend bool operator!=(const Bitset &lhs, const Bitset &rhs) { + return !operator==(lhs, rhs); + } +}; + +#endif diff --git a/systemc-components/pci/nvme_ssd/include/config.h b/systemc-components/pci/nvme_ssd/include/config.h new file mode 100644 index 00000000..2d62fe40 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/config.h @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#pragma once + +#ifndef __HIL_NVME_CONFIG__ +#define __HIL_NVME_CONFIG__ + +#include + +#include "base_config.h" +#include "interface.h" +#include "algorithm.h" + +typedef enum { + NVME_PCIE_GEN, + NVME_PCIE_LANE, + NVME_AXI_BUS_WIDTH, + NVME_AXI_CLOCK, + NVME_FIFO_UNIT, + NVME_WORK_INTERVAL, + NVME_MAX_REQUEST_COUNT, + NVME_MAX_IO_CQUEUE, + NVME_MAX_IO_SQUEUE, + NVME_WRR_HIGH, + NVME_WRR_MEDIUM, + NVME_ENABLE_DEFAULT_NAMESPACE, + NVME_LBA_SIZE, + NVME_ENABLE_DISK_IMAGE, + NVME_STRICT_DISK_SIZE, + NVME_DISK_IMAGE_PATH, + NVME_USE_COW_DISK +} NVME_CONFIG; +/* +class Config : public BaseConfig { + private: + PCIExpress::PCIE_GEN pcieGen; //!< Default: PCIE_3_X + uint8_t pcieLane; //!< Default: 4 + ARM::AXI::BUS_WIDTH axiWidth; //!< Default: BUS_128BIT + uint64_t axiClock; //!< Default: 250000000 (250MHz) + uint64_t fifoUnit; //!< Default: 4096 + uint64_t workInterval; //!< Default: 50000 (50ns) + uint64_t maxRequestCount; //!< Default: 4 + uint16_t maxIOCQueue; //!< Default: 16 + uint16_t maxIOSQueue; //!< Default: 16 + uint16_t wrrHigh; //!< Default: 2 + uint16_t wrrMedium; //!< Default: 2 + uint64_t lbaSize; //!< Default: 512 + uint16_t defaultNamespace; //!< Default: 1 + bool enableDiskImage; //!< Default: False + bool strictDiskSize; //!< Default: False + bool useCopyOnWriteDisk; //!< Default: False + std::unordered_map diskImagePaths; //!< Default: "" + + public: + Config(); + + bool setConfig(const char *, const char *) override; + void update() override; + + int64_t readInt(uint32_t) override; + uint64_t readUint(uint32_t) override; + std::string readString(uint32_t) override; + bool readBoolean(uint32_t) override; +}; +*/ +#endif diff --git a/systemc-components/pci/nvme_ssd/include/def.h b/systemc-components/pci/nvme_ssd/include/def.h new file mode 100644 index 00000000..6054df9e --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/def.h @@ -0,0 +1,391 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#pragma once + +#ifndef __HIL_NVME_DEF__ +#define __HIL_NVME_DEF__ + +#include +#include +#include "bitset.h" +#include "dma_interface.h" + +#define NSID_NONE 0x00000000 +#define NSID_LOWEST 0x00000001 +#define NSID_ALL 0xFFFFFFFF + +#define MAKE_SGL_ID(type, subtype) \ + (uint8_t)(((type << 4) & 0xF0) | (subtype & 0x0F)) +#define SGL_TYPE(id) (uint8_t)(id >> 4) +#define SGL_SUBTYPE(id) (uint8_t)(id & 0x0F) + +#define OCSSD_VENDOR 0x1D1D +#define OCSSD_SSVID_1_2 0x0102 +#define OCSSD_SSVID_2_0 0x0200 + +typedef union _HealthInfo { + uint8_t data[0x200]; + struct { + uint8_t status; + uint16_t temperature; + uint8_t availableSpare; + uint8_t spareThreshold; + uint8_t lifeUsed; + uint8_t reserved[26]; + uint64_t readL; + uint64_t readH; + uint64_t writeL; + uint64_t writeH; + uint64_t readCommandL; + uint64_t readCommandH; + uint64_t writeCommandL; + uint64_t writeCommandH; + }; + + _HealthInfo() { memset(data, 0, 0x200); } +} HealthInfo; + +typedef enum : uint8_t { + TYPE_DATA_BLOCK_DESCRIPTOR = 0x00, + TYPE_BIT_BUCKET_DESCRIPTOR = 0x01, + TYPE_SEGMENT_DESCRIPTOR = 0x02, + TYPE_LAST_SEGMENT_DESCRIPTOR = 0x03, + TYPE_KEYED_DATA_BLOCK_DESCRIPTOR = 0x04 +} SGL_DESCRIPTOR_TYPE; + +typedef enum : uint8_t { + SUBTYPE_ADDRESS = 0x00, + SUBTYPE_OFFSET = 0x01, + SUBTYPE_NVME_TRANSPORT_SPECIFIC = 0x02 +} SGL_DESCRIPTOR_SUBTYPE; + +typedef enum { + PRIORITY_URGENT, + PRIORITY_HIGH, + PRIORITY_MEDIUM, + PRIORITY_LOW +} SQUEUE_PRIORITY; + +typedef enum { + REG_CONTROLLER_CAPABILITY = 0x00, + REG_VERSION = 0x08, + REG_INTERRUPT_MASK_SET = 0x0C, + REG_INTERRUPT_MASK_CLEAR = 0x10, + REG_CONTROLLER_CONFIG = 0x14, + REG_CONTROLLER_STATUS = 0x1C, + REG_NVM_SUBSYSTEM_RESET = 0x20, // 32 + REG_ADMIN_QUEUE_ATTRIBUTE = 0x24, // 36 + REG_ADMIN_SQUEUE_BASE_ADDR = 0x28, // 40 + REG_ADMIN_CQUEUE_BASE_ADDR = 0x30, // 48 + REG_CMB_LOCATION = 0x38, // 56 + REG_CMB_SIZE = 0x3C, + REG_DOORBELL_BEGIN = 0x1000 +} COMMAND_REGISTER; + +typedef enum { ROUND_ROBIN, WEIGHTED_ROUND_ROBIN } ARBITRATION_METHOD; + +typedef enum { + OPCODE_PROPERTY_SET = 0x00, + OPCODE_CONNECT = 0x01, + OPCODE_PROPERTY_GET = 0x04, + OPCODE_AUTHENTICATION_SEND = 0x05, + OPCODE_AUTHENTICATION_RECEIVE = 0x06 +} FABRIC_OPCODE; + +typedef enum { + OPCODE_DELETE_IO_SQUEUE = 0x00, + OPCODE_CREATE_IO_SQUEUE = 0x01, + OPCODE_GET_LOG_PAGE = 0x02, + OPCODE_DELETE_IO_CQUEUE = 0x04, + OPCODE_CREATE_IO_CQUEUE = 0x05, + OPCODE_IDENTIFY = 0x06, + OPCODE_ABORT = 0x08, + OPCODE_SET_FEATURES = 0x09, + OPCODE_GET_FEATURES = 0x0A, + OPCODE_ASYNC_EVENT_REQ = 0x0C, + OPCODE_NAMESPACE_MANAGEMENT = 0x0D, + OPCODE_FIRMWARE_COMMIT = 0x10, + OPCODE_FIRMWARE_DOWNLOAD = 0x11, + OPCODE_DEVICE_SELF_TEST = 0x14, + OPCODE_NAMESPACE_ATTACHMENT = 0x15, + OPCODE_KEEPALIVE = 0x18, + OPCODE_DIRECTIVE_SEND = 0x19, + OPCODE_DIRECTIVE_RECEIVE = 0x1A, + OPCODE_VIRTUALIZATION_MANAGEMENT = 0x1C, + OPCODE_NVME_MI_SEND = 0x1D, + OPCODE_NVME_MI_RECEIVE = 0x1E, + OPCODE_DOORBELL_BUFFER_CONFIG = 0x7C, + OPCODE_FORMAT_NVM = 0x80, + OPCODE_SECURITY_SEND = 0x81, + OPCODE_SECURITY_RECEIVE = 0x82, + OPCODE_SANITIZE = 0x84, + + // OpenChannel SSD 1.2 + OPCODE_DEVICE_IDENTIFICATION = 0xE2, + OPCODE_SET_BAD_BLOCK_TABLE = 0xF1, + OPCODE_GET_BAD_BLOCK_TABLE = 0xF2, + + // OpenChannel SSD 2.0 + OPCODE_GEOMETRY = 0xE2 +} ADMIN_OPCODE; + +typedef enum { + OPCODE_FLUSH = 0x00, + OPCODE_WRITE = 0x01, + OPCODE_READ = 0x02, + OPCODE_WRITE_UNCORRECTABLE = 0x04, + OPCODE_COMPARE = 0x05, + OPCODE_WRITE_ZEROS = 0x08, + OPCODE_DATASET_MANAGEMEMT = 0x09, + OPCODE_RESERVATION_REGISTER = 0x0D, + OPCODE_RESERVATION_REPORT = 0x0E, + OPCODE_RESERVATION_ACQUIRE = 0x11, + OPCODE_RESERVATION_RELEASE = 0x15, + + // OpenChannel SSD 1.2 + OPCODE_PHYSICAL_BLOCK_ERASE = 0x90, + OPCODE_PHYSICAL_PAGE_WRITE, + OPCODE_PHYSICAL_PAGE_READ, + OPCODE_PHYSICAL_PAGE_RAW_WRITE = 0x95, + OPCODE_PHYSICAL_PAGE_RAW_READ, + + // OpenChannel SSD 2.0 + OPCODE_VECTOR_CHUNK_RESET = 0x90, + OPCODE_VECTOR_CHUNK_WRITE, + OPCODE_VECTOR_CHUNK_READ, + OPCODE_VECTOR_CHUNK_COPY +} NVM_OPCODE; + +typedef enum { + LOG_ERROR_INFORMATION = 0x01, + LOG_SMART_HEALTH_INFORMATION, + LOG_FIRMWARE_SLOT_INFORMATION, + LOG_CHANGED_NAMESPACE_LIST, + LOG_COMMAND_EFFECTS_LOG, + LOG_RESERVATION_NOTIFICATION = 0x80, + + // OpenChannel SSD + LOG_CHUNK_INFORMATION = 0xCA +} LOG_PAGE; + +typedef enum { + CNS_IDENTIFY_NAMESPACE = 0x00, + CNS_IDENTIFY_CONTROLLER = 0x01, + CNS_ACTIVE_NAMESPACE_LIST = 0x02, + CNS_ALLOCATED_NAMESPACE_LIST = 0x10, + CNS_IDENTIFY_ALLOCATED_NAMESPACE = 0x11, + CNS_ATTACHED_CONTROLLER_LIST = 0x12, + CNS_CONTROLLER_LIST = 0x13 +} IDENTIFY_CNS; + +typedef enum { + FEATURE_ARBITRATION = 0x01, + FEATURE_POWER_MANAGEMENT, + FEATURE_LBA_RANGE_TYPE, + FEATURE_TEMPERATURE_THRESHOLD, + FEATURE_ERROR_RECOVERY, + FEATURE_VOLATILE_WRITE_CACHE, + FEATURE_NUMBER_OF_QUEUES, + FEATURE_INTERRUPT_COALESCING, + FEATURE_INTERRUPT_VECTOR_CONFIGURATION, + FEATURE_WRITE_ATOMICITY_NORMAL, + FEATURE_ASYNC_EVENT_CONFIGURATION, + FEATURE_AUTO_POWER_STATE_TRANSITION, + FEATURE_HOST_MEMORY_BUFFER, + FEATURE_TIMESTAMP, + FEATURE_KEEPALIVE_TIMER, + FEATURE_HOST_CONTROLLED_THERMAL_MANAGEMENT, + FEATURE_NON_OPERATIONAL_POWER_STATE_CONFIG, + FEATURE_SOFTWARE_PROGRESS_MARKER = 0x80, + FEATURE_HOST_IDENTIFIER, + FEATURE_RESERVATION_NOTIFICATION_MASK, + FEATURE_RESERVATION_PERSISTANCE, + + // OpenChannel SSD + FEATURE_MEDIA_FEEDBACH = 0xCA +} FEATURE; + +typedef enum { + TYPE_GENERIC_COMMAND_STATUS, // -> NVME_STATUS_CODE + TYPE_COMMAND_SPECIFIC_STATUS, // -> NVME_ERROR_CODE + TYPE_MEDIA_AND_DATA_INTEGRITY_ERROR, +} STATUS_CODE_TYPE; + +typedef enum { + /** Generic Command Status **/ + STATUS_SUCCESS, + STATUS_INVALID_OPCODE, + STATUS_INVALID_FIELD, + STATUS_CID_CONFLICT, + STATUS_DATA_TRANSFER_ERROR, + STATUS_ABORT_DUE_TO_POWER_LOSS, + STATUS_INTERNAL_ERROR, + STATUS_ABORT_REQUESTED, + STATUS_ABORT_DUE_TO_SQ_DELETE, + STATUS_ABORT_DUE_TO_FAILED_FUSE, + STATUS_ABORT_DUE_TO_MISSING_FUSE, + STATUS_ABORT_INVALID_NAMESPACE, + STATUS_ABORT_INVALID_FORMAT = STATUS_ABORT_INVALID_NAMESPACE, + STATUS_COMMAND_SEQUENCE_ERROR, + STATUS_INVALID_SGL_SEGMENT_DESCRIPTOR, + STATUS_INVALID_NUMBER_OF_SGL_DESCRIPTOR, + STATUS_INVALID_DATA_SGL_LENGTH, + STATUS_INVALID_METADATA_SGL_LENGTH, + STATUS_INVALID_SGL_DESCRIPTOR_TYPE, + STATUS_INVALID_USE_OF_MEMORY_BUFFER, + STATUS_PRP_OFFSET_INVALID, + STATUS_ATOMIC_WRITE_UNIT_EXCEEDED, + STATUS_INVALID_SGL_OFFSET = 0x16, + STATUS_INVALID_SGL_SUB_TYPE, + STATUS_HOST_ID_INCONSISTENT_FORMAT, + STATUS_KEEPALIVE_TIMEOUT_EXPIRED, + STATUS_INVALID_KEEPALIVE_TIMEOUT, + + /** NVM Command Status **/ + STATUS_LBA_OUT_OF_RANGE = 0x80, + STATUS_CAPACITY_EXCEEDED, + STATUS_NAMESPACE_NOT_READY, + STATUE_RESERVATION_CONFLICT, + STATUS_FORMAT_IN_PROGRESS +} STATUS_CODE; + +typedef enum { + /** Generic Command Errors **/ + STATUS_INVALID_COMPLETION_QUEUE, + STATUS_INVALID_QUEUE_ID, + STATUS_INVALID_QUEUE_SIZE, + STATUS_ABORT_COMMAND_LIMIT_EXCEEDED, + STATUS_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED = 0x05, + STATUS_INVALID_FIRMWARE_SLOT, + STATUS_INVALID_FIRMWARE_IMAGE, + STATUS_INVALID_INTERRUPT_VECTOR, + STATUS_INVALID_LOG_PAGE, + STATUS_INVALID_FORMAT, + STATUS_FIRMWARE_ACTIVATION_REQUIRES_CONVENTIONAL_RESET, + STATUS_INVALID_QUEUE_DELETE, + STATUS_FEATURE_ID_NOT_SAVEABLE, + STATUS_FEATURE_NOT_CHANGABLE, + STATUS_FEATURE_NOT_NAMESPACE_SPECIFIC, + STATUS_FIRMWARE_ACTIVATION_REQUIRES_NVM_SUBSYSTEM_RESET, + STATUS_FIRMWARE_ACTIVATION_REQUIRES_RESET, + STATUS_FIRMWARE_ACTIVATION_REQUIRES_MAXIMUM_TIME_VIOLATION, + STATUS_FIRMWARE_ACTIVATION_PROHIBITED, + STATUS_OVERLAPPING_RANGE, + STATUS_NAMESPACE_INSUFFICIENT_CAPACITY, + STATUS_NAMESPACE_ID_UNAVAILABLE, + STATUS_NAMESPACE_ALREADY_ATTACHED = 0x18, + STATUS_NAMESPACE_IS_PRIVATE, + STATUS_NAMESPACE_NOT_ATTACHED, + STATUS_THIN_PROVISIONING_NOT_SUPPORTED, + STATUS_CONTROLLER_LIST_INVALID, + + /** NVM Command Errors **/ + STATUS_ATTRIBUTE_CONFLICT = 0x80, + STATUS_INVALID_PROTECTION_INFORMATION, + STATUS_WRITE_TO_READ_ONLY_RANGE, +} ERROR_CODE; + +typedef enum { + /** I/O Command Media Errors **/ + STATUS_WRITE_FAULT = 0x80, + STATUS_UNRECOVERED_READ_ERROR, + STATUS_END_TO_END_GUARD_CHECK_ERROR, + STATUS_END_TO_END_APPLICATION_TAG_CHECK_ERROR, + STATUS_END_TO_END_REFERENCE_TAG_CHECK_ERROR, + STATUS_COMPARE_FAILURE, + STATUS_ACCESS_DENIED, + STATUS_DEALLOCATED_OR_UNWRITTEN_LOGICAL_BLOCK +} MEDIA_ERROR_CODE; + + +typedef struct _LPNRange { + uint64_t slpn; + uint64_t nlp; + + _LPNRange(); + _LPNRange(uint64_t, uint64_t); +} LPNRange; + +namespace HIL { + +typedef struct _Request { + uint64_t reqID; + uint64_t reqSubID; + uint64_t offset; + uint64_t length; + LPNRange range; + + uint64_t finishedAt; + DMAFunction function; + void *context; + + _Request(); + _Request(DMAFunction &, void *); + + bool operator()(const _Request &a, const _Request &b); +} Request; + +} // namespace HIL + +namespace ICL { + +typedef struct _Request { + uint64_t reqID; + uint64_t reqSubID; + uint64_t offset; + uint64_t length; + LPNRange range; + + _Request(); + _Request(HIL::Request &); +} Request; + +} // namespace ICL + +namespace FTL { + +typedef struct _Request { + uint64_t reqID; // ID of ICL::Request + uint64_t reqSubID; + uint64_t lpn; + Bitset ioFlag; + + _Request(uint32_t); + _Request(uint32_t, ICL::Request &); +} Request; + +} // namespace FTL + +namespace PAL { + +typedef struct _Request { + uint64_t reqID; // ID of ICL::Request + uint64_t reqSubID; + uint32_t blockIndex; + uint32_t pageIndex; + Bitset ioFlag; + + _Request(uint32_t); + _Request(FTL::Request &); +} Request; + +} // namespace PAL + +#endif diff --git a/systemc-components/pci/nvme_ssd/include/disk.h b/systemc-components/pci/nvme_ssd/include/disk.h new file mode 100644 index 00000000..fb98981d --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/disk.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#pragma once + +#ifndef __UTIL_DISK__ +#define __UTIL_DISK__ + +#include +#include +#include +#include +#include + +class Disk { + protected: + std::string filename; + uint64_t diskSize; + uint32_t sectorSize; + + std::fstream disk; + + public: + Disk(); + Disk(const Disk &) = delete; + virtual ~Disk(); + + virtual uint64_t open(std::string, uint64_t, uint32_t); + virtual void close(); + + virtual uint16_t read(uint64_t, uint16_t, uint8_t *&); + virtual uint16_t write(uint64_t, uint16_t, uint8_t *); + virtual uint16_t erase(uint64_t, uint16_t); +}; + +class CoWDisk : public Disk { + private: + std::unordered_map> table; + + public: + CoWDisk(); + CoWDisk(const CoWDisk &) = delete; + ~CoWDisk(); + + void close() override; + + uint16_t read(uint64_t, uint16_t, uint8_t *&) override; + uint16_t write(uint64_t, uint16_t, uint8_t *) override; +}; + +class MemDisk : public Disk { + private: + std::unordered_map> table; + + public: + MemDisk(); + MemDisk(const MemDisk &) = delete; + ~MemDisk(); + + uint64_t open(std::string, uint64_t, uint32_t) override; + void close() override; + + uint16_t read(uint64_t, uint16_t, uint8_t *&) override; + uint16_t write(uint64_t, uint16_t, uint8_t *) override; + uint16_t erase(uint64_t, uint16_t) override; +}; + +#endif diff --git a/systemc-components/pci/nvme_ssd/include/dma.h b/systemc-components/pci/nvme_ssd/include/dma.h new file mode 100644 index 00000000..601fb73a --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/dma.h @@ -0,0 +1,163 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#pragma once + +#ifndef __HIL_NVME_DMA__ +#define __HIL_NVME_DMA__ + +#include + +#include "config.h" +#include "def.h" +#include "dma_interface.h" +//#include "config_reader.h" +//#include "util/simplessd.hh" + +#ifndef MIN +#define MIN(x, y) ((x) > (y) ? (y) : (x)) +#endif + +class Controller; +typedef uint64_t Event; +class DMAInterface; + +typedef struct { + //ConfigReader *pConfigReader; + DMAInterface *pInterface; + uint64_t memoryPageSize; + uint8_t memoryPageSizeOrder; + uint16_t maxQueueEntry; +} ConfigData; + +class DMAInterface { + protected: + DMAInterface *pInterface; + DMAFunction initFunction; + uint64_t callCounter; + void *context; + + Event immediateEvent; + + DMAFunction dmaHandler; + static void commonDMAHandler(uint64_t, void *); + + public: + DMAInterface(ConfigData &, DMAFunction &, void *); + virtual ~DMAInterface(); + + virtual void read(uint64_t, uint64_t, uint8_t *, DMAFunction &, + void * = nullptr) = 0; + virtual void write(uint64_t, uint64_t, uint8_t *, DMAFunction &, + void * = nullptr) = 0; +}; + +struct DMAInitContext { + DMAInterface *pThis; + uint64_t totalSize; + uint64_t currentSize; + uint8_t *buffer; +}; + +struct PRP { + uint64_t addr; + uint64_t size; + + PRP(); + PRP(uint64_t, uint64_t); +}; + +class PRPList : public DMAInterface { + private: + std::vector prpList; + uint64_t totalSize; + uint64_t pagesize; + bool requiresPRP2DMA = false; + + void getPRPListFromPRP(uint64_t, uint64_t); + uint64_t getPRPSize(uint64_t); + + public: + // TODO(jhieb) should the buffer be public? Or getter to fill it externally? + uint8_t *buffer; // Used for DMA requests for PRPlists. + uint64_t prpListSize; + uint64_t prpListAddr; + std::vector getPRPList(){ return prpList; } + PRPList(ConfigData &, DMAFunction &, void *, uint64_t, uint64_t, uint64_t); + PRPList(ConfigData &, DMAFunction &, void *, uint64_t, uint64_t, bool); + ~PRPList(); + + void read(uint64_t, uint64_t, uint8_t *, DMAFunction &, + void * = nullptr) override; + void write(uint64_t, uint64_t, uint8_t *, DMAFunction &, + void * = nullptr) override; + bool getPRP2ReqDMA(){ return requiresPRP2DMA; } + + void processPRPListBuffer(); +}; + +union SGLDescriptor { + uint8_t data[16]; + struct { + uint64_t address; + uint32_t length; + uint8_t reserved[3]; + uint8_t id; + }; + + SGLDescriptor(); +}; + +struct Chunk { + uint64_t addr; + uint32_t length; + + bool ignore; + + Chunk(); + Chunk(uint64_t, uint32_t, bool); +}; + +class SGL : public DMAInterface { + private: + bool requiresSGLDMA = false; + std::vector chunkList; + uint64_t totalSize; + + void parseSGLDescriptor(SGLDescriptor &); + + public: + SGL(ConfigData &, DMAFunction &, void *, uint64_t, uint64_t); + ~SGL(); + + uint64_t address; + uint32_t length; + uint8_t *buffer; // Used for DMA requests for PRPlists. + + std::vector getChunkList() { return chunkList; } + void read(uint64_t, uint64_t, uint8_t *, DMAFunction &, + void * = nullptr) override; + void write(uint64_t, uint64_t, uint8_t *, DMAFunction &, + void * = nullptr) override; + bool getReqSGLDMA(){ return requiresSGLDMA; } + + void parseSGLSegment(); +}; + +#endif diff --git a/systemc-components/pci/nvme_ssd/include/dma_interface.h b/systemc-components/pci/nvme_ssd/include/dma_interface.h new file mode 100644 index 00000000..5bf7382a --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/dma_interface.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#pragma once + +#ifndef __SIM_DMA_INTERFACE__ +#define __SIM_DMA_INTERFACE__ + +#include +#include + +typedef std::function DMAFunction; + +typedef struct _DMAContext { + int counter; + DMAFunction function; + void *context; + + _DMAContext(DMAFunction &f) : counter(0), function(f), context(nullptr) {} + _DMAContext(DMAFunction &f, void *c) : counter(0), function(f), context(c) {} +} DMAContext; + +#endif diff --git a/systemc-components/pci/nvme_ssd/include/interface.h b/systemc-components/pci/nvme_ssd/include/interface.h new file mode 100644 index 00000000..6b12baaa --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/interface.h @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#pragma once + +#ifndef __UTIL_INTERFACE__ +#define __UTIL_INTERFACE__ + +#include + +namespace PCIExpress { + +typedef enum { + PCIE_1_X, // PCI Express Gen. 1.x + PCIE_2_X, // PCI Express Gen. 2.x + PCIE_3_X, // PCI Express Gen. 3.x + PCIE_NUM +} PCIE_GEN; + +uint64_t calculateDelay(PCIE_GEN, uint8_t, uint64_t); + +} // namespace PCIExpress + +namespace SATA { + +typedef enum { + SATA_1_0, // SATA 1.0 (1.5Gbps) + SATA_2_0, // SATA 2.0 (3Gbps) + SATA_3_0, // SATA 3.0/3.1 (6Gbps) + SATA_NUM +} SATA_GEN; + +uint64_t calculateDelay(SATA_GEN, uint64_t); + +} // namespace SATA + +namespace MIPI { + +namespace M_PHY { + +typedef enum { + HS_G1, // High Speed Gear 1 + HS_G2, // High Speed Gear 2 + HS_G3, // High Speed Gear 3 + HS_G4, // High Speed Gear 4 + HS_NUM +} M_PHY_MODE; + +uint64_t calculateDelay(M_PHY_MODE, uint8_t, uint64_t); + +} // namespace M_PHY + +namespace UniPro { + +uint64_t calculateDelay(M_PHY::M_PHY_MODE, uint8_t, uint64_t); + +} // namespace UniPro + +} // namespace MIPI + +namespace ARM { + +namespace AXI { + +typedef enum { + BUS_32BIT = 4, + BUS_64BIT = 8, + BUS_128BIT = 16, + BUS_256BIT = 32, + BUS_512BIT = 64, + BUS_1024BIT = 128, +} BUS_WIDTH; + +uint64_t calculateDelay(uint64_t, BUS_WIDTH, uint64_t); + +namespace Stream { + +uint64_t calculateDelay(uint64_t, BUS_WIDTH, uint64_t); + +} // namespace Stream + +} // namespace AXI + +} // namespace ARM + +#endif diff --git a/systemc-components/pci/nvme_ssd/include/namespace.h b/systemc-components/pci/nvme_ssd/include/namespace.h new file mode 100644 index 00000000..8446567b --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/namespace.h @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#ifndef __HIL_NVME_NAMESPACE__ +#define __HIL_NVME_NAMESPACE__ + +#include + +#include "def.h" +#include "dma.h" +#include "queue.h" +#include "disk.h" +//#include "util/simplessd.hh" + +typedef union _DatasetManagementRange { + uint8_t data[0x10]; + struct { + uint32_t attr; + uint32_t nlb; + uint64_t slba; + }; +} DatasetManagementRange; + +typedef std::function RequestFunction; + +class RequestContext { + public: + DMAInterface *dma; + RequestFunction function; + CQEntryWrapper resp; + + uint8_t *buffer; + + RequestContext(RequestFunction &f, CQEntryWrapper &r) + : dma(nullptr), function(f), resp(r), buffer(nullptr) {} +}; + +class IOContext : public RequestContext { + public: + uint64_t beginAt; + uint64_t slba; + uint64_t nlb; + uint64_t tick; + + IOContext(RequestFunction &f, CQEntryWrapper &r) : RequestContext(f, r) {} +}; + +class CompareContext : public IOContext { + public: + uint8_t *hostContent; + + CompareContext(RequestFunction &f, CQEntryWrapper &r) + : IOContext(f, r), hostContent(nullptr) {} +}; + +class Namespace { + public: + typedef struct _Information { + uint64_t size; //!< NSZE + uint64_t capacity; //!< NCAP + uint64_t utilization; //!< NUSE + uint64_t sizeInByteL; //> 2) +#define R_GLBL2_PF_BARLITE_INT (0x104 >> 2) +#define R_GLBL2_PF_BARLITE_EXT (0x10C >> 2) +#define R_GLBL2_CHANNEL_MDMA (0x118 >> 2) +#define R_GLBL2_CHANNEL_QDMA_CAP (0x120 >> 2) +#define R_GLBL2_CHANNEL_FUNC_RET (0x12C >> 2) +#define R_GLBL2_MISC_CAP (0x134 >> 2) +#define R_GLBL_INTR_CFG (0x2c4 >> 2) +#define R_GLBL_INTR_CFG_INT_PEND (1UL << 1) +#define R_GLBL_INTR_CFG_INT_EN (1UL << 0) +#define R_IND_CTXT_DATA (0x804 >> 2) +/* Those registers are not at the same place for CPM4. */ +#define R_IND_CTXT_CMD(v) ((v ? 0x824 : 0x844) >> 2) +#define R_DMAP_SEL_INT_CIDX(v, n) (((v ? 0x6400 : 0x18000) \ + + (0x10 * n)) >> 2) +#define R_DMAP_SEL_H2C_DSC_PIDX(v, n) (((v ? 0x6404 : 0x18004) \ + + (0x10 * n)) >> 2) +#define R_DMAP_SEL_C2H_DSC_PIDX(v, n) (((v ? 0x6408 : 0x18008) \ + + (0x10 * n)) >> 2) +#define R_DMAP_SEL_CMPT_CIDX(v, n) (((v ? 0x640C : 0x1800C) \ + + (0x10 * n)) >> 2) + +/* About MSIX Vector mapping: + * + * For Versal HARD IP: + * 0 - User Vector (1 but # might be configurable) + * 1 - Error Vector + * 2 - Data Vector + * + * For SOFT IP: + * 0 - MailBox Vector (for IP >= 2019_1 only) + * 1 - User Vector (1 but # might be configurable) + * 2 - Error Vector + * 3 - Data Vector + */ +#if QDMA_HAS_MAILBOX +#define QDMA_PF0_FIRST_DATA_MSIX_VECTOR (3) +#else +#define QDMA_PF0_FIRST_DATA_MSIX_VECTOR (2) +#endif + +// NVMe registers +typedef union _RegisterTable { + uint8_t data[64]; + struct { + uint64_t capabilities; + uint32_t version; + uint32_t interruptMaskSet; + uint32_t interruptMaskClear; + uint32_t configuration; + uint32_t reserved; + uint32_t status; // TODO(jhieb) do i need this? + uint32_t subsystemReset; + uint32_t adminQueueAttributes; + uint64_t adminSQueueBaseAddress; + uint64_t adminCQueueBaseAddress; + uint32_t memoryBufferLocation; + uint32_t memoryBufferSize; + }; + + _RegisterTable(){ + // Optional: zero-initialize the union + memset(data, 0, sizeof(data)); + } +} RegisterTable; + +typedef struct { + uint64_t nextTime; + uint32_t requestCount; + bool valid; + bool pending; +} AggregationInfo; + +const uint32_t nLBAFormat = 4; +const uint32_t lbaFormat[nLBAFormat] = { + 0x02090000, // 512B + 0, Good performance + 0x020A0000, // 1KB + 0, Good performance + 0x010B0000, // 2KB + 0, Better performance + 0x000C0000, // 4KB + 0, Best performance +}; +const uint32_t lbaSize[nLBAFormat] = { + 512, // 512B + 1024, // 1KB + 2048, // 2KB + 4096, // 4KB +}; + +typedef struct { + uint32_t channel; //!< Total # channels + uint32_t package; //!< # packages / channel + uint32_t die; //!< # dies / package + uint32_t plane; //!< # planes / die + uint32_t block; //!< # blocks / plane + uint32_t page; //!< # pages / block + uint32_t superBlock; //!< Total super blocks + uint32_t pageSize; //!< Size of page in bytes + uint32_t superPageSize; //!< Size of super page in bytes + uint32_t pageInSuperPage; //!< # pages in one superpage +} NANDParameter; + +typedef struct { + uint64_t totalPhysicalBlocks; //!< (PAL::Parameter::superBlock) + uint64_t totalLogicalBlocks; + uint64_t pagesInBlock; //!< (PAL::Parameter::page) + uint32_t pageSize; //!< Mapping unit (PAL::Parameter::superPageSize) + uint32_t ioUnitInPage; //!< # smallest I/O unit in one page + uint32_t pageCountToMaxPerf; //!< # pages to fully utilize internal parallism +} FTLParameter; + +template +class qdma : public sc_module +{ +private: + + // Manual config items for now. + uint32_t cqsize = 16; + uint32_t sqsize = 16; + uint16_t nNamespaces = 1; + uint32_t lba = 512; + NANDParameter nandParameters; + FTLParameter ftlParameter; + uint64_t totalLogicalPages; + uint32_t logicalPageSize; + double overProvisioningRatio = 0.25; + + // TODO(jhieb) stuff to move out eventually. + /* Nvme variables */ + RegisterTable registers; //!< Table for NVMe Controller Registers + uint64_t sqstride; //!< Calculated SQ Stride + uint64_t cqstride; //!< Calculated CQ stride + uint8_t adminQueueInited; //!< Flag for initialization of Admin CQ/SQ + uint16_t arbitration; //!< Selected Arbitration Mechanism + uint32_t interruptMask; //!< Variable to store current interrupt mask + uint64_t commandCount; + ConfigData cfgdata; + bool shutdownReserved; + uint64_t registerTableBaseAddress; + int registerTableSize; + uint64_t aggregationTime; + uint32_t aggregationThreshold; + std::unordered_map aggregationMap; + uint32_t queueAllocated; + uint64_t allocatedLogicalPages; + + CQueue **ppCQueue; //!< Completion Queue array + SQueue **ppSQueue; //!< Submission Queue array + std::list lSQFIFO; //!< Internal FIFO queue for submission + std::list lCQFIFO; //!< Internal FIFO queue for completion + std::list lNamespaces; + + /* Context handling. */ + + /* The per queue HW contexts. */ + struct __attribute__((__packed__)) hw_ctx { + /* CIDX of the last fetched descriptor. */ + uint16_t hw_cidx; + /* Credit consumed. */ + uint16_t credit_used; + uint8_t rsvd; + /* CIDX != PIDX. */ + uint8_t desc_pending : 1; + uint8_t invalid_desc : 1; + uint8_t event_pending : 1; + uint8_t desc_fetch_pending : 4; + uint8_t rsvd2 : 1; + }; + + /* Register area for the contexts described above. */ + struct { + uint32_t data[QDMA_U32_PER_CONTEXT]; + } queue_contexts[QDMA_QUEUE_COUNT][QDMA_MAX_CONTEXT_SELECTOR]; + + /* MSI-X handling. */ + enum msix_status { QDMA_MSIX_LOW = 0, QDMA_MSIX_HIGH } + msix_status[NR_QDMA_IRQ]; + sc_event msix_trig[NR_QDMA_IRQ]; + + void msix_strobe(unsigned int msix_id) + { + sc_event *msix_trig; + + /* Sanity check on the number of queue. */ + if (!(msix_id < NR_QDMA_IRQ)) { + SC_REPORT_ERROR("qdma", "invalid MSIX ID"); + } + + /* Each queue has it's own event to be triggered. */ + msix_trig = &this->msix_trig[msix_id]; + while (1) { + /* Waiting for an MSIX to be triggered. */ + wait(*msix_trig); + this->irq[msix_id].write(true); + wait(10, SC_NS); + this->irq[msix_id].write(false); + } + } + + /* Context commands. */ + enum { + QDMA_CTXT_CMD_CLR = 0, + QDMA_CTXT_CMD_WR = 1, + QDMA_CTXT_CMD_RD = 2, + QDMA_CTXT_CMD_INV = 3 + }; + + /* The contexts are indirectly accessed by the driver, also they are + * slightly version dependent (CPM4 vs CPM5). */ + enum { + QDMA_CTXT_SELC_DEC_SW_C2H = 0, + QDMA_CTXT_SELC_DEC_SW_H2C = 1, + QDMA_CTXT_SELC_DEC_HW_C2H = 2, + QDMA_CTXT_SELC_DEC_HW_H2C = 3, + QDMA_CTXT_SELC_DEC_CR_C2H = 4, + QDMA_CTXT_SELC_DEC_CR_H2C = 5, + /* Write Back, also called completion queue. */ + QDMA_CTXT_SELC_WRB = 6, + QDMA_CTXT_SELC_PFTCH = 7, + /* Interrupt context. */ + QDMA_CTXT_SELC_INT_COAL = 8, + QDMA_CTXT_SELC_HOST_PROFILE = 0xA, + QDMA_CTXT_SELC_TIMER = 0xB, + QDMA_CTXT_SELC_FMAP_QID2VEC = 0xC, + QDMA_CTXT_SELC_FNC_STS = 0xD, + }; + + void handle_ctxt_cmd(uint32_t reg) { + uint32_t qid = (reg >> 7) & 0x1FFF; + uint32_t cmd = (reg >> 5) & 0x3; + uint32_t sel = (reg >> 1) & 0xF; + uint32_t *data; + + if (sel == QDMA_CTXT_SELC_INT_COAL) { + /* This one requires some special treatment. */ + this->handle_irq_ctxt_cmd(qid, cmd); + return; + } + + /* Find the context data. */ + assert(qid < QDMA_QUEUE_COUNT); + switch (sel) { + case QDMA_CTXT_SELC_FMAP_QID2VEC: + case QDMA_CTXT_SELC_PFTCH: + case QDMA_CTXT_SELC_WRB: + case QDMA_CTXT_SELC_DEC_CR_H2C: + case QDMA_CTXT_SELC_DEC_CR_C2H: + case QDMA_CTXT_SELC_DEC_HW_H2C: + case QDMA_CTXT_SELC_DEC_HW_C2H: + case QDMA_CTXT_SELC_DEC_SW_H2C: + case QDMA_CTXT_SELC_DEC_SW_C2H: + data = this->queue_contexts[qid][sel].data; + break; + default: + SC_REPORT_ERROR("qdma", "Unsupported selector"); + return; + case QDMA_CTXT_SELC_INT_COAL: + /* Handled elsewere. */ + abort(); + return; + } + + switch (cmd) { + case QDMA_CTXT_CMD_CLR: + memset(data, 0, QDMA_U32_PER_CONTEXT * 4); + break; + case QDMA_CTXT_CMD_WR: + memcpy(data, &this->regs.u32[R_IND_CTXT_DATA], + QDMA_U32_PER_CONTEXT * 4); + break; + case QDMA_CTXT_CMD_RD: + memcpy(&this->regs.u32[R_IND_CTXT_DATA], data, + QDMA_U32_PER_CONTEXT * 4); + break; + case QDMA_CTXT_CMD_INV: + break; + default: + SC_REPORT_ERROR("qdma", "Unsupported command"); + break; + } + } + + /* This one deserves a special treatment, because it has some side + * effects. */ + void handle_irq_ctxt_cmd(uint32_t ring_idx, uint32_t cmd) { + INTR_CTX *intr_ctx = + (INTR_CTX *)this->queue_contexts[ring_idx] + [QDMA_CTXT_SELC_INT_COAL].data; + + switch (cmd) { + case QDMA_CTXT_CMD_CLR: + memset(intr_ctx, 0, QDMA_U32_PER_CONTEXT * 4); + break; + case QDMA_CTXT_CMD_RD: + memcpy(&this->regs.u32[R_IND_CTXT_DATA], + intr_ctx, + QDMA_U32_PER_CONTEXT * 4); + break; + case QDMA_CTXT_CMD_WR: + { + bool valid = intr_ctx->valid; + + memcpy(intr_ctx, + &this->regs.u32[R_IND_CTXT_DATA], + QDMA_U32_PER_CONTEXT * 4); + if (intr_ctx->valid && !valid) { + /* Interrupt context validated, + * reset the ring index. */ + this->irq_ring_entry_idx + [ring_idx] = 0; + } + } + break; + case QDMA_CTXT_CMD_INV: + /* Drop the valid bit. */ + intr_ctx->valid = 0; + break; + default: + break; + } + } + + /* Descriptors: for h2c and c2h memory mapped transfer. */ + struct x2c_mm_descriptor { + uint64_t src_address : 64; + uint64_t byte_count : 28; + uint64_t rsvd0 : 36; + uint64_t dst_address : 64; + uint64_t rsvd1 : 64; + }; + + /* Status descriptor, written by the DMA at the end of the transfer. */ + struct x2c_wb_descriptor { + /* 0 No errors, 1: DMA error, 2: Descriptor fetch error. */ + uint16_t err : 2; + uint16_t rsvd0 : 14; + uint16_t cidx : 16; + uint16_t pidx : 16; + uint16_t rsvd1 : 16; + }; + + /* Transfer data from the Host 2 the Card (h2c = true), + Card 2 Host (h2c = false). */ + int do_mm_dma(uint64_t src_addr, uint64_t dst_addr, uint64_t size, + bool h2c) + { + uint64_t i; + sc_time delay(SC_ZERO_TIME); + struct { + tlm::tlm_generic_payload trans; + const char *name; + } trans_ext[2]; + uint32_t data; + + trans_ext[0].trans.set_command(tlm::TLM_READ_COMMAND); + trans_ext[0].trans.set_data_ptr((unsigned char *)&data); + trans_ext[0].trans.set_streaming_width(4); + trans_ext[0].trans.set_data_length(4); + + trans_ext[1].trans.set_command(tlm::TLM_WRITE_COMMAND); + trans_ext[1].trans.set_data_ptr((unsigned char *)&data); + trans_ext[1].trans.set_streaming_width(4); + trans_ext[1].trans.set_data_length(4); + + for (i = 0; i < size; i+=4) { + trans_ext[0].trans.set_address(src_addr); + trans_ext[1].trans.set_address(dst_addr); + src_addr += 4; + dst_addr += 4; + + if (h2c) { + this->dma->b_transport( + trans_ext[0].trans, delay); + } else { + this->card_bus->b_transport( + trans_ext[0].trans, delay); + } + + if (trans_ext[0].trans.get_response_status() != + tlm::TLM_OK_RESPONSE) { + SC_REPORT_ERROR("qdma", + "error while fetching the data"); + return -1; + } + + if (h2c) { + this->card_bus->b_transport( + trans_ext[1].trans, delay); + } else { + this->dma->b_transport( + trans_ext[1].trans, delay); + } + + if (trans_ext[1].trans.get_response_status() != + tlm::TLM_OK_RESPONSE) { + SC_REPORT_ERROR("qdma", + "error while pushing the data"); + return -1; + } + } + + return 0; + } + + /* The driver wrote the @pidx in the update register of the given qid. + Handle the request. */ + void run_mm_dma(int16_t qid, bool h2c) + { + SW_CTX *sw_ctx; + struct hw_ctx *hw_ctx; + uint16_t pidx; + uint8_t desc[QDMA_DESC_MAX_SIZE]; + int desc_size; + uint32_t ring_sizes[16] = { + 2048, 64, 128, 192, 256, 384, 512, 768, 1024, + 1536, 3072, 4096, 6144, 8192, 12288, 16384 }; + uint32_t ring_size; + struct x2c_mm_descriptor *pdesc = + (struct x2c_mm_descriptor *)desc; + struct x2c_wb_descriptor *pstatus = + (struct x2c_wb_descriptor *) desc; + + if (qid > QDMA_QUEUE_COUNT) { + SC_REPORT_ERROR("qdma", "invalid queue ID"); + return; + } + + /* Compute some useful information from the context. */ + sw_ctx = this->get_software_context(qid, h2c); + hw_ctx = + (struct hw_ctx *)this->queue_contexts[qid] + [h2c ? QDMA_CTXT_SELC_DEC_HW_H2C : + QDMA_CTXT_SELC_DEC_HW_C2H].data; + pidx = this->regs.u32 + [h2c ? R_DMAP_SEL_H2C_DSC_PIDX(this->is_cpm4(), qid) : + R_DMAP_SEL_C2H_DSC_PIDX(this->is_cpm4(), qid)] & 0xffff; + desc_size = 8 << sw_ctx->desc_size; + ring_size = ring_sizes[sw_ctx->ring_size]; + + sw_ctx->pidx = pidx; + + /* Check that the producer index is in the descriptor ring, and + isn't pointing to the status descriptor. */ + if (sw_ctx->pidx >= ring_size) { + SC_REPORT_ERROR("qdma", "Producer index outside the " + "descriptor ring."); + } + + /* Running through the remaining descriptors from CIDX to + * PIDX. Wrap around if needed */ + while (sw_ctx->pidx != hw_ctx->hw_cidx) { + this->fetch_descriptor( + ((uint64_t)sw_ctx->desc_base_high << 32) + + sw_ctx->desc_base_low + + desc_size * hw_ctx->hw_cidx, + desc_size, desc); + + this->do_mm_dma(pdesc->src_address, pdesc->dst_address, + pdesc->byte_count, h2c); + + /* Descriptor is processed, go to the next one. This + might warp around the descriptor ring, also skip the + last descriptor which is the status descriptor. */ + hw_ctx->hw_cidx = hw_ctx->hw_cidx == ring_size - 1 ? + 0 : hw_ctx->hw_cidx + 1; + + /* Sending MSIX and / or writing back status descriptor + doesn't make sense at this point since the simulator + won't notice. Do it once for all when the queue + finishes its work to gain performance. */ + if (sw_ctx->pidx != hw_ctx->hw_cidx) { + continue; + } + + /* Update the status, and write the descriptor back. */ + if (sw_ctx->writeback_en) { + /* Fetch the last descriptor, put the status in + * it, and write it back. */ + this->fetch_descriptor( + ((uint64_t)sw_ctx->desc_base_high << 32) + + sw_ctx->desc_base_low + + desc_size * ring_size, + desc_size, + desc); + + pstatus->err = 0; + pstatus->cidx = hw_ctx->hw_cidx; + pstatus->pidx = pidx; + this->descriptor_writeback( + ((uint64_t)sw_ctx->desc_base_high << 32) + + sw_ctx->desc_base_low + + desc_size * ring_size, + desc_size, + desc); + } + + /* Trigger an IRQ? */ + if ((!sw_ctx->irq_arm) || (!sw_ctx->irq_enabled)) { + /* The software is polling for the completion. + * Just get out. */ + continue; + } + + if (this->irq_aggregation_enabled(qid, h2c)) { + INTR_CTX *intr_ctx; + INTR_RING_ENTRY entry; + int ring_idx = this->get_vec(qid, h2c); + + /* Each queue has a programmable irq ring + * associated to it. */ + intr_ctx = + (INTR_CTX *)this->queue_contexts + [ring_idx] + [QDMA_CTXT_SELC_INT_COAL].data; + + /* Update the PIDX in the Interrupt Context + * Structure. */ + intr_ctx->pidx = pidx; + + if (!intr_ctx->valid) { + SC_REPORT_ERROR("qdma", + "invalid interrupt context"); + return; + } + + /* Now the controller needs to populate the IRQ + * ring. */ + entry.qid = qid; + entry.interrupt_type = h2c ? 0 : 1; + entry.coal_color = intr_ctx->color; + entry.error = 0; + entry.interrupt_state = 0; + entry.color = entry.coal_color; + entry.cidx = hw_ctx->hw_cidx; + entry.pidx = intr_ctx->pidx; + + /* Write it to the buffer. */ + this->write_irq_ring_entry(ring_idx, &entry); + + /* Send the MSI-X associated to the ring. */ + this->msix_trig[intr_ctx->vector].notify(); + } else { + /* Direct interrupt: legacy or MSI-X. */ + /* Pends an IRQ for the driver. */ +#ifdef QDMA_SOFT_IP + this->regs.u32[R_GLBL_INTR_CFG] |= + R_GLBL_INTR_CFG_INT_PEND; + this->update_legacy_irq(); +#endif + /* Send the MSI-X. */ + this->msix_trig[get_vec(qid, h2c)].notify(); + } + } + } + + /* Update the IRQ. */ + void update_legacy_irq(void) + { +#ifndef QDMA_SOFT_IP + return; +#endif + + bool irq_on; + + if (this->regs.u32[R_GLBL_INTR_CFG] & R_GLBL_INTR_CFG_INT_EN) { + /* Yes, so consider sending a legacy IRQ not an MSI-X. + */ + irq_on = this->regs.u32[R_GLBL_INTR_CFG] & + R_GLBL_INTR_CFG_INT_PEND; + + this->irq[0].write(!!irq_on); + } + } + + /* Descriptors. */ + void fetch_descriptor(uint64_t addr, uint8_t size, uint8_t *data) { + sc_time delay(SC_ZERO_TIME); + tlm::tlm_generic_payload trans; + + /* Do only 4bytes transactions. */ + trans.set_command(tlm::TLM_READ_COMMAND); + trans.set_data_length(4); + trans.set_streaming_width(4); + + for (int i = 0; i < size; i += 4) { + trans.set_address(addr + i); + trans.set_data_ptr(data + i); + this->dma->b_transport(trans, delay); + if (trans.get_response_status() != + tlm::TLM_OK_RESPONSE) { + goto err; + } + } + + return; +err: + SC_REPORT_ERROR("qdma", "error fetching the descriptor"); + } + + void descriptor_writeback(uint64_t addr, uint8_t size, uint8_t *data) { + sc_time delay(SC_ZERO_TIME); + tlm::tlm_generic_payload trans; + + trans.set_command(tlm::TLM_WRITE_COMMAND); + trans.set_address(addr); + trans.set_data_ptr(data); + trans.set_data_length(size); + trans.set_streaming_width(size); + + this->dma->b_transport(trans, delay); + + if (trans.get_response_status() != tlm::TLM_OK_RESPONSE) { + SC_REPORT_ERROR("qdma", + "error writing back the descriptor"); + } + } + + /* Write the IRQ ring entry and increment the ring pointer and the + * color in case of a warp arround. NOTE: The IRQ context is not queue + * specific, but rather each queue has a ring index which is selecting + * the interrupt context. In the CPM4 flavour it's defined in the + * qid2vec table. */ + void write_irq_ring_entry(uint32_t ring_idx, + const INTR_RING_ENTRY *entry) { + sc_time delay(SC_ZERO_TIME); + tlm::tlm_generic_payload trans; + uint64_t addr; + INTR_CTX *intr_ctx = + (INTR_CTX *)this->queue_contexts[ring_idx] + [QDMA_CTXT_SELC_INT_COAL].data; + + /* Compute the address of the entry. */ + addr = QDMA_INTR_RING_ENTRY_ADDR(intr_ctx->baddr); + addr += QDMA_INTR_RING_ENTRY_SZ + * this->irq_ring_entry_idx[ring_idx]; + + trans.set_command(tlm::TLM_WRITE_COMMAND); + trans.set_address(addr); + trans.set_data_ptr((unsigned char *)entry); + trans.set_data_length(QDMA_INTR_RING_ENTRY_SZ); + trans.set_streaming_width(QDMA_INTR_RING_ENTRY_SZ); + + this->dma->b_transport(trans, delay); + + if (trans.get_response_status() != tlm::TLM_OK_RESPONSE) { + SC_REPORT_ERROR("qdma", + "error writing to the IRQ ring"); + } + + /* Now that the entry is written increment the counter, if + * there is a wrap around, invert the color, so the driver + * doesn't risk to miss / overwrite data. */ + this->irq_ring_entry_idx[ring_idx]++; + if (this->irq_ring_entry_idx[ring_idx] * QDMA_INTR_RING_ENTRY_SZ + == (1 + intr_ctx->page_size) * 4096) { + this->irq_ring_entry_idx[ring_idx] = 0; + intr_ctx->color = intr_ctx->color ? 0 : 1; + } + } + + void axi_master_light_bar_b_transport(tlm::tlm_generic_payload &trans, + sc_time &delay) { + tlm::tlm_command cmd = trans.get_command(); + sc_dt::uint64 addr = trans.get_address(); + unsigned char *data = trans.get_data_ptr(); + unsigned int len = trans.get_data_length(); + unsigned char *byte_en = trans.get_byte_enable_ptr(); + unsigned int s_width = trans.get_streaming_width(); + uint32_t v = 0; + + if (byte_en || len > 4 || s_width < len) { + goto err; + } + + if (cmd == tlm::TLM_READ_COMMAND) { + switch (addr >> 2) { + default: + v = this->axi_regs.u32[addr >> 2]; + break; + } + memcpy(data, &v, len); + } else if (cmd == tlm::TLM_WRITE_COMMAND) { + memcpy(&v, data, len); + switch (addr >> 2) { + default: + this->axi_regs.u32[addr >> 2] = v; + break; + } + } else { + goto err; + } + + trans.set_response_status(tlm::TLM_OK_RESPONSE); + return; + +err: + SC_REPORT_WARNING("qdma", + "unsupported read / write on the axi bar"); + trans.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE); + } + + void ringCQHeadDoorbell(uint16_t qid, uint16_t head){ + CQueue *pQueue = ppCQueue[qid]; + + if (pQueue) { + //uint16_t oldhead = pQueue->getHead(); + //uint32_t oldcount = pQueue->getItemCount(); + + pQueue->setHead(head); + + // TODO(Jhieb) explore interrupts. + /*if (pQueue->interruptEnabled()) { + clearInterrupt(pQueue->getInterruptVector()); + }*/ + } + } + + void ringSQTailDoorbell(uint16_t qid, uint16_t tail){ + SQueue *pQueue = ppSQueue[qid]; + + if (pQueue) { + //uint16_t oldtail = pQueue->getTail(); + //uint32_t oldcount = pQueue->getItemCount(); + + pQueue->setTail(tail); + } + } + + void controller_identify(uint8_t *data){ + uint16_t vid, ssvid; + uint64_t totalSize; + uint64_t unallocated; + + // TODO(jhieb) support? + //pParent->getVendorID(vid, ssvid); + + totalSize = totalLogicalPages * logicalPageSize; + // TODO(jhieb) isn't this inverted wrong? + unallocated = allocatedLogicalPages * logicalPageSize; + //pSubsystem->getNVMCapacity(totalSize, unallocated); + + SC_REPORT_WARNING("qdma", "CONTROLLER IDFY"); + + unallocated = totalSize - unallocated; + + // TODO(jhieb) move all of this into a struct? + /** Controller Capabilities and Features **/ + { + // PCI Vendor ID + memcpy(data + 0x0000, &vid, 2); + + // PCI Subsystem Vendor ID + memcpy(data + 0x0002, &ssvid, 2); + + // Serial Number + memcpy(data + 0x0004, "00000000000000000000", 0x14); + + // Model Number + memcpy(data + 0x0018, "SimpleSSD NVMe Controller by CAMELab ", 0x28); + + // Firmware Revision + memcpy(data + 0x0040, "02.01.03", 0x08); + + // Recommended Arbitration Burst + data[0x0048] = 0x00; + + // IEEE OUI Identifier + { + data[0x0049] = 0x00; + data[0x004A] = 0x00; + data[0x004B] = 0x00; + } + + // Controller Multi-Path I/O and Namespace Sharing Capabilities + // [Bits ] Description + // [07:04] Reserved + // [03:03] 1 for Asymmetric Namespace Access Reporting + // [02:02] 1 for SR-IOV Virtual Function, 0 for PCI (Physical) Function + // [01:01] 1 for more than one host may connected to NVM subsystem + // [00:00] 1 for NVM subsystem may has more than one NVM subsystem port + data[0x004C] = 0x00; + + // Maximum Data Transfer Size + data[0x004D] = 0x00; // No limit + + // Controller ID + { + data[0x004E] = 0x00; + data[0x004F] = 0x00; + } + + // Version + { + data[0x0050] = 0x01; + data[0x0051] = 0x04; + data[0x0052] = 0x00; + data[0x0053] = 0x00; + } // NVM Express 1.4 Compliant Controller + + // RTD3 Resume Latency + { + data[0x0054] = 0x00; + data[0x0055] = 0x00; + data[0x0056] = 0x00; + data[0x0057] = 0x00; + } // Not reported + + // RTD3 Enter Latency + { + data[0x0058] = 0x00; + data[0x0059] = 0x00; + data[0x005A] = 0x00; + data[0x005B] = 0x00; + } // Not repotred + + // Optional Asynchronous Events Supported + { + // [Bits ] Description + // [31:15] Reserved + // [14:14] 1 for Support Endurance Group Event Aggregate Log Page Change + // Notice + // [13:13] 1 for Support LBA Status Information Notice + // [12:12] 1 for Support Predictable Latency Event Aggregate Log Change + // Notice + // [11:11] 1 for Support Asymmetric Namespace Access Change Notice + // [10:10] Reserved + // [09:09] 1 for Support Firmware Activation Notice + // [08:08] 1 for Support Namespace Attributes Notice + // [07:00] Reserved + data[0x005C] = 0x00; + data[0x005D] = 0x00; + data[0x005E] = 0x00; + data[0x005F] = 0x00; + } + + // Controller Attributes + { + // [Bits ] Description + // [31:01] Reserved + // [09:09] 1 for Support UUID List + // [08:08] 1 for Support SQ Associations + // [07:07] 1 for Support Namespace Granularity + // [06:06] 1 for Traffic Based Keep Alive Support + // [05:05] 1 for Support Predictable Latency Mode + // [04:04] 1 for Support Endurance Group + // [03:03] 1 for Support Read Recovery Levels + // [02:02] 1 for Support NVM Sets + // [01:01] 1 for Support Non-Operational Power State Permissive Mode + // [00:00] 1 for Support 128-bit Host Identifier + data[0x0060] = 0x00; + data[0x0061] = 0x00; + data[0x0062] = 0x00; + data[0x0063] = 0x00; + } + + // Read Recovery Levels Supported + { + // [Bits ] Description + // [15:15] 1 for Read Recovery Level 15 - Fast Fail + // ... + // [04:04] 1 for Read Recovery Level 4 - Default + // ... + // [00:00] 1 for Read Recovery Level 0 + data[0x0064] = 0x00; + data[0x0065] = 0x00; + } + + memset(data + 0x0066, 0, 9); // Reserved + + // Controller Type + // [Value] Description + // [ 0h] Reserved (Controller Type not reported) + // [ 1h] I/O Controller + // [ 2h] Discovery Controller + // [ 3h] Administrative Controller + // [4h to FFh] Reserved + data[0x006F] = 0x01; + + // FRU Globally Unique Identifier + memset(data + 0x0070, 0, 16); + + // Command Retry Delay Time 1 + { + data[0x0080] = 0x00; + data[0x0081] = 0x00; + } + + // Command Retry Delay Time 2 + { + data[0x0082] = 0x00; + data[0x0083] = 0x00; + } + + // Command Retry Delay Time 3 + { + data[0x0084] = 0x00; + data[0x0085] = 0x00; + } + + memset(data + 0x0086, 0, 106); // Reserved + memset(data + 0x00F0, 0, 16); // See NVMe-MI Specification + } + + /** Admin Command Set Attributes & Optional Controller Capabilities **/ + { + // Optional Admin Command Support + { + // [Bits ] Description + // [15:10] Reserved + // [09:09] 1 for SupportGet LBA Status capability + // [08:08] 1 for Support Doorbell Buffer Config command + // [07:07] 1 for Support Virtualization Management command + // [06:06] 1 for Support NVMe-MI Send and NVMe-MI Receive commands + // [05:05] 1 for Support Directives + // [04:04] 1 for Support Device Self-Test command + // [03:03] 1 for Support Namespace Management and Namespace Attachment + // commands + // [02:02] 1 for Support Firmware Commit and Firmware Image Download + // commands + // [01:01] 1 for Support Format NVM command + // [00:00] 1 for Support Security Send and Security Receive commands + data[0x0100] = 0x0A; + data[0x0101] = 0x00; + } + + // Abort Command Limit + data[0x0102] = 0x03; // Recommanded value is 4 (3 + 1) + + // Asynchronous Event Request Limit + data[0x0103] = 0x03; // Recommanded value is 4 (3 + 1)) + + // Firmware Updates + // [Bits ] Description + // [07:05] Reserved + // [04:04] 1 for Support firmware activation without a reset + // [03:01] The number of firmware slot + // [00:00] 1 for First firmware slot is read only, 0 for read/write + data[0x0104] = 0x00; + + // Log Page Attributes + // [Bits ] Description + // [07:05] Reserved + // [04:04] 1 for Support Persisten Event log + // [03:03] 1 for Support Telemetry Host-Initiated and Telemetry Controller- + // Initiated log pages and Telemetry Log Notices + // [02:02] 1 for Support extended data for Get Log Page command + // [01:01] 1 for Support Command Effects log page + // [00:00] 1 for Support S.M.A.R.T. / Health information log page per + // namespace basis + data[0x0105] = 0x01; + + // Error Log Page Entries, 0's based value + data[0x0106] = 0x63; // 64 entries + + // Number of Power States Support, 0's based value + data[0x0107] = 0x00; // 1 states + + // Admin Vendor Specific Command Configuration + // [Bits ] Description + // [07:01] Reserved + // [00:00] 1 for all vendor specific commands use the format at Figure 12. + // 0 for format is vendor specific + data[0x0108] = 0x00; + + // Autonomous Power State Transition Attributes + // [Bits ] Description + // [07:01] Reserved + // [00:00] 1 for Support autonomous power state transitions + data[0x0109] = 0x00; + + // Warning Composite Temperature Threshold + { + data[0x010A] = 0x00; + data[0x010B] = 0x00; + } + + // Critical Composite Temperature Threshold + { + data[0x010C] = 0x00; + data[0x010D] = 0x00; + } + + // Maximum Time for Firmware Activation + { + data[0x010E] = 0x00; + data[0x010F] = 0x00; + } + + // Host Memory Buffer Preferred Size + { + data[0x0110] = 0x00; + data[0x0111] = 0x00; + data[0x0112] = 0x00; + data[0x0113] = 0x00; + } + + // Host Memory Buffer Minimum Size + { + data[0x0114] = 0x00; + data[0x0115] = 0x00; + data[0x0116] = 0x00; + data[0x0117] = 0x00; + } + + // Total NVM Capacity + { + memcpy(data + 0x118, &totalSize, 8); + memset(data + 0x120, 0, 8); + } + + // Unallocated NVM Capacity + { + memcpy(data + 0x118, &unallocated, 8); + memset(data + 0x120, 0, 8); + } + + // Replay Protected Memory Block Support + { + // [Bits ] Description + // [31:24] Access Size + // [23:16] Total Size + // [15:06] Reserved + // [05:03] Authentication Method + // [02:00] Number of RPMB Units + data[0x0138] = 0x00; + data[0x0139] = 0x00; + data[0x013A] = 0x00; + data[0x013B] = 0x00; + } + + // Extended Device Self-Test Time + { + data[0x013C] = 0x00; + data[0x013D] = 0x00; + } + + // Device Self-Test Options + // [Bits ] Description + // [07:01] Reserved + // [00:00] 1 for Support only one device self-test operation in process at + // a time + data[0x013E] = 0x00; + + // Firmware Update Granularity + data[0x013F] = 0x00; + + // Keep Alive Support + { + data[0x0140] = 0x00; + data[0x0141] = 0x00; + } + + // Host Controlled Thermal Management Attributes + { + // [Bits ] Description + // [15:01] Reserved + // [00:00] 1 for Support host controlled thermal management + data[0x0142] = 0x00; + data[0x0143] = 0x00; + } + + // Minimum Thernam Management Temperature + { + data[0x0144] = 0x00; + data[0x0145] = 0x00; + } + + // Maximum Thernam Management Temperature + { + data[0x0146] = 0x00; + data[0x0147] = 0x00; + } + + // Sanitize Capabilities + { + // [Bits ] Description + // [31:30] No-Deallocate Modifies Media After Sanitize + // [29:29] No-Deallocate Inhibited + // [28:03] Reserved + // [02:02] 1 for Support Overwrite + // [01:01] 1 for Support Block Erase + // [00:00] 1 for Support Crypto Erase + data[0x0148] = 0x00; + data[0x0149] = 0x00; + data[0x014A] = 0x00; + data[0x014B] = 0x00; + } + + // Host Memory Buffer Minimum Descriptor Entry Size + { + data[0x014C] = 0x00; + data[0x014D] = 0x00; + data[0x014E] = 0x00; + data[0x014F] = 0x00; + } + + // Host Memory Maximum Descriptors Entries + { + data[0x0150] = 0x00; + data[0x0151] = 0x00; + } + + // NVM Set Identifier Maximum + { + data[0x0152] = 0x00; + data[0x0153] = 0x00; + } + + // Endurance Group Identifier Maximum + { + data[0x0154] = 0x00; + data[0x0155] = 0x00; + } + + // ANA Transition Time + data[0x0156] = 0x00; + + // Asymmetric Namespace Access Capabilities + // [Bits ] Description + // [07:07] 1 for Support non-zero ANAGRPID + // [06:06] 1 for ANAGRPID does not change while namespace is attached + // [05:05] Reserved + // [04:04] 1 for Support ANA Change state + // [03:03] 1 for Support ANA Persistent Loss state + // [02:02] 1 for Support ANA Inaccessible state + // [01:01] 1 for Support ANA Non-Optimized state + // [00:00] 1 for Support ANA Optimized state + data[0x157] = 0x00; + + // ANA Group Identifier Maximum + { + data[0x0158] = 0x00; + data[0x0159] = 0x00; + data[0x015A] = 0x00; + data[0x015B] = 0x00; + } + + // Number of ANA AGroup Identifiers + { + data[0x015C] = 0x00; + data[0x015D] = 0x00; + data[0x015E] = 0x00; + data[0x015F] = 0x00; + } + + // Persistent Event Log Size + { + data[0x0160] = 0x00; + data[0x0161] = 0x00; + data[0x0162] = 0x00; + data[0x0163] = 0x00; + } + + // Reserved + memset(data + 0x0164, 0, 156); + } + + /** NVM Command Set Attributes **/ + { + // Submission Queue Entry Size + // [Bits ] Description + // [07:04] Maximum Submission Queue Entry Size + // [03:00] Minimum Submission Queue Entry Size + data[0x0200] = 0x66; // 64Bytes, 64Bytes + + // Completion Queue Entry Size + // [Bits ] Description + // [07:04] Maximum Completion Queue Entry Size + // [03:00] Minimum Completion Queue Entry Size + data[0x0201] = 0x44; // 16Bytes, 16Bytes + + // Maximum Outstanding Commands + { + data[0x0202] = 0x00; + data[0x0203] = 0x00; + } + + // Number of Namespaces + // SimpleSSD supports infinite number of namespaces (0xFFFFFFFD) + // But kernel's DIV_ROUND_UP has problem when number is too big + // #define _KERNEL_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) + // This wrong macro introduces DIV_ROUND_UP(0xFFFFFFFD, 1024) to zero + // So we use 1024 here, for only one IDENTIFY NSLIST command + // TODO(jhieb) is this necessary?? + //*(uint32_t *)(data + 0x0204) = 1024; + *(uint32_t *)(data + 0x0204) = 1; + + // Optional NVM Command Support + { + // [Bits ] Description + // [15:08] Reserved + // [07:07] 1 for Support Verify command + // [06:06] 1 for Support Timestamp features + // [05:05] 1 for Support reservations + // [04:04] 1 for Support Save field in Set Features command and Select + // field in Get Features command + // [03:03] 1 for Support Write Zeros command + // [02:02] 1 for Support Dataset Management command + // [01:01] 1 for Support Write Uncorrectable command + // [00:00] 1 for Support Compare command + data[0x0208] = 0x05; + data[0x0209] = 0x00; + } + + // Fused Operation Support + { + // [Bits ] Description + // [15:01] Reserved + // [00:00] 1 for Support Compare and Write fused operation + data[0x020A] = 0x00; + data[0x020B] = 0x00; + } + + // Format NVM Attributes + // [Bits ] Description + // [07:03] Reserved + // [02:02] 1 for Support cryptographic erase + // [01:01] 1 for Support cryptographic erase performed on all namespaces, + // 0 for namespace basis + // [00:00] 1 for Format on specific namespace results on format on all + // namespaces, 0 for namespace basis + data[0x020C] = 0x00; + + // Volatile Write Cache + // [Bits ] Description + // [07:03] Reserved + // [02:01] Indicated Flush comand behavior if the NSID is 0xFFFFFFFF + // [00:00] 1 for volatile write cache is present + data[0x020D] = 1; + data[0x020D] |= 0x06; + + // Atomic Write Unit Normal + { + data[0x020E] = 0x00; + data[0x020F] = 0x00; + } + + // Atomic Write Unit Power Fail + { + data[0x0210] = 0x00; + data[0x0211] = 0x00; + } + + // NVM Vendor Specific Command Configuration + // [Bits ] Description + // [07:01] Reserved + // [00:00] 1 for all vendor specific commands use the format at Figure 12. + // 0 for format is vendor specific + data[0x0212] = 0x00; + + // Namespace Write Protection Capabilities + // [Bits ] Description + // [07:03] Reserved + // [02:02] 1 for Support Permenant Write Protect state + // [01:01] 1 for Support Write Protect Until Power Cycle state + // [00:00] 1 for Support No Write Protect and Write Protect state + data[0x0213] = 0x00; + + // Atomic Compare & Write Unit + { + data[0x0214] = 0x00; + data[0x0215] = 0x00; + } + + // Reserved + memset(data + 0x0216, 0, 2); + + // SGL Support + { + // [Bits ] Description + // [31:22] Reserved + // [21:21] 1 for Support Ransport SGL Data Block + // [20:20] 1 for Support Address field in SGL Data Block + // [19:19] 1 for Support MPTR containing SGL descriptor + // [18:18] 1 for Support MPTR/DPTR containing SGL with larger than amount + // of data to be trasferred + // [17:17] 1 for Support byte aligned contiguous physical buffer of + // metadata is supported + // [16:16] 1 for Support SGL Bit Bucket descriptor + // [15:03] Reserved + // [02:02] 1 for Support Keyed SGL Data Block descriptor + // [01:01] Reserved + // [00:00] 1 for Support SGLs in NVM Command Set + data[0x0218] = 0x01; + data[0x0219] = 0x00; + data[0x021A] = 0x17; + data[0x021B] = 0x00; + } + + // Maximun Number of Allowed Namespaces + *(uint32_t *)(data + 0x021C) = 0; + + // Reserved + memset(data + 0x0220, 0, 224); + + // NVM Subsystem NVMe Qualified Name + { + memset(data + 0x300, 0, 0x100); + strncpy((char *)data + 0x0300, + "nqn.2014-08.org.nvmexpress:uuid:270a1c70-962c-4116-6f1e340b9321", + 0x44); + } + + // Reserved + memset(data + 0x0400, 0, 768); + + // NVMe over Fabric + memset(data + 0x0700, 0, 256); + } + + /** Power State Descriptors **/ + // Power State 0 + /// Descriptor + { + // Maximum Power + { + data[0x0800] = 0xC4; + data[0x0801] = 0x09; + } + + // Reserved + data[0x0802] = 0x00; + + // [Bits ] Description + // [31:26] Reserved + // [25:25] Non-Operational State + // [24:24] Max Power Scale + data[0x0803] = 0x00; + + // Entry Latency + { + data[0x0804] = 0x00; + data[0x0805] = 0x00; + data[0x0806] = 0x00; + data[0x0807] = 0x00; + } + + // Exit Latency + { + data[0x0808] = 0x00; + data[0x0809] = 0x00; + data[0x080A] = 0x00; + data[0x080B] = 0x00; + } + + // [Bits ] Description + // [103:101] Reserved + // [100:096] Relative Read Throughput + data[0x080C] = 0x00; + + // [Bits ] Description + // [111:109] Reserved + // [108:104] Relative Read Latency + data[0x080D] = 0x00; + + // [Bits ] Description + // [119:117] Reserved + // [116:112] Relative Write Throughput + data[0x080E] = 0x00; + + // [Bits ] Description + // [127:125] Reserved + // [124:120] Relative Write Latency + data[0x080E] = 0x00; + + // Idle Power + { + data[0x080F] = 0x00; + data[0x0810] = 0x00; + } + + // [Bits ] Description + // [151:150] Idle Power Scale + // [149:144] Reserved + data[0x0811] = 0x00; + + // Reserved + data[0x0812] = 0x00; + + // Active Power + { + data[0x0813] = 0x00; + data[0x0814] = 0x00; + } + + // [Bits ] Description + // [183:182] Active Power Scale + // [181:179] Reserved + // [178:176] Active Power Workload + data[0x0815] = 0x00; + + // Reserved + memset(data + 0x0816, 0, 9); + } + + // PSD1 ~ PSD31 + memset(data + 0x0820, 0, 992); + + // Vendor specific area + memset(data + 0x0C00, 0, 1024); + + } + + void read_dma_host(uint64_t size, uint64_t addr, uint8_t *buffer){ + // TODO(jhieb) this has problems with the memory being outside of the card memory space need to figure it out. + // Currently hacking by using the descriptor writeback in chunks. + //this->do_mm_dma(reinterpret_cast(buffer), req.entry.data1, 0x1000, false); + // Write the identify buffer to host memory piece by piece + const size_t chunk_size = 64; // Write 64 bytes at a time, max desc size. + const size_t total_size = size; + for (size_t offset = 0; offset < total_size; offset += chunk_size) { + size_t remaining = total_size - offset; + size_t current_chunk = (remaining < chunk_size) ? remaining : chunk_size; + this->fetch_descriptor( + addr + offset, // Host PRP address + offset + current_chunk, // Current chunk size + buffer + offset // Buffer + offset + ); + } + } + + void write_dma_prp_list_host(PRPList* prpList, uint8_t *buffer){ + std::vector prpVec = prpList->getPRPList(); + for(size_t i=0;i < prpVec.size();i++){ + write_dma_host(prpVec[i].size, prpVec[i].addr, buffer); + // move buffer ptr forward by size of prp that we just wrote for the next one. + buffer += prpVec[i].size / 8; + } + } + + void write_dma_sgl_list_host(SGL* sgl, uint8_t *buffer){ + std::vector chunkVec = sgl->getChunkList(); + for(size_t i=0;i < chunkVec.size();i++){ + write_dma_host(chunkVec[i].length, chunkVec[i].addr, buffer); + // move buffer ptr forward by size of chunk that we just wrote for the next one. + buffer += chunkVec[i].length / 8; + } + } + + void write_dma_host(uint64_t size, uint64_t addr, uint8_t *buffer){ + // TODO(jhieb) this has problems with the memory being outside of the card memory space need to figure it out. + // Currently hacking by using the descriptor writeback in chunks. + //this->do_mm_dma(reinterpret_cast(buffer), req.entry.data1, 0x1000, false); + // Write the identify buffer to host memory piece by piece + const size_t chunk_size = 64; // Write 64 bytes at a time, max desc size. + const size_t total_size = size; + for (size_t offset = 0; offset < total_size; offset += chunk_size) { + size_t remaining = total_size - offset; + size_t current_chunk = (remaining < chunk_size) ? remaining : chunk_size; + + this->descriptor_writeback( + addr + offset, // Host PRP address + offset + current_chunk, // Current chunk size + buffer + offset // Buffer + offset + ); + } + } + + void fillIdentifyNamespace(uint8_t *buffer, + Namespace::Information *info) { + // Namespace Size + memcpy(buffer + 0, &info->size, 8); + + // Namespace Capacity + memcpy(buffer + 8, &info->capacity, 8); + + // Namespace Utilization + // This function also called from OpenChannelSSD subsystem + // TODO(jhieb) support? + /*if (pHIL) { + info->utilization = + pHIL->getUsedPageCount(info->range.slpn, + info->range.slpn + info->range.nlp) * + logicalPageSize / info->lbaSize; + }*/ + + memcpy(buffer + 16, &info->utilization, 8); + + // Namespace Features + // [Bits ] Description + // [07:05] Reserved + // [04:04] NVM Set capabilities + // [03:03] Reuse of NGUID field + // [02:02] 1 for Support Deallocated or Unwritten Logical Block error + // [01:01] 1 for NAWUN, NAWUPF, NACWU are defined + // [00:00] 1 for Support Thin Provisioning + buffer[24] = 0x04; // Trim supported + + // Number of LBA Formats + buffer[25] = nLBAFormat - 1; // 0's based + + // Formatted LBA Size + buffer[26] = info->lbaFormatIndex; + + // End-to-end Data Protection Capabilities + buffer[28] = info->dataProtectionSettings; + + // Namespace Multi-path I/O and Namespace Sharing Capabilities + buffer[30] = info->namespaceSharingCapabilities; + + // NVM capacity + memcpy(buffer + 48, &info->sizeInByteL, 8); + memcpy(buffer + 56, &info->sizeInByteH, 8); + + // LBA Formats + for (uint32_t i = 0; i < nLBAFormat; i++) { + memcpy(buffer + 128 + i * 4, lbaFormat + i, 4); + } + + // OpenChannel SSD + /*if (!pHIL) { + buffer[384] = 0x01; + }*/ + } + + bool identify(SQEntryWrapper &req) { + bool err = false; + + SC_REPORT_WARNING("qdma", "IDENTIFY"); + CQEntryWrapper resp(req); + uint8_t cns = req.entry.dword10 & 0xFF; + //uint16_t cntid = (req.entry.dword10 & 0xFFFF0000) >> 16; + bool ret = true; + + //RequestContext *pContext = new RequestContext(func, resp); + //uint16_t idx = 0; + + //uint8_t* buffer = (uint8_t *)calloc(0x1000, sizeof(uint8_t)); + uint8_t buffer[0x1000]; //local storage. + memset(buffer, 0, sizeof(buffer)); + //pContext->buffer = (uint8_t *)calloc(0x1000, sizeof(uint8_t)); + //debugprint(LOG_HIL_NVME, "ADMIN | Identify | CNS %d | CNTID %d | NSID %d", + // cns, cntid, req.entry.namespaceID); + std::ostringstream msg; + switch (cns) { + case CNS_IDENTIFY_NAMESPACE: + msg << "ctrl idfy nsid " << req.entry.namespaceID; + SC_REPORT_INFO("qdma", msg.str().c_str()); + + if (req.entry.namespaceID == NSID_ALL) { + // FIXME: Not called by current(4.9.32) NVMe driver + } + else { + for (auto &iter : lNamespaces) { + if (iter->isAttached() && iter->getNSID() == req.entry.namespaceID) { + // TODO(jhieb) does this work with multiple namespaces with the buffer pointer? + fillIdentifyNamespace(buffer, iter->getInfo()); + } + } + } + + break; + case CNS_IDENTIFY_CONTROLLER: + controller_identify(buffer); + break; + case CNS_ACTIVE_NAMESPACE_LIST: + SC_REPORT_ERROR("qdma", "unsupported idfy"); + /* + if (req.entry.namespaceID >= NSID_ALL - 1) { + err = true; + resp.makeStatus(true, false, TYPE_GENERIC_COMMAND_STATUS, + STATUS_ABORT_INVALID_NAMESPACE); + } + else { + for (auto &iter : lNamespaces) { + if (iter->isAttached() && iter->getNSID() > req.entry.namespaceID) { + ((uint32_t *)pContext->buffer)[idx++] = iter->getNSID(); + } + } + } + */ + break; + case CNS_ALLOCATED_NAMESPACE_LIST: + SC_REPORT_ERROR("qdma", "unsupported idfy"); + /* + if (req.entry.namespaceID >= NSID_ALL - 1) { + err = true; + resp.makeStatus(true, false, TYPE_GENERIC_COMMAND_STATUS, + STATUS_ABORT_INVALID_NAMESPACE); + } + else { + for (auto &iter : lNamespaces) { + if (iter->getNSID() > req.entry.namespaceID) { + ((uint32_t *)pContext->buffer)[idx++] = iter->getNSID(); + } + } + }*/ + + break; + case CNS_IDENTIFY_ALLOCATED_NAMESPACE: + SC_REPORT_ERROR("qdma", "unsupported idfy"); + /* + for (auto &iter : lNamespaces) { + if (iter->getNSID() == req.entry.namespaceID) { + fillIdentifyNamespace(pContext->buffer, iter->getInfo()); + } + }*/ + + break; + case CNS_ATTACHED_CONTROLLER_LIST: + SC_REPORT_ERROR("qdma", "unsupported idfy"); + /* + // Only one controller + if (cntid == 0) { + ((uint16_t *)pContext->buffer)[idx++] = 1; + }*/ + + break; + case CNS_CONTROLLER_LIST: + SC_REPORT_ERROR("qdma", "unsupported idfy"); + /* + // Only one controller + if (cntid == 0) { + ((uint16_t *)pContext->buffer)[idx++] = 1; + }*/ + + break; + } + + if (ret && !err) { + write_dma_host(4096, req.entry.data1, buffer); + SC_REPORT_WARNING("qdma", "CTRL IDFY CMPL STS"); + resp.makeStatus(false, false, TYPE_GENERIC_COMMAND_STATUS, STATUS_SUCCESS); + write_cq_entry(resp); + // TODO(jhieb) support SGL. + /*if (req.useSGL) { + pContext->dma = + new SGL(cfgdata, doWrite, pContext, req.entry.data1, req.entry.data2); + } + else { + pContext->dma = new PRPList(cfgdata, doWrite, pContext, req.entry.data1, + req.entry.data2, (uint64_t)0x1000); + }*/ + } + else { + //func(resp); + //free(buffer); + //delete pContext; + } + + return ret; + } + + + bool setFeatures(SQEntryWrapper &req) { + bool err = false; + + CQEntryWrapper resp(req); + uint16_t fid = req.entry.dword10 & 0x00FF; + bool save = req.entry.dword10 & 0x80000000; + + if (save) { + err = true; + resp.makeStatus(false, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_FEATURE_ID_NOT_SAVEABLE); + } + + if (!err) { + switch (fid) { + case FEATURE_NUMBER_OF_QUEUES: + if ((req.entry.dword11 & 0xFFFF) == 0xFFFF || + (req.entry.dword11 & 0xFFFF0000) == 0xFFFF0000) { + resp.makeStatus(false, false, TYPE_GENERIC_COMMAND_STATUS, STATUS_INVALID_FIELD); + } + else { + if ((req.entry.dword11 & 0xFFFF) >= sqsize) { + req.entry.dword11 = (req.entry.dword11 & 0xFFFF0000) | (sqsize - 1); + } + if (((req.entry.dword11 & 0xFFFF0000) >> 16) >= cqsize) { + req.entry.dword11 = + (req.entry.dword11 & 0xFFFF) | ((uint32_t)(cqsize - 1) << 16); + } + + resp.entry.dword0 = req.entry.dword11; + queueAllocated = resp.entry.dword0; + resp.makeStatus(false, false, TYPE_GENERIC_COMMAND_STATUS, STATUS_SUCCESS); + } + + break; + case FEATURE_INTERRUPT_COALESCING: + setCoalescingParameter((req.entry.dword11 >> 8) & 0xFF, + req.entry.dword11 & 0xFF); + resp.makeStatus(false, false, TYPE_GENERIC_COMMAND_STATUS, STATUS_SUCCESS); + break; + case FEATURE_INTERRUPT_VECTOR_CONFIGURATION: + setCoalescing(req.entry.dword11 & 0xFFFF, + req.entry.dword11 & 0x10000); + resp.makeStatus(false, false, TYPE_GENERIC_COMMAND_STATUS, STATUS_SUCCESS); + break; + default: + resp.makeStatus(true, false, TYPE_GENERIC_COMMAND_STATUS, + STATUS_INVALID_FIELD); + break; + } + } + SC_REPORT_WARNING("qdma", "set features CMPL STS"); + write_cq_entry(resp); + + return true; + } + + void setCoalescingParameter(uint8_t time, uint8_t thres) { + aggregationTime = time * 100000000; + aggregationThreshold = thres; + } + + void setCoalescing(uint16_t iv, bool enable) { + auto iter = aggregationMap.find(iv); + + if (iter != aggregationMap.end()) { + iter->second.valid = enable; + iter->second.nextTime = 0; + iter->second.requestCount = 0; + iter->second.pending = false; + } + } + + int createCQueueCtrl(uint16_t cqid, uint16_t size, uint16_t iv, + bool ien, bool pc, uint64_t prp1) { + int ret = 1; // Invalid Queue ID + static DMAFunction empty = [](uint64_t, void *) {}; + if (ppCQueue[cqid] == NULL) { + ppCQueue[cqid] = new CQueue(iv, ien, cqid, size); + ppCQueue[cqid]->setBase(new PRPList(cfgdata, empty, nullptr, prp1, size * cqstride, pc), cqstride, prp1); + ret = 0; + // Interrupt coalescing config + auto iter = aggregationMap.find(iv); + AggregationInfo info; + + info.valid = false; + info.nextTime = 0; + info.requestCount = 0; + + if (iter == aggregationMap.end()) { + aggregationMap.insert({iv, info}); + } + else { + iter->second = info; + } + } + + return ret; + } + + bool createCQueue(SQEntryWrapper &req) { + CQEntryWrapper resp(req); + bool err = false; + + uint16_t cqid = req.entry.dword10 & 0xFFFF; + uint16_t entrySize = ((req.entry.dword10 & 0xFFFF0000) >> 16) + 1; + uint16_t iv = (req.entry.dword11 & 0xFFFF0000) >> 16; + bool ien = req.entry.dword11 & 0x02; + bool pc = req.entry.dword11 & 0x01; + + if (entrySize > cfgdata.maxQueueEntry) { + err = true; + resp.makeStatus(false, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_INVALID_QUEUE_SIZE); + } + + if (!err) { + int ret = createCQueueCtrl(cqid, entrySize, iv, ien, pc, + req.entry.data1); + + if (ret == 1) { + resp.makeStatus(false, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_INVALID_QUEUE_ID); + } else { + resp.makeStatus(false, false, TYPE_GENERIC_COMMAND_STATUS, STATUS_SUCCESS); + } + } + SC_REPORT_WARNING("qdma", "create Cqueue CMPL STS"); + write_cq_entry(resp); + + return true; + } + + int createSQueueCtrl(uint16_t sqid, uint16_t cqid, uint16_t size, + uint8_t priority, bool pc, uint64_t prp1) { + int ret = 1; // Invalid Queue ID + static DMAFunction empty = [](uint64_t, void *) {}; + if (ppSQueue[sqid] == NULL) { + if (ppCQueue[cqid] != NULL) { + ppSQueue[sqid] = new SQueue(cqid, priority, sqid, size); + ppSQueue[sqid]->setBase(new PRPList(cfgdata, empty, nullptr, prp1, size * sqstride, pc), sqstride, prp1); + ret = 0; + } + else { + ret = 2; // Invalid CQueue + } + } + + return ret; + } + + + bool createSQueue(SQEntryWrapper &req) { + CQEntryWrapper resp(req); + bool err = false; + + uint16_t sqid = req.entry.dword10 & 0xFFFF; + uint16_t entrySize = ((req.entry.dword10 & 0xFFFF0000) >> 16) + 1; + uint16_t cqid = (req.entry.dword11 & 0xFFFF0000) >> 16; + uint8_t priority = (req.entry.dword11 & 0x06) >> 1; + bool pc = req.entry.dword11 & 0x01; + + if (entrySize > cfgdata.maxQueueEntry) { + err = true; + resp.makeStatus(false, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_INVALID_QUEUE_SIZE); + } + + if (!err) { + int ret = createSQueueCtrl(sqid, cqid, entrySize, priority, pc, + req.entry.data1); + + if (ret == 1) { + resp.makeStatus(false, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_INVALID_QUEUE_ID); + } + else if (ret == 2) { + resp.makeStatus(false, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_INVALID_COMPLETION_QUEUE); + } else { + resp.makeStatus(false, false, TYPE_GENERIC_COMMAND_STATUS, STATUS_SUCCESS); + } + } + SC_REPORT_WARNING("qdma", "create Squeue CMPL STS"); + write_cq_entry(resp); + + return true; + } + + + bool getLogPage(SQEntryWrapper &req) { + CQEntryWrapper resp(req); + uint16_t numdl = (req.entry.dword10 & 0xFFFF0000) >> 16; + uint16_t lid = req.entry.dword10 & 0xFFFF; + uint16_t numdu = req.entry.dword11 & 0xFFFF; + uint32_t lopl = req.entry.dword12; + uint32_t lopu = req.entry.dword13; + bool ret = false; + bool submit = true; + + uint32_t req_size = (((uint32_t)numdu << 16 | numdl) + 1) * 4; + uint64_t offset = ((uint64_t)lopu << 32) | lopl; + + /* + DMAFunction smartInfo = [offset](uint64_t, void *context) { + RequestContext *pContext = (RequestContext *)context; + + pContext->dma->write(offset, 512, pContext->buffer, dmaDone, context); + };*/ + + switch (lid) { + case LOG_ERROR_INFORMATION: + SC_REPORT_WARNING("qdma", "LOG ERROR INFORMATION"); + ret = true; + break; + case LOG_SMART_HEALTH_INFORMATION: { + SC_REPORT_WARNING("qdma", "LOG SMART HEALTH"); + if (req.entry.namespaceID == NSID_ALL) { + ret = true; + submit = false; + + //pContext->buffer = globalHealth.data; + + /*if (req.useSGL) { + pContext->dma = new SGL(cfgdata, smartInfo, pContext, req.entry.data1, + req.entry.data2); + } + else { + pContext->dma = + new PRPList(cfgdata, smartInfo, pContext, req.entry.data1, + req.entry.data2, (uint64_t)req_size); + }*/ + } + } break; + case LOG_FIRMWARE_SLOT_INFORMATION: + SC_REPORT_WARNING("qdma", "LOG FIRMWARE SLOT"); + ret = true; + break; + default: + ret = true; + resp.makeStatus(true, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_INVALID_LOG_PAGE); + break; + } + + if (ret) { + //write_dma_host(4096, req.entry.data1, buffer); + SC_REPORT_WARNING("qdma", "GET LOG PAGE CMPL"); + resp.makeStatus(false, false, TYPE_GENERIC_COMMAND_STATUS, STATUS_SUCCESS); + write_cq_entry(resp); + // TODO(jhieb) support SGL. + /*if (req.useSGL) { + pContext->dma = + new SGL(cfgdata, doWrite, pContext, req.entry.data1, req.entry.data2); + } + else { + pContext->dma = new PRPList(cfgdata, doWrite, pContext, req.entry.data1, + req.entry.data2, (uint64_t)0x1000); + }*/ + } + else { + //func(resp); + //free(buffer); + //delete pContext; + } + + return ret; + } + + void submitCommand(SQEntryWrapper &req) { + CQEntryWrapper resp(req); + bool processed = false; + + commandCount++; + + // Admin command + if (req.sqID == 0) { + switch (req.entry.dword0.opcode) { + case OPCODE_DELETE_IO_SQUEUE: + //processed = deleteSQueue(req, func); + SC_REPORT_ERROR("qdma", "unsupported opcode DELETE IO SQUEUE"); + break; + case OPCODE_CREATE_IO_SQUEUE: + processed = createSQueue(req); + break; + case OPCODE_GET_LOG_PAGE: + processed = getLogPage(req); + break; + case OPCODE_DELETE_IO_CQUEUE: + //processed = deleteCQueue(req, func); + SC_REPORT_ERROR("qdma", "unsupported opcode DELETE IO CQUEUE"); + break; + case OPCODE_CREATE_IO_CQUEUE: + processed = createCQueue(req); + break; + case OPCODE_IDENTIFY: + processed = identify(req); + break; + case OPCODE_ABORT: + //processed = abort(req, func); + SC_REPORT_ERROR("qdma", "unsupported opcode ABORT"); + break; + case OPCODE_SET_FEATURES: + processed = setFeatures(req); + break; + case OPCODE_GET_FEATURES: + //processed = getFeatures(req, func); + SC_REPORT_ERROR("qdma", "unsupported opcode GET FEATURES"); + break; + case OPCODE_ASYNC_EVENT_REQ: + break; + case OPCODE_NAMESPACE_MANAGEMENT: + //processed = namespaceManagement(req, func); + SC_REPORT_ERROR("qdma", "unsupported opcode"); + break; + case OPCODE_NAMESPACE_ATTACHMENT: + //processed = namespaceAttachment(req, func); + SC_REPORT_ERROR("qdma", "unsupported opcode"); + break; + case OPCODE_FORMAT_NVM: + //processed = formatNVM(req, func); + SC_REPORT_ERROR("qdma", "unsupported opcode"); + break; + default: + resp.makeStatus(true, false, TYPE_GENERIC_COMMAND_STATUS, + STATUS_INVALID_OPCODE); + break; + } + } else { + if (req.entry.namespaceID < NSID_ALL) { + for (auto &iter : lNamespaces) { + if (iter->getNSID() == req.entry.namespaceID) { + uint8_t *buffer = nullptr; + uint64_t slba; + uint16_t nlb; + uint32_t lbaSize; + uint64_t totalBlockSize; + std::ostringstream msg; + PRPList* list; + SGL* sgl; + static DMAFunction empty = [](uint64_t, void *) {}; + CQEntryWrapper resp = iter->submitCommand(req, buffer); + // TODO(jhieb) how do i know if the command needs to write the buffer back? Pass reference to QDMA to namespace so it can do the write? + // option two pass back a counter of how much buffer needs to be written and to where?? + switch (req.entry.dword0.opcode){ + case OPCODE_READ: + slba = ((uint64_t)req.entry.dword11 << 32) | req.entry.dword10; + nlb = (req.entry.dword12 & 0xFFFF) + 1; + lbaSize = iter->getLBASize(); + totalBlockSize = lbaSize * nlb; + + if(req.useSGL){ + sgl = new SGL(cfgdata, empty, nullptr, req.entry.data1, req.entry.data2); + while(sgl->getReqSGLDMA()){ + read_dma_host(sgl->length, sgl->address, sgl->buffer); + sgl->parseSGLSegment(); + } + write_dma_sgl_list_host(sgl, buffer); + } else { + list = new PRPList(cfgdata, empty, nullptr, req.entry.data1, req.entry.data2, totalBlockSize); + // We need to DMA read the PRPList so that we can finish building our PRPList. + if(list->getPRP2ReqDMA()){ + read_dma_host(list->prpListSize, list->prpListAddr, list->buffer); + list->processPRPListBuffer(); + } + write_dma_prp_list_host(list, buffer); + delete list; + } + + msg << "read slba " << slba << " nlb " << nlb; + SC_REPORT_INFO("qdma", msg.str().c_str()); + // delete list; + free(buffer); + buffer = NULL; + break; + default: + break; + } + write_cq_entry(resp); + processed = true; + break; + } + } + } else { + // TODO(jhieb) when does this make sense? + if (lNamespaces.size() > 0) { + uint8_t *buffer = nullptr; + CQEntryWrapper resp = lNamespaces.front()->submitCommand(req, buffer); + processed = true; + } + } + } + + // failed to do work. Invalid namespace status for now? + if (!processed) { + resp.makeStatus(false, false, TYPE_GENERIC_COMMAND_STATUS, + STATUS_ABORT_INVALID_NAMESPACE); + SC_REPORT_ERROR("qdma", "FAILED TO SUBMIT COMMAND"); + //func(resp); + } + } + + void handle_request(){ + // Check SQFIFO + if (lSQFIFO.size() > 0) { + SQEntryWrapper *front = new SQEntryWrapper(lSQFIFO.front()); + lSQFIFO.pop_front(); + submitCommand(*front); + } + } + + void write_cq_entry(CQEntryWrapper resp){ + CQueue *pQueue = ppCQueue[resp.cqID]; + uint64_t addr = pQueue->getBaseAddr() + pQueue->getTail() * cqstride; + pQueue->setData(&(resp.entry)); + this->descriptor_writeback(addr, cqstride, resp.entry.data); + SC_REPORT_WARNING("qdma", "CQ Entry Push MSIX"); + this->msix_trig[resp.cqID].notify(); + } + + SQEntryWrapper read_sq_entry(int qid){ + uint8_t local_buffer[QDMA_DESC_MAX_SIZE]; //local storage. + SQueue *pQueue = ppSQueue[qid]; + uint64_t addr = pQueue->getBaseAddr() + pQueue->getHead() * sqstride; + this->fetch_descriptor(addr, sqstride, local_buffer); + SQEntry entry; + memcpy(&entry, local_buffer, sizeof(SQEntry)); + // increment the head. + pQueue->getData(&entry); + // TODO(jhieb) squid do i need it? + SQEntryWrapper new_cmd(entry, qid, pQueue->getCQID(), pQueue->getHead(), 0); // entry, sqid, cqid, sqhead, squid; + return new_cmd; + } + + void nvme_config_write(sc_dt::uint64 addr, uint32_t data) { + // Admin queue doorbells. + if(addr >= REG_DOORBELL_BEGIN){ + const int dstrd = 4; // doorbell stride + int offset = addr - REG_DOORBELL_BEGIN; + + // Calculate queue type and queue ID from offset + uint32_t uiTemp, uiMask, sqid; + uiTemp = offset / dstrd; + uiMask = (uiTemp & 0x00000001); // 0 for submission queue tail doorbell + // 1 for completion queue head doorbell + sqid = (uiTemp >> 1); // queue ID + + if(uiMask){ + ringCQHeadDoorbell(sqid, data); + SC_REPORT_WARNING("qdma", "RING CQ TAIL DOORBELL"); + } else { + ringSQTailDoorbell(sqid, data); + SC_REPORT_WARNING("qdma", "RING SQ TAIL DOORBELL"); + lSQFIFO.push_back(read_sq_entry(sqid)); + handle_request(); + } + } else { + // non admin queue commands. Primarily nvme config registers at the start of the BAR. + nvme_config_register_write(addr, data); + } + } + + void nvme_config_register_write(sc_dt::uint64 addr, uint32_t data) { + static DMAFunction empty = [](uint64_t, void *) {}; + + switch(addr){ + case REG_CONTROLLER_CONFIG: + + registers.configuration &= 0xFF00000E; + registers.configuration |= (data & 0x00FFFFF1); + + // Update entry size + sqstride = (int)powf(2.f, (registers.configuration & 0x000F0000) >> 16); + cqstride = (int)powf(2.f, (registers.configuration & 0x00F00000) >> 20); + // TODO(jhieb) should assert if these don't match the sizeof(sqentry) and sizeof(cqentry) + + // Update Memory Page Size + cfgdata.memoryPageSizeOrder = + ((registers.configuration & 0x780) >> 7) + 11; // CC.MPS + 12 - 1 + cfgdata.memoryPageSize = + (int)powf(2.f, cfgdata.memoryPageSizeOrder + 1); + + // Update Arbitration Mechanism + arbitration = (registers.configuration & 0x00003800) >> 11; + + // create the admin completion and submission queues. + // Apply to admin queue + if (ppCQueue[0]) { + ppCQueue[0]->setBase( + new PRPList(cfgdata, empty, nullptr, + registers.adminCQueueBaseAddress, + ppCQueue[0]->getSize() * cqstride, true), + cqstride, registers.adminCQueueBaseAddress); + } + if (ppSQueue[0]) { + ppSQueue[0]->setBase( + new PRPList(cfgdata, empty, nullptr, + registers.adminSQueueBaseAddress, + ppSQueue[0]->getSize() * sqstride, true), + sqstride, registers.adminSQueueBaseAddress); + } + // Shutdown notification + if (registers.configuration & 0x0000C000) { + registers.status &= 0xFFFFFFF2; // RDY = 1 + registers.status |= 0x00000005; // Shutdown processing occurring + shutdownReserved = true; + } else if(data & 0x00000001 ){ + // If CFG.EN = 1, Set CSTS.RDY = 1 + this->regs.u32[REG_CONTROLLER_STATUS >> 2] |= 0x00000001; + // TODO(Jhieb) disabling CSTS for testing. + //this->regs.u32[REG_CONTROLLER_STATUS >> 2] &= 0xFFFFFFFE; + } else { + // If CFG.EN = 0, Set CSTS.RDY = 0 + this->regs.u32[REG_CONTROLLER_STATUS >> 2] &= 0xFFFFFFFE; + } + break; + case REG_ADMIN_QUEUE_ATTRIBUTE: + registers.adminQueueAttributes &= 0xF000F000; + registers.adminQueueAttributes |= (data & 0x0FFF0FFF); + break; + case REG_ADMIN_CQUEUE_BASE_ADDR: + registers.adminCQueueBaseAddress = data; + adminQueueInited++; + break; + case REG_ADMIN_CQUEUE_BASE_ADDR + 4: + // upper 32 bytes + registers.adminCQueueBaseAddress = (static_cast(data) << 32) | registers.adminCQueueBaseAddress; + adminQueueInited++; + break; + case REG_ADMIN_SQUEUE_BASE_ADDR: + registers.adminSQueueBaseAddress = data; + adminQueueInited++; + break; + case REG_ADMIN_SQUEUE_BASE_ADDR + 4: + // upper 32 bytes + registers.adminSQueueBaseAddress = (static_cast(data) << 32) | registers.adminSQueueBaseAddress; + adminQueueInited++; + break; + default: + SC_REPORT_WARNING("qdma", "unsupported register write"); + break; + } + + + // If we got all the admin queue details then create the initial queues. + if (adminQueueInited == 4) { + uint16_t entrySize = 0; + adminQueueInited = 0; + entrySize = ((registers.adminQueueAttributes & 0x0FFF0000) >> 16) + 1; + ppCQueue[0] = new CQueue(0, true, 0, entrySize); + entrySize = (registers.adminQueueAttributes & 0x0FFF) + 1; + ppSQueue[0] = new SQueue(0, 0, 0, entrySize); + } + } + + void config_bar_b_transport(tlm::tlm_generic_payload &trans, + sc_time &delay) { + tlm::tlm_command cmd = trans.get_command(); + sc_dt::uint64 addr = trans.get_address(); + unsigned char *data = trans.get_data_ptr(); + unsigned int len = trans.get_data_length(); + unsigned char *byte_en = trans.get_byte_enable_ptr(); + unsigned int s_width = trans.get_streaming_width(); + uint32_t v = 0; + + if (byte_en || len > 4 || s_width < len) { + goto err; + } + if (cmd == tlm::TLM_READ_COMMAND) { + switch (addr >> 2) { + case R_GLBL2_MISC_CAP: + v = this->qdma_get_version(); + break; + default: + v = this->regs.u32[addr >> 2]; + break; + } + memcpy(data, &v, len); + } else if (cmd == tlm::TLM_WRITE_COMMAND) { + bool done = true; + memcpy(&v, data, len); + + // TODO(jhieb) this shouldn't live in the qdma module but ok for now. + nvme_config_write(addr, v); + + /* There is some differences in the register set for + * the cpm4 flavour, handle those register appart. */ + if (this->is_cpm4()) { + switch (addr >> 2) { + case R_DMAP_SEL_INT_CIDX(1, 0) ... + R_DMAP_SEL_CMPT_CIDX(1, QDMA_QUEUE_COUNT): + { + int qid = (addr + - (R_DMAP_SEL_INT_CIDX(1, 0) << 2)) + / 0x10; + + this->regs.u32[addr >> 2] = v; + + switch (addr % 0x10) { + case 0x4: + /* R_DMAP_SEL_H2C_DSC_PIDX(n) */ + this->run_mm_dma(qid, + true); + break; + case 0x8: + /* R_DMAP_SEL_C2H_DSC_PIDX(n) */ + this->run_mm_dma(qid, + false); + break; + default: + break; + } + break; + } + case R_IND_CTXT_CMD(1): + this->handle_ctxt_cmd(v); + /* Drop the busy bit. */ + this->regs.u32[addr >> 2] = + v & 0xFFFFFFFE; + break; + default: + done = false; + break; + } + } else { + switch (addr >> 2) { + case R_DMAP_SEL_INT_CIDX(0, 0) ... + R_DMAP_SEL_CMPT_CIDX(0, QDMA_QUEUE_COUNT): + { + int qid = (addr + - (R_DMAP_SEL_INT_CIDX(0, 0) << 2)) + / 0x10; + + this->regs.u32[addr >> 2] = v; + + switch (addr % 0x10) { + case 0x4: + /* R_DMAP_SEL_H2C_DSC_PIDX(n) */ + this->run_mm_dma(qid, + true); + break; + case 0x8: + /* R_DMAP_SEL_C2H_DSC_PIDX(n) */ + this->run_mm_dma(qid, + false); + break; + default: + break; + } + break; + } + case R_IND_CTXT_CMD(0): + this->handle_ctxt_cmd(v); + /* Drop the busy bit. */ + this->regs.u32[addr >> 2] = + v & 0xFFFFFFFE; + break; + default: + done = false; + break; + } + } + + if (!done) { + switch (addr >> 2) { + case R_CONFIG_BLOCK_IDENT: + case R_GLBL2_MISC_CAP: + /* Read Only register. */ + break; + case R_GLBL_INTR_CFG: + /* W1C */ + if (v + & R_GLBL_INTR_CFG_INT_PEND) { + v &= + ~R_GLBL_INTR_CFG_INT_PEND; + } + this->regs.u32[addr >> 2] = v; + this->update_legacy_irq(); + break; + default: + this->regs.u32[addr >> 2] = v; + break; + } + } + } else { + goto err; + } + + trans.set_response_status(tlm::TLM_OK_RESPONSE); + return; +err: + SC_REPORT_WARNING("qdma", + "unsupported read / write on the config bar"); + trans.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE); + return; + } + + union { + struct { + uint32_t config_block_ident; + }; + uint32_t u32[0x18000]; + } regs; + + struct { + uint32_t u32[0xA8]; + } axi_regs; + + /* Reset the IP. */ + void reset(void) { + this->regs.config_block_ident = 0x1FD30001; + /* One bar mapped for PF0. */ + this->regs.u32[R_GLBL2_PF_BARLITE_INT] = 0x01; + this->regs.u32[R_GLBL2_CHANNEL_QDMA_CAP] = + QDMA_QUEUE_COUNT; + this->regs.u32[R_GLBL2_CHANNEL_MDMA] = 0x00030f0f; + this->regs.u32[R_GLBL2_CHANNEL_FUNC_RET] = 0; + this->regs.u32[R_GLBL2_PF_BARLITE_EXT] = 1 + << QDMA_USER_BAR_ID; + } + + void init_msix() + { + for (int i = 0; i < NR_QDMA_IRQ; i++) { + sc_spawn(sc_bind(&qdma::msix_strobe, + this, + i)); + } + } + + bool createNamespace(uint32_t nsid, Namespace::Information *info) { + std::list allocated; + std::list unallocated; + + // Allocate LPN + uint64_t requestedLogicalPages = + info->size / logicalPageSize * lbaSize[info->lbaFormatIndex]; + uint64_t unallocatedLogicalPages = totalLogicalPages - allocatedLogicalPages; + + if (requestedLogicalPages > unallocatedLogicalPages) { + return false; + } + + // Collect allocated slots + for (auto &iter : lNamespaces) { + allocated.push_back(iter->getInfo()->range); + } + + // Sort + allocated.sort([](const LPNRange &a, const LPNRange &b) -> bool { + return a.slpn < b.slpn; + }); + + // Merge + auto iter = allocated.begin(); + + if (iter != allocated.end()) { + auto next = iter; + + while (true) { + next++; + + if (iter != allocated.end() || next == allocated.end()) { + break; + } + + if (iter->slpn + iter->nlp == next->slpn) { + iter = allocated.erase(iter); + next = iter; + } + else { + iter++; + } + } + } + + // Invert + unallocated.push_back(LPNRange(0, totalLogicalPages)); + + for (auto &iter : allocated) { + // Split last item + auto &last = unallocated.back(); + + if (last.slpn <= iter.slpn && + last.slpn + last.nlp >= iter.slpn + iter.nlp) { + unallocated.push_back(LPNRange( + iter.slpn + iter.nlp, last.slpn + last.nlp - iter.slpn - iter.nlp)); + last.nlp = iter.slpn - last.slpn; + } + else { + SC_REPORT_WARNING("qdma", "create namespace bug"); + } + } + + // Allocated unallocated area to namespace + for (auto iter = unallocated.begin(); iter != unallocated.end(); iter++) { + if (iter->nlp >= requestedLogicalPages) { + info->range = *iter; + info->range.nlp = requestedLogicalPages; + + break; + } + } + + if (info->range.nlp == 0) { + return false; + } + + allocatedLogicalPages += requestedLogicalPages; + + // Fill Information + info->sizeInByteL = requestedLogicalPages * logicalPageSize; + info->sizeInByteH = 0; + + // Create namespace + Namespace *pNS = new Namespace(cfgdata); + pNS->setData(nsid, info); + + lNamespaces.push_back(pNS); + /* + debugprint(LOG_HIL_NVME, + "NS %-5d| CREATE | LBA size %" PRIu32 " | Capacity %" PRIu64, nsid, + info->lbaSize, info->size); + */ + + lNamespaces.sort([](Namespace *lhs, Namespace *rhs) -> bool { + return lhs->getNSID() < rhs->getNSID(); + }); + + return true; + } + + virtual bool is_cpm4() = 0; + virtual uint32_t qdma_get_version() = 0; + virtual bool irq_aggregation_enabled(int qid, bool h2c) = 0; + /* Either get the MSIX Vector (in direct interrupt mode), or the IRQ + * Ring Index (in indirect interrupt mode). */ + virtual int get_vec(int qid, bool h2c) = 0; +protected: + SW_CTX *get_software_context(int qid, bool h2c) + { + return (SW_CTX *)this->queue_contexts[qid] + [h2c ? QDMA_CTXT_SELC_DEC_SW_H2C + : QDMA_CTXT_SELC_DEC_SW_C2H].data; + } + + qid2vec_ctx_cpm4 *get_qid2vec_context(int qid) + { + return (qid2vec_ctx_cpm4 *)(this->queue_contexts[qid] + [QDMA_CTXT_SELC_FMAP_QID2VEC].data); + } + + /* Current entry in a given interrupt ring. */ + int irq_ring_entry_idx[QDMA_QUEUE_COUNT]; +public: + SC_HAS_PROCESS(qdma); + sc_in rst; + tlm_utils::simple_initiator_socket card_bus; + + /* Interface to toward PCIE. */ + tlm_utils::simple_target_socket config_bar; + tlm_utils::simple_target_socket user_bar; + tlm_utils::simple_initiator_socket dma; + sc_vector > irq; + + qdma(sc_core::sc_module_name name) : + rst("rst"), + card_bus("card_initiator_socket"), + config_bar("config_bar"), + user_bar("user_bar"), + dma("dma"), + irq("irq", NR_QDMA_IRQ) + { + memset(®s, 0, sizeof regs); + + SC_METHOD(reset); + dont_initialize(); + sensitive << rst; + + config_bar.register_b_transport(this, + &qdma::config_bar_b_transport); + user_bar.register_b_transport(this, + &qdma::axi_master_light_bar_b_transport); + this->init_msix(); + + // Nvme setup stuff. + // Allocate array for Command Queues + // MaxIOCQueue = 16 + // MaxIOSQueue = 16 + adminQueueInited = 0; + ppCQueue = (CQueue **)calloc(cqsize+1, sizeof(CQueue *)); + ppSQueue = (SQueue **)calloc(sqsize+1, sizeof(SQueue *)); + + // [Bits ] Name : Description : Current Setting + // [63:56] Reserved + // [55:52] MPSMZX: Memory Page Size Maximum : 2^14 Bytes + // [51:48] MPSMIN: Memory Page Size Minimum : 2^12 Bytes + // [47:45] Reserved + // [44:37] CSS : Command Sets Supported : NVM command set + // [36:36] NSSRS : NVM Subsystem Reset Supported : No + // [35:32] DSTRD : Doorbell Stride : 0 (4 bytes) + // [31:24] TO : Timeout : 40 * 500ms + // [23:19] Reserved + // [18:17] AMS : Arbitration Mechanism Supported : Weighted Round Robin + // [16:16] CQR : Contiguous Queues Required : Yes + // [15:00] MQES : Maximum Queue Entries Supported : 4096 Entries + registers.capabilities = 0x0020002028010FFF; + registers.version = 0x00010201; // NVMe 1.2.1 + + cfgdata.maxQueueEntry = (registers.capabilities & 0xFFFF) + 1; + + allocatedLogicalPages = 0; + + // TODO(jhieb) hard coded config for now. + nandParameters.channel = 8; + nandParameters.package = 4; + nandParameters.die = 2; + nandParameters.plane = 2; + nandParameters.block = 512; + nandParameters.page = 512; + nandParameters.pageSize = 16384; + + nandParameters.superBlock = nandParameters.block * nandParameters.channel * nandParameters.package * nandParameters.die * nandParameters.plane; + nandParameters.pageInSuperPage = 1 * nandParameters.channel * nandParameters.package * nandParameters.die * nandParameters.plane; + // nandParameters.superPageSize = nandParameters.superPageSize; // ? + + ftlParameter.totalPhysicalBlocks = nandParameters.superBlock; + ftlParameter.totalLogicalBlocks = nandParameters.superBlock * (1 - overProvisioningRatio); + ftlParameter.pagesInBlock = nandParameters.page; + ftlParameter.pageSize = nandParameters.pageSize; + // TODO(jhieb) what is this? + ftlParameter.pageCountToMaxPerf = nandParameters.superBlock / nandParameters.block; + + totalLogicalPages = ftlParameter.totalLogicalBlocks * ftlParameter.pagesInBlock; + logicalPageSize = ftlParameter.pageSize; + + if (nNamespaces > 0) { + Namespace::Information info; + uint64_t totalSize; + + for (info.lbaFormatIndex = 0; info.lbaFormatIndex < nLBAFormat;info.lbaFormatIndex++) { + if (lbaSize[info.lbaFormatIndex] == lba) { + info.lbaSize = lbaSize[info.lbaFormatIndex]; + break; + } + } + + if (info.lbaFormatIndex == nLBAFormat) { + SC_REPORT_WARNING("qdma", "Failed to setting LBA size (512B ~ 4KB)"); + } + + // Fill Namespace information + totalSize = totalLogicalPages * logicalPageSize / lba; + info.dataProtectionSettings = 0x00; + info.namespaceSharingCapabilities = 0x00; + + for (uint16_t i = 0; i < nNamespaces; i++) { + info.size = totalSize / nNamespaces; + info.capacity = info.size; + + if (createNamespace(NSID_LOWEST + i, &info)) { + lNamespaces.back()->attach(true); + } + else { + SC_REPORT_WARNING("qdma", "Failed to create namespace"); + } + } + } + } +}; + +class qdma_cpm5: + public qdma +{ +private: + bool is_cpm4() + { + return false; + } + + bool irq_aggregation_enabled(int qid, bool h2c) + { + struct sw_ctx_cpm5 *sw_ctx = this->get_software_context(qid, + h2c); + return (sw_ctx->int_aggr != 0); + } + + int get_vec(int qid, bool h2c) + { + struct sw_ctx_cpm5 *sw_ctx + = this->get_software_context(qid, h2c); + return sw_ctx->vec; + } + + uint32_t qdma_get_version() + { + return QDMA_VERSION_FIELD(QDMA_CPM5_DEVICE_ID, 1, 0); + } +public: + qdma_cpm5(sc_core::sc_module_name name): + qdma(name) + { + } +}; + +class qdma_cpm4: + public qdma +{ +private: + bool is_cpm4() + { + return true; + } + + bool irq_aggregation_enabled(int qid, bool h2c) + { + struct qid2vec_ctx_cpm4 *qid2vec_ctx = + this->get_qid2vec_context(qid); + + return h2c ? qid2vec_ctx->h2c_en_coal + : qid2vec_ctx->c2h_en_coal; + } + + int get_vec(int qid, bool h2c) + { + struct qid2vec_ctx_cpm4 *qid2vec_ctx + = this->get_qid2vec_context(qid); + + return h2c ? qid2vec_ctx->h2c_vector : qid2vec_ctx->c2h_vector; + } + + uint32_t qdma_get_version() + { + return QDMA_VERSION_FIELD(QDMA_CPM4_DEVICE_ID, 0, 0); + } +public: + qdma_cpm4(sc_core::sc_module_name name): + qdma(name) + { + } +}; + +#endif /* __PCI_QDMA_H__ */ diff --git a/systemc-components/pci/nvme_ssd/include/nvme_ssd.h b/systemc-components/pci/nvme_ssd/include/nvme_ssd.h new file mode 100644 index 00000000..307a1ce1 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/nvme_ssd.h @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Author: jhieb@micron.com 2025 + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _GREENSOCS_BASE_COMPONENTS_NVME_SSD_H +#define _GREENSOCS_BASE_COMPONENTS_NVME_SSD_H + + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "tlm-modules/pcie-controller.h" +#include "nvme_controller.h" + +#include + +#ifndef _WIN32 +#include +#include +#include +#endif + +#include "pci_ids.h" + +#define QDMA_TYPE qdma_cpm5 + +#define PCI_VENDOR_ID_SAMSUNG (0x144D) +#define PCI_SUBSYSTEM_VENDOR_ID_INTEL (0x3704) // 2E Intel 750 series nvme ssd +#define PCI_SUBSYSTEM_ID_INTEL (0x8086) // Intel corporation +#define PCI_DEVICE_ID_XILINX_EF100 (0xd004) +#define PCI_SUBSYSTEM_ID_XILINX_TEST (0x000A) + +#define PCI_CLASS_BASE_MASS_STORAGE_CONTROLLER (0x01) +#define PCI_CLASS_BASE_NETWORK_CONTROLLER (0x02) + +#ifndef PCI_EXP_LNKCAP_ASPM_L0S +#define PCI_EXP_LNKCAP_ASPM_L0S 0x00000400 /* ASPM L0s Support */ +#endif + +#define KiB (1024) +#define RAM_SIZE (4 * KiB) + +#define NR_MMIO_BAR 6 +#define NR_IRQ NR_QDMA_IRQ + + +PhysFuncConfig getSSDPhysFuncConfig(){ + PhysFuncConfig cfg; + PMCapability pmCap; + PCIExpressCapability pcieCap; + MSIXCapability msixCap; + + // 32 bit BARs. With BAR 0,4,5 it freezes with 64 bit BARs?? + uint32_t bar_flags = PCI_BASE_ADDRESS_MEM_TYPE_32; + uint32_t msixTableSz = 16; // 512 + // TODO(jhieb) how do i set the table offset and PBA correctly so the PBA and msix table are on different BARs? + // TODO(jhieb) the bar5 doesn't get set as a MSIX bar unless the table is on it. + uint32_t tableOffset = 0x0 | 4; // Table offset: 0, BIR: 4 + uint32_t pba = 0x2000 | 4; // Table offset: 8192, BIR: 4 + + cfg.SetPCIVendorID(PCI_VENDOR_ID_SAMSUNG); + // QDMA + cfg.SetPCIDeviceID(0x2001); + + cfg.SetPCIClassProgIF(0x02); + cfg.SetPCIClassDevice(0x08); + cfg.SetPCIClassBase(PCI_CLASS_BASE_MASS_STORAGE_CONTROLLER); + + cfg.SetPCIBAR0(8 * KiB, bar_flags); + cfg.SetPCIBAR4(16 * KiB, bar_flags); + // TODO(jhieb) the bar5 doesn't get set as a MSIX bar unless the table is on it. + //cfg.SetPCIBAR5(4 * KiB, bar_flags); + + cfg.SetPCISubsystemVendorID(PCI_SUBSYSTEM_VENDOR_ID_INTEL); + cfg.SetPCISubsystemID(PCI_SUBSYSTEM_ID_INTEL); + cfg.SetPCIExpansionROMBAR(0, 0); + + // TODO(jhieb) seeing if disabling this gets me closer to the right setup? + pmCap.SetCapabilityID(PCI_CAP_ID_PM); + pmCap.SetNextCapPtr(0xC0); + pmCap.SetPMCapabilities(0x02); + cfg.AddPCICapability(pmCap); + + // TODO(jhieb) since the capability size/start isn't DWORD aligned had to modify pf-config.h line 108 to this from 0x8. + // PCIExpressCapSize = PCI_CAP_EXP_ENDPOINT_SIZEOF_V2 + 0xA, + pcieCap.SetCapabilityID(PCI_CAP_ID_EXP); + //pcieCap.SetNextCapPtr(0xBC); + pcieCap.SetDeviceCapabilities(PCI_EXP_DEVCAP_FLR | PCI_EXP_DEVCAP_RBER); + pcieCap.SetLinkCapabilities(0x0); + pcieCap.SetLinkStatus(PCI_EXP_LNKSTA_CLS_2_5GB); + pcieCap.SetDeviceControl(0x0); + pcieCap.SetDeviceStatus(0x0); + pcieCap.SetLinkStatus(0x1); + pcieCap.SetLinkControl(0x0); + pcieCap.SetPCIECapabilities(0x0002); + cfg.AddPCICapability(pcieCap); + + //msixCap. + msixCap.SetCapabilityID(PCI_CAP_ID_MSIX); + //msixCap.SetNextCapPtr(0x00); + msixCap.SetMessageControl(msixTableSz); + msixCap.SetTableOffsetBIR(tableOffset); + msixCap.SetPendingBitArray(pba); + cfg.AddPCICapability(msixCap); + + return cfg; +} + +namespace gs { + + +template +class pcie_versal : public pci_device_base +{ +private: + QDMA_t qdma; + + // BARs towards the QDMA + tlm_utils::simple_initiator_socket user_bar_init_socket; + tlm_utils::simple_initiator_socket cfg_init_socket; + + // QDMA towards PCIe interface (host) + tlm_utils::simple_target_socket brdg_dma_tgt_socket; + + // MSI-X propagation + sc_vector > signals_irq; + + + // TODO(jhieb) this is a hack till i figure out the iconnect/memory part. + tlm_utils::simple_target_socket card_bus; + + // + // Nothing to attach to the QDMA yet, just add a dummy memory. + // With that the testcase will be able to check what has been + // written in the memory.. Add an interconnect, so we can map + // it anywhere. + // + // TODO(jhieb) evaluate how we fix this for DMA and what not. + //iconnect<1, 1> bus; + //memory sbi_dummy; + + void bar_b_transport(int bar_nr, tlm::tlm_generic_payload &trans, + sc_time &delay) + { + switch (bar_nr) { + case QDMA_USER_BAR_ID: + user_bar_init_socket->b_transport(trans, delay); + break; + case 0: + cfg_init_socket->b_transport(trans, delay); + break; + default: + SC_REPORT_ERROR("pcie_versal", + "writing to an unimplemented bar"); + trans.set_response_status( + tlm::TLM_GENERIC_ERROR_RESPONSE); + break; + } + } + + // + // Forward DMA requests received from the CPM5 QDMA + // + void fwd_dma_b_transport(tlm::tlm_generic_payload& trans, + sc_time& delay) + { + dma->b_transport(trans, delay); + } + + // + // MSI-X propagation + // + void irq_thread(unsigned int i) + { + while (true) { + wait(signals_irq[i].value_changed_event()); + irq[i].write(signals_irq[i].read()); + } + } + +public: + SC_HAS_PROCESS(pcie_versal); + + pcie_versal(sc_core::sc_module_name name) : + + pci_device_base(name, NR_MMIO_BAR, NR_IRQ), + + qdma("qdma"), + + user_bar_init_socket("user_bar_init_socket"), + cfg_init_socket("cfg_init_socket"), + brdg_dma_tgt_socket("brdg-dma-tgt-socket"), + + signals_irq("signals_irq", NR_IRQ) + + //bus("bus"), + //sbi_dummy("sbi_dummy", sc_time(0, SC_NS), RAM_SIZE) + { + // + // QDMA connections + // + user_bar_init_socket.bind(qdma.user_bar); + cfg_init_socket.bind(qdma.config_bar); + + // Setup DMA forwarding path (qdma.dma -> upstream to host) + qdma.dma.bind(brdg_dma_tgt_socket); + brdg_dma_tgt_socket.register_b_transport( + this, &pcie_versal::fwd_dma_b_transport); + + // TODO(jhieb) evaluate fixes to the memory mapping. + // Connect the SBI dummy RAM + //bus.memmap(0x102100000ULL, 0x1000 - 1, + // ADDRMODE_RELATIVE, -1, sbi_dummy.socket); + //qdma.card_bus.bind((*bus.t_sk[0])); + // TODO(jhieb) hack to bind the card_bus for now. + qdma.card_bus.bind(card_bus); + + // Setup MSI-X propagation + for (unsigned int i = 0; i < NR_IRQ; i++) { + qdma.irq[i](signals_irq[i]); + sc_spawn(sc_bind(&pcie_versal::irq_thread, this, i)); + } + } + + void card_bus_b_transport(tlm::tlm_generic_payload& trans, sc_time& delay){ + // TODO(jhieb) hack for binding card bus. + delay = sc_core::sc_time(1, sc_core::SC_NS); + } + + void rst(sc_signal& rst) + { + qdma.rst(rst); + } +}; + + +/** + * @class nvme_ssd + * + * @brief A nvme_ssd PCI based nvme storage device. + * + * @details TODO(jhieb) document + */ +template +class nvme_ssd : public sc_core::sc_module +{ + SCP_LOGGER(()); + +public: + + std::shared_ptr pcie_ctlr; + pcie_versal nvme_ctlr; + sc_signal rst; + + nvme_ssd(const sc_core::sc_module_name& name, sc_core::sc_object* gpex) + : nvme_ssd(name, *(dynamic_cast*>(gpex))) + { + } + + nvme_ssd(sc_core::sc_module_name name, gs_gpex& gpex) : + pcie_ctlr(std::make_shared("pcie_ctlr", getSSDPhysFuncConfig())), + nvme_ctlr("nvme_ctlr") + { + //pcie_ctlr = std::make_shared("pcie_ctlr", getSSDPhysFuncConfig()); + //nvme_ctlr = std::make_shared("nvme_ctlr"); + SCP_DEBUG(()) << "nvme_ssd constructor"; + + pcie_ctlr->bind(nvme_ctlr); + nvme_ctlr.rst(rst); + // TODO(jhieb) create a PCI base class that we pass to gpex? + gpex.add_device(pcie_ctlr); + + SC_THREAD(pull_reset); + } + + void pull_reset(void) { + /* Pull the reset signal. */ + rst.write(true); + wait(1, SC_US); + rst.write(false); + } + + void before_end_of_elaboration() + { + } + + nvme_ssd() = delete; + nvme_ssd(const nvme_ssd&) = delete; + + ~nvme_ssd() {} +}; +} // namespace gs + +extern "C" void module_register(); +#endif diff --git a/systemc-components/pci/nvme_ssd/include/pci-device-base.h b/systemc-components/pci/nvme_ssd/include/pci-device-base.h new file mode 100644 index 00000000..866acbee --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/pci-device-base.h @@ -0,0 +1,65 @@ +/* + * TLM-2.0 PCI device base class. + * + * Copyright (c) 2020 Xilinx Inc. + * Written by Edgar E. Iglesias. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef PCI_DEVICE_BASE_H__ +#define PCI_DEVICE_BASE_H__ + +// This is a base class representing a PCIe device. +// Actual models may extend on this. +class pci_device_base : public sc_core::sc_module +{ +public: + // Config Address-Space + tlm_utils::simple_target_socket config; + // Up to 6 BAR ports. + sc_vector > bar; + // A DMA port for end-point access into the host. + tlm_utils::simple_initiator_socket dma; + + tlm_utils::simple_initiator_socket ats_req; + tlm_utils::simple_target_socket ats_inv; + + // Interrupts. Normally this vector will be of size 1. + // Depending on where device connects there may be infrastructure + // to handle multiple interrupts (mapping these signals into MSI). + // + // If not, the DMA port can be used to send MSI/MSI-X interrupts. + sc_vector > irq; + + pci_device_base(sc_module_name name, + unsigned int nr_bars, + unsigned int nr_irqs); + +protected: + virtual void config_b_transport(tlm::tlm_generic_payload& trans, + sc_time& delay); + + // Tagged callback for BAR access. ID represents the BAR nr. + virtual void bar_b_transport(int bar_nr, tlm::tlm_generic_payload& trans, + sc_time& delay); + + virtual void b_transport_ats_inv(tlm::tlm_generic_payload& trans, + sc_time& delay); +}; +#endif diff --git a/systemc-components/pci/nvme_ssd/include/qdma-ctx.inc b/systemc-components/pci/nvme_ssd/include/qdma-ctx.inc new file mode 100644 index 00000000..944600c9 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/qdma-ctx.inc @@ -0,0 +1,164 @@ +/* + * This models the context for the QDMA. + * + * Copyright (c) 2022 Xilinx Inc. + * Written by Fred Konrad + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/* The per queue SW contexts. */ +#ifdef QDMA_CPM4 +struct __attribute__((__packed__)) sw_ctx_cpm4 { +#else /* !QDMA_CPM4 */ +struct __attribute__((__packed__)) sw_ctx_cpm5 { +#endif + /* Producer index. */ + uint16_t pidx; + /* The queue is allowed to raise an IRQ. */ + uint16_t irq_arm : 1; +#ifdef QDMA_CPM4 + uint32_t rsv : 15; +#else /* !QDMA_CPM4 */ + uint16_t fn_id : 12; + uint16_t rsv : 3; +#endif /* QDMA_CPM4 */ + /* Queue is enabled. */ + uint16_t enabled : 1; + uint16_t fcrd_en : 1; + uint16_t wbi_chk : 1; + uint16_t wbi_int_en : 1; +#ifdef QDMA_CPM4 + uint16_t fn_id : 8; +#else /* !QDMA_CPM4 */ + uint16_t at : 1; + uint16_t fetch_max : 4; + uint16_t reserved2 : 3; +#endif /* QDMA_CPM4 */ + /* Number of the descriptor for this context. */ + uint16_t ring_size : 4; + /* desc_size: 0: 8 bytes ... 3: 64 bytes. */ + uint16_t desc_size : 2; + uint16_t bypass : 1; + /* 0. */ + uint16_t mm_chan : 1; + /* Write to the status descriptor upon status update. */ + uint16_t writeback_en : 1; + /* Send an irq upon status update. */ + uint16_t irq_enabled : 1; + uint16_t port_id : 3; + uint16_t irq_no_last : 1; + uint16_t err : 2; + uint16_t err_wb_sent : 1; + uint16_t irq_req : 1; + uint16_t mrkr : 1; + uint16_t is_mm : 1; + /* Descriptor base address, descriptor for CIDX will be at + * BASE + CIDX. */ + uint32_t desc_base_low; + uint32_t desc_base_high; +#ifndef QDMA_CPM4 + /* MSI-X vector (direct irq), IRQ Ring index (indirect irq). */ + uint16_t vec : 11; + uint16_t int_aggr : 1; + uint16_t dis_intr_on_vf : 1; + uint16_t virtio_en : 1; + uint16_t pack_byp_out : 1; + uint16_t irq_byp : 1; + uint16_t host_id : 4; + uint16_t pasid_low : 12; + uint16_t pasid_high : 10; + uint16_t pasid_en : 1; + uint16_t virtio_desc_base_low : 12; + uint32_t virtio_desc_base_med; + uint16_t virtio_desc_base_high : 11; +#endif /* !QDMA_CPM4 */ +}; + +/* Interrupt Aggregation. */ +/* Interrupt contexts. */ +#ifdef QDMA_CPM4 +struct __attribute__((__packed__)) intr_ctx_cpm4 { +#else /* !QDMA_CPM4 */ +struct __attribute__((__packed__)) intr_ctx_cpm5 { +#endif + uint32_t valid : 1; +#ifdef QDMA_CPM4 + uint32_t vector : 5; +#else /* !QDMA_CPM4 */ + uint32_t vector : 11; +#endif /* QDMA_CPM4 */ + uint32_t rsvd : 1; + uint32_t status : 1; + uint32_t color : 1; + uint64_t baddr : 52; + uint32_t page_size : 3; + uint32_t pidx : 12; +#ifndef QDMA_CPM4 + uint32_t at : 1; + uint32_t host_id : 4; + uint32_t pasid : 22; + uint32_t pasid_en : 1; + uint32_t rsvd2 : 4; + uint32_t func : 12; +#endif /* !QDMA_CPM4 */ +}; + +/* Interrupt entry as written in the ring. */ +#ifdef QDMA_CPM4 +struct __attribute__((__packed__)) intr_ring_entry_cpm4 { +#else /* !QDMA_CPM4 */ +struct __attribute__((__packed__)) intr_ring_entry_cpm5 { +#endif /* QDMA_CPM4 */ + uint16_t pidx : 16; + uint16_t cidx : 16; + uint16_t color : 1; + uint16_t interrupt_state : 2; +#ifdef QDMA_CPM4 + uint16_t error : 4; + uint32_t rsvd : 11; + uint16_t err_int : 1; +#else /* !QDMA_CPM4 */ + uint16_t error : 2; + uint32_t rsvd : 1; +#endif /* QDMA_CPM4 */ + uint32_t interrupt_type : 1; +#ifdef QDMA_CPM4 + uint32_t qid : 11; +#else /* !QDMA_CPM4 */ + uint32_t qid : 24; +#endif /* QDMA_CPM4 */ + uint32_t coal_color : 1; +}; + +#ifdef QDMA_CPM4 +struct __attribute__((__packed__)) qid2vec_ctx_cpm4 { + /* For direct IRQ: + * MSI-X vector index. + * For indirect IRQ: + * Ring Index. */ + uint16_t c2h_vector : 8; + /* 0: direct irq, 1: indirect irq. */ + uint16_t c2h_en_coal : 1; + /* Likewise for h2c. */ + uint16_t h2c_vector : 8; + uint16_t h2c_en_coal : 1; + uint16_t rsvd : 14; +}; +#endif /* QDMA_CPM4 */ diff --git a/systemc-components/pci/nvme_ssd/include/queue.h b/systemc-components/pci/nvme_ssd/include/queue.h new file mode 100644 index 00000000..7d4638c6 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/include/queue.h @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#pragma once + +#ifndef __HIL_NVME_QUEUE__ +#define __HIL_NVME_QUEUE__ + +#include "def.h" +#include "dma.h" +//#include "util/simplessd.hh" + +typedef union _SQEntry { + uint8_t data[64]; + struct { + struct { + uint8_t opcode; + uint8_t fuse; + uint16_t commandID; + } dword0; + uint32_t namespaceID; + uint32_t reserved1; + uint32_t reserved2; + uint64_t metadata; + uint64_t data1; + uint64_t data2; + uint32_t dword10; + uint32_t dword11; + uint32_t dword12; + uint32_t dword13; + uint32_t dword14; + uint32_t dword15; + }; + + _SQEntry(); +} SQEntry; + +typedef union _CQEntry { + uint8_t data[16]; + struct { + uint32_t dword0; + uint32_t reserved; + struct { + uint16_t sqHead; + uint16_t sqID; + } dword2; + struct { + uint16_t commandID; + uint16_t status; + } dword3; + }; + + _CQEntry(); +} CQEntry; + +typedef struct _SQEntryWrapper { + SQEntry entry; + + uint16_t sqID; + uint16_t cqID; + uint16_t sqHead; + uint16_t sqUID; //!< HEAD - 1 + + bool useSGL; + + _SQEntryWrapper(SQEntry &, uint16_t, uint16_t, uint16_t, uint16_t); +} SQEntryWrapper; + +typedef struct _CQEntryWrapper { + CQEntry entry; + + uint64_t submitAt; + + uint16_t cqID; //!< sqID at entry.dword2 + uint16_t sqUID; //!< Head - 1 + + _CQEntryWrapper(SQEntryWrapper &); + void makeStatus(bool, bool, STATUS_CODE_TYPE, int); + uint16_t getStatus(); // Used for copying status between responses. + void setStatus(uint16_t status); // Used for copying status between responses. +} CQEntryWrapper; + +class Queue { + protected: + uint16_t id; + + uint16_t head; + uint16_t tail; + + uint16_t size; + uint64_t stride; + uint64_t base_addr; + + DMAInterface *base; + + public: + Queue(uint16_t, uint16_t); + ~Queue(); + + uint16_t getID(); + uint16_t getItemCount(); + uint16_t getHead(); + uint16_t getTail(); + uint16_t getSize(); + uint64_t getBaseAddr(); + void setBase(DMAInterface *, uint64_t, uint64_t); +}; + +class CQueue : public Queue { + protected: + bool ien; + bool phase; + uint16_t interruptVector; + + public: + CQueue(uint16_t, bool, uint16_t, uint16_t); + + void setData(CQEntry *); + uint16_t incHead(); + void setHead(uint16_t); + bool interruptEnabled(); + uint16_t getInterruptVector(); +}; + +class SQueue : public Queue { + protected: + uint16_t cqID; + uint8_t priority; + + public: + SQueue(uint16_t, uint8_t, uint16_t, uint16_t); + + uint16_t getCQID(); + void setTail(uint16_t); + void getData(SQEntry *);// , DMAFunction, void *); + uint8_t getPriority(); +}; + +#endif diff --git a/systemc-components/pci/nvme_ssd/src/bitset.cc b/systemc-components/pci/nvme_ssd/src/bitset.cc new file mode 100644 index 00000000..98df1767 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/src/bitset.cc @@ -0,0 +1,222 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#include "bitset.h" + +#include +#include + +#include "algorithm.h" + +Bitset::Bitset() : data(nullptr), dataSize(0), allocSize(0) {} + +Bitset::Bitset(uint32_t size) : Bitset() { + if (size > 0) { + dataSize = size; + allocSize = DIVCEIL(dataSize, 8); + data = (uint8_t *)calloc(allocSize, 1); + } +} + +Bitset::Bitset(const Bitset &rhs) : Bitset(rhs.dataSize) { + if (rhs.data) { + memcpy(data, rhs.data, allocSize); + } +} + +Bitset::Bitset(Bitset &&rhs) noexcept { + dataSize = std::move(rhs.dataSize); + allocSize = std::move(rhs.allocSize); + data = std::move(rhs.data); + + rhs.dataSize = 0; + rhs.allocSize = 0; + rhs.data = nullptr; +} + +Bitset::~Bitset() { + free(data); + + data = nullptr; + dataSize = 0; + allocSize = 0; +} + +bool Bitset::test(uint32_t idx) noexcept { + return data[idx / 8] & (0x01 << (idx % 8)); +} + +bool Bitset::all() noexcept { + uint8_t ret = 0xFF; + uint8_t mask = 0xFF << (dataSize + 8 - allocSize * 8); + + for (uint32_t i = 0; i < allocSize - 1; i++) { + ret &= data[i]; + } + + ret &= data[allocSize - 1] | mask; + + return ret == 0xFF; +} + +bool Bitset::any() noexcept { + return !none(); +} + +bool Bitset::none() noexcept { + uint8_t ret = 0x00; + + for (uint32_t i = 0; i < allocSize; i++) { + ret |= data[i]; + } + + return ret == 0x00; +} + +uint32_t Bitset::count() noexcept { + uint32_t count = 0; + + for (uint32_t i = 0; i < allocSize; i++) { + count += popcount(data[i]); + } + + return count; +} + +uint32_t Bitset::size() noexcept { + return dataSize; +} + +void Bitset::set() noexcept { + uint8_t mask = 0xFF >> (allocSize * 8 - dataSize); + + for (uint32_t i = 0; i < allocSize - 1; i++) { + data[i] = 0xFF; + } + + data[allocSize - 1] = mask; +} + +void Bitset::set(uint32_t idx, bool value) noexcept { + data[idx / 8] &= ~(0x01 << (idx % 8)); + + if (value) { + data[idx / 8] = data[idx / 8] | (0x01 << (idx % 8)); + } +} + +void Bitset::reset() noexcept { + for (uint32_t i = 0; i < allocSize; i++) { + data[i] = 0x00; + } +} + +void Bitset::reset(uint32_t idx) noexcept { + data[idx / 8] &= ~(0x01 << (idx % 8)); +} + +void Bitset::flip() noexcept { + uint8_t mask = 0xFF >> (allocSize * 8 - dataSize); + + for (uint32_t i = 0; i < allocSize; i++) { + data[i] = ~data[i]; + } + + data[allocSize - 1] &= mask; +} + +void Bitset::flip(uint32_t idx) noexcept { + data[idx / 8] = (~data[idx / 8] & (0x01 << (idx % 8))) | + (data[idx / 8] & ~(0x01 << (idx % 8))); +} + +bool Bitset::operator[](uint32_t idx) noexcept { + return test(idx); +} + +Bitset &Bitset::operator&=(const Bitset &rhs) { + if (dataSize != rhs.dataSize) { + SC_REPORT_WARNING("bitset", "Size does not match"); + } + + for (uint32_t i = 0; i < allocSize; i++) { + data[i] &= rhs.data[i]; + } + + return *this; +} + +Bitset &Bitset::operator|=(const Bitset &rhs) { + if (dataSize != rhs.dataSize) { + SC_REPORT_WARNING("bitset", "Size does not match"); + } + + for (uint32_t i = 0; i < allocSize; i++) { + data[i] |= rhs.data[i]; + } + + return *this; +} + +Bitset &Bitset::operator^=(const Bitset &rhs) { + if (dataSize != rhs.dataSize) { + SC_REPORT_WARNING("bitset", "Size does not match"); + } + + for (uint32_t i = 0; i < allocSize; i++) { + data[i] ^= rhs.data[i]; + } + + return *this; +} + +Bitset &Bitset::operator=(const Bitset &rhs) { + if (this != &rhs) { + free(data); + data = nullptr; + + *this = Bitset(rhs); + } + + return *this; +} + +Bitset &Bitset::operator=(Bitset &&rhs) noexcept { + if (this != &rhs) { + free(data); + + dataSize = std::move(rhs.dataSize); + allocSize = std::move(rhs.allocSize); + data = std::move(rhs.data); + + rhs.dataSize = 0; + rhs.allocSize = 0; + rhs.data = nullptr; + } + + return *this; +} + +Bitset Bitset::operator~() const { + Bitset ret(*this); + + ret.flip(); + + return ret; +} diff --git a/systemc-components/pci/nvme_ssd/src/config.cc b/systemc-components/pci/nvme_ssd/src/config.cc new file mode 100644 index 00000000..0df0bc7f --- /dev/null +++ b/systemc-components/pci/nvme_ssd/src/config.cc @@ -0,0 +1,273 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#include "config.h" +#include "systemc.h" +//#include "util/algorithm.hh" +//#include "util/simplessd.hh" + +const char NAME_PCIE_GEN[] = "PCIEGeneration"; +const char NAME_PCIE_LANE[] = "PCIELane"; +const char NAME_AXI_BUS_WIDTH[] = "AXIBusWidth"; +const char NAME_AXI_CLOCK[] = "AXIClock"; +const char NAME_FIFO_UNIT[] = "FIFOTransferUnit"; +const char NAME_WORK_INTERVAL[] = "WorkInterval"; +const char NAME_MAX_REQUEST_COUNT[] = "MaxRequestCount"; +const char NAME_MAX_IO_CQUEUE[] = "MaxIOCQueue"; +const char NAME_MAX_IO_SQUEUE[] = "MaxIOSQueue"; +const char NAME_WRR_HIGH[] = "WRRHigh"; +const char NAME_WRR_MEDIUM[] = "WRRMedium"; +const char NAME_ENABLE_DEFAULT_NAMESPACE[] = "DefaultNamespace"; +const char NAME_LBA_SIZE[] = "LBASize"; +const char NAME_ENABLE_DISK_IMAGE[] = "EnableDiskImage"; +const char NAME_STRICT_DISK_SIZE[] = "StrictSizeCheck"; +const char NAME_DISK_IMAGE_PATH[] = "DiskImageFile"; +const char NAME_USE_COW_DISK[] = "UseCopyOnWriteDisk"; + +/* +Config::Config() { + pcieGen = PCIExpress::PCIE_3_X; + pcieLane = 4; + axiWidth = ARM::AXI::BUS_128BIT; + axiClock = 250000000; + fifoUnit = 4096; + workInterval = 50000; + maxRequestCount = 4; + maxIOCQueue = 16; + maxIOSQueue = 16; + wrrHigh = 2; + wrrMedium = 2; + lbaSize = 512; + defaultNamespace = 1; + enableDiskImage = false; + strictDiskSize = false; + useCopyOnWriteDisk = false; +} + +bool Config::setConfig(const char *name, const char *value) { + bool ret = true; + + if (MATCH_NAME(NAME_PCIE_GEN)) { + switch (strtoul(value, nullptr, 10)) { + case 0: + pcieGen = PCIExpress::PCIE_1_X; + break; + case 1: + pcieGen = PCIExpress::PCIE_2_X; + break; + case 2: + pcieGen = PCIExpress::PCIE_3_X; + break; + default: + SC_REPORT_WARNING("config", "Invalid PCI Express Generation"); + break; + } + } + else if (MATCH_NAME(NAME_PCIE_LANE)) { + pcieLane = (uint8_t)strtoul(value, nullptr, 10); + } + else if (MATCH_NAME(NAME_AXI_BUS_WIDTH)) { + switch (strtoul(value, nullptr, 10)) { + case 0: + axiWidth = ARM::AXI::BUS_32BIT; + break; + case 1: + axiWidth = ARM::AXI::BUS_64BIT; + break; + case 2: + axiWidth = ARM::AXI::BUS_128BIT; + break; + case 3: + axiWidth = ARM::AXI::BUS_256BIT; + break; + case 4: + axiWidth = ARM::AXI::BUS_512BIT; + break; + case 5: + axiWidth = ARM::AXI::BUS_1024BIT; + break; + default: + SC_REPORT_WARNING("config", "Invalid AXI Stream Bus Width"); + break; + } + } + else if (MATCH_NAME(NAME_AXI_CLOCK)) { + axiClock = strtoul(value, nullptr, 10); + } + else if (MATCH_NAME(NAME_FIFO_UNIT)) { + fifoUnit = strtoul(value, nullptr, 10); + } + else if (MATCH_NAME(NAME_WORK_INTERVAL)) { + workInterval = strtoul(value, nullptr, 10); + } + else if (MATCH_NAME(NAME_MAX_REQUEST_COUNT)) { + maxRequestCount = strtoul(value, nullptr, 10); + } + else if (MATCH_NAME(NAME_MAX_IO_CQUEUE)) { + maxIOCQueue = (uint16_t)strtoul(value, nullptr, 10); + } + else if (MATCH_NAME(NAME_MAX_IO_SQUEUE)) { + maxIOSQueue = (uint16_t)strtoul(value, nullptr, 10); + } + else if (MATCH_NAME(NAME_WRR_HIGH)) { + wrrHigh = (uint16_t)strtoul(value, nullptr, 10); + } + else if (MATCH_NAME(NAME_WRR_MEDIUM)) { + wrrMedium = (uint16_t)strtoul(value, nullptr, 10); + } + else if (MATCH_NAME(NAME_ENABLE_DEFAULT_NAMESPACE)) { + defaultNamespace = (uint16_t)strtoul(value, nullptr, 10); + } + else if (MATCH_NAME(NAME_LBA_SIZE)) { + lbaSize = strtoul(value, nullptr, 10); + } + else if (MATCH_NAME(NAME_ENABLE_DISK_IMAGE)) { + enableDiskImage = convertBool(value); + } + else if (MATCH_NAME(NAME_STRICT_DISK_SIZE)) { + strictDiskSize = convertBool(value); + } + else if (strncmp(name, NAME_DISK_IMAGE_PATH, strlen(NAME_DISK_IMAGE_PATH)) == + 0) { + uint16_t index = + (uint16_t)strtoul(name + strlen(NAME_DISK_IMAGE_PATH), nullptr, 10); + + if (index > 0) { + auto find = diskImagePaths.find(index); + + if (find == diskImagePaths.end()) { + diskImagePaths.insert({index, value}); + } + else { + find->second = value; + } + } + } + else if (MATCH_NAME(NAME_USE_COW_DISK)) { + useCopyOnWriteDisk = convertBool(value); + } + else { + ret = false; + } + + return ret; +} + +void Config::update() { + if (popcount(lbaSize) != 1 || lbaSize < 512) { + SC_REPORT_WARNING("config", "Invalid LBA size"); + } + if (maxRequestCount == 0) { + SC_REPORT_WARNING("config", "MaxRequestCount should be larger then 0"); + } + if (fifoUnit > 4096) { + SC_REPORT_WARNING("config", "FIFOTransferUnit should be less than or equal to 4096"); + } +} + +int64_t Config::readInt(uint32_t idx) { + int64_t ret = 0; + + switch (idx) { + case NVME_PCIE_GEN: + ret = pcieGen; + break; + case NVME_AXI_BUS_WIDTH: + ret = axiWidth; + break; + } + + return ret; +} + +uint64_t Config::readUint(uint32_t idx) { + uint64_t ret = 0; + + switch (idx) { + case NVME_PCIE_LANE: + ret = pcieLane; + break; + case NVME_AXI_CLOCK: + ret = axiClock; + break; + case NVME_FIFO_UNIT: + ret = fifoUnit; + break; + case NVME_WORK_INTERVAL: + ret = workInterval; + break; + case NVME_MAX_REQUEST_COUNT: + ret = maxRequestCount; + break; + case NVME_MAX_IO_CQUEUE: + ret = maxIOCQueue; + break; + case NVME_MAX_IO_SQUEUE: + ret = maxIOSQueue; + break; + case NVME_WRR_HIGH: + ret = wrrHigh; + break; + case NVME_WRR_MEDIUM: + ret = wrrMedium; + break; + case NVME_ENABLE_DEFAULT_NAMESPACE: + ret = defaultNamespace; + break; + case NVME_LBA_SIZE: + ret = lbaSize; + break; + } + + return ret; +} + +std::string Config::readString(uint32_t idx) { + std::string ret(""); + + if (idx >= NVME_DISK_IMAGE_PATH) { + idx -= NVME_DISK_IMAGE_PATH; + + auto find = diskImagePaths.find((uint16_t)idx); + + if (find != diskImagePaths.end()) { + ret = find->second; + } + } + + return ret; +} + +bool Config::readBoolean(uint32_t idx) { + bool ret = false; + + switch (idx) { + case NVME_ENABLE_DISK_IMAGE: + ret = enableDiskImage; + break; + case NVME_STRICT_DISK_SIZE: + ret = strictDiskSize; + break; + case NVME_USE_COW_DISK: + ret = useCopyOnWriteDisk; + break; + } + + return ret; +}*/ diff --git a/systemc-components/pci/nvme_ssd/src/def.cc b/systemc-components/pci/nvme_ssd/src/def.cc new file mode 100644 index 00000000..77a0d3ca --- /dev/null +++ b/systemc-components/pci/nvme_ssd/src/def.cc @@ -0,0 +1,75 @@ + +#include "def.h" + + +LPNRange::_LPNRange() : slpn(0), nlp(0) {} + +LPNRange::_LPNRange(uint64_t s, uint64_t n) : slpn(s), nlp(n) {} + +namespace HIL { + +Request::_Request() + : reqID(0), + reqSubID(0), + offset(0), + length(0), + finishedAt(0), + context(nullptr) {} + +Request::_Request(DMAFunction &f, void *c) + : reqID(0), + reqSubID(0), + offset(0), + length(0), + finishedAt(0), + function(f), + context(c) {} + +bool Request::operator()(const Request &a, const Request &b) { + return a.finishedAt > b.finishedAt; +} + +} // namespace HIL + +namespace ICL { + +Request::_Request() : reqID(0), reqSubID(0), offset(0), length(0) {} + +Request::_Request(HIL::Request &r) + : reqID(r.reqID), + reqSubID(r.reqSubID), + offset(r.offset), + length(r.length), + range(r.range) {} + +} // namespace ICL + +namespace FTL { + +Request::_Request(uint32_t iocount) + : reqID(0), reqSubID(0), lpn(0), ioFlag(iocount) {} + +Request::_Request(uint32_t iocount, ICL::Request &r) + : reqID(r.reqID), + reqSubID(r.reqSubID), + lpn(r.range.slpn / iocount), + ioFlag(iocount) { + ioFlag.set(r.range.slpn % iocount); +} + +} // namespace FTL + +namespace PAL { + +Request::_Request(uint32_t iocount) + : reqID(0), reqSubID(0), blockIndex(0), pageIndex(0), ioFlag(iocount) {} + +Request::_Request(FTL::Request &r) + : reqID(r.reqID), + reqSubID(r.reqSubID), + blockIndex(0), + pageIndex(0), + ioFlag(r.ioFlag) {} + +} // namespace PAL + diff --git a/systemc-components/pci/nvme_ssd/src/disk.cc b/systemc-components/pci/nvme_ssd/src/disk.cc new file mode 100644 index 00000000..55239e63 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/src/disk.cc @@ -0,0 +1,312 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#include "disk.h" + +#include + +#ifdef _MSC_VER +#include +#else +#include +#include +#endif + +Disk::Disk() : diskSize(0), sectorSize(0) {} + +Disk::~Disk() { + close(); +} + +uint64_t Disk::open(std::string path, uint64_t desiredSize, uint32_t lbaSize) { + filename = path; + sectorSize = lbaSize; + + // Validate size +#ifdef _MSC_VER + LARGE_INTEGER size; + HANDLE hFile = CreateFileA(path.c_str(), GENERIC_READ | GENERIC_WRITE, NULL, + NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + + if (GetLastError() == ERROR_ALREADY_EXISTS) { + if (GetFileSizeEx(hFile, &size)) { + diskSize = size.QuadPart; + } + else { + // panic("Get file size failed!"); + } + } + else { + size.QuadPart = desiredSize; + + if (SetFilePointerEx(hFile, size, nullptr, FILE_BEGIN)) { + if (SetEndOfFile(hFile)) { + diskSize = desiredSize; + } + else { + // panic("SetEndOfFile failed"); + } + } + else { + // panic("SetFilePointerEx failed"); + } + } + + CloseHandle(hFile); +#else + struct stat s; + + if (stat(filename.c_str(), &s) == 0) { + // File exists + if (S_ISREG(s.st_mode)) { + diskSize = s.st_size; + } + else { + // panic("nvme_disk: Specified file %s is not regular file.\n", + // filename.c_str()); + } + } + else { + // Create zero-sized file + disk.open(filename, std::ios::out | std::ios::binary); + disk.close(); + + // Set file size + if (truncate(filename.c_str(), desiredSize) == 0) { + diskSize = desiredSize; + } + else { + // panic("nvme_disk: Failed to set disk size %" PRIu64 " errno=%d\n", + // diskSize, errno); + } + } +#endif + + // Open file + disk.open(filename, std::ios::in | std::ios::out | std::ios::binary); + + if (!disk.is_open()) { + // panic("failed to open file"); + } + + return diskSize; +} + +void Disk::close() { + if (disk.is_open()) { + disk.close(); + } +} + +uint16_t Disk::read(uint64_t slba, uint16_t nlblk, uint8_t *&buffer) { + uint16_t ret = 0; + + if (disk.is_open()) { + uint64_t avail; + + slba *= sectorSize; + avail = nlblk * sectorSize; + + if (slba + avail > diskSize) { + if (slba >= diskSize) { + avail = 0; + } + else { + avail = diskSize - slba; + } + } + + if (avail > 0) { + disk.seekg(slba, std::ios::beg); + if (!disk.good()) { + // panic("nvme_disk: Fail to seek to %" PRIu64 "\n", slba); + } + + disk.read((char *)buffer, avail); + } + + memset(buffer + avail, 0, nlblk * sectorSize - avail); + + // DPRINTF(NVMeDisk, "DISK | READ | BYTE %016" PRIX64 " + %X\n", + // slba, nlblk * sectorSize); + + ret = nlblk; + } + + return ret; +} + +uint16_t Disk::write(uint64_t slba, uint16_t nlblk, uint8_t *buffer) { + uint16_t ret = 0; + + if (disk.is_open()) { + slba *= sectorSize; + + disk.seekp(slba, std::ios::beg); + if (!disk.good()) { + // panic("nvme_disk: Fail to seek to %" PRIu64 "\n", slba); + } + + uint64_t offset = disk.tellp(); + disk.write((char *)buffer, sectorSize * nlblk); + offset = (uint64_t)disk.tellp() - offset; + + // DPRINTF(NVMeDisk, "DISK | WRITE | BYTE %016" PRIX64 " + %X\n", slba, + // offset); + + ret = offset / sectorSize; + } + + return ret; +} + +uint16_t Disk::erase(uint64_t, uint16_t nlblk) { + return nlblk; +} + +CoWDisk::CoWDisk() {} + +CoWDisk::~CoWDisk() { + close(); +} + +void CoWDisk::close() { + table.clear(); + + Disk::close(); +} + +uint16_t CoWDisk::read(uint64_t slba, uint16_t nlblk, uint8_t *&buffer) { + uint16_t read = 0; + + for (uint64_t i = 0; i < nlblk; i++) { + auto block = table.find(slba + i); + + if (block != table.end()) { + memcpy(buffer + i * sectorSize, block->second.data(), sectorSize); + read++; + } + else { + uint8_t* sectorPtr = buffer + i * sectorSize; + read += Disk::read(slba + i, 1, sectorPtr); + } + } + + return read; +} + +uint16_t CoWDisk::write(uint64_t slba, uint16_t nlblk, uint8_t *buffer) { + uint16_t write = 0; + + for (uint64_t i = 0; i < nlblk; i++) { + auto block = table.find(slba + i); + + if (block != table.end()) { + memcpy(block->second.data(), buffer + i * sectorSize, sectorSize); + } + else { + std::vector data; + + data.resize(sectorSize); + memcpy(data.data(), buffer + i * sectorSize, sectorSize); + + table.insert({slba + i, data}); + } + + write++; + } + + return write; +} + +uint64_t MemDisk::open(std::string, uint64_t size, uint32_t lbaSize) { + diskSize = size; + sectorSize = lbaSize; + + return size; +} + +void MemDisk::close() { + table.clear(); +} + +MemDisk::MemDisk() {} + +MemDisk::~MemDisk() { + close(); +} + +uint16_t MemDisk::read(uint64_t slba, uint16_t nlblk, uint8_t *&buffer) { + uint16_t read = 0; + + for (uint64_t i = 0; i < nlblk; i++) { + auto block = table.find(slba + i); + + // TODO(Jhieb) fix the double pointer stuff. + if (block != table.end()) { + memcpy(buffer + i * sectorSize, block->second.data(), sectorSize); + } + else { + memset(buffer + i * sectorSize, 0, sectorSize); + } + + read++; + } + + return read; +} + +uint16_t MemDisk::write(uint64_t slba, uint16_t nlblk, uint8_t *buffer) { + uint16_t write = 0; + + for (uint64_t i = 0; i < nlblk; i++) { + auto block = table.find(slba + i); + + if (block != table.end()) { + memcpy(block->second.data(), buffer + i * sectorSize, sectorSize); + } + else { + std::vector data; + + data.resize(sectorSize); + memcpy(data.data(), buffer + i * sectorSize, sectorSize); + + table.insert({slba + i, data}); + } + + write++; + } + + return write; +} +uint16_t MemDisk::erase(uint64_t slba, uint16_t nlblk) { + uint16_t erase = 0; + + for (uint64_t i = 0; i < nlblk; i++) { + auto block = table.find(slba + i); + + if (block != table.end()) { + table.erase(block); + } + + erase++; + } + + return erase; +} diff --git a/systemc-components/pci/nvme_ssd/src/dma.cc b/systemc-components/pci/nvme_ssd/src/dma.cc new file mode 100644 index 00000000..57de090b --- /dev/null +++ b/systemc-components/pci/nvme_ssd/src/dma.cc @@ -0,0 +1,490 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#include "dma.h" +#include "systemc.h" + +//#include "hil/nvme/controller.hh" +//#include "util/algorithm.hh" + +DMAInterface::DMAInterface(ConfigData &cfg, DMAFunction &f, void *c) + : pInterface(cfg.pInterface), + initFunction(f), + callCounter(0), + context(c), + dmaHandler(commonDMAHandler) { + + // TODO(jhieb) does this matter? + //immediateEvent = + // allocate([this](uint64_t now) { initFunction(now, context); }); +} + +DMAInterface::~DMAInterface() {} + +void DMAInterface::commonDMAHandler(uint64_t now, void *context) { + DMAContext *pContext = (DMAContext *)context; + + pContext->counter--; + + if (pContext->counter == 0) { + pContext->function(now, pContext->context); + delete pContext; + } +} + +PRP::PRP() : addr(0), size(0) {} + +PRP::PRP(uint64_t address, uint64_t s) : addr(address), size(s) {} + +PRPList::PRPList(ConfigData &cfg, DMAFunction &f, void *c, uint64_t prp1, + uint64_t prp2, uint64_t size) + : DMAInterface(cfg, f, c), totalSize(size), pagesize(cfg.memoryPageSize) { + bool immediate = true; + uint64_t prp1Size = getPRPSize(prp1); + uint64_t prp2Size = getPRPSize(prp2); + + // Determine PRP1 and PRP2 + if (totalSize <= pagesize) { + if (totalSize <= prp1Size) { + // PRP1 is PRP pointer, PRP2 is not used + prpList.push_back(PRP(prp1, totalSize)); + } + else { + // PRP1 is PRP pointer, PRP2 is PRP pointer + prpList.push_back(PRP(prp1, prp1Size)); + prpList.push_back(PRP(prp2, prp2Size)); + + if (prp1Size + prp2Size < totalSize) { + SC_REPORT_WARNING("prplist", "Invalid DPTR size"); + } + } + } + else if (totalSize <= pagesize * 2) { + if (prp1Size == pagesize) { + // PRP1 is PRP pointer, PRP2 is PRP pointer + prpList.push_back(PRP(prp1, prp1Size)); + prpList.push_back(PRP(prp2, prp2Size)); + + if (prp1Size + prp2Size < totalSize) { + SC_REPORT_WARNING("prplist", "Invalid DPTR size"); + } + } + else { + immediate = false; + requiresPRP2DMA = true; + + // PRP1 is PRP pointer, PRP2 is PRP list + prpList.push_back(PRP(prp1, prp1Size)); + getPRPListFromPRP(prp2, totalSize - prp1Size); + } + } + else { + immediate = false; + requiresPRP2DMA = true; + + // PRP1 is PRP pointer, PRP2 is PRP list + prpList.push_back(PRP(prp1, prp1Size)); + getPRPListFromPRP(prp2, totalSize - prp1Size); + } + + // TODO(jhieb) does it matter? + /* + if (immediate) { + schedule(immediateEvent, getTick()); + }*/ +} + +PRPList::PRPList(ConfigData &cfg, DMAFunction &f, void *c, uint64_t base, + uint64_t size, bool cont) + : DMAInterface(cfg, f, c), totalSize(size), pagesize(cfg.memoryPageSize) { + if (cont) { + prpList.push_back(PRP(base, size)); + + // TODO(jhieb) does it matter? + // schedule(immediateEvent, getTick()); + } + else { + getPRPListFromPRP(base, size); + } +} + +PRPList::~PRPList() {} + +void PRPList::processPRPListBuffer(){ + + uint64_t listPRP; + uint64_t listPRPSize; + uint64_t currentSize = prpList.size() * pagesize; + + for (size_t i = 0; i < prpListSize; i += 8) { + listPRP = *((uint64_t *)(buffer + i)); + listPRPSize = getPRPSize(listPRP); + currentSize += listPRPSize; + + if (listPRP == 0) { + SC_REPORT_WARNING("prplist", "Invalid PRP in PRP List"); + } + + prpList.push_back(PRP(listPRP, listPRPSize)); + + if (currentSize >= totalSize) { + break; + } + // TODO(jhieb) need to test if we need a chained PRP here. + } +} + +void PRPList::getPRPListFromPRP(uint64_t base, uint64_t size) { + // Get PRP size + prpListSize = getPRPSize(base); + prpListAddr = base; + // Allocate buffer + buffer = (uint8_t *)malloc(prpListSize); +} + +uint64_t PRPList::getPRPSize(uint64_t addr) { + return pagesize - (addr & (pagesize - 1)); +} + +void PRPList::read(uint64_t offset, uint64_t length, uint8_t *buffer, + DMAFunction &func, void *context) { + DMAFunction doRead = [this, offset, length, buffer](uint64_t, void *context) { + uint64_t totalRead = 0; + uint64_t currentOffset = 0; + uint64_t read; + bool begin = false; + + DMAContext *readContext = (DMAContext *)context; + + for (auto &iter : prpList) { + if (begin) { + read = MIN(iter.size, length - totalRead); + readContext->counter++; + // TODO(jhieb) how to do the DMA read in qemu?? + /* pInterface->dmaRead(iter.addr, read, buffer ? buffer + totalRead : NULL, + dmaHandler, readContext);*/ + totalRead += read; + + if (totalRead == length) { + break; + } + } + + if (!begin && currentOffset + iter.size > offset) { + begin = true; + totalRead = offset - currentOffset; + read = MIN(iter.size - totalRead, length); + readContext->counter++; + // TODO(jhieb) how to do the DMA read in qemu?? + /*pInterface->dmaRead(iter.addr + totalRead, read, buffer, dmaHandler, + readContext);*/ + totalRead = read; + } + + currentOffset += iter.size; + } + + if (readContext->counter == 0) { + readContext->counter = 1; + + // TODO(jhieb) does this matter? + // dmaHandler(getTick(), readContext); + } + }; + + DMAContext *readContext = new DMAContext(func, context); + + // TODO(jhieb) how to do DMA reads in QEMU? + //execute(CPU::NVME__PRPLIST, CPU::READ, doRead, readContext); +} + +void PRPList::write(uint64_t offset, uint64_t length, uint8_t *buffer, + DMAFunction &func, void *context) { + DMAFunction doWrite = [this, offset, length, buffer](uint64_t, + void *context) { + uint64_t totalWritten = 0; + uint64_t currentOffset = 0; + uint64_t written; + bool begin = false; + + DMAContext *writeContext = (DMAContext *)context; + + for (auto &iter : prpList) { + if (begin) { + written = MIN(iter.size, length - totalWritten); + writeContext->counter++; + // TODO(jhieb) how to do the DMA write in QEMU?? + /* + pInterface->dmaWrite(iter.addr, written, + buffer ? buffer + totalWritten : NULL, dmaHandler, + writeContext);*/ + totalWritten += written; + + if (totalWritten == length) { + break; + } + } + + if (!begin && currentOffset + iter.size > offset) { + begin = true; + totalWritten = offset - currentOffset; + written = MIN(iter.size - totalWritten, length); + writeContext->counter++; + // TODO(jhieb) how to do the DMA write in QEMU?? + /*pInterface->dmaWrite(iter.addr + totalWritten, written, buffer, + dmaHandler, writeContext);*/ + totalWritten = written; + } + + currentOffset += iter.size; + } + + if (writeContext->counter == 0) { + writeContext->counter = 1; + // TODO(jhieb) does this matter? + //dmaHandler(getTick(), writeContext); + } + }; + + DMAContext *writeContext = new DMAContext(func, context); + // TODO(jhieb) how to do the DMA write in QEMU?? + //execute(CPU::NVME__PRPLIST, CPU::WRITE, doWrite, writeContext); +} + +SGLDescriptor::SGLDescriptor() { + memset(data, 0, 16); +} + +Chunk::Chunk() : addr(0), length(0), ignore(true) {} + +Chunk::Chunk(uint64_t a, uint32_t l, bool i) : addr(a), length(l), ignore(i) {} + +SGL::SGL(ConfigData &cfg, DMAFunction &f, void *c, uint64_t prp1, uint64_t prp2) + : DMAInterface(cfg, f, c), totalSize(0) { + SGLDescriptor desc; + + // Create first SGL descriptor from PRP pointers + memcpy(desc.data, &prp1, 8); + memcpy(desc.data + 8, &prp2, 8); + + // Check type + if (SGL_TYPE(desc.id) == TYPE_DATA_BLOCK_DESCRIPTOR || + SGL_TYPE(desc.id) == TYPE_KEYED_DATA_BLOCK_DESCRIPTOR) { + // This is entire buffer + parseSGLDescriptor(desc); + + // TODO(jhieb) does this matter? + //schedule(immediateEvent, getTick()); + } + else if (SGL_TYPE(desc.id) == TYPE_SEGMENT_DESCRIPTOR || + SGL_TYPE(desc.id) == TYPE_LAST_SEGMENT_DESCRIPTOR) { + // This points segment + requiresSGLDMA = true; + address = desc.address; + length = desc.length; + buffer = (uint8_t *)malloc(length); + //parseSGLSegment(desc.address, desc.length); + } +} + +SGL::~SGL() {} + +void SGL::parseSGLDescriptor(SGLDescriptor &desc) { + switch (SGL_TYPE(desc.id)) { + case TYPE_DATA_BLOCK_DESCRIPTOR: + case TYPE_KEYED_DATA_BLOCK_DESCRIPTOR: + chunkList.push_back(Chunk(desc.address, desc.length, false)); + totalSize += desc.length; + + break; + case TYPE_BIT_BUCKET_DESCRIPTOR: + chunkList.push_back(Chunk(desc.address, desc.length, true)); + totalSize += desc.length; + + break; + default: + SC_REPORT_WARNING("sgl", "Invalid SGL descriptor"); + break; + } + + if (SGL_SUBTYPE(desc.id) != SUBTYPE_ADDRESS) { + SC_REPORT_WARNING("sgl", "Unexpected SGL subtype"); + } +} + +void SGL::parseSGLSegment() { + SGLDescriptor desc; + DMAInitContext *pContext = (DMAInitContext *)context; + //SGL *pThis = (SGL *)pContext->pThis; + bool next = false; + requiresSGLDMA = false; + for (uint32_t i = 0; i < length; i += 16) { + memcpy(desc.data, buffer + i, 16); + + switch (SGL_TYPE(desc.id)) { + case TYPE_DATA_BLOCK_DESCRIPTOR: + case TYPE_KEYED_DATA_BLOCK_DESCRIPTOR: + case TYPE_BIT_BUCKET_DESCRIPTOR: + parseSGLDescriptor(desc); + break; + case TYPE_SEGMENT_DESCRIPTOR: + case TYPE_LAST_SEGMENT_DESCRIPTOR: + if (i != pContext->currentSize - 16) { + SC_REPORT_WARNING("sgl", "Invalid SGL segment"); + } + next = true; + + break; + } + } + + free(buffer); + + // Go to next + if (next) { + requiresSGLDMA = true; + address = desc.address; + length = desc.length; + buffer = (uint8_t *)malloc(length); + } +} + +void SGL::read(uint64_t offset, uint64_t length, uint8_t *buffer, + DMAFunction &func, void *context) { + DMAFunction doRead = [this, offset, length, buffer](uint64_t, void *context) { + uint64_t totalRead = 0; + uint64_t currentOffset = 0; + uint64_t read; + bool begin = false; + + DMAContext *readContext = (DMAContext *)context; + + for (auto &iter : chunkList) { + if (begin) { + read = MIN(iter.length, length - totalRead); + + if (!iter.ignore) { + readContext->counter++; + // TODO(jhieb) how to do DMA reads in qemu? + /* + pInterface->dmaRead(iter.addr, read, + buffer ? buffer + totalRead : NULL, dmaHandler, + readContext);*/ + } + + totalRead += read; + + if (totalRead == length) { + break; + } + } + + if (!begin && currentOffset + iter.length > offset) { + begin = true; + totalRead = offset - currentOffset; + read = MIN(iter.length - totalRead, length); + + if (!iter.ignore) { + readContext->counter++; + // TODO(jhieb) how to do DMA reads in qemu? + /* + pInterface->dmaRead(iter.addr + totalRead, read, buffer, dmaHandler, + readContext);*/ + } + + totalRead = read; + } + + currentOffset += iter.length; + } + + if (readContext->counter == 0) { + readContext->counter = 1; + // TODO(jhieb) Does this matter? + //dmaHandler(getTick(), readContext); + } + }; + + DMAContext *readContext = new DMAContext(func, context); + + // TODO(jhieb) how to do DMA reads in qemu? + //execute(CPU::NVME__SGL, CPU::READ, doRead, readContext); +} + +void SGL::write(uint64_t offset, uint64_t length, uint8_t *buffer, + DMAFunction &func, void *context) { + DMAFunction doWrite = [this, offset, length, buffer](uint64_t, + void *context) { + uint64_t totalWritten = 0; + uint64_t currentOffset = 0; + uint64_t written; + bool begin = false; + + DMAContext *writeContext = (DMAContext *)context; + + for (auto &iter : chunkList) { + if (begin) { + written = MIN(iter.length, length - totalWritten); + + if (!iter.ignore) { + writeContext->counter++; + // TODO(jhieb) how to do DMA writes in qemu? + /* + pInterface->dmaWrite(iter.addr, written, + buffer ? buffer + totalWritten : NULL, + dmaHandler, writeContext);*/ + } + + totalWritten += written; + + if (totalWritten == length) { + break; + } + } + + if (!begin && currentOffset + iter.length > offset) { + begin = true; + totalWritten = offset - currentOffset; + written = MIN(iter.length - totalWritten, length); + + if (!iter.ignore) { + writeContext->counter++; + // TODO(jhieb) how to do DMA writes in qemu? + /* + pInterface->dmaWrite(iter.addr + totalWritten, written, buffer, + dmaHandler, writeContext);*/ + } + + totalWritten = written; + } + + currentOffset += iter.length; + } + + if (writeContext->counter == 0) { + writeContext->counter = 1; + // TODO(jhieb) Does this matter? + //dmaHandler(getTick(), writeContext); + } + }; + + DMAContext *writeContext = new DMAContext(func, context); + // TODO(jhieb) how to do DMA writes in qemu? + //execute(CPU::NVME__SGL, CPU::WRITE, doWrite, writeContext); +} diff --git a/systemc-components/pci/nvme_ssd/src/interface.cc b/systemc-components/pci/nvme_ssd/src/interface.cc new file mode 100644 index 00000000..4fad31c9 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/src/interface.cc @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#include "interface.h" +#include "algorithm.h" + +namespace PCIExpress { + +static const uint32_t maxPayloadSize = 4096; // Bytes +static const uint32_t packetOverhead = 36; // Bytes +static const uint32_t internalDelay[PCIE_NUM] = {19, 70, 115}; // Symbol +static const float encoding[PCIE_NUM] = {1.25f, 1.25f, 1.015625f}; // Ratio +static const uint32_t delay[PCIE_NUM] = {3200, 1600, 1000}; // pico-second + +uint64_t calculateDelay(PCIE_GEN gen, uint8_t lane, uint64_t bytesize) { + uint64_t nTLP = MAX(DIVCEIL(bytesize, maxPayloadSize), 1); + uint64_t nSymbol; + + nSymbol = bytesize + nTLP * (packetOverhead); + nSymbol = DIVCEIL(nSymbol, lane) + 1 + nTLP * internalDelay[gen]; + + return (uint64_t)(delay[gen] * encoding[gen] * nSymbol + 0.5f); +} + +} // namespace PCIExpress + +namespace SATA { + +// Each primitive is 1 DWORD (4bytes) +// One Fram contains SOF/EOF/CRC and HOLD/A primitives +// Just assume no HOLD/HOLDA + +static const uint32_t packetOverhead = 3; // Primitive +static const uint32_t internalDelay = 12; // DWORD +static const float encoding = 1.25f; // Ratio +static const uint32_t delay[SATA_NUM] = {5336, 2667, 1333}; // pico-second + +uint64_t calculateDelay(SATA_GEN gen, uint64_t bytesize) { + uint64_t dwords = DIVCEIL(bytesize, 4) + internalDelay + packetOverhead; + + dwords = dwords * 4; + + return (uint64_t)(delay[gen] * encoding * dwords + 0.5f); +} + +} // namespace SATA + +namespace MIPI { + +namespace M_PHY { + +// See M-PHY spec v4.0 section 4.7.2 BURST, Figure 14 and 15 +static const uint32_t tLine = 7000; // pico-second +static const uint32_t nPrepare = 15; // Bytes +static const uint32_t nSync = 15; // Bytes +static const uint32_t nStall = 7; // Bytes +static const float encoding = 1.25f; // Ratio +static const uint32_t delay[HS_NUM] = {6410, 3205, 1603, 801}; // pico-second + +uint64_t calculateDelay(M_PHY_MODE mode, uint8_t lane, uint64_t symbol) { + uint64_t nSymbol = nPrepare + nSync + 1 + symbol * 2 + nStall; + + nSymbol = (nSymbol - 1) / lane + 2; + + return (uint64_t)(delay[mode] * encoding * nSymbol + 0.5f + tLine); +} + +} // namespace M_PHY + +namespace UniPro { + +static const uint32_t maxPayloadSize = 254; // Bytes +static const uint32_t packetOverhead = 7; // Symbol + +uint64_t calculateDelay(M_PHY::M_PHY_MODE mode, uint8_t lane, + uint64_t bytesize) { + uint64_t nPacket = MAX((bytesize - 1) / maxPayloadSize + 1, 1); + uint64_t nSymbol = (bytesize + 1) / 2; + + nSymbol += nPacket * packetOverhead; + + return M_PHY::calculateDelay(mode, lane, nSymbol); +} + +} // namespace UniPro + +} // namespace MIPI + +namespace ARM { + +namespace AXI { + +// Two cycles per one data beat +// Two cycles for address, one cycle for write response +// One burst request should contain 1, 2, 4, 8 .. of beats + +static const uint32_t maxPayloadSize = 4096; // Bytes + +uint64_t calculateDelay(uint64_t clock, BUS_WIDTH width, uint64_t bytesize) { + uint32_t nBeats = MAX(DIVCEIL(bytesize, width), 1); + uint32_t nBursts = popcount(nBeats) + (bytesize - 1) / maxPayloadSize; + uint32_t nClocks = nBeats * 2 + nBursts * 3; + uint64_t period = (uint64_t)(1000000000000.0 / clock + 0.5); + + return nClocks * period; +} + +namespace Stream { + +// Unlimited bursts +// If master and slave can handle data sufficiently fast, +// one cycle per one data beat +// No address and responses + +uint64_t calculateDelay(uint64_t clock, BUS_WIDTH width, uint64_t bytesize) { + uint64_t period = (uint64_t)(1000000000000.0 / clock + 0.5); + + return bytesize / width * period; +} + +} // namespace Stream + +} // namespace AXI + +} // namespace ARM diff --git a/systemc-components/pci/nvme_ssd/src/namespace.cc b/systemc-components/pci/nvme_ssd/src/namespace.cc new file mode 100644 index 00000000..1bc65976 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/src/namespace.cc @@ -0,0 +1,654 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#include "namespace.h" +#include "algorithm.h" +#include "systemc.h" +#include + +Namespace::Namespace(ConfigData &c) + : //pParent(p), + pDisk(nullptr), + cfgdata(c), + nsid(NSID_NONE), + attached(false), + allocated(false), + formatFinishedAt(0) {} + +Namespace::~Namespace() { + if (pDisk) { + delete pDisk; + } +} + +CQEntryWrapper Namespace::submitCommand(SQEntryWrapper &req, uint8_t *&buffer) { + CQEntryWrapper resp(req); + + // TODO(jhieb) timing fixes? + /*if (getTick() < formatFinishedAt) { + resp.makeStatus(false, false, TYPE_GENERIC_COMMAND_STATUS, + STATUS_FORMAT_IN_PROGRESS); + + response = true; + }*/ + //else { + // Admin commands + if (req.sqID == 0) { + switch (req.entry.dword0.opcode) { + case OPCODE_GET_LOG_PAGE: + getLogPage(req); + break; + default: + resp.makeStatus(true, false, TYPE_GENERIC_COMMAND_STATUS, + STATUS_INVALID_OPCODE); + break; + } + } + + // NVM commands + else { + CQEntryWrapper rd_rsp(req); + switch (req.entry.dword0.opcode) { + case OPCODE_FLUSH: + flush(req); + break; + case OPCODE_WRITE: + write(req); + break; + case OPCODE_READ: + rd_rsp = read(req, buffer); + resp.setStatus(rd_rsp.getStatus()); + break; + case OPCODE_COMPARE: + compare(req); + break; + case OPCODE_DATASET_MANAGEMEMT: + datasetManagement(req); + break; + default: + resp.makeStatus(true, false, TYPE_GENERIC_COMMAND_STATUS, + STATUS_INVALID_OPCODE); + break; + } + } + return resp; +} + +uint32_t Namespace::getLBASize(){ + return info.lbaSize; +} + +void Namespace::setData(uint32_t id, Information *data) { + nsid = id; + memcpy(&info, data, sizeof(Information)); + + if (NVME_ENABLE_DISK_IMAGE) { + uint64_t diskSize; + + std::string filename = NVME_DISK_IMAGE_PATH + std::to_string(nsid); + + if (filename.length() == 0) { + pDisk = new MemDisk(); + } + else if (NVME_USE_COW_DISK) { + pDisk = new CoWDisk(); + } + else { + pDisk = new Disk(); + } + + diskSize = pDisk->open(filename, info.size * info.lbaSize, info.lbaSize); + + if (diskSize == 0) { + SC_REPORT_WARNING("namespace", "Failed to open disk image"); + } + else if (diskSize != info.size * info.lbaSize) { + if (NVME_STRICT_DISK_SIZE) { + SC_REPORT_WARNING("namespace", "Disk size not match"); + } + } + + if (filename.length() > 0) { + std::ostringstream msg; + msg << "Using disk image at " << filename << " for NSID " << nsid; + SC_REPORT_INFO("namespace", msg.str().c_str()); + } + } + + allocated = true; +} + +void Namespace::attach(bool attach) { + attached = attach; +} + +uint32_t Namespace::getNSID() { + return nsid; +} + +Namespace::Information *Namespace::getInfo() { + return &info; +} + +bool Namespace::isAttached() { + return attached; +} + +void Namespace::format(uint64_t tick) { + formatFinishedAt = tick; + + health = HealthInfo(); + + if (pDisk) { + delete pDisk; + pDisk = nullptr; + } +} + +void Namespace::getLogPage(SQEntryWrapper &req) { + CQEntryWrapper resp(req); + uint16_t numdl = (req.entry.dword10 & 0xFFFF0000) >> 16; + uint16_t lid = req.entry.dword10 & 0xFFFF; + uint16_t numdu = req.entry.dword11 & 0xFFFF; + uint32_t lopl = req.entry.dword12; + uint32_t lopu = req.entry.dword13; + bool submit = true; + + uint32_t req_size = (((uint32_t)numdu << 16 | numdl) + 1) * 4; + uint64_t offset = ((uint64_t)lopu << 32) | lopl; + + /* + debugprint(LOG_HIL_NVME, + "ADMIN | Get Log Page | Log %d | Size %d | NSID %d", lid, + req_size, nsid); + */ + + static DMAFunction dmaDone = [](uint64_t, void *context) { + RequestContext *pContext = (RequestContext *)context; + + pContext->function(pContext->resp); + + delete pContext->dma; + delete pContext; + }; + DMAFunction smartInfo = [offset](uint64_t, void *context) { + RequestContext *pContext = (RequestContext *)context; + + pContext->dma->write(offset, 512, pContext->buffer, dmaDone, context); + }; + + switch (lid) { + case LOG_SMART_HEALTH_INFORMATION: + if (req.entry.namespaceID == nsid) { + submit = false; + //RequestContext *pContext = new RequestContext(func, resp); + //pContext->buffer = health.data; + + // TODO(jhieb) fix? + /* + if (req.useSGL) { + pContext->dma = new SGL(cfgdata, smartInfo, pContext, req.entry.data1, + req.entry.data2); + } + else { + pContext->dma = + new PRPList(cfgdata, smartInfo, pContext, req.entry.data1, + req.entry.data2, (uint64_t)req_size); + } + */ + } + else { + resp.makeStatus(true, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_NAMESPACE_NOT_ATTACHED); + } + + break; + default: + resp.makeStatus(true, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_INVALID_LOG_PAGE); + break; + } + + /* + if (submit) { + func(resp); + }*/ +} + +void Namespace::flush(SQEntryWrapper &req) { + bool err = false; + + CQEntryWrapper resp(req); + + if (!attached) { + err = true; + resp.makeStatus(true, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_NAMESPACE_NOT_ATTACHED); + + //func(resp); + } + + if (!err) { + /* + DMAFunction begin = [this](uint64_t, void *context) { + DMAFunction doFlush = [this](uint64_t now, void *context) { + IOContext *pContext = (IOContext *)context; + + + debugprint( + LOG_HIL_NVME, + "NVM | FLUSH | CQ %u | SQ %u:%u | CID %u | NSID %-5d | %" PRIu64 + " - %" PRIu64 " (%" PRIu64 ")", + pContext->resp.cqID, pContext->resp.entry.dword2.sqID, + pContext->resp.sqUID, pContext->resp.entry.dword3.commandID, nsid, + pContext->beginAt, now, now - pContext->beginAt); + + pContext->function(pContext->resp); + + delete pContext; + }; + + //pParent->flush(this, doFlush, context); + }; + */ + + /* + debugprint(LOG_HIL_NVME, "NVM | FLUSH | SQ %u:%u | CID %u | NSID %-5d", + req.sqID, req.sqUID, req.entry.dword0.commandID, nsid); + */ + + //IOContext *pContext = new IOContext(func, resp); + + //pContext->beginAt = getTick(); + + //execute(CPU::NVME__NAMESPACE, CPU::FLUSH, begin, pContext); + } +} + +void Namespace::write(SQEntryWrapper &req) { + bool err = false; + + CQEntryWrapper resp(req); + uint64_t slba = ((uint64_t)req.entry.dword11 << 32) | req.entry.dword10; + uint16_t nlb = (req.entry.dword12 & 0xFFFF) + 1; + + if (!attached) { + err = true; + resp.makeStatus(true, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_NAMESPACE_NOT_ATTACHED); + } + if (nlb == 0) { + err = true; + SC_REPORT_WARNING("namespace", "host tried to write 0 blocks"); + } + + /* + debugprint(LOG_HIL_NVME, + "NVM | WRITE | SQ %u:%u | CID %u | NSID %-5d | %" PRIX64 + " + %d", + req.sqID, req.sqUID, req.entry.dword0.commandID, nsid, slba, nlb); + */ + + if (!err) { + /* + DMAFunction doRead = [this](uint64_t tick, void *context) { + DMAFunction dmaDone = [this](uint64_t tick, void *context) { + IOContext *pContext = (IOContext *)context; + + pContext->beginAt++; + + if (pContext->beginAt == 2) { + debugprint( + LOG_HIL_NVME, + "NVM | WRITE | CQ %u | SQ %u:%u | CID %u | NSID %-5d | " + "%" PRIX64 " + %d | %" PRIu64 " - %" PRIu64 " (%" PRIu64 ")", + pContext->resp.cqID, pContext->resp.entry.dword2.sqID, + pContext->resp.sqUID, pContext->resp.entry.dword3.commandID, nsid, + pContext->slba, pContext->nlb, pContext->tick, tick, + tick - pContext->tick); + pContext->function(pContext->resp); + + if (pContext->buffer) { + pDisk->write(pContext->slba, pContext->nlb, pContext->buffer); + + free(pContext->buffer); + } + + delete pContext->dma; + delete pContext; + } + }; + + IOContext *pContext = (IOContext *)context; + + pContext->tick = tick; + pContext->beginAt = 0; + + if (pDisk) { + pContext->buffer = (uint8_t *)calloc(pContext->nlb, info.lbaSize); + + pContext->dma->read(0, pContext->nlb * info.lbaSize, pContext->buffer, + dmaDone, context); + } + else { + pContext->dma->read(0, pContext->nlb * info.lbaSize, nullptr, dmaDone, + context); + } + + pParent->write(this, pContext->slba, pContext->nlb, dmaDone, context); + }; + */ + + //IOContext *pContext = new IOContext(func, resp); + + // TODO(jhieb) support timing. + //pContext->beginAt = getTick(); + //pContext->slba = slba; + //pContext->nlb = nlb; + + //CPUContext *pCPU = + // new CPUContext(doRead, pContext, CPU::NVME__NAMESPACE, CPU::WRITE); + /* + static DMAFunction empty = [](uint64_t, void *) {}; + if (req.useSGL) { + pContext->dma = + new SGL(cfgdata, empty, nullptr, req.entry.data1, req.entry.data2); + } + else { + pContext->dma = + new PRPList(cfgdata, empty, nullptr, req.entry.data1, + req.entry.data2, (uint64_t)nlb * info.lbaSize); + }*/ + } + //else { + //func(resp); + //} +} + +CQEntryWrapper Namespace::read(SQEntryWrapper &req, uint8_t *&buffer) { + bool err = false; + + CQEntryWrapper resp(req); + uint64_t slba = ((uint64_t)req.entry.dword11 << 32) | req.entry.dword10; + uint16_t nlb = (req.entry.dword12 & 0xFFFF) + 1; + // bool fua = req.entry.dword12 & 0x40000000; + + if (!attached) { + err = true; + resp.makeStatus(true, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_NAMESPACE_NOT_ATTACHED); + } + if (nlb == 0) { + err = true; + resp.makeStatus(true, false, TYPE_COMMAND_SPECIFIC_STATUS, STATUS_NAMESPACE_NOT_ATTACHED); + SC_REPORT_WARNING("namespace", "host tried to read 0 blocks"); + } + + /* + debugprint(LOG_HIL_NVME, + "NVM | READ | SQ %u:%u | CID %u | NSID %-5d | %" PRIX64 + " + %d", + req.sqID, req.sqUID, req.entry.dword0.commandID, nsid, slba, nlb); + */ + + if (!err) { + // TODO(jhieb) fix? + //pParent->read(this, pContext->slba, pContext->nlb, dmaDone, pContext); + + buffer = (uint8_t *)calloc(nlb, info.lbaSize); + + if (pDisk) { + pDisk->read(slba, nlb, buffer); + } + resp.makeStatus(false, false, TYPE_GENERIC_COMMAND_STATUS, STATUS_SUCCESS); + /*pContext->dma->write(0, pContext->nlb * info.lbaSize, pContext->buffer, + dmaDone, context);*/ + } + return resp; +} + +void Namespace::compare(SQEntryWrapper &req) { + bool err = false; + + CQEntryWrapper resp(req); + uint64_t slba = ((uint64_t)req.entry.dword11 << 32) | req.entry.dword10; + uint16_t nlb = (req.entry.dword12 & 0xFFFF) + 1; + // bool fua = req.entry.dword12 & 0x40000000; + + if (!attached) { + err = true; + resp.makeStatus(true, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_NAMESPACE_NOT_ATTACHED); + } + if (nlb == 0) { + err = true; + SC_REPORT_WARNING("namespace", "host tried to read 0 blocks"); + } + + /* + debugprint(LOG_HIL_NVME, + "NVM | COMP | SQ %u:%u | CID %u | NSID %-5d | %" PRIX64 + " + %d", + req.sqID, req.sqUID, req.entry.dword0.commandID, nsid, slba, nlb); + */ + + if (!err) { + DMAFunction doRead = [this](uint64_t tick, void *context) { + DMAFunction dmaDone = [this](uint64_t tick, void *context) { + CompareContext *pContext = (CompareContext *)context; + + pContext->beginAt++; + + if (pContext->beginAt == 2) { + // Compare buffer! + // Always success if no disk + if (pDisk && memcmp(pContext->buffer, pContext->hostContent, + pContext->nlb * info.lbaSize) != 0) { + pContext->resp.makeStatus(false, false, + TYPE_MEDIA_AND_DATA_INTEGRITY_ERROR, + STATUS_COMPARE_FAILURE); + } + + /* + debugprint( + LOG_HIL_NVME, + "NVM | COMP | CQ %u | SQ %u:%u | CID %u | NSID %-5d | " + "%" PRIX64 " + %d | %" PRIu64 " - %" PRIu64 " (%" PRIu64 ")", + pContext->resp.cqID, pContext->resp.entry.dword2.sqID, + pContext->resp.sqUID, pContext->resp.entry.dword3.commandID, nsid, + pContext->slba, pContext->nlb, pContext->tick, tick, + tick - pContext->tick); + */ + + pContext->function(pContext->resp); + + if (pContext->buffer) { + free(pContext->buffer); + } + if (pContext->hostContent) { + free(pContext->hostContent); + } + + delete pContext->dma; + delete pContext; + } + }; + + CompareContext *pContext = (CompareContext *)context; + + pContext->tick = tick; + pContext->beginAt = 0; + + // TODO(jhieb) fix + //pParent->read(this, pContext->slba, pContext->nlb, dmaDone, pContext); + + pContext->buffer = (uint8_t *)calloc(pContext->nlb, info.lbaSize); + pContext->hostContent = (uint8_t *)calloc(pContext->nlb, info.lbaSize); + + if (pDisk) { + pDisk->read(pContext->slba, pContext->nlb, pContext->buffer); + } + + pContext->dma->read(0, pContext->nlb * info.lbaSize, + pContext->hostContent, dmaDone, context); + }; + + //CompareContext *pContext = new CompareContext(func, resp); + + // TODO(Jhieb)timing? + //pContext->beginAt = getTick(); + /* + pContext->slba = slba; + pContext->nlb = nlb; + + //CPUContext *pCPU = + // new CPUContext(doRead, pContext, CPU::NVME__NAMESPACE, CPU::READ); + static DMAFunction empty = [](uint64_t, void *) {}; + if (req.useSGL) { + pContext->dma = + new SGL(cfgdata, empty, nullptr, req.entry.data1, req.entry.data2); + } + else { + pContext->dma = + new PRPList(cfgdata, empty, nullptr, req.entry.data1, + req.entry.data2, pContext->nlb * info.lbaSize); + } + */ + } + //else { + // func(resp); + //} +} + +void Namespace::datasetManagement(SQEntryWrapper &req) { + bool err = false; + + CQEntryWrapper resp(req); + int nr = (req.entry.dword10 & 0xFF) + 1; + bool ad = req.entry.dword11 & 0x04; + + if (!attached) { + err = true; + resp.makeStatus(true, false, TYPE_COMMAND_SPECIFIC_STATUS, + STATUS_NAMESPACE_NOT_ATTACHED); + } + if (!ad) { + err = true; + // Just ignore + } + + /* + debugprint( + LOG_HIL_NVME, + "NVM | TRIM | SQ %u:%u | CID %u | NSID %-5d| %d ranges | Attr %1X", + req.sqID, req.sqUID, req.entry.dword0.commandID, nsid, nr, + req.entry.dword11 & 0x0F); + */ + + if (!err) { + static DMAFunction eachTrimDone = [](uint64_t tick, void *context) { + DMAContext *pContext = (DMAContext *)context; + + pContext->counter--; + + if (pContext->counter == 0) { + pContext->function(tick, pContext->context); + } + + delete pContext; + }; + DMAFunction doTrim = [this](uint64_t, void *context) { + DMAFunction dmaDone = [this](uint64_t, void *context) { + DMAFunction trimDone = [this](uint64_t tick, void *context) { + IOContext *pContext = (IOContext *)context; + + /* + debugprint(LOG_HIL_NVME, + "NVM | TRIM | CQ %u | SQ %u:%u | CID %u | NSID %-5d| " + "%" PRIu64 " - %" PRIu64 " (%" PRIu64 ")", + pContext->resp.cqID, pContext->resp.entry.dword2.sqID, + pContext->resp.sqUID, + pContext->resp.entry.dword3.commandID, nsid, + pContext->beginAt, tick, tick - pContext->beginAt); + */ + + pContext->function(pContext->resp); + + delete pContext; + }; + + DatasetManagementRange range; + IOContext *pContext = (IOContext *)context; + DMAContext *pDMA = new DMAContext(trimDone); + + pDMA->context = context; + + for (uint64_t i = 0; i < pContext->slba; i++) { + memcpy(range.data, + pContext->buffer + i * sizeof(DatasetManagementRange), + sizeof(DatasetManagementRange)); + + pDMA->counter++; + // TODO(jhieb) fix. + //pParent->trim(this, range.slba, range.nlb, eachTrimDone, pDMA); + } + + if (pDMA->counter == 0) { + pDMA->counter = 1; + + // TODO(Jhieb) fix? + //eachTrimDone(getTick(), pDMA); + } + + free(pContext->buffer); + delete pContext->dma; + }; + + IOContext *pContext = (IOContext *)context; + + pContext->buffer = + (uint8_t *)calloc(pContext->slba, sizeof(DatasetManagementRange)); + + pContext->dma->read(0, pContext->slba * sizeof(DatasetManagementRange), + pContext->buffer, dmaDone, context); + }; + + /* + IOContext *pContext = new IOContext(func, resp); + + // TODO(jhieb) fix? + //pContext->beginAt = getTick(); + pContext->slba = nr; + + //CPUContext *pCPU = new CPUContext(doTrim, pContext, CPU::NVME__NAMESPACE, + // CPU::DATASET_MANAGEMENT); + static DMAFunction empty = [](uint64_t, void *) {}; + if (req.useSGL) { + pContext->dma = + new SGL(cfgdata, empty, nullptr, req.entry.data1, req.entry.data2); + } + else { + pContext->dma = new PRPList(cfgdata, empty, nullptr, req.entry.data1, + req.entry.data2, (uint64_t)nr * 0x10); + }*/ + } + //else { + // func(resp); + //} +} diff --git a/systemc-components/pci/nvme_ssd/src/nvme_ssd.cc b/systemc-components/pci/nvme_ssd/src/nvme_ssd.cc new file mode 100644 index 00000000..9559f750 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/src/nvme_ssd.cc @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All Rights Reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include "nvme_ssd.h" + +typedef gs::nvme_ssd<> nvme_ssd; + +void module_register() { GSC_MODULE_REGISTER_C(nvme_ssd, sc_core::sc_object*); } \ No newline at end of file diff --git a/systemc-components/pci/nvme_ssd/src/pci-device-base.cc b/systemc-components/pci/nvme_ssd/src/pci-device-base.cc new file mode 100644 index 00000000..7d075853 --- /dev/null +++ b/systemc-components/pci/nvme_ssd/src/pci-device-base.cc @@ -0,0 +1,81 @@ +/* + * TLM-2.0 PCI device base class. + * + * Copyright (c) 2020 Xilinx Inc. + * Written by Edgar E. Iglesias. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "systemc.h" + +#include "tlm_utils/simple_initiator_socket.h" +#include "tlm_utils/simple_target_socket.h" + +#include "pci-device-base.h" + +pci_device_base::pci_device_base(sc_module_name name, + unsigned int nr_bars, + unsigned int nr_irqs) + : sc_module(name), + config("config"), + bar("bar", nr_bars), + dma("dma"), + ats_req("ats_req"), + ats_inv("ats_inv"), + irq("irq", nr_irqs) +{ + unsigned int i; + + config.register_b_transport(this, &pci_device_base::config_b_transport); + + for (i = 0; i < bar.size(); i++) { + bar[i].register_b_transport(this, + &pci_device_base::bar_b_transport, + i); + } + + ats_inv.register_b_transport(this, + &pci_device_base::b_transport_ats_inv); +} + +void pci_device_base::config_b_transport(tlm::tlm_generic_payload& trans, + sc_time& delay) +{ + if (trans.is_read()) { + // User needs to implement an infrastructure for CONFIG-space + // if reads are forwarded to us. + trans.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE); + } else { + trans.set_response_status(tlm::TLM_OK_RESPONSE); + } +} + +void pci_device_base::bar_b_transport(int bar_nr, + tlm::tlm_generic_payload& trans, + sc_time& delay) +{ + trans.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE); +} + +void pci_device_base::b_transport_ats_inv(tlm::tlm_generic_payload& trans, + sc_time& delay) +{ + trans.set_response_status(tlm::TLM_OK_RESPONSE); +} diff --git a/systemc-components/pci/nvme_ssd/src/queue.cc b/systemc-components/pci/nvme_ssd/src/queue.cc new file mode 100644 index 00000000..221bc10f --- /dev/null +++ b/systemc-components/pci/nvme_ssd/src/queue.cc @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2017 CAMELab + * + * This file is part of SimpleSSD. + * + * SimpleSSD is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SimpleSSD is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SimpleSSD. If not, see . + */ + +#include "queue.h" +#include "systemc.h" + +SQEntry::_SQEntry() { + memset(data, 0, 64); +} + +CQEntry::_CQEntry() { + memset(data, 0, 16); +} + +SQEntryWrapper::_SQEntryWrapper(SQEntry &sqdata, uint16_t sqid, uint16_t cqid, + uint16_t sqhead, uint16_t squid) + : entry(sqdata), + sqID(sqid), + cqID(cqid), + sqHead(sqhead), + sqUID(squid), + useSGL(true) { + if ((sqdata.dword0.fuse >> 6) == 0x00) { + useSGL = false; + } +} + +CQEntryWrapper::_CQEntryWrapper(SQEntryWrapper &sqew) { + cqID = sqew.cqID; + sqUID = sqew.sqUID; + entry.dword2.sqHead = sqew.sqHead; + entry.dword2.sqID = sqew.sqID; + entry.dword3.commandID = sqew.entry.dword0.commandID; +} + +void CQEntryWrapper::makeStatus(bool dnr, bool more, STATUS_CODE_TYPE sct, + int sc) { + entry.dword3.status = 0x0000; + + entry.dword3.status = ((dnr ? 1 : 0) << 15); + entry.dword3.status |= ((more ? 1 : 0) << 14); + entry.dword3.status |= ((sct & 0x07) << 9); + entry.dword3.status |= ((sc & 0xFF) << 1); +} + +uint16_t CQEntryWrapper::getStatus(){ + return entry.dword3.status; +} + +void CQEntryWrapper::setStatus(uint16_t status){ + entry.dword3.status = status; +} + +Queue::Queue(uint16_t qid, uint16_t length) + : id(qid), head(0), tail(0), size(length), stride(0), base(nullptr) {} + +Queue::~Queue() { + if (base) { + delete base; + } +} + +uint16_t Queue::getID() { + return id; +} + +uint16_t Queue::getItemCount() { + if (tail >= head) { + return tail - head; + } + else { + return (size - head) + tail; + } +} + +uint16_t Queue::getHead() { + return head; +} + +uint16_t Queue::getTail() { + return tail; +} + +uint16_t Queue::getSize() { + return size; +} + +void Queue::setBase(DMAInterface *p, uint64_t s, uint64_t b_addr) { + base = p; + stride = s; + base_addr = b_addr; +} + +uint64_t Queue::getBaseAddr(){ + return base_addr; +} + + +CQueue::CQueue(uint16_t iv, bool en, uint16_t qid, uint16_t size) + : Queue(qid, size), ien(en), phase(true), interruptVector(iv) {} + +void CQueue::setData(CQEntry *entry) { + if (entry) { + // Set phase + entry->dword3.status &= 0xFFFE; + entry->dword3.status |= (phase ? 0x0001 : 0x0000); + + // Write entry + // TODO(jhieb) support DMA writes in the queue set function? + // base->write(tail * stride, 0x10, entry->data, func, context); + + // Increase tail + tail++; + + if (tail == size) { + tail = 0; + phase = !phase; + } + + if (head == tail) { + SC_REPORT_WARNING("CQueue", "Completion queue overflow"); + } + } +} + +uint16_t CQueue::incHead() { + head++; + + if (head == size) { + head = 0; + } + + return head; +} + +void CQueue::setHead(uint16_t newHead) { + head = newHead; +} + +bool CQueue::interruptEnabled() { + return ien; +} + +uint16_t CQueue::getInterruptVector() { + return interruptVector; +} + +SQueue::SQueue(uint16_t cqid, uint8_t pri, uint16_t qid, uint16_t size) + : Queue(qid, size), cqID(cqid), priority(pri) {} + +uint16_t SQueue::getCQID() { + return cqID; +} + +void SQueue::setTail(uint16_t newTail) { + tail = newTail; +} + +void SQueue::getData(SQEntry *entry){ // , DMAFunction func, void *context) { + if (entry && head != tail) { + // TODO(jhieb) do i want to add this back in?? + // Read entry + //base->read(head * stride, 0x40, entry->data, func, context); + + // Increase head + head++; + + if (head == size) { + head = 0; + } + } +} + +uint8_t SQueue::getPriority() { + return priority; +}