From 30b8e6c448c981eef57fc929d94c561344c7ac16 Mon Sep 17 00:00:00 2001 From: icpp Date: Wed, 23 Jul 2025 12:39:54 -0400 Subject: [PATCH 1/4] Exposes file creation timestamp via query Adds a new query endpoint to retrieve the file creation timestamp in nanoseconds and its age in seconds. --- README.md | 7 ++- src/db_chats.cpp | 1 + src/files.cpp | 116 +++++++++++++++++++++++++++++++++++++++++---- src/files.h | 12 ++++- src/llama_cpp.did | 19 +++++++- src/utils.cpp | 6 --- src/utils.h | 5 +- test/test_files.py | 61 +++++++++++++++++++++++- 8 files changed, 203 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index bbd8c1f..8486364 100644 --- a/README.md +++ b/README.md @@ -612,10 +612,13 @@ dfx canister call llama_cpp recursive_dir_content_update '(record {dir = ".canis dfx canister call llama_cpp recursive_dir_content_update '(record {dir = ".canister_cache"; max_entries = 5000 : nat64})' --output json # Get the size of a file in bytes -dfx canister --network $NETWORK call $llm filesystem_file_size '(record {filename = ""})' +dfx canister call llama_cpp filesystem_file_size '(record {filename = ""})' --output json + +# Get the creation timestamp of a file in nanoseconds (also returns age of file in seconds) +dfx canister call llama_cpp get_creation_timestamp_ns '(record {filename = ""})' --output json # remove a file or empty directory -dfx canister --network $NETWORK call $llm filesystem_remove '(record {filename = ""})' +dfx canister call llama_cpp filesystem_remove '(record {filename = ""})' ``` # Appendix A: max_tokens diff --git a/src/db_chats.cpp b/src/db_chats.cpp index 0f3d934..debaec1 100644 --- a/src/db_chats.cpp +++ b/src/db_chats.cpp @@ -1,6 +1,7 @@ #include "db_chats.h" #include "auth.h" #include "common.h" +#include "files.h" #include "http.h" #include "main_.h" #include "max_tokens.h" diff --git a/src/files.cpp b/src/files.cpp index 3c970fe..98487a4 100644 --- a/src/files.cpp +++ b/src/files.cpp @@ -7,6 +7,7 @@ // This library is included with icpp-pro #include "hash-library/sha256.h" +#include #include #include #include @@ -31,6 +32,20 @@ void filesystem_file_size() { filesystem_file_size_(ic_api, filename, true); } +void get_creation_timestamp_ns() { + IC_API ic_api(CanisterQuery{std::string(__func__)}, false); + if (!is_caller_a_controller(ic_api)) return; + + // Get filename + std::string filename{""}; + + CandidTypeRecord r_in; + r_in.append("filename", CandidTypeText{&filename}); + ic_api.from_wire(r_in); + + get_creation_timestamp_ns_(ic_api, filename, true); +} + void filesystem_remove() { IC_API ic_api(CanisterUpdate{std::string(__func__)}, false); if (!is_caller_a_controller(ic_api)) return; @@ -175,7 +190,8 @@ std::uint64_t filesystem_file_size_(IC_API &ic_api, const std::string &filename, error = true; msg = "Error: " + ec.message() + "\n"; } else if (!exists) { - msg = "File does not exist: " + filename + "\n"; + error = true; + msg = "File does not exist: " + filename; } else { msg = "File exists: " + filename + "\n"; // Use the non-throwing version of std::filesystem::file_size @@ -193,18 +209,93 @@ std::uint64_t filesystem_file_size_(IC_API &ic_api, const std::string &filename, << std::endl; if (to_wire) { - // Return the filesize over the wire (caller must immediately return from endpoint) - CandidTypeRecord filesystem_file_size_record; - filesystem_file_size_record.append("exists", CandidTypeBool{exists}); - filesystem_file_size_record.append("filename", CandidTypeText{filename}); - filesystem_file_size_record.append("filesize", CandidTypeNat64{filesize}); - filesystem_file_size_record.append("msg", CandidTypeText{msg}); - ic_api.to_wire( - CandidTypeVariant{"Ok", CandidTypeRecord{filesystem_file_size_record}}); + if (error) { + ic_api.to_wire(CandidTypeVariant{ + "Err", CandidTypeVariant{"Other", CandidTypeText{msg}}}); + } else { + // Return the filesize over the wire (caller must immediately return from endpoint) + CandidTypeRecord filesystem_file_size_record; + filesystem_file_size_record.append("exists", CandidTypeBool{exists}); + filesystem_file_size_record.append("filename", CandidTypeText{filename}); + filesystem_file_size_record.append("filesize", CandidTypeNat64{filesize}); + filesystem_file_size_record.append("msg", CandidTypeText{msg}); + ic_api.to_wire(CandidTypeVariant{ + "Ok", CandidTypeRecord{filesystem_file_size_record}}); + } } return filesize; } +std::uint64_t get_creation_timestamp_ns_(IC_API &ic_api, + const std::string &filename, + bool to_wire) { + std::error_code ec; + std::string msg; + bool error = false; + std::uint64_t timestamp_ns = 0; + std::uint64_t age_seconds = 0; + + bool exists = std::filesystem::exists(filename, ec); + if (ec) { + error = true; + msg = "Error: " + ec.message() + "\n"; + } else if (!exists) { + error = true; + msg = "File does not exist: " + filename; + } else { + msg = "File exists: " + filename + "\n"; + // Use the non-throwing version of std::filesystem::last_write_time + // NOTE: on the IC, this is actually the creation time, not the last write time + auto ftime = std::filesystem::last_write_time(filename, ec); + if (ec) { + error = true; + msg += "Error: " + ec.message() + "\n"; + } else { + // Get time since epoch in nanoseconds + auto nanos_since_epoch = + std::chrono::duration_cast( + ftime.time_since_epoch()) + .count(); + timestamp_ns = static_cast(nanos_since_epoch); + + // Calculate age in seconds + auto now = std::chrono::system_clock::now(); + auto age = std::chrono::duration_cast( + now.time_since_epoch() - ftime.time_since_epoch()) + .count(); + age_seconds = static_cast(age); + + msg += + "File creation timestamp_ns: " + std::to_string(nanos_since_epoch) + + " ns" + "\n" + "File creation age: " + std::to_string(age_seconds) + + " seconds" + "\n"; + } + } + + std::cout << "llama_cpp: " << std::string(__func__) << " - " << msg + << std::endl; + + if (to_wire) { + if (error) { + ic_api.to_wire(CandidTypeVariant{ + "Err", CandidTypeVariant{"Other", CandidTypeText{msg}}}); + } else { + // Return the filesize over the wire (caller must immediately return from endpoint) + CandidTypeRecord filesystem_timestamp_record; + filesystem_timestamp_record.append("exists", CandidTypeBool{exists}); + filesystem_timestamp_record.append("filename", CandidTypeText{filename}); + filesystem_timestamp_record.append("timestamp_ns", + CandidTypeNat64{timestamp_ns}); + filesystem_timestamp_record.append("age_seconds", + CandidTypeNat64{age_seconds}); + filesystem_timestamp_record.append("msg", CandidTypeText{msg}); + ic_api.to_wire(CandidTypeVariant{ + "Ok", CandidTypeRecord{filesystem_timestamp_record}}); + } + } + return timestamp_ns; +} + std::vector list_directory_contents(const std::filesystem::path &dir, const std::uint64_t &max_entries) { @@ -252,3 +343,10 @@ list_directory_contents(const std::filesystem::path &dir, return entries; } + +// Helper function to retrieve the last write time of a file +// NOTE: on the IC, this is actually the creation time, not the last write time +std::filesystem::file_time_type +get_last_write_time(const std::filesystem::path &file, std::error_code &ec) { + return std::filesystem::last_write_time(file, ec); +} \ No newline at end of file diff --git a/src/files.h b/src/files.h index 30fffa0..2e32b5f 100644 --- a/src/files.h +++ b/src/files.h @@ -12,6 +12,8 @@ void filesystem_remove() WASM_SYMBOL_EXPORTED("canister_update filesystem_remove"); void filesystem_file_size() WASM_SYMBOL_EXPORTED("canister_query filesystem_file_size"); +void get_creation_timestamp_ns() + WASM_SYMBOL_EXPORTED("canister_query get_creation_timestamp_ns"); void recursive_dir_content_query() WASM_SYMBOL_EXPORTED("canister_query recursive_dir_content_query"); void recursive_dir_content_update() @@ -22,6 +24,9 @@ bool filesystem_remove_(IC_API &ic_api, const std::string &filename, bool all, bool to_wire = true); std::uint64_t filesystem_file_size_(IC_API &ic_api, const std::string &filename, bool to_wire = true); +std::uint64_t get_creation_timestamp_ns_(IC_API &ic_api, + const std::string &filename, + bool to_wire = true); struct FileEntry { std::string filename; @@ -29,4 +34,9 @@ struct FileEntry { }; std::vector list_directory_contents(const std::filesystem::path &dir, - const std::uint64_t &max_entries); \ No newline at end of file + const std::uint64_t &max_entries); + +// Helper function to retrieve the last write time of a file +// NOTE: on the IC, this is actually the creation time, not the last write time +std::filesystem::file_time_type +get_last_write_time(const std::filesystem::path &file, std::error_code &ec); diff --git a/src/llama_cpp.did b/src/llama_cpp.did index 7c9a3ee..4168cbc 100644 --- a/src/llama_cpp.did +++ b/src/llama_cpp.did @@ -94,7 +94,7 @@ type GetChatsRecordResult = variant { }; type GetChatsRecord = record { chats : vec record { - timestamp : text; + timestamp_ns : text; chat : text } }; @@ -156,6 +156,22 @@ type FilesystemFileSizeRecord = record { msg : text }; +// ----------------------------------------------------- +type FilesystemTimestampInputRecord = record { + filename : text +}; +type FilesystemTimestampRecordResult = variant { + Err : ApiError; + Ok : FilesystemTimestampRecord +}; +type FilesystemTimestampRecord = record { + exists : bool; + filename : text; + timestamp_ns : nat64; // the file creation timestamp in nanoseconds + age_seconds : nat64; // the file creation age in seconds + msg : text +}; + // ----------------------------------------------------- type FilesystemRemoveInputRecord = record { filename : text @@ -218,6 +234,7 @@ service : { // Files - general utility endpoints exposing std::filesystem functions filesystem_remove : (FilesystemRemoveInputRecord) -> (FilesystemRemoveRecordResult); filesystem_file_size : (FilesystemFileSizeInputRecord) -> (FilesystemFileSizeRecordResult) query; + get_creation_timestamp_ns : (FilesystemTimestampInputRecord) -> (FilesystemTimestampRecordResult) query; recursive_dir_content_query : (DirContentInputRecord) -> (DirContentRecordResult) query; recursive_dir_content_update : (DirContentInputRecord) -> (DirContentRecordResult); diff --git a/src/utils.cpp b/src/utils.cpp index 777b468..1584a53 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -86,10 +86,4 @@ void send_output_record_result_error_to_wire(IC_API &ic_api, r_out.append("prompt_remaining", CandidTypeText{""}); r_out.append("generated_eog", CandidTypeBool{false}); ic_api.to_wire(CandidTypeVariant{"Err", r_out}); -} - -// Helper function to retrieve the last write time of a file -std::filesystem::file_time_type -get_last_write_time(const std::filesystem::path &file, std::error_code &ec) { - return std::filesystem::last_write_time(file, ec); } \ No newline at end of file diff --git a/src/utils.h b/src/utils.h index b03c4e2..6de7262 100644 --- a/src/utils.h +++ b/src/utils.h @@ -20,7 +20,4 @@ bool my_create_directory(const std::filesystem::path &dir_path, void send_output_record_result_error_to_wire(IC_API &ic_api, uint16_t http_status_code, - const std::string &error_msg); - -std::filesystem::file_time_type -get_last_write_time(const std::filesystem::path &file, std::error_code &ec); \ No newline at end of file + const std::string &error_msg); \ No newline at end of file diff --git a/test/test_files.py b/test/test_files.py index 3a1f8d3..860827c 100644 --- a/test/test_files.py +++ b/test/test_files.py @@ -151,7 +151,7 @@ def test__filesystem_file_size_non_existing(network: str, principal: str) -> Non canister_argument='(record {filename = "does_not_exist.bin"})', network=network, ) - expected_response = f'(variant {{ Ok = record {{ msg = "File does not exist: does_not_exist.bin\\n"; filename = "does_not_exist.bin"; filesize = 0 : nat64; exists = false;}} }})' + expected_response = '(variant { Err = variant { Other = "File does not exist: does_not_exist.bin" } })' assert response == expected_response def test__filesystem_file_size_anonymous(identity_anonymous: Dict[str, str], network: str) -> None: @@ -201,6 +201,65 @@ def test__filesystem_file_size_controller(network: str, principal: str) -> None: expected_response = f'(variant {{ Ok = record {{ msg = "File exists: .canister_cache/{principal}/sessions/prompt.cache\\nFile size: 5 bytes\\n"; filename = ".canister_cache/{principal}/sessions/prompt.cache"; filesize = 5 : nat64; exists = true;}} }})' assert response == expected_response +# ------------------------------------------------------------------ +def test__get_creation_timestamp_ns_non_existing(network: str, principal: str) -> None: + response = call_canister_api( + dfx_json_path=DFX_JSON_PATH, + canister_name=CANISTER_NAME, + canister_method="get_creation_timestamp_ns", + canister_argument='(record {filename = "does_not_exist.bin"})', + network=network, + ) + expected_response = '(variant { Err = variant { Other = "File does not exist: does_not_exist.bin" } })' + assert response == expected_response + +def test__get_creation_timestamp_ns_anonymous(identity_anonymous: Dict[str, str], network: str) -> None: + # double check the identity_anonymous fixture worked + assert identity_anonymous["identity"] == "anonymous" + assert identity_anonymous["principal"] == "2vxsx-fae" + + principal = identity_anonymous["principal"] + filename = f".canister_cache/{principal}/sessions/prompt.cache" + + response = call_canister_api( + dfx_json_path=DFX_JSON_PATH, + canister_name=CANISTER_NAME, + canister_method="get_creation_timestamp_ns", + canister_argument=f'(record {{filename = "{filename}"}})', + network=network, + ) + expected_response = f'(variant {{ Err = variant {{ Other = "Access Denied" }} }})' + assert response == expected_response + +# This test requires to run the test with non-default identity --> TODO: qa script must run with non-default identity +# +# def test__get_creation_timestamp_ns_non_controller(identity_default: Dict[str, str], network: str) -> None: +# principal = identity_default["principal"] +# filename = f".canister_cache/{principal}/sessions/prompt.cache" + +# response = call_canister_api( +# dfx_json_path=DFX_JSON_PATH, +# canister_name=CANISTER_NAME, +# canister_method="get_creation_timestamp_ns", +# canister_argument=f'(record {{filename = "{filename}"}})', +# network=network, +# ) +# expected_response = f'(variant {{ Err = variant {{ Other = "Access Denied" }} }})' +# assert response == expected_response + +def test__get_creation_timestamp_ns_controller(network: str, principal: str) -> None: + filename = f".canister_cache/{principal}/sessions/prompt.cache" + + response = call_canister_api( + dfx_json_path=DFX_JSON_PATH, + canister_name=CANISTER_NAME, + canister_method="get_creation_timestamp_ns", + canister_argument=f'(record {{filename = "{filename}"}})', + network=network, + ) + expected_response_startswith = f'(variant {{ Ok = record {{ msg = "File exists: .canister_cache/{principal}/sessions/prompt.cache\\nFile creation timestamp_ns: ' + assert response.startswith(expected_response_startswith) + # ------------------------------------------------------------------ def test__filesystem_remove_non_existing(network: str, principal: str) -> None: response = call_canister_api( From 94bfeeb0b9472eca1124cf6980eaf25e3a6ed259 Mon Sep 17 00:00:00 2001 From: icpp Date: Wed, 23 Jul 2025 13:05:43 -0400 Subject: [PATCH 2/4] Fix QA test for filesystem_file_size - non-existing file --- native/test_files.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native/test_files.cpp b/native/test_files.cpp index 33bc9d8..a412adb 100644 --- a/native/test_files.cpp +++ b/native/test_files.cpp @@ -230,7 +230,7 @@ void test_files(MockIC &mockIC) { std::string(__func__) + ": " + "filesystem_file_size - non-existing file", filesystem_file_size, "4449444c016c01c7dda8bb0771010012646f65735f6e6f745f65786973742e62696e", - "4449444c026c04c1b4cc0271c7dda8bb0771bdbaf9d50778dcbb80ff0b7e6b01bc8a01000101002846696c6520646f6573206e6f742065786973743a20646f65735f6e6f745f65786973742e62696e0a12646f65735f6e6f745f65786973742e62696e000000000000000000", + "4449444c026b01b0ad8fcd0c716b01c5fed20100010100002746696c6520646f6573206e6f742065786973743a20646f65735f6e6f745f65786973742e62696e", silent_on_trap, my_principal); // ----------------------------------------------------------------------------- From e388eb884bf52249290d3d67f37977b3d6f7464f Mon Sep 17 00:00:00 2001 From: icpp Date: Wed, 23 Jul 2025 13:44:14 -0400 Subject: [PATCH 3/4] use file_clock iso system_clock to calculate age --- src/files.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/files.cpp b/src/files.cpp index 98487a4..b2ee227 100644 --- a/src/files.cpp +++ b/src/files.cpp @@ -259,7 +259,7 @@ std::uint64_t get_creation_timestamp_ns_(IC_API &ic_api, timestamp_ns = static_cast(nanos_since_epoch); // Calculate age in seconds - auto now = std::chrono::system_clock::now(); + auto now = std::chrono::file_clock::now(); auto age = std::chrono::duration_cast( now.time_since_epoch() - ftime.time_since_epoch()) .count(); From c81967483177b9cf34b8c126cfb4ae8ee75b8cc8 Mon Sep 17 00:00:00 2001 From: icpp Date: Wed, 23 Jul 2025 13:45:52 -0400 Subject: [PATCH 4/4] Update comment --- src/files.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/files.cpp b/src/files.cpp index b2ee227..b4a14bb 100644 --- a/src/files.cpp +++ b/src/files.cpp @@ -280,7 +280,7 @@ std::uint64_t get_creation_timestamp_ns_(IC_API &ic_api, ic_api.to_wire(CandidTypeVariant{ "Err", CandidTypeVariant{"Other", CandidTypeText{msg}}}); } else { - // Return the filesize over the wire (caller must immediately return from endpoint) + // Return the timestamp & age over the wire (caller must immediately return from endpoint) CandidTypeRecord filesystem_timestamp_record; filesystem_timestamp_record.append("exists", CandidTypeBool{exists}); filesystem_timestamp_record.append("filename", CandidTypeText{filename});