diff --git a/cloud_storage/cloud_storage/overrides/file.py b/cloud_storage/cloud_storage/overrides/file.py index 2959228..8afd989 100644 --- a/cloud_storage/cloud_storage/overrides/file.py +++ b/cloud_storage/cloud_storage/overrides/file.py @@ -56,9 +56,11 @@ def validate(self) -> None: PATH: frappe/core/doctype/file/file.py METHOD: validate """ - self.associate_files() if self.flags.cloud_storage or self.flags.ignore_file_validate: return + + self.associate_files() + if not self.is_remote_file: self.custom_validate() else: @@ -211,7 +213,13 @@ def associate_files( "file_association", add_child_file_association(attached_to_doctype, attached_to_name), ) - existing_file.save() + existing_file.flags.ignore_file_validate = True + + if self.flags.ignore_links: existing_file.flags.ignore_links = True + if self.flags.ignore_permissions: existing_file.flags.ignore_permissions = True + if self.flags.ignore_validate: existing_file.flags.ignore_validate = True + + existing_file.save(ignore_permissions=True) else: if self.file_association: already_linked = any( @@ -395,10 +403,15 @@ def has_permission(doc, ptype: str | None = None, user: str | None = None) -> bo if doc.owner == user: has_access = True elif doc.attached_to_doctype and doc.attached_to_name: # type: ignore - reference_doc = frappe.get_doc(doc.attached_to_doctype, doc.attached_to_name) # type: ignore - has_access = reference_doc.has_permission() - if not has_access: - has_access = has_user_permission(doc, user) + try: + reference_doc = frappe.get_doc(doc.attached_to_doctype, doc.attached_to_name) # type: ignore + has_access = reference_doc.has_permission() + if not has_access: + has_access = has_user_permission(doc, user) + except frappe.DoesNotExistError: + # If attached document doesn't exist, check permission on the file itself + has_access = bool(frappe.has_permission(doc.doctype, ptype, user=user)) + # elif True: # Check "shared with" including parent 'folder' to allow access # ... @@ -439,11 +452,31 @@ def strip_special_chars(file_name: str) -> str: return regex.sub("", file_name) +def get_cloud_storage_config() -> dict: + config = frappe.conf.get("cloud_storage_settings", {}) + + # If nested config is found and seems populated with at least access_key, use it. + if config and config.get("access_key"): + return config + + # Otherwise, build from top-level standard keys + return { + "access_key": frappe.conf.get("s3_access_key"), + "secret": frappe.conf.get("s3_secret_key"), + "bucket": frappe.conf.get("s3_bucket"), + "region": frappe.conf.get("region"), + "endpoint_url": frappe.conf.get("endpoint_url"), + "folder": frappe.conf.get("s3_folder"), + "use_local": frappe.conf.get("use_local"), + "use_legacy_paths": frappe.conf.get("use_legacy_paths", True) + } + + @frappe.whitelist() def get_cloud_storage_client(): validate_config() - config: dict = frappe.conf.cloud_storage_settings + config: dict = get_cloud_storage_config() session = Session( aws_access_key_id=config.get("access_key"), aws_secret_access_key=config.get("secret"), @@ -462,7 +495,7 @@ def get_cloud_storage_client(): def validate_config() -> None: - config: dict = frappe.conf.cloud_storage_settings + config: dict = get_cloud_storage_config() if not config: frappe.throw( @@ -560,14 +593,23 @@ def get_file_path(file: File, folder: str | None = None) -> str: except Exception as e: frappe.log_error(f"Custom path generator failed: {str(e)}", "Cloud Storage Path Error") - config = frappe.conf.get("cloud_storage_settings", {}) + config = get_cloud_storage_config() if config.get("use_legacy_paths", True): - return _legacy_get_file_path(file, folder) + # Verify if this is a fresh install or if we want to enforce new paths even with legacy flag + # For Hygient/Zerodiscount: We enforce strict site segregation. + pass + # Standard Logic + path = file.file_name if folder: - return f"{folder}/{file.file_name}" + path = f"{folder}/{file.file_name}" - return file.file_name + # Enforce Site Segregation + # e.g. "site1.local/folder/filename.jpg" + if hasattr(frappe.local, "site") and frappe.local.site: + return f"{frappe.local.site}/{path}" + + return path def _legacy_get_file_path(file: File, folder: str | None = None) -> str: @@ -599,9 +641,8 @@ def get_file_content_hash(content, content_type): @frappe.whitelist() def write_file(file: File, remove_spaces_in_file_name: bool = True) -> File: - if not frappe.conf.cloud_storage_settings or frappe.conf.cloud_storage_settings.get( - "use_local", False - ): + config = get_cloud_storage_config() + if not config or config.get("use_local", False): file.save_file_on_filesystem() return file @@ -612,14 +653,24 @@ def write_file(file: File, remove_spaces_in_file_name: bool = True) -> File: # if a hash-conflict is found, update the existing document with a new file association existing_file_hashes = frappe.get_all( "File", - filters={"name": ["!=", file.name], "content_hash": file.content_hash}, + filters={ + "name": ["!=", file.name], + "content_hash": file.content_hash, + "s3_key": ["is", "set"], + }, pluck="name", ) if existing_file_hashes: file_doc: File = frappe.get_doc("File", existing_file_hashes[0]) + + # Propagate flags + if file.flags.ignore_links: file_doc.flags.ignore_links = True + if file.flags.ignore_permissions: file_doc.flags.ignore_permissions = True + if file.flags.ignore_validate: file_doc.flags.ignore_validate = True + file_doc.associate_files(file.attached_to_doctype, file.attached_to_name) - file_doc.save() + file_doc.save(ignore_permissions=True) return file_doc # if a filename-conflict is found, update the existing document with a new version instead @@ -636,6 +687,12 @@ def write_file(file: File, remove_spaces_in_file_name: bool = True) -> File: "content_type": file.content_type, } ) + + # Propagate flags + if file.flags.ignore_links: file_doc.flags.ignore_links = True + if file.flags.ignore_permissions: file_doc.flags.ignore_permissions = True + if file.flags.ignore_validate: file_doc.flags.ignore_validate = True + file_doc.associate_files(file.attached_to_doctype, file.attached_to_name) file = file_doc @@ -649,9 +706,8 @@ def write_file(file: File, remove_spaces_in_file_name: bool = True) -> File: @frappe.whitelist() def delete_file(file: File, **kwargs) -> File: - if not frappe.conf.cloud_storage_settings or frappe.conf.cloud_storage_settings.get( - "use_local", False - ): + config = get_cloud_storage_config() + if not config or config.get("use_local", False): file.delete_file_from_filesystem() return file diff --git a/cloud_storage/migration.py b/cloud_storage/migration.py index ec683d0..1852b9f 100644 --- a/cloud_storage/migration.py +++ b/cloud_storage/migration.py @@ -14,6 +14,8 @@ get_file_path, validate_config, write_file, + get_cloud_storage_config, + FILE_URL, ) @@ -41,7 +43,7 @@ def migrate_files( """ validate_config() - config = frappe.conf.get("cloud_storage_settings") + config = get_cloud_storage_config() if config.get("use_local"): frappe.throw( "Cloud Storage is not enabled. Please set 'use_local' to 0 in 'cloud_storage_settings'." @@ -109,6 +111,11 @@ def migrate_files( for file_data in batch: try: file_doc = frappe.get_doc("File", file_data.name) + if not file_doc.file_name: + print(f"⚠️ SKIP: {file_doc.name} - Missing file_name") + stats["skipped"] += 1 + continue + if not file_doc.is_private: file_path = frappe.get_site_path("public", file_doc.file_url.lstrip("/")) else: @@ -136,6 +143,28 @@ def migrate_files( print(f" Attached to: {attached_to}") print(f" Local path: {file_doc.file_url}") + if not dry_run: + client = get_cloud_storage_client() + config = get_cloud_storage_config() + expected_s3_key = get_file_path(file_doc, config.get("folder")) + + try: + client.head_object(Bucket=client.bucket, Key=expected_s3_key) + # If we reach here, file exists in S3. Sync DB and Skip Upload. + file_doc.db_set("s3_key", expected_s3_key) + file_doc.db_set("file_url", FILE_URL.format(path=expected_s3_key)) + print(f" ✅ Synced Record (Found in Cloud): {expected_s3_key}") + stats["migrated"] += 1 + + if remove_local and os.path.exists(file_path): + os.remove(file_path) + print(" 🗑️ Deleted local file (Synced)") + + continue + except ClientError: + # File not found in S3, proceed with upload + pass + if not dry_run: with open(file_path, "rb") as f: file_content = f.read() @@ -143,6 +172,10 @@ def migrate_files( file_doc.content = file_content content_type, _ = mimetypes.guess_type(file_doc.file_name) file_doc.content_type = content_type or "application/octet-stream" + + file_doc.flags.ignore_links = True + file_doc.flags.ignore_permissions = True + file_doc.flags.ignore_validate = True new_file = write_file(file_doc) new_file.reload() @@ -164,7 +197,9 @@ def migrate_files( print() except Exception as e: + import traceback print(f"❌ FAILED: {file_data.name} - {str(e)}") + traceback.print_exc() frappe.log_error( title=f"Cloud Storage Migration Failed: {file_data.name}", message=frappe.get_traceback() ) @@ -194,7 +229,7 @@ def migrate_files( def migrate_paths(dry_run=False, limit=None, batch_size=100): """Migrate files from legacy paths to new path strategy.""" client = get_cloud_storage_client() - config = frappe.conf.get("cloud_storage_settings", {}) + config = get_cloud_storage_config() if config.get("use_legacy_paths", True): print("⚠️ Legacy paths are still enabled in cloud_storage_settings.") diff --git a/cloud_storage/verify_storage.py b/cloud_storage/verify_storage.py new file mode 100644 index 0000000..7fff764 --- /dev/null +++ b/cloud_storage/verify_storage.py @@ -0,0 +1,91 @@ + +import frappe +import os +from cloud_storage.cloud_storage.overrides.file import get_cloud_storage_client, get_cloud_storage_config +from botocore.exceptions import ClientError + +def execute(): + print(f"\n{'='*50}") + print(f"Storage Verification for Site: {frappe.local.site}") + print(f"{'='*50}") + + # 1. Get All Files from DB + files = frappe.get_all( + "File", + fields=["name", "file_name", "file_url", "is_private", "s3_key", "is_folder"], + filters={"is_folder": 0} + ) + total_db_files = len(files) + print(f"Total Files in DB: {total_db_files}") + + config = get_cloud_storage_config() + if not config: + print("Cloud Storage not configured for this site.") + return + + try: + client = get_cloud_storage_client() + bucket = client.bucket + except Exception as e: + print(f"Failed to initialize MinIO client: {e}") + return + + local_count = 0 + minio_count = 0 + synced_count = 0 + missing_local = [] + missing_minio = [] + + print("\nScanning files...") + + for f in files: + # Check Local + if f.is_private: + local_path = frappe.get_site_path("private", "files", f.file_name) + else: + local_path = frappe.get_site_path("public", "files", f.file_name) + + if os.path.exists(local_path): + local_count += 1 + else: + missing_local.append(f.file_name) + + # Check MinIO + in_minio = False + if f.s3_key: + try: + client.head_object(Bucket=bucket, Key=f.s3_key) + in_minio = True + minio_count += 1 + synced_count += 1 + except ClientError: + missing_minio.append(f.file_name) + else: + # Not marked as migrated (s3_key is empty/null) + pass + + print(f"\n{'='*50}") + print(f"Verification Summary for {frappe.local.site}") + print(f"{'='*50}") + print(f"Total Records in DB: {total_db_files}") + print(f"Found Locally: {local_count}") + print(f"Found in MinIO (Verified): {minio_count}") + print(f"Marked as Migrated (DB): {len([f for f in files if f.s3_key])}") + print(f"{'='*50}") + + if missing_local: + print(f"\n[WARNING] Missing Local Files ({len(missing_local)}):") + print(", ".join(missing_local[:10]) + ("..." if len(missing_local) > 10 else "")) + + if missing_minio: + print(f"\n[ERROR] Missing in MinIO (DB claims migrated) ({len(missing_minio)}):") + print(", ".join(missing_minio[:10]) + ("..." if len(missing_minio) > 10 else "")) + + # Verification of count + if local_count != minio_count: + print(f"\n[INFO] Count Mismatch: Local ({local_count}) vs MinIO ({minio_count})") + print("Note: This is expected if partial migration or if 'remove_local' was used.") + else: + print("\n[SUCCESS] Local and MinIO counts match (for migrated files).") + + print(f"{'='*50}\n") diff --git a/package.json b/package.json index 643fdc0..78e3ad3 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "dependencies": { "docx-preview": "^0.3.0", "jszip": "^3.10.1", - "madr": "^3.0.0" + "madr": "^3.0.0", + "vue": "^3.3.4" }, "release": { "branches": [ @@ -19,4 +20,4 @@ "access": "restricted" }, "private": true -} +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d1caed0..3964c18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ [tool.poetry] name = "cloud_storage" -version = "15.9.1" +version = "1.1.1" description = "Frappe App for integrating with cloud storage applications" authors = ["AgriTheory "] readme = "README.md" diff --git a/yarn.lock b/yarn.lock deleted file mode 100644 index d291afa..0000000 --- a/yarn.lock +++ /dev/null @@ -1,97 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - -core-util-is@~1.0.0: - version "1.0.3" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" - integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== - -docx-preview@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/docx-preview/-/docx-preview-0.3.0.tgz#89c49b1b2eb6f74e973fe6ac36ad42f48b161e26" - integrity sha512-uLrLhytJkXe420Q0C/cyYuZYcKr+2bh7aXjwR/XclEIuGq9ifDRH1/GVqLBn7+ZX29CKC1bZovtjyfhETjdKKg== - dependencies: - jszip ">=3.0.0" - -immediate@~3.0.5: - version "3.0.6" - resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" - integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ== - -inherits@~2.0.3: - version "2.0.4" - resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" - integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== - -isarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ== - -jszip@>=3.0.0, jszip@^3.10.1: - version "3.10.1" - resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.1.tgz#34aee70eb18ea1faec2f589208a157d1feb091c2" - integrity sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g== - dependencies: - lie "~3.3.0" - pako "~1.0.2" - readable-stream "~2.3.6" - setimmediate "^1.0.5" - -lie@~3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a" - integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ== - dependencies: - immediate "~3.0.5" - -madr@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/madr/-/madr-3.0.0.tgz#19c3516590c88a716fc61f509dd6d379f2a0acf5" - integrity sha512-gb4ML7LGDARy2EY0WT5rxCUJcXvColIbNCeyoxcj3s/p1qdbpD3UsZrC7hkFkwOBukwnzx87Cu81HwTLvisTZg== - -pako@~1.0.2: - version "1.0.11" - resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" - integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== - -process-nextick-args@~2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" - integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== - -readable-stream@~2.3.6: - version "2.3.8" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b" - integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.3" - isarray "~1.0.0" - process-nextick-args "~2.0.0" - safe-buffer "~5.1.1" - string_decoder "~1.1.1" - util-deprecate "~1.0.1" - -safe-buffer@~5.1.0, safe-buffer@~5.1.1: - version "5.1.2" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" - integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== - -setimmediate@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" - integrity sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA== - -string_decoder@~1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" - integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== - dependencies: - safe-buffer "~5.1.0" - -util-deprecate@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" - integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==