Add Monthly Report export

.github/workflows/GenerateReport-Debug.yml

@@ -0,0 +1,97 @@
+name: Generate Report (Debug)
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - MonthlyReport
+    paths:
+      - '**.py'
+      - 'GenerateReport-Debug.yml'
+      - '.github/workflows/PythonSetup/**'
+      - '.github/workflows/RcloneSetup/**'
+      - '.env'
+      - 'src/**'
+
+permissions:
+  contents: write
+
+jobs:
+  healthcheck:
+    name: Health Check
+    runs-on: ubuntu-latest
+    environment: WT_WeeklyTriggerEnv
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: MonthlyReport
+
+      - name: Call setup-python
+        id: setup-python
+        uses: ./.github/workflows/PythonSetup
+
+      - name: DEBUG - Check pip health
+        run: |
+          pip --version
+          python -c "import pip._vendor.resolvelib; print(pip._vendor.resolvelib.__file__)"
+
+      - name: DEBUG - Print config.env
+        run: grep -E '^(FULL_RELOAD_PACKAGES|BASE_PACKAGE_CSV|REQUIREMENTS_FILE)=' .env || true
+
+  generate-report:
+    name: Generate Weekly Report
+    runs-on: ubuntu-latest
+    needs: healthcheck
+    environment: WT_WeeklyTriggerEnv
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: MonthlyReport
+
+      - name: Call setup-python
+        id: setup-python
+        uses: ./.github/workflows/PythonSetup
+
+      - name: DEBUG - Check pip health
+        run: |
+          pip --version
+          python -c "import pip._vendor.resolvelib; print(pip._vendor.resolvelib.__file__)"
+
+      - name: DEBUG - Print config.env
+        run: grep -E '^(FULL_RELOAD_PACKAGES|BASE_PACKAGE_CSV|REQUIREMENTS_FILE)=' .env || true
+
+      - name: Run weekly report script
+        run: python GenerateReport.py
+
+      - name: DEBUG - Show changed files
+        run: |
+          git status
+          git diff --name-only
+
+      - name: Commit and push changes
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          git add WeeklyReport/* || true
+          git commit -m "📝 Update WeeklyReport on $(date '+%Y-%m-%d %H:%M')" || echo "No changes in WeeklyReport"
+
+          git add MonthlyReport/* || true
+          git commit -m "📊 Update MonthlyReport on $(date '+%Y-%m-%d %H:%M')" || echo "No changes in MonthlyReport"
+
+          git add src/BasePackageWithDependencies.csv || true
+          git commit -m "📦 Update BasePackageWithDependencies on $(date '+%Y-%m-%d %H:%M')" || echo "No changes in BasePackageWithDependencies"
+
+          # Pull remote changes before pushing to avoid non-fast-forward errors
+          git fetch origin ${{ github.ref_name }}
+          git rebase origin/${{ github.ref_name }} || {
+            echo "❌ Rebase failed due to conflict. Please resolve manually."
+            exit 1
+          }
+
+          git push origin HEAD:${{ github.ref_name }} || {
+            echo "❌ Push failed. Someone else may have pushed changes. Please re-run the workflow."
+            exit 1
+          }

.github/workflows/GenerateReport.yml

@@ -4,14 +4,6 @@ on:
   schedule:
     - cron: '0 0 * * MON,THU'
   workflow_dispatch:
-  push:
-    branches:
-      - main
-    paths:
-      - '**.py'
-      - '**.yml'
-      - '.env'
-      - 'src/**'
 
 permissions:
   contents: write

GenerateReport.py

@@ -23,6 +23,7 @@
 from utils.ConfigUtils import(
     get_report_paths,
     get_report_output_folder,
+    get_monthly_report_output_folder,
     load_base_packages,
     parse_requirements
 )
@@ -84,6 +85,7 @@ def main() -> None:
     """
     paths = get_report_paths()
     report_dir = get_report_output_folder()
+    monthly_report_dir = get_monthly_report_output_folder()
 
     OUTPUT_CSV = paths["csv"]
     logger.debug(f"CSV Path: {OUTPUT_CSV}")
@@ -274,7 +276,55 @@ def main() -> None:
             logger.warning(f"⚠️ {len(failed_versions)} package versions failed vulnerability check. Saved to {OUTPUT_FAILED}.txt")
         except Exception as e:
             print(f"❌ Failed to write failed packages list: {e}")
-
+
+    # Monthly Report
+    import pandas as pd
+    from datetime import datetime
+    monthly_df = pd.DataFrame(rows)[[
+        'Package Name', 'Package Type', 'Custodian', 'Current Version',
+        'Dependencies for Current', 'Newer Versions', 'Dependencies for Latest',
+        'Latest Version', 'Current Version Vulnerable?', 'Current Version Vulnerability Details',
+        'Upgrade Version Vulnerable?', 'Upgrade Vulnerability Details',
+        'Suggested Upgrade', 'Remarks'
+    ]]
+    # Overview Sheet
+    total_packages = len(monthly_df)
+    base_count = (monthly_df['Package Type'] == 'Base Package').sum()
+    dep_count = total_packages - base_count
+    custodian_summary = monthly_df.groupby(['Custodian', 'Package Type']).size().unstack(fill_value=0)
+    custodian_summary['Total'] = custodian_summary.sum(axis=1)
+    overview_df = pd.DataFrame({
+        "Metric": ["Total Packages", "Base Packages", "Dependency Packages"],
+        "Count": [total_packages, base_count, dep_count]
+    })
+    # Custodian Sheet
+    custodian_raw_df = pd.read_csv(CUSTODIAN_LIST)
+    custodian_map_rev = {
+        "1": decode_base64_env("CUSTODIAN_1"),
+        "2": decode_base64_env("CUSTODIAN_2")
+    }
+    custodian_raw_df['Custodian'] = custodian_raw_df['Custodian'].astype(str).map(custodian_map_rev)
+
+    now = datetime.now().strftime("%Y%m-%d-%H%M")
+    monthly_file_path = os.path.join(monthly_report_dir, f"MonthlyReport-{now}.xlsx")
+    YearMonth = datetime.now().strftime("%Y%m")
+
+    # Write to Monthly Report
+    try:
+        with pd.ExcelWriter(monthly_file_path, engine='xlsxwriter') as writer:
+            overview_df.to_excel(writer, sheet_name='Overview', index=False)
+            custodian_summary.reset_index().to_excel(writer, sheet_name='Overview', startrow=5, index=False)
+            custodian_raw_df.to_excel(writer, sheet_name='Custodian', index=False)
+            monthly_df.to_excel(writer, sheet_name=f'Monthly Report - {YearMonth}', index=False)
+            # Hide specific columns in Excel
+            worksheet = writer.sheets[f'Monthly Report - {YearMonth}']
+            col_indices = monthly_df.columns.get_indexer(['Dependencies for Current', 'Dependencies for Latest'])
+            for col_idx in col_indices:
+                worksheet.set_column(col_idx, col_idx, None, None, {'hidden': True})
+        print(f"\U0001F4C4 Monthly Excel report saved to {monthly_file_path}")
+    except Exception as e:
+        print(f"\u274C Failed to write monthly Excel report: {e}")
+
     # Summary logging
     total = len(rows)
     base_count = sum(1 for r in rows if r['Package Type'] == 'Base Package')
@@ -294,4 +344,4 @@ def main() -> None:
         main()
     except KeyboardInterrupt:
         print("\n❌ Execution interrupted by user.")
-        sys.exit(1)
+        sys.exit(1)

WeeklyReport/2025-06-02/WeeklyReport_20250602_165317.csv

@@ -0,0 +1,6 @@
+Package Name,Package Type,Custodian,Current Version,Dependencies for Current,Newer Versions,Dependencies for Latest,Latest Version,Current Version Vulnerable?,Current Version Vulnerability Details,Upgrade Version Vulnerable?,Upgrade Vulnerability Details,Suggested Upgrade,Upgrade Instruction,Remarks
+adlfs,Base Package,EY,2024.4.1,"azure-core<2.0.0,>=1.28.0; azure-datalake-store<0.1,>=0.0.53; azure-identity; azure-storage-blob>=12.17.0; fsspec>=2023.12.0; aiohttp>=3.7.0; sphinx; extra == ""docs""; myst-parser; extra == ""docs""; furo; extra == ""docs""; numpydoc; extra == ""docs""; pytest; extra == ""tests""; docker; extra == ""tests""; pytest-mock; extra == ""tests""; arrow; extra == ""tests""; dask[dataframe]; extra == ""tests""","2024.7.0, 2024.12.0","azure-core<2.0.0,>=1.28.0; azure-datalake-store<0.1,>=0.0.53; azure-identity; azure-storage-blob>=12.17.0; fsspec>=2023.12.0; aiohttp>=3.7.0; sphinx; extra == ""docs""; myst-parser; extra == ""docs""; furo; extra == ""docs""; numpydoc; extra == ""docs""; pytest; extra == ""tests""; docker; extra == ""tests""; pytest-mock; extra == ""tests""; arrow; extra == ""tests""; dask[dataframe]; extra == ""tests""",2024.12.0,No,,No,None,2024.12.0,"{'base_package': 'adlfs==2024.12.0', 'dependencies': ['azure-core==1.34.0', 'azure-datalake-store==0.0.53', 'azure-identity==1.23.0', 'azure-storage-blob==12.26.0b1', 'fsspec==2023.12.2', 'aiohttp==3.12.6', 'sphinx==8.3.0', 'myst-parser==4.0.1', 'furo==2024.8.6', 'numpydoc==1.8.0', 'pytest==8.3.5', 'docker==7.1.0', 'pytest-mock==3.14.1', 'arrow==1.3.0', 'dask==2025.5.1']}",
+adal,Dependency Package,EY,1.2.7,"PyJWT (<3,>=1.0.0); requests (<3,>=2.0.0); python-dateutil (<3,>=2.1.0); cryptography (>=1.1.0)",,"PyJWT (<3,>=1.0.0); requests (<3,>=2.0.0); python-dateutil (<3,>=2.1.0); cryptography (>=1.1.0)",1.2.7,No,,No,None,1.2.7,"{'base_package': 'adal==1.2.7', 'dependencies': ['requests==2.32.3', 'python-dateutil==2.9.0.post0']}",
+aiofiles,Dependency Package,EY,24.1.0,,,,24.1.0,No,,No,None,24.1.0,"{'base_package': 'aiofiles==24.1.0', 'dependencies': []}",
+aiohappyeyeballs,Dependency Package,EY,2.4.6,,"2.4.7, 2.4.8, 2.5.0, 2.6.0, 2.6.1",,2.6.1,No,,No,None,2.6.1,"{'base_package': 'aiohappyeyeballs==2.6.1', 'dependencies': []}",
+absl-py,Dependency Package,I&S,2.1.0,,"2.2.0, 2.2.1, 2.2.2, 2.3.0",,2.3.0,No,,No,None,2.3.0,"{'base_package': 'absl-py==2.3.0', 'dependencies': []}",