diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0b576ad --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,100 @@ +name: CI + +on: + push: + branches: [ master, main ] + pull_request: + branches: [ master, main ] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + pip install -r requirements.txt + - name: Run pylint + run: | + pylint --recursive=y . + - name: Markdown Lint + uses: nosborn/github-action-markdown-cli@v3.3.0 + with: + files: . + config_file: .markdownlint.json + + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.x"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Run tests + run: | + python -m unittest discover tests + + build-docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build the Docker image + run: docker build . --file Dockerfile --tag my-image-name:$(date +%s) + + build-exe-windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pyinstaller + pip install -r requirements.txt + - name: Build EXE + run: | + pyinstaller --onefile import-mailbox-to-gmail.py + - name: Verify EXE runs + run: | + ./dist/import-mailbox-to-gmail.exe --help + + build-exe-macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pyinstaller + pip install -r requirements.txt + - name: Build EXE + run: | + export TOOL_pyinstaller=$(which pyinstaller) + export TOOL_pyi_makespec=$(which pyi-makespec) + chmod +x build-exe.mac.sh + ./build-exe.mac.sh + - name: Verify EXE runs + run: | + ./build/exe/macos/import-mailbox-to-gmail --help diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..28dd2bb --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,58 @@ +name: Release + +on: + release: + types: [created] + workflow_dispatch: + +jobs: + build-exe-windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Build EXE + run: | + pyinstaller --onefile import-mailbox-to-gmail.py + - name: Verify EXE runs + run: | + ./dist/import-mailbox-to-gmail.exe --help + - name: Upload Windows Binary + uses: actions/upload-artifact@v4 + with: + name: import-mailbox-to-gmail-windows + path: dist/import-mailbox-to-gmail.exe + + build-exe-macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Build EXE + run: | + export TOOL_pyinstaller=$(which pyinstaller) + export TOOL_pyi_makespec=$(which pyi-makespec) + chmod +x build-exe.mac.sh + ./build-exe.mac.sh + - name: Verify EXE runs + run: | + ./build/exe/macos/import-mailbox-to-gmail --help + - name: Upload macOS Binary + uses: actions/upload-artifact@v4 + with: + name: import-mailbox-to-gmail-macos + path: build/exe/macos/import-mailbox-to-gmail diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..147c64c --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +build/ +dist/ +*.spec diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 0000000..c3af26a --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,8 @@ +{ + "default": true, + "MD013": { + "line_length": 80, + "code_blocks": false, + "tables": false + } +} diff --git a/.markdownlintignore b/.markdownlintignore new file mode 100644 index 0000000..a1892e3 --- /dev/null +++ b/.markdownlintignore @@ -0,0 +1 @@ +CONTRIBUTING.md diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..f4df598 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,5 @@ +[BASIC] +good-names=import-mailbox-to-gmail + +[FORMAT] +indent-string=' ' diff --git a/Dockerfile b/Dockerfile index 0cf32b2..4d27779 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2 +FROM python:3-slim WORKDIR /usr/src/app COPY requirements.txt ./ @@ -7,4 +7,4 @@ RUN pip install --no-cache-dir -r requirements.txt COPY import-mailbox-to-gmail.py . ENTRYPOINT [ "python", "import-mailbox-to-gmail.py" ] -CMD [ "-h" ] +CMD [ "--help" ] diff --git a/README.md b/README.md index ce19f21..e62d758 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Import .mbox files to Google Workspace (formerly G Suite / Google Apps) -This script allows Google Workspace admins to import mbox files in bulk for their -users. +This script allows Google Workspace admins to import mbox files in bulk for +their users. **DISCLAIMER**: This is not an official Google product. @@ -11,95 +11,74 @@ If you want to migrate from Mozilla Thunderbird, try You only authorize it once using a service account, and then it can import mail into the mailboxes of all users in your domain. -### A. Creating and authorizing a service account for Gmail API +## A. Creating and authorizing a service account for Gmail API -1. Go to the [Developers Console](https://console.developers.google.com/project) - and log in as a domain super administrator. +The easiest way is to +[use the automated script to authorize GWMME](https://support.google.com/a/answer/6291304#script). +The resulting JSON service account key file will work with this script as well. +It will allow more API scopes than are needed, so you might want to remove them +after it's created and verified. -2. Create a new project. +If you don't want to use the automated script, you can follow the manual +instructions on the same page, but you'll only need to enable Gmail API +(not all of the other APIs), and only the two Gmail scopes: - * If you have not used the API console before, select **Create a project** from - the **Select a project** dropdown list. - * If this is not your first project, use the **Create Project** button. +```text +https://www.googleapis.com/auth/gmail.insert, https://www.googleapis.com/auth/gmail.labels +``` -3. Enter "Gmail API" (or any name you prefer) as the project name and press the - **Create** button. If this is your first project you must agree to the Terms of - Service at this point. +At the end of either option, you will have a JSON service account key file, +that you can use to authorize programs to access the Gmail API "insert" and +"label" scopes of all users in your Google Workspace domain. -4. Click the **Enable and manage APIs** link in the **Use Google APIs** box. +**Remember to store this key safely, and don't share it with anyone.** -5. Enable the Gmail API - Select the **Gmail API** link and press the **Enable - API** button. You can leave the default APIs enabled - it doesn't matter. +## B. Importing mbox files -6. Click the 3-line icon (**≡**) in the top left corner of the console. +**Important**: If you're planning to import mail from Apple Mail.app, see the +notes below. -7. Click **IAM & Admin** and select **Service accounts**. +You can either run the pre-compiled executable (easiest) or run the Python +script directly. -8. Click **Create service account**. +### Option 1: Using the executable (Recommended for Windows) -9. Enter a name (for example, "import-mailbox") in the **Name** field. +1. Download `import-mailbox-to-gmail.exe` from + [the latest release](https://github.com/google/import-mailbox-to-gmail/releases/latest). + **Note**: Executables are provided for Windows only. macOS and Linux users + should use Option 2. -10. Check the **Furnish a new private key** box and ensure the key type is set - to JSON. +2. Open a **Command Prompt** (CMD) window. -11. Check the **Enable G Suite Domain-wide Delegation** box and enter a name - in the **Product name for the consent screen** field. +3. Create a folder for the mbox files, for example `C:\mbox` (see step 5 below). -12. Click **Create**. You will see a confirmation message advising that the - Service account JSON file has been downloaded to your computer. Make a note - of the location and name of this file. **This JSON file contains a private - key that potentially allows access to all users in your domain. Protect it - like you'd protect your admin password. Don't share it with anyone.** +4. Follow steps 6-8 below, replacing `python import-mailbox-to-gmail.py` with + the path to your downloaded executable (usually + `%USERPROFILE%\Downloads\import-mailbox-to-gmail.exe`). -13. Click **Close**. +### Option 2: Running the Python script -14. Click the **View Client ID** link in the **Options** column. +1. Download the script `import-mailbox-to-gmail.py` from + [the latest release](https://github.com/google/import-mailbox-to-gmail/releases/latest). -15. Copy the **Client ID** value. You will need this later. - -16. Go to [the **Domain-wide Delegation** page of the Admin console for your - Google Workspace domain](https://admin.google.com/ac/owl/domainwidedelegation). - -17. Under **Client ID**, enter the Client ID collected in step 15. - -18. Under **OAuth Scopes**, enter the following: - ``` - https://www.googleapis.com/auth/gmail.insert, https://www.googleapis.com/auth/gmail.labels - ``` -19. Click **Authorize**. - -You can now use the JSON file to authorize programs to access the Gmail API -"insert" and "label" scopes of all users in your Google Workspace domain. - -### B. Importing mbox files using import-mailbox-to-gmail.py - -**Important**: If you're planning to import mail from Apple Mail.app, see the notes below. - -1. Download the script - [import-mailbox-to-gmail.py](https://github.com/google/import-mailbox-to-gmail/releases/download/v1.5/import-mailbox-to-gmail.py). - -2. [Download](https://www.python.org/downloads/) and install Python 2.7 (not - Python 3.x) for your operating system if needed. +2. [Download](https://www.python.org/downloads/) and install Python 3 (latest + version) for your operating system if needed. 3. Open a **Command Prompt** (CMD) window (on Windows) / **Terminal** window - (on Linux). - -4. Install the Google API Client Libraries for Python and their dependencies by - running, all in one line: + (on macOS/Linux). - Mac/Linux: - ``` - sudo pip install --upgrade google-api-python-client PyOpenSSL - ``` +4. Install the Google API Client Libraries for Python and their dependencies. + Ensure you have a `requirements.txt` file (you can download it from the repo) + in the same directory, then run: - Windows: - ``` - C:\Python27\Scripts\pip install --upgrade google-api-python-client PyOpenSSL + ```bash + pip3 install -r requirements.txt ``` **Note**: On Windows, you may need to do this on a Command Prompt window that was run as Administrator. -5. Create a folder for the mbox files, for example `C:\mbox`. +5. Create a folder for the mbox files, for example `C:\mbox` or `~/mbox`. 6. Under that folder, create a folder for each of the users into which you intend to import the mbox files. The folder names should be the users' full @@ -110,66 +89,90 @@ You can now use the JSON file to authorize programs to access the Gmail API messages to go into a label called "Imported messages", name the file "Imported messages.mbox". - Your final folder and file structure should look like this (for example): - ``` - C:\mbox - C:\mbox\user1@domain.com - C:\mbox\user1@domain.com\Imported messages.mbox - C:\mbox\user1@domain.com\Other imported messages.mbox - C:\mbox\user2@domain.com - C:\mbox\user2@domain.com\Imported messages.mbox - C:\mbox\user2@domain.com\Other imported messages.mbox - ``` - - IMPORTANT: It's essential to test the migration before migrating into the real - users' mailboxes. First, migrate the mbox files into a test user, to make sure - the messages are imported correctly. + Your final folder and file structure should look like this (for example): + + ```text + C:\mbox + C:\mbox\user1@domain.com + C:\mbox\user1@domain.com\Imported messages.mbox + C:\mbox\user1@domain.com\Other imported messages.mbox + C:\mbox\user2@domain.com + C:\mbox\user2@domain.com\Imported messages.mbox + C:\mbox\user2@domain.com\Other imported messages.mbox + ``` + + IMPORTANT: It's essential to test the migration before migrating into the + real users' mailboxes. First, migrate the mbox files into a test user, to + make sure the messages are imported correctly. 8. To start the migration, run the following command (one line): - Mac/Linux: - ``` - python import-mailbox-to-gmail.py --json Credentials.json --dir C:\mbox + macOS/Linux: + + ```bash + python3 import-mailbox-to-gmail.py --json Credentials.json --dir ~/mbox ``` Windows: - ``` - C:\Python27\python import-mailbox-to-gmail.py --json Credentials.json --dir C:\mbox + + ```cmd + python import-mailbox-to-gmail.py --json Credentials.json --dir C:\mbox ``` - * Replace `import-mailbox-to-gmail.py` with the full path of import-mailbox-to-gmail.py - - usually `~/Downloads/import-mailbox-to-gmail.py` on Mac/Linux or - `%USERPROFILE%\Downloads\import-mailbox-to-gmail.py` on Windows. - * Replace `Credentials.json` with the path to the JSON file from step 12 - above. - * Replace `C:\mbox` with the path to the folder you created in step 5. + * Replace `import-mailbox-to-gmail.py` with the full path of + import-mailbox-to-gmail.py - usually + `~/Downloads/import-mailbox-to-gmail.py` on Mac/Linux or + `%USERPROFILE%\Downloads\import-mailbox-to-gmail.py` on Windows. + * Replace `Credentials.json` with the path of the JSON service account key + file created in the previous step. + * Replace `C:\mbox` with the path to the folder you created in step 5. The mbox files will now be imported, one by one, into the users' mailboxes. You can monitor the migration by looking at the output, and inspect errors by viewing the `import-mailbox-to-gmail.log` file. -### Options and notes +## Options and notes + +* Use the `--from_message` parameter to start the upload from a particular + message. This allows you to resume an upload if the process previously + stopped. (Affects _all_ users and _all_ mbox files) -* Use the `--from_message` parameter to start the upload from a particular message. - This allows you to resume an upload if the process previously stopped. (Affects - _all_ users and _all_ mbox files) + e.g. `python3 import-mailbox-to-gmail.py --from_message 74336` + +* If any of the folders have a ".mbox" extension, it will be dropped when + creating the label for it in Gmail. + +* To import mail from Apple Mail.app, make sure you export it first - the raw + Apple Mail files can't be imported. You can export a folder by right clicking + it in Apple Mail and choosing "Export Mailbox". + +* This script can import nested folders. In order to do so, it is necessary to + preserve the email folders' hierarchy when exporting them as mbox files. In + Apple Mail.app, this can be done by expanding all subfolders, selecting both + parents and subfolders at the same time, and exporting them by right clicking + the selection and choosing "Export Mailbox". + +* If any of the folders have a ".mbox" extension and a file named "mbox" in + them, the contents of the "mbox" file will be imported to the label named as + the folder. This is how Apple Mail exports are structured. - e.g. `./import-mailbox-to-gmail.py --from_message 74336` -* If any of the folders have a ".mbox" extension, it will be dropped when creating the label for it in Gmail. -* To import mail from Apple Mail.app, make sure you export it first - the raw Apple Mail files can't be imported. You can export a folder by right clicking it in Apple Mail and choosing "Export Mailbox". -* This script can import nested folders. In order to do so, it is necessary to preserve the email folders' hierarchy when exporting them as mbox files. In Apple Mail.app, this can be done by expanding all subfolders, selecting both parents and subfolders at the same time, and exporting them by right clicking the selection and choosing "Export Mailbox". -* If any of the folders have a ".mbox" extension and a file named "mbox" in them, the contents of the "mbox" file will be imported to the label named as the folder. This is how Apple Mail exports are structured. * To run under [Docker](https://www.docker.com/): - 1. Build the image: - ``` - docker build -t google/import-mailbox-to-gmail . - ``` - 2. Run the import command: - ``` - docker run --rm -it \ - -v "/local/path/to/auth.json:/auth.json" \ - -v "/local/path/to/mbox:/mbox" \ - google/import-mailbox-to-gmail --json "/auth.json" --dir "/mbox" - ``` - - **Note** `-v` is mounting a local file/directory */local/path/to/auth.json* in the container as `/auth.json`. The command is then using it within the container `--json "/auth.json"`. For more help, see [Volume in Docker Run](https://docs.docker.com/engine/reference/commandline/run/#mount-volume--v---read-only). + 1. Build the image: + + ```bash + docker build -t import-mailbox-to-gmail https://github.com/google/import-mailbox-to-gmail.git + ``` + + 2. Run the import command: + + ```bash + docker run --rm -it \ + -v "/local/path/to/auth.json:/auth.json" \ + -v "/local/path/to/mbox:/mbox" \ + import-mailbox-to-gmail --json "/auth.json" --dir "/mbox" + ``` + + **Note** `-v` is mounting a local file/directory _/local/path/to/auth.json_ + in the container as `/auth.json`. The command is then using it within the + container `--json "/auth.json"`. For more help, see + [Volume in Docker Run](https://docs.docker.com/engine/reference/commandline/run/#mount-volume--v---read-only). diff --git a/build-exe.mac.sh b/build-exe.mac.sh index f064e74..028a283 100755 --- a/build-exe.mac.sh +++ b/build-exe.mac.sh @@ -24,7 +24,7 @@ if [[ -z "${TOOL_pyinstaller}" || -z "${TOOL_pyi_makespec}" ]]; then exit 1 fi -python2 \ +python3 \ "${TOOL_pyi_makespec}" \ --name "${NAME}" \ --specpath "${BUILD_DIR}" \ @@ -38,8 +38,8 @@ if [[ "${_exit_code}" -ne 0 ]]; then echo "Spec file generation failed" >&2 exit ${_exit_code} fi - -python2 \ + +python3 \ "${TOOL_pyinstaller}" \ --noconfirm \ --clean \ @@ -52,4 +52,3 @@ if [[ "${_exit_code}" -ne 0 ]]; then echo "Pyinstaller invocation failed" >&2 exit ${_exit_code} fi - diff --git a/build.py b/build.py new file mode 100644 index 0000000..5332993 --- /dev/null +++ b/build.py @@ -0,0 +1,38 @@ +"""Build and test script for import-mailbox-to-gmail.""" +import unittest +import subprocess +import sys + +def run_tests(): + """Runs the unit tests.""" + loader = unittest.TestLoader() + suite = loader.discover('tests', top_level_dir='.') + runner = unittest.TextTestRunner() + result = runner.run(suite) + return len(result.failures) == 0 and len(result.errors) == 0 + +def create_executable(): + """Creates the executable using PyInstaller.""" + try: + import PyInstaller # pylint: disable=unused-import,import-outside-toplevel + except ImportError: + print("PyInstaller not found, installing...") + try: + subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'pyinstaller']) + except subprocess.CalledProcessError as e: + print(f"Failed to install PyInstaller: {e}") + sys.exit(1) + + print("Running PyInstaller...") + subprocess.run( + [sys.executable, '-m', 'PyInstaller', '--onefile', 'import-mailbox-to-gmail.py'], + check=True + ) + +if __name__ == '__main__': + if run_tests(): + print("Tests passed, creating executable...") + create_executable() + else: + print("Tests failed, not creating executable.") + sys.exit(1) diff --git a/import-mailbox-to-gmail.py b/import-mailbox-to-gmail.py index 5d6ad6b..c3f2fb2 100755 --- a/import-mailbox-to-gmail.py +++ b/import-mailbox-to-gmail.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- """Import mbox files to a specified label for many users. @@ -20,25 +20,23 @@ """ import argparse -import base64 import io -import json import logging import logging.handlers import mailbox import os import sys -from apiclient import discovery -from apiclient.http import set_user_agent import httplib2 -from apiclient.http import MediaIoBaseUpload -from oauth2client.service_account import ServiceAccountCredentials -import oauth2client.tools -import OpenSSL # Required by Google API library, but not checked by it +import google.auth +from google.oauth2 import service_account +from google_auth_httplib2 import AuthorizedHttp +from googleapiclient import discovery +from googleapiclient.http import MediaIoBaseUpload +from googleapiclient.http import set_user_agent APPLICATION_NAME = 'import-mailbox-to-gmail' -APPLICATION_VERSION = '1.5' +APPLICATION_VERSION = '2.0' SCOPES = ['https://www.googleapis.com/auth/gmail.insert', 'https://www.googleapis.com/auth/gmail.labels'] @@ -47,7 +45,6 @@ parser = argparse.ArgumentParser( description='Import mbox files to a specified label for many users.', formatter_class=argparse.RawDescriptionHelpFormatter, - parents=[oauth2client.tools.argparser], epilog= """ * The directory needs to have a subdirectory for each user (with the full @@ -62,10 +59,13 @@ * See the README at https://goo.gl/JnFt0x for more usage information. """) -parser.add_argument( +group = parser.add_mutually_exclusive_group(required=True) +group.add_argument( '--json', - required=True, help='Path to the JSON key file from https://console.developers.google.com') +group.add_argument( + '--service_account_email', + help='The email address of the service account to use for signing JWTs') parser.add_argument( '--dir', required=True, @@ -97,15 +97,10 @@ parser.add_argument( '--log', required=False, - default='%s-%d.log' % (APPLICATION_NAME, os.getpid()), + default=f'{APPLICATION_NAME}-{os.getpid()}.log', help= - 'Optional: Path to a the log file (default: %s-####.log in the current ' - 'directory, where #### is the process ID)' % APPLICATION_NAME) -parser.add_argument( - '--httplib2debuglevel', - default=0, - type=int, - help='Debug level of the HTTP library: 0=None (default), 4=Maximum.') + f'Optional: Path to a the log file (default: {APPLICATION_NAME}-####.log in the current ' + 'directory, where #### is the process ID)') parser.add_argument( '--from_message', default=0, @@ -113,9 +108,14 @@ help= 'Message number to resume from, affects ALL users and ALL ' 'mbox files (default: 0)') +parser.add_argument( + '--httplib2debuglevel', + default=0, + type=int, + help='Debug level of the HTTP library: 0=None (default), 4=Maximum.') parser.set_defaults(fix_msgid=True, replace_quoted_printable=True, logging_level='INFO') -args = parser.parse_args() +ARGS = None def get_credentials(username): @@ -126,9 +126,19 @@ def get_credentials(username): Returns: Credentials, the obtained credential. """ - credentials = ServiceAccountCredentials.from_json_keyfile_name( - args.json, - scopes=SCOPES).create_delegated(username) + if ARGS.json: + credentials = service_account.Credentials.from_service_account_file( + ARGS.json, + scopes=SCOPES, + subject=username) + else: + # Use Application Default Credentials to sign a JWT + source_credentials, _ = google.auth.default(scopes=SCOPES) + credentials = google.auth.impersonated_credentials.Credentials( + source_credentials=source_credentials, + target_principal=ARGS.service_account_email, + target_scopes=SCOPES, + subject=username) return credentials @@ -151,16 +161,110 @@ def get_label_id_from_name(service, username, labels, labelname): } label = service.users().labels().create( userId=username, - body=label_object).execute(num_retries=args.num_retries) + body=label_object).execute(num_retries=ARGS.num_retries) logging.info("Label '%s' created", labelname) labels.append(label) return label['id'] - except Exception: + except Exception: # pylint: disable=broad-exception-caught logging.exception("Can't create label '%s' for user %s", labelname, username) raise -def process_mbox_files(username, service, labels): +def import_message(service, username, message, label_id): + """Imports a single message to Gmail.""" + metadata_object = {'labelIds': [label_id]} + try: + # Use media upload to allow messages more than 5mb. + # See https://developers.google.com/api-client-library/python/guide/media_upload + # and http://google-api-python-client.googlecode.com/hg/docs/epy/ + # apiclient.http.MediaIoBaseUpload-class.html. + message_data = io.BytesIO(message.as_string().encode('utf-8')) + media = MediaIoBaseUpload(message_data, mimetype='message/rfc822') + message_response = service.users().messages().import_( + userId=username, + fields='id', + neverMarkSpam=True, + processForCalendar=False, + internalDateSource='dateHeader', + body=metadata_object, + media_body=media).execute(num_retries=ARGS.num_retries) + logging.debug("Imported mbox message '%s' to Gmail ID %s", + message.get_from(), + message_response['id']) + return True + except Exception: # pylint: disable=broad-exception-caught + logging.exception('Failed to import mbox message') + return False + + +def process_message_headers(message): + """Fixes message headers.""" + try: + if (ARGS.replace_quoted_printable and + 'Content-Type' in message and + 'text/quoted-printable' in message['Content-Type']): + message.replace_header( + 'Content-Type', message['Content-Type'].replace( + 'text/quoted-printable', 'text/plain')) + logging.info('Replaced text/quoted-printable with text/plain') + except Exception: # pylint: disable=broad-exception-caught + logging.exception( + 'Failed to replace text/quoted-printable with text/plain ' + 'in Content-Type header') + + try: + if ARGS.fix_msgid and 'Message-ID' in message: + msgid = message['Message-ID'] + if msgid[0] != '<': + msgid = '<' + msgid + logging.info('Added < to Message-ID: %s', msgid) + if msgid[-1] != '>': + msgid += '>' + logging.info('Added > to Message-ID: %s', msgid) + message.replace_header('Message-ID', msgid) + except Exception: # pylint: disable=broad-exception-caught + logging.exception('Failed to fix brackets in Message-ID header') + + +def process_mbox_file(full_filename, labelname, service, username, labels): + """Imports all messages from a single mbox file.""" + try: + label_id = get_label_id_from_name(service, username, labels, labelname) + except Exception: # pylint: disable=broad-exception-caught + logging.error("Skipping label '%s' because it can't be created", labelname) + return None + + logging.info("Using label name '%s', ID '%s'", labelname, label_id) + + number_of_successes_in_label = 0 + number_of_failures_in_label = 0 + + mbox = mailbox.mbox(full_filename) + try: + for index, message in enumerate(mbox): + if index < ARGS.from_message: + continue + logging.info("Processing message %d in label '%s'", index, labelname) + + process_message_headers(message) + + if import_message(service, username, message, label_id): + number_of_successes_in_label += 1 + else: + number_of_failures_in_label += 1 + finally: + mbox.close() + + logging.info("Finished processing '%s'. %d messages imported " + "successfully, %d messages failed.", + full_filename, + number_of_successes_in_label, + number_of_failures_in_label) + + return number_of_successes_in_label, number_of_failures_in_label + + +def process_mbox_files(username, service, labels): # pylint: disable=too-many-locals """Iterates over the mbox files found in the user's subdir and imports them. Args: @@ -179,18 +283,18 @@ def process_mbox_files(username, service, labels): number_of_labels_failed = 0 number_of_messages_imported_without_error = 0 number_of_messages_failed = 0 - base_path = os.path.join(args.dir, username) + base_path = os.path.join(ARGS.dir, username) for root, dirs, files in os.walk(base_path): - for dir in dirs: + for dirname in dirs: try: - labelname = os.path.join(root[len(base_path) + 1:], dir) + labelname = os.path.join(root[len(base_path) + 1:], dirname) get_label_id_from_name(service, username, labels, labelname) - except Exception: - logging.error("Labels under '%s' may not nest correctly", dir) + except Exception: # pylint: disable=broad-exception-caught + logging.error("Labels under '%s' may not nest correctly", dirname) for file in files: filename = root[len(base_path) + 1:] if filename: - filename += u'/' + filename += '/' filename += file labelname, ext = os.path.splitext(filename) full_filename = os.path.join(root, file) @@ -212,81 +316,24 @@ def process_mbox_files(username, service, labels): full_filename += os.path.join(full_filename, 'mbox') logging.info("Using '%s' instead of the directory", full_filename) logging.info("Starting processing of '%s'", full_filename) - number_of_successes_in_label = 0 - number_of_failures_in_label = 0 - mbox = mailbox.mbox(full_filename) - try: - label_id = get_label_id_from_name(service, username, labels, labelname) - except Exception: - logging.error("Skipping label '%s' because it can't be created", labelname) + + result = process_mbox_file( + full_filename, labelname, service, username, labels) + + if result is None: + number_of_labels_failed += 1 continue - logging.info("Using label name '%s', ID '%s'", labelname, label_id) - for index, message in enumerate(mbox): - if index < args.from_message: - continue - logging.info("Processing message %d in label '%s'", index, labelname) - try: - if (args.replace_quoted_printable and - 'Content-Type' in message and - 'text/quoted-printable' in message['Content-Type']): - message.replace_header( - 'Content-Type', message['Content-Type'].replace( - 'text/quoted-printable', 'text/plain')) - logging.info('Replaced text/quoted-printable with text/plain') - except Exception: - logging.exception( - 'Failed to replace text/quoted-printable with text/plain ' - 'in Content-Type header') - try: - if args.fix_msgid and 'Message-ID' in message: - msgid = message['Message-ID'] - if msgid[0] != '<': - msgid = '<' + msgid - logging.info('Added < to Message-ID: %s', msgid) - if msgid[-1] != '>': - msgid += '>' - logging.info('Added > to Message-ID: %s', msgid) - message.replace_header('Message-ID', msgid) - except Exception: - logging.exception('Failed to fix brackets in Message-ID header') - metadata_object = {'labelIds': [label_id]} - try: - # Use media upload to allow messages more than 5mb. - # See https://developers.google.com/api-client-library/python/guide/media_upload - # and http://google-api-python-client.googlecode.com/hg/docs/epy/apiclient.http.MediaIoBaseUpload-class.html. - if sys.version_info.major == 2: - message_data = io.BytesIO(message.as_string()) - else: - message_data = io.StringIO(message.as_string()) - media = MediaIoBaseUpload(message_data, mimetype='message/rfc822') - message_response = service.users().messages().import_( - userId=username, - fields='id', - neverMarkSpam=True, - processForCalendar=False, - internalDateSource='dateHeader', - body=metadata_object, - media_body=media).execute(num_retries=args.num_retries) - number_of_successes_in_label += 1 - logging.debug("Imported mbox message '%s' to Gmail ID %s", - message.get_from(), - message_response['id']) - except Exception: - number_of_failures_in_label += 1 - logging.exception('Failed to import mbox message') - logging.info("Finished processing '%s'. %d messages imported " - "successfully, %d messages failed.", - full_filename, - number_of_successes_in_label, - number_of_failures_in_label) - if number_of_failures_in_label == 0: + + successes, failures = result + + if failures == 0: number_of_labels_imported_without_error += 1 - elif number_of_successes_in_label > 0: + elif successes > 0: number_of_labels_imported_with_some_errors += 1 else: number_of_labels_failed += 1 - number_of_messages_imported_without_error += number_of_successes_in_label - number_of_messages_failed += number_of_failures_in_label + number_of_messages_imported_without_error += successes + number_of_messages_failed += failures return (number_of_labels_imported_without_error, # 0 number_of_labels_imported_with_some_errors, # 1 number_of_labels_failed, # 2 @@ -294,16 +341,11 @@ def process_mbox_files(username, service, labels): number_of_messages_failed) # 4 -def main(): - """Import multiple users' mbox files to Gmail. - - """ - httplib2.debuglevel = args.httplib2debuglevel - # Use args.logging_level if defined. - try: - logging_level = args.logging_level - except AttributeError: - logging_level = 'INFO' +def setup_logging(): + """Configures logging.""" + httplib2.debuglevel = ARGS.httplib2debuglevel + # Use ARGS.logging_level if defined. + logging_level = getattr(ARGS, 'logging_level', 'INFO') # Default logging to standard output logging.basicConfig( @@ -312,7 +354,7 @@ def main(): datefmt='%H:%M:%S') # More detailed logging to file - file_handler = logging.handlers.RotatingFileHandler(args.log, + file_handler = logging.handlers.RotatingFileHandler(ARGS.log, maxBytes=1024 * 1024 * 32, backupCount=8) file_formatter = logging.Formatter( @@ -327,9 +369,58 @@ def main(): APPLICATION_VERSION, sys.version) logging.info('Arguments:') - for arg, value in sorted(vars(args).items()): + for arg, value in sorted(vars(ARGS).items()): logging.info('\t%s: %r', arg, value) + +def process_user(username): + """Imports for a single user. Returns stats tuple.""" + try: + logging.info('Processing user %s', username) + try: + credentials = get_credentials(username) + http = set_user_agent( + httplib2.Http(), + f'{APPLICATION_NAME}-{APPLICATION_VERSION}') + authed_http = AuthorizedHttp(credentials, http=http) + service = discovery.build('gmail', 'v1', http=authed_http, + cache_discovery=False) # pylint: disable=no-member + except Exception: # pylint: disable=broad-exception-caught + logging.error("Can't get access token for user %s", username) + raise + + try: + # pylint: disable=no-member + results = service.users().labels().list( + userId=username, + fields='labels(id,name)').execute(num_retries=ARGS.num_retries) + labels = results.get('labels', []) + except Exception: # pylint: disable=broad-exception-caught + logging.error("Can't get labels for user %s", username) + raise + + try: + result = process_mbox_files(username, service, labels) + except Exception: # pylint: disable=broad-exception-caught + logging.error("Can't process mbox files for user %s", username) + raise + + return result + + except Exception: # pylint: disable=broad-exception-caught + logging.exception("Can't process user %s", username) + return None + + +def main(argv): # pylint: disable=too-many-locals + """Import multiple users' mbox files to Gmail. + + """ + global ARGS # pylint: disable=global-statement + ARGS = parser.parse_args(argv) + + setup_logging() + number_of_labels_imported_without_error = 0 number_of_labels_imported_with_some_errors = 0 number_of_labels_failed = 0 @@ -339,33 +430,9 @@ def main(): number_of_users_imported_with_some_errors = 0 number_of_users_failed = 0 - for username in next(os.walk(unicode(args.dir)))[1]: - try: - logging.info('Processing user %s', username) - try: - credentials = get_credentials(username) - http = credentials.authorize(set_user_agent( - httplib2.Http(), - '%s-%s' % (APPLICATION_NAME, APPLICATION_VERSION))) - service = discovery.build('gmail', 'v1', http=http) - except Exception: - logging.error("Can't get access token for user %s", username) - raise - - try: - results = service.users().labels().list( - userId=username, - fields='labels(id,name)').execute(num_retries=args.num_retries) - labels = results.get('labels', []) - except Exception: - logging.error("Can't get labels for user %s", username) - raise - - try: - result = process_mbox_files(username, service, labels) - except Exception: - logging.error("Can't process mbox files for user %s", username) - raise + for username in next(os.walk(ARGS.dir))[1]: + result = process_user(username) + if result: if result[2] == 0 and result[4] == 0: number_of_users_imported_without_error += 1 elif result[0] > 0 or result[3] > 0: @@ -385,10 +452,10 @@ def main(): result[2], result[3], result[4]) - except Exception: + else: number_of_users_failed += 1 - logging.exception("Can't process user %s", username) - logging.info("*** Done importing all users from directory '%s'", args.dir) + + logging.info("*** Done importing all users from directory '%s'", ARGS.dir) logging.info('*** Import summary:') logging.info(' %d users imported with no failures', number_of_users_imported_without_error) @@ -408,9 +475,9 @@ def main(): number_of_messages_failed) if (number_of_messages_failed + number_of_labels_failed + number_of_users_failed > 0): - logging.info('*** Check log file %s for detailed errors.', args.log) + logging.info('*** Check log file %s for detailed errors.', ARGS.log) logging.info('Finished.\n\n') if __name__ == '__main__': - main() + main(sys.argv[1:]) diff --git a/requirements.txt b/requirements.txt index 0c3ba37..1114205 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ google-api-python-client +google-auth +google-auth-httplib2 +google-auth-oauthlib PyOpenSSL -oauth2client +pyinstaller diff --git a/tests/run_real_import_test.py b/tests/run_real_import_test.py new file mode 100644 index 0000000..539f9fe --- /dev/null +++ b/tests/run_real_import_test.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +""" +Interactive Real Import Test Script. + +This script allows users to test the import functionality using real credentials +and generated test data, and then verifies the import by checking the Gmail API. +""" +import os +import sys +import shutil +import tempfile +import subprocess +import mailbox +import time +import email.message +import uuid +import random +import logging +import traceback + +from google.oauth2 import service_account +from googleapiclient import discovery +from googleapiclient.errors import HttpError + +# Scopes needed for import and verification +SCOPES = [ + 'https://www.googleapis.com/auth/gmail.insert', + 'https://www.googleapis.com/auth/gmail.labels', + 'https://www.googleapis.com/auth/gmail.readonly' # Added for verification +] + +# Configure logging for the test script +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + +def get_service(creds_path, user_email): + """Authenticates and returns the Gmail API service.""" + creds = service_account.Credentials.from_service_account_file( + creds_path, scopes=SCOPES, subject=user_email + ) + return discovery.build('gmail', 'v1', credentials=creds) + +def execute_with_retry(request, num_retries=5): + """Executes an API request with exponential backoff retries.""" + for n in range(num_retries): + try: + return request.execute() + except (HttpError, OSError) as e: + if n == num_retries - 1: + raise + sleep_time = (2 ** n) + random.random() + logging.warning( + "Request failed with %s, retrying in %.2f seconds...", e, sleep_time + ) + time.sleep(sleep_time) + return None + +def create_dummy_mbox(filepath, message_id, date_string): + """Creates a dummy mbox file with one message having specific headers.""" + print(f"Creating dummy mbox file at {filepath}...") + mbox = mailbox.mbox(filepath) + msg = email.message.Message() + msg['Subject'] = 'Test Import Message' + msg['From'] = 'sender@example.com' + msg['To'] = 'recipient@example.com' + msg['Date'] = date_string + msg['Message-ID'] = message_id + msg.set_payload('This is a test message body for verifying import functionality.') + mbox.add(msg) + mbox.flush() + mbox.close() + +def get_label_id(service, user_email, label_name): + """Finds the label ID for a given label name.""" + response = execute_with_retry( + service.users().labels().list(userId=user_email) + ) + labels = response.get('labels', []) + for label in labels: + if label['name'].lower() == label_name.lower(): + return label['id'] + return None + +def verify_import(service, user_email, label_name, message_id, expected_date, expected_subject): # pylint: disable=too-many-arguments,too-many-positional-arguments + """Verifies that the imported message exists in Gmail with correct attributes.""" + print(f"\nVerifying import for user: {user_email}") + + # 1. Find the label ID + label_id = get_label_id(service, user_email, label_name) + + if not label_id: + print(f"Error: Label '{label_name}' not found in Gmail.") + return False + + print(f"Found label '{label_name}' with ID: {label_id}") + + # 2. Search for the message by Message-ID + query = f"rfc822msgid:{message_id}" + print(f"Searching for message with query: {query}") + + response = execute_with_retry( + service.users().messages().list( + userId=user_email, q=query, includeSpamTrash=True + ) + ) + + messages = response.get('messages', []) + + if not messages: + print("FAILURE: Message not found by Message-ID.") + return False + + # Get full message details + msg_id = messages[0]['id'] + print(f"Found message with Gmail ID: {msg_id}") + + msg_detail = execute_with_retry( + service.users().messages().get(userId=user_email, id=msg_id) + ) + + # Check Label + if label_id not in msg_detail.get('labelIds', []): + print(f"FAILURE: Message does not have the expected label ID {label_id}.") + print(f"Actual labels: {msg_detail.get('labelIds')}") + return False + + # Check Headers (Subject, Date) + headers = msg_detail.get('payload', {}).get('headers', []) + subject = next((h['value'] for h in headers if h['name'] == 'Subject'), None) + date = next((h['value'] for h in headers if h['name'] == 'Date'), None) + + if subject != expected_subject: + print(f"FAILURE: Subject mismatch. Expected: '{expected_subject}', Found: '{subject}'") + return False + + if date != expected_date: + print(f"FAILURE: Date mismatch. Expected: '{expected_date}', Found: '{date}'") + return False + + print("SUCCESS: Import verification passed! Message found with correct Label, Subject, and Date.") + return True + + +def get_user_inputs(): + """Gets credentials path and target email from args or input.""" + if len(sys.argv) > 1: + creds_path = sys.argv[1] + else: + creds_path = input("Enter path to Credentials.json: ").strip() + + if not os.path.exists(creds_path): + print(f"Error: File '{creds_path}' not found.") + sys.exit(1) + + if len(sys.argv) > 2: + target_email = sys.argv[2] + else: + target_email = input("Enter target email address: ").strip() + + if not target_email: + print("Error: Target email is required.") + sys.exit(1) + + return creds_path, target_email + + +def generate_test_data(user_dir): + """Generates test mbox data.""" + dst_mbox_name = "Test Import.mbox" + dst_mbox_path = os.path.join(user_dir, dst_mbox_name) + + # Generate test data + message_id = f"<{uuid.uuid4()}@test.local>" + date_string = "Mon, 20 Jan 2025 12:00:00 -0000" + + # Always create a fresh dummy mbox to ensure controlled test data + create_dummy_mbox(dst_mbox_path, message_id, date_string) + + # Append invalid message to Test Import.mbox + print(f"Appending invalid message to {dst_mbox_path}...") + mbox = mailbox.mbox(dst_mbox_path) + msg = email.message.Message() + msg['Subject'] = 'Invalid Headers Message' + msg['Date'] = date_string + msg['Message-ID'] = f"<{uuid.uuid4()}@test.local>" + msg['To'] = 'recipient@example.com' + # Add duplicate From headers + msg.add_header('From', 'sender1@example.com') + msg.add_header('From', 'sender2@example.com') + msg.set_payload('This message has two From headers.') + mbox.add(msg) + mbox.flush() + mbox.close() + + # Create Outbox.mbox (invalid label) + outbox_path = os.path.join(user_dir, "Outbox.mbox") + create_dummy_mbox(outbox_path, f"<{uuid.uuid4()}@test.local>", date_string) + + return message_id, date_string + + +def run_and_verify(creds_path, target_email, temp_dir, user_dir): + """Runs the import and verifies log output.""" + message_id, date_string = generate_test_data(user_dir) + subject_string = "Test Import Message" + + print(f"\nPrepared test data in {temp_dir}") + print(f"Message-ID: {message_id}") + print(f"Importing into {target_email} with label 'Test Import'...") + + # Log file + log_file = os.path.join(temp_dir, 'import.log') + + # Run import + parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + script_path = os.path.join(parent_dir, "import-mailbox-to-gmail.py") + cmd = [ + sys.executable, + script_path, + "--json", creds_path, + "--dir", temp_dir, + "--log", log_file + ] + + subprocess.check_call(cmd) + + print("\nImport script finished. Verifying logs and result...") + + # Read log file + with open(log_file, 'r', encoding='utf-8') as f: + log_content = f.read() + + # Check for skipped Outbox label + if "Skipping label 'Outbox' because it can't be created" not in log_content: + print("FAILURE: Log does not indicate that 'Outbox' label was skipped.") + sys.exit(1) + else: + print("SUCCESS: Log indicates 'Outbox' label was skipped.") + + # Check for failed message + if "Failed to import mbox message" not in log_content: + print("FAILURE: Log does not indicate message failure.") + sys.exit(1) + else: + print("SUCCESS: Log indicates message failure.") + + return message_id, date_string, subject_string + + +def main(): + """Main function to run the interactive test.""" + print("Interactive Real Import Test") + print("----------------------------") + + creds_path, target_email = get_user_inputs() + + # Setup temp directory + temp_dir = tempfile.mkdtemp(prefix="import_test_") + try: + user_dir = os.path.join(temp_dir, target_email) + os.makedirs(user_dir) + + message_id, date_string, subject_string = run_and_verify( + creds_path, target_email, temp_dir, user_dir) + + # Verify successful import + try: + service = get_service(creds_path, target_email) + # The label name is derived from the filename "Test Import.mbox" + label_name = "Test Import" + if verify_import(service, target_email, label_name, message_id, + date_string, subject_string): + sys.exit(0) + else: + sys.exit(1) + except Exception: # pylint: disable=broad-exception-caught + print("Verification failed with error:") + traceback.print_exc() + print("Note: Ensure your service account has " + "'https://www.googleapis.com/auth/gmail.readonly' scope authorized.") + sys.exit(1) + + except subprocess.CalledProcessError as e: + print(f"\nImport process failed with exit code {e.returncode}") + sys.exit(e.returncode) + except Exception as e: # pylint: disable=broad-exception-caught + print(f"\nAn error occurred: {e}") + sys.exit(1) + finally: + # Cleanup + if os.path.exists(temp_dir): + print(f"Cleaning up {temp_dir}...") + try: + shutil.rmtree(temp_dir) + except OSError as e: + print(f"Warning: Failed to clean up {temp_dir}: {e}") + +if __name__ == "__main__": + main() diff --git a/tests/sample.mbox b/tests/sample.mbox new file mode 100644 index 0000000..9008f39 --- /dev/null +++ b/tests/sample.mbox @@ -0,0 +1,17 @@ +From MAILER-DAEMON Fri Jul 8 12:08:34 2005 +From: John Doe +To: Jane Doe +Subject: Test message 1 +Date: Fri, 8 Jul 2005 12:08:34 -0700 +Message-ID: <12345@local.machine.example> + +This is the first test message. + +From MAILER-DAEMON Fri Jul 8 12:09:15 2005 +From: John Doe +To: Jane Doe +Subject: Test message 2 +Date: Fri, 8 Jul 2005 12:09:15 -0700 +Message-ID: <67890@local.machine.example> + +This is the second test message. diff --git a/tests/test_import.py b/tests/test_import.py new file mode 100644 index 0000000..e4e866e --- /dev/null +++ b/tests/test_import.py @@ -0,0 +1,384 @@ +"""Unit tests for import-mailbox-to-gmail.""" +import unittest +import os +import shutil +import tempfile +import importlib.util +import sys +import mailbox +import email.message +import logging +from unittest.mock import patch, MagicMock + +# Load the module dynamically +parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +script_path = os.path.join(parent_dir, "import-mailbox-to-gmail.py") +spec = importlib.util.spec_from_file_location("import_mailbox_to_gmail", script_path) +import_mailbox_to_gmail = importlib.util.module_from_spec(spec) +sys.modules["import_mailbox_to_gmail"] = import_mailbox_to_gmail +spec.loader.exec_module(import_mailbox_to_gmail) + +class TestImport(unittest.TestCase): + """Test case for import logic.""" + + def setUp(self): + logging.disable(logging.CRITICAL) + self.test_dir = tempfile.mkdtemp() + self.username = 'testuser@example.com' + self.user_dir = os.path.join(self.test_dir, self.username) + os.makedirs(self.user_dir) + self.mbox_path = os.path.join(self.user_dir, 'test.mbox') + sample_mbox_path = os.path.join(os.path.dirname(__file__), 'sample.mbox') + shutil.copyfile(sample_mbox_path, self.mbox_path) + + + def tearDown(self): + logging.disable(logging.NOTSET) + try: + shutil.rmtree(self.test_dir) + except OSError as e: + print(f"Warning: Failed to clean up {self.test_dir}: {e}") + + @patch('import_mailbox_to_gmail.discovery.build') + def test_import(self, mock_build): + """Test the import process.""" + # Mock the service and its methods + mock_service = MagicMock() + mock_build.return_value = mock_service + + # Mock the labels().list() call to return no existing labels + mock_service.users().labels().list().execute.return_value = {'labels': []} + + # Mock the labels().create() call to return a new label + mock_service.users().labels().create().execute.return_value = { + 'id': 'LABEL_1', + 'name': 'test' + } + + # Mock the messages().import_() call + mock_service.users().messages().import_().execute.return_value = {'id': 'MSG_ID'} + + # Set up the arguments for the script + args = MagicMock() + args.dir = self.test_dir + args.from_message = 0 + args.fix_msgid = True + args.replace_quoted_printable = True + args.num_retries = 3 + args.log = 'test.log' + args.httplib2debuglevel = 0 + + import_mailbox_to_gmail.ARGS = args + + # Call the function that processes the mbox files + result = import_mailbox_to_gmail.process_mbox_files( + self.username, mock_service, []) + + # Assertions + self.assertEqual(result[3], 2) # 2 messages imported + self.assertEqual(result[4], 0) # 0 messages failed + + # Check that the label was created + self.assertEqual(mock_service.users().labels().create.call_count, 2) + mock_service.users().labels().create.assert_any_call( + userId=self.username, + body={'messageListVisibility': 'show', 'name': 'test', 'labelListVisibility': 'labelShow'} + ) + + # Check that the messages were imported + self.assertEqual(mock_service.users().messages().import_().execute.call_count, 2) + + @patch('import_mailbox_to_gmail.discovery.build') + def test_import_label_failure(self, mock_build): + """Test that failed label creation increments the failure counter.""" + # Mock the service and its methods + mock_service = MagicMock() + mock_build.return_value = mock_service + + # Mock the labels().list() call to return no existing labels + mock_service.users().labels().list().execute.return_value = {'labels': []} + + # Mock the labels().create() call to raise an exception + mock_service.users().labels().create().execute.side_effect = Exception("400 Bad Request") + + # Set up the arguments for the script + args = MagicMock() + args.dir = self.test_dir + args.from_message = 0 + args.fix_msgid = True + args.replace_quoted_printable = True + args.num_retries = 1 + args.log = 'test.log' + args.httplib2debuglevel = 0 + + import_mailbox_to_gmail.ARGS = args + + # Call the function that processes the mbox files + result = import_mailbox_to_gmail.process_mbox_files( + self.username, mock_service, []) + + # Assertions + # We expect 1 failed label (the one corresponding to test.mbox) + self.assertEqual(result[2], 1) # result[2] is number_of_labels_failed + + @patch('import_mailbox_to_gmail.discovery.build') + def test_import_message_failure(self, mock_build): + """Test that failed message import increments the failure counter.""" + mock_service = MagicMock() + mock_build.return_value = mock_service + + mock_service.users().labels().list().execute.return_value = {'labels': []} + mock_service.users().labels().create().execute.return_value = { + 'id': 'LABEL_1', 'name': 'test'} + + # Mock import_ to raise exception + mock_service.users().messages().import_().execute.side_effect = Exception( + "Import Failed") + + args = MagicMock() + args.dir = self.test_dir + args.from_message = 0 + args.fix_msgid = True + args.replace_quoted_printable = True + args.num_retries = 1 + args.log = 'test.log' + args.httplib2debuglevel = 0 + import_mailbox_to_gmail.ARGS = args + + result = import_mailbox_to_gmail.process_mbox_files( + self.username, mock_service, []) + + # Expect 2 messages failed (sample.mbox has 2 messages) + self.assertEqual(result[4], 2) # result[4] is number_of_messages_failed + self.assertEqual(result[3], 0) # result[3] is number_of_messages_imported_without_error + + @patch('import_mailbox_to_gmail.process_user') + @patch('import_mailbox_to_gmail.setup_logging') + @patch('os.walk') + def test_main_user_failure_counter(self, mock_walk, _, mock_process_user): + """Test that failed user processing increments the user failure counter.""" + mock_process_user.return_value = None # Simulate failure + + # Mock os.walk to return one user + mock_walk.return_value = iter([ + (self.test_dir, [self.username], []) # root + ]) + + with patch('logging.info') as mock_logging_info: + # We need to simulate arguments passed to main + import_mailbox_to_gmail.main( + ['--dir', self.test_dir, '--json', 'creds.json']) + + # Check for user failure logging + found = False + for call in mock_logging_info.call_args_list: + args, _ = call + if len(args) > 1 and args[0] == ' %d users failed' and args[1] == 1: + found = True + break + self.assertTrue(found, "Did not find expected logging for user failure count") + + @patch('import_mailbox_to_gmail.discovery.build') + def test_args_noreplaceqp(self, mock_build): + """Test --noreplaceqp argument behavior.""" + mock_service = MagicMock() + mock_build.return_value = mock_service + mock_service.users().labels().list().execute.return_value = {'labels': []} + mock_service.users().labels().create().execute.return_value = { + 'id': 'LABEL_1', 'name': 'test'} + mock_service.users().messages().import_().execute.return_value = { + 'id': 'MSG_ID'} + + # Create a mbox with quoted-printable content type + mbox_path = os.path.join(self.user_dir, 'qp.mbox') + mbox = mailbox.mbox(mbox_path) + msg = email.message.Message() + msg['Subject'] = 'Test QP' + msg['Content-Type'] = 'text/quoted-printable' + msg.set_payload('Test') + mbox.add(msg) + mbox.flush() + mbox.close() + + # Test with replace_quoted_printable=True (default) + args = MagicMock() + args.dir = self.test_dir + args.from_message = 0 + args.fix_msgid = True + args.replace_quoted_printable = True + args.num_retries = 1 + args.log = 'test.log' + args.httplib2debuglevel = 0 + import_mailbox_to_gmail.ARGS = args + + with patch('import_mailbox_to_gmail.import_message') as mock_import_message: + mock_import_message.return_value = True + import_mailbox_to_gmail.process_mbox_files(self.username, mock_service, []) + + # Verify call arguments + # First call, first message (sample.mbox) - we skip it as we are testing qp.mbox + # Actually process_mbox_files processes all mbox files. + # We should probably clear user dir first or only have qp.mbox + # But sample.mbox is there from setUp. + + # Find the call for qp.mbox message + found_replaced = False + for call in mock_import_message.call_args_list: + msg_arg = call[0][2] + if msg_arg['Subject'] == 'Test QP': + if 'text/plain' in msg_arg['Content-Type']: + found_replaced = True + self.assertTrue(found_replaced, "Should have replaced text/quoted-printable with text/plain") + + # Test with replace_quoted_printable=False + args.replace_quoted_printable = False + + with patch('import_mailbox_to_gmail.import_message') as mock_import_message: + mock_import_message.return_value = True + import_mailbox_to_gmail.process_mbox_files(self.username, mock_service, []) + + found_original = False + for call in mock_import_message.call_args_list: + msg_arg = call[0][2] + if msg_arg['Subject'] == 'Test QP': + if 'text/quoted-printable' in msg_arg['Content-Type']: + found_original = True + self.assertTrue(found_original, "Should NOT have replaced text/quoted-printable") + + @patch('import_mailbox_to_gmail.discovery.build') + def test_args_no_fix_msgid(self, mock_build): + """Test --no-fix-msgid argument behavior.""" + mock_service = MagicMock() + mock_build.return_value = mock_service + mock_service.users().labels().list().execute.return_value = {'labels': []} + mock_service.users().labels().create().execute.return_value = { + 'id': 'LABEL_1', 'name': 'test'} + mock_service.users().messages().import_().execute.return_value = { + 'id': 'MSG_ID'} + + # Create a mbox with missing brackets in Message-ID + mbox_path = os.path.join(self.user_dir, 'nomsgid.mbox') + mbox = mailbox.mbox(mbox_path) + msg = email.message.Message() + msg['Subject'] = 'Test NoMsgID' + msg['Message-ID'] = 'no-brackets@example.com' + msg.set_payload('Test') + mbox.add(msg) + mbox.flush() + mbox.close() + + # Test with fix_msgid=True (default) + args = MagicMock() + args.dir = self.test_dir + args.from_message = 0 + args.fix_msgid = True + args.replace_quoted_printable = True + args.num_retries = 1 + args.log = 'test.log' + args.httplib2debuglevel = 0 + import_mailbox_to_gmail.ARGS = args + + with patch('import_mailbox_to_gmail.import_message') as mock_import_message: + mock_import_message.return_value = True + import_mailbox_to_gmail.process_mbox_files(self.username, mock_service, []) + + found_fixed = False + for call in mock_import_message.call_args_list: + msg_arg = call[0][2] + if msg_arg['Subject'] == 'Test NoMsgID': + if msg_arg['Message-ID'] == '': + found_fixed = True + self.assertTrue(found_fixed, "Should have fixed Message-ID brackets") + + # Test with fix_msgid=False + args.fix_msgid = False + + with patch('import_mailbox_to_gmail.import_message') as mock_import_message: + mock_import_message.return_value = True + import_mailbox_to_gmail.process_mbox_files(self.username, mock_service, []) + + found_original = False + for call in mock_import_message.call_args_list: + msg_arg = call[0][2] + if msg_arg['Subject'] == 'Test NoMsgID': + if msg_arg['Message-ID'] == 'no-brackets@example.com': + found_original = True + self.assertTrue(found_original, "Should NOT have fixed Message-ID brackets") + + @patch('import_mailbox_to_gmail.discovery.build') + def test_args_from_message(self, mock_build): + """Test --from_message argument behavior.""" + mock_service = MagicMock() + mock_build.return_value = mock_service + mock_service.users().labels().list().execute.return_value = {'labels': []} + mock_service.users().labels().create().execute.return_value = { + 'id': 'LABEL_1', 'name': 'test'} + mock_service.users().messages().import_().execute.return_value = { + 'id': 'MSG_ID'} + + # Use sample.mbox which has 2 messages. + # Set from_message=1, should import only the second message (index 1) + + args = MagicMock() + args.dir = self.test_dir + args.from_message = 1 + args.fix_msgid = True + args.replace_quoted_printable = True + args.num_retries = 1 + args.log = 'test.log' + args.httplib2debuglevel = 0 + import_mailbox_to_gmail.ARGS = args + + with patch('import_mailbox_to_gmail.import_message') as mock_import_message: + mock_import_message.return_value = True + import_mailbox_to_gmail.process_mbox_files(self.username, mock_service, []) + + # Should be called once for sample.mbox (2nd message) + # Note: process_mbox_files loops over all mbox files. + # Ensure we only have sample.mbox or count correctly. + # setUp copies sample.mbox to test.mbox. + + count = 0 + for call in mock_import_message.call_args_list: + # Check if this call is for test.mbox (label 'test') + # process_mbox_files doesn't pass filename to import_message, but label_id. + # We can check the message subject if needed. + msg_arg = call[0][2] + if msg_arg['Subject'] == 'Test message 2': + count += 1 + if msg_arg['Subject'] == 'Test message 1': + self.fail("Should have skipped Test message 1") + + self.assertEqual(count, 1) + + @patch('import_mailbox_to_gmail.process_mbox_files') + @patch('import_mailbox_to_gmail.discovery.build') + @patch('import_mailbox_to_gmail.AuthorizedHttp') + @patch('import_mailbox_to_gmail.get_credentials') + @patch('import_mailbox_to_gmail.set_user_agent') + def test_process_user_builds_service_without_cache( + # pylint: disable=too-many-arguments,too-many-positional-arguments + self, mock_set_ua, mock_get_creds, mock_authed_http, mock_build, + mock_process_mbox_files): + """Test that process_user builds the service with cache_discovery=False.""" + mock_get_creds.return_value = MagicMock() + mock_set_ua.return_value = MagicMock() + mock_authed_http.return_value = MagicMock() + + # Mock service to avoid further errors + mock_service = MagicMock() + mock_build.return_value = mock_service + mock_service.users().labels().list().execute.return_value = {'labels': []} + + # Mock process_mbox_files to return success + mock_process_mbox_files.return_value = (0, 0, 0, 0, 0) + + import_mailbox_to_gmail.process_user(self.username) + + # Verify discovery.build was called with cache_discovery=False + mock_build.assert_called_with( + 'gmail', 'v1', http=mock_authed_http.return_value, + cache_discovery=False) + +if __name__ == '__main__': + unittest.main()