v3.2.0 release

Author: Sid Mohan

README.md

@@ -27,12 +27,10 @@ DataFog is an open-source DevSecOps platform that lets you scan and redact Perso
 
 ![image](https://github.com/DataFog/datafog-python/assets/61345237/57fba4e5-21cc-458f-ac6a-6fbbb70a8de1)
 
-
 ### How it works
 
 ![image](https://github.com/DataFog/datafog-python/assets/61345237/91f4634a-8a9f-4621-81bc-09930feda78a)
 
-
 ## Installation
 
 DataFog can be installed via pip:
@@ -41,7 +39,82 @@ DataFog can be installed via pip:
 pip install datafog
 ```
 
-## Examples - Updated for v3.1
+## Examples -
+
+### v3.2.0 NEW
+
+Based on the provided test cases, here's a suitable "Getting Started" section for the documentation:
+
+## Getting Started
+
+The DataFog library provides functionality for text and image processing, including PII (Personally Identifiable Information) annotation and OCR (Optical Character Recognition) capabilities.
+
+### Installation
+
+To install the DataFog library, use the following command:
+
+```
+pip install datafog
+```
+
+### Usage
+
+Here are some examples of how to use the DataFog library:
+
+#### Text PII Annotation
+
+To annotate PII in a given text, lets start with a set of clinical notes:
+
+```
+!git clone https://gist.github.com/b43b72693226422bac5f083c941ecfdb.git
+```
+
+```python
+from datafog import TextPIIAnnotator
+
+text = "John Doe lives at 1234 Elm St, Springfield."
+text_annotator = TextPIIAnnotator()
+annotated_text = text_annotator.run(text)
+print(annotated_text)
+```
+
+This will output the annotated text with PII labeled, such as `{"LOC": ["Springfield"]}`.
+
+#### Image Text Extraction and Annotation
+
+To extract text from an image and perform PII annotation, you can use the `DataFog` class:
+
+```python
+from datafog import DataFog
+
+image_url = "https://example.com/image.png"
+datafog = DataFog()
+annotated_text = await datafog.run_ocr_pipeline([image_url])
+print(annotated_text)
+```
+
+This will download the image, extract the text using OCR, and annotate any PII found in the extracted text.
+
+#### Text Processing
+
+To process and annotate text using the DataFog pipeline, you can use the `DataFog` class:
+
+```python
+from datafog import DataFog
+
+text = ["Tokyo is the capital of Japan"]
+datafog = DataFog()
+annotated_text = await datafog.run_text_pipeline(text)
+print(annotated_text)
+```
+
+This will process the given text and annotate entities such as person names and locations.
+
+For more detailed usage and examples, please refer to the API documentation.
+
+Note: The DataFog library uses asynchronous programming, so make sure to use the `async`/`await` syntax when calling the appropriate methods.
+
+### v3.1.0
 
 ### Base case: PII annotation of text-files
 

datafog/__about__.py

@@ -1 +1 @@
-__version__ = "3.2.0b5"
+__version__ = "3.2.0"

datafog/__init__.py

@@ -1,15 +1,14 @@
-from .main import (DataFog, OCRPIIAnnotator, TextPIIAnnotator
-)
+from .config import OperationType
+from .main import DataFog, OCRPIIAnnotator, TextPIIAnnotator
 from .processing.image_processing.donut_processor import DonutProcessor
 from .processing.image_processing.image_downloader import ImageDownloader
 from .processing.image_processing.pytesseract_processor import PytesseractProcessor
 from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator
-
-
 from .services.image_service import ImageService
 from .services.spark_service import SparkService
 from .services.text_service import TextService
-from .config import OperationType
+
+# from .__about__ import __version__
 
 __all__ = [
     "DonutProcessor",
@@ -23,4 +22,5 @@
     "SpacyPIIAnnotator",
     "ImageDownloader",
     "PytesseractProcessor",
-]
+    # "__version__",
+]

datafog/main.py

@@ -1,21 +1,28 @@
 import asyncio
-import json
-from typing import List
 import importlib
-import aiohttp
+import json
 import subprocess
 import sys
+from typing import List
+
+import aiohttp
+
 from .config import OperationType
+from .processing.image_processing.donut_processor import DonutProcessor
+from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator
 from .services.image_service import ImageService
 from .services.spark_service import SparkService
 from .services.text_service import TextService
 
-from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator
-from .processing.image_processing.donut_processor import DonutProcessor
-
 
 class DataFog:
-    def __init__(self, image_service = ImageService(), text_service = TextService(), spark_service = None, operations: List[OperationType] = [OperationType.ANNOTATE_PII]):
+    def __init__(
+        self,
+        image_service=ImageService(),
+        text_service=TextService(),
+        spark_service=None,
+        operations: List[OperationType] = [OperationType.ANNOTATE_PII],
+    ):
         self.image_service = image_service
         self.text_service = text_service
         self.spark_service: SparkService = spark_service
@@ -45,7 +52,6 @@ def __init__(self):
         self.text_annotator = SpacyPIIAnnotator.create()
         self.spark_service: SparkService = None
 
-
     async def run(self, image_urls: List[str], output_path=None):
         try:
             # Download and process the image to extract text

datafog/processing/__init__.py

@@ -1,5 +1,5 @@
 from .image_processing.donut_processor import DonutProcessor
 from .image_processing.image_downloader import ImageDownloader
 from .image_processing.pytesseract_processor import PytesseractProcessor
-from .text_processing.spacy_pii_annotator import SpacyPIIAnnotator
 from .spark_processing.pyspark_udfs import broadcast_pii_annotator_udf, pii_annotator
+from .text_processing.spacy_pii_annotator import SpacyPIIAnnotator

datafog/processing/image_processing/donut_processor.py

@@ -1,9 +1,10 @@
+import importlib
 import json
 import re
-from io import BytesIO
-import importlib
 import subprocess
 import sys
+from io import BytesIO
+
 import requests
 from PIL import Image
 
@@ -13,12 +14,12 @@
 class DonutProcessor:
     def __init__(self, model_path="naver-clova-ix/donut-base-finetuned-cord-v2"):
 
+        self.ensure_installed("torch")
+        self.ensure_installed("transformers")
 
-        self.ensure_installed('torch')
-        self.ensure_installed('transformers')
-
-        from transformers import VisionEncoderDecoderModel, DonutProcessor as TransformersDonutProcessor
         import torch
+        from transformers import DonutProcessor as TransformersDonutProcessor
+        from transformers import VisionEncoderDecoderModel
 
         self.processor = TransformersDonutProcessor.from_pretrained(model_path)
         self.model = VisionEncoderDecoderModel.from_pretrained(model_path)
@@ -31,7 +32,9 @@ def ensure_installed(self, package_name):
         try:
             importlib.import_module(package_name)
         except ImportError:
-            subprocess.check_call([sys.executable, "-m", "pip", "install", package_name])
+            subprocess.check_call(
+                [sys.executable, "-m", "pip", "install", package_name]
+            )
 
     async def parse_image(self, image: Image) -> str:
         """Process the image using DonutProcessor and VisionEncoderDecoderModel and extract text."""
@@ -72,4 +75,4 @@ def download_image(self, url: str) -> Image:
         """Download an image from URL."""
         response = requests.get(url)
         image = Image.open(BytesIO(response.content))
-        return image
+        return image

datafog/processing/image_processing/pytesseract_processor.py

@@ -10,4 +10,3 @@ def __init__(self):
     async def extract_text_from_image(image: Image) -> str:
         """Extract text from an image using pytesseract."""
         return pytesseract.image_to_string(image)
-

datafog/processing/spark_processing/__init__.py

@@ -1,2 +1 @@
 from .pyspark_udfs import broadcast_pii_annotator_udf, pii_annotator
-

datafog/services/__init__.py

@@ -1,4 +1,3 @@
 from .image_service import ImageService
 from .spark_service import SparkService
 from .text_service import TextService
-

datafog/services/image_service.py

@@ -29,28 +29,25 @@ async def ocr_extract(
         self,
         image_urls: List[str],
         image_files: List[Image.Image] = None,
-
     ) -> List[str]:
         if image_files is None:
             image_files = await self.download_images(image_urls)
 
         if self.use_donut and self.use_tesseract:
             raise ValueError("Both OCR processors cannot be selected simultaneously")
-        
+
         if not self.use_donut and not self.use_tesseract:
             raise ValueError("No OCR processor selected")
-        
+
         if self.use_donut:
             return await asyncio.gather(
                 *[self.donut_processor.parse_image(image) for image in image_files]
             )
-        
+
         if self.use_tesseract:
             return await asyncio.gather(
                 *[
                     self.tesseract_processor.extract_text_from_image(image)
                     for image in image_files
                 ]
             )
-
-