structured_extractor.py

import re
from typing import Dict, List, Tuple
from pdf_analyzer import PDFAnalyzer

class StructuredDataExtractor:
    def __init__(self, analyzer: PDFAnalyzer):
        self.analyzer = analyzer
    
    def extract_structured_data(self, text: str) -> Dict:
        """Extract structured data from text using spaCy"""
        if not self.analyzer.nlp:
            return self._extract_fallback_data(text)
        
        doc = self.analyzer.nlp(text)
        
        return {
            "sentences": self._extract_sentences(doc),
            "entities": self._extract_entities(doc),
            "key_concepts": self._extract_key_concepts(doc),
            "relationships": self._extract_relationships(doc),
            "facts": self._extract_facts(doc),
            "metadata": self._extract_metadata(doc)
        }
    
    def _extract_sentences(self, doc) -> List[str]:
        """Extract meaningful sentences"""
        sentences = []
        for sent in doc.sents:
            text = sent.text.strip()
            if len(text) > 20 and not text.startswith(('Figure', 'Table', 'Chapter')):
                sentences.append(text)
        return sentences
    
    def _extract_entities(self, doc) -> List[Tuple[str, str]]:
        """Extract named entities with their types"""
        entities = []
        for ent in doc.ents:
            if ent.label_ in ['PERSON', 'ORG', 'GPE', 'PRODUCT', 'EVENT', 'DATE', 'QUANTITY']:
                entities.append((ent.text, ent.label_))
        return entities
    
    def _extract_key_concepts(self, doc) -> List[str]:
        """Extract key concepts and noun phrases"""
        concepts = []
        
        # Noun chunks (complete noun phrases)
        for chunk in doc.noun_chunks:
            if len(chunk.text) > 3 and chunk.text.lower() not in ['the text', 'this document']:
                concepts.append(chunk.text.lower())
        
        # Important nouns and adjectives
        for token in doc:
            if (token.pos_ in ['NOUN', 'PROPN', 'ADJ'] and 
                not token.is_stop and 
                len(token.text) > 3 and
                token.text.lower() not in ['text', 'page', 'chapter', 'section']):
                concepts.append(token.text.lower())
        
        # Remove duplicates and return most frequent
        concept_counts = {}
        for concept in concepts:
            concept_counts[concept] = concept_counts.get(concept, 0) + 1
        
        sorted_concepts = sorted(concept_counts.items(), key=lambda x: x[1], reverse=True)
        return [concept for concept, count in sorted_concepts[:20]]
    
    def _extract_relationships(self, doc) -> List[Dict]:
        """Extract semantic relationships between concepts"""
        relationships = []
        
        # Subject-verb-object relationships
        for token in doc:
            if token.dep_ == "nsubj" and token.head.pos_ == "VERB":
                subject = token.text
                verb = token.head.text
                
                # Find object
                for child in token.head.children:
                    if child.dep_ in ["dobj", "pobj"]:
                        obj = child.text
                        relationships.append({
                            "type": "action",
                            "subject": subject,
                            "action": verb,
                            "object": obj
                        })
        
        # Adjective-noun relationships
        for token in doc:
            if token.pos_ == "ADJ":
                for child in token.children:
                    if child.pos_ == "NOUN":
                        relationships.append({
                            "type": "attribute",
                            "concept": child.text,
                            "attribute": token.text
                        })
        
        return relationships
    
    def _extract_facts(self, doc) -> List[str]:
        """Extract factual statements"""
        facts = []
        
        for sent in doc.sents:
            # Look for factual patterns
            text = sent.text.lower()
            
            # Contains numbers or dates
            if re.search(r'\d+', text):
                facts.append(sent.text)
            
            # Contains comparison words
            elif any(word in text for word in ['more', 'less', 'better', 'worse', 'higher', 'lower']):
                facts.append(sent.text)
            
            # Contains cause-effect words
            elif any(word in text for word in ['causes', 'leads to', 'results in', 'affects']):
                facts.append(sent.text)
        
        return facts[:10]  # Limit to top 10 facts
    
    def _extract_metadata(self, doc) -> Dict:
        """Extract document metadata"""
        return {
            "total_sentences": len(list(doc.sents)),
            "total_tokens": len(doc),
            "language": doc.lang_,
            "key_topics": self._extract_topics(doc)
        }
    
    def _extract_topics(self, doc) -> List[str]:
        """Extract main topics from the document"""
        topics = []
        
        # Find most frequent noun phrases
        noun_phrases = {}
        for chunk in doc.noun_chunks:
            phrase = chunk.text.lower()
            if len(phrase) > 5:
                noun_phrases[phrase] = noun_phrases.get(phrase, 0) + 1
        
        # Return top topics
        sorted_topics = sorted(noun_phrases.items(), key=lambda x: x[1], reverse=True)
        return [topic for topic, count in sorted_topics[:5]]
    
    def _extract_fallback_data(self, text: str) -> Dict:
        """Fallback extraction when spaCy is not available"""
        sentences = re.split(r'[.!?]+', text)
        sentences = [s.strip() for s in sentences if len(s.strip()) > 20]
        
        words = text.lower().split()
        concepts = [w for w in words if len(w) > 4 and w.isalpha()]
        
        return {
            "sentences": sentences,
            "entities": [],
            "key_concepts": concepts[:10],
            "relationships": [],
            "facts": sentences[:5],
            "metadata": {
                "total_sentences": len(sentences),
                "total_tokens": len(words),
                "language": "unknown",
                "key_topics": concepts[:5]
            }
        }