Chunks Modular

Text / Documents
       │
       ▼
  ChunkManager          ← Orchestrator (recommended entry point)
       │
       ├── QueryPatternAnalyzer   (optional: picks strategy from queries)
       │
       ├── Chunker selection:
       │     ├── SemanticChunker         (embedding-based splits)
       │     ├── AdaptiveChunker         (density-adaptive sizing)
       │     ├── StructureAwareChunker   (respects headers, code, tables)
       │     ├── ContextAwareChunker     (sliding-window context)
       │     └── Hybrid                  (structural + semantic)
       │
       ├── MetadataEnricher       (keywords, scores)
       ├── Deduplicator           (hash + similarity)
       └── Finalize               (position, total_chunks)
             │
             ▼
      List[DocumentChunk]

from fennec_community.chunks import ChunkManager, ChunkConfig

# 1. Basic usage — auto mode
manager = ChunkManager()
chunks = manager.process("Your document text here...", source="my_doc.pdf")

# 2. Semantic chunking with custom config
config = ChunkConfig(
    use_semantic_chunking=True,
    chunk_size=512,
    overlap=128,
    extract_keywords=True,
    deduplication_enabled=True,
)
manager = ChunkManager(config=config)
chunks = manager.process(text, source="report.pdf")

# 3. Inspect results
for chunk in chunks:
    print(chunk.text[:80])
    print(f"  keywords: {chunk.metadata.keywords}")
    print(f"  score:    {chunk.metadata.score:.3f}")

ChunkManager(
    config: Optional[ChunkConfig] = None,
    mode: ChunkMode = ChunkMode.AUTO,
    *,
    device: Optional[str] = None,
    language: str = "auto",
)

def process(
    text: str,
    doc_id: Optional[str] = None,
    source: str = "",
    *,
    sample_queries: Optional[List[str]] = None,
    mode_override: Optional[ChunkMode] = None,
) -> List[DocumentChunk]

chunks = manager.process(
    text="Large document content...",
    source="annual_report_2024.pdf",
    sample_queries=["What is the revenue?", "Who are the key executives?"],
)

def process_documents(
    documents: List[Document],
    *,
    sample_queries: Optional[List[str]] = None,
    mode_override: Optional[ChunkMode] = None,
) -> List[DocumentChunk]

from fennec_community.chunks import Document

docs = [
    Document(page_content="Doc 1 text...", metadata={"source": "file1.pdf"}),
    Document(page_content="Doc 2 text...", metadata={"source": "file2.pdf"}),
]
all_chunks = manager.process_documents(docs)

def process_with_query_optimization(
    text: str,
    queries: List[str],
    doc_id: Optional[str] = None,
    source: str = "",
) -> List[DocumentChunk]

chunks = manager.process_with_query_optimization(
    text=document_text,
    queries=["How does the API authenticate?", "What are the rate limits?"],
    source="api_docs.md",
)

def get_stats(chunks: List[DocumentChunk]) -> Dict[str, Any]

stats = manager.get_stats(chunks)
print(stats)
# {'count': 42, 'avg_chars': 487.3, 'avg_score': 0.6812, ...}

def chunk(self, text: str, doc_id: str = "", source: str = "") -> List[DocumentChunk]

def chunk_documents(self, documents: List[Document]) -> List[DocumentChunk]

SemanticChunker(
    config: Optional[ChunkConfig] = None,
    *,
    similarity_threshold: float = 0.75,
    min_chunk_size: int = 50,
    max_chunk_size: int = 2048,
    model_name: Optional[str] = None,
    device: Optional[str] = None,
    language: str = "auto",
)

from fennec_community.chunks import SemanticChunker

chunker = SemanticChunker(similarity_threshold=0.70)
chunks = chunker.chunk(text, source="article.txt")

AdaptiveChunker(
    config: Optional[ChunkConfig] = None,
    *,
    base_size: int = 512,
    min_size: int = 100,
    max_size: int = 1500,
    overlap: int = 128,
    technical_threshold: float = 0.5,
    language: str = "auto",
)

StructureAwareChunker(
    config: Optional[ChunkConfig] = None,
    *,
    max_chunk_size: int = 1024,
    min_chunk_size: int = 50,
    language: str = "auto",
    split_on_headers: bool = True,
    preserve_code_blocks: bool = True,
)

ContextAwareChunker(
    config: Optional[ChunkConfig] = None,
    *,
    chunk_size: int = 512,
    overlap: int = 128,
    window_size: int = 2,
    min_chunk_size: int = 50,
    language: str = "auto",
)

ArabicTextChunker(
    chunk_size: int = 512,
    overlap: int = 128,
    min_chunk_size: int = 50,
    model_name: str = "CAMeL-Lab/bert-base-arabic-camelbert-mix",
    use_semantic_chunking: bool = False,
    device: Optional[str] = None,
    preserve_formatting: bool = True,
    fix_spacing: bool = True,
    strict_size_limit: bool = True,
    size_tolerance: float = 0.05,
    use_smart_overlap: bool = True,
    smart_overlap_threshold: float = 0.7,
)

@classmethod
def create_safely(cls, **kwargs) -> ArabicTextChunker

@classmethod
def get_available_parameters(cls) -> List[str]

def get_arabic_ratio(self, text: str) -> float

def is_arabic_dominant(self, text: str, threshold: float = 0.3) -> bool

MultilanguageTextChunker(
    chunk_size: int = 512,
    overlap: int = 128,
    min_chunk_size: int = 50,
    model_name: Optional[str] = None,
    language: str = "auto",
    use_semantic_chunking: bool = False,
    device: Optional[str] = None,
    use_smart_overlap: bool = True,
    smart_overlap_threshold: float = 0.7,
    strict_size_limit: bool = True,
    size_tolerance: float = 0.05,
)

def get_supported_languages(self) -> List[str]

chunker = MultilanguageTextChunker()
print(chunker.get_supported_languages())
# ['multilingual', 'english', 'arabic', 'chinese', 'french', ...]

def switch_language(self, language: str) -> None

chunker = MultilanguageTextChunker(language="english")
chunks_en = chunker.chunk(english_text)

chunker.switch_language("arabic")
chunks_ar = chunker.chunk(arabic_text)

def split_documents(self, documents: List[Document]) -> List[Document]

def create_documents(
    self,
    texts: List[str],
    metadatas: Optional[List[Dict[str, Any]]] = None,
) -> List[Document]

from fennec_community.chunks import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(chunk_size=300, chunk_overlap=50)

docs = splitter.create_documents(
    texts=["Long document one...", "Long document two..."],
    metadatas=[{"source": "file1.txt"}, {"source": "file2.txt"}],
)

CharacterTextSplitter(
    separator: str = "\n\n",
    chunk_size: int = 1000,
    chunk_overlap: int = 200,
    length_function: Callable = len,
)

def split_text(self, text: str) -> List[str]

RecursiveCharacterTextSplitter(
    separators: Optional[List[str]] = None,
    arabic_mode: bool = False,
    chunk_size: int = 1000,
    chunk_overlap: int = 200,
    length_function: Callable = len,
)

def split_text(self, text: str) -> List[str]

TokenTextSplitter(
    encoding_name: str = "cl100k_base",
    chunk_size: int = 1000,
    chunk_overlap: int = 200,
    length_function: Callable = len,
)

def split_text(self, text: str) -> List[str]

SentenceTextSplitter(
    chunk_size: int = 1000,
    chunk_overlap: int = 200,
    length_function: Callable = len,
)

def split_text(self, text: str) -> List[str]

from fennec_community.chunks import cosine_similarity_matrix

cosine_similarity_matrix(A: np.ndarray, B: np.ndarray) -> np.ndarray

import numpy as np
from fennec_community.chunks import cosine_similarity_matrix, EmbeddingProvider

provider = EmbeddingProvider("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
texts_a = ["Hello world", "Machine learning is great"]
texts_b = ["Hi there", "AI is fascinating", "Cooking recipes"]

A = np.array(provider.embed_batch(texts_a))
B = np.array(provider.embed_batch(texts_b))

sim_matrix = cosine_similarity_matrix(A, B)
# sim_matrix.shape == (2, 3)
print(sim_matrix)

EmbeddingProvider(
    model_name: str,
    device: Optional[str] = None,
    cache_size: int = 10_000,
)

def embed(self, text: str) -> np.ndarray

def embed_batch(self, texts: List[str], batch_size: int = 64) -> List[np.ndarray]

def pairwise_similarity(self, a: str, b: str) -> float

def adjacent_similarities(self, sentences: List[str]) -> List[float]

MetadataEnricher(
    max_keywords: int = 10,
    header_score_boost: float = 1.5,
)

def enrich(self, chunks: List[DocumentChunk]) -> List[DocumentChunk]

def enrich_single(self, chunk: DocumentChunk) -> DocumentChunk

Deduplicator(
    use_hash: bool = True,
    use_similarity: bool = False,
    similarity_threshold: float = 0.95,
    embedding_model: Optional[str] = None,
    device: Optional[str] = None,
)

def deduplicate(self, chunks: List[DocumentChunk]) -> List[DocumentChunk]

DocumentStructureParser()

def detect_document_type(self, text: str) -> DocumentType

def parse(self, text: str, doc_type: DocumentType) -> List[StructuredSection]

QueryPatternAnalyzer(default_strategy: str = "general")

def analyze_queries(self, queries: List[str]) -> ChunkingStrategy

def analyze_document(self, text: str) -> ChunkingStrategy

def analyze(
    self,
    text: str,
    sample_queries: Optional[List[str]] = None,
) -> ChunkingStrategy

def get_strategy(self, name: str) -> ChunkingStrategy

@dataclass
class DocumentChunk:
    text: str                          # The chunk's text content
    chunk_id: str                      # UUID (auto-generated)
    doc_id: str                        # Parent document ID
    embedding: Optional[np.ndarray]    # Vector embedding (if computed)
    metadata: ChunkMetadata            # Rich metadata

@dataclass
class ChunkMetadata:
    source: str            # Document origin (filename, URL)
    page: Optional[int]    # Page number (if applicable)
    section: Optional[str] # Section/heading title
    position: int          # Index of this chunk in the document (0-based)
    total_chunks: int      # Total chunks in the parent document
    chunk_type: ChunkType  # Type classification
    document_type: DocumentType
    language: str
    keywords: List[str]    # Top TF-IDF keywords
    score: float           # Importance score [0.0 – 1.0]
    keyword_density: float # Ratio of meaningful words
    is_header: bool        # True if this chunk is a document heading
    heading_level: int     # 0 = not a header; 1–6 = H1–H6
    char_start: int        # Character offset in original document
    char_end: int          # Character offset end in original document
    extra: Dict[str, Any]  # Chunker-specific extra data

@dataclass
class StructuredSection:
    text: str                  # The section's raw text content
    heading: str               # Title of the nearest parent heading (empty if none)
    heading_level: int         # 0 = body text; 1–6 = H1–H6
    page: Optional[int]        # Page number if available (e.g. from PDF parsers)
    section_index: int         # Sequential index of this section within the document
    is_code_block: bool        # True if the section is a fenced code block
    is_list: bool              # True if the section is a list (ordered or unordered)
    is_table: bool             # True if the section is a table
    char_start: int            # Character offset of the section start in the original text
    char_end: int              # Character offset of the section end in the original text
    extra: Dict[str, Any]      # Parser-specific additional data

from fennec_community.chunks import DocumentStructureParser

parser = DocumentStructureParser()
doc_type = parser.detect_document_type(markdown_text)
sections = parser.parse(markdown_text, doc_type)

for s in sections:
    print(f"[H{s.heading_level}] {s.heading!r} — code={s.is_code_block} list={s.is_list}")
    print(f"  chars [{s.char_start}:{s.char_end}]: {s.text[:60]}")

@dataclass
class Document:
    page_content: str            # The raw text
    metadata: Dict[str, Any]     # Arbitrary key-value metadata
    doc_id: str                  # Auto-generated UUID

from fennec_community.chunks import BaseChunker, DocumentChunk
from typing import List

class MyChunker(BaseChunker):
    def _chunk_impl(self, text: str, doc_id: str, source: str) -> List[DocumentChunk]:
        # implement your splitting logic here
        ...

from fennec_community.chunks import TextSplitter
from typing import List

class MyTextSplitter(TextSplitter):
    def split_text(self, text: str) -> List[str]:
        # implement your splitting logic here
        ...

from fennec_community.chunks import ChunkConfig

config = ChunkConfig(
    chunk_size=512,
    overlap=128,
    use_semantic_chunking=True,
    extract_keywords=True,
    deduplication_enabled=True,
)

config = ChunkConfig(
    use_semantic_chunking=True,
    arabic="CAMeL-Lab/bert-base-arabic-camelbert-ca",   # swap Arabic model
    french="camembert/camembert-large",                  # swap French model
)

from fennec_community.chunks import ChunkMode

ChunkMode.AUTO        # ChunkManager picks strategy based on config flags
ChunkMode.SEMANTIC    # Force SemanticChunker
ChunkMode.ADAPTIVE    # Force AdaptiveChunker
ChunkMode.STRUCTURAL  # Force StructureAwareChunker
ChunkMode.CONTEXTUAL  # Force ContextAwareChunker
ChunkMode.HYBRID      # StructureAwareChunker → SemanticChunker on large sections

from fennec_community.chunks import ChunkManager, ChunkConfig, ChunkMode

config = ChunkConfig(
    chunk_size=512,
    overlap=128,
    use_semantic_chunking=True,
    extract_keywords=True,
    deduplication_enabled=True,
    query_aware=True,
)

manager = ChunkManager(config=config, mode=ChunkMode.AUTO)

with open("report.md", "r", encoding="utf-8") as f:
    text = f.read()

chunks = manager.process_with_query_optimization(
    text=text,
    queries=["What are the financial results?", "Who is the CEO?"],
    source="report.md",
)

print(manager.get_stats(chunks))

for chunk in chunks:
    vector = embed_model.encode(chunk.text)   # your embedding model
    vector_db.insert(chunk.chunk_id, vector, chunk.to_dict())

from fennec_community.chunks import ArabicTextChunker

chunker = ArabicTextChunker(
    chunk_size=400,
    overlap=100,
    use_semantic_chunking=True,
    fix_spacing=True,
)

chunks = chunker.chunk(arabic_text, source="arabic_doc.pdf")

for ch in chunks:
    print(f"[{ch.metadata.position}] {ch.text[:60]}...")
    print(f"  keywords: {ch.metadata.keywords}")

from fenenc_community.chunks import ChunkManager, Document

manager = ChunkManager(language="auto")

documents = [
    Document(page_content=english_text, metadata={"source": "en.txt", "lang": "en"}),
    Document(page_content=arabic_text,  metadata={"source": "ar.txt", "lang": "ar"}),
    Document(page_content=french_text,  metadata={"source": "fr.txt", "lang": "fr"}),
]

all_chunks = manager.process_documents(documents)
print(f"Total chunks: {len(all_chunks)}")

from fenenc_community.chunks import RecursiveCharacterTextSplitter, TokenTextSplitter

# Token-aware splitting for OpenAI models
splitter = TokenTextSplitter(
    encoding_name="cl100k_base",
    chunk_size=500,
    chunk_overlap=50,
)
text_chunks = splitter.split_text(long_text)

# Arabic recursive splitting
arabic_splitter = RecursiveCharacterTextSplitter(
    arabic_mode=True,
    chunk_size=400,
    chunk_overlap=80,
)
arabic_chunks = arabic_splitter.split_text(arabic_text)

from fenenc_community.chunks import MetadataEnricher, Deduplicator

enricher = MetadataEnricher(max_keywords=8, header_score_boost=2.0)
deduplicator = Deduplicator(use_hash=True, use_similarity=False)

# After any chunking operation:
chunks = enricher.enrich(chunks)
chunks = deduplicator.deduplicate(chunks)

# Sort by importance score
chunks.sort(key=lambda c: c.metadata.score, reverse=True)
top_chunks = chunks[:10]

# ChunkManager always disables similarity dedup internally.
# Use Deduplicator directly when you need near-duplicate removal:
from fenenc_community.chunks import ChunkManager, Deduplicator

manager = ChunkManager()
chunks = manager.process(text, source="doc.pdf")

deduplicator = Deduplicator(
    use_hash=True,
    use_similarity=True,
    similarity_threshold=0.92,
    embedding_model="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
)
chunks = deduplicator.deduplicate(chunks)

from fenenc_community.chunks import ChunkConfig, MultilanguageTextChunker

config = ChunkConfig(
    use_semantic_chunking=True,
    arabic="CAMeL-Lab/bert-base-arabic-camelbert-ca",
    french="camembert/camembert-large",
    embedding_batch_size=32,   # lower batch size on limited GPU memory
    embedding_cache_size=5_000,
)

chunker = MultilanguageTextChunker(language="arabic")
chunks = chunker.chunk(arabic_text, source="doc.pdf")

from fennec_community.chunks import DocumentStructureParser

parser = DocumentStructureParser()
doc_type = parser.detect_document_type(md_text)
sections = parser.parse(md_text, doc_type)

code_sections = [s for s in sections if s.is_code_block]
headers = [s for s in sections if s.heading_level > 0]

print(f"Found {len(code_sections)} code blocks and {len(headers)} headings")