# Batch Processing Rules

## Core Principles
1. **Queue Everything**: Process through queues, not direct calls
2. **Fail Independently**: One file failure doesn't stop the batch
3. **Track Progress**: Real-time visibility into batch status
4. **Recover Gracefully**: Resume from last successful point

## Queue Management Rules

### Job Structure
```python
class BatchJob:
    """Every batch job MUST have these fields"""
    id: UUID
    status: Literal["pending", "processing", "completed", "failed", "partial"]
    total_files: int
    processed_files: int
    successful_files: int
    failed_files: int
    start_time: datetime
    end_time: Optional[datetime]
    error_log: List[Dict]
    settings: Dict  # Processing configuration
```

### Priority Rules
```python
# Priority levels (higher number = higher priority)
PRIORITY_IMMEDIATE = 100  # Single files, <5 min
PRIORITY_HIGH = 75       # Small batches, <10 files
PRIORITY_NORMAL = 50     # Standard batches, <50 files
PRIORITY_LOW = 25        # Large batches, >50 files
PRIORITY_BACKGROUND = 0  # Maintenance, re-processing

# Assignment rules
if file_count == 1:
    priority = PRIORITY_IMMEDIATE
elif file_count <= 10:
    priority = PRIORITY_HIGH
elif file_count <= 50:
    priority = PRIORITY_NORMAL
else:
    priority = PRIORITY_LOW
```

## Processing Rules

### Parallel Processing
```python
# REQUIRED: Respect system limits
MAX_PARALLEL_WORKERS = min(cpu_count(), 8)
MAX_MEMORY_PER_WORKER = 2 * GB
MAX_QUEUE_DEPTH = 1000

# REQUIRED: Worker configuration
worker_config = {
    "timeout": 600,  # 10 minutes max per file
    "retry_count": 3,
    "backoff_factor": 2,
    "memory_limit": MAX_MEMORY_PER_WORKER
}
```

### Batch Optimization
```python
# REQUIRED: Batch files by characteristics
def optimize_batch(files: List[Path]) -> List[List[Path]]:
    """Group files for efficient processing"""
    batches = {
        "small": [],    # <10MB, process many in parallel
        "medium": [],   # 10-100MB, standard processing
        "large": [],    # >100MB, process sequentially
    }
    
    for file in files:
        size = file.stat().st_size
        if size < 10 * MB:
            batches["small"].append(file)
        elif size < 100 * MB:
            batches["medium"].append(file)
        else:
            batches["large"].append(file)
    
    return batches
```

### Progress Tracking
```python
# REQUIRED: Update progress atomically
async def update_progress(job_id: UUID, file: Path, status: str):
    async with db.transaction():
        job = await get_job(job_id)
        job.processed_files += 1
        
        if status == "success":
            job.successful_files += 1
        else:
            job.failed_files += 1
            job.error_log.append({
                "file": str(file),
                "error": status,
                "timestamp": datetime.utcnow()
            })
        
        # Update percentage
        job.progress = (job.processed_files / job.total_files) * 100
        
        # Check if complete
        if job.processed_files == job.total_files:
            job.status = "completed" if job.failed_files == 0 else "partial"
            job.end_time = datetime.utcnow()
        
        await save_job(job)
```

## Error Handling Rules

### File-Level Errors
```python
# REQUIRED: Capture and continue
try:
    result = await process_file(file)
    await mark_success(job_id, file, result)
except Exception as e:
    await mark_failure(job_id, file, str(e))
    # Continue with next file
    continue  # NEVER break the batch
```

### Batch-Level Errors
```python
# REQUIRED: Graceful degradation
if worker_crashed:
    # Restart worker
    restart_worker(worker_id)
    # Requeue unprocessed files
    requeue_files(worker.pending_files)
    # Continue with reduced capacity
    continue_with_fewer_workers()

if database_error:
    # Switch to file-based tracking
    write_progress_to_file(job_id)
    # Continue processing
    continue_without_db()
    # Sync when database returns
    schedule_sync_task()
```

### Recovery Rules
```python
# REQUIRED: Resume capability
def resume_batch(job_id: UUID):
    """Resume a failed or stopped batch"""
    job = load_job(job_id)
    
    # Find unprocessed files
    processed = set(job.completed_files + job.failed_files)
    all_files = set(job.all_files)
    remaining = all_files - processed
    
    # Reset job status
    job.status = "processing"
    job.resume_count += 1
    
    # Process remaining
    return process_files(remaining, job_id)
```

## Resource Management Rules

### Memory Management
```python
# REQUIRED: Enforce memory limits
@memory_limit(2 * GB)
async def process_with_limit(file: Path):
    """Process with memory cap"""
    # Monitor memory during processing
    if get_memory_usage() > 1.8 * GB:
        # Flush caches
        clear_caches()
        # Garbage collect
        gc.collect()
        # If still high, chunk the file
        if get_memory_usage() > 1.8 * GB:
            return process_in_chunks(file)
    
    return process_normal(file)
```

### Disk Management
```python
# REQUIRED: Clean up continuously
async def cleanup_temp_files(job_id: UUID):
    """Remove temporary files during processing"""
    temp_dir = Path(f"/tmp/trax/{job_id}")
    
    # Clean up completed files immediately
    for file in temp_dir.glob("*.completed"):
        file.unlink()
    
    # Clean up old files (>1 hour)
    for file in temp_dir.glob("*"):
        if file.stat().st_mtime < time.time() - 3600:
            file.unlink()
```

## Monitoring Rules

### Required Metrics
```python
batch_metrics = {
    # Performance
    "files_per_second": processed_files / elapsed_time,
    "average_file_time": total_time / processed_files,
    "queue_depth": len(pending_files),
    
    # Reliability
    "success_rate": successful_files / processed_files,
    "error_rate": failed_files / processed_files,
    "retry_rate": retried_files / processed_files,
    
    # Resources
    "cpu_utilization": current_cpu_percent,
    "memory_usage": current_memory_mb,
    "disk_usage": temp_directory_size,
    
    # Business
    "cost_per_file": total_api_cost / processed_files,
    "throughput": completed_batches / hour
}
```

### Alert Thresholds
```python
alerts = {
    "queue_depth_high": queue_depth > 500,
    "success_rate_low": success_rate < 0.95,
    "processing_slow": files_per_second < 0.1,
    "memory_high": memory_usage > 0.9 * limit,
    "disk_full": disk_usage > 0.9 * available,
    "cost_high": cost_per_file > 0.02
}
```

## Testing Rules

### Batch Test Scenarios
```python
# REQUIRED: Test these scenarios
test_scenarios = [
    "empty_batch",           # 0 files
    "single_file",          # 1 file
    "small_batch",          # 10 files
    "medium_batch",         # 50 files
    "large_batch",          # 100+ files
    "mixed_formats",        # Various file types
    "mixed_sizes",          # 1KB to 1GB files
    "with_failures",        # 20% fail rate
    "with_timeout",         # Slow processing
    "parallel_batches",     # Multiple concurrent
    "resume_partial",       # Resume after crash
    "out_of_memory",        # Memory pressure
    "disk_full",           # Storage limits
]
```

### Performance Benchmarks
```python
benchmarks = {
    "10_files": {"expected": 60, "max": 120},      # seconds
    "50_files": {"expected": 300, "max": 600},
    "100_files": {"expected": 600, "max": 1200},
    "500_files": {"expected": 3000, "max": 6000},
}
```

## Database Rules

### Schema Requirements
```sql
-- Batch jobs table
CREATE TABLE batch_jobs (
    id UUID PRIMARY KEY,
    status VARCHAR(20) NOT NULL,
    total_files INTEGER NOT NULL,
    processed_files INTEGER DEFAULT 0,
    successful_files INTEGER DEFAULT 0,
    failed_files INTEGER DEFAULT 0,
    progress FLOAT DEFAULT 0.0,
    settings JSONB,
    error_log JSONB DEFAULT '[]',
    created_at TIMESTAMP DEFAULT NOW(),
    updated_at TIMESTAMP DEFAULT NOW(),
    start_time TIMESTAMP,
    end_time TIMESTAMP,
    
    -- Indexes for performance
    INDEX idx_status (status),
    INDEX idx_created (created_at),
    INDEX idx_progress (progress)
);

-- Batch files table
CREATE TABLE batch_files (
    id UUID PRIMARY KEY,
    job_id UUID REFERENCES batch_jobs(id),
    file_path TEXT NOT NULL,
    status VARCHAR(20) NOT NULL,
    result JSONB,
    error TEXT,
    processing_time FLOAT,
    created_at TIMESTAMP DEFAULT NOW(),
    
    -- Indexes
    INDEX idx_job_status (job_id, status),
    UNIQUE INDEX idx_job_file (job_id, file_path)
);
```

## CLI Rules

### Required Commands
```bash
# Start batch
trax batch process /path/to/files --parallel 4

# Check status
trax batch status <job_id>

# List all batches
trax batch list --status processing

# Resume failed batch
trax batch resume <job_id>

# Cancel batch
trax batch cancel <job_id>

# Retry failed files
trax batch retry <job_id> --failed-only

# Export results
trax batch export <job_id> --format json
```

### Output Requirements
```
Processing batch job: 7d4f3a2b-8c9e-4f1a-b3d2-1a8e9c7f6d5e
[████████████████░░░░] 80% | 40/50 files | 2 failed | ETA: 2m 15s
Current: processing_audio_lecture.mp4 (3.2 MB)
Speed: 0.8 files/sec | Avg time: 1.25s/file
Errors: 2 files failed (see logs for details)
```

---

*These rules ensure reliable, efficient batch processing at scale.*