mirror of
https://git.datalinker.icu/deepseek-ai/DeepSeek-V3.git
synced 2025-12-09 04:44:28 +08:00
e44e45bfc5
Comprehensive intelligence retrieval system for collecting and aggregating information about Nairobi, Kenya from multiple sources. Features: - Multi-source data collection (news, social media, government, tourism, business) - RESTful API with FastAPI - Automated scheduling for continuous data collection - Intelligence brief generation - Real-time trending topics tracking - Alert system for important updates - Web scraping with rate limiting and caching - Social media integration (Twitter, Instagram) - NLP-powered categorization and processing - Docker support for easy deployment - CLI for manual operations Components: - Data models with SQLAlchemy - Base collector class with extensible architecture - Source-specific collectors (news, social, government, tourism, business) - Data processor for brief generation - Scheduler for automated collection - Comprehensive API endpoints - CLI interface for manual control Documentation: - Complete README with setup instructions - Quick start guide - Example usage scripts - Docker Compose configuration - Environment configuration templates
73 lines
1.5 KiB
Python
73 lines
1.5 KiB
Python
"""
|
|
Database connection and initialization
|
|
"""
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import sessionmaker, Session
|
|
from typing import Generator
|
|
import logging
|
|
|
|
from app.config import get_settings
|
|
from app.models.data_models import Base
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
settings = get_settings()
|
|
|
|
# Create database engine
|
|
engine = create_engine(
|
|
settings.database_url,
|
|
echo=settings.debug,
|
|
pool_pre_ping=True,
|
|
pool_size=10,
|
|
max_overflow=20
|
|
)
|
|
|
|
# Create session factory
|
|
SessionLocal = sessionmaker(
|
|
autocommit=False,
|
|
autoflush=False,
|
|
bind=engine
|
|
)
|
|
|
|
|
|
def get_db() -> Generator[Session, None, None]:
|
|
"""
|
|
Get database session
|
|
|
|
Yields:
|
|
Database session
|
|
"""
|
|
db = SessionLocal()
|
|
try:
|
|
yield db
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def init_db() -> None:
|
|
"""
|
|
Initialize database - create all tables
|
|
"""
|
|
try:
|
|
logger.info("Creating database tables...")
|
|
Base.metadata.create_all(bind=engine)
|
|
logger.info("Database tables created successfully!")
|
|
except Exception as e:
|
|
logger.error(f"Error creating database tables: {e}")
|
|
raise
|
|
|
|
|
|
def drop_db() -> None:
|
|
"""
|
|
Drop all database tables (use with caution!)
|
|
"""
|
|
logger.warning("Dropping all database tables...")
|
|
Base.metadata.drop_all(bind=engine)
|
|
logger.info("Database tables dropped!")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Initialize database when run directly
|
|
logging.basicConfig(level=logging.INFO)
|
|
init_db()
|