Page History

import os
import time
import logging
from datetime import datetime, timedelta, timezone

import requests
import pymongo
from dotenv import load_dotenv

load_dotenv()

# Налаштування
MONGO_URI = os.getenv("MONGO_URI", "mongodb://localhost:27017/")
DB_NAME = os.getenv("DB_NAME", "testdb")
COLLECTION_NAME = os.getenv("COLLECTION_NAME", "cbd3_procedures")
BASE_URL = "https://procedure.prozorro.sale/api/search/byDateModified/"
START_DATE = "2021-10-01T00:00:00.000001Z"
LIMIT = 100
REQUEST_TIMEOUT = 30
NO_PROGRESS_LIMIT = 5  # скільки разів підряд можна "не просунутись", перш ніж зупинитись

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)

client = pymongo.MongoClient(MONGO_URI)
db = client[DB_NAME]
collection = db[COLLECTION_NAME]

def parse_z(dt_str: str) -> datetime:
    # "2025-05-07T17:25:32.907000Z" -> aware datetime UTC
    if dt_str.endswith("Z"):
        dt_str = dt_str[:-1]
    return datetime.fromisoformat(dt_str).replace(tzinfo=timezone.utc)

def to_z(dt: datetime) -> str:
    return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")

def fetch_and_store_data():
    date_modified = parse_z(START_DATE)
    no_progress_streak = 0
    last_url = None

    try:
        while True:
            url = f"{BASE_URL}{to_z(date_modified)}?limit={LIMIT}"
            if url == last_url:
                no_progress_streak += 1
                logger.warning(f"Same URL encountered again ({no_progress_streak}/{NO_PROGRESS_LIMIT}).")
                if no_progress_streak >= NO_PROGRESS_LIMIT:
                    logger.info("No progress guard triggered. Stopping.")
                    break
            else:
                no_progress_streak = 0

            last_url = url
            logger.info(f"Fetching data from: {url}")

            try:
                resp = requests.get(url, timeout=REQUEST_TIMEOUT)
                resp.raise_for_status()
            except requests.RequestException as e:
                logger.error(f"Request failed: {e}")
                break

            data = resp.json()

            # Порожній або не список — фініш
            if not isinstance(data, list) or not data:
                logger.info("No more data to fetch. Stopping execution.")
                break

            # Записуємо/оновлюємо
            inserted_count = 0
            for item in data:
                res = collection.update_one(
                    {"_id": item["_id"]},
                    {"$set": item},
                    upsert=True
                )
                if res.upserted_id:
                    inserted_count += 1

            logger.info(f"Inserted {inserted_count} new records into MongoDB.")

            # Вираховуємо наступний курсор
            # Беремо максимальний dateModified з поточної порції
            try:
                max_date_str = max(obj["dateModified"] for obj in data)
                batch_max = parse_z(max_date_str)
            except (KeyError, ValueError, TypeError) as e:
                logger.error(f"Bad dateModified in response: {e}. Stopping.")
                break

            next_cursor = max(date_modified, batch_max) + timedelta(microseconds=1)

            if next_cursor <= date_modified:
                # safety (по ідеї не має спрацьовувати, але лишимо)
                no_progress_streak += 1
                logger.warning(f"Cursor did not advance (streak {no_progress_streak}/{NO_PROGRESS_LIMIT}).")
                if no_progress_streak >= NO_PROGRESS_LIMIT:
                    logger.info("No progress guard triggered. Stopping.")
                    break
            else:
                date_modified = next_cursor
                logger.info(f"Next dateModified: {to_z(date_modified)}")

            time.sleep(1)  # не душимо API
    finally:
        client.close()

if __name__ == "__main__":
    fetch_and_store_data()