Merge branch 'add-parser' into 'master'

poetry.lock

@@ -185,6 +185,44 @@ tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothe
 tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""]
 
 [[package]]
+name = "beautifulsoup4"
+version = "4.13.4"
+description = "Screen-scraping library"
+optional = false
+python-versions = ">=3.7.0"
+groups = ["main"]
+files = [
+    {file = "beautifulsoup4-4.13.4-py3-none-any.whl", hash = "sha256:9bbbb14bfde9d79f38b8cd5f8c7c85f4b8f2523190ebed90e950a8dea4cb1c4b"},
+    {file = "beautifulsoup4-4.13.4.tar.gz", hash = "sha256:dbb3c4e1ceae6aefebdaf2423247260cd062430a410e38c66f2baa50a8437195"},
+]
+
+[package.dependencies]
+soupsieve = ">1.2"
+typing-extensions = ">=4.0.0"
+
+[package.extras]
+cchardet = ["cchardet"]
+chardet = ["chardet"]
+charset-normalizer = ["charset-normalizer"]
+html5lib = ["html5lib"]
+lxml = ["lxml"]
+
+[[package]]
+name = "bs4"
+version = "0.0.2"
+description = "Dummy package for Beautiful Soup (beautifulsoup4)"
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = [
+    {file = "bs4-0.0.2-py2.py3-none-any.whl", hash = "sha256:abf8742c0805ef7f662dce4b51cca104cffe52b835238afc169142ab9b3fbccc"},
+    {file = "bs4-0.0.2.tar.gz", hash = "sha256:a48685c58f50fe127722417bae83fe6badf500d54b55f7e39ffe43b798653925"},
+]
+
+[package.dependencies]
+beautifulsoup4 = "*"
+
+[[package]]
 name = "certifi"
 version = "2025.7.9"
 description = "Python package for providing Mozilla's CA Bundle."
@@ -1034,6 +1072,18 @@ files = [
 cli = ["click (>=5.0)"]
 
 [[package]]
+name = "soupsieve"
+version = "2.7"
+description = "A modern CSS selector implementation for Beautiful Soup."
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "soupsieve-2.7-py3-none-any.whl", hash = "sha256:6e60cc5c1ffaf1cebcc12e8188320b72071e922c2e897f737cadce79ad5d30c4"},
+    {file = "soupsieve-2.7.tar.gz", hash = "sha256:ad282f9b6926286d2ead4750552c8a6142bc4c783fd66b0293547c8fe6ae126a"},
+]
+
+[[package]]
 name = "telegrinder"
 version = "0.5.1"
 description = "Modern visionary telegram bot framework."
@@ -1228,4 +1278,4 @@ propcache = ">=0.2.1"
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.12,<4.0"
-content-hash = "a510528d6af1e3839834d33b34c1b07dd8842f5755c5d861429e32bf4432e0af"
+content-hash = "a931f25d5878e10204c31cdf053d05a4587b3b5c55056e719c2f1489525c1084"

pyproject.toml

@@ -12,7 +12,8 @@ dependencies = [
     "telegrinder (>=0.5.1,<0.6.0)",
     "pydantic (>=2.11.7,<3.0.0)",
     "pydantic-settings (>=2.10.1,<3.0.0)",
-    "peewee (>=3.18.2,<4.0.0)"
+    "peewee (>=3.18.2,<4.0.0)",
+    "bs4 (>=0.0.2,<0.0.3)"
 ]
 
 [project.urls]

src/altrepo/parser/methods.py

 import aiohttp
 
-from typing import Literal
-
 from . import models
+from .news import urls_parser, packages_parser, bugs_parser
 
 
 class BaseParser:
@@ -10,21 +9,39 @@ class BaseParser:
     def __init__(self, session: aiohttp.ClientSession):
         self.session = session
 
-    async def get(self, url: str):
+    async def get(self, url: str, encoding: str = "utf-8"):
         async with self.session.get(url) as resp:
             resp.raise_for_status()
-            return await resp.text()
+            return await resp.text(encoding=encoding)
 
 
 class NewsInfo:
     def __init__(self, client: BaseParser):
         self.client = client
 
-    async def news_url(self, news_type: Literal["sisyphus", "p11", "bugs"]):
-        ...
-
-    async def sisyphus(self):
-        ...
+    async def news_urls(self) -> models.NewsURL:
+        return await urls_parser(self.client)
+
+    async def bugs(self) -> models.BugsModel:
+        url = (await self.news_urls()).bugs
+        if not url:
+            return None
+        html = await self.client.get(url, "koi8-r")
+        return await bugs_parser(html, url)
+
+    async def sisyphus(self) -> models.PackagesModel | None:
+        url = (await self.news_urls()).sisyphus
+        if not url:
+            return None
+        html = await self.client.get(url, "koi8-r")
+        return await packages_parser(html, url)
+        
+    async def p11(self) -> models.PackagesModel | None:
+        url = (await self.news_urls()).p11
+        if not url:
+            return None
+        html = await self.client.get(url, "koi8-r")
+        return await packages_parser(html, url)
 
 
 class ALTRepoParser:

src/altrepo/parser/models.py

 from pydantic import BaseModel
 from typing import List, Dict, Any
+
+
+class NewsURL(BaseModel):
+    sisyphus: str | None = None
+    p11: str | None = None
+    bugs: str | None = None
+
+
+class BugsElementModel(BaseModel):
+    id: int
+    component: str
+    priority: str
+    status: str | None = None
+    summary: str
+
+
+class BugsModel(BaseModel):
+    url: str
+    quickly_resolved: List[BugsElementModel] | None = None
+    new: List[BugsElementModel] | None = None
+    old: List[BugsElementModel] | None = None
+    resolved: List[BugsElementModel] | None = None
+    reopened: List[BugsElementModel] | None = None
+    random: List[BugsElementModel] | None = None
+
+
+class RemovedPackageElementModel(BaseModel):
+    name: str
+    version: str
+
+
+class PackageElementModel(BaseModel):
+    name: str
+    description: str
+    maintainer_name: str
+    maintainer_nick: str
+
+
+class PackagesModel(BaseModel):
+    url: str
+    added: List[PackageElementModel] | None = None
+    removed: List[RemovedPackageElementModel] | None = None
+    updated: List[PackageElementModel] | None = None

src/altrepo/parser/news/__init__.py

+from .urls import urls_parser
+from .packages import packages_parser
+from .bugs import bugs_parser

src/altrepo/parser/news/bugs.py

+from bs4 import BeautifulSoup
+
+import re
+
+from .. import models
+
+async def bugs_parser(html: str, url: str):
+    soup = BeautifulSoup(html, "html.parser")
+    pre_tag = soup.find("pre")
+    if not pre_tag:
+        return {}
+    text = pre_tag.get_text()
+
+    lines = text.strip().splitlines()
+    data = {}
+    section_name = None
+    bug_pattern = re.compile(r"#(\d+)\s+([^\t]+)\s+([^\t]+)\s+([^\t]+)")
+    description_buffer = ""
+    current_bug = None
+
+    for line in lines:
+        header_match = re.match(
+            r"^\s*(\d+)\s+(NEW|RESOLVED|REOPENED|RANDOM)\s+bugs?.*", line)
+        if header_match:
+            if current_bug and section_name:
+                current_bug["summary"] = description_buffer.strip()
+                data.setdefault(section_name, []).append(current_bug)
+                current_bug = None
+                description_buffer = ""
+
+            section_name = await _get_bug_section_name(line)
+            continue
+
+        bug_match = bug_pattern.match(line)
+        if bug_match:
+            if current_bug and section_name:
+                current_bug["summary"] = description_buffer.strip()
+                data.setdefault(section_name, []).append(current_bug)
+
+            status = bug_match.group(4).strip()
+            if status == "---":
+                status = None
+
+            current_bug = {
+                "id": int(bug_match.group(1)),
+                "component": bug_match.group(2).strip(),
+                "priority": bug_match.group(3).strip(),
+                "status": status
+            }
+            description_buffer = ""
+        elif current_bug:
+            if not line.strip():
+                continue
+            if re.match(r"^Total\s+\d+\s+pending bugs", line.strip(), re.IGNORECASE):
+                continue
+            description_buffer += line.strip() + " "
+
+    if current_bug and section_name:
+        current_bug["summary"] = description_buffer.strip()
+        data.setdefault(section_name, []).append(current_bug)
+    
+    data["url"] = url
+
+    return models.BugsModel(**data)
+
+
+async def _get_bug_section_name(line: str) -> str:
+    line = line.lower()
+    if "new" in line and "resolved" in line:
+        return "quickly_resolved"
+    elif "new" in line:
+        return "new"
+    elif "old" in line:
+        return "old"
+    elif "resolved" in line:
+        return "resolved"
+    elif "reopened" in line:
+        return "reopened"
+    elif "random" in line:
+        return "random"

src/altrepo/parser/news/packages.py

+from bs4 import BeautifulSoup
+
+import re
+
+from .. import models
+
+async def packages_parser(html: str, url:  str):
+    soup = BeautifulSoup(html, "html.parser")
+    pre_tag = soup.find("pre")
+    if not pre_tag:
+        return {}
+    text = pre_tag.get_text()
+
+    lines = text.strip().splitlines()
+    sections = {"added": [], "removed": [], "updated": []}
+    current_section = None
+    current_package = {}
+
+    for line in lines:
+        # Смена секции
+        if re.search(r"^\s*\d+\s+ADDED package[s]?", line):
+            if current_package and current_section:
+                sections[current_section].append(current_package)
+            current_section = "added"
+            current_package = {}
+            continue
+        elif re.search(r"^\s*\d+\s+REMOVED package[s]?", line):
+            if current_package and current_section:
+                sections[current_section].append(current_package)
+            current_section = "removed"
+            current_package = {}
+            continue
+        elif re.search(r"^\s*\d+\s+UPDATED packages[s]?", line):
+            if current_package and current_section:
+                sections[current_section].append(current_package)
+            current_section = "updated"
+            current_package = {}
+            continue
+
+        if current_section == "removed":
+            if line.strip():
+                parts = line.strip().split("\t")
+                if len(parts) == 2:
+                    name, version = parts
+                    sections["removed"].append({
+                        "name": name.strip(),
+                        "version": version.strip()
+                    })
+            continue
+
+        match = re.match(r"^(\S+)\s+-\s+(.*)", line)
+        if match:
+            if current_package:
+                sections[current_section].append(current_package)
+                current_package = {}
+
+            current_package["name"] = match.group(1)
+
+            current_package["description"] = await _clean_description(match.group(2))
+            continue
+
+        match = re.match(
+            r"^\*\s+\w+\s+\w+\s+\d+\s+\d+\s+(.+?) <([a-z0-9._-]+) на altlinux>", line
+        )
+        if match and current_section != "removed":
+            if current_package:
+                current_package["maintainer_name"] = match.group(1).strip()
+                current_package["maintainer_nick"] = match.group(2).strip()
+    if current_package and current_section:
+        sections[current_section].append(current_package)
+    
+    sections["url"] = url
+
+    return models.PackagesModel(**sections)
+
+
+async def _clean_description(desc: str):
+    desc = desc.strip()
+    desc = re.sub(r'\s+', ' ', desc)
+    desc = re.sub(r'\[\d+[KMG]?\]', '', desc).strip()
+    return desc

src/altrepo/parser/news/urls.py

+from bs4 import BeautifulSoup
+
+from datetime import datetime
+import re
+
+from .. import models
+
+from config import CYBERTALK_URL
+
+
+async def urls_parser(client):
+
+    now = datetime.now()
+    year_month = f"{now.year}-{now.strftime("%B")}"
+    today = now.strftime("%Y%m%d")
+    base_url = CYBERTALK_URL.format(year_month)
+    html = await client.get(f"{base_url}date.html", "koi8-r")
+    soup = BeautifulSoup(html, "html.parser")
+
+    sisyphus_id = 0
+    result = {"sisyphus": None, "bugs": None, "p11": None}
+
+    for li in soup.find_all("li"):
+        a = li.find("a")
+        if not a or not a.get("href"):
+            continue
+
+        href = a["href"]
+        text = a.get_text(strip=True)
+
+        match = re.search(r"(\d+)\.html", href)
+        if not match:
+            continue
+        msg_id = int(match.group(1))
+        url = base_url + href
+
+        if f"Sisyphus-{today} packages" in text:
+            result["sisyphus"] = url
+            sisyphus_id = msg_id
+        elif f"Sisyphus-{today} bugs" in text:
+            result["bugs"] = url
+        elif "p11/branch packages" in text and sisyphus_id:
+            if msg_id > sisyphus_id:
+                result["p11"] = url
+
+    if result["sisyphus"] is None:
+        result["p11"] = None
+
+    return models.NewsURL(**result)

src/config.py

@@ -18,3 +18,5 @@ DEFAUIL_BRANCHES = [
 ]
 
 BUGS_URL = "https://bugzilla.altlinux.org/"
+PACKAGES_URL = "https://packages.altlinux.org/ru/{repo}/"
+CYBERTALK_URL = "https://lists.altlinux.org/pipermail/sisyphus-cybertalk/{}/"
\ No newline at end of file

src/handlers/news.py

+from telegrinder import Dispatch, Message
+from telegrinder.rules import Command, Argument
+
+from altrepo import altrepo
+
+from services.news import format_packages, format_bugs
+
+dp = Dispatch()
+
+@dp.message(Command("news", Argument("news_type", optional=True), Argument("translate", optional=True)))
+async def news_handler(m: Message, news_type: str | None, translate: str | None) -> None:
+    news_type = news_type or "sisyphus"
+    
+    if news_type == "bugs":
+        bugs_data = await altrepo.parser.news.bugs()
+        if not bugs_data:
+            await m.answer("Новостей пока нет.")
+            return
+        bugs = await format_bugs(bugs_data)
+        await m.answer(bugs)
+    else:
+        if news_type == "sisyphus":
+            packages_data = await altrepo.parser.news.sisyphus()
+        elif news_type == "p11":
+            packages_data = await altrepo.parser.news.p11()
+        else:
+            return
+        
+        if not packages_data:
+            await m.answer("Новостей пока нет.")
+            return
+        
+        added, removed, updated, info_message = await format_packages(packages_data, bool(translate))
+        
+        await m.answer(f"{added}{removed}{info_message}")
+        for msg in updated:
+            res = await m.answer(msg)

src/handlers/statistics.py

@@ -29,10 +29,8 @@ async def statistics_handler(m: Message, user: User | None, branch: str | None =
         
     statistics_data = (await altrepo.api.packageset.repository_statistics(branch)).branches[0]
 
-    locale.setlocale(locale.LC_TIME, 'ru_RU.UTF-8')
-
     statistics_date = (datetime.fromisoformat(
-        statistics_data.date_update)).strftime("%-d %B %Y г.")
+        statistics_data.date_update)).strftime("%F")
     
     if "_" in statistics_data.branch:
         branch, arch = statistics_data.branch.split("_")

src/services/news/__init__.py

+from .packages import format_packages
+from .bugs import format_bugs
\ No newline at end of file

src/services/news/bugs.py

+from telegrinder.tools.formatting import HTMLFormatter, link, bold, block_quote
+
+from altrepo.parser.models import BugsModel, BugsElementModel
+
+from config import BUGS_URL, PACKAGES_URL
+
+
+repo = PACKAGES_URL.format(repo="sisyphus")
+
+
+async def format_bugs(bugs: BugsModel):
+    bug_message = HTMLFormatter(bold("Новости Багзиллы\n\n"))
+
+    categories = [
+        ("Быстрозакрытые", bugs.quickly_resolved),
+        ("Новые", bugs.new),
+        ("Устаревшие", bugs.old),
+        ("Закрытые", bugs.resolved),
+        ("Переоткрытые", bugs.reopened),
+        ("Рандомные", bugs.random),
+    ]
+
+    for title, bug_list in categories:
+        if bug_list:
+            bug_message += f"{title}:\n"
+            for bug in bug_list:
+                bug_message += await _format_bug(bug)
+
+    bug_message += (
+        HTMLFormatter(bold("Сделаем Багзиллу чище!\n")) +
+        "— Приглашаем активных участников к проверке багов на воспроизводимость.\n\n"
+    )
+
+    bug_message += (
+        "Источник: " +
+        HTMLFormatter(link(bugs.url, text="sisyphus-cybertalk")) +
+        "\n\n#Репорты\n\n"
+        "Сформировано при помощи " +
+        HTMLFormatter(bold(link("https://t.me/altlinux_packages_bot", text="ALT Packages Bot")))
+    )
+
+    return bug_message
+
+async def _format_bug(bug: BugsElementModel):
+
+    message = HTMLFormatter(
+        bold(link(f"{BUGS_URL}{bug.id}", text=f"#{bug.id}"))) + " | "
+    message += HTMLFormatter(
+        bold(link(f"{repo}{bug.component}", text=f"{bug.component}"))) + " | "
+    message += f"{bug.priority}"
+    message += f" | {bug.status}\n" if bug.status else "\n"
+    message += f"{bug.summary}\n\n"
+
+    return message

src/services/news/packages.py

+from telegrinder.tools.formatting import HTMLFormatter, link, bold
+
+import random
+from collections import defaultdict
+
+from altrepo.parser.models import PackagesModel, PackageElementModel, RemovedPackageElementModel
+
+from config import PACKAGES_URL
+from services.utils import translate_async, chunk_list
+
+
+repo = PACKAGES_URL.format(repo="sisyphus")
+
+
+async def format_packages(packages: PackagesModel, translate: bool | None = None):
+
+    added_message = ""
+    removed_message = ""
+    updated_messages = []
+
+    if packages.added:
+        added_message += "Добавлены:\n"
+        for package in packages.added:
+            added_message += await _format_package(package, translate)
+
+    if packages.removed:
+        removed_message += "Удалены:\n"
+        for package in packages.removed:
+            removed_message += await _format_removed_package(package)
+
+    if packages.updated:
+        updated_messages = await _format_updated_packages(packages, translate)
+
+    info_message = (
+        "Источник: " +
+        HTMLFormatter(link(packages.url, text="sisyphus-cybertalk")) +
+        "\n\n"
+        "Сформировано при помощи " +
+        HTMLFormatter(bold(link("https://t.me/altlinux_packages_bot", text="ALT Packages Bot")))
+    )
+
+    return added_message, removed_message, updated_messages, info_message
+
+
+async def _format_package(package: PackageElementModel, translate: bool):
+    message = HTMLFormatter(
+        bold(link(f"{repo}srpms/{package.name}", text=package.name))
+    ) + f" - {(await translate_async(package.description)) if translate else package.description}\n"
+    message += f"{(await translate_async(package.maintainer_name)) if translate else package.maintainer_name} ({package.maintainer_nick})\n\n"
+
+    return message
+
+
+async def _format_removed_package(package: RemovedPackageElementModel):
+
+    message = HTMLFormatter(
+        bold(link(f"{repo}srpms/{package.name}", text=package.name))) + "\n\n"
+    return message
+
+
+async def _format_updated_packages(packages_model: PackagesModel, translate: bool):
+    updated = packages_model.updated or []
+    total_updated = len(updated)
+
+    grouped = defaultdict(list)
+    for pkg in updated:
+        grouped[(pkg.maintainer_nick, pkg.maintainer_name)].append(pkg)
+    sorted_groups = sorted(grouped.items(), key=lambda x: x[0][0])
+
+    result_pages = []
+
+    for chunk in chunk_list(sorted_groups, 8):
+        lines = [f"Обновлено {total_updated} пакетов, в том числе:"]
+        for (nick, name), pkgs in chunk:
+            selected_pkgs = random.sample(pkgs, min(8, len(pkgs)))
+
+            maintainer_display = (
+                await translate_async(name) if translate else name
+            ) + f" ({nick})"
+
+            for pkg in selected_pkgs:
+                description = (
+                    await translate_async(pkg.description)
+                    if translate else pkg.description
+                )
+                pkg_link = link(f"{repo}srpms/{pkg.name}", text=pkg.name)
+                pkg_name = HTMLFormatter(bold(pkg_link))
+                lines.append(f"{pkg_name} — {description}")
+
+            lines.append(f"* {maintainer_display}\n")
+
+        result_pages.append("\n".join(lines).strip())
+
+    return result_pages

src/services/utils.py

+import asyncio
+
+async def translate_async(text):
+    process = await asyncio.create_subprocess_exec(
+        "trans", "-b", "-e", "bing", text,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE
+    )
+    
+    stdout, stderr = await process.communicate()
+    
+    if process.returncode != 0:
+        raise RuntimeError(f"Translate error: {stderr.decode().strip()}")
+    
+    return stdout.decode().strip()
+
+def chunk_list(lst, size):
+    return [lst[i:i + size] for i in range(0, len(lst), size)]