349 lines
12 KiB
Python
349 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""海信 9 月项目 + qa_raw / qa_product_raw 查询（resource-api / localhost）。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import sys
|
||
import time
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[4] # ai_script
|
||
SKILL_DATA = Path(__file__).resolve().parents[1] / "data"
|
||
DEFAULT_BASE = "https://resource-api.cemeta.cn"
|
||
REQUIRED_END_MONTH = "2026-09"
|
||
|
||
|
||
def load_haoxian_config() -> dict:
|
||
cfg_path = ROOT / "config" / "config.dev.yaml"
|
||
if not cfg_path.exists():
|
||
raise SystemExit(f"缺少配置: {cfg_path}")
|
||
section = None
|
||
hao: dict[str, str] = {}
|
||
for line in cfg_path.read_text().splitlines():
|
||
if line.startswith("haoxian:"):
|
||
section = "hao"
|
||
continue
|
||
if line and not line.startswith(" ") and line.endswith(":"):
|
||
section = None
|
||
continue
|
||
if section == "hao" and ":" in line and line.startswith(" "):
|
||
k, v = line.split(":", 1)
|
||
hao[k.strip()] = v.strip().strip('"').strip("'")
|
||
if not hao.get("token"):
|
||
raise SystemExit("config.dev.yaml 缺少 haoxian.token")
|
||
if not hao.get("base_url"):
|
||
hao["base_url"] = DEFAULT_BASE
|
||
return hao
|
||
|
||
|
||
def http_get(url: str, token: str, retries: int = 4) -> dict:
|
||
last: Exception | None = None
|
||
for i in range(retries):
|
||
try:
|
||
req = urllib.request.Request(url, headers={"x-token": token})
|
||
with urllib.request.urlopen(req, timeout=180) as resp:
|
||
return json.loads(resp.read())
|
||
except Exception as e: # noqa: BLE001
|
||
last = e
|
||
time.sleep(1.5 * (i + 1))
|
||
raise RuntimeError(f"GET failed after retries: {url}: {last}")
|
||
|
||
|
||
def list_sept_projects(base: str, token: str) -> list[dict]:
|
||
url = f"{base.rstrip('/')}/api/agentic-project/project?page=1&page_size=999&listed=all"
|
||
data = http_get(url, token)
|
||
items = (data.get("data") or {}).get("list") or []
|
||
out = []
|
||
for p in items:
|
||
if (
|
||
p.get("listed") == 1
|
||
and str(p.get("status") or "").lower() == "running"
|
||
and p.get("end_month") == REQUIRED_END_MONTH
|
||
):
|
||
out.append(
|
||
{
|
||
"id": p.get("id"),
|
||
"project_name": p.get("project_name"),
|
||
"brand": p.get("brand"),
|
||
"category": p.get("category"),
|
||
"end_month": p.get("end_month"),
|
||
"status": p.get("status"),
|
||
"listed": p.get("listed"),
|
||
}
|
||
)
|
||
out.sort(key=lambda x: int(x["id"]) if str(x["id"]).isdigit() else 0, reverse=True)
|
||
return out
|
||
|
||
|
||
def fetch_l1_table(
|
||
base: str,
|
||
token: str,
|
||
table: str,
|
||
project_id: int | str,
|
||
batch: str,
|
||
page_size: int = 50,
|
||
) -> dict:
|
||
"""分页拉取 /api/geo_label/l1/{table}，返回 {total, code, ok, crawl_batch_id, items}。"""
|
||
crawl = f"HISENSE-HIST-{project_id}-{batch}"
|
||
all_items: list = []
|
||
total = None
|
||
code = 0
|
||
page = 1
|
||
while True:
|
||
url = (
|
||
f"{base.rstrip('/')}/api/geo_label/l1/{table}"
|
||
f"?project_id={project_id}&crawl_batch_id={crawl}"
|
||
f"&page={page}&page_size={page_size}"
|
||
)
|
||
data = http_get(url, token)
|
||
code = data.get("code", 0)
|
||
if code != 0:
|
||
return {
|
||
"id": int(project_id) if str(project_id).isdigit() else project_id,
|
||
"total": 0,
|
||
"code": code,
|
||
"ok": False,
|
||
"crawl_batch_id": crawl,
|
||
"items": [],
|
||
"message": data.get("message"),
|
||
}
|
||
d = data.get("data") or {}
|
||
if total is None:
|
||
total = d.get("total", 0) or 0
|
||
page_items = d.get("items") or []
|
||
all_items.extend(page_items)
|
||
if len(all_items) >= total or not page_items:
|
||
break
|
||
page += 1
|
||
if page > 100:
|
||
break
|
||
return {
|
||
"id": int(project_id) if str(project_id).isdigit() else project_id,
|
||
"total": total,
|
||
"code": code,
|
||
"ok": True,
|
||
"crawl_batch_id": crawl,
|
||
"fetched": len(all_items),
|
||
"items": all_items,
|
||
}
|
||
|
||
|
||
def fetch_qa_raw(base: str, token: str, project_id: int | str, batch: str, page_size: int = 50) -> dict:
|
||
return fetch_l1_table(base, token, "qa_raw", project_id, batch, page_size)
|
||
|
||
|
||
def fetch_qa_product_raw(base: str, token: str, project_id: int | str, batch: str, page_size: int = 50) -> dict:
|
||
return fetch_l1_table(base, token, "qa_product_raw", project_id, batch, page_size)
|
||
|
||
|
||
def cmd_list_projects(args: argparse.Namespace) -> None:
|
||
cfg = load_haoxian_config()
|
||
base = args.base or cfg["base_url"]
|
||
projects = list_sept_projects(base, cfg["token"])
|
||
SKILL_DATA.mkdir(parents=True, exist_ok=True)
|
||
out = SKILL_DATA / "sept_projects_202609.json"
|
||
out.write_text(json.dumps(projects, ensure_ascii=False, indent=2))
|
||
print(json.dumps({"count": len(projects), "path": str(out), "projects": projects}, ensure_ascii=False, indent=2))
|
||
|
||
|
||
def cmd_batch_totals(args: argparse.Namespace) -> None:
|
||
cfg = load_haoxian_config()
|
||
base = args.base or cfg["base_url"]
|
||
batch = args.batch
|
||
projects = list_sept_projects(base, cfg["token"])
|
||
cache_dir = SKILL_DATA / f"qa_raw_{batch}"
|
||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
rows = []
|
||
for i, p in enumerate(projects, 1):
|
||
pid = p["id"]
|
||
cache = cache_dir / f"{pid}.json"
|
||
if cache.exists() and not args.refresh:
|
||
payload = json.loads(cache.read_text())
|
||
total = payload.get("total", 0)
|
||
items = payload.get("items") or []
|
||
row = {
|
||
"id": pid,
|
||
"project_name": p["project_name"],
|
||
"total": total,
|
||
"code": 0,
|
||
"ok": True,
|
||
"crawl_batch_id": payload.get("crawl_batch_id") or f"HISENSE-HIST-{pid}-{batch}",
|
||
"fetched": len(items),
|
||
}
|
||
else:
|
||
try:
|
||
raw = fetch_qa_raw(base, cfg["token"], pid, batch, page_size=args.page_size)
|
||
except Exception as e: # noqa: BLE001
|
||
row = {
|
||
"id": pid,
|
||
"project_name": p["project_name"],
|
||
"total": -1,
|
||
"code": None,
|
||
"ok": False,
|
||
"err": str(e),
|
||
}
|
||
rows.append(row)
|
||
print(f"[{i}/{len(projects)}] FAIL id={pid}: {e}", file=sys.stderr)
|
||
continue
|
||
items = raw.get("items") or []
|
||
payload = {
|
||
"project_id": pid,
|
||
"project_name": p["project_name"],
|
||
"crawl_batch_id": raw["crawl_batch_id"],
|
||
"batch_date": batch,
|
||
"total": raw["total"],
|
||
"fetched": raw.get("fetched", len(items)),
|
||
"items": items,
|
||
}
|
||
cache.write_text(json.dumps(payload, ensure_ascii=False))
|
||
row = {
|
||
"id": pid,
|
||
"project_name": p["project_name"],
|
||
"total": raw["total"],
|
||
"code": raw.get("code", 0),
|
||
"ok": raw.get("ok", True),
|
||
"crawl_batch_id": raw["crawl_batch_id"],
|
||
"fetched": payload["fetched"],
|
||
}
|
||
rows.append(row)
|
||
if i % 20 == 0 or i == len(projects):
|
||
print(f"[{i}/{len(projects)}] ...", file=sys.stderr)
|
||
|
||
summary = {
|
||
"batch_date": batch,
|
||
"base_url": base,
|
||
"count": len(rows),
|
||
"ok": sum(1 for r in rows if r.get("ok")),
|
||
"total_sum": sum(r["total"] for r in rows if r.get("ok") and r["total"] >= 0),
|
||
"projects": rows,
|
||
}
|
||
(SKILL_DATA / f"qa_raw_{batch}_summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2))
|
||
tsv = ["project_id\tproject_name\ttotal\tok\tcrawl_batch_id"]
|
||
for r in rows:
|
||
tsv.append(
|
||
f"{r.get('id')}\t{r.get('project_name')}\t{r.get('total')}\t{r.get('ok')}\t{r.get('crawl_batch_id','')}"
|
||
)
|
||
(SKILL_DATA / f"qa_raw_{batch}_totals.tsv").write_text("\n".join(tsv) + "\n")
|
||
print(json.dumps({"batch_date": batch, "count": len(rows), "projects": rows}, ensure_ascii=False, indent=2))
|
||
|
||
|
||
def cmd_project(args: argparse.Namespace) -> None:
|
||
cfg = load_haoxian_config()
|
||
base = args.base or cfg["base_url"]
|
||
batch = args.batch
|
||
pid = args.id
|
||
cache_dir = SKILL_DATA / f"qa_raw_{batch}"
|
||
cache = cache_dir / f"{pid}.json"
|
||
product_cache = cache_dir / f"{pid}_product_raw.json"
|
||
|
||
name = None
|
||
projects_path = SKILL_DATA / "sept_projects_202609.json"
|
||
if projects_path.exists():
|
||
for p in json.loads(projects_path.read_text()):
|
||
if str(p.get("id")) == str(pid):
|
||
name = p.get("project_name")
|
||
break
|
||
|
||
if cache.exists() and not args.refresh:
|
||
payload = json.loads(cache.read_text())
|
||
qa_raw_list = payload.get("items") or payload.get("qa_raw_list") or []
|
||
crawl = payload.get("crawl_batch_id") or f"HISENSE-HIST-{pid}-{batch}"
|
||
total = payload.get("total")
|
||
code = 0
|
||
ok = True
|
||
if not name:
|
||
name = payload.get("project_name")
|
||
else:
|
||
raw = fetch_qa_raw(base, cfg["token"], pid, batch, page_size=args.page_size)
|
||
qa_raw_list = raw.get("items") or []
|
||
crawl = raw["crawl_batch_id"]
|
||
total = raw["total"]
|
||
code = raw.get("code", 0)
|
||
ok = raw.get("ok", True)
|
||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||
cache.write_text(
|
||
json.dumps(
|
||
{
|
||
"project_id": raw["id"],
|
||
"project_name": name,
|
||
"crawl_batch_id": crawl,
|
||
"batch_date": batch,
|
||
"total": total,
|
||
"fetched": len(qa_raw_list),
|
||
"items": qa_raw_list,
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
)
|
||
|
||
if product_cache.exists() and not args.refresh:
|
||
prod_payload = json.loads(product_cache.read_text())
|
||
qa_product_raw_list = (
|
||
prod_payload.get("items") or prod_payload.get("qa_product_raw_list") or []
|
||
)
|
||
else:
|
||
prod = fetch_qa_product_raw(base, cfg["token"], pid, batch, page_size=args.page_size)
|
||
qa_product_raw_list = prod.get("items") or []
|
||
if not prod.get("ok", True):
|
||
ok = False
|
||
code = prod.get("code", code)
|
||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||
product_cache.write_text(
|
||
json.dumps(
|
||
{
|
||
"project_id": int(pid) if str(pid).isdigit() else pid,
|
||
"project_name": name,
|
||
"crawl_batch_id": crawl,
|
||
"batch_date": batch,
|
||
"total": prod.get("total", len(qa_product_raw_list)),
|
||
"fetched": len(qa_product_raw_list),
|
||
"items": qa_product_raw_list,
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
)
|
||
|
||
out = {
|
||
"id": int(pid) if str(pid).isdigit() else pid,
|
||
"project_name": name,
|
||
"total": total,
|
||
"code": code,
|
||
"ok": ok,
|
||
"crawl_batch_id": crawl,
|
||
"qa_raw_list": qa_raw_list,
|
||
"qa_product_raw_list": qa_product_raw_list,
|
||
}
|
||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description="Hisense Sept qa_raw query")
|
||
parser.add_argument("--base", default=None, help="API base, default from config / resource-api")
|
||
parser.add_argument("--page-size", type=int, default=50)
|
||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||
|
||
p1 = sub.add_parser("list-projects", help="List Sept running projects")
|
||
p1.set_defaults(func=cmd_list_projects)
|
||
|
||
p2 = sub.add_parser("batch-totals", help="Fetch/totals for all Sept projects in a batch")
|
||
p2.add_argument("--batch", required=True, help="Batch date YYYYMMDD, e.g. 20260919")
|
||
p2.add_argument("--refresh", action="store_true")
|
||
p2.set_defaults(func=cmd_batch_totals)
|
||
|
||
p3 = sub.add_parser("project", help="Fetch qa_raw + qa_product_raw; stdout uses qa_raw_list")
|
||
p3.add_argument("--batch", required=True)
|
||
p3.add_argument("--id", required=True)
|
||
p3.add_argument("--refresh", action="store_true")
|
||
p3.set_defaults(func=cmd_project)
|
||
|
||
args = parser.parse_args()
|
||
args.func(args)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|