"""从人工结构化且保留官方原文的CSV导入省级控制线。""" import argparse import csv import hashlib import os import subprocess import uuid from decimal import Decimal from pathlib import Path def sql_string(value): if value is None or value == "": return "NULL" return "'" + str(value).replace("\\", "\\\\").replace("'", "\\'") + "'" def mysql_run(args, sql, capture=False): command = [args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port), "-u", args.user, "--default-character-set=utf8mb4", "-D", args.database] if capture: command.extend(["-N", "-B"]) result = subprocess.run(command, input=sql, text=True, encoding="utf-8", capture_output=True) if result.returncode: raise RuntimeError(result.stderr.strip() or result.stdout.strip()) return result.stdout def main(): parser = argparse.ArgumentParser(description="导入省级考试主管部门官方控制线") parser.add_argument("--province", required=True) parser.add_argument("--year", required=True, type=int) parser.add_argument("--publisher", required=True) parser.add_argument("--title", required=True) parser.add_argument("--landing-page-url", required=True) parser.add_argument("--published-on", required=True) parser.add_argument("--mapping-file", required=True) parser.add_argument("--raw-file", required=True) parser.add_argument("--mysql-bin", required=True) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=3306) parser.add_argument("--user", default="root") parser.add_argument("--database", default="zhiyuantong") args = parser.parse_args() if not os.environ.get("MYSQL_PWD"): raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码") mapping_path = Path(args.mapping_file) raw_path = Path(args.raw_file) with mapping_path.open(encoding="utf-8-sig", newline="") as stream: rows = list(csv.DictReader(stream)) expected = {"admission_category", "subject_category", "education_level", "culture_score", "professional_score", "notes"} if not rows or set(rows[0]) != expected: raise ValueError("控制线CSV字段不符合预期") keys = [(row["admission_category"], row["subject_category"], row["education_level"]) for row in rows] if len(set(keys)) != len(keys): raise ValueError("控制线CSV存在重复业务键") for row in rows: culture = Decimal(row["culture_score"]) professional = Decimal(row["professional_score"]) if row["professional_score"] else None if not (0 <= culture <= 750) or (professional is not None and not 0 <= professional <= 750): raise ValueError(f"分数超出有效范围：{row}") sha256 = hashlib.sha256(raw_path.read_bytes()).hexdigest() run_id = str(uuid.uuid4()) values = ",\n".join( "(" + ",".join([ sql_string(args.province), str(args.year), sql_string(row["admission_category"]), sql_string(row["subject_category"]), sql_string(row["education_level"]), row["culture_score"], row["professional_score"] or "NULL", sql_string(row["notes"]), "@source_id", ]) + ")" for row in rows ) sql = f""" SET NAMES utf8mb4; SET time_zone='+08:00'; START TRANSACTION; INSERT INTO data_sources ( source_type,publisher,title,landing_page_url,published_on,retrieved_at,sha256,raw_file_path,notes ) VALUES ( 'official_html',{sql_string(args.publisher)},{sql_string(args.title)}, {sql_string(args.landing_page_url)},{sql_string(args.published_on)},NOW(), {sql_string(sha256)},{sql_string(str(raw_path.resolve()))}, '官方页面已原样存档；CSV仅作结构化映射，文化分和专业分分字段保存。' ) AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at, raw_file_path=incoming.raw_file_path,landing_page_url=incoming.landing_page_url,notes=incoming.notes; SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(sha256)}); INSERT INTO ingestion_runs ( ingestion_run_id,source_id,dataset_name,dataset_year,started_at,status,rows_read,rows_rejected ) VALUES ({sql_string(run_id)},@source_id,'official_control_score_lines',{args.year},NOW(),'running',{len(rows)},0); SET @before_count=(SELECT COUNT(*) FROM control_score_lines WHERE province_name={sql_string(args.province)} AND admission_year={args.year}); INSERT INTO control_score_lines ( province_name,admission_year,admission_category,subject_category,education_level, culture_score,professional_score,notes,source_id ) VALUES {values} AS incoming ON DUPLICATE KEY UPDATE culture_score=incoming.culture_score, professional_score=incoming.professional_score,notes=incoming.notes,source_id=incoming.source_id; DELETE old_line FROM control_score_lines old_line JOIN control_score_lines official_line ON official_line.province_name=old_line.province_name AND official_line.admission_year=old_line.admission_year AND official_line.admission_category=old_line.admission_category AND official_line.subject_category=old_line.subject_category AND REPLACE(official_line.education_level,'高职（专科）','专科') =REPLACE(old_line.education_level,'高职（专科）','专科') AND official_line.source_id=@source_id WHERE old_line.province_name={sql_string(args.province)} AND old_line.admission_year={args.year} AND old_line.source_id<>@source_id; SET @after_count=(SELECT COUNT(*) FROM control_score_lines WHERE province_name={sql_string(args.province)} AND admission_year={args.year}); SET @inserted=GREATEST(@after_count-@before_count,0); INSERT INTO dataset_coverage ( dataset_name,province_name,admission_year,admission_category,subject_category, segment_code,coverage_status,record_count,notes,source_id ) VALUES ( '普通类控制线',{sql_string(args.province)},{args.year},'普通类','','', 'complete_public_snapshot',@after_count, CONCAT('已按省级考试主管部门官方公告结构化',@after_count,'条，包含普通、艺术和体育类别公开口径。'), @source_id ) AS incoming ON DUPLICATE KEY UPDATE coverage_status=incoming.coverage_status, record_count=incoming.record_count,notes=incoming.notes,source_id=incoming.source_id; UPDATE ingestion_runs SET finished_at=NOW(),status='success',rows_inserted=@inserted, rows_updated={len(rows)}-@inserted, message=CONCAT('结构化官方控制线{len(rows)}条；新增',@inserted,'条，更新',{len(rows)}-@inserted,'条。') WHERE ingestion_run_id={sql_string(run_id)}; COMMIT; """ mysql_run(args, sql) print(mysql_run(args, "SELECT message FROM ingestion_runs WHERE ingestion_run_id=" + sql_string(run_id) + ";", True).strip()) if __name__ == "__main__": main()