You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
141 lines
6.9 KiB
141 lines
6.9 KiB
"""从人工结构化且保留官方原文的CSV导入省级控制线。"""
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import os
|
|
import subprocess
|
|
import uuid
|
|
from decimal import Decimal
|
|
from pathlib import Path
|
|
|
|
|
|
def sql_string(value):
|
|
if value is None or value == "":
|
|
return "NULL"
|
|
return "'" + str(value).replace("\\", "\\\\").replace("'", "\\'") + "'"
|
|
|
|
|
|
def mysql_run(args, sql, capture=False):
|
|
command = [args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port),
|
|
"-u", args.user, "--default-character-set=utf8mb4", "-D", args.database]
|
|
if capture:
|
|
command.extend(["-N", "-B"])
|
|
result = subprocess.run(command, input=sql, text=True, encoding="utf-8", capture_output=True)
|
|
if result.returncode:
|
|
raise RuntimeError(result.stderr.strip() or result.stdout.strip())
|
|
return result.stdout
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="导入省级考试主管部门官方控制线")
|
|
parser.add_argument("--province", required=True)
|
|
parser.add_argument("--year", required=True, type=int)
|
|
parser.add_argument("--publisher", required=True)
|
|
parser.add_argument("--title", required=True)
|
|
parser.add_argument("--landing-page-url", required=True)
|
|
parser.add_argument("--published-on", required=True)
|
|
parser.add_argument("--mapping-file", required=True)
|
|
parser.add_argument("--raw-file", required=True)
|
|
parser.add_argument("--mysql-bin", required=True)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--port", type=int, default=3306)
|
|
parser.add_argument("--user", default="root")
|
|
parser.add_argument("--database", default="zhiyuantong")
|
|
args = parser.parse_args()
|
|
if not os.environ.get("MYSQL_PWD"):
|
|
raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码")
|
|
|
|
mapping_path = Path(args.mapping_file)
|
|
raw_path = Path(args.raw_file)
|
|
with mapping_path.open(encoding="utf-8-sig", newline="") as stream:
|
|
rows = list(csv.DictReader(stream))
|
|
expected = {"admission_category", "subject_category", "education_level", "culture_score", "professional_score", "notes"}
|
|
if not rows or set(rows[0]) != expected:
|
|
raise ValueError("控制线CSV字段不符合预期")
|
|
keys = [(row["admission_category"], row["subject_category"], row["education_level"]) for row in rows]
|
|
if len(set(keys)) != len(keys):
|
|
raise ValueError("控制线CSV存在重复业务键")
|
|
for row in rows:
|
|
culture = Decimal(row["culture_score"])
|
|
professional = Decimal(row["professional_score"]) if row["professional_score"] else None
|
|
if not (0 <= culture <= 750) or (professional is not None and not 0 <= professional <= 750):
|
|
raise ValueError(f"分数超出有效范围：{row}")
|
|
|
|
sha256 = hashlib.sha256(raw_path.read_bytes()).hexdigest()
|
|
run_id = str(uuid.uuid4())
|
|
values = ",\n".join(
|
|
"(" + ",".join([
|
|
sql_string(args.province), str(args.year),
|
|
sql_string(row["admission_category"]), sql_string(row["subject_category"]),
|
|
sql_string(row["education_level"]), row["culture_score"],
|
|
row["professional_score"] or "NULL", sql_string(row["notes"]), "@source_id",
|
|
]) + ")" for row in rows
|
|
)
|
|
sql = f"""
|
|
SET NAMES utf8mb4;
|
|
SET time_zone='+08:00';
|
|
START TRANSACTION;
|
|
INSERT INTO data_sources (
|
|
source_type,publisher,title,landing_page_url,published_on,retrieved_at,sha256,raw_file_path,notes
|
|
) VALUES (
|
|
'official_html',{sql_string(args.publisher)},{sql_string(args.title)},
|
|
{sql_string(args.landing_page_url)},{sql_string(args.published_on)},NOW(),
|
|
{sql_string(sha256)},{sql_string(str(raw_path.resolve()))},
|
|
'官方页面已原样存档；CSV仅作结构化映射，文化分和专业分分字段保存。'
|
|
) AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at,
|
|
raw_file_path=incoming.raw_file_path,landing_page_url=incoming.landing_page_url,notes=incoming.notes;
|
|
SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(sha256)});
|
|
INSERT INTO ingestion_runs (
|
|
ingestion_run_id,source_id,dataset_name,dataset_year,started_at,status,rows_read,rows_rejected
|
|
) VALUES ({sql_string(run_id)},@source_id,'official_control_score_lines',{args.year},NOW(),'running',{len(rows)},0);
|
|
|
|
SET @before_count=(SELECT COUNT(*) FROM control_score_lines
|
|
WHERE province_name={sql_string(args.province)} AND admission_year={args.year});
|
|
INSERT INTO control_score_lines (
|
|
province_name,admission_year,admission_category,subject_category,education_level,
|
|
culture_score,professional_score,notes,source_id
|
|
) VALUES
|
|
{values}
|
|
AS incoming ON DUPLICATE KEY UPDATE culture_score=incoming.culture_score,
|
|
professional_score=incoming.professional_score,notes=incoming.notes,source_id=incoming.source_id;
|
|
|
|
DELETE old_line FROM control_score_lines old_line
|
|
JOIN control_score_lines official_line
|
|
ON official_line.province_name=old_line.province_name
|
|
AND official_line.admission_year=old_line.admission_year
|
|
AND official_line.admission_category=old_line.admission_category
|
|
AND official_line.subject_category=old_line.subject_category
|
|
AND REPLACE(official_line.education_level,'高职（专科）','专科')
|
|
=REPLACE(old_line.education_level,'高职（专科）','专科')
|
|
AND official_line.source_id=@source_id
|
|
WHERE old_line.province_name={sql_string(args.province)}
|
|
AND old_line.admission_year={args.year}
|
|
AND old_line.source_id<>@source_id;
|
|
SET @after_count=(SELECT COUNT(*) FROM control_score_lines
|
|
WHERE province_name={sql_string(args.province)} AND admission_year={args.year});
|
|
SET @inserted=GREATEST(@after_count-@before_count,0);
|
|
|
|
INSERT INTO dataset_coverage (
|
|
dataset_name,province_name,admission_year,admission_category,subject_category,
|
|
segment_code,coverage_status,record_count,notes,source_id
|
|
) VALUES (
|
|
'普通类控制线',{sql_string(args.province)},{args.year},'普通类','','',
|
|
'complete_public_snapshot',@after_count,
|
|
CONCAT('已按省级考试主管部门官方公告结构化',@after_count,'条，包含普通、艺术和体育类别公开口径。'),
|
|
@source_id
|
|
) AS incoming ON DUPLICATE KEY UPDATE coverage_status=incoming.coverage_status,
|
|
record_count=incoming.record_count,notes=incoming.notes,source_id=incoming.source_id;
|
|
|
|
UPDATE ingestion_runs SET finished_at=NOW(),status='success',rows_inserted=@inserted,
|
|
rows_updated={len(rows)}-@inserted,
|
|
message=CONCAT('结构化官方控制线{len(rows)}条；新增',@inserted,'条，更新',{len(rows)}-@inserted,'条。')
|
|
WHERE ingestion_run_id={sql_string(run_id)};
|
|
COMMIT;
|
|
"""
|
|
mysql_run(args, sql)
|
|
print(mysql_run(args, "SELECT message FROM ingestion_runs WHERE ingestion_run_id=" + sql_string(run_id) + ";", True).strip())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|