ZhiYuanTong/scripts/import_adult_school_subset.py

"""导入招生数据实际引用的教育部成人高校，并补齐稳定学校ID。"""

import argparse
import csv
import hashlib
import os
import subprocess
import uuid
from pathlib import Path


def sql_string(value):
    if value is None or value == "":
        return "NULL"
    return "'" + str(value).replace("\\", "\\\\").replace("'", "\\'") + "'"


def run_mysql(args, sql, capture=False):
    command = [args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port),
               "-u", args.user, "--default-character-set=utf8mb4", "-D", args.database]
    if capture:
        command.extend(["-N", "-B"])
    result = subprocess.run(command, input=sql, text=True, encoding="utf-8", capture_output=True)
    if result.returncode:
        raise RuntimeError(result.stderr.strip() or result.stdout.strip())
    return result.stdout


def main():
    parser = argparse.ArgumentParser(description="导入2025教育部成人高校招生引用子集")
    parser.add_argument("--mapping-file", required=True)
    parser.add_argument("--raw-file", required=True)
    parser.add_argument("--mysql-bin", required=True)
    parser.add_argument("--host", default="127.0.0.1")
    parser.add_argument("--port", type=int, default=3306)
    parser.add_argument("--user", default="root")
    parser.add_argument("--database", default="zhiyuantong")
    args = parser.parse_args()
    if not os.environ.get("MYSQL_PWD"):
        raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码")

    mapping_path = Path(args.mapping_file)
    raw_path = Path(args.raw_file)
    with mapping_path.open(encoding="utf-8-sig", newline="") as stream:
        rows = list(csv.DictReader(stream))
    expected = {"official_name", "moe_school_code", "authority_name", "province_name", "city_name", "notes"}
    if not rows or set(rows[0]) != expected:
        raise ValueError("映射文件字段不符合预期")
    if any(len(row["moe_school_code"]) != 10 for row in rows):
        raise ValueError("学校标识码必须为10位")
    sha256 = hashlib.sha256(raw_path.read_bytes()).hexdigest()
    run_id = str(uuid.uuid4())
    values = ",\n".join(
        "(" + ",".join(sql_string(row[key]) for key in [
            "official_name", "moe_school_code", "authority_name", "province_name", "city_name", "notes"
        ]) + ")" for row in rows
    )
    sql = f"""
    SET NAMES utf8mb4;
    SET time_zone='+08:00';
    START TRANSACTION;
    INSERT INTO data_sources (
      source_type,publisher,title,landing_page_url,file_url,published_on,retrieved_at,sha256,raw_file_path,notes
    ) VALUES (
      'official_xls','中华人民共和国教育部','全国成人高等学校名单（截至2025年6月20日）',
      'https://www.moe.gov.cn/jyb_xxgk/s5743/s5744/A03/202506/t20250627_1195683.html',
      'https://www.moe.gov.cn/jyb_xxgk/s5743/s5744/A03/202506/W020250627301230143042.xls',
      '2025-06-20',NOW(),{sql_string(sha256)},{sql_string(str(raw_path.resolve()))},
      '原始名单共248所；本批仅导入现有普通高考招生来源数据实际引用的2所成人高校，避免扩大普通高校基础库口径。'
    ) AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at,
      raw_file_path=incoming.raw_file_path,landing_page_url=incoming.landing_page_url,
      file_url=incoming.file_url,notes=incoming.notes;
    SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(sha256)});
    INSERT INTO ingestion_runs (
      ingestion_run_id,source_id,dataset_name,dataset_year,started_at,status,rows_read,rows_rejected
    ) VALUES ({sql_string(run_id)},@source_id,'adult_school_master_referenced_subset',2025,NOW(),'running',{len(rows)},0);

    CREATE TEMPORARY TABLE tmp_adult_schools (
      official_name VARCHAR(255) NOT NULL COMMENT '教育部成人高校名单中的学校名称',
      moe_school_code CHAR(10) NOT NULL COMMENT '教育部学校标识码',
      authority_name VARCHAR(255) NOT NULL COMMENT '名单所列主管部门',
      province_name VARCHAR(64) NOT NULL COMMENT '学校所在省份',
      city_name VARCHAR(128) NOT NULL COMMENT '学校所在城市',
      notes VARCHAR(512) NOT NULL COMMENT '入库范围和依据说明',
      PRIMARY KEY(moe_school_code)
    ) ENGINE=InnoDB COMMENT='招生数据引用成人高校导入临时表';
    INSERT INTO tmp_adult_schools VALUES
    {values};

    SET @schools_before=(SELECT COUNT(*) FROM schools);
    INSERT INTO schools (
      school_id,moe_school_code,official_name,province_name,city_name,authority_name,
      education_level,source_remark,ownership_type,school_status,first_seen_year,last_seen_year,source_id
    )
    SELECT UUID(),t.moe_school_code,t.official_name,t.province_name,t.city_name,t.authority_name,
      '成人高等教育',t.notes,'公办','active',2025,2025,@source_id
    FROM tmp_adult_schools t
    WHERE NOT EXISTS (SELECT 1 FROM schools s WHERE s.moe_school_code=t.moe_school_code);

    INSERT IGNORE INTO school_names (school_id,school_name,name_type,valid_from_year,source_id)
    SELECT s.school_id,t.official_name,'official',2025,@source_id
    FROM tmp_adult_schools t JOIN schools s ON s.moe_school_code=t.moe_school_code;

    SET @groups_before=(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL);
    UPDATE admission_groups target JOIN tmp_adult_schools t ON t.official_name=target.school_name_raw
      JOIN schools s ON s.moe_school_code=t.moe_school_code
    SET target.school_id=s.school_id WHERE target.school_id IS NULL;
    SET @matched_groups=@groups_before-(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL);

    SET @major_before=(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL);
    UPDATE admission_major_lines target JOIN tmp_adult_schools t ON t.official_name=target.school_name_raw
      JOIN schools s ON s.moe_school_code=t.moe_school_code
    SET target.school_id=s.school_id WHERE target.school_id IS NULL;
    SET @matched_major=@major_before-(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL);

    SET @cutoff_before=(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL);
    UPDATE admission_cutoff_lines target JOIN tmp_adult_schools t ON t.official_name=target.school_name_raw
      JOIN schools s ON s.moe_school_code=t.moe_school_code
    SET target.school_id=s.school_id WHERE target.school_id IS NULL;
    SET @matched_cutoff=@cutoff_before-(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL);

    SET @schools_inserted=(SELECT COUNT(*) FROM schools)-@schools_before;
    UPDATE ingestion_runs SET finished_at=NOW(),status='success',rows_inserted=@schools_inserted,
      rows_updated=@matched_groups+@matched_major+@matched_cutoff,
      message=CONCAT('新增成人高校',@schools_inserted,'所；补匹配专业组',@matched_groups,
        '条、专业线',@matched_major,'条、投档线',@matched_cutoff,'条。')
    WHERE ingestion_run_id={sql_string(run_id)};
    COMMIT;
    """
    run_mysql(args, sql)
    print(run_mysql(args, "SELECT message FROM ingestion_runs WHERE ingestion_run_id=" + sql_string(run_id) + ";", True).strip())


if __name__ == "__main__":
    main()