"""导入招生数据实际引用的教育部成人高校,并补齐稳定学校ID。""" import argparse import csv import hashlib import os import subprocess import uuid from pathlib import Path def sql_string(value): if value is None or value == "": return "NULL" return "'" + str(value).replace("\\", "\\\\").replace("'", "\\'") + "'" def run_mysql(args, sql, capture=False): command = [args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port), "-u", args.user, "--default-character-set=utf8mb4", "-D", args.database] if capture: command.extend(["-N", "-B"]) result = subprocess.run(command, input=sql, text=True, encoding="utf-8", capture_output=True) if result.returncode: raise RuntimeError(result.stderr.strip() or result.stdout.strip()) return result.stdout def main(): parser = argparse.ArgumentParser(description="导入2025教育部成人高校招生引用子集") parser.add_argument("--mapping-file", required=True) parser.add_argument("--raw-file", required=True) parser.add_argument("--mysql-bin", required=True) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=3306) parser.add_argument("--user", default="root") parser.add_argument("--database", default="zhiyuantong") args = parser.parse_args() if not os.environ.get("MYSQL_PWD"): raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码") mapping_path = Path(args.mapping_file) raw_path = Path(args.raw_file) with mapping_path.open(encoding="utf-8-sig", newline="") as stream: rows = list(csv.DictReader(stream)) expected = {"official_name", "moe_school_code", "authority_name", "province_name", "city_name", "notes"} if not rows or set(rows[0]) != expected: raise ValueError("映射文件字段不符合预期") if any(len(row["moe_school_code"]) != 10 for row in rows): raise ValueError("学校标识码必须为10位") sha256 = hashlib.sha256(raw_path.read_bytes()).hexdigest() run_id = str(uuid.uuid4()) values = ",\n".join( "(" + ",".join(sql_string(row[key]) for key in [ "official_name", "moe_school_code", "authority_name", "province_name", "city_name", "notes" ]) + ")" for row in rows ) sql = f""" SET NAMES utf8mb4; SET time_zone='+08:00'; START TRANSACTION; INSERT INTO data_sources ( source_type,publisher,title,landing_page_url,file_url,published_on,retrieved_at,sha256,raw_file_path,notes ) VALUES ( 'official_xls','中华人民共和国教育部','全国成人高等学校名单(截至2025年6月20日)', 'https://www.moe.gov.cn/jyb_xxgk/s5743/s5744/A03/202506/t20250627_1195683.html', 'https://www.moe.gov.cn/jyb_xxgk/s5743/s5744/A03/202506/W020250627301230143042.xls', '2025-06-20',NOW(),{sql_string(sha256)},{sql_string(str(raw_path.resolve()))}, '原始名单共248所;本批仅导入现有普通高考招生来源数据实际引用的2所成人高校,避免扩大普通高校基础库口径。' ) AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at, raw_file_path=incoming.raw_file_path,landing_page_url=incoming.landing_page_url, file_url=incoming.file_url,notes=incoming.notes; SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(sha256)}); INSERT INTO ingestion_runs ( ingestion_run_id,source_id,dataset_name,dataset_year,started_at,status,rows_read,rows_rejected ) VALUES ({sql_string(run_id)},@source_id,'adult_school_master_referenced_subset',2025,NOW(),'running',{len(rows)},0); CREATE TEMPORARY TABLE tmp_adult_schools ( official_name VARCHAR(255) NOT NULL COMMENT '教育部成人高校名单中的学校名称', moe_school_code CHAR(10) NOT NULL COMMENT '教育部学校标识码', authority_name VARCHAR(255) NOT NULL COMMENT '名单所列主管部门', province_name VARCHAR(64) NOT NULL COMMENT '学校所在省份', city_name VARCHAR(128) NOT NULL COMMENT '学校所在城市', notes VARCHAR(512) NOT NULL COMMENT '入库范围和依据说明', PRIMARY KEY(moe_school_code) ) ENGINE=InnoDB COMMENT='招生数据引用成人高校导入临时表'; INSERT INTO tmp_adult_schools VALUES {values}; SET @schools_before=(SELECT COUNT(*) FROM schools); INSERT INTO schools ( school_id,moe_school_code,official_name,province_name,city_name,authority_name, education_level,source_remark,ownership_type,school_status,first_seen_year,last_seen_year,source_id ) SELECT UUID(),t.moe_school_code,t.official_name,t.province_name,t.city_name,t.authority_name, '成人高等教育',t.notes,'公办','active',2025,2025,@source_id FROM tmp_adult_schools t WHERE NOT EXISTS (SELECT 1 FROM schools s WHERE s.moe_school_code=t.moe_school_code); INSERT IGNORE INTO school_names (school_id,school_name,name_type,valid_from_year,source_id) SELECT s.school_id,t.official_name,'official',2025,@source_id FROM tmp_adult_schools t JOIN schools s ON s.moe_school_code=t.moe_school_code; SET @groups_before=(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL); UPDATE admission_groups target JOIN tmp_adult_schools t ON t.official_name=target.school_name_raw JOIN schools s ON s.moe_school_code=t.moe_school_code SET target.school_id=s.school_id WHERE target.school_id IS NULL; SET @matched_groups=@groups_before-(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL); SET @major_before=(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL); UPDATE admission_major_lines target JOIN tmp_adult_schools t ON t.official_name=target.school_name_raw JOIN schools s ON s.moe_school_code=t.moe_school_code SET target.school_id=s.school_id WHERE target.school_id IS NULL; SET @matched_major=@major_before-(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL); SET @cutoff_before=(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL); UPDATE admission_cutoff_lines target JOIN tmp_adult_schools t ON t.official_name=target.school_name_raw JOIN schools s ON s.moe_school_code=t.moe_school_code SET target.school_id=s.school_id WHERE target.school_id IS NULL; SET @matched_cutoff=@cutoff_before-(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL); SET @schools_inserted=(SELECT COUNT(*) FROM schools)-@schools_before; UPDATE ingestion_runs SET finished_at=NOW(),status='success',rows_inserted=@schools_inserted, rows_updated=@matched_groups+@matched_major+@matched_cutoff, message=CONCAT('新增成人高校',@schools_inserted,'所;补匹配专业组',@matched_groups, '条、专业线',@matched_major,'条、投档线',@matched_cutoff,'条。') WHERE ingestion_run_id={sql_string(run_id)}; COMMIT; """ run_mysql(args, sql) print(run_mysql(args, "SELECT message FROM ingestion_runs WHERE ingestion_run_id=" + sql_string(run_id) + ";", True).strip()) if __name__ == "__main__": main()