"""导入不在教育部普通高校名单内、但确有普通高考招生记录的特殊院校。""" import argparse import csv import hashlib import os import subprocess import uuid from pathlib import Path def sql_string(value): if value is None or value == "": return "NULL" return "'" + str(value).replace("\\", "\\\\").replace("'", "\\'") + "'" def mysql_run(args, sql, capture=False): command = [ args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port), "-u", args.user, "--default-character-set=utf8mb4", "-D", args.database, ] if capture: command.extend(["-N", "-B"]) process = subprocess.run(command, input=sql, text=True, encoding="utf-8", capture_output=True) if process.returncode: raise RuntimeError(process.stderr.strip() or process.stdout.strip()) return process.stdout def main(): parser = argparse.ArgumentParser(description="导入2025军队等特殊招生院校并关联稳定ID") parser.add_argument("--mapping-file", required=True) parser.add_argument("--raw-file", required=True) parser.add_argument("--landing-page-url", required=True) parser.add_argument("--publisher", required=True) parser.add_argument("--source-title", required=True) parser.add_argument("--dataset-name", default="special_school_master_military") parser.add_argument("--published-on", default="2025-06-03") parser.add_argument("--source-notes", default="特殊招生院校使用官方名录或招生政策确认主体;地点待核验项不用于自动地理分析。") parser.add_argument("--mysql-bin", required=True) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=3306) parser.add_argument("--user", default="root") parser.add_argument("--database", default="zhiyuantong") args = parser.parse_args() if not os.environ.get("MYSQL_PWD"): raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码") mapping_path = Path(args.mapping_file) raw_path = Path(args.raw_file) if not raw_path.is_file(): raise FileNotFoundError(raw_path) with mapping_path.open(encoding="utf-8-sig", newline="") as stream: rows = list(csv.DictReader(stream)) required = { "official_name", "school_kind", "authority_name", "education_level", "province_name", "city_name", "aliases", "notes", } if not rows or set(rows[0]) != required: raise ValueError("映射文件字段不符合预期") if len({row["official_name"] for row in rows}) != len(rows): raise ValueError("特殊院校正式名称重复") sha256 = hashlib.sha256(raw_path.read_bytes()).hexdigest() run_id = str(uuid.uuid4()) row_values = ",\n".join( "(" + ",".join(sql_string(row[key]) for key in [ "official_name", "school_kind", "authority_name", "education_level", "province_name", "city_name", "aliases", "notes", ]) + ")" for row in rows ) sql = f""" SET NAMES utf8mb4; SET time_zone='+08:00'; START TRANSACTION; INSERT INTO data_sources ( source_type,publisher,title,landing_page_url,published_on,retrieved_at, sha256,raw_file_path,notes ) VALUES ( 'official_html',{sql_string(args.publisher)},{sql_string(args.source_title)}, {sql_string(args.landing_page_url)},{sql_string(args.published_on)},NOW(),{sql_string(sha256)}, {sql_string(str(raw_path.resolve()))}, {sql_string(args.source_notes)} ) AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at,raw_file_path=incoming.raw_file_path, landing_page_url=incoming.landing_page_url,notes=incoming.notes; SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(sha256)}); INSERT INTO ingestion_runs ( ingestion_run_id,source_id,dataset_name,dataset_year,started_at,status,rows_read,rows_rejected ) VALUES ({sql_string(run_id)},@source_id,{sql_string(args.dataset_name)},2025,NOW(),'running',{len(rows)},0); CREATE TEMPORARY TABLE tmp_special_schools ( official_name VARCHAR(255) NOT NULL COMMENT '招生院校官方名称', school_kind VARCHAR(32) NOT NULL COMMENT '特殊院校类别', authority_name VARCHAR(255) NOT NULL COMMENT '主管单位', education_level VARCHAR(32) NOT NULL COMMENT '招生培养层次', province_name VARCHAR(64) NOT NULL COMMENT '院校所在省份;无法确认时为待核验', city_name VARCHAR(128) NOT NULL COMMENT '院校所在城市;无法确认时为待核验', aliases TEXT NULL COMMENT '竖线分隔的招生来源别名', notes VARCHAR(512) NULL COMMENT '来源及口径备注', PRIMARY KEY(official_name) ) ENGINE=InnoDB COMMENT='特殊招生院校导入临时表'; INSERT INTO tmp_special_schools VALUES {row_values}; SET @schools_before=(SELECT COUNT(*) FROM schools); INSERT INTO schools ( school_id,moe_school_code,official_name,province_name,city_name, authority_name,education_level,source_remark,ownership_type,school_status, first_seen_year,last_seen_year,source_id ) SELECT UUID(),NULL,t.official_name,t.province_name,t.city_name,t.authority_name, t.education_level,CONCAT(t.school_kind,';',t.notes), CASE WHEN t.school_kind LIKE '%自资%' THEN '民办' ELSE '公办' END, 'active',2025,2025,@source_id FROM tmp_special_schools t WHERE NOT EXISTS (SELECT 1 FROM schools s WHERE s.official_name=t.official_name); UPDATE schools s JOIN tmp_special_schools t ON t.official_name=s.official_name SET s.authority_name=t.authority_name,s.education_level=t.education_level, s.province_name=t.province_name,s.city_name=t.city_name, s.source_remark=CONCAT(t.school_kind,';',t.notes),s.school_status='active', s.first_seen_year=LEAST(s.first_seen_year,2025),s.last_seen_year=GREATEST(s.last_seen_year,2025), s.source_id=@source_id WHERE s.moe_school_code IS NULL; SET @names_before=(SELECT COUNT(*) FROM school_names); INSERT IGNORE INTO school_names (school_id,school_name,name_type,valid_from_year,source_id) SELECT s.school_id,t.official_name,'official',2025,@source_id FROM tmp_special_schools t JOIN schools s ON s.official_name=t.official_name; INSERT IGNORE INTO school_names (school_id,school_name,name_type,valid_from_year,source_id) WITH RECURSIVE alias_parts AS ( SELECT official_name,TRIM(SUBSTRING_INDEX(aliases,'|',1)) alias_name, CASE WHEN aliases LIKE '%|%' THEN SUBSTRING(aliases,LOCATE('|',aliases)+1) ELSE '' END rest FROM tmp_special_schools WHERE aliases IS NOT NULL AND aliases<>'' UNION ALL SELECT official_name,TRIM(SUBSTRING_INDEX(rest,'|',1)), CASE WHEN rest LIKE '%|%' THEN SUBSTRING(rest,LOCATE('|',rest)+1) ELSE '' END FROM alias_parts WHERE rest<>'' ) SELECT s.school_id,a.alias_name,'alias',2025,@source_id FROM alias_parts a JOIN schools s ON s.official_name=a.official_name WHERE a.alias_name<>''; CREATE TEMPORARY TABLE tmp_unique_school_names ( normalized_name VARCHAR(255) NOT NULL COMMENT '统一括号并移除空格后的唯一招生名称', school_id CHAR(36) NOT NULL COMMENT '系统稳定学校UUID', PRIMARY KEY(normalized_name) ) ENGINE=InnoDB COMMENT='招生记录稳定学校ID关联临时表'; INSERT INTO tmp_unique_school_names SELECT normalized_name,MIN(school_id) FROM ( SELECT REPLACE(REPLACE(REPLACE(TRIM(official_name),'(','('),')',')'),' ','') normalized_name,school_id FROM schools UNION ALL SELECT REPLACE(REPLACE(REPLACE(TRIM(school_name),'(','('),')',')'),' ','') normalized_name,school_id FROM school_names ) all_names GROUP BY normalized_name HAVING COUNT(DISTINCT school_id)=1; SET @groups_before=(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL); UPDATE admission_groups target JOIN tmp_unique_school_names mapped ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'(','('),')',')'),' ','') SET target.school_id=mapped.school_id WHERE target.school_id IS NULL; SET @matched_groups=@groups_before-(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL); SET @major_before=(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL); UPDATE admission_major_lines target JOIN tmp_unique_school_names mapped ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'(','('),')',')'),' ','') SET target.school_id=mapped.school_id WHERE target.school_id IS NULL; SET @matched_major=@major_before-(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL); SET @cutoff_before=(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL); UPDATE admission_cutoff_lines target JOIN tmp_unique_school_names mapped ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'(','('),')',')'),' ','') SET target.school_id=mapped.school_id WHERE target.school_id IS NULL; SET @matched_cutoff=@cutoff_before-(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL); SET @schools_inserted=(SELECT COUNT(*) FROM schools)-@schools_before; SET @names_inserted=(SELECT COUNT(*) FROM school_names)-@names_before; UPDATE ingestion_runs SET finished_at=NOW(),status='success',rows_inserted=@schools_inserted, rows_updated=@matched_groups+@matched_major+@matched_cutoff, message=CONCAT('新增特殊院校',@schools_inserted,'所、院校名称',@names_inserted, '个;补匹配专业组',@matched_groups,'条、专业线',@matched_major,'条、投档线',@matched_cutoff,'条。') WHERE ingestion_run_id={sql_string(run_id)}; COMMIT; """ mysql_run(args, sql) result = mysql_run( args, "SELECT message FROM ingestion_runs WHERE ingestion_run_id=" + sql_string(run_id) + ";", capture=True, ).strip() print(result) if __name__ == "__main__": main()