"""用可审计的校区/来源名称映射补齐招生记录的稳定学校ID。""" import argparse import csv import hashlib import os import subprocess import uuid from pathlib import Path def sql_string(value): if value is None or value == "": return "NULL" value = str(value).replace("\\", "\\\\").replace("'", "\\'") return f"'{value}'" def mysql_run(args, sql, capture=False): command = [ args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port), "-u", args.user, "--default-character-set=utf8mb4", "-D", args.database, ] if capture: command.extend(["-N", "-B"]) process = subprocess.run(command, input=sql, text=True, encoding="utf-8", capture_output=True) if process.returncode: raise RuntimeError(process.stderr.strip() or process.stdout.strip()) return process.stdout def main(): parser = argparse.ArgumentParser(description="补充校区招生别名并重新关联稳定学校ID") parser.add_argument("--mapping-file", required=True) parser.add_argument("--mysql-bin", required=True) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=3306) parser.add_argument("--user", default="root") parser.add_argument("--database", default="zhiyuantong") args = parser.parse_args() if not os.environ.get("MYSQL_PWD"): raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码") mapping_path = Path(args.mapping_file) with mapping_path.open(encoding="utf-8-sig", newline="") as stream: rows = list(csv.DictReader(stream)) required = {"alias_name", "parent_school_name", "campus_name", "province_name", "city_name", "notes"} if not rows or set(rows[0]) != required: raise ValueError("映射文件字段不符合预期") if len({row["alias_name"] for row in rows}) != len(rows): raise ValueError("映射文件存在重复招生名称") parents = sorted({row["parent_school_name"] for row in rows}) parent_list = ",".join(sql_string(value) for value in parents) output = mysql_run( args, f"SELECT official_name,COUNT(*) FROM schools WHERE official_name IN ({parent_list}) GROUP BY official_name;", capture=True, ) found = {line.split("\t")[0]: int(line.split("\t")[1]) for line in output.strip().splitlines()} invalid = [parent for parent in parents if found.get(parent) != 1] if invalid: raise ValueError(f"父级学校不存在或不唯一:{invalid}") sha256 = hashlib.sha256(mapping_path.read_bytes()).hexdigest() run_id = str(uuid.uuid4()) values = ",\n".join( "(" + ",".join(sql_string(row[key]) for key in [ "alias_name", "parent_school_name", "campus_name", "province_name", "city_name", "notes" ]) + ")" for row in rows ) sql = f""" SET NAMES utf8mb4; SET time_zone='+08:00'; START TRANSACTION; INSERT INTO data_sources ( source_type, publisher, title, retrieved_at, sha256, raw_file_path, notes ) VALUES ( 'curated_mapping', '志愿通数据治理', '2025招生来源校区名称与教育部学校主体映射', NOW(), {sql_string(sha256)}, {sql_string(str(mapping_path.resolve()))}, '仅映射明确属于同一学校主体的医学部、分校区和分支教学单位;全部进入待审核状态。' ) AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at, raw_file_path=incoming.raw_file_path, notes=incoming.notes; SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(sha256)}); INSERT INTO ingestion_runs ( ingestion_run_id, source_id, dataset_name, dataset_year, started_at, status, rows_read, rows_rejected ) VALUES ({sql_string(run_id)},@source_id,'school_parent_alias_mapping',2025,NOW(),'running',{len(rows)},0); CREATE TEMPORARY TABLE tmp_parent_aliases ( alias_name VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '招生数据中出现的学校或校区名称', parent_school_name VARCHAR(255) NOT NULL COMMENT '教育部名单中的学校主体名称', campus_name VARCHAR(255) NULL COMMENT '招生名称对应的校区或医学部名称', province_name VARCHAR(64) NULL COMMENT '校区所在省份', city_name VARCHAR(128) NULL COMMENT '校区所在城市', notes VARCHAR(512) NULL COMMENT '映射依据说明', PRIMARY KEY(alias_name) ) ENGINE=InnoDB COMMENT='稳定学校ID补匹配使用的校区名称临时映射'; INSERT INTO tmp_parent_aliases VALUES {values}; SET @alias_before=(SELECT COUNT(*) FROM school_names); INSERT IGNORE INTO school_names ( school_id, school_name, name_type, valid_from_year, source_id ) SELECT school.school_id, mapping.alias_name, 'campus_alias', 2025, @source_id FROM tmp_parent_aliases mapping JOIN schools school ON school.official_name=mapping.parent_school_name; INSERT IGNORE INTO school_campuses ( campus_id, school_id, campus_name, province_name, city_name, is_main_campus, verification_status, source_id ) SELECT UUID(), school.school_id, mapping.campus_name, mapping.province_name, mapping.city_name, 0, 'pending', @source_id FROM tmp_parent_aliases mapping JOIN schools school ON school.official_name=mapping.parent_school_name WHERE mapping.campus_name IS NOT NULL; CREATE TEMPORARY TABLE tmp_raw_school_names ( raw_name VARCHAR(255) NOT NULL COMMENT '尚未关联稳定ID的招生来源学校名称', PRIMARY KEY(raw_name) ) ENGINE=InnoDB COMMENT='自动识别尾部地区或原校名注记的临时来源名称表'; INSERT IGNORE INTO tmp_raw_school_names SELECT school_name_raw FROM admission_groups WHERE school_id IS NULL; INSERT IGNORE INTO tmp_raw_school_names SELECT school_name_raw FROM admission_major_lines WHERE school_id IS NULL; INSERT IGNORE INTO tmp_raw_school_names SELECT school_name_raw FROM admission_cutoff_lines WHERE school_id IS NULL; INSERT IGNORE INTO school_names ( school_id, school_name, name_type, valid_from_year, source_id ) SELECT MIN(school.school_id), raw.raw_name, 'source_variant', 2025, @source_id FROM tmp_raw_school_names raw JOIN schools school ON school.official_name=TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(raw.raw_name,'(',1),'(',1),'[',1)) WHERE raw.raw_name<>school.official_name AND CHAR_LENGTH(TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(raw.raw_name,'(',1),'(',1),'[',1)))>=4 GROUP BY raw.raw_name HAVING COUNT(DISTINCT school.school_id)=1; CREATE TEMPORARY TABLE tmp_unique_base_names ( normalized_base VARCHAR(255) NOT NULL COMMENT '统一括号并移除空格后的现有正式名或别名', school_id CHAR(36) NOT NULL COMMENT '系统稳定学校UUID', PRIMARY KEY(normalized_base) ) ENGINE=InnoDB COMMENT='保留学校名内部括号的最长前缀匹配基础表'; INSERT INTO tmp_unique_base_names SELECT normalized_base,MIN(school_id) FROM ( SELECT REPLACE(REPLACE(REPLACE(TRIM(official_name),'(','('),')',')'),' ','') normalized_base,school_id FROM schools UNION ALL SELECT REPLACE(REPLACE(REPLACE(TRIM(school_name),'(','('),')',')'),' ','') normalized_base,school_id FROM school_names ) names GROUP BY normalized_base HAVING COUNT(DISTINCT school_id)=1; CREATE TEMPORARY TABLE tmp_prefix_matches ( raw_name VARCHAR(255) NOT NULL COMMENT '招生来源完整学校名称', school_id CHAR(36) NOT NULL COMMENT '候选稳定学校UUID', base_length SMALLINT UNSIGNED NOT NULL COMMENT '候选基础名称长度', KEY idx_prefix_raw(raw_name), KEY idx_prefix_best(raw_name,base_length) ) ENGINE=InnoDB COMMENT='尾随地区性质计划注记的最长前缀候选表'; INSERT INTO tmp_prefix_matches SELECT raw.raw_name,base.school_id,CHAR_LENGTH(base.normalized_base) FROM tmp_raw_school_names raw JOIN tmp_unique_base_names base ON REPLACE(REPLACE(REPLACE(TRIM(raw.raw_name),'(','('),')',')'),' ','') LIKE CONCAT(base.normalized_base,'%') WHERE CHAR_LENGTH(REPLACE(REPLACE(REPLACE(TRIM(raw.raw_name),'(','('),')',')'),' ','')) >CHAR_LENGTH(base.normalized_base) AND SUBSTRING( REPLACE(REPLACE(REPLACE(TRIM(raw.raw_name),'(','('),')',')'),' ',''), CHAR_LENGTH(base.normalized_base)+1,1 ) IN ('(', '['); CREATE TEMPORARY TABLE tmp_prefix_best ( raw_name VARCHAR(255) NOT NULL COMMENT '招生来源完整学校名称', best_length SMALLINT UNSIGNED NOT NULL COMMENT '唯一候选中的最长基础名称长度', PRIMARY KEY(raw_name) ) ENGINE=InnoDB COMMENT='尾随注记最长前缀选择表'; INSERT INTO tmp_prefix_best SELECT raw_name,MAX(base_length) FROM tmp_prefix_matches GROUP BY raw_name; INSERT IGNORE INTO school_names ( school_id,school_name,name_type,valid_from_year,source_id ) SELECT MIN(candidate.school_id),candidate.raw_name,'source_variant',2025,@source_id FROM tmp_prefix_matches candidate JOIN tmp_prefix_best best ON best.raw_name=candidate.raw_name AND best.best_length=candidate.base_length GROUP BY candidate.raw_name HAVING COUNT(DISTINCT candidate.school_id)=1; INSERT IGNORE INTO school_names ( school_id, school_name, name_type, valid_from_year, valid_to_year, source_id ) SELECT school_id, TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(school_name,'(原',-1),')',1)), 'former', 2025, 2025, @source_id FROM school_names WHERE source_id=@source_id AND school_name LIKE '%(原%)%' AND CHAR_LENGTH(TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(school_name,'(原',-1),')',1)))>=4; CREATE TEMPORARY TABLE tmp_unique_school_names ( normalized_name VARCHAR(255) NOT NULL COMMENT '统一全半角括号并去空格后的唯一学校名称', school_id CHAR(36) NOT NULL COMMENT '系统稳定学校UUID', PRIMARY KEY(normalized_name) ) ENGINE=InnoDB COMMENT='招生记录关联稳定学校ID的唯一名称临时表'; INSERT INTO tmp_unique_school_names SELECT normalized_name,MIN(school_id) FROM ( SELECT REPLACE(REPLACE(REPLACE(TRIM(official_name),'(','('),')',')'),' ','') normalized_name,school_id FROM schools UNION ALL SELECT REPLACE(REPLACE(REPLACE(TRIM(school_name),'(','('),')',')'),' ','') normalized_name,school_id FROM school_names ) names GROUP BY normalized_name HAVING COUNT(DISTINCT school_id)=1; SET @before_groups=(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL); UPDATE admission_groups target JOIN tmp_unique_school_names mapped ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'(','('),')',')'),' ','') SET target.school_id=mapped.school_id WHERE target.school_id IS NULL; SET @matched_groups=@before_groups-(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL); SET @before_major=(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL); UPDATE admission_major_lines target JOIN tmp_unique_school_names mapped ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'(','('),')',')'),' ','') SET target.school_id=mapped.school_id WHERE target.school_id IS NULL; SET @matched_major=@before_major-(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL); SET @before_cutoff=(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL); UPDATE admission_cutoff_lines target JOIN tmp_unique_school_names mapped ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'(','('),')',')'),' ','') SET target.school_id=mapped.school_id WHERE target.school_id IS NULL; SET @matched_cutoff=@before_cutoff-(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL); SET @alias_inserted=(SELECT COUNT(*) FROM school_names)-@alias_before; UPDATE ingestion_runs SET finished_at=NOW(),status='success',rows_inserted=@alias_inserted, rows_updated=@matched_groups+@matched_major+@matched_cutoff, message=CONCAT('新增招生来源别名',@alias_inserted,'个、校区', (SELECT COUNT(*) FROM school_campuses WHERE source_id=@source_id),'个;补匹配专业组',@matched_groups, '条、专业线',@matched_major,'条、投档线',@matched_cutoff,'条。') WHERE ingestion_run_id={sql_string(run_id)}; COMMIT; """ mysql_run(args, sql) print(f"完成{len(rows)}个校区名称映射及安全来源名称归一化。") if __name__ == "__main__": main()