"""导入教育部历史年度高校名单,不回退较新年度的学校主数据。""" import argparse import hashlib import os import uuid from pathlib import Path from import_moe_schools import read_rows, run_mysql, sql_string, values_sql def main(): parser = argparse.ArgumentParser(description="导入教育部高校名单历史快照并补充历史校名") parser.add_argument("--source-file", required=True) parser.add_argument("--year", type=int, required=True) parser.add_argument("--published-on", required=True) parser.add_argument("--expected-count", type=int, required=True) parser.add_argument("--landing-page-url", required=True) parser.add_argument("--file-url", required=True) parser.add_argument("--mysql-bin", required=True) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=3306) parser.add_argument("--user", default="root") parser.add_argument("--database", default="zhiyuantong") args = parser.parse_args() source = Path(args.source_file) records, rejected = read_rows(source) if rejected: raise ValueError(f"教育部名单存在{len(rejected)}条校验失败记录") if len(records) != args.expected_count: raise ValueError(f"教育部名单应有{args.expected_count}所,实际解析{len(records)}所") sha256 = hashlib.sha256(source.read_bytes()).hexdigest() run_id = str(uuid.uuid4()) statements = [ "SET NAMES utf8mb4;", "SET time_zone='+08:00';", f"USE `{args.database}`;", "START TRANSACTION;", f""" INSERT INTO data_sources ( source_type, publisher, title, landing_page_url, file_url, published_on, retrieved_at, sha256, raw_file_path, notes ) VALUES ( 'official_xls', '中华人民共和国教育部', {sql_string(f'全国普通高等学校名单(截至{args.published_on})')}, {sql_string(args.landing_page_url)}, {sql_string(args.file_url)}, {sql_string(args.published_on)}, NOW(), {sql_string(sha256)}, {sql_string(str(source.resolve()))}, '历史年度全国普通高校名单;用于保留招生年度有效校名,不覆盖较新年度主数据。' ) AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at, raw_file_path=incoming.raw_file_path, notes=incoming.notes; SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(sha256)}); INSERT INTO ingestion_runs ( ingestion_run_id, source_id, dataset_name, dataset_year, started_at, status, rows_read, rows_rejected ) VALUES ({sql_string(run_id)}, @source_id, 'moe_regular_higher_education_institutions_historical', {args.year}, NOW(), 'running', {len(records)}, 0); """, ] for start in range(0, len(records), 400): statements.append(f""" INSERT INTO stg_moe_schools ( ingestion_run_id, source_row_number, moe_school_code, official_name, province_name, city_name, authority_name, education_level, source_remark ) VALUES {values_sql(records[start:start + 400], run_id)}; """) statements.extend([ f""" SET @existing_count=( SELECT COUNT(*) FROM schools s JOIN stg_moe_schools stg ON stg.moe_school_code=s.moe_school_code AND stg.ingestion_run_id={sql_string(run_id)} ); INSERT IGNORE INTO school_names ( school_id, school_name, name_type, valid_from_year, valid_to_year, source_id ) SELECT s.school_id, stg.official_name, 'former', {args.year}, {args.year}, @source_id FROM stg_moe_schools stg JOIN schools s ON s.moe_school_code=stg.moe_school_code WHERE stg.ingestion_run_id={sql_string(run_id)} AND s.official_name<>stg.official_name; UPDATE schools s JOIN stg_moe_schools stg ON stg.moe_school_code=s.moe_school_code SET s.first_seen_year=LEAST(s.first_seen_year,{args.year}) WHERE stg.ingestion_run_id={sql_string(run_id)}; INSERT INTO schools ( school_id, moe_school_code, official_name, province_name, city_name, authority_name, education_level, source_remark, ownership_type, school_status, first_seen_year, last_seen_year, source_id ) SELECT UUID(), stg.moe_school_code, stg.official_name, stg.province_name, stg.city_name, stg.authority_name, stg.education_level, stg.source_remark, CASE WHEN stg.source_remark='民办' THEN '民办' WHEN stg.source_remark LIKE '%中外合作办学%' THEN '合作办学' WHEN stg.source_remark LIKE '%境外高等教育机构%' THEN '境外办学' ELSE '公办' END, 'inactive', {args.year}, {args.year}, @source_id FROM stg_moe_schools stg LEFT JOIN schools s ON s.moe_school_code=stg.moe_school_code WHERE stg.ingestion_run_id={sql_string(run_id)} AND s.school_id IS NULL; INSERT INTO school_names ( school_id, school_name, name_type, valid_from_year, valid_to_year, source_id ) SELECT s.school_id, stg.official_name, 'official', {args.year}, CASE WHEN s.last_seen_year={args.year} AND s.school_status='inactive' THEN {args.year} ELSE NULL END, @source_id FROM stg_moe_schools stg JOIN schools s ON s.moe_school_code=stg.moe_school_code WHERE stg.ingestion_run_id={sql_string(run_id)} ON DUPLICATE KEY UPDATE valid_from_year=LEAST(valid_from_year,VALUES(valid_from_year)), source_id=VALUES(source_id); DROP TEMPORARY TABLE IF EXISTS tmp_unique_school_names; CREATE TEMPORARY TABLE tmp_unique_school_names ( normalized_name VARCHAR(255) NOT NULL COMMENT '统一括号和空格后的学校名称', school_id CHAR(36) NOT NULL COMMENT '唯一对应的系统稳定学校UUID', PRIMARY KEY (normalized_name) ) ENGINE=InnoDB COMMENT='本次历史校名导入使用的唯一名称映射临时表'; INSERT INTO tmp_unique_school_names (normalized_name, school_id) SELECT normalized_name, MIN(school_id) FROM ( SELECT REPLACE(REPLACE(REPLACE(TRIM(official_name),'(','('),')',')'),' ','') normalized_name, school_id FROM schools UNION ALL SELECT REPLACE(REPLACE(REPLACE(TRIM(school_name),'(','('),')',')'),' ','') normalized_name, school_id FROM school_names ) names GROUP BY normalized_name HAVING COUNT(DISTINCT school_id)=1; SET @before_groups=(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL); UPDATE admission_groups target JOIN tmp_unique_school_names mapped ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'(','('),')',')'),' ','') SET target.school_id=mapped.school_id WHERE target.school_id IS NULL; SET @matched_groups=@before_groups-(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL); SET @before_major=(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL); UPDATE admission_major_lines target JOIN tmp_unique_school_names mapped ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'(','('),')',')'),' ','') SET target.school_id=mapped.school_id WHERE target.school_id IS NULL; SET @matched_major=@before_major-(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL); SET @before_cutoff=(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL); UPDATE admission_cutoff_lines target JOIN tmp_unique_school_names mapped ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'(','('),')',')'),' ','') SET target.school_id=mapped.school_id WHERE target.school_id IS NULL; SET @matched_cutoff=@before_cutoff-(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL); UPDATE ingestion_runs SET finished_at=NOW(), status='success', rows_inserted={len(records)}-@existing_count, rows_updated=@existing_count, message=CONCAT('历史快照校验通过;新增历史学校', {len(records)}-@existing_count, '所;补匹配专业组',@matched_groups,'条、专业线',@matched_major,'条、投档线',@matched_cutoff,'条。') WHERE ingestion_run_id={sql_string(run_id)}; COMMIT; """, ]) run_mysql(args.mysql_bin, args.host, args.port, args.user, "\n".join(statements)) print(f"教育部{args.year}年历史快照导入完成:{len(records)}所。") if __name__ == "__main__": main()