You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
245 lines
12 KiB
245 lines
12 KiB
"""用可审计的校区/来源名称映射补齐招生记录的稳定学校ID。"""
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import os
|
|
import subprocess
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
|
|
def sql_string(value):
|
|
if value is None or value == "":
|
|
return "NULL"
|
|
value = str(value).replace("\\", "\\\\").replace("'", "\\'")
|
|
return f"'{value}'"
|
|
|
|
|
|
def mysql_run(args, sql, capture=False):
|
|
command = [
|
|
args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port),
|
|
"-u", args.user, "--default-character-set=utf8mb4", "-D", args.database,
|
|
]
|
|
if capture:
|
|
command.extend(["-N", "-B"])
|
|
process = subprocess.run(command, input=sql, text=True, encoding="utf-8", capture_output=True)
|
|
if process.returncode:
|
|
raise RuntimeError(process.stderr.strip() or process.stdout.strip())
|
|
return process.stdout
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="补充校区招生别名并重新关联稳定学校ID")
|
|
parser.add_argument("--mapping-file", required=True)
|
|
parser.add_argument("--mysql-bin", required=True)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--port", type=int, default=3306)
|
|
parser.add_argument("--user", default="root")
|
|
parser.add_argument("--database", default="zhiyuantong")
|
|
args = parser.parse_args()
|
|
if not os.environ.get("MYSQL_PWD"):
|
|
raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码")
|
|
|
|
mapping_path = Path(args.mapping_file)
|
|
with mapping_path.open(encoding="utf-8-sig", newline="") as stream:
|
|
rows = list(csv.DictReader(stream))
|
|
required = {"alias_name", "parent_school_name", "campus_name", "province_name", "city_name", "notes"}
|
|
if not rows or set(rows[0]) != required:
|
|
raise ValueError("映射文件字段不符合预期")
|
|
if len({row["alias_name"] for row in rows}) != len(rows):
|
|
raise ValueError("映射文件存在重复招生名称")
|
|
|
|
parents = sorted({row["parent_school_name"] for row in rows})
|
|
parent_list = ",".join(sql_string(value) for value in parents)
|
|
output = mysql_run(
|
|
args,
|
|
f"SELECT official_name,COUNT(*) FROM schools WHERE official_name IN ({parent_list}) GROUP BY official_name;",
|
|
capture=True,
|
|
)
|
|
found = {line.split("\t")[0]: int(line.split("\t")[1]) for line in output.strip().splitlines()}
|
|
invalid = [parent for parent in parents if found.get(parent) != 1]
|
|
if invalid:
|
|
raise ValueError(f"父级学校不存在或不唯一：{invalid}")
|
|
|
|
sha256 = hashlib.sha256(mapping_path.read_bytes()).hexdigest()
|
|
run_id = str(uuid.uuid4())
|
|
values = ",\n".join(
|
|
"(" + ",".join(sql_string(row[key]) for key in [
|
|
"alias_name", "parent_school_name", "campus_name", "province_name", "city_name", "notes"
|
|
]) + ")"
|
|
for row in rows
|
|
)
|
|
sql = f"""
|
|
SET NAMES utf8mb4;
|
|
SET time_zone='+08:00';
|
|
START TRANSACTION;
|
|
INSERT INTO data_sources (
|
|
source_type, publisher, title, retrieved_at, sha256, raw_file_path, notes
|
|
) VALUES (
|
|
'curated_mapping', '志愿通数据治理', '2025招生来源校区名称与教育部学校主体映射',
|
|
NOW(), {sql_string(sha256)}, {sql_string(str(mapping_path.resolve()))},
|
|
'仅映射明确属于同一学校主体的医学部、分校区和分支教学单位；全部进入待审核状态。'
|
|
) AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at,
|
|
raw_file_path=incoming.raw_file_path, notes=incoming.notes;
|
|
SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(sha256)});
|
|
INSERT INTO ingestion_runs (
|
|
ingestion_run_id, source_id, dataset_name, dataset_year, started_at,
|
|
status, rows_read, rows_rejected
|
|
) VALUES ({sql_string(run_id)},@source_id,'school_parent_alias_mapping',2025,NOW(),'running',{len(rows)},0);
|
|
|
|
CREATE TEMPORARY TABLE tmp_parent_aliases (
|
|
alias_name VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '招生数据中出现的学校或校区名称',
|
|
parent_school_name VARCHAR(255) NOT NULL COMMENT '教育部名单中的学校主体名称',
|
|
campus_name VARCHAR(255) NULL COMMENT '招生名称对应的校区或医学部名称',
|
|
province_name VARCHAR(64) NULL COMMENT '校区所在省份',
|
|
city_name VARCHAR(128) NULL COMMENT '校区所在城市',
|
|
notes VARCHAR(512) NULL COMMENT '映射依据说明',
|
|
PRIMARY KEY(alias_name)
|
|
) ENGINE=InnoDB COMMENT='稳定学校ID补匹配使用的校区名称临时映射';
|
|
INSERT INTO tmp_parent_aliases VALUES
|
|
{values};
|
|
|
|
SET @alias_before=(SELECT COUNT(*) FROM school_names);
|
|
INSERT IGNORE INTO school_names (
|
|
school_id, school_name, name_type, valid_from_year, source_id
|
|
)
|
|
SELECT school.school_id, mapping.alias_name, 'campus_alias', 2025, @source_id
|
|
FROM tmp_parent_aliases mapping JOIN schools school
|
|
ON school.official_name=mapping.parent_school_name;
|
|
|
|
INSERT IGNORE INTO school_campuses (
|
|
campus_id, school_id, campus_name, province_name, city_name,
|
|
is_main_campus, verification_status, source_id
|
|
)
|
|
SELECT UUID(), school.school_id, mapping.campus_name, mapping.province_name,
|
|
mapping.city_name, 0, 'pending', @source_id
|
|
FROM tmp_parent_aliases mapping JOIN schools school
|
|
ON school.official_name=mapping.parent_school_name
|
|
WHERE mapping.campus_name IS NOT NULL;
|
|
|
|
CREATE TEMPORARY TABLE tmp_raw_school_names (
|
|
raw_name VARCHAR(255) NOT NULL COMMENT '尚未关联稳定ID的招生来源学校名称',
|
|
PRIMARY KEY(raw_name)
|
|
) ENGINE=InnoDB COMMENT='自动识别尾部地区或原校名注记的临时来源名称表';
|
|
INSERT IGNORE INTO tmp_raw_school_names
|
|
SELECT school_name_raw FROM admission_groups WHERE school_id IS NULL;
|
|
INSERT IGNORE INTO tmp_raw_school_names
|
|
SELECT school_name_raw FROM admission_major_lines WHERE school_id IS NULL;
|
|
INSERT IGNORE INTO tmp_raw_school_names
|
|
SELECT school_name_raw FROM admission_cutoff_lines WHERE school_id IS NULL;
|
|
|
|
INSERT IGNORE INTO school_names (
|
|
school_id, school_name, name_type, valid_from_year, source_id
|
|
)
|
|
SELECT MIN(school.school_id), raw.raw_name, 'source_variant', 2025, @source_id
|
|
FROM tmp_raw_school_names raw JOIN schools school
|
|
ON school.official_name=TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(raw.raw_name,'（',1),'(',1),'[',1))
|
|
WHERE raw.raw_name<>school.official_name
|
|
AND CHAR_LENGTH(TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(raw.raw_name,'（',1),'(',1),'[',1)))>=4
|
|
GROUP BY raw.raw_name
|
|
HAVING COUNT(DISTINCT school.school_id)=1;
|
|
|
|
CREATE TEMPORARY TABLE tmp_unique_base_names (
|
|
normalized_base VARCHAR(255) NOT NULL COMMENT '统一括号并移除空格后的现有正式名或别名',
|
|
school_id CHAR(36) NOT NULL COMMENT '系统稳定学校UUID',
|
|
PRIMARY KEY(normalized_base)
|
|
) ENGINE=InnoDB COMMENT='保留学校名内部括号的最长前缀匹配基础表';
|
|
INSERT INTO tmp_unique_base_names
|
|
SELECT normalized_base,MIN(school_id) FROM (
|
|
SELECT REPLACE(REPLACE(REPLACE(TRIM(official_name),'（','('),'）',')'),' ','') normalized_base,school_id FROM schools
|
|
UNION ALL
|
|
SELECT REPLACE(REPLACE(REPLACE(TRIM(school_name),'（','('),'）',')'),' ','') normalized_base,school_id FROM school_names
|
|
) names GROUP BY normalized_base HAVING COUNT(DISTINCT school_id)=1;
|
|
|
|
CREATE TEMPORARY TABLE tmp_prefix_matches (
|
|
raw_name VARCHAR(255) NOT NULL COMMENT '招生来源完整学校名称',
|
|
school_id CHAR(36) NOT NULL COMMENT '候选稳定学校UUID',
|
|
base_length SMALLINT UNSIGNED NOT NULL COMMENT '候选基础名称长度',
|
|
KEY idx_prefix_raw(raw_name),
|
|
KEY idx_prefix_best(raw_name,base_length)
|
|
) ENGINE=InnoDB COMMENT='尾随地区性质计划注记的最长前缀候选表';
|
|
INSERT INTO tmp_prefix_matches
|
|
SELECT raw.raw_name,base.school_id,CHAR_LENGTH(base.normalized_base)
|
|
FROM tmp_raw_school_names raw JOIN tmp_unique_base_names base
|
|
ON REPLACE(REPLACE(REPLACE(TRIM(raw.raw_name),'（','('),'）',')'),' ','')
|
|
LIKE CONCAT(base.normalized_base,'%')
|
|
WHERE CHAR_LENGTH(REPLACE(REPLACE(REPLACE(TRIM(raw.raw_name),'（','('),'）',')'),' ',''))
|
|
>CHAR_LENGTH(base.normalized_base)
|
|
AND SUBSTRING(
|
|
REPLACE(REPLACE(REPLACE(TRIM(raw.raw_name),'（','('),'）',')'),' ',''),
|
|
CHAR_LENGTH(base.normalized_base)+1,1
|
|
) IN ('(', '[');
|
|
|
|
CREATE TEMPORARY TABLE tmp_prefix_best (
|
|
raw_name VARCHAR(255) NOT NULL COMMENT '招生来源完整学校名称',
|
|
best_length SMALLINT UNSIGNED NOT NULL COMMENT '唯一候选中的最长基础名称长度',
|
|
PRIMARY KEY(raw_name)
|
|
) ENGINE=InnoDB COMMENT='尾随注记最长前缀选择表';
|
|
INSERT INTO tmp_prefix_best
|
|
SELECT raw_name,MAX(base_length) FROM tmp_prefix_matches GROUP BY raw_name;
|
|
|
|
INSERT IGNORE INTO school_names (
|
|
school_id,school_name,name_type,valid_from_year,source_id
|
|
)
|
|
SELECT MIN(candidate.school_id),candidate.raw_name,'source_variant',2025,@source_id
|
|
FROM tmp_prefix_matches candidate JOIN tmp_prefix_best best
|
|
ON best.raw_name=candidate.raw_name AND best.best_length=candidate.base_length
|
|
GROUP BY candidate.raw_name
|
|
HAVING COUNT(DISTINCT candidate.school_id)=1;
|
|
|
|
INSERT IGNORE INTO school_names (
|
|
school_id, school_name, name_type, valid_from_year, valid_to_year, source_id
|
|
)
|
|
SELECT school_id,
|
|
TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(school_name,'（原',-1),'）',1)),
|
|
'former', 2025, 2025, @source_id
|
|
FROM school_names
|
|
WHERE source_id=@source_id AND school_name LIKE '%（原%）%'
|
|
AND CHAR_LENGTH(TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(school_name,'（原',-1),'）',1)))>=4;
|
|
|
|
CREATE TEMPORARY TABLE tmp_unique_school_names (
|
|
normalized_name VARCHAR(255) NOT NULL COMMENT '统一全半角括号并去空格后的唯一学校名称',
|
|
school_id CHAR(36) NOT NULL COMMENT '系统稳定学校UUID',
|
|
PRIMARY KEY(normalized_name)
|
|
) ENGINE=InnoDB COMMENT='招生记录关联稳定学校ID的唯一名称临时表';
|
|
INSERT INTO tmp_unique_school_names
|
|
SELECT normalized_name,MIN(school_id) FROM (
|
|
SELECT REPLACE(REPLACE(REPLACE(TRIM(official_name),'（','('),'）',')'),' ','') normalized_name,school_id FROM schools
|
|
UNION ALL
|
|
SELECT REPLACE(REPLACE(REPLACE(TRIM(school_name),'（','('),'）',')'),' ','') normalized_name,school_id FROM school_names
|
|
) names GROUP BY normalized_name HAVING COUNT(DISTINCT school_id)=1;
|
|
|
|
SET @before_groups=(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL);
|
|
UPDATE admission_groups target JOIN tmp_unique_school_names mapped
|
|
ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'（','('),'）',')'),' ','')
|
|
SET target.school_id=mapped.school_id WHERE target.school_id IS NULL;
|
|
SET @matched_groups=@before_groups-(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL);
|
|
|
|
SET @before_major=(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL);
|
|
UPDATE admission_major_lines target JOIN tmp_unique_school_names mapped
|
|
ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'（','('),'）',')'),' ','')
|
|
SET target.school_id=mapped.school_id WHERE target.school_id IS NULL;
|
|
SET @matched_major=@before_major-(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL);
|
|
|
|
SET @before_cutoff=(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL);
|
|
UPDATE admission_cutoff_lines target JOIN tmp_unique_school_names mapped
|
|
ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'（','('),'）',')'),' ','')
|
|
SET target.school_id=mapped.school_id WHERE target.school_id IS NULL;
|
|
SET @matched_cutoff=@before_cutoff-(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL);
|
|
|
|
SET @alias_inserted=(SELECT COUNT(*) FROM school_names)-@alias_before;
|
|
UPDATE ingestion_runs SET finished_at=NOW(),status='success',rows_inserted=@alias_inserted,
|
|
rows_updated=@matched_groups+@matched_major+@matched_cutoff,
|
|
message=CONCAT('新增招生来源别名',@alias_inserted,'个、校区',
|
|
(SELECT COUNT(*) FROM school_campuses WHERE source_id=@source_id),'个；补匹配专业组',@matched_groups,
|
|
'条、专业线',@matched_major,'条、投档线',@matched_cutoff,'条。')
|
|
WHERE ingestion_run_id={sql_string(run_id)};
|
|
COMMIT;
|
|
"""
|
|
mysql_run(args, sql)
|
|
print(f"完成{len(rows)}个校区名称映射及安全来源名称归一化。")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|