You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
185 lines
10 KiB
185 lines
10 KiB
"""用人工复核映射修正招生来源学校名，并删除确定误入的数据行。"""
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import os
|
|
import subprocess
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
|
|
def sql_string(value):
|
|
if value is None or value == "":
|
|
return "NULL"
|
|
return "'" + str(value).replace("\\", "\\\\").replace("'", "\\'") + "'"
|
|
|
|
|
|
def mysql_run(args, sql, capture=False):
|
|
command = [args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port),
|
|
"-u", args.user, "--default-character-set=utf8mb4", "-D", args.database]
|
|
if capture:
|
|
command.extend(["-N", "-B"])
|
|
result = subprocess.run(command, input=sql, text=True, encoding="utf-8", capture_output=True)
|
|
if result.returncode:
|
|
raise RuntimeError(result.stderr.strip() or result.stdout.strip())
|
|
return result.stdout
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="清理2025招生来源学校名称异常")
|
|
parser.add_argument("--corrections-file", required=True)
|
|
parser.add_argument("--invalid-file", required=True)
|
|
parser.add_argument("--mysql-bin", required=True)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--port", type=int, default=3306)
|
|
parser.add_argument("--user", default="root")
|
|
parser.add_argument("--database", default="zhiyuantong")
|
|
args = parser.parse_args()
|
|
if not os.environ.get("MYSQL_PWD"):
|
|
raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码")
|
|
|
|
corrections_path = Path(args.corrections_file)
|
|
invalid_path = Path(args.invalid_file)
|
|
with corrections_path.open(encoding="utf-8-sig", newline="") as stream:
|
|
corrections = list(csv.DictReader(stream))
|
|
with invalid_path.open(encoding="utf-8-sig", newline="") as stream:
|
|
invalid = list(csv.DictReader(stream))
|
|
if not corrections or set(corrections[0]) != {"raw_name", "target_school_name", "correction_type", "notes"}:
|
|
raise ValueError("名称修正规则字段不符合预期")
|
|
if not invalid or set(invalid[0]) != {"raw_name", "reason"}:
|
|
raise ValueError("无效行规则字段不符合预期")
|
|
if len({row["raw_name"] for row in corrections}) != len(corrections):
|
|
raise ValueError("名称修正规则存在重复来源名称")
|
|
|
|
targets = sorted({row["target_school_name"] for row in corrections})
|
|
target_sql = ",".join(sql_string(name) for name in targets)
|
|
found_text = mysql_run(
|
|
args,
|
|
f"SELECT official_name,COUNT(*) FROM schools WHERE official_name IN ({target_sql}) GROUP BY official_name;",
|
|
capture=True,
|
|
)
|
|
found = {line.split("\t")[0]: int(line.split("\t")[1]) for line in found_text.strip().splitlines()}
|
|
missing = [name for name in targets if found.get(name) != 1]
|
|
if missing:
|
|
raise ValueError(f"目标学校不存在或不唯一：{missing}")
|
|
|
|
digest = hashlib.sha256(corrections_path.read_bytes() + b"\n" + invalid_path.read_bytes()).hexdigest()
|
|
run_id = str(uuid.uuid4())
|
|
correction_values = ",\n".join(
|
|
"(" + ",".join(sql_string(row[key]) for key in [
|
|
"raw_name", "target_school_name", "correction_type", "notes"
|
|
]) + ")" for row in corrections
|
|
)
|
|
invalid_values = ",\n".join(
|
|
"(" + ",".join(sql_string(row[key]) for key in ["raw_name", "reason"]) + ")"
|
|
for row in invalid
|
|
)
|
|
sql = f"""
|
|
SET NAMES utf8mb4;
|
|
SET time_zone='+08:00';
|
|
START TRANSACTION;
|
|
INSERT INTO data_sources (
|
|
source_type,publisher,title,retrieved_at,sha256,raw_file_path,notes
|
|
) VALUES (
|
|
'curated_mapping','志愿通数据治理','2025招生来源学校名称异常复核规则',NOW(),
|
|
{sql_string(digest)},{sql_string(str(corrections_path.resolve()) + ';' + str(invalid_path.resolve()))},
|
|
'逐条保留错别字、截断、旧称、二级机构和无效行的复核依据；无效行删除前已有全库备份。'
|
|
) AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at,
|
|
raw_file_path=incoming.raw_file_path,notes=incoming.notes;
|
|
SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(digest)});
|
|
INSERT INTO ingestion_runs (
|
|
ingestion_run_id,source_id,dataset_name,dataset_year,started_at,status,rows_read,rows_rejected
|
|
) VALUES ({sql_string(run_id)},@source_id,'school_name_anomaly_cleanup',2025,NOW(),'running',
|
|
{len(corrections) + len(invalid)},0);
|
|
|
|
CREATE TEMPORARY TABLE tmp_school_name_corrections (
|
|
raw_name VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '招生来源中的异常学校名称原值',
|
|
target_school_name VARCHAR(255) NOT NULL COMMENT '人工复核后的学校主体正式名称',
|
|
correction_type VARCHAR(32) NOT NULL COMMENT '异常类型：旧称错字截断标点或二级机构',
|
|
notes VARCHAR(512) NOT NULL COMMENT '人工复核依据',
|
|
PRIMARY KEY(raw_name)
|
|
) ENGINE=InnoDB COMMENT='招生来源学校名称异常修正临时表';
|
|
INSERT INTO tmp_school_name_corrections VALUES
|
|
{correction_values};
|
|
|
|
INSERT IGNORE INTO school_names (
|
|
school_id,school_name,name_type,valid_from_year,valid_to_year,source_id
|
|
)
|
|
SELECT s.school_id,c.raw_name,
|
|
CASE WHEN c.correction_type='former' THEN 'former'
|
|
WHEN c.correction_type='secondary_unit' THEN 'campus_alias'
|
|
ELSE 'source_variant' END,
|
|
2025,CASE WHEN c.correction_type='former' THEN 2025 ELSE NULL END,@source_id
|
|
FROM tmp_school_name_corrections c JOIN schools s ON s.official_name=c.target_school_name;
|
|
|
|
SET @groups_before=(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL);
|
|
UPDATE admission_groups target JOIN tmp_school_name_corrections c ON c.raw_name=target.school_name_raw
|
|
JOIN schools s ON s.official_name=c.target_school_name
|
|
SET target.school_id=s.school_id WHERE target.school_id IS NULL;
|
|
SET @matched_groups=@groups_before-(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL);
|
|
|
|
SET @major_before=(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL);
|
|
UPDATE admission_major_lines target JOIN tmp_school_name_corrections c ON c.raw_name=target.school_name_raw
|
|
JOIN schools s ON s.official_name=c.target_school_name
|
|
SET target.school_id=s.school_id WHERE target.school_id IS NULL;
|
|
SET @matched_major=@major_before-(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL);
|
|
|
|
SET @cutoff_before=(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL);
|
|
UPDATE admission_cutoff_lines target JOIN tmp_school_name_corrections c ON c.raw_name=target.school_name_raw
|
|
JOIN schools s ON s.official_name=c.target_school_name
|
|
SET target.school_id=s.school_id WHERE target.school_id IS NULL;
|
|
SET @matched_cutoff=@cutoff_before-(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL);
|
|
|
|
CREATE TEMPORARY TABLE tmp_invalid_school_rows (
|
|
raw_name VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '误入学校名称字段的原始文本',
|
|
reason VARCHAR(512) NOT NULL COMMENT '判定无效数据行的原因',
|
|
PRIMARY KEY(raw_name)
|
|
) ENGINE=InnoDB COMMENT='确认无效的招生来源行清理规则临时表';
|
|
INSERT INTO tmp_invalid_school_rows VALUES
|
|
{invalid_values};
|
|
|
|
SET @deleted_groups=(SELECT COUNT(*) FROM admission_groups g JOIN tmp_invalid_school_rows i ON i.raw_name=g.school_name_raw);
|
|
DELETE g FROM admission_groups g JOIN tmp_invalid_school_rows i ON i.raw_name=g.school_name_raw;
|
|
SET @deleted_major=(SELECT COUNT(*) FROM admission_major_lines m JOIN tmp_invalid_school_rows i ON i.raw_name=m.school_name_raw);
|
|
DELETE m FROM admission_major_lines m JOIN tmp_invalid_school_rows i ON i.raw_name=m.school_name_raw;
|
|
SET @deleted_cutoff=(SELECT COUNT(*) FROM admission_cutoff_lines c JOIN tmp_invalid_school_rows i ON i.raw_name=c.school_name_raw);
|
|
DELETE c FROM admission_cutoff_lines c JOIN tmp_invalid_school_rows i ON i.raw_name=c.school_name_raw;
|
|
|
|
INSERT INTO data_quality_issues (
|
|
entity_type,entity_key,issue_code,severity,field_name,issue_message,
|
|
issue_fingerprint,resolution_status,detected_at,resolved_at,resolved_by,resolution_note
|
|
)
|
|
SELECT 'admission_group',CAST(g.admission_group_id AS CHAR),'COMPOSITE_ADMISSION_UNIT','info','school_id',
|
|
'该招生单元下的专业分别由多所民航院校承办，不应绑定单一学校ID。',
|
|
SHA2(CONCAT('admission_group|',g.admission_group_id,'|COMPOSITE_ADMISSION_UNIT'),256),
|
|
'false_positive',NOW(),NOW(),'system_import',
|
|
'保留school_id为空；7个专业计划的承办院校记录在专业名称与备注中。'
|
|
FROM admission_groups g
|
|
WHERE g.admission_year=2025 AND g.province_name='上海市'
|
|
AND g.school_name_raw='民航飞行技术类'
|
|
ON DUPLICATE KEY UPDATE resolution_status='false_positive',resolved_at=NOW(),
|
|
resolved_by='system_import',resolution_note=VALUES(resolution_note);
|
|
|
|
UPDATE dataset_coverage SET record_count=(
|
|
SELECT COUNT(*) FROM admission_cutoff_lines l
|
|
WHERE l.admission_year=2025 AND l.province_name='山西省'
|
|
),notes='GaokaoCompass 2025院校投档线CSV原始1097条；259条缺院校名称未入业务表，31条误入表头已清理，保留807条有效记录且全部关联稳定学校ID。'
|
|
WHERE dataset_name='院校专业组投档线' AND admission_year=2025
|
|
AND province_name='山西省';
|
|
|
|
UPDATE ingestion_runs SET finished_at=NOW(),status='success',
|
|
rows_updated=@matched_groups+@matched_major+@matched_cutoff,
|
|
rows_rejected=@deleted_groups+@deleted_major+@deleted_cutoff,
|
|
message=CONCAT('补匹配专业组',@matched_groups,'条、专业线',@matched_major,
|
|
'条、投档线',@matched_cutoff,'条；删除误入表头或类别行',
|
|
@deleted_groups+@deleted_major+@deleted_cutoff,'条。')
|
|
WHERE ingestion_run_id={sql_string(run_id)};
|
|
COMMIT;
|
|
"""
|
|
mysql_run(args, sql)
|
|
print(mysql_run(args, "SELECT message FROM ingestion_runs WHERE ingestion_run_id=" + sql_string(run_id) + ";", True).strip())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|