You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
198 lines
9.9 KiB
198 lines
9.9 KiB
"""导入不在教育部普通高校名单内、但确有普通高考招生记录的特殊院校。"""
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import os
|
|
import subprocess
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
|
|
def sql_string(value):
|
|
if value is None or value == "":
|
|
return "NULL"
|
|
return "'" + str(value).replace("\\", "\\\\").replace("'", "\\'") + "'"
|
|
|
|
|
|
def mysql_run(args, sql, capture=False):
|
|
command = [
|
|
args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port),
|
|
"-u", args.user, "--default-character-set=utf8mb4", "-D", args.database,
|
|
]
|
|
if capture:
|
|
command.extend(["-N", "-B"])
|
|
process = subprocess.run(command, input=sql, text=True, encoding="utf-8", capture_output=True)
|
|
if process.returncode:
|
|
raise RuntimeError(process.stderr.strip() or process.stdout.strip())
|
|
return process.stdout
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="导入2025军队等特殊招生院校并关联稳定ID")
|
|
parser.add_argument("--mapping-file", required=True)
|
|
parser.add_argument("--raw-file", required=True)
|
|
parser.add_argument("--landing-page-url", required=True)
|
|
parser.add_argument("--publisher", required=True)
|
|
parser.add_argument("--source-title", required=True)
|
|
parser.add_argument("--dataset-name", default="special_school_master_military")
|
|
parser.add_argument("--published-on", default="2025-06-03")
|
|
parser.add_argument("--source-notes", default="特殊招生院校使用官方名录或招生政策确认主体；地点待核验项不用于自动地理分析。")
|
|
parser.add_argument("--mysql-bin", required=True)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--port", type=int, default=3306)
|
|
parser.add_argument("--user", default="root")
|
|
parser.add_argument("--database", default="zhiyuantong")
|
|
args = parser.parse_args()
|
|
if not os.environ.get("MYSQL_PWD"):
|
|
raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码")
|
|
|
|
mapping_path = Path(args.mapping_file)
|
|
raw_path = Path(args.raw_file)
|
|
if not raw_path.is_file():
|
|
raise FileNotFoundError(raw_path)
|
|
with mapping_path.open(encoding="utf-8-sig", newline="") as stream:
|
|
rows = list(csv.DictReader(stream))
|
|
required = {
|
|
"official_name", "school_kind", "authority_name", "education_level",
|
|
"province_name", "city_name", "aliases", "notes",
|
|
}
|
|
if not rows or set(rows[0]) != required:
|
|
raise ValueError("映射文件字段不符合预期")
|
|
if len({row["official_name"] for row in rows}) != len(rows):
|
|
raise ValueError("特殊院校正式名称重复")
|
|
|
|
sha256 = hashlib.sha256(raw_path.read_bytes()).hexdigest()
|
|
run_id = str(uuid.uuid4())
|
|
row_values = ",\n".join(
|
|
"(" + ",".join(sql_string(row[key]) for key in [
|
|
"official_name", "school_kind", "authority_name", "education_level",
|
|
"province_name", "city_name", "aliases", "notes",
|
|
]) + ")"
|
|
for row in rows
|
|
)
|
|
|
|
sql = f"""
|
|
SET NAMES utf8mb4;
|
|
SET time_zone='+08:00';
|
|
START TRANSACTION;
|
|
INSERT INTO data_sources (
|
|
source_type,publisher,title,landing_page_url,published_on,retrieved_at,
|
|
sha256,raw_file_path,notes
|
|
) VALUES (
|
|
'official_html',{sql_string(args.publisher)},{sql_string(args.source_title)},
|
|
{sql_string(args.landing_page_url)},{sql_string(args.published_on)},NOW(),{sql_string(sha256)},
|
|
{sql_string(str(raw_path.resolve()))},
|
|
{sql_string(args.source_notes)}
|
|
) AS incoming ON DUPLICATE KEY UPDATE
|
|
retrieved_at=incoming.retrieved_at,raw_file_path=incoming.raw_file_path,
|
|
landing_page_url=incoming.landing_page_url,notes=incoming.notes;
|
|
SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(sha256)});
|
|
INSERT INTO ingestion_runs (
|
|
ingestion_run_id,source_id,dataset_name,dataset_year,started_at,status,rows_read,rows_rejected
|
|
) VALUES ({sql_string(run_id)},@source_id,{sql_string(args.dataset_name)},2025,NOW(),'running',{len(rows)},0);
|
|
|
|
CREATE TEMPORARY TABLE tmp_special_schools (
|
|
official_name VARCHAR(255) NOT NULL COMMENT '招生院校官方名称',
|
|
school_kind VARCHAR(32) NOT NULL COMMENT '特殊院校类别',
|
|
authority_name VARCHAR(255) NOT NULL COMMENT '主管单位',
|
|
education_level VARCHAR(32) NOT NULL COMMENT '招生培养层次',
|
|
province_name VARCHAR(64) NOT NULL COMMENT '院校所在省份；无法确认时为待核验',
|
|
city_name VARCHAR(128) NOT NULL COMMENT '院校所在城市；无法确认时为待核验',
|
|
aliases TEXT NULL COMMENT '竖线分隔的招生来源别名',
|
|
notes VARCHAR(512) NULL COMMENT '来源及口径备注',
|
|
PRIMARY KEY(official_name)
|
|
) ENGINE=InnoDB COMMENT='特殊招生院校导入临时表';
|
|
INSERT INTO tmp_special_schools VALUES
|
|
{row_values};
|
|
|
|
SET @schools_before=(SELECT COUNT(*) FROM schools);
|
|
INSERT INTO schools (
|
|
school_id,moe_school_code,official_name,province_name,city_name,
|
|
authority_name,education_level,source_remark,ownership_type,school_status,
|
|
first_seen_year,last_seen_year,source_id
|
|
)
|
|
SELECT UUID(),NULL,t.official_name,t.province_name,t.city_name,t.authority_name,
|
|
t.education_level,CONCAT(t.school_kind,'；',t.notes),
|
|
CASE WHEN t.school_kind LIKE '%自资%' THEN '民办' ELSE '公办' END,
|
|
'active',2025,2025,@source_id
|
|
FROM tmp_special_schools t
|
|
WHERE NOT EXISTS (SELECT 1 FROM schools s WHERE s.official_name=t.official_name);
|
|
|
|
UPDATE schools s JOIN tmp_special_schools t ON t.official_name=s.official_name
|
|
SET s.authority_name=t.authority_name,s.education_level=t.education_level,
|
|
s.province_name=t.province_name,s.city_name=t.city_name,
|
|
s.source_remark=CONCAT(t.school_kind,'；',t.notes),s.school_status='active',
|
|
s.first_seen_year=LEAST(s.first_seen_year,2025),s.last_seen_year=GREATEST(s.last_seen_year,2025),
|
|
s.source_id=@source_id
|
|
WHERE s.moe_school_code IS NULL;
|
|
|
|
SET @names_before=(SELECT COUNT(*) FROM school_names);
|
|
INSERT IGNORE INTO school_names (school_id,school_name,name_type,valid_from_year,source_id)
|
|
SELECT s.school_id,t.official_name,'official',2025,@source_id
|
|
FROM tmp_special_schools t JOIN schools s ON s.official_name=t.official_name;
|
|
|
|
INSERT IGNORE INTO school_names (school_id,school_name,name_type,valid_from_year,source_id)
|
|
WITH RECURSIVE alias_parts AS (
|
|
SELECT official_name,TRIM(SUBSTRING_INDEX(aliases,'|',1)) alias_name,
|
|
CASE WHEN aliases LIKE '%|%' THEN SUBSTRING(aliases,LOCATE('|',aliases)+1) ELSE '' END rest
|
|
FROM tmp_special_schools WHERE aliases IS NOT NULL AND aliases<>''
|
|
UNION ALL
|
|
SELECT official_name,TRIM(SUBSTRING_INDEX(rest,'|',1)),
|
|
CASE WHEN rest LIKE '%|%' THEN SUBSTRING(rest,LOCATE('|',rest)+1) ELSE '' END
|
|
FROM alias_parts WHERE rest<>''
|
|
)
|
|
SELECT s.school_id,a.alias_name,'alias',2025,@source_id
|
|
FROM alias_parts a JOIN schools s ON s.official_name=a.official_name
|
|
WHERE a.alias_name<>'';
|
|
|
|
CREATE TEMPORARY TABLE tmp_unique_school_names (
|
|
normalized_name VARCHAR(255) NOT NULL COMMENT '统一括号并移除空格后的唯一招生名称',
|
|
school_id CHAR(36) NOT NULL COMMENT '系统稳定学校UUID',
|
|
PRIMARY KEY(normalized_name)
|
|
) ENGINE=InnoDB COMMENT='招生记录稳定学校ID关联临时表';
|
|
INSERT INTO tmp_unique_school_names
|
|
SELECT normalized_name,MIN(school_id) FROM (
|
|
SELECT REPLACE(REPLACE(REPLACE(TRIM(official_name),'（','('),'）',')'),' ','') normalized_name,school_id FROM schools
|
|
UNION ALL
|
|
SELECT REPLACE(REPLACE(REPLACE(TRIM(school_name),'（','('),'）',')'),' ','') normalized_name,school_id FROM school_names
|
|
) all_names GROUP BY normalized_name HAVING COUNT(DISTINCT school_id)=1;
|
|
|
|
SET @groups_before=(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL);
|
|
UPDATE admission_groups target JOIN tmp_unique_school_names mapped
|
|
ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'（','('),'）',')'),' ','')
|
|
SET target.school_id=mapped.school_id WHERE target.school_id IS NULL;
|
|
SET @matched_groups=@groups_before-(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL);
|
|
|
|
SET @major_before=(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL);
|
|
UPDATE admission_major_lines target JOIN tmp_unique_school_names mapped
|
|
ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'（','('),'）',')'),' ','')
|
|
SET target.school_id=mapped.school_id WHERE target.school_id IS NULL;
|
|
SET @matched_major=@major_before-(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL);
|
|
|
|
SET @cutoff_before=(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL);
|
|
UPDATE admission_cutoff_lines target JOIN tmp_unique_school_names mapped
|
|
ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'（','('),'）',')'),' ','')
|
|
SET target.school_id=mapped.school_id WHERE target.school_id IS NULL;
|
|
SET @matched_cutoff=@cutoff_before-(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL);
|
|
|
|
SET @schools_inserted=(SELECT COUNT(*) FROM schools)-@schools_before;
|
|
SET @names_inserted=(SELECT COUNT(*) FROM school_names)-@names_before;
|
|
UPDATE ingestion_runs SET finished_at=NOW(),status='success',rows_inserted=@schools_inserted,
|
|
rows_updated=@matched_groups+@matched_major+@matched_cutoff,
|
|
message=CONCAT('新增特殊院校',@schools_inserted,'所、院校名称',@names_inserted,
|
|
'个；补匹配专业组',@matched_groups,'条、专业线',@matched_major,'条、投档线',@matched_cutoff,'条。')
|
|
WHERE ingestion_run_id={sql_string(run_id)};
|
|
COMMIT;
|
|
"""
|
|
mysql_run(args, sql)
|
|
result = mysql_run(
|
|
args,
|
|
"SELECT message FROM ingestion_runs WHERE ingestion_run_id=" + sql_string(run_id) + ";",
|
|
capture=True,
|
|
).strip()
|
|
print(result)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|