You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
169 lines
8.7 KiB
169 lines
8.7 KiB
"""导入教育部历史年度高校名单，不回退较新年度的学校主数据。"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import os
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
from import_moe_schools import read_rows, run_mysql, sql_string, values_sql
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="导入教育部高校名单历史快照并补充历史校名")
|
|
parser.add_argument("--source-file", required=True)
|
|
parser.add_argument("--year", type=int, required=True)
|
|
parser.add_argument("--published-on", required=True)
|
|
parser.add_argument("--expected-count", type=int, required=True)
|
|
parser.add_argument("--landing-page-url", required=True)
|
|
parser.add_argument("--file-url", required=True)
|
|
parser.add_argument("--mysql-bin", required=True)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--port", type=int, default=3306)
|
|
parser.add_argument("--user", default="root")
|
|
parser.add_argument("--database", default="zhiyuantong")
|
|
args = parser.parse_args()
|
|
|
|
source = Path(args.source_file)
|
|
records, rejected = read_rows(source)
|
|
if rejected:
|
|
raise ValueError(f"教育部名单存在{len(rejected)}条校验失败记录")
|
|
if len(records) != args.expected_count:
|
|
raise ValueError(f"教育部名单应有{args.expected_count}所，实际解析{len(records)}所")
|
|
|
|
sha256 = hashlib.sha256(source.read_bytes()).hexdigest()
|
|
run_id = str(uuid.uuid4())
|
|
statements = [
|
|
"SET NAMES utf8mb4;",
|
|
"SET time_zone='+08:00';",
|
|
f"USE `{args.database}`;",
|
|
"START TRANSACTION;",
|
|
f"""
|
|
INSERT INTO data_sources (
|
|
source_type, publisher, title, landing_page_url, file_url, published_on,
|
|
retrieved_at, sha256, raw_file_path, notes
|
|
) VALUES (
|
|
'official_xls', '中华人民共和国教育部',
|
|
{sql_string(f'全国普通高等学校名单（截至{args.published_on}）')},
|
|
{sql_string(args.landing_page_url)}, {sql_string(args.file_url)},
|
|
{sql_string(args.published_on)}, NOW(), {sql_string(sha256)},
|
|
{sql_string(str(source.resolve()))},
|
|
'历史年度全国普通高校名单；用于保留招生年度有效校名，不覆盖较新年度主数据。'
|
|
) AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at,
|
|
raw_file_path=incoming.raw_file_path, notes=incoming.notes;
|
|
SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(sha256)});
|
|
INSERT INTO ingestion_runs (
|
|
ingestion_run_id, source_id, dataset_name, dataset_year, started_at,
|
|
status, rows_read, rows_rejected
|
|
) VALUES ({sql_string(run_id)}, @source_id,
|
|
'moe_regular_higher_education_institutions_historical', {args.year},
|
|
NOW(), 'running', {len(records)}, 0);
|
|
""",
|
|
]
|
|
for start in range(0, len(records), 400):
|
|
statements.append(f"""
|
|
INSERT INTO stg_moe_schools (
|
|
ingestion_run_id, source_row_number, moe_school_code, official_name,
|
|
province_name, city_name, authority_name, education_level, source_remark
|
|
) VALUES
|
|
{values_sql(records[start:start + 400], run_id)};
|
|
""")
|
|
|
|
statements.extend([
|
|
f"""
|
|
SET @existing_count=(
|
|
SELECT COUNT(*) FROM schools s JOIN stg_moe_schools stg
|
|
ON stg.moe_school_code=s.moe_school_code
|
|
AND stg.ingestion_run_id={sql_string(run_id)}
|
|
);
|
|
|
|
INSERT IGNORE INTO school_names (
|
|
school_id, school_name, name_type, valid_from_year, valid_to_year, source_id
|
|
)
|
|
SELECT s.school_id, stg.official_name, 'former', {args.year}, {args.year}, @source_id
|
|
FROM stg_moe_schools stg JOIN schools s ON s.moe_school_code=stg.moe_school_code
|
|
WHERE stg.ingestion_run_id={sql_string(run_id)}
|
|
AND s.official_name<>stg.official_name;
|
|
|
|
UPDATE schools s JOIN stg_moe_schools stg ON stg.moe_school_code=s.moe_school_code
|
|
SET s.first_seen_year=LEAST(s.first_seen_year,{args.year})
|
|
WHERE stg.ingestion_run_id={sql_string(run_id)};
|
|
|
|
INSERT INTO schools (
|
|
school_id, moe_school_code, official_name, province_name, city_name,
|
|
authority_name, education_level, source_remark, ownership_type,
|
|
school_status, first_seen_year, last_seen_year, source_id
|
|
)
|
|
SELECT UUID(), stg.moe_school_code, stg.official_name, stg.province_name,
|
|
stg.city_name, stg.authority_name, stg.education_level, stg.source_remark,
|
|
CASE WHEN stg.source_remark='民办' THEN '民办'
|
|
WHEN stg.source_remark LIKE '%中外合作办学%' THEN '合作办学'
|
|
WHEN stg.source_remark LIKE '%境外高等教育机构%' THEN '境外办学'
|
|
ELSE '公办' END,
|
|
'inactive', {args.year}, {args.year}, @source_id
|
|
FROM stg_moe_schools stg LEFT JOIN schools s ON s.moe_school_code=stg.moe_school_code
|
|
WHERE stg.ingestion_run_id={sql_string(run_id)} AND s.school_id IS NULL;
|
|
|
|
INSERT INTO school_names (
|
|
school_id, school_name, name_type, valid_from_year, valid_to_year, source_id
|
|
)
|
|
SELECT s.school_id, stg.official_name, 'official', {args.year},
|
|
CASE WHEN s.last_seen_year={args.year} AND s.school_status='inactive' THEN {args.year} ELSE NULL END,
|
|
@source_id
|
|
FROM stg_moe_schools stg JOIN schools s ON s.moe_school_code=stg.moe_school_code
|
|
WHERE stg.ingestion_run_id={sql_string(run_id)}
|
|
ON DUPLICATE KEY UPDATE
|
|
valid_from_year=LEAST(valid_from_year,VALUES(valid_from_year)),
|
|
source_id=VALUES(source_id);
|
|
|
|
DROP TEMPORARY TABLE IF EXISTS tmp_unique_school_names;
|
|
CREATE TEMPORARY TABLE tmp_unique_school_names (
|
|
normalized_name VARCHAR(255) NOT NULL COMMENT '统一括号和空格后的学校名称',
|
|
school_id CHAR(36) NOT NULL COMMENT '唯一对应的系统稳定学校UUID',
|
|
PRIMARY KEY (normalized_name)
|
|
) ENGINE=InnoDB COMMENT='本次历史校名导入使用的唯一名称映射临时表';
|
|
INSERT INTO tmp_unique_school_names (normalized_name, school_id)
|
|
SELECT normalized_name, MIN(school_id)
|
|
FROM (
|
|
SELECT REPLACE(REPLACE(REPLACE(TRIM(official_name),'（','('),'）',')'),' ','') normalized_name,
|
|
school_id
|
|
FROM schools
|
|
UNION ALL
|
|
SELECT REPLACE(REPLACE(REPLACE(TRIM(school_name),'（','('),'）',')'),' ','') normalized_name,
|
|
school_id
|
|
FROM school_names
|
|
) names
|
|
GROUP BY normalized_name HAVING COUNT(DISTINCT school_id)=1;
|
|
|
|
SET @before_groups=(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL);
|
|
UPDATE admission_groups target JOIN tmp_unique_school_names mapped
|
|
ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'（','('),'）',')'),' ','')
|
|
SET target.school_id=mapped.school_id WHERE target.school_id IS NULL;
|
|
SET @matched_groups=@before_groups-(SELECT COUNT(*) FROM admission_groups WHERE school_id IS NULL);
|
|
|
|
SET @before_major=(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL);
|
|
UPDATE admission_major_lines target JOIN tmp_unique_school_names mapped
|
|
ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'（','('),'）',')'),' ','')
|
|
SET target.school_id=mapped.school_id WHERE target.school_id IS NULL;
|
|
SET @matched_major=@before_major-(SELECT COUNT(*) FROM admission_major_lines WHERE school_id IS NULL);
|
|
|
|
SET @before_cutoff=(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL);
|
|
UPDATE admission_cutoff_lines target JOIN tmp_unique_school_names mapped
|
|
ON mapped.normalized_name=REPLACE(REPLACE(REPLACE(TRIM(target.school_name_raw),'（','('),'）',')'),' ','')
|
|
SET target.school_id=mapped.school_id WHERE target.school_id IS NULL;
|
|
SET @matched_cutoff=@before_cutoff-(SELECT COUNT(*) FROM admission_cutoff_lines WHERE school_id IS NULL);
|
|
|
|
UPDATE ingestion_runs SET finished_at=NOW(), status='success',
|
|
rows_inserted={len(records)}-@existing_count, rows_updated=@existing_count,
|
|
message=CONCAT('历史快照校验通过；新增历史学校', {len(records)}-@existing_count,
|
|
'所；补匹配专业组',@matched_groups,'条、专业线',@matched_major,'条、投档线',@matched_cutoff,'条。')
|
|
WHERE ingestion_run_id={sql_string(run_id)};
|
|
COMMIT;
|
|
""",
|
|
])
|
|
run_mysql(args.mysql_bin, args.host, args.port, args.user, "\n".join(statements))
|
|
print(f"教育部{args.year}年历史快照导入完成：{len(records)}所。")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|