You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
356 lines
12 KiB
356 lines
12 KiB
import argparse
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import uuid
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
|
|
EXPECTED_COLUMNS = [
|
|
"序号",
|
|
"学校名称",
|
|
"学校标识码",
|
|
"主管部门",
|
|
"所在地",
|
|
"办学层次",
|
|
"备注",
|
|
]
|
|
|
|
|
|
def sql_string(value):
|
|
if value is None:
|
|
return "NULL"
|
|
text = str(value)
|
|
text = (
|
|
text.replace("\\", "\\\\")
|
|
.replace("'", "\\'")
|
|
.replace("\x00", "")
|
|
.replace("\r", " ")
|
|
.replace("\n", " ")
|
|
)
|
|
return f"'{text}'"
|
|
|
|
|
|
def clean_cell(value):
|
|
if value is None or pd.isna(value):
|
|
return None
|
|
value = str(value).strip()
|
|
return value or None
|
|
|
|
|
|
def normalize_code(value):
|
|
value = clean_cell(value)
|
|
if value is None:
|
|
return None
|
|
value = re.sub(r"\.0$", "", value)
|
|
return value
|
|
|
|
|
|
def read_rows(path):
|
|
frame = pd.read_excel(path, sheet_name=0, header=2, dtype=str, engine="xlrd")
|
|
if list(frame.columns) != EXPECTED_COLUMNS:
|
|
raise ValueError(f"Unexpected columns: {list(frame.columns)!r}")
|
|
|
|
province = None
|
|
records = []
|
|
rejected = []
|
|
province_pattern = re.compile(r"^(.+?)[（(]\d+所[）)]$")
|
|
|
|
for dataframe_index, row in frame.iterrows():
|
|
source_row_number = dataframe_index + 4
|
|
sequence = clean_cell(row["序号"])
|
|
name = clean_cell(row["学校名称"])
|
|
code = normalize_code(row["学校标识码"])
|
|
|
|
province_match = province_pattern.match(sequence or "")
|
|
if province_match and not name and not code:
|
|
province = province_match.group(1)
|
|
continue
|
|
|
|
if not name and not code:
|
|
continue
|
|
|
|
record = {
|
|
"source_row_number": source_row_number,
|
|
"moe_school_code": code,
|
|
"official_name": name,
|
|
"province_name": province,
|
|
"city_name": clean_cell(row["所在地"]),
|
|
"authority_name": clean_cell(row["主管部门"]),
|
|
"education_level": clean_cell(row["办学层次"]),
|
|
"source_remark": clean_cell(row["备注"]),
|
|
}
|
|
|
|
errors = []
|
|
if not re.fullmatch(r"\d{10}", code or ""):
|
|
errors.append("invalid_moe_school_code")
|
|
for required in [
|
|
"official_name",
|
|
"province_name",
|
|
"city_name",
|
|
"authority_name",
|
|
"education_level",
|
|
]:
|
|
if not record[required]:
|
|
errors.append(f"missing_{required}")
|
|
if errors:
|
|
rejected.append((source_row_number, errors, record))
|
|
else:
|
|
records.append(record)
|
|
|
|
codes = [row["moe_school_code"] for row in records]
|
|
if len(codes) != len(set(codes)):
|
|
raise ValueError("Duplicate Ministry of Education school codes in source file")
|
|
return records, rejected
|
|
|
|
|
|
def values_sql(rows, run_id):
|
|
values = []
|
|
for row in rows:
|
|
values.append(
|
|
"(" + ",".join(
|
|
[
|
|
sql_string(run_id),
|
|
str(row["source_row_number"]),
|
|
sql_string(row["moe_school_code"]),
|
|
sql_string(row["official_name"]),
|
|
sql_string(row["province_name"]),
|
|
sql_string(row["city_name"]),
|
|
sql_string(row["authority_name"]),
|
|
sql_string(row["education_level"]),
|
|
sql_string(row["source_remark"]),
|
|
]
|
|
) + ")"
|
|
)
|
|
return ",\n".join(values)
|
|
|
|
|
|
def build_sql(args, records, rejected, digest, run_id):
|
|
raw_path = str(Path(args.source_file).resolve())
|
|
source_title = f"全国普通高等学校名单（截至{args.published_on}）"
|
|
statements = [
|
|
"SET NAMES utf8mb4;",
|
|
"SET time_zone = '+08:00';",
|
|
f"USE `{args.database}`;",
|
|
"START TRANSACTION;",
|
|
"""
|
|
INSERT INTO data_sources (
|
|
source_type, publisher, title, landing_page_url, file_url,
|
|
published_on, retrieved_at, sha256, raw_file_path, notes
|
|
) VALUES (
|
|
"""
|
|
+ ",".join(
|
|
[
|
|
sql_string("official_xls"),
|
|
sql_string("中华人民共和国教育部"),
|
|
sql_string(source_title),
|
|
sql_string(args.landing_page_url),
|
|
sql_string(args.file_url),
|
|
sql_string(args.published_on),
|
|
"NOW()",
|
|
sql_string(digest),
|
|
sql_string(raw_path),
|
|
sql_string("全国普通高校，不含成人高校及港澳台高校"),
|
|
]
|
|
)
|
|
+ ") AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at, raw_file_path=incoming.raw_file_path;",
|
|
f"SET @source_id = (SELECT source_id FROM data_sources WHERE sha256={sql_string(digest)});",
|
|
f"DELETE FROM stg_moe_schools WHERE ingestion_run_id={sql_string(run_id)};",
|
|
f"""
|
|
INSERT INTO ingestion_runs (
|
|
ingestion_run_id, source_id, dataset_name, dataset_year, started_at,
|
|
status, rows_read, rows_rejected
|
|
) VALUES (
|
|
{sql_string(run_id)}, @source_id, 'moe_regular_higher_education_institutions',
|
|
{args.year}, NOW(), 'running', {len(records) + len(rejected)}, {len(rejected)}
|
|
);
|
|
""",
|
|
]
|
|
|
|
batch_size = 400
|
|
for start in range(0, len(records), batch_size):
|
|
batch = records[start : start + batch_size]
|
|
statements.append(
|
|
"""
|
|
INSERT INTO stg_moe_schools (
|
|
ingestion_run_id, source_row_number, moe_school_code, official_name,
|
|
province_name, city_name, authority_name, education_level, source_remark
|
|
) VALUES
|
|
"""
|
|
+ values_sql(batch, run_id)
|
|
+ ";"
|
|
)
|
|
|
|
statements.extend(
|
|
[
|
|
f"""
|
|
SET @existing_count = (
|
|
SELECT COUNT(*)
|
|
FROM schools s
|
|
JOIN stg_moe_schools stg
|
|
ON stg.moe_school_code=s.moe_school_code
|
|
AND stg.ingestion_run_id={sql_string(run_id)}
|
|
);
|
|
""",
|
|
f"""
|
|
INSERT IGNORE INTO school_names (
|
|
school_id, school_name, name_type, valid_from_year,
|
|
valid_to_year, source_id
|
|
)
|
|
SELECT s.school_id, s.official_name, 'former', s.first_seen_year,
|
|
{args.year - 1}, s.source_id
|
|
FROM schools s
|
|
JOIN stg_moe_schools stg
|
|
ON stg.moe_school_code = s.moe_school_code
|
|
AND stg.ingestion_run_id = {sql_string(run_id)}
|
|
WHERE s.official_name <> stg.official_name;
|
|
""",
|
|
f"""
|
|
INSERT INTO schools (
|
|
school_id, moe_school_code, official_name, province_name, city_name,
|
|
authority_name, education_level, source_remark, ownership_type,
|
|
school_status, first_seen_year, last_seen_year, source_id
|
|
)
|
|
SELECT incoming.school_id, incoming.moe_school_code, incoming.official_name,
|
|
incoming.province_name, incoming.city_name, incoming.authority_name,
|
|
incoming.education_level, incoming.source_remark, incoming.ownership_type,
|
|
incoming.school_status, incoming.first_seen_year,
|
|
incoming.last_seen_year, incoming.source_id
|
|
FROM (
|
|
SELECT UUID() AS school_id, moe_school_code, official_name,
|
|
province_name, city_name, authority_name, education_level,
|
|
source_remark,
|
|
CASE
|
|
WHEN source_remark='民办' THEN '民办'
|
|
WHEN source_remark LIKE '%中外合作办学%' THEN '合作办学'
|
|
WHEN source_remark LIKE '%境外高等教育机构%' THEN '境外办学'
|
|
ELSE '公办'
|
|
END AS ownership_type,
|
|
'active' AS school_status,
|
|
{args.year} AS first_seen_year, {args.year} AS last_seen_year,
|
|
@source_id AS source_id
|
|
FROM stg_moe_schools
|
|
WHERE ingestion_run_id = {sql_string(run_id)}
|
|
) AS incoming
|
|
ON DUPLICATE KEY UPDATE
|
|
official_name=incoming.official_name,
|
|
province_name=incoming.province_name,
|
|
city_name=incoming.city_name,
|
|
authority_name=incoming.authority_name,
|
|
education_level=incoming.education_level,
|
|
source_remark=incoming.source_remark,
|
|
ownership_type=incoming.ownership_type,
|
|
school_status='active',
|
|
last_seen_year=incoming.last_seen_year,
|
|
source_id=incoming.source_id;
|
|
""",
|
|
f"""
|
|
INSERT IGNORE INTO school_names (
|
|
school_id, school_name, name_type, valid_from_year, source_id
|
|
)
|
|
SELECT s.school_id, s.official_name, 'official', {args.year}, @source_id
|
|
FROM schools s
|
|
JOIN stg_moe_schools stg
|
|
ON stg.moe_school_code = s.moe_school_code
|
|
AND stg.ingestion_run_id = {sql_string(run_id)};
|
|
""",
|
|
f"""
|
|
UPDATE schools s
|
|
SET s.school_status='inactive'
|
|
WHERE s.last_seen_year < {args.year}
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM stg_moe_schools stg
|
|
WHERE stg.ingestion_run_id={sql_string(run_id)}
|
|
AND stg.moe_school_code=s.moe_school_code
|
|
);
|
|
""",
|
|
f"""
|
|
SET @loaded_count = (
|
|
SELECT COUNT(*) FROM stg_moe_schools
|
|
WHERE ingestion_run_id={sql_string(run_id)}
|
|
);
|
|
UPDATE ingestion_runs
|
|
SET finished_at=NOW(), status='success',
|
|
rows_inserted=@loaded_count-@existing_count,
|
|
rows_updated=@existing_count,
|
|
message={sql_string(f'Validated {len(records)} records; rejected {len(rejected)} records')}
|
|
WHERE ingestion_run_id={sql_string(run_id)};
|
|
""",
|
|
"COMMIT;",
|
|
]
|
|
)
|
|
return "\n".join(statements)
|
|
|
|
|
|
def run_mysql(mysql_bin, host, port, user, sql):
|
|
if not os.environ.get("MYSQL_PWD"):
|
|
raise RuntimeError("MYSQL_PWD environment variable is required")
|
|
process = subprocess.run(
|
|
[
|
|
mysql_bin,
|
|
"--protocol=TCP",
|
|
"-h",
|
|
host,
|
|
"-P",
|
|
str(port),
|
|
"-u",
|
|
user,
|
|
"--default-character-set=utf8mb4",
|
|
],
|
|
input=sql,
|
|
text=True,
|
|
encoding="utf-8",
|
|
capture_output=True,
|
|
)
|
|
if process.returncode:
|
|
raise RuntimeError(process.stderr.strip() or process.stdout.strip())
|
|
return process.stdout.strip(), process.stderr.strip()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--source-file", required=True)
|
|
parser.add_argument("--year", type=int, required=True)
|
|
parser.add_argument("--published-on", type=date.fromisoformat, required=True)
|
|
parser.add_argument("--expected-count", type=int, required=True)
|
|
parser.add_argument("--landing-page-url", required=True)
|
|
parser.add_argument("--file-url", required=True)
|
|
parser.add_argument("--mysql-bin", required=True)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--port", type=int, default=3306)
|
|
parser.add_argument("--user", default="root")
|
|
parser.add_argument("--database", default="zhiyuantong")
|
|
args = parser.parse_args()
|
|
|
|
source_path = Path(args.source_file)
|
|
digest = hashlib.sha256(source_path.read_bytes()).hexdigest()
|
|
records, rejected = read_rows(source_path)
|
|
if rejected:
|
|
first = rejected[0]
|
|
raise ValueError(f"Rejected source rows: {len(rejected)}; first={first!r}")
|
|
if len(records) != args.expected_count:
|
|
raise ValueError(
|
|
f"Expected {args.expected_count} schools, parsed {len(records)}"
|
|
)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
sql = build_sql(args, records, rejected, digest, run_id)
|
|
stdout, stderr = run_mysql(
|
|
args.mysql_bin, args.host, args.port, args.user, sql
|
|
)
|
|
print(
|
|
f"run_id={run_id} parsed={len(records)} rejected={len(rejected)} sha256={digest}"
|
|
)
|
|
if stdout:
|
|
print(stdout)
|
|
if stderr:
|
|
print(stderr, file=sys.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|