import argparse import hashlib import os import re import subprocess import urllib.request import uuid from collections import defaultdict from io import StringIO from pathlib import Path import pandas as pd YEAR = 2025 USER_AGENT = "Mozilla/5.0 (compatible; ZhiyuantongDataAudit/1.0)" CONTROL_SOURCE_URL = "https://www.gk100.com/read_17157488.htm" SCORE_INDEX_URL = "https://www.eol.cn/e_html/gk/gkfsd/2025.shtml" PROVINCES = { "北京": ("北京市", "beijing"), "天津": ("天津市", "tianjin"), "河北": ("河北省", "hebei"), "山西": ("山西省", "shanxi"), "内蒙古": ("内蒙古自治区", "inner_mongolia"), "辽宁": ("辽宁省", "liaoning"), "吉林": ("吉林省", "jilin"), "黑龙江": ("黑龙江省", "heilongjiang"), "上海": ("上海市", "shanghai"), "江苏": ("江苏省", "jiangsu"), "浙江": ("浙江省", "zhejiang"), "安徽": ("安徽省", "anhui"), "福建": ("福建省", "fujian"), "江西": ("江西省", "jiangxi"), "山东": ("山东省", "shandong"), "河南": ("河南省", "henan"), "湖北": ("湖北省", "hubei"), "湖南": ("湖南省", "hunan"), "广东": ("广东省", "guangdong"), "广西": ("广西壮族自治区", "guangxi"), "海南": ("海南省", "hainan"), "重庆": ("重庆市", "chongqing"), "四川": ("四川省", "sichuan"), "贵州": ("贵州省", "guizhou"), "云南": ("云南省", "yunnan"), "西藏": ("西藏自治区", "tibet"), "陕西": ("陕西省", "shaanxi"), "甘肃": ("甘肃省", "gansu"), "青海": ("青海省", "qinghai"), "宁夏": ("宁夏回族自治区", "ningxia"), "新疆": ("新疆维吾尔自治区", "xinjiang"), } # 这些转载页均在全国一分一段导航中列出，页面正文注明数据来自对应省级考试机构。 # 这里只纳入能逐行解析并校验“本分人数 + 上一行累计人数 = 本行累计人数”的表格。 SCORE_SOURCES = [ ("北京市", "beijing", "综合", "https://gaokao.eol.cn/bei_jing/dongtai/202506/t20250625_2676934.shtml"), ("重庆市", "chongqing", "历史类", "https://gaokao.eol.cn/chong_qing/dongtai/202506/t20250624_2676749.shtml"), ("重庆市", "chongqing", "物理类", "https://gaokao.eol.cn/chong_qing/dongtai/202506/t20250624_2676788.shtml"), ("山东省", "shandong", "综合", "https://gaokao.eol.cn/shan_dong/dongtai/202506/t20250625_2677092.shtml"), ("四川省", "sichuan", "物理类", "https://gaokao.eol.cn/si_chuan/dongtai/202507/t20250702_2678480.shtml"), ("四川省", "sichuan", "历史类", "https://gaokao.eol.cn/si_chuan/dongtai/202507/t20250702_2678481.shtml"), ("江苏省", "jiangsu", "物理类", "https://gaokao.eol.cn/jiang_su/dongtai/202506/t20250625_2676969.shtml"), ("江苏省", "jiangsu", "历史类", "https://gaokao.eol.cn/jiang_su/dongtai/202506/t20250625_2676968.shtml"), ("安徽省", "anhui", "物理类", "https://gaokao.eol.cn/an_hui/dongtai/202506/t20250625_2676962.shtml"), ("安徽省", "anhui", "历史类", "https://gaokao.eol.cn/an_hui/dongtai/202506/t20250625_2676963.shtml"), ("福建省", "fujian", "物理类", "https://gaokao.eol.cn/fu_jian/dongtai/202507/t20250702_2678456.shtml"), ("福建省", "fujian", "历史类", "https://gaokao.eol.cn/fu_jian/dongtai/202507/t20250702_2678457.shtml"), ("河南省", "henan", "历史类", "https://gaokao.eol.cn/he_nan/dongtai/202506/t20250625_2676858.shtml"), ("河南省", "henan", "物理类", "https://gaokao.eol.cn/he_nan/dongtai/202506/t20250625_2676859.shtml"), ("湖南省", "hunan", "历史类", "https://gaokao.eol.cn/hu_nan/dongtai/202506/t20250625_2676955.shtml"), ("湖南省", "hunan", "物理类", "https://gaokao.eol.cn/hu_nan/dongtai/202506/t20250625_2676956.shtml"), ("广西壮族自治区", "guangxi", "物理类", "https://gaokao.eol.cn/guang_xi/dongtai/202506/t20250625_2677014.shtml"), ("广西壮族自治区", "guangxi", "历史类", "https://gaokao.eol.cn/guang_xi/dongtai/202506/t20250625_2677013.shtml"), ("山西省", "shanxi", "物理类", "https://gaokao.eol.cn/shan_xi/dongtai/202506/t20250626_2677293.shtml"), ("陕西省", "shaanxi", "物理类", "https://gaokao.eol.cn/shan_xi_sheng/dongtai/202506/t20250625_2677115.shtml"), ("陕西省", "shaanxi", "历史类", "https://gaokao.eol.cn/shan_xi_sheng/dongtai/202506/t20250625_2677119.shtml"), ("内蒙古自治区", "inner_mongolia", "历史类", "https://gaokao.eol.cn/nei_meng/dongtai/202506/t20250625_2676868.shtml"), ("内蒙古自治区", "inner_mongolia", "物理类", "https://gaokao.eol.cn/nei_meng/dongtai/202506/t20250625_2676869.shtml"), ("黑龙江省", "heilongjiang", "物理类", "https://gaokao.eol.cn/hei_long_jiang/dongtai/202506/t20250624_2676681.shtml"), ("黑龙江省", "heilongjiang", "历史类", "https://gaokao.eol.cn/hei_long_jiang/dongtai/202506/t20250624_2676679.shtml"), ("吉林省", "jilin", "历史类", "https://gaokao.eol.cn/ji_lin/dongtai/202506/t20250625_2677007.shtml"), ("吉林省", "jilin", "物理类", "https://gaokao.eol.cn/ji_lin/dongtai/202506/t20250625_2677006.shtml"), ("辽宁省", "liaoning", "物理类", "https://gaokao.eol.cn/liao_ning/dongtai/202506/t20250624_2676792.shtml"), ("辽宁省", "liaoning", "历史类", "https://gaokao.eol.cn/liao_ning/dongtai/202506/t20250624_2676782.shtml"), ("宁夏回族自治区", "ningxia", "物理类", "https://gaokao.eol.cn/ning_xia/dongtai/202506/t20250625_2677009.shtml"), ("宁夏回族自治区", "ningxia", "历史类", "https://gaokao.eol.cn/ning_xia/dongtai/202506/t20250625_2677008.shtml"), ("青海省", "qinghai", "历史类", "https://gaokao.eol.cn/qing_hai/dongtai/202506/t20250625_2677002.shtml"), ("青海省", "qinghai", "物理类", "https://gaokao.eol.cn/qing_hai/dongtai/202506/t20250625_2677001.shtml"), ("海南省", "hainan", "综合", "https://gaokao.eol.cn/hai_nan/dongtai/202507/t20250702_2678468.shtml"), ] def sql_string(value): if value is None or value == "": return "NULL" text = str(value).replace("\\", "\\\\").replace("'", "\\'") text = text.replace("\x00", "").replace("\r", " ").replace("\n", " ") return f"'{text}'" def fetch(url, path, refresh=False): path.parent.mkdir(parents=True, exist_ok=True) if path.exists() and not refresh: return path.read_bytes() request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) with urllib.request.urlopen(request, timeout=45) as response: content = response.read() path.write_bytes(content) return content def digest(path): return hashlib.sha256(path.read_bytes()).hexdigest() def source_upsert(path, title, landing_url, publisher, notes): sha256 = digest(path) return f""" INSERT INTO data_sources ( source_type, publisher, title, landing_page_url, retrieved_at, sha256, raw_file_path, notes ) VALUES ('official_republication_html', {sql_string(publisher)}, {sql_string(title)}, {sql_string(landing_url)}, NOW(), {sql_string(sha256)}, {sql_string(str(path.resolve()))}, {sql_string(notes)}) AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at, raw_file_path=incoming.raw_file_path, notes=incoming.notes; SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(sha256)}); """ def ingestion_start(dataset_name, rows_read): run_id = str(uuid.uuid4()) sql = f""" INSERT INTO ingestion_runs ( ingestion_run_id, source_id, dataset_name, dataset_year, started_at, status, rows_read, rows_rejected ) VALUES ({sql_string(run_id)}, @source_id, {sql_string(dataset_name)}, {YEAR}, NOW(), 'running', {rows_read}, 0); """ return run_id, sql def ingestion_finish(run_id, rows_inserted, message): return f""" UPDATE ingestion_runs SET finished_at=NOW(), status='success', rows_inserted={rows_inserted}, message={sql_string(message)} WHERE ingestion_run_id={sql_string(run_id)}; """ def coverage_sql(province, name, status, count, notes, subject="", category="普通类"): return f""" INSERT INTO dataset_coverage ( dataset_name, province_name, admission_year, admission_category, subject_category, segment_code, coverage_status, record_count, notes, source_id ) VALUES ({sql_string(name)}, {sql_string(province)}, {YEAR}, {sql_string(category) if category else "''"}, {sql_string(subject) if subject else "''"}, '', {sql_string(status)}, {count}, {sql_string(notes)}, @source_id) AS incoming ON DUPLICATE KEY UPDATE coverage_status=incoming.coverage_status, record_count=incoming.record_count, notes=incoming.notes, source_id=incoming.source_id; """ def normalize_subject(province, raw): value = re.sub(r"\s+", "", str(raw)) if "综合" in value: return "综合" old_style = province in {"西藏自治区", "新疆维吾尔自治区"} if "历史" in value or "文科" in value: return "文史类" if old_style else "历史类" if "物理" in value or "理科" in value: return "理工类" if old_style else "物理类" raise ValueError(f"无法识别控制线科类: {province} {raw}") def normalize_level(raw): value = re.sub(r"\s+", "", str(raw)).replace("(", "（").replace(")", "）") return { "专科线": "高职专科", "专科批": "高职专科", "本科线": "本科", "特控线": "特殊类型", "一段线": "一段", "二段线": "二段", }.get(value, value) def parse_control_lines(html_text): table = pd.read_html(StringIO(html_text))[0] records = [] for _, row in table.iterrows(): short_name = re.sub(r"\s+", "", str(row.iloc[0])) if short_name not in PROVINCES: continue province = PROVINCES[short_name][0] score_match = re.search(r"\d+(?:\.\d+)?", str(row.iloc[3])) if not score_match: raise ValueError(f"控制线分数无法识别: {row.tolist()}") score = score_match.group(0) records.append(( province, normalize_subject(province, row.iloc[2]), normalize_level(row.iloc[1]), score, re.sub(r"\s+", " ", str(row.iloc[1])).strip(), )) provinces = {row[0] for row in records} if provinces != {value[0] for value in PROVINCES.values()}: raise ValueError(f"控制线省份不完整: {sorted(provinces)}") return records def parse_score_rows(html_text, url): candidates = [] for table in pd.read_html(StringIO(html_text)): if table.shape[1] not in {3, 7}: continue rows = [] triples = [(0, 1, 2)] if table.shape[1] == 3 else [(0, 1, 2), (4, 5, 6)] for score_column, same_column, cumulative_column in triples: for _, row in table.iterrows(): # 控制线可能直接注记在分数后；但“101分以下及缺考”之类汇总行 # 无法映射到单一分数，必须排除并在覆盖说明中保留公开范围。 score_label = str(row.iloc[score_column]) if "以下" in score_label or "缺考" in score_label: continue match = re.match(r"^\s*(\d+)(?:\s*[-—]\s*\d+)?", score_label) if not match: continue try: score = int(match.group(1)) same = int(float(row.iloc[same_column])) cumulative = int(float(row.iloc[cumulative_column])) except (TypeError, ValueError): continue rows.append((score, same, cumulative)) if len(rows) >= 100: candidates.append(rows) if len(candidates) != 1: raise ValueError(f"一分一段表数量异常: {url} 候选表{len(candidates)}个") rows = candidates[0] if len({row[0] for row in rows}) != len(rows): raise ValueError(f"一分一段存在重复分数: {url}") if any(rows[index][2] != rows[index - 1][2] + rows[index][1] for index in range(1, len(rows))): raise ValueError(f"一分一段累计人数校验失败: {url}") return [( score, same, cumulative, max(1, cumulative - same + 1), cumulative, index + 1, ) for index, (score, same, cumulative) in enumerate(rows)] def chunks(values, size=300): for index in range(0, len(values), size): yield values[index:index + size] def mysql_run(args, sql): if not os.environ.get("MYSQL_PWD"): raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码") process = subprocess.run([ args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port), "-u", args.user, "--default-character-set=utf8mb4", ], input=sql, text=True, encoding="utf-8", capture_output=True) if process.returncode: raise RuntimeError(process.stderr.strip() or process.stdout.strip()) def main(): parser = argparse.ArgumentParser(description="导入全国31省级地区2025年公开招录数据首批快照") parser.add_argument("--mysql-bin", required=True) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=3306) parser.add_argument("--user", default="root") parser.add_argument("--database", default="zhiyuantong") parser.add_argument("--root", default=str(Path(__file__).resolve().parents[1])) parser.add_argument("--refresh", action="store_true", help="重新下载已经归档的原始网页") args = parser.parse_args() root = Path(args.root) national_raw = root / "data/raw/national/2025" control_path = national_raw / "control_lines_31_provinces_2025.html" score_index_path = national_raw / "score_rank_index_2025.html" control_html = fetch(CONTROL_SOURCE_URL, control_path, args.refresh).decode("utf-8") fetch(SCORE_INDEX_URL, score_index_path, args.refresh) controls = parse_control_lines(control_html) statements = ["SET NAMES utf8mb4;", "SET time_zone='+08:00';", f"USE `{args.database}`;", "START TRANSACTION;"] statements.append(source_upsert( control_path, "2025年全国31省级地区普通高考录取控制线汇总", CONTROL_SOURCE_URL, "高考100（汇总各省级考试机构公开数据）", "普通类控制线与阳光高考2025批次线汇总交叉核对；保留原始网页及哈希。", )) run_id, start_sql = ingestion_start("national_2025_control_score_lines", len(controls)) statements.append(start_sql) controls_by_province = defaultdict(list) for record in controls: controls_by_province[record[0]].append(record) for province, records in controls_by_province.items(): if province == "甘肃省": continue statements.append(f"DELETE FROM control_score_lines WHERE province_name={sql_string(province)} AND admission_year={YEAR} AND admission_category='普通类';") values = [] for _, subject, level, score, raw_level in records: values.append("(" + ",".join([ sql_string(province), str(YEAR), sql_string("普通类"), sql_string(subject), sql_string(level), score, "NULL", sql_string(f"全国批次线汇总原口径：{raw_level}"), "@source_id", ]) + ")") statements.append(""" INSERT INTO control_score_lines ( province_name, admission_year, admission_category, subject_category, education_level, culture_score, professional_score, notes, source_id ) VALUES """ + ",\n".join(values) + ";") statements.append(coverage_sql( province, "普通类控制线", "complete_public_snapshot", len(records), "2025年普通类主要批次控制线已按全国汇总页逐行导入，并与阳光高考批次线汇总交叉核对。", )) statements.append(ingestion_finish(run_id, len(controls), "全国31省级地区2025年普通类控制线完成解析；甘肃保留更完整的普通及艺体数据")) score_counts = defaultdict(int) imported_score_provinces = set() for province, slug, subject, url in SCORE_SOURCES: path = root / "data/raw" / slug / "2025" / f"score_rank_{subject}_2025.html" html_text = fetch(url, path, args.refresh).decode("utf-8") rows = parse_score_rows(html_text, url) imported_score_provinces.add(province) score_counts[province] += len(rows) statements.append(source_upsert( path, f"{province}2025年普通高考一分一段表（{subject}）", url, "中国教育在线（转载省级考试机构公开数据）", "从全国2025一分一段导航定位；仅导入可逐行解析并通过累计人数校验的公开表格。", )) score_run_id, score_start = ingestion_start(f"{slug}_2025_score_rank_{subject}", len(rows)) statements.extend([ score_start, f"DELETE FROM score_rank_points WHERE province_name={sql_string(province)} AND admission_year={YEAR} AND admission_category='普通类' AND subject_category={sql_string(subject)};", f"DELETE FROM dataset_coverage WHERE province_name={sql_string(province)} AND admission_year={YEAR} AND dataset_name='一分一段' AND subject_category='';", ]) values = ["(" + ",".join([ sql_string(province), str(YEAR), sql_string("普通类"), sql_string(subject), str(score), str(same), str(cumulative), str(rank_from), str(rank_to), str(sequence), "@source_id", ]) + ")" for score, same, cumulative, rank_from, rank_to, sequence in rows] for batch in chunks(values): statements.append(""" INSERT INTO score_rank_points ( province_name, admission_year, admission_category, subject_category, score, same_score_count, cumulative_count, rank_from, rank_to, source_sequence, source_id ) VALUES """ + ",\n".join(batch) + ";") first, last = rows[0], rows[-1] top_note = "；最高分段存在隐私合并，精确位次按页面累计人数保存" if first[2] != first[1] else "" statements.append(coverage_sql( province, "一分一段", "complete_public_snapshot", len(rows), f"完整导入该公开表的{len(rows)}个分数点，范围{first[0]}至{last[0]}分，末行累计{last[2]}人；累计人数逐行校验通过{top_note}。", subject, )) statements.append(ingestion_finish(score_run_id, len(rows), f"{province}{subject}一分一段公开表导入并校验通过")) all_provinces = {value[0] for value in PROVINCES.values()} gansu = "甘肃省" gap_note = "本轮已建立2025年全国省级数据域并完成控制线/可校验一分一段；该分类仍需按本省考试院文件继续结构化，不能据此判定官方未公开。" for province in sorted(all_provinces - {gansu}): if province not in imported_score_provinces: statements.append(coverage_sql( province, "一分一段", "partial_public_snapshot", 0, "已定位2025年公开来源，但当前来源为图片、附件或非标准表格，尚未通过逐行结构化校验。", category="普通类", )) for dataset_name in ["批次与志愿规则", "初始招生计划", "院校专业组", "专业招生计划", "院校专业组投档线"]: statements.append(coverage_sql( province, dataset_name, "partial_public_snapshot", 0, gap_note, category="", )) control_count = len(controls_by_province[province]) score_count = score_counts[province] note = f"已导入2025年普通类控制线{control_count}条" if score_count: note += f"、一分一段{score_count}条" else: note += "；一分一段公开源尚待结构化复核" note += "。招生规则、计划、专业组和投档线继续按省级官方来源补充。" statements.append(f""" UPDATE collection_scopes SET collection_status='partial', priority_level='high', status_note={sql_string(note)}, first_collected_at=COALESCE(first_collected_at, NOW()), last_collected_at=NOW() WHERE province_name={sql_string(province)} AND admission_year={YEAR}; """) statements.append("COMMIT;") mysql_run(args, "\n".join(statements)) print( f"导入完成: 全国控制线 {len(controls)} 条（甘肃保留原34条），" f"新增一分一段 {sum(score_counts.values())} 条，覆盖 {len(imported_score_provinces)} 个省级地区；" f"31省级地区2025范围均已建立覆盖缺口。" ) if __name__ == "__main__": main()