You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
409 lines
21 KiB
409 lines
21 KiB
import argparse
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import urllib.request
|
|
import uuid
|
|
from collections import defaultdict
|
|
from io import StringIO
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
|
|
YEAR = 2025
|
|
USER_AGENT = "Mozilla/5.0 (compatible; ZhiyuantongDataAudit/1.0)"
|
|
CONTROL_SOURCE_URL = "https://www.gk100.com/read_17157488.htm"
|
|
SCORE_INDEX_URL = "https://www.eol.cn/e_html/gk/gkfsd/2025.shtml"
|
|
|
|
PROVINCES = {
|
|
"北京": ("北京市", "beijing"),
|
|
"天津": ("天津市", "tianjin"),
|
|
"河北": ("河北省", "hebei"),
|
|
"山西": ("山西省", "shanxi"),
|
|
"内蒙古": ("内蒙古自治区", "inner_mongolia"),
|
|
"辽宁": ("辽宁省", "liaoning"),
|
|
"吉林": ("吉林省", "jilin"),
|
|
"黑龙江": ("黑龙江省", "heilongjiang"),
|
|
"上海": ("上海市", "shanghai"),
|
|
"江苏": ("江苏省", "jiangsu"),
|
|
"浙江": ("浙江省", "zhejiang"),
|
|
"安徽": ("安徽省", "anhui"),
|
|
"福建": ("福建省", "fujian"),
|
|
"江西": ("江西省", "jiangxi"),
|
|
"山东": ("山东省", "shandong"),
|
|
"河南": ("河南省", "henan"),
|
|
"湖北": ("湖北省", "hubei"),
|
|
"湖南": ("湖南省", "hunan"),
|
|
"广东": ("广东省", "guangdong"),
|
|
"广西": ("广西壮族自治区", "guangxi"),
|
|
"海南": ("海南省", "hainan"),
|
|
"重庆": ("重庆市", "chongqing"),
|
|
"四川": ("四川省", "sichuan"),
|
|
"贵州": ("贵州省", "guizhou"),
|
|
"云南": ("云南省", "yunnan"),
|
|
"西藏": ("西藏自治区", "tibet"),
|
|
"陕西": ("陕西省", "shaanxi"),
|
|
"甘肃": ("甘肃省", "gansu"),
|
|
"青海": ("青海省", "qinghai"),
|
|
"宁夏": ("宁夏回族自治区", "ningxia"),
|
|
"新疆": ("新疆维吾尔自治区", "xinjiang"),
|
|
}
|
|
|
|
# 这些转载页均在全国一分一段导航中列出，页面正文注明数据来自对应省级考试机构。
|
|
# 这里只纳入能逐行解析并校验“本分人数 + 上一行累计人数 = 本行累计人数”的表格。
|
|
SCORE_SOURCES = [
|
|
("北京市", "beijing", "综合", "https://gaokao.eol.cn/bei_jing/dongtai/202506/t20250625_2676934.shtml"),
|
|
("重庆市", "chongqing", "历史类", "https://gaokao.eol.cn/chong_qing/dongtai/202506/t20250624_2676749.shtml"),
|
|
("重庆市", "chongqing", "物理类", "https://gaokao.eol.cn/chong_qing/dongtai/202506/t20250624_2676788.shtml"),
|
|
("山东省", "shandong", "综合", "https://gaokao.eol.cn/shan_dong/dongtai/202506/t20250625_2677092.shtml"),
|
|
("四川省", "sichuan", "物理类", "https://gaokao.eol.cn/si_chuan/dongtai/202507/t20250702_2678480.shtml"),
|
|
("四川省", "sichuan", "历史类", "https://gaokao.eol.cn/si_chuan/dongtai/202507/t20250702_2678481.shtml"),
|
|
("江苏省", "jiangsu", "物理类", "https://gaokao.eol.cn/jiang_su/dongtai/202506/t20250625_2676969.shtml"),
|
|
("江苏省", "jiangsu", "历史类", "https://gaokao.eol.cn/jiang_su/dongtai/202506/t20250625_2676968.shtml"),
|
|
("安徽省", "anhui", "物理类", "https://gaokao.eol.cn/an_hui/dongtai/202506/t20250625_2676962.shtml"),
|
|
("安徽省", "anhui", "历史类", "https://gaokao.eol.cn/an_hui/dongtai/202506/t20250625_2676963.shtml"),
|
|
("福建省", "fujian", "物理类", "https://gaokao.eol.cn/fu_jian/dongtai/202507/t20250702_2678456.shtml"),
|
|
("福建省", "fujian", "历史类", "https://gaokao.eol.cn/fu_jian/dongtai/202507/t20250702_2678457.shtml"),
|
|
("河南省", "henan", "历史类", "https://gaokao.eol.cn/he_nan/dongtai/202506/t20250625_2676858.shtml"),
|
|
("河南省", "henan", "物理类", "https://gaokao.eol.cn/he_nan/dongtai/202506/t20250625_2676859.shtml"),
|
|
("湖南省", "hunan", "历史类", "https://gaokao.eol.cn/hu_nan/dongtai/202506/t20250625_2676955.shtml"),
|
|
("湖南省", "hunan", "物理类", "https://gaokao.eol.cn/hu_nan/dongtai/202506/t20250625_2676956.shtml"),
|
|
("广西壮族自治区", "guangxi", "物理类", "https://gaokao.eol.cn/guang_xi/dongtai/202506/t20250625_2677014.shtml"),
|
|
("广西壮族自治区", "guangxi", "历史类", "https://gaokao.eol.cn/guang_xi/dongtai/202506/t20250625_2677013.shtml"),
|
|
("山西省", "shanxi", "物理类", "https://gaokao.eol.cn/shan_xi/dongtai/202506/t20250626_2677293.shtml"),
|
|
("陕西省", "shaanxi", "物理类", "https://gaokao.eol.cn/shan_xi_sheng/dongtai/202506/t20250625_2677115.shtml"),
|
|
("陕西省", "shaanxi", "历史类", "https://gaokao.eol.cn/shan_xi_sheng/dongtai/202506/t20250625_2677119.shtml"),
|
|
("内蒙古自治区", "inner_mongolia", "历史类", "https://gaokao.eol.cn/nei_meng/dongtai/202506/t20250625_2676868.shtml"),
|
|
("内蒙古自治区", "inner_mongolia", "物理类", "https://gaokao.eol.cn/nei_meng/dongtai/202506/t20250625_2676869.shtml"),
|
|
("黑龙江省", "heilongjiang", "物理类", "https://gaokao.eol.cn/hei_long_jiang/dongtai/202506/t20250624_2676681.shtml"),
|
|
("黑龙江省", "heilongjiang", "历史类", "https://gaokao.eol.cn/hei_long_jiang/dongtai/202506/t20250624_2676679.shtml"),
|
|
("吉林省", "jilin", "历史类", "https://gaokao.eol.cn/ji_lin/dongtai/202506/t20250625_2677007.shtml"),
|
|
("吉林省", "jilin", "物理类", "https://gaokao.eol.cn/ji_lin/dongtai/202506/t20250625_2677006.shtml"),
|
|
("辽宁省", "liaoning", "物理类", "https://gaokao.eol.cn/liao_ning/dongtai/202506/t20250624_2676792.shtml"),
|
|
("辽宁省", "liaoning", "历史类", "https://gaokao.eol.cn/liao_ning/dongtai/202506/t20250624_2676782.shtml"),
|
|
("宁夏回族自治区", "ningxia", "物理类", "https://gaokao.eol.cn/ning_xia/dongtai/202506/t20250625_2677009.shtml"),
|
|
("宁夏回族自治区", "ningxia", "历史类", "https://gaokao.eol.cn/ning_xia/dongtai/202506/t20250625_2677008.shtml"),
|
|
("青海省", "qinghai", "历史类", "https://gaokao.eol.cn/qing_hai/dongtai/202506/t20250625_2677002.shtml"),
|
|
("青海省", "qinghai", "物理类", "https://gaokao.eol.cn/qing_hai/dongtai/202506/t20250625_2677001.shtml"),
|
|
("海南省", "hainan", "综合", "https://gaokao.eol.cn/hai_nan/dongtai/202507/t20250702_2678468.shtml"),
|
|
]
|
|
|
|
|
|
def sql_string(value):
|
|
if value is None or value == "":
|
|
return "NULL"
|
|
text = str(value).replace("\\", "\\\\").replace("'", "\\'")
|
|
text = text.replace("\x00", "").replace("\r", " ").replace("\n", " ")
|
|
return f"'{text}'"
|
|
|
|
|
|
def fetch(url, path, refresh=False):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
if path.exists() and not refresh:
|
|
return path.read_bytes()
|
|
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
|
with urllib.request.urlopen(request, timeout=45) as response:
|
|
content = response.read()
|
|
path.write_bytes(content)
|
|
return content
|
|
|
|
|
|
def digest(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def source_upsert(path, title, landing_url, publisher, notes):
|
|
sha256 = digest(path)
|
|
return f"""
|
|
INSERT INTO data_sources (
|
|
source_type, publisher, title, landing_page_url, retrieved_at,
|
|
sha256, raw_file_path, notes
|
|
) VALUES ('official_republication_html', {sql_string(publisher)},
|
|
{sql_string(title)}, {sql_string(landing_url)}, NOW(), {sql_string(sha256)},
|
|
{sql_string(str(path.resolve()))}, {sql_string(notes)})
|
|
AS incoming ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at,
|
|
raw_file_path=incoming.raw_file_path, notes=incoming.notes;
|
|
SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(sha256)});
|
|
"""
|
|
|
|
|
|
def ingestion_start(dataset_name, rows_read):
|
|
run_id = str(uuid.uuid4())
|
|
sql = f"""
|
|
INSERT INTO ingestion_runs (
|
|
ingestion_run_id, source_id, dataset_name, dataset_year, started_at,
|
|
status, rows_read, rows_rejected
|
|
) VALUES ({sql_string(run_id)}, @source_id, {sql_string(dataset_name)}, {YEAR},
|
|
NOW(), 'running', {rows_read}, 0);
|
|
"""
|
|
return run_id, sql
|
|
|
|
|
|
def ingestion_finish(run_id, rows_inserted, message):
|
|
return f"""
|
|
UPDATE ingestion_runs SET finished_at=NOW(), status='success',
|
|
rows_inserted={rows_inserted}, message={sql_string(message)}
|
|
WHERE ingestion_run_id={sql_string(run_id)};
|
|
"""
|
|
|
|
|
|
def coverage_sql(province, name, status, count, notes, subject="", category="普通类"):
|
|
return f"""
|
|
INSERT INTO dataset_coverage (
|
|
dataset_name, province_name, admission_year, admission_category,
|
|
subject_category, segment_code, coverage_status, record_count, notes, source_id
|
|
) VALUES ({sql_string(name)}, {sql_string(province)}, {YEAR},
|
|
{sql_string(category) if category else "''"}, {sql_string(subject) if subject else "''"},
|
|
'', {sql_string(status)}, {count}, {sql_string(notes)}, @source_id)
|
|
AS incoming ON DUPLICATE KEY UPDATE coverage_status=incoming.coverage_status,
|
|
record_count=incoming.record_count, notes=incoming.notes, source_id=incoming.source_id;
|
|
"""
|
|
|
|
|
|
def normalize_subject(province, raw):
|
|
value = re.sub(r"\s+", "", str(raw))
|
|
if "综合" in value:
|
|
return "综合"
|
|
old_style = province in {"西藏自治区", "新疆维吾尔自治区"}
|
|
if "历史" in value or "文科" in value:
|
|
return "文史类" if old_style else "历史类"
|
|
if "物理" in value or "理科" in value:
|
|
return "理工类" if old_style else "物理类"
|
|
raise ValueError(f"无法识别控制线科类: {province} {raw}")
|
|
|
|
|
|
def normalize_level(raw):
|
|
value = re.sub(r"\s+", "", str(raw)).replace("(", "（").replace(")", "）")
|
|
return {
|
|
"专科线": "高职专科",
|
|
"专科批": "高职专科",
|
|
"本科线": "本科",
|
|
"特控线": "特殊类型",
|
|
"一段线": "一段",
|
|
"二段线": "二段",
|
|
}.get(value, value)
|
|
|
|
|
|
def parse_control_lines(html_text):
|
|
table = pd.read_html(StringIO(html_text))[0]
|
|
records = []
|
|
for _, row in table.iterrows():
|
|
short_name = re.sub(r"\s+", "", str(row.iloc[0]))
|
|
if short_name not in PROVINCES:
|
|
continue
|
|
province = PROVINCES[short_name][0]
|
|
score_match = re.search(r"\d+(?:\.\d+)?", str(row.iloc[3]))
|
|
if not score_match:
|
|
raise ValueError(f"控制线分数无法识别: {row.tolist()}")
|
|
score = score_match.group(0)
|
|
records.append((
|
|
province,
|
|
normalize_subject(province, row.iloc[2]),
|
|
normalize_level(row.iloc[1]),
|
|
score,
|
|
re.sub(r"\s+", " ", str(row.iloc[1])).strip(),
|
|
))
|
|
provinces = {row[0] for row in records}
|
|
if provinces != {value[0] for value in PROVINCES.values()}:
|
|
raise ValueError(f"控制线省份不完整: {sorted(provinces)}")
|
|
return records
|
|
|
|
|
|
def parse_score_rows(html_text, url):
|
|
candidates = []
|
|
for table in pd.read_html(StringIO(html_text)):
|
|
if table.shape[1] not in {3, 7}:
|
|
continue
|
|
rows = []
|
|
triples = [(0, 1, 2)] if table.shape[1] == 3 else [(0, 1, 2), (4, 5, 6)]
|
|
for score_column, same_column, cumulative_column in triples:
|
|
for _, row in table.iterrows():
|
|
# 控制线可能直接注记在分数后；但“101分以下及缺考”之类汇总行
|
|
# 无法映射到单一分数，必须排除并在覆盖说明中保留公开范围。
|
|
score_label = str(row.iloc[score_column])
|
|
if "以下" in score_label or "缺考" in score_label:
|
|
continue
|
|
match = re.match(r"^\s*(\d+)(?:\s*[-—]\s*\d+)?", score_label)
|
|
if not match:
|
|
continue
|
|
try:
|
|
score = int(match.group(1))
|
|
same = int(float(row.iloc[same_column]))
|
|
cumulative = int(float(row.iloc[cumulative_column]))
|
|
except (TypeError, ValueError):
|
|
continue
|
|
rows.append((score, same, cumulative))
|
|
if len(rows) >= 100:
|
|
candidates.append(rows)
|
|
if len(candidates) != 1:
|
|
raise ValueError(f"一分一段表数量异常: {url} 候选表{len(candidates)}个")
|
|
rows = candidates[0]
|
|
if len({row[0] for row in rows}) != len(rows):
|
|
raise ValueError(f"一分一段存在重复分数: {url}")
|
|
if any(rows[index][2] != rows[index - 1][2] + rows[index][1] for index in range(1, len(rows))):
|
|
raise ValueError(f"一分一段累计人数校验失败: {url}")
|
|
return [(
|
|
score,
|
|
same,
|
|
cumulative,
|
|
max(1, cumulative - same + 1),
|
|
cumulative,
|
|
index + 1,
|
|
) for index, (score, same, cumulative) in enumerate(rows)]
|
|
|
|
|
|
def chunks(values, size=300):
|
|
for index in range(0, len(values), size):
|
|
yield values[index:index + size]
|
|
|
|
|
|
def mysql_run(args, sql):
|
|
if not os.environ.get("MYSQL_PWD"):
|
|
raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码")
|
|
process = subprocess.run([
|
|
args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port),
|
|
"-u", args.user, "--default-character-set=utf8mb4",
|
|
], input=sql, text=True, encoding="utf-8", capture_output=True)
|
|
if process.returncode:
|
|
raise RuntimeError(process.stderr.strip() or process.stdout.strip())
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="导入全国31省级地区2025年公开招录数据首批快照")
|
|
parser.add_argument("--mysql-bin", required=True)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--port", type=int, default=3306)
|
|
parser.add_argument("--user", default="root")
|
|
parser.add_argument("--database", default="zhiyuantong")
|
|
parser.add_argument("--root", default=str(Path(__file__).resolve().parents[1]))
|
|
parser.add_argument("--refresh", action="store_true", help="重新下载已经归档的原始网页")
|
|
args = parser.parse_args()
|
|
|
|
root = Path(args.root)
|
|
national_raw = root / "data/raw/national/2025"
|
|
control_path = national_raw / "control_lines_31_provinces_2025.html"
|
|
score_index_path = national_raw / "score_rank_index_2025.html"
|
|
control_html = fetch(CONTROL_SOURCE_URL, control_path, args.refresh).decode("utf-8")
|
|
fetch(SCORE_INDEX_URL, score_index_path, args.refresh)
|
|
controls = parse_control_lines(control_html)
|
|
|
|
statements = ["SET NAMES utf8mb4;", "SET time_zone='+08:00';", f"USE `{args.database}`;", "START TRANSACTION;"]
|
|
statements.append(source_upsert(
|
|
control_path,
|
|
"2025年全国31省级地区普通高考录取控制线汇总",
|
|
CONTROL_SOURCE_URL,
|
|
"高考100（汇总各省级考试机构公开数据）",
|
|
"普通类控制线与阳光高考2025批次线汇总交叉核对；保留原始网页及哈希。",
|
|
))
|
|
run_id, start_sql = ingestion_start("national_2025_control_score_lines", len(controls))
|
|
statements.append(start_sql)
|
|
controls_by_province = defaultdict(list)
|
|
for record in controls:
|
|
controls_by_province[record[0]].append(record)
|
|
for province, records in controls_by_province.items():
|
|
if province == "甘肃省":
|
|
continue
|
|
statements.append(f"DELETE FROM control_score_lines WHERE province_name={sql_string(province)} AND admission_year={YEAR} AND admission_category='普通类';")
|
|
values = []
|
|
for _, subject, level, score, raw_level in records:
|
|
values.append("(" + ",".join([
|
|
sql_string(province), str(YEAR), sql_string("普通类"), sql_string(subject),
|
|
sql_string(level), score, "NULL", sql_string(f"全国批次线汇总原口径：{raw_level}"), "@source_id",
|
|
]) + ")")
|
|
statements.append("""
|
|
INSERT INTO control_score_lines (
|
|
province_name, admission_year, admission_category, subject_category,
|
|
education_level, culture_score, professional_score, notes, source_id
|
|
) VALUES
|
|
""" + ",\n".join(values) + ";")
|
|
statements.append(coverage_sql(
|
|
province, "普通类控制线", "complete_public_snapshot", len(records),
|
|
"2025年普通类主要批次控制线已按全国汇总页逐行导入，并与阳光高考批次线汇总交叉核对。",
|
|
))
|
|
statements.append(ingestion_finish(run_id, len(controls), "全国31省级地区2025年普通类控制线完成解析；甘肃保留更完整的普通及艺体数据"))
|
|
|
|
score_counts = defaultdict(int)
|
|
imported_score_provinces = set()
|
|
for province, slug, subject, url in SCORE_SOURCES:
|
|
path = root / "data/raw" / slug / "2025" / f"score_rank_{subject}_2025.html"
|
|
html_text = fetch(url, path, args.refresh).decode("utf-8")
|
|
rows = parse_score_rows(html_text, url)
|
|
imported_score_provinces.add(province)
|
|
score_counts[province] += len(rows)
|
|
statements.append(source_upsert(
|
|
path,
|
|
f"{province}2025年普通高考一分一段表（{subject}）",
|
|
url,
|
|
"中国教育在线（转载省级考试机构公开数据）",
|
|
"从全国2025一分一段导航定位；仅导入可逐行解析并通过累计人数校验的公开表格。",
|
|
))
|
|
score_run_id, score_start = ingestion_start(f"{slug}_2025_score_rank_{subject}", len(rows))
|
|
statements.extend([
|
|
score_start,
|
|
f"DELETE FROM score_rank_points WHERE province_name={sql_string(province)} AND admission_year={YEAR} AND admission_category='普通类' AND subject_category={sql_string(subject)};",
|
|
f"DELETE FROM dataset_coverage WHERE province_name={sql_string(province)} AND admission_year={YEAR} AND dataset_name='一分一段' AND subject_category='';",
|
|
])
|
|
values = ["(" + ",".join([
|
|
sql_string(province), str(YEAR), sql_string("普通类"), sql_string(subject),
|
|
str(score), str(same), str(cumulative), str(rank_from), str(rank_to),
|
|
str(sequence), "@source_id",
|
|
]) + ")" for score, same, cumulative, rank_from, rank_to, sequence in rows]
|
|
for batch in chunks(values):
|
|
statements.append("""
|
|
INSERT INTO score_rank_points (
|
|
province_name, admission_year, admission_category, subject_category,
|
|
score, same_score_count, cumulative_count, rank_from, rank_to,
|
|
source_sequence, source_id
|
|
) VALUES
|
|
""" + ",\n".join(batch) + ";")
|
|
first, last = rows[0], rows[-1]
|
|
top_note = "；最高分段存在隐私合并，精确位次按页面累计人数保存" if first[2] != first[1] else ""
|
|
statements.append(coverage_sql(
|
|
province, "一分一段", "complete_public_snapshot", len(rows),
|
|
f"完整导入该公开表的{len(rows)}个分数点，范围{first[0]}至{last[0]}分，末行累计{last[2]}人；累计人数逐行校验通过{top_note}。",
|
|
subject,
|
|
))
|
|
statements.append(ingestion_finish(score_run_id, len(rows), f"{province}{subject}一分一段公开表导入并校验通过"))
|
|
|
|
all_provinces = {value[0] for value in PROVINCES.values()}
|
|
gansu = "甘肃省"
|
|
gap_note = "本轮已建立2025年全国省级数据域并完成控制线/可校验一分一段；该分类仍需按本省考试院文件继续结构化，不能据此判定官方未公开。"
|
|
for province in sorted(all_provinces - {gansu}):
|
|
if province not in imported_score_provinces:
|
|
statements.append(coverage_sql(
|
|
province, "一分一段", "partial_public_snapshot", 0,
|
|
"已定位2025年公开来源，但当前来源为图片、附件或非标准表格，尚未通过逐行结构化校验。",
|
|
category="普通类",
|
|
))
|
|
for dataset_name in ["批次与志愿规则", "初始招生计划", "院校专业组", "专业招生计划", "院校专业组投档线"]:
|
|
statements.append(coverage_sql(
|
|
province, dataset_name, "partial_public_snapshot", 0, gap_note, category="",
|
|
))
|
|
control_count = len(controls_by_province[province])
|
|
score_count = score_counts[province]
|
|
note = f"已导入2025年普通类控制线{control_count}条"
|
|
if score_count:
|
|
note += f"、一分一段{score_count}条"
|
|
else:
|
|
note += "；一分一段公开源尚待结构化复核"
|
|
note += "。招生规则、计划、专业组和投档线继续按省级官方来源补充。"
|
|
statements.append(f"""
|
|
UPDATE collection_scopes SET collection_status='partial', priority_level='high',
|
|
status_note={sql_string(note)}, first_collected_at=COALESCE(first_collected_at, NOW()),
|
|
last_collected_at=NOW()
|
|
WHERE province_name={sql_string(province)} AND admission_year={YEAR};
|
|
""")
|
|
|
|
statements.append("COMMIT;")
|
|
mysql_run(args, "\n".join(statements))
|
|
print(
|
|
f"导入完成: 全国控制线 {len(controls)} 条（甘肃保留原34条），"
|
|
f"新增一分一段 {sum(score_counts.values())} 条，覆盖 {len(imported_score_provinces)} 个省级地区；"
|
|
f"31省级地区2025范围均已建立覆盖缺口。"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|