You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
383 lines
19 KiB
383 lines
19 KiB
import argparse
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
|
|
PROVINCE = "甘肃省"
|
|
YEAR = 2025
|
|
|
|
|
|
def sql_string(value):
|
|
if value is None or value == "":
|
|
return "NULL"
|
|
text = str(value).replace("\\", "\\\\").replace("'", "\\'")
|
|
text = text.replace("\x00", "").replace("\r", " ").replace("\n", " ")
|
|
return f"'{text}'"
|
|
|
|
|
|
def file_digest(path):
|
|
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
|
|
|
|
def source_upsert(meta, path):
|
|
digest = file_digest(path)
|
|
return f"""
|
|
INSERT INTO data_sources (
|
|
source_type, publisher, title, landing_page_url, file_url, published_on,
|
|
retrieved_at, sha256, raw_file_path, notes
|
|
) VALUES (
|
|
{sql_string(meta['source_type'])}, {sql_string(meta['publisher'])},
|
|
{sql_string(meta['title'])}, {sql_string(meta.get('landing_page_url'))},
|
|
{sql_string(meta.get('file_url'))}, {sql_string(meta.get('published_on'))},
|
|
NOW(), {sql_string(digest)}, {sql_string(str(Path(path).resolve()))},
|
|
{sql_string(meta.get('notes'))}
|
|
) AS incoming
|
|
ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at,
|
|
raw_file_path=incoming.raw_file_path, notes=incoming.notes;
|
|
SET @source_id=(SELECT source_id FROM data_sources WHERE sha256={sql_string(digest)});
|
|
"""
|
|
|
|
|
|
def run_start(dataset_name, row_count):
|
|
run_id = str(uuid.uuid4())
|
|
return run_id, f"""
|
|
INSERT INTO ingestion_runs (
|
|
ingestion_run_id, source_id, dataset_name, dataset_year, started_at,
|
|
status, rows_read, rows_rejected
|
|
) VALUES ({sql_string(run_id)}, @source_id, {sql_string(dataset_name)}, {YEAR},
|
|
NOW(), 'running', {row_count}, 0);
|
|
"""
|
|
|
|
|
|
def run_finish(run_id, row_count, message):
|
|
return f"""
|
|
UPDATE ingestion_runs SET finished_at=NOW(), status='success',
|
|
rows_inserted={row_count}, message={sql_string(message)}
|
|
WHERE ingestion_run_id={sql_string(run_id)};
|
|
"""
|
|
|
|
|
|
def chunks(rows, size=300):
|
|
for index in range(0, len(rows), size):
|
|
yield rows[index:index + size]
|
|
|
|
|
|
def coverage_sql(name, category, subject, segment, status, count, notes):
|
|
return f"""
|
|
INSERT INTO dataset_coverage (
|
|
dataset_name, province_name, admission_year, admission_category,
|
|
subject_category, segment_code, coverage_status, record_count, notes, source_id
|
|
) VALUES ({sql_string(name)}, {sql_string(PROVINCE)}, {YEAR},
|
|
{sql_string(category) if category else "''"},
|
|
{sql_string(subject) if subject else "''"},
|
|
{sql_string(segment) if segment else "''"},
|
|
{sql_string(status)}, {count}, {sql_string(notes)}, @source_id)
|
|
AS incoming ON DUPLICATE KEY UPDATE coverage_status=incoming.coverage_status,
|
|
record_count=incoming.record_count, notes=incoming.notes, source_id=incoming.source_id;
|
|
"""
|
|
|
|
|
|
def build_rules_sql(raw_dir):
|
|
path = raw_dir / "admission_rules_2025.html"
|
|
meta = {
|
|
"source_type": "official_html",
|
|
"publisher": "甘肃省教育考试院",
|
|
"title": "2025年甘肃省普通高校招生网上填报志愿及征集志愿实施办法",
|
|
"landing_page_url": "https://www.ganseea.cn/html/gaokaogaozhao/1507.html",
|
|
"published_on": "2025-06-17",
|
|
"notes": "用于核验A至I段、志愿模式、专业组选报数和征集次数。A段同时包含顺序志愿与平行志愿类型。",
|
|
}
|
|
batches = [
|
|
("普通类", "本科", "A", "普通类本科提前批A段（含顺序志愿和平行志愿类型）", "顺序志愿/平行志愿", 30, 6, 1),
|
|
("普通类", "本科", "B", "普通类本科批B段", "单志愿", 1, 6, 0),
|
|
("普通类", "本科", "C", "普通类本科批C段", "平行志愿", 45, 6, 2),
|
|
("普通类", "高职专科", "D", "普通类高职（专科）提前批D段", "1+1顺序志愿", 2, 6, 1),
|
|
("普通类", "高职专科", "E", "普通类高职（专科）提前批E段", "平行志愿", 45, 6, 1),
|
|
("普通类", "高职专科", "F", "普通类高职（专科）批F段", "平行志愿", 45, 6, 2),
|
|
("艺体类", "本科", "G", "艺体类本科提前批G段", "单志愿", 1, 6, 0),
|
|
("艺体类", "本科", "H", "艺体类本科批H段", "平行志愿", 30, 6, 2),
|
|
("艺体类", "高职专科", "I", "艺体类高职（专科）批I段", "平行志愿", 30, 6, 2),
|
|
]
|
|
statements = [source_upsert(meta, path)]
|
|
run_id, start = run_start("gansu_2025_admission_rules", len(batches))
|
|
statements.append(start)
|
|
values = ["(" + ",".join([
|
|
sql_string(PROVINCE), str(YEAR), sql_string(category), sql_string(level),
|
|
sql_string(segment), sql_string(name), sql_string(mode), str(max_groups),
|
|
str(max_majors), str(solicitations), "@source_id",
|
|
]) + ")" for category, level, segment, name, mode, max_groups, max_majors, solicitations in batches]
|
|
statements.append("""
|
|
INSERT INTO admission_batches (
|
|
province_name, admission_year, admission_category, education_level,
|
|
segment_code, batch_name, volunteer_mode, max_group_volunteers,
|
|
max_majors_per_group, solicitation_count, source_id
|
|
) VALUES
|
|
""" + ",\n".join(values) + """
|
|
AS incoming ON DUPLICATE KEY UPDATE education_level=incoming.education_level,
|
|
batch_name=incoming.batch_name, volunteer_mode=incoming.volunteer_mode,
|
|
max_group_volunteers=incoming.max_group_volunteers,
|
|
max_majors_per_group=incoming.max_majors_per_group,
|
|
solicitation_count=incoming.solicitation_count, source_id=incoming.source_id;
|
|
""")
|
|
statements.append(coverage_sql(
|
|
"批次与志愿规则", "", "", "", "complete_public_snapshot", len(batches),
|
|
"考试院公开实施办法完整覆盖A至I段；A段同时存在顺序志愿和30个专业组平行志愿两种类型。",
|
|
))
|
|
statements.append(run_finish(run_id, len(batches), "官方2025年A至I段志愿规则已导入"))
|
|
return statements, len(batches)
|
|
|
|
|
|
def parse_score_rows(path):
|
|
table = pd.read_html(str(path))[0]
|
|
records = []
|
|
previous = 0
|
|
seen = set()
|
|
for _, row in table.iterrows():
|
|
score_text = str(row.iloc[0]).strip()
|
|
match = re.match(r"^(\d+)(?:\s*-\s*\d+)?", score_text)
|
|
if not match:
|
|
continue
|
|
score = int(match.group(1))
|
|
same = int(float(row.iloc[1]))
|
|
cumulative = int(float(row.iloc[2]))
|
|
if score in seen:
|
|
raise ValueError(f"一分一段出现重复分数: {path} {score}")
|
|
if cumulative != previous + same:
|
|
raise ValueError(f"一分一段累计人数校验失败: {path} {score} {same} {cumulative} {previous}")
|
|
rank_from = previous + 1 if same else cumulative
|
|
records.append((score, same, cumulative, rank_from, cumulative, len(records) + 1))
|
|
seen.add(score)
|
|
previous = cumulative
|
|
if len(records) < 600 or records[-1][0] != 0:
|
|
raise ValueError(f"一分一段记录不完整: {path} 共{len(records)}条")
|
|
return records
|
|
|
|
|
|
def build_score_sql(raw_dir, filename, subject, page_url):
|
|
path = raw_dir / filename
|
|
records = parse_score_rows(path)
|
|
meta = {
|
|
"source_type": "official_republication_html",
|
|
"publisher": "中国教育在线（数据来源：甘肃省教育考试院）",
|
|
"title": f"甘肃2025年高考一分一段表公布（{subject}）",
|
|
"landing_page_url": page_url,
|
|
"published_on": "2025-07-02",
|
|
"notes": "公开转载页明确标注数据来源为甘肃省教育考试院；顶部合并分数区间按区间下限存储。",
|
|
}
|
|
statements = [source_upsert(meta, path)]
|
|
run_id, start = run_start(f"gansu_2025_score_rank_{subject}", len(records))
|
|
statements += [start, f"DELETE FROM score_rank_points WHERE province_name={sql_string(PROVINCE)} AND admission_year={YEAR} AND admission_category='普通类' AND subject_category={sql_string(subject)};"]
|
|
values = ["(" + ",".join([
|
|
sql_string(PROVINCE), str(YEAR), sql_string("普通类"), sql_string(subject),
|
|
str(score), str(same), str(cumulative), str(rank_from), str(rank_to),
|
|
str(sequence), "@source_id",
|
|
]) + ")" for score, same, cumulative, rank_from, rank_to, sequence in records]
|
|
for batch in chunks(values):
|
|
statements.append("""
|
|
INSERT INTO score_rank_points (
|
|
province_name, admission_year, admission_category, subject_category,
|
|
score, same_score_count, cumulative_count, rank_from, rank_to,
|
|
source_sequence, source_id
|
|
) VALUES
|
|
""" + ",\n".join(batch) + ";")
|
|
statements.append(coverage_sql(
|
|
"一分一段", "普通类", subject, "", "complete_public_snapshot", len(records),
|
|
f"{subject}公开分段表从最高合并区间连续覆盖至0分，累计人数校验通过。",
|
|
))
|
|
statements.append(run_finish(run_id, len(records), f"2025年{subject}一分一段完整公开转载表已导入"))
|
|
return statements, len(records)
|
|
|
|
|
|
def build_control_sql(raw_dir):
|
|
path = raw_dir / "control_lines_republication_2025.html"
|
|
meta = {
|
|
"source_type": "official_republication_html",
|
|
"publisher": "甘肃教育在线（来源：甘肃省教育考试院）",
|
|
"title": "2025年甘肃省普通高校招生统一考试录取最低控制分数线",
|
|
"landing_page_url": "https://www.ghsi.cn/Index/news/getcontent.html?id=1726",
|
|
"published_on": "2025-06-25",
|
|
"notes": "转载页标明来源为甘肃省教育考试院；原始目录同时保存普通类、艺术体育类本科和专科控制线图片。",
|
|
}
|
|
normal = [
|
|
("普通类", "物理类", "特殊类型", 475, None),
|
|
("普通类", "物理类", "本科", 374, None),
|
|
("普通类", "物理类", "高职专科", 180, None),
|
|
("普通类", "历史类", "特殊类型", 500, None),
|
|
("普通类", "历史类", "本科", 412, None),
|
|
("普通类", "历史类", "高职专科", 160, None),
|
|
]
|
|
art_sport = [
|
|
("艺术类", "美术与设计类", 281, 200),
|
|
("艺术类", "书法类", 281, 204),
|
|
("艺术类", "音乐类（音乐表演-声乐）", 281, 234),
|
|
("艺术类", "音乐类（音乐教育-声乐）", 281, 227),
|
|
("艺术类", "音乐类（音乐表演-器乐）", 281, 210),
|
|
("艺术类", "音乐类（音乐教育-器乐）", 281, 217),
|
|
("艺术类", "舞蹈类", 281, 228),
|
|
("艺术类", "播音与主持类", 374, 211),
|
|
("艺术类", "表（导）演类（戏剧影视表演）", 281, 240),
|
|
("艺术类", "表（导）演类（戏剧影视导演）", 281, 242),
|
|
("艺术类", "表（导）演类（服装表演）", 281, 215),
|
|
("体育类", "体育类（田径）", 281, 254),
|
|
("体育类", "体育类（足球、篮球、排球）", 281, 242),
|
|
("体育类", "体育类（武术、体操）", 281, 240),
|
|
]
|
|
lines = list(normal)
|
|
lines += [(category, subject, "本科", culture, professional) for category, subject, culture, professional in art_sport]
|
|
lines += [(category, subject, "高职专科", 160, 160) for category, subject, _, _ in art_sport]
|
|
statements = [source_upsert(meta, path)]
|
|
run_id, start = run_start("gansu_2025_control_score_lines", len(lines))
|
|
statements += [start, f"DELETE FROM control_score_lines WHERE province_name={sql_string(PROVINCE)} AND admission_year={YEAR};"]
|
|
values = []
|
|
for category, subject, level, culture, professional in lines:
|
|
note = "普通类文化课控制线" if category == "普通类" else "艺术体育类统考文化课/专业课控制线"
|
|
values.append("(" + ",".join([
|
|
sql_string(PROVINCE), str(YEAR), sql_string(category), sql_string(subject),
|
|
sql_string(level), str(culture), "NULL" if professional is None else str(professional),
|
|
sql_string(note), "@source_id",
|
|
]) + ")")
|
|
statements.append("""
|
|
INSERT INTO control_score_lines (
|
|
province_name, admission_year, admission_category, subject_category,
|
|
education_level, culture_score, professional_score, notes, source_id
|
|
) VALUES
|
|
""" + ",\n".join(values) + ";")
|
|
statements.append(coverage_sql(
|
|
"普通类控制线", "普通类", "", "", "complete_public_snapshot", len(normal),
|
|
"物理类、历史类各含特殊类型、本科和高职专科控制线。",
|
|
))
|
|
statements.append(coverage_sql(
|
|
"艺体类控制线", "艺体类", "", "", "complete_public_snapshot", len(lines) - len(normal),
|
|
"本科和高职专科各覆盖11个艺术统考类别及3个体育统考类别。",
|
|
))
|
|
statements.append(run_finish(run_id, len(lines), "2025年普通类及艺术体育类控制线已导入"))
|
|
return statements, len(lines)
|
|
|
|
|
|
def build_restricted_data_sql(raw_dir):
|
|
releases = [
|
|
(1, 1532, "本科提前批A段征集", "2025-07-12"),
|
|
(2, 1533, "艺体类本科批H段第一次征集", "2025-07-14"),
|
|
(3, 1535, "艺体类本科批H段第二次征集", "2025-07-16"),
|
|
(4, 1545, "普通类本科批C段第一次征集", "2025-07-24"),
|
|
(5, 1547, "普通类本科批C段第二次征集", "2025-07-26"),
|
|
(6, 1552, "高职（专科）提前批D、E段征集", "2025-08-01"),
|
|
(7, 1556, "艺体类高职（专科）批I段第一次征集", "2025-08-10"),
|
|
(8, 1557, "艺体类高职（专科）批I段第二次征集", "2025-08-11"),
|
|
(9, 1558, "高职（专科）批F段第一次征集", "2025-08-15"),
|
|
(10, 1560, "高职（专科）批F段第二次征集", "2025-08-17"),
|
|
]
|
|
statements = []
|
|
for number, page_id, round_name, published_on in releases:
|
|
path = raw_dir / f"solicitation_{number:02d}_2025.html"
|
|
meta = {
|
|
"source_type": "official_html",
|
|
"publisher": "甘肃省教育考试院",
|
|
"title": f"2025年甘肃省普通高校招生征集志愿第{number}号公告",
|
|
"landing_page_url": f"https://www.ganseea.cn/shouyegonggao/{page_id}.html",
|
|
"published_on": published_on,
|
|
"notes": "公告元数据可公开；院校专业组剩余计划明细仅在登录后的志愿填报辅助平台提供。",
|
|
}
|
|
statements.append(source_upsert(meta, path))
|
|
run_id, start = run_start(f"gansu_2025_solicitation_notice_{number}", 1)
|
|
statements.append(start)
|
|
statements.append(f"""
|
|
INSERT INTO admission_plan_releases (
|
|
province_name, admission_year, release_title, plan_round,
|
|
coverage_scope, published_on, source_id
|
|
) VALUES ({sql_string(PROVINCE)}, {YEAR}, {sql_string(meta['title'])},
|
|
{sql_string(round_name)}, 'login_only', {sql_string(published_on)}, @source_id)
|
|
AS incoming ON DUPLICATE KEY UPDATE release_title=incoming.release_title,
|
|
plan_round=incoming.plan_round, coverage_scope=incoming.coverage_scope,
|
|
published_on=incoming.published_on;
|
|
""")
|
|
statements.append(run_finish(run_id, 1, "征集公告元数据已导入；剩余计划明细需登录查询"))
|
|
statements.append(coverage_sql(
|
|
"初始招生计划", "", "", "", "unavailable_publicly", 0,
|
|
"2025年初始招生专业目录以纸质目录及登录后志愿填报辅助平台提供，未发现公开下载的全量明细。",
|
|
))
|
|
statements.append(coverage_sql(
|
|
"征集专业计划", "", "", "", "unavailable_publicly", 0,
|
|
"考试院10次征集公告均要求登录志愿填报辅助平台查询学校专业组剩余计划，公开公告无明细附件。",
|
|
))
|
|
|
|
cutoff_path = raw_dir / "cutoff_access_policy_2025.html"
|
|
cutoff_meta = {
|
|
"source_type": "official_html",
|
|
"publisher": "甘肃省教育考试院",
|
|
"title": "2025年甘肃省普通高校招生本科批（C段）投档最低分及录取信息查询公告",
|
|
"landing_page_url": "https://www.ganseea.cn/shouyegonggao/1544.html",
|
|
"published_on": "2025-07-21",
|
|
"notes": "公告明确自2025年起投档及录取信息仅向考生本人和在甘招生高校提供，不再向其他单位和个人开放。",
|
|
}
|
|
statements.append(source_upsert(cutoff_meta, cutoff_path))
|
|
run_id, start = run_start("gansu_2025_cutoff_access_policy", 1)
|
|
statements.append(start)
|
|
statements.append(coverage_sql(
|
|
"院校专业组投档线", "", "", "", "unavailable_publicly", 0,
|
|
"考试院明确自2025年起不再公开完整投档及录取信息；仅可由考生或在甘招生高校在登录系统内查询。",
|
|
))
|
|
statements.append(coverage_sql(
|
|
"院校专业组", "", "", "", "unavailable_publicly", 0,
|
|
"院校专业组及对应投档线只能在需要身份登录的官方平台按条件查询，未发现公开全量附件。",
|
|
))
|
|
statements.append(run_finish(run_id, 1, "投档信息公开限制政策已存档"))
|
|
return statements, len(releases)
|
|
|
|
|
|
def mysql_run(args, sql):
|
|
if not os.environ.get("MYSQL_PWD"):
|
|
raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码")
|
|
process = subprocess.run([
|
|
args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port),
|
|
"-u", args.user, "--default-character-set=utf8mb4",
|
|
], input=sql, text=True, encoding="utf-8", capture_output=True)
|
|
if process.returncode:
|
|
raise RuntimeError(process.stderr.strip() or process.stdout.strip())
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="导入甘肃省2025年可公开取得的招录数据")
|
|
parser.add_argument("--mysql-bin", required=True)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--port", type=int, default=3306)
|
|
parser.add_argument("--user", default="root")
|
|
parser.add_argument("--database", default="zhiyuantong")
|
|
parser.add_argument("--root", default=str(Path(__file__).resolve().parents[1]))
|
|
args = parser.parse_args()
|
|
raw_dir = Path(args.root) / "data/raw/gansu/2025"
|
|
|
|
statements = ["SET NAMES utf8mb4;", "SET time_zone='+08:00';", f"USE `{args.database}`;", "START TRANSACTION;"]
|
|
rules_sql, rule_count = build_rules_sql(raw_dir)
|
|
statements += rules_sql
|
|
score_count = 0
|
|
for filename, subject, url in [
|
|
("score_rank_physics_2025.html", "物理类", "https://gaokao.eol.cn/gan_su/dongtai/202507/t20250702_2678474.shtml"),
|
|
("score_rank_history_2025.html", "历史类", "https://gaokao.eol.cn/gan_su/dongtai/202507/t20250702_2678473.shtml"),
|
|
]:
|
|
score_sql, count = build_score_sql(raw_dir, filename, subject, url)
|
|
statements += score_sql
|
|
score_count += count
|
|
control_sql, control_count = build_control_sql(raw_dir)
|
|
statements += control_sql
|
|
restricted_sql, release_count = build_restricted_data_sql(raw_dir)
|
|
statements += restricted_sql
|
|
statements.append(f"""
|
|
UPDATE collection_scopes SET collection_status='partial', priority_level='high',
|
|
status_note={sql_string('已导入2025年官方批次志愿规则、普通类及艺体类控制线、物理类与历史类一分一段，以及10次征集公告元数据。考试院自2025年起不公开完整投档线，招生计划与专业组明细须登录查询，因此相关明细未导入。')},
|
|
first_collected_at=COALESCE(first_collected_at, NOW()), last_collected_at=NOW()
|
|
WHERE province_name={sql_string(PROVINCE)} AND admission_year={YEAR};
|
|
""")
|
|
statements.append("COMMIT;")
|
|
mysql_run(args, "\n".join(statements))
|
|
print(f"导入完成: 批次规则 {rule_count} 条，一分一段 {score_count} 条，控制线 {control_count} 条，征集公告 {release_count} 条；计划/专业组/投档线已按官方政策标记为公开不可得")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|