You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
541 lines
28 KiB
541 lines
28 KiB
import argparse
|
|
import csv
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import uuid
|
|
from datetime import date
|
|
from io import StringIO
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
|
|
PROVINCE = "甘肃省"
|
|
YEAR = 2024
|
|
|
|
|
|
def sql_string(value):
|
|
if value is None or value == "":
|
|
return "NULL"
|
|
text = str(value).replace("\\", "\\\\").replace("'", "\\'")
|
|
text = text.replace("\x00", "").replace("\r", " ").replace("\n", " ")
|
|
return f"'{text}'"
|
|
|
|
|
|
def sql_text(value):
|
|
return sql_string(value)
|
|
|
|
|
|
def sql_number(value):
|
|
if value is None or str(value).strip() == "":
|
|
return "NULL"
|
|
text = re.sub(r"\.0$", "", str(value).strip())
|
|
if not re.fullmatch(r"-?\d+(?:\.\d+)?", text):
|
|
raise ValueError(f"不是有效数值: {value!r}")
|
|
return text
|
|
|
|
|
|
def normalize_code(value, width):
|
|
text = re.sub(r"\.0$", "", str(value or "").strip())
|
|
if not text:
|
|
return None
|
|
return text.zfill(width) if text.isdigit() else text
|
|
|
|
|
|
def file_digest(path):
|
|
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
|
|
|
|
def read_csv_rows(path, expected_columns):
|
|
with Path(path).open("r", encoding="utf-8-sig", newline="") as handle:
|
|
rows = list(csv.DictReader(handle))
|
|
if len(rows) < 3:
|
|
raise ValueError(f"文件无有效数据: {path}")
|
|
actual = [str(rows[1].get(f"col{i}") or "").replace("\n", "").replace(" ", "") for i in range(1, len(expected_columns) + 1)]
|
|
expected = [item.replace("\n", "").replace(" ", "") for item in expected_columns]
|
|
if actual != expected:
|
|
raise ValueError(f"表头不匹配 {path}: {actual!r}")
|
|
return [row for row in rows[2:] if any(str(row.get(f"col{i}") or "").strip() for i in range(1, len(expected_columns) + 1))]
|
|
|
|
|
|
def source_upsert(meta, digest, raw_path):
|
|
return f"""
|
|
INSERT INTO data_sources (
|
|
source_type, publisher, title, landing_page_url, file_url, published_on,
|
|
retrieved_at, sha256, raw_file_path, notes
|
|
) VALUES (
|
|
{sql_string(meta['source_type'])}, {sql_string(meta['publisher'])},
|
|
{sql_string(meta['title'])}, {sql_string(meta.get('landing_page_url'))},
|
|
{sql_string(meta.get('file_url'))}, {sql_string(meta.get('published_on'))},
|
|
NOW(), {sql_string(digest)}, {sql_string(str(Path(raw_path).resolve()))},
|
|
{sql_string(meta.get('notes'))}
|
|
) AS incoming
|
|
ON DUPLICATE KEY UPDATE retrieved_at=incoming.retrieved_at,
|
|
raw_file_path=incoming.raw_file_path, notes=incoming.notes;
|
|
SET @source_id = (SELECT source_id FROM data_sources WHERE sha256={sql_string(digest)});
|
|
"""
|
|
|
|
|
|
def run_start(dataset_name, row_count):
|
|
run_id = str(uuid.uuid4())
|
|
sql = f"""
|
|
INSERT INTO ingestion_runs (
|
|
ingestion_run_id, source_id, dataset_name, dataset_year, started_at,
|
|
status, rows_read, rows_rejected
|
|
) VALUES ({sql_string(run_id)}, @source_id, {sql_string(dataset_name)}, {YEAR},
|
|
NOW(), 'running', {row_count}, 0);
|
|
"""
|
|
return run_id, sql
|
|
|
|
|
|
def run_finish(run_id, row_count, message):
|
|
return f"""
|
|
UPDATE ingestion_runs SET finished_at=NOW(), status='success',
|
|
rows_inserted={row_count}, message={sql_string(message)}
|
|
WHERE ingestion_run_id={sql_string(run_id)};
|
|
"""
|
|
|
|
|
|
def chunks(rows, size=300):
|
|
for index in range(0, len(rows), size):
|
|
yield rows[index:index + size]
|
|
|
|
|
|
def build_cutoff_sql(csv_path, raw_path, meta, subject, level, segment):
|
|
headers = [
|
|
"院校代号", "院校名称", "院校专业组", "院校专业组名称", "投档最低分",
|
|
"总分(不含政策性照顾加分)", "语文、数学之和", "语文或数学单科最高成绩",
|
|
"外语单科成绩", "首选科目单科成绩", "再选科目单科最高成绩",
|
|
"再选科目单科次高成绩", "志愿号",
|
|
]
|
|
rows = read_csv_rows(csv_path, headers)
|
|
digest = file_digest(raw_path)
|
|
statements = [source_upsert(meta, digest, raw_path)]
|
|
run_id, start_sql = run_start(f"gansu_2024_cutoff_{segment}_{subject}", len(rows))
|
|
statements += [start_sql, "DELETE FROM admission_cutoff_lines WHERE source_id=@source_id;"]
|
|
values = []
|
|
for row in rows:
|
|
school_code = normalize_code(row["col1"], 4)
|
|
group_code = normalize_code(row["col3"], 3)
|
|
values.append("(" + ",".join([
|
|
sql_string(PROVINCE), str(YEAR), sql_string("普通类"), sql_string(subject),
|
|
sql_string(level), sql_string(segment), "NULL", sql_string(school_code),
|
|
sql_string(row["col2"].strip()), sql_string(group_code), sql_string(row["col4"].strip()),
|
|
sql_number(row["col5"]), sql_number(row["col6"]), sql_number(row["col7"]),
|
|
sql_number(row["col8"]), sql_number(row["col9"]), sql_number(row["col10"]),
|
|
sql_number(row["col11"]), sql_number(row["col12"]), sql_number(row["col13"]),
|
|
str(int(row["row_number"])), "@source_id"
|
|
]) + ")")
|
|
for batch in chunks(values):
|
|
statements.append("""
|
|
INSERT INTO admission_cutoff_lines (
|
|
province_name, admission_year, admission_category, subject_category,
|
|
education_level, segment_code, school_id, source_school_code,
|
|
school_name_raw, group_code, group_name, cutoff_score, raw_total_score,
|
|
chinese_math_sum, chinese_or_math_max, foreign_language_score,
|
|
first_choice_subject_score, second_choice_subject_max,
|
|
second_choice_subject_second, volunteer_number, source_row_number, source_id
|
|
) VALUES
|
|
""" + ",\n".join(batch) + ";")
|
|
statements.append("""
|
|
UPDATE admission_cutoff_lines c
|
|
SET c.school_id=(SELECT MIN(s.school_id) FROM schools s WHERE s.official_name=c.school_name_raw)
|
|
WHERE c.source_id=@source_id;
|
|
""")
|
|
statements.append("""
|
|
INSERT INTO admission_groups (
|
|
plan_release_id, province_name, admission_year, admission_category,
|
|
subject_category, segment_code, school_id, source_school_code,
|
|
school_name_raw, group_code, group_name, source_id
|
|
)
|
|
SELECT NULL, province_name, admission_year, admission_category,
|
|
subject_category, segment_code, school_id, source_school_code,
|
|
school_name_raw, group_code, group_name, source_id
|
|
FROM admission_cutoff_lines
|
|
WHERE source_id=@source_id
|
|
ON DUPLICATE KEY UPDATE
|
|
school_id=COALESCE(admission_groups.school_id, VALUES(school_id)),
|
|
group_name=COALESCE(admission_groups.group_name, VALUES(group_name));
|
|
""")
|
|
statements.append(run_finish(run_id, len(rows), "官方平行志愿投档最低分完整公开表已导入"))
|
|
statements.append(f"""
|
|
INSERT INTO dataset_coverage (
|
|
dataset_name, province_name, admission_year, admission_category,
|
|
subject_category, segment_code, coverage_status, record_count, notes, source_id
|
|
) VALUES ('院校专业组投档线', '{PROVINCE}', {YEAR}, '普通类', {sql_string(subject)},
|
|
{sql_string(segment)}, 'complete_public_snapshot', {len(rows)},
|
|
'官方公开的该段次首轮平行志愿投档最低分完整附件。', @source_id)
|
|
AS incoming ON DUPLICATE KEY UPDATE coverage_status=incoming.coverage_status,
|
|
record_count=incoming.record_count, notes=incoming.notes, source_id=incoming.source_id;
|
|
""")
|
|
return statements, len(rows)
|
|
|
|
|
|
def category_for(subject):
|
|
if subject in ("物理类", "历史类"):
|
|
return "普通类"
|
|
return subject
|
|
|
|
|
|
def build_plan_sql(csv_path, raw_path, meta, plan_round):
|
|
headers_a = ["批次", "科类", "院校代号", "院校名称", "专业代号", "专业名称", "院校专业组", "考试科目要求", "剩余计划数", "备注"]
|
|
headers_b = ["批次", "科类", "院校代号", "院校名称", "专业代号", "专业名称", "院校专业组", "科目要求", "缺额数", "备注"]
|
|
try:
|
|
rows = read_csv_rows(csv_path, headers_a)
|
|
except ValueError:
|
|
rows = read_csv_rows(csv_path, headers_b)
|
|
digest = file_digest(raw_path)
|
|
statements = [source_upsert(meta, digest, raw_path)]
|
|
run_id, start_sql = run_start(f"gansu_2024_plan_{plan_round}", len(rows))
|
|
statements += [start_sql]
|
|
statements += [
|
|
"DELETE FROM admission_major_plans WHERE source_id=@source_id;",
|
|
f"""
|
|
INSERT INTO admission_plan_releases (
|
|
province_name, admission_year, release_title, plan_round,
|
|
coverage_scope, published_on, source_id
|
|
) VALUES ('{PROVINCE}', {YEAR}, {sql_string(meta['title'])},
|
|
{sql_string(plan_round)}, 'remaining_only', {sql_string(meta['published_on'])}, @source_id)
|
|
AS incoming ON DUPLICATE KEY UPDATE
|
|
release_title=incoming.release_title, plan_round=incoming.plan_round,
|
|
coverage_scope=incoming.coverage_scope, published_on=incoming.published_on;
|
|
SET @release_id=(SELECT plan_release_id FROM admission_plan_releases WHERE source_id=@source_id);
|
|
""",
|
|
]
|
|
group_values = {}
|
|
plan_values = []
|
|
for row in rows:
|
|
segment = row["col1"].strip()
|
|
subject = row["col2"].strip()
|
|
school_code = normalize_code(row["col3"], 4)
|
|
school_name = row["col4"].strip()
|
|
major_code = normalize_code(row["col5"], 2)
|
|
group_code = normalize_code(row["col7"], 3)
|
|
key = (subject, segment, school_code, group_code)
|
|
group_values[key] = "(" + ",".join([
|
|
"@release_id", sql_string(PROVINCE), str(YEAR), sql_string(category_for(subject)),
|
|
sql_string(subject), sql_string(segment), "NULL", sql_string(school_code),
|
|
sql_string(school_name), sql_string(group_code), "NULL", "@source_id"
|
|
]) + ")"
|
|
plan_values.append({
|
|
"key": key, "major_code": major_code, "major_name": row["col6"].strip(),
|
|
"requirements": row["col8"].strip() or None, "count": row["col9"],
|
|
"notes": row["col10"].strip() or None, "row_number": int(row["row_number"]),
|
|
})
|
|
for batch in chunks(list(group_values.values())):
|
|
statements.append("""
|
|
INSERT INTO admission_groups (
|
|
plan_release_id, province_name, admission_year, admission_category,
|
|
subject_category, segment_code, school_id, source_school_code,
|
|
school_name_raw, group_code, group_name, source_id
|
|
) VALUES
|
|
""" + ",\n".join(batch) + """
|
|
AS incoming ON DUPLICATE KEY UPDATE
|
|
plan_release_id=COALESCE(admission_groups.plan_release_id, incoming.plan_release_id),
|
|
school_id=COALESCE(admission_groups.school_id, incoming.school_id),
|
|
school_name_raw=incoming.school_name_raw;
|
|
""")
|
|
for batch in chunks(plan_values):
|
|
values = []
|
|
for item in batch:
|
|
subject, segment, school_code, group_code = item["key"]
|
|
group_id = f"(SELECT admission_group_id FROM admission_groups WHERE province_name={sql_string(PROVINCE)} AND admission_year={YEAR} AND admission_category={sql_string(category_for(subject))} AND subject_category={sql_string(subject)} AND segment_code={sql_string(segment)} AND source_school_code={sql_string(school_code)} AND group_code={sql_string(group_code)} LIMIT 1)"
|
|
values.append("(" + ",".join([
|
|
group_id, "@release_id", sql_string(item["major_code"]), sql_string(item["major_name"]),
|
|
sql_string(item["requirements"]), sql_number(item["count"]), "NULL", "NULL", "NULL",
|
|
sql_text(item["notes"]), str(item["row_number"]), "@source_id"
|
|
]) + ")")
|
|
statements.append("""
|
|
INSERT INTO admission_major_plans (
|
|
admission_group_id, plan_release_id, major_code, major_name,
|
|
subject_requirements, plan_count, tuition_yuan_per_year, study_years,
|
|
campus_name, notes, source_row_number, source_id
|
|
) VALUES
|
|
""" + ",\n".join(values) + ";")
|
|
statements.append("""
|
|
UPDATE admission_groups g
|
|
SET g.school_id=(SELECT MIN(s.school_id) FROM schools s WHERE s.official_name=g.school_name_raw)
|
|
WHERE g.plan_release_id=@release_id;
|
|
""")
|
|
statements.append(run_finish(run_id, len(rows), "官方征集志愿剩余计划已导入；不能视为初始招生计划"))
|
|
segments = sorted({row["col1"].strip() for row in rows})
|
|
statements.append(f"""
|
|
INSERT INTO dataset_coverage (
|
|
dataset_name, province_name, admission_year, admission_category,
|
|
subject_category, segment_code, coverage_status, record_count, notes, source_id
|
|
) VALUES ('专业招生计划', '{PROVINCE}', {YEAR}, '', '', {sql_string(','.join(segments))},
|
|
'partial_public_snapshot', {len(rows)},
|
|
'仅为官方公开的征集志愿剩余或缺额计划，不是2024年初始全量招生专业目录。', @source_id)
|
|
AS incoming ON DUPLICATE KEY UPDATE coverage_status=incoming.coverage_status,
|
|
record_count=incoming.record_count, notes=incoming.notes, source_id=incoming.source_id;
|
|
""")
|
|
return statements, len(rows), len(group_values)
|
|
|
|
|
|
def build_score_rank_sql(html_path, meta):
|
|
text = Path(html_path).read_text(encoding="utf-8")
|
|
table = pd.read_html(StringIO(text))[0]
|
|
records = []
|
|
for subject, offset in (("物理类", 0), ("历史类", 3)):
|
|
points = []
|
|
for _, row in table.iloc[2:].iterrows():
|
|
sequence, score, cumulative = row.iloc[offset:offset + 3]
|
|
if pd.isna(score) or pd.isna(cumulative):
|
|
continue
|
|
points.append((int(float(sequence)), int(float(score)), int(float(cumulative))))
|
|
points.sort(key=lambda item: item[1], reverse=True)
|
|
previous = 0
|
|
for sequence, score, cumulative in points:
|
|
if cumulative < previous:
|
|
raise ValueError(f"{subject}累计人数不是单调递增: {score}")
|
|
same = cumulative - previous
|
|
rank_from = previous + 1 if same else cumulative
|
|
records.append((subject, score, same, cumulative, rank_from, cumulative, sequence))
|
|
previous = cumulative
|
|
digest = file_digest(html_path)
|
|
statements = [source_upsert(meta, digest, html_path)]
|
|
run_id, start_sql = run_start("gansu_2024_score_rank", len(records))
|
|
statements += [start_sql, "DELETE FROM score_rank_points WHERE source_id=@source_id;"]
|
|
values = []
|
|
for subject, score, same, cumulative, rank_from, rank_to, sequence in records:
|
|
values.append("(" + ",".join([
|
|
sql_string(PROVINCE), str(YEAR), sql_string("普通类"), sql_string(subject),
|
|
str(score), str(same), str(cumulative), str(rank_from), str(rank_to),
|
|
str(sequence), "@source_id"
|
|
]) + ")")
|
|
for batch in chunks(values):
|
|
statements.append("""
|
|
INSERT INTO score_rank_points (
|
|
province_name, admission_year, admission_category, subject_category,
|
|
score, same_score_count, cumulative_count, rank_from, rank_to,
|
|
source_sequence, source_id
|
|
) VALUES
|
|
""" + ",\n".join(batch) + ";")
|
|
statements.append(run_finish(run_id, len(records), "官方2024年物理类、历史类一分一段表已导入"))
|
|
for subject in ("物理类", "历史类"):
|
|
count = sum(1 for row in records if row[0] == subject)
|
|
statements.append(f"""
|
|
INSERT INTO dataset_coverage (
|
|
dataset_name, province_name, admission_year, admission_category,
|
|
subject_category, segment_code, coverage_status, record_count, notes, source_id
|
|
) VALUES ('一分一段', '{PROVINCE}', {YEAR}, '普通类', {sql_string(subject)}, '',
|
|
'complete_public_snapshot', {count}, '官方公开的分数至累计人数完整分段表。', @source_id)
|
|
AS incoming ON DUPLICATE KEY UPDATE coverage_status=incoming.coverage_status,
|
|
record_count=incoming.record_count, notes=incoming.notes, source_id=incoming.source_id;
|
|
""")
|
|
return statements, len(records)
|
|
|
|
|
|
def build_reference_sql(root):
|
|
implementation = root / "data/raw/gansu/2024/admission_implementation_plan_2024.html"
|
|
control = root / "data/raw/gansu/2024/control_score_lines_republication_2024.html"
|
|
implementation_meta = {
|
|
"source_type": "official_html", "publisher": "甘肃省教育考试院",
|
|
"title": "2024年甘肃省普通高等学校招生考试和录取工作实施方案",
|
|
"landing_page_url": "https://www.ganseea.cn/shouyegonggao/910.html?appPrefix=xbsfdx",
|
|
"published_on": "2023-11-08", "notes": "用于批次段次、院校专业组和志愿数量规则。",
|
|
}
|
|
statements = [source_upsert(implementation_meta, file_digest(implementation), implementation)]
|
|
batches = [
|
|
("普通类", "本科", "A", "普通类本科提前批A段", "顺序志愿", 2, 6, 1),
|
|
("普通类", "本科", "B", "普通类本科批B段", "单志愿", 1, 6, 0),
|
|
("普通类", "本科", "C", "普通类本科批C段", "平行志愿", 45, 6, 2),
|
|
("普通类", "高职专科", "D", "普通类高职（专科）提前批D段", "顺序志愿", 2, 6, 1),
|
|
("普通类", "高职专科", "E", "普通类高职（专科）提前批E段", "平行志愿", 45, 6, 2),
|
|
("普通类", "高职专科", "F", "普通类高职（专科）批F段", "平行志愿", 45, 6, 2),
|
|
("艺体类", "本科", "G", "艺体类本科提前批G段", "单志愿", 1, 6, 0),
|
|
("艺体类", "本科", "H", "艺体类本科批H段", "平行志愿", 30, 6, 2),
|
|
("艺体类", "高职专科", "I", "艺体类高职（专科）批I段", "平行志愿", 30, 6, 2),
|
|
]
|
|
values = ["(" + ",".join([
|
|
sql_string(PROVINCE), str(YEAR), sql_string(category), sql_string(level),
|
|
sql_string(segment), sql_string(name), sql_string(mode), str(max_groups),
|
|
str(max_majors), str(solicitations), "@source_id"
|
|
]) + ")" for category, level, segment, name, mode, max_groups, max_majors, solicitations in batches]
|
|
statements.append("""
|
|
INSERT INTO admission_batches (
|
|
province_name, admission_year, admission_category, education_level,
|
|
segment_code, batch_name, volunteer_mode, max_group_volunteers,
|
|
max_majors_per_group, solicitation_count, source_id
|
|
) VALUES
|
|
""" + ",\n".join(values) + """
|
|
AS incoming ON DUPLICATE KEY UPDATE education_level=incoming.education_level,
|
|
batch_name=incoming.batch_name, volunteer_mode=incoming.volunteer_mode,
|
|
max_group_volunteers=incoming.max_group_volunteers,
|
|
max_majors_per_group=incoming.max_majors_per_group,
|
|
solicitation_count=incoming.solicitation_count, source_id=incoming.source_id;
|
|
""")
|
|
statements.append(f"""
|
|
INSERT INTO dataset_coverage (
|
|
dataset_name, province_name, admission_year, admission_category,
|
|
subject_category, segment_code, coverage_status, record_count, notes, source_id
|
|
) VALUES ('初始招生计划', '{PROVINCE}', {YEAR}, '', '', '',
|
|
'unavailable_publicly', 0,
|
|
'考试院公开网页指向纸质《2024年甘肃省普通高等学校招生专业目录》和登录后平台；本次未发现可公开下载的初始全量计划附件。', @source_id)
|
|
AS incoming ON DUPLICATE KEY UPDATE coverage_status=incoming.coverage_status,
|
|
record_count=incoming.record_count, notes=incoming.notes, source_id=incoming.source_id;
|
|
""")
|
|
|
|
control_meta = {
|
|
"source_type": "official_republication_html",
|
|
"publisher": "甘肃省教育考试院微信公众号（澎湃新闻转载）",
|
|
"title": "甘肃高考分数线出炉：普通类本科历史类421、物理类370，特殊类型历史类502、物理类488",
|
|
"landing_page_url": "https://www.thepaper.cn/newsDetail_forward_27849388",
|
|
"published_on": "2024-06-25", "notes": "公开转载甘肃省教育考试院发布内容；高职专科普通类控制线为160分。",
|
|
}
|
|
statements.append(source_upsert(control_meta, file_digest(control), control))
|
|
lines = [
|
|
("历史类", "特殊类型", 502), ("历史类", "本科", 421), ("历史类", "高职专科", 160),
|
|
("物理类", "特殊类型", 488), ("物理类", "本科", 370), ("物理类", "高职专科", 160),
|
|
]
|
|
values = ["(" + ",".join([
|
|
sql_string(PROVINCE), str(YEAR), sql_string("普通类"), sql_string(subject),
|
|
sql_string(level), str(score), "NULL", sql_string("普通类文化课控制线"), "@source_id"
|
|
]) + ")" for subject, level, score in lines]
|
|
statements.append("""
|
|
INSERT INTO control_score_lines (
|
|
province_name, admission_year, admission_category, subject_category,
|
|
education_level, culture_score, professional_score, notes, source_id
|
|
) VALUES
|
|
""" + ",\n".join(values) + """
|
|
AS incoming ON DUPLICATE KEY UPDATE culture_score=incoming.culture_score,
|
|
professional_score=incoming.professional_score, notes=incoming.notes,
|
|
source_id=incoming.source_id;
|
|
""")
|
|
statements.append(f"""
|
|
INSERT INTO dataset_coverage (
|
|
dataset_name, province_name, admission_year, admission_category,
|
|
subject_category, segment_code, coverage_status, record_count, notes, source_id
|
|
) VALUES ('普通类控制线', '{PROVINCE}', {YEAR}, '普通类', '', '',
|
|
'complete_public_snapshot', 6, '物理类、历史类各含特殊类型、本科和高职专科控制线。', @source_id)
|
|
AS incoming ON DUPLICATE KEY UPDATE coverage_status=incoming.coverage_status,
|
|
record_count=incoming.record_count, notes=incoming.notes, source_id=incoming.source_id;
|
|
""")
|
|
return statements
|
|
|
|
|
|
def mysql_run(args, sql):
|
|
if not os.environ.get("MYSQL_PWD"):
|
|
raise RuntimeError("必须通过 MYSQL_PWD 环境变量提供数据库密码")
|
|
process = subprocess.run([
|
|
args.mysql_bin, "--protocol=TCP", "-h", args.host, "-P", str(args.port),
|
|
"-u", args.user, "--default-character-set=utf8mb4",
|
|
], input=sql, text=True, encoding="utf-8", capture_output=True)
|
|
if process.returncode:
|
|
raise RuntimeError(process.stderr.strip() or process.stdout.strip())
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="导入甘肃省2024年公开招录数据")
|
|
parser.add_argument("--mysql-bin", required=True)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--port", type=int, default=3306)
|
|
parser.add_argument("--user", default="root")
|
|
parser.add_argument("--database", default="zhiyuantong")
|
|
parser.add_argument("--root", default=str(Path(__file__).resolve().parents[1]))
|
|
args = parser.parse_args()
|
|
root = Path(args.root)
|
|
raw = root / "data/raw/gansu/2024"
|
|
csv_dir = root / "data/processed/gansu/2024/csv"
|
|
|
|
source_specs = {
|
|
"cutoff_bachelor_c_history": {
|
|
"source_type": "official_xls", "publisher": "甘肃省教育考试院",
|
|
"title": "2024年甘肃省普通高校招生本科批（C段）物理类平行志愿投档最低分",
|
|
"landing_page_url": "https://www.ganseea.cn/shouyegonggao/1181.html",
|
|
"file_url": "https://www.ganseea.cn/uploads/allimg/20240719/11-240G9113416231.xls",
|
|
"published_on": "2024-07-19", "notes": "官方附件；本地文件名保留下载识别名，表内标题确认为物理类。",
|
|
},
|
|
"cutoff_bachelor_c_physics": {
|
|
"source_type": "official_xls", "publisher": "甘肃省教育考试院",
|
|
"title": "2024年甘肃省普通高校招生本科批（C段）历史类平行志愿投档最低分",
|
|
"landing_page_url": "https://www.ganseea.cn/shouyegonggao/1181.html",
|
|
"file_url": "https://www.ganseea.cn/uploads/allimg/20240811/11-240Q1105311438.xls",
|
|
"published_on": "2024-07-19", "notes": "官方附件；本地文件名保留下载识别名，表内标题确认为历史类。",
|
|
},
|
|
"cutoff_vocational_f_1": {
|
|
"source_type": "official_xls", "publisher": "甘肃省教育考试院",
|
|
"title": "2024年甘肃省普通高校招生高职（专科）批（F段）物理类平行志愿投档最低分",
|
|
"landing_page_url": "https://www.ganseea.cn/shouyegonggao/1198.html",
|
|
"file_url": "https://www.ganseea.cn/uploads/allimg/20240808/27-240PQ5223b96.xls",
|
|
"published_on": "2024-08-08", "notes": "官方附件。",
|
|
},
|
|
"cutoff_vocational_f_2": {
|
|
"source_type": "official_xls", "publisher": "甘肃省教育考试院",
|
|
"title": "2024年甘肃省普通高校招生高职（专科）批（F段）历史类平行志愿投档最低分",
|
|
"landing_page_url": "https://www.ganseea.cn/shouyegonggao/1198.html",
|
|
"file_url": "https://www.ganseea.cn/uploads/allimg/20240808/27-240PQ52250120.xls",
|
|
"published_on": "2024-08-08", "notes": "官方附件。",
|
|
},
|
|
"plan_bachelor_c_first_solicitation": {
|
|
"source_type": "official_xls", "publisher": "甘肃省教育考试院",
|
|
"title": "普通类本科批C段第一次征集志愿剩余计划统计表",
|
|
"landing_page_url": "https://www.ganseea.cn/shouyegonggao/1186.html",
|
|
"file_url": "https://www.ganseea.cn/uploads/allimg/20240722/27-240H216345JO.XLS",
|
|
"published_on": "2024-07-22", "notes": "仅为第一次征集时的剩余计划。",
|
|
},
|
|
"plan_segments_dei_solicitation": {
|
|
"source_type": "official_xls", "publisher": "甘肃省教育考试院",
|
|
"title": "普通类高职（专科）批E段、艺体类高职（专科）批I段征集志愿剩余计划统计表",
|
|
"landing_page_url": "https://www.ganseea.cn/shouyegonggao/1196.html",
|
|
"file_url": "https://www.ganseea.cn/uploads/allimg/20240803/27-240P31G3015Q.XLS",
|
|
"published_on": "2024-08-03", "notes": "第6号征集志愿公告附件；D段计划已完成，附件实际含E、I段。",
|
|
},
|
|
}
|
|
|
|
statements = ["SET NAMES utf8mb4;", "SET time_zone='+08:00';", f"USE `{args.database}`;", "START TRANSACTION;"]
|
|
statements += build_reference_sql(root)
|
|
score_meta = {
|
|
"source_type": "official_html", "publisher": "甘肃省教育考试院",
|
|
"title": "2024年甘肃普通高等学校招生考试成绩分段表",
|
|
"landing_page_url": "https://www.ganseea.cn/gaokaogaozhao/1157.html",
|
|
"published_on": "2024-06-25", "notes": "普通类物理、历史一分一段累计人数表。",
|
|
}
|
|
score_sql, score_count = build_score_rank_sql(raw / "score_rank_2024.html", score_meta)
|
|
statements += score_sql
|
|
|
|
cutoff_jobs = [
|
|
("cutoff_bachelor_c_history", "物理类", "本科", "C"),
|
|
("cutoff_bachelor_c_physics", "历史类", "本科", "C"),
|
|
("cutoff_vocational_f_1", "物理类", "高职专科", "F"),
|
|
("cutoff_vocational_f_2", "历史类", "高职专科", "F"),
|
|
]
|
|
cutoff_count = 0
|
|
for name, subject, level, segment in cutoff_jobs:
|
|
sql, count = build_cutoff_sql(csv_dir / f"{name}.csv", raw / f"{name}.xls", source_specs[name], subject, level, segment)
|
|
statements += sql
|
|
cutoff_count += count
|
|
|
|
plan_count = group_count = 0
|
|
for name, round_name in (("plan_bachelor_c_first_solicitation", "第一次征集"), ("plan_segments_dei_solicitation", "第六号公告征集")):
|
|
sql, rows, groups = build_plan_sql(csv_dir / f"{name}.csv", raw / f"{name}.xls", source_specs[name], round_name)
|
|
statements += sql
|
|
plan_count += rows
|
|
group_count += groups
|
|
|
|
statements.append("""
|
|
INSERT INTO school_external_codes (
|
|
school_id, code_system, external_code, province_name, admission_year,
|
|
admission_batch, admission_type, source_id
|
|
)
|
|
SELECT mapped.school_id, 'gansu_admission_school_code', mapped.source_school_code,
|
|
'甘肃省', 2024, '', '', MIN(mapped.source_id)
|
|
FROM (
|
|
SELECT school_id, source_school_code, source_id FROM admission_cutoff_lines
|
|
WHERE province_name='甘肃省' AND admission_year=2024 AND school_id IS NOT NULL
|
|
UNION ALL
|
|
SELECT school_id, source_school_code, source_id FROM admission_groups
|
|
WHERE province_name='甘肃省' AND admission_year=2024 AND school_id IS NOT NULL
|
|
) mapped
|
|
GROUP BY mapped.school_id, mapped.source_school_code
|
|
ON DUPLICATE KEY UPDATE school_id=VALUES(school_id), source_id=VALUES(source_id);
|
|
""")
|
|
statements += ["COMMIT;"]
|
|
mysql_run(args, "\n".join(statements))
|
|
print(f"导入完成: 一分一段 {score_count} 条，投档线 {cutoff_count} 条，专业计划 {plan_count} 条，院校专业组 {group_count} 条")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|