From 681c1678e4e8a75fbb08949eb00183354090b768 Mon Sep 17 00:00:00 2001 From: Niklas Mertsch Date: Sun, 25 May 2025 16:55:23 +0200 Subject: [PATCH 1/9] Remove outdated query '?questions=all' --- src/download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/download.py b/src/download.py index 847f751..e4350c1 100644 --- a/src/download.py +++ b/src/download.py @@ -29,8 +29,8 @@ # Build resource list dynamically based on exclusions resources = [ - "submissions?questions=all&state=confirmed", - "speakers?questions=all", + "submissions?state=confirmed", + "speakers", ] if "youtube" not in exclude: From acb289940f29489e6f61d9708d5d1c19cc5f8a01 Mon Sep 17 00:00:00 2001 From: Niklas Mertsch Date: Sun, 25 May 2025 16:56:03 +0200 Subject: [PATCH 2/9] Use '?expand=...' to request nested resources --- src/download.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/download.py b/src/download.py index e4350c1..810d1c3 100644 --- a/src/download.py +++ b/src/download.py @@ -25,12 +25,16 @@ } base_url = f"https://pretalx.com/api/events/{Config.event}/" -schedule_url = base_url + "schedules/latest/" +schedule_url = ( + base_url + + "schedules/latest?expand=" + + "slots,slots.submission,slots.submission.submission_type,slots.submission.track,slots.room" +) # Build resource list dynamically based on exclusions resources = [ - "submissions?state=confirmed", - "speakers", + "submissions?state=confirmed&expand=answers.question,submission_type,track,slots.room", + "speakers?expand=answers.question", ] if "youtube" not in exclude: From 13349970eda1d2f3b28d0ab5c0a4034684a0645d Mon Sep 17 00:00:00 2001 From: Niklas Mertsch Date: Sun, 25 May 2025 16:57:27 +0200 Subject: [PATCH 3/9] Deserialization: speaker.avatar -> speaker.avatar_url --- data/examples/pretalx/speakers.json | 2 +- src/models/pretalx.py | 2 +- src/utils/transform.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/examples/pretalx/speakers.json b/data/examples/pretalx/speakers.json index fa3e0ff..b5f5161 100644 --- a/data/examples/pretalx/speakers.json +++ b/data/examples/pretalx/speakers.json @@ -6,7 +6,7 @@ "submissions": [ "A8CD3F" ], - "avatar": "https://pretalx.com/media/avatars/picture.jpg", + "avatar_url": "https://pretalx.com/media/avatars/picture.jpg", "answers": [ { "id": 272244, diff --git a/src/models/pretalx.py b/src/models/pretalx.py index 0a3cce5..d6adae7 100644 --- a/src/models/pretalx.py +++ b/src/models/pretalx.py @@ -45,7 +45,7 @@ class PretalxSpeaker(BaseModel): code: str name: str biography: str | None = None - avatar: str + avatar_url: str submissions: list[str] answers: list[PretalxAnswer] diff --git a/src/utils/transform.py b/src/utils/transform.py index ac7d6d9..066e43b 100644 --- a/src/utils/transform.py +++ b/src/utils/transform.py @@ -83,7 +83,7 @@ def pretalx_speakers_to_europython_speakers( code=speaker.code, name=speaker.name, biography=speaker.biography, - avatar=speaker.avatar, + avatar=speaker.avatar_url, slug=speaker_code_to_slug[speaker.code], answers=speaker.answers, submissions=speaker.submissions, From 1b708dd669399d7006bfcef706b49ee31a1c9255 Mon Sep 17 00:00:00 2001 From: Niklas Mertsch Date: Sun, 25 May 2025 16:58:31 +0200 Subject: [PATCH 4/9] Deserialization: Localization 'en' -> 'name.en' --- data/examples/pretalx/submissions.json | 8 ++++++-- src/models/pretalx.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/data/examples/pretalx/submissions.json b/data/examples/pretalx/submissions.json index d3b2759..3351420 100644 --- a/data/examples/pretalx/submissions.json +++ b/data/examples/pretalx/submissions.json @@ -21,7 +21,9 @@ "submission_type": "Talk (long session)", "submission_type_id": 3961, "track": { - "en": "Software Engineering & Architecture" + "name": { + "en": "Software Engineering & Architecture" + } }, "track_id": 4493, "state": "confirmed", @@ -137,7 +139,9 @@ "submission_type": "Talk", "submission_type_id": 3961, "track": { - "en": "PyData: LLMs" + "name": { + "en": "PyData: LLMs" + } }, "track_id": 4493, "state": "confirmed", diff --git a/src/models/pretalx.py b/src/models/pretalx.py index d6adae7..b59646c 100644 --- a/src/models/pretalx.py +++ b/src/models/pretalx.py @@ -33,7 +33,7 @@ class PretalxSlot(BaseModel): @classmethod def handle_localized(cls, v) -> str | None: if isinstance(v, dict): - return v.get("en") + return v["name"].get("en") return v @@ -77,7 +77,7 @@ class PretalxSubmission(BaseModel): @classmethod def handle_localized(cls, v) -> str | None: if isinstance(v, dict): - return v.get("en") + return v["name"].get("en") return v @field_validator("duration", mode="before") From a3e667f01c6b83092254f86cf1e4a4f8711fdd4c Mon Sep 17 00:00:00 2001 From: Niklas Mertsch Date: Sun, 25 May 2025 16:59:23 +0200 Subject: [PATCH 5/9] Deserialization: Schedule slots --- src/models/pretalx.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/models/pretalx.py b/src/models/pretalx.py index b59646c..da35e72 100644 --- a/src/models/pretalx.py +++ b/src/models/pretalx.py @@ -98,8 +98,9 @@ def process_values(cls, values) -> dict: values["speakers"] = sorted([s["code"] for s in values["speakers"]]) # Set slot information - if values.get("slot"): - slot = PretalxSlot.model_validate(values["slot"]) + if values.get("slots"): + slot = PretalxSlot.model_validate(values["slots"][0]) + values["slot"] = slot values["room"] = slot.room values["start"] = slot.start values["end"] = slot.end @@ -146,3 +147,28 @@ class PretalxSchedule(BaseModel): slots: list[PretalxSubmission] breaks: list[PretalxScheduleBreak] + + @model_validator(mode="before") + @classmethod + def process_values(cls, values) -> dict: + submission_slots = [] + break_slots = [] + for slot_dict in values["slots"]: + # extract nested slot fields into slot + slot_object = PretalxSlot.model_validate(slot_dict) + slot_dict["slot"] = slot_object + slot_dict["room"] = slot_object.room + slot_dict["start"] = slot_object.start + slot_dict["end"] = slot_object.end + + if slot_dict.get("submission") is None: + break_slots.append(slot_dict) + else: + # merge submission fields into slot + slot_dict.update(slot_dict.get("submission", {})) + + submission_slots.append(slot_dict) + + values["slots"] = submission_slots + values["breaks"] = break_slots + return values From 2eb8b2c946a4331d55296e7eca16414f541b4272 Mon Sep 17 00:00:00 2001 From: Niklas Mertsch Date: Sun, 25 May 2025 16:59:46 +0200 Subject: [PATCH 6/9] Deserialization: Don't expect embedded submission.speakers --- data/examples/pretalx/submissions.json | 24 +++--------------------- src/models/pretalx.py | 2 -- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/data/examples/pretalx/submissions.json b/data/examples/pretalx/submissions.json index 3351420..ac4d835 100644 --- a/data/examples/pretalx/submissions.json +++ b/data/examples/pretalx/submissions.json @@ -2,20 +2,8 @@ { "code": "A8CD3F", "speakers": [ - { - "code": "F3DC8A", - "name": "A Speaker", - "biography": "This is a biography of F3D speaker", - "avatar": "https://pretalx.com/media/avatars/picture.jpg", - "email": "f3dc8a@example.com" - }, - { - "code": "ZXCVBN", - "name": "ZXC Speaker", - "biography": "This is a biography of ZXC speaker", - "avatar": "https://pretalx.com/media/avatars/picture.jpg", - "email": "zxcvbn@example.com" - } + "F3DC8A", + "ZXCVBN" ], "title": "This is a test talk from a test speaker about a test topic.", "submission_type": "Talk (long session)", @@ -127,13 +115,7 @@ { "code": "B8CD4F", "speakers": [ - { - "code": "G3DC8A", - "name": "Another Speaker", - "biography": "This is a biography of F3D speaker", - "avatar": "https://pretalx.com/media/avatars/picture.jpg", - "email": "g3dc8a@example.com" - } + "G3DC8A" ], "title": "A talk with shorter title", "submission_type": "Talk", diff --git a/src/models/pretalx.py b/src/models/pretalx.py index da35e72..b334922 100644 --- a/src/models/pretalx.py +++ b/src/models/pretalx.py @@ -95,8 +95,6 @@ def handle_resources(cls, v) -> list[dict[str, str]] | None: @model_validator(mode="before") @classmethod def process_values(cls, values) -> dict: - values["speakers"] = sorted([s["code"] for s in values["speakers"]]) - # Set slot information if values.get("slots"): slot = PretalxSlot.model_validate(values["slots"][0]) From 27c18923ee945bb69bf5248ed92761d263874b1a Mon Sep 17 00:00:00 2001 From: Niklas Mertsch Date: Sun, 25 May 2025 17:00:25 +0200 Subject: [PATCH 7/9] Deserialization: Handle non-expanded submission.resources Workaround for https://github.com/pretalx/pretalx/issues/2040 --- src/models/pretalx.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/models/pretalx.py b/src/models/pretalx.py index b334922..a0db4a9 100644 --- a/src/models/pretalx.py +++ b/src/models/pretalx.py @@ -90,6 +90,10 @@ def duration_to_string(cls, v) -> str: @field_validator("resources", mode="before") @classmethod def handle_resources(cls, v) -> list[dict[str, str]] | None: + if v and all(isinstance(res, int) for res in v): + # currently, ?expand=resources is broken in Pretalx + # https://github.com/pretalx/pretalx/issues/2040 + return None return v or None @model_validator(mode="before") From 628090d9402d87f74b4203dffe976d47df34ccc3 Mon Sep 17 00:00:00 2001 From: Niklas Mertsch Date: Sun, 25 May 2025 17:00:35 +0200 Subject: [PATCH 8/9] Chore: Fix typing --- src/utils/parse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/parse.py b/src/utils/parse.py index cfe53e4..efb6289 100644 --- a/src/utils/parse.py +++ b/src/utils/parse.py @@ -34,12 +34,12 @@ def publishable_speakers( js = json.load(fd) all_speakers = [PretalxSpeaker.model_validate(s) for s in js] - speakers_with_publishable_sessions: list[PretalxSubmission] = [] + speakers_with_publishable_sessions: list[PretalxSpeaker] = [] for speaker in all_speakers: if publishable_sessions := Utils.publishable_sessions_of_speaker( speaker, publishable_sessions_keys ): - speaker.submissions = publishable_sessions + speaker.submissions = sorted(publishable_sessions) speakers_with_publishable_sessions.append(speaker) publishable_speakers_by_code = { From d462963af487aa74374b1e0a7958dc2fd9d1c567 Mon Sep 17 00:00:00 2001 From: Niklas Mertsch Date: Sun, 25 May 2025 19:13:57 +0200 Subject: [PATCH 9/9] Include submission resources in session export --- src/download.py | 2 +- src/models/pretalx.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/download.py b/src/download.py index 810d1c3..4eab58a 100644 --- a/src/download.py +++ b/src/download.py @@ -33,7 +33,7 @@ # Build resource list dynamically based on exclusions resources = [ - "submissions?state=confirmed&expand=answers.question,submission_type,track,slots.room", + "submissions?state=confirmed&expand=answers.question,submission_type,track,slots.room,resources", "speakers?expand=answers.question", ] diff --git a/src/models/pretalx.py b/src/models/pretalx.py index a0db4a9..a925930 100644 --- a/src/models/pretalx.py +++ b/src/models/pretalx.py @@ -90,15 +90,19 @@ def duration_to_string(cls, v) -> str: @field_validator("resources", mode="before") @classmethod def handle_resources(cls, v) -> list[dict[str, str]] | None: - if v and all(isinstance(res, int) for res in v): - # currently, ?expand=resources is broken in Pretalx - # https://github.com/pretalx/pretalx/issues/2040 - return None return v or None @model_validator(mode="before") @classmethod def process_values(cls, values) -> dict: + # Transform resource information + if raw_resources := values.get("resources"): + resources = [ + {"description": res["description"], "resource": res["resource"]} + for res in raw_resources + ] + values["resources"] = resources + # Set slot information if values.get("slots"): slot = PretalxSlot.model_validate(values["slots"][0]) @@ -169,6 +173,9 @@ def process_values(cls, values) -> dict: # merge submission fields into slot slot_dict.update(slot_dict.get("submission", {})) + # remove resource IDs (not expandable with API, not required for schedule) + slot_dict.pop("resources", None) + submission_slots.append(slot_dict) values["slots"] = submission_slots