Support crawling complete dossier with more than 1000 results

[mastaal]

parlhist/parlhistnl/crawler/kamerdossier.py

Lines 21 to 43 in e59edcd

	def get_kamerstukken_in_kamerstukdossier(dossiernummer: str) -> list[tuple[str, str]]:
	"""Using the SRU api, get all kamerstukken in a kamerstukdossier
	Returns strings in the same formatting as denoted in other metadata, e.g. '35899;7', '35925-VII;31' or '35979;F'
	"""
	# overheid SRU documentation:
	# https://data.overheid.nl/sites/default/files/dataset/d0cca537-44ea-48cf-9880-fa21e1a7058f/resources/Handleiding%2BSRU%2B2.0.pdf
	base_query = f"https://repository.overheid.nl/sru?query=(w.dossiernummer=={dossiernummer})&maximumRecords=1000&startRecord=1"
	try:
	query_response = get_url_or_error(base_query)
	except CrawlerException as exc:
	logger.error("Received exception %s when trying to query the kamerstukken in dossier %s", exc, dossiernummer)
	raise CrawlerException(f"Received exception when trying to query the kamerstukken in dossier {dossiernummer}") from exc
	xml: ET.Element = ET.fromstring(query_response.text)
	number_of_records = int(xml.find("{http://docs.oasis-open.org/ns/search-ws/sruResponse}numberOfRecords").text)
	if number_of_records > 1000:
	logger.critical("More than 1000 results in a kamerdossier!")
	# TODO, implement this case