From c1e89bfb93bf10e0b29f9c0e9aaeeca3f368b9d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cruz=20N=C3=BA=C3=B1ez?= Date: Fri, 8 Nov 2024 13:45:42 -0500 Subject: [PATCH 1/5] add cinoh civil service commission spider --- city_scrapers/spiders/cinoh_Civil_Service.py | 80 +++++++++++++++++++ tests/files/cinoh_Civil_Service.json | 1 + tests/test_cinoh_Civil_Service.py | 81 ++++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 city_scrapers/spiders/cinoh_Civil_Service.py create mode 100644 tests/files/cinoh_Civil_Service.json create mode 100644 tests/test_cinoh_Civil_Service.py diff --git a/city_scrapers/spiders/cinoh_Civil_Service.py b/city_scrapers/spiders/cinoh_Civil_Service.py new file mode 100644 index 0000000..77e127e --- /dev/null +++ b/city_scrapers/spiders/cinoh_Civil_Service.py @@ -0,0 +1,80 @@ +from city_scrapers_core.constants import COMMISSION +from city_scrapers_core.items import Meeting +from city_scrapers_core.spiders import CityScrapersSpider +import datetime +from dateutil.parser import parse +import scrapy + + +class CinohCivilServiceSpider(CityScrapersSpider): + name = "cinoh_Civil_Service" + agency = "Cincinnati Civil Service Commission" + timezone = "America/Chicago" + committee_id = 'A9HCN931D6BA' + custom_settings = { + "ROBOTSTXT_OBEY": False, + } + + # original URL: https://go.boarddocs.com/oh/csc/Board.nsf/vpublic?open + # clicking on meetings tab takes you to meetings index and uses API + # we scrape API instead via POST request and ignore robots file + def start_requests(self): + url = 'https://go.boarddocs.com/oh/csc/Board.nsf/BD-GetMeetingsList' + form_data = { + 'current_committee_id': self.committee_id + } + # Sending the POST request + yield scrapy.FormRequest(url, formdata=form_data, callback=self.parse) + + def parse(self, response): + """ + Parse JSON response. + """ + + year = str(datetime.datetime.today().year) + + data = response.json() + + + for item in data: + numb = item.get("numberdate") + + # skip iteration if meeting is not for current year + if numb is None or year not in numb: + continue + + # if we are in current year then parse meeting + meeting = Meeting( + title=item["name"], + description="", + classification=COMMISSION, + start=parse(numb), + end=None, + all_day=False, + time_notes="", + location=self._parse_location(item), + links=self._parse_links(item), + source=self._parse_source(response), + ) + + meeting["status"] = self._get_status(meeting) + meeting["id"] = self._get_id(meeting) + + yield meeting + + def _parse_location(self, item): + """Parse or generate location.""" + return { + "name": "Cincinnati Civil Service Commission", + "address": "805 Central Ave, Suite 200, Cincinnati, OH 45202", + } + + def _parse_links(self, item): + """Parse or generate links.""" + uniq = item["unique"] + href = f"https://go.boarddocs.com/oh/csc/Board.nsf/Download-AgendaDetailed?open&id={uniq}¤t_committee_id={self.committee_id}" + return [{"title": "Agenda", "href": href}] + + def _parse_source(self, response): + """Generate source. Instead of returning API URL we return the more user-friendly web page we can see this data from.""" + return "https://go.boarddocs.com/oh/csc/Board.nsf/vpublic?open#tab-meetings" diff --git a/tests/files/cinoh_Civil_Service.json b/tests/files/cinoh_Civil_Service.json new file mode 100644 index 0000000..95b4d89 --- /dev/null +++ b/tests/files/cinoh_Civil_Service.json @@ -0,0 +1 @@ +[{"unique": "CZQLFH5631AD","name": "November 7, 2024 Civil Service Commission","current": "1","preliveoak": "","numberdate": "20241107","unid": "201D7C63A07B467885258AAC0056584B"},{"unique": "D9URJQ6E42DA","name": "Civil Service Commission Special Meeting","current": "","preliveoak": "","numberdate": "20241022","unid": "1C1833F85890632985258BB0006E4D8C"},{"unique": "CYWRT36F7B82","name": "October 17, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20241017","unid": "F3F0C589EF5A528085258A92006FA5CE"},{"unique": "CYWRSK6F68D1","name": "October 3, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20241003","unid": "905E508BE495745385258A92006F74A3"},{"unique": "CYWREE6DA0CE","name": "September 19, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240919","unid": "B2BF511B740B8D1C85258A92006DA962"},{"unique": "CYWRBN6D39CD","name": "August 15, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240815","unid": "3D209397C86404F585258A92006D458F"},{"unique": "CYWR8H6C0B24","name": "August 1, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240801","unid": "D420827DF775FD7885258A92006C197A"},{"unique": "CYWR8C6BF675","name": "July 18, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240718","unid": "BF3479F4755A348785258A92006C04B9"},{"unique": "CYWPUP664259","name": "June 27, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240627","unid": "2A86051BC0CD679C85258A920066446F"},{"unique": "D52Q6K610B68","name": "June 25, 2024 Civil Service Commission Special Meeting","current": "","preliveoak": "","numberdate": "20240625","unid": "826AB2D99C05023A85258B1600611E2C"},{"unique": "CY4S7U712BD9","name": "June 6, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240606","unid": "D90DE395C2706DFA85258A78007195CE"},{"unique": "CY4QVU6AAD7C","name": "May 16, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240516","unid": "59D8004E8DFDA57785258A78006AB79B"},{"unique": "CY4PHR64BAEB","name": "May 2, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240502","unid": "B6F2DB9976CD0DD685258A780064C39F"},{"unique": "CY4NCE5EDEED","name": "April 18, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240418","unid": "8E1BA6BE8919FE4C85258A78005F0968"},{"unique": "CY4M87596959","name": "April 4, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240404","unid": "AC12E4ECC905489085258A780059A93D"},{"unique": "CY2PBR635915","name": "March 21, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240321","unid": "34EA4F91F937BC9385258A760063F254"},{"unique": "CY2NJJ600766","name": "March 7, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240307","unid": "DD70A78276A5A00485258A76006012E8"},{"unique": "CY2LEC562E4B","name": "February 15, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240215","unid": "F2C5F261F5FE00AD85258A7600565C24"},{"unique": "CXTLU65833D0","name": "February 1, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240201","unid": "8AFF546799207B3F85258A6F00584096"},{"unique": "CZCQLR68A691","name": "January 16, 2024 Civil Service Commission Special Meeting","current": "","preliveoak": "","numberdate": "20240116","unid": "6996EB3301416E0685258AA00068C05B"},{"unique": "CXTKT3535AC1","name": "January 11, 2024 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20240111","unid": "802FBB4C75312B6385258A6F00536332"},{"unique": "CGQVWZ817E22","name": "December 21, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20231221","unid": "7D9D70C8AFF555128525888C00817E22"},{"unique": "CGQVYD818282","name": "December 7, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20231207","unid": "A7E14F5E6738C9F68525888C00818282"},{"unique": "CGQVZR818727","name": "November 16, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20231116","unid": "1DFCA99A0C5D96F48525888C00818727"},{"unique": "CGQW34818A0C","name": "November 2, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20231102","unid": "3F4CF527EFE758E58525888C00818A0C"},{"unique": "CGQW4F818CD8","name": "October 19, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20231019","unid": "1C8D891EB3B201DA8525888C00818CD8"},{"unique": "CGQW5S819037","name": "October 5, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20231005","unid": "798D9C6B1877E5738525888C00819037"},{"unique": "CGR2358192F1","name": "September 21, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230921","unid": "C36F75E2511B40CF8525888C008192F1"},{"unique": "CGR24G819583","name": "August 17, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230817","unid": "BAE4A3BC7440B24F8525888C00819583"},{"unique": "CGR25T819891","name": "August 3, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230803","unid": "507FC017BFEA53F28525888C00819891"},{"unique": "CGR276819B41","name": "July 20, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230720","unid": "C08F154E2482E3F88525888C00819B41"},{"unique": "CGR29V819FAE","name": "June 15, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230615","unid": "7123B93C57968F5A8525888C00819FAE"},{"unique": "CGR2BB81A1B7","name": "May 18, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230518","unid": "F67E7E54218DE2958525888C0081A1B7"},{"unique": "CGR2CN81A3CB","name": "May 4, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230504","unid": "B88B41F8EAE0896A8525888C0081A3CB"},{"unique": "CGR2DZ81A5C5","name": "April 20, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230420","unid": "4FC2D5BF67654CAF8525888C0081A5C5"},{"unique": "CQLPTX653F31","name": "April 18, 2023 Civil Service Commission Special Meeting","current": "","preliveoak": "","numberdate": "20230418","unid": "AB9894B8FF0DC4428525898800655E94"},{"unique": "CGR2FC81A7CD","name": "April 6, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230406","unid": "527BA86B2926FABE8525888C0081A7CD"},{"unique": "CGQVVK8164E0","name": "March 16, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230316","unid": "2B73EEE46276BCA58525888C008164E0"},{"unique": "CGQVU6815632","name": "March 2, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230302","unid": "251BDD1C363A472B8525888C00815632"},{"unique": "CNPRBL6D3730","name": "February 23, 2023 Civil Service Commission Special Meeting","current": "","preliveoak": "","numberdate": "20230223","unid": "7C4F2A3115A8FDD58525894B006D3730"},{"unique": "CGQVSR81443B","name": "February 16, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230216","unid": "3AB54F327E3F9FEA8525888C0081443B"},{"unique": "CGQVRB813098","name": "February 2, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230202","unid": "4CE4BB1FB749EFB38525888C00813098"},{"unique": "CN8NRV613B78","name": "January 31, 2023 Civil Service Commission Special Meeting","current": "","preliveoak": "","numberdate": "20230131","unid": "9750050E7A066AC48525893C00614D69"},{"unique": "CGQVPU811E09","name": "January 12, 2023 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20230112","unid": "0897EA982067E7118525888C00811E09"},{"unique": "CGQVHW80BC25","name": "December 15, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20221215","unid": "A17373A2C2F39BCB8525888C0080BC25"},{"unique": "CGQVK980BF79","name": "December 1, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20221201","unid": "DA0849C4625CA49E8525888C0080BF79"},{"unique": "CGQVLM80C32C","name": "November 17, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20221117","unid": "7A2A01452124DD9F8525888C0080C32C"},{"unique": "CGQVMY80C5A9","name": "November 3, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20221103","unid": "C67A0C97E6EA69978525888C0080C5A9"},{"unique": "CGQVGJ80B09A","name": "October 20, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20221020","unid": "1C8D5981924EE6B88525888C0080B09A"},{"unique": "CGQVDD803996","name": "October 6, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20221006","unid": "47A7AC67947097368525888C00803996"},{"unique": "C9CPF662452A","name": "September 15, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220915","unid": "F83E26444FC8C7A7852587A00062452A"},{"unique": "C9CPG5624673","name": "September 1, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220901","unid": "5A97D9A3653862DE852587A000624673"},{"unique": "C9CP7C623792","name": "August 18, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220818","unid": "DD6E7780D54C7838852587A000623792"},{"unique": "C9CP4861DDC6","name": "August 4, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220804","unid": "75633CDACF7355A4852587A00061DDC6"},{"unique": "C9CPST612B0D","name": "July 21, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220721","unid": "9E236922A38B4599852587A000612B0D"},{"unique": "C9CPTY6132B2","name": "July 7, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220707","unid": "89B6454A045A7DB7852587A0006132B2"},{"unique": "C9CPRS611630","name": "June 16, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220616","unid": "B8B373F5F62C356A852587A000611630"},{"unique": "CE6JVX4E8265","name": "June 2, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220602","unid": "B80D6E24AF1E2FCD8525883A004E8265"},{"unique": "C9CPNF60C534","name": "May 19, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220519","unid": "1965B516A14E8409852587A00060C534"},{"unique": "C9CPM660BA0D","name": "May 5, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220505","unid": "95016A025379B4AB852587A00060BA0D"},{"unique": "C9CPKA609F53","name": "April 21, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220421","unid": "318B7DFC6047A499852587A000609F53"},{"unique": "C9CPHZ609017","name": "April 7, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220407","unid": "7861C27E687ADC04852587A000609017"},{"unique": "C9CPGC607D48","name": "March 17, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220317","unid": "5DD451961B3FFEAF852587A000607D48"},{"unique": "C9CP5H606D35","name": "March 3, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220303","unid": "8D238A73EFC155BE852587A000606D35"},{"unique": "C9CP4D6061A0","name": "February 17, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220217","unid": "0B4F9F2EFD1D3250852587A0006061A0"},{"unique": "C9CP2J603C6F","name": "February 10, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220210","unid": "4BF25322F3E45D9C852587A000603C6F"},{"unique": "C9CNYF601B0F","name": "January 13, 2022 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20220113","unid": "26935AD4B1F641C7852587A000601B0F"},{"unique": "C28T7Z7224B0","name": "December 16, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20211216","unid": "48D9A9C1D7F615AD852586BC007224B0"},{"unique": "C28T987226D8","name": "December 2, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20211202","unid": "0D83958DCD0E962D852586BC007226D8"},{"unique": "C28TAH7228F7","name": "November 18, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20211118","unid": "842535686217C332852586BC007228F7"},{"unique": "C28T2P71EA58","name": "November 4, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20211104","unid": "8745C66D67C80BBA852586BC0071EA58"},{"unique": "C28T5671EE8D","name": "October 21, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20211021","unid": "9650A27EB63751C6852586BC0071EE8D"},{"unique": "C28T3W71EC1B","name": "October 7, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20211007","unid": "7378F8DF1119153B852586BC0071EC1B"},{"unique": "C28T6D71F046","name": "September 16, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210916","unid": "C600B1350A76A7C7852586BC0071F046"},{"unique": "C28SX271BBA8","name": "September 2, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210902","unid": "32B5846A81D74594852586BC0071BBA8"},{"unique": "C28SYB71BF93","name": "August 19, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210819","unid": "4B82345E0D1E6C21852586BC0071BF93"},{"unique": "C28SQE712ED1","name": "August 5, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210805","unid": "66437D443C872845852586BC00712ED1"},{"unique": "C28SSX7141B4","name": "July 15, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210715","unid": "01346E8BF287E51A852586BC007141B4"},{"unique": "C28SUS716CCC","name": "July 1, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210701","unid": "5EE57C807511522C852586BC00716CCC"},{"unique": "BZKLM356EA2D","name": "June 17, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210617","unid": "FFD9BF77B837A393852586A70057CF52"},{"unique": "BZGK6N505DA3","name": "May 20, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210520","unid": "63E34E8906B302AD852586A400506E57"},{"unique": "BZGJQR4DFE1C","name": "April 29, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210429","unid": "253C91921670FBB8852586A4004E29CD"},{"unique": "BZG3JF7033FD","name": "April 15, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210415","unid": "AA576F4F94460DFA852586A30070750F"},{"unique": "BYXMCJ594888","name": "April 1, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210401","unid": "C57419BF244C9DD6852586930059571D"},{"unique": "BY6JSQ4E9E51","name": "March 18, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210318","unid": "DDBC741680859A4B8525867A004EA7E5"},{"unique": "BY6JK64D82FB","name": "March 4, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210304","unid": "EF6F07A2B6A537B88525867A004D90F3"},{"unique": "BXH2J8010EE5","name": "February 25, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210225","unid": "113156C8D87F96838525866500011A1B"},{"unique": "BXH2CP81C4E5","name": "February 4, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210204","unid": "BE46036A45324E51852586640081D4FB"},{"unique": "BW3VA7766138","name": "January 28, 2021 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20210128","unid": "8AB48F5F35B87833852586370076725D"},{"unique": "BW3V9V763B34","name": "January 7, 2021 Civil Service Commission","current": "","preliveoak": "","numberdate": "20210107","unid": "7D404B4C0685907C852586370076581B"},{"unique": "BJ3ULZ77B2EA","name": "December 17, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20201217","unid": "792B3C5A7AA23B03852584B70077BCD0"},{"unique": "BJ3ULK779711","name": "November 24, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20201124","unid": "E8D27511DE985290852584B70077A4DD"},{"unique": "BJ3UHW76B383","name": "October 30, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20201030","unid": "BBA654FC39BBA2E8852584B70076BEB1"},{"unique": "BJ3UHP76A17A","name": "October 15, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20201015","unid": "AFA9EAB627A67822852584B70076AF08"},{"unique": "BJ3UF47624A8","name": "October 1, 2020 Civil Service Commission","current": "","preliveoak": "","numberdate": "20201001","unid": "423576F249A91EA0852584B7007631ED"},{"unique": "BJ3UBC759087","name": "September 17, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200917","unid": "F8E97D8FC7ADB5EE852584B700759C60"},{"unique": "BJ3U9L750D4A","name": "September 3, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200903","unid": "D4B9A67C6BDE7CAC852584B70075176C"},{"unique": "BJ3TZJ747441","name": "August 20, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200820","unid": "8E28B0B274ED340B852584B700747EE9"},{"unique": "BJ3TXY73F05E","name": "August 6, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200806","unid": "E53DF93845E9EFF3852584B70073FAB2"},{"unique": "BJ3TUK73667B","name": "July 16, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200716","unid": "6AFA31B190988651852584B700737039"},{"unique": "BJ3TPW72E3F2","name": "July 2, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200702","unid": "E824F84AC9C56C3D852584B70072ED3D"},{"unique": "BJ3TH6726099","name": "June 18, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200618","unid": "B0962319EF556637852584B700726AD0"},{"unique": "BJ3TDY71CEAB","name": "June 4, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200604","unid": "0E1084C96C41B5F9852584B70071D90E"},{"unique": "BJ3T6N706545","name": "May 8, 2020 (Fri) Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200508","unid": "96363953CC5DAF65852584B700706E9C"},{"unique": "BJ3RWV6BC3BF","name": "March 5, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200305","unid": "DB9517F6D70EB28F852584B7006BCE94"},{"unique": "BJ3RE86B24ED","name": "February 20, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200220","unid": "0D1A87B948826A80852584B7006B30FC"},{"unique": "BJ3RAN6A83EB","name": "February 6, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200206","unid": "98F3699C73296755852584B7006A9112"},{"unique": "BJ3QRR69B843","name": "January 23, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200123","unid": "DD983C4C3E3D8380852584B70069C838"},{"unique": "BHAPWL657D01","name": "January 9, 2020 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20200109","unid": "8ABA1B264B29BC2C8525849E00659375"},{"unique": "BC8M4S597835","name": "December 12, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20191212","unid": "36F31C1FB3D52644872583FC00598844"},{"unique": "BC8M4C5967C0","name": "December 5, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20191205","unid": "F40E6C78A3790647872583FC005974DD"},{"unique": "BC8M42595896","name": "November 21, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20191121","unid": "6D19FC2625B0D7BE872583FC0059657D"},{"unique": "BC8M3M594CA4","name": "November 7, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20191107","unid": "08138E8A29B7F4C4872583FC005955D9"},{"unique": "BC8M2F591FAC","name": "October 17, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20191017","unid": "8FD3299CC98BCEDE872583FC00592AA2"},{"unique": "BC8M26591362","name": "October 3, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20191003","unid": "7341EE6EC163B73B872583FC00591DA4"},{"unique": "BC8LYT58E0D4","name": "September 19, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190919","unid": "03DB39E6A48B3D04872583FC0058EBCC"},{"unique": "BC8LYE58D319","name": "September 5, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190905","unid": "51AF5E21D3D69170872583FC0058DE3B"},{"unique": "BC8LY658C56D","name": "August 15, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190815","unid": "95142871A5025BCD872583FC0058CFA9"},{"unique": "BC8LXQ58B077","name": "August 1, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190801","unid": "7D2EDFCAA518389F872583FC0058B8BB"},{"unique": "BC8LXD589837","name": "July 18, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190718","unid": "60A192B0D44E359A872583FC0058A977"},{"unique": "B6BSNE5E514B","name": "June 20, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190620","unid": "76E0F911463B0F088725833F005E5AC7"},{"unique": "B6BSH95DA8ED","name": "June 6, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190606","unid": "41788413F5A595AF8725833F005DB1F7"},{"unique": "B6BS6U5D19BA","name": "May 16, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190516","unid": "6F2933F1227C9A998725833F005D2254"},{"unique": "B6BS265C8530","name": "May 2, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190502","unid": "E1275B002B82CF788725833F005C8E28"},{"unique": "B6BRWZ5BFC71","name": "April 18, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190418","unid": "3AF00D9275655F368725833F005C0983"},{"unique": "B6BRNX5B61DC","name": "April 4, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190404","unid": "59A0F549A1CB8F168725833F005B6CE2"},{"unique": "B6BRC35AC35C","name": "March 21, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190321","unid": "07E42884F0ADE9B38725833F005AD877"},{"unique": "B6BQU259CA3F","name": "March 7, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190307","unid": "C52D2AE2396572418725833F0059D304"},{"unique": "B6BQNH593A26","name": "February 21, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190221","unid": "889802E4919C45B38725833F00594729"},{"unique": "B6BQKE5895F3","name": "February 7, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190207","unid": "7CCC994F644E341E8725833F0058A0BB"},{"unique": "B6BQ9S57E83A","name": "January 17, 2019 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20190117","unid": "1F754D0DF68FB2D78725833F0057FD12"},{"unique": "ATAP9R619CF1","name": "December 20, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20181220","unid": "4FADF9BC7C80FEE3872581DE0061A49B"},{"unique": "ATAP7N60FD38","name": "December 6, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20181206","unid": "A189E4C3E2890D9E872581DE00610B55"},{"unique": "ATAP5K6051F2","name": "November 15, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20181115","unid": "54C90CDC38C2A3D0872581DE00605979"},{"unique": "ATANZM5F93D1","name": "November 1, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20181101","unid": "C17D53F644144E78872581DE005FA8A4"},{"unique": "ATANU65ECEB3","name": "October 18, 2018 Civil Service Commission","current": "","preliveoak": "","numberdate": "20181018","unid": "0F3F546DF3551CD2872581DE005EDECF"},{"unique": "ATAMV95CCE5C","name": "October 4, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20181004","unid": "2425EAECF9E1F70F872581DE005CDFBD"},{"unique": "ATAMP85C25DB","name": "September 20, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180920","unid": "E3703CC4A139DA91872581DE005C2E31"},{"unique": "ATAMJJ5B7A50","name": "September 6, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180906","unid": "EFA7182EA5DFA08D872581DE005B905B"},{"unique": "ATAMG35AEACC","name": "August 16, 2018 Civil Service Commission","current": "","preliveoak": "","numberdate": "20180816","unid": "EC3D2553F722F958872581DE005AF5FA"},{"unique": "ATAMD85A4C73","name": "August 2, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180802","unid": "150CA240D62E699A872581DE005A5B32"},{"unique": "ATAM7Q59AD5C","name": "July 19, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180719","unid": "AE8AFF42B6B1862A872581DE0059BAB1"},{"unique": "AZPMGT55FE0E","name": "June 28, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180628","unid": "5A57BB8CCAAFE85F872582AB005612D2"},{"unique": "AT7R6865BB81","name": "June 21, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180621","unid": "76726FD5C4D8B7B4872581DB0065C2BE"},{"unique": "AT7R4664E619","name": "June 7, 2018 Civil Service Commission","current": "","preliveoak": "","numberdate": "20180607","unid": "45BF7A8D7EB31ABD872581DB0064F0DA"},{"unique": "AT7QX66445A8","name": "May 17, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180517","unid": "B23C99A011705AD7872581DB00644D12"},{"unique": "AT7QRG63A418","name": "May 3, 2018 Civil Service Commission","current": "","preliveoak": "","numberdate": "20180503","unid": "11B66B5D2E92D5E7872581DB0063AED6"},{"unique": "AT7QD762ED3A","name": "April 19, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180419","unid": "4E2E910831340C9D872581DB0062F397"},{"unique": "AT7Q2V623966","name": "April 5, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180405","unid": "6727BD3827AA9EEE872581DB00624B00"},{"unique": "AT7PCD617FBD","name": "March 15, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180315","unid": "257F128B1A0E4D62872581DB0061862C"},{"unique": "AT7P7460D163","name": "March 1, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180301","unid": "F760B95B9686FBED872581DB0060DCCF"},{"unique": "AT7NZ7600D89","name": "February 15, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180215","unid": "2ED60BFB66617D07872581DB006014C8"},{"unique": "AT7NT45F3365","name": "February 1, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180201","unid": "7A91ABDC3B2236F9872581DB005F5093"},{"unique": "AT7NMQ5E79CB","name": "January 18, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180118","unid": "76BF6691231BA054872581DB005E89BD"},{"unique": "AT7NFF5D7715","name": "January 4, 2018 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20180104","unid": "7DE20182E81716A6872581DB005D9E16"},{"unique": "APSSSA73A02E","name": "Dec 21, 2017 Civil Service Commission","current": "","preliveoak": "","numberdate": "20171221","unid": "DA6445F3AB4991558725816E0073AE01"},{"unique": "APSSS8739186","name": "Dec 7, 2017 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20171207","unid": "B4BE665A1B6EA7698725816E00739EA5"},{"unique": "APSSRV738502","name": "Nov 16, 2017 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20171116","unid": "F0670D4AC04DA06E8725816E00738FCE"},{"unique": "APSSP77378F1","name": "Nov 2, 2017 Civil Service Commission","current": "","preliveoak": "","numberdate": "20171102","unid": "6E19FAA761980AD08725816E00738441"},{"unique": "APSSP3735D3F","name": "Oct 5, 2017 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20171005","unid": "9B67FC4844B8D6168725816E00736996"},{"unique": "APSSP273504A","name": "Sep 21, 2017 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20170921","unid": "8D1E9C3202F500708725816E00735C46"},{"unique": "APSSNZ7339D5","name": "Sept 7, 2017 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20170907","unid": "B4EDBFC56F4441638725816E00734820"},{"unique": "APSSNY730FBD","name": "Aug 17, 2017 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20170817","unid": "2C87EDB7B615B1658725816E007324F9"},{"unique": "AN3JEU4C8C2F","name": "Aug 3, 2017 Civil Service Commission","current": "","preliveoak": "","numberdate": "20170803","unid": "B8F99FDB237A8A4F87258137004C8C2F"},{"unique": "AMFQ4G61BFE2","name": "July 20, 2017 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20170720","unid": "6BC5240CBEA28482872581230061BFE2"},{"unique": "AMFNUM61A656","name": "July 6, 2017 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20170706","unid": "64AB9A587A06F8C5872581230061A656"},{"unique": "ALQLY258A17D","name": "June 15, 2017 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20170615","unid": "63EC82A67C47E9EF8725810C0058A17D"},{"unique": "ALQLWH588A8A","name": "June 1, 2017 Civil Service Commission","current": "0","preliveoak": "","numberdate": "20170601","unid": "5C957A1153B7315F8725810C00588A8A"},{"unique": "AKRNY46203B7","name": "May 18, 2017 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20170518","unid": "E9CE58A84EC7AC1D872580ED006203B7"},{"unique": "AKRNW861E14C","name": "May 4, 2017 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20170504","unid": "1FC71160E8B8F442872580ED0061E14C"},{"unique": "AJTQCV623DEB","name": "April 20, 2017 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20170420","unid": "BEED8C2C993AAED7872580CF00623DEB"},{"unique": "AJTQBK621F50","name": "April 6, 2017 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20170406","unid": "6CE504C306135B82872580CF00621F50"},{"unique": "AKXQUF6AC29F","name": "Special Meeting of the Civil Service Commission","current": "","preliveoak": "","numberdate": "20170323","unid": "413E4E4B726EA408872580F3006AC29F"},{"unique": "AH9RLR6E1628","name": "March 16, 2017 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20170316","unid": "6E4A578C30AD4BC08725809D006E1628"},{"unique": "AH9RKT6E07DA","name": "March 2, 2017 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20170302","unid": "AE9352EB636E84518725809D006E07DA"},{"unique": "AH9RJT6DEBDA","name": "February 16, 2017 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20170216","unid": "80D2F50CB2C55EE58725809D006DEBDA"},{"unique": "AH9TJM6DBA5A","name": "February 2, 2017 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20170202","unid": "95DCCD62589EE6028725809D006DBA5A"},{"unique": "AG9ST573FED9","name": "January 19, 2017 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20170119","unid": "102A41C4BDD855F88725807D0073FED9"},{"unique": "AG9TAH74EE4E","name": "January 5, 2017 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20170105","unid": "50BC5638BB71CD5C8725807D0074EE4E"},{"unique": "AD5QJR697D7A","name": "December 15, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20161215","unid": "86C57F7F29D3009E8725801900697D7A"},{"unique": "AD5QHQ69679E","name": "December 1, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20161201","unid": "D317112B65F18D42872580190069679E"},{"unique": "AD5QGQ6946E5","name": "November 17, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20161117","unid": "1B6E078CD336828687258019006946E5"},{"unique": "AD5QDM68D3AB","name": "November 3, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20161103","unid": "5B90AC34949ED8DE872580190068D3AB"},{"unique": "ABMLVX542569","name": "October 20, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20161020","unid": "0D6BC6F0B9884DD787257FE900542569"},{"unique": "ABMLY9547D0E","name": "October 6, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20161006","unid": "A59D5DA61135B11C87257FE900547D0E"},{"unique": "ABMMAM55AD77","name": "September 22, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20160922","unid": "D9BDEB57052DDF8D87257FE90055AD77"},{"unique": "ABMM7E559476","name": "September 8, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20160908","unid": "778A6E0613DB2D9287257FE900559476"},{"unique": "ABMM66557FD7","name": "August 25, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20160825","unid": "FD3BAFC61AE60A8087257FE900557FD7"},{"unique": "ABMM4Z555AAC","name": "August 11, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20160811","unid": "9C7200EDC8735C8987257FE900555AAC"},{"unique": "A9DSJS63F77C","name": "July 28, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20160728","unid": "9CB972F274DDE05487257FA10063F77C"},{"unique": "A9DSHA63E77F","name": "July 14, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20160714","unid": "EB850053DBC054A287257FA10063E77F"},{"unique": "A9DSFW63D75F","name": "June 30, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20160630","unid": "F81E3D43ECCBED7687257FA10063D75F"},{"unique": "A9DSE763C253","name": "June 16, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20160616","unid": "E0456D593E730AB487257FA10063C253"},{"unique": "A9DS9K632D4D","name": "June 2, 2016 Civil Service Commission meeting","current": "0","preliveoak": "","numberdate": "20160602","unid": "9B3BA87668E73C6487257FA100632D4D"},{"unique": "A9CQGB60BE72","name": "May 12, 2016 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20160512","unid": "BCA3DB11AA16EADC87257FA00060CB5E"},{"unique": "A7QABV110E5A","name": "April 28, 2016 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20160428","unid": "1098FF851ED614DF87257F6C00110E5A"},{"unique": "A7G43U09A503","name": "April 14, 2016 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20160414","unid": "950EF19F0C64F40B87257F640009A503"},{"unique": "A7G44M09B8FF","name": "March 24, 2016 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20160324","unid": "2FDAB1B3691140CF87257F640009B8FF"},{"unique": "A5PNX261B73B","name": "March 10, 2016 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20160310","unid": "F750184048EB89E787257F2B0061B73B"},{"unique": "A53JGL4C3791","name": "February 25, 2016 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20160225","unid": "324393EF72A78A5187257F17004C3791"},{"unique": "A53K374E8BD6","name": "February 11, 2016 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20160211","unid": "57B4DD7CEEEF4FCA87257F17004E8BD6"},{"unique": "A53JFT4C3587","name": "January 28, 2016 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20160128","unid": "23922A9EE6F6682387257F17004C3587"},{"unique": "A53JER4C2387","name": "January 14, 2016 Civil Service Commission Meeting","current": "0","preliveoak": "","numberdate": "20160114","unid": "D7A01B89F041DDF387257F17004C2387"},{"unique": "A3BJX74AF5DE","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20151217","unid": "4E1E40E5A84E1E8C87257EDF004AF5DE"},{"unique": "A3BJVQ4ADC76","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20151203","unid": "3E8E43AF376CCE5B87257EDF004ADC76"},{"unique": "A3BJTP4AC101","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20151119","unid": "60D1908CD708D61A87257EDF004AC101"},{"unique": "A3BJD94A008F","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20151112","unid": "978BDD7BAE27F02F87257EDF004A008F"},{"unique": "A2FVP766CC30","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20151022","unid": "F2D7389EE552E05F87257EC30066CC30"},{"unique": "A2FVLL66B9A8","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20151008","unid": "A45D0FB71DCD699587257EC30066B9A8"},{"unique": "9ZCEN461043D","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20150924","unid": "0B4D2B557E2DFAD787257E9F0061043D"},{"unique": "9ZCEMA60FE59","name": "Civil Service Commission","current": "","preliveoak": "","numberdate": "20150910","unid": "EA25C37451FE796187257E9F0060FE59"},{"unique": "9ZCEPE61085B","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20150827","unid": "50C8B34B1C68C77387257E9F0061085B"},{"unique": "9XJ6LX76B50D","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20150813","unid": "9D929E4A6341A01387257E650076B50D"},{"unique": "9XJ6L676A359","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20150709","unid": "A67450AAD07D016387257E650076A359"},{"unique": "9X5L4Z4C251C","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20150625","unid": "C8733305039DD57287257E59004C251C"},{"unique": "9W947F73CBC7","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20150611","unid": "8C39B5BCE9327D6785257E3C0073CBC7"},{"unique": "9W945G73BFD7","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20150528","unid": "EDAFD80B3623F25F85257E3C0073BFD7"},{"unique": "9W944P73B340","name": "Civil Service Commission","current": "0","preliveoak": "","numberdate": "20150514","unid": "0D42BE1BB934321A85257E3C0073B340"},{}] diff --git a/tests/test_cinoh_Civil_Service.py b/tests/test_cinoh_Civil_Service.py new file mode 100644 index 0000000..a26feb9 --- /dev/null +++ b/tests/test_cinoh_Civil_Service.py @@ -0,0 +1,81 @@ +from datetime import datetime +from os.path import dirname, join + +import pytest +from city_scrapers_core.constants import COMMISSION +from city_scrapers_core.utils import file_response +from freezegun import freeze_time + +from city_scrapers.spiders.cinoh_Civil_Service import CinohCivilServiceSpider + +test_response = file_response( + join(dirname(__file__), "files", "cinoh_Civil_Service.json"), + url="https://go.boarddocs.com/oh/csc/Board.nsf/BD-GetMeetingsList", +) +spider = CinohCivilServiceSpider() + +freezer = freeze_time("2024-11-06") +freezer.start() + +parsed_items = [item for item in spider.parse(test_response)] + +freezer.stop() + + +def test_count(): + assert len(parsed_items) == 21 + + +def test_title(): + assert parsed_items[0]["title"] == "November 7, 2024 Civil Service Commission" + + +def test_description(): + assert parsed_items[0]["description"] == "" + + +def test_start(): + assert parsed_items[0]["start"] == datetime(2024, 11, 7, 0, 0) + + +def test_end(): + assert parsed_items[0]["end"] is None + + +def test_time_notes(): + assert parsed_items[0]["time_notes"] == "" + + +def test_id(): + assert parsed_items[0]["id"] == "cinoh_Civil_Service/202411070000/x/november_7_2024_civil_service_commission" + + +def test_status(): + assert parsed_items[0]["status"] == "tentative" + + +def test_location(): + assert parsed_items[0]["location"] == { + "name": "Cincinnati Civil Service Commission", + "address": "805 Central Ave, Suite 200, Cincinnati, OH 45202", + } + + +def test_source(): + assert parsed_items[0]["source"] == "https://go.boarddocs.com/oh/csc/Board.nsf/vpublic?open#tab-meetings" + + +def test_links(): + assert parsed_items[0]["links"] == [{ + "title": "Agenda", + "href": "https://go.boarddocs.com/oh/csc/Board.nsf/Download-AgendaDetailed?open&id=CZQLFH5631AD¤t_committee_id=A9HCN931D6BA", + }] + + +def test_classification(): + assert parsed_items[0]["classification"] == COMMISSION + + +@pytest.mark.parametrize("item", parsed_items) +def test_all_day(item): + assert item["all_day"] is False From 5ba89dfb67effad763d12b8e7b9d3494ec6f500d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cruz=20N=C3=BA=C3=B1ez?= Date: Fri, 8 Nov 2024 13:53:10 -0500 Subject: [PATCH 2/5] run linters --- city_scrapers/spiders/cinoh_Civil_Service.py | 31 +++++++++++--------- tests/test_cinoh_Civil_Service.py | 20 +++++++++---- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/city_scrapers/spiders/cinoh_Civil_Service.py b/city_scrapers/spiders/cinoh_Civil_Service.py index 77e127e..ed6eb1f 100644 --- a/city_scrapers/spiders/cinoh_Civil_Service.py +++ b/city_scrapers/spiders/cinoh_Civil_Service.py @@ -1,16 +1,17 @@ +import datetime + +import scrapy from city_scrapers_core.constants import COMMISSION from city_scrapers_core.items import Meeting from city_scrapers_core.spiders import CityScrapersSpider -import datetime from dateutil.parser import parse -import scrapy class CinohCivilServiceSpider(CityScrapersSpider): name = "cinoh_Civil_Service" agency = "Cincinnati Civil Service Commission" timezone = "America/Chicago" - committee_id = 'A9HCN931D6BA' + committee_id = "A9HCN931D6BA" custom_settings = { "ROBOTSTXT_OBEY": False, } @@ -19,11 +20,9 @@ class CinohCivilServiceSpider(CityScrapersSpider): # clicking on meetings tab takes you to meetings index and uses API # we scrape API instead via POST request and ignore robots file def start_requests(self): - url = 'https://go.boarddocs.com/oh/csc/Board.nsf/BD-GetMeetingsList' - form_data = { - 'current_committee_id': self.committee_id - } - # Sending the POST request + url = "https://go.boarddocs.com/oh/csc/Board.nsf/BD-GetMeetingsList" + form_data = {"current_committee_id": self.committee_id} + # send the POST request and use parse method when response is returned yield scrapy.FormRequest(url, formdata=form_data, callback=self.parse) def parse(self, response): @@ -35,7 +34,6 @@ def parse(self, response): data = response.json() - for item in data: numb = item.get("numberdate") @@ -63,18 +61,23 @@ def parse(self, response): yield meeting def _parse_location(self, item): - """Parse or generate location.""" + """Generate location.""" return { "name": "Cincinnati Civil Service Commission", "address": "805 Central Ave, Suite 200, Cincinnati, OH 45202", } def _parse_links(self, item): - """Parse or generate links.""" - uniq = item["unique"] - href = f"https://go.boarddocs.com/oh/csc/Board.nsf/Download-AgendaDetailed?open&id={uniq}¤t_committee_id={self.committee_id}" + """Generate links.""" + href = ( + f"https://go.boarddocs.com/oh/csc/Board.nsf/Download-AgendaDetailed?" + f"open&id={item['unique']}¤t_committee_id={self.committee_id}" + ) return [{"title": "Agenda", "href": href}] def _parse_source(self, response): - """Generate source. Instead of returning API URL we return the more user-friendly web page we can see this data from.""" + """ + Generate source. Instead of returning API URL + we return the more user-friendly web page we can see this data from. + """ return "https://go.boarddocs.com/oh/csc/Board.nsf/vpublic?open#tab-meetings" diff --git a/tests/test_cinoh_Civil_Service.py b/tests/test_cinoh_Civil_Service.py index a26feb9..88294a0 100644 --- a/tests/test_cinoh_Civil_Service.py +++ b/tests/test_cinoh_Civil_Service.py @@ -47,7 +47,10 @@ def test_time_notes(): def test_id(): - assert parsed_items[0]["id"] == "cinoh_Civil_Service/202411070000/x/november_7_2024_civil_service_commission" + assert ( + parsed_items[0]["id"] + == "cinoh_Civil_Service/202411070000/x/november_7_2024_civil_service_commission" + ) def test_status(): @@ -62,14 +65,19 @@ def test_location(): def test_source(): - assert parsed_items[0]["source"] == "https://go.boarddocs.com/oh/csc/Board.nsf/vpublic?open#tab-meetings" + assert ( + parsed_items[0]["source"] + == "https://go.boarddocs.com/oh/csc/Board.nsf/vpublic?open#tab-meetings" + ) def test_links(): - assert parsed_items[0]["links"] == [{ - "title": "Agenda", - "href": "https://go.boarddocs.com/oh/csc/Board.nsf/Download-AgendaDetailed?open&id=CZQLFH5631AD¤t_committee_id=A9HCN931D6BA", - }] + assert parsed_items[0]["links"] == [ + { + "title": "Agenda", + "href": "https://go.boarddocs.com/oh/csc/Board.nsf/Download-AgendaDetailed?open&id=CZQLFH5631AD¤t_committee_id=A9HCN931D6BA", # noqa + } + ] def test_classification(): From 0cebebed16fae33f3eb139df01bbb415415bce0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cruz=20N=C3=BA=C3=B1ez?= Date: Fri, 8 Nov 2024 14:10:56 -0500 Subject: [PATCH 3/5] Fix timezone Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- city_scrapers/spiders/cinoh_Civil_Service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/city_scrapers/spiders/cinoh_Civil_Service.py b/city_scrapers/spiders/cinoh_Civil_Service.py index ed6eb1f..dc32595 100644 --- a/city_scrapers/spiders/cinoh_Civil_Service.py +++ b/city_scrapers/spiders/cinoh_Civil_Service.py @@ -10,7 +10,7 @@ class CinohCivilServiceSpider(CityScrapersSpider): name = "cinoh_Civil_Service" agency = "Cincinnati Civil Service Commission" - timezone = "America/Chicago" + timezone = "America/New_York" committee_id = "A9HCN931D6BA" custom_settings = { "ROBOTSTXT_OBEY": False, From 548441c3d73c5543fb6fd1f7a6fdc79044619c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cruz=20N=C3=BA=C3=B1ez?= Date: Thu, 19 Dec 2024 15:29:04 -0500 Subject: [PATCH 4/5] improve date validation logic --- city_scrapers/spiders/cinoh_Civil_Service.py | 16 +++++++++++----- tests/test_cinoh_Civil_Service.py | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/city_scrapers/spiders/cinoh_Civil_Service.py b/city_scrapers/spiders/cinoh_Civil_Service.py index dc32595..28c48ee 100644 --- a/city_scrapers/spiders/cinoh_Civil_Service.py +++ b/city_scrapers/spiders/cinoh_Civil_Service.py @@ -1,10 +1,11 @@ -import datetime +from datetime import datetime import scrapy from city_scrapers_core.constants import COMMISSION from city_scrapers_core.items import Meeting from city_scrapers_core.spiders import CityScrapersSpider from dateutil.parser import parse +from dateutil.relativedelta import relativedelta class CinohCivilServiceSpider(CityScrapersSpider): @@ -30,18 +31,23 @@ def parse(self, response): Parse JSON response. """ - year = str(datetime.datetime.today().year) + lower_limit = datetime.now() - relativedelta(months=6) data = response.json() for item in data: numb = item.get("numberdate") - # skip iteration if meeting is not for current year - if numb is None or year not in numb: + # skip if no date or meeting is too old + if numb is None: continue - # if we are in current year then parse meeting + # skip if date is too old + meeting_date = parse(numb) + if meeting_date < lower_limit: + continue + + # if date is valid then parse meeting meeting = Meeting( title=item["name"], description="", diff --git a/tests/test_cinoh_Civil_Service.py b/tests/test_cinoh_Civil_Service.py index 88294a0..7eb106c 100644 --- a/tests/test_cinoh_Civil_Service.py +++ b/tests/test_cinoh_Civil_Service.py @@ -23,7 +23,7 @@ def test_count(): - assert len(parsed_items) == 21 + assert len(parsed_items) == 12 def test_title(): From afb8a217d7b9749d30daed83b031beb292124c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cruz=20N=C3=BA=C3=B1ez?= Date: Thu, 19 Dec 2024 15:29:57 -0500 Subject: [PATCH 5/5] clean up --- city_scrapers/spiders/cinoh_Civil_Service.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/city_scrapers/spiders/cinoh_Civil_Service.py b/city_scrapers/spiders/cinoh_Civil_Service.py index 28c48ee..cb7112f 100644 --- a/city_scrapers/spiders/cinoh_Civil_Service.py +++ b/city_scrapers/spiders/cinoh_Civil_Service.py @@ -41,8 +41,6 @@ def parse(self, response): # skip if no date or meeting is too old if numb is None: continue - - # skip if date is too old meeting_date = parse(numb) if meeting_date < lower_limit: continue