fix but skipping first data 3d, 3d_pro

Author: vietvudanh

data/3d.jsonl

@@ -2,18 +2,22 @@
 {"date":"2019-04-24","id":"00002","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["817","156"],"Gi\u1ea3i Nh\u1ea5t":["007","038","125","667"],"Gi\u1ea3i Nh\u00ec":["355","787","838","558","762","488"],"Gi\u1ea3i ba":["722","401","873","296","292","054","243","882"]},"page":163}
 {"date":"2019-04-26","id":"00003","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["428","336"],"Gi\u1ea3i Nh\u1ea5t":["479","232","435","033"],"Gi\u1ea3i Nh\u00ec":["071","986","348","539","464","731"],"Gi\u1ea3i ba":["428","982","057","331","545","111","873","039"]},"page":163}
 {"date":"2019-04-29","id":"00004","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["668","878"],"Gi\u1ea3i Nh\u1ea5t":["798","683","954","222"],"Gi\u1ea3i Nh\u00ec":["463","754","971","300","553","179"],"Gi\u1ea3i ba":["803","617","958","125","008","393","003","102"]},"page":163}
+{"date":"2019-05-01","id":"00005","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["702","081"],"Gi\u1ea3i Nh\u1ea5t":["294","847","766","617"],"Gi\u1ea3i Nh\u00ec":["501","037","145","148","725","384"],"Gi\u1ea3i ba":["782","847","350","078","174","651","658","469"]},"page":163}
 {"date":"2019-05-03","id":"00006","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["572","133"],"Gi\u1ea3i Nh\u1ea5t":["642","469","512","415"],"Gi\u1ea3i Nh\u00ec":["228","127","348","638","488","145"],"Gi\u1ea3i ba":["111","870","834","085","437","238","232","083"]},"page":162}
 {"date":"2019-05-06","id":"00007","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["516","143"],"Gi\u1ea3i Nh\u1ea5t":["182","748","370","160"],"Gi\u1ea3i Nh\u00ec":["459","572","334","472","714","365"],"Gi\u1ea3i ba":["477","662","210","349","105","494","063","212"]},"page":162}
 {"date":"2019-05-08","id":"00008","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["864","188"],"Gi\u1ea3i Nh\u1ea5t":["589","765","332","126"],"Gi\u1ea3i Nh\u00ec":["004","658","298","897","812","149"],"Gi\u1ea3i ba":["979","409","609","606","661","192","439","122"]},"page":162}
 {"date":"2019-05-10","id":"00009","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["656","843"],"Gi\u1ea3i Nh\u1ea5t":["042","466","498","631"],"Gi\u1ea3i Nh\u00ec":["309","898","019","729","115","696"],"Gi\u1ea3i ba":["725","756","894","424","426","209","085","635"]},"page":162}
+{"date":"2019-05-13","id":"00010","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["725","013"],"Gi\u1ea3i Nh\u1ea5t":["148","553","664","376"],"Gi\u1ea3i Nh\u00ec":["922","866","259","353","436","918"],"Gi\u1ea3i ba":["190","024","899","289","110","721","881","174"]},"page":162}
 {"date":"2019-05-15","id":"00011","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["219","211"],"Gi\u1ea3i Nh\u1ea5t":["746","004","589","811"],"Gi\u1ea3i Nh\u00ec":["377","470","369","036","455","798"],"Gi\u1ea3i ba":["684","510","751","093","215","260","545","474"]},"page":161}
 {"date":"2019-05-17","id":"00012","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["023","769"],"Gi\u1ea3i Nh\u1ea5t":["668","327","671","187"],"Gi\u1ea3i Nh\u00ec":["851","452","307","068","725","423"],"Gi\u1ea3i ba":["462","603","248","667","733","183","342","566"]},"page":161}
 {"date":"2019-05-20","id":"00013","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["300","111"],"Gi\u1ea3i Nh\u1ea5t":["913","695","037","059"],"Gi\u1ea3i Nh\u00ec":["910","284","112","294","272","887"],"Gi\u1ea3i ba":["626","881","025","066","896","263","548","251"]},"page":161}
 {"date":"2019-05-22","id":"00014","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["548","068"],"Gi\u1ea3i Nh\u1ea5t":["528","924","103","384"],"Gi\u1ea3i Nh\u00ec":["344","700","035","988","375","441"],"Gi\u1ea3i ba":["429","439","003","742","751","022","414","872"]},"page":161}
+{"date":"2019-05-24","id":"00015","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["408","771"],"Gi\u1ea3i Nh\u1ea5t":["788","327","384","314"],"Gi\u1ea3i Nh\u00ec":["334","371","919","273","927","969"],"Gi\u1ea3i ba":["784","103","593","040","622","277","935","655"]},"page":161}
 {"date":"2019-05-27","id":"00016","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["525","052"],"Gi\u1ea3i Nh\u1ea5t":["922","441","358","211"],"Gi\u1ea3i Nh\u00ec":["433","103","586","643","759","006"],"Gi\u1ea3i ba":["023","691","214","380","835","806","196","708"]},"page":160}
 {"date":"2019-05-29","id":"00017","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["936","784"],"Gi\u1ea3i Nh\u1ea5t":["575","463","403","532"],"Gi\u1ea3i Nh\u00ec":["948","068","866","411","132","720"],"Gi\u1ea3i ba":["350","439","402","723","603","264","352","913"]},"page":160}
 {"date":"2019-05-31","id":"00018","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["901","951"],"Gi\u1ea3i Nh\u1ea5t":["841","324","204","769"],"Gi\u1ea3i Nh\u00ec":["855","410","093","219","667","116"],"Gi\u1ea3i ba":["236","256","060","578","389","976","457","792"]},"page":160}
 {"date":"2019-06-03","id":"00019","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["447","179"],"Gi\u1ea3i Nh\u1ea5t":["671","249","073","217"],"Gi\u1ea3i Nh\u00ec":["560","428","908","146","012","640"],"Gi\u1ea3i ba":["946","879","156","181","905","747","802","095"]},"page":160}
+{"date":"2019-06-05","id":"00020","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["110","152"],"Gi\u1ea3i Nh\u1ea5t":["503","292","540","249"],"Gi\u1ea3i Nh\u00ec":["922","139","331","606","145","207"],"Gi\u1ea3i ba":["309","480","243","655","966","607","005","222"]},"page":160}
 {"date":"2019-06-07","id":"00021","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["355","191"],"Gi\u1ea3i Nh\u1ea5t":["623","114","540","545"],"Gi\u1ea3i Nh\u00ec":["222","482","843","289","327","588"],"Gi\u1ea3i ba":["603","639","007","937","740","222","836","123"]},"page":159}
 {"date":"2019-06-10","id":"00022","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["484","139"],"Gi\u1ea3i Nh\u1ea5t":["670","617","285","013"],"Gi\u1ea3i Nh\u00ec":["213","012","457","778","218","269"],"Gi\u1ea3i ba":["384","660","683","400","772","192","624","888"]},"page":159}
 {"date":"2019-06-12","id":"00023","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["813","400"],"Gi\u1ea3i Nh\u1ea5t":["672","507","158","053"],"Gi\u1ea3i Nh\u00ec":["249","469","646","494","888","311"],"Gi\u1ea3i ba":["989","867","067","748","588","298","028","396"]},"page":159}

data/3d_pro.jsonl

@@ -13,6 +13,7 @@
 {"date":"2021-10-12","id":"00013","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["232","426"],"Gi\u1ea3i Nh\u1ea5t":["021","626","628","436"],"Gi\u1ea3i Nh\u00ec":["297","269","073","208","335","026"],"Gi\u1ea3i ba":["865","463","554","679","515","538","407","967"]},"page":90}
 {"date":"2021-10-14","id":"00014","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["744","066"],"Gi\u1ea3i Nh\u1ea5t":["290","881","830","513"],"Gi\u1ea3i Nh\u00ec":["776","270","787","212","352","695"],"Gi\u1ea3i ba":["902","810","578","871","928","941","064","607"]},"page":90}
 {"date":"2021-10-16","id":"00015","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["362","339"],"Gi\u1ea3i Nh\u1ea5t":["864","962","784","230"],"Gi\u1ea3i Nh\u00ec":["861","188","189","085","748","623"],"Gi\u1ea3i ba":["825","062","488","288","538","720","236","108"]},"page":90}
+{"date":"2021-10-19","id":"00016","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["702","929"],"Gi\u1ea3i Nh\u1ea5t":["984","204","900","942"],"Gi\u1ea3i Nh\u00ec":["126","330","882","903","217","369"],"Gi\u1ea3i ba":["047","907","407","134","332","059","434","123"]},"page":90}
 {"date":"2021-10-21","id":"00017","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["762","717"],"Gi\u1ea3i Nh\u1ea5t":["218","060","863","078"],"Gi\u1ea3i Nh\u00ec":["320","598","187","642","165","294"],"Gi\u1ea3i ba":["769","005","605","078","619","226","678","679"]},"page":89}
 {"date":"2021-10-23","id":"00018","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["841","363"],"Gi\u1ea3i Nh\u1ea5t":["543","562","797","573"],"Gi\u1ea3i Nh\u00ec":["621","160","658","965","173","668"],"Gi\u1ea3i ba":["645","124","241","475","497","073","146","482"]},"page":89}
 {"date":"2021-10-26","id":"00019","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["164","042"],"Gi\u1ea3i Nh\u1ea5t":["487","921","590","236"],"Gi\u1ea3i Nh\u00ec":["342","880","105","458","602","738"],"Gi\u1ea3i ba":["108","973","246","081","989","534","019","704"]},"page":89}
@@ -464,3 +465,4 @@
 {"date":"2024-09-07","id":"00465","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["558","082"],"Gi\u1ea3i Nh\u1ea5t":["706","380","930","497"],"Gi\u1ea3i Nh\u00ec":["249","224","944","476","295","659"],"Gi\u1ea3i ba":["564","298","422","038","917","152","420","221"]},"page":0}
 {"date":"2024-09-10","id":"00466","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["745","263"],"Gi\u1ea3i Nh\u1ea5t":["788","044","044","654"],"Gi\u1ea3i Nh\u00ec":["823","791","160","125","965","208"],"Gi\u1ea3i ba":["971","432","215","550","834","662","018","571"]},"page":0}
 {"date":"2024-09-12","id":"00467","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["258","728"],"Gi\u1ea3i Nh\u1ea5t":["565","283","069","066"],"Gi\u1ea3i Nh\u00ec":["119","203","366","480","686","689"],"Gi\u1ea3i ba":["183","160","063","249","701","579","831","750"]},"page":0}
+{"date":"2024-09-14","id":"00468","result":{"Gi\u1ea3i \u0110\u1eb7c bi\u1ec7t":["310","884"],"Gi\u1ea3i Nh\u1ea5t":["735","926","135","528"],"Gi\u1ea3i Nh\u00ec":["235","487","648","076","320","061"],"Gi\u1ea3i ba":["702","087","344","736","687","394","179","313"]},"page":0}

src/vietlott/cli/crawl.py

@@ -15,12 +15,6 @@
 def crawl(ctx, product, run_date, index_from, index_to):
     """
     crawl a product with a given run date or from/to index page
-    :param ctx:
-    :param product:
-    :param run_date:
-    :param index_from:
-    :param index_to:
-    :return:
     """
     if product not in product_config_map:
         click.echo(f"Error:: Product must in product_map: {list(product_config_map.keys())}", err=True)

src/vietlott/crawler/products/p3d.py

@@ -42,8 +42,6 @@ def process_result(self, params, body: dict, res_json: dict, task_data: dict) ->
         # run_date_str = task_data["run_date_str"]
         data = []
         for i, tr in enumerate(soup.select("table tr")):
-            if i == 0:
-                continue
             tds = tr.find_all("td")
             row = {}