-
Notifications
You must be signed in to change notification settings - Fork 0
/
sample data reduced.json
91 lines (88 loc) · 3.47 KB
/
sample data reduced.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
{
"items":[
{
"crawl_count": 0,
"crawl_start_time": "09/02/2021 15:15:07",
"attempt_count": 0,
"tenant_name": "Tenant Admin",
"draft_status": "published",
"scraped_count": 0,
"download_delay": "0.217",
"hier": "f747323e-9a5b-49dc-a6e9-a0a4e6c5f513",
"website_name": "TestSite101",
"S3_url": "s3://inscrape-bucket-develop/tenantAdmin/testsite101-bf65e2a3-7f44-4c5e-9df1-b771e522bf81",
"is_stop_site": False,
"spider_status": "Scraping Completed",
"crawl_end_time": "09/02/2021 15:15:50",
"is_error": True,
"is_refused": True,
"website_url": "www.testsite.com",
"project_id": "1b7b9205-efbe-4cb4-bb4a-fb398095ed19",
"upload_type": "instant_upload",
"id": "bf65e2a3-7f44-4c5e-9df1-b771e522bf81",
"node_count": 0,
"site_id": 61,
"type": "url",
"project_name": "News",
"scraping_time": "0:00:43",
"uploader_time": ""
},
{
"crawl_count": 0,
"crawl_start_time": "09/24/2021 19:39:18",
"attempt_count": 469,
"tenant_name": "Tenant Admin",
"draft_status": "published",
"scraped_count": 469,
"download_delay": "0.25",
"hier": "f747323e-9a5b-49dc-a6e9-a0a4e6c5f513",
"website_name": "BCBS_TN",
"S3_url": "s3://inscrape-bucket-develop/tenantAdmin/bcbsTn-d12f0b53-66d4-4037-b9d7-6aa9ed7c2432",
"is_stop_site": False,
"spider_status": "Stopped",
"crawl_end_time": "09/24/2021 19:39:18",
"is_error": False,
"is_refused": False,
"website_url": "https://www.bcbst.com/mpmanual/!SSL!/WebHelp/Welcome.htm",
"upload_type": "instant_upload",
"project_id": "1e989559-c5ba-4d9d-8e0e-7d967f78d17d",
"id": "d12f0b53-66d4-4037-b9d7-6aa9ed7c2432",
"node_count": None,
"site_id": 4,
"type": "url",
"project_name": "Demo",
"scraping_time": "0:00:00",
"uploader_time": ""
},
{
"crawl_count": 406,
"attempt_count": 470,
"crawl_start_time": "09/02/2021 19:12:25",
"tenant_name": "Tenant Admin",
"draft_status": "published",
"scraped_count": 470,
"download_delay": "0.25",
"hier": "f747323e-9a5b-49dc-a6e9-a0a4e6c5f513",
"website_name": "Amerigroup",
"S3_url": "s3://inscrape-bucket-develop/tenantAdmin/amerigroup-09cecaf9-102d-4d87-8615-086c4ce30f92",
"is_stop_site": False,
"spider_status": "Scraping Completed",
"crawl_end_time": "09/02/2021 19:46:29",
"is_error": True,
"is_refused": False,
"website_url": "https://medpol.providers.amerigroup.com/green-provider/medical-policies-and-clinical-guidelines-full-list",
"upload_type": "instant_upload",
"project_id": "1e989559-c5ba-4d9d-8e0e-7d967f78d17d",
"id": "09cecaf9-102d-4d87-8615-086c4ce30f92",
"node_count": 1,
"site_id": 31,
"type": "url",
"project_name": "Demo",
"scraping_time": "0:34:04",
"uploader_time": ""
}
],
"msg":"hai",
"count":3,
"endpage": False
}