修复一些错误

Author: Yeonon-Wyy

CrawlBase.py

@@ -1,5 +1,5 @@
 from collections import defaultdict
-import random,re
+import random,re,os
 from PyQt5.QtCore import QThread
 import requests
 from bs4 import BeautifulSoup
@@ -113,21 +113,14 @@ def positionHandle(self):
 				f.write(str(position) + ',')
 				f.write(str(num) + '\n')
 
-
+	#保存文件到txt
 	def staffHandle(self):
 		fileName = 'staff.txt'
 		with open(self.file_path + fileName, 'w', encoding='utf-8') as f:
 			for job_info in self.job_infos:
 				f.write(str(job_info['staff']) + ',' + str(job_info['details_url'] + '\n'))
 
 
-	#保存文件，用户可提取
-	def saveFileCSV(self, fileName):
-		with open(self.file_path + fileName + '.csv', 'w', encoding='utf-8') as f:
-			f.write(str(fileName) + '\n')
-			for job_info in self.job_infos:
-				f.write(str(job_info[fileName]) + '\n')
-
 	#保存所有信息，使用sqlite3数据库存储
 	def saveAll(self, tableName, db):
 		cursor = db.cursor()
@@ -140,5 +133,20 @@ def saveAll(self, tableName, db):
 		cursor.execute("UPDATE latestType SET latest_type = ? WHERE id = ?",(tableName,1))
 		cursor.close()
 		db.commit()
-		
+
+	#初始化数据库（若文件不存在，则创建数据库）
+	def InitDB(self):
+		if os.path.isfile(os.getcwd() + '/resource/jobs.db'):
+			db = sqlite3.connect(os.getcwd() + '/resource/jobs.db')
+			return db
+		else:
+			db = sqlite3.connect(os.getcwd() + '/resource/jobs.db')
+			cursor = db.cursor()
+			cursor.execute('CREATE TABLE zhilian (staff text, salary varchar(20), position varchar(20), details_url text)')
+			cursor.execute('CREATE TABLE lagou (staff text, salary varchar(20), position varchar(20), details_url text)')
+			cursor.execute('CREATE TABLE latestType (id INTEGER, latest_type varchar(20))')
+			cursor.execute('INSERT INTO latestType (id, latest_type) values (?, ?)',(1,'lagou'))
+			cursor.close()
+			db.commit()
+			return db
 

jobs.db

@@ -37,6 +37,7 @@ def __init__(self, position, keyword, progressBar, page_number):
 
 
 
+
 	def generateUrl(self):
 		q = Queue()
 		for i in range(self.page_number):
@@ -73,12 +74,10 @@ def run(self):
 		t3.join()
 		t4.join()
 
-		db = sqlite3.connect('jobs.db')
-
+		db = self.InitDB()
 		self.salaryHandle()																	#保存文件，单独存放薪水，用于方便生成图像，下同
 		self.positionHandle()
-		self.saveAll('lagou',db)
-		self.staffHandle()																		#保存文件，单独存放职位名称和对应的URL
+		self.saveAll('lagou',db)																		#保存文件，单独存放职位名称和对应的URL
 
 
 		self.trigger.emit()

lagou/lagou.py

@@ -18,10 +18,12 @@ def __init__(self):
         QMainWindow.__init__(self)
         self.setupUi(self)
         self.type = ""
+        self.db = self.InitDB()
         self.staff_list = []
         self.InitImage()
         self.showStaff()
         self.ItemNumber = 0
+        
 
     def typeMap(self, type):
         if type == "拉勾网":
@@ -33,7 +35,6 @@ def typeMap(self, type):
 
     #初始化图片展示到GUI上
     def InitImage(self):
-        self.db = sqlite3.connect('jobs.db')
         cursor = self.db.cursor()
         cursor.execute("SELECT * FROM latestType")
         self.type = cursor.fetchall()[0][1]
@@ -45,6 +46,22 @@ def InitImage(self):
         PixMapPosition = QtGui.QPixmap(os.getcwd() + '/resource/%s/images/2.png' % self.type).scaled(500,500)
         self.PositionImage.setPixmap(PixMapPosition)
 
+    #这里代码和CrawlBase里的代码重复了，违背了软件开发的一些原则，但是因为sqlite3对象不能传递到另一个线程，所以暂时只能这样做
+    def InitDB(self):
+        if os.path.isfile(os.getcwd() + '/resource/jobs.db'):
+            db = sqlite3.connect(os.getcwd() + '/resource/jobs.db')
+            return db
+        else:
+            db = sqlite3.connect(os.getcwd() + '/resource/jobs.db')
+            cursor = db.cursor()
+            cursor.execute('CREATE TABLE zhilian (staff text, salary varchar(20), position varchar(20), details_url text)')
+            cursor.execute('CREATE TABLE lagou (staff text, salary varchar(20), position varchar(20), details_url text)')
+            cursor.execute('CREATE TABLE latestType (id INTEGER, latest_type varchar(20))')
+            cursor.execute('INSERT INTO latestType (id, latest_type) values (?, ?)',(1,'lagou'))
+            cursor.close()
+            db.commit()
+            return db
+
     #开启主线程外的另一个线程，防止UI阻塞，注意到在那个线程里爬数据的时候再次开启了多线程，这是可以的，也是python和Qt 灵活的地方
     def work(self):
         position = self.positionEdit.text()
@@ -66,14 +83,13 @@ def work(self):
         self.workTheard.start()
         self.workTheard.trigger.connect(self.showImage)
 
+    #展示staff信息到listwidget
     def showStaff(self):
         cursor = self.db.cursor()
         self.staff_list = []
         self.listWidget.clear()                                 #staff_list 常驻内存，切记每次都要初始化为空，否则列表将无限增长，最终程序崩溃
 
-        #从数据库中得到所有职位信息
-        
-        cursor.execute('SELECT * FROM %s' % (self.type))
+        cursor.execute('SELECT * FROM %s' % (self.type))        #从数据库中得到所有职位信息
         values = cursor.fetchall()
         N = 50 if len(values) >= 50 else len(values)
 
@@ -103,6 +119,7 @@ def showImage(self):
 
         del image
         gc.collect()
+
         #读取新的数据
         self.showStaff()
 
@@ -115,19 +132,34 @@ def showItem(self):
                 webbrowser.open_new(url)
 
 
+    #打开文件目录浏览器
     def openFile(self):
         DirName = QtWidgets.QFileDialog.getExistingDirectory(self.Filedialog, "浏览文件",
                             "C:",QtWidgets.QFileDialog.ShowDirsOnly)
         self.DirlineEdit.setText(DirName)
 
-
+    
+    #保存数据到EXCEL文件
     def saveFileToExcel(self):
         fileName = self.FilelineEdit.text()
         if fileName == '':
             fileName = 'default'
 
-        fileName = self.DirlineEdit.text() + '/' + fileName
+        fileName = self.DirlineEdit.text() + '/' + fileName + '.xls'
+
+        if os.path.isfile(fileName):
+            existMessage = QtWidgets.QMessageBox.warning( self, 
+                                                         '文件已存在',
+                                                         fileName + ' 已存在,是否覆盖该文件',
+                                                         QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No)
+
+            if existMessage == QtWidgets.QMessageBox.No:
+                return                                             
 
+
+        
+        
+
         cursor = self.db.cursor()
         cursor.execute('SELECT * FROM %s' % (self.type))
         values = cursor.fetchall()
@@ -143,13 +175,19 @@ def saveFileToExcel(self):
             workSheet.write(i+1, 1, values[i][1])
             workSheet.write(i+1, 2, values[i][2])
 
-        workBook.save(fileName + '.xls')
+        workBook.save(fileName)
 
         SucessMessage = QtWidgets.QMessageBox.information( self, 
                                                             '导出EXCEL',
                                                             '导出成功',
                                                             QtWidgets.QMessageBox.Yes)
 
+        
+        if SucessMessage == QtWidgets.QMessageBox.Yes:
+            self.Filedialog.close()
+
+        
+
 
 
 
@@ -166,6 +204,7 @@ def networkError(self):
             pass
 
 
+
 if __name__ == '__main__':
     app = QtWidgets.QApplication(sys.argv)
     MainWindow = Main()

main.py

@@ -159,6 +159,7 @@ def outPutFile(self):
         self.DirlineEdit = QtWidgets.QLineEdit(self.Filedialog)
         self.DirlineEdit.move(100,40)
         self.DirlineEdit.setFixedSize(350,30)
+        self.DirlineEdit.setText(os.getcwd())
 
         self.Filelabel = QtWidgets.QLabel(self.Filedialog)
         self.Filelabel.move(20,100)

mainWIndow.py

@@ -1,12 +1,13 @@
 位置
-房山区,1
-西城区,3
-None,5
-丰台区,4
-东城区,6
-朝阳区,92
-昌平区,7
-通州区,1
+朝阳区,118
+昌平区,3
+None,13
+海淀区,108
+顺义区,1
+石景山区,2
+丰台区,3
+东城区,28
+西城区,11
 大兴区,10
-海淀区,168
-石景山区,3
+房山区,2
+通州区,1

resource/jobs.db

@@ -1,6 +1,5 @@
 薪水
-5K-8K,10
-8K-12K,61
-0-5K,9
-15K-~,161
-12K-15K,59
+12K-15K,46
+15K-~,249
+8K-12K,4
+0-5K,1

resource/lagou/images/1.png

@@ -1,12 +1,15 @@
 位置
-昌平区,14
-石景山区,1
-丰台区,11
-顺义区,2
-大兴区,3
-海淀区,100
-房山区,3
-朝阳区,25
-东城区,3
-北京,428
-西城区,10
+东城区,12
+顺义区,5
+房山区,4
+海淀区,164
+西城区,34
+石景山区,6
+通州区,5
+崇文区,3
+延庆县,1
+昌平区,26
+大兴区,9
+北京,835
+丰台区,27
+朝阳区,69

resource/lagou/images/2.png

@@ -1,6 +1,6 @@
 薪水
-8K-12K,171
-5K-8K,121
-12K-15K,150
-15K-~,144
-0-5K,14
+5K-8K,325
+15K-~,238
+12K-15K,292
+8K-12K,319
+0-5K,26

resource/lagou/position_for_image.csv

@@ -70,11 +70,10 @@ def run(self):
 		t3.join()
 		t4.join()
 
-		db = sqlite3.connect('jobs.db')
+		db = self.InitDB()
 		self.salaryHandle()																	#保存文件，单独存放薪水，用于方便生成图像，下同
 		self.positionHandle()
-		self.saveAll('zhilian',db)
-		self.staffHandle()																		#保存文件，单独存放职位名称和对应的URL
+		self.saveAll('zhilian',db)																		#保存文件，单独存放职位名称和对应的URL
 
 		self.trigger.emit()																		#爬取完毕，要发送信号给UI主线程，并执行相应的槽函数