File tree Expand file tree Collapse file tree 14 files changed +46
-45
lines changed
Expand file tree Collapse file tree 14 files changed +46
-45
lines changed Original file line number Diff line number Diff line change 11import urllib .parse
22from Crawler .ImgFinder import ImgFinder
33import Models .Queue .Image
4+ import Models .Queue .Link
45
56
67class Image :
78 def __init__ (self , page_url ):
89
9- image = Models .Queue .Image .Image
10- image .fetch ()
11- self .image = image
10+ self .image = Models .Queue .Image .Image ()
11+ self .link = Models .Queue .Link .Link ()
1212
1313 self .page_url = page_url
1414 urlres = urllib .parse .urlparse (page_url )
@@ -46,6 +46,8 @@ def save(self):
4646 def save_links (self ):
4747 self .fetch_links ()
4848 self .save ()
49+ self .update_link ()
4950
50-
51-
51+ def update_link (self ):
52+ self .link .remove (self .page_url )
53+ self .link .save ()
Original file line number Diff line number Diff line change 88
99class Page :
1010 def __init__ (self , page_url ):
11-
12- link = Models .Queue .Link .Link
13- link .fetch ()
14- self .link = link
15-
11+ self .link = Models .Queue .Link .Link ()
1612 self .page_url = page_url
1713 self .link .add (page_url )
1814 urlres = urllib .parse .urlparse (page_url )
Original file line number Diff line number Diff line change @@ -5,4 +5,6 @@ class Image(Model):
55 file_path = 'storage/complete/images.txt'
66
77 def __init__ (self ):
8- pass
8+ self .links .clear ()
9+ self .fetch ()
10+ Model .__init__ (self )
Original file line number Diff line number Diff line change @@ -5,5 +5,7 @@ class Link(Model):
55 file_path = 'storage/complete/links.txt'
66
77 def __init__ (self ):
8- pass
8+ self .links .clear ()
9+ self .fetch ()
10+ Model .__init__ (self )
911
Original file line number Diff line number Diff line change 44
55
66class Model :
7- links = set ()
87 file_path = ''
98
10- @classmethod
11- def fetch (cls ):
9+ def __init__ (self ):
10+ self .links = set ()
11+
12+ def fetch (self ):
1213 """
1314 Load links from file and set to Set()
1415 :return: object
1516 """
16- cls .init_dir ()
17- with open (cls .file_path , 'r' ) as f :
17+ self .init_dir ()
18+ with open (self .file_path , 'r' ) as f :
1819 for line in f :
19- cls .links .add (line .replace ('\n ' , '' ))
20- return cls .links
20+ self .links .add (line .replace ('\n ' , '' ))
21+ return self .links
2122
22- @classmethod
23- def save (cls ):
23+ def save (self ):
2424 """
2525
2626 :type links: object
2727 """
28- with open (cls .file_path , 'w' ) as f :
29- for line in sorted (cls .links ):
28+ with open (self .file_path , 'w' ) as f :
29+ for line in sorted (self .links ):
3030 f .write (line + "\n " )
3131 return True
3232
33- @classmethod
34- def has (cls , link ):
35- if not link in cls .links :
33+ def has (self , link ):
34+ if not link in self .links :
3635 return False
3736 else :
3837 return True
3938
40- @classmethod
41- def add (cls , link ):
42- cls .links .add (link )
43- return cls
39+ def add (self , link ):
40+ self .links .add (link )
41+ return self
4442
45- @classmethod
46- def remove (cls , link ):
47- cls .links .discard (link )
48- return cls
43+ def remove (self , link ):
44+ self .links .discard (link )
45+ return self
4946
50- @classmethod
51- def init_dir (cls ):
52- dir_name = os .path .dirname (cls .file_path )
47+ def init_dir (self ):
48+ dir_name = os .path .dirname (self .file_path )
5349 if not os .path .exists (dir_name ):
5450 os .makedirs (dir_name )
55- if not os .path .isfile (cls .file_path ):
56- with open (cls .file_path , 'w' ) as f :
51+ if not os .path .isfile (self .file_path ):
52+ with open (self .file_path , 'w' ) as f :
5753 f .write ('' )
Original file line number Diff line number Diff line change @@ -5,5 +5,7 @@ class Image(Model):
55 file_path = 'storage/queue/images.txt'
66
77 def __init__ (self ):
8- pass
8+ Model .__init__ (self )
9+ self .fetch ()
10+
911
Original file line number Diff line number Diff line change @@ -5,4 +5,5 @@ class Link(Model):
55 file_path = 'storage/queue/links.txt'
66
77 def __init__ (self ):
8- pass
8+ Model .__init__ (self )
9+ self .fetch ()
You can’t perform that action at this time.
0 commit comments