From 8db1b7b8599c062036aff3e9d6e83a619d075de1 Mon Sep 17 00:00:00 2001 From: Elian Fr Date: Thu, 15 Aug 2019 18:07:45 +0200 Subject: [PATCH 1/4] Adapt the project to Lychee Laravel --- lycheesync/lycheedao.py | 79 +++++++++++++++++++++-------------------- ressources/lychee.sql | 28 +++++++-------- 2 files changed, 54 insertions(+), 53 deletions(-) diff --git a/lycheesync/lycheedao.py b/lycheesync/lycheedao.py index 5e837de..7f7d90b 100644 --- a/lycheesync/lycheedao.py +++ b/lycheesync/lycheedao.py @@ -101,7 +101,7 @@ def getUniqTimeBasedId(self): def getAlbumNameDBWidth(self): res = 50 # default value - query = "show columns from lychee_albums where Field='title'" + query = "show columns from albums where Field='title'" cur = self.db.cursor() try: cur.execute(query) @@ -127,8 +127,8 @@ def getAlbumMinMaxIds(self): """ returns min, max album ids """ - min_album_query = "select min(id) as min from lychee_albums" - max_album_query = "select max(id) as max from lychee_albums" + min_album_query = "select min(id) as min from albums" + max_album_query = "select max(id) as max from albums" try: min = -1 max = -1 @@ -166,7 +166,7 @@ def updateAlbumDate(self, albumid, newdate): res = True try: - qry = "update lychee_albums set sysstamp= '" + str(newdate) + "' where id=" + str(albumid) + qry = "update albums set created_at= '" + str(newdate) + "' where id=" + str(albumid) cur = self.db.cursor() cur.execute(qry) self.db.commit() @@ -183,8 +183,8 @@ def changeAlbumId(self, oldid, newid): Change albums id based on album titles (to affect display order) """ res = True - photo_query = "update lychee_photos set album = " + str(newid) + " where album = " + str(oldid) - album_query = "update lychee_albums set id = " + str(newid) + " where id = " + str(oldid) + photo_query = "update photos set album = " + str(newid) + " where album = " + str(oldid) + album_query = "update albums set id = " + str(newid) + " where id = " + str(oldid) try: cur = self.db.cursor() cur.execute(photo_query) @@ -206,7 +206,7 @@ def loadAlbumList(self): """ # Load album list cur = self.db.cursor() - cur.execute("SELECT title,id from lychee_albums") + cur.execute("SELECT title,id from albums") rows = cur.fetchall() for row in rows: self.albumslist[row['title']] = row['id'] @@ -218,7 +218,7 @@ def albumIdExists(self, album_id): res = False try: cur = self.db.cursor() - cur.execute("select * from lychee_albums where id=%s", (album_id)) + cur.execute("select * from albums where id=%s", (album_id)) row = cur.fetchall() if len(row) != 0: res = True @@ -243,7 +243,7 @@ def getAlbumNameFromIdsList(self, list_id): album_names = '' try: albumids = ','.join(list_id) - query = ("select title from lychee_albums where id in(" + albumids + ")") + query = ("select title from albums where id in(" + albumids + ")") cur = self.db.cursor() cur.execute(query) rows = cur.fetchall() @@ -259,7 +259,7 @@ def photoIdExists(self, photoid): res = None try: cur = self.db.cursor() - cur.execute("select id from lychee_photos where id=%s", (photoid)) + cur.execute("select id from photos where id=%s", (photoid)) row = cur.fetchall() if len(row) != 0: logger.debug("photoExistsById %s", row) @@ -273,7 +273,7 @@ def photoExistsByName(self, photo_name): res = None try: cur = self.db.cursor() - cur.execute("select id from lychee_photos where title=%s", (photo_name)) + cur.execute("select id from photos where title=%s", (photo_name)) row = cur.fetchall() if len(row) != 0: logger.debug("photoExistsByName %s", row) @@ -294,7 +294,7 @@ def photoExists(self, photo): try: cur = self.db.cursor() cur.execute( - "select * from lychee_photos where album=%s AND (title=%s OR checksum=%s)", + "select * from photos where album_id=%s AND (title=%s OR checksum=%s)", (photo.albumid, photo.originalname, photo.checksum)) @@ -306,11 +306,11 @@ def photoExists(self, photo): cur = self.db.cursor() cur.execute( - "select album from lychee_photos where (title=%s OR checksum=%s)", + "select album_id from photos where (title=%s OR checksum=%s)", (photo.originalname, photo.checksum)) rows = cur.fetchall() - album_ids = [r['album'] for r in rows] + album_ids = [r['album_id'] for r in rows] if len(album_ids) > 0: logger.warn( "a photo with this name: %s or checksum: %s already exists in at least another album: %s", @@ -334,10 +334,10 @@ def createAlbum(self, album): """ album['id'] = str(self.getUniqAlbumId()) - query = ("insert into lychee_albums (id, title, sysstamp, public, password) values ({},'{}',{},'{}',NULL)".format( + query = ("insert into albums (id, title, created_at, public, password) values ({},'{}',{},'{}',NULL)".format( album['id'], album['name'], - datetime.datetime.now().strftime('%s'), + datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), str(self.conf["publicAlbum"])) ) @@ -346,10 +346,10 @@ def createAlbum(self, album): cur = self.db.cursor() logger.debug("try to createAlbum: %s", query) # duplicate of previous query to use driver quote protection features - cur.execute("insert into lychee_albums (id, title, sysstamp, public, password) values (%s,%s,%s,%s,NULL)", (album['id'], album['name'], datetime.datetime.now().strftime('%s'), str(self.conf["publicAlbum"]))) + cur.execute("insert into albums (id, title, created_at, updated_at, public, password, description) values (%s,%s,%s,%s,%s,NULL,'')", (album['id'], album['name'], datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), str(self.conf["publicAlbum"]))) self.db.commit() - cur.execute("select id from lychee_albums where title=%s", (album['name'])) + cur.execute("select id from albums where title=%s", (album['name'])) row = cur.fetchone() self.albumslist['name'] = row['id'] album['id'] = row['id'] @@ -369,8 +369,8 @@ def eraseAlbum(self, album_id): Return list of the erased photo url """ res = [] - query = "delete from lychee_photos where album = " + str(album_id) + '' - selquery = "select url from lychee_photos where album = " + str(album_id) + '' + query = "delete from photos where album = " + str(album_id) + '' + selquery = "select url from photos where album = " + str(album_id) + '' try: cur = self.db.cursor() cur.execute(selquery) @@ -388,7 +388,7 @@ def eraseAlbum(self, album_id): def dropAlbum(self, album_id): res = False - query = "delete from lychee_albums where id = " + str(album_id) + '' + query = "delete from albums where id = " + str(album_id) + '' try: cur = self.db.cursor() cur.execute(query) @@ -403,7 +403,7 @@ def dropAlbum(self, album_id): def dropPhoto(self, photo_id): """ delete a photo. parameter: photo_id """ res = False - query = "delete from lychee_photos where id = " + str(photo_id) + '' + query = "delete from photos where id = " + str(photo_id) + '' try: cur = self.db.cursor() cur.execute(query) @@ -422,9 +422,9 @@ def get_all_photos(self, album_id=None): """ res = [] if not(album_id): - selquery = "select id, url, album from lychee_photos" + selquery = "select id, url, album from photos" else: - selquery = "select id, url, album from lychee_photos where album={}".format(album_id) + selquery = "select id, url, album from photos where album={}".format(album_id) try: cur = self.db.cursor() @@ -445,7 +445,7 @@ def get_empty_albums(self): res = [] try: # check if exists in db - sql = "select id from lychee_albums where id not in(select distinct album from lychee_photos)" + sql = "select id from albums where id not in(select distinct album from photos)" with self.db.cursor() as cursor: cursor.execute(sql) rows = cursor.fetchall() @@ -462,7 +462,7 @@ def get_album_ids_titles(self): res = None try: # check if exists in db - sql = "select id, title from lychee_albums" + sql = "select id, title from albums" with self.db.cursor() as cursor: cursor.execute(sql) rows = cursor.fetchall() @@ -483,31 +483,32 @@ def addFileToAlbum(self, photo): """ res = True try: - stamp = parse(photo.exif.takedate + ' ' + photo.exif.taketime).strftime('%s') + stamp = parse(photo.exif.takedate + ' ' + photo.exif.taketime).strftime('%Y-%m-%d %H:%M') except Exception as e: - stamp = datetime.datetime.now().strftime('%s') + stamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') - query = ("insert into lychee_photos " + + query = ("insert into photos " + "(id, url, public, type, width, height, " + "size, star, " + - "thumbUrl, album,iso, aperture, make, " + + "thumbUrl, album_id,iso, aperture, make, " + "model, shutter, focal, takestamp, " + - "description, title, checksum) " + + "description, title, checksum, tags, " + + "created_at, updated_at )" + "values " + "({}, '{}', {}, '{}', {}, {}, " + "'{}', {}, " + "'{}', '{}', '{}', '{}'," + " '{}', " + "'{}', '{}', '{}', '{}', " + - "'{}', %s, '{}')" + "'{}', %s, '{}', '{}'," + + "'{}', '{}' )" ).format(photo.id, photo.url, self.conf["publicAlbum"], photo.type, photo.width, photo.height, photo.size, photo.star, - photo.thumbUrl, photo.albumid, - photo.exif.iso, - photo.exif.aperture, + photo.thumbUrl, photo.albumid, photo.exif.iso, photo.exif.aperture, photo.exif.make, photo.exif.model, photo.exif.shutter, photo.exif.focal, stamp, - photo.description, photo.checksum) + photo.description, photo.checksum, '', + stamp, stamp) try: logger.debug(query) cur = self.db.cursor() @@ -525,7 +526,7 @@ def reinitAlbumAutoIncrement(self): min, max = self.getAlbumMinMaxIds() if max: - qry = "alter table lychee_albums AUTO_INCREMENT=" + str(max + 1) + qry = "alter table albums AUTO_INCREMENT=" + str(max + 1) try: cur = self.db.cursor() cur.execute(qry) @@ -549,8 +550,8 @@ def dropAll(self): """ try: cur = self.db.cursor() - cur.execute("delete from lychee_albums") - cur.execute("delete from lychee_photos") + cur.execute("delete from albums") + cur.execute("delete from photos") self.db.commit() except Exception as e: logger.exception(e) diff --git a/ressources/lychee.sql b/ressources/lychee.sql index f9b0855..56e289e 100644 --- a/ressources/lychee.sql +++ b/ressources/lychee.sql @@ -24,13 +24,13 @@ CREATE DATABASE /*!32312 IF NOT EXISTS*/ `lychee_ci` /*!40100 DEFAULT CHARACTER USE `lychee_ci`; -- --- Table structure for table `lychee_albums` +-- Table structure for table `albums` -- -DROP TABLE IF EXISTS `lychee_albums`; +DROP TABLE IF EXISTS `albums`; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; -CREATE TABLE `lychee_albums` ( +CREATE TABLE `albums` ( `id` bigint(14) NOT NULL, `title` varchar(100) NOT NULL DEFAULT '', `description` varchar(1000) DEFAULT '', @@ -44,12 +44,12 @@ CREATE TABLE `lychee_albums` ( /*!40101 SET character_set_client = @saved_cs_client */; -- --- Dumping data for table `lychee_albums` +-- Dumping data for table `albums` -- -LOCK TABLES `lychee_albums` WRITE; -/*!40000 ALTER TABLE `lychee_albums` DISABLE KEYS */; -/*!40000 ALTER TABLE `lychee_albums` ENABLE KEYS */; +LOCK TABLES `albums` WRITE; +/*!40000 ALTER TABLE `albums` DISABLE KEYS */; +/*!40000 ALTER TABLE `albums` ENABLE KEYS */; UNLOCK TABLES; -- @@ -80,13 +80,13 @@ LOCK TABLES `lychee_log` WRITE; UNLOCK TABLES; -- --- Table structure for table `lychee_photos` +-- Table structure for table `photos` -- -DROP TABLE IF EXISTS `lychee_photos`; +DROP TABLE IF EXISTS `photos`; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; -CREATE TABLE `lychee_photos` ( +CREATE TABLE `photos` ( `id` bigint(14) NOT NULL, `title` varchar(100) NOT NULL DEFAULT '', `description` varchar(1000) DEFAULT '', @@ -114,12 +114,12 @@ CREATE TABLE `lychee_photos` ( /*!40101 SET character_set_client = @saved_cs_client */; -- --- Dumping data for table `lychee_photos` +-- Dumping data for table `photos` -- -LOCK TABLES `lychee_photos` WRITE; -/*!40000 ALTER TABLE `lychee_photos` DISABLE KEYS */; -/*!40000 ALTER TABLE `lychee_photos` ENABLE KEYS */; +LOCK TABLES `photos` WRITE; +/*!40000 ALTER TABLE `photos` DISABLE KEYS */; +/*!40000 ALTER TABLE `photos` ENABLE KEYS */; UNLOCK TABLES; -- From e9d611d38e3ce728cc264a1065df4d0dd066f6da Mon Sep 17 00:00:00 2001 From: Elian Fr Date: Thu, 15 Aug 2019 19:31:34 +0200 Subject: [PATCH 2/4] Make it process sub-dirs as sub-albums correctly --- lycheesync/lycheedao.py | 24 ++++++++++++++++++++++-- lycheesync/lycheesyncer.py | 19 ++++++++++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/lycheesync/lycheedao.py b/lycheesync/lycheedao.py index 7f7d90b..59d289d 100644 --- a/lycheesync/lycheedao.py +++ b/lycheesync/lycheedao.py @@ -255,6 +255,21 @@ def getAlbumNameFromIdsList(self, list_id): finally: return album_names + def getAlbumIdFromName(self, title, parent_id = "NULL"): + id = "" + try: + query = ("select id from albums where parent_id" + ("=" + parent_id if (parent_id!= "NULL") else " is NULL") + " and title='" + title + "'") + cur = self.db.cursor() + cur.execute(query) + rows = cur.fetchall() + id = rows[0]['id'] + except Exception as e: + album_names = '' + logger.error('impossible to execute ' + query) + logger.exception(e) + finally: + return id + def photoIdExists(self, photoid): res = None try: @@ -334,9 +349,10 @@ def createAlbum(self, album): """ album['id'] = str(self.getUniqAlbumId()) - query = ("insert into albums (id, title, created_at, public, password) values ({},'{}',{},'{}',NULL)".format( + query = ("insert into albums (id, title, parent_id, created_at, public, password) values ({},'{}',{},'{}','{}',NULL)".format( album['id'], album['name'], + album['parent_id'], datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), str(self.conf["publicAlbum"])) ) @@ -346,7 +362,11 @@ def createAlbum(self, album): cur = self.db.cursor() logger.debug("try to createAlbum: %s", query) # duplicate of previous query to use driver quote protection features - cur.execute("insert into albums (id, title, created_at, updated_at, public, password, description) values (%s,%s,%s,%s,%s,NULL,'')", (album['id'], album['name'], datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), str(self.conf["publicAlbum"]))) + if album['parent_id']=="0": + query = "insert into albums (id, title, parent_id, created_at, updated_at, public, password, description) values ({},'{}',NULL,'{}','{}','{}',NULL,'')".format(album['id'], album['name'], datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), str(self.conf["publicAlbum"])) + else: + query = "insert into albums (id, title, parent_id, created_at, updated_at, public, password, description) values ({},'{}','{}','{}','{}','{}',NULL,'')".format(album['id'], album['name'], album['parent_id'], datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), str(self.conf["publicAlbum"])) + cur.execute(query) self.db.commit() cur.execute("select id from albums where title=%s", (album['name'])) diff --git a/lycheesync/lycheesyncer.py b/lycheesync/lycheesyncer.py index c3979e3..86cd8dc 100644 --- a/lycheesync/lycheesyncer.py +++ b/lycheesync/lycheesyncer.py @@ -379,7 +379,24 @@ def sync(self): # Fill in other album properties # albumnames start at srcdir (to avoid absolute path albumname) album['relpath'] = os.path.relpath(album['path'], self.conf['srcdir']) - album['name'] = self.getAlbumNameFromPath(album) + # album['name'] = self.getAlbumNameFromPath(album) + album['name'] = (album['relpath'].split(os.sep))[-1] + + album['parent_id']="0" + if (len(album['relpath'].split(os.sep)) != 1): + #Get parent_id + path = album['relpath'].split(os.sep) + + tmpparentid = "" + + for name in path: + if name==path[-1]: + album['parent_id'] = tmpparentid + else: + tmpparentid = self.dao.getAlbumIdFromName(name, "NULL" if name==path[0] else tmpparentid) + + + if len(album['name']) > album_name_max_width: logger.warn("album name too long, will be truncated " + album['name']) From 02839bd834e16ec219354883ee78f797728a5e83 Mon Sep 17 00:00:00 2001 From: Elian Fr Date: Sat, 17 Aug 2019 00:03:28 +0200 Subject: [PATCH 3/4] Fix issue of inexisting parent album --- lycheesync/lycheedao.py | 11 +++++++---- lycheesync/lycheemodel.py | 2 +- lycheesync/lycheesyncer.py | 16 +++++++++++----- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/lycheesync/lycheedao.py b/lycheesync/lycheedao.py index 59d289d..a5e42d4 100644 --- a/lycheesync/lycheedao.py +++ b/lycheesync/lycheedao.py @@ -70,6 +70,7 @@ def getUniqPhotoId(self): nbtry = 1 while (self.photoIdExists(id)): id = self.getUniqTimeBasedId() + time.sleep(1) nbtry += 1 if (nbtry == 5): raise Exception("didn't manage to create uniq id") @@ -80,6 +81,7 @@ def getUniqAlbumId(self): nbtry = 1 while (self.albumIdExists(id)): id = self.getUniqTimeBasedId() + time.sleep(1) nbtry += 1 if (nbtry == 5): raise Exception("didn't manage to create uniq id") @@ -90,8 +92,8 @@ def getUniqTimeBasedId(self): id = str(int(time.time())) # not precise enough length = len(id) - if length < 14: - missing_char = 14 - length + if length < 10: + missing_char = 10 - length r = random.random() r = str(r) # last missing_char char @@ -256,13 +258,14 @@ def getAlbumNameFromIdsList(self, list_id): return album_names def getAlbumIdFromName(self, title, parent_id = "NULL"): - id = "" + id = False try: query = ("select id from albums where parent_id" + ("=" + parent_id if (parent_id!= "NULL") else " is NULL") + " and title='" + title + "'") cur = self.db.cursor() cur.execute(query) rows = cur.fetchall() - id = rows[0]['id'] + if len(rows) != 0: + id = rows[0]['id'] except Exception as e: album_names = '' logger.error('impossible to execute ' + query) diff --git a/lycheesync/lycheemodel.py b/lycheesync/lycheemodel.py index 0a26a9a..8fda0a5 100644 --- a/lycheesync/lycheemodel.py +++ b/lycheesync/lycheemodel.py @@ -145,7 +145,7 @@ def __init__(self, id, conf, photoname, album): if ('star' in self.originalname) or ('cover' in self.originalname): self.star = 1 - assert len(self.id) == 14, "id {} is not 14 character long: {}".format(self.id, str(len(self.id))) + assert len(self.id) == 10, "id {} is not 10 character long: {}".format(self.id, str(len(self.id))) # Compute file storage url m = hashlib.md5() diff --git a/lycheesync/lycheesyncer.py b/lycheesync/lycheesyncer.py index 86cd8dc..f2f227b 100644 --- a/lycheesync/lycheesyncer.py +++ b/lycheesync/lycheesyncer.py @@ -381,21 +381,27 @@ def sync(self): album['relpath'] = os.path.relpath(album['path'], self.conf['srcdir']) # album['name'] = self.getAlbumNameFromPath(album) album['name'] = (album['relpath'].split(os.sep))[-1] - + album['parent_id']="0" if (len(album['relpath'].split(os.sep)) != 1): #Get parent_id path = album['relpath'].split(os.sep) - tmpparentid = "" + tmpparentid = "0" for name in path: if name==path[-1]: album['parent_id'] = tmpparentid else: - tmpparentid = self.dao.getAlbumIdFromName(name, "NULL" if name==path[0] else tmpparentid) - - + tmp = self.dao.getAlbumIdFromName(name, "NULL" if name==path[0] else tmpparentid) + if tmp==False: + albumtmp={} + albumtmp['name']=name + albumtmp['parent_id']=tmpparentid + tmpparentid=self.createAlbum(albumtmp) + else: + tmpparentid=tmp + if len(album['name']) > album_name_max_width: From 36c2a5ee4cb6b774e3a9e5c30a1ee13ee0ee430b Mon Sep 17 00:00:00 2001 From: Elian Fr Date: Sat, 17 Aug 2019 01:09:07 +0200 Subject: [PATCH 4/4] Another fix for parent album --- lycheesync/lycheedao.py | 4 ++-- lycheesync/lycheesyncer.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lycheesync/lycheedao.py b/lycheesync/lycheedao.py index a5e42d4..217e771 100644 --- a/lycheesync/lycheedao.py +++ b/lycheesync/lycheedao.py @@ -258,9 +258,9 @@ def getAlbumNameFromIdsList(self, list_id): return album_names def getAlbumIdFromName(self, title, parent_id = "NULL"): - id = False + id = 0 try: - query = ("select id from albums where parent_id" + ("=" + parent_id if (parent_id!= "NULL") else " is NULL") + " and title='" + title + "'") + query = ("select id from albums where parent_id" + ("=" + str(parent_id) if (parent_id!= "NULL") else " is NULL") + " and title='" + title + "'") cur = self.db.cursor() cur.execute(query) rows = cur.fetchall() diff --git a/lycheesync/lycheesyncer.py b/lycheesync/lycheesyncer.py index f2f227b..81b82fe 100644 --- a/lycheesync/lycheesyncer.py +++ b/lycheesync/lycheesyncer.py @@ -381,7 +381,6 @@ def sync(self): album['relpath'] = os.path.relpath(album['path'], self.conf['srcdir']) # album['name'] = self.getAlbumNameFromPath(album) album['name'] = (album['relpath'].split(os.sep))[-1] - album['parent_id']="0" if (len(album['relpath'].split(os.sep)) != 1): #Get parent_id @@ -394,7 +393,7 @@ def sync(self): album['parent_id'] = tmpparentid else: tmp = self.dao.getAlbumIdFromName(name, "NULL" if name==path[0] else tmpparentid) - if tmp==False: + if tmp==0: albumtmp={} albumtmp['name']=name albumtmp['parent_id']=tmpparentid