fixed code that was not closing the document when necessary and add f…

…unction in cli and api for stat and clean wrong open lo doc

.env

@@ -1,2 +1,3 @@
 NB_WORKERS=6
+MAXTIME=60
 SECRET_KEY=DEFAULT_KEY

API/utils.py

@@ -15,10 +15,13 @@
 port='200'
 gworkers=0
 scannedjson=''
-def start_soffice(workers,jsondir):
+maxtime=60
+def start_soffice(workers,jsondir,maxt=60):
     global gworkers
     global my_lo
     global scannedjson
+    global maxtime
+    maxtime=maxt
     scannedjson=jsondir
     gworkers=workers
     os.makedirs("uploads", exist_ok=True)

app.py

@@ -12,7 +12,7 @@
 from os import listdir
 from API import utils
 from lotemplate.utils import get_cached_json
-
+from lotemplate import statistic_open_document,clean_old_open_document
 
 app = Flask(__name__)
 
@@ -37,6 +37,22 @@ def main_route():
         return jsonify(os.listdir("uploads"))
 
 
+@app.route("/stats")
+def stats_route():
+     if request.headers.get('secretkey', '') != os.environ.get('SECRET_KEY', ''):
+        return utils.error_sim(
+            'ApiError', 'invalid_secretkey', "The secret key is invalid or not given", {'key': 'secret_key'}), 401
+     else:
+        return statistic_open_document(utils.my_lo,utils.maxtime)
+
+@app.route("/clean_lo")
+def clean_route():
+     if request.headers.get('secretkey', '') != os.environ.get('SECRET_KEY', ''):
+        return utils.error_sim(
+            'ApiError', 'invalid_secretkey', "The secret key is invalid or not given", {'key': 'secret_key'}), 401
+     else:
+        return clean_old_open_document(utils.my_lo,utils.maxtime)
+
 @app.route("/<directory>", methods=['PUT', 'DELETE', 'PATCH', 'GET'])
 def directory_route(directory):
     if request.headers.get('secretkey', '') != os.environ.get('SECRET_KEY', ''):

docker-compose.yml

@@ -11,4 +11,5 @@ services:
     environment:
       - SECRET_KEY=$SECRET_KEY
       - NB_WORKERS=$NB_WORKERS
+      - MAXTIME=$MAXTIME
     command: "gunicorn -b 0.0.0.0:8000 app:app"

gunicorn.conf.py

@@ -2,8 +2,9 @@
 import os
 
 workers=int(os.environ.get('NB_WORKERS', 4))
+maxtime=int(os.environ.get('MAXTIME', 60))
 my_lo=[]
 scannedjson='uploads/scannnedjson'
 def on_starting(server):
 
-    utils.start_soffice(workers,scannedjson)
+    utils.start_soffice(workers,scannedjson,maxtime)

lotemplate/CalcTemplate.py

@@ -89,6 +89,7 @@ def search_error(self, json_vars: dict[str, dict[str, Union[str, list[str]]]]) -
         if  not notdiff:
             return
         else:
+           self.close()
            raise errors.JsonComparaisonError(
                 'missing_required_variable',
                 f"The json are not the same here is the diff "
@@ -100,6 +101,7 @@ def search_error(self, json_vars: dict[str, dict[str, Union[str, list[str]]]]) -
         # So we check if types are equals or if type in json is "html" while type in template is "text"
         json_incorrect = [key for key in self.variables if (json_vars[key]['type'] != self.variables[key]['type']) and (json_vars[key]['type'] != "html" or self.variables[key]['type']!="text")]
         if json_incorrect:
+            self.close()
             raise errors.JsonComparaisonError(
                 'incorrect_value_type',
                 f"The variable {json_incorrect[0]!r} should be of type "

lotemplate/Statement/IfStatement.py

@@ -132,10 +132,12 @@ def compute_if(x_found, x_found_endif):
 
             match = re.search(IfStatement.start_regex, if_cursor.String, re.IGNORECASE)
             if match is None:
+                c_string=dict_of(if_cursor.String)
+                doc.close(True)
                 raise errors.TemplateError(
                     'syntax_error_in_if_statement',
-                    f"The statement {if_cursor.String} has a Syntax Error",
-                    dict_of(if_cursor.String)
+                    f"The statement {c_string} has a Syntax Error",
+                    dict_of(c_string)
                 )
 
             if_cursor.String = ''
@@ -163,10 +165,12 @@ def find_if_to_compute(doc, search, x_found):
             x_found_endif = doc.findNext(x_found.End, endif_search)
             if x_found_endif is None:
                 cursor = x_found.getText().createTextCursorByRange(x_found)
+                c_string=dict_of(cursor.String)
+                doc.close(True)
                 raise errors.TemplateError(
                     'no_endif_found',
-                    f"The statement {cursor.String} has no endif",
-                    dict_of(cursor.String)
+                    f"The statement {c_string} has no endif",
+                    c_string
                 )
             compute_if(x_found, x_found_endif)
 
@@ -186,6 +190,7 @@ def find_if_to_compute(doc, search, x_found):
         str = globalCursor.String
         match = re.search(IfStatement.end_regex, str, re.IGNORECASE)
         if match is not None:
+            doc.close(True)
             raise errors.TemplateError(
                 'too_many_endif_found',
                 "The document has too many endif",
@@ -251,10 +256,12 @@ def find_if_to_compute(doc, search, x_found):
             x_found_endif = doc.findNext(x_found.End, endif_search)
             if x_found_endif is None:
                 cursor = x_found.getText().createTextCursorByRange(x_found)
+                c_string=dict_of(cursor.String)
+                doc.close(True)
                 raise errors.TemplateError(
                     'no_endif_found',
-                    f"The statement {cursor.String} has no endif",
-                    dict_of(cursor.String)
+                    f"The statement {c_string} has no endif",
+                    dict_of(c_string)
                 )
             compute_if(x_found, x_found_endif)
 

lotemplate/Template.py

@@ -27,7 +27,6 @@
 import shutil
 import json
 
-
 class Template:
 
     TMPDIR='/tmp'
@@ -41,6 +40,13 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 
+    def __del__(self):
+        #print('destroy#########################################################')
+        self.close()
+        if os.path.exists(self.tmp_file):
+            os.remove(self.tmp_file)
+
+
     def __str__(self):
         return str(self.file_name)
 
@@ -83,7 +89,8 @@ def open_doc_from_url(self):
     def validDocType(self,doc):
            pass
 
-    def __init__(self, file_path: str, cnx, should_scan: bool, json_cache_dir=None):
+    def __init__(self, file_path: str, cnx, should_scan: bool,
+                 json_cache_dir=None, author=''):
         """
         An object representing a LibreOffice/OpenOffice template that you can fill, scan, export and more
 
@@ -105,6 +112,9 @@ def __init__(self, file_path: str, cnx, should_scan: bool, json_cache_dir=None):
             self.variables = None
             self.doc = None
             self.doc = self.open_doc_from_url()
+            self.doc.getDocumentProperties().resetUserData(author)
+            #print("number of opendocument"+str(len(list(self.cnx.desktop.getComponents()))))
+            #print([print(a.getURL()) for a in list(self.cnx.desktop.getComponents())])
             if json_cache_dir:
                 cachedjson=get_cached_json(json_cache_dir,file_path)
                 if os.path.exists(cachedjson) and should_scan :
@@ -119,6 +129,7 @@ def __init__(self, file_path: str, cnx, should_scan: bool, json_cache_dir=None):
                 with open(cachedjson, 'w') as f:
                     json.dump(self.variables, f, ensure_ascii=False)
         else:
+            self.close()
             raise errors.FileNotFoundError(
                 'file_not_found',
                 f"the given file does not exist or has not been found (file {file_path!r})",
@@ -145,6 +156,7 @@ def search_error(self, json_vars: dict[str, dict[str, Union[str, list[str]]]]) -
 
         json_missing = [key for key in set(self.variables) - set(json_vars)]
         if json_missing:
+            self.close()
             raise errors.JsonComparaisonError(
                 'missing_required_variable',
                 f"The variable {json_missing[0]!r}, present in the template, "
@@ -156,6 +168,7 @@ def search_error(self, json_vars: dict[str, dict[str, Union[str, list[str]]]]) -
         # So we check if types are equals or if type in json is "html" while type in template is "text"
         json_incorrect = [key for key in self.variables if (json_vars[key]['type'] != self.variables[key]['type']) and (json_vars[key]['type'] != "html" or self.variables[key]['type']!="text")]
         if json_incorrect:
+            self.close()
             raise errors.JsonComparaisonError(
                 'incorrect_value_type',
                 f"The variable {json_incorrect[0]!r} should be of type "
@@ -179,11 +192,13 @@ def close(self) -> None:
 
         :return: None
         """
-
         if not self:
             return
-        if self.doc:
-            self.doc.close(True)
+        try:
+            if self.doc:
+                 self.doc.close(True)
+        except Exception:
+            pass 
         try:
             os.remove(self.tmp_file)
             os.remove(Template.TMPDIR+ "/.~lock." + self.file_tmp_name + "#")
@@ -210,9 +225,11 @@ def export(self, filename: str, dirname=None, no_uid=None )  -> Union[str, None]
         try:
             self.doc.storeToURL(url, (PropertyValue("FilterName", 0, self.formats[file_type], 0),))
         except KeyError:
+            self.close()
             raise errors.ExportError('invalid_format',
                                      f"Invalid export format {file_type!r}.", dict_of(file_type)) from None
         except IOException as error:
+            self.close()
 
             raise errors.ExportError(
                 'unknown_error',
@@ -222,7 +239,4 @@ def export(self, filename: str, dirname=None, no_uid=None )  -> Union[str, None]
 
         return path
 
-    def __del__(self):
-        if os.path.exists(self.tmp_file):
-            os.remove(self.tmp_file)
 

lotemplate/WriterTemplate.py

@@ -53,6 +53,7 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 
+
     def __str__(self):
         return str(self.file_name)
 
@@ -66,7 +67,8 @@ def validDocType(self,doc):
 
 
         if not doc or not doc.supportsService('com.sun.star.text.GenericTextDocument'):
-            self.close()
+            if doc:
+                doc.close(True)
             raise errors.TemplateError(
                 'invalid_format',
                 f"The given format ({self.file_name.split('.')[-1]!r}) is invalid, or the file is already open by "
@@ -105,7 +107,7 @@ def scan(self, **kwargs) -> dict[str: dict[str, Union[str, list[str]]]]:
         :return: list containing all the variables founded in the template
         """
 
-        should_close = kwargs.get("should_close", False)
+        #should_close = kwargs.get("should_close", False)
 
         texts = TextStatement.scan_text(self.doc)
         # we use another document for if statement scanning because it modifies the file
@@ -122,8 +124,7 @@ def scan(self, **kwargs) -> dict[str: dict[str, Union[str, list[str]]]]:
         if duplicates:
             first_type = "text" if duplicates[0] in texts.keys() else "image"
             second_type = "table" if duplicates[0] in tables.keys() else "image"
-            if should_close:
-                self.close()
+            self.close()
             raise errors.TemplateError(
                 'duplicated_variable',
                 f"The variable {duplicates[0]!r} is mentioned two times, but "

lotemplate/__init__.py

@@ -23,11 +23,13 @@
     'TemplateFromExt',
     'start_multi_office',
     'randomConnexion',
+    'clean_old_open_document',
+    'statistic_open_document',
 )
 
 from .connexion import Connexion
 from .utils import convert_to_datas_template,is_network_based,get_file_url
 from .Template import Template
 from .WriterTemplate import WriterTemplate
 from .CalcTemplate import CalcTemplate
-from .lofunction import TemplateFromExt,start_multi_office,randomConnexion
+from .lofunction import TemplateFromExt,start_multi_office,randomConnexion,clean_old_open_document,statistic_open_document

lotemplate/lofunction.py

@@ -16,6 +16,8 @@
 from .CalcTemplate import CalcTemplate 
 from .connexion import Connexion,start_office 
 import random
+from datetime import datetime
+
 def TemplateFromExt(file_path: str, cnx, should_scan: bool,json_cache_dir=None):
 
         filename, file_extension = os.path.splitext(file_path)
@@ -29,8 +31,7 @@ def TemplateFromExt(file_path: str, cnx, should_scan: bool,json_cache_dir=None):
 
 def randomConnexion(lstOffice):
        host,port,lodir = random.choice(lstOffice)
-       return Connexion(host,port) 
-
+       return Connexion(host,port)
 
 def start_multi_office(host:str="localhost",start_port:int=2000,nb_env:int=1):
     """
@@ -39,7 +40,7 @@ def start_multi_office(host:str="localhost",start_port:int=2000,nb_env:int=1):
     :param host:  define host in the UNO connect-string --accept
     :param port:   define port in the UNO connect-string --accept
     :param nb_env: number of process to launch
-    :return: list of (host,port,lo dir) 
+    :return: list of (host,port,lo dir)
     """
     if nb_env <= 0:
        raise TypeError("%s is an invalid positive int value" % nb_env)
@@ -50,3 +51,53 @@ def start_multi_office(host:str="localhost",start_port:int=2000,nb_env:int=1):
         port=port+1
     return soffices
 
+def clean_old_open_document(lstOffice, max_time):
+    """
+    clean open document open for too long and where a sure are not use anymore
+    :lstOffice list of host,port
+    :max_time:  Maximum time in second afeter wich were consider the document
+    not use anymore
+    :checkfile:   add a check to see if the file still exist if not just close
+    the doc.
+    """
+    counter=0
+    for host,port,lodir  in lstOffice:
+        cnx=Connexion(host,port)
+        for doc  in list(cnx.desktop.getComponents()):
+            #print("number of opendocument"+str(len(list(cnx.desktop.getComponents()))))
+            #print(doc)
+            url=doc.getURL()
+            file=url[7:]
+            try:
+                delta=(datetime.now() -
+                       datetime.fromtimestamp(os.path.getmtime(file))).seconds
+                if delta > int(max_time):
+                    counter += 1
+                    doc.close(True)
+                    os.remove(file)
+            except FileNotFoundError:
+                counter += 1
+                doc.close(True) 
+    return({"nb_clean":counter})
+
+def statistic_open_document(lstOffice, max_time,):
+    mylist=[]
+    for host,port,lodir  in lstOffice:
+        cnx=Connexion(host,port)
+        baddoc={"tooold":[],"missing":[]}
+        cnxdict={"maxtime":max_time,"hosts":host,"port":port,}
+        for doc  in list(cnx.desktop.getComponents()):
+            #print(doc)
+            url=doc.getURL()
+            file=url[7:]
+            try:
+                delta=(datetime.now() -
+                       datetime.fromtimestamp(os.path.getmtime(file))).seconds
+                if delta > int(max_time):
+                    baddoc["tooold"].append(file)
+            except FileNotFoundError:
+                    baddoc["missing"].append(file)
+            cnxdict["baddoc"]=baddoc
+        mylist.append(cnxdict)
+    return mylist
+