diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1f2e2c8..5467b73 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,11 @@
## CHANGELOG
+### [25.2.0] - Jan 6, 2025
+- Added a "Delete Link" button to allow DataTemplates to be deleted
+- Added support so a specific DataSet can be selected via the DataTemplate URL using `?ds=` or `/dt/
/`
+- Don't add a hash symbol to the end of the URL when clicking on links
+- Fixed an issue where the "Protect Link" button was losing it's icon under certain conditions
+
### [25.1.1] - Jan 1, 2025
- Rewritten Web Log authentication so it will now prompt you for a password
@@ -368,6 +374,7 @@
- Initial release
+[25.2.0]: https://github.com/cmason3/jinjafx_server/compare/25.1.1...25.2.0
[25.1.1]: https://github.com/cmason3/jinjafx_server/compare/25.1.0...25.1.1
[25.1.0]: https://github.com/cmason3/jinjafx_server/compare/24.12.1...25.1.0
[24.12.1]: https://github.com/cmason3/jinjafx_server/compare/24.12.0...24.12.1
diff --git a/README.md b/README.md
index 508bee7..1c6f730 100644
--- a/README.md
+++ b/README.md
@@ -95,6 +95,8 @@ As well as supporting the standard CodeMirror shortcut keys for the "data.csv",
The DataSet feature allows you to include multiple different "data.csv" and "vars.yml" contents while maintaining the same "template.j2". When enabled it also enables a "global.yml" pane for specifying global variables which are common across all DataSets (any variables which are redefined in "vars.yml" will overwrite variables in "global.yml"). This is to support scenarios where you have different DataSets for your Live vs your Test environments, but the template should be the same. There are no limits on the number of different DataSets that can be added to a single DataTemplate (the name must start with a letter and only contain alphanumerical, "-", " " or "_" characters). When you click "Generate" it will use the currently active DataSet to generate the output - clicking on the name of the current DataSet (by default there is a single "Default" DataSet) allows you to switch between the different DataSets.
+You can also select a specific DataSet when you load a DataTemplate using the `ds` query string parameter (i.e. `?ds='`) or by adding an additional path element (i.e. `/dt/
/`).
+
### Output Formats
JinjaFx Server supports the ability to use "output" tags to create different outputs with different names like JinjaFx, but it also allows you to optionally specify how you want the output to be rendered. By default, the output is rendered as "text" but you also have the option to specify "html" and "markdown" (for GitHub Flavoured Markdown), which will result in the output being rendered appropriately, e.g:
diff --git a/jinjafx_server.py b/jinjafx_server.py
index 71e8d18..9417ad5 100755
--- a/jinjafx_server.py
+++ b/jinjafx_server.py
@@ -26,7 +26,7 @@
import re, argparse, hashlib, traceback, glob, hmac, uuid, struct, binascii, gzip, requests, ctypes, subprocess
import cmarkgfm, emoji
-__version__ = '25.1.1'
+__version__ = '25.2.0'
llock = threading.RLock()
rlock = threading.RLock()
@@ -247,7 +247,7 @@ def do_GET(self, head=False, cache=True, versioned=False):
fpath = '/index.html'
self.hide = not verbose
- if re.search(r'^/dt/[A-Za-z0-9_-]{1,24}$', fpath):
+ if re.search(r'^/dt/[A-Za-z0-9_-]{1,24}(?:/[A-Za-z][A-Za-z0-9_ %-]*)?$', fpath):
fpath = '/index.html'
if re.search(r'^/[a-f0-9]{8}/', fpath):
@@ -621,10 +621,11 @@ def html_escape(text):
else:
r = [ 'text/plain', 400, '400 Bad Request\r\n', sys._getframe().f_lineno ]
- elif fpath == '/get_link':
+ elif fpath == '/get_link' or fpath == '/delete_link':
if aws_s3_url or github_url or repository:
if self.headers['Content-Type'] == 'application/json':
try:
+ dt_yml = ''
dt_password = ''
dt_opassword = ''
dt_mpassword = ''
@@ -650,88 +651,132 @@ def html_escape(text):
dt_revision = int(self.headers['X-Dt-Revision'])
if not self.ratelimit(remote_addr, 1, False):
- dt = json.loads(postdata.decode('utf-8'))
-
- vdt = {}
-
- dt_yml = '---\n'
- dt_yml += 'dt:\n'
+ def authenticate_dt(rdt, r):
+ mm = re.search(r'dt_mpassword: "(\S+)"', rdt)
+ mo = re.search(r'dt_password: "(\S+)"', rdt)
- if 'datasets' in dt:
- if 'global' in dt:
- vdt['global'] = self.d(dt['global']).decode('utf-8') if 'global' in dt and len(dt['global'].strip()) > 0 else ''
+ if mm != None or mo != None:
+ if dt_password != '':
+ rpassword = mm.group(1) if mm != None else mo.group(1)
+ t = binascii.unhexlify(rpassword.encode('utf-8'))
+ if t != self.derive_key(dt_password, t[2:int(t[1]) + 2], t[0]):
+ cheaders['X-Dt-Authentication'] = 'Modify' if (mm != None) else 'Open'
+ r = [ 'text/plain', 401, '401 Unauthorized\r\n', sys._getframe().f_lineno ]
- if vdt['global'] == '':
- dt_yml += ' global: ""\n\n'
else:
- dt_yml += ' global: |2\n'
- dt_yml += re.sub('^', ' ' * 4, vdt['global'].rstrip(), flags=re.MULTILINE) + '\n\n'
-
- dt_yml += ' datasets:\n'
+ cheaders['X-Dt-Authentication'] = 'Modify' if (mm != None) else 'Open'
+ r = [ 'text/plain', 401, '401 Unauthorized\r\n', sys._getframe().f_lineno ]
- for ds in dt['datasets']:
- vdt['data'] = self.d(dt['datasets'][ds]['data']).decode('utf-8') if 'data' in dt['datasets'][ds] and len(dt['datasets'][ds]['data'].strip()) > 0 else ''
- vdt['vars'] = self.d(dt['datasets'][ds]['vars']).decode('utf-8') if 'vars' in dt['datasets'][ds] and len(dt['datasets'][ds]['vars'].strip()) > 0 else ''
+ return r
- dt_yml += ' "' + ds + '":\n'
+ if fpath == '/get_link':
+ dt = json.loads(postdata.decode('utf-8'))
+ vdt = {}
+ dt_yml += '---\n'
+ dt_yml += 'dt:\n'
+
+ if 'datasets' in dt:
+ if 'global' in dt:
+ vdt['global'] = self.d(dt['global']).decode('utf-8') if 'global' in dt and len(dt['global'].strip()) > 0 else ''
+
+ if vdt['global'] == '':
+ dt_yml += ' global: ""\n\n'
+ else:
+ dt_yml += ' global: |2\n'
+ dt_yml += re.sub('^', ' ' * 4, vdt['global'].rstrip(), flags=re.MULTILINE) + '\n\n'
+
+ dt_yml += ' datasets:\n'
+
+ for ds in dt['datasets']:
+ vdt['data'] = self.d(dt['datasets'][ds]['data']).decode('utf-8') if 'data' in dt['datasets'][ds] and len(dt['datasets'][ds]['data'].strip()) > 0 else ''
+ vdt['vars'] = self.d(dt['datasets'][ds]['vars']).decode('utf-8') if 'vars' in dt['datasets'][ds] and len(dt['datasets'][ds]['vars'].strip()) > 0 else ''
+
+ dt_yml += ' "' + ds + '":\n'
+
+ if vdt['data'] == '':
+ dt_yml += ' data: ""\n'
+ else:
+ dt_yml += ' data: |2\n'
+ dt_yml += re.sub('^', ' ' * 8, vdt['data'].rstrip(), flags=re.MULTILINE) + '\n\n'
+
+ if vdt['vars'] == '':
+ dt_yml += ' vars: ""\n\n'
+ else:
+ dt_yml += ' vars: |2\n'
+ dt_yml += re.sub('^', ' ' * 8, vdt['vars'].rstrip(), flags=re.MULTILINE) + '\n\n'
+
+ else :
+ vdt['data'] = self.d(dt['data']).decode('utf-8') if 'data' in dt and len(dt['data'].strip()) > 0 else ''
+ vdt['vars'] = self.d(dt['vars']).decode('utf-8') if 'vars' in dt and len(dt['vars'].strip()) > 0 else ''
+
if vdt['data'] == '':
- dt_yml += ' data: ""\n'
+ dt_yml += ' data: ""\n'
else:
- dt_yml += ' data: |2\n'
- dt_yml += re.sub('^', ' ' * 8, vdt['data'].rstrip(), flags=re.MULTILINE) + '\n\n'
-
+ dt_yml += ' data: |2\n'
+ dt_yml += re.sub('^', ' ' * 4, vdt['data'].rstrip(), flags=re.MULTILINE) + '\n\n'
+
if vdt['vars'] == '':
- dt_yml += ' vars: ""\n\n'
+ dt_yml += ' vars: ""\n\n'
else:
- dt_yml += ' vars: |2\n'
- dt_yml += re.sub('^', ' ' * 8, vdt['vars'].rstrip(), flags=re.MULTILINE) + '\n\n'
-
- else :
- vdt['data'] = self.d(dt['data']).decode('utf-8') if 'data' in dt and len(dt['data'].strip()) > 0 else ''
- vdt['vars'] = self.d(dt['vars']).decode('utf-8') if 'vars' in dt and len(dt['vars'].strip()) > 0 else ''
-
- if vdt['data'] == '':
- dt_yml += ' data: ""\n'
- else:
- dt_yml += ' data: |2\n'
- dt_yml += re.sub('^', ' ' * 4, vdt['data'].rstrip(), flags=re.MULTILINE) + '\n\n'
-
- if vdt['vars'] == '':
- dt_yml += ' vars: ""\n\n'
+ dt_yml += ' vars: |2\n'
+ dt_yml += re.sub('^', ' ' * 4, vdt['vars'].rstrip(), flags=re.MULTILINE) + '\n\n'
+
+ if isinstance(dt['template'], dict):
+ dt_yml += ' template:\n'
+
+ for t in dt['template']:
+ te = self.d(dt['template'][t]).decode('utf-8') if len(dt['template'][t].strip()) > 0 else ''
+
+ if te == '':
+ dt_yml += ' "' + t + '": ""\n'
+ else:
+ dt_yml += ' "' + t + '": |2\n'
+ dt_yml += re.sub('^', ' ' * 6, te, flags=re.MULTILINE) + '\n\n'
+
else:
- dt_yml += ' vars: |2\n'
- dt_yml += re.sub('^', ' ' * 4, vdt['vars'].rstrip(), flags=re.MULTILINE) + '\n\n'
-
- if isinstance(dt['template'], dict):
- dt_yml += ' template:\n'
-
- for t in dt['template']:
- te = self.d(dt['template'][t]).decode('utf-8') if len(dt['template'][t].strip()) > 0 else ''
-
+ te = self.d(dt['template']).decode('utf-8') if len(dt['template'].strip()) > 0 else ''
+
if te == '':
- dt_yml += ' "' + t + '": ""\n'
+ dt_yml += ' template: ""\n'
else:
- dt_yml += ' "' + t + '": |2\n'
- dt_yml += re.sub('^', ' ' * 6, te, flags=re.MULTILINE) + '\n\n'
-
- else:
- te = self.d(dt['template']).decode('utf-8') if len(dt['template'].strip()) > 0 else ''
-
- if te == '':
- dt_yml += ' template: ""\n'
- else:
- dt_yml += ' template: |2\n'
- dt_yml += re.sub('^', ' ' * 4, te, flags=re.MULTILINE) + '\n\n'
-
- if not dt_yml.endswith('\n\n'):
- dt_yml += '\n'
-
- dt_yml += 'revision: ' + str(dt_revision) + '\n'
- dt_yml += 'dataset: "' + dt['dataset'] + '"\n'
-
- if dt_encrypted:
- dt_yml += 'encrypted: 1\n'
+ dt_yml += ' template: |2\n'
+ dt_yml += re.sub('^', ' ' * 4, te, flags=re.MULTILINE) + '\n\n'
+
+ if not dt_yml.endswith('\n\n'):
+ dt_yml += '\n'
+
+ dt_yml += 'revision: ' + str(dt_revision) + '\n'
+ dt_yml += 'dataset: "' + dt['dataset'] + '"\n'
+
+ if dt_encrypted:
+ dt_yml += 'encrypted: 1\n'
+
+ def update_dt(rdt, dt_yml, r):
+ r = authenticate_dt(rdt, r)
+
+ if r[1] != 401:
+ if dt_protected:
+ if dt_opassword != '' or dt_mpassword != '':
+ if dt_opassword != '':
+ dt_yml += 'dt_password: "' + binascii.hexlify(self.derive_key(dt_opassword)).decode('utf-8') + '"\n'
+
+ if dt_mpassword != '':
+ dt_yml += 'dt_mpassword: "' + binascii.hexlify(self.derive_key(dt_mpassword)).decode('utf-8') + '"\n'
+
+ else:
+ if mo != None:
+ dt_yml += 'dt_password: "' + mo.group(1) + '"\n'
+
+ if mm != None:
+ dt_yml += 'dt_mpassword: "' + mm.group(1) + '"\n'
+
+ return dt_yml, r
+
+ def add_client_fields(dt_yml, remote_addr):
+ dt_yml += 'remote_addr: "' + remote_addr + '"\n'
+ dt_yml += 'updated: "' + str(int(time.time())) + '"\n'
+ return dt_yml
if 'id' in params:
if re.search(r'^[A-Za-z0-9_-]{1,24}$', params['id']):
@@ -740,50 +785,14 @@ def html_escape(text):
else:
raise Exception("invalid link format")
+ elif fpath == '/delete_link':
+ raise Exception("link id is required")
+
else:
dt_link = self.encode_link(hashlib.sha256((str(uuid.uuid1()) + ':' + dt_yml).encode('utf-8')).digest()[:6])
dt_filename = 'jfx_' + dt_link + '.yml'
- def update_dt(rdt, dt_yml, r):
- mm = re.search(r'dt_mpassword: "(\S+)"', rdt)
- mo = re.search(r'dt_password: "(\S+)"', rdt)
-
- if mm != None or mo != None:
- if dt_password != '':
- rpassword = mm.group(1) if mm != None else mo.group(1)
- t = binascii.unhexlify(rpassword.encode('utf-8'))
- if t != self.derive_key(dt_password, t[2:int(t[1]) + 2], t[0]):
- cheaders['X-Dt-Authentication'] = 'Modify' if (mm != None) else 'Open'
- r = [ 'text/plain', 401, '401 Unauthorized\r\n', sys._getframe().f_lineno ]
-
- else:
- cheaders['X-Dt-Authentication'] = 'Modify' if (mm != None) else 'Open'
- r = [ 'text/plain', 401, '401 Unauthorized\r\n', sys._getframe().f_lineno ]
-
- if r[1] != 401:
- if dt_protected:
- if dt_opassword != '' or dt_mpassword != '':
- if dt_opassword != '':
- dt_yml += 'dt_password: "' + binascii.hexlify(self.derive_key(dt_opassword)).decode('utf-8') + '"\n'
-
- if dt_mpassword != '':
- dt_yml += 'dt_mpassword: "' + binascii.hexlify(self.derive_key(dt_mpassword)).decode('utf-8') + '"\n'
-
- else:
- if mo != None:
- dt_yml += 'dt_password: "' + mo.group(1) + '"\n'
-
- if mm != None:
- dt_yml += 'dt_mpassword: "' + mm.group(1) + '"\n'
-
- return dt_yml, r
-
- def add_client_fields(dt_yml, remote_addr):
- dt_yml += 'remote_addr: "' + remote_addr + '"\n'
- dt_yml += 'updated: "' + str(int(time.time())) + '"\n'
- return dt_yml
-
if aws_s3_url:
rr = aws_s3_get(aws_s3_url, dt_filename)
if rr.status_code == 200:
@@ -808,9 +817,22 @@ def add_client_fields(dt_yml, remote_addr):
r = [ 'text/plain', 409, '409 Conflict\r\n', sys._getframe().f_lineno ]
if r[1] != 409:
- dt_yml, r = update_dt(rr.text, dt_yml, r)
+ if fpath == '/get_link':
+ dt_yml, r = update_dt(rr.text, dt_yml, r)
+
+ elif fpath == '/delete_link':
+ r = authenticate_dt(rr.text, r)
+
+ if r[1] != 401:
+ rr = aws_s3_delete(aws_s3_url, dt_filename)
- if r[1] == 500 or r[1] == 200:
+ if rr.status_code == 204:
+ r = [ 'text/plain', 200, '200 OK\r\n', sys._getframe().f_lineno ]
+
+ elif rr.status_code == 403:
+ r = [ 'text/plain', 403, '403 Forbidden\r\n', sys._getframe().f_lineno ]
+
+ if fpath != '/delete_link' and (r[1] == 500 or r[1] == 200):
dt_yml = add_client_fields(dt_yml, remote_addr)
if dt_encrypted:
@@ -866,9 +888,22 @@ def add_client_fields(dt_yml, remote_addr):
r = [ 'text/plain', 409, '409 Conflict\r\n', sys._getframe().f_lineno ]
if r[1] != 409:
- dt_yml, r = update_dt(content, dt_yml, r)
+ if fpath == '/get_link':
+ dt_yml, r = update_dt(content, dt_yml, r)
+
+ elif fpath == '/delete_link':
+ r = authenticate_dt(content, r)
+
+ if r[1] != 401:
+ rr = github_delete(github_url, dt_filename, sha)
- if r[1] == 500 or r[1] == 200:
+ if str(rr.status_code).startswith('2'):
+ r = [ 'text/plain', 200, '200 OK\r\n', sys._getframe().f_lineno ]
+
+ elif rr.status_code == 401:
+ r = [ 'text/plain', 403, '403 Forbidden\r\n', sys._getframe().f_lineno ]
+
+ if fpath != '/delete_link' and (r[1] == 500 or r[1] == 200):
dt_yml = add_client_fields(dt_yml, remote_addr)
if dt_encrypted:
@@ -890,9 +925,6 @@ def add_client_fields(dt_yml, remote_addr):
elif rr.status_code == 401:
r = [ 'text/plain', 403, '403 Forbidden\r\n', sys._getframe().f_lineno ]
- else:
- print(rr.text)
-
else:
dt_filename = os.path.normpath(repository + '/' + dt_filename)
@@ -918,9 +950,17 @@ def add_client_fields(dt_yml, remote_addr):
r = [ 'text/plain', 409, '409 Conflict\r\n', sys._getframe().f_lineno ]
if r[1] != 409:
- dt_yml, r = update_dt(rr, dt_yml, r)
+ if fpath == '/get_link':
+ dt_yml, r = update_dt(rr, dt_yml, r)
+
+ elif fpath == '/delete_link':
+ r = authenticate_dt(rr, r)
- if r[1] == 500 or r[1] == 200:
+ if r[1] != 401:
+ os.remove(dt_filename)
+ r = [ 'text/plain', 200, '200 OK\r\n', sys._getframe().f_lineno ]
+
+ if fpath != '/delete_link' and (r[1] == 500 or r[1] == 200):
dt_yml = add_client_fields(dt_yml, remote_addr)
if dt_encrypted:
@@ -933,11 +973,11 @@ def add_client_fields(dt_yml, remote_addr):
else:
r = [ 'text/plain', 400, '400 Bad Request\r\n', sys._getframe().f_lineno ]
- if r[1] == 500 or r[1] == 200:
- with open(dt_filename, 'w') as f:
- f.write(dt_yml)
+ if r[1] != 400:
+ with open(dt_filename, 'w') as f:
+ f.write(dt_yml)
- r = [ 'text/plain', 200, dt_link + '\r\n', sys._getframe().f_lineno ]
+ r = [ 'text/plain', 200, dt_link + '\r\n', sys._getframe().f_lineno ]
else:
r = [ 'text/plain', 429, '429 Too Many Requests\r\n', sys._getframe().f_lineno ]
@@ -1229,6 +1269,17 @@ def aws_s3_authorization(method, fname, region, headers):
return headers
+def aws_s3_delete(s3_url, fname):
+ headers = {
+ 'Host': s3_url,
+ 'Content-Type': 'text/plain',
+ 'x-amz-content-sha256': hashlib.sha256(b'').hexdigest(),
+ 'x-amz-date': datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ')
+ }
+ headers = aws_s3_authorization('DELETE', fname, s3_url.split('.')[2], headers)
+ return requests.delete('https://' + s3_url + '/' + fname, headers=headers)
+
+
def aws_s3_put(s3_url, fname, content, ctype):
content = gzip.compress(content.encode('utf-8'))
headers = {
@@ -1254,6 +1305,25 @@ def aws_s3_get(s3_url, fname):
return requests.get('https://' + s3_url + '/' + fname, headers=headers)
+def github_delete(github_url, fname, sha=None):
+ headers = {
+ 'Authorization': 'Token ' + github_token,
+ 'Content-Type': 'application/json'
+ }
+
+ data = {
+ 'message': 'Delete ' + fname
+ }
+
+ if ':' in github_url:
+ github_url, data['branch'] = github_url.split(':', 1)
+
+ if sha is not None:
+ data['sha'] = sha
+
+ return requests.delete('https://api.github.com/repos/' + github_url + '/contents/' + fname, headers=headers, data=json.dumps(data))
+
+
def github_put(github_url, fname, content, sha=None):
headers = {
'Authorization': 'Token ' + github_token,
diff --git a/www/index.html b/www/index.html
index fe4cd1b..e26c250 100644
--- a/www/index.html
+++ b/www/index.html
@@ -32,7 +32,7 @@
-
+
@@ -100,17 +100,25 @@