From c566b79d517017ffd5dac47ba0a0e6f5e4e25e8c Mon Sep 17 00:00:00 2001 From: Dzuk <32108484+dzuk-mutant@users.noreply.github.com> Date: Thu, 30 Jan 2020 13:34:49 +0000 Subject: [PATCH 1/6] Refactor of image exporting code The image exporting code that used to just exist in export_thread.py has been broken out into export_task.py and imacr_proc.py. In these new files, the image exporting code has been entirely refactored to read much better and to no longer have repeating lines of code. - Improved some readability of code in export.py. - paths.py renamed to dest_paths.py. --- paths.py => dest_paths.py | 0 export.py | 34 ++++-- export_task.py | 129 +++++++++++++++++++++ export_thread.py | 237 ++------------------------------------ image_proc.py | 110 ++++++++++++++++++ 5 files changed, 270 insertions(+), 240 deletions(-) rename paths.py => dest_paths.py (100%) create mode 100644 export_task.py create mode 100644 image_proc.py diff --git a/paths.py b/dest_paths.py similarity index 100% rename from paths.py rename to dest_paths.py diff --git a/export.py b/export.py index f7b22dd..5ac651c 100644 --- a/export.py +++ b/export.py @@ -4,7 +4,7 @@ from export_thread import ExportThread from exception import FilterException -from paths import format_path, format_resolve +from dest_paths import format_path, format_resolve import log import exif import svg @@ -40,22 +40,33 @@ def export(m, filtered_emoji, input_path, formats, path, src_size, if 'src' not in e: raise ValueError(f"The emoji '{short}' is missing an 'src' attribute. It needs to have one.") - srcpath = os.path.join(m.homedir, input_path, e['src']) + + + # try to see if this exists + srcpath = os.path.join(m.homedir, input_path, e['src']) try: emoji_svg = open(srcpath, 'r').read() except Exception: raise ValueError(f"This source image for emoji '{short}' could not be loaded: {srcpath}") + + # the SVG size check (-q) if src_size is not None: - imgsize = svg.get_viewbox_size(emoji_svg) - if imgsize != src_size: - raise ValueError("The source image size for emoji '{}' is not what was expected. It's supposed to be {}, but it's actually {}.".format( - short, - str(src_size[0]) + 'x' + str(src_size[1]), - str(imgsize[0]) + 'x' + str(imgsize[1]) - )) + img_size = svg.get_viewbox_size(emoji_svg) + + if img_size != src_size: + raise ValueError("""The source image size for emoji '{}' is not what + was expected. It's supposed to be {}, but it's actually + {}.""".format( + short, + str(src_size[0]) + 'x' + str(src_size[1]), + str(img_size[0]) + 'x' + str(img_size[1]) + )) + + + # add the emoji to exporting_emoji if it's passed all the tests. exporting_emoji.append(e) @@ -74,8 +85,9 @@ def export(m, filtered_emoji, input_path, formats, path, src_size, # -------------------------------------------------------------------------- # declare some specs of this export. log.out("Exporting emoji...", 36) - log.out(f"- {', '.join(formats)}") - log.out(f"- to '{path}'") + log.out(f"- {', '.join(formats)}") # print formats + log.out(f"- to '{path}'") # print out path + if num_threads > 1: log.out(f"- {num_threads} threads") else: diff --git a/export_task.py b/export_task.py new file mode 100644 index 0000000..c1f20ca --- /dev/null +++ b/export_task.py @@ -0,0 +1,129 @@ + +import os +import pathlib +import subprocess + +import svg +import image_proc + +def to_svg(emoji_svg, out_path, license=None): + """ + SVG exporting function. Doesn't create temporary files. + Will append license if requested. + """ + if license: + final_svg = svg.add_license(emoji_svg, license) + else: + final_svg = emoji_svg + + # write SVG out to file + try: + out = open(out_path, 'w') + out.write(final_svg) + out.close() + except Exception as e: + raise Exception(f'Could not write SVG to file: {e}') + + + + + + + +def to_png(emoji_svg, out_path, renderer, size, name): + """ + PNG Exporting function. Creates temporary SVGs + first before converting to PNG. + """ + # saving SVG to a temporary file + tmp_name = '.tmp' + name + '.svg' + + # try to write a temporary SVG. + image_proc.write_temp_svg(emoji_svg, tmp_name) + # try to render the SVG. + image_proc.render_svg(tmp_name, out_path, renderer, size) + + # delete temporary files + os.remove(tmp_name) + + + + + + + +def to_webp(emoji_svg, out_path, renderer, size, name): + """ + WebP Exporting function. Creates temporary SVGs and PNGs + first before converting to WebP. + """ + + tmp_svg_path = '.tmp' + name + '.svg' + tmp_png_path = '.tmp' + name + '.png' + + # try to write a temporary SVG. + image_proc.write_temp_svg(emoji_svg, tmp_svg_path) + # try to render the SVG. + image_proc.render_svg(tmp_svg_path, tmp_png_path, renderer, size) + # try to convert to WebP. + image_proc.convert_webp(tmp_png_path, out_path) + + + + # delete temporary files + os.remove(tmp_svg_path) + os.remove(tmp_png_path) + + + + + + +def to_avif(emoji_svg, out_path, renderer, size, name): + """ + Lossless AVIF Exporting function. Creates temporary SVGs and PNGs + first before converting to AVIF. + """ + + tmp_svg_path = '.tmp' + name + '.svg' + tmp_png_path = '.tmp' + name + '.png' + + + # try to write a temporary SVG. + image_proc.write_temp_svg(emoji_svg, tmp_svg_path) + # try to render the SVG. + image_proc.render_svg(tmp_svg_path, tmp_png_path, renderer, size) + # try to convert to AVIF. + image_proc.convert_avif(tmp_png_path, out_path) + + + # delete temporary files + os.remove(tmp_svg_path) + os.remove(tmp_png_path) + + + + + + +def to_flif(emoji_svg, out_path, renderer, size, name): + """ + FLIF Exporting function. Creates temporary SVGs and PNGs + first before converting to FLIF. + """ + + tmp_svg_path = '.tmp' + name + '.svg' + tmp_png_path = '.tmp' + name + '.png' + + # try to write a temporary SVG. + image_proc.write_temp_svg(emoji_svg, tmp_svg_path) + # try to render the SVG. + image_proc.render_svg(tmp_svg_path, tmp_png_path, renderer, size) + # try to convert to WebP. + image_proc.convert_flif(tmp_png_path, out_path) + + + + # delete temporary files + os.remove(tmp_svg_path) + os.remove(tmp_png_path) diff --git a/export_thread.py b/export_thread.py index fd44a47..48b94a2 100644 --- a/export_thread.py +++ b/export_thread.py @@ -5,7 +5,8 @@ import threading from exception import FilterException -from paths import format_path +from dest_paths import format_path +import export_task import svg import log @@ -25,7 +26,7 @@ def __init__(self, queue, name, total, m, input_path, formats, path, self.path = path self.renderer = renderer self.err = None - # this essentially tells self.run() to stop running if it is + # this essentially tells self.run() to stop running if it is True self.kill_flag = False # the actual thread part of this thread self.thread = threading.Thread(target=self.run) @@ -53,228 +54,6 @@ def msg(self, s, color=37, indent=0): - def export_svg(self, emoji_svg, path, license=None): - """ - SVG exporting function - """ - if license: - final_svg = svg.add_license(emoji_svg, license) - else: - final_svg = emoji_svg - - # write SVG out to file - try: - out = open(path, 'w') - out.write(final_svg) - out.close() - except Exception: - raise Exception('Could not write to file: ' + path) - - def export_png(self, emoji_svg, size, path): - - # saving SVG to a temporary file - tmp_name = '.tmp' + self.name + '.svg' - try: - f = open(tmp_name, 'w') - f.write(emoji_svg) - f.close() - except IOError: - raise Exception('Could not write to temporary file: ' + tmp_name) - - # export the SVG to a PNG based on the user's renderer - if self.renderer == 'inkscape': - cmd = ['inkscape', os.path.abspath(tmp_name), - '--export-png=' + os.path.abspath(path), - '-h', str(size), '-w', str(size)] - elif self.renderer == 'rendersvg': - cmd = ['rendersvg', '-w', str(size), '-h', str(size), - os.path.abspath(tmp_name), os.path.abspath(path)] - elif self.renderer == 'imagemagick': - cmd = ['convert', '-background', 'none', '-density', str(size / 32 * 128), - '-resize', str(size) + 'x' + str(size), os.path.abspath(tmp_name), os.path.abspath(path)] - else: - raise AssertionError - try: - r = subprocess.run(cmd, stdout=subprocess.DEVNULL).returncode - except Exception as e: - raise Exception('Rasteriser invocation failed: ' + str(e)) - if r: - raise Exception('Rasteriser returned error code: ' + str(r)) - - # delete temporary files - os.remove(tmp_name) - - def export_flif(self, emoji_svg, size, path): - """ - FLIF Exporting function. Creates temporary PNGs first before converting to WebP. - """ - - tmp_svg_name = '.tmp' + self.name + '.svg' - tmp_png_name = '.tmp' + self.name + '.png' - - - # try to write temporary SVG - try: - f = open(tmp_svg_name, 'w') - f.write(emoji_svg) - f.close() - - except IOError: - raise Exception('Could not write to temporary file: ' + tmp_svg_name) - - - # export the SVG to a temporary PNG based on the user's renderer - if self.renderer == 'inkscape': - cmd_png = ['inkscape', os.path.abspath(tmp_svg_name), - '--export-png=' + os.path.abspath(tmp_png_name), - '-h', str(size), '-w', str(size)] - elif self.renderer == 'rendersvg': - cmd_png = ['rendersvg', '-w', str(size), '-h', str(size), - os.path.abspath(tmp_svg_name), os.path.abspath(tmp_png_name)] - elif self.renderer == 'imagemagick': - cmd_png = ['convert', '-background', 'none', '-density', str(size / 32 * 128), - '-resize', str(size) + 'x' + str(size), os.path.abspath(tmp_svg_name), os.path.abspath(tmp_png_name)] - else: - raise AssertionError - - try: - r = subprocess.run(cmd_png, stdout=subprocess.DEVNULL).returncode - except Exception as e: - raise Exception('PNG rasteriser invocation failed: ' + str(e)) - if r: - raise Exception('PNG rasteriser returned error code: ' + str(r)) - - - # try to export FLIF - cmd_flif = ['flif', '-e', '--overwrite', '-Q100', os.path.abspath(tmp_png_name), os.path.abspath(path)] - - try: - r = subprocess.run(cmd_flif, stdout=subprocess.DEVNULL).returncode - except Exception as e: - raise Exception('FLIF converter invocation failed: ' + str(e)) - if r: - raise Exception('FLIF converter returned error code: ' + str(r)) - - - - # delete temporary files - os.remove(tmp_svg_name) - os.remove(tmp_png_name) - - def export_webp(self, emoji_svg, size, path): - """ - WebP Exporting function. Creates temporary PNGs first before converting to WebP. - """ - - tmp_svg_name = '.tmp' + self.name + '.svg' - tmp_png_name = '.tmp' + self.name + '.png' - - - # try to write a temporary SVG - try: - f = open(tmp_svg_name, 'w') - f.write(emoji_svg) - f.close() - - except IOError: - raise Exception('Could not write to temporary file: ' + tmp_svg_name) - - - # export the SVG to a temporary PNG based on the user's renderer - if self.renderer == 'inkscape': - cmd_png = ['inkscape', os.path.abspath(tmp_svg_name), - '--export-png=' + os.path.abspath(tmp_png_name), - '-h', str(size), '-w', str(size)] - elif self.renderer == 'rendersvg': - cmd_png = ['rendersvg', '-w', str(size), '-h', str(size), - os.path.abspath(tmp_svg_name), os.path.abspath(tmp_png_name)] - elif self.renderer == 'imagemagick': - cmd_png = ['convert', '-background', 'none', '-density', str(size / 32 * 128), - '-resize', str(size) + 'x' + str(size), os.path.abspath(tmp_svg_name), os.path.abspath(tmp_png_name)] - else: - raise AssertionError - - - try: - r = subprocess.run(cmd_png, stdout=subprocess.DEVNULL).returncode - except Exception as e: - raise Exception('PNG rasteriser invocation failed: ' + str(e)) - if r: - raise Exception('PNG rasteriser returned error code: ' + str(r)) - - - # try to export WebP - cmd_webp = ['cwebp', '-lossless', '-quiet', os.path.abspath(tmp_png_name), '-o', os.path.abspath(path)] - - try: - r = subprocess.run(cmd_webp, stdout=subprocess.DEVNULL).returncode - except Exception as e: - raise Exception('WebP converter invocation failed: ' + str(e)) - if r: - raise Exception('WebP converter returned error code: ' + str(r)) - - - - # delete temporary files - os.remove(tmp_svg_name) - os.remove(tmp_png_name) - - def export_avif(self, emoji_svg, size, path): - """ - Lossless AVIF Exporting function. Creates temporary PNGs first before converting to AVIF. - """ - - tmp_svg_name = '.tmp' + self.name + '.svg' - tmp_png_name = '.tmp' + self.name + '.png' - - - # try to write a temporary SVG - try: - f = open(tmp_svg_name, 'w') - f.write(emoji_svg) - f.close() - - except IOError: - raise Exception('Could not write to temporary file: ' + tmp_svg_name) - - - # export the SVG to a temporary PNG based on the user's renderer - if self.renderer == 'inkscape': - cmd_png = ['inkscape', os.path.abspath(tmp_svg_name), - '--export-png=' + os.path.abspath(tmp_png_name), - '-h', str(size), '-w', str(size)] - elif self.renderer == 'rendersvg': - cmd_png = ['rendersvg', '-w', str(size), '-h', str(size), - os.path.abspath(tmp_svg_name), os.path.abspath(tmp_png_name)] - elif self.renderer == 'imagemagick': - cmd_png = ['convert', '-background', 'none', '-density', str(size / 32 * 128), - '-resize', str(size) + 'x' + str(size), os.path.abspath(tmp_svg_name), os.path.abspath(tmp_png_name)] - else: - raise AssertionError - - - try: - r = subprocess.run(cmd_png, stdout=subprocess.DEVNULL).returncode - except Exception as e: - raise Exception('PNG rasteriser invocation failed: ' + str(e)) - if r: - raise Exception('PNG rasteriser returned error code: ' + str(r)) - - - # try to export AVIF - cmd_avif = ['avif', '-e', os.path.abspath(tmp_png_name), '-o', os.path.abspath(path), '--lossless'] - - try: - r = subprocess.run(cmd_avif, stdout=subprocess.DEVNULL).returncode - except Exception as e: - raise Exception('AVIF converter invocation failed: ' + str(e)) - if r: - raise Exception('AVIF converter returned error code: ' + str(r)) - - - # delete temporary files - os.remove(tmp_svg_name) - os.remove(tmp_png_name) def export_emoji(self, emoji, emoji_svg, f, path, license): """ @@ -293,28 +72,28 @@ def export_emoji(self, emoji, emoji_svg, f, path, license): # run a format-specific export task on the emoji. if f == 'svg': - self.export_svg(emoji_svg, final_path, license.get('svg')) + export_task.to_svg(emoji_svg, final_path, license.get('svg')) elif f.startswith('png-'): try: size = int(f[4:]) except ValueError: raise ValueError(f"The end ('{f[4:]}') of a format you gave ('{f}') isn't a number. It must be a number.") - self.export_png(emoji_svg, size, final_path) + export_task.to_png(emoji_svg, final_path, self.renderer, size, self.name) elif f.startswith('flif-'): try: size = int(f[5:]) except ValueError: raise ValueError(f"The end ('{f[5:]}') of a format you gave ('{f}') isn't a number. It must be a number.") - self.export_flif(emoji_svg, size, final_path) + export_task.to_flif(emoji_svg, final_path, self.renderer, size, self.name) elif f.startswith('webp-'): try: size = int(f[5:]) except ValueError: raise ValueError(f"The end ('{f[5:]}') of a format you gave ('{f}') isn't a number. It must be a number.") - self.export_webp(emoji_svg, size, final_path) + export_task.to_webp(emoji_svg, final_path, self.renderer, size, self.name) elif f.startswith('avif-'): @@ -322,7 +101,7 @@ def export_emoji(self, emoji, emoji_svg, f, path, license): size = int(f[5:]) except ValueError: raise ValueError(f"The end ('{f[5:]}') of a format you gave ('{f}') isn't a number. It must be a number.") - self.export_avif(emoji_svg, size, final_path) + export_task.to_avif(emoji_svg, final_path, self.renderer, size, self.name) else: raise ValueError('Invalid format: ' + f) diff --git a/image_proc.py b/image_proc.py new file mode 100644 index 0000000..9eb1dee --- /dev/null +++ b/image_proc.py @@ -0,0 +1,110 @@ +import os +import pathlib +import subprocess + + + +def write_temp_svg(emoji_svg, out_path): + """ + Tries to write a temporary SVG out for exporting. + """ + try: + f = open(out_path, 'w') + f.write(emoji_svg) + f.close() + + except IOError: + raise Exception('Could not write SVG to temporary file: ' + tmp_name) + + + + + + +def render_svg(svg_in, png_out, renderer, size): + """ + Export a given SVG to a PNG based on the user's renderer choice. + """ + + if renderer == 'inkscape': + cmd = ['inkscape', os.path.abspath(svg_in), + '--export-png=' + os.path.abspath(png_out), + '-h', str(size), '-w', str(size)] + + elif renderer == 'rendersvg': + cmd = ['rendersvg', '-w', str(size), '-h', str(size), + os.path.abspath(svg_in), os.path.abspath(png_out)] + + elif renderer == 'imagemagick': + cmd = ['convert', '-background', 'none', '-density', str(size / 32 * 128), + '-resize', str(size) + 'x' + str(size), os.path.abspath(svg_in), os.path.abspath(png_out)] + else: + raise AssertionError + + + try: + r = subprocess.run(cmd, stdout=subprocess.DEVNULL).returncode + + except Exception as e: + raise Exception('Rasteriser invocation failed: ' + str(e)) + if r: + raise Exception('Rasteriser returned error code: ' + str(r)) + + + + + + + +def convert_webp(png_in, webp_out): + """ + Converts a PNG at `png_in` to a Lossless WebP at `webp_out`. + + Will raise an exception if trying to invoke the converter failed. + """ + cmd_webp = ['cwebp', '-lossless', '-quiet', os.path.abspath(png_in), '-o', os.path.abspath(webp_out)] + + try: + r = subprocess.run(cmd_webp, stdout=subprocess.DEVNULL).returncode + except Exception as e: + raise Exception('WebP converter invocation failed: ' + str(e)) + if r: + raise Exception('WebP converter returned error code: ' + str(r)) + + + + + + +def convert_avif(png_in, avif_out): + """ + Converts a PNG at `png_in` to a Lossless AVIF at `avif_out`. + + Will raise an exception if trying to invoke the converter failed. + """ + cmd_avif = ['avif', '-e', os.path.abspath(png_in), '-o', os.path.abspath(avif_out), '--lossless'] + + try: + r = subprocess.run(cmd_avif, stdout=subprocess.DEVNULL).returncode + except Exception as e: + raise Exception('AVIF converter invocation failed: ' + str(e)) + if r: + raise Exception('AVIF converter returned error code: ' + str(r)) + + + + +def convert_flif(png_in, flif_out): + """ + Converts a PNG at `png_in` to a FLIF at `flif_out`. + + Will raise an exception if trying to invoke the converter failed. + """ + cmd_flif = ['flif', '-e', '--overwrite', '-Q100', os.path.abspath(png_in), os.path.abspath(flif_out)] + + try: + r = subprocess.run(cmd_flif, stdout=subprocess.DEVNULL).returncode + except Exception as e: + raise Exception('FLIF converter invocation failed: ' + str(e)) + if r: + raise Exception('FLIF converter returned error code: ' + str(r)) From 7782fe0613a9baa40339cf5168712a60daba91ac Mon Sep 17 00:00:00 2001 From: Dzuk <32108484+dzuk-mutant@users.noreply.github.com> Date: Thu, 30 Jan 2020 13:52:17 +0000 Subject: [PATCH 2/6] Further refactor of image exporting code All of the raster image out functions in export_task.py have been condensed down into one to_raster() function. - Improved an error printout in orxport.py. --- export_task.py | 102 +++++++++-------------------------------------- export_thread.py | 11 ++--- orxport.py | 2 +- 3 files changed, 26 insertions(+), 89 deletions(-) diff --git a/export_task.py b/export_task.py index c1f20ca..c6fd9ab 100644 --- a/export_task.py +++ b/export_task.py @@ -29,101 +29,37 @@ def to_svg(emoji_svg, out_path, license=None): - -def to_png(emoji_svg, out_path, renderer, size, name): - """ - PNG Exporting function. Creates temporary SVGs - first before converting to PNG. - """ - # saving SVG to a temporary file - tmp_name = '.tmp' + name + '.svg' - - # try to write a temporary SVG. - image_proc.write_temp_svg(emoji_svg, tmp_name) - # try to render the SVG. - image_proc.render_svg(tmp_name, out_path, renderer, size) - - # delete temporary files - os.remove(tmp_name) - - - - - - - -def to_webp(emoji_svg, out_path, renderer, size, name): - """ - WebP Exporting function. Creates temporary SVGs and PNGs - first before converting to WebP. - """ +def to_raster(emoji_svg, out_path, renderer, format, size, name): tmp_svg_path = '.tmp' + name + '.svg' tmp_png_path = '.tmp' + name + '.png' # try to write a temporary SVG. image_proc.write_temp_svg(emoji_svg, tmp_svg_path) - # try to render the SVG. - image_proc.render_svg(tmp_svg_path, tmp_png_path, renderer, size) - # try to convert to WebP. - image_proc.convert_webp(tmp_png_path, out_path) - - # delete temporary files - os.remove(tmp_svg_path) - os.remove(tmp_png_path) - - - - - - -def to_avif(emoji_svg, out_path, renderer, size, name): - """ - Lossless AVIF Exporting function. Creates temporary SVGs and PNGs - first before converting to AVIF. - """ - - tmp_svg_path = '.tmp' + name + '.svg' - tmp_png_path = '.tmp' + name + '.png' + if format == "png": + # single-step process + image_proc.render_svg(tmp_svg_path, out_path, renderer, size) + else: + # multi-step process + image_proc.render_svg(tmp_svg_path, tmp_png_path, renderer, size) - # try to write a temporary SVG. - image_proc.write_temp_svg(emoji_svg, tmp_svg_path) - # try to render the SVG. - image_proc.render_svg(tmp_svg_path, tmp_png_path, renderer, size) - # try to convert to AVIF. - image_proc.convert_avif(tmp_png_path, out_path) + if format == "webp": + image_proc.convert_webp(tmp_png_path, out_path) + elif format == "avif": + image_proc.convert_avif(tmp_png_path, out_path) + elif format == "flif": + image_proc.convert_flif(tmp_png_path, out_path) + else: + os.remove(tmp_svg_path) + os.remove(tmp_png_path) + raise ValueError(f"This function wasn't given a correct format! ({format})") # delete temporary files os.remove(tmp_svg_path) - os.remove(tmp_png_path) - - - - - -def to_flif(emoji_svg, out_path, renderer, size, name): - """ - FLIF Exporting function. Creates temporary SVGs and PNGs - first before converting to FLIF. - """ - - tmp_svg_path = '.tmp' + name + '.svg' - tmp_png_path = '.tmp' + name + '.png' - - # try to write a temporary SVG. - image_proc.write_temp_svg(emoji_svg, tmp_svg_path) - # try to render the SVG. - image_proc.render_svg(tmp_svg_path, tmp_png_path, renderer, size) - # try to convert to WebP. - image_proc.convert_flif(tmp_png_path, out_path) - - - - # delete temporary files - os.remove(tmp_svg_path) - os.remove(tmp_png_path) + if format != "png": + os.remove(tmp_png_path) diff --git a/export_thread.py b/export_thread.py index 48b94a2..8023fb7 100644 --- a/export_thread.py +++ b/export_thread.py @@ -74,34 +74,35 @@ def export_emoji(self, emoji, emoji_svg, f, path, license): if f == 'svg': export_task.to_svg(emoji_svg, final_path, license.get('svg')) + + elif f.startswith('png-'): try: size = int(f[4:]) except ValueError: raise ValueError(f"The end ('{f[4:]}') of a format you gave ('{f}') isn't a number. It must be a number.") - export_task.to_png(emoji_svg, final_path, self.renderer, size, self.name) + export_task.to_raster(emoji_svg, final_path, self.renderer, "png", size, self.name) elif f.startswith('flif-'): try: size = int(f[5:]) except ValueError: raise ValueError(f"The end ('{f[5:]}') of a format you gave ('{f}') isn't a number. It must be a number.") - export_task.to_flif(emoji_svg, final_path, self.renderer, size, self.name) + export_task.to_raster(emoji_svg, final_path, self.renderer, "flif", size, self.name) elif f.startswith('webp-'): try: size = int(f[5:]) except ValueError: raise ValueError(f"The end ('{f[5:]}') of a format you gave ('{f}') isn't a number. It must be a number.") - export_task.to_webp(emoji_svg, final_path, self.renderer, size, self.name) - + export_task.to_raster(emoji_svg, final_path, self.renderer, "webp", size, self.name) elif f.startswith('avif-'): try: size = int(f[5:]) except ValueError: raise ValueError(f"The end ('{f[5:]}') of a format you gave ('{f}') isn't a number. It must be a number.") - export_task.to_avif(emoji_svg, final_path, self.renderer, size, self.name) + export_task.to_raster(emoji_svg, final_path, self.renderer, "avif", size, self.name) else: raise ValueError('Invalid format: ' + f) diff --git a/orxport.py b/orxport.py index d296af4..eb1db2d 100755 --- a/orxport.py +++ b/orxport.py @@ -195,7 +195,7 @@ def main(): # validate basic input that can't be checked while in progress if renderer not in RENDERERS: - raise Exception(f"{renderer} is not a renderer you can use in orxporter.") + raise Exception(f"There's a mistake in your command arguments. '{renderer}' is not a renderer you can use in orxporter.") From d63036ce1fe5c8918b2d60ad7f0a5d2e8423b36f Mon Sep 17 00:00:00 2001 From: Dzuk <32108484+dzuk-mutant@users.noreply.github.com> Date: Thu, 30 Jan 2020 13:53:50 +0000 Subject: [PATCH 3/6] Added documentation export_task.to to_raster(). --- export_task.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/export_task.py b/export_task.py index c6fd9ab..d020934 100644 --- a/export_task.py +++ b/export_task.py @@ -30,7 +30,10 @@ def to_svg(emoji_svg, out_path, license=None): def to_raster(emoji_svg, out_path, renderer, format, size, name): - + """ + Raster exporting function. Can export to any of orxporter's supported raster formats. + Creates and deletes temporary SVG files. Might also create and delete temporary PNG files depending on the format. + """ tmp_svg_path = '.tmp' + name + '.svg' tmp_png_path = '.tmp' + name + '.png' From adb76f9ce702371463a7063430078765198eb33b Mon Sep 17 00:00:00 2001 From: Dzuk <32108484+dzuk-mutant@users.noreply.github.com> Date: Fri, 31 Jan 2020 00:57:10 +0000 Subject: [PATCH 4/6] export_thread and export refactoring - Done more refactoring of export_thread.py to remove even more repetitious code. - Moved the bulk of the initial (re: light) emoji checking process in export.export() into a new module and a new function that will likely be expanded on in the future - check.emoji(). - Added more and better documentation and comments here and there. --- check.py | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ export.py | 63 +++++++-------------------------------- export_task.py | 2 +- export_thread.py | 66 +++++++++++++++++++---------------------- image_proc.py | 8 ----- 5 files changed, 120 insertions(+), 96 deletions(-) create mode 100644 check.py diff --git a/check.py b/check.py new file mode 100644 index 0000000..e378fd5 --- /dev/null +++ b/check.py @@ -0,0 +1,77 @@ +import os + +from dest_paths import format_path, format_resolve +from exception import FilterException +import svg + +def emoji(m, filtered_emoji, input_path, formats, path, src_size, + num_threads, renderer, max_batch, verbose): + """ + Checks all emoji in a very light validation as well as checking if emoji + aren't filtered out by user choices. + + It only checks: + - If the emoji has been filtered out by user exporting options. + + - If the source SVG file exists. + (Will throw an Exception if not the case.) + + - If the shortcode ('short') attribute exists. + (Will throw an Exception if not the case.) + + - If the svg size is consistent (if a -q flag is used). + (Will throw an Exception if not the case.) + + It this doesn't result in an Exception, it returns dict containing + a list of emoji that aren't filtered out, as well as a count + of emoji that were skipped. + """ + + exporting_emoji = [] + skipped_emoji_count = 0 + + for i, e in enumerate(filtered_emoji): + + short = e.get("code", "") # to provide info on possible error printouts + + try: + format_path(path, e, 'svg') + except FilterException as ex: + if verbose: + log.out(f"- - Skipped emoji: {short} - {ex}", 34) + skipped_emoji_count += 1 + continue # skip if filtered out + + if 'src' not in e: + raise ValueError(f"The emoji '{short}' is missing an 'src' attribute. It needs to have one.") + + + + # try to see if the source SVG file exists + srcpath = os.path.join(m.homedir, input_path, e['src']) + try: + emoji_svg = open(srcpath, 'r').read() + except Exception: + raise ValueError(f"This source image for emoji '{short}' could not be loaded: {srcpath}") + + + + # the SVG size check (-q) + if src_size is not None: + img_size = svg.get_viewbox_size(emoji_svg) + + if img_size != src_size: + raise ValueError("""The source image size for emoji '{}' is not what + was expected. It's supposed to be {}, but it's actually + {}.""".format( + short, + str(src_size[0]) + 'x' + str(src_size[1]), + str(img_size[0]) + 'x' + str(img_size[1]) + )) + + # add the emoji to exporting_emoji if it's passed all the tests. + exporting_emoji.append(e) + + return { "exporting_emoji" : exporting_emoji + , "skipped_emoji_count" : skipped_emoji_count + } diff --git a/export.py b/export.py index 5ac651c..5ee6961 100644 --- a/export.py +++ b/export.py @@ -2,12 +2,14 @@ import queue import time -from export_thread import ExportThread +import check from exception import FilterException -from dest_paths import format_path, format_resolve -import log import exif -import svg +from export_thread import ExportThread +from dest_paths import format_path +import log + + @@ -19,57 +21,14 @@ def export(m, filtered_emoji, input_path, formats, path, src_size, validation of emoji metadata and running the tasks associated with exporting. """ - # verify emoji + # verify emoji (in a very basic way) # -------------------------------------------------------------------------- log.out('Checking emoji...', 36) + check_result = check.emoji(m, filtered_emoji, input_path, formats, path, src_size, + num_threads, renderer, max_batch, verbose) - - exporting_emoji = [] - skipped_emoji_count = 0 - for i, e in enumerate(filtered_emoji): - - short = e.get("code", "") # to provide info on possible error printouts - - try: - format_path(path, e, 'svg') - except FilterException as ex: - if verbose: - log.out(f"- - Skipped emoji: {short} - {ex}", 34) - skipped_emoji_count += 1 - continue #skip if filtered out - - if 'src' not in e: - raise ValueError(f"The emoji '{short}' is missing an 'src' attribute. It needs to have one.") - - - - # try to see if this exists - srcpath = os.path.join(m.homedir, input_path, e['src']) - try: - emoji_svg = open(srcpath, 'r').read() - except Exception: - raise ValueError(f"This source image for emoji '{short}' could not be loaded: {srcpath}") - - - - # the SVG size check (-q) - if src_size is not None: - img_size = svg.get_viewbox_size(emoji_svg) - - if img_size != src_size: - raise ValueError("""The source image size for emoji '{}' is not what - was expected. It's supposed to be {}, but it's actually - {}.""".format( - short, - str(src_size[0]) + 'x' + str(src_size[1]), - str(img_size[0]) + 'x' + str(img_size[1]) - )) - - - - - # add the emoji to exporting_emoji if it's passed all the tests. - exporting_emoji.append(e) + exporting_emoji = check_result["exporting_emoji"] + skipped_emoji_count = check_result["skipped_emoji_count"] if skipped_emoji_count > 0: log.out(f"- {skipped_emoji_count} emoji have been skipped, leaving {len(exporting_emoji)} emoji to export.", 34) diff --git a/export_task.py b/export_task.py index d020934..60e9d7f 100644 --- a/export_task.py +++ b/export_task.py @@ -44,8 +44,8 @@ def to_raster(emoji_svg, out_path, renderer, format, size, name): if format == "png": # single-step process image_proc.render_svg(tmp_svg_path, out_path, renderer, size) - else: + else: # multi-step process image_proc.render_svg(tmp_svg_path, tmp_png_path, renderer, size) diff --git a/export_thread.py b/export_thread.py index 8023fb7..82fc03c 100644 --- a/export_thread.py +++ b/export_thread.py @@ -5,7 +5,7 @@ import threading from exception import FilterException -from dest_paths import format_path +import dest_paths import export_task import svg import log @@ -49,17 +49,13 @@ def join(self): """ self.thread.join() - def msg(self, s, color=37, indent=0): - log.out(s, color, indent, self.name) - - - def export_emoji(self, emoji, emoji_svg, f, path, license): """ Runs a single export batch. """ - final_path = format_path(path, emoji, f) + + final_path = dest_paths.format_path(path, emoji, f) # try to make the directory for this particular export batch. try: @@ -70,42 +66,39 @@ def export_emoji(self, emoji, emoji_svg, f, path, license): raise Exception('Could not create directory: ' + dirname) - # run a format-specific export task on the emoji. + # svg format doesn't involve a resolution so it can go straight to export. if f == 'svg': export_task.to_svg(emoji_svg, final_path, license.get('svg')) - - - elif f.startswith('png-'): + else: + # any format other than svg is a raster, therefore it needs + # to have a number separated by a dash. + raster_format = f.split("-") try: - size = int(f[4:]) + size = int(raster_format[1]) except ValueError: - raise ValueError(f"The end ('{f[4:]}') of a format you gave ('{f}') isn't a number. It must be a number.") - export_task.to_raster(emoji_svg, final_path, self.renderer, "png", size, self.name) + self.err = Exception(f"""A format you gave ('{f}') isn't correct. All formats + that aren't svg must have a number separated by a dash. + (ie 'png-32', 'webp-128')""") - elif f.startswith('flif-'): - try: - size = int(f[5:]) - except ValueError: - raise ValueError(f"The end ('{f[5:]}') of a format you gave ('{f}') isn't a number. It must be a number.") - export_task.to_raster(emoji_svg, final_path, self.renderer, "flif", size, self.name) + # now the size has been retrieved, try image + # conversion based on the format. + if raster_format[0] == "png": + export_task.to_raster(emoji_svg, final_path, self.renderer, "png", size, self.name) - elif f.startswith('webp-'): - try: - size = int(f[5:]) - except ValueError: - raise ValueError(f"The end ('{f[5:]}') of a format you gave ('{f}') isn't a number. It must be a number.") - export_task.to_raster(emoji_svg, final_path, self.renderer, "webp", size, self.name) + elif raster_format[0] == "webp": + export_task.to_raster(emoji_svg, final_path, self.renderer, "webp", size, self.name) - elif f.startswith('avif-'): - try: - size = int(f[5:]) - except ValueError: - raise ValueError(f"The end ('{f[5:]}') of a format you gave ('{f}') isn't a number. It must be a number.") - export_task.to_raster(emoji_svg, final_path, self.renderer, "avif", size, self.name) + elif raster_format[0] == "flif": + export_task.to_raster(emoji_svg, final_path, self.renderer, "flif", size, self.name) - else: - raise ValueError('Invalid format: ' + f) + elif raster_format[0] == "avif": + export_task.to_raster(emoji_svg, final_path, self.renderer, "avif", size, self.name) + + else: + self.err = Exception(f"""A format you gave ('{f}') uses a file format + ('{raster_format[0]}') that orxporter + doesn't support.""") @@ -123,14 +116,17 @@ def run(self): while not self.kill_flag: # try to get an item from the queue. + # break the loop if nothing is left. try: i, emoji = self.queue.get_nowait() except queue.Empty: break # compose the file path of the emoji. - format_path(self.path, emoji, 'svg') + dest_paths.format_path(self.path, emoji, 'svg') + # check if the src attribute is in the emoji. + # if so, make a proper path out of it. if 'src' not in emoji: raise ValueError('Missing src attribute') diff --git a/image_proc.py b/image_proc.py index 9eb1dee..73ec53e 100644 --- a/image_proc.py +++ b/image_proc.py @@ -53,13 +53,9 @@ def render_svg(svg_in, png_out, renderer, size): - - - def convert_webp(png_in, webp_out): """ Converts a PNG at `png_in` to a Lossless WebP at `webp_out`. - Will raise an exception if trying to invoke the converter failed. """ cmd_webp = ['cwebp', '-lossless', '-quiet', os.path.abspath(png_in), '-o', os.path.abspath(webp_out)] @@ -74,12 +70,9 @@ def convert_webp(png_in, webp_out): - - def convert_avif(png_in, avif_out): """ Converts a PNG at `png_in` to a Lossless AVIF at `avif_out`. - Will raise an exception if trying to invoke the converter failed. """ cmd_avif = ['avif', '-e', os.path.abspath(png_in), '-o', os.path.abspath(avif_out), '--lossless'] @@ -97,7 +90,6 @@ def convert_avif(png_in, avif_out): def convert_flif(png_in, flif_out): """ Converts a PNG at `png_in` to a FLIF at `flif_out`. - Will raise an exception if trying to invoke the converter failed. """ cmd_flif = ['flif', '-e', '--overwrite', '-Q100', os.path.abspath(png_in), os.path.abspath(flif_out)] From e09027009a1c174a8a2be38928429171347aea54 Mon Sep 17 00:00:00 2001 From: Dzuk <32108484+dzuk-mutant@users.noreply.github.com> Date: Fri, 31 Jan 2020 02:58:17 +0000 Subject: [PATCH 5/6] Optimised SVGs - Optimised SVG support has now been added. - File writing functionality has been placed in one single function in a new module - files.try_write(). - Updated and improved readme and Dzuk's image format documentation. --- README.md | 4 +- dest_paths.py | 2 + docs/dzuk/image_formats.md | 21 +++++++---- export_task.py | 26 +++++++------ export_thread.py | 4 +- files.py | 17 +++++++++ image_proc.py | 75 +++++++++++++++++++++++--------------- orxport.py | 1 + 8 files changed, 100 insertions(+), 50 deletions(-) create mode 100644 files.py diff --git a/README.md b/README.md index 827684c..600bc89 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,8 @@ Mutant Standard's [code of conduct](docs/code_of_conduct.md). - Color mapping (recoloring) - Outputting emoji both as shortcode-named files (ie. 'ice_cream') and unicode codepoint-named files (ie. '1f368'). - SVG, PNG, Lossless WebP, Lossless AVIF and FLIF exports -- Supports multiple SVG renderers: rendersvg, inkscape and imagemagick +- Optional space-saving optimisations for SVG files +- Supports multiple SVG renderers: rendersvg, Inkscape and ImageMagick - Output options including emoji filtering, customisable export directory structure and filenaming - JSON output of emoji set metadata @@ -51,6 +52,7 @@ Mutant Standard's [code of conduct](docs/code_of_conduct.md). - Inkscape - ImageMagick - exiftool (Optional; for embedding EXIF license metadata) +- [svgcleaner](https://github.com/RazrFalcon/svgcleaner) (Optional; for Optimised SVG output) - [FLIF](https://github.com/FLIF-hub/FLIF) (Optional; for FLIF output) - [libwebp](https://developers.google.com/speed/webp/docs/precompiled) (Optional; for Lossless WebP output) - [go-avif](https://github.com/Kagami/go-avif) (Optional; for AVIF output) (Experimental; does not currently produce truly lossless images. We're trying to figure out why that is.) diff --git a/dest_paths.py b/dest_paths.py index 263b4d6..dcf5785 100644 --- a/dest_paths.py +++ b/dest_paths.py @@ -62,6 +62,8 @@ def format_path(path, emoji, format): # (also acts as a format check) if format == 'svg': res = res + '.svg' + elif format == 'svgo': + res = res + '.svg' elif format.startswith('png-'): res = res + '.png' elif format.startswith('flif-'): diff --git a/docs/dzuk/image_formats.md b/docs/dzuk/image_formats.md index 2fd0edc..1293782 100644 --- a/docs/dzuk/image_formats.md +++ b/docs/dzuk/image_formats.md @@ -1,21 +1,28 @@ # Formats -### Out of the box support: - +### Vector - `svg` SVG -- `png` PNG +- `svgo` Optimised SVG (requires svgcleaner) + +*(check the [readme](../../readme.md) for all the information on dependencies)* + +#### `svgo` Optimised SVG +Optimised SVG is the same format as SVG, but it's losslessly compressed to create a smaller file size (in Mutant Standard tests, optimised SVGs are 30-40% smaller). It requires an extra processing stage, and it needs the dependency listed. Check the [svgcleaner](https://github.com/RazrFalcon/svgcleaner) repo to see the documentation for it so you can see what it does to the SVG files. -### Requires extra software to install: +orxporter uses svgcleaner with the following settings: +`svgcleaner --remove-metadata=no --quiet` + + +### Raster +- `png` PNG - `webp` Lossless WebP (requires cwebp) - `avif` Lossless AVIF (requires go-avif) - `flif` FLIF (requires flif) *(check the [readme](../../readme.md) for all the information on dependencies)* -Crushed PNGs and Optimised SVGs are the same formats as PNG and SVG, but their data is arranged in such a way to create a smaller file size, they require an extra processing stage, and they need the dependencies listed. - -When choosing raster images (any format but `svg`), you have to add a size as well. you do this by adding a hyphen followed by the square size in pixels. +When choosing raster images, you have to add a size as well. you do this by adding a hyphen followed by the square size in pixels. eg. diff --git a/export_task.py b/export_task.py index 60e9d7f..492e5d0 100644 --- a/export_task.py +++ b/export_task.py @@ -3,13 +3,16 @@ import pathlib import subprocess +import files import svg import image_proc -def to_svg(emoji_svg, out_path, license=None): +def to_svg(emoji_svg, out_path, name, license=None, optimise=False): """ SVG exporting function. Doesn't create temporary files. Will append license if requested. + + Can optimise the output (ie, output to svgo) if requested. """ if license: final_svg = svg.add_license(emoji_svg, license) @@ -17,14 +20,13 @@ def to_svg(emoji_svg, out_path, license=None): final_svg = emoji_svg # write SVG out to file - try: - out = open(out_path, 'w') - out.write(final_svg) - out.close() - except Exception as e: - raise Exception(f'Could not write SVG to file: {e}') - - + if not optimise: # (svg) + files.try_write(final_svg, out_path, "final SVG") + else: # (svgo) + tmp_svg_path = '.tmp' + name + '.svg' + files.try_write(final_svg, tmp_svg_path, "temporary SVG") + image_proc.optimise_svg(tmp_svg_path, out_path) + os.remove(tmp_svg_path) @@ -38,15 +40,15 @@ def to_raster(emoji_svg, out_path, renderer, format, size, name): tmp_png_path = '.tmp' + name + '.png' # try to write a temporary SVG. - image_proc.write_temp_svg(emoji_svg, tmp_svg_path) + files.try_write(emoji_svg, tmp_svg_path, "temporary SVG") if format == "png": - # single-step process + # one-step process image_proc.render_svg(tmp_svg_path, out_path, renderer, size) else: - # multi-step process + # two-step process image_proc.render_svg(tmp_svg_path, tmp_png_path, renderer, size) if format == "webp": diff --git a/export_thread.py b/export_thread.py index 82fc03c..a4a2859 100644 --- a/export_thread.py +++ b/export_thread.py @@ -68,7 +68,9 @@ def export_emoji(self, emoji, emoji_svg, f, path, license): # svg format doesn't involve a resolution so it can go straight to export. if f == 'svg': - export_task.to_svg(emoji_svg, final_path, license.get('svg')) + export_task.to_svg(emoji_svg, final_path, self.name, license.get('svg'), optimise=False) + elif f == 'svgo': + export_task.to_svg(emoji_svg, final_path, self.name, license.get('svg'), optimise=True) else: # any format other than svg is a raster, therefore it needs diff --git a/files.py b/files.py new file mode 100644 index 0000000..fdaf63a --- /dev/null +++ b/files.py @@ -0,0 +1,17 @@ +import os + +def try_write(data, out_path, obj_name): + """ + Tries to write a file out. Creates an exception if it fails. + + 'obj_name' is a name to give the thing you're trying to export so that + if it fails, the error message that's given is more specific. + """ + + try: + f = open(out_path, 'w') + f.write(data) + f.close() + + except IOError as e: + raise Exception(f"Could not write {obj_name} to path '{out_path}'. More info: {e}") diff --git a/image_proc.py b/image_proc.py index 73ec53e..2b834f3 100644 --- a/image_proc.py +++ b/image_proc.py @@ -4,23 +4,6 @@ -def write_temp_svg(emoji_svg, out_path): - """ - Tries to write a temporary SVG out for exporting. - """ - try: - f = open(out_path, 'w') - f.write(emoji_svg) - f.close() - - except IOError: - raise Exception('Could not write SVG to temporary file: ' + tmp_name) - - - - - - def render_svg(svg_in, png_out, renderer, size): """ Export a given SVG to a PNG based on the user's renderer choice. @@ -58,14 +41,14 @@ def convert_webp(png_in, webp_out): Converts a PNG at `png_in` to a Lossless WebP at `webp_out`. Will raise an exception if trying to invoke the converter failed. """ - cmd_webp = ['cwebp', '-lossless', '-quiet', os.path.abspath(png_in), '-o', os.path.abspath(webp_out)] + cmd = ['cwebp', '-lossless', '-quiet', os.path.abspath(png_in), '-o', os.path.abspath(webp_out)] try: - r = subprocess.run(cmd_webp, stdout=subprocess.DEVNULL).returncode + r = subprocess.run(cmd, stdout=subprocess.DEVNULL).returncode except Exception as e: - raise Exception('WebP converter invocation failed: ' + str(e)) + raise Exception('Invoking the WebP converter (cwebp) failed: ' + str(e)) if r: - raise Exception('WebP converter returned error code: ' + str(r)) + raise Exception('The WebP converter returned the following: ' + str(r)) @@ -75,14 +58,14 @@ def convert_avif(png_in, avif_out): Converts a PNG at `png_in` to a Lossless AVIF at `avif_out`. Will raise an exception if trying to invoke the converter failed. """ - cmd_avif = ['avif', '-e', os.path.abspath(png_in), '-o', os.path.abspath(avif_out), '--lossless'] + cmd = ['avif', '-e', os.path.abspath(png_in), '-o', os.path.abspath(avif_out), '--lossless'] try: - r = subprocess.run(cmd_avif, stdout=subprocess.DEVNULL).returncode + r = subprocess.run(cmd, stdout=subprocess.DEVNULL).returncode except Exception as e: - raise Exception('AVIF converter invocation failed: ' + str(e)) + raise Exception('Invoking the AVIF converter (avif) failed: ' + str(e)) if r: - raise Exception('AVIF converter returned error code: ' + str(r)) + raise Exception('The AVIF converter returned the following: ' + str(r)) @@ -92,11 +75,45 @@ def convert_flif(png_in, flif_out): Converts a PNG at `png_in` to a FLIF at `flif_out`. Will raise an exception if trying to invoke the converter failed. """ - cmd_flif = ['flif', '-e', '--overwrite', '-Q100', os.path.abspath(png_in), os.path.abspath(flif_out)] + cmd = ['flif', '-e', '--overwrite', '-Q100', os.path.abspath(png_in), os.path.abspath(flif_out)] try: - r = subprocess.run(cmd_flif, stdout=subprocess.DEVNULL).returncode + r = subprocess.run(cmd, stdout=subprocess.DEVNULL).returncode + except Exception as e: + raise Exception('Invoking the FLIF converter (flif) failed: ' + str(e)) + if r: + raise Exception('The FLIF converter returned the following: ' + str(r)) + + + + +def optimise_svg(svg_in, svgo_out): + """ + Optimises an SVG at `svg_in` to `svgo_out`. + Will raise an exception if trying to invoke the optimiser failed. + """ + cmd = ['svgcleaner', os.path.abspath(svg_in), os.path.abspath(svgo_out), '--remove-metadata=no', '--quiet'] + + try: + r = subprocess.run(cmd, stdout=subprocess.DEVNULL).returncode + except Exception as e: + raise Exception('Invoking the SVG optimiser (svgcleaner) failed: ' + str(e)) + if r: + raise Exception('The SVG optimiser returned the following: ' + str(r)) + + + + +def crush_png(png_in, pngc_out): + """ + Crushes a PNG at `png_in` to `png_out`. + Will raise an exception if trying to invoke the optimiser failed. + """ + cmd = ['oxipng', os.path.abspath(png_in), '--out', os.path.abspath(pngc_out), '--quiet'] + + try: + r = subprocess.run(cmd, stdout=subprocess.DEVNULL).returncode except Exception as e: - raise Exception('FLIF converter invocation failed: ' + str(e)) + raise Exception('Invoking the PNG crusher (oxipng) failed: ' + str(e)) if r: - raise Exception('FLIF converter returned error code: ' + str(r)) + raise Exception('The PNG crusher returned the following: ' + str(r)) diff --git a/orxport.py b/orxport.py index eb1db2d..1f23815 100755 --- a/orxport.py +++ b/orxport.py @@ -54,6 +54,7 @@ -F Format (default: {DEF_OUTPUT_FORMATS[0]}) comma separated with no spaces (ie. 'svg,png-64,flif-128') - svg + - svgo - png-SIZE - flif-SIZE - webp-SIZE From 5745f88525dc18d8c2c48be4a07ea7a124058c20 Mon Sep 17 00:00:00 2001 From: Dzuk <32108484+dzuk-mutant@users.noreply.github.com> Date: Fri, 31 Jan 2020 03:39:38 +0000 Subject: [PATCH 6/6] Crushed PNGs, EXIF embedding for AVIF - Support for Crushed PNGs has been added. - EXIF metadata is now embedded in AVIF files as well as PNG. - Documentation has been updated to reflect the new crushed PNG format. - New metadata documentation for showing how to add licensing metadata to images. --- README.md | 9 +++--- dest_paths.py | 2 ++ docs/dzuk/howto.md | 6 ++++ docs/dzuk/image_formats.md | 17 ++++++++-- docs/dzuk/metadata.md | 66 ++++++++++++++++++++++++++++++++++++++ docs/kiilas/manifest.md | 2 +- export.py | 3 +- export_task.py | 4 ++- export_thread.py | 3 ++ orxport.py | 13 ++++---- 10 files changed, 110 insertions(+), 15 deletions(-) create mode 100644 docs/dzuk/metadata.md diff --git a/README.md b/README.md index 600bc89..c673bd9 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,10 @@ Mutant Standard's [code of conduct](docs/code_of_conduct.md). - Declarative language for defining semantics of emoji set - Color mapping (recoloring) -- Outputting emoji both as shortcode-named files (ie. 'ice_cream') and unicode codepoint-named files (ie. '1f368'). -- SVG, PNG, Lossless WebP, Lossless AVIF and FLIF exports -- Optional space-saving optimisations for SVG files -- Supports multiple SVG renderers: rendersvg, Inkscape and ImageMagick +- Outputting emoji both as shortcode-named files (ie. 'ice_cream') and unicode codepoint-named files (ie. '1f368') +- Export to SVG, PNG, Lossless WebP, Lossless AVIF and FLIF +- Optional lossless crushing of SVG and PNG files +- Supports multiple SVG renderers (rendersvg, Inkscape and ImageMagick) - Output options including emoji filtering, customisable export directory structure and filenaming - JSON output of emoji set metadata @@ -53,6 +53,7 @@ Mutant Standard's [code of conduct](docs/code_of_conduct.md). - ImageMagick - exiftool (Optional; for embedding EXIF license metadata) - [svgcleaner](https://github.com/RazrFalcon/svgcleaner) (Optional; for Optimised SVG output) +- [oxipng](https://github.com/shssoichiro/oxipng) (Optional; for Crushed PNG output) - [FLIF](https://github.com/FLIF-hub/FLIF) (Optional; for FLIF output) - [libwebp](https://developers.google.com/speed/webp/docs/precompiled) (Optional; for Lossless WebP output) - [go-avif](https://github.com/Kagami/go-avif) (Optional; for AVIF output) (Experimental; does not currently produce truly lossless images. We're trying to figure out why that is.) diff --git a/dest_paths.py b/dest_paths.py index dcf5785..d9d9037 100644 --- a/dest_paths.py +++ b/dest_paths.py @@ -66,6 +66,8 @@ def format_path(path, emoji, format): res = res + '.svg' elif format.startswith('png-'): res = res + '.png' + elif format.startswith('pngc-'): + res = res + '.png' elif format.startswith('flif-'): res = res + '.flif' elif format.startswith('webp-'): diff --git a/docs/dzuk/howto.md b/docs/dzuk/howto.md index 7132553..7123d76 100644 --- a/docs/dzuk/howto.md +++ b/docs/dzuk/howto.md @@ -60,6 +60,12 @@ Using this means that you're just exporting JSON for this command, you can't use ---- +# [Metadata](metadata.md) + +forc can embed SVG or EXIF metadata into your exported emoji sets. + +--- + # Extra flags ## Force Text Descriptions (`--force-desc`) diff --git a/docs/dzuk/image_formats.md b/docs/dzuk/image_formats.md index 1293782..109790c 100644 --- a/docs/dzuk/image_formats.md +++ b/docs/dzuk/image_formats.md @@ -7,15 +7,18 @@ *(check the [readme](../../readme.md) for all the information on dependencies)* #### `svgo` Optimised SVG -Optimised SVG is the same format as SVG, but it's losslessly compressed to create a smaller file size (in Mutant Standard tests, optimised SVGs are 30-40% smaller). It requires an extra processing stage, and it needs the dependency listed. Check the [svgcleaner](https://github.com/RazrFalcon/svgcleaner) repo to see the documentation for it so you can see what it does to the SVG files. +Optimised SVG is the same format as SVG, but it's losslessly compressed to create a smaller file size. It requires an extra processing stage (which is very cheap on CPU), and it needs the dependency listed. Check the [svgcleaner](https://github.com/RazrFalcon/svgcleaner) repo to see the documentation for it so you can see what it does to the SVG files. -orxporter uses svgcleaner with the following settings: +orxporter uses svgcleaner with the following command: `svgcleaner --remove-metadata=no --quiet` +In Mutant Standard tests, optimised SVGs are 30-40% smaller than normal SVGs. Unless you're doing something in particular, it's probably worth using Optimise SVGs by default instead of SVGs in your workflow. + ### Raster - `png` PNG +- `pngc` Crushed PNG (requires oxipng) - `webp` Lossless WebP (requires cwebp) - `avif` Lossless AVIF (requires go-avif) - `flif` FLIF (requires flif) @@ -32,3 +35,13 @@ avif-128 webp-512 png-60 ``` + +#### `pngc` Crushed PNG +Crushed PNG is the same format as PNG, but it's losslessly compressed to create a smaller file size. It requires an extra processing stage (which can be quite expensive on CPU at very large sizes like 512px), and it needs the dependency listed. Check the [oxipng](https://github.com/shssoichiro/oxipng) repo to see the documentation for it so you can see what it does to the PNG files. + +orxporter uses oxipng with the following command: + +`oxipng --out, --quiet` + +In Mutant Standard tests, Crushed PNGs have a file size reduction of about 25% compared to normal PNGs, but only on larger sizes (128px upwards). +At 32px, crushing PNGs only has an average file size reduction of 3%. diff --git a/docs/dzuk/metadata.md b/docs/dzuk/metadata.md new file mode 100644 index 0000000..2df09d1 --- /dev/null +++ b/docs/dzuk/metadata.md @@ -0,0 +1,66 @@ +# Metadata (Licenses) + +Orxporter can embed metadata into your resulting emoji. This is useful if you want to put author or license information into your work for public distribution. + +You add metadata to your manifests using the orx keyword `license`, as shown below: + +``` +license svg = license/svg.xml exif = license/exif.json +``` + +Metadata is automatically embedded in your resulting images, but if you want to leave it out... + +- Use the flag `-l` if you're using the [simple exporting method](image_easy,md). +- Use `license = no` in your Parameters file if you're using the [advanced exporting method](image_advanced), + +Metadata embedding is only done with certain formats: + +- EXIF metadata can be embedded in PNG and AVIF files. +- SVG metadata can be embedded in SVG files. + +--- + +## `svg`: SVG Metadata + +SVG Metadata is an XML file you create that contains stuff that gets inserted in the `` tag of your resulting SVGs. + +Below is an example XML file with Mutant Standard's SVG metadata: + +``` + + + + Mutant Standard emoji v0.4.1 + + + + + Dzuk + http://mutant.tech/ + + + +``` + +--- + +## `exif`: EXIF Metadata + +EXIF Metadata is a JSON file you create that contains stuff that gets inserted in the EXIF metadata of your resulting raster images. + +Below is an example JSON file with Mutant Standard's EXIF metadata: + +``` +{ + "XMP-dc:title": "Mutant Standard emoji v0.4.1", + "XMP-dc:rights": "This work is licensed to the public under the Attribution-NonCommercial-ShareAlike 4.0 International license https://creativecommons.org/licenses/by-nc-sa/4.0/", + "XMP-xmpRights:UsageTerms": "This work is licensed to the public under the Attribution-NonCommercial-ShareAlike 4.0 International license https://creativecommons.org/licenses/by-nc-sa/4.0/", + "XMP-cc:AttributionName": "Dzuk", + "XMP-cc:AttributionURL": "mutant.tech", + "XMP-cc:License": "https://creativecommons.org/licenses/by-nc-sa/4.0/" +} + +``` diff --git a/docs/kiilas/manifest.md b/docs/kiilas/manifest.md index 9473911..ae84c65 100644 --- a/docs/kiilas/manifest.md +++ b/docs/kiilas/manifest.md @@ -258,7 +258,7 @@ The *svg* parameter must point to a file containing the desired string to be inserted inside each SVG file's *metadata* tag. The *exif* parameter must point to a JSON file containing a single object with -the desired EXIF tags to be written to each PNG file. +the desired EXIF tags to be written to each PNG or AVIF file. Text descriptions ----------------- diff --git a/export.py b/export.py index 5ee6961..bbfd522 100644 --- a/export.py +++ b/export.py @@ -129,7 +129,8 @@ def export(m, filtered_emoji, input_path, formats, path, src_size, png_files = [] for e in exporting_emoji: for f in formats: - if f.startswith('png-'): + # png, pngc or avif + if f.startswith('png') or f.startswith('avif-'): try: png_files.append(format_path(path, e, f)) except FilterException: diff --git a/export_task.py b/export_task.py index 492e5d0..5018143 100644 --- a/export_task.py +++ b/export_task.py @@ -51,7 +51,9 @@ def to_raster(emoji_svg, out_path, renderer, format, size, name): # two-step process image_proc.render_svg(tmp_svg_path, tmp_png_path, renderer, size) - if format == "webp": + if format == "pngc": + image_proc.crush_png(tmp_png_path, out_path) + elif format == "webp": image_proc.convert_webp(tmp_png_path, out_path) elif format == "avif": image_proc.convert_avif(tmp_png_path, out_path) diff --git a/export_thread.py b/export_thread.py index a4a2859..ead5e98 100644 --- a/export_thread.py +++ b/export_thread.py @@ -88,6 +88,9 @@ def export_emoji(self, emoji, emoji_svg, f, path, license): if raster_format[0] == "png": export_task.to_raster(emoji_svg, final_path, self.renderer, "png", size, self.name) + elif raster_format[0] == "pngc": + export_task.to_raster(emoji_svg, final_path, self.renderer, "pngc", size, self.name) + elif raster_format[0] == "webp": export_task.to_raster(emoji_svg, final_path, self.renderer, "webp", size, self.name) diff --git a/orxport.py b/orxport.py index 1f23815..786242e 100755 --- a/orxport.py +++ b/orxport.py @@ -53,12 +53,13 @@ ---------------------------------------------------- -F Format (default: {DEF_OUTPUT_FORMATS[0]}) comma separated with no spaces (ie. 'svg,png-64,flif-128') - - svg - - svgo - - png-SIZE - - flif-SIZE - - webp-SIZE - - avif-SIZE + - svg (SVG) + - svgo (Optimised SVG) + - png-SIZE (PNG) + - pngc-SIZE (Crushed PNG) + - flif-SIZE (FLIF) + - webp-SIZE (Lossless WebP) + - avif-SIZE (Lossless AVIF) -f Directory/filename naming system for output (default: {DEF_OUTPUT_NAMING}) See the documentation for how this works.