Merge remote-tracking branch 'origin/atomic-exports' into upstream

README.md

@@ -32,9 +32,10 @@ Mutant Standard's [code of conduct](docs/code_of_conduct.md).
 
 - Declarative language for defining semantics of emoji set
 - Color mapping (recoloring)
-- Outputting emoji both as shortcode-named files (ie. 'ice_cream') and unicode codepoint-named files (ie. '1f368').
-- SVG, PNG, Lossless WebP, Lossless AVIF and FLIF exports
-- Supports multiple SVG renderers: rendersvg, inkscape and imagemagick
+- Outputting emoji both as shortcode-named files (ie. 'ice_cream') and unicode codepoint-named files (ie. '1f368')
+- Export to SVG, PNG, Lossless WebP, Lossless AVIF and FLIF
+- Optional lossless crushing of SVG and PNG files
+- Supports multiple SVG renderers (rendersvg, Inkscape and ImageMagick)
 - Output options including emoji filtering, customisable export directory
   structure and filenaming
 - JSON output of emoji set metadata
@@ -51,6 +52,8 @@ Mutant Standard's [code of conduct](docs/code_of_conduct.md).
     - Inkscape
     - ImageMagick
 - exiftool (Optional; for embedding EXIF license metadata)
+- [svgcleaner](https://github.com/RazrFalcon/svgcleaner) (Optional; for Optimised SVG output)
+- [oxipng](https://github.com/shssoichiro/oxipng) (Optional; for Crushed PNG output)
 - [FLIF](https://github.com/FLIF-hub/FLIF) (Optional; for FLIF output)
 - [libwebp](https://developers.google.com/speed/webp/docs/precompiled) (Optional; for Lossless WebP output)
 - [go-avif](https://github.com/Kagami/go-avif) (Optional; for AVIF output) (Experimental; does not currently produce truly lossless images. We're trying to figure out why that is.)

check.py

@@ -0,0 +1,77 @@
+import os
+
+from dest_paths import format_path, format_resolve
+from exception import FilterException
+import svg
+
+def emoji(m, filtered_emoji, input_path, formats, path, src_size,
+           num_threads, renderer, max_batch, verbose):
+    """
+    Checks all emoji in a very light validation as well as checking if emoji
+    aren't filtered out by user choices.
+
+    It only checks:
+    - If the emoji has been filtered out by user exporting options.
+
+    - If the source SVG file exists.
+      (Will throw an Exception if not the case.)
+
+    - If the shortcode ('short') attribute exists.
+      (Will throw an Exception if not the case.)
+
+    - If the svg size is consistent (if a -q flag is used).
+      (Will throw an Exception if not the case.)
+
+    It this doesn't result in an Exception, it returns dict containing
+    a list of emoji that aren't filtered out, as well as a count
+    of emoji that were skipped.
+    """
+
+    exporting_emoji = []
+    skipped_emoji_count = 0
+
+    for i, e in enumerate(filtered_emoji):
+
+        short = e.get("code", "<UNNAMED>") # to provide info on possible error printouts
+
+        try:
+            format_path(path, e, 'svg')
+        except FilterException as ex:
+            if verbose:
+                log.out(f"- - Skipped emoji: {short} - {ex}", 34)
+            skipped_emoji_count += 1
+            continue # skip if filtered out
+
+        if 'src' not in e:
+            raise ValueError(f"The emoji '{short}' is missing an 'src' attribute. It needs to have one.")
+
+
+
+        # try to see if the source SVG file exists
+        srcpath = os.path.join(m.homedir, input_path, e['src'])
+        try:
+            emoji_svg = open(srcpath, 'r').read()
+        except Exception:
+            raise ValueError(f"This source image for emoji '{short}' could not be loaded: {srcpath}")
+
+
+
+        # the SVG size check (-q)
+        if src_size is not None:
+            img_size = svg.get_viewbox_size(emoji_svg)
+
+            if img_size != src_size:
+                raise ValueError("""The source image size for emoji '{}' is not what
+                                was expected. It's supposed to be {}, but it's actually
+                                {}.""".format(
+                                    short,
+                                    str(src_size[0]) + 'x' + str(src_size[1]),
+                                    str(img_size[0]) + 'x' + str(img_size[1])
+                                    ))
+
+        # add the emoji to exporting_emoji if it's passed all the tests.
+        exporting_emoji.append(e)
+
+    return { "exporting_emoji" : exporting_emoji
+           , "skipped_emoji_count" : skipped_emoji_count
+           }

paths.py → dest_paths.py

@@ -62,8 +62,12 @@ def format_path(path, emoji, format):
     # (also acts as a format check)
     if format == 'svg':
         res = res + '.svg'
+    elif format == 'svgo':
+        res = res + '.svg'
     elif format.startswith('png-'):
         res = res + '.png'
+    elif format.startswith('pngc-'):
+        res = res + '.png'
     elif format.startswith('flif-'):
         res = res + '.flif'
     elif format.startswith('webp-'):

docs/dzuk/howto.md

@@ -60,6 +60,12 @@ Using this means that you're just exporting JSON for this command, you can't use
 
 ----
 
+# [Metadata](metadata.md)
+
+forc can embed SVG or EXIF metadata into your exported emoji sets.
+
+---
+
 # Extra flags
 
 ## Force Text Descriptions (`--force-desc`)

docs/dzuk/image_formats.md

@@ -1,21 +1,31 @@
 # Formats
 
-### Out of the box support:
-
+### Vector
 - `svg` SVG
-- `png` PNG
+- `svgo` Optimised SVG (requires svgcleaner)
+
+*(check the [readme](../../readme.md) for all the information on dependencies)*
+
+#### `svgo` Optimised SVG
+Optimised SVG is the same format as SVG, but it's losslessly compressed to create a smaller file size. It requires an extra processing stage (which is very cheap on CPU), and it needs the dependency listed. Check the [svgcleaner](https://github.com/RazrFalcon/svgcleaner) repo to see the documentation for it so you can see what it does to the SVG files.
+
+orxporter uses svgcleaner with the following command:
+
+`svgcleaner <in file> <out file> --remove-metadata=no --quiet`
 
-### Requires extra software to install:
+In Mutant Standard tests, optimised SVGs are 30-40% smaller than normal SVGs. Unless you're doing something in particular, it's probably worth using Optimise SVGs by default instead of SVGs in your workflow.
 
+
+### Raster
+- `png` PNG
+- `pngc` Crushed PNG (requires oxipng)
 - `webp` Lossless WebP (requires cwebp)
 - `avif` Lossless AVIF (requires go-avif)
 - `flif` FLIF (requires flif)
 
 *(check the [readme](../../readme.md) for all the information on dependencies)*
 
-Crushed PNGs and Optimised SVGs are the same formats as PNG and SVG, but their data is arranged in such a way to create a smaller file size, they require an extra processing stage, and they need the dependencies listed.
-
-When choosing raster images (any format but `svg`), you have to add a size as well. you do this by adding a hyphen followed by the square size in pixels.
+When choosing raster images, you have to add a size as well. you do this by adding a hyphen followed by the square size in pixels.
 
 eg.
 
@@ -25,3 +35,13 @@ avif-128
 webp-512
 png-60
 ```
+
+#### `pngc` Crushed PNG
+Crushed PNG is the same format as PNG, but it's losslessly compressed to create a smaller file size. It requires an extra processing stage (which can be quite expensive on CPU at very large sizes like 512px), and it needs the dependency listed. Check the [oxipng](https://github.com/shssoichiro/oxipng) repo to see the documentation for it so you can see what it does to the PNG files.
+
+orxporter uses oxipng with the following command:
+
+`oxipng <in file> --out, <out file> --quiet`
+
+In Mutant Standard tests, Crushed PNGs have a file size reduction of about 25% compared to normal PNGs, but only on larger sizes (128px upwards).
+At 32px, crushing PNGs only has an average file size reduction of 3%.

docs/dzuk/metadata.md

@@ -0,0 +1,66 @@
+# Metadata (Licenses)
+
+Orxporter can embed metadata into your resulting emoji. This is useful if you want to put author or license information into your work for public distribution.
+
+You add metadata to your manifests using the orx keyword `license`, as shown below:
+
+```
+license svg = license/svg.xml		exif = license/exif.json
+```
+
+Metadata is automatically embedded in your resulting images, but if you want to leave it out...
+
+- Use the flag `-l` if you're using the [simple exporting method](image_easy,md).
+- Use `license = no` in your Parameters file if you're using the [advanced exporting method](image_advanced),
+
+Metadata embedding is only done with certain formats:
+
+- EXIF metadata can be embedded in PNG and AVIF files.
+- SVG metadata can be embedded in SVG files.
+
+---
+
+## `svg`: SVG Metadata
+
+SVG Metadata is an XML file you create that contains stuff that gets inserted in the `<metadata>` tag of your resulting SVGs.
+
+Below is an example XML file with Mutant Standard's SVG metadata:
+
+```
+<rdf:RDF xmlns:cc="http://web.resource.org/cc/"
+         xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+         xmlns:dc = "http://purl.org/dc/elements/1.1/"
+         >
+
+     <rdf:Description rdf:about="">
+       <dc:title>Mutant Standard emoji v0.4.1</dc:title>
+     </rdf:Description>
+
+    <cc:work rdf:about="">
+        <cc:license rdf:resource="http://creativecommons.org/licenses/by-nc-sa/4.0/"/>
+        <cc:attributionName>Dzuk</cc:attributionName>
+        <cc:attributionURL>http://mutant.tech/</cc:attributionURL>
+    </cc:work>
+
+</rdf:RDF>
+```
+
+---
+
+## `exif`: EXIF Metadata
+
+EXIF Metadata is a JSON file you create that contains stuff that gets inserted in the EXIF metadata of your resulting raster images.
+
+Below is an example JSON file with Mutant Standard's EXIF metadata:
+
+```
+{
+    "XMP-dc:title": "Mutant Standard emoji v0.4.1",
+    "XMP-dc:rights": "This work is licensed to the public under the Attribution-NonCommercial-ShareAlike 4.0 International license https://creativecommons.org/licenses/by-nc-sa/4.0/",
+    "XMP-xmpRights:UsageTerms": "This work is licensed to the public under the Attribution-NonCommercial-ShareAlike 4.0 International license https://creativecommons.org/licenses/by-nc-sa/4.0/",
+    "XMP-cc:AttributionName": "Dzuk",
+    "XMP-cc:AttributionURL": "mutant.tech",
+    "XMP-cc:License": "https://creativecommons.org/licenses/by-nc-sa/4.0/"
+}
+
+```

docs/kiilas/manifest.md

@@ -258,7 +258,7 @@ The *svg* parameter must point to a file containing the desired string to be
 inserted inside each SVG file's *metadata* tag.
 
 The *exif* parameter must point to a JSON file containing a single object with
-the desired EXIF tags to be written to each PNG file.
+the desired EXIF tags to be written to each PNG or AVIF file.
 
 Text descriptions
 -----------------

export.py

@@ -2,12 +2,14 @@
 import queue
 import time
 
-from export_thread import ExportThread
+import check
 from exception import FilterException
-from paths import format_path, format_resolve
-import log
 import exif
-import svg
+from export_thread import ExportThread
+from dest_paths import format_path
+import log
+
+
 
 
 
@@ -19,46 +21,14 @@ def export(m, filtered_emoji, input_path, formats, path, src_size,
     validation of emoji metadata and running the tasks associated with exporting.
     """
 
-    # verify emoji
+    # verify emoji (in a very basic way)
     # --------------------------------------------------------------------------
     log.out('Checking emoji...', 36)
+    check_result = check.emoji(m, filtered_emoji, input_path, formats, path, src_size,
+               num_threads, renderer, max_batch, verbose)
 
-
-    exporting_emoji = []
-    skipped_emoji_count = 0
-    for i, e in enumerate(filtered_emoji):
-
-        short = e.get("code", "<UNNAMED>") # to provide info on possible error printouts
-
-        try:
-            format_path(path, e, 'svg')
-        except FilterException as ex:
-            if verbose:
-                log.out(f"- - Skipped emoji: {short} - {ex}", 34)
-            skipped_emoji_count += 1
-            continue #skip if filtered out
-
-        if 'src' not in e:
-            raise ValueError(f"The emoji '{short}' is missing an 'src' attribute. It needs to have one.")
-        srcpath = os.path.join(m.homedir, input_path, e['src'])
-
-        try:
-            emoji_svg = open(srcpath, 'r').read()
-        except Exception:
-            raise ValueError(f"This source image for emoji '{short}' could not be loaded: {srcpath}")
-
-        # the SVG size check (-q)
-        if src_size is not None:
-            imgsize = svg.get_viewbox_size(emoji_svg)
-            if imgsize != src_size:
-                raise ValueError("The source image size for emoji '{}' is not what was expected. It's supposed to be {}, but it's actually {}.".format(
-                    short,
-                    str(src_size[0]) + 'x' + str(src_size[1]),
-                    str(imgsize[0]) + 'x' + str(imgsize[1])
-                    ))
-
-        # add the emoji to exporting_emoji if it's passed all the tests.
-        exporting_emoji.append(e)
+    exporting_emoji = check_result["exporting_emoji"]
+    skipped_emoji_count = check_result["skipped_emoji_count"]
 
     if skipped_emoji_count > 0:
         log.out(f"- {skipped_emoji_count} emoji have been skipped, leaving {len(exporting_emoji)} emoji to export.", 34)
@@ -74,8 +44,9 @@ def export(m, filtered_emoji, input_path, formats, path, src_size,
     # --------------------------------------------------------------------------
     # declare some specs of this export.
     log.out("Exporting emoji...", 36)
-    log.out(f"- {', '.join(formats)}")
-    log.out(f"- to '{path}'")
+    log.out(f"- {', '.join(formats)}") # print formats
+    log.out(f"- to '{path}'") # print out path
+
     if num_threads > 1:
         log.out(f"- {num_threads} threads")
     else:
@@ -158,7 +129,8 @@ def export(m, filtered_emoji, input_path, formats, path, src_size,
         png_files = []
         for e in exporting_emoji:
             for f in formats:
-                if f.startswith('png-'):
+                # png, pngc or avif
+                if f.startswith('png') or f.startswith('avif-'):
                     try:
                         png_files.append(format_path(path, e, f))
                     except FilterException: