Dynamic content type mapping based on filename extension. (#347)

* Add configDir to make testing easier

* Helper class to aid testing

* Remove config poller for partnership

* Check and load Content-Type mappings at system level.
Make openas2.properties.file a static for testing purposes.

* Check and load Content-Type mappings at partnership level.

* Sample file for Content-Type dynamic mappings.

* Enable Content-Type mapping in server to server test

* Additional utility methods for file operations

* Fix outbox to support multiple sending <-> receiving partner
combinations

* Use a more appropriately named variable for readability

* Extract common server test stuff to base class

* Tests for dynamic Content-Type mapping

* Helper methods for file related stuff

* Fix spelling error

* Enhance to support dynamic Content-Type setting based on filename
extension

* Change visibility scope of static to support testing

* Use partnership based polller instead of config

* Support system level Content-Type mapping

* Minor cleanup

* Fix formatting to use spaces instead of tabs

* Release notes and associated documentation.

* Version updates.

* Add debug for Microsoft crap as usual

* Escape backslash in file path so it works on Windows

* Remove the custom properties file and associated system property to
avoid affecting other tests.

* Escape the backslash in Windows paths when setting the custom Openas2
property file system property

RELEASE-NOTES.md

@@ -1,21 +1,19 @@
 #              OpenAS2 Server
-#              Version 3.7.0
+#              Version 3.8.0
 #              RELEASE NOTES
 -----
-The OpenAS2 project is pleased to announce the release of OpenAS2 3.7.0
+The OpenAS2 project is pleased to announce the release of OpenAS2 3.8.0
 
-The release download file is: OpenAS2Server-3.7.0.zip
+The release download file is: OpenAS2Server-3.8.0.zip
 
 The zip file contains a PDF document (OpenAS2HowTo.pdf) providing information on installing and using the application.
 ## NOTE: Testing covers Java 8 to 17. The application should work for older versions down to Java 7 but they are not tested as part of the CI/CD pipeline.
 
-Version 3.7.0 - 2023-09-12
-This is an enhancement and bugfix release:
+Version 3.8.0 - 2023-11-07
+This is an enhancement release:
        **IMPORTANT NOTE**: Please review upgrade notes below if you are upgrading
 
-  1. Support parallel mode processing for the directory polling configuration to achieve high volume throughput.
-  2. Enhance error handling when chacking for files that never received an DMN response.
-  3. Added logging to indicate reading a fixed byte count message from HTTP stream to aid debugging.
+  1. Support for configurable dynamic Content-Type based on the file extension. See documentation section 7.5 "Setting Content Type"
 
 ##Upgrade Notes
  See the openAS2HowTo appendix for the general process on upgrading OpenAS2.

Remote/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>net.sf.openas2</groupId>
         <artifactId>OpenAS2</artifactId>
-        <version>3.7.0</version>
+        <version>3.8.0</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

Server/pom.xml

@@ -7,7 +7,7 @@
         <!-- DO NOT CHANGE THIS "groupId" WITHOUT CHANGING XMLSession.getManifestAttributes.MANIFEST_VENDOR_ID_ATTRIB -->
         <groupId>net.sf.openas2</groupId>
         <artifactId>OpenAS2</artifactId>
-        <version>3.7.0</version>
+        <version>3.8.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

Server/src/config/content_type_mappings.properties

@@ -0,0 +1,3 @@
+xml=application/xml
+edi=application/edifact
+txt=text/plain

Server/src/main/java/org/openas2/XMLSession.java

@@ -12,11 +12,13 @@
 import org.openas2.params.InvalidParameterException;
 import org.openas2.params.ParameterParser;
 import org.openas2.message.MessageFactory;
+import org.openas2.partner.Partnership;
 import org.openas2.partner.PartnershipFactory;
 import org.openas2.processor.Processor;
 import org.openas2.processor.ProcessorModule;
 import org.openas2.processor.receiver.PollingModule;
 import org.openas2.schedule.SchedulerComponent;
+import org.openas2.util.FileUtil;
 import org.openas2.util.Properties;
 import org.openas2.util.XMLUtil;
 import org.w3c.dom.Document;
@@ -141,8 +143,9 @@ protected void load(InputStream in) throws Exception {
      * 
      * @param propNode - the "properties" element of the configuration file containing property values
      * @throws InvalidParameterException 
+     * @throws IOException 
      */
-    private void loadProperties(Node propNode) throws InvalidParameterException {
+    private void loadProperties(Node propNode) throws InvalidParameterException, IOException {
         LOGGER.info("Loading properties...");
 
         Map<String, String> properties = XMLUtil.mapAttributes(propNode, false);
@@ -151,7 +154,7 @@ private void loadProperties(Node propNode) throws InvalidParameterException {
         properties.put(Properties.APP_TITLE_PROP, getAppTitle());
         properties.put(Properties.APP_VERSION_PROP, getAppVersion());
         Properties.setProperties(properties);
-        String appPropsFile = System.getProperty("openas2.properties.file");
+        String appPropsFile = System.getProperty(Properties.OPENAS2_PROPERTIES_FILE_PROP);
         if (appPropsFile != null && appPropsFile.length() > 1) {
             java.util.Properties appProps = new java.util.Properties();
             FileInputStream fis = null;
@@ -210,6 +213,11 @@ private void loadProperties(Node propNode) throws InvalidParameterException {
                 Properties.setProperty(key, entry.getValue());
             }
         }
+        // Now check if we need to load Content-Type mappings
+        String contentTypeMapFilename = Properties.getProperty(Partnership.PA_CONTENT_TYPE_MAPPING_FILE, null);
+        if (contentTypeMapFilename != null) {
+            Properties.setContentTypeMap(FileUtil.loadProperties(contentTypeMapFilename));
+        }
     }
 
     private void loadCertificates(Node rootNode) throws OpenAS2Exception {

Server/src/main/java/org/openas2/app/partner/AddPartnershipCommand.java

@@ -16,7 +16,7 @@
 import java.util.regex.Pattern;
 
 /**
- * adds a new partnership entry in partneship store
+ * adds a new partnership entry in partnership store
  *
  * @author joseph mcverry
  */
@@ -56,7 +56,7 @@ public CommandResult execute(PartnershipFactory partFx, Object[] params) throws
 
             for (int i = 0; i < params.length; i++) {
                 String param = (String) params[i];
-                int pos = param.indexOf('=');
+                int equalsPos = param.indexOf('=');
                 if (i == 0) {
                     partnershipRoot.setAttribute("name", param);
                 } else if (i == 1) {
@@ -67,9 +67,9 @@ public CommandResult execute(PartnershipFactory partFx, Object[] params) throws
                     Element elem = doc.createElement(Partnership.PCFG_RECEIVER);
                     elem.setAttribute("name", param);
                     partnershipRoot.appendChild(elem);
-                } else if (pos == 0) {
+                } else if (equalsPos == 0) {
                     return new CommandResult(CommandResult.TYPE_ERROR, "incoming parameter missing name");
-                } else if (pos > 0) {
+                } else if (equalsPos > 0) {
                     if (param.startsWith("pollerConfig.")) {
                         // Add a pollerConfig element
                         String regex = "^pollerConfig.([^=]*)=((?:[^\"']+)|'(?:[^']*)'|\"(?:[^\"]*)\")";
@@ -86,8 +86,8 @@ public CommandResult execute(PartnershipFactory partFx, Object[] params) throws
                         pollerConfigElem.setAttribute(name, val);
                     } else {
                         Element elem = doc.createElement("attribute");
-                        elem.setAttribute("name", param.substring(0, pos));
-                        elem.setAttribute("value", param.substring(pos + 1));
+                        elem.setAttribute("name", param.substring(0, equalsPos));
+                        elem.setAttribute("value", param.substring(equalsPos + 1));
                         partnershipRoot.appendChild(elem);
                     }
                 } else {

Server/src/main/java/org/openas2/message/FileAttribute.java

@@ -11,4 +11,5 @@ public interface FileAttribute {
     String MA_ERROR_FILENAME = "errorfilename";
     String MA_SENT_DIR = "sentdir";
     String MA_SENT_FILENAME = "sentfilename";
+    String MA_FILENAME_EXTENSION = "filename_extension";
 }

Server/src/main/java/org/openas2/partner/Partnership.java

@@ -2,8 +2,10 @@
 
 import org.openas2.OpenAS2Exception;
 import org.openas2.cert.CertificateNotFoundException;
+import org.openas2.util.FileUtil;
 import org.openas2.util.Properties;
 
+import java.io.IOException;
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -37,11 +39,13 @@ public class Partnership implements Serializable {
     /* partnership definition attributes */
     public static final String PA_SUBJECT = "subject"; // Subject sent in messages    
     public static final String PA_CONTENT_TYPE = "content_type"; // optional content type for mime parts
+    public static final String PA_USE_DYNAMIC_CONTENT_TYPE_MAPPING = "use_dynamic_content_type_mapping"; // use file extension to Content-Type mapping
+    public static final String PA_CONTENT_TYPE_MAPPING_FILE = "content_type_mapping_file"; // file containing file extension to Content-Type mapping
     public static final String PA_CONTENT_TRANSFER_ENCODING = "content_transfer_encoding"; // optional content transfer enc value
     public static final String PA_SET_CONTENT_TRANSFER_ENCODING_HTTP = "set_content_transfer_encoding_http_header"; // See as an HTTP header
     public static final String PA_REMOVE_PROTECTION_ATTRIB = "remove_cms_algorithm_protection_attrib"; // Some AS2 systems do not support the attribute
     public static final String PA_SET_CONTENT_TRANSFER_ENCODING_OMBP = "set_content_transfer_encoding_on_outer_mime_bodypart"; // optional content transfer enc value
-    public static final String PA_RESEND_REQUIRES_NEW_MESSAGE_ID = "resend_requires_new_message_id"; // list of nme/value pairs for setting custom mime headers
+    public static final String PA_RESEND_REQUIRES_NEW_MESSAGE_ID = "resend_requires_new_message_id"; // list of name/value pairs for setting custom mime headers
     public static final String PA_COMPRESSION_TYPE = "compression";
     public static final String PA_SIGNATURE_ALGORITHM = "sign";
     public static final String PA_ENCRYPTION_ALGORITHM = "encrypt";
@@ -80,6 +84,9 @@ public class Partnership implements Serializable {
     private Map<String, Object> receiverIDs;
     private Map<String, Object> senderIDs;
     private String name;
+    private java.util.Properties overrideContentTypeFromFileExtensionMap = null;
+    private java.util.Properties contentTypeFromFileExtensionMap = null;
+    private boolean useDynamicContentTypeLookup = false;
 
     public String getName() {
         return name;
@@ -173,7 +180,59 @@ public boolean matches(Partnership partnership) {
 
     }
 
-    public String getAlias(String partnershipType) throws OpenAS2Exception {
+     public boolean isUseDynamicContentTypeLookup() {
+        return useDynamicContentTypeLookup;
+    }
+
+    /** This method is called if the partnership is configured to use dynamic mappings.
+     *  It will check that there are either system or partnership specific mappings available
+     *  load them into a partnership mapping cache.
+     * @param useDynamicContentTypeLookup - if true then enable dynamic mapping
+     * @throws OpenAS2Exception
+     * @throws IOException
+     */
+    public void setUseDynamicContentTypeLookup(boolean useDynamicContentTypeLookup) throws OpenAS2Exception, IOException {
+        if (useDynamicContentTypeLookup) {
+            // Make sure there is a lookup available
+            // If there is a partnership specific override then make the partnership use 
+            // that otherwise point it at the system mapping if available
+            String contentTypeMapFilename = getAttribute(Partnership.PA_CONTENT_TYPE_MAPPING_FILE);
+            if (contentTypeMapFilename != null) {
+                if (Properties.getContentTypeMap() != null) {
+                    // Copy the system level mapping in first then override/add the custom mappings
+                    overrideContentTypeFromFileExtensionMap = new java.util.Properties();
+                    overrideContentTypeFromFileExtensionMap.putAll(Properties.getContentTypeMap());
+                      overrideContentTypeFromFileExtensionMap.putAll(FileUtil.loadProperties(contentTypeMapFilename));
+                  } else {
+                    // Get the override map
+                    setOverrideContentTypeFromFileExtension(FileUtil.loadProperties(contentTypeMapFilename));
+                  }
+                // Configure this partnership to use the override lookup
+                contentTypeFromFileExtensionMap = overrideContentTypeFromFileExtensionMap;
+            } else {
+                // Set the partnership to use the global map
+                contentTypeFromFileExtensionMap = Properties.getContentTypeMap();
+            }
+            // If there is no map to do the lookup throw an excpetion
+            if (this.contentTypeFromFileExtensionMap == null) {
+                throw new OpenAS2Exception("Trying to use Content-Type mapping functionality but no mappings loaded.");
+            }
+        }
+        this.useDynamicContentTypeLookup = useDynamicContentTypeLookup;
+    }
+
+    public String getContentTypeFromFileExtension(String key) {
+        if (contentTypeFromFileExtensionMap == null) {
+            return null;
+        }
+        return (String) contentTypeFromFileExtensionMap.get(key);
+     }
+
+     public void setOverrideContentTypeFromFileExtension(java.util.Properties contentTypeFromFileExtension) {
+         this.overrideContentTypeFromFileExtensionMap = contentTypeFromFileExtension;
+     }
+
+   public String getAlias(String partnershipType) throws OpenAS2Exception {
         String alias = null;
 
         if (partnershipType == PTYPE_RECEIVER) {

Server/src/main/java/org/openas2/partner/XMLPartnershipFactory.java

@@ -209,7 +209,15 @@ public void loadPartnership(Map<String, Object> partners, List<Partnership> part
 
         // read in the partnership attributes
         loadAttributes(node, partnership);
-
+        // Now check if we need to enable Content-Type mappings for this partnership
+        if ("true".equalsIgnoreCase(partnership.getAttributeOrProperty(Partnership.PA_USE_DYNAMIC_CONTENT_TYPE_MAPPING, "false"))) {
+            try {
+                partnership.setUseDynamicContentTypeLookup(true);
+            } catch (IOException e) {
+                logger.error("Error setting up dynamic Content-Type lookup: " + e.getMessage(), e);
+                throw new OpenAS2Exception("Partnership failed to be set up correctly for dynamic Content-Type lookup: " + getName());
+            }
+        }
         // add the partnership to the list of available partnerships
         partnerships.add(partnership);
 

Server/src/main/java/org/openas2/processor/receiver/MessageBuilderModule.java

@@ -20,6 +20,7 @@
 import org.openas2.processor.resender.ResenderModule;
 import org.openas2.processor.sender.SenderModule;
 import org.openas2.util.AS2Util;
+import org.openas2.util.FileUtil;
 import org.openas2.util.IOUtil;
 import org.openas2.util.Properties;
 
@@ -280,6 +281,8 @@ public Message buildBaseMessage(String filename) throws OpenAS2Exception {
     public void addMessageMetadata(Message msg, String filename) throws OpenAS2Exception {
         msg.setAttribute(FileAttribute.MA_FILENAME, filename);
         msg.setPayloadFilename(filename);
+        // Set the filename extension if it has one
+        msg.setAttribute(FileAttribute.MA_FILENAME_EXTENSION, FileUtil.getFilenameExtension(filename));
         // Set a new message ID
         msg.updateMessageID();
         // Set the sender and receiver in the Message object headers
@@ -352,24 +355,35 @@ public void buildMessageData(Message msg, DataSource dataSource, String contentT
         msg.setData(body);
     }
 
-    private String getMessageContentType(Message msg) throws OpenAS2Exception {
+    public String getMessageContentType(Message msg) throws OpenAS2Exception {
         MessageParameters params = new MessageParameters(msg);
 
-            // Allow Content-Type to be overridden at partnership level or as property
-            String contentType = msg.getPartnership().getAttributeOrProperty(Partnership.PA_CONTENT_TYPE, null);
-            if (contentType == null) {
-                contentType = getParameter(PARAM_MIMETYPE, false);
-            }
-            if (contentType == null) {
-                contentType = "application/octet-stream";
-            } else {
-                try {
-                    contentType = ParameterParser.parse(contentType, params);
-                } catch (InvalidParameterException e) {
-                    throw new OpenAS2Exception("Bad content-type" + contentType, e);
+        // Allow Content-Type to be overridden at partnership level or as property
+        String contentType = msg.getPartnership().getAttributeOrProperty(Partnership.PA_CONTENT_TYPE, null);
+        // The content type could be determined dynamically based on filename extension
+        if (msg.getPartnership().isUseDynamicContentTypeLookup()) {
+            String fileExtension = msg.getAttribute(FileAttribute.MA_FILENAME_EXTENSION);
+            if (fileExtension != null) {
+                String dynamicContentType = msg.getPartnership().getContentTypeFromFileExtension(fileExtension);
+                if (dynamicContentType != null) {
+                    // Dynamic override found so use it
+                    contentType = dynamicContentType;
                 }
             }
-            return contentType;
+        }
+        if (contentType == null) {
+            contentType = getParameter(PARAM_MIMETYPE, false);
+        }
+        if (contentType == null) {
+            contentType = "application/octet-stream";
+        } else {
+            try {
+                contentType = ParameterParser.parse(contentType, params);
+            } catch (InvalidParameterException e) {
+                throw new OpenAS2Exception("Bad content-type" + contentType, e);
+            }
+        }
+        return contentType;
     }
 
     private void setAdditionalMetaData(Message msg, MimeBodyPart mimeBodyPart) throws OpenAS2Exception {

Server/src/main/java/org/openas2/processor/receiver/PollingModule.java

@@ -10,7 +10,7 @@
 
 
 public abstract class PollingModule extends MessageBuilderModule {
-    private static final String PARAM_POLLING_INTERVAL = "interval";
+    protected final String PARAM_POLLING_INTERVAL = "interval";
     private Timer timer;
     private boolean busy;
     private String outboxDir;

Server/src/main/java/org/openas2/util/FileUtil.java

@@ -5,16 +5,46 @@
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.util.Properties;
 
 
 public class FileUtil {
 
     //private static final Log logger = LogFactory.getLog(FileUtil.class.getSimpleName());
 
+    public static Properties loadProperties(String filename) throws IOException {
+        Properties fileProps = new java.util.Properties();
+        FileInputStream fis = null;
+        fis = new FileInputStream(filename);
+        try {
+            fileProps.load(fis);
+        } finally {
+            if (fis != null) {
+                    fis.close();
+            }
+        }
+        return fileProps;
+    }
+
+    /** Attempts to extract the filename extension by searching for the last occurrence
+     *  of a period and returning all characters following that period.
+     *  If no period is found then it returns null.
+     * @param filename - the full name of the file including extension
+     * @return the extension of the filename excluding the period
+     */
+    public static String getFilenameExtension(String filename) {
+        int period_index = filename.lastIndexOf(".");
+        if (period_index == -1) {
+            return null;
+        }
+        return filename.substring(filename.lastIndexOf(".") + 1);
+    }
+
     public static void splitLineBasedFile(File sourceFile, String outputDir, long maxFileSize, boolean containsHeaderRow, String newFileBaseName, String filenamePrefix) throws OpenAS2Exception {
         FileReader fileReader;
         try {