BL-642 #resolve

deserializeJSON not handling white space / control characters

src/main/bx/ModuleConfig.bx

@@ -84,6 +84,9 @@
 			engine = "lucee",
 			isLucee = false,
 			isAdobe = false,
+			// JSON control character auto-escaping flag
+			// IF you turn to true, be aware that the entire JSON serialization will be escaped and be slower.
+			jsonEscapeControlCharacters = true,
 			// This simulates the query to empty value that Adobe/Lucee do when NOT in full null support
 			// We default it to true to simulate Adobe/Lucee behavior
 			queryNullToEmpty = true,

src/main/java/ortus/boxlang/modules/compat/bifs/conversion/JSONDeserialize.java

@@ -0,0 +1,97 @@
+package ortus.boxlang.modules.compat.bifs.conversion;
+
+import ortus.boxlang.modules.compat.util.KeyDictionary;
+import ortus.boxlang.runtime.bifs.BoxBIF;
+import ortus.boxlang.runtime.bifs.BoxMember;
+import ortus.boxlang.runtime.context.IBoxContext;
+import ortus.boxlang.runtime.dynamic.casters.BooleanCaster;
+import ortus.boxlang.runtime.modules.ModuleRecord;
+import ortus.boxlang.runtime.scopes.ArgumentsScope;
+import ortus.boxlang.runtime.scopes.Key;
+import ortus.boxlang.runtime.types.BoxLangType;
+
+@BoxBIF
+@BoxMember( type = BoxLangType.STRING )
+public class JSONDeserialize extends ortus.boxlang.runtime.bifs.global.conversion.JSONDeserialize {
+
+	public static final Key moduleName = Key.of( "compat-cfml" );
+
+	/**
+	 * Converts a JSON (JavaScript Object Notation) string data representation into data, such as a structure or array.
+	 *
+	 * @param context   The context in which the BIF is being invoked.
+	 * @param arguments Argument scope for the BIF.
+	 *
+	 * @argument.json The JSON string to convert to data.
+	 *
+	 * @argument.strictMapping A Boolean value that specifies whether to convert the JSON strictly. If true, everything becomes structures.
+	 *
+	 * @argument.useCustomSerializer A string that specifies the name of a custom serializer to use. (Not used)
+	 *
+	 * @return The data representation of the JSON string.
+	 */
+	@Override
+	public Object _invoke( IBoxContext context, ArgumentsScope arguments ) {
+		String			json					= arguments.getAsString( Key.json );
+		ModuleRecord	moduleRecord			= moduleService.getModuleRecord( moduleName );
+		Object			moduleSetting			= moduleRecord.settings.get( KeyDictionary.jsonEscapeControlCharacters );
+		Boolean			escapeControlCharacters	= BooleanCaster
+		    .attempt( moduleSetting )
+		    .getOrDefault( false );
+
+		if ( escapeControlCharacters ) {
+			// Escape all control characters in the JSON string
+			arguments.put( Key.json, escapeControlCharacters( json ) );
+		}
+
+		return super._invoke( context, arguments );
+	}
+
+	/**
+	 * Escape all control characters in ASCII range 0-31 and
+	 * convert them to their escaped representation.
+	 *
+	 * @param json The JSON string to escape control characters in.
+	 *
+	 * @return The JSON string with control characters escaped.
+	 */
+	private String escapeControlCharacters( String json ) {
+		StringBuilder	escapedJson	= new StringBuilder();
+		boolean			inQuotes	= false; // Flag to track if we're inside quotes (single or double)
+
+		for ( int i = 0; i < json.length(); i++ ) {
+			char c = json.charAt( i );
+
+			// Check for single or double quotes and toggle the inQuotes flag
+			if ( c == '\"' || c == '\'' ) {
+				escapedJson.append( c );
+				inQuotes = !inQuotes;
+			} else if ( inQuotes && c < 32 ) {
+				// Only escape control characters if inside quotes
+				switch ( c ) {
+					case 9 : // Tab
+						escapedJson.append( "\\t" );
+						break;
+					case 10 : // Newline
+						escapedJson.append( "\\n" );
+						break;
+					case 13 : // Carriage return
+						escapedJson.append( "\\r" );
+						break;
+					case 12 : // Form feed
+						escapedJson.append( "\\f" );
+						break;
+					default :
+						escapedJson.append( String.format( "\\u%04x", ( int ) c ) );
+						break;
+				}
+			} else {
+				// If not in quotes, append character as-is
+				escapedJson.append( c );
+			}
+		}
+
+		return escapedJson.toString();
+	}
+
+}

src/main/java/ortus/boxlang/modules/compat/util/KeyDictionary.java

@@ -18,8 +18,10 @@
 
 public class KeyDictionary {
 
-	public static final Key	filterOrTags	= Key.of( "filterOrTags" );
-	public static final Key	isKey			= Key.of( "isKey" );
-	public static final Key	objectType		= Key.of( "objectType" );
+	public static final Key	filterOrTags				= Key.of( "filterOrTags" );
+	public static final Key	isKey						= Key.of( "isKey" );
+	public static final Key	objectType					= Key.of( "objectType" );
+	public static final Key	moduleName					= Key.of( "compat-cfml" );
+	public static final Key	jsonEscapeControlCharacters	= Key.of( "jsonEscapeControlCharacters" );
 
 }

src/test/java/ortus/boxlang/modules/compat/BaseIntegrationTest.java

@@ -0,0 +1,65 @@
+/**
+ * [BoxLang]
+ *
+ * Copyright [2023] [Ortus Solutions, Corp]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package ortus.boxlang.modules.compat;
+
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+
+import ortus.boxlang.runtime.BoxRuntime;
+import ortus.boxlang.runtime.context.ScriptingRequestBoxContext;
+import ortus.boxlang.runtime.modules.ModuleRecord;
+import ortus.boxlang.runtime.scopes.IScope;
+import ortus.boxlang.runtime.scopes.Key;
+import ortus.boxlang.runtime.scopes.VariablesScope;
+import ortus.boxlang.runtime.services.ModuleService;
+
+public class BaseIntegrationTest {
+
+	protected static BoxRuntime				runtime;
+	protected static ModuleService			moduleService;
+	protected static Key					result		= new Key( "result" );
+	protected static Key					moduleName	= new Key( "compat-cfml" );
+	protected ScriptingRequestBoxContext	context;
+	protected IScope						variables;
+
+	@BeforeAll
+	public static void setup() {
+		runtime			= BoxRuntime.getInstance( true );
+		moduleService	= runtime.getModuleService();
+	}
+
+	@BeforeEach
+	public void setupEach() {
+		context		= new ScriptingRequestBoxContext();
+		variables	= context.getScopeNearby( VariablesScope.name );
+	}
+
+	@SuppressWarnings( "unused" )
+	protected void loadModule() {
+		String			physicalPath	= Paths.get( "./build/module" ).toAbsolutePath().toString();
+		ModuleRecord	moduleRecord	= new ModuleRecord( physicalPath );
+
+		// When
+		moduleRecord
+		    .loadDescriptor( context )
+		    .register( context )
+		    .activate( context );
+
+		moduleService.getRegistry().put( moduleName, moduleRecord );
+	}
+
+}

src/test/java/ortus/boxlang/modules/compat/IntegrationTest.java

@@ -2,60 +2,19 @@
 
 import static com.google.common.truth.Truth.assertThat;
 
-import java.nio.file.Paths;
-
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 
-import ortus.boxlang.runtime.BoxRuntime;
-import ortus.boxlang.runtime.context.IBoxContext;
-import ortus.boxlang.runtime.context.ScriptingRequestBoxContext;
-import ortus.boxlang.runtime.modules.ModuleRecord;
-import ortus.boxlang.runtime.scopes.IScope;
-import ortus.boxlang.runtime.scopes.Key;
-import ortus.boxlang.runtime.scopes.VariablesScope;
-import ortus.boxlang.runtime.services.ModuleService;
-
 /**
  * This loads the module and runs an integration test on the module.
  */
-public class IntegrationTest {
-
-	static BoxRuntime		runtime;
-	static ModuleService	moduleService;
-	static Key				result	= new Key( "result" );
-	IBoxContext				context;
-	IScope					variables;
-
-	@BeforeAll
-	public static void setup() {
-		runtime			= BoxRuntime.getInstance( true );
-		moduleService	= runtime.getModuleService();
-	}
-
-	@BeforeEach
-	public void setupEach() {
-		context		= new ScriptingRequestBoxContext();
-		variables	= context.getScopeNearby( VariablesScope.name );
-	}
+public class IntegrationTest extends BaseIntegrationTest {
 
 	@DisplayName( "Test the module loads in BoxLang" )
 	@Test
 	public void testModuleLoads() {
 		// Given
-		Key				moduleName		= new Key( "compat" );
-		String			physicalPath	= Paths.get( "./build/module" ).toAbsolutePath().toString();
-		ModuleRecord	moduleRecord	= new ModuleRecord( physicalPath );
-
-		// When
-		moduleRecord
-		    .loadDescriptor( context )
-		    .register( context )
-		    .activate( context );
-
-		moduleService.getRegistry().put( moduleName, moduleRecord );
+		loadModule();
 
 		// Then
 		assertThat( moduleService.getRegistry().containsKey( moduleName ) ).isTrue();

src/test/java/ortus/boxlang/modules/compat/JSONTest.java

@@ -0,0 +1,34 @@
+package ortus.boxlang.modules.compat;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * This loads the module and runs an integration test on the module.
+ */
+public class JSONTest extends BaseIntegrationTest {
+
+	@DisplayName( "Test control characters in JSON" )
+	@Test
+	public void testJSONControlCharacters() {
+		// Given
+		loadModule();
+
+		// @formatter:off
+		runtime.executeSource( """
+			seed = '{
+						"COLUMNS":
+							["EIN","FIRST NAME","LAST NAME"," EMAIL","MANAGER EIN","MANAGER NAME","MANAGER:EMAIL","TEAM"," COMPANY"," GENDER ","N-LEVEL","COUNTRY","LETTER REQUIRED!","	TEMPLATETOSEND","^INVITATION METHOD$","!COMPLETION@METHOD%"],
+						"DATA":
+							[
+								["01234","Amy","Adams","a-a@anyco.co.uk","","","","CEO","Any Co","Male","","United Kingdom","","Batch 2 - UK & I ","Online","Online"],
+								[56789,"Barry","Bocum","b.b@anyco.co.uk","01234","Charlie Chalk","c.c@anyco.co.uk","Division","Division  Ireland","Male",-5.0,"Ireland","","Batch 2 - UK & I ","Online","Online"]
+							]
+					}';
+			result = jsonDeserialize( seed, false );
+			"""
+			, context );
+		// @formatter:on
+
+	}
+}