From 3af152482806247789c4a7ceeed02f2247b0e51f Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Tue, 15 Oct 2024 17:25:04 +0200 Subject: [PATCH] BL-642 #resolve deserializeJSON not handling white space / control characters --- src/main/bx/ModuleConfig.bx | 3 + .../bifs/conversion/JSONDeserialize.java | 97 +++++++++++++++++++ .../modules/compat/util/KeyDictionary.java | 8 +- .../modules/compat/BaseIntegrationTest.java | 65 +++++++++++++ .../modules/compat/IntegrationTest.java | 45 +-------- .../boxlang/modules/compat/JSONTest.java | 34 +++++++ 6 files changed, 206 insertions(+), 46 deletions(-) create mode 100644 src/main/java/ortus/boxlang/modules/compat/bifs/conversion/JSONDeserialize.java create mode 100644 src/test/java/ortus/boxlang/modules/compat/BaseIntegrationTest.java create mode 100644 src/test/java/ortus/boxlang/modules/compat/JSONTest.java diff --git a/src/main/bx/ModuleConfig.bx b/src/main/bx/ModuleConfig.bx index b4cd104..5ed1228 100644 --- a/src/main/bx/ModuleConfig.bx +++ b/src/main/bx/ModuleConfig.bx @@ -84,6 +84,9 @@ engine = "lucee", isLucee = false, isAdobe = false, + // JSON control character auto-escaping flag + // IF you turn to true, be aware that the entire JSON serialization will be escaped and be slower. + jsonEscapeControlCharacters = true, // This simulates the query to empty value that Adobe/Lucee do when NOT in full null support // We default it to true to simulate Adobe/Lucee behavior queryNullToEmpty = true, diff --git a/src/main/java/ortus/boxlang/modules/compat/bifs/conversion/JSONDeserialize.java b/src/main/java/ortus/boxlang/modules/compat/bifs/conversion/JSONDeserialize.java new file mode 100644 index 0000000..2e12228 --- /dev/null +++ b/src/main/java/ortus/boxlang/modules/compat/bifs/conversion/JSONDeserialize.java @@ -0,0 +1,97 @@ +package ortus.boxlang.modules.compat.bifs.conversion; + +import ortus.boxlang.modules.compat.util.KeyDictionary; +import ortus.boxlang.runtime.bifs.BoxBIF; +import ortus.boxlang.runtime.bifs.BoxMember; +import ortus.boxlang.runtime.context.IBoxContext; +import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; +import ortus.boxlang.runtime.modules.ModuleRecord; +import ortus.boxlang.runtime.scopes.ArgumentsScope; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.BoxLangType; + +@BoxBIF +@BoxMember( type = BoxLangType.STRING ) +public class JSONDeserialize extends ortus.boxlang.runtime.bifs.global.conversion.JSONDeserialize { + + public static final Key moduleName = Key.of( "compat-cfml" ); + + /** + * Converts a JSON (JavaScript Object Notation) string data representation into data, such as a structure or array. + * + * @param context The context in which the BIF is being invoked. + * @param arguments Argument scope for the BIF. + * + * @argument.json The JSON string to convert to data. + * + * @argument.strictMapping A Boolean value that specifies whether to convert the JSON strictly. If true, everything becomes structures. + * + * @argument.useCustomSerializer A string that specifies the name of a custom serializer to use. (Not used) + * + * @return The data representation of the JSON string. + */ + @Override + public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { + String json = arguments.getAsString( Key.json ); + ModuleRecord moduleRecord = moduleService.getModuleRecord( moduleName ); + Object moduleSetting = moduleRecord.settings.get( KeyDictionary.jsonEscapeControlCharacters ); + Boolean escapeControlCharacters = BooleanCaster + .attempt( moduleSetting ) + .getOrDefault( false ); + + if ( escapeControlCharacters ) { + // Escape all control characters in the JSON string + arguments.put( Key.json, escapeControlCharacters( json ) ); + } + + return super._invoke( context, arguments ); + } + + /** + * Escape all control characters in ASCII range 0-31 and + * convert them to their escaped representation. + * + * @param json The JSON string to escape control characters in. + * + * @return The JSON string with control characters escaped. + */ + private String escapeControlCharacters( String json ) { + StringBuilder escapedJson = new StringBuilder(); + boolean inQuotes = false; // Flag to track if we're inside quotes (single or double) + + for ( int i = 0; i < json.length(); i++ ) { + char c = json.charAt( i ); + + // Check for single or double quotes and toggle the inQuotes flag + if ( c == '\"' || c == '\'' ) { + escapedJson.append( c ); + inQuotes = !inQuotes; + } else if ( inQuotes && c < 32 ) { + // Only escape control characters if inside quotes + switch ( c ) { + case 9 : // Tab + escapedJson.append( "\\t" ); + break; + case 10 : // Newline + escapedJson.append( "\\n" ); + break; + case 13 : // Carriage return + escapedJson.append( "\\r" ); + break; + case 12 : // Form feed + escapedJson.append( "\\f" ); + break; + default : + escapedJson.append( String.format( "\\u%04x", ( int ) c ) ); + break; + } + } else { + // If not in quotes, append character as-is + escapedJson.append( c ); + } + } + + return escapedJson.toString(); + } + +} diff --git a/src/main/java/ortus/boxlang/modules/compat/util/KeyDictionary.java b/src/main/java/ortus/boxlang/modules/compat/util/KeyDictionary.java index 63795e6..5705447 100644 --- a/src/main/java/ortus/boxlang/modules/compat/util/KeyDictionary.java +++ b/src/main/java/ortus/boxlang/modules/compat/util/KeyDictionary.java @@ -18,8 +18,10 @@ public class KeyDictionary { - public static final Key filterOrTags = Key.of( "filterOrTags" ); - public static final Key isKey = Key.of( "isKey" ); - public static final Key objectType = Key.of( "objectType" ); + public static final Key filterOrTags = Key.of( "filterOrTags" ); + public static final Key isKey = Key.of( "isKey" ); + public static final Key objectType = Key.of( "objectType" ); + public static final Key moduleName = Key.of( "compat-cfml" ); + public static final Key jsonEscapeControlCharacters = Key.of( "jsonEscapeControlCharacters" ); } diff --git a/src/test/java/ortus/boxlang/modules/compat/BaseIntegrationTest.java b/src/test/java/ortus/boxlang/modules/compat/BaseIntegrationTest.java new file mode 100644 index 0000000..56e1ed5 --- /dev/null +++ b/src/test/java/ortus/boxlang/modules/compat/BaseIntegrationTest.java @@ -0,0 +1,65 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.modules.compat; + +import java.nio.file.Paths; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; + +import ortus.boxlang.runtime.BoxRuntime; +import ortus.boxlang.runtime.context.ScriptingRequestBoxContext; +import ortus.boxlang.runtime.modules.ModuleRecord; +import ortus.boxlang.runtime.scopes.IScope; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.scopes.VariablesScope; +import ortus.boxlang.runtime.services.ModuleService; + +public class BaseIntegrationTest { + + protected static BoxRuntime runtime; + protected static ModuleService moduleService; + protected static Key result = new Key( "result" ); + protected static Key moduleName = new Key( "compat-cfml" ); + protected ScriptingRequestBoxContext context; + protected IScope variables; + + @BeforeAll + public static void setup() { + runtime = BoxRuntime.getInstance( true ); + moduleService = runtime.getModuleService(); + } + + @BeforeEach + public void setupEach() { + context = new ScriptingRequestBoxContext(); + variables = context.getScopeNearby( VariablesScope.name ); + } + + @SuppressWarnings( "unused" ) + protected void loadModule() { + String physicalPath = Paths.get( "./build/module" ).toAbsolutePath().toString(); + ModuleRecord moduleRecord = new ModuleRecord( physicalPath ); + + // When + moduleRecord + .loadDescriptor( context ) + .register( context ) + .activate( context ); + + moduleService.getRegistry().put( moduleName, moduleRecord ); + } + +} diff --git a/src/test/java/ortus/boxlang/modules/compat/IntegrationTest.java b/src/test/java/ortus/boxlang/modules/compat/IntegrationTest.java index a17e51c..099d17d 100644 --- a/src/test/java/ortus/boxlang/modules/compat/IntegrationTest.java +++ b/src/test/java/ortus/boxlang/modules/compat/IntegrationTest.java @@ -2,60 +2,19 @@ import static com.google.common.truth.Truth.assertThat; -import java.nio.file.Paths; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import ortus.boxlang.runtime.BoxRuntime; -import ortus.boxlang.runtime.context.IBoxContext; -import ortus.boxlang.runtime.context.ScriptingRequestBoxContext; -import ortus.boxlang.runtime.modules.ModuleRecord; -import ortus.boxlang.runtime.scopes.IScope; -import ortus.boxlang.runtime.scopes.Key; -import ortus.boxlang.runtime.scopes.VariablesScope; -import ortus.boxlang.runtime.services.ModuleService; - /** * This loads the module and runs an integration test on the module. */ -public class IntegrationTest { - - static BoxRuntime runtime; - static ModuleService moduleService; - static Key result = new Key( "result" ); - IBoxContext context; - IScope variables; - - @BeforeAll - public static void setup() { - runtime = BoxRuntime.getInstance( true ); - moduleService = runtime.getModuleService(); - } - - @BeforeEach - public void setupEach() { - context = new ScriptingRequestBoxContext(); - variables = context.getScopeNearby( VariablesScope.name ); - } +public class IntegrationTest extends BaseIntegrationTest { @DisplayName( "Test the module loads in BoxLang" ) @Test public void testModuleLoads() { // Given - Key moduleName = new Key( "compat" ); - String physicalPath = Paths.get( "./build/module" ).toAbsolutePath().toString(); - ModuleRecord moduleRecord = new ModuleRecord( physicalPath ); - - // When - moduleRecord - .loadDescriptor( context ) - .register( context ) - .activate( context ); - - moduleService.getRegistry().put( moduleName, moduleRecord ); + loadModule(); // Then assertThat( moduleService.getRegistry().containsKey( moduleName ) ).isTrue(); diff --git a/src/test/java/ortus/boxlang/modules/compat/JSONTest.java b/src/test/java/ortus/boxlang/modules/compat/JSONTest.java new file mode 100644 index 0000000..f2d7c41 --- /dev/null +++ b/src/test/java/ortus/boxlang/modules/compat/JSONTest.java @@ -0,0 +1,34 @@ +package ortus.boxlang.modules.compat; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * This loads the module and runs an integration test on the module. + */ +public class JSONTest extends BaseIntegrationTest { + + @DisplayName( "Test control characters in JSON" ) + @Test + public void testJSONControlCharacters() { + // Given + loadModule(); + + // @formatter:off + runtime.executeSource( """ + seed = '{ + "COLUMNS": + ["EIN","FIRST NAME","LAST NAME"," EMAIL","MANAGER EIN","MANAGER NAME","MANAGER:EMAIL","TEAM"," COMPANY"," GENDER ","N-LEVEL","COUNTRY","LETTER REQUIRED!"," TEMPLATETOSEND","^INVITATION METHOD$","!COMPLETION@METHOD%"], + "DATA": + [ + ["01234","Amy","Adams","a-a@anyco.co.uk","","","","CEO","Any Co","Male","","United Kingdom","","Batch 2 - UK & I ","Online","Online"], + [56789,"Barry","Bocum","b.b@anyco.co.uk","01234","Charlie Chalk","c.c@anyco.co.uk","Division","Division Ireland","Male",-5.0,"Ireland","","Batch 2 - UK & I ","Online","Online"] + ] + }'; + result = jsonDeserialize( seed, false ); + """ + , context ); + // @formatter:on + + } +}