From 01bb33d3ea8629261a64c600ded260f6158773f1 Mon Sep 17 00:00:00 2001 From: Michael Balser Date: Thu, 26 May 2011 15:34:13 +0200 Subject: [PATCH 1/2] Added support for storing large strings as multiple attribute/value pairs --- .../simplejpa/SplitValueTests.java | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 core/src/test/java/com/spaceprogram/simplejpa/SplitValueTests.java diff --git a/core/src/test/java/com/spaceprogram/simplejpa/SplitValueTests.java b/core/src/test/java/com/spaceprogram/simplejpa/SplitValueTests.java new file mode 100644 index 0000000..5b9a8ec --- /dev/null +++ b/core/src/test/java/com/spaceprogram/simplejpa/SplitValueTests.java @@ -0,0 +1,77 @@ +package com.spaceprogram.simplejpa; + +import javax.persistence.EntityManager; + +import junit.framework.Assert; + +import org.junit.Test; + +public class SplitValueTests extends BaseTestClass { + + private void doTest(String value) { + EntityManager em = factory.createEntityManager(); + MyTestObject object = new MyTestObject(); + object.setName(value); + em.persist(object); + em.close(); + + em = factory.createEntityManager(); + object = em.find(MyTestObject.class, object.getId()); + Assert.assertEquals(value, object.getName()); + em.remove(object); + em.close(); + } + + @Test + public void testSmallStringValue() { + doTest("Test"); + } + + @Test + public void testMaximumValueWithNoSplitting() { + StringBuffer s = new StringBuffer(); + for (int i = 0; i < 1024; i ++) { + s.append((char) ('a' + i % 26)); + } + doTest(s.toString()); + } + + @Test + public void testMinimumValueWithSplitting() { + StringBuffer s = new StringBuffer(); + for (int i = 0; i < 1025; i ++) { + s.append((char) ('a' + i % 26)); + } + doTest(s.toString()); + } + + @Test + public void testValueWithLargeNumberOfSplits() { + StringBuffer s = new StringBuffer(); + for (int i = 0; i < 1024 * 12 + 1; i ++) { + s.append((char) ('a' + i % 26)); + } + doTest(s.toString()); + } + + @Test + public void testValueWithSpecialCharactersAtEvenOffset() { + StringBuffer s = new StringBuffer(); + for (int i = 0; i < 1024 * 2; i ++) { + s.append('\u00e4'); + } + doTest(s.toString()); + } + + + @Test + public void testValueWithSpecialCharactersAtOddOffset() { + StringBuffer s = new StringBuffer(); + s.append('a'); + for (int i = 0; i < 1024 * 2; i ++) { + s.append('\u00e4'); + } + doTest(s.toString()); + } + +} From 5df06fd93b2135e6868b150b9a536755cabc819c Mon Sep 17 00:00:00 2001 From: Michael Balser Date: Thu, 26 May 2011 15:37:13 +0200 Subject: [PATCH 2/2] Added support for storing large strings as multiple attribute/value pairs (Part 2) --- core/pom.xml | 2 +- .../spaceprogram/simplejpa/ObjectBuilder.java | 31 ++++++- .../simplejpa/operations/Save.java | 88 +++++++++++++------ 3 files changed, 89 insertions(+), 32 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 68a1da7..0cbd4f4 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -2,7 +2,7 @@ 4.0.0 SimpleJPA SimpleJPA - 1.6-SNAPSHOT + 1.6.1-SNAPSHOT SimpleJPA http://code.google.com/p/simplejpa diff --git a/core/src/main/java/com/spaceprogram/simplejpa/ObjectBuilder.java b/core/src/main/java/com/spaceprogram/simplejpa/ObjectBuilder.java index a448ca8..ef81edf 100644 --- a/core/src/main/java/com/spaceprogram/simplejpa/ObjectBuilder.java +++ b/core/src/main/java/com/spaceprogram/simplejpa/ObjectBuilder.java @@ -15,6 +15,7 @@ import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.logging.Logger; @@ -26,6 +27,7 @@ * * Additional Contributions * - Yair Ben-Meir reformy@gmail.com + * - Michael Balser michael@die-balsers.de */ public class ObjectBuilder { @@ -165,14 +167,37 @@ private static String getIdForManyToOne(EntityManagerSimpleJPA em, Method getter private static String getValueToSet(List atts, String propertyName, String columnName) { if(columnName != null) propertyName = columnName; + // Retrieve all matching attributes. + List matchingAtts = new ArrayList(); for (Attribute att : atts) { String attName = att.getName(); if (attName.equals(propertyName)) { - String val = att.getValue(); - return val; + matchingAtts.add(att); } } - return null; + if (matchingAtts.size() == 0) { + return null; + } else if (matchingAtts.size() == 1) { + // Value has not been split into multiple chunks. + // Simply return value. + String val = matchingAtts.get(0).getValue(); + return val; + } else { + // Value has been split into multiple chunks. + // 1. Order chunks according to attached counter. + String[] chunks = new String[matchingAtts.size()]; + for (int i = 0; i < matchingAtts.size(); i ++) { + String chunk = matchingAtts.get(i).getValue(); + int counter = Integer.parseInt("" + chunk.charAt(chunk.length() - 4)) * 1000 + Integer.parseInt("" + chunk.charAt(chunk.length() - 3)) * 100 + Integer.parseInt("" + chunk.charAt(chunk.length() - 2)) * 10 + Integer.parseInt("" + chunk.charAt(chunk.length() - 1)); + chunks[counter] = chunk.substring(0, chunk.length() - 4); + } + // 2. Append chunks. + StringBuffer val = new StringBuffer(); + for (int i = 0; i < chunks.length; i ++) { + val.append(chunks[i]); + } + return val.toString(); + } } diff --git a/core/src/main/java/com/spaceprogram/simplejpa/operations/Save.java b/core/src/main/java/com/spaceprogram/simplejpa/operations/Save.java index c9d09aa..7c7ce89 100644 --- a/core/src/main/java/com/spaceprogram/simplejpa/operations/Save.java +++ b/core/src/main/java/com/spaceprogram/simplejpa/operations/Save.java @@ -1,39 +1,15 @@ package com.spaceprogram.simplejpa.operations; -import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.simpledb.model.Attribute; -import com.amazonaws.services.simpledb.model.DeleteAttributesRequest; -import com.amazonaws.services.simpledb.model.Item; -import com.amazonaws.services.simpledb.model.PutAttributesRequest; -import com.amazonaws.services.simpledb.model.ReplaceableAttribute; -import com.spaceprogram.simplejpa.AnnotationInfo; -import com.spaceprogram.simplejpa.DomainHelper; -import com.spaceprogram.simplejpa.EntityManagerFactoryImpl; -import com.spaceprogram.simplejpa.EntityManagerSimpleJPA; -import com.spaceprogram.simplejpa.LazyInterceptor; -import com.spaceprogram.simplejpa.NamingHelper; -import net.sf.cglib.proxy.Factory; - -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.Id; -import javax.persistence.Lob; -import javax.persistence.ManyToOne; -import javax.persistence.OneToMany; -import javax.persistence.PersistenceException; -import javax.persistence.PostPersist; -import javax.persistence.PostUpdate; -import javax.persistence.PrePersist; -import javax.persistence.PreUpdate; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.ObjectOutputStream; +import java.io.UnsupportedEncodingException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.UUID; @@ -41,10 +17,39 @@ import java.util.logging.Level; import java.util.logging.Logger; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Id; +import javax.persistence.Lob; +import javax.persistence.ManyToOne; +import javax.persistence.OneToMany; +import javax.persistence.PersistenceException; +import javax.persistence.PostPersist; +import javax.persistence.PostUpdate; +import javax.persistence.PrePersist; +import javax.persistence.PreUpdate; + +import net.sf.cglib.proxy.Factory; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.simpledb.model.Attribute; +import com.amazonaws.services.simpledb.model.DeleteAttributesRequest; +import com.amazonaws.services.simpledb.model.PutAttributesRequest; +import com.amazonaws.services.simpledb.model.ReplaceableAttribute; +import com.spaceprogram.simplejpa.AnnotationInfo; +import com.spaceprogram.simplejpa.EntityManagerFactoryImpl; +import com.spaceprogram.simplejpa.EntityManagerSimpleJPA; +import com.spaceprogram.simplejpa.LazyInterceptor; +import com.spaceprogram.simplejpa.NamingHelper; + /** * User: treeder * Date: Apr 1, 2008 * Time: 11:51:16 AM + * + * Additional Contributions + * - Michael Balser michael@die-balsers.de */ public class Save implements Callable { private static Logger logger = Logger.getLogger(Save.class.getName()); @@ -200,8 +205,35 @@ else if(getter.getAnnotation(Id.class) != null) } else { String toSet = ob != null ? em.padOrConvertIfRequired(ob) : ""; - // todo: throw an exception if this is going to exceed maximum size, suggest using @Lob - attsToPut.add(new ReplaceableAttribute(columnName, toSet, true)); + + try { + // Check size of encoded value. + byte[] bytes = toSet.getBytes("UTF-8"); + if (bytes.length > 1024) { + // Maximum size is exceeded; split value into multiple chunks. + int i = 0, pos = 0; + while (pos < bytes.length) { + int size = 1020; + // Beware: do not split encoded characters. + // (Additional bytes of an encoded character follow the pattern 10xxxxxx.) + while (pos + size < bytes.length && (bytes[pos + size] & 0xc0) == 0x80) { + size --; + } + String chunk = new String(Arrays.copyOfRange(bytes, pos, Math.min(pos + size, bytes.length)), "UTF-8"); + // Add four digit counter. + String counter = Integer.toString(i / 1000 % 10) + Integer.toString(i / 100 % 10) + Integer.toString(i / 10 % 10) + Integer.toString(i % 10); + attsToPut.add(new ReplaceableAttribute(columnName, chunk + counter, i == 0)); + i ++; + pos += size; + } + } else { + // Simply store string as single value. + attsToPut.add(new ReplaceableAttribute(columnName, toSet, true)); + } + } catch (UnsupportedEncodingException x) { + // should never happen + throw new PersistenceException("Encoding 'UTF-8' is not supported!", x); + } } }