Skip to content

Commit

Permalink
Merge pull request #37 from tkrs/improve-performance
Browse files Browse the repository at this point in the history 
Improve performance of msg-pack
  • Loading branch information
Takeru Sato authored Dec 5, 2016
2 parents cfba736 + 12d72dc commit cf7fb11
Show file tree
Hide file tree
Showing 4 changed files with 245 additions and 136 deletions.
5 changes: 4 additions & 1 deletion core/src/main/scala/fluflu/Message.scala
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package fluflu

import io.circe.{ Encoder, Json }
import java.nio.charset.StandardCharsets

import cats.syntax.either._
import io.circe.{ Encoder, Json, Printer }
import io.circe.syntax._

import scala.util.{ Either => \/ }
Expand Down
307 changes: 207 additions & 100 deletions msgpack/src/main/scala/fluflu/msgpack/MessagePacker.scala
View file
Original file line number Diff line number Diff line change
@@ -1,91 +1,175 @@
package fluflu
package msgpack

import java.nio.charset.StandardCharsets.UTF_8
import java.lang.Double.doubleToLongBits
import java.nio.{ ByteBuffer, CharBuffer }
import java.nio.charset.{ CharsetEncoder, StandardCharsets }

import cats.syntax.either._
import io.circe.{ Encoder, Json, JsonNumber, JsonObject }
import io.circe.{ Encoder, Json }

import scala.annotation.tailrec
import scala.util.{ Either => \/ }

object MessagePacker {

final val `0x3d` = BigInt(0xd3.toLong)
final val `0xcf` = BigInt(0xcf.toLong)
val encoder: CharsetEncoder = StandardCharsets.UTF_8.newEncoder()

final val `0xc3`: Byte = 0xc3.toByte
final val `0xc2`: Byte = 0xc2.toByte

final val `0xcb`: Byte = 0xcb.toByte
final val `0xcf`: Byte = 0xcf.toByte
final val `0xce`: Byte = 0xce.toByte
final val `0xcd`: Byte = 0xcd.toByte
final val `0xcc`: Byte = 0xcc.toByte
final val `0xd0`: Byte = 0xd0.toByte
final val `0xd1`: Byte = 0xd1.toByte
final val `0xd2`: Byte = 0xd2.toByte
final val `0xd3`: Byte = 0xd3.toByte
final val `0x3d`: Byte = 0xd3.toByte

final val `0xda`: Byte = 0xda.toByte
final val `0xdb`: Byte = 0xdb.toByte
final val `0xdd`: Byte = 0xdd.toByte

// final val `0x90`: Byte = 0x90.toByte
final val `0xdc`: Byte = 0xdc.toByte
final val `0xde`: Byte = 0xde.toByte
final val `0xdf`: Byte = 0xdf.toByte

def apply() = new MessagePacker()

def formatArrayFamilyHeader(size: Int): Vector[Byte] =
def formatArrayFamilyHeader(size: Int, builder: ByteBuffer): Unit = {
if (size < 16)
Vector((0x90 | size).toByte)
else if (size < 65536)
Vector(0xdc, size >>> 8, size >>> 0).map(_.toByte)
else
Vector(0xdd, size >>> 24, size >>> 16, size >>> 8, size >>> 0).map(_.toByte)
builder.put((0x90 | size).toByte)
else if (size < 65536) {
builder.put(`0xdc`)
builder.put((size >>> 8).toByte)
builder.put((size >>> 0).toByte)
} else {
builder.put(`0xdd`)
builder.put((size >>> 24).toByte)
builder.put((size >>> 16).toByte)
builder.put((size >>> 8).toByte)
builder.put((size >>> 0).toByte)
}
}

def formatMapFamilyHeader(sz: Int): Vector[Byte] =
def formatMapFamilyHeader(sz: Int, builder: ByteBuffer): Unit = {
if (sz < 16)
Vector((0x80 | sz).toByte)
else if (sz < 65536)
Vector(0xde, sz >>> 8, sz >>> 0).map(_.toByte)
else
Vector(0xdf, sz >>> 24, sz >>> 16, sz >>> 8, sz >>> 0).map(_.toByte)
builder.put((0x80 | sz).toByte)
else if (sz < 65536) {
builder.put(`0xde`)
builder.put((sz >>> 8).toByte)
builder.put((sz >>> 0).toByte)
} else {
builder.put(`0xdf`)
builder.put((sz >>> 24).toByte)
builder.put((sz >>> 16).toByte)
builder.put((sz >>> 8).toByte)
builder.put((sz >>> 0).toByte)
}
}

def formatStrFamilyHeader(sz: Int): Vector[Byte] =
def formatStrFamilyHeader(sz: Int, builder: ByteBuffer): Unit =
if (sz < 32)
Vector((0xa0 | sz).toByte)
else if (sz < 65536)
Vector(0xda, sz >>> 8, sz >>> 0).map(_.toByte)
else
Vector(0xdb, sz >>> 24, sz >>> 16, sz >>> 8, sz >>> 0).map(_.toByte)

def formatNil: Vector[Byte] = Vector(0xc0.toByte)

def formatBoolFamily(v: Boolean): Vector[Byte] =
if (v) Vector(0xc3.toByte) else Vector(0xc2.toByte)

def formatIntFamily(l: Long): Vector[Byte] =
if (4294967296L <= l) formatLong(0xcf, l)
else if (65536L <= l) formatInt(0xce, l.toInt)
else if (256L <= l) formatShort(0xcd, l.toInt)
else if (128 <= l) formatByte(0xcc, l.toInt)
else if (0 <= l) Vector(l.toByte)
else if (l >= -32L) formatByte(0xe0 | (l + 32).toInt)
else if (l >= Byte.MinValue.toLong) formatByte(0xd0, l.toInt)
else if (l >= Short.MinValue.toLong) formatShort(0xd1, l.toInt)
else if (l >= Int.MinValue.toLong) formatInt(0xd2, l.toInt)
else formatLong(0xd3, l)

def formatIntFamily(t: BigInt, v: BigInt): Vector[Byte] =
Vector(t, v >> 56, v >> 48, v >> 40, v >> 32, v >> 24, v >> 16, v >> 8, v >> 0).map(_.toByte)

def formatFloatFamily(v: Double): Vector[Byte] =
(doubleToLongBits _ andThen (x => Vector(0xcb.toLong, x >>> 56, x >>> 48, x >>> 40, x >>> 32, x >>> 24, x >>> 16, x >>> 8, x >>> 0)))(v).map(_.toByte)

def formatStrFamily(v: String): Vector[Byte] =
formatStrFamilyHeader(strSize(v.toCharArray)) ++ v.getBytes(UTF_8)

def formatByte(v: Int): Vector[Byte] =
Vector(v.toByte)
def formatByte(t: Int, v: Int): Vector[Byte] =
Vector(t, v).map(_.toByte)
def formatShort(t: Int, v: Int): Vector[Byte] =
Vector(t, v >>> 8, v >>> 0).map(_.toByte)
def formatInt(t: Int, v: Int): Vector[Byte] =
Vector(t, v >>> 24, v >>> 16, v >>> 8, v >>> 0).map(_.toByte)
def formatLong(t: Long, v: Long): Vector[Byte] =
Vector(t, v >>> 56, v >>> 48, v >>> 40, v >>> 32, v >>> 24, v >>> 16, v >>> 8, v >>> 0).map(_.toByte)

def strSize(value: Array[Char]): Int = strSize(value, 0)
@tailrec def strSize(value: Array[Char], count: Int): Int = value match {
case a if a.isEmpty => 0
case Array(h) => count + charSize(h)
case _ => strSize(value.tail, count + charSize(value(0)))
builder.put((0xa0 | sz).toByte)
else if (sz < 65536) {
builder.put(`0xda`)
builder.put((sz >>> 8).toByte)
builder.put((sz >>> 0).toByte)
} else {
builder.put(`0xdb`)
builder.put((sz >>> 24).toByte)
builder.put((sz >>> 16).toByte)
builder.put((sz >>> 8).toByte)
builder.put((sz >>> 0).toByte)
}

val formatNil: Byte = 0xc0.toByte

def formatBoolFamily(v: Boolean, builder: ByteBuffer): Unit =
builder.put(if (v) `0xc3` else `0xc2`)

def formatIntFamily(l: Long, builder: ByteBuffer): Unit =
if (4294967296L <= l) formatLong(`0xcf`, l, builder)
else if (65536L <= l) formatInt(`0xce`, l.toInt, builder)
else if (256L <= l) formatShort(`0xcd`, l.toInt, builder)
else if (128 <= l) formatByte(`0xcc`, l.toByte, builder)
else if (0 <= l) formatByte(l.toByte, builder)
else if (l >= -32L) formatByte((0xe0 | (l + 32)).toByte, builder)
else if (l >= Byte.MinValue.toLong) formatByte(`0xd0`, l.toInt.toByte, builder)
else if (l >= Short.MinValue.toLong) formatShort(`0xd1`, l.toInt, builder)
else if (l >= Int.MinValue.toLong) formatInt(`0xd2`, l.toInt, builder)
else formatLong(`0xd3`, l, builder)

def formatIntFamily(t: Byte, v: BigInt, builder: ByteBuffer): Unit = {
builder.put(t)
builder.put((v >> 56).toByte)
builder.put((v >> 48).toByte)
builder.put((v >> 40).toByte)
builder.put((v >> 32).toByte)
builder.put((v >> 24).toByte)
builder.put((v >> 16).toByte)
builder.put((v >> 8).toByte)
builder.put((v >> 0).toByte)
}

def formatFloatFamily(v: Double, builder: ByteBuffer): Unit = {
val x = doubleToLongBits(v)
builder.put(`0xcb`)
builder.put((x >>> 56).toByte)
builder.put((x >>> 48).toByte)
builder.put((x >>> 40).toByte)
builder.put((x >>> 32).toByte)
builder.put((x >>> 24).toByte)
builder.put((x >>> 16).toByte)
builder.put((x >>> 8).toByte)
builder.put((x >>> 0).toByte)
}

def formatStrFamily(v: String, builder: ByteBuffer): Unit = {
val cb = CharBuffer.wrap(v)
val buf = encoder.encode(cb)
formatStrFamilyHeader(strSize(cb), builder)
builder.put(buf)
cb.clear()
}

@inline def charSize(ch: Char): Int =
def formatByte(v: Byte, builder: ByteBuffer): Unit = builder.put(v)

def formatByte(t: Byte, v: Byte, builder: ByteBuffer): Unit = {
builder.put(t)
builder.put(v)
}
def formatShort(t: Byte, v: Int, builder: ByteBuffer): Unit = {
builder.put(t)
builder.put((v >>> 8).toByte)
builder.put((v >>> 0).toByte)
}
def formatInt(t: Byte, v: Int, builder: ByteBuffer): Unit = {
builder.put(t)
builder.put((v >>> 24).toByte)
builder.put((v >>> 16).toByte)
builder.put((v >>> 8).toByte)
builder.put((v >>> 0).toByte)
}
def formatLong(t: Byte, v: Long, builder: ByteBuffer): Unit = {
builder.put(t)
builder.put((v >>> 56).toByte)
builder.put((v >>> 48).toByte)
builder.put((v >>> 40).toByte)
builder.put((v >>> 32).toByte)
builder.put((v >>> 24).toByte)
builder.put((v >>> 16).toByte)
builder.put((v >>> 8).toByte)
builder.put((v >>> 0).toByte)
}

def strSize(cb: CharBuffer): Int =
(0 until cb.capacity()).foldLeft(0)((l, r) => l + charSize(cb.get(r)))

def charSize(ch: Char): Int =
if (ch < 0x80) 1
else if (ch < 0x800) 2
else if (Character isHighSurrogate ch) 2
Expand All @@ -99,41 +183,64 @@ final class MessagePacker {

def encode[A](a: A)(implicit A: Encoder[A]): Throwable \/ Array[Byte] = pack(A(a))

def pack(doc: Json): Throwable \/ Array[Byte] =
\/.catchNonFatal(go(doc).toArray)
def pack(doc: Json): Throwable \/ Array[Byte] = {
val acc: ByteBuffer = ByteBuffer.allocate(1048576)
\/.catchNonFatal {
go(doc, acc)
val i = acc.position()
val arr = Array.ofDim[Byte](i)
acc.flip()
acc.get(arr)
acc.clear()
arr
}
}

def double(x: BigDecimal): Boolean = x.scale != 0

val go: (Json) => Vector[Byte] = _.fold(
{
formatNil
},
{ x: Boolean =>
formatBoolFamily(x)
},
{ x: JsonNumber =>
val n = x.toBigDecimal
n match {
case None => throw new ArithmeticException()
case Some(v) if double(v) => formatFloatFamily(v.toDouble)
case Some(v) if v.isValidLong => formatIntFamily(v.toLong)
case Some(v) if v.signum == -1 => formatIntFamily(`0x3d`, v.toBigInt())
case Some(v) => formatIntFamily(`0xcf`, v.toBigInt())
}
},
{ xs: String =>
formatStrFamily(xs)
},
{ xs: List[Json] =>
formatArrayFamilyHeader(xs.size) ++ xs.foldLeft(Vector.empty[Byte])(_ ++ go(_))
},
{ x: JsonObject =>
val xs = x.toList
val vec = formatMapFamilyHeader(xs.size)
vec ++ xs.foldLeft(Vector.empty[Byte]) {
case (acc, (key, v)) =>
acc ++ formatStrFamily(key) ++ go(v)
}
def go(json: Json, acc: ByteBuffer): Unit =
if (json.isNull)
acc.put(formatNil)
else if (json.isBoolean) json.asBoolean match {
case None => ()
case Some(x) => formatBoolFamily(x, acc)
}
else if (json.isNumber) json.asNumber match {
case None => ()
case Some(x) =>
val n = x.toBigDecimal
n match {
case None => ()
case Some(v) if double(v) =>
formatFloatFamily(v.toDouble, acc)
case Some(v) if v.isValidLong =>
formatIntFamily(v.toLong, acc)
case Some(v) if v.signum == -1 =>
formatIntFamily(`0x3d`, v.toBigInt(), acc)
case Some(v) =>
formatIntFamily(`0xcf`, v.toBigInt(), acc)
}
}
else if (json.isString) json.asString match {
case None => ()
case Some(x) =>
formatStrFamily(x, acc)
}
else if (json.isArray) json.asArray match {
case None => ()
case Some(xs) =>
formatArrayFamilyHeader(xs.size, acc)
xs.foreach(go(_, acc))
}
else if (json.isObject) json.asObject match {
case None => ()
case Some(x) =>
val xs = x.toList
formatMapFamilyHeader(xs.size, acc)
xs.foreach {
case (key, (v)) =>
formatStrFamily(key, acc)
go(v, acc)
}
}
)
}
Loading

0 comments on commit cf7fb11

Please sign in to comment.