diff --git a/.gitignore b/.gitignore index f406b2b0..023c8d07 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ Gemfile* pkg test/debug.log *~ +*.swp /rdoc tmp .classpath diff --git a/ext/java/org/msgpack/jruby/Decoder.java b/ext/java/org/msgpack/jruby/Decoder.java index 8cb625b4..d7df7578 100644 --- a/ext/java/org/msgpack/jruby/Decoder.java +++ b/ext/java/org/msgpack/jruby/Decoder.java @@ -12,6 +12,7 @@ import org.jruby.RubyClass; import org.jruby.RubyBignum; import org.jruby.RubyString; +import org.jruby.RubyArray; import org.jruby.RubyHash; import org.jruby.exceptions.RaiseException; import org.jruby.runtime.builtin.IRubyObject; @@ -29,33 +30,58 @@ public class Decoder implements Iterator { private final Encoding utf8Encoding; private final RubyClass unpackErrorClass; private final RubyClass underflowErrorClass; + private final RubyClass malformedFormatErrorClass; + private final RubyClass stackErrorClass; private final RubyClass unexpectedTypeErrorClass; + private final RubyClass unknownExtTypeErrorClass; + private ExtensionRegistry registry; private ByteBuffer buffer; private boolean symbolizeKeys; + private boolean allowUnknownExt; public Decoder(Ruby runtime) { - this(runtime, new byte[] {}, 0, 0); + this(runtime, null, new byte[] {}, 0, 0, false, false); + } + + public Decoder(Ruby runtime, ExtensionRegistry registry) { + this(runtime, registry, new byte[] {}, 0, 0, false, false); } public Decoder(Ruby runtime, byte[] bytes) { - this(runtime, bytes, 0, bytes.length); + this(runtime, null, bytes, 0, bytes.length, false, false); + } + + public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes) { + this(runtime, registry, bytes, 0, bytes.length, false, false); + } + + public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, boolean symbolizeKeys, boolean allowUnknownExt) { + this(runtime, registry, bytes, 0, bytes.length, symbolizeKeys, allowUnknownExt); + } + + public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length) { + this(runtime, registry, bytes, offset, length, false, false); } - public Decoder(Ruby runtime, byte[] bytes, int offset, int length) { + public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length, boolean symbolizeKeys, boolean allowUnknownExt) { this.runtime = runtime; + this.registry = registry; + this.symbolizeKeys = symbolizeKeys; + this.allowUnknownExt = allowUnknownExt; this.binaryEncoding = runtime.getEncodingService().getAscii8bitEncoding(); this.utf8Encoding = UTF8Encoding.INSTANCE; this.unpackErrorClass = runtime.getModule("MessagePack").getClass("UnpackError"); this.underflowErrorClass = runtime.getModule("MessagePack").getClass("UnderflowError"); + this.malformedFormatErrorClass = runtime.getModule("MessagePack").getClass("MalformedFormatError"); + this.stackErrorClass = runtime.getModule("MessagePack").getClass("StackError"); this.unexpectedTypeErrorClass = runtime.getModule("MessagePack").getClass("UnexpectedTypeError"); + this.unknownExtTypeErrorClass = runtime.getModule("MessagePack").getClass("UnknownExtTypeError"); + this.symbolizeKeys = symbolizeKeys; + this.allowUnknownExt = allowUnknownExt; feed(bytes, offset, length); } - public void symbolizeKeys(boolean symbolize) { - this.symbolizeKeys = symbolize; - } - public void feed(byte[] bytes) { feed(bytes, 0, bytes.length); } @@ -73,7 +99,7 @@ public void feed(byte[] bytes, int offset, int length) { } public void reset() { - buffer.rewind(); + buffer = null; } public int offset() { @@ -118,7 +144,20 @@ private IRubyObject consumeHash(int size) { private IRubyObject consumeExtension(int size) { int type = buffer.get(); byte[] payload = readBytes(size); - return ExtensionValue.newExtensionValue(runtime, type, payload); + + if (registry != null) { + IRubyObject proc = registry.lookupUnpackerByTypeId(type); + if (proc != null) { + ByteList byteList = new ByteList(payload, runtime.getEncodingService().getAscii8bitEncoding()); + return proc.callMethod(runtime.getCurrentContext(), "call", runtime.newString(byteList)); + } + } + + if (this.allowUnknownExt) { + return ExtensionValue.newExtensionValue(runtime, type, payload); + } + + throw runtime.newRaiseException(unknownExtTypeErrorClass, "unexpected extension type"); } private byte[] readBytes(int size) { @@ -142,11 +181,11 @@ public IRubyObject read_array_header() { try { byte b = buffer.get(); if ((b & 0xf0) == 0x90) { - return runtime.newFixnum(b & 0x0f); + return runtime.newFixnum(b & 0x0f); } else if (b == ARY16) { - return runtime.newFixnum(buffer.getShort() & 0xffff); + return runtime.newFixnum(buffer.getShort() & 0xffff); } else if (b == ARY32) { - return runtime.newFixnum(buffer.getInt()); + return runtime.newFixnum(buffer.getInt()); } throw runtime.newRaiseException(unexpectedTypeErrorClass, "unexpected type"); } catch (RaiseException re) { @@ -163,11 +202,11 @@ public IRubyObject read_map_header() { try { byte b = buffer.get(); if ((b & 0xf0) == 0x80) { - return runtime.newFixnum(b & 0x0f); + return runtime.newFixnum(b & 0x0f); } else if (b == MAP16) { - return runtime.newFixnum(buffer.getShort() & 0xffff); + return runtime.newFixnum(buffer.getShort() & 0xffff); } else if (b == MAP32) { - return runtime.newFixnum(buffer.getInt()); + return runtime.newFixnum(buffer.getInt()); } throw runtime.newRaiseException(unexpectedTypeErrorClass, "unexpected type"); } catch (RaiseException re) { @@ -233,7 +272,7 @@ public IRubyObject next() { default: return runtime.newFixnum(b); } buffer.position(position); - throw runtime.newRaiseException(unpackErrorClass, "Illegal byte sequence"); + throw runtime.newRaiseException(malformedFormatErrorClass, "Illegal byte sequence"); } catch (RaiseException re) { buffer.position(position); throw re; diff --git a/ext/java/org/msgpack/jruby/Encoder.java b/ext/java/org/msgpack/jruby/Encoder.java index 0df9673c..f803b849 100644 --- a/ext/java/org/msgpack/jruby/Encoder.java +++ b/ext/java/org/msgpack/jruby/Encoder.java @@ -34,15 +34,21 @@ public class Encoder { private final Encoding binaryEncoding; private final Encoding utf8Encoding; private final boolean compatibilityMode; + private final ExtensionRegistry registry; private ByteBuffer buffer; - public Encoder(Ruby runtime, boolean compatibilityMode) { + public Encoder(Ruby runtime, boolean compatibilityMode, ExtensionRegistry registry) { this.runtime = runtime; this.buffer = ByteBuffer.allocate(CACHE_LINE_SIZE - ARRAY_HEADER_SIZE); this.binaryEncoding = runtime.getEncodingService().getAscii8bitEncoding(); this.utf8Encoding = UTF8Encoding.INSTANCE; this.compatibilityMode = compatibilityMode; + this.registry = registry; + } + + public boolean isCompatibilityMode() { + return compatibilityMode; } private void ensureRemainingCapacity(int c) { @@ -107,7 +113,7 @@ private void appendObject(IRubyObject object, IRubyObject destination) { } else if (object instanceof ExtensionValue) { appendExtensionValue((ExtensionValue) object); } else { - appendCustom(object, destination); + appendOther(object, destination); } } @@ -295,12 +301,7 @@ public void visit(IRubyObject key, IRubyObject value) { } } - private void appendExtensionValue(ExtensionValue object) { - long type = ((RubyFixnum)object.get_type()).getLongValue(); - if (type < -128 || type > 127) { - throw object.getRuntime().newRangeError(String.format("integer %d too big to convert to `signed char'", type)); - } - ByteList payloadBytes = ((RubyString)object.payload()).getByteList(); + private void appendExt(int type, ByteList payloadBytes) { int payloadSize = payloadBytes.length(); int outputSize = 0; boolean fixSize = payloadSize == 1 || payloadSize == 2 || payloadSize == 4 || payloadSize == 8 || payloadSize == 16; @@ -338,6 +339,28 @@ private void appendExtensionValue(ExtensionValue object) { buffer.put(payloadBytes.unsafeBytes(), payloadBytes.begin(), payloadSize); } + private void appendExtensionValue(ExtensionValue object) { + long type = ((RubyFixnum)object.get_type()).getLongValue(); + if (type < -128 || type > 127) { + throw object.getRuntime().newRangeError(String.format("integer %d too big to convert to `signed char'", type)); + } + ByteList payloadBytes = ((RubyString)object.payload()).getByteList(); + appendExt((int) type, payloadBytes); + } + + private void appendOther(IRubyObject object, IRubyObject destination) { + if (registry != null) { + IRubyObject[] pair = registry.lookupPackerByClass(object.getType()); + if (pair != null) { + RubyString bytes = pair[0].callMethod(runtime.getCurrentContext(), "call", object).asString(); + int type = (int) ((RubyFixnum) pair[1]).getLongValue(); + appendExt(type, bytes.getByteList()); + return; + } + } + appendCustom(object, destination); + } + private void appendCustom(IRubyObject object, IRubyObject destination) { if (destination == null) { IRubyObject result = object.callMethod(runtime.getCurrentContext(), "to_msgpack"); diff --git a/ext/java/org/msgpack/jruby/ExtensionRegistry.java b/ext/java/org/msgpack/jruby/ExtensionRegistry.java new file mode 100644 index 00000000..2ffa4be3 --- /dev/null +++ b/ext/java/org/msgpack/jruby/ExtensionRegistry.java @@ -0,0 +1,159 @@ +package org.msgpack.jruby; + +import org.jruby.Ruby; +import org.jruby.RubyHash; +import org.jruby.RubyArray; +import org.jruby.RubyClass; +import org.jruby.RubyFixnum; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; + +import java.util.Map; +import java.util.HashMap; + +public class ExtensionRegistry { + private final Map extensionsByClass; + private final Map extensionsByAncestor; + private final ExtensionEntry[] extensionsByTypeId; + + public ExtensionRegistry() { + this(new HashMap()); + } + + private ExtensionRegistry(Map extensionsByClass) { + this.extensionsByClass = new HashMap(extensionsByClass); + this.extensionsByAncestor = new HashMap(); + this.extensionsByTypeId = new ExtensionEntry[256]; + for (ExtensionEntry entry : extensionsByClass.values()) { + if (entry.hasUnpacker()) { + extensionsByTypeId[entry.getTypeId() + 128] = entry; + } + } + } + + public ExtensionRegistry dup() { + return new ExtensionRegistry(extensionsByClass); + } + + public IRubyObject toInternalPackerRegistry(ThreadContext ctx) { + RubyHash hash = RubyHash.newHash(ctx.getRuntime()); + for (RubyClass extensionClass : extensionsByClass.keySet()) { + ExtensionEntry entry = extensionsByClass.get(extensionClass); + if (entry.hasPacker()) { + hash.put(extensionClass, entry.toPackerTuple(ctx)); + } + } + return hash; + } + + public IRubyObject toInternalUnpackerRegistry(ThreadContext ctx) { + RubyHash hash = RubyHash.newHash(ctx.getRuntime()); + for (int typeIdIndex = 0 ; typeIdIndex < 256 ; typeIdIndex++) { + ExtensionEntry entry = extensionsByTypeId[typeIdIndex]; + if (entry != null && entry.hasUnpacker()) { + IRubyObject typeId = RubyFixnum.newFixnum(ctx.getRuntime(), typeIdIndex - 128); + hash.put(typeId, entry.toUnpackerTuple(ctx)); + } + } + return hash; + } + + public void put(RubyClass cls, int typeId, IRubyObject packerProc, IRubyObject packerArg, IRubyObject unpackerProc, IRubyObject unpackerArg) { + ExtensionEntry entry = new ExtensionEntry(cls, typeId, packerProc, packerArg, unpackerProc, unpackerArg); + extensionsByClass.put(cls, entry); + extensionsByTypeId[typeId + 128] = entry; + extensionsByAncestor.clear(); + } + + public IRubyObject lookupUnpackerByTypeId(int typeId) { + ExtensionEntry e = extensionsByTypeId[typeId + 128]; + if (e != null && e.hasUnpacker()) { + return e.getUnpackerProc(); + } else { + return null; + } + } + + public IRubyObject[] lookupPackerByClass(RubyClass cls) { + ExtensionEntry e = extensionsByClass.get(cls); + if (e == null) { + e = extensionsByAncestor.get(cls); + } + if (e == null) { + e = findEntryByClassOrAncestor(cls); + if (e != null) { + extensionsByAncestor.put(e.getExtensionClass(), e); + } + } + if (e != null && e.hasPacker()) { + return e.toPackerProcTypeIdPair(cls.getRuntime().getCurrentContext()); + } else { + return null; + } + } + + private ExtensionEntry findEntryByClassOrAncestor(final RubyClass cls) { + ThreadContext ctx = cls.getRuntime().getCurrentContext(); + for (RubyClass extensionClass : extensionsByClass.keySet()) { + RubyArray ancestors = (RubyArray) cls.callMethod(ctx, "ancestors"); + if (ancestors.callMethod(ctx, "include?", extensionClass).isTrue()) { + return extensionsByClass.get(extensionClass); + } + } + return null; + } + + private static class ExtensionEntry { + private final RubyClass cls; + private final int typeId; + private final IRubyObject packerProc; + private final IRubyObject packerArg; + private final IRubyObject unpackerProc; + private final IRubyObject unpackerArg; + + public ExtensionEntry(RubyClass cls, int typeId, IRubyObject packerProc, IRubyObject packerArg, IRubyObject unpackerProc, IRubyObject unpackerArg) { + this.cls = cls; + this.typeId = typeId; + this.packerProc = packerProc; + this.packerArg = packerArg; + this.unpackerProc = unpackerProc; + this.unpackerArg = unpackerArg; + } + + public RubyClass getExtensionClass() { + return cls; + } + + public int getTypeId() { + return typeId; + } + + public boolean hasPacker() { + return packerProc != null; + } + + public boolean hasUnpacker() { + return unpackerProc != null; + } + + public IRubyObject getPackerProc() { + return packerProc; + } + + public IRubyObject getUnpackerProc() { + return unpackerProc; + } + + public RubyArray toPackerTuple(ThreadContext ctx) { + return RubyArray.newArray(ctx.getRuntime(), new IRubyObject[] {RubyFixnum.newFixnum(ctx.getRuntime(), typeId), packerProc, packerArg}); + } + + public RubyArray toUnpackerTuple(ThreadContext ctx) { + return RubyArray.newArray(ctx.getRuntime(), new IRubyObject[] {cls, unpackerProc, unpackerArg}); + } + + public IRubyObject[] toPackerProcTypeIdPair(ThreadContext ctx) { + return new IRubyObject[] {packerProc, RubyFixnum.newFixnum(ctx.getRuntime(), typeId)}; + } + } +} diff --git a/ext/java/org/msgpack/jruby/Factory.java b/ext/java/org/msgpack/jruby/Factory.java new file mode 100644 index 00000000..e5f5d0be --- /dev/null +++ b/ext/java/org/msgpack/jruby/Factory.java @@ -0,0 +1,117 @@ +package org.msgpack.jruby; + + +import org.jruby.Ruby; +import org.jruby.RubyClass; +import org.jruby.RubyObject; +import org.jruby.RubyArray; +import org.jruby.RubyHash; +import org.jruby.RubyIO; +import org.jruby.RubyInteger; +import org.jruby.RubyFixnum; +import org.jruby.RubyString; +import org.jruby.RubySymbol; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.anno.JRubyClass; +import org.jruby.anno.JRubyMethod; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.ObjectAllocator; +import org.jruby.util.ByteList; + +import static org.jruby.runtime.Visibility.PRIVATE; + +@JRubyClass(name="MessagePack::Factory") +public class Factory extends RubyObject { + private final Ruby runtime; + private final ExtensionRegistry extensionRegistry; + + public Factory(Ruby runtime, RubyClass type) { + super(runtime, type); + this.runtime = runtime; + this.extensionRegistry = new ExtensionRegistry(); + } + + static class FactoryAllocator implements ObjectAllocator { + public IRubyObject allocate(Ruby runtime, RubyClass type) { + return new Factory(runtime, type); + } + } + + public ExtensionRegistry extensionRegistry() { + return extensionRegistry.dup(); + } + + @JRubyMethod(name = "initialize") + public IRubyObject initialize(ThreadContext ctx) { + return this; + } + + @JRubyMethod(name = "packer", optional = 1) + public Packer packer(ThreadContext ctx, IRubyObject[] args) { + return Packer.newPacker(ctx, extensionRegistry(), args); + } + + @JRubyMethod(name = "unpacker", optional = 1) + public Unpacker unpacker(ThreadContext ctx, IRubyObject[] args) { + return Unpacker.newUnpacker(ctx, extensionRegistry(), args); + } + + @JRubyMethod(name = "registered_types_internal", visibility = PRIVATE) + public IRubyObject registeredTypesInternal(ThreadContext ctx) { + return RubyArray.newArray(ctx.getRuntime(), new IRubyObject[] { + extensionRegistry.toInternalPackerRegistry(ctx), + extensionRegistry.toInternalUnpackerRegistry(ctx) + }); + } + + @JRubyMethod(name = "register_type", required = 2, optional = 1) + public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args) { + Ruby runtime = ctx.getRuntime(); + IRubyObject type = args[0]; + IRubyObject klass = args[1]; + + IRubyObject packerArg; + IRubyObject unpackerArg; + if (args.length == 2) { + packerArg = runtime.newSymbol("to_msgpack_ext"); + unpackerArg = runtime.newSymbol("from_msgpack_ext"); + } else if (args.length == 3) { + if (args[args.length - 1] instanceof RubyHash) { + RubyHash options = (RubyHash) args[args.length - 1]; + packerArg = options.fastARef(runtime.newSymbol("packer")); + unpackerArg = options.fastARef(runtime.newSymbol("unpacker")); + } else { + throw runtime.newArgumentError(String.format("expected Hash but found %s.", args[args.length - 1].getType().getName())); + } + } else { + throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 2..3)", 2 + args.length)); + } + + long typeId = ((RubyFixnum) type).getLongValue(); + if (typeId < -128 || typeId > 127) { + throw runtime.newRangeError(String.format("integer %d too big to convert to `signed char'", typeId)); + } + + if (!(klass instanceof RubyClass)) { + throw runtime.newArgumentError(String.format("expected Class but found %s.", klass.getType().getName())); + } + RubyClass extClass = (RubyClass) klass; + + IRubyObject packerProc = runtime.getNil(); + IRubyObject unpackerProc = runtime.getNil(); + if (packerArg != null) { + packerProc = packerArg.callMethod(ctx, "to_proc"); + } + if (unpackerArg != null) { + if (unpackerArg instanceof RubyString || unpackerArg instanceof RubySymbol) { + unpackerProc = extClass.method(unpackerArg.callMethod(ctx, "to_sym")); + } else { + unpackerProc = unpackerArg.callMethod(ctx, "method", runtime.newSymbol("call")); + } + } + + extensionRegistry.put(extClass, (int) typeId, packerProc, packerArg, unpackerProc, unpackerArg); + + return runtime.getNil(); + } +} diff --git a/ext/java/org/msgpack/jruby/MessagePackLibrary.java b/ext/java/org/msgpack/jruby/MessagePackLibrary.java index d3e266ca..90f6c245 100644 --- a/ext/java/org/msgpack/jruby/MessagePackLibrary.java +++ b/ext/java/org/msgpack/jruby/MessagePackLibrary.java @@ -20,15 +20,20 @@ public class MessagePackLibrary implements Library { + public static Factory defaultFactory; + public void load(Ruby runtime, boolean wrap) { RubyModule msgpackModule = runtime.defineModule("MessagePack"); msgpackModule.defineAnnotatedMethods(MessagePackModule.class); RubyClass standardErrorClass = runtime.getStandardError(); RubyClass unpackErrorClass = msgpackModule.defineClassUnder("UnpackError", standardErrorClass, standardErrorClass.getAllocator()); RubyClass underflowErrorClass = msgpackModule.defineClassUnder("UnderflowError", unpackErrorClass, unpackErrorClass.getAllocator()); + RubyClass malformedFormatErrorClass = msgpackModule.defineClassUnder("MalformedFormatError", unpackErrorClass, unpackErrorClass.getAllocator()); + RubyClass stackErrorClass = msgpackModule.defineClassUnder("StackError", unpackErrorClass, unpackErrorClass.getAllocator()); RubyModule typeErrorModule = msgpackModule.defineModuleUnder("TypeError"); - RubyClass unexpectedTypeErrorClass = msgpackModule.defineClassUnder("UnexpetedTypeError", unpackErrorClass, standardErrorClass.getAllocator()); + RubyClass unexpectedTypeErrorClass = msgpackModule.defineClassUnder("UnexpectedTypeError", unpackErrorClass, unpackErrorClass.getAllocator()); unexpectedTypeErrorClass.includeModule(typeErrorModule); + RubyClass unknownExtTypeErrorClass = msgpackModule.defineClassUnder("UnknownExtTypeError", unpackErrorClass, unpackErrorClass.getAllocator()); RubyClass extensionValueClass = msgpackModule.defineClassUnder("ExtensionValue", runtime.getObject(), new ExtensionValue.ExtensionValueAllocator()); extensionValueClass.defineAnnotatedMethods(ExtensionValue.class); RubyClass packerClass = msgpackModule.defineClassUnder("Packer", runtime.getObject(), new Packer.PackerAllocator()); @@ -37,6 +42,11 @@ public void load(Ruby runtime, boolean wrap) { unpackerClass.defineAnnotatedMethods(Unpacker.class); RubyClass bufferClass = msgpackModule.defineClassUnder("Buffer", runtime.getObject(), new Buffer.BufferAllocator()); bufferClass.defineAnnotatedMethods(Buffer.class); + RubyClass factoryClass = msgpackModule.defineClassUnder("Factory", runtime.getObject(), new Factory.FactoryAllocator()); + factoryClass.defineAnnotatedMethods(Factory.class); + defaultFactory = new Factory(runtime, factoryClass); + defaultFactory.initialize(runtime.getCurrentContext()); + msgpackModule.defineConstant("DefaultFactory", defaultFactory); installCoreExtensions(runtime); } @@ -100,20 +110,27 @@ public static IRubyObject pack(ThreadContext ctx, IRubyObject recv, IRubyObject[ extraArgs = new IRubyObject[args.length - 1]; System.arraycopy(args, 1, extraArgs, 0, args.length - 1); } - Packer packer = new Packer(ctx.getRuntime(), ctx.getRuntime().getModule("MessagePack").getClass("Packer")); - packer.initialize(ctx, extraArgs); + Packer packer = MessagePackLibrary.defaultFactory.packer(ctx, extraArgs); packer.write(ctx, args[0]); return packer.toS(ctx); } @JRubyMethod(module = true, required = 1, optional = 1, alias = {"load"}) public static IRubyObject unpack(ThreadContext ctx, IRubyObject recv, IRubyObject[] args) { - Decoder decoder = new Decoder(ctx.getRuntime(), args[0].asString().getBytes()); + ExtensionRegistry registry = MessagePackLibrary.defaultFactory.extensionRegistry(); + + boolean symbolizeKeys = false; + boolean allowUnknownExt = false; if (args.length > 1 && !args[args.length - 1].isNil()) { RubyHash hash = args[args.length - 1].convertToHash(); - IRubyObject symbolizeKeys = hash.fastARef(ctx.getRuntime().newSymbol("symbolize_keys")); - decoder.symbolizeKeys(symbolizeKeys != null && symbolizeKeys.isTrue()); + IRubyObject symbolizeKeysVal = hash.fastARef(ctx.getRuntime().newSymbol("symbolize_keys")); + symbolizeKeys = symbolizeKeysVal != null && symbolizeKeysVal.isTrue(); + IRubyObject allowUnknownExtVal = hash.fastARef(ctx.getRuntime().newSymbol("allow_unknown_ext")); + allowUnknownExt = (allowUnknownExtVal != null && allowUnknownExtVal.isTrue()); } + byte[] bytes = args[0].asString().getBytes(); + Decoder decoder = new Decoder(ctx.getRuntime(), registry, bytes, 0, bytes.length, symbolizeKeys, allowUnknownExt); + return decoder.next(); } } diff --git a/ext/java/org/msgpack/jruby/Packer.java b/ext/java/org/msgpack/jruby/Packer.java index fd326477..aa10ed14 100644 --- a/ext/java/org/msgpack/jruby/Packer.java +++ b/ext/java/org/msgpack/jruby/Packer.java @@ -4,9 +4,12 @@ import org.jruby.Ruby; import org.jruby.RubyClass; import org.jruby.RubyObject; +import org.jruby.RubyArray; import org.jruby.RubyHash; import org.jruby.RubyIO; import org.jruby.RubyInteger; +import org.jruby.RubyFixnum; +import org.jruby.runtime.Block; import org.jruby.runtime.builtin.IRubyObject; import org.jruby.anno.JRubyClass; import org.jruby.anno.JRubyMethod; @@ -14,19 +17,22 @@ import org.jruby.runtime.ObjectAllocator; import org.jruby.util.ByteList; +import static org.jruby.runtime.Visibility.PRIVATE; @JRubyClass(name="MessagePack::Packer") public class Packer extends RubyObject { + public ExtensionRegistry registry; private Buffer buffer; private Encoder encoder; - public Packer(Ruby runtime, RubyClass type) { + public Packer(Ruby runtime, RubyClass type, ExtensionRegistry registry) { super(runtime, type); + this.registry = registry; } static class PackerAllocator implements ObjectAllocator { public IRubyObject allocate(Ruby runtime, RubyClass type) { - return new Packer(runtime, type); + return new Packer(runtime, type, null); } } @@ -35,15 +41,68 @@ public IRubyObject initialize(ThreadContext ctx, IRubyObject[] args) { boolean compatibilityMode = false; if (args.length > 0 && args[args.length - 1] instanceof RubyHash) { RubyHash options = (RubyHash) args[args.length - 1]; - compatibilityMode = options.fastARef(ctx.getRuntime().newSymbol("compatibility_mode")).isTrue(); + IRubyObject mode = options.fastARef(ctx.getRuntime().newSymbol("compatibility_mode")); + compatibilityMode = (mode != null) && mode.isTrue(); } - this.encoder = new Encoder(ctx.getRuntime(), compatibilityMode); + this.encoder = new Encoder(ctx.getRuntime(), compatibilityMode, registry); this.buffer = new Buffer(ctx.getRuntime(), ctx.getRuntime().getModule("MessagePack").getClass("Buffer")); this.buffer.initialize(ctx, args); + this.registry = new ExtensionRegistry(); return this; } - @JRubyMethod(name = "write") + public static Packer newPacker(ThreadContext ctx, ExtensionRegistry extRegistry, IRubyObject[] args) { + Packer packer = new Packer(ctx.getRuntime(), ctx.getRuntime().getModule("MessagePack").getClass("Packer"), extRegistry); + packer.initialize(ctx, args); + return packer; + } + + @JRubyMethod(name = "compatibility_mode?") + public IRubyObject isCompatibilityMode(ThreadContext ctx) { + return encoder.isCompatibilityMode() ? ctx.getRuntime().getTrue() : ctx.getRuntime().getFalse(); + } + + @JRubyMethod(name = "registered_types_internal", visibility = PRIVATE) + public IRubyObject registeredTypesInternal(ThreadContext ctx) { + return registry.toInternalPackerRegistry(ctx); + } + + @JRubyMethod(name = "register_type", required = 2, optional = 1) + public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args, final Block block) { + Ruby runtime = ctx.getRuntime(); + IRubyObject type = args[0]; + IRubyObject klass = args[1]; + + IRubyObject arg; + IRubyObject proc; + if (args.length == 2) { + if (! block.isGiven()) { + throw runtime.newLocalJumpErrorNoBlock(); + } + proc = block.getProcObject(); + arg = proc; + } else if (args.length == 3) { + arg = args[2]; + proc = arg.callMethod(ctx, "to_proc"); + } else { + throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 2..3)", 2 + args.length)); + } + + long typeId = ((RubyFixnum) type).getLongValue(); + if (typeId < -128 || typeId > 127) { + throw runtime.newRangeError(String.format("integer %d too big to convert to `signed char'", typeId)); + } + + if (!(klass instanceof RubyClass)) { + throw runtime.newArgumentError(String.format("expected Class but found %s.", klass.getType().getName())); + } + RubyClass extClass = (RubyClass) klass; + + registry.put(extClass, (int) typeId, proc, arg, null, null); + return runtime.getNil(); + } + + @JRubyMethod(name = "write", alias = { "pack" }) public IRubyObject write(ThreadContext ctx, IRubyObject obj) { buffer.write(ctx, encoder.encode(obj, this)); return this; @@ -69,7 +128,7 @@ public IRubyObject writeMapHeader(ThreadContext ctx, IRubyObject size) { return this; } - @JRubyMethod(name = "to_s") + @JRubyMethod(name = "to_s", alias = { "to_str" }) public IRubyObject toS(ThreadContext ctx) { return buffer.toS(ctx); } diff --git a/ext/java/org/msgpack/jruby/Unpacker.java b/ext/java/org/msgpack/jruby/Unpacker.java index 4ecb6ec6..f2a622e7 100644 --- a/ext/java/org/msgpack/jruby/Unpacker.java +++ b/ext/java/org/msgpack/jruby/Unpacker.java @@ -1,12 +1,16 @@ package org.msgpack.jruby; +import java.util.Arrays; import org.jruby.Ruby; import org.jruby.RubyClass; import org.jruby.RubyString; import org.jruby.RubyObject; +import org.jruby.RubyArray; import org.jruby.RubyHash; import org.jruby.RubyNumeric; +import org.jruby.RubyFixnum; +import org.jruby.RubyProc; import org.jruby.RubyIO; import org.jruby.exceptions.RaiseException; import org.jruby.runtime.builtin.IRubyObject; @@ -20,17 +24,24 @@ import static org.jruby.runtime.Visibility.PRIVATE; - @JRubyClass(name="MessagePack::Unpacker") public class Unpacker extends RubyObject { + private final ExtensionRegistry registry; + private IRubyObject stream; private IRubyObject data; private Decoder decoder; private final RubyClass underflowErrorClass; private boolean symbolizeKeys; + private boolean allowUnknownExt; public Unpacker(Ruby runtime, RubyClass type) { + this(runtime, type, new ExtensionRegistry()); + } + + public Unpacker(Ruby runtime, RubyClass type, ExtensionRegistry registry) { super(runtime, type); + this.registry = registry; this.underflowErrorClass = runtime.getModule("MessagePack").getClass("UnderflowError"); } @@ -43,6 +54,7 @@ public IRubyObject allocate(Ruby runtime, RubyClass klass) { @JRubyMethod(name = "initialize", optional = 1, visibility = PRIVATE) public IRubyObject initialize(ThreadContext ctx, IRubyObject[] args) { symbolizeKeys = false; + allowUnknownExt = false; if (args.length > 0) { if (args[args.length - 1] instanceof RubyHash) { RubyHash options = (RubyHash) args[args.length - 1]; @@ -50,6 +62,10 @@ public IRubyObject initialize(ThreadContext ctx, IRubyObject[] args) { if (sk != null) { symbolizeKeys = sk.isTrue(); } + IRubyObject au = options.fastARef(ctx.getRuntime().newSymbol("allow_unknown_ext")); + if (au != null) { + allowUnknownExt = au.isTrue(); + } } else if (!(args[0] instanceof RubyHash)) { setStream(ctx, args[0]); } @@ -57,6 +73,61 @@ public IRubyObject initialize(ThreadContext ctx, IRubyObject[] args) { return this; } + public static Unpacker newUnpacker(ThreadContext ctx, ExtensionRegistry extRegistry, IRubyObject[] args) { + Unpacker unpacker = new Unpacker(ctx.getRuntime(), ctx.getRuntime().getModule("MessagePack").getClass("Unpacker"), extRegistry); + unpacker.initialize(ctx, args); + return unpacker; + } + + @JRubyMethod(name = "symbolize_keys?") + public IRubyObject isSymbolizeKeys(ThreadContext ctx) { + return symbolizeKeys ? ctx.getRuntime().getTrue() : ctx.getRuntime().getFalse(); + } + + @JRubyMethod(name = "allow_unknown_ext?") + public IRubyObject isAllowUnknownExt(ThreadContext ctx) { + return allowUnknownExt ? ctx.getRuntime().getTrue() : ctx.getRuntime().getFalse(); + } + + @JRubyMethod(name = "registered_types_internal", visibility = PRIVATE) + public IRubyObject registeredTypesInternal(ThreadContext ctx) { + return registry.toInternalUnpackerRegistry(ctx); + } + + @JRubyMethod(name = "register_type", required = 1, optional = 2) + public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args, final Block block) { + Ruby runtime = ctx.getRuntime(); + IRubyObject type = args[0]; + + RubyClass extClass; + IRubyObject arg; + IRubyObject proc; + if (args.length == 1) { + if (! block.isGiven()) { + throw runtime.newLocalJumpErrorNoBlock(); + } + proc = RubyProc.newProc(runtime, block, block.type); + if (proc == null) + System.err.println("proc from Block is null"); + arg = proc; + extClass = null; + } else if (args.length == 3) { + extClass = (RubyClass) args[1]; + arg = args[2]; + proc = extClass.method(arg); + } else { + throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 1 or 3)", 2 + args.length)); + } + + long typeId = ((RubyFixnum) type).getLongValue(); + if (typeId < -128 || typeId > 127) { + throw runtime.newRangeError(String.format("integer %d too big to convert to `signed char'", typeId)); + } + + registry.put(extClass, (int) typeId, null, null, proc, arg); + return runtime.getNil(); + } + @JRubyMethod(required = 2) public IRubyObject execute(ThreadContext ctx, IRubyObject data, IRubyObject offset) { return executeLimit(ctx, data, offset, null); @@ -71,11 +142,10 @@ public IRubyObject executeLimit(ThreadContext ctx, IRubyObject str, IRubyObject if (limit == -1) { limit = byteList.length() - offset; } - Decoder decoder = new Decoder(ctx.getRuntime(), byteList.unsafeBytes(), byteList.begin() + offset, limit); - decoder.symbolizeKeys(symbolizeKeys); + Decoder decoder = new Decoder(ctx.getRuntime(), registry, byteList.unsafeBytes(), byteList.begin() + offset, limit, symbolizeKeys, allowUnknownExt); try { - this.data = null; - this.data = decoder.next(); + data = null; + data = decoder.next(); } catch (RaiseException re) { if (re.getException().getType() != underflowErrorClass) { throw re; @@ -102,8 +172,7 @@ public IRubyObject finished_p(ThreadContext ctx) { public IRubyObject feed(ThreadContext ctx, IRubyObject data) { ByteList byteList = data.asString().getByteList(); if (decoder == null) { - decoder = new Decoder(ctx.getRuntime(), byteList.unsafeBytes(), byteList.begin(), byteList.length()); - decoder.symbolizeKeys(symbolizeKeys); + decoder = new Decoder(ctx.getRuntime(), registry, byteList.unsafeBytes(), byteList.begin(), byteList.length(), symbolizeKeys, allowUnknownExt); } else { decoder.feed(byteList.unsafeBytes(), byteList.begin(), byteList.length()); } @@ -113,8 +182,12 @@ public IRubyObject feed(ThreadContext ctx, IRubyObject data) { @JRubyMethod(name = "feed_each", required = 1) public IRubyObject feedEach(ThreadContext ctx, IRubyObject data, Block block) { feed(ctx, data); - each(ctx, block); - return ctx.getRuntime().getNil(); + if (block.isGiven()) { + each(ctx, block); + return ctx.getRuntime().getNil(); + } else { + return callMethod(ctx, "to_enum"); + } } @JRubyMethod @@ -150,20 +223,30 @@ public IRubyObject reset(ThreadContext ctx) { return ctx.getRuntime().getNil(); } - @JRubyMethod + @JRubyMethod(name = "read", alias = { "unpack" }) public IRubyObject read(ThreadContext ctx) { - if (decoder != null) { - try { - return decoder.next(); - } catch (RaiseException re) { - if (re.getException().getType() != underflowErrorClass) { - throw re; - } else { - throw ctx.getRuntime().newEOFError(); - } + if (decoder == null) { + throw ctx.getRuntime().newEOFError(); + } + try { + return decoder.next(); + } catch (RaiseException re) { + if (re.getException().getType() != underflowErrorClass) { + throw re; + } else { + throw ctx.getRuntime().newEOFError(); } } - return ctx.getRuntime().getNil(); + } + + @JRubyMethod(name = "skip") + public IRubyObject skip(ThreadContext ctx) { + throw ctx.getRuntime().newNotImplementedError("Not supported yet in JRuby implementation"); + } + + @JRubyMethod(name = "skip_nil") + public IRubyObject skipNil(ThreadContext ctx) { + throw ctx.getRuntime().newNotImplementedError("Not supported yet in JRuby implementation"); } @JRubyMethod @@ -220,8 +303,7 @@ public IRubyObject setStream(ThreadContext ctx, IRubyObject stream) { ByteList byteList = str.getByteList(); this.stream = stream; this.decoder = null; - this.decoder = new Decoder(ctx.getRuntime(), byteList.unsafeBytes(), byteList.begin(), byteList.length()); - decoder.symbolizeKeys(symbolizeKeys); + this.decoder = new Decoder(ctx.getRuntime(), registry, byteList.unsafeBytes(), byteList.begin(), byteList.length(), symbolizeKeys, allowUnknownExt); return getStream(ctx); } } diff --git a/ext/msgpack/packer_class.c b/ext/msgpack/packer_class.c index 79552400..36bdf674 100644 --- a/ext/msgpack/packer_class.c +++ b/ext/msgpack/packer_class.c @@ -109,6 +109,12 @@ VALUE MessagePack_Packer_initialize(int argc, VALUE* argv, VALUE self) return self; } +static VALUE Packer_compatibility_mode_p(VALUE self) +{ + PACKER(self, pk); + return pk->compatibility_mode ? Qtrue : Qfalse; +} + static VALUE Packer_buffer(VALUE self) { PACKER(self, pk); @@ -330,6 +336,7 @@ void MessagePack_Packer_module_init(VALUE mMessagePack) rb_define_alloc_func(cMessagePack_Packer, MessagePack_Packer_alloc); rb_define_method(cMessagePack_Packer, "initialize", MessagePack_Packer_initialize, -1); + rb_define_method(cMessagePack_Packer, "compatibility_mode?", Packer_compatibility_mode_p, 0); rb_define_method(cMessagePack_Packer, "buffer", Packer_buffer, 0); rb_define_method(cMessagePack_Packer, "write", Packer_write, 1); rb_define_alias(cMessagePack_Packer, "pack", "write"); diff --git a/ext/msgpack/packer_ext_registry.c b/ext/msgpack/packer_ext_registry.c index de5b46c0..8601cc6e 100644 --- a/ext/msgpack/packer_ext_registry.c +++ b/ext/msgpack/packer_ext_registry.c @@ -77,11 +77,3 @@ VALUE msgpack_packer_ext_registry_put(msgpack_packer_ext_registry_t* pkrg, #endif return rb_hash_aset(pkrg->hash, ext_class, e); } - -VALUE msgpack_packer_ext_registry_call(msgpack_packer_ext_registry_t* pkrg, - VALUE proc, VALUE ext_value) -{ - VALUE string = rb_funcall(proc, s_call, 1, ext_value); - StringValue(string); - return string; -} diff --git a/ext/msgpack/packer_ext_registry.h b/ext/msgpack/packer_ext_registry.h index 5fb216b0..570dde57 100644 --- a/ext/msgpack/packer_ext_registry.h +++ b/ext/msgpack/packer_ext_registry.h @@ -95,7 +95,4 @@ static inline VALUE msgpack_packer_ext_registry_lookup(msgpack_packer_ext_regist return Qnil; } -VALUE msgpack_packer_ext_registry_call(msgpack_packer_ext_registry_t* pkrg, - VALUE proc, VALUE ext_value); - #endif diff --git a/ext/msgpack/unpacker_class.c b/ext/msgpack/unpacker_class.c index 7128821f..d0458793 100644 --- a/ext/msgpack/unpacker_class.c +++ b/ext/msgpack/unpacker_class.c @@ -116,6 +116,18 @@ VALUE MessagePack_Unpacker_initialize(int argc, VALUE* argv, VALUE self) return self; } +static VALUE Unpacker_symbolized_keys_p(VALUE self) +{ + UNPACKER(self, uk); + return uk->symbolize_keys ? Qtrue : Qfalse; +} + +static VALUE Unpacker_allow_unknown_ext_p(VALUE self) +{ + UNPACKER(self, uk); + return uk->allow_unknown_ext ? Qtrue : Qfalse; +} + static void raise_unpacker_error(int r) { switch(r) { @@ -455,6 +467,8 @@ void MessagePack_Unpacker_module_init(VALUE mMessagePack) rb_define_alloc_func(cMessagePack_Unpacker, MessagePack_Unpacker_alloc); rb_define_method(cMessagePack_Unpacker, "initialize", MessagePack_Unpacker_initialize, -1); + rb_define_method(cMessagePack_Unpacker, "symbolize_keys?", Unpacker_symbolized_keys_p, 0); + rb_define_method(cMessagePack_Unpacker, "allow_unknown_ext?", Unpacker_allow_unknown_ext_p, 0); rb_define_method(cMessagePack_Unpacker, "buffer", Unpacker_buffer, 0); rb_define_method(cMessagePack_Unpacker, "read", Unpacker_read, 0); rb_define_alias(cMessagePack_Unpacker, "unpack", "read"); diff --git a/ext/msgpack/unpacker_ext_registry.c b/ext/msgpack/unpacker_ext_registry.c index 9a8b78b3..94c6694b 100644 --- a/ext/msgpack/unpacker_ext_registry.c +++ b/ext/msgpack/unpacker_ext_registry.c @@ -60,9 +60,3 @@ VALUE msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t* ukrg, ukrg->array[ext_type + 128] = e; return before; } - -VALUE msgpack_unpacker_ext_registry_call(msgpack_unpacker_ext_registry_t* ukrg, - VALUE proc, VALUE ext_data) -{ - return rb_funcall(proc, s_call, 1, ext_data); -} diff --git a/ext/msgpack/unpacker_ext_registry.h b/ext/msgpack/unpacker_ext_registry.h index 683213dd..fddef9d2 100644 --- a/ext/msgpack/unpacker_ext_registry.h +++ b/ext/msgpack/unpacker_ext_registry.h @@ -56,7 +56,4 @@ static inline VALUE msgpack_unpacker_ext_registry_lookup(msgpack_unpacker_ext_re return rb_ary_entry(e, 1); } -VALUE msgpack_unpacker_ext_registry_call(msgpack_unpacker_ext_registry_t* ukrg, - VALUE proc, VALUE ext_data); - #endif diff --git a/msgpack.gemspec b/msgpack.gemspec index 826dae8d..4ea7d289 100644 --- a/msgpack.gemspec +++ b/msgpack.gemspec @@ -25,7 +25,9 @@ Gem::Specification.new do |s| s.add_development_dependency 'bundler', ['~> 1.0'] s.add_development_dependency 'rake', ['~> 0.9.2'] s.add_development_dependency 'rake-compiler', ['~> 0.9.4'] - s.add_development_dependency 'rake-compiler-dock', ['~> 0.4.3'] + if /java/ !~ RUBY_PLATFORM + s.add_development_dependency 'rake-compiler-dock', ['~> 0.4.3'] + end s.add_development_dependency 'rspec', ['~> 3.3'] s.add_development_dependency 'yard', ['~> 0.8.2'] s.add_development_dependency 'json' diff --git a/spec/cruby/packer_spec.rb b/spec/cruby/packer_spec.rb deleted file mode 100644 index 59fde8b1..00000000 --- a/spec/cruby/packer_spec.rb +++ /dev/null @@ -1,138 +0,0 @@ -# encoding: ascii-8bit -require 'spec_helper' - -require 'stringio' -if defined?(Encoding) - Encoding.default_external = 'ASCII-8BIT' -end - -describe Packer do - let :packer do - Packer.new - end - - it 'initialize' do - Packer.new - Packer.new(nil) - Packer.new(StringIO.new) - Packer.new({}) - Packer.new(StringIO.new, {}) - end - - #it 'Packer' do - # Packer(packer).object_id.should == packer.object_id - # Packer(nil).class.should == Packer - # Packer('').class.should == Packer - # Packer('initbuf').to_s.should == 'initbuf' - #end - - it 'write' do - packer.write([]) - packer.to_s.should == "\x90" - end - - it 'write_nil' do - packer.write_nil - packer.to_s.should == "\xc0" - end - - it 'write_array_header 0' do - packer.write_array_header(0) - packer.to_s.should == "\x90" - end - - it 'write_array_header 1' do - packer.write_array_header(1) - packer.to_s.should == "\x91" - end - - it 'write_map_header 0' do - packer.write_map_header(0) - packer.to_s.should == "\x80" - end - - it 'write_map_header 1' do - packer.write_map_header(1) - packer.to_s.should == "\x81" - end - - it 'flush' do - io = StringIO.new - pk = Packer.new(io) - pk.write_nil - pk.flush - pk.to_s.should == '' - io.string.should == "\xc0" - end - - it 'to_msgpack returns String' do - nil.to_msgpack.class.should == String - true.to_msgpack.class.should == String - false.to_msgpack.class.should == String - 1.to_msgpack.class.should == String - 1.0.to_msgpack.class.should == String - "".to_msgpack.class.should == String - Hash.new.to_msgpack.class.should == String - Array.new.to_msgpack.class.should == String - end - - class CustomPack01 - def to_msgpack(pk=nil) - return MessagePack.pack(self, pk) unless pk.class == MessagePack::Packer - pk.write_array_header(2) - pk.write(1) - pk.write(2) - return pk - end - end - - class CustomPack02 - def to_msgpack(pk=nil) - [1,2].to_msgpack(pk) - end - end - - it 'calls custom to_msgpack method' do - MessagePack.pack(CustomPack01.new).should == [1,2].to_msgpack - MessagePack.pack(CustomPack02.new).should == [1,2].to_msgpack - CustomPack01.new.to_msgpack.should == [1,2].to_msgpack - CustomPack02.new.to_msgpack.should == [1,2].to_msgpack - end - - it 'calls custom to_msgpack method with io' do - s01 = StringIO.new - MessagePack.pack(CustomPack01.new, s01) - s01.string.should == [1,2].to_msgpack - - s02 = StringIO.new - MessagePack.pack(CustomPack02.new, s02) - s02.string.should == [1,2].to_msgpack - - s03 = StringIO.new - CustomPack01.new.to_msgpack(s03) - s03.string.should == [1,2].to_msgpack - - s04 = StringIO.new - CustomPack02.new.to_msgpack(s04) - s04.string.should == [1,2].to_msgpack - end - - context 'in compatibility mode' do - it 'does not use the bin types' do - packed = MessagePack.pack('hello'.force_encoding(Encoding::BINARY), compatibility_mode: true) - packed.should eq("\xA5hello") - packed = MessagePack.pack(('hello' * 100).force_encoding(Encoding::BINARY), compatibility_mode: true) - packed.should start_with("\xDA\x01\xF4") - - packer = MessagePack::Packer.new(compatibility_mode: 1) - packed = packer.pack(('hello' * 100).force_encoding(Encoding::BINARY)) - packed.to_str.should start_with("\xDA\x01\xF4") - end - - it 'does not use the str8 type' do - packed = MessagePack.pack('x' * 32, compatibility_mode: true) - packed.should start_with("\xDA\x00\x20") - end - end -end - diff --git a/spec/cruby/unpacker_spec.rb b/spec/cruby/unpacker_spec.rb index 62ecfddb..2f55c4d1 100644 --- a/spec/cruby/unpacker_spec.rb +++ b/spec/cruby/unpacker_spec.rb @@ -10,60 +10,6 @@ Packer.new end - # TODO initialize - - it 'read_array_header succeeds' do - unpacker.feed("\x91") - unpacker.read_array_header.should == 1 - end - - it 'read_array_header fails' do - unpacker.feed("\x81") - lambda { - unpacker.read_array_header - }.should raise_error(MessagePack::TypeError) # TypeError is included in UnexpectedTypeError - lambda { - unpacker.read_array_header - }.should raise_error(MessagePack::UnexpectedTypeError) - end - - it 'read_map_header converts an map to key-value sequence' do - packer.write_array_header(2) - packer.write("e") - packer.write(1) - unpacker = Unpacker.new - unpacker.feed(packer.to_s) - unpacker.read_array_header.should == 2 - unpacker.read.should == "e" - unpacker.read.should == 1 - end - - it 'read_map_header succeeds' do - unpacker.feed("\x81") - unpacker.read_map_header.should == 1 - end - - it 'read_map_header converts an map to key-value sequence' do - packer.write_map_header(1) - packer.write("k") - packer.write("v") - unpacker = Unpacker.new - unpacker.feed(packer.to_s) - unpacker.read_map_header.should == 1 - unpacker.read.should == "k" - unpacker.read.should == "v" - end - - it 'read_map_header fails' do - unpacker.feed("\x91") - lambda { - unpacker.read_map_header - }.should raise_error(MessagePack::TypeError) # TypeError is included in UnexpectedTypeError - lambda { - unpacker.read_map_header - }.should raise_error(MessagePack::UnexpectedTypeError) - end - it 'skip_nil succeeds' do unpacker.feed("\xc0") unpacker.skip_nil.should == true @@ -91,12 +37,6 @@ unpacker.read.should == 5 end - it 'read raises EOFError' do - lambda { - unpacker.read - }.should raise_error(EOFError) - end - it 'skip raises EOFError' do lambda { unpacker.skip @@ -109,88 +49,6 @@ }.should raise_error(EOFError) end - let :sample_object do - [1024, {["a","b"]=>["c","d"]}, ["e","f"], "d", 70000, 4.12, 1.5, 1.5, 1.5] - end - - it 'feed and each continue internal state' do - raw = sample_object.to_msgpack.to_s * 4 - objects = [] - - raw.split(//).each do |b| - unpacker.feed(b) - unpacker.each {|c| - objects << c - } - end - - objects.should == [sample_object] * 4 - end - - it 'feed_each continues internal state' do - raw = sample_object.to_msgpack.to_s * 4 - objects = [] - - raw.split(//).each do |b| - unpacker.feed_each(b) {|c| - objects << c - } - end - - objects.should == [sample_object] * 4 - end - - it 'feed_each enumerator' do - raw = sample_object.to_msgpack.to_s * 4 - - unpacker.feed_each(raw).to_a.should == [sample_object] * 4 - end - - it 'reset clears internal buffer' do - # 1-element array - unpacker.feed("\x91") - unpacker.reset - unpacker.feed("\x01") - - unpacker.each.map {|x| x }.should == [1] - end - - it 'reset clears internal state' do - # 1-element array - unpacker.feed("\x91") - unpacker.each.map {|x| x }.should == [] - - unpacker.reset - - unpacker.feed("\x01") - unpacker.each.map {|x| x }.should == [1] - end - - it 'frozen short strings' do - raw = sample_object.to_msgpack.to_s.force_encoding('UTF-8') - lambda { - unpacker.feed_each(raw.freeze) { } - }.should_not raise_error - end - - it 'frozen long strings' do - raw = (sample_object.to_msgpack.to_s * 10240).force_encoding('UTF-8') - lambda { - unpacker.feed_each(raw.freeze) { } - }.should_not raise_error - end - - it 'read raises level stack too deep error' do - 512.times { packer.write_array_header(1) } - packer.write(nil) - - unpacker = Unpacker.new - unpacker.feed(packer.to_s) - lambda { - unpacker.read - }.should raise_error(MessagePack::StackError) - end - it 'skip raises level stack too deep error' do 512.times { packer.write_array_header(1) } packer.write(nil) @@ -202,116 +60,11 @@ }.should raise_error(MessagePack::StackError) end - it 'read raises invalid byte error' do - unpacker.feed("\xc1") - lambda { - unpacker.read - }.should raise_error(MessagePack::MalformedFormatError) - end - it 'skip raises invalid byte error' do unpacker.feed("\xc1") lambda { unpacker.skip }.should raise_error(MessagePack::MalformedFormatError) end - - it "gc mark" do - raw = sample_object.to_msgpack.to_s * 4 - - n = 0 - raw.split(//).each do |b| - GC.start - unpacker.feed_each(b) {|o| - GC.start - o.should == sample_object - n += 1 - } - GC.start - end - - n.should == 4 - end - - it "buffer" do - orig = "a"*32*1024*4 - raw = orig.to_msgpack.to_s - - n = 655 - times = raw.size / n - times += 1 unless raw.size % n == 0 - - off = 0 - parsed = false - - times.times do - parsed.should == false - - seg = raw[off, n] - off += seg.length - - unpacker.feed_each(seg) {|obj| - parsed.should == false - obj.should == orig - parsed = true - } - end - - parsed.should == true - end - - it 'MessagePack.unpack symbolize_keys' do - symbolized_hash = {:a => 'b', :c => 'd'} - MessagePack.load(MessagePack.pack(symbolized_hash), :symbolize_keys => true).should == symbolized_hash - MessagePack.unpack(MessagePack.pack(symbolized_hash), :symbolize_keys => true).should == symbolized_hash - end - - it 'Unpacker#unpack symbolize_keys' do - unpacker = Unpacker.new(:symbolize_keys => true) - symbolized_hash = {:a => 'b', :c => 'd'} - unpacker.feed(MessagePack.pack(symbolized_hash)).read.should == symbolized_hash - end - - it "msgpack str 8 type" do - MessagePack.unpack([0xd9, 0x00].pack('C*')).should == "" - MessagePack.unpack([0xd9, 0x00].pack('C*')).encoding.should == Encoding::UTF_8 - MessagePack.unpack([0xd9, 0x01].pack('C*') + 'a').should == "a" - MessagePack.unpack([0xd9, 0x02].pack('C*') + 'aa').should == "aa" - end - - it "msgpack str 16 type" do - MessagePack.unpack([0xda, 0x00, 0x00].pack('C*')).should == "" - MessagePack.unpack([0xda, 0x00, 0x00].pack('C*')).encoding.should == Encoding::UTF_8 - MessagePack.unpack([0xda, 0x00, 0x01].pack('C*') + 'a').should == "a" - MessagePack.unpack([0xda, 0x00, 0x02].pack('C*') + 'aa').should == "aa" - end - - it "msgpack str 32 type" do - MessagePack.unpack([0xdb, 0x00, 0x00, 0x00, 0x00].pack('C*')).should == "" - MessagePack.unpack([0xdb, 0x00, 0x00, 0x00, 0x00].pack('C*')).encoding.should == Encoding::UTF_8 - MessagePack.unpack([0xdb, 0x00, 0x00, 0x00, 0x01].pack('C*') + 'a').should == "a" - MessagePack.unpack([0xdb, 0x00, 0x00, 0x00, 0x02].pack('C*') + 'aa').should == "aa" - end - - it "msgpack bin 8 type" do - MessagePack.unpack([0xc4, 0x00].pack('C*')).should == "" - MessagePack.unpack([0xc4, 0x00].pack('C*')).encoding.should == Encoding::ASCII_8BIT - MessagePack.unpack([0xc4, 0x01].pack('C*') + 'a').should == "a" - MessagePack.unpack([0xc4, 0x02].pack('C*') + 'aa').should == "aa" - end - - it "msgpack bin 16 type" do - MessagePack.unpack([0xc5, 0x00, 0x00].pack('C*')).should == "" - MessagePack.unpack([0xc5, 0x00, 0x00].pack('C*')).encoding.should == Encoding::ASCII_8BIT - MessagePack.unpack([0xc5, 0x00, 0x01].pack('C*') + 'a').should == "a" - MessagePack.unpack([0xc5, 0x00, 0x02].pack('C*') + 'aa').should == "aa" - end - - it "msgpack bin 32 type" do - MessagePack.unpack([0xc6, 0x00, 0x00, 0x00, 0x00].pack('C*')).should == "" - MessagePack.unpack([0xc6, 0x0, 0x00, 0x00, 0x000].pack('C*')).encoding.should == Encoding::ASCII_8BIT - MessagePack.unpack([0xc6, 0x00, 0x00, 0x00, 0x01].pack('C*') + 'a').should == "a" - MessagePack.unpack([0xc6, 0x00, 0x00, 0x00, 0x02].pack('C*') + 'aa').should == "aa" - end end diff --git a/spec/factory_spec.rb b/spec/factory_spec.rb index fca8b486..9bd058ce 100644 --- a/spec/factory_spec.rb +++ b/spec/factory_spec.rb @@ -1,8 +1,6 @@ # encoding: ascii-8bit require 'spec_helper' -eval("return") if java? - describe MessagePack::Factory do subject do described_class.new @@ -220,7 +218,6 @@ class MyType2 < MyType require_relative 'exttypes' it 'should be referred by MessagePack.pack and MessagePack.unpack' do - skip("not supported yet in JRuby implementation") if java? MessagePack::DefaultFactory.register_type(DummyTimeStamp1::TYPE, DummyTimeStamp1) MessagePack::DefaultFactory.register_type(DummyTimeStamp2::TYPE, DummyTimeStamp2, packer: :serialize, unpacker: :deserialize) diff --git a/spec/jruby/msgpack/unpacker_spec.rb b/spec/jruby/unpacker_spec.rb similarity index 51% rename from spec/jruby/msgpack/unpacker_spec.rb rename to spec/jruby/unpacker_spec.rb index bc620783..445f2417 100644 --- a/spec/jruby/msgpack/unpacker_spec.rb +++ b/spec/jruby/unpacker_spec.rb @@ -2,43 +2,35 @@ require 'stringio' require 'tempfile' -require 'spec_helper' +require 'spec_helper' -describe ::MessagePack::Unpacker do - def flatten(struct, results = []) - case struct - when Array - struct.each { |v| flatten(v, results) } - when Hash - struct.each { |k, v| flatten(v, flatten(k, results)) } - else - results << struct - end - results +describe MessagePack::Unpacker do + let :unpacker do + MessagePack::Unpacker.new end - subject do - described_class.new + let :packer do + MessagePack::Packer.new end - + let :buffer1 do MessagePack.pack(:foo => 'bar') end - + let :buffer2 do MessagePack.pack(:hello => {:world => [1, 2, 3]}) end - + let :buffer3 do MessagePack.pack(:x => 'y') end - + describe '#execute/#execute_limit/#finished?' do let :buffer do buffer1 + buffer2 + buffer3 end - + it 'extracts an object from the buffer' do subject.execute(buffer, 0) subject.data.should == {'foo' => 'bar'} @@ -53,7 +45,7 @@ def flatten(struct, results = []) subject.execute_limit(buffer, buffer1.length, buffer2.length) subject.data.should == {'hello' => {'world' => [1, 2, 3]}} end - + it 'extracts nothing if the limit cuts an object in half' do subject.execute_limit(buffer, buffer1.length, 3) subject.data.should be_nil @@ -63,63 +55,17 @@ def flatten(struct, results = []) subject.execute(buffer, 0).should == buffer1.length subject.execute(buffer, buffer1.length).should == buffer1.length + buffer2.length end - + it 'is finished if #data returns an object' do subject.execute_limit(buffer, buffer1.length, buffer2.length) subject.should be_finished - + subject.execute_limit(buffer, buffer1.length, 3) subject.should_not be_finished end end - describe '#read' do - context 'with a buffer' do - it 'reads objects' do - objects = [] - subject.feed(buffer1) - subject.feed(buffer2) - subject.feed(buffer3) - objects << subject.read - objects << subject.read - objects << subject.read - objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}] - end - - it 'reads map header' do - subject.feed({}.to_msgpack) - subject.read_map_header.should == 0 - end - - it 'reads array header' do - subject.feed([].to_msgpack) - subject.read_array_header.should == 0 - end - end - end - describe '#each' do - context 'with a buffer' do - it 'yields each object in the buffer' do - objects = [] - subject.feed(buffer1) - subject.feed(buffer2) - subject.feed(buffer3) - subject.each do |obj| - objects << obj - end - objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}] - end - - it 'returns an enumerator when no block is given' do - subject.feed(buffer1) - subject.feed(buffer2) - subject.feed(buffer3) - enum = subject.each - enum.map { |obj| obj.keys.first }.should == %w[foo hello x] - end - end - context 'with a StringIO stream' do it 'yields each object in the stream' do objects = [] @@ -130,7 +76,7 @@ def flatten(struct, results = []) objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}] end end - + context 'with a File stream' do it 'yields each object in the stream' do objects = [] @@ -146,40 +92,8 @@ def flatten(struct, results = []) objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}] end end - - context 'with a stream passed to the constructor' do - it 'yields each object in the stream' do - objects = [] - unpacker = described_class.new(StringIO.new(buffer1 + buffer2 + buffer3)) - unpacker.each do |obj| - objects << obj - end - objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}] - end - end end - - describe '#feed_each' do - it 'feeds the buffer then runs #each' do - objects = [] - subject.feed_each(buffer1 + buffer2 + buffer3) do |obj| - objects << obj - end - objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}] - end - it 'handles chunked data' do - objects = [] - buffer = buffer1 + buffer2 + buffer3 - buffer.chars.each do |ch| - subject.feed_each(ch) do |obj| - objects << obj - end - end - objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}] - end - end - describe '#fill' do it 'is a no-op' do subject.stream = StringIO.new(buffer1 + buffer2 + buffer3) @@ -187,22 +101,17 @@ def flatten(struct, results = []) subject.each { |obj| } end end - - describe '#reset' do - context 'with a buffer' do - it 'is unclear what it is supposed to do' - end - - context 'with a stream' do - it 'is unclear what it is supposed to do' - end - end - context 'regressions' do - it 'handles massive arrays (issue #2)' do - array = ['foo'] * 10_000 - MessagePack.unpack(MessagePack.pack(array)).should have(10_000).items + def flatten(struct, results = []) + case struct + when Array + struct.each { |v| flatten(v, results) } + when Hash + struct.each { |k, v| flatten(v, flatten(k, results)) } + else + results << struct end + results end context 'encoding', :encodings do @@ -229,15 +138,15 @@ def flatten(struct, results = []) Encoding.default_external = Encoding::ISO_8859_1 end - it 'produces results with default internal encoding' do + it 'produces results with encoding as binary or string(utf8)' do unpacker.execute(buffer, 0) strings = flatten(unpacker.data).grep(String) - strings.map(&:encoding).uniq.should == [Encoding.default_internal] + strings.map(&:encoding).uniq.sort{|a,b| a.to_s <=> b.to_s}.should == [Encoding::ASCII_8BIT, Encoding::UTF_8] end it 'recodes to internal encoding' do unpacker.execute(buffer, 0) - unpacker.data['nested'][1].keys.should == ["sk\xC3\xA5l".force_encoding(Encoding.default_internal)] + unpacker.data['nested'][1].keys.should == ["sk\xC3\xA5l".force_encoding(Encoding::UTF_8)] end end @@ -255,23 +164,6 @@ def flatten(struct, results = []) unpacker.execute(buffer, 0) unpacker.data.should == {:hello => 'world', :nested => ['object', {:structure => true}]} end - - it 'can symbolize keys when using #each' do - objs = [] - unpacker.feed(buffer) - unpacker.each do |obj| - objs << obj - end - objs.should == [{:hello => 'world', :nested => ['object', {:structure => true}]}] - end - - it 'can symbolize keys when using #feed_each' do - objs = [] - unpacker.feed_each(buffer) do |obj| - objs << obj - end - objs.should == [{:hello => 'world', :nested => ['object', {:structure => true}]}] - end end context 'encoding', :encodings do @@ -280,35 +172,14 @@ def flatten(struct, results = []) end let :unpacker do - described_class.new(:encoding => 'UTF-8') + described_class.new() end - it 'can hardcode encoding when using #execute' do + it 'decode binary as ascii-8bit string when using #execute' do unpacker.execute(buffer, 0) strings = flatten(unpacker.data).grep(String) strings.should == %w[hello world nested object structure] - strings.map(&:encoding).uniq.should == [Encoding::UTF_8] - end - - it 'can hardcode encoding when using #each' do - objs = [] - unpacker.feed(buffer) - unpacker.each do |obj| - objs << obj - end - strings = flatten(objs).grep(String) - strings.should == %w[hello world nested object structure] - strings.map(&:encoding).uniq.should == [Encoding::UTF_8] - end - - it 'can hardcode encoding when using #feed_each' do - objs = [] - unpacker.feed_each(buffer) do |obj| - objs << obj - end - strings = flatten(objs).grep(String) - strings.should == %w[hello world nested object structure] - strings.map(&:encoding).uniq.should == [Encoding::UTF_8] + strings.map(&:encoding).uniq.should == [Encoding::ASCII_8BIT] end end end diff --git a/spec/msgpack_spec.rb b/spec/msgpack_spec.rb index 2c41ff75..3be6777b 100644 --- a/spec/msgpack_spec.rb +++ b/spec/msgpack_spec.rb @@ -116,7 +116,7 @@ def asciienc(str) end it 'rasies an error on #unpack with garbage' do - skip "???" + skip "but nothing was raised. why?" expect { MessagePack.unpack('asdka;sd') }.to raise_error(MessagePack::UnpackError) end end diff --git a/spec/packer_spec.rb b/spec/packer_spec.rb index 94813cb0..48096a74 100644 --- a/spec/packer_spec.rb +++ b/spec/packer_spec.rb @@ -1,6 +1,141 @@ +# encoding: ascii-8bit require 'spec_helper' +require 'stringio' +if defined?(Encoding) + Encoding.default_external = 'ASCII-8BIT' +end + describe MessagePack::Packer do + let :packer do + MessagePack::Packer.new + end + + it 'initialize' do + MessagePack::Packer.new + MessagePack::Packer.new(nil) + MessagePack::Packer.new(StringIO.new) + MessagePack::Packer.new({}) + MessagePack::Packer.new(StringIO.new, {}) + end + + it 'gets options to specify how to pack values' do + u1 = MessagePack::Packer.new + u1.compatibility_mode?.should == false + + u2 = MessagePack::Packer.new(compatibility_mode: true) + u2.compatibility_mode?.should == true + end + + it 'write' do + packer.write([]) + packer.to_s.should == "\x90" + end + + it 'write_nil' do + packer.write_nil + packer.to_s.should == "\xc0" + end + + it 'write_array_header 0' do + packer.write_array_header(0) + packer.to_s.should == "\x90" + end + + it 'write_array_header 1' do + packer.write_array_header(1) + packer.to_s.should == "\x91" + end + + it 'write_map_header 0' do + packer.write_map_header(0) + packer.to_s.should == "\x80" + end + + it 'write_map_header 1' do + packer.write_map_header(1) + packer.to_s.should == "\x81" + end + + it 'flush' do + io = StringIO.new + pk = MessagePack::Packer.new(io) + pk.write_nil + pk.flush + pk.to_s.should == '' + io.string.should == "\xc0" + end + + it 'to_msgpack returns String' do + nil.to_msgpack.class.should == String + true.to_msgpack.class.should == String + false.to_msgpack.class.should == String + 1.to_msgpack.class.should == String + 1.0.to_msgpack.class.should == String + "".to_msgpack.class.should == String + Hash.new.to_msgpack.class.should == String + Array.new.to_msgpack.class.should == String + end + + class CustomPack01 + def to_msgpack(pk=nil) + return MessagePack.pack(self, pk) unless pk.class == MessagePack::Packer + pk.write_array_header(2) + pk.write(1) + pk.write(2) + return pk + end + end + + class CustomPack02 + def to_msgpack(pk=nil) + [1,2].to_msgpack(pk) + end + end + + it 'calls custom to_msgpack method' do + MessagePack.pack(CustomPack01.new).should == [1,2].to_msgpack + MessagePack.pack(CustomPack02.new).should == [1,2].to_msgpack + CustomPack01.new.to_msgpack.should == [1,2].to_msgpack + CustomPack02.new.to_msgpack.should == [1,2].to_msgpack + end + + it 'calls custom to_msgpack method with io' do + s01 = StringIO.new + MessagePack.pack(CustomPack01.new, s01) + s01.string.should == [1,2].to_msgpack + + s02 = StringIO.new + MessagePack.pack(CustomPack02.new, s02) + s02.string.should == [1,2].to_msgpack + + s03 = StringIO.new + CustomPack01.new.to_msgpack(s03) + s03.string.should == [1,2].to_msgpack + + s04 = StringIO.new + CustomPack02.new.to_msgpack(s04) + s04.string.should == [1,2].to_msgpack + end + + context 'in compatibility mode' do + it 'does not use the bin types' do + packed = MessagePack.pack('hello'.force_encoding(Encoding::BINARY), compatibility_mode: true) + packed.should eq("\xA5hello") + packed = MessagePack.pack(('hello' * 100).force_encoding(Encoding::BINARY), compatibility_mode: true) + packed.should start_with("\xDA\x01\xF4") + + packer = MessagePack::Packer.new(compatibility_mode: 1) + packed = packer.pack(('hello' * 100).force_encoding(Encoding::BINARY)) + packed.to_str.should start_with("\xDA\x01\xF4") + end + + it 'does not use the str8 type' do + packed = MessagePack.pack('x' * 32, compatibility_mode: true) + packed.should start_with("\xDA\x00\x20") + end + end + class ValueOne def initialize(num) @num = num @@ -33,14 +168,12 @@ def self.from_msgpack_ext(data) describe '#type_registered?' do it 'receive Class or Integer, and return bool' do - skip("not supported yet in JRuby implementation") if java? expect(subject.type_registered?(0x00)).to be_falsy expect(subject.type_registered?(0x01)).to be_falsy expect(subject.type_registered?(::ValueOne)).to be_falsy end it 'returns true if specified type or class is already registered' do - skip("not supported yet in JRuby implementation") if java? subject.register_type(0x30, ::ValueOne, :to_msgpack_ext) subject.register_type(0x31, ::ValueTwo, :to_msgpack_ext) @@ -56,7 +189,6 @@ def self.from_msgpack_ext(data) describe '#register_type' do it 'get type and class mapping for packing' do - skip("not supported yet in JRuby implementation") if java? packer = MessagePack::Packer.new packer.register_type(0x01, ValueOne){|obj| obj.to_msgpack_ext } packer.register_type(0x02, ValueTwo){|obj| obj.to_msgpack_ext } @@ -71,7 +203,6 @@ def self.from_msgpack_ext(data) end it 'returns a Hash which contains map of Class and type' do - skip("not supported yet in JRuby implementation") if java? packer = MessagePack::Packer.new packer.register_type(0x01, ValueOne, :to_msgpack_ext) packer.register_type(0x02, ValueTwo, :to_msgpack_ext) diff --git a/spec/unpacker_spec.rb b/spec/unpacker_spec.rb index e5b5f23d..25f0fe72 100644 --- a/spec/unpacker_spec.rb +++ b/spec/unpacker_spec.rb @@ -1,6 +1,265 @@ +# encoding: ascii-8bit + +require 'stringio' +require 'tempfile' + require 'spec_helper' describe MessagePack::Unpacker do + let :unpacker do + MessagePack::Unpacker.new + end + + let :packer do + MessagePack::Packer.new + end + + it 'gets options to specify how to unpack values' do + u1 = MessagePack::Unpacker.new + u1.symbolize_keys?.should == false + u1.allow_unknown_ext?.should == false + + u2 = MessagePack::Unpacker.new(symbolize_keys: true, allow_unknown_ext: true) + u2.symbolize_keys?.should == true + u2.allow_unknown_ext?.should == true + end + + it 'read_array_header succeeds' do + unpacker.feed("\x91") + unpacker.read_array_header.should == 1 + end + + it 'read_array_header fails' do + unpacker.feed("\x81") + lambda { + unpacker.read_array_header + }.should raise_error(MessagePack::TypeError) # TypeError is included in UnexpectedTypeError + lambda { + unpacker.read_array_header + }.should raise_error(MessagePack::UnexpectedTypeError) + end + + it 'read_map_header converts an map to key-value sequence' do + packer.write_array_header(2) + packer.write("e") + packer.write(1) + unpacker = MessagePack::Unpacker.new + unpacker.feed(packer.to_s) + unpacker.read_array_header.should == 2 + unpacker.read.should == "e" + unpacker.read.should == 1 + end + + it 'read_map_header succeeds' do + unpacker.feed("\x81") + unpacker.read_map_header.should == 1 + end + + it 'read_map_header converts an map to key-value sequence' do + packer.write_map_header(1) + packer.write("k") + packer.write("v") + unpacker = MessagePack::Unpacker.new + unpacker.feed(packer.to_s) + unpacker.read_map_header.should == 1 + unpacker.read.should == "k" + unpacker.read.should == "v" + end + + it 'read_map_header fails' do + unpacker.feed("\x91") + lambda { + unpacker.read_map_header + }.should raise_error(MessagePack::TypeError) # TypeError is included in UnexpectedTypeError + lambda { + unpacker.read_map_header + }.should raise_error(MessagePack::UnexpectedTypeError) + end + + it 'read raises EOFError before feeding' do + lambda { + unpacker.read + }.should raise_error(EOFError) + end + + let :sample_object do + [1024, {["a","b"]=>["c","d"]}, ["e","f"], "d", 70000, 4.12, 1.5, 1.5, 1.5] + end + + it 'feed and each continue internal state' do + raw = sample_object.to_msgpack.to_s * 4 + objects = [] + + raw.split(//).each do |b| + unpacker.feed(b) + unpacker.each {|c| + objects << c + } + end + + objects.should == [sample_object] * 4 + end + + it 'feed_each continues internal state' do + raw = sample_object.to_msgpack.to_s * 4 + objects = [] + + raw.split(//).each do |b| + unpacker.feed_each(b) {|c| + objects << c + } + end + + objects.should == [sample_object] * 4 + end + + it 'feed_each enumerator' do + raw = sample_object.to_msgpack.to_s * 4 + + enum = unpacker.feed_each(raw) + enum.should be_instance_of(Enumerator) + enum.to_a.should == [sample_object] * 4 + end + + it 'reset clears internal buffer' do + # 1-element array + unpacker.feed("\x91") + unpacker.reset + unpacker.feed("\x01") + + unpacker.each.map {|x| x }.should == [1] + end + + it 'reset clears internal state' do + # 1-element array + unpacker.feed("\x91") + unpacker.each.map {|x| x }.should == [] + + unpacker.reset + + unpacker.feed("\x01") + unpacker.each.map {|x| x }.should == [1] + end + + it 'frozen short strings' do + raw = sample_object.to_msgpack.to_s.force_encoding('UTF-8') + lambda { + unpacker.feed_each(raw.freeze) { } + }.should_not raise_error + end + + it 'frozen long strings' do + raw = (sample_object.to_msgpack.to_s * 10240).force_encoding('UTF-8') + lambda { + unpacker.feed_each(raw.freeze) { } + }.should_not raise_error + end + + it 'read raises invalid byte error' do + unpacker.feed("\xc1") + lambda { + unpacker.read + }.should raise_error(MessagePack::MalformedFormatError) + end + + it "gc mark" do + raw = sample_object.to_msgpack.to_s * 4 + + n = 0 + raw.split(//).each do |b| + GC.start + unpacker.feed_each(b) {|o| + GC.start + o.should == sample_object + n += 1 + } + GC.start + end + + n.should == 4 + end + + it "buffer" do + orig = "a"*32*1024*4 + raw = orig.to_msgpack.to_s + + n = 655 + times = raw.size / n + times += 1 unless raw.size % n == 0 + + off = 0 + parsed = false + + times.times do + parsed.should == false + + seg = raw[off, n] + off += seg.length + + unpacker.feed_each(seg) {|obj| + parsed.should == false + obj.should == orig + parsed = true + } + end + + parsed.should == true + end + + it 'MessagePack.unpack symbolize_keys' do + symbolized_hash = {:a => 'b', :c => 'd'} + MessagePack.load(MessagePack.pack(symbolized_hash), :symbolize_keys => true).should == symbolized_hash + MessagePack.unpack(MessagePack.pack(symbolized_hash), :symbolize_keys => true).should == symbolized_hash + end + + it 'Unpacker#unpack symbolize_keys' do + unpacker = MessagePack::Unpacker.new(:symbolize_keys => true) + symbolized_hash = {:a => 'b', :c => 'd'} + unpacker.feed(MessagePack.pack(symbolized_hash)).read.should == symbolized_hash + end + + it "msgpack str 8 type" do + MessagePack.unpack([0xd9, 0x00].pack('C*')).should == "" + MessagePack.unpack([0xd9, 0x00].pack('C*')).encoding.should == Encoding::UTF_8 + MessagePack.unpack([0xd9, 0x01].pack('C*') + 'a').should == "a" + MessagePack.unpack([0xd9, 0x02].pack('C*') + 'aa').should == "aa" + end + + it "msgpack str 16 type" do + MessagePack.unpack([0xda, 0x00, 0x00].pack('C*')).should == "" + MessagePack.unpack([0xda, 0x00, 0x00].pack('C*')).encoding.should == Encoding::UTF_8 + MessagePack.unpack([0xda, 0x00, 0x01].pack('C*') + 'a').should == "a" + MessagePack.unpack([0xda, 0x00, 0x02].pack('C*') + 'aa').should == "aa" + end + + it "msgpack str 32 type" do + MessagePack.unpack([0xdb, 0x00, 0x00, 0x00, 0x00].pack('C*')).should == "" + MessagePack.unpack([0xdb, 0x00, 0x00, 0x00, 0x00].pack('C*')).encoding.should == Encoding::UTF_8 + MessagePack.unpack([0xdb, 0x00, 0x00, 0x00, 0x01].pack('C*') + 'a').should == "a" + MessagePack.unpack([0xdb, 0x00, 0x00, 0x00, 0x02].pack('C*') + 'aa').should == "aa" + end + + it "msgpack bin 8 type" do + MessagePack.unpack([0xc4, 0x00].pack('C*')).should == "" + MessagePack.unpack([0xc4, 0x00].pack('C*')).encoding.should == Encoding::ASCII_8BIT + MessagePack.unpack([0xc4, 0x01].pack('C*') + 'a').should == "a" + MessagePack.unpack([0xc4, 0x02].pack('C*') + 'aa').should == "aa" + end + + it "msgpack bin 16 type" do + MessagePack.unpack([0xc5, 0x00, 0x00].pack('C*')).should == "" + MessagePack.unpack([0xc5, 0x00, 0x00].pack('C*')).encoding.should == Encoding::ASCII_8BIT + MessagePack.unpack([0xc5, 0x00, 0x01].pack('C*') + 'a').should == "a" + MessagePack.unpack([0xc5, 0x00, 0x02].pack('C*') + 'aa').should == "aa" + end + + it "msgpack bin 32 type" do + MessagePack.unpack([0xc6, 0x00, 0x00, 0x00, 0x00].pack('C*')).should == "" + MessagePack.unpack([0xc6, 0x0, 0x00, 0x00, 0x000].pack('C*')).encoding.should == Encoding::ASCII_8BIT + MessagePack.unpack([0xc6, 0x00, 0x00, 0x00, 0x01].pack('C*') + 'a').should == "a" + MessagePack.unpack([0xc6, 0x00, 0x00, 0x00, 0x02].pack('C*') + 'aa').should == "aa" + end + class ValueOne attr_reader :num def initialize(num) @@ -41,14 +300,12 @@ def self.from_msgpack_ext(data) describe '#type_registered?' do it 'receive Class or Integer, and return bool' do - skip("not supported yet in JRuby implementation") if java? expect(subject.type_registered?(0x00)).to be_falsy expect(subject.type_registered?(0x01)).to be_falsy expect(subject.type_registered?(::ValueOne)).to be_falsy end it 'returns true if specified type or class is already registered' do - skip("not supported yet in JRuby implementation") if java? subject.register_type(0x30, ::ValueOne, :from_msgpack_ext) subject.register_type(0x31, ::ValueTwo, :from_msgpack_ext) @@ -62,7 +319,6 @@ def self.from_msgpack_ext(data) end it 'cannot detect unpack rule with block, not method' do - skip("not supported yet in JRuby implementation") if java? subject.register_type(0x40){|data| ValueOne.from_msgpack_ext(data) } expect(subject.type_registered?(0x40)).to be_truthy @@ -72,7 +328,6 @@ def self.from_msgpack_ext(data) context 'with ext definitions' do it 'get type and class mapping for packing' do - skip("not supported yet in JRuby implementation") if java? unpacker = MessagePack::Unpacker.new unpacker.register_type(0x01){|data| ValueOne.from_msgpack_ext } unpacker.register_type(0x02){|data| ValueTwo.from_msgpack_ext(data) } @@ -83,7 +338,6 @@ def self.from_msgpack_ext(data) end it 'returns a Array of Hash which contains :type, :class and :unpacker' do - skip("not supported yet in JRuby implementation") if java? unpacker = MessagePack::Unpacker.new unpacker.register_type(0x02, ValueTwo, :from_msgpack_ext) unpacker.register_type(0x01, ValueOne, :from_msgpack_ext) @@ -107,7 +361,6 @@ def self.from_msgpack_ext(data) end it 'returns a Array of Hash, which contains nil for class if block unpacker specified' do - skip("not supported yet in JRuby implementation") if java? unpacker = MessagePack::Unpacker.new unpacker.register_type(0x01){|data| ValueOne.from_msgpack_ext } unpacker.register_type(0x02, &ValueTwo.method(:from_msgpack_ext)) @@ -130,4 +383,210 @@ def self.from_msgpack_ext(data) expect(two[:unpacker]).to be_instance_of(Proc) end end + + def flatten(struct, results = []) + case struct + when Array + struct.each { |v| flatten(v, results) } + when Hash + struct.each { |k, v| flatten(v, flatten(k, results)) } + else + results << struct + end + results + end + + subject do + described_class.new + end + + let :buffer1 do + MessagePack.pack(:foo => 'bar') + end + + let :buffer2 do + MessagePack.pack(:hello => {:world => [1, 2, 3]}) + end + + let :buffer3 do + MessagePack.pack(:x => 'y') + end + + describe '#read' do + context 'with a buffer' do + it 'reads objects' do + objects = [] + subject.feed(buffer1) + subject.feed(buffer2) + subject.feed(buffer3) + objects << subject.read + objects << subject.read + objects << subject.read + objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}] + end + + it 'reads map header' do + subject.feed({}.to_msgpack) + subject.read_map_header.should == 0 + end + + it 'reads array header' do + subject.feed([].to_msgpack) + subject.read_array_header.should == 0 + end + end + end + + describe '#each' do + context 'with a buffer' do + it 'yields each object in the buffer' do + objects = [] + subject.feed(buffer1) + subject.feed(buffer2) + subject.feed(buffer3) + subject.each do |obj| + objects << obj + end + objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}] + end + + it 'returns an enumerator when no block is given' do + subject.feed(buffer1) + subject.feed(buffer2) + subject.feed(buffer3) + enum = subject.each + enum.map { |obj| obj.keys.first }.should == %w[foo hello x] + end + end + + context 'with a stream passed to the constructor' do + it 'yields each object in the stream' do + objects = [] + unpacker = described_class.new(StringIO.new(buffer1 + buffer2 + buffer3)) + unpacker.each do |obj| + objects << obj + end + objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}] + end + end + end + + describe '#feed_each' do + it 'feeds the buffer then runs #each' do + objects = [] + subject.feed_each(buffer1 + buffer2 + buffer3) do |obj| + objects << obj + end + objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}] + end + + it 'handles chunked data' do + objects = [] + buffer = buffer1 + buffer2 + buffer3 + buffer.chars.each do |ch| + subject.feed_each(ch) do |obj| + objects << obj + end + end + objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}] + end + end + + context 'regressions' do + it 'handles massive arrays (issue #2)' do + array = ['foo'] * 10_000 + MessagePack.unpack(MessagePack.pack(array)).size.should == 10_000 + end + end + + context 'extensions' do + context 'symbolized keys' do + let :buffer do + MessagePack.pack({'hello' => 'world', 'nested' => ['object', {'structure' => true}]}) + end + + let :unpacker do + described_class.new(:symbolize_keys => true) + end + + it 'can symbolize keys when using #each' do + objs = [] + unpacker.feed(buffer) + unpacker.each do |obj| + objs << obj + end + objs.should == [{:hello => 'world', :nested => ['object', {:structure => true}]}] + end + + it 'can symbolize keys when using #feed_each' do + objs = [] + unpacker.feed_each(buffer) do |obj| + objs << obj + end + objs.should == [{:hello => 'world', :nested => ['object', {:structure => true}]}] + end + end + + context 'binary encoding', :encodings do + let :buffer do + MessagePack.pack({'hello' => 'world', 'nested' => ['object', {'structure' => true}]}) + end + + let :unpacker do + described_class.new() + end + + it 'decodes binary as ascii-8bit when using #feed' do + objs = [] + unpacker.feed(buffer) + unpacker.each do |obj| + objs << obj + end + strings = flatten(objs).grep(String) + strings.should == %w[hello world nested object structure] + strings.map(&:encoding).uniq.should == [Encoding::ASCII_8BIT] + end + + it 'decodes binary as ascii-8bit when using #feed_each' do + objs = [] + unpacker.feed_each(buffer) do |obj| + objs << obj + end + strings = flatten(objs).grep(String) + strings.should == %w[hello world nested object structure] + strings.map(&:encoding).uniq.should == [Encoding::ASCII_8BIT] + end + end + + context 'string encoding', :encodings do + let :buffer do + MessagePack.pack({'hello'.force_encoding(Encoding::UTF_8) => 'world'.force_encoding(Encoding::UTF_8), 'nested'.force_encoding(Encoding::UTF_8) => ['object'.force_encoding(Encoding::UTF_8), {'structure'.force_encoding(Encoding::UTF_8) => true}]}) + end + + let :unpacker do + described_class.new() + end + + it 'decodes string as utf-8 when using #feed' do + objs = [] + unpacker.feed(buffer) + unpacker.each do |obj| + objs << obj + end + strings = flatten(objs).grep(String) + strings.should == %w[hello world nested object structure] + strings.map(&:encoding).uniq.should == [Encoding::UTF_8] + end + + it 'decodes binary as ascii-8bit when using #feed_each' do + objs = [] + unpacker.feed_each(buffer) do |obj| + objs << obj + end + strings = flatten(objs).grep(String) + strings.should == %w[hello world nested object structure] + strings.map(&:encoding).uniq.should == [Encoding::UTF_8] + end + end + end end