From: gregor herrmann Date: Sun, 3 Aug 2014 21:21:35 +0000 (+0200) Subject: Imported Upstream version 3.001.006 X-Git-Tag: archive/raspbian/4.017+ds-1+rpi1~1^2~3^2~21 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=6dcca9fc1cf397acbfe632d0a8fa6efa5b1da9f2;p=libsereal-encoder-perl.git Imported Upstream version 3.001.006 --- diff --git a/Changes b/Changes index 012aff8..590b16d 100644 --- a/Changes +++ b/Changes @@ -3,6 +3,14 @@ Revision history for Perl extension Sereal-Encoder * Warning: For a seamless upgrade, upgrade to version 3 * of the decoder before upgrading to version 3 of the * encoder! +3.001_006 Sun, Aug 03 2014 + - Rework bulk tests so we test more, but report less tests. + The test infrastructure doesn't play well with lots of tests + in a file. Similarly, if we fail one of the methods in the bulk + tests we stop testing the rest. + - Add a canonical mode to the encoder. + - More tests. + 3.001_005 Mon, July 28 2014 - Fixup how MakeMaker runs the tests. diff --git a/MANIFEST b/MANIFEST index e7b1b3a..595d9d2 100644 --- a/MANIFEST +++ b/MANIFEST @@ -35,6 +35,7 @@ srl_inline.h srl_protocol.h t/001_load.t t/002_constants.t +t/002_testset.t t/003_ptable.t t/010_desperate.t t/011_aliased_dedupe.t @@ -48,20 +49,28 @@ t/200_bulk.t t/300_fail.t t/400_evil.t t/700_roundtrip/v1/plain.t +t/700_roundtrip/v1/plain_canon.t t/700_roundtrip/v1/snappy.t +t/700_roundtrip/v1/snappy_canon.t t/700_roundtrip/v2/dedudep_strings.t t/700_roundtrip/v2/freeze_thaw.t t/700_roundtrip/v2/plain.t +t/700_roundtrip/v2/plain_canon.t t/700_roundtrip/v2/readonly.t t/700_roundtrip/v2/snappy.t +t/700_roundtrip/v2/snappy_canon.t t/700_roundtrip/v2/snappy_incr.t +t/700_roundtrip/v2/snappy_incr_canon.t t/700_roundtrip/v2/sort_keys.t t/700_roundtrip/v3/dedudep_strings.t t/700_roundtrip/v3/freeze_thaw.t t/700_roundtrip/v3/plain.t +t/700_roundtrip/v3/plain_canon.t t/700_roundtrip/v3/readonly.t t/700_roundtrip/v3/snappy.t +t/700_roundtrip/v3/snappy_canon.t t/700_roundtrip/v3/snappy_incr.t +t/700_roundtrip/v3/snappy_incr_canon.t t/700_roundtrip/v3/sort_keys.t t/700_roundtrip/v3/zlib.t t/700_roundtrip/v3/zlib_force.t diff --git a/META.json b/META.json index 4e4ea9b..042b86d 100644 --- a/META.json +++ b/META.json @@ -55,5 +55,5 @@ "url" : "git://github.com/Sereal/Sereal.git" } }, - "version" : "3.001_005" + "version" : "3.001_006" } diff --git a/META.yml b/META.yml index eed72e1..8b59bce 100644 --- a/META.yml +++ b/META.yml @@ -32,4 +32,4 @@ requires: resources: bugtracker: https://github.com/Sereal/Sereal/issues repository: git://github.com/Sereal/Sereal.git -version: 3.001_005 +version: 3.001_006 diff --git a/Makefile.PL b/Makefile.PL index f4fab2a..25f9bca 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -11,7 +11,8 @@ my $in_source_repo = -d "../../.git" and -d $shared_dir; my $module = "Sereal::Encoder"; -unshift @INC, '.', 'inc'; +unshift @INC, ".", "./inc"; +unshift @INC, $shared_dir, "$shared_dir/inc" if $in_source_repo; require inc::Sereal::BuildTools; inc::Sereal::BuildTools::link_files($shared_dir) if $in_source_repo; inc::Sereal::BuildTools::generate_constant_includes($module) if $in_source_repo; diff --git a/author_tools/hobodecoder.pl b/author_tools/hobodecoder.pl index 19264b1..831f7ae 100644 --- a/author_tools/hobodecoder.pl +++ b/author_tools/hobodecoder.pl @@ -4,13 +4,16 @@ use warnings; use Data::Dumper; use Getopt::Long qw(GetOptions); +our @constants; BEGIN { my $err; eval ' use Sereal::Encoder::Constants qw(:all); + @constants= @Sereal::Encoder::Constants::EXPORT_OK; 1; ' or do { $err= $@; eval ' use Sereal::Decoder::Constants qw(:all); + @constants= @Sereal::Decoder::Constants::EXPORT_OK; 1; ' } or die "No encoder/decoder constants: $err\n$@"; } @@ -19,7 +22,6 @@ my $done; my $data; my $hlen; my $indent = ""; -my %const_names = map {$_ => eval "$_"} @Sereal::Constants::EXPORT_OK; sub parse_header { $data =~ s/^(=[s\xF3]rl)(.)// or die "invalid header: $data"; @@ -113,6 +115,9 @@ sub parse_sv { if ($o == SRL_HDR_VARINT) { printf "VARINT: %u\n", varint(); } + elsif ($o == SRL_HDR_ZIGZAG) { + printf "ZIGZAG: %d\n", zigzag(); + } elsif (SRL_HDR_POS_LOW <= $o && $o <= SRL_HDR_POS_HIGH) { printf "POS: %u\n", $o; } @@ -227,7 +232,8 @@ sub parse_sv { } else { printf "\n"; - die "unsupported type: $o ($t): $const_names{$o}"; + die sprintf "unsupported type: 0x%02x (%d) %s: %s", $o, $o, + Data::Dumper::qquote($t), Data::Dumper->new([$TAG_INFO_ARRAY[$o]])->Terse(1)->Dump(); } return 0; } @@ -280,6 +286,14 @@ sub varint { return $x; } +BEGIN{ +my $_shift= length(pack"j",0) * 8 - 1; +sub zigzag { + my $n= varint(); + return ($n >> 1) ^ (-($n & 1)); +} +} + GetOptions( my $opt = {}, 'e|stderr', @@ -290,8 +304,6 @@ if ($opt->{e}) { select(STDERR); } -#print Dumper \%const_names; exit; - local $/ = undef; $data = ; diff --git a/author_tools/update_from_header.pl b/author_tools/update_from_header.pl index 1b5836d..4f009d2 100644 --- a/author_tools/update_from_header.pl +++ b/author_tools/update_from_header.pl @@ -3,6 +3,7 @@ use strict; use warnings; use Data::Dumper; my ( + @meta, %name_to_value, # just the names in the srl_protocol.h %name_to_value_expanded, # names from srl_protocol, but with the LOW/HIGH data expanded %value_to_name_expanded, # values from srl_protocol_expanded, mapping back, note value points at FIRST name @@ -15,25 +16,48 @@ sub fill_ranges { $pfx=~s/_LOW//; defined(my $ofs= $name_to_value_expanded{$pfx}) or die "unknown $pfx"; - for my $i ( $name_to_value_expanded{$pfx . "_LOW"} .. $name_to_value_expanded{$pfx . "_HIGH"}) { - my $n= $pfx=~/NEG/ ? abs($i - 32) : $i - $ofs; - $name_to_value_expanded{ $pfx . "_" . $n } ||= $i; - $value_to_name_expanded{ $i } = $pfx . "_". $n; - $value_to_comment_expanded{ $i } ||= ''; + for my $value ( $name_to_value_expanded{$pfx . "_LOW"} .. $name_to_value_expanded{$pfx . "_HIGH"}) { + my $n= $pfx=~/NEG/ ? abs($value - 32) : $value - $ofs; + my $name= $pfx . "_" . $n; + $name_to_value_expanded{ $name } ||= $value; + $value_to_name_expanded{ $value } = $name; + $value_to_comment_expanded{ $value } ||= ''; + + $meta[$value]{name}= $name; + $meta[$value]{value}= $value; + $meta[$value]{type_name}= $pfx; + $meta[$value]{type_value}= $ofs; + #$meta[$value]{comment}= $value_to_comment_expanded{ $ofs } + # if exists $value_to_comment_expanded{ $ofs }; + + $meta[$value]{masked_val}= $n; + $meta[$value]{masked}= 1; + } $value_to_comment_expanded{ $name_to_value_expanded{$pfx . "_HIGH"} } = $value_to_comment_expanded{ $ofs }; } sub read_protocol { open my $fh,"<", "Perl/shared/srl_protocol.h" or die "Perl/shared/srl_protocol.h: $!"; + my @fill; while (<$fh>) { if(m!^#define\s+SRL_HDR_(\S+)\s+\(\(char\)(\d+)\)\s*(?:/\*\s*(.*?)\s*\*/)?\s*\z!i) { - $name_to_value{$1}= $2; - $name_to_value_expanded{$1}= $2; - $value_to_name_expanded{$2} ||= $1; - $value_to_comment_expanded{$2} ||= $3; - push @fill, $1 if substr($1,-4) eq '_LOW'; + my ($name, $value, $comment)= ($1, $2, $3); + $value= 0+$value; + $name_to_value{$name}= $value; + $name_to_value_expanded{$name}= $value; + $value_to_name_expanded{$value} ||= $name; + $value_to_comment_expanded{$value} ||= $comment; + push @fill, $name if substr($name, -4) eq '_LOW'; + + if ( $value < 128 ) { + $meta[$value]{name}= $name; + $meta[$value]{value}= $value; + $meta[$value]{type_name}= $name; + $meta[$value]{type_value}= $value; + $meta[$value]{comment}= $comment if defined $comment; + } } } close $fh; @@ -42,6 +66,7 @@ sub read_protocol { $max_name_length= length($pfx) if $max_name_length < length($pfx); } } + sub open_swap { my $file= shift; open my $fh,"<", $file @@ -58,12 +83,12 @@ sub replace_block { my ($in,$out)= open_swap($file); while (<$in>) { print $out $_; - last if /^=for autoupdater start/; + last if /^=for autoupdater start/ || /^# start autoupdated section/; } $blob=~s/\s+$//mg; print $out "\n$blob\n\n"; while (<$in>) { - if (/^=for autoupdater stop/) { + if (/^=for autoupdater stop/ || /^# stop autoupdated section/) { print $out $_; last; } @@ -74,6 +99,18 @@ sub replace_block { close $out; close $in; } +sub update_buildtools { + my $dump= Data::Dumper->new([\@meta],['*TAG_INFO_ARRAY'])->Indent(1)->Dump(); + $dump =~ s/^(\s*)\{/$1# autoupdated by $0 do not modify directly!\n$1\{/mg; + return replace_block( + "Perl/shared/inc/Sereal/BuildTools.pm", + join "\n", + "our (%TAG_INFO_HASH, \@TAG_INFO_ARRAY);", + $dump, + "\$TAG_INFO_HASH{chr \$_}= \$TAG_INFO_ARRAY[\$_] for 0 .. 127;", + "push \@EXPORT_OK, qw(%TAG_INFO_HASH \@TAG_INFO_ARRAY);", + ) +} sub update_srl_decoder_h { replace_block("Perl/Decoder/srl_decoder.h", join("\n", @@ -135,6 +172,7 @@ chomp($git_dir); chdir "$git_dir/.." or die "Failed to chdir to root of repo '$git_dir/..': $!"; read_protocol(); +update_buildtools(); update_srl_decoder_h(); update_table("sereal_spec.pod"); update_table("Perl/shared/srl_protocol.h"); diff --git a/inc/Sereal/BuildTools.pm b/inc/Sereal/BuildTools.pm index 9a72963..530efa3 100644 --- a/inc/Sereal/BuildTools.pm +++ b/inc/Sereal/BuildTools.pm @@ -101,9 +101,1160 @@ HERE }; } -sub SRL_MAGIC_STRING () {"=srl"} -sub SRL_MAGIC_STRING_HIGHBIT () {"=\xF3rl"} -push @EXPORT_OK, qw(SRL_MAGIC_STRING SRL_MAGIC_STRING_HIGHBIT); +sub SRL_MAGIC_STRING () { "=srl" } +sub SRL_MAGIC_STRING_HIGHBIT () { "=\xF3rl" } +sub SRL_MAGIC_STRING_HIGHBIT_UTF8 () { "=\xC3\xB3rl" } + +push @EXPORT_OK, qw( + SRL_MAGIC_STRING + SRL_MAGIC_STRING_HIGHBIT + SRL_MAGIC_STRING_HIGHBIT_UTF8 +); + +# start autoupdated section - do not modify directly + +our (%TAG_INFO_HASH, @TAG_INFO_ARRAY); +@TAG_INFO_ARRAY = ( + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'comment' => 'small positive integer - value in low 4 bits (identity)', + 'value' => 0, + 'name' => 'POS_0', + 'masked_val' => 0, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 1, + 'name' => 'POS_1', + 'masked_val' => 1, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 2, + 'name' => 'POS_2', + 'masked_val' => 2, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 3, + 'name' => 'POS_3', + 'masked_val' => 3, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 4, + 'name' => 'POS_4', + 'masked_val' => 4, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 5, + 'name' => 'POS_5', + 'masked_val' => 5, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 6, + 'name' => 'POS_6', + 'masked_val' => 6, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 7, + 'name' => 'POS_7', + 'masked_val' => 7, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 8, + 'name' => 'POS_8', + 'masked_val' => 8, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 9, + 'name' => 'POS_9', + 'masked_val' => 9, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 10, + 'name' => 'POS_10', + 'masked_val' => 10, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 11, + 'name' => 'POS_11', + 'masked_val' => 11, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 12, + 'name' => 'POS_12', + 'masked_val' => 12, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 13, + 'name' => 'POS_13', + 'masked_val' => 13, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 14, + 'name' => 'POS_14', + 'masked_val' => 14, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'comment' => 'small positive integer - value in low 4 bits (identity)', + 'value' => 15, + 'name' => 'POS_15', + 'masked_val' => 15, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'comment' => 'small negative integer - value in low 4 bits (k+32)', + 'value' => 16, + 'name' => 'NEG_16', + 'masked_val' => 16, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 17, + 'name' => 'NEG_15', + 'masked_val' => 15, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 18, + 'name' => 'NEG_14', + 'masked_val' => 14, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 19, + 'name' => 'NEG_13', + 'masked_val' => 13, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 20, + 'name' => 'NEG_12', + 'masked_val' => 12, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 21, + 'name' => 'NEG_11', + 'masked_val' => 11, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 22, + 'name' => 'NEG_10', + 'masked_val' => 10, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 23, + 'name' => 'NEG_9', + 'masked_val' => 9, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 24, + 'name' => 'NEG_8', + 'masked_val' => 8, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 25, + 'name' => 'NEG_7', + 'masked_val' => 7, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 26, + 'name' => 'NEG_6', + 'masked_val' => 6, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 27, + 'name' => 'NEG_5', + 'masked_val' => 5, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 28, + 'name' => 'NEG_4', + 'masked_val' => 4, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 29, + 'name' => 'NEG_3', + 'masked_val' => 3, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 30, + 'name' => 'NEG_2', + 'masked_val' => 2, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'comment' => 'small negative integer - value in low 4 bits (k+32)', + 'value' => 31, + 'name' => 'NEG_1', + 'masked_val' => 1, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'VARINT', + 'comment' => ' - Varint variable length integer', + 'value' => 32, + 'name' => 'VARINT', + 'type_value' => 32 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ZIGZAG', + 'comment' => ' - Zigzag variable length integer', + 'value' => 33, + 'name' => 'ZIGZAG', + 'type_value' => 33 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'FLOAT', + 'comment' => '', + 'value' => 34, + 'name' => 'FLOAT', + 'type_value' => 34 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'DOUBLE', + 'comment' => '', + 'value' => 35, + 'name' => 'DOUBLE', + 'type_value' => 35 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'LONG_DOUBLE', + 'comment' => '', + 'value' => 36, + 'name' => 'LONG_DOUBLE', + 'type_value' => 36 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'UNDEF', + 'comment' => 'None - Perl undef var; eg my $var= undef;', + 'value' => 37, + 'name' => 'UNDEF', + 'type_value' => 37 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'BINARY', + 'comment' => ' - binary/(latin1) string', + 'value' => 38, + 'name' => 'BINARY', + 'type_value' => 38 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'STR_UTF8', + 'comment' => ' - utf8 string', + 'value' => 39, + 'name' => 'STR_UTF8', + 'type_value' => 39 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'REFN', + 'comment' => ' - ref to next item', + 'value' => 40, + 'name' => 'REFN', + 'type_value' => 40 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'REFP', + 'comment' => ' - ref to previous item stored at offset', + 'value' => 41, + 'name' => 'REFP', + 'type_value' => 41 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASH', + 'comment' => ' [ ...] - count followed by key/value pairs', + 'value' => 42, + 'name' => 'HASH', + 'type_value' => 42 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAY', + 'comment' => ' [ ...] - count followed by items', + 'value' => 43, + 'name' => 'ARRAY', + 'type_value' => 43 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'OBJECT', + 'comment' => ' - class, object-item', + 'value' => 44, + 'name' => 'OBJECT', + 'type_value' => 44 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'OBJECTV', + 'comment' => ' - offset of previously used classname tag - object-item', + 'value' => 45, + 'name' => 'OBJECTV', + 'type_value' => 45 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ALIAS', + 'comment' => ' - alias to item defined at offset', + 'value' => 46, + 'name' => 'ALIAS', + 'type_value' => 46 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'COPY', + 'comment' => ' - copy of item defined at offset', + 'value' => 47, + 'name' => 'COPY', + 'type_value' => 47 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'WEAKEN', + 'comment' => ' - Weaken the following reference', + 'value' => 48, + 'name' => 'WEAKEN', + 'type_value' => 48 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'REGEXP', + 'comment' => ' ', + 'value' => 49, + 'name' => 'REGEXP', + 'type_value' => 49 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'OBJECT_FREEZE', + 'comment' => ' - class, object-item. Need to call "THAW" method on class after decoding', + 'value' => 50, + 'name' => 'OBJECT_FREEZE', + 'type_value' => 50 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'OBJECTV_FREEZE', + 'comment' => ' - (OBJECTV_FREEZE is to OBJECT_FREEZE as OBJECTV is to OBJECT)', + 'value' => 51, + 'name' => 'OBJECTV_FREEZE', + 'type_value' => 51 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'RESERVED', + 'masked' => 1, + 'comment' => 'reserved', + 'value' => 52, + 'name' => 'RESERVED_0', + 'masked_val' => 0, + 'type_value' => 52 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'RESERVED', + 'masked' => 1, + 'value' => 53, + 'name' => 'RESERVED_1', + 'masked_val' => 1, + 'type_value' => 52 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'RESERVED', + 'masked' => 1, + 'value' => 54, + 'name' => 'RESERVED_2', + 'masked_val' => 2, + 'type_value' => 52 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'RESERVED', + 'masked' => 1, + 'value' => 55, + 'name' => 'RESERVED_3', + 'masked_val' => 3, + 'type_value' => 52 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'RESERVED', + 'masked' => 1, + 'value' => 56, + 'name' => 'RESERVED_4', + 'masked_val' => 4, + 'type_value' => 52 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'CANONICAL_UNDEF', + 'comment' => 'undef (PL_sv_undef) - "the" Perl undef (see notes)', + 'value' => 57, + 'name' => 'CANONICAL_UNDEF', + 'type_value' => 57 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'FALSE', + 'comment' => 'false (PL_sv_no)', + 'value' => 58, + 'name' => 'FALSE', + 'type_value' => 58 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'TRUE', + 'comment' => 'true (PL_sv_yes)', + 'value' => 59, + 'name' => 'TRUE', + 'type_value' => 59 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'MANY', + 'comment' => ' - repeated tag (not done yet, will be implemented in version 3)', + 'value' => 60, + 'name' => 'MANY', + 'type_value' => 60 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'PACKET_START', + 'comment' => '(first byte of magic string in header)', + 'value' => 61, + 'name' => 'PACKET_START', + 'type_value' => 61 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'EXTEND', + 'comment' => ' - for additional tags', + 'value' => 62, + 'name' => 'EXTEND', + 'type_value' => 62 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'PAD', + 'comment' => '(ignored tag, skip to next byte)', + 'value' => 63, + 'name' => 'PAD', + 'type_value' => 63 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'comment' => '[ ...] - count of items in low 4 bits (ARRAY must be refcnt=1)', + 'value' => 64, + 'name' => 'ARRAYREF_0', + 'masked_val' => 0, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 65, + 'name' => 'ARRAYREF_1', + 'masked_val' => 1, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 66, + 'name' => 'ARRAYREF_2', + 'masked_val' => 2, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 67, + 'name' => 'ARRAYREF_3', + 'masked_val' => 3, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 68, + 'name' => 'ARRAYREF_4', + 'masked_val' => 4, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 69, + 'name' => 'ARRAYREF_5', + 'masked_val' => 5, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 70, + 'name' => 'ARRAYREF_6', + 'masked_val' => 6, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 71, + 'name' => 'ARRAYREF_7', + 'masked_val' => 7, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 72, + 'name' => 'ARRAYREF_8', + 'masked_val' => 8, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 73, + 'name' => 'ARRAYREF_9', + 'masked_val' => 9, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 74, + 'name' => 'ARRAYREF_10', + 'masked_val' => 10, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 75, + 'name' => 'ARRAYREF_11', + 'masked_val' => 11, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 76, + 'name' => 'ARRAYREF_12', + 'masked_val' => 12, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 77, + 'name' => 'ARRAYREF_13', + 'masked_val' => 13, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 78, + 'name' => 'ARRAYREF_14', + 'masked_val' => 14, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 79, + 'name' => 'ARRAYREF_15', + 'masked_val' => 15, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'comment' => '[ ...] - count in low 4 bits, key/value pairs (HASH must be refcnt=1)', + 'value' => 80, + 'name' => 'HASHREF_0', + 'masked_val' => 0, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 81, + 'name' => 'HASHREF_1', + 'masked_val' => 1, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 82, + 'name' => 'HASHREF_2', + 'masked_val' => 2, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 83, + 'name' => 'HASHREF_3', + 'masked_val' => 3, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 84, + 'name' => 'HASHREF_4', + 'masked_val' => 4, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 85, + 'name' => 'HASHREF_5', + 'masked_val' => 5, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 86, + 'name' => 'HASHREF_6', + 'masked_val' => 6, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 87, + 'name' => 'HASHREF_7', + 'masked_val' => 7, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 88, + 'name' => 'HASHREF_8', + 'masked_val' => 8, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 89, + 'name' => 'HASHREF_9', + 'masked_val' => 9, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 90, + 'name' => 'HASHREF_10', + 'masked_val' => 10, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 91, + 'name' => 'HASHREF_11', + 'masked_val' => 11, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 92, + 'name' => 'HASHREF_12', + 'masked_val' => 12, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 93, + 'name' => 'HASHREF_13', + 'masked_val' => 13, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 94, + 'name' => 'HASHREF_14', + 'masked_val' => 14, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 95, + 'name' => 'HASHREF_15', + 'masked_val' => 15, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'comment' => ' - binary/latin1 string, length encoded in low 5 bits of tag', + 'value' => 96, + 'name' => 'SHORT_BINARY_0', + 'masked_val' => 0, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 97, + 'name' => 'SHORT_BINARY_1', + 'masked_val' => 1, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 98, + 'name' => 'SHORT_BINARY_2', + 'masked_val' => 2, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 99, + 'name' => 'SHORT_BINARY_3', + 'masked_val' => 3, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 100, + 'name' => 'SHORT_BINARY_4', + 'masked_val' => 4, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 101, + 'name' => 'SHORT_BINARY_5', + 'masked_val' => 5, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 102, + 'name' => 'SHORT_BINARY_6', + 'masked_val' => 6, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 103, + 'name' => 'SHORT_BINARY_7', + 'masked_val' => 7, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 104, + 'name' => 'SHORT_BINARY_8', + 'masked_val' => 8, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 105, + 'name' => 'SHORT_BINARY_9', + 'masked_val' => 9, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 106, + 'name' => 'SHORT_BINARY_10', + 'masked_val' => 10, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 107, + 'name' => 'SHORT_BINARY_11', + 'masked_val' => 11, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 108, + 'name' => 'SHORT_BINARY_12', + 'masked_val' => 12, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 109, + 'name' => 'SHORT_BINARY_13', + 'masked_val' => 13, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 110, + 'name' => 'SHORT_BINARY_14', + 'masked_val' => 14, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 111, + 'name' => 'SHORT_BINARY_15', + 'masked_val' => 15, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 112, + 'name' => 'SHORT_BINARY_16', + 'masked_val' => 16, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 113, + 'name' => 'SHORT_BINARY_17', + 'masked_val' => 17, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 114, + 'name' => 'SHORT_BINARY_18', + 'masked_val' => 18, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 115, + 'name' => 'SHORT_BINARY_19', + 'masked_val' => 19, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 116, + 'name' => 'SHORT_BINARY_20', + 'masked_val' => 20, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 117, + 'name' => 'SHORT_BINARY_21', + 'masked_val' => 21, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 118, + 'name' => 'SHORT_BINARY_22', + 'masked_val' => 22, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 119, + 'name' => 'SHORT_BINARY_23', + 'masked_val' => 23, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 120, + 'name' => 'SHORT_BINARY_24', + 'masked_val' => 24, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 121, + 'name' => 'SHORT_BINARY_25', + 'masked_val' => 25, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 122, + 'name' => 'SHORT_BINARY_26', + 'masked_val' => 26, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 123, + 'name' => 'SHORT_BINARY_27', + 'masked_val' => 27, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 124, + 'name' => 'SHORT_BINARY_28', + 'masked_val' => 28, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 125, + 'name' => 'SHORT_BINARY_29', + 'masked_val' => 29, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 126, + 'name' => 'SHORT_BINARY_30', + 'masked_val' => 30, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 127, + 'name' => 'SHORT_BINARY_31', + 'masked_val' => 31, + 'type_value' => 96 + } +); +$TAG_INFO_HASH{chr $_}= $TAG_INFO_ARRAY[$_] for 0 .. 127; +push @EXPORT_OK, qw(%TAG_INFO_HASH @TAG_INFO_ARRAY); + +# stop autoupdated section - do not modify directly! + + our %EXPORT_TAGS=(all => \@EXPORT_OK); HERE close $ofh; diff --git a/lib/Sereal/Encoder.pm b/lib/Sereal/Encoder.pm index 0a377d5..36f7e6d 100644 --- a/lib/Sereal/Encoder.pm +++ b/lib/Sereal/Encoder.pm @@ -5,7 +5,7 @@ use warnings; use Carp qw/croak/; use XSLoader; -our $VERSION = '3.001_005'; # Don't forget to update the TestCompat set for testing against installed decoders! +our $VERSION = '3.001_006'; # Don't forget to update the TestCompat set for testing against installed decoders! our $XS_VERSION = $VERSION; $VERSION= eval $VERSION; # not for public consumption, just for testing. @@ -236,6 +236,12 @@ do so. Do note that the setting is somewhat approximate. Setting it to 10000 may break at somewhere between 9997 and 10003 nested structures depending on their types. +=head3 canonical_refs + +Normally C will ARRAYREF and HASHREF tags when the item contains +less than 16 items, and and is not referenced more than once. This flag will +override this optimization and use a standard REFN ARRAY style tag output. + =head3 sort_keys Normally C will output hashes in whatever order is convenient, @@ -490,6 +496,10 @@ the issues well enough for you to decide if it is suitable for your needs. This can be enabled via C, see above. +=item Sereal output is sensitive to refcounts + +This can be somewhat mitigated by the use of C, see above. + =item There are multiple valid Sereal documents that you can produce for the same Perl data structure. Just L is not enough. A trivial example is PAD bytes which diff --git a/lib/Sereal/Encoder/Constants.pm b/lib/Sereal/Encoder/Constants.pm index 7379889..307dd92 100644 --- a/lib/Sereal/Encoder/Constants.pm +++ b/lib/Sereal/Encoder/Constants.pm @@ -90,7 +90,1158 @@ BEGIN { @EXPORT_OK = qw( }; } -sub SRL_MAGIC_STRING () {"=srl"} -sub SRL_MAGIC_STRING_HIGHBIT () {"=\xF3rl"} -push @EXPORT_OK, qw(SRL_MAGIC_STRING SRL_MAGIC_STRING_HIGHBIT); +sub SRL_MAGIC_STRING () { "=srl" } +sub SRL_MAGIC_STRING_HIGHBIT () { "=\xF3rl" } +sub SRL_MAGIC_STRING_HIGHBIT_UTF8 () { "=\xC3\xB3rl" } + +push @EXPORT_OK, qw( + SRL_MAGIC_STRING + SRL_MAGIC_STRING_HIGHBIT + SRL_MAGIC_STRING_HIGHBIT_UTF8 +); + +# start autoupdated section - do not modify directly + +our (%TAG_INFO_HASH, @TAG_INFO_ARRAY); +@TAG_INFO_ARRAY = ( + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'comment' => 'small positive integer - value in low 4 bits (identity)', + 'value' => 0, + 'name' => 'POS_0', + 'masked_val' => 0, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 1, + 'name' => 'POS_1', + 'masked_val' => 1, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 2, + 'name' => 'POS_2', + 'masked_val' => 2, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 3, + 'name' => 'POS_3', + 'masked_val' => 3, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 4, + 'name' => 'POS_4', + 'masked_val' => 4, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 5, + 'name' => 'POS_5', + 'masked_val' => 5, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 6, + 'name' => 'POS_6', + 'masked_val' => 6, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 7, + 'name' => 'POS_7', + 'masked_val' => 7, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 8, + 'name' => 'POS_8', + 'masked_val' => 8, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 9, + 'name' => 'POS_9', + 'masked_val' => 9, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 10, + 'name' => 'POS_10', + 'masked_val' => 10, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 11, + 'name' => 'POS_11', + 'masked_val' => 11, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 12, + 'name' => 'POS_12', + 'masked_val' => 12, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 13, + 'name' => 'POS_13', + 'masked_val' => 13, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'value' => 14, + 'name' => 'POS_14', + 'masked_val' => 14, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'POS', + 'masked' => 1, + 'comment' => 'small positive integer - value in low 4 bits (identity)', + 'value' => 15, + 'name' => 'POS_15', + 'masked_val' => 15, + 'type_value' => 0 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'comment' => 'small negative integer - value in low 4 bits (k+32)', + 'value' => 16, + 'name' => 'NEG_16', + 'masked_val' => 16, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 17, + 'name' => 'NEG_15', + 'masked_val' => 15, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 18, + 'name' => 'NEG_14', + 'masked_val' => 14, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 19, + 'name' => 'NEG_13', + 'masked_val' => 13, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 20, + 'name' => 'NEG_12', + 'masked_val' => 12, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 21, + 'name' => 'NEG_11', + 'masked_val' => 11, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 22, + 'name' => 'NEG_10', + 'masked_val' => 10, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 23, + 'name' => 'NEG_9', + 'masked_val' => 9, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 24, + 'name' => 'NEG_8', + 'masked_val' => 8, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 25, + 'name' => 'NEG_7', + 'masked_val' => 7, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 26, + 'name' => 'NEG_6', + 'masked_val' => 6, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 27, + 'name' => 'NEG_5', + 'masked_val' => 5, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 28, + 'name' => 'NEG_4', + 'masked_val' => 4, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 29, + 'name' => 'NEG_3', + 'masked_val' => 3, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'value' => 30, + 'name' => 'NEG_2', + 'masked_val' => 2, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'NEG', + 'masked' => 1, + 'comment' => 'small negative integer - value in low 4 bits (k+32)', + 'value' => 31, + 'name' => 'NEG_1', + 'masked_val' => 1, + 'type_value' => 16 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'VARINT', + 'comment' => ' - Varint variable length integer', + 'value' => 32, + 'name' => 'VARINT', + 'type_value' => 32 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ZIGZAG', + 'comment' => ' - Zigzag variable length integer', + 'value' => 33, + 'name' => 'ZIGZAG', + 'type_value' => 33 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'FLOAT', + 'comment' => '', + 'value' => 34, + 'name' => 'FLOAT', + 'type_value' => 34 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'DOUBLE', + 'comment' => '', + 'value' => 35, + 'name' => 'DOUBLE', + 'type_value' => 35 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'LONG_DOUBLE', + 'comment' => '', + 'value' => 36, + 'name' => 'LONG_DOUBLE', + 'type_value' => 36 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'UNDEF', + 'comment' => 'None - Perl undef var; eg my $var= undef;', + 'value' => 37, + 'name' => 'UNDEF', + 'type_value' => 37 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'BINARY', + 'comment' => ' - binary/(latin1) string', + 'value' => 38, + 'name' => 'BINARY', + 'type_value' => 38 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'STR_UTF8', + 'comment' => ' - utf8 string', + 'value' => 39, + 'name' => 'STR_UTF8', + 'type_value' => 39 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'REFN', + 'comment' => ' - ref to next item', + 'value' => 40, + 'name' => 'REFN', + 'type_value' => 40 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'REFP', + 'comment' => ' - ref to previous item stored at offset', + 'value' => 41, + 'name' => 'REFP', + 'type_value' => 41 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASH', + 'comment' => ' [ ...] - count followed by key/value pairs', + 'value' => 42, + 'name' => 'HASH', + 'type_value' => 42 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAY', + 'comment' => ' [ ...] - count followed by items', + 'value' => 43, + 'name' => 'ARRAY', + 'type_value' => 43 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'OBJECT', + 'comment' => ' - class, object-item', + 'value' => 44, + 'name' => 'OBJECT', + 'type_value' => 44 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'OBJECTV', + 'comment' => ' - offset of previously used classname tag - object-item', + 'value' => 45, + 'name' => 'OBJECTV', + 'type_value' => 45 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ALIAS', + 'comment' => ' - alias to item defined at offset', + 'value' => 46, + 'name' => 'ALIAS', + 'type_value' => 46 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'COPY', + 'comment' => ' - copy of item defined at offset', + 'value' => 47, + 'name' => 'COPY', + 'type_value' => 47 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'WEAKEN', + 'comment' => ' - Weaken the following reference', + 'value' => 48, + 'name' => 'WEAKEN', + 'type_value' => 48 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'REGEXP', + 'comment' => ' ', + 'value' => 49, + 'name' => 'REGEXP', + 'type_value' => 49 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'OBJECT_FREEZE', + 'comment' => ' - class, object-item. Need to call "THAW" method on class after decoding', + 'value' => 50, + 'name' => 'OBJECT_FREEZE', + 'type_value' => 50 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'OBJECTV_FREEZE', + 'comment' => ' - (OBJECTV_FREEZE is to OBJECT_FREEZE as OBJECTV is to OBJECT)', + 'value' => 51, + 'name' => 'OBJECTV_FREEZE', + 'type_value' => 51 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'RESERVED', + 'masked' => 1, + 'comment' => 'reserved', + 'value' => 52, + 'name' => 'RESERVED_0', + 'masked_val' => 0, + 'type_value' => 52 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'RESERVED', + 'masked' => 1, + 'value' => 53, + 'name' => 'RESERVED_1', + 'masked_val' => 1, + 'type_value' => 52 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'RESERVED', + 'masked' => 1, + 'value' => 54, + 'name' => 'RESERVED_2', + 'masked_val' => 2, + 'type_value' => 52 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'RESERVED', + 'masked' => 1, + 'value' => 55, + 'name' => 'RESERVED_3', + 'masked_val' => 3, + 'type_value' => 52 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'RESERVED', + 'masked' => 1, + 'value' => 56, + 'name' => 'RESERVED_4', + 'masked_val' => 4, + 'type_value' => 52 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'CANONICAL_UNDEF', + 'comment' => 'undef (PL_sv_undef) - "the" Perl undef (see notes)', + 'value' => 57, + 'name' => 'CANONICAL_UNDEF', + 'type_value' => 57 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'FALSE', + 'comment' => 'false (PL_sv_no)', + 'value' => 58, + 'name' => 'FALSE', + 'type_value' => 58 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'TRUE', + 'comment' => 'true (PL_sv_yes)', + 'value' => 59, + 'name' => 'TRUE', + 'type_value' => 59 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'MANY', + 'comment' => ' - repeated tag (not done yet, will be implemented in version 3)', + 'value' => 60, + 'name' => 'MANY', + 'type_value' => 60 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'PACKET_START', + 'comment' => '(first byte of magic string in header)', + 'value' => 61, + 'name' => 'PACKET_START', + 'type_value' => 61 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'EXTEND', + 'comment' => ' - for additional tags', + 'value' => 62, + 'name' => 'EXTEND', + 'type_value' => 62 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'PAD', + 'comment' => '(ignored tag, skip to next byte)', + 'value' => 63, + 'name' => 'PAD', + 'type_value' => 63 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'comment' => '[ ...] - count of items in low 4 bits (ARRAY must be refcnt=1)', + 'value' => 64, + 'name' => 'ARRAYREF_0', + 'masked_val' => 0, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 65, + 'name' => 'ARRAYREF_1', + 'masked_val' => 1, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 66, + 'name' => 'ARRAYREF_2', + 'masked_val' => 2, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 67, + 'name' => 'ARRAYREF_3', + 'masked_val' => 3, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 68, + 'name' => 'ARRAYREF_4', + 'masked_val' => 4, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 69, + 'name' => 'ARRAYREF_5', + 'masked_val' => 5, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 70, + 'name' => 'ARRAYREF_6', + 'masked_val' => 6, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 71, + 'name' => 'ARRAYREF_7', + 'masked_val' => 7, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 72, + 'name' => 'ARRAYREF_8', + 'masked_val' => 8, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 73, + 'name' => 'ARRAYREF_9', + 'masked_val' => 9, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 74, + 'name' => 'ARRAYREF_10', + 'masked_val' => 10, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 75, + 'name' => 'ARRAYREF_11', + 'masked_val' => 11, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 76, + 'name' => 'ARRAYREF_12', + 'masked_val' => 12, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 77, + 'name' => 'ARRAYREF_13', + 'masked_val' => 13, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 78, + 'name' => 'ARRAYREF_14', + 'masked_val' => 14, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'ARRAYREF', + 'masked' => 1, + 'value' => 79, + 'name' => 'ARRAYREF_15', + 'masked_val' => 15, + 'type_value' => 64 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'comment' => '[ ...] - count in low 4 bits, key/value pairs (HASH must be refcnt=1)', + 'value' => 80, + 'name' => 'HASHREF_0', + 'masked_val' => 0, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 81, + 'name' => 'HASHREF_1', + 'masked_val' => 1, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 82, + 'name' => 'HASHREF_2', + 'masked_val' => 2, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 83, + 'name' => 'HASHREF_3', + 'masked_val' => 3, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 84, + 'name' => 'HASHREF_4', + 'masked_val' => 4, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 85, + 'name' => 'HASHREF_5', + 'masked_val' => 5, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 86, + 'name' => 'HASHREF_6', + 'masked_val' => 6, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 87, + 'name' => 'HASHREF_7', + 'masked_val' => 7, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 88, + 'name' => 'HASHREF_8', + 'masked_val' => 8, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 89, + 'name' => 'HASHREF_9', + 'masked_val' => 9, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 90, + 'name' => 'HASHREF_10', + 'masked_val' => 10, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 91, + 'name' => 'HASHREF_11', + 'masked_val' => 11, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 92, + 'name' => 'HASHREF_12', + 'masked_val' => 12, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 93, + 'name' => 'HASHREF_13', + 'masked_val' => 13, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 94, + 'name' => 'HASHREF_14', + 'masked_val' => 14, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'HASHREF', + 'masked' => 1, + 'value' => 95, + 'name' => 'HASHREF_15', + 'masked_val' => 15, + 'type_value' => 80 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'comment' => ' - binary/latin1 string, length encoded in low 5 bits of tag', + 'value' => 96, + 'name' => 'SHORT_BINARY_0', + 'masked_val' => 0, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 97, + 'name' => 'SHORT_BINARY_1', + 'masked_val' => 1, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 98, + 'name' => 'SHORT_BINARY_2', + 'masked_val' => 2, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 99, + 'name' => 'SHORT_BINARY_3', + 'masked_val' => 3, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 100, + 'name' => 'SHORT_BINARY_4', + 'masked_val' => 4, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 101, + 'name' => 'SHORT_BINARY_5', + 'masked_val' => 5, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 102, + 'name' => 'SHORT_BINARY_6', + 'masked_val' => 6, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 103, + 'name' => 'SHORT_BINARY_7', + 'masked_val' => 7, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 104, + 'name' => 'SHORT_BINARY_8', + 'masked_val' => 8, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 105, + 'name' => 'SHORT_BINARY_9', + 'masked_val' => 9, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 106, + 'name' => 'SHORT_BINARY_10', + 'masked_val' => 10, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 107, + 'name' => 'SHORT_BINARY_11', + 'masked_val' => 11, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 108, + 'name' => 'SHORT_BINARY_12', + 'masked_val' => 12, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 109, + 'name' => 'SHORT_BINARY_13', + 'masked_val' => 13, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 110, + 'name' => 'SHORT_BINARY_14', + 'masked_val' => 14, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 111, + 'name' => 'SHORT_BINARY_15', + 'masked_val' => 15, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 112, + 'name' => 'SHORT_BINARY_16', + 'masked_val' => 16, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 113, + 'name' => 'SHORT_BINARY_17', + 'masked_val' => 17, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 114, + 'name' => 'SHORT_BINARY_18', + 'masked_val' => 18, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 115, + 'name' => 'SHORT_BINARY_19', + 'masked_val' => 19, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 116, + 'name' => 'SHORT_BINARY_20', + 'masked_val' => 20, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 117, + 'name' => 'SHORT_BINARY_21', + 'masked_val' => 21, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 118, + 'name' => 'SHORT_BINARY_22', + 'masked_val' => 22, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 119, + 'name' => 'SHORT_BINARY_23', + 'masked_val' => 23, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 120, + 'name' => 'SHORT_BINARY_24', + 'masked_val' => 24, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 121, + 'name' => 'SHORT_BINARY_25', + 'masked_val' => 25, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 122, + 'name' => 'SHORT_BINARY_26', + 'masked_val' => 26, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 123, + 'name' => 'SHORT_BINARY_27', + 'masked_val' => 27, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 124, + 'name' => 'SHORT_BINARY_28', + 'masked_val' => 28, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 125, + 'name' => 'SHORT_BINARY_29', + 'masked_val' => 29, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 126, + 'name' => 'SHORT_BINARY_30', + 'masked_val' => 30, + 'type_value' => 96 + }, + # autoupdated by author_tools/update_from_header.pl do not modify directly! + { + 'type_name' => 'SHORT_BINARY', + 'masked' => 1, + 'value' => 127, + 'name' => 'SHORT_BINARY_31', + 'masked_val' => 31, + 'type_value' => 96 + } +); +$TAG_INFO_HASH{chr $_}= $TAG_INFO_ARRAY[$_] for 0 .. 127; +push @EXPORT_OK, qw(%TAG_INFO_HASH @TAG_INFO_ARRAY); + +# stop autoupdated section - do not modify directly! + + our %EXPORT_TAGS=(all => \@EXPORT_OK); diff --git a/srl_encoder.c b/srl_encoder.c index 3c319d3..3b7d9a5 100644 --- a/srl_encoder.c +++ b/srl_encoder.c @@ -472,9 +472,17 @@ srl_build_encoder_struct(pTHX_ HV *opt) } svp = hv_fetchs(opt, "sort_keys", 0); + if ( !svp ) + svp = hv_fetchs(opt, "canonical",0); if ( svp && SvTRUE(*svp) ) SRL_ENC_SET_OPTION(enc, SRL_F_SORT_KEYS); + svp = hv_fetchs(opt, "canonical_refs", 0); + if ( !svp ) + svp = hv_fetchs(opt, "canonical",0); + if ( svp && SvTRUE(*svp) ) + SRL_ENC_SET_OPTION(enc, SRL_F_CANONICAL_REFS); + svp = hv_fetchs(opt, "aliased_dedupe_strings", 0); if ( svp && SvTRUE(*svp) ) SRL_ENC_SET_OPTION(enc, SRL_F_ALIASED_DEDUPE_STRINGS | SRL_F_DEDUPE_STRINGS); @@ -725,7 +733,7 @@ srl_dump_ivuv(pTHX_ srl_encoder_t *enc, SV *src) /* FIXME find a way to express the condition without repeated SvIV/SvUV */ if (expect_true( SvIOK_UV(src) || SvIV(src) >= 0 )) { const UV num = SvUV(src); /* FIXME is SvUV_nomg good enough because of the GET magic in dump_sv? SvUVX after having checked the flags? */ - if (num < 16) { + if (num <= 15) { /* encodable as POS */ hdr = SRL_HDR_POS_LOW | (unsigned char)num; srl_buf_cat_char(enc, hdr); @@ -736,7 +744,7 @@ srl_dump_ivuv(pTHX_ srl_encoder_t *enc, SV *src) } else { const IV num = SvIV(src); - if (num > -17) { + if (num >= -16) { /* encodable as NEG */ hdr = SRL_HDR_NEG_LOW | ((unsigned char)num + 32); srl_buf_cat_char(enc, hdr); @@ -1161,7 +1169,7 @@ srl_dump_av(pTHX_ srl_encoder_t *enc, AV *src, U32 refcount) /* heuristic: n is virtually the min. size of any element */ BUF_SIZE_ASSERT(enc, 2 + SRL_MAX_VARINT_LENGTH + n); - if (n < 16 && refcount == 1) { + if (n < 16 && refcount == 1 && !SRL_ENC_HAVE_OPTION(enc,SRL_F_CANONICAL_REFS)) { enc->buf.pos--; /* backup over previous REFN */ srl_buf_cat_char_nocheck(enc, SRL_HDR_ARRAYREF + n); } else { @@ -1236,7 +1244,7 @@ srl_dump_hv(pTHX_ srl_encoder_t *enc, HV *src, U32 refcount) * + 2*n = very conservative min size of n hashkeys if all COPY */ BUF_SIZE_ASSERT(enc, 2 + SRL_MAX_VARINT_LENGTH + 3*n); - if (n < 16 && refcount == 1) { + if (n < 16 && refcount == 1 && !SRL_ENC_HAVE_OPTION(enc,SRL_F_CANONICAL_REFS)) { enc->buf.pos--; /* back up over the previous REFN */ srl_buf_cat_char_nocheck(enc, SRL_HDR_HASHREF + n); } else { @@ -1302,7 +1310,7 @@ srl_dump_hv(pTHX_ srl_encoder_t *enc, HV *src, U32 refcount) /* heuristic: n = ~min size of n values; * + 2*n = very conservative min size of n hashkeys if all COPY */ BUF_SIZE_ASSERT(enc, 2 + SRL_MAX_VARINT_LENGTH + 3*n); - if (n < 16 && refcount == 1) { + if (n < 16 && refcount == 1 && !SRL_ENC_HAVE_OPTION(enc,SRL_F_CANONICAL_REFS)) { enc->buf.pos--; /* backup over the previous REFN */ srl_buf_cat_char_nocheck(enc, SRL_HDR_HASHREF + n); } else { @@ -1402,7 +1410,7 @@ srl_dump_svpv(pTHX_ srl_encoder_t *enc, SV *src) if (SvIOK(ofs_sv)) { /* emit copy or alias */ if (out_tag == SRL_HDR_ALIAS) - SRL_SET_FBIT(*(enc->buf.body_pos + SvUV(ofs_sv))); + SRL_SET_TRACK_FLAG(*(enc->buf.body_pos + SvUV(ofs_sv))); srl_buf_cat_varint(aTHX_ enc, out_tag, SvIV(ofs_sv)); return; } else if (SvUOK(ofs_sv)) { @@ -1531,7 +1539,7 @@ redo_dump: if (DEBUGHACK) warn("alias to %p as %lu", src, (long unsigned int)oldoffset); srl_buf_cat_varint(aTHX_ enc, SRL_HDR_ALIAS, (UV)oldoffset); } - SRL_SET_FBIT(*(enc->buf.body_pos + oldoffset)); + SRL_SET_TRACK_FLAG(*(enc->buf.body_pos + oldoffset)); --enc->recursion_depth; return; } diff --git a/srl_encoder.h b/srl_encoder.h index 448f54f..6297ec1 100644 --- a/srl_encoder.h +++ b/srl_encoder.h @@ -59,57 +59,63 @@ SV *srl_dump_data_structure_mortal_sv(pTHX_ srl_encoder_t *enc, SV *src, SV *use /* Will default to "on". If set, hash keys will be shared using COPY. * Corresponds to the inverse of constructor option "no_shared_hashkeys" */ -#define SRL_F_SHARED_HASHKEYS 0x00001UL +#define SRL_F_SHARED_HASHKEYS 0x00001UL /* If set, then we're using the OO interface and we shouldn't destroy the * encoder struct during SAVEDESTRUCTOR_X time */ -#define SRL_F_REUSE_ENCODER 0x00002UL +#define SRL_F_REUSE_ENCODER 0x00002UL /* If set in flags, then we rather croak than serialize an object. * Corresponds to the 'croak_on_bless' option to the Perl constructor. */ -#define SRL_F_CROAK_ON_BLESS 0x00004UL +#define SRL_F_CROAK_ON_BLESS 0x00004UL /* If set in flags, then we will emit for all data types * that aren't supported. Corresponds to the 'undef_unknown' option. */ -#define SRL_F_UNDEF_UNKNOWN 0x00008UL +#define SRL_F_UNDEF_UNKNOWN 0x00008UL /* If set in flags, then we will stringify (SvPV) all data types * that aren't supported. Corresponds to the 'stringify_unknown' option. */ -#define SRL_F_STRINGIFY_UNKNOWN 0x00010UL +#define SRL_F_STRINGIFY_UNKNOWN 0x00010UL /* If set in flags, then we warn() when trying to serialize an unsupported * data structure. Applies only if stringify_unknown or undef_unknown are * set since we otherwise croak. Corresponds to the 'warn_unknown' option. */ -#define SRL_F_WARN_UNKNOWN 0x00020UL +#define SRL_F_WARN_UNKNOWN 0x00020UL /* WARNING: This is different from the protocol bit SRL_PROTOCOL_ENCODING_SNAPPY in that it's * a flag on the encoder struct indicating that we want to use Snappy. */ -#define SRL_F_COMPRESS_SNAPPY 0x00040UL -#define SRL_F_COMPRESS_SNAPPY_INCREMENTAL 0x00080UL +#define SRL_F_COMPRESS_SNAPPY 0x00040UL +#define SRL_F_COMPRESS_SNAPPY_INCREMENTAL 0x00080UL /* WARNING: This is different from the protocol bit SRL_PROTOCOL_ENCODING_ZLIB in that it's * a flag on the encoder struct indicating that we want to use ZLIB. */ -#define SRL_F_COMPRESS_ZLIB 0x00100UL +#define SRL_F_COMPRESS_ZLIB 0x00100UL /* Only meaningful if SRL_F_WARN_UNKNOWN also set. If this one is set, then we don't warn * if the unsupported item has string overloading. */ -#define SRL_F_NOWARN_UNKNOWN_OVERLOAD 0x00200UL +#define SRL_F_NOWARN_UNKNOWN_OVERLOAD 0x00200UL /* Only meaningful if SRL_F_WARN_UNKNOWN also set. If this one is set, then we don't warn * if the unsupported item has string overloading. */ -#define SRL_F_SORT_KEYS 0x00400UL +#define SRL_F_SORT_KEYS 0x00400UL /* If set, use a hash to emit COPY() tags for all duplicated strings * (slow, but great compression) */ -#define SRL_F_DEDUPE_STRINGS 0x00800UL +#define SRL_F_DEDUPE_STRINGS 0x00800UL /* Like SRL_F_DEDUPE_STRINGS but emits ALIAS() instead of COPY() for * non-class-name, non-hash-key strings that are deduped. If set, * supersedes SRL_F_DEDUPE_STRINGS. */ -#define SRL_F_ALIASED_DEDUPE_STRINGS 0x01000UL +#define SRL_F_ALIASED_DEDUPE_STRINGS 0x01000UL /* If set in flags, then we serialize objects without class information. * Corresponds to the 'no_bless_objects' flag found in the Decoder. */ -#define SRL_F_NO_BLESS_OBJECTS 0x02000UL +#define SRL_F_NO_BLESS_OBJECTS 0x02000UL /* If set in flags, then support calling FREEZE method on objects. */ -#define SRL_F_ENABLE_FREEZE_SUPPORT 0x04000UL +#define SRL_F_ENABLE_FREEZE_SUPPORT 0x04000UL +/* if set in flags, then do not use ARRAYREF or HASHREF ever */ +#define SRL_F_CANONICAL_REFS 0x08000UL + +/* ==================================================================== + * oper flags + */ /* Set while the encoder is in active use / dirty */ #define SRL_OF_ENCODER_DIRTY 1UL diff --git a/srl_protocol.h b/srl_protocol.h index 23815df..a297766 100644 --- a/srl_protocol.h +++ b/srl_protocol.h @@ -246,6 +246,6 @@ /* TODO */ -#define SRL_SET_FBIT(where) ((where) |= SRL_HDR_TRACK_FLAG) +#define SRL_SET_TRACK_FLAG(where) ((where) |= SRL_HDR_TRACK_FLAG) #endif diff --git a/t/002_testset.t b/t/002_testset.t new file mode 100644 index 0000000..25b8fa6 --- /dev/null +++ b/t/002_testset.t @@ -0,0 +1,32 @@ +#!perl +use strict; +use warnings; +use Sereal::Decoder; +use Data::Dumper; +use File::Spec; + +# test our test framework + +use lib File::Spec->catdir(qw(t lib)); +BEGIN { + lib->import('lib') + if !-d 't'; +} + +use Sereal::TestSet qw(:all); +use Test::More; +# needs more tests +ok(_deep_cmp(["x"],{})); +ok(_deep_cmp({"x"=>1},{"y"=>1})); +ok(_deep_cmp({"x"=>1},{"x"=>2})); +ok(_deep_cmp({"x"=>1},{"x"=>2,"y"=>1})); +ok(!_deep_cmp({"x"=>1},{"x"=>1})); +ok(!_deep_cmp(["x"],["x"])); +ok(_deep_cmp(["x"],["y","p"])); +ok(_deep_cmp(["a","x"],["y"])); +ok(_test_str("test","foo","bar")); +ok(!_test_str("test","aaa","aaa")); + +pass(); +done_testing(); + diff --git a/t/700_roundtrip/v1/plain.t b/t/700_roundtrip/v1/plain.t index 282424e..49f7e03 100644 --- a/t/700_roundtrip/v1/plain.t +++ b/t/700_roundtrip/v1/plain.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v1/plain_canon.t b/t/700_roundtrip/v1/plain_canon.t new file mode 100644 index 0000000..5f95299 --- /dev/null +++ b/t/700_roundtrip/v1/plain_canon.t @@ -0,0 +1,28 @@ +#!perl +use strict; +use warnings; +use Sereal::Decoder; +use Data::Dumper; +use File::Spec; + +use lib File::Spec->catdir(qw(t lib)); +BEGIN { + lib->import('lib') + if !-d 't'; +} + +use Sereal::TestSet qw(:all); +use Test::More; + +my $ok = have_encoder_and_decoder(); +$ok= 0 if $Sereal::Encoder::VERSION < 3.001006; +if (not $ok) { + plan skip_all => 'Did not find right version of encoder'; +} +else { + run_roundtrip_tests("plain_canon",{canonical => 1}); +} + +pass(); +done_testing(); + diff --git a/t/700_roundtrip/v1/snappy.t b/t/700_roundtrip/v1/snappy.t index 5e8beb9..a9735e0 100644 --- a/t/700_roundtrip/v1/snappy.t +++ b/t/700_roundtrip/v1/snappy.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v1/snappy_canon.t b/t/700_roundtrip/v1/snappy_canon.t new file mode 100644 index 0000000..9b8deb5 --- /dev/null +++ b/t/700_roundtrip/v1/snappy_canon.t @@ -0,0 +1,29 @@ +#!perl +use strict; +use warnings; +use Sereal::Decoder; +use Data::Dumper; +use File::Spec; + +use lib File::Spec->catdir(qw(t lib)); +BEGIN { + lib->import('lib') + if !-d 't'; +} + +use Sereal::TestSet qw(:all); +use Test::More; + +my $ok = have_encoder_and_decoder(); +$ok= 0 if $Sereal::Encoder::VERSION < 3.001006; +if (not $ok) { + plan skip_all => 'Did not find right version of encoder'; +} +else { + run_roundtrip_tests('snappy_canon', { snappy => 1, canonical => 1 } ); +} + + +pass(); +done_testing(); + diff --git a/t/700_roundtrip/v2/dedudep_strings.t b/t/700_roundtrip/v2/dedudep_strings.t index 83ebd4c..41473a6 100644 --- a/t/700_roundtrip/v2/dedudep_strings.t +++ b/t/700_roundtrip/v2/dedudep_strings.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v2/freeze_thaw.t b/t/700_roundtrip/v2/freeze_thaw.t index e41cb9e..35913aa 100644 --- a/t/700_roundtrip/v2/freeze_thaw.t +++ b/t/700_roundtrip/v2/freeze_thaw.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v2/plain.t b/t/700_roundtrip/v2/plain.t index 282424e..49f7e03 100644 --- a/t/700_roundtrip/v2/plain.t +++ b/t/700_roundtrip/v2/plain.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v2/plain_canon.t b/t/700_roundtrip/v2/plain_canon.t new file mode 100644 index 0000000..5f95299 --- /dev/null +++ b/t/700_roundtrip/v2/plain_canon.t @@ -0,0 +1,28 @@ +#!perl +use strict; +use warnings; +use Sereal::Decoder; +use Data::Dumper; +use File::Spec; + +use lib File::Spec->catdir(qw(t lib)); +BEGIN { + lib->import('lib') + if !-d 't'; +} + +use Sereal::TestSet qw(:all); +use Test::More; + +my $ok = have_encoder_and_decoder(); +$ok= 0 if $Sereal::Encoder::VERSION < 3.001006; +if (not $ok) { + plan skip_all => 'Did not find right version of encoder'; +} +else { + run_roundtrip_tests("plain_canon",{canonical => 1}); +} + +pass(); +done_testing(); + diff --git a/t/700_roundtrip/v2/readonly.t b/t/700_roundtrip/v2/readonly.t index 4f0b1d6..08a2916 100644 --- a/t/700_roundtrip/v2/readonly.t +++ b/t/700_roundtrip/v2/readonly.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v2/snappy.t b/t/700_roundtrip/v2/snappy.t index 5e8beb9..a9735e0 100644 --- a/t/700_roundtrip/v2/snappy.t +++ b/t/700_roundtrip/v2/snappy.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v2/snappy_canon.t b/t/700_roundtrip/v2/snappy_canon.t new file mode 100644 index 0000000..9b8deb5 --- /dev/null +++ b/t/700_roundtrip/v2/snappy_canon.t @@ -0,0 +1,29 @@ +#!perl +use strict; +use warnings; +use Sereal::Decoder; +use Data::Dumper; +use File::Spec; + +use lib File::Spec->catdir(qw(t lib)); +BEGIN { + lib->import('lib') + if !-d 't'; +} + +use Sereal::TestSet qw(:all); +use Test::More; + +my $ok = have_encoder_and_decoder(); +$ok= 0 if $Sereal::Encoder::VERSION < 3.001006; +if (not $ok) { + plan skip_all => 'Did not find right version of encoder'; +} +else { + run_roundtrip_tests('snappy_canon', { snappy => 1, canonical => 1 } ); +} + + +pass(); +done_testing(); + diff --git a/t/700_roundtrip/v2/snappy_incr.t b/t/700_roundtrip/v2/snappy_incr.t index 9dfb91f..ddd4bb9 100644 --- a/t/700_roundtrip/v2/snappy_incr.t +++ b/t/700_roundtrip/v2/snappy_incr.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v2/snappy_incr_canon.t b/t/700_roundtrip/v2/snappy_incr_canon.t new file mode 100644 index 0000000..dd1d622 --- /dev/null +++ b/t/700_roundtrip/v2/snappy_incr_canon.t @@ -0,0 +1,31 @@ +#!perl +use strict; +use warnings; +use Sereal::Decoder; +use Data::Dumper; +use File::Spec; + +use lib File::Spec->catdir(qw(t lib)); +BEGIN { + lib->import('lib') + if !-d 't'; +} + +use Sereal::TestSet qw(:all); +use Test::More; + +my $ok = have_encoder_and_decoder(); +$ok= 0 if $Sereal::Encoder::VERSION < 3.001006; +if (not $ok) { + plan skip_all => 'Did not find right version of encoder'; +} +else { + run_roundtrip_tests( + 'snappy_incr_canon', { snappy_incr => 1, canonical => 1 } + ); +} + + +pass(); +done_testing(); + diff --git a/t/700_roundtrip/v2/sort_keys.t b/t/700_roundtrip/v2/sort_keys.t index 57aa81b..61f5439 100644 --- a/t/700_roundtrip/v2/sort_keys.t +++ b/t/700_roundtrip/v2/sort_keys.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v3/dedudep_strings.t b/t/700_roundtrip/v3/dedudep_strings.t index 83ebd4c..41473a6 100644 --- a/t/700_roundtrip/v3/dedudep_strings.t +++ b/t/700_roundtrip/v3/dedudep_strings.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v3/freeze_thaw.t b/t/700_roundtrip/v3/freeze_thaw.t index e41cb9e..35913aa 100644 --- a/t/700_roundtrip/v3/freeze_thaw.t +++ b/t/700_roundtrip/v3/freeze_thaw.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v3/plain.t b/t/700_roundtrip/v3/plain.t index 282424e..49f7e03 100644 --- a/t/700_roundtrip/v3/plain.t +++ b/t/700_roundtrip/v3/plain.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v3/plain_canon.t b/t/700_roundtrip/v3/plain_canon.t new file mode 100644 index 0000000..5f95299 --- /dev/null +++ b/t/700_roundtrip/v3/plain_canon.t @@ -0,0 +1,28 @@ +#!perl +use strict; +use warnings; +use Sereal::Decoder; +use Data::Dumper; +use File::Spec; + +use lib File::Spec->catdir(qw(t lib)); +BEGIN { + lib->import('lib') + if !-d 't'; +} + +use Sereal::TestSet qw(:all); +use Test::More; + +my $ok = have_encoder_and_decoder(); +$ok= 0 if $Sereal::Encoder::VERSION < 3.001006; +if (not $ok) { + plan skip_all => 'Did not find right version of encoder'; +} +else { + run_roundtrip_tests("plain_canon",{canonical => 1}); +} + +pass(); +done_testing(); + diff --git a/t/700_roundtrip/v3/readonly.t b/t/700_roundtrip/v3/readonly.t index 4f0b1d6..08a2916 100644 --- a/t/700_roundtrip/v3/readonly.t +++ b/t/700_roundtrip/v3/readonly.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v3/snappy.t b/t/700_roundtrip/v3/snappy.t index 5e8beb9..a9735e0 100644 --- a/t/700_roundtrip/v3/snappy.t +++ b/t/700_roundtrip/v3/snappy.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v3/snappy_canon.t b/t/700_roundtrip/v3/snappy_canon.t new file mode 100644 index 0000000..9b8deb5 --- /dev/null +++ b/t/700_roundtrip/v3/snappy_canon.t @@ -0,0 +1,29 @@ +#!perl +use strict; +use warnings; +use Sereal::Decoder; +use Data::Dumper; +use File::Spec; + +use lib File::Spec->catdir(qw(t lib)); +BEGIN { + lib->import('lib') + if !-d 't'; +} + +use Sereal::TestSet qw(:all); +use Test::More; + +my $ok = have_encoder_and_decoder(); +$ok= 0 if $Sereal::Encoder::VERSION < 3.001006; +if (not $ok) { + plan skip_all => 'Did not find right version of encoder'; +} +else { + run_roundtrip_tests('snappy_canon', { snappy => 1, canonical => 1 } ); +} + + +pass(); +done_testing(); + diff --git a/t/700_roundtrip/v3/snappy_incr.t b/t/700_roundtrip/v3/snappy_incr.t index 9dfb91f..ddd4bb9 100644 --- a/t/700_roundtrip/v3/snappy_incr.t +++ b/t/700_roundtrip/v3/snappy_incr.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v3/snappy_incr_canon.t b/t/700_roundtrip/v3/snappy_incr_canon.t new file mode 100644 index 0000000..dd1d622 --- /dev/null +++ b/t/700_roundtrip/v3/snappy_incr_canon.t @@ -0,0 +1,31 @@ +#!perl +use strict; +use warnings; +use Sereal::Decoder; +use Data::Dumper; +use File::Spec; + +use lib File::Spec->catdir(qw(t lib)); +BEGIN { + lib->import('lib') + if !-d 't'; +} + +use Sereal::TestSet qw(:all); +use Test::More; + +my $ok = have_encoder_and_decoder(); +$ok= 0 if $Sereal::Encoder::VERSION < 3.001006; +if (not $ok) { + plan skip_all => 'Did not find right version of encoder'; +} +else { + run_roundtrip_tests( + 'snappy_incr_canon', { snappy_incr => 1, canonical => 1 } + ); +} + + +pass(); +done_testing(); + diff --git a/t/700_roundtrip/v3/sort_keys.t b/t/700_roundtrip/v3/sort_keys.t index 57aa81b..61f5439 100644 --- a/t/700_roundtrip/v3/sort_keys.t +++ b/t/700_roundtrip/v3/sort_keys.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v3/zlib.t b/t/700_roundtrip/v3/zlib.t index 6dffeb7..b541a46 100644 --- a/t/700_roundtrip/v3/zlib.t +++ b/t/700_roundtrip/v3/zlib.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/700_roundtrip/v3/zlib_force.t b/t/700_roundtrip/v3/zlib_force.t index 6556aa8..3764001 100644 --- a/t/700_roundtrip/v3/zlib_force.t +++ b/t/700_roundtrip/v3/zlib_force.t @@ -5,16 +5,6 @@ use Sereal::Decoder; use Data::Dumper; use File::Spec; -# These tests use an installed Decoder (or respectively Encoder) to do -# round-trip testing. There are two strategies, both with drawbacks: -# - Test::More's is_deeply is waaaay too lenient to catch all the -# subtleties that Sereal is supposed to encode. -# - Serialize - Deserialize - Serialize, then do a string compare. -# This won't catch if the first serialization has bogus output -# but the subsequent de- & serialization work for the already -# bogus output. -# These tests can't replace carefully crafted manual tests, I fear. - use lib File::Spec->catdir(qw(t lib)); BEGIN { lib->import('lib') diff --git a/t/lib/Sereal/TestSet.pm b/t/lib/Sereal/TestSet.pm index 283bd4a..726ac71 100644 --- a/t/lib/Sereal/TestSet.pm +++ b/t/lib/Sereal/TestSet.pm @@ -10,7 +10,8 @@ use Test::More; use Test::LongString; #use Data::Dumper; # MUST BE LOADED *AFTER* THIS FILE (BUG IN PERL) use Devel::Peek; -use Encode qw(encode_utf8); +use Encode qw(encode_utf8 is_utf8); +use Scalar::Util qw(reftype blessed refaddr); # Dynamically load constants from whatever is being tested our ($Class, $ConstClass); @@ -36,7 +37,7 @@ our @ISA = qw(Exporter); our @EXPORT_OK = qw( @BasicTests $Class $ConstClass Header - FBIT + TRACK_FLAG hobodecode integer short_string varint array array_fbit hash dump_bless @@ -45,12 +46,15 @@ our @EXPORT_OK = qw( write_test_files $use_objectv setup_tests + _deep_cmp + _test + _test_str ); our %EXPORT_TAGS = (all => \@EXPORT_OK); our $use_objectv = 1; -use constant FBIT => 128; +use constant TRACK_FLAG => 128; sub hobodecode { return unless defined $_[0]; @@ -72,7 +76,7 @@ sub array { sub array_fbit { chr(SRL_HDR_REFN). - chr(SRL_HDR_ARRAY+FBIT) . varint(0+@_) . join("", @_) + chr(SRL_HDR_ARRAY+TRACK_FLAG) . varint(0+@_) . join("", @_) } sub hash_head { @@ -318,7 +322,7 @@ sub setup_tests { [ $weak_thing, chr(SRL_HDR_REFN) - . chr(SRL_HDR_ARRAY + FBIT) . varint(2) + . chr(SRL_HDR_ARRAY + TRACK_FLAG) . varint(2) . chr(SRL_HDR_PAD) . chr(SRL_HDR_REFN) . chr(SRL_HDR_REFP) . varint(offseti(1)) . chr(0b0000_0001) @@ -328,7 +332,7 @@ sub setup_tests { [ \$weak_thing, chr(SRL_HDR_REFN) - . chr(SRL_HDR_REFN + FBIT) + . chr(SRL_HDR_REFN + TRACK_FLAG) . chr(SRL_HDR_ARRAY) . varint(2) .chr(SRL_HDR_WEAKEN) . chr(SRL_HDR_REFP) . varint(offseti(1)) .chr(0b0000_0001) @@ -337,7 +341,7 @@ sub setup_tests { ], sub { \@_ } ->( $weak_thing, - chr(SRL_HDR_REFN + FBIT) + chr(SRL_HDR_REFN + TRACK_FLAG) .chr(SRL_HDR_ARRAY).varint(2) .chr(SRL_HDR_WEAKEN).chr(SRL_HDR_REFP).varint(offseti(0)) .chr(0b0000_0001) @@ -350,8 +354,8 @@ sub setup_tests { my $content= array_head(2); my $pos= offset($content); $content - . chr(SRL_HDR_REFN + FBIT) - . chr(SRL_HDR_REFP + FBIT) + . chr(SRL_HDR_REFN + TRACK_FLAG) + . chr(SRL_HDR_REFP + TRACK_FLAG) . varint( $pos ) . chr(SRL_HDR_ALIAS) . varint($pos + 1) @@ -364,9 +368,9 @@ sub setup_tests { my $content= array_head(2); my $pos= offset($content); $content - . chr(SRL_HDR_WEAKEN + FBIT) + . chr(SRL_HDR_WEAKEN + TRACK_FLAG) . chr(SRL_HDR_REFN) - . chr(SRL_HDR_WEAKEN + FBIT) + . chr(SRL_HDR_WEAKEN + TRACK_FLAG) . chr(SRL_HDR_REFP) . varint($pos) . chr(SRL_HDR_ALIAS) @@ -388,7 +392,7 @@ sub setup_tests { chr(SRL_HDR_OBJECT), short_string("bar"), chr(SRL_HDR_REFN), - chr(SRL_HDR_REGEXP + FBIT), + chr(SRL_HDR_REGEXP + TRACK_FLAG), short_string("foo"), short_string("ix"), chr(SRL_HDR_REFP), @@ -404,10 +408,10 @@ sub setup_tests { my $pos= offset($content); join("",$content, short_string("foo"), - chr(SRL_HDR_REFN).chr(SRL_HDR_ARRAY + FBIT),varint(0), + chr(SRL_HDR_REFN).chr(SRL_HDR_ARRAY + TRACK_FLAG),varint(0), chr( SRL_HDR_OBJECT + $use_objectv), $use_objectv ? () : chr(SRL_HDR_COPY), varint($pos), - chr(SRL_HDR_REFN).chr(SRL_HDR_ARRAY + FBIT), varint(0), + chr(SRL_HDR_REFN).chr(SRL_HDR_ARRAY + TRACK_FLAG), varint(0), chr(SRL_HDR_REFP),varint($pos + 5), chr(SRL_HDR_REFP),varint($pos + 10), ) @@ -536,6 +540,7 @@ sub get_git_top_dir { } sub have_encoder_and_decoder { + my ($min_v)= @_; # $Class is the already-loaded class, so the one we're testing my $need = $Class =~ /Encoder/ ? "Decoder" : "Encoder"; my $need_class = "Sereal::$need"; @@ -555,6 +560,11 @@ sub have_encoder_and_decoder { return(); }; my $cmp_v = $need_class->VERSION; + if ($min_v and $cmp_v <= $min_v) { + note("Could not load correct version of $need_class for testing " + ."(got: $cmp_v, needed at least $min_v)"); + return; + } $cmp_v =~ s/_//; $cmp_v = sprintf("%.2f", int($cmp_v*100)/100); if (not defined $cmp_v or not exists $compat_versions{$cmp_v}) { @@ -562,7 +572,6 @@ sub have_encoder_and_decoder { ."(got: $cmp_v, needed any of ".join(", ", keys %compat_versions).")"); return(); } - return 1; } @@ -596,7 +605,9 @@ our @ScalarRoundtripTests = ( ["small int", 3], ["small negative int", -8], ["largeish int", 100000], - ["largeish negative int", -302001], + ["largeish negative int -302001", -302001], + ["largeish negative int -1234567", -1234567], + ["largeish negative int -12345678", -12345678], ( map {["integer: $_", $_]} ( @@ -618,9 +629,29 @@ our @ScalarRoundtripTests = ( ["float", 0.2], ["short ascii string", "fooo"], ["short latin1 string", "Müller"], - ["short utf8 string", do {use utf8; " עדיין ח"}], + ["short utf8 string", do {use utf8; " עדיין ח"} ], + + (map { [ "long ascii string 'a' x $_", do{"a" x $_} ] } ( + 9999,10000,10001, + 1023,1024,1025, + 8191,8192,8193, + )), + (map { [ "long ascii string 'ab' x $_", do{"ab" x $_} ] } ( + 9999,10000,10001, + 1023,1024,1025, + 8191,8192,8193, + )), + (map { [ "long ascii string 'abc' x $_", do{"abc" x $_} ] } ( + 9999,10000,10001, + 1023,1024,1025, + 8191,8192,8193, + )), + (map { [ "long ascii string 'abcd' x $_", do{"abcd" x $_} ] } ( + 9999,10000,10001, + 1023,1024,1025, + 8191,8192,8193, + )), - ["long ascii string", do{"abc" x 10000}], ["long latin1 string", "üll" x 10000], ["long utf8 string", do {use utf8; " עדיין חשב" x 10000}], ["long utf8 string with only ascii", do {use utf8; "foo" x 10000}], @@ -690,6 +721,10 @@ our @RoundtripTests = ( (map {["hash ref to " . $_->[0], ({foo => $_->[1]})]} @ScalarRoundtripTests), # --- (map {["array ref to duplicate " . $_->[0], ([$_->[1], $_->[1]])]} @ScalarRoundtripTests), + (map {[ + "AoA of duplicates " . $_->[0], + ( [ $_->[1], [ $_->[1], $_->[1] ], $_->[1], [ $_->[1], $_->[1], $_->[1] ], $_->[1] ] ) + ]} @ScalarRoundtripTests), # --- (map {["array ref to aliases " . $_->[0], (sub {\@_}->($_->[1], $_->[1]))]} @ScalarRoundtripTests), (map {["array ref to scalar refs to same " . $_->[0], ([\($_->[1]), \($_->[1])])]} @ScalarRoundtripTests), @@ -731,34 +766,156 @@ sub run_roundtrip_tests { run_roundtrip_tests_internal($name . $suffix, $opts); } +sub _test { + my ($msg, $v1, $v2)= @_; + if ($v1 ne $v2) { + my $q1= Data::Dumper::qquote($v1); + my $q2= Data::Dumper::qquote($v2); + return "msg: $q1 ne $q2" + } + return; +} +sub _test_str { + my ($msg, $v1, $v2)= @_; + if (is_utf8($v1) != is_utf8($v2)) { + return "$msg: utf8 flag mismatch"; + } + if ($v1 eq $v2) { + return; + } + my $diff_start= 0; + $diff_start++ while $diff_start < length($v1) + and $diff_start < length($v2) + and substr($v1, $diff_start,1) eq substr($v2, $diff_start,1); + my $diff_end= $diff_start; + $diff_end++ while $diff_end < length($v1) + and $diff_end < length($v2) + and substr($v1, $diff_end,1) ne substr($v2, $diff_end,1); + my $length_to_show= $diff_end - $diff_start; + $length_to_show= 30 if $length_to_show > 30; + + my $q1= Data::Dumper::qquote(substr($v1, $diff_start, $length_to_show )); + my $q2= Data::Dumper::qquote(substr($v2, $diff_start, $length_to_show )); + my $context_start= $diff_start > 10 ? $diff_start - 10 : 0; + + if ($context_start < $diff_start) { + $q1 = Data::Dumper::qquote(substr($v1,$context_start,10)) . " . " . $q1; + $q2 = Data::Dumper::qquote(substr($v2,$context_start,10)) . " . " . $q2; + } + if ($context_start > 0) { + $q1 = "...$q1"; + $q2 = "...$q2"; + } + if ($length_to_show < 30) { + $q1 .= " . " . Data::Dumper::qquote(substr($v1, $diff_start + $length_to_show, 30-$length_to_show)); + $q2 .= " . " . Data::Dumper::qquote(substr($v2, $diff_start + $length_to_show, 30-$length_to_show)); + } + if ( $diff_start + 30 < length($v1) ) { + $q1 .= "..." + } + if ( $diff_start + 30 < length($v2) ) { + $q2 .= "..." + } + return ($msg, sprintf("%s at offset %d\nv1 = %s (length %d)\nv2 = %s (length %d)\n", + $msg, $diff_start, $q1, length($v1), $q2, length($v2))); +} + +sub _deep_cmp { + my ($x, $y, $seenx, $seeny)= @_; + $seenx||={}; + $seeny||={}; + my $cmp; + + $cmp= _test("defined mismatch",defined($x),defined($y)) + and return $cmp; + defined($x) + or return ""; + $cmp= _test("seen scalar ", ++$seenx->{refaddr \$_[0]}, ++$seeny->{refaddr \$_[1]}) + || _test("boolean mismatch",!!$x, !!$y) + || _test("isref mismatch",!!ref($x), !!ref($y)) + and return $cmp; + + if (ref $x) { + $cmp= _test("seen ref", ++$seenx->{refaddr $x}, ++$seeny->{refaddr $y}) + || _test("reftype mismatch",reftype($x), reftype($y)) + || _test("class mismatch", !blessed($x), !blessed($y)) + || _test("class different", blessed($x)//"", blessed($y)//"") + and return $cmp; + return "" if $x == $y + or $seenx->{refaddr $x} > 1; + + if (reftype($x) eq "HASH") { + $cmp= _test("keycount mismatch",0+keys(%$x),0+keys(%$y)) + and return $cmp; + foreach my $key (keys %$x) { + return "key missing '$key'" unless exists $y->{$key}; + $cmp= _deep_cmp($x->{$key},$y->{$key}, $seenx, $seeny) + and return $cmp; + } + } elsif (reftype($x) eq "ARRAY") { + $cmp= _test("arraysize mismatch",0+@$x,0+@$y) + and return $cmp; + foreach my $idx (0..$#$x) { + $cmp= _deep_cmp($x->[$idx], $y->[$idx], $seenx, $seeny) + and return $cmp; + } + } elsif (reftype($x) eq "SCALAR" or reftype($x) eq "REF") { + return _deep_cmp($$x, $$y, $seenx, $seeny); + } elsif (reftype($x) eq "REGEXP") { + $cmp= _test("regexp different","$x","$y") + and return $cmp; + } else { + die "Unknown reftype '",reftype($x)."'"; + } + } else { + $cmp= _test_str("strings differ",$x,$y) + and return $cmp; + } + return "" +} + +sub deep_cmp { + my ($v1, $v2, $name)= @_; + my $diff= _deep_cmp($v1, $v2); + if ($diff) { + my ($reason,$diag)= split /\n/, $diff, 2; + fail("$name - $reason"); + diag("$reason\n$diag") if $diag; + return; + } + return 1; +} + + sub run_roundtrip_tests_internal { my ($ename, $opt, $encode_decode_callbacks) = @_; my $decoder = Sereal::Decoder->new($opt); my $encoder = Sereal::Encoder->new($opt); - - foreach my $meth ( - ['functional simple', - sub {Sereal::Encoder::encode_sereal($_[0], $opt)}, - sub {Sereal::Decoder::decode_sereal($_[0], $opt)}], - ['object-oriented', - sub {$encoder->encode($_[0])}, - sub {$decoder->decode($_[0])}], - ['functional with object', - sub {Sereal::Encoder::sereal_encode_with_object($encoder, $_[0])}, - sub {Sereal::Decoder::sereal_decode_with_object($decoder, $_[0])}], - ['header-body', - sub {$encoder->encode($_[0], 123456789)}, # header data is abitrary to stand out for debugging - sub {$decoder->decode($_[0])}], - ['header-only', - sub {$encoder->encode(987654321, $_[0])}, # body data is abitrary to stand out for debugging - sub {$decoder->decode_only_header($_[0])}], - ) - { - my ($mname, $enc, $dec) = @$meth; - next if $mname =~ /header/ and $opt->{use_protocol_v1}; - - foreach my $rt (@RoundtripTests) { - my ($name, $data) = @$rt; + my %seen_name; + + foreach my $rt (@RoundtripTests) { + my ($name, $data) = @$rt; + + foreach my $meth ( + ['object-oriented', + sub {$encoder->encode($_[0])}, + sub {$decoder->decode($_[0])}], + ['functional simple', + sub {Sereal::Encoder::encode_sereal($_[0], $opt)}, + sub {Sereal::Decoder::decode_sereal($_[0], $opt)}], + ['functional with object', + sub {Sereal::Encoder::sereal_encode_with_object($encoder, $_[0])}, + sub {Sereal::Decoder::sereal_decode_with_object($decoder, $_[0])}], + ['header-body', + sub {$encoder->encode($_[0], 123456789)}, # header data is abitrary to stand out for debugging + sub {$decoder->decode($_[0])}], + ['header-only', + sub {$encoder->encode(987654321, $_[0])}, # body data is abitrary to stand out for debugging + sub {$decoder->decode_only_header($_[0])}], + ) { + my ($mname, $enc, $dec) = @$meth; + + next if $mname =~ /header/ and $opt->{use_protocol_v1}; my $encoded; eval {$encoded = $enc->($data); 1} @@ -766,21 +923,26 @@ sub run_roundtrip_tests_internal { my $err = $@ || 'Zombie error'; diag("Got error while encoding: $err"); }; - ok(defined $encoded, "$name ($ename, $mname, encoded defined)") + + defined($encoded) or do { + fail("$name ($ename, $mname, encoded defined)"); debug_checks(\$data, \$encoded, undef); - next; + last; }; + my $decoded; eval {$decoded = $dec->($encoded); 1} or do { my $err = $@ || 'Zombie error'; diag("Got error while decoding: $err"); }; - ok( defined($decoded) == defined($data), "$name ($ename, $mname, decoded definedness)") + + defined($decoded) == defined($data) or do { + fail("$name ($ename, $mname, decoded definedness)"); debug_checks(\$data, \$encoded, undef); - next; + last; }; # Second roundtrip @@ -790,10 +952,12 @@ sub run_roundtrip_tests_internal { my $err = $@ || 'Zombie error'; diag("Got error while encoding the second time: $err"); }; - ok(defined $encoded2, "$name ($ename, $mname, encoded2 defined)") + + defined $encoded2 or do { + fail("$name ($ename, $mname, encoded2 defined)"); debug_checks(\$data, \$encoded, \$decoded); - next; + last; }; my $decoded2; @@ -803,42 +967,81 @@ sub run_roundtrip_tests_internal { diag("Got error while encoding the second time: $err"); }; - ok(defined($decoded2) == defined($data), "$name ($ename, $mname, decoded2 defined)") - or next; - is_deeply($decoded, $data, "$name ($ename, $mname, decoded vs data)") + defined($decoded2) == defined($data) or do { - debug_checks(\$data, undef, \$decoded, "debug"); + fail("$name ($ename, $mname, decoded2 defined)"); + last; }; - is_deeply($decoded2, $data, "$name ($ename, $mname, decoded2 vs data)") + + # Third roundtrip + my $encoded3; + eval {$encoded3 = $enc->($decoded2); 1} or do { - debug_checks(\$data, undef, \$decoded2, "debug"); + my $err = $@ || 'Zombie error'; + diag("Got error while encoding the third time: $err"); }; - is_deeply($decoded2, $decoded, "$name ($ename, $mname, decoded vs decoded2)") + + defined $encoded3 or do { - debug_checks(\$decoded, undef, \$decoded2, "debug"); + fail("$name ($ename, $mname, encoded3 defined)"); + debug_checks(\$data, \$encoded, \$decoded); + last; + }; + + my $decoded3; + eval {$decoded3 = $dec->($encoded3); 1} + or do { + my $err = $@ || 'Zombie error'; + diag("Got error while encoding the third time: $err"); }; - if (0) { - # It isnt really safe to test this way right now. The exact output - # of two runs of Sereal is not guaranteed to be the same due to the effect of - # refcounts. We could disable ARRAYREF/HASHREF as an option, - # and then skip these tests. We should probably do that just to test - # that we can handle both representations properly at all times. - my $ret; - if ($name=~/complex/) { - SKIP: { - skip "Encoded string length tests for complex hashes and compression depends on hash key ordering", 1 if $opt->{snappy}; - $ret = is(length($encoded2), length($encoded),"$name ($ename, $mname, length encoded2 vs length encoded)"); + defined($decoded3) == defined($data) + or do { + fail("$name ($ename, $mname, decoded3 defined)"); + last; + }; + + deep_cmp($decoded, $data, "$name ($ename, $mname, decoded vs data)") or last; + deep_cmp($decoded2, $data, "$name ($ename, $mname, decoded2 vs data)") or last; + deep_cmp($decoded2, $decoded, "$name ($ename, $mname, decoded2 vs decoded)") or last; + + deep_cmp($decoded3, $data, "$name ($ename, $mname, decoded3 vs data)") or last; + deep_cmp($decoded3, $decoded, "$name ($ename, $mname, decoded3 vs decoded)") or last; + deep_cmp($decoded3, $decoded2, "$name ($ename, $mname, decoded3 vs decoded2)") or last; + + if ( $ename =~ /canon/) { + deep_cmp($encoded2, $encoded, "$name ($ename, $mname, encoded2 vs encoded)") or last; + deep_cmp($encoded3, $encoded, "$name ($ename, $mname, encoded3 vs encoded)") or last; + deep_cmp($encoded3, $encoded2, "$name ($ename, $mname, encoded3 vs encoded2)") or last; + + if ($ENV{SEREAL_TEST_SAVE_OUTPUT} and $mname eq 'object-oriented') { + use File::Path; + my $combined_name= "$ename - $name"; + if (!$seen_name{$combined_name}) { + my @clean= ($ename, $name); + s/[^\w.-]+/_/g, s/__+/_/g for @clean; + my $cleaned= join "/", @clean; + my $dir= $0; + $dir=~s!/[^/]+\z!/data/$clean[0]!; + mkpath $dir unless -d $dir; + my $base= "$dir/$clean[1].enc"; + $seen_name{$combined_name}= $base; + for my $f ( [ "", $encoded ], $encoded ne $encoded2 ? [ "2", $encoded2 ] : ()) { + my $file= $base . $f->[0]; + next if -e $file; + open my $fh, ">", $file + or die "Can't open '$file' for writing: $!"; + binmode($fh); + print $fh $f->[1]; + close $fh; + } + diag "Wrote sample files for '$combined_name' to $base"; } - } else { - $ret = is_string($encoded2, $encoded, "$name ($ename, $mname, encoded2 vs encoded)"); } - $ret or do { - debug_checks(\$data, \$encoded, \$decoded); - }; } - } - } # end serialization method iteration + } # end method type + pass("$name ($ename)"); + } # end test type }