almost working parser (unicode broken)

3 files changed, 432 insertions, 0 deletions

diff --git a/ext/ffi_yajl/ext/parser/Makefile b/ext/ffi_yajl/ext/parser/Makefile
new file mode 100644
index 0000000..840cc31
--- /dev/null
+++ b/ext/ffi_yajl/ext/parser/Makefile
@@ -0,0 +1,238 @@
+
+SHELL = /bin/sh
+
+# V=0 quiet, V=1 verbose.  other values don't work.
+V = 0
+Q1 = $(V:1=)
+Q = $(Q1:0=@)
+ECHO1 = $(V:1=@:)
+ECHO = $(ECHO1:0=@echo)
+
+#### Start of system configuration section. ####
+
+srcdir = .
+topdir = /Users/lamont/.rvm/rubies/ruby-2.0.0-p247/include/ruby-2.0.0
+hdrdir = $(topdir)
+arch_hdrdir = /Users/lamont/.rvm/rubies/ruby-2.0.0-p247/include/ruby-2.0.0/x86_64-darwin12.4.0
+PATH_SEPARATOR = :
+VPATH = $(srcdir):$(arch_hdrdir)/ruby:$(hdrdir)/ruby
+prefix = /Users/lamont/.rvm/rubies/ruby-2.0.0-p247
+rubysitearchprefix = $(rubylibprefix)/$(sitearch)
+rubyarchprefix = $(rubylibprefix)/$(arch)
+rubylibprefix = $(libdir)/$(RUBY_BASE_NAME)
+exec_prefix = $(prefix)
+vendorarchhdrdir = $(vendorhdrdir)/$(sitearch)
+sitearchhdrdir = $(sitehdrdir)/$(sitearch)
+rubyarchhdrdir = $(rubyhdrdir)/$(arch)
+vendorhdrdir = $(rubyhdrdir)/vendor_ruby
+sitehdrdir = $(rubyhdrdir)/site_ruby
+rubyhdrdir = $(includedir)/$(RUBY_VERSION_NAME)
+vendorarchdir = $(vendorlibdir)/$(sitearch)
+vendorlibdir = $(vendordir)/$(ruby_version)
+vendordir = $(rubylibprefix)/vendor_ruby
+sitearchdir = $(sitelibdir)/$(sitearch)
+sitelibdir = $(sitedir)/$(ruby_version)
+sitedir = $(rubylibprefix)/site_ruby
+rubyarchdir = $(rubylibdir)/$(arch)
+rubylibdir = $(rubylibprefix)/$(ruby_version)
+sitearchincludedir = $(includedir)/$(sitearch)
+archincludedir = $(includedir)/$(arch)
+sitearchlibdir = $(libdir)/$(sitearch)
+archlibdir = $(libdir)/$(arch)
+ridir = $(datarootdir)/$(RI_BASE_NAME)
+mandir = $(datarootdir)/man
+localedir = $(datarootdir)/locale
+libdir = $(exec_prefix)/lib
+psdir = $(docdir)
+pdfdir = $(docdir)
+dvidir = $(docdir)
+htmldir = $(docdir)
+infodir = $(datarootdir)/info
+docdir = $(datarootdir)/doc/$(PACKAGE)
+oldincludedir = /usr/include
+includedir = $(prefix)/include
+localstatedir = $(prefix)/var
+sharedstatedir = $(prefix)/com
+sysconfdir = $(prefix)/etc
+datadir = $(datarootdir)
+datarootdir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+sbindir = $(exec_prefix)/sbin
+bindir = $(exec_prefix)/bin
+archdir = $(rubyarchdir)
+
+
+CC = /usr/bin/clang
+CXX = clang++
+LIBRUBY = $(LIBRUBY_SO)
+LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
+LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME)
+LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static
+empty =
+OUTFLAG = -o $(empty)
+COUTFLAG = -o $(empty)
+
+RUBY_EXTCONF_H = 
+cflags   =  $(optflags) $(debugflags) $(warnflags)
+optflags = -O3 -fno-fast-math
+debugflags = -ggdb3
+warnflags = -Wall -Wextra -Wno-unused-parameter -Wno-parentheses -Wno-long-long -Wno-missing-field-initializers -Wunused-variable -Wpointer-arith -Wwrite-strings -Wdeclaration-after-statement -Wshorten-64-to-32 -Wimplicit-function-declaration
+CCDLFLAGS = -fno-common
+CFLAGS   = $(CCDLFLAGS) -I/Users/lamont/git/ffi-yajl/include -L/Users/lamont/git/ffi-yajl/lib -O3 -march=nocona -O2 -pipe -fno-common -Wall $(ARCH_FLAG)
+INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir)/ruby/backward -I$(hdrdir) -I$(srcdir)
+DEFS     = 
+CPPFLAGS =  -D_XOPEN_SOURCE -D_DARWIN_C_SOURCE -D_DARWIN_UNLIMITED_SELECT -D_REENTRANT -I/usr/local/opt/libyaml/include -I/usr/local/opt/readline/include -I/usr/local/opt/libksba/include -I/usr/local/opt/openssl/include $(DEFS) $(cppflags)
+CXXFLAGS = $(CCDLFLAGS) $(cxxflags) $(ARCH_FLAG)
+ldflags  = -L/Users/lamont/git/ffi-yajl/lib -I/Users/lamont/git/ffi-yajl/include -L/Users/lamont/git/ffi-yajl/lib -O3 -march=nocona -O2 -pipe -fno-common -lyajl
+dldflags = -Wl,-undefined,dynamic_lookup -Wl,-multiply_defined,suppress -L/usr/local/opt/libyaml/lib -L/usr/local/opt/readline/lib -L/usr/local/opt/libksba/lib -L/usr/local/opt/openssl/lib  
+ARCH_FLAG = 
+DLDFLAGS = $(ldflags) $(dldflags) $(ARCH_FLAG)
+LDSHARED = $(CC) -dynamic -bundle
+LDSHAREDXX = $(CXX) -dynamic -bundle
+AR = ar
+EXEEXT = 
+
+RUBY_INSTALL_NAME = ruby
+RUBY_SO_NAME = ruby.2.0.0
+RUBYW_INSTALL_NAME = 
+RUBY_VERSION_NAME = $(RUBY_BASE_NAME)-$(ruby_version)
+RUBYW_BASE_NAME = rubyw
+RUBY_BASE_NAME = ruby
+
+arch = x86_64-darwin12.4.0
+sitearch = $(arch)
+ruby_version = 2.0.0
+ruby = $(bindir)/ruby
+RUBY = $(ruby)
+ruby_headers = $(hdrdir)/ruby.h $(hdrdir)/ruby/defines.h $(arch_hdrdir)/ruby/config.h
+
+RM = rm -f
+RM_RF = $(RUBY) -run -e rm -- -rf
+RMDIRS = rmdir -p
+MAKEDIRS = mkdir -p
+INSTALL = /usr/bin/install -c
+INSTALL_PROG = $(INSTALL) -m 0755
+INSTALL_DATA = $(INSTALL) -m 644
+COPY = cp
+TOUCH = exit >
+
+#### End of system configuration section. ####
+
+preload = 
+
+libpath = . $(libdir) /usr/local/opt/libyaml/lib /usr/local/opt/readline/lib /usr/local/opt/libksba/lib /usr/local/opt/openssl/lib
+LIBPATH =  -L. -L$(libdir) -L/usr/local/opt/libyaml/lib -L/usr/local/opt/readline/lib -L/usr/local/opt/libksba/lib -L/usr/local/opt/openssl/lib
+DEFFILE = 
+
+CLEANFILES = mkmf.log
+DISTCLEANFILES = 
+DISTCLEANDIRS = 
+
+extout = 
+extout_prefix = 
+target_prefix = /ffi_yajl/ext
+LOCAL_LIBS = 
+LIBS = $(LIBRUBYARG_SHARED)  -lpthread -ldl -lobjc 
+ORIG_SRCS = parser.c
+SRCS = $(ORIG_SRCS) 
+OBJS = parser.o
+HDRS = 
+TARGET = parser
+TARGET_NAME = parser
+TARGET_ENTRY = Init_$(TARGET_NAME)
+DLLIB = $(TARGET).bundle
+EXTSTATIC = 
+STATIC_LIB = 
+
+BINDIR        = $(DESTDIR)$(bindir)
+RUBYCOMMONDIR = $(DESTDIR)$(sitedir)$(target_prefix)
+RUBYLIBDIR    = $(DESTDIR)$(sitelibdir)$(target_prefix)
+RUBYARCHDIR   = $(DESTDIR)$(sitearchdir)$(target_prefix)
+HDRDIR        = $(DESTDIR)$(rubyhdrdir)/ruby$(target_prefix)
+ARCHHDRDIR    = $(DESTDIR)$(rubyhdrdir)/$(arch)/ruby$(target_prefix)
+
+TARGET_SO     = $(DLLIB)
+CLEANLIBS     = $(TARGET).bundle 
+CLEANOBJS     = *.o  *.bak
+
+all:    $(DLLIB)
+static: $(STATIC_LIB)
+.PHONY: all install static install-so install-rb
+.PHONY: clean clean-so clean-static clean-rb
+
+clean-static::
+clean-rb-default::
+clean-rb::
+clean-so::
+clean: clean-so clean-static clean-rb-default clean-rb
+		-$(Q)$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) .*.time
+
+distclean-rb-default::
+distclean-rb::
+distclean-so::
+distclean-static::
+distclean: clean distclean-so distclean-static distclean-rb-default distclean-rb
+		-$(Q)$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
+		-$(Q)$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
+		-$(Q)$(RMDIRS) $(DISTCLEANDIRS) 2> /dev/null || true
+
+realclean: distclean
+install: install-so install-rb
+
+install-so: $(DLLIB) ./.RUBYARCHDIR.time
+	$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
+clean-static::
+	-$(Q)$(RM) $(STATIC_LIB)
+install-rb: pre-install-rb install-rb-default
+install-rb-default: pre-install-rb-default
+pre-install-rb: Makefile
+pre-install-rb-default: Makefile
+pre-install-rb-default:
+	$(ECHO) installing default parser libraries
+./.RUBYARCHDIR.time:
+	$(Q) $(MAKEDIRS) $(RUBYARCHDIR)
+	$(Q) $(TOUCH) $@
+
+site-install: site-install-so site-install-rb
+site-install-so: install-so
+site-install-rb: install-rb
+
+.SUFFIXES: .c .m .cc .mm .cxx .cpp .C .o
+
+.cc.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $<
+
+.mm.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $<
+
+.cxx.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $<
+
+.cpp.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $<
+
+.C.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $<
+
+.c.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $<
+
+.m.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $<
+
+$(DLLIB): $(OBJS) Makefile
+	$(ECHO) linking shared-object ffi_yajl/ext/$(DLLIB)
+	-$(Q)$(RM) $(@)
+	$(Q) $(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
+	$(Q) test -z '$(RUBY_CODESIGN)' || codesign -s '$(RUBY_CODESIGN)' -f $@
+
+
+
+$(OBJS): $(HDRS) $(ruby_headers)
diff --git a/ext/ffi_yajl/ext/parser/extconf.rb b/ext/ffi_yajl/ext/parser/extconf.rb
new file mode 100644
index 0000000..e78e7a2
--- /dev/null
+++ b/ext/ffi_yajl/ext/parser/extconf.rb
@@ -0,0 +1,29 @@
+require 'mkmf'
+
+# the customer is always right, ruby is always compiled to be stupid
+$CFLAGS = ENV['CFLAGS'] if ENV['CFLAGS']
+$LDFLAGS = ENV['LDFLAGS'] if ENV['LDFLAGS']
+RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC'] if ENV['CC']
+
+# search our gem root first to pick up libyajl2 that we vendored
+gem_root = File.expand_path(File.join(File.dirname(__FILE__), "../../../.."))
+$CFLAGS = "-I#{gem_root}/include -L#{gem_root}/lib #{$CFLAGS}"
+$LDFLAGS = "-L#{gem_root}/lib #{$CFLAGS}"
+
+puts gem_root
+puts $CFLAGS
+puts $LDFLAGS
+
+# except if you're doing an unoptimized gcc install we're going to help you out a bit
+if RbConfig::MAKEFILE_CONFIG['CC'] =~ /gcc|clang/
+  $CFLAGS << " -O3" unless $CFLAGS[/-O\d/]
+  # how many people realize that -Wall is a compiler-specific flag???
+  # apparently not many based on reading lots of shitty extconf.rb's out there
+  $CFLAGS << " -Wall"
+end
+
+$LDFLAGS << " -lyajl"
+
+dir_config 'parser'
+
+create_makefile 'ffi_yajl/ext/parser'
diff --git a/ext/ffi_yajl/ext/parser/parser.c b/ext/ffi_yajl/ext/parser/parser.c
new file mode 100644
index 0000000..7793a20
--- /dev/null
+++ b/ext/ffi_yajl/ext/parser/parser.c
@@ -0,0 +1,165 @@
+#include <ruby.h>
+#include <yajl/yajl_parse.h>
+
+static VALUE mFFI_Yajl, mExt, mParser, cParseError;
+
+typedef struct {
+  VALUE finished;
+  VALUE stack;
+  VALUE key_stack;
+  VALUE key;
+} CTX;
+
+void set_value(CTX *ctx, VALUE val) {
+  long len = RARRAY_LEN(ctx->stack);
+  VALUE last = rb_ary_entry(ctx->stack, len-1);
+  switch (TYPE(last)) {
+    case T_ARRAY:
+      rb_ary_push(last, val);
+      break;
+    case T_HASH:
+      rb_hash_aset(last, ctx->key, val);
+      break;
+    default:
+      break;
+  }
+}
+
+void set_key(CTX *ctx, VALUE key) {
+  ctx->key = key;
+}
+
+void start_object(CTX *ctx, VALUE obj) {
+  rb_ary_push(ctx->key_stack, ctx->key);
+  rb_ary_push(ctx->stack, obj);
+}
+
+void end_object(CTX *ctx) {
+  ctx->key = rb_ary_pop(ctx->key_stack);
+  if ( RARRAY_LEN(ctx->stack) > 1 ) {
+    set_value(ctx, rb_ary_pop(ctx->stack));
+  } else {
+    ctx->finished = rb_ary_pop(ctx->stack);
+  }
+}
+
+int null_callback(void *ctx) {
+  set_value(ctx, Qnil);
+  return 1;
+}
+
+int boolean_callback(void *ctx, int boolean) {
+  set_value(ctx, boolean ? Qtrue : Qfalse);
+  return 1;
+}
+
+int integer_callback(void *ctx, long long intVal) {
+  set_value(ctx, LONG2NUM(intVal));
+  return 1;
+}
+
+int double_callback(void *ctx, double doubleVal) {
+  set_value(ctx, rb_float_new(doubleVal));
+  return 1;
+}
+
+int number_callback(void *ctx, const char *numberVal, size_t numberLen) {
+  char buf[numberLen+1];
+  buf[numberLen] = 0;
+  memcpy(buf, numberVal, numberLen);
+  if (memchr(buf, '.', numberLen) ||
+    memchr(buf, 'e', numberLen) ||
+    memchr(buf, 'E', numberLen)) {
+    set_value(ctx, rb_float_new(strtod(buf, NULL)));
+  } else {
+    set_value(ctx, rb_cstr2inum(buf, 10));
+  }
+  return 1;
+}
+
+int string_callback(void *ctx, const unsigned char *stringVal, size_t stringLen) {
+  char buf[stringLen+1];
+  buf[stringLen] = 0;
+  memcpy(buf, stringVal, stringLen);
+  set_value(ctx,rb_str_new2(buf));
+  return 1;
+}
+
+int start_map_callback(void *ctx) {
+  start_object(ctx,rb_hash_new());
+  return 1;
+}
+
+int map_key_callback(void *ctx, const unsigned char *stringVal, size_t stringLen) {
+  char buf[stringLen+1];
+  buf[stringLen] = 0;
+  memcpy(buf, stringVal, stringLen);
+  set_key(ctx,rb_str_new2(buf));
+  return 1;
+}
+
+int end_map_callback(void *ctx) {
+  end_object(ctx);
+  return 1;
+}
+
+int start_array_callback(void *ctx) {
+  start_object(ctx,rb_ary_new());
+  return 1;
+}
+
+int end_array_callback(void *ctx) {
+  end_object(ctx);
+  return 1;
+}
+
+static yajl_callbacks callbacks = {
+  null_callback,
+  boolean_callback,
+  integer_callback,
+  double_callback,
+  number_callback,
+  string_callback,
+  start_map_callback,
+  map_key_callback,
+  end_map_callback,
+  start_array_callback,
+  end_array_callback,
+};
+
+static VALUE mParser_do_yajl_parse(VALUE self, VALUE str, VALUE opts) {
+  yajl_handle hand;
+  yajl_status stat;
+  unsigned char *err;
+  CTX ctx;
+
+  ctx.stack = rb_ary_new();
+  ctx.key_stack = rb_ary_new();
+
+  hand = yajl_alloc(&callbacks, NULL, &ctx);
+  if ((stat = yajl_parse(hand, (unsigned char *)RSTRING_PTR(str), RSTRING_LEN(str))) != yajl_status_ok) {
+    err = yajl_get_error(hand, 1, (unsigned char *)RSTRING_PTR(str), RSTRING_LEN(str));
+    goto raise;
+  }
+  if ((stat = yajl_complete_parse(hand)) != yajl_status_ok) {
+    err = yajl_get_error(hand, 1, (unsigned char *)RSTRING_PTR(str), RSTRING_LEN(str));
+    goto raise;
+  }
+  yajl_free(hand);
+  return ctx.finished;
+
+raise:
+  if (hand) {
+    yajl_free(hand);
+  }
+  rb_raise(cParseError, "%s", err);
+}
+
+void Init_parser() {
+  mFFI_Yajl = rb_define_module("FFI_Yajl");
+  cParseError = rb_define_class_under(mFFI_Yajl, "ParseError", rb_eStandardError);
+  mExt = rb_define_module_under(mFFI_Yajl, "Ext");
+  mParser = rb_define_module_under(mExt, "Parser");
+  rb_define_method(mParser, "do_yajl_parse", mParser_do_yajl_parse, 2);
+}
+