diff options
author | Lamont Granquist <lamont@scriptkiddie.org> | 2013-12-20 14:38:57 -0800 |
---|---|---|
committer | Lamont Granquist <lamont@scriptkiddie.org> | 2013-12-20 14:38:57 -0800 |
commit | 11911fc8d992260297e706a0168e4089e7d65eb8 (patch) | |
tree | 84eb7446375877311015c1a5027ae89e22367ea2 /ext/ffi_yajl | |
parent | f5a670d49b26764dbe805f1a8ad03c9d106eafea (diff) | |
download | ffi-yajl-11911fc8d992260297e706a0168e4089e7d65eb8.tar.gz |
almost working parser (unicode broken)
Diffstat (limited to 'ext/ffi_yajl')
-rw-r--r-- | ext/ffi_yajl/ext/parser/Makefile | 238 | ||||
-rw-r--r-- | ext/ffi_yajl/ext/parser/extconf.rb | 29 | ||||
-rw-r--r-- | ext/ffi_yajl/ext/parser/parser.c | 165 |
3 files changed, 432 insertions, 0 deletions
diff --git a/ext/ffi_yajl/ext/parser/Makefile b/ext/ffi_yajl/ext/parser/Makefile new file mode 100644 index 0000000..840cc31 --- /dev/null +++ b/ext/ffi_yajl/ext/parser/Makefile @@ -0,0 +1,238 @@ + +SHELL = /bin/sh + +# V=0 quiet, V=1 verbose. other values don't work. +V = 0 +Q1 = $(V:1=) +Q = $(Q1:0=@) +ECHO1 = $(V:1=@:) +ECHO = $(ECHO1:0=@echo) + +#### Start of system configuration section. #### + +srcdir = . +topdir = /Users/lamont/.rvm/rubies/ruby-2.0.0-p247/include/ruby-2.0.0 +hdrdir = $(topdir) +arch_hdrdir = /Users/lamont/.rvm/rubies/ruby-2.0.0-p247/include/ruby-2.0.0/x86_64-darwin12.4.0 +PATH_SEPARATOR = : +VPATH = $(srcdir):$(arch_hdrdir)/ruby:$(hdrdir)/ruby +prefix = /Users/lamont/.rvm/rubies/ruby-2.0.0-p247 +rubysitearchprefix = $(rubylibprefix)/$(sitearch) +rubyarchprefix = $(rubylibprefix)/$(arch) +rubylibprefix = $(libdir)/$(RUBY_BASE_NAME) +exec_prefix = $(prefix) +vendorarchhdrdir = $(vendorhdrdir)/$(sitearch) +sitearchhdrdir = $(sitehdrdir)/$(sitearch) +rubyarchhdrdir = $(rubyhdrdir)/$(arch) +vendorhdrdir = $(rubyhdrdir)/vendor_ruby +sitehdrdir = $(rubyhdrdir)/site_ruby +rubyhdrdir = $(includedir)/$(RUBY_VERSION_NAME) +vendorarchdir = $(vendorlibdir)/$(sitearch) +vendorlibdir = $(vendordir)/$(ruby_version) +vendordir = $(rubylibprefix)/vendor_ruby +sitearchdir = $(sitelibdir)/$(sitearch) +sitelibdir = $(sitedir)/$(ruby_version) +sitedir = $(rubylibprefix)/site_ruby +rubyarchdir = $(rubylibdir)/$(arch) +rubylibdir = $(rubylibprefix)/$(ruby_version) +sitearchincludedir = $(includedir)/$(sitearch) +archincludedir = $(includedir)/$(arch) +sitearchlibdir = $(libdir)/$(sitearch) +archlibdir = $(libdir)/$(arch) +ridir = $(datarootdir)/$(RI_BASE_NAME) +mandir = $(datarootdir)/man +localedir = $(datarootdir)/locale +libdir = $(exec_prefix)/lib +psdir = $(docdir) +pdfdir = $(docdir) +dvidir = $(docdir) +htmldir = $(docdir) +infodir = $(datarootdir)/info +docdir = $(datarootdir)/doc/$(PACKAGE) +oldincludedir = /usr/include +includedir = $(prefix)/include +localstatedir = $(prefix)/var +sharedstatedir = $(prefix)/com +sysconfdir = $(prefix)/etc +datadir = $(datarootdir) +datarootdir = $(prefix)/share +libexecdir = $(exec_prefix)/libexec +sbindir = $(exec_prefix)/sbin +bindir = $(exec_prefix)/bin +archdir = $(rubyarchdir) + + +CC = /usr/bin/clang +CXX = clang++ +LIBRUBY = $(LIBRUBY_SO) +LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a +LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME) +LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static +empty = +OUTFLAG = -o $(empty) +COUTFLAG = -o $(empty) + +RUBY_EXTCONF_H = +cflags = $(optflags) $(debugflags) $(warnflags) +optflags = -O3 -fno-fast-math +debugflags = -ggdb3 +warnflags = -Wall -Wextra -Wno-unused-parameter -Wno-parentheses -Wno-long-long -Wno-missing-field-initializers -Wunused-variable -Wpointer-arith -Wwrite-strings -Wdeclaration-after-statement -Wshorten-64-to-32 -Wimplicit-function-declaration +CCDLFLAGS = -fno-common +CFLAGS = $(CCDLFLAGS) -I/Users/lamont/git/ffi-yajl/include -L/Users/lamont/git/ffi-yajl/lib -O3 -march=nocona -O2 -pipe -fno-common -Wall $(ARCH_FLAG) +INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir)/ruby/backward -I$(hdrdir) -I$(srcdir) +DEFS = +CPPFLAGS = -D_XOPEN_SOURCE -D_DARWIN_C_SOURCE -D_DARWIN_UNLIMITED_SELECT -D_REENTRANT -I/usr/local/opt/libyaml/include -I/usr/local/opt/readline/include -I/usr/local/opt/libksba/include -I/usr/local/opt/openssl/include $(DEFS) $(cppflags) +CXXFLAGS = $(CCDLFLAGS) $(cxxflags) $(ARCH_FLAG) +ldflags = -L/Users/lamont/git/ffi-yajl/lib -I/Users/lamont/git/ffi-yajl/include -L/Users/lamont/git/ffi-yajl/lib -O3 -march=nocona -O2 -pipe -fno-common -lyajl +dldflags = -Wl,-undefined,dynamic_lookup -Wl,-multiply_defined,suppress -L/usr/local/opt/libyaml/lib -L/usr/local/opt/readline/lib -L/usr/local/opt/libksba/lib -L/usr/local/opt/openssl/lib +ARCH_FLAG = +DLDFLAGS = $(ldflags) $(dldflags) $(ARCH_FLAG) +LDSHARED = $(CC) -dynamic -bundle +LDSHAREDXX = $(CXX) -dynamic -bundle +AR = ar +EXEEXT = + +RUBY_INSTALL_NAME = ruby +RUBY_SO_NAME = ruby.2.0.0 +RUBYW_INSTALL_NAME = +RUBY_VERSION_NAME = $(RUBY_BASE_NAME)-$(ruby_version) +RUBYW_BASE_NAME = rubyw +RUBY_BASE_NAME = ruby + +arch = x86_64-darwin12.4.0 +sitearch = $(arch) +ruby_version = 2.0.0 +ruby = $(bindir)/ruby +RUBY = $(ruby) +ruby_headers = $(hdrdir)/ruby.h $(hdrdir)/ruby/defines.h $(arch_hdrdir)/ruby/config.h + +RM = rm -f +RM_RF = $(RUBY) -run -e rm -- -rf +RMDIRS = rmdir -p +MAKEDIRS = mkdir -p +INSTALL = /usr/bin/install -c +INSTALL_PROG = $(INSTALL) -m 0755 +INSTALL_DATA = $(INSTALL) -m 644 +COPY = cp +TOUCH = exit > + +#### End of system configuration section. #### + +preload = + +libpath = . $(libdir) /usr/local/opt/libyaml/lib /usr/local/opt/readline/lib /usr/local/opt/libksba/lib /usr/local/opt/openssl/lib +LIBPATH = -L. -L$(libdir) -L/usr/local/opt/libyaml/lib -L/usr/local/opt/readline/lib -L/usr/local/opt/libksba/lib -L/usr/local/opt/openssl/lib +DEFFILE = + +CLEANFILES = mkmf.log +DISTCLEANFILES = +DISTCLEANDIRS = + +extout = +extout_prefix = +target_prefix = /ffi_yajl/ext +LOCAL_LIBS = +LIBS = $(LIBRUBYARG_SHARED) -lpthread -ldl -lobjc +ORIG_SRCS = parser.c +SRCS = $(ORIG_SRCS) +OBJS = parser.o +HDRS = +TARGET = parser +TARGET_NAME = parser +TARGET_ENTRY = Init_$(TARGET_NAME) +DLLIB = $(TARGET).bundle +EXTSTATIC = +STATIC_LIB = + +BINDIR = $(DESTDIR)$(bindir) +RUBYCOMMONDIR = $(DESTDIR)$(sitedir)$(target_prefix) +RUBYLIBDIR = $(DESTDIR)$(sitelibdir)$(target_prefix) +RUBYARCHDIR = $(DESTDIR)$(sitearchdir)$(target_prefix) +HDRDIR = $(DESTDIR)$(rubyhdrdir)/ruby$(target_prefix) +ARCHHDRDIR = $(DESTDIR)$(rubyhdrdir)/$(arch)/ruby$(target_prefix) + +TARGET_SO = $(DLLIB) +CLEANLIBS = $(TARGET).bundle +CLEANOBJS = *.o *.bak + +all: $(DLLIB) +static: $(STATIC_LIB) +.PHONY: all install static install-so install-rb +.PHONY: clean clean-so clean-static clean-rb + +clean-static:: +clean-rb-default:: +clean-rb:: +clean-so:: +clean: clean-so clean-static clean-rb-default clean-rb + -$(Q)$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) .*.time + +distclean-rb-default:: +distclean-rb:: +distclean-so:: +distclean-static:: +distclean: clean distclean-so distclean-static distclean-rb-default distclean-rb + -$(Q)$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log + -$(Q)$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES) + -$(Q)$(RMDIRS) $(DISTCLEANDIRS) 2> /dev/null || true + +realclean: distclean +install: install-so install-rb + +install-so: $(DLLIB) ./.RUBYARCHDIR.time + $(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR) +clean-static:: + -$(Q)$(RM) $(STATIC_LIB) +install-rb: pre-install-rb install-rb-default +install-rb-default: pre-install-rb-default +pre-install-rb: Makefile +pre-install-rb-default: Makefile +pre-install-rb-default: + $(ECHO) installing default parser libraries +./.RUBYARCHDIR.time: + $(Q) $(MAKEDIRS) $(RUBYARCHDIR) + $(Q) $(TOUCH) $@ + +site-install: site-install-so site-install-rb +site-install-so: install-so +site-install-rb: install-rb + +.SUFFIXES: .c .m .cc .mm .cxx .cpp .C .o + +.cc.o: + $(ECHO) compiling $(<) + $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $< + +.mm.o: + $(ECHO) compiling $(<) + $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $< + +.cxx.o: + $(ECHO) compiling $(<) + $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $< + +.cpp.o: + $(ECHO) compiling $(<) + $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $< + +.C.o: + $(ECHO) compiling $(<) + $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $< + +.c.o: + $(ECHO) compiling $(<) + $(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $< + +.m.o: + $(ECHO) compiling $(<) + $(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $< + +$(DLLIB): $(OBJS) Makefile + $(ECHO) linking shared-object ffi_yajl/ext/$(DLLIB) + -$(Q)$(RM) $(@) + $(Q) $(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS) + $(Q) test -z '$(RUBY_CODESIGN)' || codesign -s '$(RUBY_CODESIGN)' -f $@ + + + +$(OBJS): $(HDRS) $(ruby_headers) diff --git a/ext/ffi_yajl/ext/parser/extconf.rb b/ext/ffi_yajl/ext/parser/extconf.rb new file mode 100644 index 0000000..e78e7a2 --- /dev/null +++ b/ext/ffi_yajl/ext/parser/extconf.rb @@ -0,0 +1,29 @@ +require 'mkmf' + +# the customer is always right, ruby is always compiled to be stupid +$CFLAGS = ENV['CFLAGS'] if ENV['CFLAGS'] +$LDFLAGS = ENV['LDFLAGS'] if ENV['LDFLAGS'] +RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC'] if ENV['CC'] + +# search our gem root first to pick up libyajl2 that we vendored +gem_root = File.expand_path(File.join(File.dirname(__FILE__), "../../../..")) +$CFLAGS = "-I#{gem_root}/include -L#{gem_root}/lib #{$CFLAGS}" +$LDFLAGS = "-L#{gem_root}/lib #{$CFLAGS}" + +puts gem_root +puts $CFLAGS +puts $LDFLAGS + +# except if you're doing an unoptimized gcc install we're going to help you out a bit +if RbConfig::MAKEFILE_CONFIG['CC'] =~ /gcc|clang/ + $CFLAGS << " -O3" unless $CFLAGS[/-O\d/] + # how many people realize that -Wall is a compiler-specific flag??? + # apparently not many based on reading lots of shitty extconf.rb's out there + $CFLAGS << " -Wall" +end + +$LDFLAGS << " -lyajl" + +dir_config 'parser' + +create_makefile 'ffi_yajl/ext/parser' diff --git a/ext/ffi_yajl/ext/parser/parser.c b/ext/ffi_yajl/ext/parser/parser.c new file mode 100644 index 0000000..7793a20 --- /dev/null +++ b/ext/ffi_yajl/ext/parser/parser.c @@ -0,0 +1,165 @@ +#include <ruby.h> +#include <yajl/yajl_parse.h> + +static VALUE mFFI_Yajl, mExt, mParser, cParseError; + +typedef struct { + VALUE finished; + VALUE stack; + VALUE key_stack; + VALUE key; +} CTX; + +void set_value(CTX *ctx, VALUE val) { + long len = RARRAY_LEN(ctx->stack); + VALUE last = rb_ary_entry(ctx->stack, len-1); + switch (TYPE(last)) { + case T_ARRAY: + rb_ary_push(last, val); + break; + case T_HASH: + rb_hash_aset(last, ctx->key, val); + break; + default: + break; + } +} + +void set_key(CTX *ctx, VALUE key) { + ctx->key = key; +} + +void start_object(CTX *ctx, VALUE obj) { + rb_ary_push(ctx->key_stack, ctx->key); + rb_ary_push(ctx->stack, obj); +} + +void end_object(CTX *ctx) { + ctx->key = rb_ary_pop(ctx->key_stack); + if ( RARRAY_LEN(ctx->stack) > 1 ) { + set_value(ctx, rb_ary_pop(ctx->stack)); + } else { + ctx->finished = rb_ary_pop(ctx->stack); + } +} + +int null_callback(void *ctx) { + set_value(ctx, Qnil); + return 1; +} + +int boolean_callback(void *ctx, int boolean) { + set_value(ctx, boolean ? Qtrue : Qfalse); + return 1; +} + +int integer_callback(void *ctx, long long intVal) { + set_value(ctx, LONG2NUM(intVal)); + return 1; +} + +int double_callback(void *ctx, double doubleVal) { + set_value(ctx, rb_float_new(doubleVal)); + return 1; +} + +int number_callback(void *ctx, const char *numberVal, size_t numberLen) { + char buf[numberLen+1]; + buf[numberLen] = 0; + memcpy(buf, numberVal, numberLen); + if (memchr(buf, '.', numberLen) || + memchr(buf, 'e', numberLen) || + memchr(buf, 'E', numberLen)) { + set_value(ctx, rb_float_new(strtod(buf, NULL))); + } else { + set_value(ctx, rb_cstr2inum(buf, 10)); + } + return 1; +} + +int string_callback(void *ctx, const unsigned char *stringVal, size_t stringLen) { + char buf[stringLen+1]; + buf[stringLen] = 0; + memcpy(buf, stringVal, stringLen); + set_value(ctx,rb_str_new2(buf)); + return 1; +} + +int start_map_callback(void *ctx) { + start_object(ctx,rb_hash_new()); + return 1; +} + +int map_key_callback(void *ctx, const unsigned char *stringVal, size_t stringLen) { + char buf[stringLen+1]; + buf[stringLen] = 0; + memcpy(buf, stringVal, stringLen); + set_key(ctx,rb_str_new2(buf)); + return 1; +} + +int end_map_callback(void *ctx) { + end_object(ctx); + return 1; +} + +int start_array_callback(void *ctx) { + start_object(ctx,rb_ary_new()); + return 1; +} + +int end_array_callback(void *ctx) { + end_object(ctx); + return 1; +} + +static yajl_callbacks callbacks = { + null_callback, + boolean_callback, + integer_callback, + double_callback, + number_callback, + string_callback, + start_map_callback, + map_key_callback, + end_map_callback, + start_array_callback, + end_array_callback, +}; + +static VALUE mParser_do_yajl_parse(VALUE self, VALUE str, VALUE opts) { + yajl_handle hand; + yajl_status stat; + unsigned char *err; + CTX ctx; + + ctx.stack = rb_ary_new(); + ctx.key_stack = rb_ary_new(); + + hand = yajl_alloc(&callbacks, NULL, &ctx); + if ((stat = yajl_parse(hand, (unsigned char *)RSTRING_PTR(str), RSTRING_LEN(str))) != yajl_status_ok) { + err = yajl_get_error(hand, 1, (unsigned char *)RSTRING_PTR(str), RSTRING_LEN(str)); + goto raise; + } + if ((stat = yajl_complete_parse(hand)) != yajl_status_ok) { + err = yajl_get_error(hand, 1, (unsigned char *)RSTRING_PTR(str), RSTRING_LEN(str)); + goto raise; + } + yajl_free(hand); + return ctx.finished; + +raise: + if (hand) { + yajl_free(hand); + } + rb_raise(cParseError, "%s", err); +} + +void Init_parser() { + mFFI_Yajl = rb_define_module("FFI_Yajl"); + cParseError = rb_define_class_under(mFFI_Yajl, "ParseError", rb_eStandardError); + mExt = rb_define_module_under(mFFI_Yajl, "Ext"); + mParser = rb_define_module_under(mExt, "Parser"); + rb_define_method(mParser, "do_yajl_parse", mParser_do_yajl_parse, 2); +} + |