1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
|
/* -*- Mode: c; c-basic-offset: 2 -*-
*
* turtle_lexer.l - Raptor Turtle lexer - making tokens for turtle grammar generator
*
* $Id$
*
* Copyright (C) 2003-2004 David Beckett - http://purl.org/net/dajobe/
* Institute for Learning and Research Technology - http://www.ilrt.bris.ac.uk/
* University of Bristol - http://www.bristol.ac.uk/
*
* This package is Free Software or Open Source available under the
* following licenses (these are alternatives):
* 1. GNU Lesser General Public License (LGPL)
* 2. GNU General Public License (GPL)
* 3. Mozilla Public License (MPL)
*
* See LICENSE.html or LICENSE.txt at the top of this package for the
* full license terms.
*
* Turtle is defined in http://www.ilrt.bristol.ac.uk/discovery/2004/01/turtle/
*
* To generate the C files from this source, rather than use the
* shipped turtle_lexer.c/.h needs a patched version of flex 2.5.31 such
* as the one available in Debian GNU/Linux. Details below
* near the %option descriptions.
*
*/
/* recognise 8-bits */
%option 8bit
%option warn nodefault
/* all symbols prefixed by this */
%option prefix="turtle_lexer_"
/* This is not needed, flex is invoked -oturtle_lexer.c */
/* %option outfile="turtle_lexer.c" */
/* Emit a C header file for prototypes
* Only available in flex 2.5.13 or newer.
* It was renamed to header-file in flex 2.5.19
*/
%option header-file="turtle_lexer.h"
/* Do not emit #include <unistd.h>
* Only available in flex 2.5.7 or newer.
* Broken in flex 2.5.31 without patches.
*/
%option nounistd
/* Never interactive */
/* No isatty() check */
%option never-interactive
/* Batch scanner */
%option batch
/* Never use yyunput */
%option nounput
%option reentrant
/* definitions */
%{
/* NOTE: These headers are NOT included here but are inserted by
* fix-flex since otherwise it appears far too late in the generated C
*/
/*
#ifdef HAVE_CONFIG_H
#include <raptor_config.h>
#endif
#ifdef WIN32
#include <win32_raptor_config.h>
#endif
*/
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <stdarg.h>
#ifdef HAVE_ERRNO_H
#include <errno.h>
#endif
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif
#include <raptor.h>
#include <raptor_internal.h>
#include <turtle_parser.h>
#include <turtle_common.h>
static unsigned char *copy_token(unsigned char *text, size_t len);
unsigned char *copy_string_token(raptor_parser* rdf_parser, unsigned char *text, size_t len, int delim);
char *filename=NULL;
#ifdef RAPTOR_DEBUG
const char * turtle_token_print(int token, YYSTYPE *lval);
#endif
int turtle_lexer_lex (YYSTYPE *turtle_parser_lval, yyscan_t yyscanner);
#define YY_DECL int turtle_lexer_lex (YYSTYPE *turtle_parser_lval, yyscan_t yyscanner)
#ifdef __cplusplus
#define INPUT_FN yyinput
#else
#define INPUT_FN input
#endif
/* Missing turtle_lexer.c/h prototypes */
int turtle_lexer_get_column(yyscan_t yyscanner);
void turtle_lexer_set_column(int column_no , yyscan_t yyscanner);
%}
%x PREF
%%
/* rules */
int c;
raptor_parser *rdf_parser=(raptor_parser*)yyextra;
raptor_turtle_parser* turtle_parser=(raptor_turtle_parser*)rdf_parser->context;
\r\n|\r|\n { turtle_parser->lineno++; }
[\ \t\v]+ { /* empty */ }
"a" { return A; }
"." { return DOT; }
"," { return COMMA; }
";" { return SEMICOLON; }
"[" { return LEFT_SQUARE; }
"]" { return RIGHT_SQUARE; }
"@prefix" { BEGIN(PREF); return PREFIX; }
"@" { return AT; }
"^^" { return HAT; }
"(" { return LEFT_ROUND; }
")" { return RIGHT_ROUND; }
'([^'\\\n\r]|\\[^\n\r])*' { turtle_parser_lval->string=copy_string_token(rdf_parser, (unsigned char*)yytext+1, yyleng-2, '\"'); /* ' */
return STRING_LITERAL; }
\"([^"\\\n\r]|\\[^\n\r])*\" { turtle_parser_lval->string=copy_string_token(rdf_parser, (unsigned char*)yytext+1, yyleng-2, '"'); /* ' */
return STRING_LITERAL; }
_:[A-Za-z_][A-Za-z0-9_]* { turtle_parser_lval->string=copy_token((unsigned char*)yytext+2, yyleng-2);
return BLANK_LITERAL; }
[A-Za-z][-A-Za-z0-9_]*:[A-Za-z_][-A-Za-z0-9_]* { turtle_parser_lval->uri=turtle_qname_to_uri(rdf_parser, (unsigned char*)yytext, yyleng);
return QNAME_LITERAL; }
:[A-Za-z_][-A-Za-z0-9_]* { turtle_parser_lval->uri=turtle_qname_to_uri(rdf_parser, (unsigned char*)yytext, yyleng);
return QNAME_LITERAL; }
[A-Za-z][-A-Za-z0-9_]*: { turtle_parser_lval->uri=turtle_qname_to_uri(rdf_parser, (unsigned char*)yytext, yyleng);
return QNAME_LITERAL; }
: { turtle_parser_lval->uri=turtle_qname_to_uri(rdf_parser, (unsigned char*)yytext, yyleng);
return QNAME_LITERAL; }
[0-9]+ { turtle_parser_lval->integer=atoi(yytext);
return INTEGER_LITERAL; }
<PREF>[\ \t\v]+ { /* eat up leading whitespace */ }
<PREF>(:|[A-Za-z][-A-Za-z0-9_]*:) { turtle_parser_lval->string=copy_token((unsigned char*)yytext, yyleng);
BEGIN(INITIAL);
return IDENTIFIER; }
<PREF>. { BEGIN(INITIAL);
if (*yytext == EOF)
return EOF;
turtle_syntax_error(rdf_parser, "syntax error at '%c' - @prefix name must end in :", *yytext);
return ERROR; }
\<[^>]*\> { if(yyleng == 2)
turtle_parser_lval->uri=raptor_uri_copy(rdf_parser->base_uri);
else {
yytext[yyleng-1]='\0';
turtle_parser_lval->uri=raptor_new_uri_relative_to_base(rdf_parser->base_uri, (const unsigned char*)yytext+1);
}
return URI_LITERAL; }
[A-Za-z][-A-Za-z0-9_]* { turtle_parser_lval->string=copy_token((unsigned char*)yytext, yyleng);
return IDENTIFIER; }
\# { while((c=INPUT_FN(yyscanner)) != EOF && c != '\n' && c != '\r' && c)
;
turtle_parser->lineno++;
if(c == EOF)
return EOF;
}
. { if (*yytext == EOF)
return EOF;
turtle_syntax_error(rdf_parser, "syntax error at '%c'", *yytext);
return ERROR;
}
%%
/* user code */
int
yywrap (yyscan_t yyscanner) {
return 1;
}
unsigned char *
copy_token(unsigned char *text, size_t len) {
unsigned char *s;
if(!len)
len=strlen((const char*)text);
s=(unsigned char *)malloc(len+1);
strncpy((char*)s, (const char*)text, len);
s[len] = '\0';
return s;
}
unsigned char *
copy_string_token(raptor_parser* rdf_parser, unsigned char *text, size_t len, int delim) {
size_t i;
unsigned char *s, *d;
unsigned char *string=(unsigned char *)malloc(len+1);
for(s=text, d=string, i=0; i<len; s++, i++) {
unsigned char c=*s;
if(c == '\\' ) {
s++; i++;
c=*s;
if(c == 'n')
*d++= '\n';
else if(c == 'r')
*d++= '\r';
else if(c == 't')
*d++= '\t';
else if(c == '\\' || c == delim)
*d++=c;
else if (c == 'u' || c == 'U') {
size_t ulen=(c == 'u') ? 4 : 8;
unsigned long unichar=0;
s++; i++;
if(i+ulen > len) {
turtle_syntax_error(rdf_parser, "\\%c over end of line", c);
free(string);
return NULL;
}
sscanf((const char*)s, ((ulen == 4) ? "%04lx" : "%08lx"), &unichar);
s+= ulen-1;
i+= ulen-1;
if(unichar < 0 || unichar > 0x10ffff) {
turtle_syntax_error(rdf_parser, "Illegal Unicode character with code point #x%lX.", unichar);
free(string);
return NULL;
}
d+=raptor_unicode_char_to_utf8(unichar, d);
} else {
/* don't handle \x where x isn't one of: \n \r \\ (delim) */
turtle_syntax_error(rdf_parser, "Illegal string escape \\%c in \"%s\"", c, text);
}
} else
*d++=c;
}
*d='\0';
return string;
}
#ifdef RAPTOR_DEBUG
const char *
turtle_token_print(int token, YYSTYPE *lval)
{
static char buffer[2048];
if(!token)
return "<<EOF>>";
switch(token) {
case PREFIX:
return "PREFIX";
case WS:
return "WS";
case A:
return "A";
case DOT:
return "DOT";
case COMMA:
return "COMMA";
case SEMICOLON:
return "SEMICOLON";
case LEFT_SQUARE:
return "LEFT_SQUARE";
case RIGHT_SQUARE:
return "RIGHT_SQUARE";
case AT:
return "AT";
case HAT:
return "HAT";
case STRING_LITERAL:
sprintf(buffer, "STRING_LITERAL(%s)", lval->string);
return buffer;
case URI_LITERAL:
sprintf(buffer, "URI_LITERAL(%s)",
(lval->uri ? (char*)raptor_uri_as_string(lval->uri) : ""));
return buffer;
case BLANK_LITERAL:
sprintf(buffer, "BLANK_LITERAL(%s)", lval->string);
return buffer;
case QNAME_LITERAL:
sprintf(buffer, "QNAME_LITERAL(%s)",
(lval->uri ? (char*)raptor_uri_as_string(lval->uri) : ""));
return buffer;
case INTEGER_LITERAL:
sprintf(buffer, "INTEGER_LITERAL(%d)", lval->integer);
return buffer;
case IDENTIFIER:
sprintf(buffer, "IDENTIFIER(%s)",
(lval->string ? (char*)lval->string : ""));
return buffer;
case ERROR:
return "ERROR";
default:
fprintf(stderr, "turtle_token_print: UNKNOWN token %d - add a new case\n", token);
abort();
}
}
#endif
#ifdef STANDALONE
static void
turtle_token_free(int token, YYSTYPE *lval)
{
if(!token)
return;
switch(token) {
case STRING_LITERAL:
case BLANK_LITERAL:
case IDENTIFIER:
if(lval->string)
free(lval->string);
break;
case URI_LITERAL:
case QNAME_LITERAL:
if(lval->uri)
raptor_free_uri(lval->uri);
break;
default:
break;
}
}
int
main(int argc, char *argv[])
{
raptor_parser rdf_parser;
raptor_turtle_parser turtle_parser;
yyscan_t scanner;
int token=EOF;
FILE *fh;
YYSTYPE lval;
const char *uri_string;
raptor_init();
if(argc > 1) {
filename=argv[1];
fh=fopen(filename, "r");
if(!fh) {
fprintf(stderr, "%s: Cannot open file %s - %s\n", argv[0], filename,
strerror(errno));
exit(1);
}
} else {
filename="<stdin>";
fh=stdin;
}
memset(&rdf_parser, 0, sizeof(raptor_parser));
memset(&turtle_parser, 0, sizeof(raptor_turtle_parser));
yylex_init(&turtle_parser.scanner);
scanner=turtle_parser.scanner;
turtle_lexer_set_in(fh, scanner);
turtle_lexer_set_extra(&rdf_parser, scanner);
/* Initialise enough of the parser and locator to get error messages */
rdf_parser.context=&turtle_parser;
turtle_parser.lineno=1;
rdf_parser.locator.file=filename;
rdf_parser.locator.column= -1;
uri_string=raptor_uri_filename_to_uri_string(filename);
rdf_parser.base_uri=raptor_new_uri(uri_string);
#if defined (RAPTOR_DEBUG) && defined(HAVE_DMALLOC_H)
#undef free
#endif
free((void*)uri_string);
while(1) {
memset(&lval, 0, sizeof(YYSTYPE));
if(turtle_lexer_get_text(scanner) != NULL)
printf("yyinput '%s'\n", turtle_lexer_get_text(scanner));
token=yylex(&lval, scanner);
#ifdef RAPTOR_DEBUG
printf("token %s\n", turtle_token_print(token, &lval));
#else
printf("token %d\n", token);
#endif
turtle_token_free(token, &lval);
if(!token || token == EOF || token == ERROR)
break;
}
yylex_destroy(scanner);
raptor_free_uri(rdf_parser.base_uri);
raptor_finish();
if(token==ERROR)
return 1;
return 0;
}
#endif
|