diogo/yap-6.3
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix get_num init issues

Browse Source
This commit is contained in:
Vítor Santos Costa 2015-10-18 11:40:12 +01:00
parent 5762aa846c
commit 9fb7325f19
1 changed files with 1347 additions and 1299 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

238
C/scanner.c
View File

@ -437,17 +437,15 @@ writing, writing a BOM can be requested using the option
static Term float_send(char *, int); static Term float_send(char *, int);
static Term get_num(int *, int *, struct stream_desc *, char *, UInt, int); static Term get_num(int *, int *, struct stream_desc *, char *, UInt, int);
static void static void Yap_setCurrentSourceLocation(struct stream_desc *s) {
Yap_setCurrentSourceLocation( struct stream_desc *s )
{
CACHE_REGS CACHE_REGS
#if HAVE_SOCKET #if HAVE_SOCKET
if (s->status & Socket_Stream_f) if (s->status & Socket_Stream_f)
LOCAL_SourceFileName = AtomSocket; LOCAL_SourceFileName = AtomSocket;
else else
#endif #endif
if (s->status & Pipe_Stream_f) if (s->status & Pipe_Stream_f)
LOCAL_SourceFileName =AtomPipe; LOCAL_SourceFileName = AtomPipe;
else if (s->status & InMemory_Stream_f) else if (s->status & InMemory_Stream_f)
LOCAL_SourceFileName = AtomCharsio; LOCAL_SourceFileName = AtomCharsio;
else else
@ -469,7 +467,8 @@ static char chtype0[NUMBER_OF_CHARS + 1] = {
BS, BS, BS, BS, BS, BS, BS, BS, BS, BS, BS, BS, BS, BS,
/* sp ! " # $ % & ' ( ) * + , - . / */ /* sp ! " # $ % & ' ( ) * + , - . / */
BS, SL, DC, SY, SY, CC, SY, QT, BK, BK, SY, SY, BK, SY, SY, SY, BS, SL, DC, SY, SY, CC, SY, QT, BK,
BK, SY, SY, BK, SY, SY, SY,
/* 0 1 2 3 4 5 6 7 8 9 : ; < = > ? */ /* 0 1 2 3 4 5 6 7 8 9 : ; < = > ? */
NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU,
@ -513,27 +512,27 @@ static char chtype0[NUMBER_OF_CHARS + 1] = {
UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC,
UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC,
/* Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß */ /* Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß */
#ifdef vms #ifdef vms
UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC, UC,
UC, UC, UC, UC, UC, UC, LC, UC, UC, UC, UC, UC, UC, LC,
#else #else
UC, UC, UC, UC, UC, UC, UC, SY, UC, UC, UC, UC, UC, UC, UC, UC, SY, UC,
UC, UC, UC, UC, UC, UC, LC, UC, UC, UC, UC, UC, UC, LC,
#endif #endif
/* à á â ã ä å æ ç è é ê ë ì í î ï */ /* à á â ã ä å æ ç è é ê ë ì í î ï */
LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC,
LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC,
/* ð ñ ò ó ô õ ö ÷ ø ù ú û ü cannot write the last /* ð ñ ò ó ô õ ö ÷ ø ù ú û ü cannot write the last
* three because of lcc */ * three because of lcc */
#ifdef vms #ifdef vms
LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC, LC,
LC, LC, LC, LC, LC, LC, LC LC, LC, LC, LC, LC, LC, LC
#else #else
LC, LC, LC, LC, LC, LC, LC, SY, LC, LC, LC, LC, LC, LC, LC, LC, SY, LC,
LC, LC, LC, LC, LC, LC, LC LC, LC, LC, LC, LC, LC, LC
#endif #endif
}; };
typedef struct scanner_internals { typedef struct scanner_internals {
@ -567,13 +566,13 @@ int Yap_wide_chtype(Int ch) {
// standard get char, uses conversion table // standard get char, uses conversion table
// and converts to wide // and converts to wide
#define getchr(inp) inp->stream_wgetc_for_read(inp-GLOBAL_Stream) #define getchr(inp) inp->stream_wgetc_for_read(inp - GLOBAL_Stream)
// get char for quoted data, eg, quoted atoms and so on // get char for quoted data, eg, quoted atoms and so on
// converts to wide // converts to wide
#define getchrq(inp) inp->stream_wgetc(inp-GLOBAL_Stream) #define getchrq(inp) inp->stream_wgetc(inp - GLOBAL_Stream)
// get char for UTF-8 quoted data, eg, quoted strings // get char for UTF-8 quoted data, eg, quoted strings
// reads bytes // reads bytes
#define getchru(inp) inp->stream_getc_utf8(inp-GLOBAL_Stream) #define getchru(inp) inp->stream_getc_utf8(inp - GLOBAL_Stream)
/* in case there is an overflow */ /* in case there is an overflow */
typedef struct scanner_extra_alloc { typedef struct scanner_extra_alloc {
@ -581,6 +580,14 @@ typedef struct scanner_extra_alloc {
void *filler; void *filler;
} ScannerExtraBlock; } ScannerExtraBlock;
static void InitScannerMemory(void) {
CACHE_REGS
LOCAL_ErrorMessage = NULL;
LOCAL_Error_Size = 0;
LOCAL_ScannerStack = (char *)TR;
LOCAL_ScannerExtraBlocks = NULL;
}
static char *AllocScannerMemory(unsigned int size) { static char *AllocScannerMemory(unsigned int size) {
CACHE_REGS CACHE_REGS
char *AuxSpScan; char *AuxSpScan;
@ -687,10 +694,11 @@ static int send_error_message(char s[]) {
return 0; return 0;
} }
static wchar_t read_quoted_char(int *scan_nextp, struct stream_desc* inp_stream) { static wchar_t read_quoted_char(int *scan_nextp,
struct stream_desc *inp_stream) {
int ch; int ch;
/* escape sequence */ /* escape sequence */
do_switch: do_switch:
ch = getchrq(inp_stream); ch = getchrq(inp_stream);
switch (ch) { switch (ch) {
@ -1095,38 +1103,91 @@ Term Yap_scan_num(StreamDesc *inp) {
Term out; Term out;
int sign = 1; int sign = 1;
int ch, cherr; int ch, cherr;
char *ptr; char *ptr, *mp;
int kind;
void *old_tr = TR;
LOCAL_ErrorMessage = NULL; InitScannerMemory();
LOCAL_ScannerStack = (char *)TR; LOCAL_VarTable = LOCAL_AnonVarTable = NULL;
LOCAL_ScannerExtraBlocks = NULL;
if (!(ptr = AllocScannerMemory(4096))) { if (!(ptr = AllocScannerMemory(4096))) {
LOCAL_ErrorMessage = "Trail Overflow"; LOCAL_ErrorMessage = "Trail Overflow";
LOCAL_Error_TYPE = RESOURCE_ERROR_TRAIL; LOCAL_Error_TYPE = RESOURCE_ERROR_TRAIL;
return TermNil; return 0;
} }
TokEntry *tokptr = (TokEntry *)AllocScannerMemory(sizeof(TokEntry));
tokptr->TokPos = GetCurInpPos(inp);
ch = getchr(inp); ch = getchr(inp);
while (chtype(ch) == BS) {
ch = getchr(inp);
}
if (ch == '-') { if (ch == '-') {
sign = -1; sign = -1;
ch = getchr(inp); ch = getchr(inp);
} else if (ch == '+') { } else if (ch == '+') {
ch = getchr(inp); ch = getchr(inp);
} }
if (chtype(ch) != NU) { if (chtype(ch) == NU) {
Yap_clean_tokenizer((TokEntry *)LOCAL_ScannerStack, NULL, NULL);
return TermNil;
}
cherr = '\0'; cherr = '\0';
if (ASP - HR < 1024) if (ASP - HR < 1024) {
return TermNil; Yap_clean_tokenizer(old_tr, NULL, NULL);
LOCAL_ErrorMessage = "Stack Overflow";
LOCAL_Error_TYPE = RESOURCE_ERROR_STACK;
return 0;
}
out = get_num(&ch, &cherr, inp, ptr, 4096, sign); /* */ out = get_num(&ch, &cherr, inp, ptr, 4096, sign); /* */
}
if (LOCAL_ErrorMessage != NULL || ch != -1 || cherr) {
CACHE_REGS
char *s = ptr;
int sign = 1;
out = 0;
if (s[0] == '+') {
s++;
}
if (s[0] == '-') {
s++;
sign = -1;
}
if (strcmp(s, "inf") == 0) {
if (sign > 0) {
out = MkFloatTerm(INFINITY);
} else {
out = MkFloatTerm(-INFINITY);
}
}
if (strcmp(s, "nan") == 0) {
if (sign > 0) {
out = MkFloatTerm(NAN);
} else {
out = MkFloatTerm(-NAN);
}
}
if (out == 0) {
TokEntry *e, *ef;
size_t len = strlen(ptr);
mp = AllocScannerMemory(len + 1);
tokptr->Tok = Ord(kind = String_tok);
tokptr->TokInfo = Unsigned(mp);
e = (TokEntry *)AllocScannerMemory(sizeof(TokEntry));
ef = (TokEntry *)AllocScannerMemory(sizeof(TokEntry));
tokptr->TokNext = e;
e->Tok = Error_tok;
if (!LOCAL_ErrorMessage)
LOCAL_ErrorMessage =
"syntax error while converting from a string to a number";
e->TokInfo = MkAtomTerm(Yap_LookupAtom(LOCAL_ErrorMessage));
e->TokPos = GetCurInpPos(inp);
e->TokNext = ef;
ef->Tok = Ord(kind = eot_tok);
ef->TokPos = GetCurInpPos(inp);
ef->TokNext = NULL;
LOCAL_tokptr = tokptr;
LOCAL_toktide = e;
LOCAL_ErrorMessage = NULL;
LOCAL_Error_Term = Yap_syntax_error(e, inp - GLOBAL_Stream);
LOCAL_Error_TYPE = SYNTAX_ERROR;
}
}
PopScannerMemory(ptr, 4096); PopScannerMemory(ptr, 4096);
Yap_clean_tokenizer((TokEntry *)LOCAL_ScannerStack, NULL, NULL); Yap_clean_tokenizer(old_tr, NULL, NULL);
if (LOCAL_ErrorMessage != NULL || ch != -1 || cherr)
return TermNil;
return out; return out;
} }
@ -1141,26 +1202,25 @@ Term Yap_scan_num(StreamDesc *inp) {
return l; \ return l; \
} }
const char * const char *Yap_tokRep(TokEntry *tokptr) {
Yap_tokRep(TokEntry *tokptr)
{
CACHE_REGS CACHE_REGS
Term info = tokptr->TokInfo; Term info = tokptr->TokInfo;
char *b, *buf = LOCAL_FileNameBuf2; char *b, *buf = LOCAL_FileNameBuf2;
size_t length, sze = YAP_FILENAME_MAX-1; size_t length, sze = YAP_FILENAME_MAX - 1;
UInt flags = 0; UInt flags = 0;
switch (tokptr->Tok) { switch (tokptr->Tok) {
case Name_tok: case Name_tok:
return (char *)RepAtom((Atom)info)->StrOfAE; return (char *)RepAtom((Atom)info)->StrOfAE;
case Number_tok: case Number_tok:
if ((b = Yap_TermToString(info, buf, sze, &length, &LOCAL_encoding, flags)) != buf) { if ((b = Yap_TermToString(info, buf, sze, &length, &LOCAL_encoding,
if (b) free(b); flags)) != buf) {
if (b)
free(b);
return NULL; return NULL;
} }
return buf; return buf;
case Var_tok: case Var_tok: {
{
VarEntry *varinfo = (VarEntry *)info; VarEntry *varinfo = (VarEntry *)info;
return varinfo->VarRep; return varinfo->VarRep;
} }
@ -1168,22 +1228,21 @@ Yap_tokRep(TokEntry *tokptr)
case BQString_tok: case BQString_tok:
return (char *)info; return (char *)info;
case WString_tok: case WString_tok:
case WBQString_tok: case WBQString_tok: {
{ wchar_t *op = (wchar_t *)info; wchar_t *op = (wchar_t *)info;
wchar_t c; wchar_t c;
unsigned char *bp = (unsigned char *)buf; unsigned char *bp = (unsigned char *)buf;
while ((c=*op++) ){ while ((c = *op++)) {
bp += put_utf8(bp, c); bp += put_utf8(bp, c);
} }
bp[0]='\0'; bp[0] = '\0';
return buf; return buf;
} }
case Error_tok: case Error_tok:
return "<ERR>"; return "<ERR>";
case eot_tok: case eot_tok:
return "<EOT>"; return "<EOT>";
case Ponctuation_tok: case Ponctuation_tok: {
{
buf[1] = '\0'; buf[1] = '\0';
if ((info) == 'l') { if ((info) == 'l') {
buf[0] = '('; buf[0] = '(';
@ -1198,7 +1257,6 @@ Yap_tokRep(TokEntry *tokptr)
} }
} }
static void open_comment(int ch, StreamDesc *inp_stream USES_REGS) { static void open_comment(int ch, StreamDesc *inp_stream USES_REGS) {
CELL *h0 = HR; CELL *h0 = HR;
HR += 5; HR += 5;
@ -1214,8 +1272,7 @@ static void open_comment(int ch, StreamDesc *inp_stream USES_REGS) {
LOCAL_CommentsTail = h0 + 1; LOCAL_CommentsTail = h0 + 1;
h0 += 2; h0 += 2;
h0[0] = (CELL)FunctorMinus; h0[0] = (CELL)FunctorMinus;
h0[1] = Yap_StreamPosition(inp_stream-GLOBAL_Stream h0[1] = Yap_StreamPosition(inp_stream - GLOBAL_Stream);
);
h0[2] = TermNil; h0[2] = TermNil;
LOCAL_CommentsNextChar = h0 + 2; LOCAL_CommentsNextChar = h0 + 2;
LOCAL_CommentsBuff = (wchar_t *)malloc(1024 * sizeof(wchar_t)); LOCAL_CommentsBuff = (wchar_t *)malloc(1024 * sizeof(wchar_t));
@ -1244,9 +1301,7 @@ static void close_comment(USES_REGS1) {
// mark that we reached EOF, // mark that we reached EOF,
// next token will be end_of_file) // next token will be end_of_file)
static void static void mark_eof(struct stream_desc *inp_stream) {
mark_eof( struct stream_desc * inp_stream )
{
inp_stream->status |= Push_Eof_Stream_f; inp_stream->status |= Push_Eof_Stream_f;
} }
@ -1289,7 +1344,7 @@ static wchar_t *ch_to_wide(char *base, char *charp) {
} }
#define add_ch_to_utf8_buff(ch) \ #define add_ch_to_utf8_buff(ch) \
{ \ { \
if ((ch & 0xff) == ch) { \ if ((ch & 0xff) == ch) { \
*charp++ = ch; \ *charp++ = ch; \
} else { \ } else { \
@ -1297,8 +1352,8 @@ static wchar_t *ch_to_wide(char *base, char *charp) {
} \ } \
} }
TokEntry *Yap_tokenizer( struct stream_desc *inp_stream, TokEntry *Yap_tokenizer(struct stream_desc *inp_stream, bool store_comments,
bool store_comments, Term *tposp) { Term *tposp) {
CACHE_REGS CACHE_REGS
TokEntry *t, *l, *p; TokEntry *t, *l, *p;
@ -1308,19 +1363,16 @@ TokEntry *Yap_tokenizer( struct stream_desc *inp_stream,
wchar_t *wcharp; wchar_t *wcharp;
struct qq_struct_t *cur_qq = NULL; struct qq_struct_t *cur_qq = NULL;
LOCAL_ErrorMessage = NULL; InitScannerMemory();
LOCAL_Error_Size = 0;
LOCAL_VarTable = NULL; LOCAL_VarTable = NULL;
LOCAL_AnonVarTable = NULL; LOCAL_AnonVarTable = NULL;
LOCAL_ScannerStack = (char *)TR;
LOCAL_ScannerExtraBlocks = NULL;
l = NULL; l = NULL;
p = NULL; /* Just to make lint happy */ p = NULL; /* Just to make lint happy */
ch = getchr(inp_stream); ch = getchr(inp_stream);
while (chtype(ch) == BS) { while (chtype(ch) == BS) {
ch = getchr(inp_stream); ch = getchr(inp_stream);
} }
*tposp = Yap_StreamPosition(inp_stream-GLOBAL_Stream); *tposp = Yap_StreamPosition(inp_stream - GLOBAL_Stream);
Yap_setCurrentSourceLocation(inp_stream); Yap_setCurrentSourceLocation(inp_stream);
LOCAL_StartLineCount = inp_stream->linecount; LOCAL_StartLineCount = inp_stream->linecount;
LOCAL_StartLinePos = inp_stream->linepos; LOCAL_StartLinePos = inp_stream->linepos;
@ -1346,7 +1398,7 @@ TokEntry *Yap_tokenizer( struct stream_desc *inp_stream,
else else
p->TokNext = t; p->TokNext = t;
p = t; p = t;
restart: restart:
while (chtype(ch) == BS) { while (chtype(ch) == BS) {
ch = getchr(inp_stream); ch = getchr(inp_stream);
} }
@ -1358,7 +1410,7 @@ restart:
if (store_comments) { if (store_comments) {
CHECK_SPACE(); CHECK_SPACE();
open_comment(ch, inp_stream PASS_REGS); open_comment(ch, inp_stream PASS_REGS);
continue_comment: continue_comment:
while ((ch = getchr(inp_stream)) != 10 && chtype(ch) != EF) { while ((ch = getchr(inp_stream)) != 10 && chtype(ch) != EF) {
CHECK_SPACE(); CHECK_SPACE();
extend_comment(ch PASS_REGS); extend_comment(ch PASS_REGS);
@ -1385,7 +1437,7 @@ continue_comment:
ch = getchr(inp_stream); ch = getchr(inp_stream);
} }
CHECK_SPACE(); CHECK_SPACE();
*tposp = Yap_StreamPosition(inp_stream-GLOBAL_Stream); *tposp = Yap_StreamPosition(inp_stream - GLOBAL_Stream);
Yap_setCurrentSourceLocation(inp_stream); Yap_setCurrentSourceLocation(inp_stream);
} }
goto restart; goto restart;
@ -1400,7 +1452,7 @@ continue_comment:
case LC: case LC:
och = ch; och = ch;
ch = getchr(inp_stream); ch = getchr(inp_stream);
scan_name: scan_name:
TokImage = (char *)((AtomEntry *)(Yap_PreAllocCodeSpace()))->StrOfAE; TokImage = (char *)((AtomEntry *)(Yap_PreAllocCodeSpace()))->StrOfAE;
charp = TokImage; charp = TokImage;
wcharp = NULL; wcharp = NULL;
@ -1408,7 +1460,7 @@ scan_name:
add_ch_to_buff(och); add_ch_to_buff(och);
for (; chtype(ch) <= NU; ch = getchr(inp_stream)) { for (; chtype(ch) <= NU; ch = getchr(inp_stream)) {
if (charp == (char *)AuxSp - 1024) { if (charp == (char *)AuxSp - 1024) {
huge_var_error: huge_var_error:
/* huge atom or variable, we are in trouble */ /* huge atom or variable, we are in trouble */
LOCAL_ErrorMessage = "Code Space Overflow due to huge atom"; LOCAL_ErrorMessage = "Code Space Overflow due to huge atom";
LOCAL_Error_TYPE = RESOURCE_ERROR_AUXILIARY_STACK; LOCAL_Error_TYPE = RESOURCE_ERROR_AUXILIARY_STACK;
@ -1420,7 +1472,8 @@ huge_var_error:
} }
add_ch_to_buff(ch); add_ch_to_buff(ch);
} }
while (ch == '\'' && isvar &&trueGlobalPrologFlag(VARIABLE_NAMES_MAY_END_WITH_QUOTES_FLAG)) { while (ch == '\'' && isvar &&
trueGlobalPrologFlag(VARIABLE_NAMES_MAY_END_WITH_QUOTES_FLAG)) {
if (charp == (char *)AuxSp - 1024) { if (charp == (char *)AuxSp - 1024) {
goto huge_var_error; goto huge_var_error;
} }
@ -1573,7 +1626,7 @@ huge_var_error:
case QT: case QT:
case DC: case DC:
quoted_string: quoted_string:
TokImage = (char *)((AtomEntry *)(Yap_PreAllocCodeSpace()))->StrOfAE; TokImage = (char *)((AtomEntry *)(Yap_PreAllocCodeSpace()))->StrOfAE;
charp = TokImage; charp = TokImage;
quote = ch; quote = ch;
@ -1634,7 +1687,7 @@ quoted_string:
} else { } else {
*charp = '\0'; *charp = '\0';
} }
if (quote == '"'||quote == '`') { if (quote == '"' || quote == '`') {
if (wcharp) { if (wcharp) {
mp = AllocScannerMemory(sizeof(wchar_t) * (len + 1)); mp = AllocScannerMemory(sizeof(wchar_t) * (len + 1));
} else { } else {
@ -1666,7 +1719,6 @@ quoted_string:
} else { } else {
t->Tok = Ord(kind = BQString_tok); t->Tok = Ord(kind = BQString_tok);
} }
} }
} else { } else {
if (wcharp) { if (wcharp) {
@ -1700,7 +1752,7 @@ quoted_string:
if (ch == '`') if (ch == '`')
goto quoted_string; goto quoted_string;
if (ch == '.') { if (ch == '.') {
int nch = Yap_peek(inp_stream-GLOBAL_Stream); int nch = Yap_peek(inp_stream - GLOBAL_Stream);
if (chtype(nch) == BS || chtype(nch) == EF || nch == '%') { if (chtype(nch) == BS || chtype(nch) == EF || nch == '%') {
t->Tok = Ord(kind = eot_tok); t->Tok = Ord(kind = eot_tok);
if (chtype(ch) == EF) if (chtype(ch) == EF)
@ -1743,15 +1795,14 @@ quoted_string:
ch = getchr(inp_stream); ch = getchr(inp_stream);
} }
CHECK_SPACE(); CHECK_SPACE();
*tposp = Yap_StreamPosition(inp_stream-GLOBAL_Stream); *tposp = Yap_StreamPosition(inp_stream - GLOBAL_Stream);
Yap_setCurrentSourceLocation(inp_stream); Yap_setCurrentSourceLocation(inp_stream);
} }
} }
goto restart; goto restart;
} }
enter_symbol: enter_symbol:
if (och == '.' && if (och == '.' && (chtype(ch) == BS || chtype(ch) == EF || ch == '%')) {
(chtype(ch) == BS || chtype(ch) == EF || ch == '%')) {
t->Tok = Ord(kind = eot_tok); t->Tok = Ord(kind = eot_tok);
if (chtype(ch) == EF) if (chtype(ch) == EF)
mark_eof(inp_stream); mark_eof(inp_stream);
@ -1860,9 +1911,9 @@ enter_symbol:
cur_qq = qq; cur_qq = qq;
} }
t->TokInfo = (CELL)qq; t->TokInfo = (CELL)qq;
if (inp_stream->status & Seekable_Stream_f ) { if (inp_stream->status & Seekable_Stream_f) {
qq->start.byteno = fseek (inp_stream->file, 0, 0); qq->start.byteno = fseek(inp_stream->file, 0, 0);
}else { } else {
qq->start.byteno = inp_stream->charcount - 1; qq->start.byteno = inp_stream->charcount - 1;
} }
qq->start.lineno = inp_stream->linecount; qq->start.lineno = inp_stream->linecount;
@ -1895,9 +1946,9 @@ enter_symbol:
} }
cur_qq = NULL; cur_qq = NULL;
t->TokInfo = (CELL)qq; t->TokInfo = (CELL)qq;
if (inp_stream->status & Seekable_Stream_f ) { if (inp_stream->status & Seekable_Stream_f) {
qq->mid.byteno = fseek (inp_stream->file, 0, 0); qq->mid.byteno = fseek(inp_stream->file, 0, 0);
}else { } else {
qq->mid.byteno = inp_stream->charcount - 1; qq->mid.byteno = inp_stream->charcount - 1;
} }
qq->mid.lineno = inp_stream->linecount; qq->mid.lineno = inp_stream->linecount;
@ -1925,8 +1976,8 @@ enter_symbol:
ch = getchrq(inp_stream); ch = getchrq(inp_stream);
if (ch != '}') { if (ch != '}') {
} else { } else {
charp = ( char *)put_utf8((unsigned char *)charp, och); charp = (char *)put_utf8((unsigned char *)charp, och);
charp = ( char *)put_utf8((unsigned char *)charp, ch); charp = (char *)put_utf8((unsigned char *)charp, ch);
/* we're done */ /* we're done */
break; break;
} }
@ -1936,7 +1987,7 @@ enter_symbol:
t->Tok = Ord(kind = eot_tok); t->Tok = Ord(kind = eot_tok);
break; break;
} else { } else {
charp = ( char *)put_utf8((unsigned char *)charp, ch); charp = (char *)put_utf8((unsigned char *)charp, ch);
ch = getchrq(inp_stream); ch = getchrq(inp_stream);
} }
if (charp > (char *)AuxSp - 1024) { if (charp > (char *)AuxSp - 1024) {
@ -1961,9 +2012,9 @@ enter_symbol:
strncpy(mp, TokImage, len + 1); strncpy(mp, TokImage, len + 1);
qq->text = (unsigned char *)mp; qq->text = (unsigned char *)mp;
Yap_ReleasePreAllocCodeSpace((CODEADDR)TokImage); Yap_ReleasePreAllocCodeSpace((CODEADDR)TokImage);
if (inp_stream->status & Seekable_Stream_f ) { if (inp_stream->status & Seekable_Stream_f) {
qq->end.byteno = fseek (inp_stream->file, 0, 0); qq->end.byteno = fseek(inp_stream->file, 0, 0);
}else { } else {
qq->end.byteno = inp_stream->charcount - 1; qq->end.byteno = inp_stream->charcount - 1;
} }
qq->end.lineno = inp_stream->linecount; qq->end.lineno = inp_stream->linecount;
@ -1989,7 +2040,6 @@ enter_symbol:
t->Tok = Ord(kind = eot_tok); t->Tok = Ord(kind = eot_tok);
return l; return l;
default: default:
#if DEBUG #if DEBUG
fprintf(stderr, "\n++++ token: wrong char type %c %d\n", ch, chtype(ch)); fprintf(stderr, "\n++++ token: wrong char type %c %d\n", ch, chtype(ch));
@ -1998,8 +2048,7 @@ enter_symbol:
} }
#if DEBUG #if DEBUG
if (GLOBAL_Option[2]) if (GLOBAL_Option[2])
fprintf(stderr, "[Token %d %s]", Ord(kind), fprintf(stderr, "[Token %d %s]", Ord(kind), Yap_tokRep(t));
Yap_tokRep( t ));
#endif #endif
if (LOCAL_ErrorMessage) { if (LOCAL_ErrorMessage) {
/* insert an error token to inform the system of what happened */ /* insert an error token to inform the system of what happened */
@ -2024,8 +2073,7 @@ enter_symbol:
return (l); return (l);
} }
void Yap_clean_tokenizer(TokEntry *tokstart, void Yap_clean_tokenizer(TokEntry *tokstart, VarEntry *vartable,
VarEntry *vartable,
VarEntry *anonvartable) { VarEntry *anonvartable) {
CACHE_REGS CACHE_REGS
struct scanner_extra_alloc *ptr = LOCAL_ScannerExtraBlocks; struct scanner_extra_alloc *ptr = LOCAL_ScannerExtraBlocks;