fix get_num init issues
This commit is contained in:
Vítor Santos Costa
parent
5762aa846c
commit
9fb7325f19
238
C/scanner.c
238
C/scanner.c
@ -437,17 +437,15 @@ writing, writing a BOM can be requested using the option
|
||||
static Term float_send(char *, int);
|
||||
static Term get_num(int *, int *, struct stream_desc *, char *, UInt, int);
|
||||
|
||||
static void
|
||||
Yap_setCurrentSourceLocation( struct stream_desc *s )
|
||||
{
|
||||
static void Yap_setCurrentSourceLocation(struct stream_desc *s) {
|
||||
CACHE_REGS
|
||||
#if HAVE_SOCKET
|
||||
#if HAVE_SOCKET
|
||||
if (s->status & Socket_Stream_f)
|
||||
LOCAL_SourceFileName = AtomSocket;
|
||||
else
|
||||
#endif
|
||||
if (s->status & Pipe_Stream_f)
|
||||
LOCAL_SourceFileName =AtomPipe;
|
||||
LOCAL_SourceFileName = AtomPipe;
|
||||
else if (s->status & InMemory_Stream_f)
|
||||
LOCAL_SourceFileName = AtomCharsio;
|
||||
else
|
||||
@ -469,7 +467,8 @@ static char chtype0[NUMBER_OF_CHARS + 1] = {
|
||||
BS, BS, BS, BS, BS, BS, BS,
|
||||
|
||||
/* sp ! " # $ % & ' ( ) * + , - . / */
|
||||
BS, SL, DC, SY, SY, CC, SY, QT, BK, BK, SY, SY, BK, SY, SY, SY,
|
||||
BS, SL, DC, SY, SY, CC, SY, QT, BK,
|
||||
BK, SY, SY, BK, SY, SY, SY,
|
||||
|
||||
/* 0 1 2 3 4 5 6 7 8 9 : ; < = > ? */
|
||||
NU, NU, NU, NU, NU, NU, NU, NU, NU,
|
||||
@ -513,27 +512,27 @@ static char chtype0[NUMBER_OF_CHARS + 1] = {
|
||||
UC, UC, UC, UC, UC, UC, UC, UC, UC,
|
||||
UC, UC, UC, UC, UC, UC, UC,
|
||||
|
||||
/* Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß */
|
||||
#ifdef vms
|
||||
/* Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß */
|
||||
#ifdef vms
|
||||
UC, UC, UC, UC, UC, UC, UC, UC, UC,
|
||||
UC, UC, UC, UC, UC, UC, LC,
|
||||
#else
|
||||
#else
|
||||
UC, UC, UC, UC, UC, UC, UC, SY, UC,
|
||||
UC, UC, UC, UC, UC, UC, LC,
|
||||
#endif
|
||||
#endif
|
||||
/* à á â ã ä å æ ç è é ê ë ì í î ï */
|
||||
LC, LC, LC, LC, LC, LC, LC, LC, LC,
|
||||
LC, LC, LC, LC, LC, LC, LC,
|
||||
|
||||
/* ð ñ ò ó ô õ ö ÷ ø ù ú û ü cannot write the last
|
||||
/* ð ñ ò ó ô õ ö ÷ ø ù ú û ü cannot write the last
|
||||
* three because of lcc */
|
||||
#ifdef vms
|
||||
#ifdef vms
|
||||
LC, LC, LC, LC, LC, LC, LC, LC, LC,
|
||||
LC, LC, LC, LC, LC, LC, LC
|
||||
#else
|
||||
#else
|
||||
LC, LC, LC, LC, LC, LC, LC, SY, LC,
|
||||
LC, LC, LC, LC, LC, LC, LC
|
||||
#endif
|
||||
#endif
|
||||
};
|
||||
|
||||
typedef struct scanner_internals {
|
||||
@ -567,13 +566,13 @@ int Yap_wide_chtype(Int ch) {
|
||||
|
||||
// standard get char, uses conversion table
|
||||
// and converts to wide
|
||||
#define getchr(inp) inp->stream_wgetc_for_read(inp-GLOBAL_Stream)
|
||||
#define getchr(inp) inp->stream_wgetc_for_read(inp - GLOBAL_Stream)
|
||||
// get char for quoted data, eg, quoted atoms and so on
|
||||
// converts to wide
|
||||
#define getchrq(inp) inp->stream_wgetc(inp-GLOBAL_Stream)
|
||||
#define getchrq(inp) inp->stream_wgetc(inp - GLOBAL_Stream)
|
||||
// get char for UTF-8 quoted data, eg, quoted strings
|
||||
// reads bytes
|
||||
#define getchru(inp) inp->stream_getc_utf8(inp-GLOBAL_Stream)
|
||||
#define getchru(inp) inp->stream_getc_utf8(inp - GLOBAL_Stream)
|
||||
|
||||
/* in case there is an overflow */
|
||||
typedef struct scanner_extra_alloc {
|
||||
@ -581,6 +580,14 @@ typedef struct scanner_extra_alloc {
|
||||
void *filler;
|
||||
} ScannerExtraBlock;
|
||||
|
||||
static void InitScannerMemory(void) {
|
||||
CACHE_REGS
|
||||
LOCAL_ErrorMessage = NULL;
|
||||
LOCAL_Error_Size = 0;
|
||||
LOCAL_ScannerStack = (char *)TR;
|
||||
LOCAL_ScannerExtraBlocks = NULL;
|
||||
}
|
||||
|
||||
static char *AllocScannerMemory(unsigned int size) {
|
||||
CACHE_REGS
|
||||
char *AuxSpScan;
|
||||
@ -687,10 +694,11 @@ static int send_error_message(char s[]) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static wchar_t read_quoted_char(int *scan_nextp, struct stream_desc* inp_stream) {
|
||||
static wchar_t read_quoted_char(int *scan_nextp,
|
||||
struct stream_desc *inp_stream) {
|
||||
int ch;
|
||||
|
||||
/* escape sequence */
|
||||
/* escape sequence */
|
||||
do_switch:
|
||||
ch = getchrq(inp_stream);
|
||||
switch (ch) {
|
||||
@ -1095,38 +1103,91 @@ Term Yap_scan_num(StreamDesc *inp) {
|
||||
Term out;
|
||||
int sign = 1;
|
||||
int ch, cherr;
|
||||
char *ptr;
|
||||
char *ptr, *mp;
|
||||
int kind;
|
||||
void *old_tr = TR;
|
||||
|
||||
LOCAL_ErrorMessage = NULL;
|
||||
LOCAL_ScannerStack = (char *)TR;
|
||||
LOCAL_ScannerExtraBlocks = NULL;
|
||||
InitScannerMemory();
|
||||
LOCAL_VarTable = LOCAL_AnonVarTable = NULL;
|
||||
if (!(ptr = AllocScannerMemory(4096))) {
|
||||
LOCAL_ErrorMessage = "Trail Overflow";
|
||||
LOCAL_Error_TYPE = RESOURCE_ERROR_TRAIL;
|
||||
return TermNil;
|
||||
return 0;
|
||||
}
|
||||
TokEntry *tokptr = (TokEntry *)AllocScannerMemory(sizeof(TokEntry));
|
||||
tokptr->TokPos = GetCurInpPos(inp);
|
||||
ch = getchr(inp);
|
||||
while (chtype(ch) == BS) {
|
||||
ch = getchr(inp);
|
||||
}
|
||||
if (ch == '-') {
|
||||
sign = -1;
|
||||
ch = getchr(inp);
|
||||
} else if (ch == '+') {
|
||||
ch = getchr(inp);
|
||||
}
|
||||
if (chtype(ch) != NU) {
|
||||
Yap_clean_tokenizer((TokEntry *)LOCAL_ScannerStack, NULL, NULL);
|
||||
return TermNil;
|
||||
}
|
||||
if (chtype(ch) == NU) {
|
||||
cherr = '\0';
|
||||
if (ASP - HR < 1024)
|
||||
return TermNil;
|
||||
if (ASP - HR < 1024) {
|
||||
Yap_clean_tokenizer(old_tr, NULL, NULL);
|
||||
LOCAL_ErrorMessage = "Stack Overflow";
|
||||
LOCAL_Error_TYPE = RESOURCE_ERROR_STACK;
|
||||
return 0;
|
||||
}
|
||||
out = get_num(&ch, &cherr, inp, ptr, 4096, sign); /* */
|
||||
}
|
||||
if (LOCAL_ErrorMessage != NULL || ch != -1 || cherr) {
|
||||
CACHE_REGS
|
||||
char *s = ptr;
|
||||
int sign = 1;
|
||||
|
||||
out = 0;
|
||||
if (s[0] == '+') {
|
||||
s++;
|
||||
}
|
||||
if (s[0] == '-') {
|
||||
s++;
|
||||
sign = -1;
|
||||
}
|
||||
if (strcmp(s, "inf") == 0) {
|
||||
if (sign > 0) {
|
||||
out = MkFloatTerm(INFINITY);
|
||||
} else {
|
||||
out = MkFloatTerm(-INFINITY);
|
||||
}
|
||||
}
|
||||
if (strcmp(s, "nan") == 0) {
|
||||
if (sign > 0) {
|
||||
out = MkFloatTerm(NAN);
|
||||
} else {
|
||||
out = MkFloatTerm(-NAN);
|
||||
}
|
||||
}
|
||||
if (out == 0) {
|
||||
TokEntry *e, *ef;
|
||||
size_t len = strlen(ptr);
|
||||
mp = AllocScannerMemory(len + 1);
|
||||
tokptr->Tok = Ord(kind = String_tok);
|
||||
tokptr->TokInfo = Unsigned(mp);
|
||||
e = (TokEntry *)AllocScannerMemory(sizeof(TokEntry));
|
||||
ef = (TokEntry *)AllocScannerMemory(sizeof(TokEntry));
|
||||
tokptr->TokNext = e;
|
||||
e->Tok = Error_tok;
|
||||
if (!LOCAL_ErrorMessage)
|
||||
LOCAL_ErrorMessage =
|
||||
"syntax error while converting from a string to a number";
|
||||
e->TokInfo = MkAtomTerm(Yap_LookupAtom(LOCAL_ErrorMessage));
|
||||
e->TokPos = GetCurInpPos(inp);
|
||||
e->TokNext = ef;
|
||||
ef->Tok = Ord(kind = eot_tok);
|
||||
ef->TokPos = GetCurInpPos(inp);
|
||||
ef->TokNext = NULL;
|
||||
LOCAL_tokptr = tokptr;
|
||||
LOCAL_toktide = e;
|
||||
LOCAL_ErrorMessage = NULL;
|
||||
LOCAL_Error_Term = Yap_syntax_error(e, inp - GLOBAL_Stream);
|
||||
LOCAL_Error_TYPE = SYNTAX_ERROR;
|
||||
}
|
||||
}
|
||||
PopScannerMemory(ptr, 4096);
|
||||
Yap_clean_tokenizer((TokEntry *)LOCAL_ScannerStack, NULL, NULL);
|
||||
if (LOCAL_ErrorMessage != NULL || ch != -1 || cherr)
|
||||
return TermNil;
|
||||
Yap_clean_tokenizer(old_tr, NULL, NULL);
|
||||
return out;
|
||||
}
|
||||
|
||||
@ -1141,26 +1202,25 @@ Term Yap_scan_num(StreamDesc *inp) {
|
||||
return l; \
|
||||
}
|
||||
|
||||
const char *
|
||||
Yap_tokRep(TokEntry *tokptr)
|
||||
{
|
||||
const char *Yap_tokRep(TokEntry *tokptr) {
|
||||
CACHE_REGS
|
||||
Term info = tokptr->TokInfo;
|
||||
char *b, *buf = LOCAL_FileNameBuf2;
|
||||
size_t length, sze = YAP_FILENAME_MAX-1;
|
||||
size_t length, sze = YAP_FILENAME_MAX - 1;
|
||||
UInt flags = 0;
|
||||
|
||||
switch (tokptr->Tok) {
|
||||
case Name_tok:
|
||||
return (char *)RepAtom((Atom)info)->StrOfAE;
|
||||
case Number_tok:
|
||||
if ((b = Yap_TermToString(info, buf, sze, &length, &LOCAL_encoding, flags)) != buf) {
|
||||
if (b) free(b);
|
||||
if ((b = Yap_TermToString(info, buf, sze, &length, &LOCAL_encoding,
|
||||
flags)) != buf) {
|
||||
if (b)
|
||||
free(b);
|
||||
return NULL;
|
||||
}
|
||||
return buf;
|
||||
case Var_tok:
|
||||
{
|
||||
case Var_tok: {
|
||||
VarEntry *varinfo = (VarEntry *)info;
|
||||
return varinfo->VarRep;
|
||||
}
|
||||
@ -1168,22 +1228,21 @@ Yap_tokRep(TokEntry *tokptr)
|
||||
case BQString_tok:
|
||||
return (char *)info;
|
||||
case WString_tok:
|
||||
case WBQString_tok:
|
||||
{ wchar_t *op = (wchar_t *)info;
|
||||
case WBQString_tok: {
|
||||
wchar_t *op = (wchar_t *)info;
|
||||
wchar_t c;
|
||||
unsigned char *bp = (unsigned char *)buf;
|
||||
while ((c=*op++) ){
|
||||
while ((c = *op++)) {
|
||||
bp += put_utf8(bp, c);
|
||||
}
|
||||
bp[0]='\0';
|
||||
bp[0] = '\0';
|
||||
return buf;
|
||||
}
|
||||
case Error_tok:
|
||||
return "<ERR>";
|
||||
case eot_tok:
|
||||
return "<EOT>";
|
||||
case Ponctuation_tok:
|
||||
{
|
||||
case Ponctuation_tok: {
|
||||
buf[1] = '\0';
|
||||
if ((info) == 'l') {
|
||||
buf[0] = '(';
|
||||
@ -1198,7 +1257,6 @@ Yap_tokRep(TokEntry *tokptr)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void open_comment(int ch, StreamDesc *inp_stream USES_REGS) {
|
||||
CELL *h0 = HR;
|
||||
HR += 5;
|
||||
@ -1214,8 +1272,7 @@ static void open_comment(int ch, StreamDesc *inp_stream USES_REGS) {
|
||||
LOCAL_CommentsTail = h0 + 1;
|
||||
h0 += 2;
|
||||
h0[0] = (CELL)FunctorMinus;
|
||||
h0[1] = Yap_StreamPosition(inp_stream-GLOBAL_Stream
|
||||
);
|
||||
h0[1] = Yap_StreamPosition(inp_stream - GLOBAL_Stream);
|
||||
h0[2] = TermNil;
|
||||
LOCAL_CommentsNextChar = h0 + 2;
|
||||
LOCAL_CommentsBuff = (wchar_t *)malloc(1024 * sizeof(wchar_t));
|
||||
@ -1244,9 +1301,7 @@ static void close_comment(USES_REGS1) {
|
||||
|
||||
// mark that we reached EOF,
|
||||
// next token will be end_of_file)
|
||||
static void
|
||||
mark_eof( struct stream_desc * inp_stream )
|
||||
{
|
||||
static void mark_eof(struct stream_desc *inp_stream) {
|
||||
inp_stream->status |= Push_Eof_Stream_f;
|
||||
}
|
||||
|
||||
@ -1289,7 +1344,7 @@ static wchar_t *ch_to_wide(char *base, char *charp) {
|
||||
}
|
||||
|
||||
#define add_ch_to_utf8_buff(ch) \
|
||||
{ \
|
||||
{ \
|
||||
if ((ch & 0xff) == ch) { \
|
||||
*charp++ = ch; \
|
||||
} else { \
|
||||
@ -1297,8 +1352,8 @@ static wchar_t *ch_to_wide(char *base, char *charp) {
|
||||
} \
|
||||
}
|
||||
|
||||
TokEntry *Yap_tokenizer( struct stream_desc *inp_stream,
|
||||
bool store_comments, Term *tposp) {
|
||||
TokEntry *Yap_tokenizer(struct stream_desc *inp_stream, bool store_comments,
|
||||
Term *tposp) {
|
||||
|
||||
CACHE_REGS
|
||||
TokEntry *t, *l, *p;
|
||||
@ -1308,19 +1363,16 @@ TokEntry *Yap_tokenizer( struct stream_desc *inp_stream,
|
||||
wchar_t *wcharp;
|
||||
struct qq_struct_t *cur_qq = NULL;
|
||||
|
||||
LOCAL_ErrorMessage = NULL;
|
||||
LOCAL_Error_Size = 0;
|
||||
InitScannerMemory();
|
||||
LOCAL_VarTable = NULL;
|
||||
LOCAL_AnonVarTable = NULL;
|
||||
LOCAL_ScannerStack = (char *)TR;
|
||||
LOCAL_ScannerExtraBlocks = NULL;
|
||||
l = NULL;
|
||||
p = NULL; /* Just to make lint happy */
|
||||
ch = getchr(inp_stream);
|
||||
while (chtype(ch) == BS) {
|
||||
ch = getchr(inp_stream);
|
||||
}
|
||||
*tposp = Yap_StreamPosition(inp_stream-GLOBAL_Stream);
|
||||
*tposp = Yap_StreamPosition(inp_stream - GLOBAL_Stream);
|
||||
Yap_setCurrentSourceLocation(inp_stream);
|
||||
LOCAL_StartLineCount = inp_stream->linecount;
|
||||
LOCAL_StartLinePos = inp_stream->linepos;
|
||||
@ -1346,7 +1398,7 @@ TokEntry *Yap_tokenizer( struct stream_desc *inp_stream,
|
||||
else
|
||||
p->TokNext = t;
|
||||
p = t;
|
||||
restart:
|
||||
restart:
|
||||
while (chtype(ch) == BS) {
|
||||
ch = getchr(inp_stream);
|
||||
}
|
||||
@ -1358,7 +1410,7 @@ restart:
|
||||
if (store_comments) {
|
||||
CHECK_SPACE();
|
||||
open_comment(ch, inp_stream PASS_REGS);
|
||||
continue_comment:
|
||||
continue_comment:
|
||||
while ((ch = getchr(inp_stream)) != 10 && chtype(ch) != EF) {
|
||||
CHECK_SPACE();
|
||||
extend_comment(ch PASS_REGS);
|
||||
@ -1385,7 +1437,7 @@ continue_comment:
|
||||
ch = getchr(inp_stream);
|
||||
}
|
||||
CHECK_SPACE();
|
||||
*tposp = Yap_StreamPosition(inp_stream-GLOBAL_Stream);
|
||||
*tposp = Yap_StreamPosition(inp_stream - GLOBAL_Stream);
|
||||
Yap_setCurrentSourceLocation(inp_stream);
|
||||
}
|
||||
goto restart;
|
||||
@ -1400,7 +1452,7 @@ continue_comment:
|
||||
case LC:
|
||||
och = ch;
|
||||
ch = getchr(inp_stream);
|
||||
scan_name:
|
||||
scan_name:
|
||||
TokImage = (char *)((AtomEntry *)(Yap_PreAllocCodeSpace()))->StrOfAE;
|
||||
charp = TokImage;
|
||||
wcharp = NULL;
|
||||
@ -1408,7 +1460,7 @@ scan_name:
|
||||
add_ch_to_buff(och);
|
||||
for (; chtype(ch) <= NU; ch = getchr(inp_stream)) {
|
||||
if (charp == (char *)AuxSp - 1024) {
|
||||
huge_var_error:
|
||||
huge_var_error:
|
||||
/* huge atom or variable, we are in trouble */
|
||||
LOCAL_ErrorMessage = "Code Space Overflow due to huge atom";
|
||||
LOCAL_Error_TYPE = RESOURCE_ERROR_AUXILIARY_STACK;
|
||||
@ -1420,7 +1472,8 @@ huge_var_error:
|
||||
}
|
||||
add_ch_to_buff(ch);
|
||||
}
|
||||
while (ch == '\'' && isvar &&trueGlobalPrologFlag(VARIABLE_NAMES_MAY_END_WITH_QUOTES_FLAG)) {
|
||||
while (ch == '\'' && isvar &&
|
||||
trueGlobalPrologFlag(VARIABLE_NAMES_MAY_END_WITH_QUOTES_FLAG)) {
|
||||
if (charp == (char *)AuxSp - 1024) {
|
||||
goto huge_var_error;
|
||||
}
|
||||
@ -1573,7 +1626,7 @@ huge_var_error:
|
||||
|
||||
case QT:
|
||||
case DC:
|
||||
quoted_string:
|
||||
quoted_string:
|
||||
TokImage = (char *)((AtomEntry *)(Yap_PreAllocCodeSpace()))->StrOfAE;
|
||||
charp = TokImage;
|
||||
quote = ch;
|
||||
@ -1634,7 +1687,7 @@ quoted_string:
|
||||
} else {
|
||||
*charp = '\0';
|
||||
}
|
||||
if (quote == '"'||quote == '`') {
|
||||
if (quote == '"' || quote == '`') {
|
||||
if (wcharp) {
|
||||
mp = AllocScannerMemory(sizeof(wchar_t) * (len + 1));
|
||||
} else {
|
||||
@ -1666,7 +1719,6 @@ quoted_string:
|
||||
} else {
|
||||
t->Tok = Ord(kind = BQString_tok);
|
||||
}
|
||||
|
||||
}
|
||||
} else {
|
||||
if (wcharp) {
|
||||
@ -1700,7 +1752,7 @@ quoted_string:
|
||||
if (ch == '`')
|
||||
goto quoted_string;
|
||||
if (ch == '.') {
|
||||
int nch = Yap_peek(inp_stream-GLOBAL_Stream);
|
||||
int nch = Yap_peek(inp_stream - GLOBAL_Stream);
|
||||
if (chtype(nch) == BS || chtype(nch) == EF || nch == '%') {
|
||||
t->Tok = Ord(kind = eot_tok);
|
||||
if (chtype(ch) == EF)
|
||||
@ -1743,15 +1795,14 @@ quoted_string:
|
||||
ch = getchr(inp_stream);
|
||||
}
|
||||
CHECK_SPACE();
|
||||
*tposp = Yap_StreamPosition(inp_stream-GLOBAL_Stream);
|
||||
*tposp = Yap_StreamPosition(inp_stream - GLOBAL_Stream);
|
||||
Yap_setCurrentSourceLocation(inp_stream);
|
||||
}
|
||||
}
|
||||
goto restart;
|
||||
}
|
||||
enter_symbol:
|
||||
if (och == '.' &&
|
||||
(chtype(ch) == BS || chtype(ch) == EF || ch == '%')) {
|
||||
enter_symbol:
|
||||
if (och == '.' && (chtype(ch) == BS || chtype(ch) == EF || ch == '%')) {
|
||||
t->Tok = Ord(kind = eot_tok);
|
||||
if (chtype(ch) == EF)
|
||||
mark_eof(inp_stream);
|
||||
@ -1860,9 +1911,9 @@ enter_symbol:
|
||||
cur_qq = qq;
|
||||
}
|
||||
t->TokInfo = (CELL)qq;
|
||||
if (inp_stream->status & Seekable_Stream_f ) {
|
||||
qq->start.byteno = fseek (inp_stream->file, 0, 0);
|
||||
}else {
|
||||
if (inp_stream->status & Seekable_Stream_f) {
|
||||
qq->start.byteno = fseek(inp_stream->file, 0, 0);
|
||||
} else {
|
||||
qq->start.byteno = inp_stream->charcount - 1;
|
||||
}
|
||||
qq->start.lineno = inp_stream->linecount;
|
||||
@ -1895,9 +1946,9 @@ enter_symbol:
|
||||
}
|
||||
cur_qq = NULL;
|
||||
t->TokInfo = (CELL)qq;
|
||||
if (inp_stream->status & Seekable_Stream_f ) {
|
||||
qq->mid.byteno = fseek (inp_stream->file, 0, 0);
|
||||
}else {
|
||||
if (inp_stream->status & Seekable_Stream_f) {
|
||||
qq->mid.byteno = fseek(inp_stream->file, 0, 0);
|
||||
} else {
|
||||
qq->mid.byteno = inp_stream->charcount - 1;
|
||||
}
|
||||
qq->mid.lineno = inp_stream->linecount;
|
||||
@ -1925,8 +1976,8 @@ enter_symbol:
|
||||
ch = getchrq(inp_stream);
|
||||
if (ch != '}') {
|
||||
} else {
|
||||
charp = ( char *)put_utf8((unsigned char *)charp, och);
|
||||
charp = ( char *)put_utf8((unsigned char *)charp, ch);
|
||||
charp = (char *)put_utf8((unsigned char *)charp, och);
|
||||
charp = (char *)put_utf8((unsigned char *)charp, ch);
|
||||
/* we're done */
|
||||
break;
|
||||
}
|
||||
@ -1936,7 +1987,7 @@ enter_symbol:
|
||||
t->Tok = Ord(kind = eot_tok);
|
||||
break;
|
||||
} else {
|
||||
charp = ( char *)put_utf8((unsigned char *)charp, ch);
|
||||
charp = (char *)put_utf8((unsigned char *)charp, ch);
|
||||
ch = getchrq(inp_stream);
|
||||
}
|
||||
if (charp > (char *)AuxSp - 1024) {
|
||||
@ -1961,9 +2012,9 @@ enter_symbol:
|
||||
strncpy(mp, TokImage, len + 1);
|
||||
qq->text = (unsigned char *)mp;
|
||||
Yap_ReleasePreAllocCodeSpace((CODEADDR)TokImage);
|
||||
if (inp_stream->status & Seekable_Stream_f ) {
|
||||
qq->end.byteno = fseek (inp_stream->file, 0, 0);
|
||||
}else {
|
||||
if (inp_stream->status & Seekable_Stream_f) {
|
||||
qq->end.byteno = fseek(inp_stream->file, 0, 0);
|
||||
} else {
|
||||
qq->end.byteno = inp_stream->charcount - 1;
|
||||
}
|
||||
qq->end.lineno = inp_stream->linecount;
|
||||
@ -1989,7 +2040,6 @@ enter_symbol:
|
||||
t->Tok = Ord(kind = eot_tok);
|
||||
return l;
|
||||
|
||||
|
||||
default:
|
||||
#if DEBUG
|
||||
fprintf(stderr, "\n++++ token: wrong char type %c %d\n", ch, chtype(ch));
|
||||
@ -1998,8 +2048,7 @@ enter_symbol:
|
||||
}
|
||||
#if DEBUG
|
||||
if (GLOBAL_Option[2])
|
||||
fprintf(stderr, "[Token %d %s]", Ord(kind),
|
||||
Yap_tokRep( t ));
|
||||
fprintf(stderr, "[Token %d %s]", Ord(kind), Yap_tokRep(t));
|
||||
#endif
|
||||
if (LOCAL_ErrorMessage) {
|
||||
/* insert an error token to inform the system of what happened */
|
||||
@ -2024,8 +2073,7 @@ enter_symbol:
|
||||
return (l);
|
||||
}
|
||||
|
||||
void Yap_clean_tokenizer(TokEntry *tokstart,
|
||||
VarEntry *vartable,
|
||||
void Yap_clean_tokenizer(TokEntry *tokstart, VarEntry *vartable,
|
||||
VarEntry *anonvartable) {
|
||||
CACHE_REGS
|
||||
struct scanner_extra_alloc *ptr = LOCAL_ScannerExtraBlocks;
|
||||
|
||||
Reference in New Issue
Block a user