diogo/yap-6.3
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

support for option comments(X) in read_term/1.

Browse Source
This commit is contained in:
Vitor Santos Costa
2011-06-12 17:23:10 +01:00
parent 787ca8fd05
commit 261e02b43e
12 changed files with 201 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
C/c_interface.c
View File

@@ -2527,20 +2527,20 @@ YAP_Read(IOSTREAM *inp)
BACKUP_MACHINE_REGS();
tokstart = LOCAL_tokptr = LOCAL_toktide = Yap_tokenizer(inp, &tpos);
tokstart = LOCAL_tokptr = LOCAL_toktide = Yap_tokenizer(inp, FALSE, &tpos);
if (LOCAL_ErrorMessage)
{
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
RECOVER_MACHINE_REGS();
return 0;
}
if (inp->flags & (SIO_FEOF|SIO_FEOF2)) {
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
RECOVER_MACHINE_REGS();
return MkAtomTerm (AtomEof);
}
t = Yap_Parse();
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
RECOVER_MACHINE_REGS();
return t;

55
C/iopreds.c
View File

@@ -402,19 +402,19 @@ Yap_StringToTerm(char *s,Term *tp)
if (sno == NULL)
return FALSE;
TR_before_parse = TR;
tokstart = LOCAL_tokptr = LOCAL_toktide = Yap_tokenizer(sno, &tpos);
tokstart = LOCAL_tokptr = LOCAL_toktide = Yap_tokenizer(sno, FALSE, &tpos);
if (tokstart == NIL || tokstart->Tok == Ord (eot_tok)) {
if (tp) {
*tp = MkAtomTerm(AtomEOFBeforeEOT);
}
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
Sclose(sno);
return FALSE;
} else if (LOCAL_ErrorMessage) {
if (tp) {
*tp = MkAtomTerm(Yap_LookupAtom(LOCAL_ErrorMessage));
}
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
Sclose(sno);
return FALSE;
}
@@ -422,11 +422,11 @@ Yap_StringToTerm(char *s,Term *tp)
TR = TR_before_parse;
if (!t || LOCAL_ErrorMessage) {
GenerateSyntaxError(tp, tokstart, sno PASS_REGS);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
Sclose(sno);
return FALSE;
}
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
Sclose(sno);
return t;
}
@@ -512,25 +512,25 @@ Yap_readTerm(void *st0, Term *tp, Term *varnames, Term *terror, Term *tpos)
if (st == NULL) {
return FALSE;
}
tokstart = LOCAL_tokptr = LOCAL_toktide = Yap_tokenizer(st, tpos);
tokstart = LOCAL_tokptr = LOCAL_toktide = Yap_tokenizer(st, FALSE, tpos);
if (LOCAL_ErrorMessage)
{
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
if (terror)
*terror = MkAtomTerm(Yap_LookupAtom(LOCAL_ErrorMessage));
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
return FALSE;
}
pt = Yap_Parse();
if (LOCAL_ErrorMessage || pt == (CELL)0) {
GenerateSyntaxError(terror, tokstart, st PASS_REGS);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
return FALSE;
}
if (varnames) {
*varnames = Yap_VarNames(LOCAL_VarTable, TermNil);
if (!*varnames) {
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
return FALSE;
}
}
@@ -548,6 +548,7 @@ Yap_readTerm(void *st0, Term *tp, Term *varnames, Term *terror, Term *tpos)
Vars: ARG4
Pos: ARG5
Err: ARG6
Comments: ARG7
*/
static Int
do_read(IOSTREAM *inp_stream, int nargs USES_REGS)
@@ -556,6 +557,8 @@ static Int
TokEntry *tokstart;
Term tmod = Deref(ARG3), OCurrentModule = CurrentModule, tpos;
extern void Yap_setCurrentSourceLocation(IOSTREAM **s);
Term tcomms = Deref(ARG7);
int store_comments = IsVarTerm(tcomms);
Yap_setCurrentSourceLocation(&inp_stream);
if (IsVarTerm(tmod)) {
@@ -583,10 +586,12 @@ static Int
while (TRUE) {
old_H = H;
tpos = Yap_StreamPosition(inp_stream);
tokstart = LOCAL_tokptr = LOCAL_toktide = Yap_tokenizer(inp_stream, &tpos);
LOCAL_Comments = TermNil;
LOCAL_CommentsNextChar = LOCAL_CommentsTail = NULL;
tokstart = LOCAL_tokptr = LOCAL_toktide = Yap_tokenizer(inp_stream, store_comments, &tpos);
if (LOCAL_Error_TYPE != YAP_NO_ERROR && seekable) {
H = old_H;
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
if (seekable) {
Sseek64(inp_stream, cpos, SIO_SEEK_SET);
}
@@ -624,10 +629,10 @@ static Int
/* did we get the end of file from an abort? */
if (LOCAL_ErrorMessage &&
!strcmp(LOCAL_ErrorMessage,"Abort")) {
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
return FALSE;
} else {
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
return Yap_unify_constant(ARG2, MkAtomTerm (AtomEof))
&& Yap_unify_constant(ARG4, TermNil);
@@ -670,7 +675,7 @@ static Int
}
if (ParserErrorStyle == QUIET_ON_PARSER_ERROR) {
/* just fail */
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
return FALSE;
} else if (ParserErrorStyle == CONTINUE_ON_PARSER_ERROR) {
LOCAL_ErrorMessage = NULL;
@@ -682,14 +687,14 @@ static Int
LOCAL_ErrorMessage = "SYNTAX ERROR";
if (ParserErrorStyle == EXCEPTION_ON_PARSER_ERROR) {
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
Yap_Error(SYNTAX_ERROR,terr,LOCAL_ErrorMessage);
return FALSE;
} else /* FAIL ON PARSER ERROR */ {
Term t[2];
t[0] = terr;
t[1] = MkAtomTerm(Yap_LookupAtom(LOCAL_ErrorMessage));
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
return Yap_unify(ARG6,Yap_MkApplTerm(Yap_MkFunctor(AtomError,2),2,t));
}
}
@@ -701,6 +706,8 @@ static Int
}
if (!Yap_unify(t, ARG2))
return FALSE;
if (store_comments && !Yap_unify(LOCAL_Comments, ARG7))
return FALSE;
if (AtomOfTerm (Deref (ARG1)) == AtomTrue) {
while (TRUE) {
CELL *old_H = H;
@@ -721,10 +728,10 @@ static Int
TR = old_TR;
}
}
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
return Yap_unify (v, ARG4);
} else {
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable);
Yap_clean_tokenizer(tokstart, LOCAL_VarTable, LOCAL_AnonVarTable, LOCAL_Comments);
return TRUE;
}
}
@@ -732,7 +739,7 @@ static Int
static Int
p_read ( USES_REGS1 )
{ /* '$read'(+Flag,?Term,?Module,?Vars,-Pos,-Err) */
return do_read(NULL, 6 PASS_REGS);
return do_read(NULL, 7 PASS_REGS);
}
extern int Yap_getInputStream(Int, IOSTREAM **);
@@ -743,10 +750,10 @@ p_read2 ( USES_REGS1 )
IOSTREAM *inp_stream;
Int out;
if (!Yap_getInputStream(Yap_InitSlot(Deref(ARG7) PASS_REGS), &inp_stream)) {
if (!Yap_getInputStream(Yap_InitSlot(Deref(ARG8) PASS_REGS), &inp_stream)) {
return(FALSE);
}
out = do_read(inp_stream, 7 PASS_REGS);
out = do_read(inp_stream, 8 PASS_REGS);
return out;
}
@@ -1108,8 +1115,8 @@ Yap_InitIOPreds(void)
/* here the Input/Output predicates */
Yap_InitCPred ("$set_read_error_handler", 1, p_set_read_error_handler, SafePredFlag|SyncPredFlag|HiddenPredFlag);
Yap_InitCPred ("$get_read_error_handler", 1, p_get_read_error_handler, SafePredFlag|SyncPredFlag|HiddenPredFlag);
Yap_InitCPred ("$read", 6, p_read, SyncPredFlag|HiddenPredFlag|UserCPredFlag);
Yap_InitCPred ("$read", 7, p_read2, SyncPredFlag|HiddenPredFlag|UserCPredFlag);
Yap_InitCPred ("$read", 7, p_read, SyncPredFlag|HiddenPredFlag|UserCPredFlag);
Yap_InitCPred ("$read", 8, p_read2, SyncPredFlag|HiddenPredFlag|UserCPredFlag);
Yap_InitCPred ("$start_line", 1, p_startline, SafePredFlag|SyncPredFlag|HiddenPredFlag);
Yap_InitCPred ("$change_type_of_char", 2, p_change_type_of_char, SafePredFlag|SyncPredFlag|HiddenPredFlag);
Yap_InitCPred ("$type_of_char", 2, p_type_of_char, SafePredFlag|SyncPredFlag|HiddenPredFlag);

140
C/scanner.c
View File

@@ -8,7 +8,7 @@
* *
**************************************************************************
* *
* File: %W% %G% *
* File: %W% %G% *
* Last rev: 22-1-03 *
* mods: *
* comments: Prolog's scanner *
@@ -746,7 +746,7 @@ Yap_scan_num(IOSTREAM *inp)
ch = getchr(inp);
}
if (chtype(ch) != NU) {
Yap_clean_tokenizer(NULL, NULL, NULL);
Yap_clean_tokenizer(NULL, NULL, NULL, 0L);
return TermNil;
}
cherr = '\0';
@@ -754,13 +754,66 @@ Yap_scan_num(IOSTREAM *inp)
return TermNil;
out = get_num(&ch, &cherr, inp, ptr, 4096, sign); /* */
PopScannerMemory(ptr, 4096);
Yap_clean_tokenizer(NULL, NULL, NULL);
Yap_clean_tokenizer(NULL, NULL, NULL, 0L);
if (LOCAL_ErrorMessage != NULL || ch != -1 || cherr)
return TermNil;
return out;
}
#define CHECK_SPACE() \
if (ASP-H < 1024) { \
LOCAL_ErrorMessage = "Stack Overflow"; \
LOCAL_Error_TYPE = OUT_OF_STACK_ERROR; \
LOCAL_Error_Size = 0L; \
if (p) \
p->Tok = Ord(kind = eot_tok); \
/* serious error now */ \
return l; \
}
static void
open_comment(int ch, IOSTREAM *inp_stream) {
CELL *h0 = H;
H += 5;
h0[0] = AbsAppl(h0+2);
h0[1] = TermNil;
if (!LOCAL_CommentsTail) {
/* first comment */
LOCAL_Comments = AbsPair(h0);
} else {
/* extra comment */
*LOCAL_CommentsTail = AbsPair(h0);
}
LOCAL_CommentsTail = h0+1;
h0 += 2;
h0[0] = (CELL)FunctorMinus;
h0[1] = Yap_StreamPosition(inp_stream);
h0[2] = TermNil;
LOCAL_CommentsNextChar = h0+2;
LOCAL_CommentsBuff = (wchar_t *)malloc(1024*sizeof(wchar_t));
LOCAL_CommentsBuffLim = 1024;
LOCAL_CommentsBuff[0] = ch;
LOCAL_CommentsBuffPos = 1;
}
static void
extend_comment(int ch) {
LOCAL_CommentsBuff[LOCAL_CommentsBuffPos] = ch;
LOCAL_CommentsBuffPos++;
if (LOCAL_CommentsBuffPos == LOCAL_CommentsBuffLim-1) {
LOCAL_CommentsBuff = (wchar_t *)realloc(LOCAL_CommentsBuff,sizeof(wchar_t)*(LOCAL_CommentsBuffLim+4096));
}
}
static void
close_comment(void) {
LOCAL_CommentsBuff[LOCAL_CommentsBuffPos] = '\0';
*LOCAL_CommentsNextChar = Yap_MkBlobWideStringTerm(LOCAL_CommentsBuff, LOCAL_CommentsBuffPos);
free(LOCAL_CommentsBuff);
}
static wchar_t *
ch_to_wide(char *base, char *charp)
{
@@ -791,7 +844,7 @@ ch_to_wide(char *base, char *charp)
}
TokEntry *
Yap_tokenizer(IOSTREAM *inp_stream, Term *tposp)
Yap_tokenizer(IOSTREAM *inp_stream, int store_comments, Term *tposp)
{
CACHE_REGS
TokEntry *t, *l, *p;
@@ -846,7 +899,27 @@ Yap_tokenizer(IOSTREAM *inp_stream, Term *tposp)
switch (chtype(ch)) {
case CC:
while ((ch = getchr(inp_stream)) != 10 && chtype(ch) != EF);
if (store_comments) {
CHECK_SPACE();
open_comment(ch, inp_stream);
continue_comment:
while ((ch = getchr(inp_stream)) != 10 && chtype(ch) != EF) {
CHECK_SPACE();
extend_comment(ch);
}
CHECK_SPACE();
extend_comment(ch);
if (chtype(ch) != EF) {
ch = getchr(inp_stream);
if (chtype(ch) == CC) {
extend_comment(ch);
goto continue_comment;
}
}
close_comment();
} else {
while ((ch = getchr(inp_stream)) != 10 && chtype(ch) != EF);
}
if (chtype(ch) != EF) {
/* blank space */
if (t == l) {
@@ -854,15 +927,7 @@ Yap_tokenizer(IOSTREAM *inp_stream, Term *tposp)
while (chtype(ch) == BS) {
ch = getchr(inp_stream);
}
if (ASP-H < 1024) {
LOCAL_ErrorMessage = "Stack Overflow";
LOCAL_Error_TYPE = OUT_OF_STACK_ERROR;
LOCAL_Error_Size = 0L;
if (p)
p->Tok = Ord(kind = eot_tok);
/* serious error now */
return l;
}
CHECK_SPACE();
*tposp = Yap_StreamPosition(inp_stream);
}
goto restart;
@@ -947,15 +1012,7 @@ Yap_tokenizer(IOSTREAM *inp_stream, Term *tposp)
/* serious error now */
return l;
}
if (ASP-H < 1024) {
LOCAL_ErrorMessage = "Stack Overflow";
LOCAL_Error_TYPE = OUT_OF_STACK_ERROR;
LOCAL_Error_Size = 0L;
if (p)
p->Tok = Ord(kind = eot_tok);
/* serious error now */
return l;
}
CHECK_SPACE();
if ((t->TokInfo = get_num(&cha,&cherr,inp_stream,ptr,4096,1)) == 0L) {
if (p)
p->Tok = Ord(kind = eot_tok);
@@ -1157,28 +1214,37 @@ Yap_tokenizer(IOSTREAM *inp_stream, Term *tposp)
och = ch;
ch = getchr(inp_stream);
if (och == '/' && ch == '*') {
while ((och != '*' || ch != '/') && chtype(ch) != EF) {
och = ch;
ch = getchr(inp_stream);
if (store_comments) {
CHECK_SPACE();
open_comment('/', inp_stream);
while ((och != '*' || ch != '/') && chtype(ch) != EF) {
och = ch;
CHECK_SPACE();
extend_comment(ch);
ch = getchr(inp_stream);
}
if (chtype(ch) != EF) {
CHECK_SPACE();
extend_comment(ch);
}
close_comment();
} else {
while ((och != '*' || ch != '/') && chtype(ch) != EF) {
och = ch;
ch = getchr(inp_stream);
}
}
if (chtype(ch) == EF) {
t->Tok = Ord(kind = eot_tok);
}
/* leave comments */
ch = getchr(inp_stream);
if (t == l) {
/* we found a comment before reading characters */
while (chtype(ch) == BS) {
ch = getchr(inp_stream);
}
if (ASP-H < 1024) {
LOCAL_ErrorMessage = "Stack Overflow";
LOCAL_Error_TYPE = OUT_OF_STACK_ERROR;
LOCAL_Error_Size = 0L;
if (p)
p->Tok = Ord(kind = eot_tok);
/* serious error now */
return l;
}
CHECK_SPACE();
*tposp = Yap_StreamPosition(inp_stream);
}
goto restart;
@@ -1293,7 +1359,7 @@ Yap_tokenizer(IOSTREAM *inp_stream, Term *tposp)
}
void
Yap_clean_tokenizer(TokEntry *tokstart, VarEntry *vartable, VarEntry *anonvartable)
Yap_clean_tokenizer(TokEntry *tokstart, VarEntry *vartable, VarEntry *anonvartable, Term commentable)
{
CACHE_REGS
struct scanner_extra_alloc *ptr = LOCAL_ScannerExtraBlocks;
@@ -1302,5 +1368,7 @@ Yap_clean_tokenizer(TokEntry *tokstart, VarEntry *vartable, VarEntry *anonvartab
free(ptr);
ptr = next;
}
LOCAL_Comments = TermNil;
LOCAL_CommentsNextChar = LOCAL_CommentsTail = NULL;
}