text handling/UNICODE

C/atomic.c

@@ -309,42 +309,40 @@ static Int char_code(USES_REGS1) {
 
   */
   static Int name(USES_REGS1) { /* name(?Atomic,?String)		 */
-  Term t = Deref(ARG2), NewT, AtomNameT = Deref(ARG1);
+  Term t2 = Deref(ARG2), NewT, t1 = Deref(ARG1);
   LOCAL_MAX_SIZE = 1024;
 
   int l = push_text_stack();
 restart_aux:
-  if (Yap_IsGroundTerm(AtomNameT)) {
+  if (Yap_IsGroundTerm(t1)) {
-    if (!IsVarTerm(t) && !IsPairTerm(t) && t != TermNil) {
+    if (!IsVarTerm(t2) && !IsPairTerm(t2) && t2 != TermNil) {
       Yap_Error(TYPE_ERROR_LIST, ARG2, "name/2");
-      pop_text_stack(l);
       ReleaseAndReturn(FALSE);
     }
     // verify if an atom, int, float or bi§gnnum
-    NewT = Yap_AtomicToListOfCodes(AtomNameT PASS_REGS);
+    NewT = Yap_AtomicToListOfCodes(t1 PASS_REGS);
     if (NewT) {
-      pop_text_stack(l);
       ReleaseAndReturn(Yap_unify(NewT, ARG2));
     }
     // else
-  } else if (IsVarTerm(t)) {
+  } else if (IsVarTerm(t2)) {
-    Yap_Error(INSTANTIATION_ERROR, t, "name/2");
+    Yap_Error(INSTANTIATION_ERROR, t2, "name/2");
     pop_text_stack(l);
-    return FALSE;
+    return false;
   } else {
-    Term at = Yap_ListToAtomic(t PASS_REGS);
+    Term at = Yap_ListToAtomic(t2 PASS_REGS);
     if (at) {
       pop_text_stack(l);
       ReleaseAndReturn(Yap_unify(at, ARG1));
     }
   }
   if (LOCAL_Error_TYPE && Yap_HandleError("atom/2")) {
-    AtomNameT = Deref(ARG1);
+    t1 = Deref(ARG1);
-    t = Deref(ARG2);
+    t2 = Deref(ARG2);
     goto restart_aux;
   }
   pop_text_stack(l);
-  ReleaseAndReturn(FALSE);
+  ReleaseAndReturn(false);
 }
 
 static Int string_to_atomic(

C/c_interface.c

@@ -2221,7 +2221,7 @@ X_API char *YAP_WriteBuffer(Term t, char *buf, size_t sze, int flags) {
 
     BACKUP_MACHINE_REGS();
     inp.val.t = t;
-    inp.type = YAP_STRING_TERM;
+    inp.type = YAP_STRING_TERM|YAP_STRING_DATUM;
     out.type = YAP_STRING_CHARS;
     out.val.c = buf;
     out.max = sze - 1;

H/YapText.h

@@ -660,6 +660,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.val.t = t0;
    inp.type = YAP_STRING_ATOM;
    out.val.uc = NULL;
+   out.enc = ENC_ISO_UTF8;
    out.type = YAP_STRING_INT | YAP_STRING_FLOAT | YAP_STRING_BIG;
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
@@ -687,6 +688,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
      YAP_STRING_FLOAT | YAP_STRING_BIG | YAP_STRING_ATOMS_CODES;
    out.val.uc = NULL;
    out.type = YAP_STRING_STRING;
+   out.enc = ENC_ISO_UTF8;
 
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
@@ -715,6 +717,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
      YAP_STRING_FLOAT | YAP_STRING_BIG | YAP_STRING_TERM;
    out.val.uc = NULL;
    out.type = mod_to_type(mod PASS_REGS);
+   out.enc = ENC_ISO_UTF8;
 
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
@@ -754,6 +757,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.type = YAP_STRING_CHARS;
    out.val.uc = NULL;
    out.type = YAP_STRING_ATOM;
+   out.enc = ENC_ISO_UTF8;
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
    return out.val.a;
@@ -925,6 +929,8 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.type = YAP_STRING_ATOMS;
    out.type = YAP_STRING_ATOM;
    out.val.uc = NULL;
+   out.enc = ENC_ISO_UTF8;
+   out.enc = ENC_ISO_UTF8;
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return (Atom)NULL;
    return out.val.a;
@@ -937,6 +943,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    out.type =
      YAP_STRING_INT | YAP_STRING_FLOAT | YAP_STRING_BIG | YAP_STRING_TERM;
    out.val.uc = NULL;
+   out.enc = ENC_ISO_UTF8;
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
    return out.val.t;
@@ -948,6 +955,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.type = YAP_STRING_ATOMS;
    out.type = YAP_STRING_STRING;
    out.val.uc = NULL;
+   out.enc = ENC_ISO_UTF8;
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
    return out.val.t;
@@ -959,6 +967,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.type = YAP_STRING_CODES;
    out.type = YAP_STRING_ATOM;
    out.val.uc = NULL;
+   out.enc = ENC_ISO_UTF8;
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return (Atom)NULL;
    return out.val.a;
@@ -969,6 +978,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.val.t = t0;
    inp.type = YAP_STRING_CODES;
    out.type = YAP_STRING_INT | YAP_STRING_FLOAT | YAP_STRING_BIG;
+   out.enc = ENC_ISO_UTF8;
    out.val.uc = NULL;
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
@@ -981,6 +991,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.type = YAP_STRING_CODES;
    out.val.uc = NULL;
    out.type = YAP_STRING_STRING;
+   out.enc = ENC_ISO_UTF8;
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
    return out.val.t;
@@ -992,6 +1003,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.type = YAP_STRING_ATOMS_CODES;
    out.val.uc = NULL;
    out.type = YAP_STRING_ATOM;
+   out.enc = ENC_ISO_UTF8;
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
    return out.val.a;
@@ -1002,6 +1014,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.val.t = t0;
    inp.type = YAP_STRING_STRING | YAP_STRING_ATOMS_CODES | YAP_STRING_TERM;
    out.val.uc = NULL;
+   out.enc = ENC_ISO_UTF8;
    out.type = YAP_STRING_ATOM | YAP_STRING_INT | YAP_STRING_FLOAT |
      YAP_STRING_BIG | YAP_STRING_OUTPUT_TERM;
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
@@ -1015,6 +1028,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.val.t = t0;
    inp.type = YAP_STRING_STRING | YAP_STRING_ATOMS_CODES;
    out.val.uc = NULL;
+   out.enc = ENC_ISO_UTF8;
    out.type = YAP_STRING_INT | YAP_STRING_FLOAT | YAP_STRING_BIG;
 
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
@@ -1029,6 +1043,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.type = YAP_STRING_STRING | YAP_STRING_ATOMS_CODES | YAP_STRING_TERM;
    out.val.uc = NULL;
    out.type = YAP_STRING_STRING;
+   out.enc = ENC_ISO_UTF8;
 
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
@@ -1044,6 +1059,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
      YAP_STRING_OUTPUT_TERM;
    out.val.uc = NULL;
    out.type = YAP_STRING_STRING;
+   out.enc = ENC_ISO_UTF8;
 
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
@@ -1057,6 +1073,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.type = YAP_STRING_STRING | YAP_STRING_ATOMS_CODES | YAP_STRING_TERM;
    out.val.uc = NULL;
    out.type = mod_to_type(mod PASS_REGS);
+   out.enc = ENC_ISO_UTF8;
 
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
@@ -1070,6 +1087,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.type = YAP_STRING_STRING | YAP_STRING_ATOMS_CODES | YAP_STRING_TERM;
    out.val.uc = NULL;
    out.type = mod_to_bqtype(mod PASS_REGS);
+   out.enc = ENC_ISO_UTF8;
 
    if (!Yap_CVT_Text(&inp, &out PASS_REGS))
      return 0L;
@@ -1083,6 +1101,7 @@ static inline size_t Yap_AtomToLength(Term t0 USES_REGS) {
    inp.val.c0 = s;
    inp.type = YAP_STRING_CHARS | YAP_STRING_NCHARS;
    inp.enc = enc;
+   out.enc = ENC_ISO_UTF8;
    out.type = YAP_STRING_ATOM;
    out.val.uc = NULL;
    out.max = len;

include/YapDefs.h

@@ -379,8 +379,8 @@ typedef void (*YAP_halt_hook)(int exit_code, void *closure);
 /* each type has a tag */
 typedef YAP_Int YAP_opaque_tag_t;
 
-typedef YAP_Bool (*YAP_Opaque_CallOnFail)(void *);
+typedef YAP_Bool (*YAP_Opaque_CallOnFail)(YAP_Term);
-typedef YAP_Bool (*YAP_Opaque_CallOnCut)(void *);
+typedef YAP_Bool (*YAP_Opaque_CallOnCut)(YAP_Term);
 typedef YAP_Bool (*YAP_Opaque_CallOnWrite)(FILE *, YAP_opaque_tag_t, void *,
                                            int);
 typedef YAP_Int (*YAP_Opaque_CallOnGCMark)(YAP_opaque_tag_t, void *, YAP_Term *,

library/dialect/swi/fli/swi.c

@@ -140,16 +140,16 @@ static UInt cvtFlags(unsigned flags) {
     inptype |= YAP_STRING_FLOAT;
   }
   if (flags & CVT_VARIABLE) {
-    inptype |= YAP_STRING_TERM;
+    inptype |= YAP_STRING_DATUM;
   }
   if (flags & CVT_WRITE) {
-    inptype |= YAP_STRING_TERM;
+    inptype |= YAP_STRING_DATUM;
   }
   if (flags & CVT_WRITEQ) {
-    inptype |= YAP_STRING_TERM | YAP_STRING_WQ;
+    inptype |= YAP_STRING_DATUM | YAP_STRING_WQ;
   }
   if (flags & CVT_WRITE_CANONICAL) {
-    inptype |= YAP_STRING_TERM | YAP_STRING_WC;
+    inptype |= YAP_STRING_DATUM | YAP_STRING_WC;
   }
   return inptype;
 }

os/fmem.c

@@ -116,18 +116,19 @@ static char SccsId[] = "%W% %G%";
 }
 
 
-bool Yap_set_stream_to_buf(StreamDesc *st, const char *buf, size_t nchars) {
+bool Yap_set_stream_to_buf(StreamDesc *st, const char *buf, encoding_t enc, size_t nchars) {
   FILE *f;
 
   // like any file stream.
   st->file = f = fmemopen((void *)buf, nchars, "r");
   st->status = Input_Stream_f | InMemory_Stream_f | Seekable_Stream_f;
   st->vfs = NULL;
+          st->encoding = enc;
   Yap_DefaultStreamOps(st);
   return true;
 }
 
-int Yap_open_buf_read_stream(const char *buf, size_t nchars, encoding_t *encp,
+int                 Yap_open_buf_read_stream(const char *buf, size_t nchars, encoding_t *encp,
                              memBufSource src) {
   CACHE_REGS
   int sno;

os/yapio.h

@@ -110,7 +110,7 @@ extern Term Yap_StringToNumberTerm(const char *s, encoding_t *encp,
 extern int Yap_FormatFloat(Float f, char **s, size_t sz);
 extern int Yap_open_buf_read_stream(const char *buf, size_t nchars,
                                     encoding_t *encp, memBufSource src);
-extern bool Yap_set_stream_to_buf(struct stream_desc *st, const char *buf,
+extern bool Yap_set_stream_to_buf(struct stream_desc *st, const char *buf, encoding_t enc,
                                   size_t nchars);
 extern int Yap_open_buf_write_stream(encoding_t enc, memBufSource src);
 extern Term Yap_BufferToTerm(const unsigned char *s, size_t sz, Term opts);

packages/swig/yap.i

@@ -279,10 +279,352 @@ class YAPEngine;
                           /* Put header files here or function declarations like below */
 #include "yapi.hh"
 
+
+
                           extern "C" {
 
                             #if THREADS
                             #define Yap_regp regcache
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                             #endif
 
                             // we cannot consult YapInterface.h, that conflicts with what we