From f4e422013042c3fc0a264f20b87a027c13169665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Santos=20Costa?= Date: Thu, 18 Jun 2015 01:32:33 +0100 Subject: [PATCH] in-memory streams --- os/fmemopen.c | 112 +++++++++++++ os/fmemopen.h | 55 +++++++ os/mem.c | 387 ++++++++++++++++++++++++++++++++++++++++++++ os/open_memstream.c | 194 ++++++++++++++++++++++ 4 files changed, 748 insertions(+) create mode 100644 os/fmemopen.c create mode 100644 os/fmemopen.h create mode 100644 os/mem.c create mode 100644 os/open_memstream.c diff --git a/os/fmemopen.c b/os/fmemopen.c new file mode 100644 index 000000000..33f27b204 --- /dev/null +++ b/os/fmemopen.c @@ -0,0 +1,112 @@ +// +// Copyright 2011-2014 NimbusKit +// Originally ported from https://github.com/ingenuitas/python-tesseract/blob/master/fmemopen.c +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifdef __APPLE__ + +#include +#include +#include +#include + +FILE *fmemopen(void *buf, size_t size, const char *mode); + +struct fmem { + size_t pos; + size_t size; + char *buffer; +}; +typedef struct fmem fmem_t; + +static int readfn(void *handler, char *buf, int size) { + fmem_t *mem = handler; + size_t available = mem->size - mem->pos; + + if (size > available) { + size = available; + } + memcpy(buf, mem->buffer + mem->pos, sizeof(char) * size); + mem->pos += size; + + return size; +} + +static int writefn(void *handler, const char *buf, int size) { + fmem_t *mem = handler; + size_t available = mem->size - mem->pos; + + if (size > available) { + size = available; + } + memcpy(mem->buffer + mem->pos, buf, sizeof(char) * size); + mem->pos += size; + + return size; +} + +static fpos_t seekfn(void *handler, fpos_t offset, int whence) { + size_t pos; + fmem_t *mem = handler; + + switch (whence) { + case SEEK_SET: { + if (offset >= 0) { + pos = (size_t)offset; + } else { + pos = 0; + } + break; + } + case SEEK_CUR: { + if (offset >= 0 || (size_t)(-offset) <= mem->pos) { + pos = mem->pos + (size_t)offset; + } else { + pos = 0; + } + break; + } + case SEEK_END: pos = mem->size + (size_t)offset; break; + default: return -1; + } + + if (pos > mem->size) { + return -1; + } + + mem->pos = pos; + return (fpos_t)pos; +} + +static int closefn(void *handler) { + free(handler); + return 0; +} + +FILE *fmemopen(void *buf, size_t size, const char *mode) { + // This data is released on fclose. + fmem_t* mem = (fmem_t *) malloc(sizeof(fmem_t)); + + // Zero-out the structure. + memset(mem, 0, sizeof(fmem_t)); + + mem->size = size; + mem->buffer = buf; + + // funopen's man page: https://developer.apple.com/library/mac/#documentation/Darwin/Reference/ManPages/man3/funopen.3.html + return funopen(mem, readfn, writefn, seekfn, closefn); +} + +#endif diff --git a/os/fmemopen.h b/os/fmemopen.h new file mode 100644 index 000000000..283306912 --- /dev/null +++ b/os/fmemopen.h @@ -0,0 +1,55 @@ +// +// Copyright 2011-2014 NimbusKit +// Originally ported from https://github.com/ingenuitas/python-tesseract/blob/master/fmemopen.c +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifdef __APPLE__ +#ifndef FMEMOPEN_H_ +#define FMEMOPEN_H_ + +#if defined __cplusplus +extern "C" { +#endif + +/** + * A BSD port of the fmemopen Linux method using funopen. + * + * man docs for fmemopen: + * http://linux.die.net/man/3/fmemopen + * + * man docs for funopen: + * https://developer.apple.com/library/mac/#documentation/Darwin/Reference/ManPages/man3/funopen.3.html + * + * This method is ported from ingenuitas' python-tesseract project. + * + * You must call fclose on the returned file pointer or memory will be leaked. + * + * @param buf The data that will be used to back the FILE* methods. Must be at least + * @c size bytes. + * @param size The size of the @c buf data. + * @param mode The permitted stream operation modes. + * @return A pointer that can be used in the fread/fwrite/fseek/fclose family of methods. + * If a failure occurred NULL will be returned. + * @ingroup NimbusMemoryMappping + */ +FILE *fmemopen(void *buf, size_t size, const char *mode); + +#ifdef __cplusplus +} +#endif + +#endif // #ifndef FMEMOPEN_H_ +#endif + diff --git a/os/mem.c b/os/mem.c new file mode 100644 index 000000000..93d6018d7 --- /dev/null +++ b/os/mem.c @@ -0,0 +1,387 @@ +/************************************************************************* +* * +* YAP Prolog * +* * +* Yap Prolog was developed at NCCUP - Universidade do Porto * +* * +* Copyright L.Damas, V.S.Costa and Universidade do Porto 1985-1997 * +* * +************************************************************************** +* * +* File: sockets.c * +* Last rev: 5/2/88 * +* mods: * +* comments: Input/Output C implemented predicates * +* * +*************************************************************************/ +#ifdef SCCS +static char SccsId[] = "%W% %G%"; +#endif + +/* + * This file includes the definition of a socket related IO. + * + */ + +#include "Yap.h" +#include "Yatom.h" +#include "YapHeap.h" +#include "yapio.h" +#include +#if HAVE_UNISTD_H +#include +#endif +#if HAVE_STDARG_H +#include +#endif +#ifdef _WIN32 +#if HAVE_IO_H +/* Windows */ +#include +#endif +#if HAVE_SOCKET +#include +#endif +#include +#ifndef S_ISDIR +#define S_ISDIR(x) (((x)&_S_IFDIR)==_S_IFDIR) +#endif +#endif +#include "iopreds.h" +#if __APPLE__ +#include "fmemopen.h" +#define HAVE_FMEMOPEN 1 +#define HAVE_OPEN_MEMSTREAM 1 +FILE * open_memstream (char **buf, size_t *len); +#endif + +#if HAVE_FMEMOPEN +#define MAY_READ 1 +#endif + +#if HAVE_OPEN_MEMSTREAM +#define MAY_READ 1 +#define MAY_WRITE 1 +#endif + +#if !MAY_READ +static int MemGetc( int); + +/* read from memory */ +static int +MemGetc (int sno) +{ + register StreamDesc *s = &GLOBAL_Stream[sno]; + Int ch; + int spos; + + spos = s->u.mem_string.pos; + if (spos == s->u.mem_string.max_size) { + return post_process_eof(s); + } else { + ch = s->u.mem_string.buf[spos]; + s->u.mem_string.pos = ++spos; + } + return post_process_read_char(ch, s); +} +#endif + +#if !MAY_WRITE +static int MemPutc( int, int); + +/* static */ +static int +MemPutc(int sno, int ch) +{ + StreamDesc *s = &GLOBAL_Stream[sno]; +#if MAC || _MSC_VER + if (ch == 10) + { + ch = '\n'; + } +#endif + s->u.mem_string.buf[s->u.mem_string.pos++] = ch; + if (s->u.mem_string.pos >= s->u.mem_string.max_size -8) { + int old_src = s->u.mem_string.src, new_src; + + /* oops, we have reached an overflow */ + Int new_max_size = s->u.mem_string.max_size + Yap_page_size; + char *newbuf; + + if (old_src == MEM_BUF_CODE && + (newbuf = Yap_AllocAtomSpace(new_max_size*sizeof(char))) != NULL) { + new_src = MEM_BUF_CODE; +#if HAVE_MEMMOVE + memmove((void *)newbuf, (void *)s->u.mem_string.buf, (size_t)((s->u.mem_string.pos)*sizeof(char))); +#else + { + Int n = s->u.mem_string.pos; + char *to = newbuf; + char *from = s->u.mem_string.buf; + while (n-- >= 0) { + *to++ = *from++; + } + } +#endif + Yap_FreeAtomSpace(s->u.mem_string.buf); +#if !HAVE_SYSTEM_MALLOC + } else if ((newbuf = (ADDR)realloc(s->u.mem_string.buf, new_max_size*sizeof(char))) != NULL) { + new_src = MEM_BUF_MALLOC; +#endif + } else { + if (GLOBAL_Stream[sno].u.mem_string.error_handler) { + CACHE_REGS + LOCAL_Error_Size = new_max_size*sizeof(char); + save_machine_regs(); + longjmp(*(jmp_buf *)GLOBAL_Stream[sno].u.mem_string.error_handler,1); + } else { + Yap_Error(OUT_OF_HEAP_ERROR, TermNil, "YAP could not grow heap for writing to string"); + } + return -1; + } + if (old_src == MEM_BUF_CODE) { + } + s->u.mem_string.buf = newbuf; + s->u.mem_string.max_size = new_max_size; + s->u.mem_string.src = new_src; + } + count_output_char(ch,s); + return ((int) ch); +} + +#endif + + + int + Yap_open_buf_read_stream(const char *nbuf, size_t nchars, encoding_t enc, memBufSource src) +{ + int sno; + StreamDesc *st; + + sno = GetFreeStreamD(); + if (sno < 0) + return (PlIOError (RESOURCE_ERROR_MAX_STREAMS,TermNil, "new stream not available for open_mem_read_stream/1")); + st = &GLOBAL_Stream[sno]; + Yap_DefaultStreamOps( st ); +#if MAY_READ + // like any file stream. + st->status = Input_Stream_f | InMemory_Stream_f | Seekable_Stream_f; + st->file = fmemopen( (void *)nbuf, nchars, "r"); +#else + /* currently these streams are not seekable */ + st->status = Input_Stream_f | InMemory_Stream_f; + st->u.mem_string.pos = 0; + st->u.mem_string.buf = (char *)nbuf; + st->u.mem_string.max_size = nchars; + st->u.mem_string.error_handler = NULL; + st->u.mem_string.src = src; +#endif + Yap_MemOps( st ); + st->linepos = 0; + st->charcount = 0; + st->linecount = 1; + st->encoding = enc; + UNLOCK(st->streamlock); + return sno; +} + +static Int +open_mem_read_stream (USES_REGS1) /* $open_mem_read_stream(+List,-Stream) */ +{ + Term t, ti; + int sno; + Int sl = 0, nchars = 0; + char *nbuf; + + ti = Deref(ARG1); + while (ti != TermNil) { + if (IsVarTerm(ti)) { + Yap_Error(INSTANTIATION_ERROR, ti, "open_mem_read_stream"); + return (FALSE); + } else if (!IsPairTerm(ti)) { + Yap_Error(TYPE_ERROR_LIST, ti, "open_mem_read_stream"); + return (FALSE); + } else { + sl++; + ti = TailOfTerm(ti); + } + } + while ((nbuf = (char *)Yap_AllocAtomSpace((sl+1)*sizeof(char))) == NULL) { + if (!Yap_growheap(FALSE, (sl+1)*sizeof(char), NULL)) { + Yap_Error(OUT_OF_HEAP_ERROR, TermNil, LOCAL_ErrorMessage); + return(FALSE); + } + } + ti = Deref(ARG1); + while (ti != TermNil) { + Term ts = HeadOfTerm(ti); + + if (IsVarTerm(ts)) { + Yap_Error(INSTANTIATION_ERROR, ARG1, "open_mem_read_stream"); + return (FALSE); + } else if (!IsIntTerm(ts)) { + Yap_Error(TYPE_ERROR_INTEGER, ARG1, "open_mem_read_stream"); + return (FALSE); + } + nbuf[nchars++] = IntOfTerm(ts); + ti = TailOfTerm(ti); + } + nbuf[nchars] = '\0'; + sno = Yap_open_buf_read_stream(nbuf, nchars, LOCAL_encoding, MEM_BUF_CODE); + t = Yap_MkStream (sno); + return (Yap_unify (ARG2, t)); +} + +int +Yap_open_buf_write_stream(char **nbufp, size_t *ncharsp) +{ + int sno; + StreamDesc *st; + char *nbuf = NULL; + size_t nchars = 0; + + if (nbufp) + nbuf = *nbufp; + if (ncharsp) + nchars = *ncharsp; + if (!nchars) + nchars = 256; + if (!nbuf) { + if (!nchars) { + nchars = Yap_page_size; + } + nbuf = malloc( nchars ); + if(!nbuf) { + UNLOCK(st->streamlock); + return -1; + } + } + sno = GetFreeStreamD(); + if (sno < 0) + return -1; + st = &GLOBAL_Stream[sno]; + /* currently these streams are not seekable */ + st->linepos = 0; + st->charcount = 0; + st->linecount = 1; + Yap_DefaultStreamOps( st ); +#if MAY_WRITE + st->file = open_memstream(&nbuf, &nchars); + st->status = Output_Stream_f | InMemory_Stream_f|Seekable_Stream_f; +#else + st->u.mem_string.pos = 0; + st->u.mem_string.buf = nbuf; + st->u.mem_string.max_size = nchars; + st->status = Output_Stream_f | InMemory_Stream_f; + #endif + Yap_MemOps( st ); + UNLOCK(st->streamlock); + *nbufp = nbuf; + return sno; +} + +int +Yap_OpenBufWriteStream( USES_REGS1 ) +{ + char *nbuf; + size_t sz = Yap_page_size; + + + while ((nbuf = (char *)Yap_AllocAtomSpace(Yap_page_size*sizeof(char))) == NULL) { + if (!Yap_growheap(FALSE, Yap_page_size*sizeof(char), NULL)) { + Yap_Error(OUT_OF_HEAP_ERROR, TermNil, LOCAL_ErrorMessage); + return -1; + } + } + return Yap_open_buf_write_stream(&nbuf, &sz); +} + +static Int +open_mem_write_stream (USES_REGS1) /* $open_mem_write_stream(-Stream) */ +{ + Term t; + int sno; + + sno = Yap_OpenBufWriteStream( PASS_REGS1 ); + if (sno == -1) + return (PlIOError (SYSTEM_ERROR,TermNil, "new stream not available for open_mem_read_stream/1")); + t = Yap_MkStream (sno); + return (Yap_unify (ARG1, t)); +} + +/** + * Yap_PeekMemwriteStream() shows the current buffer for a memory stream. + * + * @param sno, the in-memory stream + * + * @return temporary buffer, discarded by close and may be moved away + * by other writes.. + */ +memHandle * +Yap_MemExportStreamPtrs( int sno ) +{ + return &GLOBAL_Stream[sno].u.mem_string; +} + + +static Int +peek_mem_write_stream ( USES_REGS1 ) +{ /* '$peek_mem_write_stream'(+GLOBAL_Stream,?S0,?S) */ + Int sno = Yap_CheckStream (ARG1, (Output_Stream_f | InMemory_Stream_f), "close/2"); + Int i = GLOBAL_Stream[sno].u.mem_string.pos; + Term tf = ARG2; + CELL *HI; + + if (sno < 0) + return (FALSE); + restart: + HI = HR; + while (i > 0) { + --i; + tf = MkPairTerm(MkIntTerm(GLOBAL_Stream[sno].u.mem_string.buf[i]),tf); + if (HR + 1024 >= ASP) { + UNLOCK(GLOBAL_Stream[sno].streamlock); + HR = HI; + if (!Yap_gcl((ASP-HI)*sizeof(CELL), 3, ENV, gc_P(P,CP))) { + UNLOCK(GLOBAL_Stream[sno].streamlock); + Yap_Error(OUT_OF_STACK_ERROR, TermNil, LOCAL_ErrorMessage); + return(FALSE); + } + i = GLOBAL_Stream[sno].u.mem_string.pos; + tf = ARG2; + LOCK(GLOBAL_Stream[sno].streamlock); + goto restart; + } + } + UNLOCK(GLOBAL_Stream[sno].streamlock); + return (Yap_unify(ARG3,tf)); +} + +void + Yap_MemOps( StreamDesc *st ) +{ +#if MAY_WRITE + st->stream_putc = FilePutc; +#else + st->stream_putc = MemPutc; +#endif + +#if MAY_READ + st->stream_getc = PlGetc; +#else + st->stream_getc = MemGetc; +#endif +} + +void +Yap_InitMems( void ) +{ + CACHE_REGS + Term cm = CurrentModule; + CurrentModule = CHARSIO_MODULE; + Yap_InitCPred ("open_mem_read_stream", 2, open_mem_read_stream, SyncPredFlag); + Yap_InitCPred ("open_mem_write_stream", 1, open_mem_write_stream, SyncPredFlag); + Yap_InitCPred ("peek_mem_write_stream", 3, peek_mem_write_stream, SyncPredFlag); + CurrentModule = cm; +} + diff --git a/os/open_memstream.c b/os/open_memstream.c new file mode 100644 index 000000000..14e3e7524 --- /dev/null +++ b/os/open_memstream.c @@ -0,0 +1,194 @@ +/* Open a write stream around a malloc'd string. + Copyright (C) 2010 Free Software Foundation, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +/* Written by Eric Blake , 2010. */ + + +#include "config.h" + +/* Specification. */ +#include + +#include +#include +#include +#include + +// #include "verify.h" + +#if !HAVE_FUNOPEN +# error Sorry, not ported to your platform yet +#else + +FILE * open_memstream (char **buf, size_t *len); + +# define INITIAL_ALLOC 64 + +struct data +{ + char **buf; /* User's argument. */ + size_t *len; /* User's argument. Smaller of pos or eof. */ + size_t pos; /* Current position. */ + size_t eof; /* End-of-file position. */ + size_t allocated; /* Allocated size of *buf, always > eof. */ + char c; /* Temporary storage for byte overwritten by NUL, if pos < eof. */ +}; +typedef struct data data; + +/* Stupid BSD interface uses int/int instead of ssize_t/size_t. */ +//verify (sizeof (int) <= sizeof (size_t)); +//verify (sizeof (int) <= sizeof (ssize_t)); + +static int +mem_write (void *c, const char *buf, int n) +{ + data *cookie = c; + char *cbuf = *cookie->buf; + + /* Be sure we don't overflow. */ + if ((ssize_t) (cookie->pos + n) < 0) + { + errno = EFBIG; + return EOF; + } + /* Grow the buffer, if necessary. Use geometric growth to avoid + quadratic realloc behavior. Overallocate, to accomodate the + requirement to always place a trailing NUL not counted by length. + Thus, we want max(prev_size*1.5, cookie->posn1). */ + if (cookie->allocated <= cookie->pos + n) + { + size_t newsize = cookie->allocated * 3 / 2; + if (newsize < cookie->pos + n + 1) + newsize = cookie->pos + n + 1; + cbuf = realloc (cbuf, newsize); + if (!cbuf) + return EOF; + *cookie->buf = cbuf; + cookie->allocated = newsize; + } + /* If we have previously done a seek beyond eof, ensure all + intermediate bytges are NUL. */ + if (cookie->eof < cookie->pos) + memset (cbuf + cookie->eof, '\0', cookie->pos - cookie->eof); + memcpy (cbuf + cookie->pos, buf, n); + cookie->pos = n; + /* If the user has previously written beyond the current position, + remember what the trailing NUL is overwriting. Otherwise, + extend the stream. */ + if (cookie->eof < cookie->pos) + cookie->eof = cookie->pos; + else + cookie->c = cbuf[cookie->pos]; + cbuf[cookie->pos] = '\0'; + *cookie->len = cookie->pos; + return n; +} + +static fpos_t +mem_seek (void *c, fpos_t pos, int whence) +{ + data *cookie = c; + off_t offset = pos; + + if (whence == SEEK_CUR) + offset = cookie->pos; + else if (whence == SEEK_END) + offset = cookie->eof; + if (offset < 0) + { + errno = EINVAL; + offset = -1; + } + else if ((size_t) offset != offset) + { + errno = ENOSPC; + offset = -1; + } + else + { + if (cookie->pos < cookie->eof) + { + (*cookie->buf)[cookie->pos] = cookie->c; + cookie->c = '\0'; + } + cookie->pos = offset; + if (cookie->pos < cookie->eof) + { + cookie->c = (*cookie->buf)[cookie->pos]; + (*cookie->buf)[cookie->pos] = '\0'; + *cookie->len = cookie->pos; + } + else + *cookie->len = cookie->eof; + } + return offset; +} + +static int +mem_close (void *c) +{ + data *cookie = c; + char *buf; + + /* Be nice and try to reduce excess memory. */ + buf = realloc (*cookie->buf, *cookie->len + 1); + if (buf) + *cookie->buf = buf; + free (cookie); + return 0; +} + +FILE * +open_memstream (char **buf, size_t *len) +{ + FILE *f; + data *cookie; + + if (!buf || !len) + { + errno = EINVAL; + return NULL; + } + if (!(cookie = malloc (sizeof *cookie))) + return NULL; + if (!(*buf = malloc (INITIAL_ALLOC))) + { + free (cookie); + errno = ENOMEM; + return NULL; + } + **buf = '\0'; + *len = 0; + + f = funopen (cookie, NULL, mem_write, mem_seek, mem_close); + if (!f) + { + int saved_errno = errno; + free (cookie); + errno = saved_errno; + } + else + { + cookie->buf = buf; + cookie->len = len; + cookie->pos = 0; + cookie->eof = 0; + cookie->c = '\0'; + cookie->allocated = INITIAL_ALLOC; + } + return f; +} +#endif /* HAVE_FUNOPEN */