in-memory streams

This commit is contained in:
Vítor Santos Costa 2015-06-18 01:32:33 +01:00
parent 3c0987ef55
commit f4e4220130
4 changed files with 748 additions and 0 deletions

112
os/fmemopen.c Normal file
View File

@ -0,0 +1,112 @@
//
// Copyright 2011-2014 NimbusKit
// Originally ported from https://github.com/ingenuitas/python-tesseract/blob/master/fmemopen.c
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#ifdef __APPLE__
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
FILE *fmemopen(void *buf, size_t size, const char *mode);
struct fmem {
size_t pos;
size_t size;
char *buffer;
};
typedef struct fmem fmem_t;
static int readfn(void *handler, char *buf, int size) {
fmem_t *mem = handler;
size_t available = mem->size - mem->pos;
if (size > available) {
size = available;
}
memcpy(buf, mem->buffer + mem->pos, sizeof(char) * size);
mem->pos += size;
return size;
}
static int writefn(void *handler, const char *buf, int size) {
fmem_t *mem = handler;
size_t available = mem->size - mem->pos;
if (size > available) {
size = available;
}
memcpy(mem->buffer + mem->pos, buf, sizeof(char) * size);
mem->pos += size;
return size;
}
static fpos_t seekfn(void *handler, fpos_t offset, int whence) {
size_t pos;
fmem_t *mem = handler;
switch (whence) {
case SEEK_SET: {
if (offset >= 0) {
pos = (size_t)offset;
} else {
pos = 0;
}
break;
}
case SEEK_CUR: {
if (offset >= 0 || (size_t)(-offset) <= mem->pos) {
pos = mem->pos + (size_t)offset;
} else {
pos = 0;
}
break;
}
case SEEK_END: pos = mem->size + (size_t)offset; break;
default: return -1;
}
if (pos > mem->size) {
return -1;
}
mem->pos = pos;
return (fpos_t)pos;
}
static int closefn(void *handler) {
free(handler);
return 0;
}
FILE *fmemopen(void *buf, size_t size, const char *mode) {
// This data is released on fclose.
fmem_t* mem = (fmem_t *) malloc(sizeof(fmem_t));
// Zero-out the structure.
memset(mem, 0, sizeof(fmem_t));
mem->size = size;
mem->buffer = buf;
// funopen's man page: https://developer.apple.com/library/mac/#documentation/Darwin/Reference/ManPages/man3/funopen.3.html
return funopen(mem, readfn, writefn, seekfn, closefn);
}
#endif

55
os/fmemopen.h Normal file
View File

@ -0,0 +1,55 @@
//
// Copyright 2011-2014 NimbusKit
// Originally ported from https://github.com/ingenuitas/python-tesseract/blob/master/fmemopen.c
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#ifdef __APPLE__
#ifndef FMEMOPEN_H_
#define FMEMOPEN_H_
#if defined __cplusplus
extern "C" {
#endif
/**
* A BSD port of the fmemopen Linux method using funopen.
*
* man docs for fmemopen:
* http://linux.die.net/man/3/fmemopen
*
* man docs for funopen:
* https://developer.apple.com/library/mac/#documentation/Darwin/Reference/ManPages/man3/funopen.3.html
*
* This method is ported from ingenuitas' python-tesseract project.
*
* You must call fclose on the returned file pointer or memory will be leaked.
*
* @param buf The data that will be used to back the FILE* methods. Must be at least
* @c size bytes.
* @param size The size of the @c buf data.
* @param mode The permitted stream operation modes.
* @return A pointer that can be used in the fread/fwrite/fseek/fclose family of methods.
* If a failure occurred NULL will be returned.
* @ingroup NimbusMemoryMappping
*/
FILE *fmemopen(void *buf, size_t size, const char *mode);
#ifdef __cplusplus
}
#endif
#endif // #ifndef FMEMOPEN_H_
#endif

387
os/mem.c Normal file
View File

@ -0,0 +1,387 @@
/*************************************************************************
* *
* YAP Prolog *
* *
* Yap Prolog was developed at NCCUP - Universidade do Porto *
* *
* Copyright L.Damas, V.S.Costa and Universidade do Porto 1985-1997 *
* *
**************************************************************************
* *
* File: sockets.c *
* Last rev: 5/2/88 *
* mods: *
* comments: Input/Output C implemented predicates *
* *
*************************************************************************/
#ifdef SCCS
static char SccsId[] = "%W% %G%";
#endif
/*
* This file includes the definition of a socket related IO.
*
*/
#include "Yap.h"
#include "Yatom.h"
#include "YapHeap.h"
#include "yapio.h"
#include <stdlib.h>
#if HAVE_UNISTD_H
#include <unistd.h>
#endif
#if HAVE_STDARG_H
#include <stdarg.h>
#endif
#ifdef _WIN32
#if HAVE_IO_H
/* Windows */
#include <io.h>
#endif
#if HAVE_SOCKET
#include <winsock2.h>
#endif
#include <windows.h>
#ifndef S_ISDIR
#define S_ISDIR(x) (((x)&_S_IFDIR)==_S_IFDIR)
#endif
#endif
#include "iopreds.h"
#if __APPLE__
#include "fmemopen.h"
#define HAVE_FMEMOPEN 1
#define HAVE_OPEN_MEMSTREAM 1
FILE * open_memstream (char **buf, size_t *len);
#endif
#if HAVE_FMEMOPEN
#define MAY_READ 1
#endif
#if HAVE_OPEN_MEMSTREAM
#define MAY_READ 1
#define MAY_WRITE 1
#endif
#if !MAY_READ
static int MemGetc( int);
/* read from memory */
static int
MemGetc (int sno)
{
register StreamDesc *s = &GLOBAL_Stream[sno];
Int ch;
int spos;
spos = s->u.mem_string.pos;
if (spos == s->u.mem_string.max_size) {
return post_process_eof(s);
} else {
ch = s->u.mem_string.buf[spos];
s->u.mem_string.pos = ++spos;
}
return post_process_read_char(ch, s);
}
#endif
#if !MAY_WRITE
static int MemPutc( int, int);
/* static */
static int
MemPutc(int sno, int ch)
{
StreamDesc *s = &GLOBAL_Stream[sno];
#if MAC || _MSC_VER
if (ch == 10)
{
ch = '\n';
}
#endif
s->u.mem_string.buf[s->u.mem_string.pos++] = ch;
if (s->u.mem_string.pos >= s->u.mem_string.max_size -8) {
int old_src = s->u.mem_string.src, new_src;
/* oops, we have reached an overflow */
Int new_max_size = s->u.mem_string.max_size + Yap_page_size;
char *newbuf;
if (old_src == MEM_BUF_CODE &&
(newbuf = Yap_AllocAtomSpace(new_max_size*sizeof(char))) != NULL) {
new_src = MEM_BUF_CODE;
#if HAVE_MEMMOVE
memmove((void *)newbuf, (void *)s->u.mem_string.buf, (size_t)((s->u.mem_string.pos)*sizeof(char)));
#else
{
Int n = s->u.mem_string.pos;
char *to = newbuf;
char *from = s->u.mem_string.buf;
while (n-- >= 0) {
*to++ = *from++;
}
}
#endif
Yap_FreeAtomSpace(s->u.mem_string.buf);
#if !HAVE_SYSTEM_MALLOC
} else if ((newbuf = (ADDR)realloc(s->u.mem_string.buf, new_max_size*sizeof(char))) != NULL) {
new_src = MEM_BUF_MALLOC;
#endif
} else {
if (GLOBAL_Stream[sno].u.mem_string.error_handler) {
CACHE_REGS
LOCAL_Error_Size = new_max_size*sizeof(char);
save_machine_regs();
longjmp(*(jmp_buf *)GLOBAL_Stream[sno].u.mem_string.error_handler,1);
} else {
Yap_Error(OUT_OF_HEAP_ERROR, TermNil, "YAP could not grow heap for writing to string");
}
return -1;
}
if (old_src == MEM_BUF_CODE) {
}
s->u.mem_string.buf = newbuf;
s->u.mem_string.max_size = new_max_size;
s->u.mem_string.src = new_src;
}
count_output_char(ch,s);
return ((int) ch);
}
#endif
int
Yap_open_buf_read_stream(const char *nbuf, size_t nchars, encoding_t enc, memBufSource src)
{
int sno;
StreamDesc *st;
sno = GetFreeStreamD();
if (sno < 0)
return (PlIOError (RESOURCE_ERROR_MAX_STREAMS,TermNil, "new stream not available for open_mem_read_stream/1"));
st = &GLOBAL_Stream[sno];
Yap_DefaultStreamOps( st );
#if MAY_READ
// like any file stream.
st->status = Input_Stream_f | InMemory_Stream_f | Seekable_Stream_f;
st->file = fmemopen( (void *)nbuf, nchars, "r");
#else
/* currently these streams are not seekable */
st->status = Input_Stream_f | InMemory_Stream_f;
st->u.mem_string.pos = 0;
st->u.mem_string.buf = (char *)nbuf;
st->u.mem_string.max_size = nchars;
st->u.mem_string.error_handler = NULL;
st->u.mem_string.src = src;
#endif
Yap_MemOps( st );
st->linepos = 0;
st->charcount = 0;
st->linecount = 1;
st->encoding = enc;
UNLOCK(st->streamlock);
return sno;
}
static Int
open_mem_read_stream (USES_REGS1) /* $open_mem_read_stream(+List,-Stream) */
{
Term t, ti;
int sno;
Int sl = 0, nchars = 0;
char *nbuf;
ti = Deref(ARG1);
while (ti != TermNil) {
if (IsVarTerm(ti)) {
Yap_Error(INSTANTIATION_ERROR, ti, "open_mem_read_stream");
return (FALSE);
} else if (!IsPairTerm(ti)) {
Yap_Error(TYPE_ERROR_LIST, ti, "open_mem_read_stream");
return (FALSE);
} else {
sl++;
ti = TailOfTerm(ti);
}
}
while ((nbuf = (char *)Yap_AllocAtomSpace((sl+1)*sizeof(char))) == NULL) {
if (!Yap_growheap(FALSE, (sl+1)*sizeof(char), NULL)) {
Yap_Error(OUT_OF_HEAP_ERROR, TermNil, LOCAL_ErrorMessage);
return(FALSE);
}
}
ti = Deref(ARG1);
while (ti != TermNil) {
Term ts = HeadOfTerm(ti);
if (IsVarTerm(ts)) {
Yap_Error(INSTANTIATION_ERROR, ARG1, "open_mem_read_stream");
return (FALSE);
} else if (!IsIntTerm(ts)) {
Yap_Error(TYPE_ERROR_INTEGER, ARG1, "open_mem_read_stream");
return (FALSE);
}
nbuf[nchars++] = IntOfTerm(ts);
ti = TailOfTerm(ti);
}
nbuf[nchars] = '\0';
sno = Yap_open_buf_read_stream(nbuf, nchars, LOCAL_encoding, MEM_BUF_CODE);
t = Yap_MkStream (sno);
return (Yap_unify (ARG2, t));
}
int
Yap_open_buf_write_stream(char **nbufp, size_t *ncharsp)
{
int sno;
StreamDesc *st;
char *nbuf = NULL;
size_t nchars = 0;
if (nbufp)
nbuf = *nbufp;
if (ncharsp)
nchars = *ncharsp;
if (!nchars)
nchars = 256;
if (!nbuf) {
if (!nchars) {
nchars = Yap_page_size;
}
nbuf = malloc( nchars );
if(!nbuf) {
UNLOCK(st->streamlock);
return -1;
}
}
sno = GetFreeStreamD();
if (sno < 0)
return -1;
st = &GLOBAL_Stream[sno];
/* currently these streams are not seekable */
st->linepos = 0;
st->charcount = 0;
st->linecount = 1;
Yap_DefaultStreamOps( st );
#if MAY_WRITE
st->file = open_memstream(&nbuf, &nchars);
st->status = Output_Stream_f | InMemory_Stream_f|Seekable_Stream_f;
#else
st->u.mem_string.pos = 0;
st->u.mem_string.buf = nbuf;
st->u.mem_string.max_size = nchars;
st->status = Output_Stream_f | InMemory_Stream_f;
#endif
Yap_MemOps( st );
UNLOCK(st->streamlock);
*nbufp = nbuf;
return sno;
}
int
Yap_OpenBufWriteStream( USES_REGS1 )
{
char *nbuf;
size_t sz = Yap_page_size;
while ((nbuf = (char *)Yap_AllocAtomSpace(Yap_page_size*sizeof(char))) == NULL) {
if (!Yap_growheap(FALSE, Yap_page_size*sizeof(char), NULL)) {
Yap_Error(OUT_OF_HEAP_ERROR, TermNil, LOCAL_ErrorMessage);
return -1;
}
}
return Yap_open_buf_write_stream(&nbuf, &sz);
}
static Int
open_mem_write_stream (USES_REGS1) /* $open_mem_write_stream(-Stream) */
{
Term t;
int sno;
sno = Yap_OpenBufWriteStream( PASS_REGS1 );
if (sno == -1)
return (PlIOError (SYSTEM_ERROR,TermNil, "new stream not available for open_mem_read_stream/1"));
t = Yap_MkStream (sno);
return (Yap_unify (ARG1, t));
}
/**
* Yap_PeekMemwriteStream() shows the current buffer for a memory stream.
*
* @param sno, the in-memory stream
*
* @return temporary buffer, discarded by close and may be moved away
* by other writes..
*/
memHandle *
Yap_MemExportStreamPtrs( int sno )
{
return &GLOBAL_Stream[sno].u.mem_string;
}
static Int
peek_mem_write_stream ( USES_REGS1 )
{ /* '$peek_mem_write_stream'(+GLOBAL_Stream,?S0,?S) */
Int sno = Yap_CheckStream (ARG1, (Output_Stream_f | InMemory_Stream_f), "close/2");
Int i = GLOBAL_Stream[sno].u.mem_string.pos;
Term tf = ARG2;
CELL *HI;
if (sno < 0)
return (FALSE);
restart:
HI = HR;
while (i > 0) {
--i;
tf = MkPairTerm(MkIntTerm(GLOBAL_Stream[sno].u.mem_string.buf[i]),tf);
if (HR + 1024 >= ASP) {
UNLOCK(GLOBAL_Stream[sno].streamlock);
HR = HI;
if (!Yap_gcl((ASP-HI)*sizeof(CELL), 3, ENV, gc_P(P,CP))) {
UNLOCK(GLOBAL_Stream[sno].streamlock);
Yap_Error(OUT_OF_STACK_ERROR, TermNil, LOCAL_ErrorMessage);
return(FALSE);
}
i = GLOBAL_Stream[sno].u.mem_string.pos;
tf = ARG2;
LOCK(GLOBAL_Stream[sno].streamlock);
goto restart;
}
}
UNLOCK(GLOBAL_Stream[sno].streamlock);
return (Yap_unify(ARG3,tf));
}
void
Yap_MemOps( StreamDesc *st )
{
#if MAY_WRITE
st->stream_putc = FilePutc;
#else
st->stream_putc = MemPutc;
#endif
#if MAY_READ
st->stream_getc = PlGetc;
#else
st->stream_getc = MemGetc;
#endif
}
void
Yap_InitMems( void )
{
CACHE_REGS
Term cm = CurrentModule;
CurrentModule = CHARSIO_MODULE;
Yap_InitCPred ("open_mem_read_stream", 2, open_mem_read_stream, SyncPredFlag);
Yap_InitCPred ("open_mem_write_stream", 1, open_mem_write_stream, SyncPredFlag);
Yap_InitCPred ("peek_mem_write_stream", 3, peek_mem_write_stream, SyncPredFlag);
CurrentModule = cm;
}

194
os/open_memstream.c Normal file
View File

@ -0,0 +1,194 @@
/* Open a write stream around a malloc'd string.
Copyright (C) 2010 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. */
/* Written by Eric Blake <e...@byu.net>, 2010. */
#include "config.h"
/* Specification. */
#include <stdio.h>
#include <assert.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
// #include "verify.h"
#if !HAVE_FUNOPEN
# error Sorry, not ported to your platform yet
#else
FILE * open_memstream (char **buf, size_t *len);
# define INITIAL_ALLOC 64
struct data
{
char **buf; /* User's argument. */
size_t *len; /* User's argument. Smaller of pos or eof. */
size_t pos; /* Current position. */
size_t eof; /* End-of-file position. */
size_t allocated; /* Allocated size of *buf, always > eof. */
char c; /* Temporary storage for byte overwritten by NUL, if pos < eof. */
};
typedef struct data data;
/* Stupid BSD interface uses int/int instead of ssize_t/size_t. */
//verify (sizeof (int) <= sizeof (size_t));
//verify (sizeof (int) <= sizeof (ssize_t));
static int
mem_write (void *c, const char *buf, int n)
{
data *cookie = c;
char *cbuf = *cookie->buf;
/* Be sure we don't overflow. */
if ((ssize_t) (cookie->pos + n) < 0)
{
errno = EFBIG;
return EOF;
}
/* Grow the buffer, if necessary. Use geometric growth to avoid
quadratic realloc behavior. Overallocate, to accomodate the
requirement to always place a trailing NUL not counted by length.
Thus, we want max(prev_size*1.5, cookie->posn1). */
if (cookie->allocated <= cookie->pos + n)
{
size_t newsize = cookie->allocated * 3 / 2;
if (newsize < cookie->pos + n + 1)
newsize = cookie->pos + n + 1;
cbuf = realloc (cbuf, newsize);
if (!cbuf)
return EOF;
*cookie->buf = cbuf;
cookie->allocated = newsize;
}
/* If we have previously done a seek beyond eof, ensure all
intermediate bytges are NUL. */
if (cookie->eof < cookie->pos)
memset (cbuf + cookie->eof, '\0', cookie->pos - cookie->eof);
memcpy (cbuf + cookie->pos, buf, n);
cookie->pos = n;
/* If the user has previously written beyond the current position,
remember what the trailing NUL is overwriting. Otherwise,
extend the stream. */
if (cookie->eof < cookie->pos)
cookie->eof = cookie->pos;
else
cookie->c = cbuf[cookie->pos];
cbuf[cookie->pos] = '\0';
*cookie->len = cookie->pos;
return n;
}
static fpos_t
mem_seek (void *c, fpos_t pos, int whence)
{
data *cookie = c;
off_t offset = pos;
if (whence == SEEK_CUR)
offset = cookie->pos;
else if (whence == SEEK_END)
offset = cookie->eof;
if (offset < 0)
{
errno = EINVAL;
offset = -1;
}
else if ((size_t) offset != offset)
{
errno = ENOSPC;
offset = -1;
}
else
{
if (cookie->pos < cookie->eof)
{
(*cookie->buf)[cookie->pos] = cookie->c;
cookie->c = '\0';
}
cookie->pos = offset;
if (cookie->pos < cookie->eof)
{
cookie->c = (*cookie->buf)[cookie->pos];
(*cookie->buf)[cookie->pos] = '\0';
*cookie->len = cookie->pos;
}
else
*cookie->len = cookie->eof;
}
return offset;
}
static int
mem_close (void *c)
{
data *cookie = c;
char *buf;
/* Be nice and try to reduce excess memory. */
buf = realloc (*cookie->buf, *cookie->len + 1);
if (buf)
*cookie->buf = buf;
free (cookie);
return 0;
}
FILE *
open_memstream (char **buf, size_t *len)
{
FILE *f;
data *cookie;
if (!buf || !len)
{
errno = EINVAL;
return NULL;
}
if (!(cookie = malloc (sizeof *cookie)))
return NULL;
if (!(*buf = malloc (INITIAL_ALLOC)))
{
free (cookie);
errno = ENOMEM;
return NULL;
}
**buf = '\0';
*len = 0;
f = funopen (cookie, NULL, mem_write, mem_seek, mem_close);
if (!f)
{
int saved_errno = errno;
free (cookie);
errno = saved_errno;
}
else
{
cookie->buf = buf;
cookie->len = len;
cookie->pos = 0;
cookie->eof = 0;
cookie->c = '\0';
cookie->allocated = INITIAL_ALLOC;
}
return f;
}
#endif /* HAVE_FUNOPEN */