From a19a216bc60160c162e616145ef091dd18ce4e61 Mon Sep 17 00:00:00 2001 From: Skip Montanaro Date: Tue, 16 Feb 2021 14:40:46 -0600 Subject: Python 0.9.1 as posted in alt.sources --- src/tokenizer.h | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/tokenizer.h (limited to 'src/tokenizer.h') diff --git a/src/tokenizer.h b/src/tokenizer.h new file mode 100644 index 0000000..aec5ef3 --- /dev/null +++ b/src/tokenizer.h @@ -0,0 +1,53 @@ +/*********************************************************** +Copyright 1991 by Stichting Mathematisch Centrum, Amsterdam, The +Netherlands. + + All Rights Reserved + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the names of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +******************************************************************/ + +/* Tokenizer interface */ + +#include "token.h" /* For token types */ + +#define MAXINDENT 100 /* Max indentation level */ + +/* Tokenizer state */ +struct tok_state { + /* Input state; buf <= cur <= inp <= end */ + /* NB an entire token must fit in the buffer */ + char *buf; /* Input buffer */ + char *cur; /* Next character in buffer */ + char *inp; /* End of data in buffer */ + char *end; /* End of input buffer */ + int done; /* 0 normally, 1 at EOF, -1 after error */ + FILE *fp; /* Rest of input; NULL if tokenizing a string */ + int tabsize; /* Tab spacing */ + int indent; /* Current indentation index */ + int indstack[MAXINDENT]; /* Stack of indents */ + int atbol; /* Nonzero if at begin of new line */ + int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ + char *prompt, *nextprompt; /* For interactive prompting */ + int lineno; /* Current line number */ +}; + +extern struct tok_state *tok_setups PROTO((char *)); +extern struct tok_state *tok_setupf PROTO((FILE *, char *ps1, char *ps2)); +extern void tok_free PROTO((struct tok_state *)); +extern int tok_get PROTO((struct tok_state *, char **, char **)); -- cgit v1.2.3