package acl2s.plugin.parse;
	
	import java.util.ArrayList;
	
	import org.eclipse.jface.text.BadLocationException;
	import org.eclipse.jface.text.DocumentEvent;
	import org.eclipse.jface.text.IDocument;
	import org.eclipse.jface.text.IDocumentListener;
	import org.eclipse.jface.text.IRegion;
	import org.eclipse.jface.text.Region;
	import org.eclipse.jface.text.source.IAnnotationModel;
	import org.peterd.util.Misc;
	
	import acl2s.lib.contentassist.*;
	import acl2s.lib.parse.obj.Sym;
	import acl2s.plugin.Acl2sPlugin;
	import acl2s.plugin.editors.A2sDocument;
	
	public class LispTokens implements IDocumentListener {
		protected class PackageRange {
			public int start, end, index;
			public PackageRange (int start, int end, int index) { this.start = start; this.end = end; this.index = index; }
		}
		protected LispToken[] tokens = null;
		protected int size = 0;
		protected int sizePow2 = 0; // just a view; largest pow2 not greater than size
		protected IDocument doc = null;
		protected IAnnotationModel model = null;
		protected ArrayList<PackageRange> packageRanges; 
		
		protected transient final ArrayList<LispToken> tokenq = new ArrayList<LispToken>(100);
		protected transient IRegion damage;
		protected transient DocumentParser dp = null;
		
		public void connect(IDocument doc) {
			if (this.doc != null) throw new IllegalStateException();
			this.doc = doc;
	
			dp = new DocumentParser(null,doc);
			tokens = new LispToken[100];
			packageRanges = new ArrayList<PackageRange>(3);
			reparse();
			doc.addPrenotifiedDocumentListener(this);
		}
		
		public void disconnect() {
			doc.removeDocumentListener(this);
			doc = null;
			tokens = null;
			size = 0;
			dp = null;
			model = null;
		}
		
		public void reparse() {
			for (int i = 0; i < tokens.length; i++) {
				if (tokens[i] != null) {
					tokens[i].removeAllAnnotations(model);
					tokens[i] = null;
				}
			}
			tokens[0] = new LispToken();
			tokens[0].type = LispToken.EOF_TOK;
			size = 1;
			sizePow2 = 1;
			documentChanged(new DocumentEvent(doc,0,0,doc.get()));
		}
		
		private int packageIndex(int offset) {
			for (PackageRange p : packageRanges) {
				if (p.start <= offset && (p.end < 0 || offset < p.end)) // p.end == -1 means no end has been prescribed
					return p.index;
			}
			return -1;
		}
		
		private void setSizePow2() {
			int v = 1;
			while (v << 1 <= size) v <<= 1;
			sizePow2 = v;
		}
		
		public int getTokenCount() { return size; }
		
		public LispToken getToken(int idx) {
			if (idx < 0 || idx >= size) throw new IndexOutOfBoundsException();
			return tokens[idx];
		}
	
		public int getTokenIdxContaining(int docOffset) {
			if (size == 0) throw new IllegalStateException();
			int docLen = tokens[size-1].absoluteEnd();
			if (docLen == 0) {
				return 0;
			}
			
			int i = 0;
			for (int delta = sizePow2; delta > 0; delta >>= 1) { // binary search
				int newi = i + delta;
				if (newi < size && tokens[newi].absoluteStart() <= docOffset) {
					i = newi;
				}
			}
			return i;
		}
		
		public LispToken getTokenAtOffset(int docOffset)
		{
			int ind = getTokenIdxContaining(docOffset);
			return (tokens[ind].absoluteEnd() >= docOffset) ? tokens[ind] : null;
		}
		
		public int matchParenTok(int idx) {
			if (idx < 0 || idx >= size) throw new IllegalArgumentException();
			int delta = tokens[idx].parenDelta();
			if (delta == 0) throw new IllegalArgumentException();
			int balance = delta;
			int i;
			for (i = idx + delta; i >= 0 && i < size; i += delta) {
				balance += tokens[i].parenDelta();
				if (balance == 0) return i;
			}
			return i;
		}
		
		public LispToken matchCommaTok(LispToken comma) {
			if (comma.type != LispToken.COMMA_TOK &&
					comma.type != LispToken.COMAT_TOK) return null;
			return LispToken.matchComma(comma.ctx,1);
		}
		
		public boolean hasErrors() {
			for (int i = 0; i < size; i++) {
				if (tokens[i].hasErrors()) return true;
			}
			return false;
		}
		
		public void documentAboutToBeChanged(DocumentEvent event) { }
	
		protected LispToken parseToken(int pos, IAnnotationModel model) {
			dp.setPos(pos);
			return LispToken.parse(dp,model);
		}
		
		protected void invalidate(int tokIdx) {
			LispToken target = tokens[tokIdx];
			for (TokenInvalidationListener l : invalidationListeners) {
				l.tokenAboutToBeInvalidated(doc, this, target);
			}
			target.offset = -1;
			target.removeAllAnnotations(model);
		}
		
		private String []getSignature(int fromInd) throws BadLocationException {
			ArrayList<String> buildSig = new ArrayList<String>();
			int i = fromInd;
			// skip opening paren (error if more than one, but who am I to judge?
			while (i < tokens.length && tokens[i] != null && tokens[i].type == LispToken.OPAREN_TOK) ++i;
			if (i != fromInd && i < tokens.length) { // == means no open paren. Must be nil.
				// add all following symbols. Stops at closing paren
				while (i < tokens.length && tokens[i] != null && tokens[i].type == LispToken.SYM_TOK) {
					buildSig.add(doc.get(tokens[i].offset, tokens[i].length));
					++i;
				}
			}
			return buildSig.toArray(new String[buildSig.size()]);
		}
		
		protected static final LispToken preToken = new LispToken();
		protected static final boolean tokenize = true;
		
		public void documentChanged(DocumentEvent event) {
			if (doc instanceof A2sDocument) {
				model = ((A2sDocument) doc).getAnnotationModel();
			}
			
			int docLen = doc.getLength();
			//long stime = System.currentTimeMillis();
			if (!tokenize || model == null) {
				dp.setPos(docLen);
				tokens[0] = LispToken.parse(dp,model);
				size = 1;
				damage = new Region(event.fOffset,event.fLength);
				return;
			}
			
			final int eventReplStart = event.fOffset;
			final int eventReplEnd = event.fOffset + event.fLength;
			
			// find first modified token
			int i;
			if (eventReplStart > 0) {
				i = getTokenIdxContaining(eventReplStart - 1); 
			} else {
				i = getTokenIdxContaining(eventReplStart); // i = 0; ?
			}
			
			// TODO change this for nested encapsulates
			int topLevelTokenIdx = i;
			while (topLevelTokenIdx > 0 && tokens[topLevelTokenIdx].ctx != null) --topLevelTokenIdx;
			
			// update indexed symbols
			WorkspaceDictionary.removeSymbolsInRange(doc, tokens[topLevelTokenIdx].offset, eventReplEnd-1);
			
			final int firstIdxInRegion = i;
			
			int damageStart = tokens[firstIdxInRegion].absoluteStart();
				
			// invalidate tokens within replaced area
			// => (old offset == new offset) -> same parse 
			invalidate(i);
			i++;
			for (/*i = first*/; i < size && tokens[i].absoluteStart() < eventReplEnd; i++) {
				invalidate(i);
			}
			
			final int firstIdxPastRegion = i;
			
			if (Acl2sPlugin.DEBUG && firstIdxPastRegion == firstIdxInRegion) {
				//System.out.println("Interesting...");
				throw new IllegalStateException();
			}
			
			// shift those after
			final int shift = event.fText.length() - event.fLength;
			if (shift != 0) {
				for (; i < size; i++) {
					tokens[i].offset += shift;
				}
			}
			
			// do magic
			//tokenq.clear(); // already done
			LispToken prev = (firstIdxInRegion == 0) ? preToken : tokens[firstIdxInRegion-1];
			i = firstIdxPastRegion;
			for (;;) {
				int pos = prev.absoluteEnd();
	
				LispToken t = parseToken(pos,model);
				
				t.setCtx(prev.getPostCtx(), model);
	
				while (i < size && tokens[i].offset < t.offset) {
					invalidate(i);
					i++;
				}
				if (i < size && tokens[i].offset == t.offset) {
					tokens[i].removeAllAnnotations(model);
					tokens[i] = t;
					break;
				}
				tokenq.add(t);
				if (t.type == LispToken.EOF_TOK) {
					break;
				}
				prev = t;
			}
			final int meetIdx = i;
			
			/* Annotation debugging:
			System.err.println("Annotations:");
			Iterator foo = model.getAnnotationIterator();
			while (foo.hasNext()) {
				System.err.println(foo.next().toString());
			}//*/
			
			// damage area
			int damageEnd = i < size ? tokens[i].absoluteEnd() : docLen;
			damage = new Region(damageStart,damageEnd-damageStart);
					
			final int oldLen = meetIdx - firstIdxInRegion;
			final int newLen = tokenq.size();
			
			// compute number of indices to shift meetIdx...size-1
			final int idxShift = newLen - oldLen;
			if (idxShift > 0) {
				// grow token array if necessary
				if (size + idxShift > tokens.length) {
					LispToken[] newTokens = new LispToken[(size + idxShift) * 2];
					System.arraycopy(tokens,0,newTokens,0,size);
					tokens = newTokens;
				}
				
				// do shift
				for (i = size - 1; i >= meetIdx; i--) {
					tokens[i + idxShift] = tokens[i];
				}
				size += idxShift;
				i += idxShift;
				
				// now, after shift, meetIdx-firstIdxInRegion == tokenq.size()
				// fill that area from tokenq
				for (;i >= firstIdxInRegion; i--) {
					tokens[i] = tokenq.get(i-firstIdxInRegion);
				}
			} else {
				// fill from tokenq
				final int max = firstIdxInRegion + newLen;
				for (i = firstIdxInRegion; i < max; i++) {
					tokens[i] = tokenq.get(i-firstIdxInRegion);
				}
				
				if (idxShift < 0) {
					// do shift
					size += idxShift;  // actually DECREASES size
					for (; i < size; i++) {
						tokens[i] = tokens[i - idxShift];
					}
					
					// optional:
					for (; i < size - idxShift; i++) {
						tokens[i] = null;
					}
				} else {
					//no shift!
				}
			}
			// TODO: go through tokens from firstIdxInRegion to firstIdxInRegion+newLen and index symbols
			// want first top level token from damaged area
			i = topLevelTokenIdx;
			int currentPackageIndex = packageIndex(tokens[i].absoluteStart());
			int endIdx = firstIdxInRegion + newLen;
			while (i < endIdx) {
				// look for top level tokens. Encapsulate special case.
				if (tokens[i] != null && tokens[i].ctx == null) {
					// skip open parens to get to first symbol
					while (i < endIdx && tokens[i] != null && tokens[i].type == LispToken.OPAREN_TOK) ++i;
					if (i >= endIdx || tokens[i] == null || tokens[i].type != LispToken.SYM_TOK) { ++i; continue; }
					
					try {

						String symbol = doc.get(tokens[i].absoluteStart(), tokens[i].length).toLowerCase();
						do {
							++i; 
						} while (i < endIdx && tokens[i] != null && tokens[i].type == LispToken.SPACE_TOK);
						if (i >= endIdx || tokens[i] == null) { ++i; continue; }
						String name = doc.get(tokens[i].offset, tokens[i].length);
						if (name.trim().length() == 0) { ++i; continue; }
						String []signature = null;
						if (symbol.equals("defun")) {
							signature = getSignature(i+1);
							WorkspaceDictionary.addSymbol(doc, name, FileDictionary.CharacterTreeNodeType.DEFUN, currentPackageIndex, tokens[i].offset, signature);
						} else if (symbol.equals("defmacro")) {
							signature = getSignature(i+1);
							WorkspaceDictionary.addSymbol(doc, name, FileDictionary.CharacterTreeNodeType.DEFMACRO, currentPackageIndex, tokens[i].offset, signature);
						} else if (symbol.equals("defthm")) {
							WorkspaceDictionary.addSymbol(doc, name, FileDictionary.CharacterTreeNodeType.DEFTHM, currentPackageIndex, tokens[i].offset, signature);						
						} else if (symbol.equals("defconst")) {
							WorkspaceDictionary.addSymbol(doc, name, FileDictionary.CharacterTreeNodeType.DEFCONST, currentPackageIndex, tokens[i].offset, signature);						
						} else if (symbol.equals("local")) {
							// TODO handle mid level common names
						} else if (symbol.equals("encapsulate")) {
							// TODO handle encapsulate with locals and nested encapsulates
						} else if (symbol.equals("in-package")) {
							// TODO create a new package range and close the previous package range
						}
					} catch (BadLocationException ble) {
						break;
					}
				}
				++i;
			}
			// update contexts
			//*
			i = firstIdxInRegion + newLen + 1;
			for (;;) {
				if (i >= size) break;
				LispToken newCtx = tokens[i-1].getPostCtx();
				// FIXME: disabled following line to fix problem with not updating offsets of pseudo-tokens (context only)
				//if (newCtx == null && tokens[i].ctx == null) break;
				tokens[i].setCtx(newCtx,model);
				i++;
			}
			//*/
			final int ctxMeet = i;
			
			
			if (Acl2sPlugin.DEBUG) {
				int prevEnd = 0;
				LispToken postCtx = null;
				for (i = 0; i < size; i++) {
					if (tokens[i].absoluteStart() != prevEnd) {
						LispToken thisOne = tokens[i];
						LispToken prevOne = (i > 0) ? tokens[i-1] : null;
						throw new IllegalStateException("" + thisOne.absoluteStart() + " != " + prevOne.absoluteEnd());
					}
					if (tokens[i].absoluteEnd() > docLen) {
						LispToken thisOne = tokens[i];
						LispToken prevOne = (i > 0) ? tokens[i-1] : null;
						throw new IllegalStateException("" + thisOne.absoluteStart() + " != " + prevOne.absoluteEnd());
					}
					prevEnd = tokens[i].absoluteEnd();
					if (!Misc.equal(tokens[i].ctx,postCtx)) {
						LispToken ctx = tokens[i].ctx;
						throw new IllegalStateException("Contexts differ " + ctx);
					}
					postCtx = tokens[i].getPostCtx();
				}
			}
			
			tokenq.clear();
			setSizePow2();
			
			//System.err.println(System.currentTimeMillis()-stime);
			
			for (TokenListener l : listeners) {
				l.tokenizationChanged(doc,this,firstIdxInRegion,oldLen,newLen,ctxMeet-firstIdxInRegion);
			}
		}
	
		public IRegion getLastDamageRegion() {
			return damage;
		}
	
		public int getDocLength() {
			if (tokens[size-1].type != LispToken.EOF_TOK) {
				throw new IllegalStateException();
			}
			return tokens[size-1].offset;
		}
	
		public int formSearch(int offset, boolean fwd) {
			int lastForm = 0;
			boolean inKeyword = false;
			for (int i = 0; i < size; i++) {
				LispToken tok = tokens[i];
				if (tok.ctx == null) {
					if (inKeyword && tok.hasWhitespaceNewline()) {
						inKeyword = false;
					}
					if (!inKeyword) {
						if (!fwd && tok.offset >= offset) break;
						lastForm = tok.offset;
						if (fwd && lastForm > offset) break;
					}
					try {
						if (tok.type == LispToken.SYM_TOK && doc.getChar(tok.offset) == ':') {
							inKeyword = true;
						}
					} catch (BadLocationException e) {
						// won't happen
					}
				}
			}
			return lastForm;
		}
		
		public int tokenSearch(int offset, boolean fwd) {
			int tIdx;
			
			if (fwd) {
				if (offset >= doc.getLength()) return doc.getLength();
				tIdx = getTokenIdxContaining(offset);
				if (tokens[tIdx].offset <= offset) {
					tIdx++;
				}
			} else {
				if (offset <= 0) return 0;
				tIdx = getTokenIdxContaining(offset-1);
				if (offset <= tokens[tIdx].offset) {
					tIdx--;
				}
			}
			
			if (tIdx < 0) {
				return 0;
			} else if (tIdx >= size) {
				return doc.getLength();
			} else {
				return tokens[tIdx].offset;
			}
		}
		
		public int nextSexpToken(int offset) {
			if (offset >= doc.getLength()) return -1;
			int tIdx = getTokenIdxContaining(offset);
			LispToken ctx = tokens[tIdx].ctx;
			int ctxOff = LispToken.contextOffset(ctx);
			
			int endIdx = tIdx + 1;
			while (endIdx < size &&
					LispToken.contextOffset(tokens[endIdx].ctx) != ctxOff &&  
					LispToken.meetContexts(ctx,tokens[endIdx].ctx) == ctx) endIdx++;
	
			if (endIdx >= size) {
				return -1;
			} else {
				return endIdx;
			}		
		}
		
		public int nextSexp(int offset) {
			int ind = nextSexpToken(offset);
			return (ind < 0) ? doc.getLength() : tokens[ind].absoluteStart();
		}
		
		public int prevSexpToken(int offset) {
			int tIdx;
			if (offset >= doc.getLength()) {
				tIdx = size - 1;
			} else {
				tIdx = getTokenIdxContaining(offset);
			}
			
			LispToken ctx = tokens[tIdx].ctx;
			int ctxOff = LispToken.contextOffset(ctx);
			
			int startIdx;
			if (offset <= tokens[tIdx].offset) {
				startIdx = tIdx - 1;
			} else {
				return tIdx;  // go to beginning of token
			}
			while (startIdx >= 0 &&
					LispToken.contextOffset(tokens[startIdx].ctx) != ctxOff &&  
					LispToken.meetContexts(ctx,tokens[startIdx].ctx) == ctx) startIdx--;
	
			if (startIdx < 0) {
				return -1;
			} else {
				return startIdx;
			}
		}
		
		public int prevSexp(int offset) {
			int ind = prevSexpToken(offset);
			return (ind < 0) ? 0 : tokens[ind].offset;
		}
		
		public int pastBeginBookForm() {
			boolean inKeyword = false;
			boolean beginBookSeen = false;
			for (int i = 0; i < size; i++) {
				LispToken tok = tokens[i];
				if (tok.ctx == null) {
					if (beginBookSeen) {
						return tok.absoluteStart();
					}
					if (inKeyword && tok.hasWhitespaceNewline()) {
						inKeyword = false;
					}
					do {
						if (inKeyword || i + 2 >= size) break;
						if (!tok.isOParen()) break;
						LispToken tok2 = tokens[i+1];
						Sym fn = tok2.asSym(dp);
						if (fn == null) break; // not a symbol
						if (!fn.sym.equals("BEGIN-BOOK")) break;
						beginBookSeen = true;
					} while (false); // for break;
					try {
						if (tok.type == LispToken.SYM_TOK && doc.getChar(tok.offset) == ':') {
							inKeyword = true;
						}
					} catch (BadLocationException e) {
						// won't happen
					}
				}
			}
			return -1;
		}
		
		public IRegion expandSelection(IRegion existing) {
			int tokFirst;
			int tokLast;
			
			int off = existing.getOffset();
			int preOff = off == 0 ? 0 : off - 1;
			int len = existing.getLength();
			int post = off + len;
			
			if (len == 0) { // if single point
				tokFirst = getTokenIdxContaining(preOff);
				tokLast  = getTokenIdxContaining(off);
				if (tokLast > tokFirst) { // we need to chose just one
					if (tokens[tokLast].offset > off || // whitespace to the right
							tokens[tokLast].type == LispToken.EOF_TOK) { // or end of file
						tokLast = tokFirst;
					} else {
						tokFirst = tokLast;
					}
				}
			} else {
				tokFirst = getTokenIdxContaining(off);
				tokLast  = getTokenIdxContaining(post - 1);
			}
			
			LispToken ctx = LispToken.meetContexts(tokens[tokFirst],tokens[tokLast]);
			if (ctx == null) return existing;
			
			IRegion r = getCtxSpan(ctx);
			int rOff = r.getOffset();
			int rPost = rOff + r.getLength();
			
			if (rOff < off || rPost > post) {
				return r; // expand to fill current context; 
			}
			// otherwise, go up one level (...probably)
			if (ctx.ctx == null) {
				// let's not expand to fill the whole file
				// instead, make sure whitespace is also included
				rOff = ctx.absoluteStart();
				return new Region(rOff,rPost-rOff);
			}
			
			// yes, go up one level
			return getCtxSpan(ctx.ctx);
		}
		
		public IRegion getCtxSpan(LispToken ctx) {
			int ctxOff = LispToken.contextOffset(ctx);
			int startIdx = getTokenIdxContaining(ctxOff);
			int pastIdx = startIdx + 1;
			while (pastIdx < size && tokens[pastIdx].hasContextOffset(ctxOff)) pastIdx++;
			int offset = tokens[startIdx].offset;
			int length = tokens[pastIdx-1].absoluteEnd() - offset;
			return new Region(offset, length);
		}
	
		public void lowercaseFromOffset(int offset) {
			int tokIdx = getTokenIdxContaining(offset);
			LispToken tok = getToken(tokIdx);
			lowercaseRegion(getCtxSpan(tok));
		}
		
		public void lowercaseRegion(IRegion sel) {
			if (sel.getLength() < 1) throw new IllegalArgumentException("Only works on non-zero length.");
			
			int tokIdxStart = getTokenIdxContaining(sel.getOffset());
			int tokIdxEnd = getTokenIdxContaining(sel.getOffset() + sel.getLength() - 1);
			
			// TODO: make undoable with a single Ctrl+Z
			for (int cur = tokIdxStart; cur <= tokIdxEnd; cur++) {
				try {
					tokens[cur].toLowercase(doc);
				} catch (BadLocationException e) {
					// shouldn't happen
				}
			}
		}
	
		
		// *********** LISTENER STUFF **************
		
		protected final ArrayList<TokenListener> listeners = new ArrayList<TokenListener>();
		public void addTokenListener(TokenListener l) {
			listeners.add(l);
		}
		
		public void removeTokenListener(TokenListener l) {
			listeners.remove(l);
		}
		
		public static interface TokenListener {
			void tokenizationChanged(IDocument doc, LispTokens tokens, int start, int oldLen, int newLen, int ctxLen);
		}
	
		
		protected final ArrayList<TokenInvalidationListener> invalidationListeners = new ArrayList<TokenInvalidationListener>();
		public void addTokenInvalidationListener(TokenInvalidationListener l) {
			invalidationListeners.add(l);
		}
		
		public void removeTokenInvalidationListener(TokenInvalidationListener l) {
			invalidationListeners.remove(l);
		}
		
		public static interface TokenInvalidationListener {
			void tokenAboutToBeInvalidated(IDocument doc, LispTokens tokens, LispToken target);
		}
	
	}