Categories:
Audio (13)
Biotech (29)
Bytecode (36)
Database (77)
Framework (7)
Game (7)
General (507)
Graphics (53)
I/O (35)
IDE (2)
JAR Tools (101)
JavaBeans (21)
JDBC (121)
JDK (426)
JSP (20)
Logging (108)
Mail (58)
Messaging (8)
Network (84)
PDF (97)
Report (7)
Scripting (84)
Security (32)
Server (121)
Servlet (26)
SOAP (24)
Testing (54)
Web (15)
XML (309)
Collections:
Other Resources:
ANTLR Runtime Source Code
ANTLR is a powerful parser generator for multiple programming languages including Java.
ANTLR contains 2 major modules:
ANTLR Runtime Source Code files are provided in the distribution packge (antlr4-4.10.1.zip). You can download them at ANTLR Website.
You can also browse the source code below:
✍: FYIcenter
⏎ org/antlr/v4/runtime/UnbufferedTokenStream.java
/* * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ package org.antlr.v4.runtime; import org.antlr.v4.runtime.misc.Interval; import java.util.Arrays; public class UnbufferedTokenStream<T extends Token> implements TokenStream { protected TokenSource tokenSource; /** * A moving window buffer of the data being scanned. While there's a marker, * we keep adding to buffer. Otherwise, {@link #consume consume()} resets so * we start filling at index 0 again. */ protected Token[] tokens; /** * The number of tokens currently in {@link #tokens tokens}. * * <p>This is not the buffer capacity, that's {@code tokens.length}.</p> */ protected int n; /** * 0..n-1 index into {@link #tokens tokens} of next token. * * <p>The {@code LT(1)} token is {@code tokens[p]}. If {@code p == n}, we are * out of buffered tokens.</p> */ protected int p=0; /** * Count up with {@link #mark mark()} and down with * {@link #release release()}. When we {@code release()} the last mark, * {@code numMarkers} reaches 0 and we reset the buffer. Copy * {@code tokens[p]..tokens[n-1]} to {@code tokens[0]..tokens[(n-1)-p]}. */ protected int numMarkers = 0; /** * This is the {@code LT(-1)} token for the current position. */ protected Token lastToken; /** * When {@code numMarkers > 0}, this is the {@code LT(-1)} token for the * first token in {@link #tokens}. Otherwise, this is {@code null}. */ protected Token lastTokenBufferStart; /** * Absolute token index. It's the index of the token about to be read via * {@code LT(1)}. Goes from 0 to the number of tokens in the entire stream, * although the stream size is unknown before the end is reached. * * <p>This value is used to set the token indexes if the stream provides tokens * that implement {@link WritableToken}.</p> */ protected int currentTokenIndex = 0; public UnbufferedTokenStream(TokenSource tokenSource) { this(tokenSource, 256); } public UnbufferedTokenStream(TokenSource tokenSource, int bufferSize) { this.tokenSource = tokenSource; tokens = new Token[bufferSize]; n = 0; fill(1); // prime the pump } @Override public Token get(int i) { // get absolute index int bufferStartIndex = getBufferStartIndex(); if (i < bufferStartIndex || i >= bufferStartIndex + n) { throw new IndexOutOfBoundsException("get("+i+") outside buffer: "+ bufferStartIndex+".."+(bufferStartIndex+n)); } return tokens[i - bufferStartIndex]; } @Override public Token LT(int i) { if ( i==-1 ) { return lastToken; } sync(i); int index = p + i - 1; if ( index < 0 ) { throw new IndexOutOfBoundsException("LT("+i+") gives negative index"); } if ( index >= n ) { assert n > 0 && tokens[n-1].getType() == Token.EOF; return tokens[n-1]; } return tokens[index]; } @Override public int LA(int i) { return LT(i).getType(); } @Override public TokenSource getTokenSource() { return tokenSource; } @Override public String getText() { return ""; } @Override public String getText(RuleContext ctx) { return getText(ctx.getSourceInterval()); } @Override public String getText(Token start, Token stop) { return getText(Interval.of(start.getTokenIndex(), stop.getTokenIndex())); } @Override public void consume() { if (LA(1) == Token.EOF) { throw new IllegalStateException("cannot consume EOF"); } // buf always has at least tokens[p==0] in this method due to ctor lastToken = tokens[p]; // track last token for LT(-1) // if we're at last token and no markers, opportunity to flush buffer if ( p == n-1 && numMarkers==0 ) { n = 0; p = -1; // p++ will leave this at 0 lastTokenBufferStart = lastToken; } p++; currentTokenIndex++; sync(1); } /** Make sure we have 'need' elements from current position {@link #p p}. Last valid * {@code p} index is {@code tokens.length-1}. {@code p+need-1} is the tokens index 'need' elements * ahead. If we need 1 element, {@code (p+1-1)==p} must be less than {@code tokens.length}. */ protected void sync(int want) { int need = (p+want-1) - n + 1; // how many more elements we need? if ( need > 0 ) { fill(need); } } /** * Add {@code n} elements to the buffer. Returns the number of tokens * actually added to the buffer. If the return value is less than {@code n}, * then EOF was reached before {@code n} tokens could be added. */ protected int fill(int n) { for (int i=0; i<n; i++) { if (this.n > 0 && tokens[this.n-1].getType() == Token.EOF) { return i; } Token t = tokenSource.nextToken(); add(t); } return n; } protected void add(Token t) { if ( n>=tokens.length ) { tokens = Arrays.copyOf(tokens, tokens.length * 2); } if (t instanceof WritableToken) { ((WritableToken)t).setTokenIndex(getBufferStartIndex() + n); } tokens[n++] = t; } /** * Return a marker that we can release later. * * <p>The specific marker value used for this class allows for some level of * protection against misuse where {@code seek()} is called on a mark or * {@code release()} is called in the wrong order.</p> */ @Override public int mark() { if (numMarkers == 0) { lastTokenBufferStart = lastToken; } int mark = -numMarkers - 1; numMarkers++; return mark; } @Override public void release(int marker) { int expectedMark = -numMarkers; if ( marker!=expectedMark ) { throw new IllegalStateException("release() called with an invalid marker."); } numMarkers--; if ( numMarkers==0 ) { // can we release buffer? if (p > 0) { // Copy tokens[p]..tokens[n-1] to tokens[0]..tokens[(n-1)-p], reset ptrs // p is last valid token; move nothing if p==n as we have no valid char System.arraycopy(tokens, p, tokens, 0, n - p); // shift n-p tokens from p to 0 n = n - p; p = 0; } lastTokenBufferStart = lastToken; } } @Override public int index() { return currentTokenIndex; } @Override public void seek(int index) { // seek to absolute index if (index == currentTokenIndex) { return; } if (index > currentTokenIndex) { sync(index - currentTokenIndex); index = Math.min(index, getBufferStartIndex() + n - 1); } int bufferStartIndex = getBufferStartIndex(); int i = index - bufferStartIndex; if ( i < 0 ) { throw new IllegalArgumentException("cannot seek to negative index " + index); } else if (i >= n) { throw new UnsupportedOperationException("seek to index outside buffer: "+ index+" not in "+ bufferStartIndex +".."+(bufferStartIndex +n)); } p = i; currentTokenIndex = index; if (p == 0) { lastToken = lastTokenBufferStart; } else { lastToken = tokens[p-1]; } } @Override public int size() { throw new UnsupportedOperationException("Unbuffered stream cannot know its size"); } @Override public String getSourceName() { return tokenSource.getSourceName(); } @Override public String getText(Interval interval) { int bufferStartIndex = getBufferStartIndex(); int bufferStopIndex = bufferStartIndex + tokens.length - 1; int start = interval.a; int stop = interval.b; if (start < bufferStartIndex || stop > bufferStopIndex) { throw new UnsupportedOperationException("interval "+interval+" not in token buffer window: "+ bufferStartIndex+".."+bufferStopIndex); } int a = start - bufferStartIndex; int b = stop - bufferStartIndex; StringBuilder buf = new StringBuilder(); for (int i = a; i <= b; i++) { Token t = tokens[i]; buf.append(t.getText()); } return buf.toString(); } protected final int getBufferStartIndex() { return currentTokenIndex - p; } }
⏎ org/antlr/v4/runtime/UnbufferedTokenStream.java
Or download all of them as a single archive file:
File name: antlr-runtime-4.10.1-sources.jar File size: 308953 bytes Release date: 2022-04-15 Download
⇐ What Is ANTLR Parser Generator
2018-10-21, 31331👍, 0💬
Popular Posts:
The Web Services Description Language for Java Toolkit (WSDL4J), Release 1.6.2, allows the creation,...
How to download and install JDK (Java Development Kit) 8? If you want to write Java applications, yo...
JAX-WS is an API for building web services and clients. It is the next generation Web Services API r...
The JSR 105 XML Digital Signature 1.0.1 FCS implementation provides an API and implementation that a...
commons-io-1.4.jar is the JAR file for Commons IO 1.4, which is a library of utilities to assist wit...