<?php
namespace Tlf\Lexer;
class Token {
/** the source string */
public $source;
protected $remainder = false;
protected $buffer = false;
/** start position in source. First time next() is called, index becomes 0*/
public $index = -1;
protected $len = 0;
protected $prevToken;
protected $matches =[];
protected $match;
/**
* The line number currently being processed.
* Ticks up WHEN next() adds a \n, so line number will be increased while \n at top of buffer.
* Ticks down WHEN rewind() is called, for EACH \n in the rewound chars.
*/
public int $line_number = 0;
public function __construct($source){
$this->source = $source;
$this->remainder = $source;
$this->len = strlen($source);
}
public function buffer(){
return $this->buffer;
}
public function remainder(){
return $this->remainder;
}
/**
*
* Set a new buffer. May corrupt the token, but remainder is unchanged.
*
* @todo update the token when the buffer is changed. (such as prevToken? index?)
* @param $newBuffer the full text of the buffer
*/
public function setBuffer($newBuffer){
$buffer = $this->buffer;
$this->buffer = $newBuffer;
return $buffer;
}
public function append($str){
$this->remainder .= $str;
$this->len += strlen($str);
}
/**
* @alias for `setBuffer('')`
*/
public function clearBuffer(){
return $this->setBuffer('');
}
public function next(){
$this->index++;
$next = substr($this->remainder,0,1);
if ($next=="\n")$this->line_number++;
$this->remainder = substr($this->remainder,1);
$this->buffer .= $next;
if ($this->index==$this->len)return false;
return $this;
}
/**
* Remove $amount characters from the buffer & prepend them to $remainder.
* @warning Its technically not a rewind, since you could change the buffer, then rewind() And have a different set of $remaining characters
*/
public function rewind(int $amount){
$chars = substr($this->buffer, -$amount);
$num_newlines = count(explode("\n", $chars)) - 1;
// echo "\n\nREWIND\n\n";
$this->line_number -= $num_newlines;
$this->buffer = substr($this->buffer, 0, -$amount);
$this->index = $this->index - $amount;
$this->remainder = $chars . $this->remainder;
}
/**
* Move the pointer forward by `$amount` chars by repeatedly calling `$this->next()`
*/
public function forward(int $amount){
$i=0;
while ($i++ < $amount)$this->next();
}
/**
*
* @return array or string or null
*/
public function match($index=false){
$ret = $index===false ? $this->match : $this->match[$index]??null;
// echo "\n\n";
// var_dump($ret);
// echo "\n\n";
// exit;
return $ret;
}
public function setMatch($match){
$prevMatch = $match;
$this->match = $match;
return $prevMatch;
}
}