| Server IP : 54.37.205.81 / Your IP : 216.73.216.76 Web Server : nginx/1.22.1 System : Linux vps-249481fa 6.1.0-50-cloud-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.176-1 (2026-07-02) x86_64 User : debian ( 1000) PHP Version : 8.2.32 Disable Function : NONE MySQL : OFF | cURL : ON | WGET : ON | Perl : ON | Python : OFF | Sudo : ON | Pkexec : OFF Directory : /lib/python3/dist-packages/lunr/ |
Upload File : |
from lunr.tokenizer import default_separator
class QueryLexer:
# TODO: use iteration protocol?
EOS = "EOS"
FIELD = "FIELD"
TERM = "TERM"
EDIT_DISTANCE = "EDIT_DISTANCE"
BOOST = "BOOST"
PRESENCE = "PRESENCE"
def __init__(self, string):
self.lexemes = []
self.string = string
self.length = len(string)
self.pos = 0
self.start = 0
self.escape_char_positions = []
@property
def width(self):
return self.pos - self.start
def ignore(self):
if self.start == self.pos:
self.pos += 1
self.start = self.pos
def backup(self):
self.pos -= 1
def accept_digit_run(self):
char = self.next()
while char != self.EOS and (47 < ord(char) < 58):
char = self.next()
if char != self.EOS:
self.backup()
def run(self):
state = self.lex_text()
while state:
state = state()
def slice_string(self):
subslices = []
slice_start = self.start
for escape_char_position in self.escape_char_positions:
subslices.append(self.string[slice_start:escape_char_position])
slice_start = escape_char_position + 1
subslices.append(self.string[slice_start : self.pos])
self.escape_char_positions = []
return "".join(subslices)
def next(self):
if self.pos >= self.length:
return self.EOS
char = self.string[self.pos]
self.pos += 1
return char
def emit(self, type_):
self.lexemes.append(
{
"type": type_,
"string": self.slice_string(),
"start": self.start,
"end": self.pos,
}
)
self.start = self.pos
def escape_character(self):
self.escape_char_positions.append(self.pos - 1)
self.pos += 1
def lex_field(self):
self.backup()
self.emit(self.FIELD)
self.ignore()
return self.lex_text
def lex_term(self):
if self.width > 1:
self.backup()
self.emit(self.TERM)
self.ignore()
return self.lex_text
def lex_edit_distance(self):
self.ignore()
self.accept_digit_run()
self.emit(self.EDIT_DISTANCE)
return self.lex_text
def lex_boost(self):
self.ignore()
self.accept_digit_run()
self.emit(self.BOOST)
return self.lex_text
def lex_EOS(self):
if self.width > 0:
self.emit(self.TERM)
def lex_text(self):
while True:
char = self.next()
if char == self.EOS:
return self.lex_EOS
if ord(char) == 92: # Escape character is '\'
self.escape_character()
continue
if char == ":":
return self.lex_field
if char == "~":
self.backup()
if self.width > 0:
self.emit(self.TERM)
return self.lex_edit_distance
if char == "^":
self.backup()
if self.width > 0:
self.emit(self.TERM)
return self.lex_boost
# '+' indicates term presence is required, check for length to
# ensure only a leading '+' is considered
if char == "+" and self.width == 1:
self.emit(self.PRESENCE)
return self.lex_text
# '-' indicates term presence is prohibited
if char == "-" and self.width == 1:
self.emit(self.PRESENCE)
return self.lex_text
if default_separator(char):
return self.lex_term