54 lines
1.8 KiB
Python
54 lines
1.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Script rápido para probar tokenización
|
|
"""
|
|
|
|
if __name__ == "__main__":
|
|
print("🧪 Prueba rápida de tokenización")
|
|
|
|
try:
|
|
# 1. Verificar registro de tipos
|
|
print("\n1. Verificando registro de tipos...")
|
|
from type_registry import discover_and_register_types, get_registered_base_context
|
|
|
|
registry_info = discover_and_register_types()
|
|
print(f"Clases registradas: {registry_info['class_count']}")
|
|
|
|
context = get_registered_base_context()
|
|
print(f"Clases en contexto: {list(context.keys())}")
|
|
|
|
# 2. Verificar si las clases tienen tokenización
|
|
print("\n2. Verificando métodos de tokenización...")
|
|
for name, cls in context.items():
|
|
if hasattr(cls, 'get_tokenization_patterns'):
|
|
patterns = cls.get_tokenization_patterns()
|
|
print(f"{name}: {len(patterns)} patrones")
|
|
for p in patterns:
|
|
print(f" - Prioridad {p['priority']}: {p['description']}")
|
|
else:
|
|
print(f"{name}: sin tokenización")
|
|
|
|
# 3. Probar tokenizador
|
|
print("\n3. Probando tokenizador...")
|
|
from tl_bracket_parser import UniversalTokenizer
|
|
|
|
tokenizer = UniversalTokenizer()
|
|
tokenizer.debug = True
|
|
print(f"Reglas cargadas: {len(tokenizer.tokenization_rules)}")
|
|
|
|
test_cases = [
|
|
"192.168.1.1",
|
|
"16#FF",
|
|
"0xFF",
|
|
"2#1010",
|
|
"10.x.1.y"
|
|
]
|
|
|
|
for case in test_cases:
|
|
result = tokenizer.preprocess_tokens(case)
|
|
print(f"'{case}' → '{result}'")
|
|
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|
|
import traceback
|
|
traceback.print_exc() |