PHP Classes

File: test.php

Recommend this page to a friend!
  Classes of Gonzalo Chumillas   Ses Tokenizer   test.php   Download  
File: test.php
Role: Example script
Content type: text/plain
Description: Examples
Class: Ses Tokenizer
Parse and split a string into tokens
Author: By
Last change: Update test.php
update
update
Date: 10 years ago
Size: 2,578 bytes
 

Contents

Class file image Download
<?php
header
("Content-Type: text/plain; charset=UTF-8");

require_once
"classes/parser/tokenizer.php";

// splits an string into 'words'
// Note that you DO NOT NEED to write an explicit regular expression. Instead of typing "/^\s*\w+/" we can write "\w+".
// In this case, the function ignores the left spaces and start searching from the current offset position.
$t = new Tokenizer("Lorem ipsum dolor sit amet");
while (list(
$token) = $t->match("\w+")) {
    echo
"$token-";
}
echo
"\n";

// In any case, you can still use explicit regular expressions:
$t = new Tokenizer("I'm 35 years old");
if (list(
$years) = $t->match("/\d+/")) {
    echo
"You are $years old\n";
}

// flags example.
// OFFSET_CAPTURE: retrieves offset positions
// CASE_SENSITIVE: switches to 'case sensitive' mode
// SEARCH_ANYWHERE: start searching from any position
$t = new Tokenizer("select * From table where id = 100", Tokenizer::OFFSET_CAPTURE|Tokenizer::CASE_SENSITIVE|Tokenizer::SEARCH_ANYWHERE);
if (
$t->match("From\s+(\w+)", $matches)) {
   
print_r($matches);
}

// parses a basic SQL sentence
$t = new Tokenizer("Select Id, Name, Age From users Where Id = 101");
if (
$t->match("select")) {
   
// columns
   
$columns = array();
    while (list(
$column) = $t->match("\w+")) {
       
array_push($columns, $column);
        if (!
$t->match(",")) {
            break;
        }
    }
   
// 'from' clause
   
if ($t->match("from\s+(\w+)", $matches)) {
       
$table_name = $matches[1];
        echo
"You want to get the columns " . implode(", ", $columns) . " from the table $table_name.\n";
    }
}

// splits a phrase into tokens
$t = new Tokenizer("Look at my horse, my horse is amazing");
while (list(
$token) = $t->token()) {
    echo
"$token-";
}
echo
"\n";

// is the next token equal to a given string?
$t = new Tokenizer("Lorem Ipsum Dolor");
if (list(
$str) = $t->eq("lorem ipsum dolor")) {
    echo
"$str Sit Amet\n";
}

// is the next token a number?
$t = new Tokenizer("125.48E-4");
if (list(
$number) = $t->number()) {
    echo
"Yes, it is a float number: $number\n";
}

// is the next token a string?
$t = new Tokenizer(' "Houston, we have a \"problem\"." ');
if (list(
$str) = $t->str()) {
    echo
"Yes, is is a string: $str\n";
}

// and the next, is a string?
$t = new Tokenizer(" 'Look at my horse, my \'horse\' is amazing' ");
if (list(
$str) = $t->str()) {
    echo
"Yes, is is a string too: $str\n";
}

// is the next token an identifier?
$t = new Tokenizer("Id: hello093");
if (
$t->eq("Id:") && list($id) = $t->identifier()) {
    echo
"Yes, it is an identifier: $id\n";
}