Tokenizer in QB64

[aurel]

I don't know how he did something else when both declare functions are same
look this :

his:
Declare function accept&(s as string)

my :
declare function tokenizer&( src as string)


so it is same ?


and i do as string variable which is of course declared with Dim

'call function tokenizer()

tokenizer&(test)

but he called function using quoted string
like :
accept&("text")

here is full code again:

Code: (Select All)

'tokenizer in QB64pe (fb) by Aurel

declare function tokenizer&( src as string)
'declare function run_tokenizer(inputCode as string) as integer

const tkNULL=0, tkPLUS=1, tkMINUS=2, tkMULTI=3, tkDIVIDE=4
const tkCOLON=5, tkCOMMA=6, tkLPAREN=7, tkRPAREN=8, tkLBRACKET=9, tkRBRACKET=10
const tkIDENT = 11 , tkNUMBER = 12 , tkQSTRING = 13, tkCOMMAND =14 ,tkEOL = 15
const tkEQUAL = 16, tkMORE = 17, tkLESS = 18, tkAND = 19, tkOR = 20, tkNOT = 21
const tkHASH=22 , tkSSTR=23, tkMOD=24 , tkSEMI=25, tkDOT=26, tkLBRACE=27, tkRBRACE=28
const  tkQUEST=29, tkMONKEY=30 , tkBACKSLAH=31, tkPOWUP=32 ,tkAPOSTR=33 , tkTILDA=34

Dim shared tokList(1024)  As string                       'token array
Dim shared typList(1024)  As integer                      'token type array
Dim shared p              As Long : p=1
Dim shared start          as Long : start = 1
Dim shared tp             as long
Dim shared tn             as long
Dim shared n              as long
Dim shared ltp            as long  : lpt = 1
Dim shared nTokens    As long                            'nTokens -> number of tokens
Dim shared lineCount As integer
Dim shared Lpar      as integer
Dim shared Rpar      as integer
Dim shared Lbrk      as integer
Dim shared Rbrk      as integer
Dim shared tokerr    as integer
Dim shared codeLen   as integer
Dim shared code      As String
Dim shared chs       As String
Dim shared tch       As String
Dim shared tk        As String
Dim shared crlf      As String
Dim shared bf        As String
Dim shared ntk       As String
crlf = chr$(13) + chr$(10)
'test string .......................................
Dim test as string  : test = "func tokenizer in QB64"
'...................................................

'call function tokenizer()
tokenizer&(test)



' *** MAIN TOKENIZER FUNCTION ***
FUNCTION tokenizer& (src as string)
print "tokenizer run:" + src
lineCount=0:ltp=start : nTokens = 0

tokenizer& = 0
END FUNCTION




do

loop until multikey(27)