String Tokenizer

[Ultraman]

I completely forgot to come back and show the tokenize function I used to use that took advantage of strtok:

Code: (Select All)

Option _Explicit



Dim As _Offset tokenized

Dim As String toTokenize: toTokenize = "36" + Chr$(9) + "Hungry Coyote Import Store" + Chr$(9) + "Yoshi Latimer" + Chr$(9) + "City Center Plaza 516 Main St" + Chr$(9) + "Elgin" + Chr$(9) + "97827" + Chr$(9) + "USA" + Chr$(10) + "37" + Chr$(9) + "Hungry Owl All-Night Grocers" + Chr$(9) + "Patricia McKenna" + Chr$(9) + "8 Johnstown Road" + Chr$(9) + "Cork" + Chr$(9) + "" + Chr$(9) + "Ireland" + Chr$(10) + "38" + Chr$(9) + "Island Trading" + Chr$(9) + "Helen Bennett" + Chr$(9) + "Garden House Crowther Way" + Chr$(9) + "Cowes" + Chr$(9) + "PO31 7PJ" + Chr$(9) + "UK" + Chr$(10) '+ Chr$(0)

Dim As String delimiter: delimiter = Chr$(9) + Chr$(10)



Dim As Long i

ReDim As String tokenized(0 To 0)

tokenize toTokenize, Chr$(9) + Chr$(10), tokenized()



For i = LBound(tokenized) To UBound(tokenized)

    Print tokenized(i)

Next



Function pointerToString$ (pointer As _Offset)

    Declare CustomType Library

        Function strlen%& (ByVal ptr As _Unsigned _Offset)

    End Declare

    Dim As _Offset length: length = strlen(pointer)

    If length Then

        Dim As _MEM pString: pString = _Mem(pointer, length)

        Dim As String ret: ret = Space$(length)

        _MemGet pString, pString.OFFSET, ret

        _MemFree pString

    End If

    pointerToString = ret

End Function



Sub tokenize (toTokenize As String, delimiters As String, StorageArray() As String)

    Declare CustomType Library

        Function strtok%& (ByVal str As _Offset, delimiters As String)

    End Declare

    Dim As _Offset tokenized

    Dim As String tokCopy: If Right$(toTokenize, 1) <> Chr$(0) Then tokCopy = toTokenize + Chr$(0) Else tokCopy = toTokenize

    Dim As String delCopy: If Right$(delimiters, 1) <> Chr$(0) Then delCopy = delimiters + Chr$(0) Else delCopy = delimiters

    Dim As _Unsigned Long lowerbound: lowerbound = LBound(StorageArray)

    Dim As _Unsigned Long i: i = lowerbound

    tokenized = strtok(_Offset(tokCopy), delCopy)

    While tokenized <> 0

        ReDim _Preserve StorageArray(lowerbound To UBound(StorageArray) + 1)

        StorageArray(i) = pointerToString(tokenized)

        tokenized = strtok(0, delCopy)

        i = i + 1

    Wend

    ReDim _Preserve StorageArray(UBound(StorageArray) - 1)

End Sub