
template <
        class TokenizerFunc = char_delimiters_separator<char>, 
        class Iterator = std::string::const_iterator,
        class Type = std::string
>
class token_iterator_generator 
  template<class Type, class Iterator, class TokenizerFunc> typename token_iterator_generator<TokenizerFunc,Iterator,Type>::type make_token_iterator(Iterator begin, Iterator end,const TokenizerFunc& fun)
The token iterator serves to provide an iterator view of the tokens in a parsed sequence.
/// simple_example_5.cpp
#include<iostream>
#include<boost/token_iterator.hpp>
#include<string>
int main(){
   using namespace std;
   using namespace boost;
   string s = "12252001";
   int offsets[] = {2,2,4};
   offset_separator f(offsets, offsets+3);
   typedef token_iterator_generator<offset_separator>::type Iter;
   Iter beg = make_token_iterator<string>(s.begin(),s.end(),f);
   Iter end = make_token_iterator<string>(s.end(),s.end(),f); 
   // The above statement could also have been what is below
   // Iter end;
   for(;beg!=end;++beg){
     cout << *beg << "\n";
   }
}
  
| Parameter | Description | 
|---|---|
| TokenizerFunc | The TokenizerFunction used to parse the sequence. | 
| Iterator | The type of the iterator the specifies the sequence. | 
| Type | The type of the token, typically string. | 
The category of Iterator, up to and including Forward Iterator. Anything higher will get scaled down to Forward Iterator.
| Type | Remarks | 
| token_iterator_generator::type | The type of the token iterator. | 
template<class Type, class Iterator, class TokenizerFunc> typename token_iterator_generator<TokenizerFunc,Iterator,Type>::type make_token_iterator(Iterator begin, Iterator end,const TokenizerFunc& fun)
| Parameter | Description | 
| begin | The beginning of the sequence to be parsed. | 
| end | Past the end of the sequence to be parsed. | 
| fun | A functor that is a model of TokenizerFunction | 
Revised 25 December, 2006
Copyright © 2001 John R. Bandela
Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)