C++11stlintermediate
std::regex_token_iterator (C++11 Addition)
C++11 · Modern Revolution · Compiler flag: -std=c++11
Explanation
An iterator that accesses specific sub-matches or the parts of a string between matches, enabling tokenization and field extraction.
Real-World Scenario
Split a CSV line into fields using a comma delimiter pattern.
Quick Snippet
snippet.cpp
#include <regex>
std::string csv = "Alice,30,Engineer";
std::regex sep(",");
std::sregex_token_iterator it(csv.begin(), csv.end(), sep, -1);
std::sregex_token_iterator end;
for (; it != end; ++it) {
std::string field = *it;
}Full Example
regex_token_iterator.cpp
// CodeNebula - C++11 Example: std::regex_token_iterator
// Compile: g++ -std=c++11 -Wall -Wextra -o example regex_token_iterator.cpp
// std::regex_token_iterator iterates over specific sub-matches or the
// parts between matches. With submatch index -1, it acts as a splitter.
// With indices 0, 1, 2..., it extracts specific capture groups.
#include <iostream>
#include <regex>
#include <string>
#include <vector>
int main() {
// === Splitting strings (submatch = -1) ===
std::string csv = "Alice,30,London,Engineer";
std::regex comma_re(",");
std::cout << "Split by comma:" << std::endl;
std::sregex_token_iterator it(csv.begin(), csv.end(), comma_re, -1);
std::sregex_token_iterator end;
for (; it != end; ++it) {
std::cout << " \"" << *it << "\"" << std::endl;
}
// Split by multiple delimiters
std::string mixed = "one;two,three:four five";
std::regex delim_re(R"([;,:\s]+)");
std::cout << "\nSplit by mixed delimiters:" << std::endl;
for (std::sregex_token_iterator t(mixed.begin(), mixed.end(), delim_re, -1);
t != end; ++t) {
std::cout << " \"" << *t << "\"" << std::endl;
}
// === Extract submatch 0 (full match) ===
std::string text = "Call 555-1234 or 555-5678 today";
std::regex phone_re(R"(\d{3}-\d{4})");
std::cout << "\nAll phone numbers (submatch 0):" << std::endl;
for (std::sregex_token_iterator t(text.begin(), text.end(), phone_re, 0);
t != end; ++t) {
std::cout << " " << *t << std::endl;
}
// === Extract specific capture groups ===
std::string pairs = "name=Alice age=30 city=London";
std::regex kv_re(R"((\w+)=(\w+))");
// Extract just keys (group 1)
std::cout << "\nKeys only (group 1):" << std::endl;
for (std::sregex_token_iterator t(pairs.begin(), pairs.end(), kv_re, 1);
t != end; ++t) {
std::cout << " " << *t << std::endl;
}
// Extract just values (group 2)
std::cout << "\nValues only (group 2):" << std::endl;
for (std::sregex_token_iterator t(pairs.begin(), pairs.end(), kv_re, 2);
t != end; ++t) {
std::cout << " " << *t << std::endl;
}
// === Multiple submatch indices ===
std::string data = "2024-01-15 2024-06-30 2024-12-25";
std::regex date_re(R"((\d{4})-(\d{2})-(\d{2}))");
// Extract year and day (groups 1 and 3), interleaved
std::cout << "\nYear and day (groups 1,3):" << std::endl;
std::vector<int> indices = {1, 3};
for (std::sregex_token_iterator t(data.begin(), data.end(), date_re, indices);
t != end; ++t) {
std::cout << " " << *t << std::endl;
}
// === Collect into a vector ===
std::string words_str = "hello world foo bar baz";
std::regex space_re(R"(\s+)");
std::sregex_token_iterator first(words_str.begin(), words_str.end(), space_re, -1);
std::vector<std::string> words(first, end);
std::cout << "\nVector of words (" << words.size() << "): ";
for (const auto& w : words) std::cout << "[" << w << "] ";
std::cout << std::endl;
return 0;
}
// Expected output:
// Split by comma:
// "Alice"
// "30"
// "London"
// "Engineer"
//
// Split by mixed delimiters:
// "one"
// "two"
// "three"
// "four"
// "five"
//
// All phone numbers (submatch 0):
// 555-1234
// 555-5678
//
// Keys only (group 1):
// name
// age
// city
//
// Values only (group 2):
// Alice
// 30
// London
//
// Year and day (groups 1,3):
// 2024
// 15
// 2024
// 30
// 2024
// 25
//
// Vector of words (5): [hello] [world] [foo] [bar] [baz]
Run command
g++ -std=c++11 -Wall -Wextra -o example regex_token_iterator.cpp