#
# I wrote this along time ago
# I can't remember how it works, but it does
# Control tokens with a trailing space seem to mean a text token follows
#
# Cal Henderson, July 2004
# <cal@iamcal.com>
#

sub tokenise_rtf{
	my ($data) = @_;

	my @pretokens = ($data =~ m!((?:\{)|(?:\}\s?)|(?:\\[^\s\\\{\}\']+\s?\n?)|(?:[^\\][^\\\{\}\n]*\n?)|(?:\s)|(?:\\\'[a-zA-Z0-9][a-zA-Z0-9]))!sixg);
	my @tokens;

	foreach $token(@pretokens){
		chomp $token;
		$token = '' if $token !~ m|[\S\ ]|;
		push @tokens, $token if $token ne '';
	};

	return \@tokens;
}