diff --git a/changelog/src/main/org/apache/maven/cvslib/EnhancedStringTokenizer.java b/changelog/src/main/org/apache/maven/cvslib/EnhancedStringTokenizer.java
new file mode 100644
index 00000000..3e11c846
--- /dev/null
+++ b/changelog/src/main/org/apache/maven/cvslib/EnhancedStringTokenizer.java
@@ -0,0 +1,144 @@
+package org.apache.maven.cvslib;
+
+/* ====================================================================
+ * The Apache Software License, Version 1.1
+ *
+ * Copyright (c) 2003 The Apache Software Foundation. All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * 3. The end-user documentation included with the redistribution,
+ * if any, must include the following acknowledgment:
+ * "This product includes software developed by the
+ * Apache Software Foundation (http://www.apache.org/)."
+ * Alternately, this acknowledgment may appear in the software itself,
+ * if and wherever such third-party acknowledgments normally appear.
+ *
+ * 4. The names "Apache" and "Apache Software Foundation" and
+ * "Apache Maven" must not be used to endorse or promote products
+ * derived from this software without prior written permission. For
+ * written permission, please contact apache@apache.org.
+ *
+ * 5. Products derived from this software may not be called "Apache",
+ * "Apache Maven", nor may "Apache" appear in their name, without
+ * prior written permission of the Apache Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation. For more
+ * information on the Apache Software Foundation, please see
+ * .
+ *
+ * ====================================================================
+ */
+
+import java.util.StringTokenizer;
+
+/**
+ * The java.util.StringTokenizer is horribly broken.
+ * Given the string 1,,,3,,4 (, delim)
+ * It will return 1,3,4
+ * Which is clearly wrong - 1,EMPTY,EMPTY,3,EMPTY,4 is what it should return
+ */
+public final class EnhancedStringTokenizer {
+ private StringTokenizer cst = null;
+ String cdelim;
+ final boolean cdelimSingleChar;
+ final char cdelimChar;
+ boolean creturnDelims;
+
+ public EnhancedStringTokenizer(String str) {
+ this(str, " \t\n\r\f", false);
+ }
+
+ public EnhancedStringTokenizer(String str, String delim) {
+ this(str, delim, false);
+ }
+
+ public EnhancedStringTokenizer(String str, String delim, boolean returnDelims) {
+ cst = new StringTokenizer(str, delim, true);
+ cdelim = delim;
+ creturnDelims = returnDelims;
+ cdelimSingleChar = (delim.length() == 1);
+ cdelimChar = delim.charAt(0);
+ }
+
+ public boolean hasMoreTokens() {
+ return cst.hasMoreTokens();
+ }
+
+ String lastToken = null;
+ boolean delimLast = true;
+ private String internalNextToken() {
+ if (lastToken != null) {
+ String last = lastToken;
+ lastToken = null;
+ return last;
+ }
+
+ String token = cst.nextToken();
+ if (isDelim(token)) {
+ if (delimLast) {
+ lastToken = token;
+ return "";
+ } else {
+ delimLast = true;
+ return token;
+ }
+ } else {
+ delimLast = false;
+ return token;
+ }
+ }
+
+ public String nextToken() {
+ String token = internalNextToken();
+ if (creturnDelims)
+ return token;
+ if (isDelim(token))
+ return hasMoreTokens() ? internalNextToken() : "";
+ else
+ return token;
+ }
+
+ private boolean isDelim(String str) {
+ if (str.length() == 1) {
+ char ch = str.charAt(0);
+ if (cdelimSingleChar) {
+ if (cdelimChar == ch) {
+ return true;
+ }
+ } else {
+ if (cdelim.indexOf(ch) >= 0) {
+ return true;
+ }
+ }
+ }
+ return false;
+
+ }
+}
\ No newline at end of file
diff --git a/changelog/src/test/org/apache/maven/cvslib/EnhancedStringTokenizerTest.java b/changelog/src/test/org/apache/maven/cvslib/EnhancedStringTokenizerTest.java
new file mode 100644
index 00000000..bd18fa86
--- /dev/null
+++ b/changelog/src/test/org/apache/maven/cvslib/EnhancedStringTokenizerTest.java
@@ -0,0 +1,142 @@
+package org.apache.maven.cvslib;
+
+/* ====================================================================
+ * The Apache Software License, Version 1.1
+ *
+ * Copyright (c) 2003 The Apache Software Foundation. All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * 3. The end-user documentation included with the redistribution,
+ * if any, must include the following acknowledgment:
+ * "This product includes software developed by the
+ * Apache Software Foundation (http://www.apache.org/)."
+ * Alternately, this acknowledgment may appear in the software itself,
+ * if and wherever such third-party acknowledgments normally appear.
+ *
+ * 4. The names "Apache" and "Apache Software Foundation" and
+ * "Apache Maven" must not be used to endorse or promote products
+ * derived from this software without prior written permission. For
+ * written permission, please contact apache@apache.org.
+ *
+ * 5. Products derived from this software may not be called "Apache",
+ * "Apache Maven", nor may "Apache" appear in their name, without
+ * prior written permission of the Apache Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation. For more
+ * information on the Apache Software Foundation, please see
+ * .
+ *
+ * ====================================================================
+ */
+
+import junit.framework.TestCase;
+
+public class EnhancedStringTokenizerTest extends TestCase {
+
+ public void testSimple() {
+ EnhancedStringTokenizer tok = new EnhancedStringTokenizer("a,b,c,d,,e,,,f,,,,,g", ",");
+ assertEquals("Token 1", "a", tok.nextToken());
+ assertEquals("Token 2", "b", tok.nextToken());
+ assertEquals("Token 3", "c", tok.nextToken());
+ assertEquals("Token 4", "d", tok.nextToken());
+ assertEquals("Token 5", "", tok.nextToken());
+ assertEquals("Token 6", "e", tok.nextToken());
+ assertEquals("Token 7", "", tok.nextToken());
+ assertEquals("Token 8", "", tok.nextToken());
+ assertEquals("Token 9", "f", tok.nextToken());
+ assertEquals("Token 10", "", tok.nextToken());
+ assertEquals("Token 11", "", tok.nextToken());
+ assertEquals("Token 12", "", tok.nextToken());
+ assertEquals("Token 13", "", tok.nextToken());
+ assertEquals("Token 14", "g", tok.nextToken());
+ assertEquals("tok.hasMoreTokens()", false, tok.hasMoreTokens());
+ }
+
+ public void testSimpleTab() {
+ EnhancedStringTokenizer tok = new EnhancedStringTokenizer("a\tb\tc\td\t\te\t\t\tf\t\t\t\t\tg", "\t");
+ assertEquals("Token 1", "a", tok.nextToken());
+ assertEquals("Token 2", "b", tok.nextToken());
+ assertEquals("Token 3", "c", tok.nextToken());
+ assertEquals("Token 4", "d", tok.nextToken());
+ assertEquals("Token 5", "", tok.nextToken());
+ assertEquals("Token 6", "e", tok.nextToken());
+ assertEquals("Token 7", "", tok.nextToken());
+ assertEquals("Token 8", "", tok.nextToken());
+ assertEquals("Token 9", "f", tok.nextToken());
+ assertEquals("Token 10", "", tok.nextToken());
+ assertEquals("Token 11", "", tok.nextToken());
+ assertEquals("Token 12", "", tok.nextToken());
+ assertEquals("Token 13", "", tok.nextToken());
+ assertEquals("Token 14", "g", tok.nextToken());
+ assertEquals("tok.hasMoreTokens()", false, tok.hasMoreTokens());
+ }
+
+ public void testNoDelim() {
+ EnhancedStringTokenizer tok = new EnhancedStringTokenizer("a,b,c,d,,e,,,f,,,,,g", "\t");
+ assertEquals("Token 1", "a,b,c,d,,e,,,f,,,,,g", tok.nextToken());
+ assertEquals("tok.hasMoreTokens()", false, tok.hasMoreTokens());
+ }
+
+ public void testNoText() {
+ EnhancedStringTokenizer tok = new EnhancedStringTokenizer("", ",");
+ assertEquals("tok.hasMoreTokens()", false, tok.hasMoreTokens());
+ }
+
+ public void testReturnDelim() {
+ EnhancedStringTokenizer tok = new EnhancedStringTokenizer("a,b,,c,d", ",", true);
+ assertEquals("Token 1", "a", tok.nextToken());
+ assertEquals("Token 2", ",", tok.nextToken());
+ assertEquals("Token 3", "b", tok.nextToken());
+ assertEquals("Token 4", ",", tok.nextToken());
+ assertEquals("Token 5", "", tok.nextToken());
+ assertEquals("Token 6", ",", tok.nextToken());
+ assertEquals("Token 7", "c", tok.nextToken());
+ assertEquals("Token 2", ",", tok.nextToken());
+ assertEquals("Token 8", "d", tok.nextToken());
+ assertEquals("tok.hasMoreTokens()", false, tok.hasMoreTokens());
+ }
+
+ public void testMultipleDelim() {
+ EnhancedStringTokenizer tok = new EnhancedStringTokenizer("a.,b.,c.d", ".,", true);
+ assertEquals("Token 1", "a", tok.nextToken());
+ assertEquals("Token 2", ".", tok.nextToken());
+ assertEquals("Token 3", "", tok.nextToken());
+ assertEquals("Token 4", ",", tok.nextToken());
+ assertEquals("Token 5", "b", tok.nextToken());
+ assertEquals("Token 6", ".", tok.nextToken());
+ assertEquals("Token 7", "", tok.nextToken());
+ assertEquals("Token 8", ",", tok.nextToken());
+ assertEquals("Token 9", "c", tok.nextToken());
+ assertEquals("Token 10", ".", tok.nextToken());
+ assertEquals("Token 11", "d", tok.nextToken());
+ assertEquals("tok.hasMoreTokens()", false, tok.hasMoreTokens());
+ }
+
+}
\ No newline at end of file