svndump.jj

options {
    STATIC = false;
    JAVA_UNICODE_ESCAPE = false;
    USER_CHAR_STREAM = true;
}

PARSER_BEGIN(SvnDumpParser)
  package com.github.cstroe.svndumpgui.generated;

  import java.io.InputStream;
  import java.io.InputStreamReader;
  import java.io.Reader;
  import java.io.IOException;
  import java.io.ByteArrayOutputStream;
  import java.io.UnsupportedEncodingException;
  import java.math.BigInteger;
  import java.nio.ByteBuffer;
  import java.nio.CharBuffer;
  import java.nio.charset.StandardCharsets;
  import java.util.Map;
  import java.util.LinkedHashMap;
  import java.security.MessageDigest;

  import com.github.cstroe.svndumpgui.api.ContentChunk;
  import com.github.cstroe.svndumpgui.api.RepositoryConsumer;
  import com.github.cstroe.svndumpgui.api.Preamble;
  import com.github.cstroe.svndumpgui.api.RepositoryWriter;
  import com.github.cstroe.svndumpgui.internal.utility.SvnDumpCharStream;
  import com.github.cstroe.svndumpgui.api.Node;
  import com.github.cstroe.svndumpgui.api.NodeHeader;
  import com.github.cstroe.svndumpgui.api.Property;
  import com.github.cstroe.svndumpgui.api.Revision;
  import com.github.cstroe.svndumpgui.internal.ContentChunkImpl;
  import com.github.cstroe.svndumpgui.internal.PreambleImpl;
  import com.github.cstroe.svndumpgui.internal.NodeImpl;
  import com.github.cstroe.svndumpgui.internal.RevisionImpl;
  import com.github.cstroe.svndumpgui.internal.utility.SvnDumpCharStream;

  public class SvnDumpParser {
    private int fileContentChunkSize = 1024 * 1024 * 4; // 4 MB buffer for file content chunks by default

    public void setFileContentChunkSize(int size) {
        this.fileContentChunkSize = size;
    }

    private String readCharacters(Integer numberOfBytes) {
        if(numberOfBytes == null) {
            return null;
        }

        try {
            byte[] bytes = ((SvnDumpCharStream) token_source.input_stream).readBytes(numberOfBytes);
            return new String(bytes, StandardCharsets.UTF_8);
        } catch(IOException ex) {
            throw new RuntimeException(ex);
        }
    }

    private void readChunks(Long length, RepositoryConsumer consumer, String md5sum, String sha1sum, String path) throws ParseException {
        if(length == null || length == 0) {
            return;
        }

        MessageDigest md5;
        MessageDigest sha1;

        try {
            md5 = MessageDigest.getInstance("MD5");
            sha1 = MessageDigest.getInstance("SHA1");
        } catch(java.security.NoSuchAlgorithmException ex) {
            throw new RuntimeException(ex);
        }

        SvnDumpCharStream stream = (SvnDumpCharStream) token_source.input_stream;
        long lengthLeft = length;
        while(lengthLeft > 0) {
            int chunkSize = (int) Math.min((long)fileContentChunkSize, lengthLeft);

            ContentChunk chunk = null;
            try {
                byte[] freshBytes = stream.readBytes(chunkSize);
                if(md5sum != null) {
                    md5.update(freshBytes);
                }
                if(sha1sum != null) {
                    sha1.update(freshBytes);
                }
                chunk = new ContentChunkImpl(freshBytes);
                lengthLeft -= chunkSize;
            } catch (IOException ex) {
                throw new RuntimeException(ex);
            }

            assert chunk != null;
            consumer.consume(chunk);
        }
        consumer.endChunks();

        String computedMd5Sum = toHex(md5.digest(), 32);
        String computedSha1Sum = toHex(sha1.digest(), 40);

        if(md5sum != null && !md5sum.equals(computedMd5Sum)) {
            throw new ParseException("MD5 sum is incorrect! Expected: " + md5sum + ", Actual: " + computedMd5Sum);
        }

        if(sha1sum != null && !sha1sum.equals(computedSha1Sum)) {
            throw new ParseException("SHA1 sum is incorrect! Expected: " + sha1sum + ", Actual: " + computedSha1Sum);
        }

        token.next = null; // we've advanced in the stream, we don't know what the next token is.
    }

    // swiped from http://stackoverflow.com/questions/415953
    private String toHex(byte[] digest, int length) {
        BigInteger bitInt = new BigInteger(1, digest);
        String hashText = bitInt.toString(16);

        while(hashText.length() < length) {
            hashText = "0" + hashText;
        }

        return hashText;
    }

    public static void consume(InputStream inputStream, RepositoryConsumer consumer) throws ParseException {
        new SvnDumpParser(new SvnDumpCharStream(inputStream)).Start(consumer);
    }
  }
PARSER_END(SvnDumpParser)

//  Lexer

SKIP: { " " }
TOKEN: { <EOL: "\n" | "\r" | "\r\n"> }
TOKEN: { <NUMBER: (["0"-"9"])+> }
TOKEN: { <  COLON: ":"> }
TOKEN: { <VERSION_KEY: "SVN-fs-dump-format-version"> }
TOKEN: { <UUID_KEY: "UUID"> }
TOKEN: { <UUID_VALUE: (["0"-"9","a"-"z"]){8} "-" (["0"-"9","a"-"z"]){4} "-" (["0"-"9","a"-"z"]){4} "-" (["0"-"9","a"-"z"]){4} "-" (["0"-"9","a"-"z"]){12}> }
TOKEN: { <REVISION_NUMBER_KEY: "Revision-number"> }
TOKEN: { <PROP_CONTENT_LENGTH_KEY: "Prop-content-length"> }
TOKEN: { <CONTENT_LENGTH_KEY: "Content-length"> }

TOKEN: { <KEY: "K"> }
TOKEN: { <VAL: "V"> }
TOKEN: { <PROPS_END: "PROPS-END"> }

TOKEN: { <NODE_PATH_KEY: "Node-path: "> : READ_PATH }
TOKEN: { <NODE_KIND_KEY: "Node-kind"> }
TOKEN: { <NODE_KIND_VALUE: ("file"|"dir")> }
TOKEN: { <NODE_ACTION_KEY: "Node-action"> }
TOKEN: { <NODE_ACTION_VALUE: ("change" | "add" | "delete" | "replace")> }
TOKEN: { <NODE_COPYFROM_REV_KEY: "Node-copyfrom-rev"> }
TOKEN: { <NODE_COPYFROM_PATH_KEY: "Node-copyfrom-path: "> : READ_PATH }

TOKEN: { <TEXT_CONTENT_LENGTH_KEY: "Text-content-length"> }
TOKEN: { <TEXT_CONTENT_MD5_KEY: "Text-content-md5"> }
TOKEN: { <MD5_VALUE: (["0"-"9","a"-"z"]){32}> }
TOKEN: { <TEXT_CONTENT_SHA1_KEY: "Text-content-sha1"> }
TOKEN: { <SHA1_VALUE: (["0"-"9","a"-"z"]){40}> }
TOKEN: { <TEXT_COPYSOURCE_MD5_KEY: "Text-copy-source-md5"> }
TOKEN: { <TEXT_COPYSOURCE_SHA1_KEY: "Text-copy-source-sha1"> }

<READ_PATH> TOKEN: { <NODE_PATH_VALUE: (~["\n","\r"])*> : DEFAULT }

// Parser

public void Start(RepositoryConsumer consumer):
{
  Token dumpVersion;
  Token uuid;
  Revision revision;
  Node node = null;
}
{
    {
        if(!(token_source.input_stream instanceof SvnDumpCharStream)) {
            throw new IllegalArgumentException("SvnDumpParser expects only an SvnDumpCharStream as the input stream.");
        }
    }
    <VERSION_KEY> <COLON> dumpVersion = <NUMBER> <EOL>
    <EOL>

    <UUID_KEY> <COLON> uuid = <UUID_VALUE> <EOL>
    <EOL>

    {
        Preamble preamble = new PreambleImpl(uuid.image);
        consumer.consume(preamble);
    }

    (
        revision = Revision()
        { consumer.consume(revision); }

        <EOL>

        (
            node = Node(revision)
            {
                consumer.consume(node);

                String textContentLength = node.get(NodeHeader.TEXT_CONTENT_LENGTH);
                String md5sum = node.get(NodeHeader.MD5);
                String sha1sum = node.get(NodeHeader.SHA1);
                if(textContentLength != null && Long.parseLong(textContentLength) != 0) {
                    readChunks(Long.parseLong(textContentLength), consumer, md5sum, sha1sum, node.get(NodeHeader.PATH));
                }
            }
            (
                <EOL>
                {
                    // really bad hack to know how many EOLs are in the stream
                    String currentValueRaw = node.getProperties().get(Property.TRAILING_NEWLINE_HINT);
                    if(currentValueRaw != null) {
                        int currentValue = Integer.parseInt(currentValueRaw);
                        currentValue++;
                        node.getProperties().put(Property.TRAILING_NEWLINE_HINT, Integer.toString(currentValue));
                    } else {
                        node.getProperties().put(Property.TRAILING_NEWLINE_HINT, "1");
                    }
                }
            )+
            {
                consumer.endNode(node);
            }
        )*
        {
            consumer.endRevision(revision);
            revision = null;
        }
    )*

    <EOF>

    { consumer.finish(); }
}

public Revision Revision():
{
    RevisionImpl revision;
    Token revisionNumber;
    Map properties;
}
{
    <REVISION_NUMBER_KEY> <COLON> revisionNumber = <NUMBER> <EOL>

    ( <PROP_CONTENT_LENGTH_KEY> <COLON> <NUMBER> <EOL> )?

    <CONTENT_LENGTH_KEY> <COLON> <NUMBER> <EOL>
    <EOL>

    { revision = new RevisionImpl(Integer.parseInt(revisionNumber.image)); }

    properties = Property()
    { revision.setProperties(properties); }

    { return revision; }
}

public Map Property():
{
    Map properties = new LinkedHashMap();
    String key, value;
    Token keyLength, valueLength;
}
{
    (
        <KEY> keyLength = <NUMBER> <EOL>
        { key = readCharacters(Integer.parseInt(keyLength.image)); }
        <EOL>

        <VAL> valueLength = <NUMBER> <EOL>
        { value = readCharacters(Integer.parseInt(valueLength.image)); }
        <EOL>

        {
            properties.put(key, value);
        }
    )*

    <PROPS_END> <EOL>

    { return properties; }
}

public Node Node(Revision revision):
{
    NodeImpl svnNode;
    Token nodePath = null;
    Token nodeKind = null;
    Token nodeAction = null;
    Token nodeMd5 = null;
    Token nodeSha1 = null;
    Token contentLength = null;
    Token textContentLength = null;
    Token propContentLength = null;
    Token copiedFromRevision = null;
    Token copiedFromPath = null;
    Token copiedFromMd5 = null;
    Token copiedFromSha1 = null;
    Map headers = new LinkedHashMap();
}
{
    (
        <NODE_PATH_KEY> nodePath = <NODE_PATH_VALUE> <EOL>
        {
            headers.put(NodeHeader.PATH, nodePath.image);
        }
    |
        <NODE_KIND_KEY> <COLON> nodeKind = <NODE_KIND_VALUE> <EOL>
        {
            headers.put(NodeHeader.KIND, nodeKind.image);
        }
    |
        <NODE_ACTION_KEY> <COLON> nodeAction = <NODE_ACTION_VALUE> <EOL>
        {
            headers.put(NodeHeader.ACTION, nodeAction.image);
        }
    |
        <PROP_CONTENT_LENGTH_KEY> <COLON> propContentLength = <NUMBER> <EOL>
        {
            headers.put(NodeHeader.PROP_CONTENT_LENGTH, propContentLength.image);
        }
    |
        <TEXT_CONTENT_LENGTH_KEY> <COLON> textContentLength = <NUMBER> <EOL>
        {
            headers.put(NodeHeader.TEXT_CONTENT_LENGTH, textContentLength.image);
        }
    |
        <TEXT_CONTENT_MD5_KEY> <COLON> nodeMd5 = <MD5_VALUE> <EOL>
        {
            headers.put(NodeHeader.MD5, nodeMd5.image);
        }
    |
        <TEXT_CONTENT_SHA1_KEY> <COLON> nodeSha1 = <SHA1_VALUE> <EOL>
        {
            headers.put(NodeHeader.SHA1, nodeSha1.image);
        }
    |
        <CONTENT_LENGTH_KEY> <COLON> contentLength = <NUMBER> <EOL>
        {
            headers.put(NodeHeader.CONTENT_LENGTH, contentLength.image);
        }
    |
        <NODE_COPYFROM_REV_KEY> <COLON> copiedFromRevision = <NUMBER> <EOL>
        {
            headers.put(NodeHeader.COPY_FROM_REV, copiedFromRevision.image);
        }
    |
        <NODE_COPYFROM_PATH_KEY> copiedFromPath = <NODE_PATH_VALUE> <EOL>
        {
            headers.put(NodeHeader.COPY_FROM_PATH, copiedFromPath.image);
        }
    |
        <TEXT_COPYSOURCE_MD5_KEY> <COLON> copiedFromMd5 =  <MD5_VALUE> <EOL>
        {
            headers.put(NodeHeader.SOURCE_MD5, copiedFromMd5.image);
        }
    |
        <TEXT_COPYSOURCE_SHA1_KEY> <COLON> copiedFromSha1 =  <SHA1_VALUE> <EOL>
        {
            headers.put(NodeHeader.SOURCE_SHA1, copiedFromSha1.image);
        }
    )+

    {
        svnNode = new NodeImpl(revision);
        svnNode.setHeaders(headers);
        headers = null;

        if(propContentLength != null) {
            jj_consume_token(EOL);
            svnNode.setProperties(Property());
        }
    }

    { return svnNode; }
}