cajparser.py

import os
import struct
from shutil import copy
from subprocess import check_output, STDOUT, CalledProcessError
from utils import fnd, fnd_all, add_outlines, fnd_rvrs, fnd_unuse_no

KDH_PASSPHRASE = b"FZHMEI"

printables = ''.join([(len(repr(chr(x)))==3) and (x != 47) and (x < 128) and chr(x) or '.' for x in range(256)])

image_type = {
    0 : "JBIG",
    1 : "JPEG",
    2 : "JPEG", # up-side-down
    3 : "JBIG2"
    }

class CAJParser(object):
    def __init__(self, filename):
        self.filename = filename
        try:
            with open(filename, "rb") as caj:
                caj_read4 = caj.read(4)
                if (caj_read4[0:1] == b'\xc8'):
                    self.format = "C8"
                    self._PAGE_NUMBER_OFFSET = 0x08
                    self._TOC_NUMBER_OFFSET = 0 # No TOC
                    self._TOC_END_OFFSET = 0x50
                    self._PAGEDATA_OFFSET = self._TOC_END_OFFSET + 20 * self.page_num
                    return
                if (caj_read4[0:2] == b'HN'):
                    if (caj.read(2) == b'\xc8\x00'): # Most of them are: 90 01, handled later
                        self.format = "HN"
                        self._PAGE_NUMBER_OFFSET = 0x90
                        self._TOC_NUMBER_OFFSET = 0
                        self._TOC_END_OFFSET = 0xD8
                        self._PAGEDATA_OFFSET = self._TOC_END_OFFSET + 20 * self.page_num
                        return
                fmt = struct.unpack("4s", caj_read4)[0].replace(b'\x00', b'').decode("gb18030")
            if fmt == "CAJ":
                self.format = "CAJ"
                self._PAGE_NUMBER_OFFSET = 0x10
                self._TOC_NUMBER_OFFSET = 0x110
            elif fmt == "HN":
                self.format = "HN"
                self._PAGE_NUMBER_OFFSET = 0x90
                self._TOC_NUMBER_OFFSET = 0x158

                # TOC = [toc_num] followed by [toc_entry * toc_num]
                # followed by [Page Info struct (20-byte) * page_num], followed by Page Data
                self._TOC_END_OFFSET = self._TOC_NUMBER_OFFSET + 4 + 0x134 * self.toc_num
                self._PAGEDATA_OFFSET = self._TOC_END_OFFSET + 20 * self.page_num
            elif fmt == "%PDF":
                self.format = "PDF"
            elif fmt == "KDH ":
                self.format = "KDH"
            elif fmt == "TEB":
                self.format = "TEB"
            else:
                self.format = None
                raise SystemExit("Unknown file type.")
        except UnicodeDecodeError:
            raise SystemExit("Unknown file type.")

    @property
    def page_num(self):
        with open(self.filename, "rb") as caj:
            caj.seek(self._PAGE_NUMBER_OFFSET)
            [page_num] = struct.unpack("i", caj.read(4))
            return page_num

    @property
    def toc_num(self):
        if (self._TOC_NUMBER_OFFSET == 0):
            return 0
        with open(self.filename, "rb") as caj:
            caj.seek(self._TOC_NUMBER_OFFSET)
            [toc_num] = struct.unpack("i", caj.read(4))
            return toc_num

    def get_toc(self, verbose=False):
        toc = []
        if (self._TOC_NUMBER_OFFSET == 0):
            return toc
        with open(self.filename, "rb") as caj:
            for i in range(self.toc_num):
                caj.seek(self._TOC_NUMBER_OFFSET + 4 + 0x134 * i)
                toc_bytes = struct.unpack("256s24s12s12si", caj.read(0x134))
                ttl_end = toc_bytes[0].find(b"\x00")
                title = toc_bytes[0][0:ttl_end].decode("gb18030").encode("utf-8")
                pg_end = toc_bytes[2].find(b"\x00")
                page = int(toc_bytes[2][0:pg_end])
                level = toc_bytes[4]
                toc_entry = {"title": title, "page": page, "level": level}
                if ( verbose ):
                    print("   " * (level -1), title.decode("utf-8"))
                toc.append(toc_entry)
            if ( verbose ):
                print("TOC END: 0x%04X" % (self._TOC_NUMBER_OFFSET + 4 + 0x134 * self.toc_num))
        return toc

    def output_toc(self, dest):
        toc_items = self.get_toc()
        with open(dest, "wb") as f:
            for toc in toc_items:
                f.write(b'    ' * (toc["level"] - 1) + toc["title"]
                        + b'    ' + str(toc["page"]).encode("utf-8") + b'\n')

    def convert(self, dest):
        if self.format == "CAJ":
            self._convert_caj(dest)
        elif self.format == "HN":
            self._convert_hn(dest)
        elif self.format == "C8":
            self._convert_hn(dest)
        elif self.format == "PDF":
            self._convert_pdf(dest)
        elif self.format == "KDH":
            self._convert_kdh(dest)

    def parse(self):
        if self.format == "CAJ":
            pass
        elif self.format == "HN":
            self._parse_hn()
        elif self.format == "C8":
            self._parse_hn()
        elif self.format == "PDF":
            pass
        elif self.format == "KDH":
            pass

    def text_extract(self):
        if self.format == "CAJ":
            pass
        if self.format == "HN":
            self._text_extract_hn()
        elif self.format == "C8":
            self._text_extract_hn()
        elif self.format == "PDF":
            pass
        elif self.format == "KDH":
            pass

    def _convert_caj(self, dest):
        caj = open(self.filename, "rb")

        # Extract original PDF data (and add header)
        caj.seek(self._PAGE_NUMBER_OFFSET + 4)
        [pdf_start_pointer] = struct.unpack("i", caj.read(4))
        caj.seek(pdf_start_pointer)
        [pdf_start] = struct.unpack("i", caj.read(4))
        pdf_end = fnd_all(caj, b"endobj")[-1] + 6
        pdf_length = pdf_end - pdf_start
        caj.seek(pdf_start)
        pdf_data = b"%PDF-1.3\r\n" + caj.read(pdf_length) + b"\r\n"
        with open("pdf.tmp", 'wb') as f:
            f.write(pdf_data)
        pdf = open("pdf.tmp", "rb")

        # deal with disordered PDF data
        endobj_addr = fnd_all(pdf, b"endobj")
        obj_no = []
        for addr in endobj_addr:
            startobj = fnd_rvrs(pdf, b" 0 obj", addr)
            startobj1 = fnd_rvrs(pdf, b"\r", startobj)
            startobj2 = fnd_rvrs(pdf, b"\n", startobj)
            startobj = max(startobj1, startobj2)
            length = fnd(pdf, b" ", startobj) - startobj
            pdf.seek(startobj)
            [no] = struct.unpack(str(length) + "s", pdf.read(length))
            if int(no) not in obj_no:
                obj_no.append(int(no))
                obj_len = addr - startobj + 6
                pdf.seek(startobj)
                [obj] = struct.unpack(str(obj_len) + "s", pdf.read(obj_len))

        # Add Catalog (find obj_no of pages)
        inds_addr = [i + 8 for i in fnd_all(pdf, b"/Parent")]
        inds = []
        for addr in inds_addr:
            length = fnd(pdf, b" ", addr) - addr
            pdf.seek(addr)
            [ind] = struct.unpack(str(length) + "s", pdf.read(length))
            inds.append(int(ind))
        # get pages_obj_no list containing distinct elements
        # & find missing pages object(s) -- top pages object(s) in pages_obj_no
        pages_obj_no = []
        top_pages_obj_no = []
        for ind in inds:
            if (ind not in pages_obj_no) and (ind not in top_pages_obj_no):
                if fnd(pdf, bytes("\r{0} 0 obj".format(ind), "utf-8")) == -1:
                    top_pages_obj_no.append(ind)
                else:
                    pages_obj_no.append(ind)
        single_pages_obj_missed = len(top_pages_obj_no) == 1
        multi_pages_obj_missed = len(top_pages_obj_no) > 1
        # generate catalog object
        catalog_obj_no = fnd_unuse_no(obj_no, top_pages_obj_no)
        obj_no.append(catalog_obj_no)
        root_pages_obj_no = None
        if multi_pages_obj_missed:
            root_pages_obj_no = fnd_unuse_no(obj_no, top_pages_obj_no)
        elif single_pages_obj_missed:
            root_pages_obj_no = top_pages_obj_no[0]
            top_pages_obj_no = pages_obj_no
        else:  # root pages object exists, then find the root pages object #
            found = False
            for pon in pages_obj_no:
                tmp_addr = fnd(pdf, bytes("\r{0} 0 obj".format(pon), 'utf-8'))
                while True:
                    pdf.seek(tmp_addr)
                    [_str] = struct.unpack("6s", pdf.read(6))
                    if _str == b"Parent":
                        break
                    elif _str == b"endobj":
                        root_pages_obj_no = pon
                        found = True
                        break
                    tmp_addr = tmp_addr + 1
                if found:
                    break
        catalog = bytes("{0} 0 obj\r<</Type /Catalog\r/Pages {1} 0 R\r>>\rendobj\r".format(
            catalog_obj_no, root_pages_obj_no), "utf-8")
        pdf_data += catalog
        pdf.close()
        with open("pdf.tmp", 'wb') as f:
            f.write(pdf_data)
        pdf = open("pdf.tmp", "rb")

        # Add Pages obj and EOF mark
        # if root pages object exist, pass
        # deal with single missing pages object
        if single_pages_obj_missed or multi_pages_obj_missed:
            inds_str = ["{0} 0 R".format(i) for i in top_pages_obj_no]
            kids_str = "[{0}]".format(" ".join(inds_str))
            pages_str = "{0} 0 obj\r<<\r/Type /Pages\r/Kids {1}\r/Count {2}\r>>\rendobj\r".format(
                root_pages_obj_no, kids_str, self.page_num)
            pdf_data += bytes(pages_str, "utf-8")
            pdf.close()
            with open("pdf.tmp", 'wb') as f:
                f.write(pdf_data)
            pdf = open("pdf.tmp", "rb")
        # deal with multiple missing pages objects
        if multi_pages_obj_missed:
            kids_dict = {i: [] for i in top_pages_obj_no}
            count_dict = {i: 0 for i in top_pages_obj_no}
            for tpon in top_pages_obj_no:
                kids_addr = fnd_all(pdf, bytes("/Parent {0} 0 R".format(tpon), "utf-8"))
                for kid in kids_addr:
                    ind = fnd_rvrs(pdf, b"obj", kid) - 4
                    addr = fnd_rvrs(pdf, b"\r", ind)
                    length = fnd(pdf, b" ", addr) - addr
                    pdf.seek(addr)
                    [ind] = struct.unpack(str(length) + "s", pdf.read(length))
                    kids_dict[tpon].append(int(ind))
                    type_addr = fnd(pdf, b"/Type", addr) + 5
                    tmp_addr = fnd(pdf, b"/", type_addr) + 1
                    pdf.seek(tmp_addr)
                    [_type] = struct.unpack("5s", pdf.read(5))
                    if _type == b"Pages":
                        cnt_addr = fnd(pdf, b"/Count ", addr) + 7
                        pdf.seek(cnt_addr)
                        [_str] = struct.unpack("1s", pdf.read(1))
                        cnt_len = 0
                        while _str not in [b" ", b"\r", b"/"]:
                            cnt_len += 1
                            pdf.seek(cnt_addr + cnt_len)
                            [_str] = struct.unpack("1s", pdf.read(1))
                        pdf.seek(cnt_addr)
                        [cnt] = struct.unpack(str(cnt_len) + "s", pdf.read(cnt_len))
                        count_dict[tpon] += int(cnt)
                    else:  # _type == b"Page"
                        count_dict[tpon] += 1
                kids_no_str = ["{0} 0 R".format(i) for i in kids_dict[tpon]]
                kids_str = "[{0}]".format(" ".join(kids_no_str))
                pages_str = "{0} 0 obj\r<<\r/Type /Pages\r/Kids {1}\r/Count {2}\r>>\rendobj\r".format(
                    tpon, kids_str, count_dict[tpon])
                pdf_data += bytes(pages_str, "utf-8")
        pdf_data += bytes("\n%%EOF\r", "utf-8")
        pdf.close()
        with open("pdf.tmp", 'wb') as f:
            f.write(pdf_data)

        # Use mutool to repair xref
        try:
            check_output(["mutool", "clean", "pdf.tmp", "pdf_toc.pdf"], stderr=STDOUT)
        except CalledProcessError as e:
            print(e.output.decode("utf-8"))
            raise SystemExit("Command mutool returned non-zero exit status " + str(e.returncode))

        # Add Outlines
        add_outlines(self.get_toc(), "pdf_toc.pdf", dest)
        os.remove("pdf.tmp")
        os.remove("pdf_toc.pdf")

    def _convert_hn(self, dest):
        caj = open(self.filename, "rb")
        image_list = []

        from pdfwutils import Colorspace, ImageFormat, convert_ImageList
        import zlib

        for i in range(self.page_num):
            caj.seek(self._TOC_END_OFFSET + i * 20)
            [page_data_offset, size_of_text_section, images_per_page, page_no, unk2, next_page_data_offset] = struct.unpack("iihhii", caj.read(20))
            caj.seek(page_data_offset)
            text_header_read32 = caj.read(32)
            if (text_header_read32[8:20] == b'COMPRESSTEXT'):
                [expanded_text_size] = struct.unpack("i", text_header_read32[20:24])
                import zlib
                caj.seek(page_data_offset + 24)
                data = caj.read(size_of_text_section - 24)
                output = zlib.decompress(data, bufsize=expanded_text_size)
                if (len(output) != expanded_text_size):
                    raise SystemExit("Unexpected:", len(output), expanded_text_size)
            else:
                caj.seek(page_data_offset)
                output = caj.read(size_of_text_section)
            from HNParsePage import HNParsePage
            page_style = (next_page_data_offset > page_data_offset)
            page_data = HNParsePage(output, page_style)

            if (images_per_page > 1):
                if (len(page_data.figures) == images_per_page):
                    image_list.append(None)
                    image_list.append(page_data.figures)
                else:
                    raise SystemExit("Image Count %d != %d" % (len(page_data.figures), images_per_page))
            current_offset = page_data_offset + size_of_text_section
            for j in range(images_per_page):
                caj.seek(current_offset)
                read32 = caj.read(32)
                [image_type_enum, offset_to_image_data, size_of_image_data] = struct.unpack("iii", read32[0:12])
                if (offset_to_image_data != current_offset + 12):
                    raise SystemExit("unusual image offset")
                caj.seek(offset_to_image_data)
                image_data = caj.read(size_of_image_data)
                current_offset = offset_to_image_data + size_of_image_data
                if (image_type[image_type_enum] == "JBIG"):
                    from jbigdec import CImage
                    cimage = CImage(image_data)
                    out = cimage.DecodeJbig()
                    # PBM is only padded to 8 rather than 32.
                    # If the padding is larger, write padded file.
                    width = cimage.width
                    if (cimage.bytes_per_line > ((cimage.width +7) >> 3)):
                        width = cimage.bytes_per_line << 3
                    image_item = (
                        Colorspace.P,
                        (300, 300),
                        ImageFormat.PBM,
                        zlib.compress(out),
                        width,
                        cimage.height,
                        [0xffffff, 0],
                        False,
                        1,
                        0
                    )
                elif (image_type[image_type_enum] == "JBIG2"):
                    from jbig2dec import CImage
                    cimage = CImage(image_data)
                    out = cimage.DecodeJbig2()
                    # PBM is only padded to 8 rather than 32.
                    # If the padding is larger, write padded file.
                    width = cimage.width
                    if (cimage.bytes_per_line > ((cimage.width +7) >> 3)):
                        width = cimage.bytes_per_line << 3
                    image_item = (
                        Colorspace.P,
                        (300, 300),
                        ImageFormat.PBM,
                        zlib.compress(out),
                        width,
                        cimage.height,
                        [0xffffff, 0],
                        False,
                        1,
                        0
                    )
                elif (image_type[image_type_enum] == "JPEG"):
                    (height, width) = struct.unpack(">HH", image_data[163:167])
                    if (image_type_enum == 1):
                        # non-inverted JPEG Images
                        height = -height
                    image_item = (
                        Colorspace.RGB,
                        (300, 300),
                        ImageFormat.JPEG,
                        image_data,
                        width,
                        height,
                        [],
                        False,
                        8,
                        0
                    )
                else:
                    raise SystemExit("Unknown Image Type %d" % (image_type_enum))
                image_list.append(image_item)
        if (len(image_list) == 0):
            raise SystemExit("File is pure-text HN; cannot convert to pdf")
        pdf_data = convert_ImageList(image_list)
        with open('pdf_toc.pdf', 'wb') as f:
            f.write(pdf_data)
        # Add Outlines
        add_outlines(self.get_toc(), "pdf_toc.pdf", dest)
        os.remove("pdf_toc.pdf")

    def _text_extract_hn(self):
        if (self._TOC_NUMBER_OFFSET > 0):
            self.get_toc(verbose=True)
        caj = open(self.filename, "rb")

        for i in range(self.page_num):
            caj.seek(self._TOC_END_OFFSET + i * 20)
            [page_data_offset, size_of_text_section, images_per_page, page_no, unk2, next_page_data_offset] = struct.unpack("iihhii", caj.read(20))
            caj.seek(page_data_offset)
            text_header_read32 = caj.read(32)
            if (text_header_read32[8:20] == b'COMPRESSTEXT'):
                [expanded_text_size] = struct.unpack("i", text_header_read32[20:24])
                import zlib
                caj.seek(page_data_offset + 24)
                data = caj.read(size_of_text_section - 24)
                output = zlib.decompress(data, bufsize=expanded_text_size)
                if (len(output) != expanded_text_size):
                    raise SystemExit("Unexpected:", len(output), expanded_text_size)
            else:
                caj.seek(page_data_offset)
                output = caj.read(size_of_text_section)
            from HNParsePage import HNParsePage
            page_style = (next_page_data_offset > page_data_offset)
            page_data = HNParsePage(output, page_style)
            print("Text on Page %d:" % (i+1))
            print(page_data.texts)
            #print("Figures:\n", page_data.figures)

    def _parse_hn(self):
        if (self._TOC_NUMBER_OFFSET > 0):
            self.get_toc(verbose=True)
        caj = open(self.filename, "rb")

        for i in range(self.page_num):
            caj.seek(self._TOC_END_OFFSET + i * 20)
            print("Reading Page Info struct #%d at offset 0x%04X" % (i+1, self._TOC_END_OFFSET + i * 20))
            [page_data_offset, size_of_text_section, images_per_page, page_no, unk2, next_page_data_offset] = struct.unpack("iihhii", caj.read(20))
            print("unknown page struct members = (%d %d)" % (unk2, next_page_data_offset))
            # All 71: 1,0,0
            print("Page Number %d Data offset = 0x%04X" % (page_no, page_data_offset))
            caj.seek(page_data_offset)
            text_header_read32 = caj.read(32)
            print("Page Text Header dump:\n", self.dump(text_header_read32), sep="")
            # The first 8 bytes are always: 03 80 XX 16 03 80 XX XX,
            # the last one 20 or 21, but the first two can be any.
            # 48/71 has: 03 80 E0 16 03 80 F7 20, the rest uniq
            if (text_header_read32[8:20] == b'COMPRESSTEXT'):
                # expanded_text_size seems to be always about 2-3 times size_of_text_section, so this is a guess.
                [expanded_text_size] = struct.unpack("i", text_header_read32[20:24])
                import zlib
                caj.seek(page_data_offset + 24)
                data = caj.read(size_of_text_section - 24)
                output = zlib.decompress(data, bufsize=expanded_text_size)
                if (len(output) != expanded_text_size):
                    print("Unexpected:", len(output), expanded_text_size)
                print("Page Text Header COMPRESSTEXT:\n", self.dump(output, GB=True), sep="")
                for x in range(len(output) >> 4):
                    try:
                        print(bytes([output[(x << 4) + 7],output[(x << 4) + 6]]).decode("gbk"), end="")
                    except UnicodeDecodeError:
                        print(self.dump(output[x << 4:(x+1) << 4]))
                print()
            else:
                caj.seek(page_data_offset)
                output = caj.read(size_of_text_section)
                print("Page Text Header non-COMPRESSTEXT:\n", self.dump(output, GB=True), sep="")
            from HNParsePage import HNParsePage
            page_style = (next_page_data_offset > page_data_offset)
            page_data = HNParsePage(output, page_style)
            print("Text:\n", page_data.texts)
            print("Figures:\n", page_data.figures)
            current_offset = page_data_offset + size_of_text_section
            for j in range(images_per_page):
                caj.seek(current_offset)
                read32 = caj.read(32)
                [image_type_enum, offset_to_image_data, size_of_image_data] = struct.unpack("iii", read32[0:12])
                if (image_type[image_type_enum] != "JPEG"):
                    read32 += caj.read(64)
                print("size of image data = %d (%s)" % (size_of_image_data, image_type[image_type_enum]))
                if (offset_to_image_data != current_offset + 12):
                    raise SystemExit("unusual image offset")
                print("Page Image Header dump:\n", self.dump(read32), sep="")
                print("Expected End of Page #%d: 0x%08X" % (i+1, current_offset + size_of_image_data + 12))
                caj.seek(offset_to_image_data)
                image_data = caj.read(size_of_image_data)
                current_offset = offset_to_image_data + size_of_image_data
                image_name = "image_dump_%04d" % (i+1)
                if (j > 0):
                    image_name = "image_dump_%04d_%04d" % (i+1, j)
                with open(image_name + ".dat", "wb") as f:
                    f.write(image_data)
                if (image_type[image_type_enum] == "JBIG"):
                    try:
                        from jbigdec import SaveJbigAsBmp
                        SaveJbigAsBmp(image_data, size_of_image_data, (image_name + ".bmp").encode('ascii'))
                    except ImportError:
                        pass
                elif (image_type[image_type_enum] == "JBIG2"):
                    try:
                        from jbigdec import SaveJbig2AsBmp
                        SaveJbig2AsBmp(image_data, size_of_image_data, (image_name + ".bmp").encode('ascii'))
                    except ImportError:
                        pass
                elif (image_type[image_type_enum] == "JPEG"):
                    with open(image_name + ".jpg", "wb") as f:
                        f.write(image_data)
        print("end 0x%08x" % self._PAGEDATA_OFFSET)

    def dump(self, src, GB=False):
        N=0
        result=[]
        while src:
            s,src = src[:16],src[16:]
            hexa = ' '.join(["%02X"% x for x in s])
            gb = ""
            if (GB):
                gb += "    "
                for x in range(len(s) >> 1):
                    try:
                        if (s[(x << 1) +1] < 128 and s[(x << 1) + 0] < 128):
                            gb += ".."
                        else:
                            gb += bytes([s[(x << 1) + 1],s[(x << 1) + 0]]).decode("gbk")
                    except UnicodeDecodeError:
                        gb += ".."
            s = ''.join(printables[x] for x in s)
            result += "%04X   %-*s   %s%s\n" % (N, 16*3, hexa, s, gb)
            N+=16
        return ''.join(result)


    def _convert_pdf(self, dest):
        copy(self.filename, dest)

    def _convert_kdh(self, dest):
        #  Read KDH file.
        fp = open(self.filename, "rb")
        origin = fp.read()
        fp.close()

        #  Decrypt.
        origin = origin[254:]
        output = []
        keycursor = 0
        for origin_byte in origin:
            output.append(origin_byte ^ KDH_PASSPHRASE[keycursor])
            keycursor += 1
            if keycursor >= len(KDH_PASSPHRASE):
                keycursor = 0
        output = bytes(output)

        #  Remove useless tail data.
        eofpos = output.rfind(b"%%EOF")
        if eofpos < 0:
            raise Exception("%%EOF mark can't be found.")
        output = output[:eofpos + 5]

        #  Write output file.
        fp = open(dest + ".tmp", "wb")
        fp.write(output)
        fp.close()

        # Use mutool to repair xref
        try:
            check_output(["mutool", "clean", dest + ".tmp", dest], stderr=STDOUT)
        except CalledProcessError as e:
            print(e.output.decode("utf-8"))
            raise SystemExit("Command mutool returned non-zero exit status " + str(e.returncode))

        os.remove(dest + ".tmp")