diff --git a/docs/structure.md b/docs/structure.md new file mode 100644 index 0000000..9791ecc --- /dev/null +++ b/docs/structure.md @@ -0,0 +1,61 @@ +# Structure Tree + +Since PDF 1.3 it is possible for a PDF to contain logical structure, +contained in a *structure tree*. In conjunction with PDF 1.2 [marked +content sections](#marked-content-sections) this forms the basis of +Tagged PDF and other accessibility features. + +Unfortunately, since all of these standards are optional and variably +implemented in PDF authoring tools, and are frequently not enabled by +default, it is not possible to rely on them to extract the structure +of a PDF and associated content. Nonetheless they can be useful as +features for a heuristic or machine-learning based system, or for +extracting particular structures such as tables. + +Since `pdfplumber`'s API is page-based, the structure is available for +a particular page, using the `structure_tree` attribute: + + with pdfplumber.open(pdffile) as pdf: + for element in pdf.pages[0].structure_tree: + print(element["type"], element["mcids"]) + for child in element.children: + print(child["type"], child["mcids"]) + +The `type` field contains the type of the structure element - the +standard structure types can be seen in section 10.7.3 of [the PDF 1.7 +reference +document](https://ghostscript.com/~robin/pdf_reference17.pdf#page=898), +but usually they are rather HTML-like, if created by a recent PDF +authoring tool (notably, older tools may simply produce `P` for +everything). + +The `mcids` field contains the list of marked content section IDs +corresponding to this element. + +The `lang` field is often present as well, and contains a language +code for the text content, e.g. `"EN-US"` or `"FR-CA"`. + +The `alt_text` field will be present if the author has helpfully added +alternate text to an image. In some cases, `actual_text` may also be +present. + +There are also various attributes that may be in the `attributes` +field. Some of these are quite useful indeed, such as ``BBox` which +gives you the bounding box of a `Table`, `Figure`, or `Image`. You +can see a full list of these [in the PDF +spec](https://ghostscript.com/~robin/pdf_reference17.pdf#page=916). +Note that the `BBox` is in PDF coordinate space with the origin at the +bottom left of the page. To convert it to `pdfplumber`'s space you +can do, for example: + + x0, y0, x1, y1 = element['attributes']['BBox'] + top = page.height - y1 + bottom = page.height - y0 + doctop = page.initial_doctop + top + bbox = (x0, top, x1, bottom) + +It is also possible to get the structure tree for the entire document. +In this case, because marked content IDs are specific to a given page, +each element will also have a `page_number` attribute, which is the +number of the page containing (partially or completely) this element, +indexed from 1 (for consistency with `pdfplumber.Page`). diff --git a/pdfplumber/cli.py b/pdfplumber/cli.py index c3f8d23..2d17f41 100644 --- a/pdfplumber/cli.py +++ b/pdfplumber/cli.py @@ -23,6 +23,12 @@ def parse_args(args_raw: List[str]) -> argparse.Namespace: "infile", nargs="?", type=argparse.FileType("rb"), default=sys.stdin.buffer ) + parser.add_argument( + "--structure", + help="Write the structure tree as JSON. " + "All arguments except --pages, --laparams, and --indent will be ignored", + action="store_true", + ) parser.add_argument("--format", choices=["csv", "json"], default="csv") parser.add_argument("--types", nargs="+") @@ -59,7 +65,9 @@ def main(args_raw: List[str] = sys.argv[1:]) -> None: args = parse_args(args_raw) with PDF.open(args.infile, pages=args.pages, laparams=args.laparams) as pdf: - if args.format == "csv": + if args.structure: + json.dump(pdf.structure_tree, sys.stdout, indent=args.indent) + elif args.format == "csv": pdf.to_csv( sys.stdout, args.types, diff --git a/pdfplumber/page.py b/pdfplumber/page.py index c86a363..05b0a06 100644 --- a/pdfplumber/page.py +++ b/pdfplumber/page.py @@ -29,6 +29,7 @@ from pdfminer.psparser import PSLiteral from . import utils from ._typing import T_bbox, T_num, T_obj, T_obj_list from .container import Container +from .structure import PDFStructTree, StructTreeMissing from .table import T_table_settings, Table, TableFinder, TableSettings from .utils import decode_text, resolve_all, resolve_and_decode from .utils.text import TextMap @@ -222,6 +223,14 @@ class Page(Container): def height(self) -> T_num: return self.bbox[3] - self.bbox[1] + @property + def structure_tree(self) -> List[Dict[str, Any]]: + """Return the structure tree for a page, if any.""" + try: + return [elem.to_dict() for elem in PDFStructTree(self.pdf, self)] + except StructTreeMissing: + return [] + @property def layout(self) -> LTPage: if hasattr(self, "_layout"): diff --git a/pdfplumber/pdf.py b/pdfplumber/pdf.py index aaf50e3..977da59 100644 --- a/pdfplumber/pdf.py +++ b/pdfplumber/pdf.py @@ -16,6 +16,7 @@ from ._typing import T_num, T_obj_list from .container import Container from .page import Page from .repair import _repair +from .structure import PDFStructTree, StructTreeMissing from .utils import resolve_and_decode logger = logging.getLogger(__name__) @@ -160,6 +161,14 @@ class PDF(Container): gen = (p.hyperlinks for p in self.pages) return list(itertools.chain(*gen)) + @property + def structure_tree(self) -> List[Dict[str, Any]]: + """Return the structure tree for the document.""" + try: + return [elem.to_dict() for elem in PDFStructTree(self)] + except StructTreeMissing: + return [] + def to_dict(self, object_types: Optional[List[str]] = None) -> Dict[str, Any]: return { "metadata": self.metadata, diff --git a/pdfplumber/structure.py b/pdfplumber/structure.py new file mode 100644 index 0000000..145fda7 --- /dev/null +++ b/pdfplumber/structure.py @@ -0,0 +1,345 @@ +from collections import deque +from dataclasses import asdict, dataclass, field +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple + +from pdfminer.data_structures import NumberTree +from pdfminer.pdfpage import PDFPage +from pdfminer.pdfparser import PDFParser +from pdfminer.pdftypes import PDFObjRef, resolve1 +from pdfminer.psparser import PSLiteral + +from .utils import decode_text + +if TYPE_CHECKING: # pragma: nocover + from .page import Page + from .pdf import PDF + + +@dataclass +class PDFStructElement: + type: str + revision: Optional[int] + id: Optional[str] + lang: Optional[str] + alt_text: Optional[str] + actual_text: Optional[str] + title: Optional[str] + page_number: Optional[int] + attributes: Dict[str, Any] = field(default_factory=dict) + mcids: List[int] = field(default_factory=list) + children: List["PDFStructElement"] = field(default_factory=list) + + def __iter__(self) -> Iterator["PDFStructElement"]: + return iter(self.children) + + def to_dict(self) -> Dict[str, Any]: + """Return a compacted dict representation.""" + r = asdict(self) + d = deque([r]) + while d: + el = d.popleft() + for k in list(el.keys()): + if el[k] is None or el[k] == [] or el[k] == {}: + del el[k] + if "children" in el: + d.extend(el["children"]) + return r + + +class StructTreeMissing(ValueError): + pass + + +class PDFStructTree: + """Parse the structure tree of a PDF. + + The constructor takes a `pdfplumber.PDF` and optionally a + `pdfplumber.Page`. To avoid creating the entire tree for a large + document it is recommended to provide a page. + + This class creates a representation of the portion of the + structure tree that reaches marked content sections, either for a + single page, or for the whole document. Note that this is slightly + different from the behaviour of other PDF libraries which will + also include structure elements with no content. + + If the PDF has no structure, the constructor will raise + `StructTreeMissing`. + + """ + + page: Optional[PDFPage] + + def __init__(self, doc: "PDF", page: Optional["Page"] = None): + self.doc = doc.doc + if "StructTreeRoot" not in self.doc.catalog: + raise StructTreeMissing("PDF has no structure") + self.root = resolve1(self.doc.catalog["StructTreeRoot"]) + self.role_map = resolve1(self.root.get("RoleMap", {})) + self.class_map = resolve1(self.root.get("ClassMap", {})) + self.children: List[PDFStructElement] = [] + + # If we have a specific page then we will work backwards from + # its ParentTree - this is because structure elements could + # span multiple pages, and the "Pg" attribute is *optional*, + # so this is the approved way to get a page's structure + if page is not None: + self.page = page.page_obj + self.page_dict = None + parent_tree = NumberTree(self.root["ParentTree"]) + # If there is no marked content in the structure tree for + # this page (which can happen even when there is a + # structure tree) then there is no `StructParents`. + # Note however that if there are XObjects in a page, + # *they* may have `StructParent` (not `StructParents`) + if "StructParents" not in self.page.attrs: + return + parent_id = self.page.attrs["StructParents"] + # NumberTree should have a `get` method like it does in pdf.js... + parent_array = resolve1( + next(array for num, array in parent_tree.values if num == parent_id) + ) + self._parse_parent_tree(parent_array) + else: + self.page = None + # Overhead of creating pages shouldn't be too bad we hope! + self.page_dict = { + page.page_obj.pageid: page.page_number for page in doc.pages + } + self._parse_struct_tree() + + def _make_attributes( + self, obj: Dict[str, Any], revision: Optional[int] + ) -> Dict[str, Any]: + attr_obj_list = [] + for key in "C", "A": + if key not in obj: + continue + attr_obj = resolve1(obj[key]) + if isinstance(attr_obj, list): + attr_obj_list.extend(attr_obj) + else: + attr_obj_list.append(attr_obj) + attr_objs = [] + prev_obj = None + for aref in attr_obj_list: + # If we find a revision number, which might "follow + # the revision object" (the spec is incredibly unclear + # about how this actually works), then use it to + # decide whether to take the previous object... + if isinstance(aref, int): # pragma: nocover + if aref == revision and prev_obj is not None: + attr_objs.append(prev_obj) + prev_obj = None + else: + if prev_obj is not None: + attr_objs.append(prev_obj) + prev_obj = resolve1(aref) + if prev_obj is not None: + attr_objs.append(prev_obj) + # Now merge all the relevant ones to a single set (FIXME: Not + # *really* sure this is how this is supposed to work... OMG) + attr = {} + for obj in attr_objs: + if isinstance(obj, PSLiteral): # OMG + key = decode_text(obj.name) + # Should be a warning at least! + if key not in self.class_map: # pragma: nocover + continue + obj = self.class_map[key] + for k, v in obj.items(): + if isinstance(v, PSLiteral): + attr[k] = decode_text(v.name) + else: + attr[k] = obj[k] + return attr + + def _make_element(self, obj: Any) -> Tuple[Optional[PDFStructElement], List[Any]]: + # We hopefully caught these earlier + assert "MCID" not in obj, "Found MCR: %s" % obj + assert "Obj" not in obj, "Found OBJR: %s" % obj + # Get page number if necessary, also (mostly) exclude unparsed + # pages (FIXME: objects on unparsed pages with no explicit + # page number will remain) + page_number = None + if self.page_dict is not None and "Pg" in obj: + page_objid = obj["Pg"].objid + if page_objid not in self.page_dict: + # Stop here, page was not parsed + return (None, []) + else: + page_number = self.page_dict[page_objid] + obj_tag = "" + if "S" in obj: + obj_tag = decode_text(obj["S"].name) + if obj_tag in self.role_map: + obj_tag = decode_text(self.role_map[obj_tag].name) + children = resolve1(obj["K"]) if "K" in obj else [] + if isinstance(children, int): # ugh... isinstance... + children = [children] + elif isinstance(children, dict): # a single object.. ugh... + children = [obj["K"]] + revision = obj.get("R") + attributes = self._make_attributes(obj, revision) + element_id = decode_text(obj["ID"]) if "ID" in obj else None + title = decode_text(obj["T"]) if "T" in obj else None + lang = decode_text(obj["Lang"]) if "Lang" in obj else None + alt_text = decode_text(obj["Alt"]) if "Alt" in obj else None + actual_text = decode_text(obj["ActualText"]) if "ActualText" in obj else None + element = PDFStructElement( + type=obj_tag, + id=element_id, + page_number=page_number, + revision=revision, + lang=lang, + title=title, + alt_text=alt_text, + actual_text=actual_text, + attributes=attributes, + ) + return element, children + + def _parse_parent_tree(self, parent_array: List[Any]) -> None: + """Populate the structure tree using the leaves of the parent tree for + a given page.""" + # First walk backwards from the leaves to the root, tracking references + d = deque(parent_array) + s = {} + found_root = False + while d: + ref = d.popleft() + # In the case where an MCID is not associated with any + # structure, there will be a "null" in the parent tree. + if ref == PDFParser.KEYWORD_NULL: + continue + if repr(ref) in s: + continue + obj = resolve1(ref) + # This is required! It's in the spec! + if "Type" in obj and decode_text(obj["Type"].name) == "StructTreeRoot": + found_root = True + else: + # We hope that these are actual elements and not + # references or marked-content sections... + element, children = self._make_element(obj) + # We have no page tree so we assume this page was parsed + assert element is not None + s[repr(ref)] = element, children + d.append(obj["P"]) + # If we didn't reach the root something is quite wrong! + assert found_root + self._resolve_children(s) + + def _parse_struct_tree(self) -> None: + """Populate the structure tree starting from the root, skipping + unparsed pages and empty elements.""" + root = resolve1(self.root["K"]) + + def on_parsed_page(obj: Dict[str, Any]) -> bool: + if self.page_dict is not None and "Pg" in obj: # pragma: nocover + page_objid = obj["Pg"].objid + return page_objid in self.page_dict + return True + + # It could just be a single object ... it's in the spec (argh) + if isinstance(root, dict): + root = [self.root["K"]] + d = deque(root) + s = {} + while d: + ref = d.popleft() + if repr(ref) in s: + continue # pragma: nocover + obj = resolve1(ref) + # Deref top-level OBJR skipping refs to unparsed pages + if isinstance(obj, dict) and "Obj" in obj: # pragma: nocover + if not on_parsed_page(obj): + continue + ref = obj["Obj"] + obj = resolve1(ref) + element, children = self._make_element(obj) + # Similar to above, delay resolving the children to avoid + # tree-recursion. + s[repr(ref)] = element, children + for child in children: + if isinstance(child, PDFObjRef): + d.append(child) + elif isinstance(child, dict) and "Obj" in child: + if on_parsed_page(child): + d.append(child["Obj"]) + + # Traverse depth-first, removing empty elements (unsure how to + # do this non-recursively) + def prune(elements: List[Any]) -> List[Any]: + next_elements = [] + for ref in elements: + if isinstance(ref, int): + next_elements.append(ref) + continue + elif isinstance(ref, dict): + if not on_parsed_page(ref): # pragma: nocover + continue + if "MCID" in ref: # pragma: nocover + next_elements.append(ref["MCID"]) + continue + elif "Obj" in ref: + ref = ref["Obj"] + elif isinstance(ref, PDFObjRef): + obj = resolve1(ref) + if isinstance(obj, dict) and "Obj" in obj: # pragma: nocover + if not on_parsed_page(obj): + continue + ref = obj["Obj"] + element, children = s[repr(ref)] + children = prune(children) + # See assertions below + if element is None or not children: + del s[repr(ref)] + else: + s[repr(ref)] = element, children + next_elements.append(ref) + return next_elements + + prune(root) + self._resolve_children(s) + + def _resolve_children(self, seen: Dict[str, Any]) -> None: + """Resolve children starting from the tree root based on references we + saw when traversing the structure tree. + """ + root = resolve1(self.root["K"]) + # It could just be a single object ... it's in the spec (argh) + if isinstance(root, dict): + root = [self.root["K"]] + d = deque(root) + while d: + ref = d.popleft() + # The pruning (or parent tree construction) done above + # should ensure we never encounter dangling references, + # *but* you never know (should emit warnings...) + if repr(ref) not in seen: # pragma: nocover + continue + element, children = seen[repr(ref)] + assert element is not None, "Unparsed element" + for child in children: + if isinstance(child, int): + element.mcids.append(child) + elif isinstance(child, dict): + # Skip out-of-page MCIDS (which are obviously wrong!) and OBJRs + if "Pg" in child and self.page is not None: # pragma: nocover + if child["Pg"].objid != self.page.pageid: + continue + if "MCID" in child: # pragma: nocover + element.mcids.append(child["MCID"]) + elif "Obj" in child: + child = child["Obj"] + # NOTE: if, not elif, in case of OBJR above + if isinstance(child, PDFObjRef): + child_element, _ = seen.get(repr(child), (None, None)) + if child_element is not None: + element.children.append(child_element) + d.append(child) + self.children = [seen[repr(ref)][0] for ref in root if repr(ref) in seen] + + def __iter__(self) -> Iterator[PDFStructElement]: + return iter(self.children) diff --git a/tests/pdfs/2023-06-20-PV.pdf b/tests/pdfs/2023-06-20-PV.pdf new file mode 100644 index 0000000..1bc3371 Binary files /dev/null and b/tests/pdfs/2023-06-20-PV.pdf differ diff --git a/tests/pdfs/chelsea_pdta.pdf b/tests/pdfs/chelsea_pdta.pdf new file mode 100644 index 0000000..e1ff984 Binary files /dev/null and b/tests/pdfs/chelsea_pdta.pdf differ diff --git a/tests/pdfs/figure_structure.pdf b/tests/pdfs/figure_structure.pdf new file mode 100644 index 0000000..2a90e12 Binary files /dev/null and b/tests/pdfs/figure_structure.pdf differ diff --git a/tests/pdfs/image_structure.pdf b/tests/pdfs/image_structure.pdf new file mode 100644 index 0000000..563b7a6 Binary files /dev/null and b/tests/pdfs/image_structure.pdf differ diff --git a/tests/pdfs/pdf_structure.pdf b/tests/pdfs/pdf_structure.pdf new file mode 100644 index 0000000..a93b028 Binary files /dev/null and b/tests/pdfs/pdf_structure.pdf differ diff --git a/tests/pdfs/word365_structure.pdf b/tests/pdfs/word365_structure.pdf new file mode 100644 index 0000000..28cea23 Binary files /dev/null and b/tests/pdfs/word365_structure.pdf differ diff --git a/tests/test_convert.py b/tests/test_convert.py index 400bdaa..4b7dafb 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -83,6 +83,12 @@ class Test(unittest.TestCase): c = self.pdf.to_csv(object_types=None) assert c.split("\r\n")[1].split(",")[0] == "line" + def test_cli_structure(self): + res = run([sys.executable, "-m", "pdfplumber.cli", self.path, "--structure"]) + c = json.loads(res) + # lol no structure + assert c == [] + def test_cli_json(self): res = run( [ diff --git a/tests/test_structure.py b/tests/test_structure.py new file mode 100644 index 0000000..7f4297a --- /dev/null +++ b/tests/test_structure.py @@ -0,0 +1,906 @@ +#!/usr/bin/env python3 + +import os +import unittest +from collections import deque + +import pdfplumber +from pdfplumber.structure import PDFStructTree + +HERE = os.path.abspath(os.path.dirname(__file__)) +TREE = [ + { + "type": "Document", + "children": [ + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + "SpaceBefore": 0.24, + "TextAlign": "Center", + }, + "mcids": [0], + }, + { + "type": "H1", + "attributes": { + "O": "Layout", + "Placement": "Block", + "SpaceBefore": 0.36, + }, + "mcids": [1], + }, + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + "SpaceBefore": 0.12, + }, + "mcids": [2], + }, + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + "SpaceBefore": 0.181, + }, + "mcids": [3, 4, 5, 6, 7], + }, + { + "type": "H2", + "attributes": { + "O": "Layout", + "Placement": "Block", + "SpaceBefore": 0.381, + }, + "mcids": [8], + }, + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + "SpaceBefore": 0.12, + }, + "mcids": [9], + }, + { + "type": "L", + "children": [ + { + "type": "LI", + "children": [ + { + "type": "LBody", + "children": [ + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + "SpaceBefore": 0.181, + "StartIndent": 0.36, + }, + "mcids": [10, 11], + } + ], + } + ], + }, + { + "type": "LI", + "children": [ + { + "type": "LBody", + "children": [ + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + "SpaceBefore": 0.181, + "StartIndent": 0.36, + }, + "mcids": [12, 13], + }, + { + "type": "L", + "children": [ + { + "type": "LI", + "children": [ + { + "type": "LBody", + "children": [ + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", # noqa: E501 + "SpaceBefore": 0.181, # noqa: E501 + "StartIndent": 0.72, # noqa: E501 + }, + "mcids": [14, 15], + } + ], + } + ], + } + ], + }, + ], + } + ], + }, + { + "type": "LI", + "children": [ + { + "type": "LBody", + "children": [ + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + "SpaceBefore": 0.181, + "StartIndent": 0.36, + }, + "mcids": [16, 17, 18, 19, 20, 21, 22, 23], + } + ], + } + ], + }, + ], + }, + { + "type": "H3", + "attributes": { + "O": "Layout", + "Placement": "Block", + "SpaceBefore": 0.321, + }, + "mcids": [24], + }, + { + "type": "Table", + "attributes": { + "O": "Layout", + "Placement": "Block", + "SpaceBefore": 0.12, + "SpaceAfter": 0.015, + "Width": 9.972, + "Height": 1.047, + "BBox": [56.7, 249.75, 555.3, 302.1], + }, + "children": [ + { + "type": "TR", + "attributes": {"O": "Layout", "Placement": "Block"}, + "children": [ + { + "type": "TH", + "attributes": { + "O": "Layout", + "Placement": "Inline", + "Width": 4.985, + "Height": 0.291, + }, + "children": [ + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + }, + "mcids": [25], + } + ], + }, + { + "type": "TH", + "attributes": { + "O": "Layout", + "Placement": "Inline", + "Width": 4.987, + "Height": 0.291, + }, + "children": [ + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + }, + "mcids": [26], + } + ], + }, + ], + }, + { + "type": "TR", + "attributes": {"O": "Layout", "Placement": "Block"}, + "children": [ + { + "type": "TD", + "attributes": { + "O": "Layout", + "Placement": "Inline", + "Width": 4.985, + "Height": 0.291, + }, + "children": [ + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + }, + "mcids": [27], + } + ], + }, + { + "type": "TD", + "attributes": { + "O": "Layout", + "Placement": "Inline", + "Width": 4.987, + "Height": 0.291, + }, + "children": [ + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + }, + "mcids": [28], + } + ], + }, + ], + }, + { + "type": "TR", + "attributes": {"O": "Layout", "Placement": "Block"}, + "children": [ + { + "type": "TD", + "attributes": { + "O": "Layout", + "Placement": "Inline", + "Width": 4.985, + "Height": 0.33, + }, + "children": [ + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + }, + "mcids": [29], + } + ], + }, + { + "type": "TD", + "attributes": { + "O": "Layout", + "Placement": "Inline", + "Width": 4.987, + "Height": 0.33, + }, + "children": [ + { + "type": "P", + "attributes": { + "O": "Layout", + "Placement": "Block", + }, + "mcids": [30], + } + ], + }, + ], + }, + ], + }, + ], + } +] + + +class Test(unittest.TestCase): + """Test a PDF specifically created to show structure.""" + + @classmethod + def setup_class(self): + path = os.path.join(HERE, "pdfs/pdf_structure.pdf") + self.pdf = pdfplumber.open(path) + + @classmethod + def teardown_class(self): + self.pdf.close() + + def test_structure_tree(self): + assert self.pdf.pages[0].structure_tree == TREE + # Add page numbers + d = deque(TREE) + while d: + el = d.popleft() + el["page_number"] = 1 + if "children" in el: + d.extend(el["children"]) + assert self.pdf.structure_tree == TREE + + +PVSTRUCT = [ + { + "type": "Sect", + "children": [ + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [0]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [1]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [2]}, + {"type": "P", "lang": "FR-FR", "page_number": 1, "mcids": [3]}, + {"type": "P", "lang": "FR-FR", "page_number": 1, "mcids": [4]}, + {"type": "P", "lang": "FR-FR", "page_number": 1, "mcids": [5]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [6]}, + {"type": "P", "lang": "FR-FR", "page_number": 1, "mcids": [7]}, + { + "type": "P", + "lang": "FR-FR", + "page_number": 1, + "mcids": [8], + "children": [ + {"type": "Span", "lang": "FR-CA", "page_number": 1, "mcids": [9]} + ], + }, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [11]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [12]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [13]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [14]}, + {"type": "P", "lang": "FR-FR", "page_number": 1, "mcids": [15]}, + {"type": "P", "lang": "FR-FR", "page_number": 1, "mcids": [16]}, + { + "type": "L", + "children": [ + { + "type": "LI", + "children": [ + { + "type": "LBody", + "lang": "FR-CA", + "page_number": 1, + "mcids": [19], + } + ], + } + ], + }, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [22]}, + {"type": "P", "lang": "FR-FR", "page_number": 1, "mcids": [23]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [24]}, + { + "type": "L", + "children": [ + { + "type": "LI", + "children": [ + { + "type": "LBody", + "lang": "FR-CA", + "page_number": 1, + "mcids": [27], + } + ], + } + ], + }, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [30]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [31]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [32]}, + { + "type": "L", + "children": [ + { + "type": "LI", + "children": [ + { + "type": "LBody", + "lang": "FR-CA", + "page_number": 1, + "mcids": [35], + } + ], + } + ], + }, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [38]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [39]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [40]}, + { + "type": "L", + "children": [ + { + "type": "LI", + "children": [ + { + "type": "LBody", + "lang": "FR-CA", + "page_number": 1, + "mcids": [43, 45], + "children": [ + { + "type": "Span", + "lang": "FR-FR", + "page_number": 1, + "mcids": [44], + } + ], + } + ], + } + ], + }, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [48]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [49]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [50]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [51]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [52]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [53]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [54]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [55]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [56]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [57]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [58]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [59]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [60]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [61]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [62]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [63]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [64]}, + {"type": "P", "lang": "FR-CA", "page_number": 1, "mcids": [65]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [0]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [1]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [2]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [3]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [4]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [5]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [6]}, + { + "type": "L", + "children": [ + { + "type": "LI", + "children": [ + { + "type": "LBody", + "lang": "FR-CA", + "page_number": 2, + "mcids": [9, 11], + "children": [ + { + "type": "Span", + "lang": "FR-FR", + "page_number": 2, + "mcids": [10], + } + ], + } + ], + } + ], + }, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [14]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [15]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [16]}, + {"type": "P", "lang": "FR-FR", "page_number": 2, "mcids": [17]}, + {"type": "P", "lang": "FR-FR", "page_number": 2, "mcids": [18]}, + {"type": "P", "lang": "FR-FR", "page_number": 2, "mcids": [19]}, + ], + } +] + + +PVSTRUCT1 = [ + { + "type": "Sect", + "children": [ + {"lang": "FR-CA", "type": "P", "mcids": [0]}, + {"lang": "FR-CA", "type": "P", "mcids": [1]}, + {"lang": "FR-CA", "type": "P", "mcids": [2]}, + {"lang": "FR-CA", "type": "P", "mcids": [3]}, + {"lang": "FR-CA", "type": "P", "mcids": [4]}, + {"lang": "FR-CA", "type": "P", "mcids": [5]}, + {"lang": "FR-CA", "type": "P", "mcids": [6]}, + { + "type": "L", + "children": [ + { + "type": "LI", + "children": [ + { + "lang": "FR-CA", + "type": "LBody", + "mcids": [9, 11], + "children": [ + {"lang": "FR-FR", "type": "Span", "mcids": [10]} + ], + } + ], + } + ], + }, + {"lang": "FR-CA", "type": "P", "mcids": [14]}, + {"lang": "FR-CA", "type": "P", "mcids": [15]}, + {"lang": "FR-CA", "type": "P", "mcids": [16]}, + {"lang": "FR-FR", "type": "P", "mcids": [17]}, + {"lang": "FR-FR", "type": "P", "mcids": [18]}, + {"lang": "FR-FR", "type": "P", "mcids": [19]}, + ], + } +] + +PVSTRUCT2 = [ + { + "type": "Sect", + "children": [ + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [0]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [1]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [2]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [3]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [4]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [5]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [6]}, + { + "type": "L", + "children": [ + { + "type": "LI", + "children": [ + { + "type": "LBody", + "lang": "FR-CA", + "page_number": 2, + "mcids": [9, 11], + "children": [ + { + "type": "Span", + "lang": "FR-FR", + "page_number": 2, + "mcids": [10], + } + ], + } + ], + } + ], + }, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [14]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [15]}, + {"type": "P", "lang": "FR-CA", "page_number": 2, "mcids": [16]}, + {"type": "P", "lang": "FR-FR", "page_number": 2, "mcids": [17]}, + {"type": "P", "lang": "FR-FR", "page_number": 2, "mcids": [18]}, + {"type": "P", "lang": "FR-FR", "page_number": 2, "mcids": [19]}, + ], + } +] + +IMAGESTRUCT = [ + { + "type": "Document", + "children": [ + {"type": "P", "mcids": [0]}, + {"type": "P", "mcids": [1]}, + { + "type": "Figure", + "alt_text": "pdfplumber on github\n\n" + "a screen capture of the github page for pdfplumber", + "mcids": [2], + }, + ], + } +] + + +WORD365 = [ + { + "type": "Document", + "children": [ + { + "type": "H1", + "children": [ + {"type": "Span", "mcids": [0]}, + {"type": "Span", "actual_text": " ", "mcids": [1]}, + ], + }, + {"type": "P", "mcids": [2]}, + { + "type": "L", + "attributes": {"O": "List", "ListNumbering": "Disc"}, + "children": [ + {"type": "LI", "children": [{"type": "LBody", "mcids": [3]}]}, + {"type": "LI", "children": [{"type": "LBody", "mcids": [4]}]}, + {"type": "LI", "children": [{"type": "LBody", "mcids": [5]}]}, + ], + }, + {"type": "P", "mcids": [6]}, + { + "type": "L", + "attributes": {"O": "List", "ListNumbering": "Decimal"}, + "children": [ + {"type": "LI", "children": [{"type": "LBody", "mcids": [7]}]}, + {"type": "LI", "children": [{"type": "LBody", "mcids": [8]}]}, + ], + }, + { + "type": "Table", + "children": [ + { + "type": "THead", + "children": [ + { + "type": "TR", + "children": [ + { + "type": "TH", + "children": [{"type": "P", "mcids": [9, 10]}], + }, + { + "type": "TH", + "children": [{"type": "P", "mcids": [11, 12]}], + }, + { + "type": "TH", + "children": [{"type": "P", "mcids": [13, 14]}], + }, + ], + } + ], + }, + { + "type": "TBody", + "children": [ + { + "type": "TR", + "children": [ + { + "type": "TD", + "children": [{"type": "P", "mcids": [15, 16]}], + }, + { + "type": "TD", + "children": [{"type": "P", "mcids": [17, 18]}], + }, + { + "type": "TD", + "children": [{"type": "P", "mcids": [19, 20]}], + }, + ], + }, + { + "type": "TR", + "children": [ + { + "type": "TD", + "children": [{"type": "P", "mcids": [21, 22]}], + }, + { + "type": "TD", + "children": [{"type": "P", "mcids": [23, 24]}], + }, + { + "type": "TD", + "children": [{"type": "P", "mcids": [25, 26]}], + }, + ], + }, + ], + }, + ], + }, + {"type": "P", "mcids": [27]}, + ], + } +] + + +SCOTUS = [ + { + "type": "Div", + "children": [ + { + "type": "P", + "page_number": 1, + "attributes": { + "LineHeight": 25.75, + "TextIndent": 21.625, + "O": "Layout", + }, + "mcids": [1], + }, + { + "type": "P", + "page_number": 1, + "attributes": { + "LineHeight": 25.75, + "StartIndent": 86.375, + "O": "Layout", + }, + "mcids": [2], + }, + { + "type": "P", + "page_number": 1, + "attributes": { + "LineHeight": 25.75, + "TextIndent": 50.375, + "O": "Layout", + }, + "mcids": [3, 4], + }, + { + "type": "P", + "page_number": 1, + # This is important, it has attributes and a class + "attributes": { + "LineHeight": 25.75, + "StartIndent": 165.625, + "EndIndent": 57.625, + "SpaceAfter": 24.5, + "O": "Layout", + }, + "mcids": [5], + }, + { + "type": "P", + "page_number": 1, + "attributes": { + "LineHeight": 25.75, + "TextIndent": 100.75, + "O": "Layout", + }, + "mcids": [6], + }, + { + "type": "P", + "page_number": 1, + # This is important, it has attributes and a class + "attributes": { + "LineHeight": 25.75, + "TextIndent": 21.625, + "EndIndent": 50.375, + "O": "Layout", + "TextAlign": "None", + "SpaceAfter": 179.125, + }, + "mcids": [7], + }, + { + "type": "P", + "page_number": 1, + # This is important, it has two attribute classes + "attributes": {"O": "Layout", "TextAlign": "Center", "SpaceAfter": 8.5}, + "mcids": [8], + }, + { + "type": "P", + "page_number": 1, + "attributes": {"O": "Layout", "TextAlign": "Center"}, + "mcids": [9], + }, + ], + } +] + + +class TestClass(unittest.TestCase): + """Test the underlying Structure tree class""" + + def test_structure_tree_class(self): + path = os.path.join(HERE, "pdfs/image_structure.pdf") + pdf = pdfplumber.open(path) + stree = PDFStructTree(pdf, pdf.pages[0]) + doc_elem = next(iter(stree)) + assert [k.type for k in doc_elem] == ["P", "P", "Figure"] + + +class TestUnparsed(unittest.TestCase): + """Test handling of PDFs with unparsed pages.""" + + def test_unparsed_pages(self): + path = os.path.join(HERE, "pdfs/2023-06-20-PV.pdf") + + pdf = pdfplumber.open(path, pages=[2]) + assert pdf.structure_tree == PVSTRUCT2 + + +class TestMany(unittest.TestCase): + """Test various PDFs.""" + + def test_no_stucture(self): + path = os.path.join(HERE, "pdfs/pdffill-demo.pdf") + pdf = pdfplumber.open(path) + assert pdf.structure_tree == [] + assert pdf.pages[0].structure_tree == [] + + def test_word365(self): + path = os.path.join(HERE, "pdfs/word365_structure.pdf") + pdf = pdfplumber.open(path) + page = pdf.pages[0] + assert page.structure_tree == WORD365 + + def test_proces_verbal(self): + path = os.path.join(HERE, "pdfs/2023-06-20-PV.pdf") + + pdf = pdfplumber.open(path) + assert pdf.structure_tree == PVSTRUCT + page = pdf.pages[1] + assert page.structure_tree == PVSTRUCT1 + + def test_image_structure(self): + path = os.path.join(HERE, "pdfs/image_structure.pdf") + + pdf = pdfplumber.open(path) + page = pdf.pages[0] + assert page.structure_tree == IMAGESTRUCT + + def test_figure_mcids(self): + path = os.path.join(HERE, "pdfs/figure_structure.pdf") + + pdf = pdfplumber.open(path) + page = pdf.pages[0] + d = deque(page.structure_tree) + while d: + el = d.popleft() + if el["type"] == "Figure": + break + if "children" in el: + d.extend(el["children"]) + # We found a Figure + assert el["type"] == "Figure" + # It has these MCIDS + assert el["mcids"] == [1, 14] + + def test_scotus(self): + # This one actually has attribute classes! + path = os.path.join(HERE, "pdfs/scotus-transcript-p1.pdf") + pdf = pdfplumber.open(path) + assert pdf.structure_tree == SCOTUS + + def test_chelsea_pdta(self): + # This one has structure elements for marked content sections + path = os.path.join(HERE, "pdfs/chelsea_pdta.pdf") + pdf = pdfplumber.open(path) + # This page has no structure tree (really!) + tree8 = pdf.pages[7].structure_tree + assert tree8 == [] + # We should also have no structure tree here + with pdfplumber.open(path, pages=[8]) as pdf8: + assert pdf8.structure_tree == [] + # This page is empty + tree3 = pdf.pages[3].structure_tree + assert tree3 == [] + # This page in particular has OBJR and MCR elements + tree1 = pdf.pages[2].structure_tree + assert tree1 # Should contain a tree! + pdf = pdfplumber.open(path, pages=[3]) + tree2 = pdf.structure_tree + assert tree2 + # Compare modulo page_number + d = deque(zip(tree1, tree2)) + while d: + el1, el2 = d.popleft() + if "page_number" in el1: + assert el1["page_number"] == 3 + assert el1 == el2 + if "children" in el1: + assert len(el1["children"]) == len(el2["children"]) + d.extend(zip(el1["children"], el2["children"]))