From 8a9191bcaee788643227bb7de680a68b9fb61e49 Mon Sep 17 00:00:00 2001 From: Jeremy Singer-Vine Date: Sat, 20 Feb 2016 19:42:37 -0500 Subject: [PATCH] Add `utils` section to README and example notebook --- README.md | 16 +- examples/notebooks/utils-nics.ipynb | 396 ++++++++++++++++++++++ examples/{ => pdfs}/background-checks.pdf | Bin 3 files changed, 411 insertions(+), 1 deletion(-) create mode 100644 examples/notebooks/utils-nics.ipynb rename examples/{ => pdfs}/background-checks.pdf (100%) diff --git a/README.md b/README.md index 24ee247..201dd00 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ pip install pdfplumber ### Basic Example ```sh -curl "https://cdn.rawgit.com/jsvine/pdfplumber/master/examples/background-checks.pdf" > background-checks.pdf +curl "https://cdn.rawgit.com/jsvine/pdfplumber/master/examples/pdfs/background-checks.pdf" > background-checks.pdf pdfplumber < background-checks.pdf > background-checks.csv ``` @@ -115,6 +115,20 @@ Each object is represented as a simple Python `dict`, with the following propert - `linewidth`: Thickness of line. - `object_type`: "rect" +### Utils / Helpers + +The `pdfplumber` Python library comes with a set of useful helper methods, accessible via `pdfplumber.utils`. They are: + +- `collate_chars(chars, x_tolerance=0, y_tolerance=0)`: Takes a list or dataframe of character objects and condenses them into a single string. Adds spaces where the difference between the `x1` of one character and the `x0` of the next is greater than `x_tolerance`. Adds newline characters where the difference between the `doctop` of one character and the `doctop` of the next is greater than `y_tolerance`. + +- `extract_columns(chars, x_tolerance=0, y_tolerance=0, gutter_min_width=5)`: Takes a list or dataframe of chars, looks for columns — vertical clumps of text separated by vertical "gutters" of non-text — and returns a representation of those columns. Passes `x_tolerance` and `y_tolerance` to `collate_chars(...)` (see above). Considers characters whose `doctop`s are within `y_tolerance` of one another to be on the same "line". For a gutter to be detected, it must be at least `gutter_min_width` pixels wide and have no character begin or end within it. + +- `within_bbox(objs, bbox)`: Takes a list or dataframe of objects (`chars`, `rects`, etc.) and returns those that are fully contained within a `bbox` of `(x0, top0, x1, top1)`. + +### Demonstrations + +- [Use `pdfplumber.utils` to extract data from the FBI's National Instant Criminal Background Check System PDFs.](examples/notebooks/utils-nics.ipynb) + ## Python Support Support for Python 3 is rough around the edges and largely dependent on the progress of [`pdfminer.six`](https://github.com/goulu/pdfminer). diff --git a/examples/notebooks/utils-nics.ipynb b/examples/notebooks/utils-nics.ipynb new file mode 100644 index 0000000..0875e31 --- /dev/null +++ b/examples/notebooks/utils-nics.ipynb @@ -0,0 +1,396 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Demonstration of `pdfplumber.utils`\n", + "\n", + "This notebook uses our [example PDF](../pdfs/background-checks.pdf) from the FBI's National Instant Criminal Background Check System to demonstrate `pdfplumber.utils`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Import `pdfplumber` and the relevant `utils`" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import pdfplumber\n", + "from pdfplumber.utils import within_bbox, extract_columns, collate_chars" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load the PDF" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "pdf = pdfplumber.from_path(\"../pdfs/background-checks.pdf\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### For use in constructing the bounding boxes later, store the the page width" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "PDF_WIDTH = pdf.pages[0].width" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Here are the first three characters stored in the PDF:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'size': 11.414, 'width': 4.642, 'pageid': 1, 'x1': 51.682, 'object_type': 'char', 'top': 67.486, 'text': 'S', 'x0': 47.04, 'upright': True, 'y1': 544.514, 'y0': 533.099, 'height': 11.414, 'doctop': 67.486, 'fontname': 'DCLTEC+Helvetica-Bold', 'adv': 0.667}, {'size': 11.414, 'width': 2.318, 'pageid': 1, 'x1': 53.916, 'object_type': 'char', 'top': 67.486, 'text': 't', 'x0': 51.599, 'upright': True, 'y1': 544.514, 'y0': 533.099, 'height': 11.414, 'doctop': 67.486, 'fontname': 'DCLTEC+Helvetica-Bold', 'adv': 0.333}, {'size': 11.414, 'width': 3.87, 'pageid': 1, 'x1': 57.863, 'object_type': 'char', 'top': 67.486, 'text': 'a', 'x0': 53.993, 'upright': True, 'y1': 544.514, 'y0': 533.099, 'height': 11.414, 'doctop': 67.486, 'fontname': 'DCLTEC+Helvetica-Bold', 'adv': 0.556}]\n" + ] + } + ], + "source": [ + "print(pdf.chars[:3])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Use `within_bbox` to focus on the main data table\n", + "\n", + "It starts around 77px from the top, and is about 410px tall. To select only these characters, we use `within_bbox`, and pass a bounding box of `(0, 77, PDF_WIDTH, 485)` as the `(x0, top0, x1, top1)` values." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'size': 7.667, 'width': 3.842, 'pageid': 1, 'x1': 47.042, 'object_type': 'char', 'y0': 525.799, 'text': 'A', 'x0': 43.2, 'upright': True, 'y1': 533.465, 'top': 78.535, 'height': 7.667, 'doctop': 78.535, 'fontname': 'WEVZII+ArialMT', 'adv': 0.667}, {'size': 7.667, 'width': 1.279, 'pageid': 1, 'x1': 48.079, 'object_type': 'char', 'y0': 525.799, 'text': 'l', 'x0': 46.8, 'upright': True, 'y1': 533.465, 'top': 78.535, 'height': 7.667, 'doctop': 78.535, 'fontname': 'WEVZII+ArialMT', 'adv': 0.222}, {'size': 7.667, 'width': 3.203, 'pageid': 1, 'x1': 51.437, 'object_type': 'char', 'y0': 525.799, 'text': 'a', 'x0': 48.234, 'upright': True, 'y1': 533.465, 'top': 78.535, 'height': 7.667, 'doctop': 78.535, 'fontname': 'WEVZII+ArialMT', 'adv': 0.556}]\n" + ] + } + ], + "source": [ + "table_chars = within_bbox(pdf.chars, (0, 77, PDF_WIDTH, 485))\n", + "print(table_chars[:3])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Use `extract_columns` to divide the characters into rows and columns\n", + "\n", + "Because side-by-side characters don't abut one another exactly, we pass `x_tolerance=2`." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{0: 'Alabama', 1: '18,870', 2: '23,022', 3: '22,650', 4: '859', 5: '1,178', 6: '0', 7: '14', 8: '15', 9: '0', 10: '2,179', 11: '2,307', 12: '11', 13: '0', 14: '0', 15: '0', 16: '', 17: '', 18: '13', 19: '14', 20: '0', 21: '3', 22: '2', 23: '0', 24: '71,137'}, {0: 'Alaska', 1: '209', 2: '3,062', 3: '3,209', 4: '191', 5: '184', 6: '0', 7: '9', 8: '3', 9: '0', 10: '100', 11: '100', 12: '0', 13: '18', 14: '9', 15: '1', 16: '', 17: '', 18: '0', 19: '0', 20: '0', 21: '0', 22: '0', 23: '0', 24: '7,095'}]\n" + ] + } + ], + "source": [ + "table = extract_columns(table_chars, x_tolerance=2)\n", + "print(table[:2])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Convert keys and values to something more useful\n", + "\n", + "The dictionary keys returned by `extract_columns` are simply numbered in order, `0, 1, 2, ...`. Let's add the actual column names, and also convert strings-representing-numbers to the numbers themselves, e.g., `\"18,870\" -> 18870`:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "COLUMNS = [\n", + " \"state\",\n", + " \"permit\",\n", + " \"handgun\",\n", + " \"long_gun\",\n", + " \"other\",\n", + " \"multiple\",\n", + " \"admin\",\n", + " \"prepawn_handgun\",\n", + " \"prepawn_long_gun\",\n", + " \"prepawn_other\",\n", + " \"redemption_handgun\",\n", + " \"redemption_long_gun\",\n", + " \"redemption_other\",\n", + " \"returned_handgun\",\n", + " \"returned_long_gun\",\n", + " \"returned_other\",\n", + " \"rentals_handgun\",\n", + " \"rentals_long_gun\",\n", + " \"private_sale_handgun\",\n", + " \"private_sale_long_gun\",\n", + " \"private_sale_other\",\n", + " \"return_to_seller_handgun\",\n", + " \"return_to_seller_long_gun\",\n", + " \"return_to_seller_other\",\n", + " \"totals\"\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def parse_value(k, x):\n", + " if k == 0: return x\n", + " if x == \"\": return None\n", + " return int(x.replace(\",\", \"\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def parse_row(row):\n", + " return dict((COLUMNS[k], parse_value(k, v)) for k, v in row.items())" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "parsed_table = [ parse_row(row) for row in table ]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here's a sample row:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'admin': 1,\n", + " 'handgun': 1745,\n", + " 'long_gun': 2372,\n", + " 'multiple': 104,\n", + " 'other': 87,\n", + " 'permit': 383,\n", + " 'prepawn_handgun': 0,\n", + " 'prepawn_long_gun': 4,\n", + " 'prepawn_other': 0,\n", + " 'private_sale_handgun': 1,\n", + " 'private_sale_long_gun': 2,\n", + " 'private_sale_other': 0,\n", + " 'redemption_handgun': 132,\n", + " 'redemption_long_gun': 184,\n", + " 'redemption_other': 0,\n", + " 'rentals_handgun': None,\n", + " 'rentals_long_gun': None,\n", + " 'return_to_seller_handgun': 0,\n", + " 'return_to_seller_long_gun': 2,\n", + " 'return_to_seller_other': 0,\n", + " 'returned_handgun': 0,\n", + " 'returned_long_gun': 0,\n", + " 'returned_other': 0,\n", + " 'state': 'Wyoming',\n", + " 'totals': 5017}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parsed_table[-2]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Sort the data\n", + "\n", + "For demonstration purposes, let's list the rows with the highest number of handgun-only background checks:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Totals: 671,330 handgun-only checks\n", + "Pennsylvania: 62,752 handgun-only checks\n", + "Texas: 56,941 handgun-only checks\n", + "Florida: 50,796 handgun-only checks\n", + "California: 41,181 handgun-only checks\n", + "Ohio: 34,878 handgun-only checks\n" + ] + } + ], + "source": [ + "for row in list(reversed(sorted(parsed_table, key=lambda x: x[\"handgun\"])))[:6]:\n", + " print(\"{state}: {handgun:,d} handgun-only checks\".format(**row))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Use `within_bbox` and `collate_chars` to extract the report month\n", + "\n", + "The month of the report is listed in an area 35px to 60px from the top of the page. The code below isolates characters in that space, and then collates their text." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "month_chars = within_bbox(pdf.chars, (0, 35, PDF_WIDTH, 60))" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'November - 2015'" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "collate_chars(month_chars, x_tolerance=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "---\n", + "\n", + "---" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.4.3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/examples/background-checks.pdf b/examples/pdfs/background-checks.pdf similarity index 100% rename from examples/background-checks.pdf rename to examples/pdfs/background-checks.pdf