#LyX 2.2 created this file. For more info see http://www.lyx.org/
\lyxformat 508
\begin_document
\begin_header
\save_transient_properties true
\origin unavailable
\textclass paper
\use_default_options true
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding auto
\fontencoding global
\font_roman "default" "default"
\font_sans "default" "default"
\font_typewriter "default" "default"
\font_math "auto" "auto"
\font_default_family default
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100 100
\font_tt_scale 100 100
\graphics default
\default_output_format default
\output_sync 0
\bibtex_command default
\index_command default
\float_placement H
\paperfontsize default
\spacing single
\use_hyperref true
\pdf_bookmarks true
\pdf_bookmarksnumbered false
\pdf_bookmarksopen false
\pdf_bookmarksopenlevel 1
\pdf_breaklinks false
\pdf_pdfborder true
\pdf_colorlinks false
\pdf_backref false
\pdf_pdfusetitle true
\papersize default
\use_geometry true
\use_package amsmath 1
\use_package amssymb 1
\use_package cancel 1
\use_package esint 1
\use_package mathdots 1
\use_package mathtools 1
\use_package mhchem 1
\use_package stackrel 1
\use_package stmaryrd 1
\use_package undertilde 1
\cite_engine basic
\cite_engine_type default
\biblio_style plain
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date true
\justification true
\use_refstyle 1
\index Index
\shortcut idx
\color #008000
\end_index
\leftmargin 3cm
\topmargin 3cm
\rightmargin 3cm
\bottommargin 3cm
\secnumdepth 3
\tocdepth 3
\paragraph_separation skip
\defskip medskip
\quotes_language english
\papercolumns 1
\papersides 1
\paperpagestyle default
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header

\begin_body

\begin_layout Title
Paperwork Hacking Guide
\end_layout

\begin_layout Standard
\begin_inset CommandInset toc
LatexCommand tableofcontents

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Newpage newpage
\end_inset


\end_layout

\begin_layout Section
This is not the document you're looking for
\end_layout

\begin_layout Standard
\begin_inset Wrap figure
lines 0
placement o
overhang 0in
width "25col%"
status open

\begin_layout Plain Layout
\begin_inset Graphics
	filename img/droid.png

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
This is an entirely technical document.
 If you're not looking to contribute to Paperwork's code, this is not the
 document you're looking for.
\end_layout

\begin_layout Standard
This document assumes you're already familiar with Git, Python 3, and GitHub.
\end_layout

\begin_layout Standard
\begin_inset Newpage newpage
\end_inset


\end_layout

\begin_layout Section
Please discuss before doing
\end_layout

\begin_layout Standard
You should always discuss what you want to do before doing it.
 Main reasons are:
\end_layout

\begin_layout Itemize
If someone else is already working on it, work will be done twice.
 And someone's work will have to be thrown away.
\end_layout

\begin_layout Itemize
You can get confirmation that what you want to do is in line with Paperwork's
 goals (easy to use, ergonomic, etc).
\end_layout

\begin_layout Itemize
You can get suggestions.
 It can strongly reduce the risk of your first pull request being rejected.
\end_layout

\begin_layout Standard
The easiest way is to open a ticket on GitHub
\begin_inset Foot
status open

\begin_layout Plain Layout
https://github.com/jflesch/paperwork/issues
\end_layout

\end_inset

.
 Just indicate you intend to work on it, and the ticket will be assigned
 to you.
\end_layout

\begin_layout Section
Git branches
\end_layout

\begin_layout Standard
In the repository, there are usually the following branches:
\end_layout

\begin_layout Itemize

\emph on
stable
\emph default
 : Matches the latest release version.
 May also include some bug fixes that will be included in the next update
 (x.y.z ; ex: 1.2.2).
 No new features allowed.
\end_layout

\begin_layout Itemize

\emph on
unstable
\emph default
 : Will become the next version (x.y.0 ; ex: 1.3.0) of Paperwork.
 New features go here.
 This branch hasn't been tested yet and things may be broken.
\end_layout

\begin_layout Itemize

\emph on
testing
\emph default
 : When the unstable branch contains all the wanted features, the branch
 
\emph on
testing
\emph default
 is created from the branch 
\emph on
unstable
\emph default
.
 Only bug fixes, translations updates and documentation updates are allowed
 in this branch.
 This branch is temporary and usually lives for about 1 month.
 After 1 month, the version is released, and this branch is merged in the
 branch 
\emph on
stable
\emph default
.
\end_layout

\begin_layout Itemize

\emph on
wip-<XXX>
\emph default
 : Branches for work in progress.
 They are temporary.
 There are no specific rules for those branches.
\end_layout

\begin_layout Section
Coding rules
\end_layout

\begin_layout Subsection
PEP8
\end_layout

\begin_layout Standard
Stick to the PEP8 as much as possible.
 Pull requests that does not respect the PEP8 will be rejected.
\end_layout

\begin_layout Itemize
Lines are at most 80 characters long (not kidding.)
\end_layout

\begin_layout Itemize
Indentation is done using 4 spaces
\end_layout

\begin_layout Standard
If you see code that does not respect the PEP8, feel free to fix it.
 However, please make a dedicated commit.
\end_layout

\begin_layout Standard
If the content of a file does not respect the PEP8 and you want to modify
 it, please 
\emph on
stick to the PEP8
\emph default
.
 Do not use the style of the file.
\end_layout

\begin_layout Subsection
Commits
\end_layout

\begin_layout Standard
One commit = one change.
\end_layout

\begin_layout Standard
Try to make your commit message clear but concise.
\end_layout

\begin_layout Standard
When writing your commit message, please keep in mind the Git history is
 about the whole repository.
 Therefore, please give some context and indicate which part of Paperwork
 has been touched by your commit.
\end_layout

\begin_layout Subsection
Comments
\end_layout

\begin_layout Standard
Too much is better than not enough.
 Other than that, there are no rules regarding comments.
 Please just use your common sense.
\end_layout

\begin_layout Standard
If your code is complex and not commented enough, it will be unmaintainable
 by others.
 It will then be 
\emph on
removed
\emph default
 as soon as you stop maintaining it.
 It already happened !
\end_layout

\begin_layout Section
Paperwork itself
\end_layout

\begin_layout Standard
The code is split in two pieces:
\end_layout

\begin_layout Itemize

\emph on
backen
\emph default
d : Everything related to document and work directory management.
 May depend on various things but 
\emph on
not
\emph default
 GTK+
\end_layout

\begin_layout Itemize

\emph on
frontend
\emph default
 (aka 
\emph on
paperwork-gui
\emph default
 or ..
 
\emph on
Paperwork
\emph default
): The GUI.
 Entirely dependent on GTK+
\end_layout

\begin_layout Standard
The following describes the code organization.
 It is 
\emph on
not
\emph default
 an API documentation.
\end_layout

\begin_layout Subsection
Paperwork-backend
\end_layout

\begin_layout Standard
See the user manual (section 
\begin_inset Quotes eld
\end_inset

advanced use
\begin_inset Quotes erd
\end_inset

) for a description of the work directory organization.
\end_layout

\begin_layout Standard
Paperwork-backend source code is available on GitHub
\begin_inset Foot
status open

\begin_layout Plain Layout
https://github.com/jflesch/paperwork-backend
\end_layout

\end_inset

.
\end_layout

\begin_layout Subsubsection
docsearch.py
\end_layout

\begin_layout Standard
Contains two classes: DummyDocSearch and DocSearch.
 DummyDocSearch can just be used as temporary placeholder for DocSearch.
 DocSearch is the root class of the backend.
\end_layout

\begin_layout Standard
DocSearch manages everything related to the index.
 DocSearch provides:
\end_layout

\begin_layout Itemize
Access to the known documents in the index (methods get(), get_doc_from_docid(),
 find_documents()).
 All documents are instances of sub-classes of common/doc.py:BasicDoc.
\end_layout

\begin_layout Itemize
A way to guess labels on a document (method guess_labels())
\end_layout

\begin_layout Itemize
A way to get keyword suggestions (method find_suggestions())
\end_layout

\begin_layout Itemize
a DocDirExaminer instance.
 This object allows to scan a work directory to find the differences with
 what is known in the index.
\end_layout

\begin_layout Itemize
a DocIndexUpdater instance.
 This object allows to update the content of the index.
\end_layout

\begin_layout Itemize
Shortcuts for label management (methods create_label(), add_label(), remove_labe
l(), update_label(), destroy_label())
\end_layout

\begin_layout Standard
When Paperwork starts, it uses the DocDirExaminer to find what has been
 changed.
 It keeps track of the changes.
 Once the scan is done, it transfer the list of documents to the DocIndexUpdater.
\end_layout

\begin_layout Standard
When a document is modified (labels, OCR re-done, etc), Paperwork uses the
 DocIndexUpdater to update the index.
\end_layout

\begin_layout Subsubsection
Common to all documents
\end_layout

\begin_layout Paragraph
Files and directories structure
\end_layout

\begin_layout Standard
Each document has its own folder.
\end_layout

\begin_layout Standard
In each of these folders, there is a file called (
\begin_inset Quotes eld
\end_inset

labels
\begin_inset Quotes erd
\end_inset

), containing the list of labels (and their color) applied to this document.
 Files 
\begin_inset Quotes eld
\end_inset

labels
\begin_inset Quotes erd
\end_inset

 are CSV files.
\end_layout

\begin_layout Standard
Folder names are arbitrary but by convention, they are dates with the following
 format: YYYYMMDD_hhmm_ss[_nn].
\end_layout

\begin_layout Standard
There may optionally be thumbnails for caching purpose : paper.<page_nb>.thumb.jpg.
\end_layout

\begin_layout Standard

\emph on
Important:
\emph default
 Files and directories structures may change in the future to solve various
 issues (atomic changes on labels, etc).
\end_layout

\begin_layout Paragraph
Programming
\end_layout

\begin_layout Standard
All documents are instances of sub-classes of common/doc.py:BasicDoc.
\end_layout

\begin_layout Standard
All pages are instances of sub-classes of common/page.py:BasicPage.
\end_layout

\begin_layout Subsubsection
Image documents
\end_layout

\begin_layout Standard
Image documents are usually created using a scanner.
 They can also be created by importing images (JPEG, PNG, etc).
\end_layout

\begin_layout Standard
Each image document is a folder.
 In this folder, there are image files (JPEG), one for each page.
\end_layout

\begin_layout Itemize
paper.<page_nb>.jpg : the scanned page
\end_layout

\begin_layout Itemize
paper.<page_nb>.words : the result of the OCR, in hOCR (HTML derivated) format
 (see the limitations of PyOCR for details)
\end_layout

\begin_layout Subsubsection
PDF documents
\end_layout

\begin_layout Standard
Paperwork takes care of never modifying PDF documents.
 Modifying them could mean losing metadata (signatures, etc).
 So, when they are imported, they are simply copied.
\end_layout

\begin_layout Standard
Only one file is specific to PDF documents: doc.pdf.
\end_layout

\begin_layout Subsubsection
docimport.py
\end_layout

\begin_layout Standard
Contains most of the code relative to document import.
 Provides various importers for various situations.
\end_layout

\begin_layout Standard
See docimport.IMPORTERS and docimport.get_possible_importers().
\end_layout

\begin_layout Subsubsection
Export
\end_layout

\begin_layout Standard
Each document and each page provides methods for export:
\end_layout

\begin_layout Itemize
get_export_formats(): returns the possible formats in which the document/page
 can be exported
\end_layout

\begin_layout Itemize
build_exporter(): returns an exporter object: provides various export settings
 and preview
\end_layout

\begin_layout Standard
Additional exporter(s) is/are available in docexport.py.
 They are mostly used when exporting many documents in one shot.
\end_layout

\begin_layout Subsection
Paperwork-shell
\end_layout

\begin_layout Standard
Paperwork-backend provides a command line utility: paperwork-shell.
\end_layout

\begin_layout Standard
Paperwork-shell provides various commands.
 Some are directly available with only paperwork-shell, some are coming
 from paperwork-gui.
\end_layout

\begin_layout Subsection
Paperwork-gui
\end_layout

\begin_layout Standard
Paperwork-gui source code is available on GitHub
\begin_inset Foot
status open

\begin_layout Plain Layout
https://github.com/jflesch/paperwork
\end_layout

\end_inset

.
\end_layout

\begin_layout Section
Development tips
\end_layout

\begin_layout Subsection
Configuration file for development only
\end_layout

\begin_layout Standard
Paperwork looks for a '
\emph on
paperwork.conf
\emph default
' in the current work directory before looking for a '
\emph on
~/.config/.paperwork.conf
\emph default
' in your home directory.
 So if you want to use a different configuration file and/or a different
 set of documents for development than for your real-life work, just copy
 your '
\emph on
~/.config/paperwork.conf
\emph default
' to '
\emph on
./paperwork.conf
\emph default
'.
 Note that the settings dialog will also take care of updating '
\emph on
./paperwork.conf
\emph default
' instead of '
\emph on
~/.config/paperwork.conf
\emph default
'.
\end_layout

\begin_layout Subsubsection
Verbose
\end_layout

\begin_layout Standard
You can use the environment variable '
\emph on
PAPERWORK_VERBOSE
\emph default
' to increase or decrease the logging level.
 The accepted values are: 
\emph on
DEBUG
\emph default
, 
\emph on
INFO
\emph default
, 
\emph on
WARNING
\emph default
, 
\emph on
ERROR
\emph default
.
\end_layout

\begin_layout Subsubsection
PEP8 and Pylint
\end_layout

\begin_layout Standard
There is a tool called '
\emph on
pep8
\emph default
' (
\emph on
apt install pep8
\emph default
).
 Use it.
\end_layout

\begin_layout Standard
Another tool is 'pylint'.
 A pylintrc is provided for Paperwork:
\end_layout

\begin_layout Verbatim
$ cd src ; pylint --rcfile=../pylintrc *
\end_layout

\begin_layout Section
Dependencies and constraints
\end_layout

\begin_layout Standard
\begin_inset Note Note
status open

\begin_layout Plain Layout
Add illustration here
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
GLib / GObject / GTK
\end_layout

\begin_layout Standard
GTK+ is not thread-safe.
\end_layout

\begin_layout Subsection
Pillow
\end_layout

\begin_layout Standard
Thread-safe.
\end_layout

\begin_layout Subsection
Whoosh
\end_layout

\begin_layout Standard
Thread-safe.
 Lock the GIL.
\end_layout

\begin_layout Subsection
Poppler
\end_layout

\begin_layout Standard
Not thread-safe.
\end_layout

\begin_layout Subsection
PyOCR
\end_layout

\begin_layout Standard
Thread-safe.
\end_layout

\begin_layout Subsection
Pyinsane2, libsane, and scanner drivers
\end_layout

\begin_layout Standard
libsane is not thread-safe (mostly depending on the driver used).
 Previous versions of libsane + HP drivers were so sensitive that you had
 to call their functions always from the very same thread.
\end_layout

\begin_layout Standard
However, Pyinsane2 is thread-safe.
\end_layout

\begin_layout Subsection
Libpillowfight / pypillowfight
\end_layout

\begin_layout Standard
Thread-safe.
\end_layout

\begin_layout Section
Threads and processes
\end_layout

\begin_layout Subsection
Thread safety
\end_layout

\begin_layout Standard
Thread safety is a major issue in Paperwork.
 We need threads to keep the GUI smooth, but unfortunately, not all Paperwork
 dependencies are thread-safe.
\end_layout

\begin_layout Standard
A job scheduling mechanism has been implemented (see 
\emph on
src/paperwork/frontend/util/jobs.py
\emph default
).
 The idea here is to run most of the jobs in the same thread (whenever possible).
\end_layout

\begin_layout Standard
Each Job object represents a task to do.
 Some jobs can be stopped and resumed later (for instance JobDocThumbnailer).
 Each Job instance is used once and only once.
\end_layout

\begin_layout Standard
JobFactories instanciate Jobs.
 They are also used to keep the job recognizable.
\end_layout

\begin_layout Standard
Jobs are passed to JobSchedulers.
 Each JobScheduler represents a thread.
 They accept jobs using the method 
\emph on
schedule()
\emph default
.
 The job with the higher priority is run first.
 If the job added to the scheduler has an higher priority than the active
 one, the scheduler will try to stop the active one and run it back later.
\end_layout

\begin_layout Standard
Jobs can be cancelled (assuming they are stoppable or not active yet).
 A single job can be canceled, or all the jobs from a given factory.
\end_layout

\begin_layout Standard
There is one main scheduler (called 
\emph on
main
\emph default
).
 The main scheduler is the one used to access all the documents contents
 and the index.
\end_layout

\begin_layout Standard
Note that there are other threads running.
 For instance, there are the thread of PyInsane and the GTK+/GLib main loop.
\end_layout

\begin_layout Subsection
Multi-processes
\end_layout

\begin_layout Subsubsection
Pyinsane and Sane
\end_layout

\begin_layout Standard
In the past, some Sane drivers have shown to be unreliable.
 Some crashes, while some other corrupts memory areas sometimes.
\end_layout

\begin_layout Standard
To work around those issues, on GNU/Linux, Pyinsane forks (see 
\emph on
pyinsane2/sane/abstract_proc.py
\emph default
).
 A dedicated process is used to scan.
 Two unix pipes are created for the communication.
\end_layout

\begin_layout Subsubsection
Paperwork-backend and python-whoosh
\end_layout

\begin_layout Standard
Python-whoosh locks Python's GIL (Global Interpreter Lock).
 When the GIL is locked, the GUI is stuck.
\end_layout

\begin_layout Standard
As a workaround, paperwork-backend uses the 
\emph on
multiprocessing
\emph default
 module : It forks itself.
 One process acts as client (
\emph on
docsearch.py
\emph default
) and the other as server (
\emph on
index.py
\emph default
)
\end_layout

\begin_layout Section
Most complex pieces
\end_layout

\begin_layout Subsection
Scan process
\end_layout

\begin_layout Standard
\begin_inset Note Note
status open

\begin_layout Plain Layout
TODO illustration
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
OCR process
\end_layout

\begin_layout Standard
\begin_inset Note Note
status open

\begin_layout Plain Layout
TODO illustration
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
Index updates
\end_layout

\begin_layout Standard
\begin_inset Note Note
status open

\begin_layout Plain Layout
TODO illustration
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
Searching
\end_layout

\begin_layout Standard
\begin_inset Note Note
status open

\begin_layout Plain Layout
TODO illustration
\end_layout

\end_inset


\end_layout

\begin_layout Section
Tests
\end_layout

\begin_layout Section
Windows build
\end_layout

\begin_layout Section
Versioning
\end_layout

\begin_layout Standard
Version have the following syntax: <M>[.<m>[.<U>[<i>][-<extra>]]]
\end_layout

\begin_layout Itemize
M = Major version : Major changes made / product completed.
\end_layout

\begin_layout Itemize
m = minor version : Minor changes made.
\end_layout

\begin_layout Itemize
U = update version : Only bug fixes
\end_layout

\begin_layout Itemize
i = installer revision : Only the installer has been changed (Windows only)
\end_layout

\begin_layout Itemize
Extra : (Git tags only) May match a Git tag done before a big change (for
 instance: before switching from Gtk 2 to Gtk 3).
 If extra == "git", indicates a version directly taken from the git repository.
 
\end_layout

\begin_layout Section
Documentation
\end_layout

\begin_layout Standard
Everything related to documentation is in the directory doc/.
\end_layout

\begin_layout Subsection
Installation
\end_layout

\begin_layout Standard
Documentation regarding the installation process is written using Markdown.
 This allows to show the documentation on GitHub
\end_layout

\begin_layout Subsection
Other
\end_layout

\begin_layout Standard
Other part of the documentations are redacted using Lyx
\begin_inset Foot
status open

\begin_layout Plain Layout
https://www.lyx.org/
\end_layout

\end_inset

.
 Lyx is a document processor with various advantages:
\end_layout

\begin_layout Itemize
Clearly structured documents
\end_layout

\begin_layout Itemize
.lyx files are ASCII files (can be versioned with Git easily)
\end_layout

\begin_layout Itemize
can be exported to PDF easily
\end_layout

\begin_layout Standard
Those documents are then packaged with Paperwork in PDF format.
 Paperwork can display them easily.
\end_layout

\begin_layout Standard
The document doc/intro_<lang>.lyx (fallback on intro.lyx if required) is the
 one automatically added and shown when there is no document in the work
 directory.
\end_layout

\end_body
\end_document
