Commit 52ac29f0 authored by Jerome Flesch's avatar Jerome Flesch

Merge branch 'wip-plugins' into 'develop'

Plugins: Implement core + 2 first plugins

See merge request !813
parents 762c8c24 d6feca8b
Pipeline #99569 failed with stage
in 4 minutes and 14 seconds
# order matters (dependencies)
ALL_COMPONENTS = paperwork-backend paperwork-gtk
ALL_COMPONENTS = openpaperwork-core paperwork-backend paperwork-gtk
build:
openpaperwork-core_install_py:
echo "Installing openpaperwork-core"
$(MAKE) -C openpaperwork-core install_py
%_install_py: openpaperwork-core_install_py
echo "Installing $(@:%_install_py=%)"
$(MAKE) -C $(@:%_install_py=%) install_py
build: $(ALL_COMPONENTS:%=%_build)
clean: $(ALL_COMPONENTS:%=%_clean)
rm -rf build dist
......@@ -10,11 +20,9 @@ clean: $(ALL_COMPONENTS:%=%_clean)
make -C sub/libpillowfight clean || true
make -C sub/pyocr clean || true
install: $(ALL_COMPONENTS:%=%_install)
install_py: $(ALL_COMPONENTS:%=%_install_py)
install_c: $(ALL_COMPONENTS:%=%_install_c)
install: install_py
uninstall: $(ALL_COMPONENTS:%=%_uninstall)
......@@ -86,34 +94,10 @@ help:
echo "Checking $(@:%_doc=%)"
$(MAKE) -C $(@:%_doc=%) doc
%_build:
echo "Building $(@:%_build=%)"
$(MAKE) -C $(@:%_build=%) build
%_clean:
echo "Building $(@:%_clean=%)"
$(MAKE) -C $(@:%_clean=%) clean
%_install:
echo "Installing $(@:%_install=%)"
$(MAKE) -C $(@:%_install=%) install
%_build_py:
echo "Building $(@:%_build_py=%)"
$(MAKE) -C $(@:%_build=%) build_py
%_install_py:
echo "Installing $(@:%_install_py=%)"
$(MAKE) -C $(@:%_build=%) install_py
%_build_c:
echo "Building $(@:%_build_c=%)"
$(MAKE) -C $(@:%_build=%) build_c
%_install_c:
echo "Installing $(@:%_install_c=%)"
$(MAKE) -C $(@:%_build=%) install_c
%_uninstall:
echo "Uninstalling $(@:%_uninstall=%)"
$(MAKE) -C $(@:%_uninstall=%) uninstall
......@@ -144,8 +128,6 @@ help:
venv:
echo "Building virtual env"
git submodule init
git submodule update --recursive --remote --init
make -C sub/libinsane build_c
virtualenv -p python3 --system-site-packages venv
......
if ! [ -e sub/libinsane/Makefile ] ; then
git submodule init
git submodule update --recursive --remote --init
fi
make venv
source venv/bin/activate
cd sub/libinsane && source ./activate_test_env.sh ; cd ../..
make -C sub/pyocr install_py
make -C sub/libpillowfight install_py
VERSION_FILE = openpaperwork_core/_version.py
PYTHON ?= python3
build: build_c build_py
install: install_py install_c
uninstall: uninstall_py
build_py: ${VERSION_FILE}
${PYTHON} ./setup.py build
build_c:
version: ${VERSION_FILE}
${VERSION_FILE}:
echo -n "version = \"" >| $@
echo -n $(shell git describe --always) >> $@
echo "\"" >> $@
doc:
check:
flake8 openpaperwork_core
test: ${VERSION_FILE}
tox
linux_exe:
windows_exe:
release:
ifeq (${RELEASE}, )
@echo "You must specify a release version (make release RELEASE=1.2.3)"
else
@echo "Will release: ${RELEASE}"
@echo "Checking release is in ChangeLog ..."
grep ${RELEASE} ChangeLog | grep -v "/xx"
endif
release_pypi:
@echo "Releasing paperwork-backend ..."
${PYTHON} ./setup.py sdist upload
@echo "All done"
clean:
rm -f ${VERSION_FILE}
rm -rf build dist *.egg-info
# PIP_ARGS is used by Flatpak build
install_py: ${VERSION_FILE}
${PYTHON} ./setup.py install ${PIP_ARGS}
install_c:
uninstall_py:
pip3 uninstall -y paperwork-backend
uninstall_c:
help:
@echo "make build || make build_py"
@echo "make check"
@echo "make help: display this message"
@echo "make install || make install_py"
@echo "make uninstall || make uninstall_py"
@echo "make release"
.PHONY: \
build \
build_c \
build_py \
check \
doc \
exe \
help \
install \
install_c \
install_py \
release \
test \
uninstall \
uninstall_c \
version
# OpenPaperwork Core
Manages Plugins, Callbacks and Interfaces.
A plugin is a Python module providing a class `Plugin`.
Callbacks are all the methods provided by the class `Plugin` (with some
exceptions, like methods starting with `_`).
Interfaces are simply conventions: Plugins pretending to implement some
interfaces must implement the corresponding methods. No check is done to ensure
they do.
## Example
### Plugin
`openpaperwork\_plugin/\_\_init\_\_.py`
```py
import openpaperwork_core
class Plugin(openpaperwork_core.PluginBase):
# indicates that users should be able to disable/enable this plugin in
# the UI (if available/possible in the application)
USER_VISIBLE = True
def __init__(self):
# do something, but the least possible
# cannot rely on dependencies here.
pass
def get_implementated_interfaces(self):
return ['interface_name_toto', 'interface_name_tutu']
def get_deps(self):
return {
'plugins': [
'module_name_a',
'module_name_b',
],
'interfaces': [
'interface_name_a',
],
}
def init(self, core):
# all the dependnecies have loaded and initialized.
# we can safely rely on them here.
pass
def some_method_a(self, arg_a):
# do something
```
### Application using the core
```py
import openpaperwork_core
core = openpaperwork_core.Core()
# load mandatory plugins
core.load("openpaperwork_plugin")
# load plugins requested by your user if any
core.load(...)
# init() will load dependencies and call method `init()` on all the plugins
core.init()
# call_all() will call all the methods with the specified name. Return values
# are ignored. You have to pass a callback as argument if you want to get
# result from all callbacks.
core.call_all('some_method_a', "random_argument")
# call_one() will call one of the methods with the specified name.
# It is assumed that only one callback has this name. Return value of the
# callback is returned as it.
return_value = core.call_one('some_method_a', "random_argument")
```
#!/usr/bin/env python3
import logging
import openpaperwork_core
LOGGER = logging.getLogger(__name__)
def main():
LOGGER.info("Start")
core = openpaperwork_core.Core()
core.load("openpaperwork_core.log_collector")
core.init()
LOGGER.info("End")
if __name__ == "__main__":
main()
import collections
import importlib
import logging
LOGGER = logging.getLogger(__name__)
class PluginBase(object):
"""
Indicates all the methods that must be implemented by any plugin
managed by OpenPaperwork core. Also provides default implementations
for each method.
"""
# Convenience for the applications: Indicates if users should be able
# to enable/disable this plugin in the UI.
USER_VISIBLE = False
def __init__(self, core):
"""
Called as soon as the module is loaded. Should be as minimal as
possible. Most of the work should be done in `init()`.
You *must* *not* rely on any dependencies here.
"""
pass
def get_interfaces(self):
"""
Indicates the list of interfaces implemented by this plugin.
Interface names are arbitrarily defined. Methods provided by each
interface are arbitrarily defined (and no checks are done).
Returns a list of string.
"""
return []
def get_deps(self):
"""
Return the dependencies required by this plugin.
"""
return {
'plugins': [],
'interfaces': [],
}
def init(self, core):
pass
class Core(object):
"""
Manage plugins and their callbacks.
"""
def __init__(self):
self.plugins = {}
self._to_initialize = set()
self._initialized = set() # avoid double-init
self.interfaces = collections.defaultdict(list)
self.callbacks = collections.defaultdict(list)
def load(self, module_name):
"""
- Load the specified module
- Instantiate the class 'Plugin()' of this module
- Register all the methods of this plugin object (except those starting
by '_' and those from the class PluginBase) as callbacks
BEWARE of dependency loops !
Arguments:
- module_name: name of the Python module to load
"""
if module_name in self.plugins:
return
LOGGER.info("Loading plugin '%s' ...", module_name)
module = importlib.import_module(module_name)
plugin = module.Plugin()
self.plugins[module_name] = plugin
for interface in plugin.get_interfaces():
LOGGER.debug("- '%s' provides '%s'", module_name, interface)
self.interfaces[interface].append(plugin)
for attr_name in dir(plugin):
if attr_name[0] == "_":
continue
if attr_name in dir(PluginBase): # ignore base methods of plugins
continue
attr = getattr(plugin, attr_name)
if not hasattr(attr, '__call__'):
continue
LOGGER.debug("- %s.%s()", module_name, attr_name)
self.callbacks[attr_name].append(attr)
self._to_initialize.add(plugin)
LOGGER.info("Plugin '%s' loaded", module_name)
return plugin
def _load_deps(self):
to_examine = list(self.plugins.values())
while len(to_examine) > 0:
plugin = to_examine[0]
to_examine = to_examine[1:]
LOGGER.info("Examining dependencies of '%s' ...", type(plugin))
deps = plugin.get_deps()
if 'plugins' in deps:
for dep_plugin in deps['plugins']:
if dep_plugin in self.plugins:
LOGGER.info("- Plugin '%s' already loaded", dep_plugin)
continue
to_examine.append(self.load(dep_plugin))
if 'interfaces' in deps:
for (dep_interface, dep_defaults) in deps['interfaces']:
if len(self.interfaces[dep_interface]) > 0:
LOGGER.info(
"- Interface '%s' already provided by %d plugins",
dep_interface, len(self.interfaces[dep_interface])
)
continue
LOGGER.info(
"Loading default plugins for interface '%s'"
" (%d plugins)",
dep_interface, len(dep_defaults)
)
for dep_default in dep_defaults:
to_examine.append(self.load(dep_default))
assert(len(self.interfaces[dep_interface]) > 0)
def _init(self, plugin):
if plugin in self._initialized:
return
deps = plugin.get_deps()
if 'plugins' in deps:
for dep_plugin in deps['plugins']:
assert(dep_plugin in self.plugins)
self._init(self.plugins[dep_plugin])
if 'interfaces' in deps:
for (dep_interface, _) in deps['interfaces']:
dep_plugins = self.interfaces[dep_interface]
assert(len(dep_plugins) > 0)
for dep_plugin in dep_plugins:
self._init(dep_plugin)
LOGGER.info("Initializing plugin '%s' ...", type(plugin))
plugin.init(self)
self._initialized.add(plugin)
def init(self):
"""
- Make sure all the dependencies of all the plugins are satisfied.
- Call the method init() of each plugin following the dependency
order (those without dependencies are called first).
BEWARE of dependency loops !
"""
LOGGER.info("Initializing all plugins")
self._load_deps()
for plugin in self._to_initialize:
self._init(plugin)
self._to_initialize = set()
LOGGER.info("All plugins initialized")
def get(self, module_name):
"""
Returns a Plugin instance based on the corresponding module name
(assuming it has been loaded).
"""
return self.plugins[module_name]
def call_all(self, callback_name, *args, **kwargs):
"""
Call all the methods of all the plugins that have `callback_name`
as name. Arguments are passed as is. Returned values are dropped
(use callbacks for return values if required)
"""
callbacks = self.callbacks[callback_name]
if len(callbacks) <= 0:
LOGGER.warning("No method '%s' available !", callback_name)
for callback in callbacks:
callback(*args, **kwargs)
def call_one(self, callback_name, *args, **kwargs):
"""
Look for a plugin method called `callback_name` and calls it.
Raises an error if no such method exists. If many exists,
raises a warning and call one at random.
Returns the value return by the callback.
You're advised to use `call_all()` instead whenever possible.
This method is only provided as convenience for when you're
fairly sure there should be only one plugin with such callback.
"""
callbacks = self.callbacks[callback_name]
if len(callbacks) <= 0:
raise IndexError(
"No method '{}' available !".format(callback_name)
)
if len(callbacks) > 1:
LOGGER.warning(
"More than one method '%s' available ! [%s]", callback_name,
", ".join([str(callback) for callback in callbacks])
)
return callbacks[0](*args, **kwargs)
"""
Manages a configuration file using configparser.
"""
import collections
import configparser
import logging
import os
import os.path
from . import PluginBase
LOGGER = logging.getLogger(__name__)
class ConfigList(object):
SEPARATOR = ", "
def __init__(self, string=None, elements=[]):
self.elements = elements
if string is not None:
elements = string.split(self.SEPARATOR, 1)
for i in elements:
(t, v) = i.split(_TYPE_SEPARATOR, 1)
self.elements.append(_STR_TO_TYPE[t](v))
def __iter__(self):
return iter(self.elements)
def __contains__(self, o):
return o in self.elements
def __get_item__(self, *args, **kwargs):
return self.elements.__get_items__(*args, **kwargs)
def __str__(self):
out = []
for e in self.elements:
out.append("{}{}{}".format(
_TYPE_TO_STR[type(e)], _TYPE_SEPARATOR, str(e)
))
return self.SEPARATOR.join(out)
_STR_TO_TYPE = {
"bool": bool,
"int": int,
"float": float,
"str": str,
"list": ConfigList,
}
_TYPE_TO_STR = {v: k for (k, v) in _STR_TO_TYPE.items()}
_TYPE_SEPARATOR = ":"
class Plugin(PluginBase):
def __init__(self):
self.config = configparser.RawConfigParser()
self.base_path = os.getenv(
"XDG_CONFIG_HOME",
os.path.expanduser("~/.config")
)
self.config_file_path_fmt = os.path.join(
"{directory}", "{app_name}.conf"
)
self.application_name = None
self.observers = collections.defaultdict(set)
self.core = None
def init(self, core):
self.core = core
def get_interfaces(self):
return ['configuration']
def config_load(self, application_name):
self.application_name = application_name
config_path = self.config_file_path_fmt.format(
directory=self.base_path,
app_name=application_name,
)
self.config = configparser.RawConfigParser()
LOGGER.info("Loading configuration '%s' ...", config_path)
with open(config_path, 'r') as fd:
self.config.read_file(fd)
for observers in self.observers.values():
for observer in observers:
observer()
def config_save(self, application_name=None):
if application_name is not None:
self.application_name = application_name
config_path = self.config_file_path_fmt.format(
directory=self.base_path,
app_name=self.application_name,
)
LOGGER.info("Writing configuration '%s' ...", config_path)
with open(config_path, 'w') as fd:
self.config.write(fd)
def config_load_plugins(self, default=[]):
"""
Load and init the plugin list from the configuration.
"""
modules = self.config_get("plugins", "modules", ConfigList(default))
LOGGER.info(
"Loading and initializing plugins from configuration: %s",
str(modules)
)
for module in modules:
self.core.load(module)
self.core.init()
def config_add_plugin(self, module_name):
LOGGER.info("Adding plugin '%s' to configuration", module_name)
modules = self.config_get("plugins", "modules", ConfigList())
modules.elements.append(module_name)
self.config_put("plugins", "modules", modules)
def config_remove_plugin(self, module_name):
LOGGER.info("Removing plugin '%s' from configuration", module_name)
modules = self.config_get("plugins", "modules", ConfigList())
modules.elements.remove(module_name)
self.config_put("plugins", "modules", modules)
def config_put(self, section, key, value):
"""
Section must be a string.
Key must be a string.
"""
LOGGER.debug("Configuration: %s:%s <-- %s", section, key, str(value))
t = _TYPE_TO_STR[type(value)]
value = "{}{}{}".format(t, _TYPE_SEPARATOR, str(value))
if section not in self.config:
self.config[section] = {key: value}
else:
self.config[section][key] = value
for observer in self.observers[section]:
observer()
def config_get(self, section, key, default=None):
try:
value = self.config[section][key]
(t, value) = value.split(_TYPE_SEPARATOR, 1)
r = _STR_TO_TYPE[t](value)
LOGGER.debug("Configuration: %s:%s --> %s", section, key, str(r))
return r
except KeyError:
if default is None:
raise KeyError(
"Configuration: {}:{} not found".format(section, key)
)
LOGGER.debug(
"Configuration: %s:%s --> %s (default value)",
section, key, str(default)
)
return default
def config_add_observer(self, section, callback):
self.observers[section].add(callback)
def config_remove_observer(self, section, callback):
self.observers[section].remove(callback)
import datetime
import logging
import sys
import tempfile
from . import PluginBase
LOGGER = logging.getLogger(__name__)
def _get_tmp_file():
date = datetime.datetime.now()
date = date.strftime("%Y%m%d_%H%M_%S")
t = tempfile.NamedTemporaryFile(
mode='w',
suffix=".txt",
prefix="openpaperwork_{}_".format(date),
encoding='utf-8'
)
if sys.stderr is not None:
sys.stderr.write("Temporary file = {}\n".format(t.name))
return t
class LogHandler(logging.Handler):
def __init__(self):
super().__init__()
self.formatter = None
self.out_fds = set()
sys.excepthook = self.on_uncatched_exception_cb
def emit(self, record):
if self.formatter is None:
return
line = self.formatter.format(record)
for fd in self.out_fds:
fd.write(line)
def on_uncatched_exception_cb(self, exc_type, exc_value, exc_tb):
LOGGER.error(
"=== UNCATCHED EXCEPTION ===",
exc_info=(exc_type, exc_value, exc_tb)
)
LOGGER.error("===========================")
class Plugin(PluginBase):
CONFIG_SECTION = 'logging'
CONFIG_LOG_LEVEL = 'level'
CONFIG_LOG_FILES = 'files'
CONFIG_LOG_FORMAT = 'format'
CONFIG_FILE_SEPARATOR = ","
LOG_LEVELS = {
'none': logging.CRITICAL,
'critical': logging.CRITICAL,
'error': logging.ERROR,