From 149f5e0bfe43066560ccd3c4e3644201107ca76d Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Sat, 4 Apr 2020 22:17:23 +0200 Subject: [PATCH] Add an initial website for Tracker, built using mkdocs Run `mkdocs` in the toplevel directory to generate the HTML. The last time Tracker had a website was 2011 (http://web.archive.org/web/20110307070049/http://projects.gnome.org/tracker) so this is very exciting. The website is published here: https://gnome.pages.gitlab.gnome.org/tracker/ The latest API reference documentation is built and included in the Tracker website, available under the /docs/api-preview/ prefix. This allows us to browse and reference the documentation during Tracker 3.0 development, as well as providing a sort of work around for https://gitlab.gnome.org/GNOME/tracker/-/issues/100. A warning is added to each documentation page advising that it is a preview built from Git. --- .gitlab-ci.yml | 26 ++++++ .gitlab-ci/checkout-tracker-miners.sh | 37 ++++++++ docs/mkdocs.yml | 10 +++ docs/website/build.py | 118 ++++++++++++++++++++++++++ docs/website/community.md | 19 +++++ docs/website/docs.md | 45 ++++++++++ docs/website/faq.md | 75 ++++++++++++++++ docs/website/index.md | 17 ++++ docs/website/overview.md | 91 ++++++++++++++++++++ 9 files changed, 438 insertions(+) create mode 100755 .gitlab-ci/checkout-tracker-miners.sh create mode 100644 docs/mkdocs.yml create mode 100755 docs/website/build.py create mode 100644 docs/website/community.md create mode 100644 docs/website/docs.md create mode 100644 docs/website/faq.md create mode 100644 docs/website/index.md create mode 100644 docs/website/overview.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 43c773bb7..aeb209be6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ variables: stages: - test - analysis + - website .test_template: &test stage: test @@ -78,3 +79,28 @@ coverity: --form description="gitlab CI build"' only: - master + +pages: + stage: website + image: registry.gitlab.gnome.org/gnome/tracker-oci-images/amd64/fedora:latest + dependencies: [] + before_script: + - dnf install -y python3-pip + - pip3 install mkdocs mkdocs-cinder + script: + # Build tracker and install. + - su tracker -c 'mkdir build; cd build; meson .. --prefix=/tmp/tracker; ninja install' + # Build tracker-miners and install any documentation from there as well. + - su tracker -c '.gitlab-ci/checkout-tracker-miners.sh' + - su tracker -c 'cd extra/tracker-miners; mkdir build; cd build; env PKG_CONFIG_PATH=/tmp/tracker/lib64/pkgconfig meson .. --prefix=/tmp/tracker; ninja install' + # Generate the website using mkdocs. + - | + tracker_commit=$CI_COMMIT_SHA1 + tracker_miners_commit=$(git -C ./extra/tracker-miners rev-parse HEAD) + ./docs/website/build.py --api-docs=/tmp/tracker/share/gtk-doc/html --tracker-commit=${tracker_commit} + artifacts: + paths: + - public + only: + - master + - sam/website diff --git a/.gitlab-ci/checkout-tracker-miners.sh b/.gitlab-ci/checkout-tracker-miners.sh new file mode 100755 index 000000000..bf1ea0b5b --- /dev/null +++ b/.gitlab-ci/checkout-tracker-miners.sh @@ -0,0 +1,37 @@ +#!/usr/bin/bash + +tracker_target= + +mkdir extra +cd extra + +git clone https://gitlab.gnome.org/GNOME/tracker-miners.git + +if [ $? -ne 0 ]; then + echo Checkout failed + exit 1 +fi + +cd tracker-miners + +if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]; then + merge_request_remote=${CI_MERGE_REQUEST_SOURCE_PROJECT_URL//tracker/tracker-miners} + merge_request_branch=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME + + echo Looking for $merge_request_branch on remote ... + if git fetch -q $merge_request_remote $merge_request_branch 2>/dev/null; then + tracker_target=FETCH_HEAD + else + tracker_target=origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + echo Using $tracker_target instead + fi +fi + +if [ -z "$tracker_target" ]; then + tracker_target=$(git branch -r -l origin/$CI_COMMIT_REF_NAME) + tracker_target=${tracker_target:-origin/master} + echo Using $tracker_target instead +fi + +git checkout -q $tracker_target + diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml new file mode 100644 index 000000000..6757bb3bc --- /dev/null +++ b/docs/mkdocs.yml @@ -0,0 +1,10 @@ +site_name: Tracker +theme: cinder +docs_dir: ./website + +nav: + - Home: 'index.md' + - Overview: 'overview.md' + - FAQ: 'faq.md' + - Community: 'community.md' + - Documentation: 'docs.md' diff --git a/docs/website/build.py b/docs/website/build.py new file mode 100755 index 000000000..49c086529 --- /dev/null +++ b/docs/website/build.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# Tracker website build script. +# +# Copyright 2020, Sam Thursfield +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. + + +import argparse +import logging +import pathlib +import shutil +import subprocess +import sys +import tempfile + +log = logging.getLogger('build.py') + +output_path = pathlib.Path('public') +mkdocs_root = pathlib.Path(__file__).parent.parent + + +def argument_parser(): + parser = argparse.ArgumentParser( + description="Tracker website build script") + parser.add_argument('--debug', dest='debug', action='store_true', + help="Enable detailed logging to stderr") + parser.add_argument('--api-docs', required=True, metavar='PATH', + help="Path that contains API documentation. Usually " + "$prefix/share/gtk-doc/html/") + parser.add_argument('--tracker-commit', required=True, metavar='COMMIT', + help="Commit ID of tracker.git repo used to build") + return parser + + +def apidocs_header(tracker_commit): + return f""" +
+

This is a documentation preview for the next version of Tracker, + generated from tracker.git commit {tracker_commit[:7]}.

+

See the Tracker website for more documentation.

+
""" + + +def add_apidocs_header(text, filename): + """Add a header to the documentation preview files.""" + + # We insert the header before any of these + markers = [ + '
', + '
', + '
', + '
', + '
', + '
', + '
', + ] + + with open(filename, encoding='utf8') as f_in: + with tempfile.NamedTemporaryFile(delete=False, mode='w', encoding='utf8') as f_out: + for line in f_in: + for marker in markers: + if line.find(marker) != -1: + f_out.write(text) + f_out.write(line) + shutil.move(f_out.name, filename) + + +def main(): + args = argument_parser().parse_args() + + if args.debug: + logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) + else: + logging.basicConfig(stream=sys.stderr, level=logging.INFO) + + if output_path.exists(): + raise RuntimeError(f"Output path '{output_path}' already exists.") + + log.info("Building website") + mkdocs_config = mkdocs_root.joinpath('mkdocs.yml') + subprocess.run(['mkdocs', 'build', '--config-file', mkdocs_config, + '--site-dir', output_path.absolute()]) + + apidocs_src = pathlib.Path(args.api_docs) + + log.info("Copying API reference documentation from %s", apidocs_src) + apidocs_dest = output_path.joinpath('docs/api-preview') + apidocs_dest.mkdir(parents=True) + for name in ['libtracker-sparql', 'ontology']: + shutil.copytree(apidocs_src.joinpath(name), apidocs_dest.joinpath(name)) + + log.info("Adding preview header to API reference documentation") + text = apidocs_header(args.tracker_commit) + for filename in apidocs_dest.rglob('*.html'): + add_apidocs_header(text, filename) + + log.info("Documentation available in public/ directory.") + + +try: + main() +except RuntimeError as e: + sys.stderr.write(f"ERROR: {e}\n") + sys.exit(1) diff --git a/docs/website/community.md b/docs/website/community.md new file mode 100644 index 000000000..18d4f94f7 --- /dev/null +++ b/docs/website/community.md @@ -0,0 +1,19 @@ +# Community + +Tracker is community-driven Free Software, released under the [GNU GPL version 2 or later](https://spdx.org/licenses/GPL-2.0-or-later). + +Development happens at gitlab.gnome.org, in the +[Tracker](https://gitlab.gnome.org/GNOME/tracker) and [Tracker Miners](https://gitlab.gnome.org/GNOME/tracker-miners) projects. +You're invited to join in! See the [help wanted label](https://gitlab.gnome.org/GNOME/tracker/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=4.%20Help%20Wanted). + +Forum discussion takes place at +[discourse.gnome.org](https://discourse.gnome.org/) -- please use the +['tracker' tag](https://discourse.gnome.org/tags/tracker) for Tracker related +posts. + +Real time discussion can be found in the #tracker channel on the GIMPnet IRC +network, also bridged to the Matrix channel #tracker:gnome.org. +[Click here to join the conversation in your browser](https://riot.im/app/#/room/#tracker:gnome.org) +using the Riot.im webapp. + +There is also a [Tracker project wiki](https://wiki.gnome.org/Projects/Tracker). diff --git a/docs/website/docs.md b/docs/website/docs.md new file mode 100644 index 000000000..beed96a56 --- /dev/null +++ b/docs/website/docs.md @@ -0,0 +1,45 @@ +# Documentation + +## User Documentation + +Tracker is a system component and most users will not need to interact with +it directly. + +GNOME has documentation on how to +[search for files in the file manager](https://developer.gnome.org/libtracker-sparql/stable/). + +The `tracker` commandline tool provides direct access to Tracker. Use the +`--help` option for documentation of this tool. + +## Developer Documentation + +Application and platform developers using Tracker will interact with Tracker +using one or more of the shared libraries it provides: + + * [libtracker-sparql](https://developer.gnome.org/libtracker-sparql/stable/) is + used to read and write data from the Tracker store using SPARQL. + * [libtracker-control](https://developer.gnome.org/libtracker-control/stable/), + is used to manage Tracker daemons. + * [libtracker-miner](https://developer.gnome.org/libtracker-miner/stable/) can + be used if you want to implement a new data provider while reusing existing + Tracker code. + +WARNING: The documention linked above is for out of date verions of Tracker 2.x. +This is due to an [infrastructure +issue](https://gitlab.gnome.org/Infrastructure/library-web/issues/50). See also +[this bug](https://gitlab.gnome.org/GNOME/tracker/-/issues/100). + +The following documentation may be useful: + + * [Tracker ontology documentation](https://developer.gnome.org/ontology/stable/). + * [Tracker documentation on wiki.gnome.org](https://wiki.gnome.org/Projects/Tracker). + +## Preview Documentation + +We provide an online version of the documentation for the latest in-development version +of Tracker. You can browse it here: + + * [libtracker-sparql](./api-preview/libtracker-sparql) + * [ontology](./api-preview/ontology) + +Be aware that some libraries from Tracker 2.0 will not be available for Tracker 3.0. diff --git a/docs/website/faq.md b/docs/website/faq.md new file mode 100644 index 000000000..473e5ec62 --- /dev/null +++ b/docs/website/faq.md @@ -0,0 +1,75 @@ +# FAQ + +## What is Tracker? + +It's a search engine, and a database. + +Tracker indexes content from your home directory automatically, so applications +can provide instant search results when you need them. + +See the [overview](/overview) for more information. + +## What files will Tracker index? + +The default configuration of Tracker is to look at files and folders in your +XDG content directories such as `Documents`, `Music`, `Pictures` and `Videos`. +It also looks at files in your home directory and `Downloads` directory, but +it doesn't recurse into folders. + +You might want to [control what Tracker indexes] so that it finds files in +other places too. + +## How can I control what Tracker indexes? + +In GNOME, you can use the Search Settings panel to control what Tracker +indexes. See [GNOME's documentation](https://help.gnome.org/users/gnome-help/unstable/files-search.html.en). + +You can control Tracker's configuration directly using +[dconf-editor](https://wiki.gnome.org/Apps/DconfEditor) or the `gsettings` CLI +tool. +The relevant schemas are `org.freedesktop.Tracker.Miner.Files` and +`org.freedesktop.Tracker.Extract`. + +## Why does Tracker consume resources on my PC? + +When you add or edit files, Tracker will update its index. This should be very +quick, but if a huge number of files are added then it may cause noticably high +CPU and IO usage until the new files have been indexed. This is normal. + +Tracker is not designed to index directories of source code or video game data. +If content like this appears in the locations Tracker is configured to index +then you might see unwanted high resource usage. + +If you see high resource usage from Tracker even when no files have changed on +disk, this probably indicates a bug in Tracker or one of its dependencies. +We can [work together](/community) to find out what the problem is. + +## How can I disable Tracker in GNOME? + +Tracker is a core dependency of GNOME, and some things will not work as +expected if you disable it completely. + +If you are experiencing performance problems, see [Why does Tracker consume +resources on my PC?](#why-does-tracker-consume-resources-on-my-pc). + +In case of a bug you may need to temporarily stop Tracker indexing. +The simplest way is to [edit the +configuration](#how-can-i-control-what-tracker-indexes) so that no directories +are indexed. This should bring resource usage to zero. + +If the Tracker store is using a lot of disk space and you are sure that +there is no unreplaceable data stored in the database, you can run `tracker +reset --hard` to delete everything stored in the database. + +## Can Tracker help me organize my music collection? + +At present, Tracker simply reads the metadata stores in your music files (often +called 'tags'). + +You may be able to use Gnome Music to correct this metadata using the +Musicbrainz database. + +Programs that fix tags and organize music collections on disk, such as +[Beets](http://beets.io/), work well with Tracker. + +[control what Tracker indexes]: #how-can-i-control-what-tracker-indexes diff --git a/docs/website/index.md b/docs/website/index.md new file mode 100644 index 000000000..92becdd03 --- /dev/null +++ b/docs/website/index.md @@ -0,0 +1,17 @@ +# Welcome to the Tracker project + +Tracker provides searching and indexing functionality for the +[GNOME desktop environment](https://www.gnome.org/) and beyond. + +Are you having problems with Tracker in GNOME? Look at the [Frequently Asked +Questions](faq). + +Are you interested in using or developing Tracker and want to find out more +about it? Read the [overview](overview) first and then take a look at the +[documentation](documentation). + +Do you want to contribute to Tracker? Join the [community](community) +and look for some bugs marked 'Help wanted' +([here](https://gitlab.gnome.org/GNOME/tracker/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=4.%20Help%20Wanted) +and +[here](https://gitlab.gnome.org/GNOME/tracker-miners/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=4.%20Help%20Wanted)). diff --git a/docs/website/overview.md b/docs/website/overview.md new file mode 100644 index 000000000..391998703 --- /dev/null +++ b/docs/website/overview.md @@ -0,0 +1,91 @@ +# Overview + +## What is Tracker? + +Tracker is an efficient search engine and +[triplestore](https://en.wikipedia.org/wiki/Triplestore) for desktop, embedded +and mobile. + +It is a middleware component aimed at desktop application developers who +want their apps to browse and search user content. It's not designed to be +used directly by desktop users, but it provides a commandline tool named +`tracker` for the adventurous. + +Tracker allows your application to instantly perform full-text searches across +all documents. This feature is used by the 'search' bar in GNOME Files, for +example. + +This is achieved by indexing the user's home directory in the background. + +Tracker also allows your application to query and list content that the user +has stored. For example, GNOME Music displays all the music files that are +found by Tracker. This means that GNOME Music doesn't need to maintain a +database of its own. + +If you need to go beyond simple searches, you'll be happy to know that +Tracker is also a [linked data](http://linkeddata.org/) endpoint and it +understands [SPARQL](https://www.w3.org/TR/2013/REC-sparql11-overview-20130321/). + +Apps can also store their own data in the Tracker database, but this feature +isn't widely used yet. The [next major version of +Tracker](https://gitlab.gnome.org/GNOME/tracker/-/milestones/1) aims to bring +improvements in this regard. + +There are several components that make up Tracker: + + * **tracker-store**, which stores the index. + * **tracker-miner-fs**, a daemon which crawls and monitors the filesystem to find content + * **tracker-extract**, a suite of modules to extract metadata and content + from many different types of file. + * the **ontologies**, which define the database schema and the linked data vocabulary. + +## Who uses Tracker? + +### GNOME + +Tracker is a core dependency of the [GNOME desktop](https://www.gnome.org/). + +Gnome's [Shell](https://wiki.gnome.org/Projects/GnomeShell) doesn't use Tracker directly. +Instead, the search results in the shell are provided by multiple apps on the system, +using the [SearchProvider](https://developer.gnome.org/SearchProvider/) API. +Some of these apps use Tracker internally, so they return search results +provided by Tracker to gnome-shell. + +The following GNOME applications use Tracker: + + * [GNOME Books](https://wiki.gnome.org/Apps/Books) (uses Tracker to find ebooks) + * [GNOME Boxes](https://wiki.gnome.org/Apps/Boxes) (uses Tracker to find VM images) + * [GNOME Documents](https://wiki.gnome.org/Apps/Documents) (uses Tracker to find documents) + * [GNOME Files](https://wiki.gnome.org/Apps/Files) (uses Tracker for full-text search within files) + * [GNOME Games](https://wiki.gnome.org/Apps/Games) (uses Tracker to find games) + * [GNOME Music](https://wiki.gnome.org/Apps/Music) (uses Tracker to find music and store playlist data) + * [GNOME Photos](https://wiki.gnome.org/Apps/Photos) (uses Tracker to find photos and store album data) + * [GNOME Videos](https://wiki.gnome.org/Apps/Videos) (uses Tracker to find video content) + +Although Tracker is able to store contacts and calendar entries, +GNOME uses [Evolution Data Server](https://developer.gnome.org/platform-overview/stable/tech-eds.html) +for this. + +### GTK + +The file chooser dialog supplied by GTK has a search interface. There's +a [Tracker backend](https://gitlab.gnome.org/GNOME/gtk/blob/master/gtk/gtksearchenginetracker.c) +for this. + +### Sailfish OS + +[Sailfish OS](https://sailfishos.org) uses Tracker for [indexing media +content](https://sailfishos.org/wiki/Core_Areas_and_APIs). + +## Related projects + +[Xapian](https://xapian.org/) provides similar functionality to Tracker. +It focuses more on scalability and less on having a lightweight footprint. +Unlike Tracker, it doesn't support SPARQL or provide a Linked Data endpoint. + +[Baloo](https://community.kde.org/Baloo) is a metadata and search framework by +KDE, implemented using Xapian. + +[Apache Lucene + Solr](http://lucene.apache.org/) is a search engine which +targets very large-scale workloads. It has a much heavier footprint compared to +Tracker. -- GitLab