Commit 3ef74c82 authored by Øyvind Kolås's avatar Øyvind Kolås

added rev to header and tiles, added sketch of collaborative

* gegl/buffer/gegl-buffer-index.h: added rev to header and tiles,
added sketch of collaborative datastructures.
* gegl/buffer/gegl-buffer-load.c: (gegl_buffer_read_header): made 
it possible to call with offset == NULL
* gegl/buffer/gegl-buffer-save.c: (gegl_tile_entry_new),
(gegl_tile_entry_destroy): do not use slice allocations for entries.
* gegl/buffer/gegl-tile-backend-file.c: (file_entry_destroy),
(get_tile), (set_tile), (flush), (load_index), (file_changed),
(gegl_tile_backend_file_constructor): monitor buffer for changes and
reload index if revision in header has changed.
* gegl/buffer/gegl-tile-backend.h: added private pointer to storage a
backend is member of, allows the backend to send messages through the
storage's bus.
* gegl/buffer/gegl-tile-handler-cache.c: (dispose), (command),
(gegl_tile_handler_cache_invalidate): remove tiles from cache when
they've changed on disk.
* gegl/buffer/gegl-tile-source.h: added macro for
gegl_tile_source_invalidated
* gegl/buffer/gegl-tile-storage.c: (gegl_tile_storage_constructor):
store pointer to storage in backend.

svn path=/trunk/; revision=2250
parent cbe530b9
2008-04-27 Øyvind Kolås <pippin@gimp.org>
* gegl/buffer/gegl-buffer-index.h: added rev to header and tiles,
added sketch of collaborative datastructures.
* gegl/buffer/gegl-buffer-load.c: (gegl_buffer_read_header): made
it possible to call with offset == NULL
* gegl/buffer/gegl-buffer-save.c: (gegl_tile_entry_new),
(gegl_tile_entry_destroy): do not use slice allocations for entries.
* gegl/buffer/gegl-tile-backend-file.c: (file_entry_destroy),
(get_tile), (set_tile), (flush), (load_index), (file_changed),
(gegl_tile_backend_file_constructor): monitor buffer for changes and
reload index if revision in header has changed.
* gegl/buffer/gegl-tile-backend.h: added private pointer to storage a
backend is member of, allows the backend to send messages through the
storage's bus.
* gegl/buffer/gegl-tile-handler-cache.c: (dispose), (command),
(gegl_tile_handler_cache_invalidate): remove tiles from cache when
they've changed on disk.
* gegl/buffer/gegl-tile-source.h: added macro for
gegl_tile_source_invalidated
* gegl/buffer/gegl-tile-storage.c: (gegl_tile_storage_constructor):
store pointer to storage in backend.
2008-04-27 Øyvind Kolås <pippin@gimp.org>
* operations/common/open-buffer.c: added new operation for opening
......
......@@ -13,7 +13,10 @@ GeglBuffer on disk representation
#define GEGL_MAGIC {'G','E','G','L'}
#define GEGL_FLAG_TILE 1
#define GEGL_FLAG_FREE_TILE 2
#define GEGL_FLAG_FREE_TILE 0xf+2
/* a VOID message, indicating that the specified tile has been rewritten */
#define GEGL_FLAG_INVALIDATED 2
/* these flags are used for the header, the lower bits of the
* header store the revision
......@@ -34,28 +37,29 @@ GeglBuffer on disk representation
* This header is the first 256 bytes of the GEGL buffer.
*/
typedef struct {
gchar magic[4]; /* - a 4 byte identifier */
guint32 flags; /* the header flags is used to encode state and revision
gchar magic[4]; /* - a 4 byte identifier */
guint32 flags; /* the header flags is used to encode state and revision
*/
guint64 next; /* offset to first GeglBufferBlock */
guint64 next; /* offset to first GeglBufferBlock */
guint32 tile_width;
guint32 tile_height;
guint16 bytes_per_pixel;
guint32 tile_width;
guint32 tile_height;
guint16 bytes_per_pixel;
gchar description[64]; /* GEGL stores the string of the babl format
* here, as well as after the \0 a debug string
* describing the buffer.
*/
gchar description[64]; /* GEGL stores the string of the babl format
* here, as well as after the \0 a debug string
* describing the buffer.
*/
/* the ROI could come as a separate block */
gint32 x; /* indication of bounding box for tiles stored. */
gint32 y; /* this isn't really needed as each GeglBuffer as */
guint32 width; /* represented on disk doesn't really have any */
guint32 height; /* dimension restriction. */
/* the ROI should come as a separate block */
gint32 x; /* indication of bounding box for tiles stored. */
gint32 y; /* this isn't really needed as each GeglBuffer as */
guint32 width; /* represented on disk doesn't really have any */
guint32 height; /* dimension restriction. */
guint32 rev; /* if it changes on disk it means the index has changed */
guint32 entry_count; /* for integrity check. */
gint32 padding[36]; /* Pad the structure to be 256 bytes long */
gint32 padding[36]; /* Pad the structure to be 256 bytes long */
} GeglBufferHeader;
/* the revision of the format is stored in the flags of the header in the
......@@ -86,14 +90,46 @@ typedef struct {
gint32 y;
gint32 z; /* mipmap subdivision level of tile (0=100%) */
/** used for shared buffers can be ignored for normal use */
guint32 rev; /* revision */
} GeglBufferTile;
/* The following structs are sketches for collaborative use of the b
typedef struct {
GeglBufferBlock block; /* The block definition for this tile entry */
guint32 rev; /* buffer rev this change belongs to */
gint32 x; /* upperleft of tile % tile_width coordinates */
gint32 y;
gint32 z; /* mipmap subdivision level of tile (0=100%) */
} GeglInvalidated;
#define MAX_CLIENTS
typedef struct {
GeglBufferBlock block; /* The block definition for this tile entry */
gchar lock; /* mmap like lock, we poll and sync to get it */
} GeglMaster;
typedef struct {
GeglBufferBlock block; /* The block definition for this tile entry */
guchar id[64]; /* a string identifying the client */
guint32 timestamp;
guint64 invalidated; /* local storage for the client to build it's
* invalidated list
*/
guint32 startreserved;
guint32 endreserved;
} GeglClient;
/* A convenience union to allow quick and simple casting */
typedef union {
guint32 length;
GeglBufferBlock block;
GeglBufferHeader header;
GeglBufferTile tile;
guint32 length;
GeglBufferBlock block;
GeglBufferHeader header;
GeglBufferTile tile;
} GeglBufferItem;
/* functions to initialize data structures */
......@@ -101,6 +137,10 @@ GeglBufferTile * gegl_tile_entry_new (gint x,
gint y,
gint z);
/* intializing the header causes the format to be written out
* as well as a hidden comment after the zero terminated format
* with additional human readable information about the header.
*/
void gegl_buffer_header_init (GeglBufferHeader *header,
gint tile_width,
gint tile_height,
......@@ -109,10 +149,10 @@ void gegl_buffer_header_init (GeglBufferHeader *header,
void gegl_tile_entry_destroy (GeglBufferTile *entry);
GeglBufferItem *gegl_buffer_read_header (GInputStream *i,
goffset *offset);
GList *gegl_buffer_read_index (GInputStream *i,
goffset *offset);
GeglBufferItem *gegl_buffer_read_header(GInputStream *i,
goffset *offset);
GList *gegl_buffer_read_index (GInputStream *i,
goffset *offset);
#define struct_check_padding(type, size) \
if (sizeof (type) != size) \
......
......@@ -101,7 +101,10 @@ GeglBufferItem *
gegl_buffer_read_header (GInputStream *i,
goffset *offset)
{
goffset placeholder;
GeglBufferItem *ret;
if (offset==0)
offset = &placeholder;
if(!g_seekable_seek (G_SEEKABLE (i), 0, G_SEEK_SET, NULL, NULL))
g_warning ("failed seeking to %i", 0);
......
......@@ -64,7 +64,7 @@ gegl_tile_entry_new (gint x,
gint y,
gint z)
{
GeglBufferTile *entry = g_slice_new0 (GeglBufferTile);
GeglBufferTile *entry = g_malloc (sizeof(GeglBufferTile));
entry->block.flags = GEGL_FLAG_TILE;
entry->block.length = sizeof (GeglBufferTile);
......@@ -77,7 +77,7 @@ gegl_tile_entry_new (gint x,
void
gegl_tile_entry_destroy (GeglBufferTile *entry)
{
g_slice_free (GeglBufferTile, entry);
g_free (entry);
}
static gsize write_block (SaveInfo *info,
......
......@@ -31,7 +31,8 @@
#include "gegl-debug.h"
/*#define HACKED_GIO_WITH_READWRITE 1
/*
#define HACKED_GIO_WITH_READWRITE 1
*/
struct _GeglTileBackendFile
......@@ -79,6 +80,14 @@ struct _GeglTileBackendFile
/* loading buffer */
GList *tiles;
/* cooperative sharing of file */
GFileMonitor *monitor; /* Before using mmap we'll use GIO's infrastructure
* for monitoring the file for changes, this should
* also be more portable.
*/
guint32 rev; /* revision of last index sync */
};
static void ensure_exist (GeglTileBackendFile *self);
......@@ -225,7 +234,7 @@ file_entry_destroy (GeglBufferTile *entry,
g_hash_table_remove (self->index, entry);
dbg_dealloc (GEGL_TILE_BACKEND (self)->tile_size);
g_slice_free (GeglBufferTile, entry);
g_free (entry);
}
static gboolean write_header (GeglTileBackendFile *self)
......@@ -381,8 +390,8 @@ get_tile (GeglTileSource *self,
return NULL;
tile = gegl_tile_new (backend->tile_size);
tile->stored_rev = 1;
tile->rev = 1;
tile->rev = entry->rev;
tile->stored_rev = entry->rev;
file_entry_read (tile_backend_file, entry, tile->data);
return tile;
......@@ -411,6 +420,7 @@ set_tile (GeglTileSource *self,
entry->z = z;
g_hash_table_insert (tile_backend_file->index, entry, entry);
}
entry->rev = tile->rev;
file_entry_write (tile_backend_file, entry, tile->data);
tile->stored_rev = tile->rev;
......@@ -447,13 +457,13 @@ exist_tile (GeglTileSource *self,
gint y,
gint z)
{
GeglTileBackend *backend;
GeglTileBackend *backend;
GeglTileBackendFile *tile_backend_file;
GeglBufferTile *entry;
GeglBufferTile *entry;
backend = GEGL_TILE_BACKEND (self);
backend = GEGL_TILE_BACKEND (self);
tile_backend_file = GEGL_TILE_BACKEND_FILE (backend);
entry = lookup_entry (tile_backend_file, x, y, z);
entry = lookup_entry (tile_backend_file, x, y, z);
return entry!=NULL?((gpointer)0x1):NULL;
}
......@@ -479,6 +489,7 @@ flush (GeglTileSource *source,
GEGL_NOTE (TILE_BACKEND, "flushing %s", self->path);
self->header.rev ++;
self->header.next = self->next_pre_alloc; /* this is the offset
we start handing
out headers from*/
......@@ -665,6 +676,88 @@ equalfunc (gconstpointer a,
return FALSE;
}
static void load_index (GeglTileBackendFile *self)
{
GeglBufferHeader new_header;
GList *iter;
GeglTileBackend *backend;
goffset offset;
goffset max=0;
/* compute total from and next pre alloc by monitoring tiles as they
* are added here
*/
/* reload header */
new_header = gegl_buffer_read_header (self->i, &offset)->header;
if (new_header.rev == self->header.rev)
{
GEGL_NOTE(TILE_BACKEND, "header not changed: %s", self->path);
return;
}
else
{
self->header=new_header;
GEGL_NOTE(TILE_BACKEND, "loading index: %s", self->path);
}
offset = self->header.next;
self->tiles = gegl_buffer_read_index (self->i, &offset);
backend = GEGL_TILE_BACKEND (self);
for (iter = self->tiles; iter; iter=iter->next)
{
GeglBufferItem *item = iter->data;
GeglBufferItem *existing = g_hash_table_lookup (self->index, item);
if (item->tile.offset > max)
max = item->tile.offset + backend->tile_size;
if (existing)
{
if (existing->tile.rev == item->tile.rev)
{
g_assert (existing->tile.offset == item->tile.offset);
existing->tile = item->tile;
g_free (item);
continue;
}
else
{
g_hash_table_remove (self->index, existing);
g_free (existing);
gegl_tile_source_invalidated (GEGL_TILE_SOURCE (backend->storage),
existing->tile.x,
existing->tile.y,
existing->tile.z);
}
}
g_hash_table_insert (self->index, iter->data, iter->data);
}
g_list_free (self->tiles);
g_slist_free (self->free_list);
self->free_list = NULL;
self->next_pre_alloc = max; /* if bigger than own? */
self->total = max;
self->tiles = NULL;
}
static void file_changed (GFileMonitor *monitor,
GFile *file,
GFile *other_file,
GFileMonitorEvent event_type,
gpointer user_data)
{
GeglTileBackendFile *self = GEGL_TILE_BACKEND_FILE (user_data);
if (event_type == G_FILE_MONITOR_EVENT_CHANGES_DONE_HINT)
/*if (event_type == G_FILE_MONITOR_EVENT_CHANGED)*/
{
load_index (self);
}
}
static GObject *
gegl_tile_backend_file_constructor (GType type,
guint n_params,
......@@ -674,59 +767,44 @@ gegl_tile_backend_file_constructor (GType type,
GeglTileBackendFile *self;
GeglTileBackend *backend;
object = G_OBJECT_CLASS (parent_class)->constructor (type, n_params, params);
self = GEGL_TILE_BACKEND_FILE (object);
object = G_OBJECT_CLASS (parent_class)->constructor (type, n_params, params);
self = GEGL_TILE_BACKEND_FILE (object);
backend = GEGL_TILE_BACKEND (object);
GEGL_NOTE (TILE_BACKEND, "constructing file backend: %s", self->path);
self->file = g_file_new_for_commandline_arg (self->path);
self->monitor = g_file_monitor_file (self->file, G_FILE_MONITOR_NONE,
NULL, NULL);
g_signal_connect (self->monitor, "changed", G_CALLBACK(file_changed), self);
self->index = g_hash_table_new (hashfunc, equalfunc);
/* if the file already exist we try to open it for appending instead of replacing */
if (g_file_query_exists (self->file, NULL))
{
goffset offset;
goffset offset=0;
#ifdef HACKED_GIO_WITH_READWRITE
self->o = G_OUTPUT_STREAM (g_file_append_to (self->file, G_FILE_CREATE_READWRITE, NULL, NULL));
self->i = g_object_get_data (G_OBJECT (self->o), "istream");
#else
g_error ("not able to open a file readwrite properly with gio");
/* don't know how to deal with this properly with normal GIO */
self->i = G_INPUT_STREAM (g_file_read (self->file, NULL, NULL));
#endif
/*self->i = G_INPUT_STREAM (g_file_read (self->file, NULL, NULL));*/
self->header = gegl_buffer_read_header (self->i, &offset)->header;
self->header.rev = self->header.rev -1;
/* we are overriding all of the work of the actual constructor here */
backend->tile_width = self->header.tile_width;
backend->tile_height = self->header.tile_height;
backend->format = babl_format (self->header.description);
/* we are overriding all of the work of the actual constructor here */
backend->px_size = backend->format->format.bytes_per_pixel;
backend->tile_size = backend->tile_width * backend->tile_height * backend->px_size;
offset = self->header.next;
self->tiles = gegl_buffer_read_index (self->i, &offset);
/* insert each of the entries into the hash table */
{
/* compute total from and next pre alloc by monitoring tiles as they
* are added here
*/
goffset max=0;
GList *iter;
for (iter = self->tiles; iter; iter=iter->next)
{
GeglBufferItem *item = iter->data;
if (item->tile.offset > max)
max = item->tile.offset + backend->tile_size;
g_hash_table_insert (self->index, iter->data, iter->data);
}
g_list_free (self->tiles);
self->next_pre_alloc = max;
self->total = max;
self->tiles = NULL;
}
load_index (self);
self->exist = TRUE;
g_assert (self->i);
g_assert (self->o);
......@@ -738,7 +816,6 @@ gegl_tile_backend_file_constructor (GType type,
g_assert (self->file);
backend->header = &self->header;
return object;
......
......@@ -42,6 +42,7 @@ struct _GeglTileBackend
/* private */
gpointer header;
gpointer storage;
};
struct _GeglTileBackendClass
......
......@@ -35,7 +35,6 @@ static gint cache_hits = 0;
static gint cache_misses = 0;
static gint cache_total = 0; /* approximate amount of bytes stored */
static gint clones_ones = 0; /* approximate amount of bytes stored */
struct _GeglTileHandlerCache
{
......@@ -76,6 +75,10 @@ static void gegl_tile_handler_cache_void (GeglTileHandlerCache *cache
gint x,
gint y,
gint z);
static void gegl_tile_handler_cache_invalidate (GeglTileHandlerCache *cache,
gint x,
gint y,
gint z);
G_DEFINE_TYPE (GeglTileHandlerCache, gegl_tile_handler_cache, GEGL_TYPE_TILE_HANDLER)
......@@ -130,7 +133,6 @@ dispose (GObject *object)
if (item->tile)
{
cache_total -= item->tile->size;
clones_ones = 0; /* XXX */
g_object_unref (item->tile);
}
g_queue_remove (cache_queue, item);
......@@ -225,11 +227,12 @@ command (GeglTileSource *tile_store,
return (gpointer)action;
break;
}
case GEGL_TILE_INVALIDATED:
gegl_tile_handler_cache_invalidate (cache, x, y, z);
break;
case GEGL_TILE_VOID:
gegl_tile_handler_cache_void (cache, x, y, z);
/*if (z!=0)
return (void*)0xdead700;*/
/* fallthrough */
break;
default:
break;
}
......@@ -355,6 +358,36 @@ gegl_tile_handler_cache_trim (GeglTileHandlerCache *cache)
return FALSE;
}
static void
gegl_tile_handler_cache_invalidate (GeglTileHandlerCache *cache,
gint x,
gint y,
gint z)
{
GList *link;
for (link = g_queue_peek_head_link (cache_queue); link; link = link->next)
{
CacheItem *item = link->data;
GeglTile *tile = item->tile;
if (tile != NULL &&
item->x == x &&
item->y == y &&
item->z == z &&
item->handler == cache)
{
cache_total -= item->tile->size;
tile->tile_storage = NULL;
tile->stored_rev = tile->rev; /* to cheat it out of being stored */
g_object_unref (tile);
g_slice_free (CacheItem, item);
g_queue_delete_link (cache_queue, link);
return;
}
}
}
static void
gegl_tile_handler_cache_void (GeglTileHandlerCache *cache,
......
......@@ -44,7 +44,7 @@ enum _GeglTileCommand
GEGL_TILE_EXIST,
GEGL_TILE_VOID,
GEGL_TILE_FLUSH,
GEGL_TILE_INVALIDATED,
GEGL_TILE_INVALIDATED, /* command sent by some backends through storage*/
GEGL_TILE_LAST_COMMAND
};
......@@ -94,20 +94,8 @@ gpointer gegl_tile_source_command (GeglTileSource *gegl_tile_source,
#define gegl_tile_source_void(source,x,y,z) \
gegl_tile_source_command(source,GEGL_TILE_VOID,x,y,z,NULL)
#define gegl_tile_source_void_tl(source,x,y,z) \
gegl_tile_source_command(source,GEGL_TILE_VOID_TL,x,y,z,NULL)
#define gegl_tile_source_void_tr(source,x,y,z) \
gegl_tile_source_command(source,GEGL_TILE_VOID_TR,x,y,z,NULL)
#define gegl_tile_source_void_bl(source,x,y,z) \
gegl_tile_source_command(source,GEGL_TILE_VOID_BL,x,y,z,NULL)
#define gegl_tile_source_void_br(source,x,y,z) \
gegl_tile_source_command(source,GEGL_TILE_VOID_BR,x,y,z,NULL)
#define gegl_tile_source_undo_start_group(source,x,y,z) \
gegl_tile_source_command(source,GEGL_TILE_UNDO_START_GROUP,x,y,z,NULL)
#define gegl_tile_source_invalidated(source,x,y,z) \
gegl_tile_source_command(source,GEGL_TILE_INVALIDATED,x,y,z,NULL)
G_END_DECLS
......
......@@ -31,6 +31,7 @@
#include "gegl-tile-handler-log.h"
G_DEFINE_TYPE (GeglTileStorage, gegl_tile_storage, GEGL_TYPE_TILE_HANDLER_CHAIN)
#define TILE_WIDTH 128
......@@ -165,6 +166,7 @@ tile_storage_idle (gpointer data)
return TRUE;
}
GeglTileBackend *gegl_buffer_backend (GObject *buffer);
static GObject *
gegl_tile_storage_constructor (GType type,
......@@ -252,6 +254,11 @@ gegl_tile_storage_constructor (GType type,
g_object_set_data (G_OBJECT (empty), "cache", cache);
g_object_set_data (G_OBJECT (zoom), "cache", cache);
{
GeglTileBackend *backend;
backend = gegl_buffer_backend (object);
backend->storage = (gpointer)object;
}
tile_storage->idle_swapper = g_timeout_add_full (G_PRIORITY_LOW,
250,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment