A while back I wanted to open this huge satellite image I downloaded from
NASA, and the Gimp required more than 2Gb of swap space. It didn't work, so I
wrote a new tile-swap.c which makes additional swap files once the current one
hits 1Gb. Also, the following diff needs to be aplpied:
--- gimp-1.3.7/app/base/tile-private.h 2001-12-03 05:44:50.000000000 -0800
+++ gimp-1.3.7-tiles/app/base/tile-private.h 2002-08-03 18:59:37.000000000 -0
700
@@ -76,7 +76,7 @@
* for swapping. swap_num 1 is always the global
* swap file.
*/
- off_t swap_offset; /* the offset within the swap file of the tile data.
+ guint64 swap_offset;/* the offset within the swap file of the tile data.
* if the tile data is in memory this will be set to -1.
*/
TileLink *tlink;
This is against 1.3.7, but should trivially apply to 1.3.8 since it's just the
wholesale swapping out of a file.
Please direct any comments, flames, etc, my way.
Tim
--
Tim Newsome nuisance@xxxxxxxxxxxxxxxx http://www.wiw.org/~drz/
I'd like to hear some funky Dixieland / Pretty mama come and take me by the hand
/* The GIMP -- an image manipulation program
* Copyright (C) 1995 Spencer Kimball and Peter Mattis
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "config.h"
#include <stdio.h> /* SEEK_SET */
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <string.h>
#include <time.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef USE_PTHREADS
#include <pthread.h>
#endif
#include <glib-object.h>
#ifdef G_OS_WIN32
#include <io.h>
#endif
#include "base-types.h"
#include "tile.h"
#include "tile-private.h"
#include "tile-swap.h"
/* The max size of a cache file. */
#define MAX_FILE_SIZE (1024 * 1024 * 1024)
/* The blocksize we use to allocate tiles in. Tile sizes get rounded up to the
* nearest multiple of this number. */
#define TILE_SIZE_INCREMENT 4096
/* Amount of files to keep open at one time. */
#define FILE_CACHE_SIZE 3
#undef SWAP_DEBUG
typedef struct _FreelistEntry {
guint64 offset;
guint64 size;
struct _FreelistEntry *next;
} FreelistEntry;
typedef struct {
guint file_number;
gint fd;
guint last_used;
} FileCacheEntry;
static FileCacheEntry file_cache[FILE_CACHE_SIZE];
static guint file_cache_hits = 0;
static guint file_cache_misses = 0;
typedef struct {
gchar *path;
FreelistEntry *freelist;
guint max_file;
guint last_used;
} SwapDir;
static SwapDir swap_dir;
#ifdef SWAP_DEBUG
static void print_freelist(void)
{
FreelistEntry *entry;
entry = swap_dir.freelist;
g_printerr("Freelist:\n");
while (entry) {
g_printerr("\t%llx -- %llx\n", entry->offset,
entry->offset + entry->size);
entry = entry->next;
}
}
#endif
/* Remove the directory/tile files. */
void tile_swap_exit(void)
{
FreelistEntry *entry, *next;
guchar *filename;
gint file;
int e;
#ifdef SWAP_DEBUG
g_printerr("tile_swap_exit()\n");
g_printerr("file cache hits: %.2f%%\n",
100. * file_cache_hits / (file_cache_hits + file_cache_misses));
#endif
for (file = 0; file < FILE_CACHE_SIZE; file++)
if (file_cache[file].fd >= 0) {
close(file_cache[file].fd);
file_cache[file].fd = -1;
}
/* free the freelist */
entry = swap_dir.freelist;
while (entry) {
next = entry->next;
g_free(entry);
entry = next;
}
swap_dir.freelist = NULL;
filename = g_new(guchar, strlen(swap_dir.path) + 50);
for (file = 0; file <= swap_dir.max_file; file++) {
strcpy(filename, swap_dir.path);
sprintf(filename + strlen(filename), "/%08x.cache", file);
#ifdef SWAP_DEBUG
g_printerr("unlink(%s)\n", filename);
#endif
e = unlink(filename);
if (e != 0) {
g_printerr("%s: unlink(%s)\n", strerror(e), filename);
}
}
g_free(filename);
/* remove the directory */
e = rmdir(swap_dir.path);
if (e) {
g_printerr("%s: rmdir(%s)\n", strerror(e), swap_dir.path);
}
}
gint tile_swap_add(gchar *filename, SwapFunc swap_func, gpointer user_data)
{
int e, i;
#ifdef SWAP_DEBUG
g_printerr("multiple-tiles-per-file swap algorithm\n");
g_printerr ("tile_swap_add(filename=%s, swap_func=%p, user_data=%p)\n",
filename, swap_func, user_data);
#endif
swap_dir.path = g_strdup (filename);
swap_dir.freelist = g_new(FreelistEntry, 1);
swap_dir.freelist->offset = 0;
swap_dir.freelist->size = ~(guint64) 0;
swap_dir.freelist->next = NULL;
swap_dir.max_file = 0;
swap_dir.last_used = 0;
for (i = 0; i < FILE_CACHE_SIZE; i++) {
file_cache[i].fd = -1;
file_cache[i].last_used = 0;
}
/* now create the directory */
e = mkdir (swap_dir.path, 0777);
if (e) {
g_printerr ("%s: mkdir(%s, 0777)\n", strerror(e), swap_dir.path);
return 0;
}
return 1;
}
void tile_swap_remove(gint swap_num)
{
#ifdef SWAP_DEBUG
g_printerr("tile_swap_remove(swap_num=%d)\n", swap_num);
#endif
}
void tile_swap_in_async(Tile *tile)
{
#ifdef SWAP_DEBUG
g_printerr("tile_swap_in_async(tile=%p)\n", tile);
#endif
}
static guchar *tile_filename(Tile *tile)
{
guchar *filename;
/* RAM is cheap. */
filename = g_new(guchar, strlen(swap_dir.path) + 50);
strcpy(filename, swap_dir.path);
sprintf(filename + strlen(filename), "/%08x.cache",
(unsigned int) (tile->swap_offset / MAX_FILE_SIZE));
swap_dir.max_file = MAX(swap_dir.max_file,
(unsigned int) (tile->swap_offset / MAX_FILE_SIZE));
return filename;
}
static gint tile_offset(Tile *tile)
{
return tile->swap_offset % MAX_FILE_SIZE;
}
static guint allocate_size(Tile *tile)
{
return tile_size(tile);
}
static void allocate_tile(Tile *tile)
{
FreelistEntry *entry, *previous, *new_entry;
guint bytes, first_chunk, second_chunk;
if (tile->swap_offset == -1) {
bytes = allocate_size(tile);
/* Find space on the freelist. There has to be space, and we can't
* cross a file boundary. */
previous = NULL;
entry = swap_dir.freelist;
while (entry) {
first_chunk = MAX_FILE_SIZE - (entry->offset % MAX_FILE_SIZE);
if (first_chunk >= entry->size) {
first_chunk = entry->size;
second_chunk = 0;
} else {
second_chunk = entry->size - first_chunk;
}
/*
g_printerr("first_chunk=%x, second_chunk=%x\n",
first_chunk, second_chunk);
*/
if (first_chunk >= bytes) {
tile->swap_offset = entry->offset;
if (entry->size == bytes) {
/* remove this entry */
if (previous)
previous->next = entry->next;
else
swap_dir.freelist = entry->next;
g_free(entry);
} else {
entry->offset += bytes;
entry->size -= bytes;
}
break;
} else if (second_chunk >= bytes) {
/* need to break the current entry up */
tile->swap_offset = entry->offset + first_chunk;
if (second_chunk == bytes) {
/* actually, just make the entry smaller */
entry->size -= bytes;
} else {
new_entry = g_new(FreelistEntry, 1);
new_entry->offset = tile->swap_offset + bytes;
new_entry->size = second_chunk - bytes;
new_entry->next = NULL;
entry->size = first_chunk;
entry->next = new_entry;
}
break;
}
previous = entry;
entry = entry->next;
}
#ifdef SWAP_DEBUG
print_freelist();
#endif
if (tile->swap_offset == -1) {
g_printerr("BAD ERROR: Couldn't allocate swap space for a tile.\n");
}
}
}
static gint tile_open_file(Tile *tile)
{
char *filename = NULL;
gint i, lru = 0;
allocate_tile(tile);
if (swap_dir.last_used > (1<<30)) {
swap_dir.last_used /= 16;
for (i = 0; i < FILE_CACHE_SIZE; i++)
file_cache[i].last_used /= 16;
}
/* see if we've got this file open */
for (i = 0; i < FILE_CACHE_SIZE; i++) {
if (file_cache[i].fd >= 0 &&
file_cache[i].file_number ==
(guint) (tile->swap_offset / MAX_FILE_SIZE)) {
file_cache[i].last_used = swap_dir.last_used++;
lseek(file_cache[i].fd, tile_offset(tile), SEEK_SET);
file_cache_hits++;
return file_cache[i].fd;
}
if (file_cache[i].last_used < file_cache[lru].last_used)
lru = i;
}
file_cache_misses++;
if (file_cache[lru].fd >= 0)
close(file_cache[lru].fd);
filename = tile_filename(tile);
#ifdef SWAP_DEBUG
g_printerr("Opening %s for cache slot %d.\n", filename, lru);
#endif
file_cache[lru].last_used = swap_dir.last_used++;
file_cache[lru].file_number = (tile->swap_offset / MAX_FILE_SIZE);
file_cache[lru].fd = open(filename, O_RDWR);
if (file_cache[lru].fd == -1) {
/* not yet in cache. create it. */
file_cache[lru].fd =
open(filename, O_CREAT | O_RDWR, S_IREAD | S_IWRITE);
if (file_cache[lru].fd == -1)
g_printerr("%s: open(%s, O_CREAT ...)\n", strerror(errno),
filename);
}
g_free(filename);
lseek(file_cache[lru].fd, tile_offset(tile), SEEK_SET);
return file_cache[lru].fd;
}
void tile_swap_in(Tile *tile)
{
gint fd;
gint bytes_read = 0, bytes, ret;
#ifdef SWAP_DEBUG
g_printerr("tile_swap_in(swap_num=%d, swap_offset=0x%llx, size=0x%x)\n",
tile->swap_num, tile->swap_offset, tile_size(tile));
#endif
if (tile->data) {
g_printerr("tile_swap_in(): tile already has data allocated\n");
return;
}
tile_alloc(tile);
if (tile->swap_offset == -1)
/* It's not in cache. Just return some uninitialized data. */
return;
fd = tile_open_file(tile);
if (fd == -1)
/* This is an error. */
return;
bytes = tile_size(tile);
while (bytes_read < bytes) {
ret = read(fd, tile->data + bytes_read, bytes - bytes_read);
if (ret == -1) {
g_printerr("%s: reading %d bytes from tile cache\n",
strerror(errno), bytes - bytes_read);
break;
}
bytes_read += ret;
}
/* tile_close_file(tile, fd); */
}
void tile_swap_out (Tile *tile)
{
gint bytes;
gint written = 0;
gint ret;
gint fd;
#ifdef SWAP_DEBUG
g_printerr("tile_swap_out(swap_num=%d, swap_offset=0x%llx, size=0x%x)\n",
tile->swap_num, tile->swap_offset, tile_size(tile));
#endif
fd = tile_open_file(tile);
if (fd == -1)
/* This is an error. */
return;
bytes = tile_size(tile);
while (written < bytes) {
ret = write(fd, tile->data + written, bytes - written);
if (ret == -1) {
g_printerr("%s: writing %d bytes to tile cache\n",
strerror(errno), bytes - written);
break;
}
written += ret;
}
/* tile_close_file(tile, fd); */
tile->dirty = FALSE;
}
void tile_swap_delete(Tile *tile)
{
FreelistEntry *entry = NULL, *left = NULL, *right = NULL;
gint bytes;
#ifdef SWAP_DEBUG
g_printerr("tile_swap_delete(swap_num=%d, swap_offset=0x%llx, size=0x%x)\n",
tile->swap_num, tile->swap_offset, tile_size(tile));
#endif
bytes = allocate_size(tile);
/* add it to the freelist */
/* find out where it should be added */
left = NULL;
right = swap_dir.freelist;
while (right && right->offset < tile->swap_offset) {
left = right;
right = right->next;
}
if (left && left->offset + left->size == tile->swap_offset) {
/* new fits snug next to left */
if (right && tile->swap_offset + bytes == right->offset) {
/* new fits snug next to right */
left->size += bytes + right->size;
left->next = right->next;
g_free(right);
} else {
/* new doesn't fit snug next to right */
left->size += bytes;
}
} else {
/* new doesn't fit snug next to left */
if (right && tile->swap_offset + bytes == right->offset) {
/* new fits snug next to right */
right->offset -= bytes;
right->size += bytes;
} else {
/* new doesn't fit snug next to right */
entry = g_new(FreelistEntry, 1);
entry->offset = tile->swap_offset;
entry->size = bytes;
entry->next = right;
if (left)
left->next = entry;
else
swap_dir.freelist = entry;
}
}
#ifdef SWAP_DEBUG
print_freelist();
#endif
}
void tile_swap_compress (gint swap_num)
{
#ifdef SWAP_DEBUG
g_printerr("tile_swap_compress(swap_num=%d)\n", swap_num);
#endif
}
