Author: grothoff
Date: 2008-02-25 00:06:55 -0700 (Mon, 25 Feb 2008)
New Revision: 6452
Modified:
GNUnet/ChangeLog
GNUnet/src/applications/fs/ecrs/download.c
GNUnet/src/applications/fs/fsui/download.c
GNUnet/src/applications/fs/fsui/downloadtest.c
GNUnet/src/applications/fs/fsui/fsui.h
GNUnet/src/applications/fs/fsui/serializetest.c
GNUnet/src/applications/fs/fsui/serializetest2.c
GNUnet/src/applications/fs/fsui/serializetest3.c
GNUnet/src/applications/fs/fsui/serializetest4.c
GNUnet/src/applications/fs/tools/gnunet-insert.c
GNUnet/src/include/gnunet_ecrs_lib.h
GNUnet/src/util/disk/storage.c
GNUnet/todo
Log:
use lstat
Modified: GNUnet/ChangeLog
===================================================================
--- GNUnet/ChangeLog 2008-02-25 06:12:45 UTC (rev 6451)
+++ GNUnet/ChangeLog 2008-02-25 07:06:55 UTC (rev 6452)
@@ -1,3 +1,9 @@
+Mon Feb 25 00:01:27 MST 2008
+ Added asynchronous search and download methods for
+ ECRS library. FSUI now can do with only one thread
+ per search or download (until now, we had two
+ threads per search / download).
+
Tue Feb 19 20:35:28 MST 2008
Updated database schemata to support O(1) operations
even if there are N files under the same keywords.
Modified: GNUnet/src/applications/fs/ecrs/download.c
===================================================================
--- GNUnet/src/applications/fs/ecrs/download.c 2008-02-25 06:12:45 UTC (rev
6451)
+++ GNUnet/src/applications/fs/ecrs/download.c 2008-02-25 07:06:55 UTC (rev
6452)
@@ -44,7 +44,7 @@
* Pointer to shared data between all nodes (request manager,
* progress data, etc.).
*/
- struct RequestManager *ctx;
+ struct GNUNET_ECRS_DownloadContext *ctx;
/**
* Previous entry in DLL.
@@ -82,7 +82,7 @@
* which queries went out with which priorities and which nodes in
* the merkle-tree are waiting for the replies.
*/
-struct RequestManager
+struct GNUNET_ECRS_DownloadContext
{
/**
@@ -208,7 +208,7 @@
* is not complete and may be resumed later.
*/
static void
-free_request_manager (struct RequestManager *rm, int unlinkTreeFiles)
+free_request_manager (struct GNUNET_ECRS_DownloadContext *rm, int
unlinkTreeFiles)
{
int i;
char *fn;
@@ -261,6 +261,7 @@
}
GNUNET_free_non_null (rm->filename);
GNUNET_free_non_null (rm->handles);
+ GNUNET_free(rm);
}
/**
@@ -274,7 +275,7 @@
* @return number of bytes read, GNUNET_SYSERR on error
*/
static int
-read_from_files (struct RequestManager *this,
+read_from_files (struct GNUNET_ECRS_DownloadContext *this,
unsigned int level,
unsigned long long pos, void *buf, unsigned int len)
{
@@ -304,7 +305,7 @@
* @return number of bytes written, GNUNET_SYSERR on error
*/
static int
-write_to_files (struct RequestManager *this,
+write_to_files (struct GNUNET_ECRS_DownloadContext *this,
unsigned int level,
unsigned long long pos, void *buf, unsigned int len)
{
@@ -338,7 +339,7 @@
static void
addRequest (struct Node *node)
{
- struct RequestManager *rm = node->ctx;
+ struct GNUNET_ECRS_DownloadContext *rm = node->ctx;
node->next = rm->head;
if (node->next != NULL)
@@ -356,9 +357,14 @@
}
static void
-signal_abort (struct RequestManager *rm)
+signal_abort (struct GNUNET_ECRS_DownloadContext *rm,
+ const char * msg)
{
rm->abortFlag = GNUNET_YES;
+ if ( (rm->head != NULL) &&
+ (rm->dpcb != NULL) )
+ rm->dpcb (rm->length+1,
+ 0, 0, 0, msg, 0, rm->dpcbClosure);
GNUNET_thread_stop_sleep (rm->main);
}
@@ -371,7 +377,7 @@
static void
delete_node (struct Node *node)
{
- struct RequestManager *rm = node->ctx;
+ struct GNUNET_ECRS_DownloadContext *rm = node->ctx;
if (node->prev == NULL)
rm->head = node->next;
@@ -383,7 +389,7 @@
node->next->prev = node->prev;
GNUNET_free (node);
if (rm->head == NULL)
- signal_abort (rm);
+ signal_abort (rm, NULL);
}
/**
@@ -439,7 +445,7 @@
notify_client_about_progress (const struct Node *node, const char *data,
unsigned int size)
{
- struct RequestManager *rm = node->ctx;
+ struct GNUNET_ECRS_DownloadContext *rm = node->ctx;
GNUNET_CronTime eta;
if ((rm->abortFlag == GNUNET_YES) || (node->level != 0))
@@ -610,7 +616,7 @@
unsigned long long uid)
{
struct Node *node = cls;
- struct RequestManager *rm = node->ctx;
+ struct GNUNET_ECRS_DownloadContext *rm = node->ctx;
struct GNUNET_GE_Context *ectx = rm->ectx;
GNUNET_HashCode hc;
unsigned int size;
@@ -643,11 +649,10 @@
{
GNUNET_free (data);
GNUNET_GE_BREAK (ectx, 0);
- GNUNET_GE_LOG (ectx, GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
- _("Decrypted content does not match key. "
- "This is either a bug or a maliciously inserted "
- "file. Download aborted.\n"));
- signal_abort (rm);
+ signal_abort (rm,
+ _("Decrypted content does not match key. "
+ "This is either a bug or a maliciously inserted "
+ "file. Download aborted.\n"));
GNUNET_mutex_unlock (rm->lock);
return GNUNET_SYSERR;
}
@@ -656,7 +661,7 @@
GNUNET_GE_LOG_STRERROR (ectx,
GNUNET_GE_ERROR | GNUNET_GE_ADMIN |
GNUNET_GE_USER | GNUNET_GE_BULK, "WRITE");
- signal_abort (rm);
+ signal_abort (rm, _("IO error."));
GNUNET_mutex_unlock (rm->lock);
return GNUNET_SYSERR;
}
@@ -727,6 +732,7 @@
/* ***************** main method **************** */
+
/**
* Download parts of a file. Note that this will store
* the blocks at the respective offset in the given file.
@@ -746,177 +752,169 @@
* @param start starting offset
* @param length length of the download (starting at offset)
*/
-int
-GNUNET_ECRS_file_download_partial (struct GNUNET_GE_Context *ectx,
- struct GNUNET_GC_Configuration *cfg,
- const struct GNUNET_ECRS_URI *uri,
- const char *filename,
- unsigned long long offset,
- unsigned long long length,
- unsigned int anonymityLevel,
- int no_temporaries,
- GNUNET_ECRS_DownloadProgressCallback dpcb,
- void *dpcbClosure,
- GNUNET_ECRS_TestTerminate tt,
- void *ttClosure)
+struct GNUNET_ECRS_DownloadContext *
+GNUNET_ECRS_file_download_partial_start (struct GNUNET_GE_Context *ectx,
+ struct GNUNET_GC_Configuration *cfg,
+ const struct GNUNET_ECRS_URI *uri,
+ const char *filename,
+ unsigned long long offset,
+ unsigned long long length,
+ unsigned int anonymityLevel,
+ int no_temporaries,
+ GNUNET_ECRS_DownloadProgressCallback
dpcb,
+ void *dpcbClosure)
{
- struct RequestManager rm;
+ struct GNUNET_ECRS_DownloadContext * rm;
struct stat buf;
struct Node *top;
char *fn;
- char *rdir;
int ret;
int i;
- int len;
if ((!GNUNET_ECRS_uri_test_chk (uri)) && (!GNUNET_ECRS_uri_test_loc (uri)))
{
GNUNET_GE_BREAK (ectx, 0);
- return GNUNET_SYSERR;
+ return NULL;
}
+ rm = GNUNET_malloc(sizeof (struct GNUNET_ECRS_DownloadContext));
+ memset (rm, 0,
+ sizeof (struct GNUNET_ECRS_DownloadContext));
+ rm->ectx = ectx;
+ rm->cfg = cfg;
+ rm->startTime = GNUNET_get_time ();
+ rm->anonymityLevel = anonymityLevel;
+ rm->offset = offset;
+ rm->length = length;
+ rm->dpcb = dpcb;
+ rm->dpcbClosure = dpcbClosure;
+ rm->main = GNUNET_thread_get_self ();
+ rm->total = GNUNET_ntohll (uri->data.fi.file_length);
+ rm->filename = get_real_download_filename (ectx, filename);
- memset (&rm, 0, sizeof (struct RequestManager));
- rm.ectx = ectx;
- rm.cfg = cfg;
- rm.startTime = GNUNET_get_time ();
- rm.anonymityLevel = anonymityLevel;
- rm.offset = offset;
- rm.length = length;
- rm.dpcb = dpcb;
- rm.dpcbClosure = dpcbClosure;
- rm.main = GNUNET_thread_get_self ();
- rm.total = GNUNET_ntohll (uri->data.fi.file_length);
- rm.filename = get_real_download_filename (ectx, filename);
-
if (GNUNET_SYSERR ==
- GNUNET_disk_directory_create_for_file (ectx, rm.filename))
+ GNUNET_disk_directory_create_for_file (ectx, rm->filename))
{
- free_request_manager (&rm, GNUNET_NO);
- return GNUNET_SYSERR;
+ free_request_manager (rm, GNUNET_NO);
+ return NULL;
}
- if (0 == rm.total)
+ if (0 == rm->total)
{
ret = GNUNET_disk_file_open (ectx,
- rm.filename,
+ rm->filename,
O_CREAT | O_WRONLY | O_TRUNC,
S_IRUSR | S_IWUSR);
if (ret == -1)
{
- free_request_manager (&rm, GNUNET_NO);
- return GNUNET_SYSERR;
+ free_request_manager (rm, GNUNET_NO);
+ return NULL;
}
CLOSE (ret);
- dpcb (0, 0, rm.startTime, 0, NULL, 0, dpcbClosure);
- free_request_manager (&rm, GNUNET_NO);
- return GNUNET_OK;
+ dpcb (0, 0, rm->startTime, 0, NULL, 0, dpcbClosure);
+ free_request_manager (rm, GNUNET_NO);
+ return NULL;
}
- rm.treedepth = GNUNET_ECRS_compute_depth (rm.total);
- rm.handles = GNUNET_malloc (sizeof (int) * (rm.treedepth + 1));
- for (i = 0; i <= rm.treedepth; i++)
- rm.handles[i] = -1;
- if ((0 == STAT (rm.filename, &buf)) && ((size_t) buf.st_size > rm.total))
+ rm->treedepth = GNUNET_ECRS_compute_depth (rm->total);
+ rm->handles = GNUNET_malloc (sizeof (int) * (rm->treedepth + 1));
+ for (i = 0; i <= rm->treedepth; i++)
+ rm->handles[i] = -1;
+ if ((0 == STAT (rm->filename, &buf)) && ((size_t) buf.st_size > rm->total))
{
/* if exists and oversized, truncate */
- if (truncate (rm.filename, rm.total) != 0)
+ if (truncate (rm->filename, rm->total) != 0)
{
GNUNET_GE_LOG_STRERROR_FILE (ectx,
GNUNET_GE_ERROR | GNUNET_GE_ADMIN |
GNUNET_GE_BULK, "truncate",
- rm.filename);
- free_request_manager (&rm, GNUNET_NO);
- return GNUNET_SYSERR;
+ rm->filename);
+ free_request_manager (rm, GNUNET_NO);
+ return NULL;
}
}
- for (i = 0; i <= rm.treedepth; i++)
+ for (i = 0; i <= rm->treedepth; i++)
{
if ((i == 0) || (no_temporaries != GNUNET_YES))
{
- fn = GNUNET_malloc (strlen (rm.filename) + 3);
- strcpy (fn, rm.filename);
+ fn = GNUNET_malloc (strlen (rm->filename) + 3);
+ strcpy (fn, rm->filename);
if (i > 0)
{
strcat (fn, ".A");
fn[strlen (fn) - 1] += i;
}
- rm.handles[i] = GNUNET_disk_file_open (ectx,
+ rm->handles[i] = GNUNET_disk_file_open (ectx,
fn,
O_CREAT | O_RDWR,
S_IRUSR | S_IWUSR);
GNUNET_free (fn);
- if (rm.handles[i] < 0)
+ if (rm->handles[i] < 0)
{
- free_request_manager (&rm, GNUNET_NO);
- return GNUNET_SYSERR;
+ free_request_manager (rm, GNUNET_NO);
+ return NULL;
}
}
}
- rm.lock = GNUNET_mutex_create (GNUNET_YES);
- rm.sctx = GNUNET_FS_create_search_context (ectx, cfg, rm.lock);
- if (rm.sctx == NULL)
+ rm->lock = GNUNET_mutex_create (GNUNET_YES);
+ rm->sctx = GNUNET_FS_create_search_context (ectx, cfg, rm->lock);
+ if (rm->sctx == NULL)
{
- free_request_manager (&rm, GNUNET_NO);
- return GNUNET_SYSERR;
+ free_request_manager (rm, GNUNET_NO);
+ return NULL;
}
if (GNUNET_ECRS_uri_test_loc (uri))
{
GNUNET_hash (&uri->data.loc.peer, sizeof (GNUNET_RSA_PublicKey),
- &rm.target.hashPubKey);
- rm.have_target = GNUNET_YES;
+ &rm->target.hashPubKey);
+ rm->have_target = GNUNET_YES;
}
top = GNUNET_malloc (sizeof (struct Node));
memset (top, 0, sizeof (struct Node));
- top->ctx = &rm;
+ top->ctx = rm;
top->chk = uri->data.fi.chk;
top->offset = 0;
- top->level = rm.treedepth;
+ top->level = rm->treedepth;
- GNUNET_mutex_lock (rm.lock);
+ GNUNET_mutex_lock (rm->lock);
if (GNUNET_NO == check_node_present (top))
addRequest (top);
else
GNUNET_free (top);
- GNUNET_mutex_unlock (rm.lock);
- while ((GNUNET_OK == tt (ttClosure)) &&
- (GNUNET_YES != GNUNET_shutdown_test ()) &&
- (rm.abortFlag == GNUNET_NO) && (rm.head != NULL))
- GNUNET_thread_sleep (5 * GNUNET_CRON_SECONDS);
- if ((rm.head == NULL) &&
- ((rm.completed == rm.total) ||
- ((rm.total != rm.length) && (rm.completed >= rm.length))))
+ GNUNET_mutex_unlock (rm->lock);
+ return rm;
+}
+
+int
+GNUNET_ECRS_file_download_partial_stop (struct GNUNET_ECRS_DownloadContext *
rm)
+{
+ int ret;
+ char * rdir;
+ int len;
+
+ if ((rm->head == NULL) &&
+ ((rm->completed == rm->total) ||
+ ((rm->total != rm->length) && (rm->completed >= rm->length))))
{
ret = GNUNET_OK;
}
else
{
#if 0
- GNUNET_GE_LOG (ectx,
+ GNUNET_GE_LOG (rm->ectx,
GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
"Download ends prematurely: %d %llu == %llu %d TT: %d\n",
- rm.requestListIndex,
- rm.completed, rm.total, rm.abortFlag, tt (ttClosure));
+ rm->requestListIndex,
+ rm->completed, rm->total, rm->abortFlag, tt (ttClosure));
#endif
- ret = GNUNET_SYSERR;
- }
-#if DEBUG_DOWNLOAD
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
- "`%s' terminating for file `%s' with result %s\n",
- __FUNCTION__, filename,
- ret == GNUNET_OK ? "SUCCESS" : "INCOMPLETE");
-#endif
- if ((ret == GNUNET_SYSERR) && (tt (ttClosure) == GNUNET_SYSERR))
- {
- if (0 != UNLINK (rm.filename))
+ if (0 != UNLINK (rm->filename))
{
- GNUNET_GE_LOG_STRERROR_FILE (ectx,
+ GNUNET_GE_LOG_STRERROR_FILE (rm->ectx,
GNUNET_GE_WARNING | GNUNET_GE_USER |
- GNUNET_GE_BULK, "unlink", rm.filename);
+ GNUNET_GE_BULK, "unlink", rm->filename);
}
else
{
/* delete empty directories */
- rdir = GNUNET_strdup (rm.filename);
+ rdir = GNUNET_strdup (rm->filename);
len = strlen (rdir);
do
{
@@ -927,15 +925,75 @@
while ((len > 0) && (0 == rmdir (rdir)));
GNUNET_free (rdir);
}
+ ret = GNUNET_SYSERR;
}
- free_request_manager (&rm,
- ((ret == GNUNET_OK) ||
- (tt (ttClosure) == GNUNET_SYSERR))
- ? GNUNET_YES : GNUNET_NO);
+#if DEBUG_DOWNLOAD
+ GNUNET_GE_LOG (ectx,
+ GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
+ "`%s' terminating for file `%s' with result %s\n",
+ __FUNCTION__, filename,
+ ret == GNUNET_OK ? "SUCCESS" : "INCOMPLETE");
+#endif
+ free_request_manager (rm,
+ (ret == GNUNET_OK) ? GNUNET_YES : GNUNET_NO);
return ret;
}
/**
+ * Download parts of a file. Note that this will store
+ * the blocks at the respective offset in the given file.
+ * Also, the download is still using the blocking of the
+ * underlying ECRS encoding. As a result, the download
+ * may *write* outside of the given boundaries (if offset
+ * and length do not match the 32k ECRS block boundaries).
+ * <p>
+ *
+ * This function should be used to focus a download towards a
+ * particular portion of the file (optimization), not to strictly
+ * limit the download to exactly those bytes.
+ *
+ * @param uri the URI of the file (determines what to download)
+ * @param filename where to store the file
+ * @param no_temporaries set to GNUNET_YES to disallow generation of temporary
files
+ * @param start starting offset
+ * @param length length of the download (starting at offset)
+ */
+int
+GNUNET_ECRS_file_download_partial (struct GNUNET_GE_Context *ectx,
+ struct GNUNET_GC_Configuration *cfg,
+ const struct GNUNET_ECRS_URI *uri,
+ const char *filename,
+ unsigned long long offset,
+ unsigned long long length,
+ unsigned int anonymityLevel,
+ int no_temporaries,
+ GNUNET_ECRS_DownloadProgressCallback dpcb,
+ void *dpcbClosure,
+ GNUNET_ECRS_TestTerminate tt,
+ void *ttClosure)
+{
+ struct GNUNET_ECRS_DownloadContext * rm;
+
+ rm = GNUNET_ECRS_file_download_partial_start (ectx,
+ cfg,
+ uri,
+ filename,
+ offset,
+ length,
+ anonymityLevel,
+ no_temporaries,
+ dpcb,
+ dpcbClosure);
+ if (rm == NULL)
+ return (length == 0) ? GNUNET_OK : GNUNET_SYSERR;
+ while ((GNUNET_OK == tt (ttClosure)) &&
+ (GNUNET_YES != GNUNET_shutdown_test ()) &&
+ (rm->abortFlag == GNUNET_NO) && (rm->head != NULL))
+ GNUNET_thread_sleep (5 * GNUNET_CRON_SECONDS);
+ return GNUNET_ECRS_file_download_partial_stop(rm);
+}
+
+/**
* Download a file (simplified API).
*
* @param uri the URI of the file (determines what to download)
Modified: GNUnet/src/applications/fs/fsui/download.c
===================================================================
--- GNUnet/src/applications/fs/fsui/download.c 2008-02-25 06:12:45 UTC (rev
6451)
+++ GNUnet/src/applications/fs/fsui/download.c 2008-02-25 07:06:55 UTC (rev
6452)
@@ -1,6 +1,6 @@
/*
This file is part of GNUnet.
- (C) 2001, 2002, 2003, 2004, 2005, 2006 Christian Grothoff (and other
contributing authors)
+ (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008 Christian Grothoff (and
other contributing authors)
GNUnet is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published
@@ -138,6 +138,74 @@
}
/**
+ * Trigger recursive download.
+ */
+static void
+download_recursive (GNUNET_FSUI_DownloadList *dl)
+{
+ char *dirBlock;
+ int fd;
+ char *fn;
+ size_t totalBytes;
+ struct GNUNET_ECRS_MetaData *md;
+
+ totalBytes = GNUNET_ECRS_uri_get_file_size (dl->fi.uri);
+ fn =
+ GNUNET_malloc (strlen (dl->filename) + strlen (GNUNET_DIRECTORY_EXT) +
+ 1);
+ strcpy (fn, dl->filename);
+ fd = strlen (fn) - 1;
+ if (fn[fd] == '/' || fn[fd] == '\\')
+ {
+ fn[fd] = '\0';
+ strcat (fn, GNUNET_DIRECTORY_EXT);
+ }
+ fd = GNUNET_disk_file_open (dl->ctx->ectx, fn, O_LARGEFILE | O_RDONLY);
+ if (fd != -1)
+ {
+ dirBlock = MMAP (NULL, totalBytes, PROT_READ, MAP_SHARED, fd, 0);
+ if (MAP_FAILED == dirBlock)
+ {
+ GNUNET_GE_LOG_STRERROR_FILE (dl->ctx->ectx,
+ GNUNET_GE_ERROR | GNUNET_GE_BULK |
+ GNUNET_GE_ADMIN | GNUNET_GE_USER,
+ "mmap", fn);
+ }
+ else
+ {
+ md = NULL;
+ GNUNET_ECRS_directory_list_contents (dl->ctx->ectx,
+ dirBlock,
+ totalBytes,
+ &md,
+ &listURIfoundDirectory,
+ dl);
+ if (md != NULL)
+ GNUNET_ECRS_meta_data_destroy (md);
+ if (dl->is_recursive)
+ {
+ /* load directory, start downloads */
+ md = NULL;
+ GNUNET_mutex_lock (dl->ctx->lock);
+ GNUNET_ECRS_directory_list_contents (dl->ctx->ectx,
+ dirBlock,
+ totalBytes,
+ &md,
+ &triggerRecursiveDownload,
+ dl);
+ GNUNET_mutex_unlock (dl->ctx->lock);
+ GNUNET_ECRS_meta_data_destroy (md);
+ MUNMAP (dirBlock, totalBytes);
+ }
+ }
+ CLOSE (fd);
+ }
+ GNUNET_free (fn);
+}
+
+
+
+/**
* Progress notification from ECRS. Tell FSUI client.
*/
static void
@@ -154,6 +222,26 @@
GNUNET_CronTime now;
GNUNET_CronTime run_time;
+ if (dl->total + 1 == totalBytes)
+ {
+ /* error! */
+ dl->state = GNUNET_FSUI_ERROR;
+ event.type = GNUNET_FSUI_download_error;
+ event.data.DownloadError.dc.pos = dl;
+ event.data.DownloadError.dc.cctx = dl->cctx;
+ event.data.DownloadError.dc.ppos =
+ dl->parent == &dl->ctx->activeDownloads ? NULL : dl->parent;
+ event.data.DownloadError.dc.pcctx = dl->parent->cctx;
+ event.data.DownloadError.dc.spos = dl->search;
+ event.data.DownloadError.dc.sctx =
+ dl->search == NULL ? NULL : dl->search->cctx;
+ event.data.DownloadError.message = lastBlock;
+ GNUNET_URITRACK_add_state (dl->ctx->ectx,
+ dl->ctx->cfg, dl->fi.uri,
+ GNUNET_URITRACK_DOWNLOAD_ABORTED);
+ dl->ctx->ecb (dl->ctx->ecbClosure, &event);
+ return;
+ }
GNUNET_GE_ASSERT (dl->ctx->ectx, dl->total == totalBytes);
dl->completed = completedBytes;
event.type = GNUNET_FSUI_download_progress;
@@ -221,56 +309,7 @@
if (md != NULL)
GNUNET_ECRS_meta_data_destroy (md);
}
-}
-
-/**
- * Check if termination of this download is desired.
- */
-static int
-testTerminate (void *cls)
-{
- GNUNET_FSUI_DownloadList *dl = cls;
-
- if ((dl->state == GNUNET_FSUI_ERROR) || (dl->state == GNUNET_FSUI_ABORTED))
- return GNUNET_SYSERR; /* aborted - delete! */
- if (dl->state != GNUNET_FSUI_ACTIVE)
- return GNUNET_NO; /* suspended */
- return GNUNET_OK;
-}
-
-/**
- * Thread that downloads a file.
- */
-static void *
-downloadThread (void *cls)
-{
- GNUNET_FSUI_DownloadList *dl = cls;
- int ret;
- GNUNET_FSUI_Event event;
- struct GNUNET_GE_Context *ectx;
- struct GNUNET_GE_Memory *mem;
- struct GNUNET_GE_Context *ee;
-
- dl->startTime = GNUNET_get_time () - dl->runTime;
- ectx = dl->ctx->ectx;
-#if DEBUG_DTM
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
- "Download thread for `%s' started...\n", dl->filename);
-#endif
- GNUNET_GE_ASSERT (ectx, dl->ctx != NULL);
- GNUNET_GE_ASSERT (ectx, dl->filename != NULL);
- mem = GNUNET_GE_memory_create (2);
- ee =
- GNUNET_GE_create_context_memory (GNUNET_GE_USER | GNUNET_GE_ADMIN |
- GNUNET_GE_ERROR | GNUNET_GE_WARNING |
- GNUNET_GE_FATAL | GNUNET_GE_BULK |
- GNUNET_GE_IMMEDIATE, mem);
- ret =
- GNUNET_ECRS_file_download (ee, dl->ctx->cfg, dl->fi.uri, dl->filename,
- dl->anonymityLevel, &downloadProgressCallback,
- dl, &testTerminate, dl);
- if (ret == GNUNET_OK)
+ if (totalBytes == completedBytes)
{
dl->state = GNUNET_FSUI_COMPLETED;
event.type = GNUNET_FSUI_download_completed;
@@ -289,128 +328,11 @@
dl->ctx->cfg,
dl->fi.uri,
GNUNET_URITRACK_DOWNLOAD_COMPLETED);
- dl->ctx->ecb (dl->ctx->ecbClosure, &event);
- }
- else if (dl->state == GNUNET_FSUI_ACTIVE)
- {
- const char *error;
-
- /* ECRS error */
- dl->state = GNUNET_FSUI_ERROR;
- event.type = GNUNET_FSUI_download_error;
- event.data.DownloadError.dc.pos = dl;
- event.data.DownloadError.dc.cctx = dl->cctx;
- event.data.DownloadError.dc.ppos =
- dl->parent == &dl->ctx->activeDownloads ? NULL : dl->parent;
- event.data.DownloadError.dc.pcctx = dl->parent->cctx;
- event.data.DownloadError.dc.spos = dl->search;
- event.data.DownloadError.dc.sctx =
- dl->search == NULL ? NULL : dl->search->cctx;
- error = GNUNET_GE_memory_get (mem, 0);
- if (error == NULL)
- error = _("Download failed (no reason given)");
- event.data.DownloadError.message = error;
- GNUNET_URITRACK_add_state (dl->ctx->ectx,
- dl->ctx->cfg, dl->fi.uri,
- GNUNET_URITRACK_DOWNLOAD_ABORTED);
- dl->ctx->ecb (dl->ctx->ecbClosure, &event);
- }
- else if (dl->state == GNUNET_FSUI_ABORTED)
- { /* aborted */
- event.type = GNUNET_FSUI_download_aborted;
- event.data.DownloadAborted.dc.pos = dl;
- event.data.DownloadAborted.dc.cctx = dl->cctx;
- event.data.DownloadAborted.dc.ppos =
- dl->parent == &dl->ctx->activeDownloads ? NULL : dl->parent;
- event.data.DownloadAborted.dc.pcctx = dl->parent->cctx;
- event.data.DownloadAborted.dc.spos = dl->search;
- event.data.DownloadAborted.dc.sctx =
- dl->search == NULL ? NULL : dl->search->cctx;
- GNUNET_URITRACK_add_state (dl->ctx->ectx, dl->ctx->cfg, dl->fi.uri,
- GNUNET_URITRACK_DOWNLOAD_ABORTED);
- dl->ctx->ecb (dl->ctx->ecbClosure, &event);
- }
- else
- {
- /* else: suspended */
- GNUNET_GE_BREAK (NULL, dl->state == GNUNET_FSUI_SUSPENDING);
- }
-
-
- if ((ret == GNUNET_OK) &&
- (dl->is_directory == GNUNET_YES)
- && (GNUNET_ECRS_uri_get_file_size (dl->fi.uri) > 0))
- {
- char *dirBlock;
- int fd;
- char *fn;
- size_t totalBytes;
- struct GNUNET_ECRS_MetaData *md;
-
- totalBytes = GNUNET_ECRS_uri_get_file_size (dl->fi.uri);
- fn =
- GNUNET_malloc (strlen (dl->filename) + strlen (GNUNET_DIRECTORY_EXT) +
- 1);
- strcpy (fn, dl->filename);
- fd = strlen (fn) - 1;
- if (fn[fd] == '/' || fn[fd] == '\\')
- {
- fn[fd] = '\0';
- strcat (fn, GNUNET_DIRECTORY_EXT);
- }
- fd = GNUNET_disk_file_open (ectx, fn, O_LARGEFILE | O_RDONLY);
- if (fd != -1)
- {
- dirBlock = MMAP (NULL, totalBytes, PROT_READ, MAP_SHARED, fd, 0);
- if (MAP_FAILED == dirBlock)
- {
- GNUNET_GE_LOG_STRERROR_FILE (ectx,
- GNUNET_GE_ERROR | GNUNET_GE_BULK |
- GNUNET_GE_ADMIN | GNUNET_GE_USER,
- "mmap", fn);
- }
- else
- {
- md = NULL;
- GNUNET_ECRS_directory_list_contents (dl->ctx->ectx,
- dirBlock,
- totalBytes,
- &md,
- &listURIfoundDirectory,
- dl);
- if (md != NULL)
- GNUNET_ECRS_meta_data_destroy (md);
-
- if (dl->is_recursive)
- {
- /* load directory, start downloads */
- md = NULL;
- GNUNET_mutex_lock (dl->ctx->lock);
- GNUNET_ECRS_directory_list_contents (dl->ctx->ectx,
- dirBlock,
- totalBytes,
- &md,
-
&triggerRecursiveDownload,
- dl);
- GNUNET_mutex_unlock (dl->ctx->lock);
- GNUNET_ECRS_meta_data_destroy (md);
- MUNMAP (dirBlock, totalBytes);
- }
- }
- CLOSE (fd);
- }
- GNUNET_free (fn);
- }
-#if DEBUG_DTM
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
- "Download thread for `%s' terminated (%s)...\n",
- dl->filename, ret == GNUNET_OK ? "COMPLETED" : "ABORTED");
-#endif
- dl->runTime = GNUNET_get_time () - dl->startTime;
- GNUNET_GE_free_context (ee);
- GNUNET_GE_memory_free (mem);
- return NULL;
+ dl->ctx->ecb (dl->ctx->ecbClosure, &event);
+ if ( (dl->is_directory == GNUNET_YES) &&
+ (GNUNET_ECRS_uri_get_file_size (dl->fi.uri) > 0) )
+ download_recursive(dl);
+ }
}
/**
@@ -525,7 +447,6 @@
{
struct GNUNET_GE_Context *ectx;
GNUNET_FSUI_DownloadList *dpos;
- void *unused;
int ret;
if (list == NULL)
@@ -553,19 +474,18 @@
list->filename);
#endif
list->state = GNUNET_FSUI_ACTIVE;
- list->handle = GNUNET_thread_create (&downloadThread, list, 128 * 1024);
+ list->startTime = GNUNET_get_time() - list->runTime;
+ list->handle = GNUNET_ECRS_file_download_partial_start(list->ctx->ectx,
list->ctx->cfg, list->fi.uri, list->filename,
+ 0,
+
GNUNET_ECRS_uri_get_file_size(list->fi.uri),
+
list->anonymityLevel,
+ GNUNET_NO,
+
&downloadProgressCallback,
+ list);
if (list->handle != NULL)
- {
- list->ctx->activeDownloadThreads++;
- }
+ list->ctx->activeDownloadThreads++;
else
- {
- GNUNET_GE_LOG_STRERROR (ectx,
- GNUNET_GE_ADMIN | GNUNET_GE_USER |
- GNUNET_GE_BULK | GNUNET_GE_ERROR,
- "pthread_create");
- list->state = GNUNET_FSUI_ERROR_JOINED;
- }
+ list->state = GNUNET_FSUI_ERROR_JOINED;
}
/* should this one be stopped? */
@@ -583,8 +503,7 @@
#endif
list->state = GNUNET_FSUI_SUSPENDING;
GNUNET_GE_ASSERT (ectx, list->handle != NULL);
- GNUNET_thread_stop_sleep (list->handle);
- GNUNET_thread_join (list->handle, &unused);
+ GNUNET_ECRS_file_download_partial_stop(list->handle);
list->handle = NULL;
list->ctx->activeDownloadThreads--;
list->state = GNUNET_FSUI_PENDING;
@@ -602,8 +521,7 @@
"Download thread manager collects inactive download of
file `%s'\n",
list->filename);
#endif
- GNUNET_thread_stop_sleep (list->handle);
- GNUNET_thread_join (list->handle, &unused);
+ GNUNET_ECRS_file_download_partial_stop(list->handle);
list->handle = NULL;
list->ctx->activeDownloadThreads--;
list->state++; /* adds _JOINED */
@@ -632,6 +550,7 @@
struct GNUNET_FSUI_DownloadList *dl)
{
struct GNUNET_FSUI_DownloadList *c;
+ GNUNET_FSUI_Event event;
GNUNET_GE_ASSERT (ctx->ectx, dl != NULL);
c = dl->child;
@@ -644,8 +563,22 @@
return GNUNET_NO;
if (dl->state == GNUNET_FSUI_ACTIVE)
{
- dl->state = GNUNET_FSUI_ABORTED;
- GNUNET_thread_stop_sleep (dl->handle);
+ dl->state = GNUNET_FSUI_ABORTED_JOINED;
+ GNUNET_ECRS_file_download_partial_stop(dl->handle);
+ dl->handle = NULL;
+ dl->runTime = GNUNET_get_time () - dl->startTime;
+ event.type = GNUNET_FSUI_download_aborted;
+ event.data.DownloadAborted.dc.pos = dl;
+ event.data.DownloadAborted.dc.cctx = dl->cctx;
+ event.data.DownloadAborted.dc.ppos =
+ dl->parent == &dl->ctx->activeDownloads ? NULL : dl->parent;
+ event.data.DownloadAborted.dc.pcctx = dl->parent->cctx;
+ event.data.DownloadAborted.dc.spos = dl->search;
+ event.data.DownloadAborted.dc.sctx =
+ dl->search == NULL ? NULL : dl->search->cctx;
+ GNUNET_URITRACK_add_state (dl->ctx->ectx, dl->ctx->cfg, dl->fi.uri,
+ GNUNET_URITRACK_DOWNLOAD_ABORTED);
+ dl->ctx->ecb (dl->ctx->ecbClosure, &event);
}
else
{
@@ -664,7 +597,6 @@
GNUNET_FSUI_download_stop (struct GNUNET_FSUI_Context *ctx,
struct GNUNET_FSUI_DownloadList *dl)
{
- void *unused;
struct GNUNET_FSUI_DownloadList *prev;
GNUNET_FSUI_Event event;
int i;
@@ -695,8 +627,9 @@
(dl->state == GNUNET_FSUI_ABORTED) || (dl->state == GNUNET_FSUI_ERROR))
{
GNUNET_GE_ASSERT (ctx->ectx, dl->handle != NULL);
- GNUNET_thread_stop_sleep (dl->handle);
- GNUNET_thread_join (dl->handle, &unused);
+ GNUNET_ECRS_file_download_partial_stop(dl->handle);
+ dl->handle = NULL;
+ dl->runTime = GNUNET_get_time () - dl->startTime;
GNUNET_mutex_lock (ctx->lock);
dl->ctx->activeDownloadThreads--;
GNUNET_mutex_unlock (ctx->lock);
Modified: GNUnet/src/applications/fs/fsui/downloadtest.c
===================================================================
--- GNUnet/src/applications/fs/fsui/downloadtest.c 2008-02-25 06:12:45 UTC
(rev 6451)
+++ GNUnet/src/applications/fs/fsui/downloadtest.c 2008-02-25 07:06:55 UTC
(rev 6452)
@@ -81,16 +81,6 @@
#endif
download = event->data.DownloadResumed.dc.pos;
break;
- case GNUNET_FSUI_search_completed:
-#if DEBUG_VERBOSE
- printf ("Search completed\n");
-#endif
- if (download == NULL)
- {
- fprintf (stderr,
- "ERROR: Search completed but download not started!\n");
- }
- break;
case GNUNET_FSUI_search_result:
#if DEBUG_VERBOSE
printf ("Received search result\n");
@@ -173,7 +163,6 @@
case GNUNET_FSUI_unindex_error:
case GNUNET_FSUI_upload_error:
case GNUNET_FSUI_download_error:
- case GNUNET_FSUI_search_error:
fprintf (stderr, "Received ERROR: %d\n", event->type);
GNUNET_GE_BREAK (ectx, 0);
break;
Modified: GNUnet/src/applications/fs/fsui/fsui.h
===================================================================
--- GNUnet/src/applications/fs/fsui/fsui.h 2008-02-25 06:12:45 UTC (rev
6451)
+++ GNUnet/src/applications/fs/fsui/fsui.h 2008-02-25 07:06:55 UTC (rev
6452)
@@ -193,9 +193,9 @@
void *cctx;
/**
- * Currently assigned thread (if any).
+ * Currently assigned ECRS context (if any).
*/
- struct GNUNET_ThreadHandle *handle;
+ struct GNUNET_ECRS_DownloadContext *handle;
/**
* FIs of completed sub-downloads.
Modified: GNUnet/src/applications/fs/fsui/serializetest.c
===================================================================
--- GNUnet/src/applications/fs/fsui/serializetest.c 2008-02-25 06:12:45 UTC
(rev 6451)
+++ GNUnet/src/applications/fs/fsui/serializetest.c 2008-02-25 07:06:55 UTC
(rev 6452)
@@ -91,7 +91,6 @@
case GNUNET_FSUI_unindex_error:
case GNUNET_FSUI_upload_error:
case GNUNET_FSUI_download_error:
- case GNUNET_FSUI_search_error:
fprintf (stderr, "Received ERROR: %d\n", event->type);
GNUNET_GE_BREAK (ectx, 0);
break;
Modified: GNUnet/src/applications/fs/fsui/serializetest2.c
===================================================================
--- GNUnet/src/applications/fs/fsui/serializetest2.c 2008-02-25 06:12:45 UTC
(rev 6451)
+++ GNUnet/src/applications/fs/fsui/serializetest2.c 2008-02-25 07:06:55 UTC
(rev 6452)
@@ -233,12 +233,6 @@
event->type, event->data.DownloadError.message);
GNUNET_GE_BREAK (ectx, 0);
break;
- case GNUNET_FSUI_search_error:
- fprintf (stderr,
- "Received ERROR: %d %s\n",
- event->type, event->data.SearchError.message);
- GNUNET_GE_BREAK (ectx, 0);
- break;
case GNUNET_FSUI_download_aborted:
#if DEBUG_VERBOSE
printf ("Received download aborted event.\n");
Modified: GNUnet/src/applications/fs/fsui/serializetest3.c
===================================================================
--- GNUnet/src/applications/fs/fsui/serializetest3.c 2008-02-25 06:12:45 UTC
(rev 6451)
+++ GNUnet/src/applications/fs/fsui/serializetest3.c 2008-02-25 07:06:55 UTC
(rev 6452)
@@ -86,7 +86,6 @@
case GNUNET_FSUI_unindex_error:
case GNUNET_FSUI_upload_error:
case GNUNET_FSUI_download_error:
- case GNUNET_FSUI_search_error:
fprintf (stderr, "Received ERROR: %d\n", event->type);
GNUNET_GE_BREAK (ectx, 0);
break;
@@ -106,7 +105,6 @@
case GNUNET_FSUI_search_started:
case GNUNET_FSUI_search_aborted:
case GNUNET_FSUI_search_stopped:
- case GNUNET_FSUI_search_completed:
case GNUNET_FSUI_unindex_started:
case GNUNET_FSUI_unindex_stopped:
break;
Modified: GNUnet/src/applications/fs/fsui/serializetest4.c
===================================================================
--- GNUnet/src/applications/fs/fsui/serializetest4.c 2008-02-25 06:12:45 UTC
(rev 6451)
+++ GNUnet/src/applications/fs/fsui/serializetest4.c 2008-02-25 07:06:55 UTC
(rev 6452)
@@ -203,7 +203,6 @@
case GNUNET_FSUI_unindex_error:
case GNUNET_FSUI_upload_error:
case GNUNET_FSUI_download_error:
- case GNUNET_FSUI_search_error:
fprintf (stderr, "Received ERROR: %d\n", event->type);
GNUNET_GE_BREAK (ectx, 0);
break;
Modified: GNUnet/src/applications/fs/tools/gnunet-insert.c
===================================================================
--- GNUnet/src/applications/fs/tools/gnunet-insert.c 2008-02-25 06:12:45 UTC
(rev 6451)
+++ GNUnet/src/applications/fs/tools/gnunet-insert.c 2008-02-25 07:06:55 UTC
(rev 6452)
@@ -398,6 +398,8 @@
}
if (uri_string == NULL)
filename = argv[i];
+ else
+ filename = NULL;
if (extract_only)
{
Modified: GNUnet/src/include/gnunet_ecrs_lib.h
===================================================================
--- GNUnet/src/include/gnunet_ecrs_lib.h 2008-02-25 06:12:45 UTC (rev
6451)
+++ GNUnet/src/include/gnunet_ecrs_lib.h 2008-02-25 07:06:55 UTC (rev
6452)
@@ -722,7 +722,11 @@
* operation.
*
* @param totalBytes number of bytes that will need to be downloaded,
- * excluding inner blocks
+ * excluding inner blocks; the value given here will
+ * be one larger than the requested download size to signal
+ * an error. In that case, all other values will be 0,
+ * except form "lastBlock" which will point to an error
+ * message describing the problem.
* @param completedBytes number of bytes that have been obtained
* @param eta absolute estimated time for the completion of the operation
* @param lastBlockOffset offset of the last block that was downloaded,
@@ -738,7 +742,45 @@
unsigned long long lastBlockOffset,
const char *lastBlock, unsigned int lastBlockSize, void *closure);
+struct GNUNET_ECRS_DownloadContext;
+
/**
+ * Download parts of a file ASYNCHRONOUSLY. Note that this will store
+ * the blocks at the respective offset in the given file. Also, the
+ * download is still using the blocking of the underlying ECRS
+ * encoding. As a result, the download may *write* outside of the
+ * given boundaries (if offset and length do not match the 32k ECRS
+ * block boundaries). <p>
+ *
+ * This function should be used to focus a download towards a
+ * particular portion of the file (optimization), not to strictly
+ * limit the download to exactly those bytes.
+ *
+ * @param uri the URI of the file (determines what to download)
+ * @param filename where to store the file
+ * @param no_temporaries set to GNUNET_YES to disallow generation of temporary
files
+ * @param start starting offset
+ * @param length length of the download (starting at offset)
+ */
+struct GNUNET_ECRS_DownloadContext *
+GNUNET_ECRS_file_download_partial_start (struct GNUNET_GE_Context *ectx,
+ struct GNUNET_GC_Configuration *cfg,
+ const struct GNUNET_ECRS_URI *uri,
+ const char *filename,
+ unsigned long long offset,
+ unsigned long long length,
+ unsigned int anonymityLevel,
+ int no_temporaries,
+ GNUNET_ECRS_DownloadProgressCallback
dpcb,
+ void *dpcbClosure);
+
+/**
+ * Stop a download (aborts if download is incomplete).
+ */
+int
+GNUNET_ECRS_file_download_partial_stop (struct GNUNET_ECRS_DownloadContext *
rm);
+
+/**
* DOWNLOAD a file.
*
* @param uri the URI of the file (determines what to download)
Modified: GNUnet/src/util/disk/storage.c
===================================================================
--- GNUnet/src/util/disk/storage.c 2008-02-25 06:12:45 UTC (rev 6451)
+++ GNUnet/src/util/disk/storage.c 2008-02-25 07:06:55 UTC (rev 6452)
@@ -609,7 +609,7 @@
{
struct stat istat;
- if (0 != STAT (fileName, &istat))
+ if (0 != LSTAT (fileName, &istat))
return GNUNET_NO; /* file may not exist... */
if (UNLINK (fileName) == 0)
return GNUNET_OK;
Modified: GNUnet/todo
===================================================================
--- GNUnet/todo 2008-02-25 06:12:45 UTC (rev 6451)
+++ GNUnet/todo 2008-02-25 07:06:55 UTC (rev 6452)
@@ -4,17 +4,20 @@
RC == Release Critical
PRE == to be done before a pre-release
-0.8.0 [6'08] (aka "advanced features"):
+0.8.0pre0 [2'08]:
+- make sure "make check" passes
+- make tcpserver IPv6 compatible
+- gnunet-gtk ready?
+- more IPv6 testing
+
+0.8.0 [4'08] (aka "advanced features"):
- clean up VPN code
-- asynchronous ECRS API [?]
- tune GAP query planning code [RC]
-- HTTP transport sometimes goes crazy with CPU usage
-- gnunet-chat (CS-only)
-- make tcpserver IPv6 compatible
-
-TESTING:
+- HTTP transport sometimes goes crazy with CPU usage [RC]
+- gnunet-chat (CS-only) [RC]
- test RPC code (write a small demo)
-- test GAP code in general (still have one unexplained crazy crash!)
+- test new asynchronous ECRS API (and resulting FSUI changes)
+- test GAP code in general; still have one unexplained crazy crash:
Program terminated with signal 11, Segmentation fault.
#0 0xb4e8e73a in GNUNET_FS_SHARED_test_valid_new_response (rl=0x8060,
primary_key=0xb343f1c8, size=820, data=0xb0b8f0e8, hc=0xb343f134) at
shared.c:94
_______________________________________________
GNUnet-SVN mailing list
GNUnet-SVN@gnu.org
http://lists.gnu.org/mailman/listinfo/gnunet-svn
